core(network-requests): add frame and preload debug data (GoogleChrome#14161)

brendankenny · web-flow · commit 34a7163fd2f9 · 2022-06-27T18:11:32.000-05:00
diff --git a/lighthouse-cli/test/smokehouse/test-definitions/oopif-requests.js b/lighthouse-cli/test/smokehouse/test-definitions/oopif-requests.js
@@ -50,19 +50,19 @@ const expectations = {
             // We want to make sure we are finding the iframe's requests (paulirish.com) *AND*
             // the iframe's iframe's iframe's requests (youtube.com/doubleclick/etc).
             _includes: [
-              {url: 'http://localhost:10200/oopif-requests.html', finished: true, statusCode: 200, resourceType: 'Document'},
+              {url: 'http://localhost:10200/oopif-requests.html', finished: true, statusCode: 200, resourceType: 'Document', experimentalFromMainFrame: true},
 
               // Paulirish iframe and subresource
-              {url: 'https://www.paulirish.com/2012/why-moving-elements-with-translate-is-better-than-posabs-topleft/', finished: true, statusCode: 200, resourceType: 'Document'},
-              {url: 'https://www.paulirish.com/avatar150.jpg', finished: true, statusCode: 200, resourceType: 'Image'},
-              {url: 'https://www.googletagmanager.com/gtag/js?id=G-PGXNGYWP8E', finished: true, statusCode: 200, resourceType: 'Script'},
-              {url: /^https:\/\/fonts\.googleapis\.com\/css/, finished: true, statusCode: 200, resourceType: 'Stylesheet'},
+              {url: 'https://www.paulirish.com/2012/why-moving-elements-with-translate-is-better-than-posabs-topleft/', finished: true, statusCode: 200, resourceType: 'Document', experimentalFromMainFrame: undefined},
+              {url: 'https://www.paulirish.com/avatar150.jpg', finished: true, statusCode: 200, resourceType: 'Image', experimentalFromMainFrame: undefined},
+              {url: 'https://www.googletagmanager.com/gtag/js?id=G-PGXNGYWP8E', finished: true, statusCode: 200, resourceType: 'Script', experimentalFromMainFrame: undefined},
+              {url: /^https:\/\/fonts\.googleapis\.com\/css/, finished: true, statusCode: 200, resourceType: 'Stylesheet', experimentalFromMainFrame: undefined},
 
               // Youtube iframe (OOPIF) and some subresources
               // FYI: Youtube has a ServiceWorker which sometimes cancels the document request. As a result, there will sometimes be multiple requests for this file.
               {url: 'https://www.youtube.com/embed/NZelrwd_iRs', finished: true, statusCode: 200, resourceType: 'Document'},
               {url: /^https:\/\/www\.youtube\.com\/.*?player.*?css/, finished: true, statusCode: 200, resourceType: 'Stylesheet'},
-              {url: /^https:\/\/www\.youtube\.com\/.*?\/embed.js/, finished: true, statusCode: 200, resourceType: 'Script'},
+              {url: /^https:\/\/www\.youtube\.com\/.*?\/embed.js/, finished: true, statusCode: 200, resourceType: 'Script', experimentalFromMainFrame: undefined},
 
               // Disqus iframe (OOPIF)
               {url: /^https:\/\/disqus\.com\/embed\/comments\//, finished: true, statusCode: 200, resourceType: 'Document'},
diff --git a/lighthouse-cli/test/smokehouse/test-definitions/perf-preload.js b/lighthouse-cli/test/smokehouse/test-definitions/perf-preload.js
@@ -78,6 +78,11 @@ const expectations = {
       'network-requests': {
         details: {
           items: {
+            _includes: [
+              {url: 'http://localhost:10200/preload.html', isLinkPreload: undefined, experimentalFromMainFrame: true},
+              {url: 'http://localhost:10200/perf/level-2.js?warning&delay=500', isLinkPreload: true, experimentalFromMainFrame: true},
+              {url: 'http://localhost:10200/perf/preload_tester.js', isLinkPreload: undefined, experimentalFromMainFrame: true},
+            ],
             length: '>5',
           },
         },
diff --git a/lighthouse-core/audits/network-requests.js b/lighthouse-core/audits/network-requests.js
@@ -8,6 +8,7 @@
 const Audit = require('./audit.js');
 const URL = require('../lib/url-shim.js');
 const NetworkRecords = require('../computed/network-records.js');
+const MainResource = require('../computed/main-resource.js');
 
 class NetworkRequests extends Audit {
   /**
@@ -19,7 +20,7 @@ class NetworkRequests extends Audit {
       scoreDisplayMode: Audit.SCORING_MODES.INFORMATIVE,
       title: 'Network Requests',
       description: 'Lists the network requests that were made during page load.',
-      requiredArtifacts: ['devtoolsLogs'],
+      requiredArtifacts: ['devtoolsLogs', 'URL', 'GatherContext'],
     };
   }
 
@@ -28,75 +29,91 @@ class NetworkRequests extends Audit {
    * @param {LH.Audit.Context} context
    * @return {Promise<LH.Audit.Product>}
    */
-  static audit(artifacts, context) {
+  static async audit(artifacts, context) {
     const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS];
-    return NetworkRecords.request(devtoolsLog, context).then(records => {
-      const earliestStartTime = records.reduce(
-        (min, record) => Math.min(min, record.startTime),
-        Infinity
-      );
+    const records = await NetworkRecords.request(devtoolsLog, context);
+    const earliestStartTime = records.reduce(
+      (min, record) => Math.min(min, record.startTime),
+      Infinity
+    );
 
-      /** @param {number} time */
-      const timeToMs = time => time < earliestStartTime || !Number.isFinite(time) ?
-        undefined : (time - earliestStartTime) * 1000;
+    // Optional mainFrameId check because the main resource is only available for
+    // navigations. TODO: https://github.com/GoogleChrome/lighthouse/issues/14157
+    // for the general solution to this.
+    /** @type {string|undefined} */
+    let mainFrameId;
+    if (artifacts.GatherContext.gatherMode === 'navigation') {
+      const mainResource = await MainResource.request({devtoolsLog, URL: artifacts.URL}, context);
+      mainFrameId = mainResource.frameId;
+    }
 
-      const results = records.map(record => {
-        const endTimeDeltaMs = record.lrStatistics?.endTimeDeltaMs;
-        const TCPMs = record.lrStatistics?.TCPMs;
-        const requestMs = record.lrStatistics?.requestMs;
-        const responseMs = record.lrStatistics?.responseMs;
+    /** @param {number} time */
+    const timeToMs = time => time < earliestStartTime || !Number.isFinite(time) ?
+      undefined : (time - earliestStartTime) * 1000;
 
-        return {
-          url: URL.elideDataURI(record.url),
-          protocol: record.protocol,
-          startTime: timeToMs(record.startTime),
-          endTime: timeToMs(record.endTime),
-          finished: record.finished,
-          transferSize: record.transferSize,
-          resourceSize: record.resourceSize,
-          statusCode: record.statusCode,
-          mimeType: record.mimeType,
-          resourceType: record.resourceType,
-          lrEndTimeDeltaMs: endTimeDeltaMs, // Only exists on Lightrider runs
-          lrTCPMs: TCPMs, // Only exists on Lightrider runs
-          lrRequestMs: requestMs, // Only exists on Lightrider runs
-          lrResponseMs: responseMs, // Only exists on Lightrider runs
-        };
-      });
-
-      // NOTE(i18n): this audit is only for debug info in the LHR and does not appear in the report.
-      /** @type {LH.Audit.Details.Table['headings']} */
-      const headings = [
-        {key: 'url', itemType: 'url', text: 'URL'},
-        {key: 'protocol', itemType: 'text', text: 'Protocol'},
-        {key: 'startTime', itemType: 'ms', granularity: 1, text: 'Start Time'},
-        {key: 'endTime', itemType: 'ms', granularity: 1, text: 'End Time'},
-        {
-          key: 'transferSize',
-          itemType: 'bytes',
-          displayUnit: 'kb',
-          granularity: 1,
-          text: 'Transfer Size',
-        },
-        {
-          key: 'resourceSize',
-          itemType: 'bytes',
-          displayUnit: 'kb',
-          granularity: 1,
-          text: 'Resource Size',
-        },
-        {key: 'statusCode', itemType: 'text', text: 'Status Code'},
-        {key: 'mimeType', itemType: 'text', text: 'MIME Type'},
-        {key: 'resourceType', itemType: 'text', text: 'Resource Type'},
-      ];
-
-      const tableDetails = Audit.makeTableDetails(headings, results);
+    const results = records.map(record => {
+      const endTimeDeltaMs = record.lrStatistics?.endTimeDeltaMs;
+      const TCPMs = record.lrStatistics?.TCPMs;
+      const requestMs = record.lrStatistics?.requestMs;
+      const responseMs = record.lrStatistics?.responseMs;
+      // Default these to undefined so omitted from JSON in the negative case.
+      const isLinkPreload = record.isLinkPreload || undefined;
+      const experimentalFromMainFrame = mainFrameId ?
+        ((record.frameId === mainFrameId) || undefined) :
+        undefined;
 
       return {
-        score: 1,
-        details: tableDetails,
+        url: URL.elideDataURI(record.url),
+        protocol: record.protocol,
+        startTime: timeToMs(record.startTime),
+        endTime: timeToMs(record.endTime),
+        finished: record.finished,
+        transferSize: record.transferSize,
+        resourceSize: record.resourceSize,
+        statusCode: record.statusCode,
+        mimeType: record.mimeType,
+        resourceType: record.resourceType,
+        isLinkPreload,
+        experimentalFromMainFrame,
+        lrEndTimeDeltaMs: endTimeDeltaMs, // Only exists on Lightrider runs
+        lrTCPMs: TCPMs, // Only exists on Lightrider runs
+        lrRequestMs: requestMs, // Only exists on Lightrider runs
+        lrResponseMs: responseMs, // Only exists on Lightrider runs
       };
     });
+
+    // NOTE(i18n): this audit is only for debug info in the LHR and does not appear in the report.
+    /** @type {LH.Audit.Details.Table['headings']} */
+    const headings = [
+      {key: 'url', itemType: 'url', text: 'URL'},
+      {key: 'protocol', itemType: 'text', text: 'Protocol'},
+      {key: 'startTime', itemType: 'ms', granularity: 1, text: 'Start Time'},
+      {key: 'endTime', itemType: 'ms', granularity: 1, text: 'End Time'},
+      {
+        key: 'transferSize',
+        itemType: 'bytes',
+        displayUnit: 'kb',
+        granularity: 1,
+        text: 'Transfer Size',
+      },
+      {
+        key: 'resourceSize',
+        itemType: 'bytes',
+        displayUnit: 'kb',
+        granularity: 1,
+        text: 'Resource Size',
+      },
+      {key: 'statusCode', itemType: 'text', text: 'Status Code'},
+      {key: 'mimeType', itemType: 'text', text: 'MIME Type'},
+      {key: 'resourceType', itemType: 'text', text: 'Resource Type'},
+    ];
+
+    const tableDetails = Audit.makeTableDetails(headings, results);
+
+    return {
+      score: 1,
+      details: tableDetails,
+    };
   }
 }
 
diff --git a/lighthouse-core/test/audits/network-requests-test.js b/lighthouse-core/test/audits/network-requests-test.js
@@ -10,12 +10,18 @@ import networkRecordsToDevtoolsLog from '../network-records-to-devtools-log.js';
 
 const cutoffLoadDevtoolsLog = readJson('../fixtures/traces/cutoff-load-m83.devtoolslog.json', import.meta);
 
+const GatherContext = {
+  gatherMode: 'navigation',
+};
+
 describe('Network requests audit', () => {
   it('should report finished and unfinished network requests', async () => {
     const artifacts = {
       devtoolsLogs: {
         [NetworkRequests.DEFAULT_PASS]: cutoffLoadDevtoolsLog,
       },
+      URL: {mainDocumentUrl: 'https://googlechrome.github.io/lighthouse/viewer/'},
+      GatherContext,
     };
 
     const output = await NetworkRequests.audit(artifacts, {computedCache: new Map()});
@@ -62,6 +68,8 @@ describe('Network requests audit', () => {
       devtoolsLogs: {
         [NetworkRequests.DEFAULT_PASS]: networkRecordsToDevtoolsLog(records),
       },
+      URL: {mainDocumentUrl: 'https://example.com/0'},
+      GatherContext,
     };
     const output = await NetworkRequests.audit(artifacts, {computedCache: new Map()});
 
@@ -75,4 +83,76 @@ describe('Network requests audit', () => {
       finished: true,
     }]);
   });
+
+  it('should report if records are from the main frame', async () => {
+    const records = [
+      {url: 'https://example.com/'},
+      {url: 'https://iframed.local/', frameId: '71D866EC199B90A2E0B2D9CF88DCBC4E'},
+    ];
+
+    const artifacts = {
+      devtoolsLogs: {
+        [NetworkRequests.DEFAULT_PASS]: networkRecordsToDevtoolsLog(records),
+      },
+      URL: {mainDocumentUrl: 'https://example.com/'},
+      GatherContext,
+    };
+    const output = await NetworkRequests.audit(artifacts, {computedCache: new Map()});
+
+    expect(output.details.items).toMatchObject([{
+      url: 'https://example.com/',
+      experimentalFromMainFrame: true,
+    }, {
+      url: 'https://iframed.local/',
+      experimentalFromMainFrame: undefined,
+    }]);
+  });
+
+  it('should not include main frame information outside of navigations', async () => {
+    const records = [
+      {url: 'https://example.com/'},
+      {url: 'https://iframed.local/', frameId: '71D866EC199B90A2E0B2D9CF88DCBC4E'},
+    ];
+
+    const artifacts = {
+      devtoolsLogs: {
+        [NetworkRequests.DEFAULT_PASS]: networkRecordsToDevtoolsLog(records),
+      },
+      URL: {mainDocumentUrl: 'https://example.com/'},
+      GatherContext: {gatherMode: 'timespan'},
+    };
+    const output = await NetworkRequests.audit(artifacts, {computedCache: new Map()});
+
+    expect(output.details.items).toMatchObject([{
+      url: 'https://example.com/',
+      experimentalFromMainFrame: undefined,
+    }, {
+      url: 'https://iframed.local/',
+      experimentalFromMainFrame: undefined,
+    }]);
+  });
+
+  it('should include if network request was preloaded', async () => {
+    const records = [
+      {url: 'https://example.com/'},
+      {url: 'https://example.com/img.jpg', isLinkPreload: true},
+    ];
+
+    const artifacts = {
+      devtoolsLogs: {
+        [NetworkRequests.DEFAULT_PASS]: networkRecordsToDevtoolsLog(records),
+      },
+      URL: {mainDocumentUrl: 'https://example.com/'},
+      GatherContext,
+    };
+    const output = await NetworkRequests.audit(artifacts, {computedCache: new Map()});
+
+    expect(output.details.items).toMatchObject([{
+      url: 'https://example.com/',
+      isLinkPreload: undefined,
+    }, {
+      url: 'https://example.com/img.jpg',
+      isLinkPreload: true,
+    }]);
+  });
 });
diff --git a/lighthouse-core/test/fixtures/fraggle-rock/reports/sample-flow-result.json b/lighthouse-core/test/fixtures/fraggle-rock/reports/sample-flow-result.json
diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json
