Skip to content

Commit 776eea0

Browse files
connorjclarkconnorjclark
authored
tests: add devtools e2e test runner (GoogleChrome#14110)
1 parent 26ff0fe commit 776eea0

File tree

13 files changed

+218
-25
lines changed

13 files changed

+218
-25
lines changed

‎.github/scripts/generate-devtools-hash.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@ cd "$LH_ROOT"
1616
bash .github/scripts/print-devtools-relevant-commits.sh
1717
md5 \
1818
lighthouse-core/test/chromium-web-tests/* \
19-
third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/**/*.*
19+
third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/**/*.* \
20+
third-party/devtools-tests/e2e/**/*.*

‎.github/workflows/devtools.yml‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,15 @@ jobs:
4545
${{ env.BLINK_TOOLS_PATH }}
4646
${{ github.workspace }}/lighthouse/.tmp/chromium-web-tests/content-shells
4747
${{ github.workspace }}/.gclient
48+
${{ github.workspace }}/.gclient_entries
4849
# This hash key changes:
4950
# 1) every monday (so invalidates once a week)
5051
# 2) every commit to CDT touching files important to Lighthouse web tests
5152
# 3) every change to file in Lighthouse repo important to running these tests.
5253
#
5354
# The number is how many times this hash key was manually updated to break the cache.
54-
key: ${{ runner.os }}-5-${{ env.WEEK_OF_THE_YEAR }}-${{ hashFiles('cdt-test-hash.txt') }}
55-
restore-keys: ${{ runner.os }}-5
55+
key: ${{ runner.os }}-7-${{ env.WEEK_OF_THE_YEAR }}-${{ hashFiles('cdt-test-hash.txt') }}
56+
restore-keys: ${{ runner.os }}-7
5657
- name: Set GHA_DEVTOOLS_CACHE_HIT
5758
if: steps.devtools-cache.outputs.cache-hit == 'true'
5859
run: echo "GHA_DEVTOOLS_CACHE_HIT=1" >> $GITHUB_ENV
@@ -73,11 +74,48 @@ jobs:
7374
uses: actions/cache@v3
7475
with:
7576
path: |
76-
${{ env.DEVTOOLS_PATH }}/out/Default/gen/front_end
77-
${{ env.DEVTOOLS_PATH }}/test/webtests
77+
${{ env.DEVTOOLS_PATH }}
7878
${{ github.workspace }}/.gclient
79+
${{ github.workspace }}/.gclient_entries
7980
key: devtools-build-artifacts-${{ github.run_id }}
8081

82+
e2e:
83+
needs: [build]
84+
runs-on: macos-latest
85+
86+
steps:
87+
- name: git clone
88+
uses: actions/checkout@v2
89+
with:
90+
path: lighthouse
91+
92+
- name: Use Node.js 14.x
93+
uses: actions/setup-node@v1
94+
with:
95+
node-version: 14.x
96+
97+
- run: yarn --frozen-lockfile --network-timeout 1000000
98+
working-directory: ${{ github.workspace }}/lighthouse
99+
100+
- name: Set $PATH
101+
run: echo "$DEPOT_TOOLS_PATH" >> $GITHUB_PATH
102+
# For vpython.
103+
- name: Download depot tools
104+
run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-depot-tools.sh
105+
106+
- name: Load build artifacts
107+
id: devtools-build-artifacts
108+
uses: actions/cache@v3
109+
with:
110+
path: |
111+
${{ env.DEVTOOLS_PATH }}
112+
${{ github.workspace }}/.gclient
113+
${{ github.workspace }}/.gclient_entries
114+
key: devtools-build-artifacts-${{ github.run_id }}
115+
116+
- name: Run e2e tests
117+
run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/run-e2e-tests.sh
118+
81119
web-tests:
82120
needs: [build]
83121
runs-on: macos-latest
@@ -98,9 +136,9 @@ jobs:
98136
uses: actions/cache@v3
99137
with:
100138
path: |
101-
${{ env.DEVTOOLS_PATH }}/out/Default/gen/front_end
102-
${{ env.DEVTOOLS_PATH }}/test/webtests
139+
${{ env.DEVTOOLS_PATH }}
103140
${{ github.workspace }}/.gclient
141+
${{ github.workspace }}/.gclient_entries
104142
key: devtools-build-artifacts-${{ github.run_id }}
105143

106144
- name: Download Blink Tools
@@ -147,9 +185,9 @@ jobs:
147185
uses: actions/cache@v3
148186
with:
149187
path: |
150-
${{ env.DEVTOOLS_PATH }}/out/Default/gen/front_end
151-
${{ env.DEVTOOLS_PATH }}/test/webtests
188+
${{ env.DEVTOOLS_PATH }}
152189
${{ github.workspace }}/.gclient
190+
${{ github.workspace }}/.gclient_entries
153191
key: devtools-build-artifacts-${{ github.run_id }}
154192

155193
- name: Download Blink Tools

‎lighthouse-core/scripts/roll-to-devtools.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,15 @@ rsync -avh "$lh_webtests_dir" "$fe_webtests_dir" --exclude="OWNERS" --delete
8080
lh_webtests_exp_dir="third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/"
8181
fe_webtests_exp_dir="$dt_dir/test/webtests/platform/generic/http/tests/devtools/lighthouse"
8282
rsync -avh "$lh_webtests_exp_dir" "$fe_webtests_exp_dir" --exclude="OWNERS" --delete
83+
84+
# copy e2e tests
85+
lh_e2e_dir="third-party/devtools-tests/e2e/lighthouse/"
86+
fe_e2e_dir="$dt_dir/test/e2e/lighthouse"
87+
rsync -avh "$lh_e2e_dir" "$fe_e2e_dir" --exclude="OWNERS" --exclude="BUILD.gn" --delete
88+
lh_e2e_res_dir="third-party/devtools-tests/e2e/resources/lighthouse/"
89+
fe_e2e_res_dir="$dt_dir/test/e2e/resources/lighthouse"
90+
rsync -avh "$lh_e2e_res_dir" "$fe_e2e_res_dir" --exclude="OWNERS" --exclude="BUILD.gn" --delete
91+
8392
echo ""
8493
echo "Done. To run the webtests: "
8594
echo " DEVTOOLS_PATH=\"$dt_dir\" yarn test-devtools"

‎lighthouse-core/test/chromium-web-tests/download-devtools.sh‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,10 @@ then
1717
git --no-pager log -1
1818

1919
# Update to keep current.
20-
if [ -z "${CI:-}" ]; then
21-
# Locally, clean everything and update to latest code.
22-
git reset --hard
23-
git clean -fd
24-
git pull --ff-only -f origin main
25-
gclient sync --delete_unversioned_trees --reset
26-
elif [ -z "${GHA_DEVTOOLS_CACHE_HIT:-}" ]; then
27-
# For CI, only run if this was a cache-miss.
28-
# The only way the folder already exists _and_ there is a cache-miss is
29-
# if actions/cache@v2 `restore-keys` has provided a partial environment for us.
30-
git reset --hard
31-
git clean -fd
32-
git pull --ff-only -f origin main
33-
gclient sync --delete_unversioned_trees --reset
34-
fi
20+
git reset --hard
21+
git clean -fd
22+
git pull --ff-only -f origin main
23+
gclient sync --delete_unversioned_trees --reset
3524

3625
exit 0
3726
fi

‎lighthouse-core/test/devtools-tests/run-e2e-tests.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
3+
##
4+
# @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
5+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
6+
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
7+
##
8+
9+
set -euo pipefail
10+
11+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
12+
export LH_ROOT="$SCRIPT_DIR/../../.."
13+
14+
cd "$DEVTOOLS_PATH"
15+
16+
TEST_PATTERN="${1:-lighthouse/*}"
17+
npm run e2etest -- "$TEST_PATTERN"

‎lighthouse-core/test/devtools-tests/setup.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
3+
##
4+
# @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
5+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
6+
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
7+
##
8+
9+
# Setup dependencies for devtools e2e tests.
10+
# Set SKIP_DOWNLOADS to skip all the downloading and just export variables.
11+
12+
set -euo pipefail
13+
14+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
15+
LH_ROOT="$SCRIPT_DIR/../../.."
16+
TEST_DIR="$LH_ROOT/.tmp/chromium-web-tests"
17+
18+
export DEVTOOLS_PATH=${DEVTOOLS_PATH:-"$TEST_DIR/devtools/devtools-frontend"}
19+
20+
if [ -z ${SKIP_DOWNLOADS+x} ]
21+
then
22+
echo "========================================"
23+
echo "Downloading latest content_shell and DevTools"
24+
echo "To skip this step, set SKIP_DOWNLOADS=1"
25+
echo "========================================"
26+
echo
27+
28+
bash "$SCRIPT_DIR/../chromium-web-tests/download-devtools.sh"
29+
bash "$SCRIPT_DIR/../chromium-web-tests/download-content-shell.sh"
30+
fi

‎lighthouse-core/test/devtools-tests/test-locally.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
3+
##
4+
# @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
5+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
6+
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
7+
##
8+
9+
# Runs the devtools e2e tests in third-party/devtools-tests using the latest
10+
# Lighthouse, DevTools, and Chrome content_shell*
11+
#
12+
# * Not exactly the latest, but close. See download-content-shell.js
13+
14+
set -euo pipefail
15+
16+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
17+
18+
source "$SCRIPT_DIR/setup.sh"
19+
bash "$SCRIPT_DIR/../chromium-web-tests/roll-devtools.sh"
20+
bash "$SCRIPT_DIR/run-e2e-tests.sh" $*

‎third-party/devtools-tests/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Devtools e2e Tests
2+
3+
These tests are rolled into the Chromium DevTools Frontend codebase. They "belong" to the devtools frontend, but are truly defined in this Lighthouse repo.
4+
5+
Run with `sh lighthouse-core/test/devtools-tests/test-locally.sh`.
6+
7+
See `lighthouse-core/test/chromium-web-tests/README.md` for more.
8+
9+
## Sync
10+
11+
```sh
12+
rsync -ahvz --exclude='OWNERS' --exclude='BUILD.gn' ~/src/devtools/devtools-frontend/test/e2e/lighthouse/ third-party/devtools-tests/e2e/lighthouse/
13+
rsync -ahvz --exclude='OWNERS' --exclude='BUILD.gn' ~/src/devtools/devtools-frontend/test/e2e/resources/lighthouse/ third-party/devtools-tests/e2e/resources/lighthouse/
14+
```

‎third-party/devtools-tests/e2e/lighthouse/generate-report_test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright 2020 The Chromium Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import {assert} from 'chai';
6+
7+
import {goToResource} from '../../shared/helper.js';
8+
import {describe, it} from '../../shared/mocha-extensions.js';
9+
import {isGenerateReportButtonDisabled, navigateToLighthouseTab} from '../helpers/lighthouse-helpers.js';
10+
11+
describe('The Lighthouse Tab', function() {
12+
this.timeout(20_000);
13+
14+
it('shows a button to generate a new report', async () => {
15+
await navigateToLighthouseTab('empty.html');
16+
17+
const disabled = await isGenerateReportButtonDisabled();
18+
assert.isFalse(disabled, 'The Generate Report button should not be disabled');
19+
});
20+
21+
// Broken on non-debug runs
22+
it.skip('[crbug.com/1057948] shows generate report button even when navigating to an unreachable page', async () => {
23+
await navigateToLighthouseTab('empty.html');
24+
25+
await goToResource('network/unreachable.rawresponse');
26+
const disabled = await isGenerateReportButtonDisabled();
27+
assert.isTrue(disabled, 'The Generate Report button should be disabled');
28+
});
29+
});

‎third-party/devtools-tests/e2e/lighthouse/indexeddb-warning_test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright 2022 The Chromium Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import {assert} from 'chai';
6+
7+
import {waitFor} from '../../shared/helper.js';
8+
import {describe, it} from '../../shared/mocha-extensions.js';
9+
import {navigateToLighthouseTab} from '../helpers/lighthouse-helpers.js';
10+
11+
describe('IndexedDB warning', function() {
12+
this.timeout(20_000);
13+
14+
it('displays when important data may affect performance', async () => {
15+
await navigateToLighthouseTab('empty.html');
16+
17+
let warningElem = await waitFor('.lighthouse-warning-text.hidden');
18+
const warningText1 = await warningElem.evaluate(node => node.textContent?.trim());
19+
assert.strictEqual(warningText1, '');
20+
21+
await navigateToLighthouseTab('lighthouse/lighthouse-storage.html');
22+
23+
warningElem = await waitFor('.lighthouse-warning-text:not(.hidden)');
24+
const expected =
25+
'There may be stored data affecting loading performance in this location: IndexedDB. Audit this page in an incognito window to prevent those resources from affecting your scores.';
26+
const warningText2 = await warningElem.evaluate(node => node.textContent?.trim());
27+
assert.strictEqual(warningText2, expected);
28+
});
29+
});

0 commit comments

Comments
 (0)