diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 000000000..42d7d4dc6 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,24 @@ +name: Performance + +on: [pull_request] + +jobs: + perf: + runs-on: Ubuntu-18.04 + steps: + - name: Checkout + uses: actions/checkout@master + with: + fetch-depth: 1 + - name: Run Benchmark + run: | + git clone https://github.com/kylef/swiftenv.git ~/.swiftenv + export SWIFTENV_ROOT="$HOME/.swiftenv" + export PATH="$SWIFTENV_ROOT/bin:$PATH" + eval "$(swiftenv init -)" + swiftenv install $TOOLCHAIN_DOWNLOAD + make perf-tester + node ci/perf-tester + env: + TOOLCHAIN_DOWNLOAD: https://github.com/swiftwasm/swift/releases/download/swift-wasm-5.3-SNAPSHOT-2020-08-10-a/swift-wasm-5.3-SNAPSHOT-2020-08-10-a-linux.tar.gz + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index 44b00a627..31000f933 100644 --- a/Makefile +++ b/Makefile @@ -22,3 +22,7 @@ benchmark_setup: .PHONY: run_benchmark run_benchmark: cd IntegrationTests && make -s run_benchmark + +.PHONY: perf-tester +perf-tester: + cd ci/perf-tester && npm install diff --git a/ci/perf-tester/package-lock.json b/ci/perf-tester/package-lock.json new file mode 100644 index 000000000..d3dbac41f --- /dev/null +++ b/ci/perf-tester/package-lock.json @@ -0,0 +1,376 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "@actions/core": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.2.tgz", + "integrity": "sha512-IbCx7oefq+Gi6FWbSs2Fnw8VkEI6Y4gvjrYprY3RV//ksq/KPMlClOerJ4jRosyal6zkUIc8R9fS/cpRMlGClg==", + "dev": true + }, + "@actions/exec": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.3.tgz", + "integrity": "sha512-TogJGnueOmM7ntCi0ASTUj4LapRRtDfj57Ja4IhPmg2fls28uVOPbAn8N+JifaOumN2UG3oEO/Ixek2A4NcYSA==", + "dev": true, + "requires": { + "@actions/io": "^1.0.1" + } + }, + "@actions/github": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-2.0.1.tgz", + "integrity": "sha512-C7dAsCkpPi1HxTzLldz+oY+9c5G+nnaK7xgk8KA83VVGlrGK7d603E3snUAFocWrqEu/uvdYD82ytggjcpYSQA==", + "dev": true, + "requires": { + "@octokit/graphql": "^4.3.1", + "@octokit/rest": "^16.15.0" + } + }, + "@actions/io": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz", + "integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg==", + "dev": true + }, + "@octokit/endpoint": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-5.5.1.tgz", + "integrity": "sha512-nBFhRUb5YzVTCX/iAK1MgQ4uWo89Gu0TH00qQHoYRCsE12dWcG1OiLd7v2EIo2+tpUKPMOQ62QFy9hy9Vg2ULg==", + "dev": true, + "requires": { + "@octokit/types": "^2.0.0", + "is-plain-object": "^3.0.0", + "universal-user-agent": "^4.0.0" + } + }, + "@octokit/graphql": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.3.1.tgz", + "integrity": "sha512-hCdTjfvrK+ilU2keAdqNBWOk+gm1kai1ZcdjRfB30oA3/T6n53UVJb7w0L5cR3/rhU91xT3HSqCd+qbvH06yxA==", + "dev": true, + "requires": { + "@octokit/request": "^5.3.0", + "@octokit/types": "^2.0.0", + "universal-user-agent": "^4.0.0" + } + }, + "@octokit/request": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.3.1.tgz", + "integrity": "sha512-5/X0AL1ZgoU32fAepTfEoggFinO3rxsMLtzhlUX+RctLrusn/CApJuGFCd0v7GMFhF+8UiCsTTfsu7Fh1HnEJg==", + "dev": true, + "requires": { + "@octokit/endpoint": "^5.5.0", + "@octokit/request-error": "^1.0.1", + "@octokit/types": "^2.0.0", + "deprecation": "^2.0.0", + "is-plain-object": "^3.0.0", + "node-fetch": "^2.3.0", + "once": "^1.4.0", + "universal-user-agent": "^4.0.0" + } + }, + "@octokit/request-error": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-1.2.0.tgz", + "integrity": "sha512-DNBhROBYjjV/I9n7A8kVkmQNkqFAMem90dSxqvPq57e2hBr7mNTX98y3R2zDpqMQHVRpBDjsvsfIGgBzy+4PAg==", + "dev": true, + "requires": { + "@octokit/types": "^2.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/rest": { + "version": "16.37.0", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-16.37.0.tgz", + "integrity": "sha512-qLPK9FOCK4iVpn6ghknNuv/gDDxXQG6+JBQvoCwWjQESyis9uemakjzN36nvvp8SCny7JuzHI2RV8ChbV5mYdQ==", + "dev": true, + "requires": { + "@octokit/request": "^5.2.0", + "@octokit/request-error": "^1.0.2", + "atob-lite": "^2.0.0", + "before-after-hook": "^2.0.0", + "btoa-lite": "^1.0.0", + "deprecation": "^2.0.0", + "lodash.get": "^4.4.2", + "lodash.set": "^4.3.2", + "lodash.uniq": "^4.5.0", + "octokit-pagination-methods": "^1.1.0", + "once": "^1.4.0", + "universal-user-agent": "^4.0.0" + } + }, + "@octokit/types": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-2.1.0.tgz", + "integrity": "sha512-n1GUYFgKm5glcy0E+U5jnqAFY2p04rnK4A0YhuM70C7Vm9Vyx+xYwd/WOTEr8nUJcbPSR/XL+/26+rirY6jJQA==", + "dev": true, + "requires": { + "@types/node": ">= 8" + } + }, + "@types/node": { + "version": "13.1.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.8.tgz", + "integrity": "sha512-6XzyyNM9EKQW4HKuzbo/CkOIjn/evtCmsU+MUM1xDfJ+3/rNjBttM1NgN7AOQvN6tP1Sl1D1PIKMreTArnxM9A==", + "dev": true + }, + "atob-lite": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/atob-lite/-/atob-lite-2.0.0.tgz", + "integrity": "sha1-D+9a1G8b16hQLGVyfwNn1e5D1pY=", + "dev": true + }, + "before-after-hook": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.1.0.tgz", + "integrity": "sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A==", + "dev": true + }, + "btoa-lite": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz", + "integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc=", + "dev": true + }, + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "requires": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==", + "dev": true + }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, + "requires": { + "once": "^1.4.0" + } + }, + "execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "dev": true, + "requires": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + } + }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, + "is-plain-object": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-3.0.0.tgz", + "integrity": "sha512-tZIpofR+P05k8Aocp7UI/2UTa9lTJSebCXpFFoR9aibpokDj/uXBsJ8luUu0tTVYKkMU6URDUuOfJZ7koewXvg==", + "dev": true, + "requires": { + "isobject": "^4.0.0" + } + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", + "dev": true + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "isobject": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-4.0.0.tgz", + "integrity": "sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA==", + "dev": true + }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", + "dev": true + }, + "lodash.set": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", + "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=", + "dev": true + }, + "lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=", + "dev": true + }, + "macos-release": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.3.0.tgz", + "integrity": "sha512-OHhSbtcviqMPt7yfw5ef5aghS2jzFVKEFyCJndQt2YpSQ9qRVSEv2axSJI1paVThEu+FFGs584h/1YhxjVqajA==", + "dev": true + }, + "nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", + "dev": true + }, + "npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "dev": true, + "requires": { + "path-key": "^2.0.0" + } + }, + "octokit-pagination-methods": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/octokit-pagination-methods/-/octokit-pagination-methods-1.1.0.tgz", + "integrity": "sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ==", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "os-name": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/os-name/-/os-name-3.1.0.tgz", + "integrity": "sha512-h8L+8aNjNcMpo/mAIBPn5PXCM16iyPGjHNWo6U1YO8sJTMHtEtyczI6QJnLoplswm6goopQkqc7OAnjhWcugVg==", + "dev": true, + "requires": { + "macos-release": "^2.2.0", + "windows-release": "^3.1.0" + } + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "dev": true + }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "dev": true + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dev": true, + "requires": { + "shebang-regex": "^1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "dev": true + }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "dev": true + }, + "strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", + "dev": true + }, + "universal-user-agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-4.0.0.tgz", + "integrity": "sha512-eM8knLpev67iBDizr/YtqkJsF3GK8gzDc6st/WKzrTuPtcsOKW/0IdL4cnMBsU69pOx0otavLWBDGTwg+dB0aA==", + "dev": true, + "requires": { + "os-name": "^3.1.0" + } + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "windows-release": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/windows-release/-/windows-release-3.2.0.tgz", + "integrity": "sha512-QTlz2hKLrdqukrsapKsINzqMgOUpQW268eJ0OaOpJN32h272waxR9fkB9VoWRtK7uKHG5EHJcTXQBD8XZVJkFA==", + "dev": true, + "requires": { + "execa": "^1.0.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + } + } +} diff --git a/ci/perf-tester/package.json b/ci/perf-tester/package.json new file mode 100644 index 000000000..c1efb42a7 --- /dev/null +++ b/ci/perf-tester/package.json @@ -0,0 +1,9 @@ +{ + "private": true, + "main": "src/index.js", + "devDependencies": { + "@actions/core": "^1.2.2", + "@actions/exec": "^1.0.3", + "@actions/github": "^2.0.1" + } +} diff --git a/ci/perf-tester/src/index.js b/ci/perf-tester/src/index.js new file mode 100644 index 000000000..e2427305a --- /dev/null +++ b/ci/perf-tester/src/index.js @@ -0,0 +1,225 @@ +/* +Adapted from preactjs/compressed-size-action, which is available under this license: + +MIT License +Copyright (c) 2020 Preact +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +const { setFailed, startGroup, endGroup, debug } = require('@actions/core'); +const { GitHub, context } = require('@actions/github'); +const { exec } = require('@actions/exec'); +const { getInput, runBenchmark, averageBenchmarks, toDiff, diffTable, toBool } = require('./utils.js'); + +async function run(octokit, context, token) { + const { number: pull_number } = context.issue; + + const pr = context.payload.pull_request; + try { + debug('pr' + JSON.stringify(pr, null, 2)); + } catch (e) { } + if (!pr) { + throw Error('Could not retrieve PR information. Only "pull_request" triggered workflows are currently supported.'); + } + + console.log(`PR #${pull_number} is targetted at ${pr.base.ref} (${pr.base.sha})`); + + const buildScript = getInput('build-script'); + startGroup(`[current] Build using '${buildScript}'`); + await exec(buildScript); + endGroup(); + + startGroup(`[current] Running benchmark`); + const newBenchmarks = await Promise.all([runBenchmark(), runBenchmark()]).then(averageBenchmarks); + endGroup(); + + startGroup(`[base] Checkout target branch`); + let baseRef; + try { + baseRef = context.payload.base.ref; + if (!baseRef) throw Error('missing context.payload.pull_request.base.ref'); + await exec(`git fetch -n origin ${context.payload.pull_request.base.ref}`); + console.log('successfully fetched base.ref'); + } catch (e) { + console.log('fetching base.ref failed', e.message); + try { + await exec(`git fetch -n origin ${pr.base.sha}`); + console.log('successfully fetched base.sha'); + } catch (e) { + console.log('fetching base.sha failed', e.message); + try { + await exec(`git fetch -n`); + } catch (e) { + console.log('fetch failed', e.message); + } + } + } + + console.log('checking out and building base commit'); + try { + if (!baseRef) throw Error('missing context.payload.base.ref'); + await exec(`git reset --hard ${baseRef}`); + } + catch (e) { + await exec(`git reset --hard ${pr.base.sha}`); + } + endGroup(); + + startGroup(`[base] Build using '${buildScript}'`); + await exec(buildScript); + endGroup(); + + startGroup(`[base] Running benchmark`); + const oldBenchmarks = await Promise.all([runBenchmark(), runBenchmark()]).then(averageBenchmarks); + endGroup(); + + const diff = toDiff(oldBenchmarks, newBenchmarks); + + const markdownDiff = diffTable(diff, { + collapseUnchanged: true, + omitUnchanged: false, + showTotal: true, + minimumChangeThreshold: parseInt(getInput('minimum-change-threshold'), 10) + }); + + let outputRawMarkdown = false; + + const commentInfo = { + ...context.repo, + issue_number: pull_number + }; + + const comment = { + ...commentInfo, + body: markdownDiff + '\n\nperformance-action' + }; + + if (toBool(getInput('use-check'))) { + if (token) { + const finish = await createCheck(octokit, context); + await finish({ + conclusion: 'success', + output: { + title: `Compressed Size Action`, + summary: markdownDiff + } + }); + } + else { + outputRawMarkdown = true; + } + } + else { + startGroup(`Updating stats PR comment`); + let commentId; + try { + const comments = (await octokit.issues.listComments(commentInfo)).data; + for (let i = comments.length; i--;) { + const c = comments[i]; + if (c.user.type === 'Bot' && /[\s\n]*performance-action/.test(c.body)) { + commentId = c.id; + break; + } + } + } + catch (e) { + console.log('Error checking for previous comments: ' + e.message); + } + + if (commentId) { + console.log(`Updating previous comment #${commentId}`) + try { + await octokit.issues.updateComment({ + ...context.repo, + comment_id: commentId, + body: comment.body + }); + } + catch (e) { + console.log('Error editing previous comment: ' + e.message); + commentId = null; + } + } + + // no previous or edit failed + if (!commentId) { + console.log('Creating new comment'); + try { + await octokit.issues.createComment(comment); + } catch (e) { + console.log(`Error creating comment: ${e.message}`); + console.log(`Submitting a PR review comment instead...`); + try { + const issue = context.issue || pr; + await octokit.pulls.createReview({ + owner: issue.owner, + repo: issue.repo, + pull_number: issue.number, + event: 'COMMENT', + body: comment.body + }); + } catch (e) { + console.log('Error creating PR review.'); + outputRawMarkdown = true; + } + } + } + endGroup(); + } + + if (outputRawMarkdown) { + console.log(` + Error: performance-action was unable to comment on your PR. + This can happen for PR's originating from a fork without write permissions. + You can copy the size table directly into a comment using the markdown below: + \n\n${comment.body}\n\n + `.replace(/^(\t| )+/gm, '')); + } + + console.log('All done!'); +} + + +// create a check and return a function that updates (completes) it +async function createCheck(octokit, context) { + const check = await octokit.checks.create({ + ...context.repo, + name: 'Compressed Size', + head_sha: context.payload.pull_request.head.sha, + status: 'in_progress', + }); + + return async details => { + await octokit.checks.update({ + ...context.repo, + check_run_id: check.data.id, + completed_at: new Date().toISOString(), + status: 'completed', + ...details + }); + }; +} + +(async () => { + try { + const token = getInput('repo-token', { required: true }); + const octokit = new GitHub(token); + await run(octokit, context, token); + } catch (e) { + setFailed(e.message); + } +})(); diff --git a/ci/perf-tester/src/utils.js b/ci/perf-tester/src/utils.js new file mode 100644 index 000000000..b92bdcd44 --- /dev/null +++ b/ci/perf-tester/src/utils.js @@ -0,0 +1,211 @@ +/* +Adapted from preactjs/compressed-size-action, which is available under this license: + +MIT License +Copyright (c) 2020 Preact +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +const fs = require('fs'); +const { exec } = require('@actions/exec'); + +const getInput = key => ({ + 'build-script': 'make bootstrap benchmark_setup', + benchmark: 'make -s run_benchmark', + 'minimum-change-threshold': 5, + 'use-check': 'no', + 'repo-token': process.env.GITHUB_TOKEN +})[key] +exports.getInput = getInput + +exports.runBenchmark = async () => { + let benchmarkBuffers = [] + await exec(getInput('benchmark'), [], { + listeners: { + stdout: data => benchmarkBuffers.push(data) + } + }); + const output = Buffer.concat(benchmarkBuffers).toString('utf8') + return parse(output) +} + +const firstLineRe = /^Running '(.+)' \.\.\.$/ +const secondLineRe = /^done ([\d.]+) ms$/ + +function parse(benchmarkData) { + const lines = benchmarkData.trim().split('\n') + const benchmarks = Object.create(null) + for (let i = 0; i < lines.length - 1; i += 2) { + const [, name] = firstLineRe.exec(lines[i]) + const [, time] = secondLineRe.exec(lines[i + 1]) + benchmarks[name] = Math.round(parseFloat(time)) + } + return benchmarks +} + +exports.averageBenchmarks = (benchmarks) => { + const result = Object.create(null) + for (const key of Object.keys(benchmarks[0])) { + result[key] = benchmarks.reduce((acc, bench) => acc + bench[key], 0) / benchmarks.length + } + return result +} + +/** + * @param {{[key: string]: number}} before + * @param {{[key: string]: number}} after + * @return {Diff[]} + */ +exports.toDiff = (before, after) => { + const names = [...new Set([...Object.keys(before), ...Object.keys(after)])] + return names.map(name => { + const timeBefore = before[name] || 0 + const timeAfter = after[name] || 0 + const delta = timeAfter - timeBefore + return { name, time: timeAfter, delta } + }) +} + + +/** + * @param {number} delta + * @param {number} difference + */ +function getDeltaText(delta, difference) { + let deltaText = (delta > 0 ? '+' : '') + delta.toLocaleString('en-US') + 'ms'; + if (delta && Math.abs(delta) > 1) { + deltaText += ` (${Math.abs(difference)}%)`; + } + return deltaText; +} + +/** + * @param {number} difference + */ +function iconForDifference(difference) { + let icon = ''; + if (difference >= 50) icon = '🆘'; + else if (difference >= 20) icon = '🚨'; + else if (difference >= 10) icon = '⚠ī¸'; + else if (difference >= 5) icon = '🔍'; + else if (difference <= -50) icon = '🏆'; + else if (difference <= -20) icon = '🎉'; + else if (difference <= -10) icon = '👏'; + else if (difference <= -5) icon = '✅'; + return icon; +} + +/** + * Create a Markdown table from text rows + * @param {string[]} rows + */ +function markdownTable(rows) { + if (rows.length == 0) { + return ''; + } + + // Skip all empty columns + while (rows.every(columns => !columns[columns.length - 1])) { + for (const columns of rows) { + columns.pop(); + } + } + + const [firstRow] = rows; + const columnLength = firstRow.length; + if (columnLength === 0) { + return ''; + } + + return [ + // Header + ['Test name', 'Duration', 'Change', ''].slice(0, columnLength), + // Align + [':---', ':---:', ':---:', ':---:'].slice(0, columnLength), + // Body + ...rows + ].map(columns => `| ${columns.join(' | ')} |`).join('\n'); +} + +/** + * @typedef {Object} Diff + * @property {string} name + * @property {number} time + * @property {number} delta + */ + +/** + * Create a Markdown table showing diff data + * @param {Diff[]} tests + * @param {object} options + * @param {boolean} [options.showTotal] + * @param {boolean} [options.collapseUnchanged] + * @param {boolean} [options.omitUnchanged] + * @param {number} [options.minimumChangeThreshold] + */ +exports.diffTable = (tests, { showTotal, collapseUnchanged, omitUnchanged, minimumChangeThreshold }) => { + let changedRows = []; + let unChangedRows = []; + + let totalTime = 0; + let totalDelta = 0; + for (const file of tests) { + const { name, time, delta } = file; + totalTime += time; + totalDelta += delta; + + const difference = ((delta / time) * 100) | 0; + const isUnchanged = Math.abs(difference) < minimumChangeThreshold; + + if (isUnchanged && omitUnchanged) continue; + + const columns = [ + name, + time.toLocaleString('en-US') + 'ms', + getDeltaText(delta, difference), + iconForDifference(difference) + ]; + if (isUnchanged && collapseUnchanged) { + unChangedRows.push(columns); + } else { + changedRows.push(columns); + } + } + + let out = markdownTable(changedRows); + + if (unChangedRows.length !== 0) { + const outUnchanged = markdownTable(unChangedRows); + out += `\n\n
ℹī¸ View Unchanged\n\n${outUnchanged}\n\n
\n\n`; + } + + if (showTotal) { + const totalDifference = ((totalDelta / totalTime) * 100) | 0; + let totalDeltaText = getDeltaText(totalDelta, totalDifference); + let totalIcon = iconForDifference(totalDifference); + out = `**Total Time:** ${totalTime.toLocaleString('en-US')}ms\n\n${out}`; + out = `**Time Change:** ${totalDeltaText} ${totalIcon}\n\n${out}`; + } + + return out; +} + +/** + * Convert a string "true"/"yes"/"1" argument value to a boolean + * @param {string} v + */ +exports.toBool = v => /^(1|true|yes)$/.test(v);