feat(build): Vendor polyfills injected during build (#5051)

Both Rollup and Sucrase add polyfills (similar to those in `tslib`) during the build process, to down-compile newer language features and help ESM and CJS modules play nicely together. Unlike with `tslib`, however, there's no option equivalent to tsc's `importHelpers` option[1], which allows those polyfills to be imports from one central place rather than copies of the full code for each necessary polyfill in every file that uses them. Having all that repeated code is a surefire way to wreck our bundle-size progress, so as part of the switch to using those tools, we will replicate the behavior of `importHelpers` in the context of our new sucrase/rollup builds.

This is the first of two PRs accomplishing that change. In this one, the polyfills we'll use have been added to the repo. In the next one, a rollup plugin will be added to replace the duplicated injected code with import statements, which will then import the polyfills this PR adds. Originally this was one PR, but it has been split into two for easier reviewing.

Notes:

- The polyfills themselves have been added to `@sentry/utils`, in a separate folder under `src/`. Various discussions have been had about where they should live[2], and they have moved around as this PR evolved. Without going into too much boring detail about about the exact pros and cons of each location, suffice it to say that my first instinct was to put them in `utils`, and as I worked on other options, each one of the reasons why I didn't actually do that to begin with (they weren't in TS, they weren't originally ours, I didn't want to pollute the `@sentry/utils` namespace) got solved, and in the end, putting them in `utils` was the simplest and most logical option.
- The added polyfills are copies of the code injected by Sucrase or Rollup, translated into TS and in some cases modified in order to make them less verbose. Since some treeshaking algorithms can only work file by file, each one is in its own file. Polyfills which were modified have had tests added, both to ensure that they do the same thing as the originals, and to ensure that what they do is actually correct.
- As we're not the original authors, attribution has been given within each file to the original source. Further, since the MIT license under which both Rollup and Sucrase are distributed requires duplication of the license blurb, it has been added to the README in the polyfills' folder.
- Because we don't want these to become part of our public API, their code is not exported by `index.ts`, meaning they are not importable directly from `@sentry/utils` and won't come up under intellisense code completions and the like. When our code imports them, it will import from `@sentry/utils/cjs/buildPolyfills` or `@sentry/utils/esm/buildPolyfills` as appropriate.
- Though not every polyfill Rollup and Sucrase can inject ends up being used in our built code, they are all included just to cover our bases. That said, in the interest of time, only the ones we use or are likely to use have had tests written for them. 
- One of the polyfills we do use is the optional chain polyfill. Typing it is tricky, because it takes a variable number of arguments, whose types are effectively `[ A, B, C, B, C, B, C,... ]` (one type for the first argument, and then alternating types for the rest). Because it's not going to be used by the public, the typing in the polyfill itself uses the generic `unknown`. For tests it was easier, because only the test cases needed to be typed, and could have a slightly modified structure: `[ A, [ B, C ][] ]`, which then gets flattened again before being passed to the optional chain polyfill. In order to be able to use `Array.prototype.flat` in tests run on older versions of Node, it itself has been polyfilled, and our testing `tsconfig` target ramped back down when running tests under those old versions.

[1] https://www.typescriptlang.org/tsconfig#importHelpers
[2] #5023 (comment)

Author: lobsterkatie

packages/utils/.eslintrc.js

@@ -1,3 +1,19 @@
 module.exports = {
   extends: ['../../.eslintrc.js'],
+  overrides: [
+    {
+      files: ['scripts/**/*.ts'],
+      parserOptions: {
+        project: ['../../tsconfig.dev.json'],
+      },
+    },
+    {
+      files: ['test/**'],
+      parserOptions: {
+        sourceType: 'module',
+      },
+    },
+  ],
+  // symlinks to the folders inside of `build`, created to simulate what's in the npm package
+  ignorePatterns: ['cjs/**', 'esm/**'],
 };

packages/utils/.gitignore

@@ -0,0 +1,6 @@
+# symlinks to the folders in `build`, needed for tests
+cjs
+esm
+
+# needed so we can test our versions of polyfills against Sucrase and Rollup's originals
+!test/buildPolyfills/originals.d.ts

packages/utils/jest.config.js

@@ -1 +1,9 @@
-module.exports = require('../../jest/jest.config.js');
+const baseConfig = require('../../jest/jest.config.js');
+
+module.exports = {
+  ...baseConfig,
+  transform: {
+    '^.+\\.ts$': 'ts-jest',
+    '^.+\\.js$': 'ts-jest',
+  },
+};

packages/utils/package.json

@@ -20,6 +20,8 @@
     "tslib": "^1.9.3"
   },
   "devDependencies": {
+    "@types/array.prototype.flat": "^1.2.1",
+    "array.prototype.flat": "^1.3.0",
     "chai": "^4.1.2"
   },
   "scripts": {
@@ -28,7 +30,7 @@
     "build:dev": "run-s build",
     "build:es5": "yarn build:cjs # *** backwards compatibility - remove in v7 ***",
     "build:esm": "tsc -p tsconfig.esm.json",
-    "build:rollup": "rollup -c rollup.npm.config.js",
+    "build:rollup": "yarn ts-node scripts/buildRollup.ts",
     "build:types": "tsc -p tsconfig.types.json",
     "build:watch": "run-p build:cjs:watch build:esm:watch build:types:watch",
     "build:cjs:watch": "tsc -p tsconfig.cjs.json --watch",
@@ -39,7 +41,7 @@
     "build:types:watch": "tsc -p tsconfig.types.json --watch",
     "build:npm": "ts-node ../../scripts/prepack.ts && npm pack ./build",
     "circularDepCheck": "madge --circular src/index.ts",
-    "clean": "rimraf build coverage",
+    "clean": "rimraf build coverage cjs esm",
     "fix": "run-s fix:eslint fix:prettier",
     "fix:eslint": "eslint . --format stylish --fix",
     "fix:prettier": "prettier --write \"{src,test,scripts}/**/*.ts\"",

packages/utils/rollup.npm.config.js

@@ -1,3 +1,9 @@
 import { makeBaseNPMConfig, makeNPMConfigVariants } from '../../rollup/index.js';
 
-export default makeNPMConfigVariants(makeBaseNPMConfig());
+export default makeNPMConfigVariants(
+  makeBaseNPMConfig({
+    // We build the polyfills separately because they're not included in the top-level exports of the package, in order
+    // to keep them out of the public API.
+    entrypoints: ['src/index.ts', 'src/buildPolyfills/index.ts'],
+  }),
+);

packages/utils/scripts/buildRollup.ts

@@ -0,0 +1,30 @@
+import * as childProcess from 'child_process';
+import * as fs from 'fs';
+
+/**
+ * Run the given shell command, piping the shell process's `stdin`, `stdout`, and `stderr` to that of the current
+ * process. Returns contents of `stdout`.
+ */
+function run(cmd: string, options?: childProcess.ExecSyncOptions): string | Buffer {
+  return childProcess.execSync(cmd, { stdio: 'inherit', ...options });
+}
+
+run('yarn rollup -c rollup.npm.config.js');
+
+// We want to distribute the README because it contains the MIT license blurb from Sucrase and Rollup
+fs.copyFileSync('src/buildPolyfills/README.md', 'build/cjs/buildPolyfills/README.md');
+fs.copyFileSync('src/buildPolyfills/README.md', 'build/esm/buildPolyfills/README.md');
+
+// Because we import our polyfills from `@sentry/utils/cjs/buildPolyfills` and `@sentry/utils/esm/buildPolyfills` rather
+// than straight from `@sentry/utils` (so as to avoid having them in the package's public API), when tests run, they'll
+// expect to find `cjs` and `esm` at the root level of the repo.
+try {
+  fs.symlinkSync('build/cjs', 'cjs');
+} catch (oO) {
+  // if we get here, it's because the symlink already exists, so we're good
+}
+try {
+  fs.symlinkSync('build/esm', 'esm');
+} catch (oO) {
+  // same as above
+}

packages/utils/src/buildPolyfills/README.md

@@ -0,0 +1,15 @@
+## Build Polyfills
+
+This is a collection of syntax and import/export polyfills either copied directly from or heavily inspired by those used by [Rollup](https://github.com/rollup/rollup) and [Sucrase](https://github.com/alangpierce/sucrase). When either tool uses one of these polyfills during a build, it injects the function source code into each file needing the function, which can lead to a great deal of duplication. For our builds, we have therefore implemented something similar to [`tsc`'s `importHelpers` behavior](https://www.typescriptlang.org/tsconfig#importHelpers): Instead of leaving the polyfills injected in multiple places, we instead replace each injected function with an `import` or `require` statement, pulling from the CJS or ESM builds as appropriate. (In other words, the injected `import` statements import from `@sentry/utils/esm/buildPolyfills` and the injected `require` statements pull from `@sentry/utils/cjs/buildPolyfills/`. Because these functions should never be part of the public API, they're not exported from the package directly.)
+
+Note that not all polyfills are currently used by the SDK, but all are included here for future compatitibility, should they ever be needed. Also, since we're never going to be calling these directly from within another TS file, their types are fairly generic. In some cases testing required more specific types, which can be found in the test files.
+
+--------
+
+_Code from both Rollup and Sucrase is used under the MIT license, copyright 2017 and 2012-2018, respectively._
+
+_Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:_
+
+_The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software._
+
+_THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE._

packages/utils/src/buildPolyfills/_asyncNullishCoalesce.ts

@@ -0,0 +1,30 @@
+// adapted from Sucrase (https://github.com/alangpierce/sucrase)
+
+import { _nullishCoalesce } from './_nullishCoalesce';
+
+/**
+ * Polyfill for the nullish coalescing operator (`??`), when used in situations where at least one of the values is the
+ * result of an async operation.
+ *
+ * Note that the RHS is wrapped in a function so that if it's a computed value, that evaluation won't happen unless the
+ * LHS evaluates to a nullish value, to mimic the operator's short-circuiting behavior.
+ *
+ * Adapted from Sucrase (https://github.com/alangpierce/sucrase)
+ *
+ * @param lhs The value of the expression to the left of the `??`
+ * @param rhsFn A function returning the value of the expression to the right of the `??`
+ * @returns The LHS value, unless it's `null` or `undefined`, in which case, the RHS value
+ */
+// eslint-disable-next-line @sentry-internal/sdk/no-async-await
+export async function _asyncNullishCoalesce(lhs: unknown, rhsFn: () => unknown): Promise<unknown> {
+  return _nullishCoalesce(lhs, rhsFn);
+}
+
+// Sucrase version:
+// async function _asyncNullishCoalesce(lhs, rhsFn) {
+//   if (lhs != null) {
+//     return lhs;
+//   } else {
+//     return await rhsFn();
+//   }
+// }

packages/utils/src/buildPolyfills/_asyncOptionalChain.ts

@@ -0,0 +1,59 @@
+import { GenericFunction } from './types';
+
+/**
+ * Polyfill for the optional chain operator, `?.`, given previous conversion of the expression into an array of values,
+ * descriptors, and functions, for situations in which at least one part of the expression is async.
+ *
+ * Adapted from Sucrase (https://github.com/alangpierce/sucrase) See
+ * https://github.com/alangpierce/sucrase/blob/265887868966917f3b924ce38dfad01fbab1329f/src/transformers/OptionalChainingNullishTransformer.ts#L15
+ *
+ * @param ops Array result of expression conversion
+ * @returns The value of the expression
+ */
+// eslint-disable-next-line @sentry-internal/sdk/no-async-await
+export async function _asyncOptionalChain(ops: unknown[]): Promise<unknown> {
+  let lastAccessLHS: unknown = undefined;
+  let value = ops[0];
+  let i = 1;
+  while (i < ops.length) {
+    const op = ops[i] as string;
+    const fn = ops[i + 1] as (intermediateValue: unknown) => Promise<unknown>;
+    i += 2;
+    // by checking for loose equality to `null`, we catch both `null` and `undefined`
+    if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
+      // really we're meaning to return `undefined` as an actual value here, but it saves bytes not to write it
+      return;
+    }
+    if (op === 'access' || op === 'optionalAccess') {
+      lastAccessLHS = value;
+      value = await fn(value);
+    } else if (op === 'call' || op === 'optionalCall') {
+      value = await fn((...args: unknown[]) => (value as GenericFunction).call(lastAccessLHS, ...args));
+      lastAccessLHS = undefined;
+    }
+  }
+  return value;
+}
+
+// Sucrase version:
+// async function _asyncOptionalChain(ops) {
+//   let lastAccessLHS = undefined;
+//   let value = ops[0];
+//   let i = 1;
+//   while (i < ops.length) {
+//     const op = ops[i];
+//     const fn = ops[i + 1];
+//     i += 2;
+//     if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
+//       return undefined;
+//     }
+//     if (op === 'access' || op === 'optionalAccess') {
+//       lastAccessLHS = value;
+//       value = await fn(value);
+//     } else if (op === 'call' || op === 'optionalCall') {
+//       value = await fn((...args) => value.call(lastAccessLHS, ...args));
+//       lastAccessLHS = undefined;
+//     }
+//   }
+//   return value;
+// }

packages/utils/src/buildPolyfills/_asyncOptionalChainDelete.ts

@@ -0,0 +1,28 @@
+import { _asyncOptionalChain } from './_asyncOptionalChain';
+
+/**
+ * Polyfill for the optional chain operator, `?.`, given previous conversion of the expression into an array of values,
+ * descriptors, and functions, in cases where the value of the expression is to be deleted.
+ *
+ * Adapted from Sucrase (https://github.com/alangpierce/sucrase) See
+ * https://github.com/alangpierce/sucrase/blob/265887868966917f3b924ce38dfad01fbab1329f/src/transformers/OptionalChainingNullishTransformer.ts#L15
+ *
+ * @param ops Array result of expression conversion
+ * @returns The return value of the `delete` operator: `true`, unless the deletion target is an own, non-configurable
+ * property (one which can't be deleted or turned into an accessor, and whose enumerability can't be changed), in which
+ * case `false`.
+ */
+// eslint-disable-next-line @sentry-internal/sdk/no-async-await
+export async function _asyncOptionalChainDelete(ops: unknown[]): Promise<boolean> {
+  const result = (await _asyncOptionalChain(ops)) as Promise<boolean | null>;
+  // If `result` is `null`, it means we didn't get to the end of the chain and so nothing was deleted (in which case,
+  // return `true` since that's what `delete` does when it no-ops). If it's non-null, we know the delete happened, in
+  // which case we return whatever the `delete` returned, which will be a boolean.
+  return result == null ? true : (result as Promise<boolean>);
+}
+
+// Sucrase version:
+// async function asyncOptionalChainDelete(ops) {
+//   const result = await ASYNC_OPTIONAL_CHAIN_NAME(ops);
+//   return result == null ? true : result;
+// }

packages/utils/src/buildPolyfills/_createNamedExportFrom.ts

@@ -0,0 +1,21 @@
+import { GenericObject } from './types';
+
+declare const exports: GenericObject;
+
+/**
+ * Copy a property from the given object into `exports`, under the given name.
+ *
+ * Adapted from Sucrase (https://github.com/alangpierce/sucrase)
+ *
+ * @param obj The object containing the property to copy.
+ * @param localName The name under which to export the property
+ * @param importedName The name under which the property lives in `obj`
+ */
+export function _createNamedExportFrom(obj: GenericObject, localName: string, importedName: string): void {
+  exports[localName] = obj[importedName];
+}
+
+// Sucrase version:
+// function _createNamedExportFrom(obj, localName, importedName) {
+//   Object.defineProperty(exports, localName, {enumerable: true, get: () => obj[importedName]});
+// }

packages/utils/src/buildPolyfills/_createStarExport.ts

@@ -0,0 +1,28 @@
+import { GenericObject } from './types';
+
+declare const exports: GenericObject;
+
+/**
+ * Copy properties from an object into `exports`.
+ *
+ * Adapted from Sucrase (https://github.com/alangpierce/sucrase)
+ *
+ * @param obj The object containing the properties to copy.
+ */
+export function _createStarExport(obj: GenericObject): void {
+  Object.keys(obj)
+    .filter(key => key !== 'default' && key !== '__esModule' && !(key in exports))
+    .forEach(key => (exports[key] = obj[key]));
+}
+
+// Sucrase version:
+// function _createStarExport(obj) {
+//   Object.keys(obj)
+//     .filter(key => key !== 'default' && key !== '__esModule')
+//     .forEach(key => {
+//       if (exports.hasOwnProperty(key)) {
+//         return;
+//       }
+//       Object.defineProperty(exports, key, { enumerable: true, get: () => obj[key] });
+//     });
+// }

packages/utils/src/buildPolyfills/_interopDefault.ts

@@ -0,0 +1,18 @@
+import { RequireResult } from './types';
+
+/**
+ * Unwraps a module if it has been wrapped in an object under the key `default`.
+ *
+ * Adapted from Rollup (https://github.com/rollup/rollup)
+ *
+ * @param requireResult The result of calling `require` on a module
+ * @returns The full module, unwrapped if necessary.
+ */
+export function _interopDefault(requireResult: RequireResult): RequireResult {
+  return requireResult.__esModule ? (requireResult.default as RequireResult) : requireResult;
+}
+
+// Rollup version:
+// function _interopDefault(e) {
+//   return e && e.__esModule ? e['default'] : e;
+// }

packages/utils/src/buildPolyfills/_interopNamespace.ts

@@ -0,0 +1,26 @@
+import { RequireResult } from './types';
+
+/**
+ * Adds a self-referential `default` property to CJS modules which aren't the result of transpilation from ESM modules.
+ *
+ * Adapted from Rollup (https://github.com/rollup/rollup)
+ *
+ * @param requireResult The result of calling `require` on a module
+ * @returns Either `requireResult` or a copy of `requireResult` with an added self-referential `default` property
+ */
+export function _interopNamespace(requireResult: RequireResult): RequireResult {
+  return requireResult.__esModule ? requireResult : { ...requireResult, default: requireResult };
+}
+
+// Rollup version (with `output.externalLiveBindings` and `output.freeze` both set to false)
+// function _interopNamespace(e) {
+//   if (e && e.__esModule) return e;
+//   var n = Object.create(null);
+//   if (e) {
+//     for (var k in e) {
+//       n[k] = e[k];
+//     }
+//   }
+//   n["default"] = e;
+//   return n;
+// }

packages/utils/src/buildPolyfills/_interopNamespaceDefaultOnly.ts

@@ -0,0 +1,24 @@
+import { RequireResult } from './types';
+
+/**
+ * Wrap a module in an object, as the value under the key `default`.
+ *
+ * Adapted from Rollup (https://github.com/rollup/rollup)
+ *
+ * @param requireResult The result of calling `require` on a module
+ * @returns An object containing the key-value pair (`default`, `requireResult`)
+ */
+export function _interopNamespaceDefaultOnly(requireResult: RequireResult): RequireResult {
+  return {
+    __proto__: null,
+    default: requireResult,
+  };
+}
+
+// Rollup version
+// function _interopNamespaceDefaultOnly(e) {
+//   return {
+//     __proto__: null,
+//     'default': e
+//   };
+// }

packages/utils/src/buildPolyfills/_interopRequireDefault.ts

@@ -0,0 +1,18 @@
+import { RequireResult } from './types';
+
+/**
+ * Wraps modules which aren't the result of transpiling an ESM module in an object under the key `default`
+ *
+ * Adapted from Sucrase (https://github.com/alangpierce/sucrase)
+ *
+ * @param requireResult The result of calling `require` on a module
+ * @returns `requireResult` or `requireResult` wrapped in an object, keyed as `default`
+ */
+export function _interopRequireDefault(requireResult: RequireResult): RequireResult {
+  return requireResult.__esModule ? requireResult : { default: requireResult };
+}
+
+// Sucrase version
+// function _interopRequireDefault(obj) {
+//   return obj && obj.__esModule ? obj : { default: obj };
+// }