diff --git a/packages/angular_devkit/build_angular/package.json b/packages/angular_devkit/build_angular/package.json index 2263c8df4789..0327d041e664 100644 --- a/packages/angular_devkit/build_angular/package.json +++ b/packages/angular_devkit/build_angular/package.json @@ -27,6 +27,7 @@ "find-cache-dir": "3.0.0", "glob": "7.1.4", "istanbul-instrumenter-loader": "3.0.1", + "jest-worker": "24.9.0", "karma-source-map-support": "1.4.0", "less": "3.9.0", "less-loader": "5.0.0", @@ -61,7 +62,6 @@ "webpack-merge": "4.2.1", "webpack-sources": "1.4.3", "webpack-subresource-integrity": "1.1.0-rc.6", - "worker-farm": "1.7.0", "worker-plugin": "3.2.0" }, "devDependencies": { diff --git a/packages/angular_devkit/build_angular/src/browser/action-executor.ts b/packages/angular_devkit/build_angular/src/browser/action-executor.ts new file mode 100644 index 000000000000..29f9d7a39b89 --- /dev/null +++ b/packages/angular_devkit/build_angular/src/browser/action-executor.ts @@ -0,0 +1,53 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import JestWorker from 'jest-worker'; +import * as os from 'os'; + +export class ActionExecutor { + private largeWorker: JestWorker; + private smallWorker: JestWorker; + + private smallThreshold = 32 * 1024; + + constructor(actionFile: string, private readonly actionName: string) { + // larger files are processed in a separate process to limit memory usage in the main process + this.largeWorker = new JestWorker(actionFile, { + exposedMethods: [actionName], + }); + + // small files are processed in a limited number of threads to improve speed + // The limited number also prevents a large increase in memory usage for an otherwise short operation + this.smallWorker = new JestWorker(actionFile, { + exposedMethods: [actionName], + numWorkers: os.cpus().length < 2 ? 1 : 2, + // Will automatically fallback to processes if not supported + enableWorkerThreads: true, + }); + } + + execute(options: Input): Promise { + if (options.size > this.smallThreshold) { + return ((this.largeWorker as unknown) as Record Promise>)[ + this.actionName + ](options); + } else { + return ((this.smallWorker as unknown) as Record Promise>)[ + this.actionName + ](options); + } + } + + executeAll(options: Input[]): Promise { + return Promise.all(options.map(o => this.execute(o))); + } + + stop() { + this.largeWorker.end(); + this.smallWorker.end(); + } +} diff --git a/packages/angular_devkit/build_angular/src/browser/index.ts b/packages/angular_devkit/build_angular/src/browser/index.ts index 8394167c8d9d..82ef60250734 100644 --- a/packages/angular_devkit/build_angular/src/browser/index.ts +++ b/packages/angular_devkit/build_angular/src/browser/index.ts @@ -33,7 +33,6 @@ import { from, of } from 'rxjs'; import { bufferCount, catchError, concatMap, map, mergeScan, switchMap } from 'rxjs/operators'; import { ScriptTarget } from 'typescript'; import * as webpack from 'webpack'; -import * as workerFarm from 'worker-farm'; import { NgBuildAnalyticsPlugin } from '../../plugins/webpack/analytics'; import { WebpackConfigOptions } from '../angular-cli-files/models/build-options'; import { @@ -80,6 +79,7 @@ import { getIndexInputFile, getIndexOutputFile, } from '../utils/webpack-browser-config'; +import { ActionExecutor } from './action-executor'; import { Schema as BrowserBuilderSchema } from './schema'; const cacache = require('cacache'); @@ -460,23 +460,20 @@ export function buildWebpackBrowser( // Attempt to get required cache entries const cacheEntries = []; + let cached = cacheKeys.length > 0; for (const key of cacheKeys) { if (key) { - cacheEntries.push(await cacache.get.info(cacheDownlevelPath, key)); + const entry = await cacache.get.info(cacheDownlevelPath, key); + if (!entry) { + cached = false; + break; + } + cacheEntries.push(entry); } else { cacheEntries.push(null); } } - // Check if required cache entries are present - let cached = cacheKeys.length > 0; - for (let i = 0; i < cacheKeys.length; ++i) { - if (cacheKeys[i] && !cacheEntries[i]) { - cached = false; - break; - } - } - // If all required cached entries are present, use the cached entries // Otherwise process the files // If SRI is enabled always process the runtime bundle @@ -598,35 +595,25 @@ export function buildWebpackBrowser( } if (processActions.length > 0) { - await new Promise((resolve, reject) => { - const workerFile = require.resolve('../utils/process-bundle'); - const workers = workerFarm( - { - maxRetries: 1, - }, - path.extname(workerFile) !== '.ts' - ? workerFile - : require.resolve('../utils/process-bundle-bootstrap'), - ['process'], - ); - let completed = 0; - const workCallback = (error: Error | null, result: ProcessBundleResult) => { - if (error) { - workerFarm.end(workers); - reject(error); - - return; - } - - processResults.push(result); - if (++completed === processActions.length) { - workerFarm.end(workers); - resolve(); - } - }; + const workerFile = require.resolve('../utils/process-bundle'); + const executor = new ActionExecutor< + ProcessBundleOptions & { size: number }, + ProcessBundleResult + >( + path.extname(workerFile) !== '.ts' + ? workerFile + : require.resolve('../utils/process-bundle-bootstrap'), + 'process', + ); - processActions.forEach(action => workers['process'](action, workCallback)); - }); + try { + const results = await executor.executeAll( + processActions.map(a => ({ ...a, size: a.code.length })), + ); + results.forEach(result => processResults.push(result)); + } finally { + executor.stop(); + } } // Runtime must be processed after all other files @@ -636,7 +623,7 @@ export function buildWebpackBrowser( runtimeData: processResults, }; processResults.push( - await import('../utils/process-bundle').then(m => m.processAsync(runtimeOptions)), + await import('../utils/process-bundle').then(m => m.process(runtimeOptions)), ); } diff --git a/packages/angular_devkit/build_angular/src/utils/process-bundle.ts b/packages/angular_devkit/build_angular/src/utils/process-bundle.ts index 1313660cc610..d6836ebf9daa 100644 --- a/packages/angular_devkit/build_angular/src/utils/process-bundle.ts +++ b/packages/angular_devkit/build_angular/src/utils/process-bundle.ts @@ -57,14 +57,7 @@ export const enum CacheKey { DownlevelMap = 3, } -export function process( - options: ProcessBundleOptions, - callback: (error: Error | null, result?: ProcessBundleResult) => void, -): void { - processAsync(options).then(result => callback(null, result), error => callback(error)); -} - -export async function processAsync(options: ProcessBundleOptions): Promise { +export async function process(options: ProcessBundleOptions): Promise { if (!options.cacheKeys) { options.cacheKeys = []; } diff --git a/yarn.lock b/yarn.lock index d14b4979bdfa..a064ff3134c6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6531,6 +6531,14 @@ jasminewd2@^2.1.0: resolved "https://registry.yarnpkg.com/jasminewd2/-/jasminewd2-2.2.0.tgz#e37cf0b17f199cce23bea71b2039395246b4ec4e" integrity sha1-43zwsX8ZnM4jvqcbIDk5Uka07E4= +jest-worker@24.9.0: + version "24.9.0" + resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-24.9.0.tgz#5dbfdb5b2d322e98567898238a9697bcce67b3e5" + integrity sha512-51PE4haMSXcHohnSMdM42anbvZANYTqMrr52tVKPqqsPJMzoP6FYYDVqahX/HrAoKEKz3uUPzSvKs9A3qR4iVw== + dependencies: + merge-stream "^2.0.0" + supports-color "^6.1.0" + jquery@^3.3.1: version "3.3.1" resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.3.1.tgz#958ce29e81c9790f31be7792df5d4d95fc57fbca" @@ -7515,6 +7523,11 @@ merge-source-map@1.0.4: dependencies: source-map "^0.5.6" +merge-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" + integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== + methods@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" @@ -12177,7 +12190,7 @@ wordwrap@~0.0.2: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107" integrity sha1-o9XabNXAvAAI03I0u68b7WMFkQc= -worker-farm@1.7.0, worker-farm@^1.7.0: +worker-farm@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8" integrity sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==