Skip to content

Commit 8bd3e5c

Browse files
authored
Make kernel/ddk the default, use new/faster kernel apis (#2124)
- Pass '--reuse-compiler-result' and '--use-incremental-compiler' along with input digests to the kernel worker and ddc worker - Add the kernel builder for the ddc platform to build_web_compilers (we could add this to build_modules?) - Change the ddc builder to use kernel by default in build_web_compilers - Various minor kernel fixes
1 parent 45bb475 commit 8bd3e5c

16 files changed

+143
-53
lines changed

.travis.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,9 @@ jobs:
6666
env: PKGS="build build_config build_daemon build_modules build_resolvers build_runner build_runner_core build_test build_vm_compilers build_web_compilers example scratch_space"
6767
script: ./tool/travis.sh dartfmt dartanalyzer_0
6868
- stage: analyze_and_format
69-
name: "SDK: 2.1.0; PKGS: build, build_daemon, build_resolvers, build_runner, build_runner_core, build_web_compilers; TASKS: `dartanalyzer --fatal-warnings .`"
69+
name: "SDK: 2.1.0; PKGS: build, build_daemon, build_resolvers, build_runner, build_runner_core; TASKS: `dartanalyzer --fatal-warnings .`"
7070
dart: "2.1.0"
71-
env: PKGS="build build_daemon build_resolvers build_runner build_runner_core build_web_compilers"
71+
env: PKGS="build build_daemon build_resolvers build_runner build_runner_core"
7272
script: ./tool/travis.sh dartanalyzer_1
7373
- stage: unit_test
7474
name: "SDK: dev; PKG: build; TASKS: `pub run build_runner test`"
@@ -116,9 +116,9 @@ jobs:
116116
env: PKGS="build_vm_compilers"
117117
script: ./tool/travis.sh test_06
118118
- stage: analyze_and_format
119-
name: "SDK: 2.1.1; PKG: build_modules; TASKS: `dartanalyzer --fatal-warnings .`"
120-
dart: "2.1.1"
121-
env: PKGS="build_modules"
119+
name: "SDK: 2.2.1-dev.3.0; PKGS: build_modules, build_web_compilers; TASKS: `dartanalyzer --fatal-warnings .`"
120+
dart: "2.2.1-dev.3.0"
121+
env: PKGS="build_modules build_web_compilers"
122122
script: ./tool/travis.sh dartanalyzer_1
123123
- stage: unit_test
124124
name: "SDK: dev; PKG: build_modules; TASKS: `dart $(pub run build_runner generate-build-script) test --delete-conflicting-outputs-- -P presubmit`"

_test/test/goldens/generated_build_script.dart

+4
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,16 @@ final _builders = <_i1.BuilderApplication>[
6868
isOptional: true,
6969
hideOutput: true,
7070
appliesBuilders: ['build_modules|module_cleanup']),
71+
_i1.apply('build_web_compilers|kernel', [_i6.ddcKernelBuilder],
72+
_i1.toNoneByDefault(),
73+
isOptional: true, hideOutput: true),
7174
_i1.apply(
7275
'build_web_compilers|ddc', [_i6.devCompilerBuilder], _i1.toAllPackages(),
7376
isOptional: true,
7477
hideOutput: true,
7578
appliesBuilders: [
7679
'build_web_compilers|dart_source_cleanup',
80+
'build_web_compilers|kernel',
7781
'build_modules|dartdevc',
7882
'build_modules|dart2js'
7983
]),

_test/test/test_integration_test.dart

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ void main() {
2323
expect(
2424
result.stdout, matches(RegExp(r'hello_world_test.dart [\d]+:[\d]+')));
2525
expect(result.stdout, isNot(contains('.js')));
26-
});
26+
}, skip: 'https://github.com/dart-lang/sdk/issues/36236');
2727

2828
group('file edits', () {
2929
setUp(() async {

build_modules/CHANGELOG.md

+7
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,16 @@
11
## 2.0.0
22

3+
### Breaking Changes
4+
35
- Remove the `merge` method from `Module` and replace with a static
46
`Module.merge`. Module instances are now immutable.
57
- Remove `jsId`, and `jsSourceMapId` from `Module`.
68

9+
### Improvements
10+
11+
- Update the kernel worker to pass input digests, along with
12+
`--reuse-compiler-result` and `--use-incremental-compiler`.
13+
714
## 1.0.10
815

916
- Fix a performance issue in the kernel_builder, especially for larger projects.

build_modules/lib/src/kernel_builder.dart

+24-13
Original file line numberDiff line numberDiff line change
@@ -119,14 +119,14 @@ Future<void> _createKernel(
119119

120120
packagesFile = await createPackagesFile(allAssetIds);
121121

122-
_addRequestArguments(request, module, kernelDeps, dartSdkDir, sdkKernelPath,
123-
outputFile, packagesFile, summaryOnly);
122+
await _addRequestArguments(request, module, kernelDeps, sdkDir,
123+
sdkKernelPath, outputFile, packagesFile, summaryOnly, buildStep);
124124
});
125125

126126
// We need to make sure and clean up the temp dir, even if we fail to compile.
127127
try {
128-
var analyzer = await buildStep.fetchResource(frontendDriverResource);
129-
var response = await analyzer.doWork(request,
128+
var frontendWorker = await buildStep.fetchResource(frontendDriverResource);
129+
var response = await frontendWorker.doWork(request,
130130
trackWork: (response) => buildStep
131131
.trackStage('Kernel Generate', () => response, isExternal: true));
132132
if (response.exitCode != EXIT_CODE_OK || !await outputFile.exists()) {
@@ -216,15 +216,16 @@ Future<Set<AssetId>> _parentsOfMissingKernelFiles(
216216

217217
/// Fills in all the required arguments for [request] in order to compile the
218218
/// kernel file for [module].
219-
void _addRequestArguments(
219+
Future<void> _addRequestArguments(
220220
WorkRequest request,
221221
Module module,
222222
Iterable<AssetId> transitiveKernelDeps,
223223
String sdkDir,
224224
String sdkKernelPath,
225225
File outputFile,
226226
File packagesFile,
227-
bool summaryOnly) {
227+
bool summaryOnly,
228+
AssetReader reader) async {
228229
request.arguments.addAll([
229230
'--dart-sdk-summary',
230231
Uri.file(p.join(sdkDir, sdkKernelPath)).toString(),
@@ -236,18 +237,28 @@ void _addRequestArguments(
236237
multiRootScheme,
237238
'--exclude-non-sources',
238239
summaryOnly ? '--summary-only' : '--no-summary-only',
240+
'--libraries-file',
241+
p.toUri(p.join(sdkDir, 'lib', 'libraries.json')).toString(),
242+
'--reuse-compiler-result',
243+
'--use-incremental-compiler',
239244
]);
245+
request.inputs.add(Input()
246+
..path = '${Uri.file(p.join(sdkDir, sdkKernelPath))}'
247+
// Sdk updates fully invalidate the build anyways.
248+
..digest = [0]);
240249

241-
// Add all summaries as summary inputs.
242-
request.arguments.addAll(transitiveKernelDeps.map((id) {
250+
// Add all kernel outlines as summary inputs, with digests.
251+
var inputs = await Future.wait(transitiveKernelDeps.map((id) async {
243252
var relativePath = p.url.relative(scratchSpace.fileFor(id).uri.path,
244253
from: scratchSpace.tempDir.uri.path);
245-
if (summaryOnly) {
246-
return '--input-summary=$multiRootScheme:///$relativePath';
247-
} else {
248-
return '--input-linked=$multiRootScheme:///$relativePath';
249-
}
254+
255+
return Input()
256+
..path = '$multiRootScheme:///$relativePath'
257+
..digest = (await reader.digest(id)).bytes;
250258
}));
259+
request.arguments.addAll(inputs
260+
.map((i) => '--input-${summaryOnly ? 'summary' : 'linked'}=${i.path}'));
261+
request.inputs.addAll(inputs);
251262

252263
request.arguments.addAll(module.sources.map((id) {
253264
var uri = id.path.startsWith('lib')

build_modules/mono_pkg.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ stages:
88
- dartanalyzer: --fatal-infos --fatal-warnings .
99
- dartanalyzer: --fatal-warnings .
1010
dart:
11-
- 2.1.1
11+
- 2.2.1-dev.3.0
1212
- unit_test:
1313
# Run the script directly - running from snapshot has issues for packages
1414
# that are depended on by build_runner itself.

build_modules/pubspec.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ author: Dart Team <misc@dartlang.org>
55
homepage: https://github.com/dart-lang/build/tree/master/build_modules
66

77
environment:
8-
sdk: ">=2.1.1-dev <3.0.0"
8+
sdk: ">=2.2.1-dev.3.0 <3.0.0"
99

1010
dependencies:
1111
analyzer: '>0.30.0 <0.36.0'

build_web_compilers/CHANGELOG.md

+5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
## 2.0.0-dev
2+
3+
- Update to run DDC in kernel mode, and consume kernel outlines instead of
4+
analyzer summaries.
5+
16
## 1.2.0
27

38
- Add a marker to inject code before the application main method is called.

build_web_compilers/build.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,16 @@
11
builders:
2+
kernel:
3+
import: "package:build_web_compilers/builders.dart"
4+
builder_factories:
5+
- ddcKernelBuilder
6+
build_extensions:
7+
.dart:
8+
- .ddc.dill
9+
is_optional: True
10+
auto_apply: none
11+
required_inputs:
12+
- .dart
13+
- .dartdevc.module
214
ddc:
315
import: "package:build_web_compilers/builders.dart"
416
builder_factories:
@@ -13,11 +25,13 @@ builders:
1325
required_inputs:
1426
- .dart
1527
- .dartdevc.module
28+
- .ddc.dill
1629
applies_builders:
1730
# We want this to apply for dart2js as well - luckily this builder doesn't
1831
# need to be disabled since it's lazy so we can use is as a handle to get
1932
# the cleanup builder applied.
2033
- build_web_compilers|dart_source_cleanup
34+
- build_web_compilers|kernel
2135
- build_modules|dartdevc
2236
- build_modules|dart2js
2337
entrypoint:

build_web_compilers/lib/builders.dart

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ import 'package:build_web_compilers/build_web_compilers.dart';
99

1010
import 'package:path/path.dart' as p;
1111

12-
Builder devCompilerBuilder(_) => DevCompilerBuilder();
12+
Builder devCompilerBuilder(_) => DevCompilerBuilder(useKernel: true);
1313
Builder webEntrypointBuilder(BuilderOptions options) =>
14-
WebEntrypointBuilder.fromOptions(options);
14+
WebEntrypointBuilder.fromOptions(options, useKernel: true);
1515
PostProcessBuilder dart2JsArchiveExtractor(BuilderOptions options) =>
1616
Dart2JsArchiveExtractor.fromOptions(options);
1717
PostProcessBuilder dartSourceCleanup(BuilderOptions options) =>

build_web_compilers/lib/src/common.dart

+1-2
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ String defaultAnalysisOptionsArg(ScratchSpace scratchSpace) =>
1818

1919
// TODO: better solution for a .packages file, today we just create a new one
2020
// for every kernel build action.
21-
Future<File> createPackagesFile(
22-
Iterable<AssetId> allAssets, ScratchSpace scratchSpace) async {
21+
Future<File> createPackagesFile(Iterable<AssetId> allAssets) async {
2322
var allPackages = allAssets.map((id) => id.package).toSet();
2423
var packagesFileDir =
2524
await Directory.systemTemp.createTemp('kernel_builder_');

build_web_compilers/lib/src/dev_compiler_builder.dart

+64-17
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,13 @@ Future _createDevCompilerModule(
8484
request.arguments.addAll([
8585
'--dart-sdk-summary=$sdkSummary',
8686
'--modules=amd',
87+
'--no-summarize',
8788
'-o',
8889
jsOutputFile.path,
8990
]);
91+
request.inputs.add(Input()
92+
..path = sdkSummary
93+
..digest = [0]);
9094

9195
if (!useKernel) {
9296
// Add the default analysis_options.
@@ -117,21 +121,35 @@ Future _createDevCompilerModule(
117121
request.arguments.add('--no-source-map');
118122
}
119123

120-
// Add all the linked summaries as summary inputs.
124+
// Add linked summaries as summary inputs.
125+
//
126+
// Also request the digests for each and add those to the inputs.
127+
//
128+
// This runs synchronously and the digest futures will be awaited later on.
129+
var digestFutures = <Future>[];
121130
for (var depModule in transitiveDeps) {
122131
var summaryId = useKernel
123132
? depModule.primarySource.changeExtension(ddcKernelExtension)
124133
: depModule.linkedSummaryId;
125-
var summaryPath = useKernel
126-
? p.url.relative(scratchSpace.fileFor(summaryId).path,
127-
from: scratchSpace.tempDir.path) +
128-
'=' +
129-
ddcModuleName(
130-
depModule.primarySource.changeExtension(jsModuleExtension))
131-
: scratchSpace.fileFor(summaryId).path;
134+
var summaryPath = scratchSpace.fileFor(summaryId).path;
135+
136+
if (useKernel) {
137+
var input = Input()..path = summaryPath;
138+
request.inputs.add(input);
139+
digestFutures.add(buildStep.digest(summaryId).then((digest) {
140+
input.digest = digest.bytes;
141+
}));
142+
var moduleName = ddcModuleName(
143+
depModule.primarySource.changeExtension(jsModuleExtension));
144+
summaryPath += '=$moduleName';
145+
}
146+
132147
request.arguments.addAll(['-s', summaryPath]);
133148
}
134149

150+
// Wait for all the digests to complete.
151+
await Future.wait(digestFutures);
152+
135153
// Add URL mappings for all the package: files to tell DartDevc where to
136154
// find them.
137155
//
@@ -154,12 +172,20 @@ Future _createDevCompilerModule(
154172
var allDeps = <AssetId>[]
155173
..addAll(module.sources)
156174
..addAll(transitiveSummaryDeps);
157-
packagesFile = await createPackagesFile(allDeps, scratchSpace);
175+
packagesFile = await createPackagesFile(allDeps);
158176
request.arguments.addAll([
159177
'--packages',
160178
packagesFile.absolute.uri.toString(),
161179
'--module-name',
162180
ddcModuleName(module.primarySource.changeExtension(jsModuleExtension)),
181+
'--multi-root-scheme',
182+
multiRootScheme,
183+
'--multi-root',
184+
'.',
185+
'--reuse-compiler-result',
186+
'--use-incremental-compiler',
187+
'--track-widget-creation',
188+
'--inline-source-map',
163189
]);
164190
}
165191

@@ -170,7 +196,9 @@ Future _createDevCompilerModule(
170196
if (uri.startsWith('package:')) {
171197
return uri;
172198
}
173-
return useKernel ? id.path : Uri.file('/${id.path}').toString();
199+
return useKernel
200+
? '$multiRootScheme:///${id.path}'
201+
: Uri.file('/${id.path}').toString();
174202
}));
175203

176204
WorkResponse response;
@@ -188,17 +216,36 @@ Future _createDevCompilerModule(
188216
// TODO(jakemac53): Fix the ddc worker mode so it always sends back a bad
189217
// status code if something failed. Today we just make sure there is an output
190218
// JS file to verify it was successful.
191-
if (response.exitCode != EXIT_CODE_OK || !jsOutputFile.existsSync()) {
192-
var message =
193-
response.output.replaceAll('${scratchSpace.tempDir.path}/', '');
194-
throw DartDevcCompilationException(jsId, '$message}');
219+
var message = response.output
220+
.replaceAll('${scratchSpace.tempDir.path}/', '')
221+
.replaceAll('$multiRootScheme:///', '');
222+
if (response.exitCode != EXIT_CODE_OK ||
223+
!jsOutputFile.existsSync() ||
224+
message.contains('Error:')) {
225+
throw DartDevcCompilationException(jsId, message);
195226
} else {
227+
if (message.isNotEmpty) {
228+
log.info('\n$message');
229+
}
196230
// Copy the output back using the buildStep.
197231
await scratchSpace.copyOutput(jsId, buildStep);
198232
if (debugMode) {
199-
await scratchSpace.copyOutput(
200-
module.primarySource.changeExtension(jsSourceMapExtension),
201-
buildStep);
233+
// We need to modify the sources in the sourcemap to remove the custom
234+
// `multiRootScheme` that we use.
235+
var sourceMapId =
236+
module.primarySource.changeExtension(jsSourceMapExtension);
237+
var file = scratchSpace.fileFor(sourceMapId);
238+
var content = await file.readAsString();
239+
var json = jsonDecode(content);
240+
json['sources'] = (json['sources'] as List).cast<String>().map((source) {
241+
var uri = Uri.parse(source);
242+
var newSegments = uri.pathSegments.first == 'packages'
243+
? uri.pathSegments
244+
: uri.pathSegments.skip(1);
245+
return Uri(path: p.url.joinAll(['/'].followedBy(newSegments)))
246+
.toString();
247+
}).toList();
248+
await buildStep.writeAsString(sourceMapId, jsonEncode(json));
202249
}
203250
}
204251
}

build_web_compilers/lib/src/web_entrypoint_builder.dart

+5-2
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@ class WebEntrypointBuilder implements Builder {
5757
this.useKernel = false,
5858
this.buildRootAppSummary = false});
5959

60-
factory WebEntrypointBuilder.fromOptions(BuilderOptions options) {
60+
factory WebEntrypointBuilder.fromOptions(BuilderOptions options,
61+
{bool useKernel}) {
62+
useKernel ??= false;
6163
validateOptions(
6264
options.config, _supportedOptions, 'build_web_compilers|entrypoint',
6365
deprecatedOptions: _deprecatedOptions);
@@ -86,7 +88,8 @@ class WebEntrypointBuilder implements Builder {
8688

8789
return WebEntrypointBuilder(compiler,
8890
buildRootAppSummary: buildRootAppSummary,
89-
dart2JsArgs: dart2JsArgs as List<String>);
91+
dart2JsArgs: dart2JsArgs as List<String>,
92+
useKernel: useKernel);
9093
}
9194

9295
@override

build_web_compilers/mono_pkg.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ stages:
88
- dartanalyzer: --fatal-infos --fatal-warnings .
99
- dartanalyzer: --fatal-warnings .
1010
dart:
11-
- 2.1.0
11+
- 2.2.1-dev.3.0
1212
- unit_test:
1313
- group:
1414
- test: -x presubmit-only

0 commit comments

Comments
 (0)