From 95439008c8b270c30408702194a4395282df9d8c Mon Sep 17 00:00:00 2001
From: Giacomo Cusinato <7659518+giacomocusinato@users.noreply.github.com>
Date: Fri, 22 Nov 2024 17:50:58 +0100
Subject: [PATCH 1/2] refactor: `generate-protocol` now fetch proto files from
`arduino_cli_{version}_proto.zip`
- Use the CLI release proto.zip to get proto files for production versions of CLI
- Extract the proto files from repo if CLI version is declared as `commitsh` or version is 1.1.0
See https://github.com/arduino/arduino-cli/pull/2761
---
.../scripts/generate-protocol.js | 256 +++++++++++-------
1 file changed, 152 insertions(+), 104 deletions(-)
diff --git a/arduino-ide-extension/scripts/generate-protocol.js b/arduino-ide-extension/scripts/generate-protocol.js
index 68a92c026..7e0cf8a55 100644
--- a/arduino-ide-extension/scripts/generate-protocol.js
+++ b/arduino-ide-extension/scripts/generate-protocol.js
@@ -3,10 +3,12 @@
(async () => {
const os = require('node:os');
const path = require('node:path');
- const { mkdirSync, promises: fs, rmSync } = require('node:fs');
+ const decompress = require('decompress');
+ const unzip = require('decompress-unzip');
+ const { mkdirSync, promises: fs, rmSync, existsSync } = require('node:fs');
const { exec } = require('./utils');
const { glob } = require('glob');
- const { SemVer, gte, valid: validSemVer, gt } = require('semver');
+ const { SemVer, gte, valid: validSemVer, eq } = require('semver');
// Use a node-protoc fork until apple arm32 is supported
// https://github.com/YePpHa/node-protoc/pull/10
const protoc = path.dirname(require('@pingghost/protoc/protoc'));
@@ -90,152 +92,198 @@
}
*/
const versionObject = JSON.parse(versionJson);
- const version = versionObject.VersionString;
- // Clone the repository and check out the tagged version
- // Return folder with proto files
- async function getProtoPath(forceCliVersion) {
- const repository = await fs.mkdtemp(path.join(os.tmpdir(), 'arduino-cli-'));
+ async function globProtos(folder, pattern = '**/*.proto') {
+ let protos = [];
+ try {
+ const matches = await glob(pattern, { cwd: folder });
+ protos = matches.map((filename) => path.join(folder, filename));
+ } catch (error) {
+ console.log(error.stack ?? error.message);
+ }
+ return protos;
+ }
+
+ async function getProtosFromRepo(
+ commitish = '',
+ version = '',
+ owner = 'arduino',
+ repo = 'arduino-cli'
+ ) {
+ const repoFolder = await fs.mkdtemp(path.join(os.tmpdir(), 'arduino-cli-'));
const url = `https://github.com/${owner}/${repo}.git`;
console.log(`>>> Cloning repository from '${url}'...`);
- exec('git', ['clone', url, repository], { logStdout: true });
+ exec('git', ['clone', url, repoFolder], { logStdout: true });
console.log(`<<< Repository cloned.`);
- let cliVersion = forceCliVersion || version;
- if (validSemVer(cliVersion)) {
+ if (validSemVer(version)) {
+ let versionTag = version;
// https://github.com/arduino/arduino-cli/pull/2374
if (
gte(new SemVer(version, { loose: true }), new SemVer('0.35.0-rc.1'))
) {
- cliVersion = `v${cliVersion}`;
+ versionTag = `v${version}`;
}
- console.log(`>>> Checking out tagged version: '${cliVersion}'...`);
- exec('git', ['-C', repository, 'fetch', '--all', '--tags'], {
+ console.log(`>>> Checking out tagged version: '${versionTag}'...`);
+ exec('git', ['-C', repoFolder, 'fetch', '--all', '--tags'], {
logStdout: true,
});
exec(
'git',
- ['-C', repository, 'checkout', `tags/${cliVersion}`, '-b', cliVersion],
+ ['-C', repoFolder, 'checkout', `tags/${versionTag}`, '-b', versionTag],
{ logStdout: true }
);
- console.log(`<<< Checked out tagged version: '${cliVersion}'.`);
- } else if (forceCliVersion) {
- console.log(`WARN: invalid semver: '${forceCliVersion}'.`);
- // If the forced version is invalid, do not proceed with fallbacks.
- return undefined;
+ console.log(`<<< Checked out tagged version: '${versionTag}'.`);
} else if (commitish) {
- console.log(
- `>>> Checking out commitish from 'package.json': '${commitish}'...`
- );
- exec('git', ['-C', repository, 'checkout', commitish], {
+ console.log(`>>> Checking out commitish: '${commitish}'...`);
+ exec('git', ['-C', repoFolder, 'checkout', commitish], {
logStdout: true,
});
- console.log(
- `<<< Checked out commitish from 'package.json': '${commitish}'.`
- );
- } else if (versionObject.Commit) {
- console.log(
- `>>> Checking out commitish from the CLI: '${versionObject.Commit}'...`
- );
- exec('git', ['-C', repository, 'checkout', versionObject.Commit], {
- logStdout: true,
- });
- console.log(
- `<<< Checked out commitish from the CLI: '${versionObject.Commit}'.`
- );
+ console.log(`<<< Checked out commitish: '${commitish}'.`);
} else {
console.log(
`WARN: no 'git checkout'. Generating from the HEAD revision.`
);
}
- return path.join(repository, 'rpc');
+ const rpcFolder = await fs.mkdtemp(
+ path.join(os.tmpdir(), 'arduino-cli-rpc')
+ );
+
+ // Copy the the repository rpc folder so we can remove the repository
+ await fs.cp(path.join(repoFolder, 'rpc'), path.join(rpcFolder), {
+ recursive: true,
+ });
+ rmSync(repoFolder, { recursive: true, maxRetries: 5, force: true });
+
+ // Patch for https://github.com/arduino/arduino-cli/issues/2755
+ // Google proto files are removed from source since v1.1.0
+ if (!existsSync(path.join(rpcFolder, 'google'))) {
+ // Include packaged google proto files from v1.1.1
+ // See https://github.com/arduino/arduino-cli/pull/2761
+ console.log(`>>> Missing google proto files. Including from v1.1.1...`);
+ const v111ProtoFolder = await getProtosFromZip('1.1.1');
+
+ // Create an return a folder name google in rpcFolder
+ const googleFolder = path.join(rpcFolder, 'google');
+ await fs.cp(path.join(v111ProtoFolder, 'google'), googleFolder, {
+ recursive: true,
+ });
+ console.log(`<<< Included google proto files from v1.1.1.`);
+ }
+
+ return rpcFolder;
}
- const protoPath = await getProtoPath();
+ async function getProtosFromZip(version) {
+ if (!version) {
+ console.log(`Could not download proto files: CLI version not provided.`);
+ process.exit(1);
+ }
+ console.log(`>>> Downloading proto files from zip for ${version}.`);
+
+ const url = `https://downloads.arduino.cc/arduino-cli/arduino-cli_${version}_proto.zip`;
+ const protos = await fs.mkdtemp(
+ path.join(os.tmpdir(), 'arduino-cli-proto')
+ );
- if (!protoPath) {
- console.log(`Could not find the proto files folder.`);
+ const { default: download } = await import('@xhmikosr/downloader');
+ /** @type {import('node:buffer').Buffer} */
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+ // @ts-ignore
+ const data = await download(url);
+
+ await decompress(data, protos, {
+ plugins: [unzip()],
+ filter: (file) => file.path.endsWith('.proto'),
+ });
+
+ console.log(
+ `<<< Finished downloading and extracting proto files for ${version}.`
+ );
+
+ return protos;
+ }
+
+ let protosFolder;
+
+ if (commitish) {
+ protosFolder = await getProtosFromRepo(commitish, undefined, owner, repo);
+ } else if (
+ versionObject.VersionString &&
+ validSemVer(versionObject.VersionString)
+ ) {
+ const version = versionObject.VersionString;
+ // v1.1.0 does not contains google proto files in zip
+ // See https://github.com/arduino/arduino-cli/issues/2755
+ const isV110 = eq(new SemVer(version, { loose: true }), '1.1.0');
+ protosFolder = isV110
+ ? await getProtosFromRepo(undefined, version)
+ : await getProtosFromZip(version);
+ } else if (versionObject.Commit) {
+ protosFolder = await getProtosFromRepo(versionObject.Commit);
+ }
+
+ if (!protosFolder) {
+ console.log(`Could not get proto files: missing commitish or version.`);
+ process.exit(1);
+ }
+
+ const protos = await globProtos(protosFolder);
+
+ if (!protos || protos.length === 0) {
+ rmSync(protosFolder, { recursive: true, maxRetries: 5, force: true });
+ console.log(`Could not find any .proto files under ${protosFolder}.`);
process.exit(1);
}
console.log('>>> Generating TS/JS API from:');
- exec('git', ['-C', protoPath, 'rev-parse', '--abbrev-ref', 'HEAD'], {
- logStdout: true,
- });
const out = path.join(__dirname, '..', 'src', 'node', 'cli-protocol');
// Must wipe the gen output folder. Otherwise, dangling service implementation remain in IDE2 code,
// although it has been removed from the proto file.
// For example, https://github.com/arduino/arduino-cli/commit/50a8bf5c3e61d5b661ccfcd6a055e82eeb510859.
- rmSync(out, { recursive: true, maxRetries: 5, force: true });
+ // rmSync(out, { recursive: true, maxRetries: 5, force: true });
mkdirSync(out, { recursive: true });
- if (gt(new SemVer(version, { loose: true }), new SemVer('1.0.4'))) {
- // Patch for https://github.com/arduino/arduino-cli/issues/2755
- // Credit https://github.com/dankeboy36/ardunno-cli-gen/pull/9/commits/64a5ac89aae605249261c8ceff7255655ecfafca
- // Download the 1.0.4 version and use the missing google/rpc/status.proto file.
- console.log('<<< Generating missing google proto files');
- const v104ProtoPath = await getProtoPath('1.0.4');
- if (!v104ProtoPath) {
- console.log(`Could not find the proto files folder for version 1.0.4.`);
- process.exit(1);
- }
- await fs.cp(
- path.join(v104ProtoPath, 'google'),
- path.join(protoPath, 'google'),
- {
- recursive: true,
- }
+ try {
+ // Generate JS code from the `.proto` files.
+ exec(
+ 'grpc_tools_node_protoc',
+ [
+ `--js_out=import_style=commonjs,binary:${out}`,
+ `--grpc_out=generate_package_definition:${out}`,
+ '-I',
+ protosFolder,
+ ...protos,
+ ],
+ { logStdout: true }
);
- console.log(`>>> Generated missing google file`);
- }
- let protos = [];
- try {
- const matches = await glob('**/*.proto', { cwd: protoPath });
- protos = matches.map((filename) => path.join(protoPath, filename));
+ // Generate the `.d.ts` files for JS.
+ exec(
+ path.join(protoc, `protoc${platform === 'win32' ? '.exe' : ''}`),
+ [
+ `--plugin=protoc-gen-ts=${path.resolve(
+ __dirname,
+ '..',
+ 'node_modules',
+ '.bin',
+ `protoc-gen-ts${platform === 'win32' ? '.cmd' : ''}`
+ )}`,
+ `--ts_out=generate_package_definition:${out}`,
+ '-I',
+ protosFolder,
+ ...protos,
+ ],
+ { logStdout: true }
+ );
} catch (error) {
- console.log(error.stack ?? error.message);
- }
-
- if (!protos || protos.length === 0) {
- console.log(`Could not find any .proto files under ${protoPath}.`);
- process.exit(1);
+ console.log(error);
+ } finally {
+ rmSync(protosFolder, { recursive: true, maxRetries: 5, force: true });
}
- // Generate JS code from the `.proto` files.
- exec(
- 'grpc_tools_node_protoc',
- [
- `--js_out=import_style=commonjs,binary:${out}`,
- `--grpc_out=generate_package_definition:${out}`,
- '-I',
- protoPath,
- ...protos,
- ],
- { logStdout: true }
- );
-
- // Generate the `.d.ts` files for JS.
- exec(
- path.join(protoc, `protoc${platform === 'win32' ? '.exe' : ''}`),
- [
- `--plugin=protoc-gen-ts=${path.resolve(
- __dirname,
- '..',
- 'node_modules',
- '.bin',
- `protoc-gen-ts${platform === 'win32' ? '.cmd' : ''}`
- )}`,
- `--ts_out=generate_package_definition:${out}`,
- '-I',
- protoPath,
- ...protos,
- ],
- { logStdout: true }
- );
-
console.log('<<< Generation was successful.');
})();
From 82fb8c5b3ec820a40893e3e914712d6089b9038a Mon Sep 17 00:00:00 2001
From: Giacomo Cusinato <7659518+giacomocusinato@users.noreply.github.com>
Date: Fri, 22 Nov 2024 17:57:47 +0100
Subject: [PATCH 2/2] feat: use Arduino CLI v1.1.1
---
arduino-ide-extension/package.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arduino-ide-extension/package.json b/arduino-ide-extension/package.json
index d900bfbfb..6dfb8e35a 100644
--- a/arduino-ide-extension/package.json
+++ b/arduino-ide-extension/package.json
@@ -171,7 +171,7 @@
],
"arduino": {
"arduino-cli": {
- "version": "1.1.0"
+ "version": "1.1.1"
},
"arduino-fwuploader": {
"version": "2.4.1"