reproducible-builds.js

// Helper for the "Reproducible builds" job.
//
// Prerequisites:
// * Node.js 18+, npm 10+
// * Git 2.11+
// * tar, shasum, gunzip (preinstalled on Linux/macOS)
'use strict';

const cp = require('child_process');
const fs = require('fs');
const path = require('path');
const util = require('util');

const utils = require('./utils.js');
const execFile = util.promisify(cp.execFile);
const tempDir = path.join(__dirname, '../temp', 'reproducible-builds');
const SRC_REPO = 'https://github.com/qunitjs/qunit.git';

/**
 * QUnit 2.17.0 and later are fully reproducible with this script.
 *
 * Known caveats:
 *
 * QUnit 2.14.1 - 2.16.0:
 *   - File headers included an uncontrolled "current" timestamp.
 *     This would have to be ignored or replaced prior to comparison.
 *   - The build wrote files to "/dist" instead of "/qunit".
 *
 * QUnit 2.15.0:
 *   - Contained some CR (\r) characters in comments from fuzzysort.js,
 *     which get normalized to LF (\n) by Git and npm, but not in the actual builds
 *     and in what we publish to the CDN. This was fixed in qunit@2.16.0 and qunit@2.17.0.
 *
 * QUnit 2.17.0 - 2.21.0:
 * - These were built and published using npm 8 or npm 9.
 *   In npm 10, upstream changed gzip encoding slightly for the npm-pack tarball (.tgz). This
 *   means a tarball from npm 10+ is not byte-for-byte identical to ones generated by npm 8 or 9.
 *   After gzip-decompression, however, the tar stream is byte-for-byte identical.
 *   Either use npm 8 or 9 to verify these, or verify the tarball after gzip decompression.
 *
 * QUnit 3.0.0-alpha.3:
 * - The package-lock.json file was a few commits behind what was actually released,
 *   thus reproducing it uees a slightly Rollup/Babel version that outputs with slightly
 *   different code formatting.
 */
const VERIFY_COUNT = 5;
const EXCLUDE = ['3.0.0-alpha.3'];

async function buildRelease (version, cacheDir = null) {
  console.log(`... ${version}: checking out the source`);

  const gitDir = path.join(tempDir, `git-${version}`);
  utils.cleanDir(gitDir);

  await execFile('git', ['clone', '-q', '-b', version, '--depth=5', SRC_REPO, gitDir]);

  // Remove any artefacts that were checked into Git
  utils.cleanDir(gitDir + '/qunit/');

  console.log(`... ${version}: installing development dependencies from npm`);
  const npmEnv = {
    npm_config_cache: cacheDir,
    npm_config_update_notifier: 'false',
    PATH: process.env.PATH,
    QUNIT_BUILD_RELEASE: '1',
    PUPPETEER_CACHE_DIR: path.join(cacheDir, 'puppeteer_download')
  };
  cp.execFileSync('npm', ['install'], {
    env: npmEnv,
    cwd: gitDir
  });

  console.log(`... ${version}: building release`);
  await execFile('npm', ['run', 'build'], {
    env: npmEnv,
    cwd: gitDir
  });

  console.log(`... ${version}: packing npm package`);
  await execFile('npm', ['pack'], {
    env: npmEnv,
    cwd: gitDir
  });

  return {
    js: {
      name: gitDir + '/qunit/qunit.js',
      contents: fs.readFileSync(gitDir + '/qunit/qunit.js', 'utf8')
    },
    css: {
      name: gitDir + '/qunit/qunit.css',
      contents: fs.readFileSync(gitDir + '/qunit/qunit.css', 'utf8')
    },
    tgz: {
      name: gitDir + `/qunit-${version}.tgz`,
      contents: cp.execSync(
        `gunzip --stdout qunit-${version}.tgz | shasum -a 256 -b`,
        { encoding: 'utf8', cwd: gitDir }
      )
    }
  };
}

const Reproducible = {
  async fetch () {
    // Fetch official releases first and store them in memory (not on disk). Only after that will
    // we run the build commands (which involve unaudited npm packages as dev deps) which could
    // modify anything on disk. Hence don't store what we want to compare against on disk.
    const releases = {};

    {
      // This may take a while locally, when removing previous builds.
      utils.cleanDir(tempDir);
    }
    {
      console.log('Fetching releases from jQuery CDN...');
      const cdnIndexUrl = 'https://releases.jquery.com/resources/cdn.json';
      const data = JSON.parse(await utils.download(cdnIndexUrl));

      for (const release of data.qunit.all.slice(0, VERIFY_COUNT)) {
        if (EXCLUDE.includes(release.version)) {
          continue;
        }
        releases[release.version] = {
          cdn: {
            js: {
              name: `https://code.jquery.com/${release.filename}`,
              contents: await utils.download(`https://code.jquery.com/${release.filename}`)
            },
            css: {
              name: `https://code.jquery.com/${release.theme}`,
              contents: await utils.download(`https://code.jquery.com/${release.theme}`)
            }
          }
        };
      }
    }
    {
      console.log('Fetching releases from npmjs.org...');
      const npmIndexUrl = 'https://registry.npmjs.org/qunit';
      const data = JSON.parse(await utils.download(npmIndexUrl));

      for (const version in releases) {
        if (!data.versions[version]) {
          throw new Error(`QUnit ${version} is missing from https://www.npmjs.com/package/qunit`);
        }
        const tarball = data.versions[version].dist.tarball;
        const tarFile = path.join(tempDir, path.basename(tarball));
        await utils.downloadFile(tarball, tarFile);

        releases[version].npm = {
          js: {
            name: `npm:${path.basename(tarball)}#package/qunit/qunit.js`,
            contents: cp.execFileSync(
              'tar', ['-xOf', tarFile, 'package/qunit/qunit.js'],
              { encoding: 'utf8' }
            )
          },
          css: {
            name: `npm:${path.basename(tarball)}#package/qunit/qunit.css`,
            contents: cp.execFileSync(
              'tar', ['-xOf', tarFile, 'package/qunit/qunit.css'],
              { encoding: 'utf8' }
            )
          },
          tgz: {
            name: `npm:${path.basename(tarball)}`,
            contents: cp.execSync(
              `gunzip --stdout ${path.basename(tarball)} | shasum -a 256 -b`,
              { encoding: 'utf8', cwd: tempDir }
            )
          }
        };
      }
    }
    {
      console.log('Reproducing release builds...');

      const cacheDir = path.join(tempDir, 'cache');
      utils.cleanDir(cacheDir);

      // Start builds in parallel and await results.
      const buildPromises = [];
      for (const version in releases) {
        buildPromises.push(
          (releases[version].buildPromise = buildRelease(version, cacheDir))
        );
      }
      await Promise.all(buildPromises);

      const diffs = [];
      for (const version in releases) {
        const release = releases[version];
        const build = await release.buildPromise;

        let verified = true;
        for (const distro in release) {
          for (const file in release[distro]) {
            if (release[distro][file].contents === build[file].contents) {
              console.log(
                `... ${version}: ${release[distro][file].name} matches ${build[file].name}`
              );
            } else {
              verified = false;
              console.error(
                `QUnit ${version} ${file} from ${distro} differs from build`
              );
              const buildFile = `qunit-${version}-build.${file}`;
              const releaseFile = `qunit-${version}-${distro}.${file}`;
              fs.writeFileSync(buildFile, utils.verboseNonPrintable(build[file].contents));
              fs.writeFileSync(releaseFile, utils.verboseNonPrintable(release[distro][file].contents));
              diffs.push(
                `--- ${build[file].name}\n+++ ${release[distro][file].name}\n`,
                utils.getDiff(buildFile, releaseFile, { ignoreWhitespace: false })
                  .split('\n').slice(2).join('\n')
              );
              fs.rmSync(buildFile);
              fs.rmSync(releaseFile);
            }
          }
        }

        if (verified) {
          console.log(`QUnit ${version} is reproducible and matches distributions!`);
        }
      }

      diffs.forEach(diff => {
        process.stdout.write(diff);
      });
      if (diffs.length) {
        throw new Error('One or more distributions differ from the reproduced build');
      }
    }
  }
};

(async function main () {
  await Reproducible.fetch();
}()).catch(e => {
  console.error(e.toString());
  process.exit(1);
});