1 python cryptography module upgrades

.github/ISSUE_TEMPLATE/ql---general.md

@@ -10,5 +10,5 @@ assignees: ''
 **Description of the issue**
 
 <!-- Please explain briefly what is the problem.
-If it is about an LGTM project, please include its URL.-->
+If it is about a GitHub project, please include its URL. -->
 

.github/ISSUE_TEMPLATE/ql--false-positive.md

@@ -1,5 +1,5 @@
 ---
-name: CodeQL False positive
+name: CodeQL false positive
 about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
 title: False positive
 labels: false-positive

.github/actions/cache-query-compilation/action.yml

@@ -0,0 +1,55 @@
+name: Cache query compilation
+description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
+
+inputs:
+  key:
+    description: 'The cache key to use - should be unique to the workflow'
+    required: true
+
+outputs:
+  cache-dir:
+    description: "The directory where the cache was stored"
+    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
+
+runs:
+  using: composite
+  steps:
+    # calculate the merge-base with main, in a way that works both on PRs and pushes to main.
+    - name: Calculate merge-base
+      shell: bash
+      if: ${{ github.event_name == 'pull_request' }}
+      env:
+        BASE_BRANCH: ${{ github.base_ref }}
+      run: |
+        MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
+        echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
+    - name: Restore read-only cache (PR)
+      if: ${{ github.event_name == 'pull_request' }}
+      uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
+      with:
+        path: '**/.cache'
+        read-only: true
+        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
+        restore-keys: |
+          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
+          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
+          codeql-compile-${{ inputs.key }}-main-
+    - name: Fill cache (push)
+      if: ${{ github.event_name != 'pull_request' }}
+      uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
+      with:
+        path: '**/.cache'
+        key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
+        restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
+          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
+          codeql-compile-${{ inputs.key }}-main-
+    - name: Fill compilation cache directory
+      id: fill-compilation-dir
+      shell: bash
+      run: |
+        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
+
+        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
+      env:
+        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir

.github/actions/cache-query-compilation/move-caches.js

@@ -0,0 +1,75 @@
+// # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+// mkdir -p ${COMBINED_CACHE_DIR}
+// rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+// # copy the contents of the .cache folders into the combined cache folder.
+// cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+// # clean up the .cache folders
+// rm -rf **/.cache/*
+
+const fs = require("fs");
+const path = require("path");
+
+// the first argv is the cache folder to create.
+const COMBINED_CACHE_DIR = process.argv[2];
+
+function* walkCaches(dir) {
+  const files = fs.readdirSync(dir, { withFileTypes: true });
+  for (const file of files) {
+    if (file.isDirectory()) {
+      const filePath = path.join(dir, file.name);
+      yield* walkCaches(filePath);
+      if (file.name === ".cache") {
+        yield filePath;
+      }
+    }
+  }
+}
+
+async function copyDir(src, dest) {
+  for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+    const srcPath = path.join(src, file.name);
+    const destPath = path.join(dest, file.name);
+    if (file.isDirectory()) {
+      if (!fs.existsSync(destPath)) {
+        fs.mkdirSync(destPath);
+      }
+      await copyDir(srcPath, destPath);
+    } else {
+      await fs.promises.copyFile(srcPath, destPath);
+    }
+  }
+}
+
+async function main() {
+  const cacheDirs = [...walkCaches(".")];
+
+  for (const dir of cacheDirs) {
+    console.log(`Found .cache dir at ${dir}`);
+  }
+
+  // mkdir -p ${COMBINED_CACHE_DIR}
+  fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+  // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+  await Promise.all(
+    cacheDirs.map((cacheDir) =>
+      (async function () {
+        await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+        await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+      })()
+    )
+  );
+
+  // # copy the contents of the .cache folders into the combined cache folder.
+  // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+  await Promise.all(
+    cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+  );
+
+  // # clean up the .cache folders
+  // rm -rf **/.cache/*
+  await Promise.all(
+    cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+  );
+}
+main();

.github/workflows/compile-queries.yml

@@ -14,58 +14,24 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
-      - name: Calculate merge-base
-        if: ${{ github.event_name == 'pull_request' }}
-        env:
-          BASE_BRANCH: ${{ github.base_ref }}
-        run: |
-          MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
-          echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
-      - name: Read CodeQL query compilation - PR
-        if: ${{ github.event_name == 'pull_request' }}
-        uses: actions/cache@v3
-        with:
-          path: '*/ql/src/.cache'
-          key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
-          restore-keys: |
-            codeql-compile-${{ github.base_ref }}-${{ env.merge-base }}
-            codeql-compile-${{ github.base_ref }}-
-            codeql-compile-main-
-      - name: Fill CodeQL query compilation cache - main
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: actions/cache@v3
-        with:
-          path: '*/ql/src/.cache'
-          key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main
-          restore-keys: | # restore from another random commit, to speed up compilation.
-            codeql-compile-${{ github.ref_name }}- 
-            codeql-compile-main-
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
         with:
           channel: 'release'
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: all-queries
       - name: check formatting
         run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/src --keep-going --warnings=error --check-only
+        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: |
-          # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-          mkdir -p ${COMBINED_CACHE_DIR}
-          rm */ql/src/.cache/{lock,size}
-          # copy the contents of the .cache folders into the combined cache folder.
-          cp -r */ql/src/.cache/* ${COMBINED_CACHE_DIR}/
-          # clean up the .cache folders
-          rm -rf */ql/src/.cache/*
-
-          # compile the queries
-          codeql query compile -j0 */ql/src --keep-going --warnings=error --compilation-cache ${COMBINED_CACHE_DIR}
-        env: 
-          COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir
+        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/csharp-qltest.yml

@@ -0,0 +1,86 @@
+name: "C#: Run QL Tests"
+
+on:
+  push:
+    paths:
+      - "csharp/**"
+      - "shared/**"
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "csharp/**"
+      - "shared/**"
+      - .github/workflows/csharp-qltest.yml
+      - .github/actions/fetch-codeql/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+      - "rc/*"
+
+defaults:
+  run:
+    working-directory: csharp
+
+jobs:
+  qlupgrade:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - name: Check DB upgrade scripts
+        run: |
+          echo >empty.trap
+          codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
+          codeql dataset upgrade testdb --additional-packs ql/lib
+          diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
+      - name: Check DB downgrade scripts
+        run: |
+          echo >empty.trap
+          rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
+          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
+           --dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
+           xargs codeql execute upgrades testdb
+          diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
+  qltest:
+    runs-on: ubuntu-latest-xl
+    strategy:
+      fail-fast: false
+      matrix:
+        slice: ["1/2", "2/2"]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: ./csharp/actions/create-extractor-pack
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: csharp-qltest-${{ matrix.slice }}
+      - name: Run QL tests
+        run: |
+          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
+          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
+          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
+          # Safe guard against using the bundled extractor
+          rm -rf "$CODEQL_PATH/csharp"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup dotnet
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 6.0.202
+      - name: Extractor unit tests
+        run: |
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"

.github/workflows/go-tests-other-os.yml

@@ -0,0 +1,80 @@
+name: "Go: Run Tests - Other OS"
+on:
+  pull_request:
+    paths:
+      - "go/**"
+      - "!go/ql/**" # don't run other-os if only ql/ files changed
+      - .github/workflows/go-tests-other-os.yml
+      - .github/actions/**
+      - codeql-workspace.yml
+jobs:
+  test-mac:
+    name: Test MacOS
+    runs-on: macos-latest
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
+
+  test-win:
+    name: Test Windows
+    runs-on: windows-latest-xl
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"