From ec5eecb246f47cf36d1263f4fe64bc2feb84eed4 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 15 Sep 2026 18:17:16 +0100 Subject: [PATCH 01/20] Use per-language CodeQL bundles Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/__bundle-toolcache.yml | 2 +- .../__per-language-bundle-validation.yml | 164 +++++ .github/workflows/codescanning-config-cli.yml | 3 +- lib/entry-points.js | 346 ++++++++-- pr-checks/checks/bundle-toolcache.yml | 2 +- .../checks/per-language-bundle-validation.yml | 117 ++++ pr-checks/sync.ts | 9 +- src/feature-flags.ts | 10 + src/init-action.ts | 8 + src/per-language-bundles.test.ts | 189 ++++++ src/per-language-bundles.ts | 142 ++++ src/setup-codeql-action.ts | 8 + src/setup-codeql.test.ts | 617 +++++++++++++++++- src/setup-codeql.ts | 219 ++++++- src/status-report.ts | 7 + src/tools-download.ts | 7 + 16 files changed, 1716 insertions(+), 134 deletions(-) create mode 100644 .github/workflows/__per-language-bundle-validation.yml create mode 100644 pr-checks/checks/per-language-bundle-validation.yml create mode 100644 src/per-language-bundles.test.ts create mode 100644 src/per-language-bundles.ts diff --git a/.github/workflows/__bundle-toolcache.yml b/.github/workflows/__bundle-toolcache.yml index 9cc983a843..d12aeb6e78 100644 --- a/.github/workflows/__bundle-toolcache.yml +++ b/.github/workflows/__bundle-toolcache.yml @@ -80,7 +80,7 @@ jobs: - id: init uses: ./../action/init with: - languages: javascript + languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - uses: ./../action/analyze with: diff --git a/.github/workflows/__per-language-bundle-validation.yml b/.github/workflows/__per-language-bundle-validation.yml new file mode 100644 index 0000000000..ea900a9e09 --- /dev/null +++ b/.github/workflows/__per-language-bundle-validation.yml @@ -0,0 +1,164 @@ +# Warning: This file is generated automatically, and should not be modified. +# Instead, please modify the template in the pr-checks directory and run: +# pr-checks/sync.sh +# to regenerate this file. + +name: PR Check - Per-language bundles +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GO111MODULE: auto +on: + push: + branches: + - main + - releases/v* + pull_request: {} + merge_group: + types: + - checks_requested + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + inputs: {} + workflow_call: + inputs: {} +defaults: + run: + shell: bash +concurrency: + cancel-in-progress: ${{ github.event_name == 'pull_request' || false }} + group: per-language-bundle-validation-${{github.ref}} +jobs: + per-language-bundle-validation: + strategy: + fail-fast: false + matrix: + include: + - language: actions + os: ubuntu-latest + version: nightly-latest + expected-extractors: actions javascript + - language: cpp + os: ubuntu-latest + version: nightly-latest + build-mode: manual + build-command: gcc -o main main.c + - language: csharp + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: go + os: ubuntu-latest + version: nightly-latest + build-mode: autobuild + - language: java + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: javascript + os: ubuntu-latest + version: nightly-latest + - language: python + os: ubuntu-latest + version: nightly-latest + - language: ruby + os: ubuntu-latest + version: nightly-latest + - language: rust + os: ubuntu-latest + version: nightly-latest + - language: swift + os: macos-latest-xlarge + version: nightly-latest + build-mode: autobuild + name: Per-language bundles + if: github.triggering_actor != 'dependabot[bot]' + permissions: + contents: read + security-events: read + timeout-minutes: 45 + runs-on: ${{ matrix.os }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Prepare test + id: prepare-test + uses: ./.github/actions/prepare-test + with: + version: ${{ matrix.version }} + use-all-platform-bundle: 'false' + setup-kotlin: 'true' + - uses: ./../action/init + id: init + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix['build-mode'] }} + tools: ${{ steps.prepare-test.outputs.tools-url }} + - name: Check that the bundle contains only the expected extractors + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + LANGUAGE: ${{ matrix.language }} + EXPECTED_EXTRACTORS: ${{ matrix['expected-extractors'] || matrix.language }} + run: | + extractors="$("$CODEQL_PATH" resolve languages --format=json | jq -r 'keys[]')" + echo "Extractors in the bundle:" + echo "$extractors" + echo "Expected: $EXPECTED_EXTRACTORS" + + for expected in $EXPECTED_EXTRACTORS; do + if ! echo "$extractors" | grep -qx "$expected"; then + echo "::error::The ${LANGUAGE} bundle does not contain the ${expected} extractor." + exit 1 + fi + done + + # If the bundle contained extractors beyond those the language needs, then it would not + # have been trimmed, and this job would be silently validating the combined bundle. + for other in actions cpp csharp go java javascript python ruby rust swift; do + if echo "$EXPECTED_EXTRACTORS" | grep -qw "$other"; then + continue + fi + if echo "$extractors" | grep -qx "$other"; then + echo "::error::The ${LANGUAGE} bundle also contains the ${other} extractor, so it is not trimmed." + exit 1 + fi + done + - name: Check that the bundle was not added to the toolcache + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + run: | + # A bundle that is missing most of its extractors must never be left in the toolcache, + # where a later job analyzing a different language could pick it up. The runner image + # ships with its own CodeQL in the toolcache, so check where this bundle was extracted to + # rather than whether the toolcache contains CodeQL at all. + echo "CodeQL is at $CODEQL_PATH" + if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then + echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH." + exit 1 + fi + if [[ "$CODEQL_PATH" != "$RUNNER_TEMP"/* ]]; then + echo "::error::Expected the per-language bundle to be extracted under $RUNNER_TEMP, but found it at $CODEQL_PATH." + exit 1 + fi + - name: Build code + if: matrix['build-command'] + run: ${{ matrix['build-command'] }} + - uses: ./../action/analyze + id: analysis + with: + upload-database: false + - name: Check that a database was created for the language + env: + DB_LOCATIONS: ${{ steps.analysis.outputs.db-locations }} + LANGUAGE: ${{ matrix.language }} + run: | + database="$(echo "$DB_LOCATIONS" | jq -r --arg lang "$LANGUAGE" '.[$lang] // empty')" + if [ -z "$database" ] || [ ! -d "$database" ]; then + echo "::error::No CodeQL database was created for ${LANGUAGE}." + echo "Databases: $DB_LOCATIONS" + exit 1 + fi + echo "Created a ${LANGUAGE} database at ${database}." + env: + CODEQL_ACTION_PER_LANGUAGE_BUNDLES: true + CODEQL_ACTION_TEST_MODE: true diff --git a/.github/workflows/codescanning-config-cli.yml b/.github/workflows/codescanning-config-cli.yml index 7bc6718e35..54474d58fb 100644 --- a/.github/workflows/codescanning-config-cli.yml +++ b/.github/workflows/codescanning-config-cli.yml @@ -75,7 +75,8 @@ jobs: uses: ./../action/.github/actions/check-codescanning-config with: expected-config-file-contents: "{}" - languages: javascript + # Request multiple languages so later checks can reuse the combined bundle. + languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - name: Packs from input diff --git a/lib/entry-points.js b/lib/entry-points.js index 4afc64403e..00f7ef9771 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -27216,8 +27216,8 @@ var require_gte = __commonJS({ "node_modules/semver/functions/gte.js"(exports2, module2) { "use strict"; var compare3 = require_compare(); - var gte7 = (a, b, loose) => compare3(a, b, loose) >= 0; - module2.exports = gte7; + var gte8 = (a, b, loose) => compare3(a, b, loose) >= 0; + module2.exports = gte8; } }); @@ -27238,7 +27238,7 @@ var require_cmp = __commonJS({ var eq = require_eq(); var neq = require_neq(); var gt = require_gt(); - var gte7 = require_gte(); + var gte8 = require_gte(); var lt2 = require_lt(); var lte2 = require_lte(); var cmp = (a, op, b, loose) => { @@ -27268,7 +27268,7 @@ var require_cmp = __commonJS({ case ">": return gt(a, b, loose); case ">=": - return gte7(a, b, loose); + return gte8(a, b, loose); case "<": return lt2(a, b, loose); case "<=": @@ -28076,7 +28076,7 @@ var require_outside = __commonJS({ var gt = require_gt(); var lt2 = require_lt(); var lte2 = require_lte(); - var gte7 = require_gte(); + var gte8 = require_gte(); var outside = (version, range2, hilo, options) => { version = new SemVer(version, options); range2 = new Range2(range2, options); @@ -28091,7 +28091,7 @@ var require_outside = __commonJS({ break; case "<": gtfn = lt2; - ltefn = gte7; + ltefn = gte8; ltfn = gt; comp = "<"; ecomp = "<="; @@ -28406,7 +28406,7 @@ var require_semver2 = __commonJS({ var lt2 = require_lt(); var eq = require_eq(); var neq = require_neq(); - var gte7 = require_gte(); + var gte8 = require_gte(); var lte2 = require_lte(); var cmp = require_cmp(); var coerce3 = require_coerce(); @@ -28445,7 +28445,7 @@ var require_semver2 = __commonJS({ lt: lt2, eq, neq, - gte: gte7, + gte: gte8, lte: lte2, cmp, coerce: coerce3, @@ -31721,7 +31721,7 @@ var require_brace_expansion = __commonJS({ function lte2(i, y) { return i <= y; } - function gte7(i, y) { + function gte8(i, y) { return i >= y; } function combine2(acc, base, pre, values, max, maxLength, dropEmpties, outBase) { @@ -31754,7 +31754,7 @@ var require_brace_expansion = __commonJS({ var reverse = y < x; if (reverse) { incr *= -1; - test = gte7; + test = gte8; } var pad = n.some(isPadded2); var length = 0; @@ -33901,8 +33901,8 @@ var require_semver3 = __commonJS({ function neq(a, b, loose) { return compare3(a, b, loose) !== 0; } - exports2.gte = gte7; - function gte7(a, b, loose) { + exports2.gte = gte8; + function gte8(a, b, loose) { return compare3(a, b, loose) >= 0; } exports2.lte = lte2; @@ -33933,7 +33933,7 @@ var require_semver3 = __commonJS({ case ">": return gt(a, b, loose); case ">=": - return gte7(a, b, loose); + return gte8(a, b, loose); case "<": return lt2(a, b, loose); case "<=": @@ -34478,7 +34478,7 @@ var require_semver3 = __commonJS({ break; case "<": gtfn = lt2; - ltefn = gte7; + ltefn = gte8; ltfn = gt; comp = "<"; ecomp = "<="; @@ -34699,7 +34699,7 @@ var require_cacheUtils = __commonJS({ var crypto3 = __importStar2(require("crypto")); var fs32 = __importStar2(require("fs")); var path30 = __importStar2(require("path")); - var semver11 = __importStar2(require_semver3()); + var semver12 = __importStar2(require_semver3()); var util3 = __importStar2(require("util")); var constants_1 = require_constants7(); var versionSalt = "1.0"; @@ -34792,7 +34792,7 @@ var require_cacheUtils = __commonJS({ function getCompressionMethod() { return __awaiter2(this, void 0, void 0, function* () { const versionOutput = yield getVersion("zstd", ["--quiet"]); - const version = semver11.clean(versionOutput); + const version = semver12.clean(versionOutput); core32.debug(`zstd version: ${version}`); if (versionOutput === "") { return constants_1.CompressionMethod.Gzip; @@ -82401,7 +82401,7 @@ var require_manifest = __commonJS({ exports2._findMatch = _findMatch; exports2._getOsVersion = _getOsVersion; exports2._readLinuxVersionFile = _readLinuxVersionFile; - var semver11 = __importStar2(require_semver2()); + var semver12 = __importStar2(require_semver2()); var core_1 = require_core(); var os7 = require("os"); var cp = require("child_process"); @@ -82415,7 +82415,7 @@ var require_manifest = __commonJS({ for (const candidate of candidates) { const version = candidate.version; (0, core_1.debug)(`check ${version} satisfies ${versionSpec}`); - if (semver11.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { + if (semver12.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { file = candidate.files.find((item) => { (0, core_1.debug)(`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`); let chk = item.arch === archFilter && item.platform === platFilter; @@ -82424,7 +82424,7 @@ var require_manifest = __commonJS({ if (osVersion === item.platform_version) { chk = true; } else { - chk = semver11.satisfies(osVersion, item.platform_version); + chk = semver12.satisfies(osVersion, item.platform_version); } } return chk; @@ -82684,7 +82684,7 @@ var require_tool_cache = __commonJS({ var os7 = __importStar2(require("os")); var path30 = __importStar2(require("path")); var httpm = __importStar2(require_lib()); - var semver11 = __importStar2(require_semver2()); + var semver12 = __importStar2(require_semver2()); var stream2 = __importStar2(require("stream")); var util3 = __importStar2(require("util")); var assert_1 = require("assert"); @@ -82957,7 +82957,7 @@ var require_tool_cache = __commonJS({ } function cacheDir2(sourceDir, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver11.clean(version) || version; + version = semver12.clean(version) || version; arch2 = arch2 || os7.arch(); core32.debug(`Caching tool ${tool} ${version} ${arch2}`); core32.debug(`source dir: ${sourceDir}`); @@ -82975,7 +82975,7 @@ var require_tool_cache = __commonJS({ } function cacheFile(sourceFile, targetFile, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver11.clean(version) || version; + version = semver12.clean(version) || version; arch2 = arch2 || os7.arch(); core32.debug(`Caching tool ${tool} ${version} ${arch2}`); core32.debug(`source file: ${sourceFile}`); @@ -83005,7 +83005,7 @@ var require_tool_cache = __commonJS({ } let toolPath = ""; if (versionSpec) { - versionSpec = semver11.clean(versionSpec) || ""; + versionSpec = semver12.clean(versionSpec) || ""; const cachePath = path30.join(_getCacheDirectory(), toolName, versionSpec, arch2); core32.debug(`checking cache: ${cachePath}`); if (fs32.existsSync(cachePath) && fs32.existsSync(`${cachePath}.complete`)) { @@ -83085,7 +83085,7 @@ var require_tool_cache = __commonJS({ } function _createToolPath(tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - const folderPath = path30.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); + const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || ""); core32.debug(`destination ${folderPath}`); const markerPath = `${folderPath}.complete`; yield io9.rmRF(folderPath); @@ -83095,15 +83095,15 @@ var require_tool_cache = __commonJS({ }); } function _completeToolPath(tool, version, arch2) { - const folderPath = path30.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); + const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || ""); const markerPath = `${folderPath}.complete`; fs32.writeFileSync(markerPath, ""); core32.debug("finished caching tool"); } function isExplicitVersion(versionSpec) { - const c = semver11.clean(versionSpec) || ""; + const c = semver12.clean(versionSpec) || ""; core32.debug(`isExplicit: ${c}`); - const valid4 = semver11.valid(c) != null; + const valid4 = semver12.valid(c) != null; core32.debug(`explicit? ${valid4}`); return valid4; } @@ -83111,14 +83111,14 @@ var require_tool_cache = __commonJS({ let version = ""; core32.debug(`evaluating ${versions.length} versions`); versions = versions.sort((a, b) => { - if (semver11.gt(a, b)) { + if (semver12.gt(a, b)) { return 1; } return -1; }); for (let i = versions.length - 1; i >= 0; i--) { const potential = versions[i]; - const satisfied = semver11.satisfies(potential, versionSpec); + const satisfied = semver12.satisfies(potential, versionSpec); if (satisfied) { version = potential; break; @@ -89595,7 +89595,7 @@ var require_brace_expansion2 = __commonJS({ function lte2(i, y) { return i <= y; } - function gte7(i, y) { + function gte8(i, y) { return i >= y; } function combine2(acc, pre, values, max, maxLength, dropEmpties) { @@ -89627,7 +89627,7 @@ var require_brace_expansion2 = __commonJS({ var reverse = y < x; if (reverse) { incr *= -1; - test = gte7; + test = gte8; } var pad = n.some(isPadded2); var length = 0; @@ -148091,6 +148091,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS", minimumVersion: void 0 }, + ["per_language_bundles" /* PerLanguageBundles */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_PER_LANGUAGE_BUNDLES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", @@ -151192,7 +151197,7 @@ var path13 = __toESM(require("path")); var core12 = __toESM(require_core()); var toolcache3 = __toESM(require_tool_cache()); var import_fast_deep_equal = __toESM(require_fast_deep_equal()); -var semver9 = __toESM(require_semver2()); +var semver10 = __toESM(require_semver2()); // src/overlay/caching.ts var fs11 = __toESM(require("fs")); @@ -151492,6 +151497,89 @@ async function getCodeQlVersionsForOverlayBaseDatabases(rawLanguages, logger) { return versions; } +// src/per-language-bundles.ts +var semver7 = __toESM(require_semver2()); +var MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; +var PER_LANGUAGE_BUNDLE_NAME = /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; +function tryGetBundleLanguageFromUrl(url2) { + let assetName; + try { + const pathname = new URL(url2).pathname; + assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); + } catch { + return void 0; + } + const match2 = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + return match2 ? parseBuiltInLanguage(match2[1]) : void 0; +} +var PER_LANGUAGE_BUNDLE_PLATFORMS = { + ["actions" /* actions */]: "linux64", + ["cpp" /* cpp */]: "linux64", + ["csharp" /* csharp */]: "linux64", + ["go" /* go */]: "linux64", + ["java" /* java */]: "linux64", + ["javascript" /* javascript */]: "linux64", + ["python" /* python */]: "linux64", + ["ruby" /* ruby */]: "linux64", + ["rust" /* rust */]: "linux64", + ["swift" /* swift */]: "osx64" +}; +async function getPerLanguageBundleLanguage(options, features, logger) { + const { + rawLanguages, + cliVersion: cliVersion2, + compressionMethod, + platform: platform2, + variant, + isNightly + } = options; + const explain = (reason) => { + logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); + return void 0; + }; + if (rawLanguages?.length !== 1) { + return explain( + `exactly one language must be requested via the 'languages' input, but ${rawLanguages?.length ?? 0} were` + ); + } + const language = parseBuiltInLanguage(rawLanguages[0]); + if (language === void 0) { + return explain(`'${rawLanguages[0]}' is not a known CodeQL language`); + } + if (compressionMethod !== "zstd") { + return explain(`the bundle would be downloaded as ${compressionMethod}`); + } + if (variant !== "GitHub.com" /* DOTCOM */) { + return explain(`we are running against ${variant}`); + } + if (!isGitHubHostedRunner()) { + return explain("the job is not running on a GitHub-hosted runner"); + } + if (!isNightly) { + if (cliVersion2 === void 0) { + return explain("the CLI version of the bundle is unknown"); + } + if (!semver7.gte(cliVersion2, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { + return explain( + `CodeQL ${cliVersion2} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the first version that publishes per-language bundles` + ); + } + } + const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language]; + if (supportedPlatform === void 0) { + return explain(`no per-language bundle is published for ${language}`); + } + if (supportedPlatform !== platform2) { + return explain( + `the ${language} bundle is only published for ${supportedPlatform}, but this job is running on ${platform2 ?? "an unknown platform"}` + ); + } + if (!await features.getValue("per_language_bundles" /* PerLanguageBundles */)) { + return explain(`the ${"per_language_bundles" /* PerLanguageBundles */} feature is disabled`); + } + return language; +} + // src/tar.ts var import_child_process = require("child_process"); var fs12 = __toESM(require("fs")); @@ -151499,7 +151587,7 @@ var stream = __toESM(require("stream")); var import_toolrunner = __toESM(require_toolrunner()); var io4 = __toESM(require_io()); var toolcache = __toESM(require_tool_cache()); -var semver7 = __toESM(require_semver2()); +var semver8 = __toESM(require_semver2()); var MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3"; var MIN_REQUIRED_GNU_TAR_VERSION = "1.31"; async function getTarVersion() { @@ -151541,9 +151629,9 @@ async function isZstdAvailable(logger) { case "gnu": return { available: foundZstdBinary && // GNU tar only uses major and minor version numbers - semver7.gte( - semver7.coerce(version), - semver7.coerce(MIN_REQUIRED_GNU_TAR_VERSION) + semver8.gte( + semver8.coerce(version), + semver8.coerce(MIN_REQUIRED_GNU_TAR_VERSION) ), foundZstdBinary, version: tarVersion @@ -151552,7 +151640,7 @@ async function isZstdAvailable(logger) { return { available: foundZstdBinary && // Do a loose comparison since these version numbers don't contain // a patch version number. - semver7.gte(version, MIN_REQUIRED_BSD_TAR_VERSION), + semver8.gte(version, MIN_REQUIRED_BSD_TAR_VERSION), foundZstdBinary, version: tarVersion }; @@ -151661,7 +151749,7 @@ var core11 = __toESM(require_core()); var import_http_client = __toESM(require_lib()); var toolcache2 = __toESM(require_tool_cache()); var import_follow_redirects = __toESM(require_follow_redirects()); -var semver8 = __toESM(require_semver2()); +var semver9 = __toESM(require_semver2()); var STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; var STREAMING_STALL_TIMEOUT_MS = 5 * 60 * 1e3; var TOOLCACHE_TOOL_NAME = "CodeQL"; @@ -151787,7 +151875,7 @@ function getToolcacheToolDirectory(env) { ); } function getToolcacheVersionDirectoryName(version) { - return semver8.clean(version) || version; + return semver9.clean(version) || version; } function getToolcacheDirectory(version) { return path12.join( @@ -151899,18 +151987,27 @@ function getCodeQLBundleExtension(compressionMethod) { assertNever(compressionMethod); } } -function getCodeQLBundleName(compressionMethod) { +function getBundlePlatform() { + switch (process.platform) { + case "win32": + return "win64"; + case "linux": + return process.arch === "arm64" ? "linux-arm64" : "linux64"; + case "darwin": + return "osx64"; + default: + return void 0; + } +} +function getCodeQLBundleName(compressionMethod, language) { const extension = getCodeQLBundleExtension(compressionMethod); - let platform2; - if (process.platform === "win32") { - platform2 = "win64"; - } else if (process.platform === "linux") { - platform2 = process.arch === "arm64" ? "linux-arm64" : "linux64"; - } else if (process.platform === "darwin") { - platform2 = "osx64"; - } else { + const platform2 = getBundlePlatform(); + if (platform2 === void 0) { return `codeql-bundle${extension}`; } + if (language !== void 0) { + return `codeql-bundle-${language}-${platform2}${extension}`; + } return `codeql-bundle-${platform2}${extension}`; } function getCodeQLActionRepository(logger) { @@ -151922,7 +152019,7 @@ function getCodeQLActionRepository(logger) { } return getRequiredEnvParam("GITHUB_ACTION_REPOSITORY"); } -async function getCodeQLBundleDownloadURL(tagName, apiDetails, compressionMethod, logger) { +async function getCodeQLBundleDownloadURL(tagName, apiDetails, codeQLBundleName, logger) { const codeQLActionRepository = getCodeQLActionRepository(logger); const potentialDownloadSources = [ // This GitHub instance, and this Action. @@ -151937,7 +152034,6 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, compressionMethod return !self2.slice(0, index2).some((other) => (0, import_fast_deep_equal.default)(source, other)); } ); - const codeQLBundleName = getCodeQLBundleName(compressionMethod); for (const downloadSource of uniqueDownloadSources) { const [apiURL, repository] = downloadSource; if (apiURL === GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) { @@ -151992,13 +152088,13 @@ function tryGetTagNameFromUrl(url2, logger) { return match2[1]; } function convertToSemVer(version, logger) { - if (!semver9.valid(version)) { + if (!semver10.valid(version)) { logger.debug( `Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.` ); version = `0.0.0-${version}`; } - const s = semver9.clean(version); + const s = semver10.clean(version); if (!s) { throw new Error(`Bundle version ${version} is not in SemVer format.`); } @@ -152126,6 +152222,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO let cliVersion2; let tagName; let url2; + let bundle; const canForceNightlyWithFF = isDynamicWorkflow() || isInTestMode(); const forceNightlyValueFF = await features.getValue("force_nightly" /* ForceNightly */); const forceNightly = forceNightlyValueFF && canForceNightlyWithFF; @@ -152156,7 +152253,8 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.` ); } - toolsInput = await getNightlyToolsUrl(logger); + bundle = await getNightlyBundle(rawLanguages, variant, features, logger); + toolsInput = bundle.url; } const forceShippedTools = toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput); if (forceShippedTools) { @@ -152207,7 +152305,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO url2 = toolsInput; if (tagName) { const bundleVersion3 = tryGetBundleVersionFromTagName(tagName, logger); - if (bundleVersion3 && semver9.valid(bundleVersion3)) { + if (bundleVersion3 && semver10.valid(bundleVersion3)) { cliVersion2 = convertToSemVer(bundleVersion3, logger); } } @@ -152310,12 +152408,38 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO let compressionMethod; if (!url2) { compressionMethod = cliVersion2 !== void 0 && await useZstdBundle(cliVersion2, tarSupportsZstd) ? "zstd" : "gzip"; - url2 = await getCodeQLBundleDownloadURL( + const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( + { + rawLanguages, + cliVersion: cliVersion2, + compressionMethod, + platform: getBundlePlatform(), + variant + }, + features, + logger + ); + const resolveBundleURL = (language) => getCodeQLBundleDownloadURL( tagName, apiDetails, - compressionMethod, + getCodeQLBundleName(compressionMethod, language), logger ); + if (perLanguageBundleLanguage !== void 0) { + logger.info( + `Downloading the ${perLanguageBundleLanguage} CodeQL bundle, since ${perLanguageBundleLanguage} is the only language being analyzed.` + ); + url2 = await resolveBundleURL(perLanguageBundleLanguage); + bundle = { + kind: "per-language", + url: url2, + language: perLanguageBundleLanguage, + combinedBundleURL: await resolveBundleURL() + }; + } else { + url2 = await resolveBundleURL(); + bundle = { kind: "combined", url: url2 }; + } } else { const method = inferCompressionMethod(url2); if (method === void 0) { @@ -152324,6 +152448,15 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO ); } compressionMethod = method; + if (bundle === void 0) { + const language = tryGetBundleLanguageFromUrl(url2); + bundle = language === void 0 ? { kind: "combined", url: url2 } : { kind: "per-language", url: url2, language }; + } + if (bundle.kind === "per-language") { + logger.info( + `${url2} appears to be a CodeQL bundle that contains only ${bundle.language}.` + ); + } } if (cliVersion2) { logger.info(`Using CodeQL CLI version ${cliVersion2} sourced from ${url2} .`); @@ -152331,7 +152464,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO logger.info(`Using CodeQL CLI sourced from ${url2} .`); } return { - bundle: { kind: "combined", url: url2 }, + bundle, bundleVersion: bundleVersion2, cliVersion: cliVersion2, compressionMethod, @@ -152383,7 +152516,7 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log writeToolcacheMarkerFile(toolcacheDestination, logger); } else { logger.debug( - `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${codeqlURL}.` + bundle.kind === "per-language" ? "Not caching the CodeQL tools because they came from a bundle that contains only a single language." : `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${codeqlURL}.` ); } return { @@ -152392,7 +152525,7 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log }; }; function getToolcacheDestination(source, logger) { - if (!source.bundleVersion) { + if (source.bundle.kind !== "combined" || !source.bundleVersion) { return void 0; } return getToolcacheDirectory( @@ -152497,24 +152630,76 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau }; } async function downloadCodeQLBundle(source, apiDetails, tarVersion, tempDir, features, logger) { + const { bundle } = source; await tryDeleteToolcacheBundles({ env: getEnv(), features, logger }); - return await downloadCodeQL(source, apiDetails, tarVersion, tempDir, logger); + try { + const result = await downloadCodeQL( + source, + apiDetails, + tarVersion, + tempDir, + logger + ); + return bundle.kind === "combined" ? result : { + ...result, + statusReport: { + ...result.statusReport, + bundleLanguage: bundle.language + } + }; + } catch (e) { + if (bundle.kind !== "per-language" || bundle.combinedBundleURL === void 0 || asHTTPError(e)?.status !== 404) { + throw e; + } + logger.warning( + `No ${bundle.language} CodeQL bundle was found at ${bundle.url}, so falling back to the bundle that contains all languages. This analysis will still produce correct results, but will take longer to set up.` + ); + const result = await downloadCodeQL( + { + ...source, + bundle: { kind: "combined", url: bundle.combinedBundleURL } + }, + apiDetails, + tarVersion, + tempDir, + logger + ); + return { + ...result, + statusReport: { + ...result.statusReport, + perLanguageBundleFallback: true + } + }; + } } async function useZstdBundle(cliVersion2, tarSupportsZstd) { return ( // In testing, gzip performs better than zstd on Windows. - process.platform !== "win32" && tarSupportsZstd && semver9.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE) + process.platform !== "win32" && tarSupportsZstd && semver10.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE) ); } function getTempExtractionDir(tempDir) { return path13.join(tempDir, v4_default()); } -async function getNightlyToolsUrl(logger) { +async function getNightlyBundle(rawLanguages, variant, features, logger) { const zstdAvailability = await isZstdAvailable(logger); const compressionMethod = await useZstdBundle( CODEQL_VERSION_ZSTD_BUNDLE, zstdAvailability.available ) ? "zstd" : "gzip"; + const language = await getPerLanguageBundleLanguage( + { + rawLanguages, + cliVersion: void 0, + compressionMethod, + platform: getBundlePlatform(), + variant, + isNightly: true + }, + features, + logger + ); try { const release2 = await getApiClient().rest.repos.listReleases({ owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER, @@ -152527,7 +152712,14 @@ async function getNightlyToolsUrl(logger) { if (!latestRelease) { throw new Error("Could not find the latest nightly release."); } - return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`; + const assetUrl = (name) => `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`; + const url2 = assetUrl(getCodeQLBundleName(compressionMethod, language)); + return language === void 0 ? { kind: "combined", url: url2 } : { + kind: "per-language", + url: url2, + language, + combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod)) + }; } catch (e) { throw new Error( `Failed to retrieve the latest nightly release: ${wrapError(e)}` @@ -152535,7 +152727,7 @@ async function getNightlyToolsUrl(logger) { } } function getLatestToolcacheVersion(logger) { - const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver9.compare(b, a)); + const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver10.compare(b, a)); logger.debug( `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( allVersions @@ -156719,7 +156911,7 @@ function isPadded(el) { function lte(i, y) { return i <= y; } -function gte6(i, y) { +function gte7(i, y) { return i >= y; } function combine(acc, pre, values, max, maxLength, dropEmpties) { @@ -156754,7 +156946,7 @@ function expandSequence(body, isAlphaSequence, max, maxLength) { const reverse = y < x; if (reverse) { incr *= -1; - test = gte6; + test = gte7; } const pad = n.some(isPadded); let length = 0; @@ -158651,7 +158843,7 @@ var import_async = __toESM(require_async(), 1); var import_path7 = require("path"); // node_modules/archiver/lib/error.js -var import_util34 = __toESM(require("util"), 1); +var import_util35 = __toESM(require("util"), 1); var ERROR_CODES = { ABORTED: "archive was aborted", DIRECTORYDIRPATHREQUIRED: "diretory dirpath argument must be a non-empty string value", @@ -158676,7 +158868,7 @@ function ArchiverError(code, data) { this.code = code; this.data = data; } -import_util34.default.inherits(ArchiverError, Error); +import_util35.default.inherits(ArchiverError, Error); // node_modules/archiver/lib/core.js var import_readable_stream2 = __toESM(require_ours(), 1); @@ -161608,7 +161800,7 @@ var fs29 = __toESM(require("fs")); var path25 = __toESM(require("path")); var core22 = __toESM(require_core()); var io7 = __toESM(require_io()); -var semver10 = __toESM(require_semver2()); +var semver11 = __toESM(require_semver2()); // src/config/inputs.ts async function getToolsInput(action, repositoryProperties) { @@ -161969,6 +162161,12 @@ async function sendCompletedStatusReport2(startedAt, config, configFile, toolsIn if (toolsDownloadStatusReport?.totalDurationMs !== void 0) { initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; } + if (toolsDownloadStatusReport?.bundleLanguage !== void 0) { + initToolsDownloadFields.tools_bundle_language = toolsDownloadStatusReport.bundleLanguage; + } + if (toolsDownloadStatusReport?.perLanguageBundleFallback !== void 0) { + initToolsDownloadFields.tools_per_language_bundle_fallback = toolsDownloadStatusReport.perLanguageBundleFallback; + } if (toolsFeatureFlagsValid !== void 0) { initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; } @@ -162088,12 +162286,12 @@ async function run3(actionState) { const experimental = "2.19.3"; const publicPreview = "2.22.1"; const actualVer = (await codeql.getVersion()).version; - if (semver10.lt(actualVer, experimental)) { + if (semver11.lt(actualVer, experimental)) { throw new ConfigurationError( `Rust analysis is supported by CodeQL CLI version ${experimental} or higher, but found version ${actualVer}` ); } - if (semver10.lt(actualVer, publicPreview)) { + if (semver11.lt(actualVer, publicPreview)) { core22.exportVariable("CODEQL_ENABLE_EXPERIMENTAL_FEATURES" /* EXPERIMENTAL_FEATURES */, "true"); logger.info("Experimental Rust analysis enabled"); } @@ -163017,6 +163215,12 @@ async function sendCompletedStatusReport3(startedAt, toolsInput, toolsDownloadSt if (toolsDownloadStatusReport?.totalDurationMs !== void 0) { initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; } + if (toolsDownloadStatusReport?.bundleLanguage !== void 0) { + initToolsDownloadFields.tools_bundle_language = toolsDownloadStatusReport.bundleLanguage; + } + if (toolsDownloadStatusReport?.perLanguageBundleFallback !== void 0) { + initToolsDownloadFields.tools_per_language_bundle_fallback = toolsDownloadStatusReport.perLanguageBundleFallback; + } if (toolsFeatureFlagsValid !== void 0) { initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; } diff --git a/pr-checks/checks/bundle-toolcache.yml b/pr-checks/checks/bundle-toolcache.yml index 83d1d7d0b5..efa1a4d76f 100644 --- a/pr-checks/checks/bundle-toolcache.yml +++ b/pr-checks/checks/bundle-toolcache.yml @@ -30,7 +30,7 @@ steps: - id: init uses: ./../action/init with: - languages: javascript + languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - uses: ./../action/analyze with: diff --git a/pr-checks/checks/per-language-bundle-validation.yml b/pr-checks/checks/per-language-bundle-validation.yml new file mode 100644 index 0000000000..21fe33e757 --- /dev/null +++ b/pr-checks/checks/per-language-bundle-validation.yml @@ -0,0 +1,117 @@ +name: Per-language bundles +description: Validates extraction and analysis using each per-language CodeQL bundle. +# TODO: Use a released bundle once releases include per-language bundles. +matrix: + include: + - language: actions + os: ubuntu-latest + version: nightly-latest + # Actions also needs the JavaScript extractor. + expected-extractors: actions javascript + - language: cpp + os: ubuntu-latest + version: nightly-latest + build-mode: manual + build-command: gcc -o main main.c + - language: csharp + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: go + os: ubuntu-latest + version: nightly-latest + build-mode: autobuild + - language: java + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: javascript + os: ubuntu-latest + version: nightly-latest + - language: python + os: ubuntu-latest + version: nightly-latest + - language: ruby + os: ubuntu-latest + version: nightly-latest + - language: rust + os: ubuntu-latest + version: nightly-latest + - language: swift + os: macos-latest-xlarge + version: nightly-latest + build-mode: autobuild +env: + CODEQL_ACTION_PER_LANGUAGE_BUNDLES: true +steps: + - uses: ./../action/init + id: init + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix['build-mode'] }} + tools: ${{ steps.prepare-test.outputs.tools-url }} + - name: Check that the bundle contains only the expected extractors + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + LANGUAGE: ${{ matrix.language }} + EXPECTED_EXTRACTORS: ${{ matrix['expected-extractors'] || matrix.language }} + run: | + extractors="$("$CODEQL_PATH" resolve languages --format=json | jq -r 'keys[]')" + echo "Extractors in the bundle:" + echo "$extractors" + echo "Expected: $EXPECTED_EXTRACTORS" + + for expected in $EXPECTED_EXTRACTORS; do + if ! echo "$extractors" | grep -qx "$expected"; then + echo "::error::The ${LANGUAGE} bundle does not contain the ${expected} extractor." + exit 1 + fi + done + + # If the bundle contained extractors beyond those the language needs, then it would not + # have been trimmed, and this job would be silently validating the combined bundle. + for other in actions cpp csharp go java javascript python ruby rust swift; do + if echo "$EXPECTED_EXTRACTORS" | grep -qw "$other"; then + continue + fi + if echo "$extractors" | grep -qx "$other"; then + echo "::error::The ${LANGUAGE} bundle also contains the ${other} extractor, so it is not trimmed." + exit 1 + fi + done + - name: Check that the bundle was not added to the toolcache + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + run: | + # A bundle that is missing most of its extractors must never be left in the toolcache, + # where a later job analyzing a different language could pick it up. The runner image + # ships with its own CodeQL in the toolcache, so check where this bundle was extracted to + # rather than whether the toolcache contains CodeQL at all. + echo "CodeQL is at $CODEQL_PATH" + if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then + echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH." + exit 1 + fi + if [[ "$CODEQL_PATH" != "$RUNNER_TEMP"/* ]]; then + echo "::error::Expected the per-language bundle to be extracted under $RUNNER_TEMP, but found it at $CODEQL_PATH." + exit 1 + fi + - name: Build code + if: matrix['build-command'] + run: ${{ matrix['build-command'] }} + - uses: ./../action/analyze + id: analysis + with: + upload-database: false + - name: Check that a database was created for the language + env: + DB_LOCATIONS: ${{ steps.analysis.outputs.db-locations }} + LANGUAGE: ${{ matrix.language }} + run: | + database="$(echo "$DB_LOCATIONS" | jq -r --arg lang "$LANGUAGE" '.[$lang] // empty')" + if [ -z "$database" ] || [ ! -d "$database" ]; then + echo "::error::No CodeQL database was created for ${LANGUAGE}." + echo "Databases: $DB_LOCATIONS" + exit 1 + fi + echo "Created a ${LANGUAGE} database at ${database}." diff --git a/pr-checks/sync.ts b/pr-checks/sync.ts index 6dde1ee48e..f0942ad2dd 100755 --- a/pr-checks/sync.ts +++ b/pr-checks/sync.ts @@ -79,6 +79,8 @@ interface Specification extends JobSpecification { useAllPlatformBundle?: string; /** Values for the `analysis-kinds` matrix dimension. */ analysisKinds?: string[]; + /** Overrides the generated job matrix using GitHub Actions matrix syntax. */ + matrix?: Record; /** Container image configuration for the job. */ container?: any; @@ -512,9 +514,6 @@ function generateJob( specDocument: yaml.Document, checkSpecification: Specification, ) { - const matrix: Array> = - generateJobMatrix(checkSpecification); - const useAllPlatformBundle = checkSpecification.useAllPlatformBundle ? checkSpecification.useAllPlatformBundle : "false"; @@ -567,8 +566,8 @@ function generateJob( const checkJob: Record = { strategy: { "fail-fast": false, - matrix: { - include: matrix, + matrix: checkSpecification.matrix ?? { + include: generateJobMatrix(checkSpecification), }, }, name: checkSpecification.name, diff --git a/src/feature-flags.ts b/src/feature-flags.ts index da7bcceade..afddaea2a4 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -164,6 +164,11 @@ export enum Feature { OverlayAnalysisStatusCheck = "overlay_analysis_status_check", /** Controls whether overlay build failures on the default branch are stored in the Actions cache. */ OverlayAnalysisStatusSave = "overlay_analysis_status_save", + /** + * Controls whether we may download a bundle containing only the single language being analysed, + * rather than the combined bundle that contains every language. + */ + PerLanguageBundles = "per_language_bundles", QaTelemetryEnabled = "qa_telemetry_enabled", /** Routes (some) API requests through the registry proxy. */ ProxyApiRequests = "proxy_api_requests", @@ -434,6 +439,11 @@ export const featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS", minimumVersion: undefined, }, + [Feature.PerLanguageBundles]: { + defaultValue: false, + envVar: "CODEQL_ACTION_PER_LANGUAGE_BUNDLES", + minimumVersion: undefined, + }, [Feature.QaTelemetryEnabled]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/src/init-action.ts b/src/init-action.ts index 8173d67aaa..dd576548dc 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -182,6 +182,14 @@ async function sendCompletedStatusReport( initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; } + if (toolsDownloadStatusReport?.bundleLanguage !== undefined) { + initToolsDownloadFields.tools_bundle_language = + toolsDownloadStatusReport.bundleLanguage; + } + if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) { + initToolsDownloadFields.tools_per_language_bundle_fallback = + toolsDownloadStatusReport.perLanguageBundleFallback; + } if (toolsFeatureFlagsValid !== undefined) { initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; } diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts new file mode 100644 index 0000000000..8242bd0144 --- /dev/null +++ b/src/per-language-bundles.test.ts @@ -0,0 +1,189 @@ +import test from "ava"; + +import { ActionsEnvVars } from "./environment"; +import { Feature } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; +import { getRunnerLogger } from "./logging"; +import { + getPerLanguageBundleLanguage, + MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + PerLanguageBundleOptions, + tryGetBundleLanguageFromUrl, +} from "./per-language-bundles"; +import { createFeatures, setupTests } from "./testing-utils"; +import { GitHubVariant } from "./util"; + +setupTests(test); + +/** Options for which we would use a per-language bundle. */ +const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = { + rawLanguages: ["java"], + // Any version at least as new as the minimum will do. + cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + compressionMethod: "zstd", + platform: "linux64", + variant: GitHubVariant.DOTCOM, +}; + +async function checkEligibility( + overrides: Partial, + enabledFeatures: Feature[] = [Feature.PerLanguageBundles], +) { + return getPerLanguageBundleLanguage( + { ...ELIGIBLE_OPTIONS, ...overrides }, + createFeatures(enabledFeatures), + getRunnerLogger(true), + ); +} + +test.beforeEach(() => { + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; +}); + +test.serial("uses Linux bundles for non-Swift languages", async (t) => { + for (const language of Object.values(BuiltInLanguage)) { + if (language === BuiltInLanguage.swift) { + continue; + } + t.is(await checkEligibility({ rawLanguages: [language] }), language); + } +}); + +test.serial("normalizes an alias before selecting a bundle", async (t) => { + t.is( + await checkEligibility({ rawLanguages: ["java-kotlin"] }), + BuiltInLanguage.java, + ); +}); + +test.serial("uses the macOS bundle for Swift", async (t) => { + t.is( + await checkEligibility({ rawLanguages: ["swift"], platform: "osx64" }), + BuiltInLanguage.swift, + ); + // Swift is only published for macOS. + t.is( + await checkEligibility({ rawLanguages: ["swift"], platform: "linux64" }), + undefined, + ); +}); + +test.serial("only publishes non-Swift languages for Linux", async (t) => { + t.is(await checkEligibility({ platform: "osx64" }), undefined); + t.is(await checkEligibility({ platform: "win64" }), undefined); + // We do not publish per-language bundles for Linux Arm64 either. + t.is(await checkEligibility({ platform: "linux-arm64" }), undefined); + t.is(await checkEligibility({ platform: undefined }), undefined); +}); + +test.serial("requires exactly one language", async (t) => { + t.is(await checkEligibility({ rawLanguages: undefined }), undefined); + t.is(await checkEligibility({ rawLanguages: [] }), undefined); + t.is(await checkEligibility({ rawLanguages: ["java", "python"] }), undefined); +}); + +test.serial("requires a language that CodeQL knows about", async (t) => { + t.is(await checkEligibility({ rawLanguages: ["cobol"] }), undefined); +}); + +test.serial("requires a zstd bundle", async (t) => { + t.is(await checkEligibility({ compressionMethod: "gzip" }), undefined); +}); + +test.serial("requires GitHub.com", async (t) => { + // Other products resolve the combined bundle against their own instance, so asking for a + // per-language bundle they do not mirror would move the download off that instance. + for (const variant of [GitHubVariant.GHES, GitHubVariant.GHEC_DR]) { + t.is(await checkEligibility({ variant }), undefined); + } +}); + +test.serial("requires a GitHub-hosted runner", async (t) => { + // A self-hosted runner may have a toolcache that persists between jobs, which is worth more than + // a smaller download. + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; + t.is(await checkEligibility({}), undefined); + + // Self-hosted runners are routinely configured to look like hosted ones, for example by mounting + // a persistent volume at `/opt/hostedtoolcache`, so we require the service to tell us explicitly. + delete process.env[ActionsEnvVars.RUNNER_ENVIRONMENT]; + process.env["RUNNER_TOOL_CACHE"] = "/opt/hostedtoolcache"; + t.is(await checkEligibility({}), undefined); +}); + +test.serial("requires a new enough CLI version", async (t) => { + t.is(await checkEligibility({ cliVersion: undefined }), undefined); + t.is(await checkEligibility({ cliVersion: "2.27.0" }), undefined); + t.is(await checkEligibility({ cliVersion: "2.27.1" }), BuiltInLanguage.java); +}); + +test.serial("requires the feature flag", async (t) => { + t.is(await checkEligibility({}, []), undefined); +}); + +test.serial("nightlies skip only the release version check", async (t) => { + const nightly = { isNightly: true, cliVersion: undefined }; + t.is(await checkEligibility(nightly), BuiltInLanguage.java); + + for (const overrides of [ + { rawLanguages: undefined }, + { rawLanguages: ["java", "python"] }, + { compressionMethod: "gzip" as const }, + { platform: "osx64" }, + { variant: GitHubVariant.GHES }, + { variant: GitHubVariant.GHEC_DR }, + ]) { + t.is(await checkEligibility({ ...nightly, ...overrides }), undefined); + } + t.is(await checkEligibility(nightly, []), undefined); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; + t.is(await checkEligibility(nightly), undefined); +}); + +test.serial("recognizes a per-language bundle from its URL", (t) => { + const url = (name: string) => + `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; + + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-java-linux64.tar.zst")), + BuiltInLanguage.java, + ); + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-swift-osx64.tar.zst")), + BuiltInLanguage.swift, + ); + // We do not publish these, but should still recognize them if we ever do. + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-csharp-win64.tar.gz")), + BuiltInLanguage.csharp, + ); + // A percent-encoded name resolves to the same asset, so it must not let a bundle that contains a + // single language pass for one that contains them all and end up in the toolcache. + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-%70ython-linux64.tar.zst")), + BuiltInLanguage.python, + ); +}); + +test.serial("does not mistake other bundles for per-language ones", (t) => { + const url = (name: string) => + `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; + + for (const name of [ + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-osx64.tar.gz", + "codeql-bundle-win64.tar.zst", + // The all-platform bundle. + "codeql-bundle.tar.gz", + // A platform we do not publish per-language bundles for, whose name also contains a hyphen. + "codeql-bundle-linux-arm64.tar.zst", + // Not a language we know about. + "codeql-bundle-cobol-linux64.tar.zst", + // A name we cannot decode must not be mistaken for a language either. + "codeql-bundle-%zz-linux64.tar.zst", + ]) { + t.is(tryGetBundleLanguageFromUrl(url(name)), undefined, name); + } + + t.is(tryGetBundleLanguageFromUrl("not a url"), undefined); +}); diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts new file mode 100644 index 0000000000..1b63e4f100 --- /dev/null +++ b/src/per-language-bundles.ts @@ -0,0 +1,142 @@ +import * as semver from "semver"; + +import { isGitHubHostedRunner } from "./actions-util"; +import { Feature, FeatureEnablement } from "./feature-flags"; +import { BuiltInLanguage, parseBuiltInLanguage } from "./languages"; +import { Logger } from "./logging"; +import * as tar from "./tar"; +import { GitHubVariant } from "./util"; + +/** Minimum CLI version for selecting a per-language release bundle. */ +export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; + +const PER_LANGUAGE_BUNDLE_NAME = + /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; + +/** Identifies per-language tools URLs that must not populate the toolcache. */ +export function tryGetBundleLanguageFromUrl( + url: string, +): BuiltInLanguage | undefined { + let assetName: string; + try { + const pathname = new URL(url).pathname; + // URL-encoded names must not bypass the toolcache safeguard. + assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); + } catch { + return undefined; + } + + const match = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + return match ? parseBuiltInLanguage(match[1]) : undefined; +} + +/** Published platform for each language; absent entries are ineligible. */ +const PER_LANGUAGE_BUNDLE_PLATFORMS: Readonly< + Partial> +> = { + [BuiltInLanguage.actions]: "linux64", + [BuiltInLanguage.cpp]: "linux64", + [BuiltInLanguage.csharp]: "linux64", + [BuiltInLanguage.go]: "linux64", + [BuiltInLanguage.java]: "linux64", + [BuiltInLanguage.javascript]: "linux64", + [BuiltInLanguage.python]: "linux64", + [BuiltInLanguage.ruby]: "linux64", + [BuiltInLanguage.rust]: "linux64", + [BuiltInLanguage.swift]: "osx64", +}; + +/** Inputs that determine whether we may download a per-language bundle. */ +export interface PerLanguageBundleOptions { + /** Explicit input only: autodetection needs a CLI instance. */ + rawLanguages: string[] | undefined; + /** CLI version, if known. Ignored for nightly bundles. */ + cliVersion: string | undefined; + compressionMethod: tar.CompressionMethod; + /** Bundle platform identifier, such as linux64. */ + platform: string | undefined; + variant: GitHubVariant; + isNightly?: boolean; +} + +/** Returns the eligible bundle language, or undefined for the combined bundle. */ +export async function getPerLanguageBundleLanguage( + options: PerLanguageBundleOptions, + features: FeatureEnablement, + logger: Logger, +): Promise { + const { + rawLanguages, + cliVersion, + compressionMethod, + platform, + variant, + isNightly, + } = options; + + const explain = (reason: string) => { + logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); + return undefined; + }; + + if (rawLanguages?.length !== 1) { + return explain( + `exactly one language must be requested via the 'languages' input, but ${ + rawLanguages?.length ?? 0 + } were`, + ); + } + + const language = parseBuiltInLanguage(rawLanguages[0]); + if (language === undefined) { + return explain(`'${rawLanguages[0]}' is not a known CodeQL language`); + } + + if (compressionMethod !== "zstd") { + // Per-language bundles are only published as zstd archives. + return explain(`the bundle would be downloaded as ${compressionMethod}`); + } + + if (variant !== GitHubVariant.DOTCOM) { + // Tenant mirrors may lack these assets, and an unreachable github.com fails with a + // connection error rather than a recoverable 404. + return explain(`we are running against ${variant}`); + } + + if (!isGitHubHostedRunner()) { + // Per-language installs stay out of the toolcache; self-hosted runners should retain + // the reusable combined bundle instead. + return explain("the job is not running on a GitHub-hosted runner"); + } + + // Nightly tags contain dates rather than comparable CLI versions. + if (!isNightly) { + if (cliVersion === undefined) { + return explain("the CLI version of the bundle is unknown"); + } + + if (!semver.gte(cliVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { + return explain( + `CodeQL ${cliVersion} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the ` + + "first version that publishes per-language bundles", + ); + } + } + + const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language]; + if (supportedPlatform === undefined) { + return explain(`no per-language bundle is published for ${language}`); + } + if (supportedPlatform !== platform) { + return explain( + `the ${language} bundle is only published for ${supportedPlatform}, but this job is ` + + `running on ${platform ?? "an unknown platform"}`, + ); + } + + if (!(await features.getValue(Feature.PerLanguageBundles))) { + return explain(`the ${Feature.PerLanguageBundles} feature is disabled`); + } + + return language; +} diff --git a/src/setup-codeql-action.ts b/src/setup-codeql-action.ts index bb6b73c9aa..3c2a191e7b 100644 --- a/src/setup-codeql-action.ts +++ b/src/setup-codeql-action.ts @@ -93,6 +93,14 @@ async function sendCompletedStatusReport( initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; } + if (toolsDownloadStatusReport?.bundleLanguage !== undefined) { + initToolsDownloadFields.tools_bundle_language = + toolsDownloadStatusReport.bundleLanguage; + } + if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) { + initToolsDownloadFields.tools_per_language_bundle_fallback = + toolsDownloadStatusReport.perLanguageBundleFallback; + } if (toolsFeatureFlagsValid !== undefined) { initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; } diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index f4dbc9d809..c7fa92abad 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -12,8 +12,10 @@ import * as api from "./api-client"; import * as diagnostics from "./diagnostics"; import { ActionsEnvVars, EnvVar, ReadOnlyEnv } from "./environment"; import { Feature } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; import { getRunnerLogger } from "./logging"; import { getCacheRestoreKeyPrefix } from "./overlay/caching"; +import { MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION } from "./per-language-bundles"; import * as setupCodeql from "./setup-codeql"; import * as tar from "./tar"; import { @@ -54,6 +56,25 @@ function stubDownloadAndExtract() { }); } +function stubHostedNightly(tagName: string) { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const client = github.getOctokit("123", { + request: { + fetch: async () => + new Response(JSON.stringify([{ tag_name: tagName }]), { + headers: { "content-type": "application/json" }, + }), + }, + }); + sinon.stub(api, "getApiClient").value(() => client); +} + test.serial("parse codeql bundle url version", (t) => { t.deepEqual( setupCodeql.getCodeQLURLVersion( @@ -373,20 +394,7 @@ test.serial( const expectedDate = "30260213"; const expectedTag = `codeql-bundle-${expectedDate}`; - // Ensure that we consistently select "zstd" for the test. - sinon.stub(process, "platform").value("linux"); - sinon.stub(tar, "isZstdAvailable").resolves({ - available: true, - foundZstdBinary: true, - }); - - const client = github.getOctokit("123"); - const listReleases = sinon.stub(client.rest.repos, "listReleases"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - listReleases.resolves({ - data: [{ tag_name: expectedTag }], - } as any); - sinon.stub(api, "getApiClient").value(() => client); + stubHostedNightly(expectedTag); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); @@ -455,20 +463,7 @@ test.serial( const expectedDate = "30260213"; const expectedTag = `codeql-bundle-${expectedDate}`; - // Ensure that we consistently select "zstd" for the test. - sinon.stub(process, "platform").value("linux"); - sinon.stub(tar, "isZstdAvailable").resolves({ - available: true, - foundZstdBinary: true, - }); - - const client = github.getOctokit("123"); - const listReleases = sinon.stub(client.rest.repos, "listReleases"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - listReleases.resolves({ - data: [{ tag_name: expectedTag }], - } as any); - sinon.stub(api, "getApiClient").value(() => client); + stubHostedNightly(expectedTag); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" }); @@ -512,6 +507,8 @@ for (const bundlePath of [ "codeql-bundle.tar.gz", "codeql-bundle.tar.zst", "codeql-bundle-/codeql-bundle.tar.gz", + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-ruby-linux64.tar.zst", ]) { test.serial( `setupCodeQLBundle reports an unknown version for ${bundlePath}`, @@ -540,6 +537,12 @@ for (const bundlePath of [ t.is(downloadSpy.firstCall.args[0].toolsVersion, "unknown"); t.is(result.toolsVersion, "unknown"); t.is(result.toolsSource, setupCodeql.ToolsSource.Download); + t.is( + result.toolsDownloadStatusReport?.bundleLanguage, + bundlePath === "codeql-bundle-ruby-linux64.tar.zst" + ? BuiltInLanguage.ruby + : undefined, + ); t.is(path.dirname(result.codeqlFolder), tmpDir); t.true(fs.existsSync(result.codeqlFolder)); t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); @@ -592,6 +595,131 @@ test.serial( }, ); +for (const toolsInput of ["nightly", "nightly-latest"]) { + test.serial( + `getCodeQLSource selects a per-language bundle for tools == ${toolsInput}`, + async (t) => { + const expectedTag = "codeql-bundle-30260213"; + const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; + stubHostedNightly(expectedTag); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + toolsInput, + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.deepEqual(source, { + sourceType: "download", + bundle: { + kind: "per-language", + language: BuiltInLanguage.java, + url: `${baseURL}/codeql-bundle-java-linux64.tar.zst`, + combinedBundleURL: `${baseURL}/codeql-bundle-linux64.tar.zst`, + }, + bundleVersion: "30260213", + cliVersion: undefined, + compressionMethod: "zstd", + toolsVersion: "0.0.0-30260213", + } satisfies setupCodeql.CodeQLDownloadSource); + }); + }, + ); +} + +test.serial( + "getCodeQLSource downloads the combined nightly bundle when not eligible", + async (t) => { + const expectedTag = "codeql-bundle-30260213"; + stubHostedNightly(expectedTag); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + for (const { languages, features } of [ + { languages: ["java"], features: createFeatures([]) }, + { + languages: ["java", "python"], + features: createFeatures([Feature.PerLanguageBundles]), + }, + ]) { + const source = await setupCodeql.getCodeQLSource( + "nightly", + SAMPLE_DEFAULT_CLI_VERSION, + languages, + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + features, + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + t.deepEqual(source.bundle, { + kind: "combined", + url: `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/codeql-bundle-linux64.tar.zst`, + }); + } + } + }); + }, +); + +for (const perLanguageBundles of [false, true]) { + test.serial( + `getCodeQLSource uses a ${perLanguageBundles ? "per-language" : "combined"} bundle for a forced nightly`, + async (t) => { + const expectedTag = "codeql-bundle-30260213"; + const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; + stubHostedNightly(expectedTag); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" }); + const source = await setupCodeql.getCodeQLSource( + undefined, // toolsInput: the nightly is selected by ForceNightly + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures( + perLanguageBundles + ? [Feature.ForceNightly, Feature.PerLanguageBundles] + : [Feature.ForceNightly], + ), + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + const combinedURL = `${baseURL}/codeql-bundle-linux64.tar.zst`; + t.deepEqual( + source.bundle, + perLanguageBundles + ? { + kind: "per-language", + language: BuiltInLanguage.java, + url: `${baseURL}/codeql-bundle-java-linux64.tar.zst`, + combinedBundleURL: combinedURL, + } + : { kind: "combined", url: combinedURL }, + ); + } + }); + }, + ); +} + test.serial( "getCodeQLSource correctly returns latest version from toolcache when tools == toolcache", async (t) => { @@ -876,6 +1004,439 @@ test.serial( }, ); +const PER_LANGUAGE_CLI_VERSION = { + enabledVersions: [ + { + cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + tagName: `codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`, + }, + ], +}; + +test.serial("getCodeQLBundleName names the per-language bundle", (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + t.is( + setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.java), + "codeql-bundle-java-linux64.tar.zst", + ); + t.is( + setupCodeql.getCodeQLBundleName("zstd"), + "codeql-bundle-linux64.tar.zst", + ); +}); + +test.serial("getCodeQLBundleName names the Swift bundle for macOS", (t) => { + sinon.stub(process, "platform").value("darwin"); + t.is( + setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.swift), + "codeql-bundle-swift-osx64.tar.zst", + ); +}); + +test.serial( + "getCodeQLSource downloads the per-language bundle for a single explicit language", + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + undefined, + PER_LANGUAGE_CLI_VERSION, + ["java-kotlin"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + t.true( + source.bundle.url.endsWith("/codeql-bundle-java-linux64.tar.zst"), + `Unexpected URL ${source.bundle.url}`, + ); + t.is(source.bundle.kind, "per-language"); + if (source.bundle.kind === "per-language") { + t.is(source.bundle.language, BuiltInLanguage.java); + t.true( + source.bundle.combinedBundleURL?.endsWith( + "/codeql-bundle-linux64.tar.zst", + ), + ); + } + } + }); + }, +); + +test.serial( + "getCodeQLSource downloads the combined bundle when the feature is disabled", + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + undefined, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([]), + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + t.true(source.bundle.url.endsWith("/codeql-bundle-linux64.tar.zst")); + t.is(source.bundle.kind, "combined"); + } + }); + }, +); + +for (const fallback of [false, true]) { + test.serial( + `setupCodeQLBundle retains the selected release identity for an opaque asset URL${fallback ? " with fallback" : ""}`, + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + sinon.stub(actionsUtil, "isRunningLocalAction").returns(false); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const tag = PER_LANGUAGE_CLI_VERSION.enabledVersions[0].tagName; + const assetURL = + "https://api.github.com/repos/codeql-testing/action-fork/releases/assets/123"; + const combinedURL = `${assetURL}4`; + const fetchRelease = sinon + .stub, ReturnType>() + .callsFake( + async () => + new Response( + JSON.stringify({ + assets: [ + { name: "codeql-bundle-java-linux64.tar.zst", url: assetURL }, + { + name: "codeql-bundle-linux64.tar.zst", + url: combinedURL, + }, + ], + }), + { headers: { "content-type": "application/json" } }, + ), + ); + const client = github.getOctokit("123", { + request: { fetch: fetchRelease }, + }); + sinon.stub(api, "getApiClient").value(() => client); + const authorizationSpy = sinon.spy(api, "getAuthorizationHeaderFor"); + const extractStub = stubDownloadAndExtract(); + if (fallback) { + extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404)); + } + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir, { + GITHUB_ACTION_REPOSITORY: "codeql-testing/action-fork", + }); + const result = await setupCodeql.setupCodeQLBundle( + undefined, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.true(fetchRelease.calledTwice); + t.is( + fetchRelease.firstCall.args[0], + `https://api.github.com/repos/codeql-testing/action-fork/releases/tags/${tag}`, + ); + t.is(extractStub.callCount, fallback ? 2 : 1); + t.is(extractStub.firstCall.args[0], assetURL); + t.is(extractStub.lastCall.args[0], fallback ? combinedURL : assetURL); + t.is(authorizationSpy.callCount, extractStub.callCount); + t.is(authorizationSpy.firstCall.args[2], assetURL); + t.is( + authorizationSpy.lastCall.args[2], + fallback ? combinedURL : assetURL, + ); + t.is(extractStub.lastCall.args[3], "token token"); + t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION); + t.is( + result.toolsDownloadStatusReport?.bundleLanguage, + fallback ? undefined : BuiltInLanguage.java, + ); + t.is( + result.toolsDownloadStatusReport?.perLanguageBundleFallback, + fallback ? true : undefined, + ); + if (fallback) { + t.is( + result.codeqlFolder, + toolcache.find("CodeQL", MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION), + ); + t.true(fs.existsSync(`${result.codeqlFolder}.complete`)); + } else { + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + } + }); + }, + ); +} + +for (const bundle of ["per-language", "combined", "fallback"] as const) { + test.serial( + `setupCodeQLBundle preserves the nightly version for a ${bundle} download`, + async (t) => { + const expectedDate = "30260213"; + const expectedTag = `codeql-bundle-${expectedDate}`; + const expectedVersion = `0.0.0-${expectedDate}`; + const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; + const combinedURL = `${baseURL}/codeql-bundle-linux64.tar.zst`; + const perLanguageURL = `${baseURL}/codeql-bundle-javascript-linux64.tar.zst`; + const loggedMessages: LoggedMessage[] = []; + const logger = getRecordingLogger(loggedMessages); + + stubHostedNightly(expectedTag); + delete process.env[EnvVar.HAS_SET_UP_CODEQL]; + + const downloadSpy = sinon.spy(setupCodeql, "downloadCodeQL"); + const extractStub = stubDownloadAndExtract(); + if (bundle === "fallback") { + extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404)); + } + const addDiagnostic = sinon.stub(diagnostics, "addNoLanguageDiagnostic"); + const features = createFeatures([ + Feature.PerLanguageBundles, + Feature.CleanupToolcacheBundles, + ]); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + "nightly", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + bundle === "combined" ? ["javascript", "python"] : ["javascript"], + false, // useOverlayAwareDefaultCliVersion + features, + logger, + ); + + const source = downloadSpy.firstCall.args[0]; + t.is(result.toolsVersion, expectedVersion); + t.is(result.toolsVersion, source.toolsVersion); + t.is( + source.bundle.kind, + bundle === "combined" ? "combined" : "per-language", + ); + t.is(result.codeqlFolder, extractStub.lastCall.args[2]); + t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); + t.is(downloadSpy.callCount, extractStub.callCount); + t.is( + extractStub.firstCall.args[0], + bundle === "combined" ? combinedURL : perLanguageURL, + ); + t.is( + extractStub.lastCall.args[0], + bundle === "per-language" ? perLanguageURL : combinedURL, + ); + t.is( + result.toolsDownloadStatusReport?.bundleLanguage, + bundle === "per-language" ? BuiltInLanguage.javascript : undefined, + ); + t.is( + result.toolsDownloadStatusReport?.perLanguageBundleFallback, + bundle === "fallback" ? true : undefined, + ); + t.is( + addDiagnostic + .getCalls() + .filter( + (call) => + call.args[1].source?.id === + "codeql-action/toolcache-bundle-cleanup", + ).length, + 1, + ); + if (bundle === "fallback") { + t.deepEqual(downloadSpy.secondCall.args[0], { + ...source, + bundle: { kind: "combined", url: combinedURL }, + }); + checkExpectedLogMessages(t, loggedMessages, [ + `No javascript CodeQL bundle was found at ${perLanguageURL}`, + ]); + } + if (bundle === "per-language") { + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + } else { + t.is( + result.codeqlFolder, + toolsDownload.getToolcacheDirectory(expectedVersion), + ); + t.true(fs.existsSync(`${result.codeqlFolder}.complete`)); + + const cachedResult = await setupCodeql.setupCodeQLBundle( + "nightly", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + ["javascript"], + false, // useOverlayAwareDefaultCliVersion + features, + logger, + ); + t.is(cachedResult.toolsSource, setupCodeql.ToolsSource.Toolcache); + t.is(cachedResult.toolsVersion, expectedVersion); + t.is(cachedResult.codeqlFolder, result.codeqlFolder); + t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); + } + }); + }, + ); +} + +for (const asset of [ + "codeql-bundle-ruby-linux64.tar.zst", + "codeql-bundle-%72uby-linux64.tar.zst", +]) { + test.serial( + `setupCodeQLBundle keeps explicitly requested ${asset} out of the toolcache`, + async (t) => { + const extractStub = stubDownloadAndExtract(); + const url = `https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/${asset}`; + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + url, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ); + + t.true(extractStub.calledOnce); + t.is(extractStub.firstCall.args[0], url); + t.is(result.toolsVersion, "9.9.9"); + t.is( + result.toolsDownloadStatusReport?.bundleLanguage, + BuiltInLanguage.ruby, + ); + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + }); + }, + ); +} + +for (const error of [ + new HTTPError("Internal Server Error", 500), + new Error("Connection reset"), +]) { + test.serial( + `setupCodeQLBundle does not fall back after ${error.message}`, + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .rejects(error); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + undefined, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ), + { is: error }, + ); + t.true(extractStub.calledOnce); + t.true( + extractStub.firstCall.args[0].endsWith( + "/codeql-bundle-java-linux64.tar.zst", + ), + ); + }); + }, + ); +} + +test.serial( + "setupCodeQLBundle does not substitute a bundle for an explicitly requested one that is missing", + async (t) => { + const error = new HTTPError("Not Found", 404); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .rejects(error); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + "https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-ruby-linux64.tar.zst", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ), + { is: error }, + ); + + t.true(extractStub.calledOnce); + }); + }, +); + test.serial( "getEnabledVersionsWithOverlayBaseDatabases returns flag-enabled versions present in cache, sorted desc", async (t) => { diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 46b9baf94f..c171a09716 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -30,8 +30,13 @@ import { Feature, FeatureEnablement, } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; import { Logger } from "./logging"; import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching"; +import { + getPerLanguageBundleLanguage, + tryGetBundleLanguageFromUrl, +} from "./per-language-bundles"; import * as tar from "./tar"; import { deleteToolcacheBundles, @@ -72,21 +77,40 @@ function getCodeQLBundleExtension( } } +/** Returns the platform component of the CodeQL bundle name for the current platform. */ +export function getBundlePlatform(): string | undefined { + switch (process.platform) { + case "win32": + return "win64"; + case "linux": + return process.arch === "arm64" ? "linux-arm64" : "linux64"; + case "darwin": + return "osx64"; + default: + return undefined; + } +} + +/** + * Returns the name of the CodeQL bundle asset to download. + * + * @param compressionMethod The compression method of the bundle. + * @param language If provided, the name of the bundle that contains only this language, rather than + * the name of the combined bundle that contains every language. + */ export function getCodeQLBundleName( compressionMethod: tar.CompressionMethod, + language?: BuiltInLanguage, ): string { const extension = getCodeQLBundleExtension(compressionMethod); + const platform = getBundlePlatform(); - let platform: string; - if (process.platform === "win32") { - platform = "win64"; - } else if (process.platform === "linux") { - platform = process.arch === "arm64" ? "linux-arm64" : "linux64"; - } else if (process.platform === "darwin") { - platform = "osx64"; - } else { + if (platform === undefined) { return `codeql-bundle${extension}`; } + if (language !== undefined) { + return `codeql-bundle-${language}-${platform}${extension}`; + } return `codeql-bundle-${platform}${extension}`; } @@ -107,7 +131,7 @@ export function getCodeQLActionRepository(logger: Logger): string { async function getCodeQLBundleDownloadURL( tagName: string, apiDetails: api.GitHubApiDetails, - compressionMethod: tar.CompressionMethod, + codeQLBundleName: string, logger: Logger, ): Promise { const codeQLActionRepository = getCodeQLActionRepository(logger); @@ -126,7 +150,6 @@ async function getCodeQLBundleDownloadURL( return !self.slice(0, index).some((other) => deepEqual(source, other)); }, ); - const codeQLBundleName = getCodeQLBundleName(compressionMethod); for (const downloadSource of uniqueDownloadSources) { const [apiURL, repository] = downloadSource; // If we've reached the final case, short-circuit the API check since we know the bundle exists and is public. @@ -215,7 +238,15 @@ export function convertToSemVer(version: string, logger: Logger): string { return s; } -type CodeQLBundle = { kind: "combined"; url: string }; +type CodeQLBundle = + | { kind: "combined"; url: string } + | { + kind: "per-language"; + url: string; + language: BuiltInLanguage; + /** Only set when the Action selected the bundle, allowing a same-version fallback. */ + combinedBundleURL?: string; + }; /** A resolved download, including its bundle identity and version. */ export interface CodeQLDownloadSource { @@ -463,6 +494,7 @@ export async function getCodeQLSource( * This does not always include a tag name. */ let url: string | undefined; + let bundle: CodeQLBundle | undefined; // We allow forcing the nightly CLI via the FF for `dynamic` events (or in test mode) where the // `tools` input cannot be adjusted to explicitly request it. @@ -471,7 +503,8 @@ export async function getCodeQLSource( const forceNightly = forceNightlyValueFF && canForceNightlyWithFF; // For advanced workflows, a value from `CODEQL_NIGHTLY_TOOLS_INPUTS` can be specified explicitly - // for the `tools` input in the workflow file. + // for the `tools` input. This is the computed input, so it may come from the repository property + // rather than the workflow file. const nightlyRequestedByToolsInput = toolsInput !== undefined && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput); @@ -505,7 +538,8 @@ export async function getCodeQLSource( `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`, ); } - toolsInput = await getNightlyToolsUrl(logger); + bundle = await getNightlyBundle(rawLanguages, variant, features, logger); + toolsInput = bundle.url; } /** @@ -730,12 +764,42 @@ export async function getCodeQLSource( ? "zstd" : "gzip"; - url = await getCodeQLBundleDownloadURL( - tagName!, - apiDetails, - compressionMethod, + const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( + { + rawLanguages, + cliVersion, + compressionMethod, + platform: getBundlePlatform(), + variant, + }, + features, logger, ); + + const resolveBundleURL = (language?: BuiltInLanguage) => + getCodeQLBundleDownloadURL( + tagName!, + apiDetails, + getCodeQLBundleName(compressionMethod, language), + logger, + ); + + if (perLanguageBundleLanguage !== undefined) { + logger.info( + `Downloading the ${perLanguageBundleLanguage} CodeQL bundle, since ${perLanguageBundleLanguage} ` + + "is the only language being analyzed.", + ); + url = await resolveBundleURL(perLanguageBundleLanguage); + bundle = { + kind: "per-language", + url, + language: perLanguageBundleLanguage, + combinedBundleURL: await resolveBundleURL(), + }; + } else { + url = await resolveBundleURL(); + bundle = { kind: "combined", url }; + } } else { const method = tar.inferCompressionMethod(url); if (method === undefined) { @@ -745,6 +809,20 @@ export async function getCodeQLSource( ); } compressionMethod = method; + + if (bundle === undefined) { + // Explicit per-language URLs must also stay out of the toolcache, but have no fallback. + const language = tryGetBundleLanguageFromUrl(url); + bundle = + language === undefined + ? { kind: "combined", url } + : { kind: "per-language", url, language }; + } + if (bundle.kind === "per-language") { + logger.info( + `${url} appears to be a CodeQL bundle that contains only ${bundle.language}.`, + ); + } } if (cliVersion) { @@ -753,7 +831,7 @@ export async function getCodeQLSource( logger.info(`Using CodeQL CLI sourced from ${url} .`); } return { - bundle: { kind: "combined", url }, + bundle, bundleVersion, cliVersion, compressionMethod, @@ -833,8 +911,11 @@ export const downloadCodeQL = async function ( writeToolcacheMarkerFile(toolcacheDestination, logger); } else { logger.debug( - "Could not cache CodeQL tools because we could not determine the bundle version from the " + - `URL ${codeqlURL}.`, + bundle.kind === "per-language" + ? "Not caching the CodeQL tools because they came from a bundle that contains only a " + + "single language." + : "Could not cache CodeQL tools because we could not determine the bundle version from the " + + `URL ${codeqlURL}.`, ); } @@ -848,7 +929,8 @@ function getToolcacheDestination( source: CodeQLDownloadSource, logger: Logger, ): string | undefined { - if (!source.bundleVersion) { + // Per-language bundles must not be stored in the toolcache. + if (source.bundle.kind !== "combined" || !source.bundleVersion) { return undefined; } @@ -1032,6 +1114,12 @@ export async function setupCodeQLBundle( }; } +/** + * Downloads the CodeQL bundle described by `source`. + * + * If `source` refers to a bundle for a single language and that bundle turns out not to exist, this + * falls back to downloading the combined bundle. + */ export async function downloadCodeQLBundle( source: CodeQLDownloadSource, apiDetails: api.GitHubApiDetails, @@ -1043,8 +1131,59 @@ export async function downloadCodeQLBundle( codeqlFolder: string; statusReport: ToolsDownloadStatusReport; }> { + const { bundle } = source; + await tryDeleteToolcacheBundles({ env: getEnv(), features, logger }); - return await downloadCodeQL(source, apiDetails, tarVersion, tempDir, logger); + + try { + const result = await downloadCodeQL( + source, + apiDetails, + tarVersion, + tempDir, + logger, + ); + return bundle.kind === "combined" + ? result + : { + ...result, + statusReport: { + ...result.statusReport, + bundleLanguage: bundle.language, + }, + }; + } catch (e) { + if ( + bundle.kind !== "per-language" || + bundle.combinedBundleURL === undefined || + util.asHTTPError(e)?.status !== 404 + ) { + throw e; + } + logger.warning( + `No ${bundle.language} CodeQL bundle was found at ${bundle.url}, so ` + + "falling back to the bundle that contains all languages. This analysis will still " + + "produce correct results, but will take longer to set up.", + ); + + const result = await downloadCodeQL( + { + ...source, + bundle: { kind: "combined", url: bundle.combinedBundleURL }, + }, + apiDetails, + tarVersion, + tempDir, + logger, + ); + return { + ...result, + statusReport: { + ...result.statusReport, + perLanguageBundleFallback: true, + }, + }; + } } async function useZstdBundle( @@ -1063,10 +1202,13 @@ function getTempExtractionDir(tempDir: string) { return path.join(tempDir, uuidV4()); } -/** - * Get the URL of the latest nightly CodeQL bundle. - */ -async function getNightlyToolsUrl(logger: Logger) { +/** Selects a bundle from the latest nightly, with a same-release fallback when applicable. */ +async function getNightlyBundle( + rawLanguages: string[] | undefined, + variant: util.GitHubVariant, + features: FeatureEnablement, + logger: Logger, +): Promise { const zstdAvailability = await tar.isZstdAvailable(logger); // The nightly is guaranteed to have a zstd bundle const compressionMethod = (await useZstdBundle( @@ -1076,6 +1218,19 @@ async function getNightlyToolsUrl(logger: Logger) { ? "zstd" : "gzip"; + const language = await getPerLanguageBundleLanguage( + { + rawLanguages, + cliVersion: undefined, + compressionMethod, + platform: getBundlePlatform(), + variant, + isNightly: true, + }, + features, + logger, + ); + try { // Since nightlies are prereleases, we can't just download the latest release // on the repository. So instead we need to find the latest pre-release @@ -1091,7 +1246,17 @@ async function getNightlyToolsUrl(logger: Logger) { if (!latestRelease) { throw new Error("Could not find the latest nightly release."); } - return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`; + const assetUrl = (name: string) => + `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`; + const url = assetUrl(getCodeQLBundleName(compressionMethod, language)); + return language === undefined + ? { kind: "combined", url } + : { + kind: "per-language", + url, + language, + combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod)), + }; } catch (e) { throw new Error( `Failed to retrieve the latest nightly release: ${util.wrapError(e)}`, diff --git a/src/status-report.ts b/src/status-report.ts index a2acd631d6..820b1c2109 100644 --- a/src/status-report.ts +++ b/src/status-report.ts @@ -645,6 +645,13 @@ export interface InitToolsDownloadFields { * Whether the relevant tools dotcom feature flags have been misconfigured. * Only populated if we attempt to determine the default version based on the dotcom feature flags. */ tools_feature_flags_valid?: boolean; + /** The language of the single-language bundle that was downloaded, if any. */ + tools_bundle_language?: string; + /** + * Whether we tried to download a single-language bundle, but it did not exist and we fell back to + * the combined bundle. + */ + tools_per_language_bundle_fallback?: boolean; } /** diff --git a/src/tools-download.ts b/src/tools-download.ts index 222a18cd91..f7b0a708ce 100644 --- a/src/tools-download.ts +++ b/src/tools-download.ts @@ -54,6 +54,13 @@ export type ToolsDownloadStatusReport = { * spent on a streaming attempt that failed and fell back to downloading before extracting. */ totalDurationMs: number; + /** The language of the single-language bundle that was downloaded, if any. */ + bundleLanguage?: string; + /** + * Whether we tried to download a single-language bundle, but it did not exist and we fell back to + * the combined bundle. + */ + perLanguageBundleFallback?: boolean; }; export async function downloadAndExtract( From eb76062ef27575487f2ed7de1a0cfa2969dd60ae Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 15 Sep 2026 19:57:17 +0100 Subject: [PATCH 02/20] Include failed bundle attempts in fallback timing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 23 +++++++++++++---------- src/setup-codeql.test.ts | 28 ++++++++++++++++++++++++---- src/setup-codeql.ts | 3 +++ 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index 00f7ef9771..564aa86828 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -4305,7 +4305,7 @@ var require_util2 = __commonJS({ var { redirectStatusSet, referrerPolicySet: referrerPolicyTokens, badPortsSet } = require_constants3(); var { getGlobalOrigin } = require_global(); var { collectASequenceOfCodePoints, collectAnHTTPQuotedString, removeChars, parseMIMEType } = require_data_url(); - var { performance: performance6 } = require("node:perf_hooks"); + var { performance: performance7 } = require("node:perf_hooks"); var { isBlobLike, ReadableStreamFrom, isValidHTTPToken, normalizedMethodRecordsBase } = require_util(); var assert = require("node:assert"); var { isUint8Array } = require("node:util/types"); @@ -4464,7 +4464,7 @@ var require_util2 = __commonJS({ }; } function coarsenedSharedCurrentTime(crossOriginIsolatedCapability) { - return coarsenTime(performance6.now(), crossOriginIsolatedCapability); + return coarsenTime(performance7.now(), crossOriginIsolatedCapability); } function createOpaqueTimingInfo(timingInfo) { return { @@ -142119,7 +142119,7 @@ module.exports = __toCommonJS(entry_points_exports); // src/analyze-action.ts var fs23 = __toESM(require("fs")); var import_path5 = __toESM(require("path")); -var import_perf_hooks4 = require("perf_hooks"); +var import_perf_hooks5 = require("perf_hooks"); var core17 = __toESM(require_core()); // src/action-common.ts @@ -148611,7 +148611,7 @@ var SarifScanOrder = [ // src/analyze.ts var fs17 = __toESM(require("fs")); var path16 = __toESM(require("path")); -var import_perf_hooks3 = require("perf_hooks"); +var import_perf_hooks4 = require("perf_hooks"); var io5 = __toESM(require_io()); // src/autobuild.ts @@ -151194,6 +151194,7 @@ async function logGeneratedFilesTelemetry(config, duration, generatedFilesCount) // src/setup-codeql.ts var fs14 = __toESM(require("fs")); var path13 = __toESM(require("path")); +var import_perf_hooks3 = require("perf_hooks"); var core12 = __toESM(require_core()); var toolcache3 = __toESM(require_tool_cache()); var import_fast_deep_equal = __toESM(require_fast_deep_equal()); @@ -152632,6 +152633,7 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau async function downloadCodeQLBundle(source, apiDetails, tarVersion, tempDir, features, logger) { const { bundle } = source; await tryDeleteToolcacheBundles({ env: getEnv(), features, logger }); + const startTime = import_perf_hooks3.performance.now(); try { const result = await downloadCodeQL( source, @@ -152668,6 +152670,7 @@ async function downloadCodeQLBundle(source, apiDetails, tarVersion, tempDir, fea ...result, statusReport: { ...result.statusReport, + totalDurationMs: Math.round(import_perf_hooks3.performance.now() - startTime), perLanguageBundleFallback: true } }; @@ -153834,10 +153837,10 @@ function dbIsFinalized(config, language, logger) { } } async function finalizeDatabaseCreation(codeql, features, config, threadsFlag, memoryFlag, logger) { - const extractionStart = import_perf_hooks3.performance.now(); + const extractionStart = import_perf_hooks4.performance.now(); await runExtraction(codeql, features, config, logger); - const extractionTime = import_perf_hooks3.performance.now() - extractionStart; - const trapImportStart = import_perf_hooks3.performance.now(); + const extractionTime = import_perf_hooks4.performance.now() - extractionStart; + const trapImportStart = import_perf_hooks4.performance.now(); for (const language of config.languages) { if (dbIsFinalized(config, language, logger)) { logger.info( @@ -153854,7 +153857,7 @@ async function finalizeDatabaseCreation(codeql, features, config, threadsFlag, m logger.endGroup(); } } - const trapImportTime = import_perf_hooks3.performance.now() - trapImportStart; + const trapImportTime = import_perf_hooks4.performance.now() - trapImportStart; return { scanned_language_extraction_duration_ms: Math.round(extractionTime), trap_import_duration_ms: Math.round(trapImportTime) @@ -156661,9 +156664,9 @@ async function run({ startedAt, logger }) { features, logger ); - const trapCacheUploadStartTime = import_perf_hooks4.performance.now(); + const trapCacheUploadStartTime = import_perf_hooks5.performance.now(); didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger); - trapCacheUploadTime = import_perf_hooks4.performance.now() - trapCacheUploadStartTime; + trapCacheUploadTime = import_perf_hooks5.performance.now() - trapCacheUploadStartTime; trapCacheCleanupTelemetry = await cleanupTrapCaches( config, features, diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index c7fa92abad..ed0f1130cc 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; +import { performance } from "perf_hooks"; import * as github from "@actions/github"; import * as toolcache from "@actions/tool-cache"; @@ -1221,10 +1222,23 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) { delete process.env[EnvVar.HAS_SET_UP_CODEQL]; const downloadSpy = sinon.spy(setupCodeql, "downloadCodeQL"); - const extractStub = stubDownloadAndExtract(); - if (bundle === "fallback") { - extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404)); - } + let elapsedMs = 1000; + sinon.stub(performance, "now").callsFake(() => elapsedMs); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .callsFake(async (_url, _compressionMethod, dest) => { + if (bundle === "fallback" && extractStub.callCount === 1) { + elapsedMs += 700.2; + throw new HTTPError("Not Found", 404); + } + elapsedMs += 300.2; + fs.mkdirSync(dest, { recursive: true }); + return { + downloadDurationMs: 200, + extractionDurationMs: 100, + totalDurationMs: 300, + }; + }); const addDiagnostic = sinon.stub(diagnostics, "addNoLanguageDiagnostic"); const features = createFeatures([ Feature.PerLanguageBundles, @@ -1253,6 +1267,12 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) { bundle === "combined" ? "combined" : "per-language", ); t.is(result.codeqlFolder, extractStub.lastCall.args[2]); + t.is( + result.toolsDownloadStatusReport?.totalDurationMs, + bundle === "fallback" ? 1000 : 300, + ); + t.is(result.toolsDownloadStatusReport?.downloadDurationMs, 200); + t.is(result.toolsDownloadStatusReport?.extractionDurationMs, 100); t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); t.is(downloadSpy.callCount, extractStub.callCount); t.is( diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index c171a09716..f500042399 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import { OutgoingHttpHeaders } from "http"; import * as path from "path"; +import { performance } from "perf_hooks"; import * as core from "@actions/core"; import * as toolcache from "@actions/tool-cache"; @@ -1135,6 +1136,7 @@ export async function downloadCodeQLBundle( await tryDeleteToolcacheBundles({ env: getEnv(), features, logger }); + const startTime = performance.now(); try { const result = await downloadCodeQL( source, @@ -1180,6 +1182,7 @@ export async function downloadCodeQLBundle( ...result, statusReport: { ...result.statusReport, + totalDurationMs: Math.round(performance.now() - startTime), perLanguageBundleFallback: true, }, }; From 289376d7ddc7e2ad4283394970db4db9ef51d336 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 15 Sep 2026 18:17:16 +0100 Subject: [PATCH 03/20] Use per-language CodeQL bundles Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/__bundle-toolcache.yml | 2 +- .../__per-language-bundle-validation.yml | 164 +++++ .github/workflows/codescanning-config-cli.yml | 3 +- lib/entry-points.js | 353 +++++++--- pr-checks/checks/bundle-toolcache.yml | 2 +- .../checks/per-language-bundle-validation.yml | 117 ++++ pr-checks/sync.ts | 9 +- src/feature-flags.ts | 10 + src/init-action.ts | 8 + src/per-language-bundles.test.ts | 189 ++++++ src/per-language-bundles.ts | 142 ++++ src/setup-codeql-action.ts | 8 + src/setup-codeql.test.ts | 617 +++++++++++++++++- src/setup-codeql.ts | 223 ++++++- src/status-report.ts | 7 + src/tools-download.ts | 7 + 16 files changed, 1715 insertions(+), 146 deletions(-) create mode 100644 .github/workflows/__per-language-bundle-validation.yml create mode 100644 pr-checks/checks/per-language-bundle-validation.yml create mode 100644 src/per-language-bundles.test.ts create mode 100644 src/per-language-bundles.ts diff --git a/.github/workflows/__bundle-toolcache.yml b/.github/workflows/__bundle-toolcache.yml index 9cc983a843..d12aeb6e78 100644 --- a/.github/workflows/__bundle-toolcache.yml +++ b/.github/workflows/__bundle-toolcache.yml @@ -80,7 +80,7 @@ jobs: - id: init uses: ./../action/init with: - languages: javascript + languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - uses: ./../action/analyze with: diff --git a/.github/workflows/__per-language-bundle-validation.yml b/.github/workflows/__per-language-bundle-validation.yml new file mode 100644 index 0000000000..ea900a9e09 --- /dev/null +++ b/.github/workflows/__per-language-bundle-validation.yml @@ -0,0 +1,164 @@ +# Warning: This file is generated automatically, and should not be modified. +# Instead, please modify the template in the pr-checks directory and run: +# pr-checks/sync.sh +# to regenerate this file. + +name: PR Check - Per-language bundles +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GO111MODULE: auto +on: + push: + branches: + - main + - releases/v* + pull_request: {} + merge_group: + types: + - checks_requested + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + inputs: {} + workflow_call: + inputs: {} +defaults: + run: + shell: bash +concurrency: + cancel-in-progress: ${{ github.event_name == 'pull_request' || false }} + group: per-language-bundle-validation-${{github.ref}} +jobs: + per-language-bundle-validation: + strategy: + fail-fast: false + matrix: + include: + - language: actions + os: ubuntu-latest + version: nightly-latest + expected-extractors: actions javascript + - language: cpp + os: ubuntu-latest + version: nightly-latest + build-mode: manual + build-command: gcc -o main main.c + - language: csharp + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: go + os: ubuntu-latest + version: nightly-latest + build-mode: autobuild + - language: java + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: javascript + os: ubuntu-latest + version: nightly-latest + - language: python + os: ubuntu-latest + version: nightly-latest + - language: ruby + os: ubuntu-latest + version: nightly-latest + - language: rust + os: ubuntu-latest + version: nightly-latest + - language: swift + os: macos-latest-xlarge + version: nightly-latest + build-mode: autobuild + name: Per-language bundles + if: github.triggering_actor != 'dependabot[bot]' + permissions: + contents: read + security-events: read + timeout-minutes: 45 + runs-on: ${{ matrix.os }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Prepare test + id: prepare-test + uses: ./.github/actions/prepare-test + with: + version: ${{ matrix.version }} + use-all-platform-bundle: 'false' + setup-kotlin: 'true' + - uses: ./../action/init + id: init + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix['build-mode'] }} + tools: ${{ steps.prepare-test.outputs.tools-url }} + - name: Check that the bundle contains only the expected extractors + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + LANGUAGE: ${{ matrix.language }} + EXPECTED_EXTRACTORS: ${{ matrix['expected-extractors'] || matrix.language }} + run: | + extractors="$("$CODEQL_PATH" resolve languages --format=json | jq -r 'keys[]')" + echo "Extractors in the bundle:" + echo "$extractors" + echo "Expected: $EXPECTED_EXTRACTORS" + + for expected in $EXPECTED_EXTRACTORS; do + if ! echo "$extractors" | grep -qx "$expected"; then + echo "::error::The ${LANGUAGE} bundle does not contain the ${expected} extractor." + exit 1 + fi + done + + # If the bundle contained extractors beyond those the language needs, then it would not + # have been trimmed, and this job would be silently validating the combined bundle. + for other in actions cpp csharp go java javascript python ruby rust swift; do + if echo "$EXPECTED_EXTRACTORS" | grep -qw "$other"; then + continue + fi + if echo "$extractors" | grep -qx "$other"; then + echo "::error::The ${LANGUAGE} bundle also contains the ${other} extractor, so it is not trimmed." + exit 1 + fi + done + - name: Check that the bundle was not added to the toolcache + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + run: | + # A bundle that is missing most of its extractors must never be left in the toolcache, + # where a later job analyzing a different language could pick it up. The runner image + # ships with its own CodeQL in the toolcache, so check where this bundle was extracted to + # rather than whether the toolcache contains CodeQL at all. + echo "CodeQL is at $CODEQL_PATH" + if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then + echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH." + exit 1 + fi + if [[ "$CODEQL_PATH" != "$RUNNER_TEMP"/* ]]; then + echo "::error::Expected the per-language bundle to be extracted under $RUNNER_TEMP, but found it at $CODEQL_PATH." + exit 1 + fi + - name: Build code + if: matrix['build-command'] + run: ${{ matrix['build-command'] }} + - uses: ./../action/analyze + id: analysis + with: + upload-database: false + - name: Check that a database was created for the language + env: + DB_LOCATIONS: ${{ steps.analysis.outputs.db-locations }} + LANGUAGE: ${{ matrix.language }} + run: | + database="$(echo "$DB_LOCATIONS" | jq -r --arg lang "$LANGUAGE" '.[$lang] // empty')" + if [ -z "$database" ] || [ ! -d "$database" ]; then + echo "::error::No CodeQL database was created for ${LANGUAGE}." + echo "Databases: $DB_LOCATIONS" + exit 1 + fi + echo "Created a ${LANGUAGE} database at ${database}." + env: + CODEQL_ACTION_PER_LANGUAGE_BUNDLES: true + CODEQL_ACTION_TEST_MODE: true diff --git a/.github/workflows/codescanning-config-cli.yml b/.github/workflows/codescanning-config-cli.yml index 7bc6718e35..54474d58fb 100644 --- a/.github/workflows/codescanning-config-cli.yml +++ b/.github/workflows/codescanning-config-cli.yml @@ -75,7 +75,8 @@ jobs: uses: ./../action/.github/actions/check-codescanning-config with: expected-config-file-contents: "{}" - languages: javascript + # Request multiple languages so later checks can reuse the combined bundle. + languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - name: Packs from input diff --git a/lib/entry-points.js b/lib/entry-points.js index 35c18d8af8..88ad90dcfa 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -27216,8 +27216,8 @@ var require_gte = __commonJS({ "node_modules/semver/functions/gte.js"(exports2, module2) { "use strict"; var compare3 = require_compare(); - var gte7 = (a, b, loose) => compare3(a, b, loose) >= 0; - module2.exports = gte7; + var gte8 = (a, b, loose) => compare3(a, b, loose) >= 0; + module2.exports = gte8; } }); @@ -27238,7 +27238,7 @@ var require_cmp = __commonJS({ var eq = require_eq(); var neq = require_neq(); var gt = require_gt(); - var gte7 = require_gte(); + var gte8 = require_gte(); var lt2 = require_lt(); var lte2 = require_lte(); var cmp = (a, op, b, loose) => { @@ -27268,7 +27268,7 @@ var require_cmp = __commonJS({ case ">": return gt(a, b, loose); case ">=": - return gte7(a, b, loose); + return gte8(a, b, loose); case "<": return lt2(a, b, loose); case "<=": @@ -28076,7 +28076,7 @@ var require_outside = __commonJS({ var gt = require_gt(); var lt2 = require_lt(); var lte2 = require_lte(); - var gte7 = require_gte(); + var gte8 = require_gte(); var outside = (version, range2, hilo, options) => { version = new SemVer(version, options); range2 = new Range2(range2, options); @@ -28091,7 +28091,7 @@ var require_outside = __commonJS({ break; case "<": gtfn = lt2; - ltefn = gte7; + ltefn = gte8; ltfn = gt; comp = "<"; ecomp = "<="; @@ -28406,7 +28406,7 @@ var require_semver2 = __commonJS({ var lt2 = require_lt(); var eq = require_eq(); var neq = require_neq(); - var gte7 = require_gte(); + var gte8 = require_gte(); var lte2 = require_lte(); var cmp = require_cmp(); var coerce3 = require_coerce(); @@ -28445,7 +28445,7 @@ var require_semver2 = __commonJS({ lt: lt2, eq, neq, - gte: gte7, + gte: gte8, lte: lte2, cmp, coerce: coerce3, @@ -31721,7 +31721,7 @@ var require_brace_expansion = __commonJS({ function lte2(i, y) { return i <= y; } - function gte7(i, y) { + function gte8(i, y) { return i >= y; } function combine2(acc, base, pre, values, max, maxLength, dropEmpties, outBase) { @@ -31754,7 +31754,7 @@ var require_brace_expansion = __commonJS({ var reverse = y < x; if (reverse) { incr *= -1; - test = gte7; + test = gte8; } var pad = n.some(isPadded2); var length = 0; @@ -33901,8 +33901,8 @@ var require_semver3 = __commonJS({ function neq(a, b, loose) { return compare3(a, b, loose) !== 0; } - exports2.gte = gte7; - function gte7(a, b, loose) { + exports2.gte = gte8; + function gte8(a, b, loose) { return compare3(a, b, loose) >= 0; } exports2.lte = lte2; @@ -33933,7 +33933,7 @@ var require_semver3 = __commonJS({ case ">": return gt(a, b, loose); case ">=": - return gte7(a, b, loose); + return gte8(a, b, loose); case "<": return lt2(a, b, loose); case "<=": @@ -34478,7 +34478,7 @@ var require_semver3 = __commonJS({ break; case "<": gtfn = lt2; - ltefn = gte7; + ltefn = gte8; ltfn = gt; comp = "<"; ecomp = "<="; @@ -34699,7 +34699,7 @@ var require_cacheUtils = __commonJS({ var crypto3 = __importStar2(require("crypto")); var fs32 = __importStar2(require("fs")); var path30 = __importStar2(require("path")); - var semver11 = __importStar2(require_semver3()); + var semver12 = __importStar2(require_semver3()); var util3 = __importStar2(require("util")); var constants_1 = require_constants7(); var versionSalt = "1.0"; @@ -34792,7 +34792,7 @@ var require_cacheUtils = __commonJS({ function getCompressionMethod() { return __awaiter2(this, void 0, void 0, function* () { const versionOutput = yield getVersion("zstd", ["--quiet"]); - const version = semver11.clean(versionOutput); + const version = semver12.clean(versionOutput); core32.debug(`zstd version: ${version}`); if (versionOutput === "") { return constants_1.CompressionMethod.Gzip; @@ -82401,7 +82401,7 @@ var require_manifest = __commonJS({ exports2._findMatch = _findMatch; exports2._getOsVersion = _getOsVersion; exports2._readLinuxVersionFile = _readLinuxVersionFile; - var semver11 = __importStar2(require_semver2()); + var semver12 = __importStar2(require_semver2()); var core_1 = require_core(); var os7 = require("os"); var cp = require("child_process"); @@ -82415,7 +82415,7 @@ var require_manifest = __commonJS({ for (const candidate of candidates) { const version = candidate.version; (0, core_1.debug)(`check ${version} satisfies ${versionSpec}`); - if (semver11.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { + if (semver12.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { file = candidate.files.find((item) => { (0, core_1.debug)(`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`); let chk = item.arch === archFilter && item.platform === platFilter; @@ -82424,7 +82424,7 @@ var require_manifest = __commonJS({ if (osVersion === item.platform_version) { chk = true; } else { - chk = semver11.satisfies(osVersion, item.platform_version); + chk = semver12.satisfies(osVersion, item.platform_version); } } return chk; @@ -82684,7 +82684,7 @@ var require_tool_cache = __commonJS({ var os7 = __importStar2(require("os")); var path30 = __importStar2(require("path")); var httpm = __importStar2(require_lib()); - var semver11 = __importStar2(require_semver2()); + var semver12 = __importStar2(require_semver2()); var stream2 = __importStar2(require("stream")); var util3 = __importStar2(require("util")); var assert_1 = require("assert"); @@ -82957,7 +82957,7 @@ var require_tool_cache = __commonJS({ } function cacheDir2(sourceDir, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver11.clean(version) || version; + version = semver12.clean(version) || version; arch2 = arch2 || os7.arch(); core32.debug(`Caching tool ${tool} ${version} ${arch2}`); core32.debug(`source dir: ${sourceDir}`); @@ -82975,7 +82975,7 @@ var require_tool_cache = __commonJS({ } function cacheFile(sourceFile, targetFile, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver11.clean(version) || version; + version = semver12.clean(version) || version; arch2 = arch2 || os7.arch(); core32.debug(`Caching tool ${tool} ${version} ${arch2}`); core32.debug(`source file: ${sourceFile}`); @@ -83005,7 +83005,7 @@ var require_tool_cache = __commonJS({ } let toolPath = ""; if (versionSpec) { - versionSpec = semver11.clean(versionSpec) || ""; + versionSpec = semver12.clean(versionSpec) || ""; const cachePath = path30.join(_getCacheDirectory(), toolName, versionSpec, arch2); core32.debug(`checking cache: ${cachePath}`); if (fs32.existsSync(cachePath) && fs32.existsSync(`${cachePath}.complete`)) { @@ -83085,7 +83085,7 @@ var require_tool_cache = __commonJS({ } function _createToolPath(tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - const folderPath = path30.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); + const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || ""); core32.debug(`destination ${folderPath}`); const markerPath = `${folderPath}.complete`; yield io9.rmRF(folderPath); @@ -83095,15 +83095,15 @@ var require_tool_cache = __commonJS({ }); } function _completeToolPath(tool, version, arch2) { - const folderPath = path30.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); + const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || ""); const markerPath = `${folderPath}.complete`; fs32.writeFileSync(markerPath, ""); core32.debug("finished caching tool"); } function isExplicitVersion(versionSpec) { - const c = semver11.clean(versionSpec) || ""; + const c = semver12.clean(versionSpec) || ""; core32.debug(`isExplicit: ${c}`); - const valid4 = semver11.valid(c) != null; + const valid4 = semver12.valid(c) != null; core32.debug(`explicit? ${valid4}`); return valid4; } @@ -83111,14 +83111,14 @@ var require_tool_cache = __commonJS({ let version = ""; core32.debug(`evaluating ${versions.length} versions`); versions = versions.sort((a, b) => { - if (semver11.gt(a, b)) { + if (semver12.gt(a, b)) { return 1; } return -1; }); for (let i = versions.length - 1; i >= 0; i--) { const potential = versions[i]; - const satisfied = semver11.satisfies(potential, versionSpec); + const satisfied = semver12.satisfies(potential, versionSpec); if (satisfied) { version = potential; break; @@ -89595,7 +89595,7 @@ var require_brace_expansion2 = __commonJS({ function lte2(i, y) { return i <= y; } - function gte7(i, y) { + function gte8(i, y) { return i >= y; } function combine2(acc, pre, values, max, maxLength, dropEmpties) { @@ -89627,7 +89627,7 @@ var require_brace_expansion2 = __commonJS({ var reverse = y < x; if (reverse) { incr *= -1; - test = gte7; + test = gte8; } var pad = n.some(isPadded2); var length = 0; @@ -148091,6 +148091,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS", minimumVersion: void 0 }, + ["per_language_bundles" /* PerLanguageBundles */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_PER_LANGUAGE_BUNDLES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", @@ -151192,7 +151197,7 @@ var path13 = __toESM(require("path")); var core12 = __toESM(require_core()); var toolcache3 = __toESM(require_tool_cache()); var import_fast_deep_equal = __toESM(require_fast_deep_equal()); -var semver9 = __toESM(require_semver2()); +var semver10 = __toESM(require_semver2()); // src/overlay/caching.ts var fs11 = __toESM(require("fs")); @@ -151492,6 +151497,89 @@ async function getCodeQlVersionsForOverlayBaseDatabases(rawLanguages, logger) { return versions; } +// src/per-language-bundles.ts +var semver7 = __toESM(require_semver2()); +var MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; +var PER_LANGUAGE_BUNDLE_NAME = /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; +function tryGetBundleLanguageFromUrl(url2) { + let assetName; + try { + const pathname = new URL(url2).pathname; + assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); + } catch { + return void 0; + } + const match2 = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + return match2 ? parseBuiltInLanguage(match2[1]) : void 0; +} +var PER_LANGUAGE_BUNDLE_PLATFORMS = { + ["actions" /* actions */]: "linux64", + ["cpp" /* cpp */]: "linux64", + ["csharp" /* csharp */]: "linux64", + ["go" /* go */]: "linux64", + ["java" /* java */]: "linux64", + ["javascript" /* javascript */]: "linux64", + ["python" /* python */]: "linux64", + ["ruby" /* ruby */]: "linux64", + ["rust" /* rust */]: "linux64", + ["swift" /* swift */]: "osx64" +}; +async function getPerLanguageBundleLanguage(options, features, logger) { + const { + rawLanguages, + cliVersion: cliVersion2, + compressionMethod, + platform: platform2, + variant, + isNightly + } = options; + const explain = (reason) => { + logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); + return void 0; + }; + if (rawLanguages?.length !== 1) { + return explain( + `exactly one language must be requested via the 'languages' input, but ${rawLanguages?.length ?? 0} were` + ); + } + const language = parseBuiltInLanguage(rawLanguages[0]); + if (language === void 0) { + return explain(`'${rawLanguages[0]}' is not a known CodeQL language`); + } + if (compressionMethod !== "zstd") { + return explain(`the bundle would be downloaded as ${compressionMethod}`); + } + if (variant !== "GitHub.com" /* DOTCOM */) { + return explain(`we are running against ${variant}`); + } + if (!isGitHubHostedRunner()) { + return explain("the job is not running on a GitHub-hosted runner"); + } + if (!isNightly) { + if (cliVersion2 === void 0) { + return explain("the CLI version of the bundle is unknown"); + } + if (!semver7.gte(cliVersion2, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { + return explain( + `CodeQL ${cliVersion2} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the first version that publishes per-language bundles` + ); + } + } + const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language]; + if (supportedPlatform === void 0) { + return explain(`no per-language bundle is published for ${language}`); + } + if (supportedPlatform !== platform2) { + return explain( + `the ${language} bundle is only published for ${supportedPlatform}, but this job is running on ${platform2 ?? "an unknown platform"}` + ); + } + if (!await features.getValue("per_language_bundles" /* PerLanguageBundles */)) { + return explain(`the ${"per_language_bundles" /* PerLanguageBundles */} feature is disabled`); + } + return language; +} + // src/tar.ts var import_child_process = require("child_process"); var fs12 = __toESM(require("fs")); @@ -151499,7 +151587,7 @@ var stream = __toESM(require("stream")); var import_toolrunner = __toESM(require_toolrunner()); var io4 = __toESM(require_io()); var toolcache = __toESM(require_tool_cache()); -var semver7 = __toESM(require_semver2()); +var semver8 = __toESM(require_semver2()); var MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3"; var MIN_REQUIRED_GNU_TAR_VERSION = "1.31"; async function getTarVersion() { @@ -151541,9 +151629,9 @@ async function isZstdAvailable(logger) { case "gnu": return { available: foundZstdBinary && // GNU tar only uses major and minor version numbers - semver7.gte( - semver7.coerce(version), - semver7.coerce(MIN_REQUIRED_GNU_TAR_VERSION) + semver8.gte( + semver8.coerce(version), + semver8.coerce(MIN_REQUIRED_GNU_TAR_VERSION) ), foundZstdBinary, version: tarVersion @@ -151552,7 +151640,7 @@ async function isZstdAvailable(logger) { return { available: foundZstdBinary && // Do a loose comparison since these version numbers don't contain // a patch version number. - semver7.gte(version, MIN_REQUIRED_BSD_TAR_VERSION), + semver8.gte(version, MIN_REQUIRED_BSD_TAR_VERSION), foundZstdBinary, version: tarVersion }; @@ -151661,7 +151749,7 @@ var core11 = __toESM(require_core()); var import_http_client = __toESM(require_lib()); var toolcache2 = __toESM(require_tool_cache()); var import_follow_redirects = __toESM(require_follow_redirects()); -var semver8 = __toESM(require_semver2()); +var semver9 = __toESM(require_semver2()); var STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; var STREAMING_STALL_TIMEOUT_MS = 5 * 60 * 1e3; var TOOLCACHE_TOOL_NAME = "CodeQL"; @@ -151787,7 +151875,7 @@ function getToolcacheToolDirectory(env) { ); } function getToolcacheVersionDirectoryName(version) { - return semver8.clean(version) || version; + return semver9.clean(version) || version; } function getToolcacheDirectory(version) { return path12.join( @@ -151899,18 +151987,27 @@ function getCodeQLBundleExtension(compressionMethod) { assertNever(compressionMethod); } } -function getCodeQLBundleName(compressionMethod) { +function getBundlePlatform() { + switch (process.platform) { + case "win32": + return "win64"; + case "linux": + return process.arch === "arm64" ? "linux-arm64" : "linux64"; + case "darwin": + return "osx64"; + default: + return void 0; + } +} +function getCodeQLBundleName(compressionMethod, language) { const extension = getCodeQLBundleExtension(compressionMethod); - let platform2; - if (process.platform === "win32") { - platform2 = "win64"; - } else if (process.platform === "linux") { - platform2 = process.arch === "arm64" ? "linux-arm64" : "linux64"; - } else if (process.platform === "darwin") { - platform2 = "osx64"; - } else { + const platform2 = getBundlePlatform(); + if (platform2 === void 0) { return `codeql-bundle${extension}`; } + if (language !== void 0) { + return `codeql-bundle-${language}-${platform2}${extension}`; + } return `codeql-bundle-${platform2}${extension}`; } function getCodeQLActionRepository(logger) { @@ -151922,7 +152019,7 @@ function getCodeQLActionRepository(logger) { } return getRequiredEnvParam("GITHUB_ACTION_REPOSITORY"); } -async function getCodeQLBundleDownloadURL(tagName, apiDetails, compressionMethod, logger) { +async function getCodeQLBundleDownloadURL(tagName, apiDetails, codeQLBundleName, logger) { const codeQLActionRepository = getCodeQLActionRepository(logger); const potentialDownloadSources = [ // This GitHub instance, and this Action. @@ -151937,7 +152034,6 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, compressionMethod return !self2.slice(0, index2).some((other) => (0, import_fast_deep_equal.default)(source, other)); } ); - const codeQLBundleName = getCodeQLBundleName(compressionMethod); for (const downloadSource of uniqueDownloadSources) { const [apiURL, repository] = downloadSource; if (apiURL === GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) { @@ -151992,13 +152088,13 @@ function tryGetTagNameFromUrl(url2, logger) { return match2[1]; } function convertToSemVer(version, logger) { - if (!semver9.valid(version)) { + if (!semver10.valid(version)) { logger.debug( `Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.` ); version = `0.0.0-${version}`; } - const s = semver9.clean(version); + const s = semver10.clean(version); if (!s) { throw new Error(`Bundle version ${version} is not in SemVer format.`); } @@ -152126,6 +152222,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO let cliVersion2; let tagName; let url2; + let bundle; const canForceNightlyWithFF = isDynamicWorkflow() || isInTestMode(); const forceNightlyValueFF = await features.getValue("force_nightly" /* ForceNightly */); const forceNightly = forceNightlyValueFF && canForceNightlyWithFF; @@ -152156,7 +152253,8 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.` ); } - toolsInput = await getNightlyToolsUrl(logger); + bundle = await getNightlyBundle(rawLanguages, variant, features, logger); + toolsInput = bundle.url; } const forceShippedTools = toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput); if (forceShippedTools) { @@ -152207,7 +152305,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO url2 = toolsInput; if (tagName) { const bundleVersion3 = tryGetBundleVersionFromTagName(tagName, logger); - if (bundleVersion3 !== void 0 && semver9.valid(bundleVersion3)) { + if (bundleVersion3 !== void 0 && semver10.valid(bundleVersion3)) { cliVersion2 = convertToSemVer(bundleVersion3, logger); } } @@ -152310,12 +152408,38 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO let compressionMethod; if (!url2) { compressionMethod = cliVersion2 !== void 0 && await useZstdBundle(cliVersion2, tarSupportsZstd) ? "zstd" : "gzip"; - url2 = await getCodeQLBundleDownloadURL( + const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( + { + rawLanguages, + cliVersion: cliVersion2, + compressionMethod, + platform: getBundlePlatform(), + variant + }, + features, + logger + ); + const resolveBundleURL = (language) => getCodeQLBundleDownloadURL( tagName, apiDetails, - compressionMethod, + getCodeQLBundleName(compressionMethod, language), logger ); + if (perLanguageBundleLanguage !== void 0) { + logger.info( + `Downloading the ${perLanguageBundleLanguage} CodeQL bundle, since ${perLanguageBundleLanguage} is the only language being analyzed.` + ); + url2 = await resolveBundleURL(perLanguageBundleLanguage); + bundle = { + kind: "per-language", + url: url2, + language: perLanguageBundleLanguage, + combinedBundleURL: await resolveBundleURL() + }; + } else { + url2 = await resolveBundleURL(); + bundle = { kind: "combined", url: url2 }; + } } else { const method = inferCompressionMethod(url2); if (method === void 0) { @@ -152324,6 +152448,15 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO ); } compressionMethod = method; + if (bundle === void 0) { + const language = tryGetBundleLanguageFromUrl(url2); + bundle = language === void 0 ? { kind: "combined", url: url2 } : { kind: "per-language", url: url2, language }; + } + if (bundle.kind === "per-language") { + logger.info( + `${url2} appears to be a CodeQL bundle that contains only ${bundle.language}.` + ); + } } if (cliVersion2) { logger.info(`Using CodeQL CLI version ${cliVersion2} sourced from ${url2} .`); @@ -152331,7 +152464,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO logger.info(`Using CodeQL CLI sourced from ${url2} .`); } return { - bundle: { kind: "combined", url: url2 }, + bundle, bundleVersion: bundleVersion2, cliVersion: cliVersion2, compressionMethod, @@ -152383,7 +152516,7 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log writeToolcacheMarkerFile(toolcacheDestination, logger); } else { logger.debug( - `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${codeqlURL}.` + bundle.kind === "per-language" ? "Not caching the CodeQL tools because they came from a bundle that contains only a single language." : `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${codeqlURL}.` ); } return { @@ -152392,7 +152525,7 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log }; }; function getToolcacheDestination(source, logger) { - if (!source.bundleVersion) { + if (source.bundle.kind !== "combined" || !source.bundleVersion) { return void 0; } return getToolcacheDirectory( @@ -152496,30 +152629,77 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau }; } async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, tempDir) { + const { bundle } = source; + const { logger } = action; await tryDeleteToolcacheBundles(action); - return await downloadCodeQL( - source, - apiDetails, - tarVersion, - tempDir, - action.logger - ); + try { + const result = await downloadCodeQL( + source, + apiDetails, + tarVersion, + tempDir, + logger + ); + return bundle.kind === "combined" ? result : { + ...result, + statusReport: { + ...result.statusReport, + bundleLanguage: bundle.language + } + }; + } catch (e) { + if (bundle.kind !== "per-language" || bundle.combinedBundleURL === void 0 || asHTTPError(e)?.status !== 404) { + throw e; + } + logger.warning( + `No ${bundle.language} CodeQL bundle was found at ${bundle.url}, so falling back to the bundle that contains all languages. This analysis will still produce correct results, but will take longer to set up.` + ); + const result = await downloadCodeQL( + { + ...source, + bundle: { kind: "combined", url: bundle.combinedBundleURL } + }, + apiDetails, + tarVersion, + tempDir, + logger + ); + return { + ...result, + statusReport: { + ...result.statusReport, + perLanguageBundleFallback: true + } + }; + } } async function useZstdBundle(cliVersion2, tarSupportsZstd) { return ( // In testing, gzip performs better than zstd on Windows. - process.platform !== "win32" && tarSupportsZstd && semver9.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE) + process.platform !== "win32" && tarSupportsZstd && semver10.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE) ); } function getTempExtractionDir(tempDir) { return path13.join(tempDir, v4_default()); } -async function getNightlyToolsUrl(logger) { +async function getNightlyBundle(rawLanguages, variant, features, logger) { const zstdAvailability = await isZstdAvailable(logger); const compressionMethod = await useZstdBundle( CODEQL_VERSION_ZSTD_BUNDLE, zstdAvailability.available ) ? "zstd" : "gzip"; + const language = await getPerLanguageBundleLanguage( + { + rawLanguages, + cliVersion: void 0, + compressionMethod, + platform: getBundlePlatform(), + variant, + isNightly: true + }, + features, + logger + ); try { const release2 = await getApiClient().rest.repos.listReleases({ owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER, @@ -152532,7 +152712,14 @@ async function getNightlyToolsUrl(logger) { if (!latestRelease) { throw new Error("Could not find the latest nightly release."); } - return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`; + const assetUrl = (name) => `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`; + const url2 = assetUrl(getCodeQLBundleName(compressionMethod, language)); + return language === void 0 ? { kind: "combined", url: url2 } : { + kind: "per-language", + url: url2, + language, + combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod)) + }; } catch (e) { throw new Error( `Failed to retrieve the latest nightly release: ${wrapError(e)}` @@ -152540,7 +152727,7 @@ async function getNightlyToolsUrl(logger) { } } function getLatestToolcacheVersion(logger) { - const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver9.compare(b, a)); + const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver10.compare(b, a)); logger.debug( `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( allVersions @@ -156724,7 +156911,7 @@ function isPadded(el) { function lte(i, y) { return i <= y; } -function gte6(i, y) { +function gte7(i, y) { return i >= y; } function combine(acc, pre, values, max, maxLength, dropEmpties) { @@ -156759,7 +156946,7 @@ function expandSequence(body, isAlphaSequence, max, maxLength) { const reverse = y < x; if (reverse) { incr *= -1; - test = gte6; + test = gte7; } const pad = n.some(isPadded); let length = 0; @@ -158656,7 +158843,7 @@ var import_async = __toESM(require_async(), 1); var import_path7 = require("path"); // node_modules/archiver/lib/error.js -var import_util34 = __toESM(require("util"), 1); +var import_util35 = __toESM(require("util"), 1); var ERROR_CODES = { ABORTED: "archive was aborted", DIRECTORYDIRPATHREQUIRED: "diretory dirpath argument must be a non-empty string value", @@ -158681,7 +158868,7 @@ function ArchiverError(code, data) { this.code = code; this.data = data; } -import_util34.default.inherits(ArchiverError, Error); +import_util35.default.inherits(ArchiverError, Error); // node_modules/archiver/lib/core.js var import_readable_stream2 = __toESM(require_ours(), 1); @@ -161613,7 +161800,7 @@ var fs29 = __toESM(require("fs")); var path25 = __toESM(require("path")); var core22 = __toESM(require_core()); var io7 = __toESM(require_io()); -var semver10 = __toESM(require_semver2()); +var semver11 = __toESM(require_semver2()); // src/config/inputs.ts async function getToolsInput(action, repositoryProperties) { @@ -161974,6 +162161,12 @@ async function sendCompletedStatusReport2(startedAt, config, configFile, toolsIn if (toolsDownloadStatusReport?.totalDurationMs !== void 0) { initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; } + if (toolsDownloadStatusReport?.bundleLanguage !== void 0) { + initToolsDownloadFields.tools_bundle_language = toolsDownloadStatusReport.bundleLanguage; + } + if (toolsDownloadStatusReport?.perLanguageBundleFallback !== void 0) { + initToolsDownloadFields.tools_per_language_bundle_fallback = toolsDownloadStatusReport.perLanguageBundleFallback; + } if (toolsFeatureFlagsValid !== void 0) { initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; } @@ -162093,12 +162286,12 @@ async function run3(actionState) { const experimental = "2.19.3"; const publicPreview = "2.22.1"; const actualVer = (await codeql.getVersion()).version; - if (semver10.lt(actualVer, experimental)) { + if (semver11.lt(actualVer, experimental)) { throw new ConfigurationError( `Rust analysis is supported by CodeQL CLI version ${experimental} or higher, but found version ${actualVer}` ); } - if (semver10.lt(actualVer, publicPreview)) { + if (semver11.lt(actualVer, publicPreview)) { core22.exportVariable("CODEQL_ENABLE_EXPERIMENTAL_FEATURES" /* EXPERIMENTAL_FEATURES */, "true"); logger.info("Experimental Rust analysis enabled"); } @@ -163022,6 +163215,12 @@ async function sendCompletedStatusReport3(startedAt, toolsInput, toolsDownloadSt if (toolsDownloadStatusReport?.totalDurationMs !== void 0) { initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; } + if (toolsDownloadStatusReport?.bundleLanguage !== void 0) { + initToolsDownloadFields.tools_bundle_language = toolsDownloadStatusReport.bundleLanguage; + } + if (toolsDownloadStatusReport?.perLanguageBundleFallback !== void 0) { + initToolsDownloadFields.tools_per_language_bundle_fallback = toolsDownloadStatusReport.perLanguageBundleFallback; + } if (toolsFeatureFlagsValid !== void 0) { initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; } diff --git a/pr-checks/checks/bundle-toolcache.yml b/pr-checks/checks/bundle-toolcache.yml index 83d1d7d0b5..efa1a4d76f 100644 --- a/pr-checks/checks/bundle-toolcache.yml +++ b/pr-checks/checks/bundle-toolcache.yml @@ -30,7 +30,7 @@ steps: - id: init uses: ./../action/init with: - languages: javascript + languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - uses: ./../action/analyze with: diff --git a/pr-checks/checks/per-language-bundle-validation.yml b/pr-checks/checks/per-language-bundle-validation.yml new file mode 100644 index 0000000000..21fe33e757 --- /dev/null +++ b/pr-checks/checks/per-language-bundle-validation.yml @@ -0,0 +1,117 @@ +name: Per-language bundles +description: Validates extraction and analysis using each per-language CodeQL bundle. +# TODO: Use a released bundle once releases include per-language bundles. +matrix: + include: + - language: actions + os: ubuntu-latest + version: nightly-latest + # Actions also needs the JavaScript extractor. + expected-extractors: actions javascript + - language: cpp + os: ubuntu-latest + version: nightly-latest + build-mode: manual + build-command: gcc -o main main.c + - language: csharp + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: go + os: ubuntu-latest + version: nightly-latest + build-mode: autobuild + - language: java + os: ubuntu-latest + version: nightly-latest + build-mode: none + - language: javascript + os: ubuntu-latest + version: nightly-latest + - language: python + os: ubuntu-latest + version: nightly-latest + - language: ruby + os: ubuntu-latest + version: nightly-latest + - language: rust + os: ubuntu-latest + version: nightly-latest + - language: swift + os: macos-latest-xlarge + version: nightly-latest + build-mode: autobuild +env: + CODEQL_ACTION_PER_LANGUAGE_BUNDLES: true +steps: + - uses: ./../action/init + id: init + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix['build-mode'] }} + tools: ${{ steps.prepare-test.outputs.tools-url }} + - name: Check that the bundle contains only the expected extractors + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + LANGUAGE: ${{ matrix.language }} + EXPECTED_EXTRACTORS: ${{ matrix['expected-extractors'] || matrix.language }} + run: | + extractors="$("$CODEQL_PATH" resolve languages --format=json | jq -r 'keys[]')" + echo "Extractors in the bundle:" + echo "$extractors" + echo "Expected: $EXPECTED_EXTRACTORS" + + for expected in $EXPECTED_EXTRACTORS; do + if ! echo "$extractors" | grep -qx "$expected"; then + echo "::error::The ${LANGUAGE} bundle does not contain the ${expected} extractor." + exit 1 + fi + done + + # If the bundle contained extractors beyond those the language needs, then it would not + # have been trimmed, and this job would be silently validating the combined bundle. + for other in actions cpp csharp go java javascript python ruby rust swift; do + if echo "$EXPECTED_EXTRACTORS" | grep -qw "$other"; then + continue + fi + if echo "$extractors" | grep -qx "$other"; then + echo "::error::The ${LANGUAGE} bundle also contains the ${other} extractor, so it is not trimmed." + exit 1 + fi + done + - name: Check that the bundle was not added to the toolcache + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + run: | + # A bundle that is missing most of its extractors must never be left in the toolcache, + # where a later job analyzing a different language could pick it up. The runner image + # ships with its own CodeQL in the toolcache, so check where this bundle was extracted to + # rather than whether the toolcache contains CodeQL at all. + echo "CodeQL is at $CODEQL_PATH" + if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then + echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH." + exit 1 + fi + if [[ "$CODEQL_PATH" != "$RUNNER_TEMP"/* ]]; then + echo "::error::Expected the per-language bundle to be extracted under $RUNNER_TEMP, but found it at $CODEQL_PATH." + exit 1 + fi + - name: Build code + if: matrix['build-command'] + run: ${{ matrix['build-command'] }} + - uses: ./../action/analyze + id: analysis + with: + upload-database: false + - name: Check that a database was created for the language + env: + DB_LOCATIONS: ${{ steps.analysis.outputs.db-locations }} + LANGUAGE: ${{ matrix.language }} + run: | + database="$(echo "$DB_LOCATIONS" | jq -r --arg lang "$LANGUAGE" '.[$lang] // empty')" + if [ -z "$database" ] || [ ! -d "$database" ]; then + echo "::error::No CodeQL database was created for ${LANGUAGE}." + echo "Databases: $DB_LOCATIONS" + exit 1 + fi + echo "Created a ${LANGUAGE} database at ${database}." diff --git a/pr-checks/sync.ts b/pr-checks/sync.ts index 6dde1ee48e..f0942ad2dd 100755 --- a/pr-checks/sync.ts +++ b/pr-checks/sync.ts @@ -79,6 +79,8 @@ interface Specification extends JobSpecification { useAllPlatformBundle?: string; /** Values for the `analysis-kinds` matrix dimension. */ analysisKinds?: string[]; + /** Overrides the generated job matrix using GitHub Actions matrix syntax. */ + matrix?: Record; /** Container image configuration for the job. */ container?: any; @@ -512,9 +514,6 @@ function generateJob( specDocument: yaml.Document, checkSpecification: Specification, ) { - const matrix: Array> = - generateJobMatrix(checkSpecification); - const useAllPlatformBundle = checkSpecification.useAllPlatformBundle ? checkSpecification.useAllPlatformBundle : "false"; @@ -567,8 +566,8 @@ function generateJob( const checkJob: Record = { strategy: { "fail-fast": false, - matrix: { - include: matrix, + matrix: checkSpecification.matrix ?? { + include: generateJobMatrix(checkSpecification), }, }, name: checkSpecification.name, diff --git a/src/feature-flags.ts b/src/feature-flags.ts index da7bcceade..afddaea2a4 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -164,6 +164,11 @@ export enum Feature { OverlayAnalysisStatusCheck = "overlay_analysis_status_check", /** Controls whether overlay build failures on the default branch are stored in the Actions cache. */ OverlayAnalysisStatusSave = "overlay_analysis_status_save", + /** + * Controls whether we may download a bundle containing only the single language being analysed, + * rather than the combined bundle that contains every language. + */ + PerLanguageBundles = "per_language_bundles", QaTelemetryEnabled = "qa_telemetry_enabled", /** Routes (some) API requests through the registry proxy. */ ProxyApiRequests = "proxy_api_requests", @@ -434,6 +439,11 @@ export const featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS", minimumVersion: undefined, }, + [Feature.PerLanguageBundles]: { + defaultValue: false, + envVar: "CODEQL_ACTION_PER_LANGUAGE_BUNDLES", + minimumVersion: undefined, + }, [Feature.QaTelemetryEnabled]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/src/init-action.ts b/src/init-action.ts index 8173d67aaa..dd576548dc 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -182,6 +182,14 @@ async function sendCompletedStatusReport( initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; } + if (toolsDownloadStatusReport?.bundleLanguage !== undefined) { + initToolsDownloadFields.tools_bundle_language = + toolsDownloadStatusReport.bundleLanguage; + } + if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) { + initToolsDownloadFields.tools_per_language_bundle_fallback = + toolsDownloadStatusReport.perLanguageBundleFallback; + } if (toolsFeatureFlagsValid !== undefined) { initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; } diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts new file mode 100644 index 0000000000..8242bd0144 --- /dev/null +++ b/src/per-language-bundles.test.ts @@ -0,0 +1,189 @@ +import test from "ava"; + +import { ActionsEnvVars } from "./environment"; +import { Feature } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; +import { getRunnerLogger } from "./logging"; +import { + getPerLanguageBundleLanguage, + MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + PerLanguageBundleOptions, + tryGetBundleLanguageFromUrl, +} from "./per-language-bundles"; +import { createFeatures, setupTests } from "./testing-utils"; +import { GitHubVariant } from "./util"; + +setupTests(test); + +/** Options for which we would use a per-language bundle. */ +const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = { + rawLanguages: ["java"], + // Any version at least as new as the minimum will do. + cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + compressionMethod: "zstd", + platform: "linux64", + variant: GitHubVariant.DOTCOM, +}; + +async function checkEligibility( + overrides: Partial, + enabledFeatures: Feature[] = [Feature.PerLanguageBundles], +) { + return getPerLanguageBundleLanguage( + { ...ELIGIBLE_OPTIONS, ...overrides }, + createFeatures(enabledFeatures), + getRunnerLogger(true), + ); +} + +test.beforeEach(() => { + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; +}); + +test.serial("uses Linux bundles for non-Swift languages", async (t) => { + for (const language of Object.values(BuiltInLanguage)) { + if (language === BuiltInLanguage.swift) { + continue; + } + t.is(await checkEligibility({ rawLanguages: [language] }), language); + } +}); + +test.serial("normalizes an alias before selecting a bundle", async (t) => { + t.is( + await checkEligibility({ rawLanguages: ["java-kotlin"] }), + BuiltInLanguage.java, + ); +}); + +test.serial("uses the macOS bundle for Swift", async (t) => { + t.is( + await checkEligibility({ rawLanguages: ["swift"], platform: "osx64" }), + BuiltInLanguage.swift, + ); + // Swift is only published for macOS. + t.is( + await checkEligibility({ rawLanguages: ["swift"], platform: "linux64" }), + undefined, + ); +}); + +test.serial("only publishes non-Swift languages for Linux", async (t) => { + t.is(await checkEligibility({ platform: "osx64" }), undefined); + t.is(await checkEligibility({ platform: "win64" }), undefined); + // We do not publish per-language bundles for Linux Arm64 either. + t.is(await checkEligibility({ platform: "linux-arm64" }), undefined); + t.is(await checkEligibility({ platform: undefined }), undefined); +}); + +test.serial("requires exactly one language", async (t) => { + t.is(await checkEligibility({ rawLanguages: undefined }), undefined); + t.is(await checkEligibility({ rawLanguages: [] }), undefined); + t.is(await checkEligibility({ rawLanguages: ["java", "python"] }), undefined); +}); + +test.serial("requires a language that CodeQL knows about", async (t) => { + t.is(await checkEligibility({ rawLanguages: ["cobol"] }), undefined); +}); + +test.serial("requires a zstd bundle", async (t) => { + t.is(await checkEligibility({ compressionMethod: "gzip" }), undefined); +}); + +test.serial("requires GitHub.com", async (t) => { + // Other products resolve the combined bundle against their own instance, so asking for a + // per-language bundle they do not mirror would move the download off that instance. + for (const variant of [GitHubVariant.GHES, GitHubVariant.GHEC_DR]) { + t.is(await checkEligibility({ variant }), undefined); + } +}); + +test.serial("requires a GitHub-hosted runner", async (t) => { + // A self-hosted runner may have a toolcache that persists between jobs, which is worth more than + // a smaller download. + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; + t.is(await checkEligibility({}), undefined); + + // Self-hosted runners are routinely configured to look like hosted ones, for example by mounting + // a persistent volume at `/opt/hostedtoolcache`, so we require the service to tell us explicitly. + delete process.env[ActionsEnvVars.RUNNER_ENVIRONMENT]; + process.env["RUNNER_TOOL_CACHE"] = "/opt/hostedtoolcache"; + t.is(await checkEligibility({}), undefined); +}); + +test.serial("requires a new enough CLI version", async (t) => { + t.is(await checkEligibility({ cliVersion: undefined }), undefined); + t.is(await checkEligibility({ cliVersion: "2.27.0" }), undefined); + t.is(await checkEligibility({ cliVersion: "2.27.1" }), BuiltInLanguage.java); +}); + +test.serial("requires the feature flag", async (t) => { + t.is(await checkEligibility({}, []), undefined); +}); + +test.serial("nightlies skip only the release version check", async (t) => { + const nightly = { isNightly: true, cliVersion: undefined }; + t.is(await checkEligibility(nightly), BuiltInLanguage.java); + + for (const overrides of [ + { rawLanguages: undefined }, + { rawLanguages: ["java", "python"] }, + { compressionMethod: "gzip" as const }, + { platform: "osx64" }, + { variant: GitHubVariant.GHES }, + { variant: GitHubVariant.GHEC_DR }, + ]) { + t.is(await checkEligibility({ ...nightly, ...overrides }), undefined); + } + t.is(await checkEligibility(nightly, []), undefined); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; + t.is(await checkEligibility(nightly), undefined); +}); + +test.serial("recognizes a per-language bundle from its URL", (t) => { + const url = (name: string) => + `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; + + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-java-linux64.tar.zst")), + BuiltInLanguage.java, + ); + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-swift-osx64.tar.zst")), + BuiltInLanguage.swift, + ); + // We do not publish these, but should still recognize them if we ever do. + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-csharp-win64.tar.gz")), + BuiltInLanguage.csharp, + ); + // A percent-encoded name resolves to the same asset, so it must not let a bundle that contains a + // single language pass for one that contains them all and end up in the toolcache. + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-%70ython-linux64.tar.zst")), + BuiltInLanguage.python, + ); +}); + +test.serial("does not mistake other bundles for per-language ones", (t) => { + const url = (name: string) => + `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; + + for (const name of [ + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-osx64.tar.gz", + "codeql-bundle-win64.tar.zst", + // The all-platform bundle. + "codeql-bundle.tar.gz", + // A platform we do not publish per-language bundles for, whose name also contains a hyphen. + "codeql-bundle-linux-arm64.tar.zst", + // Not a language we know about. + "codeql-bundle-cobol-linux64.tar.zst", + // A name we cannot decode must not be mistaken for a language either. + "codeql-bundle-%zz-linux64.tar.zst", + ]) { + t.is(tryGetBundleLanguageFromUrl(url(name)), undefined, name); + } + + t.is(tryGetBundleLanguageFromUrl("not a url"), undefined); +}); diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts new file mode 100644 index 0000000000..1b63e4f100 --- /dev/null +++ b/src/per-language-bundles.ts @@ -0,0 +1,142 @@ +import * as semver from "semver"; + +import { isGitHubHostedRunner } from "./actions-util"; +import { Feature, FeatureEnablement } from "./feature-flags"; +import { BuiltInLanguage, parseBuiltInLanguage } from "./languages"; +import { Logger } from "./logging"; +import * as tar from "./tar"; +import { GitHubVariant } from "./util"; + +/** Minimum CLI version for selecting a per-language release bundle. */ +export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; + +const PER_LANGUAGE_BUNDLE_NAME = + /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; + +/** Identifies per-language tools URLs that must not populate the toolcache. */ +export function tryGetBundleLanguageFromUrl( + url: string, +): BuiltInLanguage | undefined { + let assetName: string; + try { + const pathname = new URL(url).pathname; + // URL-encoded names must not bypass the toolcache safeguard. + assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); + } catch { + return undefined; + } + + const match = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + return match ? parseBuiltInLanguage(match[1]) : undefined; +} + +/** Published platform for each language; absent entries are ineligible. */ +const PER_LANGUAGE_BUNDLE_PLATFORMS: Readonly< + Partial> +> = { + [BuiltInLanguage.actions]: "linux64", + [BuiltInLanguage.cpp]: "linux64", + [BuiltInLanguage.csharp]: "linux64", + [BuiltInLanguage.go]: "linux64", + [BuiltInLanguage.java]: "linux64", + [BuiltInLanguage.javascript]: "linux64", + [BuiltInLanguage.python]: "linux64", + [BuiltInLanguage.ruby]: "linux64", + [BuiltInLanguage.rust]: "linux64", + [BuiltInLanguage.swift]: "osx64", +}; + +/** Inputs that determine whether we may download a per-language bundle. */ +export interface PerLanguageBundleOptions { + /** Explicit input only: autodetection needs a CLI instance. */ + rawLanguages: string[] | undefined; + /** CLI version, if known. Ignored for nightly bundles. */ + cliVersion: string | undefined; + compressionMethod: tar.CompressionMethod; + /** Bundle platform identifier, such as linux64. */ + platform: string | undefined; + variant: GitHubVariant; + isNightly?: boolean; +} + +/** Returns the eligible bundle language, or undefined for the combined bundle. */ +export async function getPerLanguageBundleLanguage( + options: PerLanguageBundleOptions, + features: FeatureEnablement, + logger: Logger, +): Promise { + const { + rawLanguages, + cliVersion, + compressionMethod, + platform, + variant, + isNightly, + } = options; + + const explain = (reason: string) => { + logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); + return undefined; + }; + + if (rawLanguages?.length !== 1) { + return explain( + `exactly one language must be requested via the 'languages' input, but ${ + rawLanguages?.length ?? 0 + } were`, + ); + } + + const language = parseBuiltInLanguage(rawLanguages[0]); + if (language === undefined) { + return explain(`'${rawLanguages[0]}' is not a known CodeQL language`); + } + + if (compressionMethod !== "zstd") { + // Per-language bundles are only published as zstd archives. + return explain(`the bundle would be downloaded as ${compressionMethod}`); + } + + if (variant !== GitHubVariant.DOTCOM) { + // Tenant mirrors may lack these assets, and an unreachable github.com fails with a + // connection error rather than a recoverable 404. + return explain(`we are running against ${variant}`); + } + + if (!isGitHubHostedRunner()) { + // Per-language installs stay out of the toolcache; self-hosted runners should retain + // the reusable combined bundle instead. + return explain("the job is not running on a GitHub-hosted runner"); + } + + // Nightly tags contain dates rather than comparable CLI versions. + if (!isNightly) { + if (cliVersion === undefined) { + return explain("the CLI version of the bundle is unknown"); + } + + if (!semver.gte(cliVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { + return explain( + `CodeQL ${cliVersion} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the ` + + "first version that publishes per-language bundles", + ); + } + } + + const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language]; + if (supportedPlatform === undefined) { + return explain(`no per-language bundle is published for ${language}`); + } + if (supportedPlatform !== platform) { + return explain( + `the ${language} bundle is only published for ${supportedPlatform}, but this job is ` + + `running on ${platform ?? "an unknown platform"}`, + ); + } + + if (!(await features.getValue(Feature.PerLanguageBundles))) { + return explain(`the ${Feature.PerLanguageBundles} feature is disabled`); + } + + return language; +} diff --git a/src/setup-codeql-action.ts b/src/setup-codeql-action.ts index bb6b73c9aa..3c2a191e7b 100644 --- a/src/setup-codeql-action.ts +++ b/src/setup-codeql-action.ts @@ -93,6 +93,14 @@ async function sendCompletedStatusReport( initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; } + if (toolsDownloadStatusReport?.bundleLanguage !== undefined) { + initToolsDownloadFields.tools_bundle_language = + toolsDownloadStatusReport.bundleLanguage; + } + if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) { + initToolsDownloadFields.tools_per_language_bundle_fallback = + toolsDownloadStatusReport.perLanguageBundleFallback; + } if (toolsFeatureFlagsValid !== undefined) { initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; } diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index 973beef5eb..f7caf575f3 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -12,8 +12,10 @@ import * as api from "./api-client"; import * as diagnostics from "./diagnostics"; import { ActionsEnvVars, EnvVar, getEnv, ReadOnlyEnv } from "./environment"; import { Feature } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; import { getRunnerLogger } from "./logging"; import { getCacheRestoreKeyPrefix } from "./overlay/caching"; +import { MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION } from "./per-language-bundles"; import * as setupCodeql from "./setup-codeql"; import * as tar from "./tar"; import { @@ -55,6 +57,25 @@ function stubDownloadAndExtract() { }); } +function stubHostedNightly(tagName: string) { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const client = github.getOctokit("123", { + request: { + fetch: async () => + new Response(JSON.stringify([{ tag_name: tagName }]), { + headers: { "content-type": "application/json" }, + }), + }, + }); + sinon.stub(api, "getApiClient").value(() => client); +} + test.serial("parse codeql bundle url version", (t) => { t.deepEqual( setupCodeql.getCodeQLURLVersion( @@ -374,20 +395,7 @@ test.serial( const expectedDate = "30260213"; const expectedTag = `codeql-bundle-${expectedDate}`; - // Ensure that we consistently select "zstd" for the test. - sinon.stub(process, "platform").value("linux"); - sinon.stub(tar, "isZstdAvailable").resolves({ - available: true, - foundZstdBinary: true, - }); - - const client = github.getOctokit("123"); - const listReleases = sinon.stub(client.rest.repos, "listReleases"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - listReleases.resolves({ - data: [{ tag_name: expectedTag }], - } as any); - sinon.stub(api, "getApiClient").value(() => client); + stubHostedNightly(expectedTag); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); @@ -456,20 +464,7 @@ test.serial( const expectedDate = "30260213"; const expectedTag = `codeql-bundle-${expectedDate}`; - // Ensure that we consistently select "zstd" for the test. - sinon.stub(process, "platform").value("linux"); - sinon.stub(tar, "isZstdAvailable").resolves({ - available: true, - foundZstdBinary: true, - }); - - const client = github.getOctokit("123"); - const listReleases = sinon.stub(client.rest.repos, "listReleases"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - listReleases.resolves({ - data: [{ tag_name: expectedTag }], - } as any); - sinon.stub(api, "getApiClient").value(() => client); + stubHostedNightly(expectedTag); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" }); @@ -513,6 +508,8 @@ for (const bundlePath of [ "codeql-bundle.tar.gz", "codeql-bundle.tar.zst", "codeql-bundle-/codeql-bundle.tar.gz", + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-ruby-linux64.tar.zst", ]) { test.serial( `setupCodeQLBundle reports an unknown version for ${bundlePath}`, @@ -542,6 +539,12 @@ for (const bundlePath of [ t.is(downloadSpy.firstCall.args[0].toolsVersion, "unknown"); t.is(result.toolsVersion, "unknown"); t.is(result.toolsSource, setupCodeql.ToolsSource.Download); + t.is( + result.toolsDownloadStatusReport?.bundleLanguage, + bundlePath === "codeql-bundle-ruby-linux64.tar.zst" + ? BuiltInLanguage.ruby + : undefined, + ); t.is(path.dirname(result.codeqlFolder), tmpDir); t.true(fs.existsSync(result.codeqlFolder)); t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); @@ -594,6 +597,131 @@ test.serial( }, ); +for (const toolsInput of ["nightly", "nightly-latest"]) { + test.serial( + `getCodeQLSource selects a per-language bundle for tools == ${toolsInput}`, + async (t) => { + const expectedTag = "codeql-bundle-30260213"; + const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; + stubHostedNightly(expectedTag); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + toolsInput, + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.deepEqual(source, { + sourceType: "download", + bundle: { + kind: "per-language", + language: BuiltInLanguage.java, + url: `${baseURL}/codeql-bundle-java-linux64.tar.zst`, + combinedBundleURL: `${baseURL}/codeql-bundle-linux64.tar.zst`, + }, + bundleVersion: "30260213", + cliVersion: undefined, + compressionMethod: "zstd", + toolsVersion: "0.0.0-30260213", + } satisfies setupCodeql.CodeQLDownloadSource); + }); + }, + ); +} + +test.serial( + "getCodeQLSource downloads the combined nightly bundle when not eligible", + async (t) => { + const expectedTag = "codeql-bundle-30260213"; + stubHostedNightly(expectedTag); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + for (const { languages, features } of [ + { languages: ["java"], features: createFeatures([]) }, + { + languages: ["java", "python"], + features: createFeatures([Feature.PerLanguageBundles]), + }, + ]) { + const source = await setupCodeql.getCodeQLSource( + "nightly", + SAMPLE_DEFAULT_CLI_VERSION, + languages, + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + features, + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + t.deepEqual(source.bundle, { + kind: "combined", + url: `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/codeql-bundle-linux64.tar.zst`, + }); + } + } + }); + }, +); + +for (const perLanguageBundles of [false, true]) { + test.serial( + `getCodeQLSource uses a ${perLanguageBundles ? "per-language" : "combined"} bundle for a forced nightly`, + async (t) => { + const expectedTag = "codeql-bundle-30260213"; + const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; + stubHostedNightly(expectedTag); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" }); + const source = await setupCodeql.getCodeQLSource( + undefined, // toolsInput: the nightly is selected by ForceNightly + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures( + perLanguageBundles + ? [Feature.ForceNightly, Feature.PerLanguageBundles] + : [Feature.ForceNightly], + ), + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + const combinedURL = `${baseURL}/codeql-bundle-linux64.tar.zst`; + t.deepEqual( + source.bundle, + perLanguageBundles + ? { + kind: "per-language", + language: BuiltInLanguage.java, + url: `${baseURL}/codeql-bundle-java-linux64.tar.zst`, + combinedBundleURL: combinedURL, + } + : { kind: "combined", url: combinedURL }, + ); + } + }); + }, + ); +} + test.serial( "getCodeQLSource correctly returns latest version from toolcache when tools == toolcache", async (t) => { @@ -878,6 +1006,439 @@ test.serial( }, ); +const PER_LANGUAGE_CLI_VERSION = { + enabledVersions: [ + { + cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + tagName: `codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`, + }, + ], +}; + +test.serial("getCodeQLBundleName names the per-language bundle", (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + t.is( + setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.java), + "codeql-bundle-java-linux64.tar.zst", + ); + t.is( + setupCodeql.getCodeQLBundleName("zstd"), + "codeql-bundle-linux64.tar.zst", + ); +}); + +test.serial("getCodeQLBundleName names the Swift bundle for macOS", (t) => { + sinon.stub(process, "platform").value("darwin"); + t.is( + setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.swift), + "codeql-bundle-swift-osx64.tar.zst", + ); +}); + +test.serial( + "getCodeQLSource downloads the per-language bundle for a single explicit language", + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + undefined, + PER_LANGUAGE_CLI_VERSION, + ["java-kotlin"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + t.true( + source.bundle.url.endsWith("/codeql-bundle-java-linux64.tar.zst"), + `Unexpected URL ${source.bundle.url}`, + ); + t.is(source.bundle.kind, "per-language"); + if (source.bundle.kind === "per-language") { + t.is(source.bundle.language, BuiltInLanguage.java); + t.true( + source.bundle.combinedBundleURL?.endsWith( + "/codeql-bundle-linux64.tar.zst", + ), + ); + } + } + }); + }, +); + +test.serial( + "getCodeQLSource downloads the combined bundle when the feature is disabled", + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + undefined, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([]), + getRunnerLogger(true), + ); + + t.is(source.sourceType, "download"); + if (source.sourceType === "download") { + t.true(source.bundle.url.endsWith("/codeql-bundle-linux64.tar.zst")); + t.is(source.bundle.kind, "combined"); + } + }); + }, +); + +for (const fallback of [false, true]) { + test.serial( + `setupCodeQLBundle retains the selected release identity for an opaque asset URL${fallback ? " with fallback" : ""}`, + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + sinon.stub(actionsUtil, "isRunningLocalAction").returns(false); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const tag = PER_LANGUAGE_CLI_VERSION.enabledVersions[0].tagName; + const assetURL = + "https://api.github.com/repos/codeql-testing/action-fork/releases/assets/123"; + const combinedURL = `${assetURL}4`; + const fetchRelease = sinon + .stub, ReturnType>() + .callsFake( + async () => + new Response( + JSON.stringify({ + assets: [ + { name: "codeql-bundle-java-linux64.tar.zst", url: assetURL }, + { + name: "codeql-bundle-linux64.tar.zst", + url: combinedURL, + }, + ], + }), + { headers: { "content-type": "application/json" } }, + ), + ); + const client = github.getOctokit("123", { + request: { fetch: fetchRelease }, + }); + sinon.stub(api, "getApiClient").value(() => client); + const authorizationSpy = sinon.spy(api, "getAuthorizationHeaderFor"); + const extractStub = stubDownloadAndExtract(); + if (fallback) { + extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404)); + } + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir, { + GITHUB_ACTION_REPOSITORY: "codeql-testing/action-fork", + }); + const result = await setupCodeql.setupCodeQLBundle( + undefined, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.true(fetchRelease.calledTwice); + t.is( + fetchRelease.firstCall.args[0], + `https://api.github.com/repos/codeql-testing/action-fork/releases/tags/${tag}`, + ); + t.is(extractStub.callCount, fallback ? 2 : 1); + t.is(extractStub.firstCall.args[0], assetURL); + t.is(extractStub.lastCall.args[0], fallback ? combinedURL : assetURL); + t.is(authorizationSpy.callCount, extractStub.callCount); + t.is(authorizationSpy.firstCall.args[2], assetURL); + t.is( + authorizationSpy.lastCall.args[2], + fallback ? combinedURL : assetURL, + ); + t.is(extractStub.lastCall.args[3], "token token"); + t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION); + t.is( + result.toolsDownloadStatusReport?.bundleLanguage, + fallback ? undefined : BuiltInLanguage.java, + ); + t.is( + result.toolsDownloadStatusReport?.perLanguageBundleFallback, + fallback ? true : undefined, + ); + if (fallback) { + t.is( + result.codeqlFolder, + toolcache.find("CodeQL", MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION), + ); + t.true(fs.existsSync(`${result.codeqlFolder}.complete`)); + } else { + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + } + }); + }, + ); +} + +for (const bundle of ["per-language", "combined", "fallback"] as const) { + test.serial( + `setupCodeQLBundle preserves the nightly version for a ${bundle} download`, + async (t) => { + const expectedDate = "30260213"; + const expectedTag = `codeql-bundle-${expectedDate}`; + const expectedVersion = `0.0.0-${expectedDate}`; + const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; + const combinedURL = `${baseURL}/codeql-bundle-linux64.tar.zst`; + const perLanguageURL = `${baseURL}/codeql-bundle-javascript-linux64.tar.zst`; + const loggedMessages: LoggedMessage[] = []; + const logger = getRecordingLogger(loggedMessages); + + stubHostedNightly(expectedTag); + delete process.env[EnvVar.HAS_SET_UP_CODEQL]; + + const downloadSpy = sinon.spy(setupCodeql, "downloadCodeQL"); + const extractStub = stubDownloadAndExtract(); + if (bundle === "fallback") { + extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404)); + } + const addDiagnostic = sinon.stub(diagnostics, "addNoLanguageDiagnostic"); + const features = createFeatures([ + Feature.PerLanguageBundles, + Feature.CleanupToolcacheBundles, + ]); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + "nightly", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + bundle === "combined" ? ["javascript", "python"] : ["javascript"], + false, // useOverlayAwareDefaultCliVersion + features, + logger, + ); + + const source = downloadSpy.firstCall.args[0]; + t.is(result.toolsVersion, expectedVersion); + t.is(result.toolsVersion, source.toolsVersion); + t.is( + source.bundle.kind, + bundle === "combined" ? "combined" : "per-language", + ); + t.is(result.codeqlFolder, extractStub.lastCall.args[2]); + t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); + t.is(downloadSpy.callCount, extractStub.callCount); + t.is( + extractStub.firstCall.args[0], + bundle === "combined" ? combinedURL : perLanguageURL, + ); + t.is( + extractStub.lastCall.args[0], + bundle === "per-language" ? perLanguageURL : combinedURL, + ); + t.is( + result.toolsDownloadStatusReport?.bundleLanguage, + bundle === "per-language" ? BuiltInLanguage.javascript : undefined, + ); + t.is( + result.toolsDownloadStatusReport?.perLanguageBundleFallback, + bundle === "fallback" ? true : undefined, + ); + t.is( + addDiagnostic + .getCalls() + .filter( + (call) => + call.args[1].source?.id === + "codeql-action/toolcache-bundle-cleanup", + ).length, + 1, + ); + if (bundle === "fallback") { + t.deepEqual(downloadSpy.secondCall.args[0], { + ...source, + bundle: { kind: "combined", url: combinedURL }, + }); + checkExpectedLogMessages(t, loggedMessages, [ + `No javascript CodeQL bundle was found at ${perLanguageURL}`, + ]); + } + if (bundle === "per-language") { + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + } else { + t.is( + result.codeqlFolder, + toolsDownload.getToolcacheDirectory(expectedVersion), + ); + t.true(fs.existsSync(`${result.codeqlFolder}.complete`)); + + const cachedResult = await setupCodeql.setupCodeQLBundle( + "nightly", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + ["javascript"], + false, // useOverlayAwareDefaultCliVersion + features, + logger, + ); + t.is(cachedResult.toolsSource, setupCodeql.ToolsSource.Toolcache); + t.is(cachedResult.toolsVersion, expectedVersion); + t.is(cachedResult.codeqlFolder, result.codeqlFolder); + t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); + } + }); + }, + ); +} + +for (const asset of [ + "codeql-bundle-ruby-linux64.tar.zst", + "codeql-bundle-%72uby-linux64.tar.zst", +]) { + test.serial( + `setupCodeQLBundle keeps explicitly requested ${asset} out of the toolcache`, + async (t) => { + const extractStub = stubDownloadAndExtract(); + const url = `https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/${asset}`; + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + url, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ); + + t.true(extractStub.calledOnce); + t.is(extractStub.firstCall.args[0], url); + t.is(result.toolsVersion, "9.9.9"); + t.is( + result.toolsDownloadStatusReport?.bundleLanguage, + BuiltInLanguage.ruby, + ); + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + }); + }, + ); +} + +for (const error of [ + new HTTPError("Internal Server Error", 500), + new Error("Connection reset"), +]) { + test.serial( + `setupCodeQLBundle does not fall back after ${error.message}`, + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .rejects(error); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + undefined, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ), + { is: error }, + ); + t.true(extractStub.calledOnce); + t.true( + extractStub.firstCall.args[0].endsWith( + "/codeql-bundle-java-linux64.tar.zst", + ), + ); + }); + }, + ); +} + +test.serial( + "setupCodeQLBundle does not substitute a bundle for an explicitly requested one that is missing", + async (t) => { + const error = new HTTPError("Not Found", 404); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .rejects(error); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + "https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-ruby-linux64.tar.zst", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ), + { is: error }, + ); + + t.true(extractStub.calledOnce); + }); + }, +); + test.serial( "getEnabledVersionsWithOverlayBaseDatabases returns flag-enabled versions present in cache, sorted desc", async (t) => { diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index e5d6a77a94..85639ff93b 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -30,8 +30,13 @@ import { Feature, FeatureEnablement, } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; import { Logger } from "./logging"; import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching"; +import { + getPerLanguageBundleLanguage, + tryGetBundleLanguageFromUrl, +} from "./per-language-bundles"; import * as tar from "./tar"; import { deleteToolcacheBundles, @@ -72,21 +77,40 @@ function getCodeQLBundleExtension( } } +/** Returns the platform component of the CodeQL bundle name for the current platform. */ +export function getBundlePlatform(): string | undefined { + switch (process.platform) { + case "win32": + return "win64"; + case "linux": + return process.arch === "arm64" ? "linux-arm64" : "linux64"; + case "darwin": + return "osx64"; + default: + return undefined; + } +} + +/** + * Returns the name of the CodeQL bundle asset to download. + * + * @param compressionMethod The compression method of the bundle. + * @param language If provided, the name of the bundle that contains only this language, rather than + * the name of the combined bundle that contains every language. + */ export function getCodeQLBundleName( compressionMethod: tar.CompressionMethod, + language?: BuiltInLanguage, ): string { const extension = getCodeQLBundleExtension(compressionMethod); + const platform = getBundlePlatform(); - let platform: string; - if (process.platform === "win32") { - platform = "win64"; - } else if (process.platform === "linux") { - platform = process.arch === "arm64" ? "linux-arm64" : "linux64"; - } else if (process.platform === "darwin") { - platform = "osx64"; - } else { + if (platform === undefined) { return `codeql-bundle${extension}`; } + if (language !== undefined) { + return `codeql-bundle-${language}-${platform}${extension}`; + } return `codeql-bundle-${platform}${extension}`; } @@ -107,7 +131,7 @@ export function getCodeQLActionRepository(logger: Logger): string { async function getCodeQLBundleDownloadURL( tagName: string, apiDetails: api.GitHubApiDetails, - compressionMethod: tar.CompressionMethod, + codeQLBundleName: string, logger: Logger, ): Promise { const codeQLActionRepository = getCodeQLActionRepository(logger); @@ -126,7 +150,6 @@ async function getCodeQLBundleDownloadURL( return !self.slice(0, index).some((other) => deepEqual(source, other)); }, ); - const codeQLBundleName = getCodeQLBundleName(compressionMethod); for (const downloadSource of uniqueDownloadSources) { const [apiURL, repository] = downloadSource; // If we've reached the final case, short-circuit the API check since we know the bundle exists and is public. @@ -216,7 +239,15 @@ export function convertToSemVer(version: string, logger: Logger): string { } /** Describes the contents and location of a downloadable CodeQL bundle. */ -type CodeQLBundle = { kind: "combined"; url: string }; +type CodeQLBundle = + | { kind: "combined"; url: string } + | { + kind: "per-language"; + url: string; + language: BuiltInLanguage; + /** Only set when the Action selected the bundle, allowing a same-version fallback. */ + combinedBundleURL?: string; + }; /** A resolved download, including its bundle identity and version. */ export interface CodeQLDownloadSource { @@ -467,6 +498,7 @@ export async function getCodeQLSource( * This does not always include a tag name. */ let url: string | undefined; + let bundle: CodeQLBundle | undefined; // We allow forcing the nightly CLI via the FF for `dynamic` events (or in test mode) where the // `tools` input cannot be adjusted to explicitly request it. @@ -475,7 +507,8 @@ export async function getCodeQLSource( const forceNightly = forceNightlyValueFF && canForceNightlyWithFF; // For advanced workflows, a value from `CODEQL_NIGHTLY_TOOLS_INPUTS` can be specified explicitly - // for the `tools` input in the workflow file. + // for the `tools` input. This is the computed input, so it may come from the repository property + // rather than the workflow file. const nightlyRequestedByToolsInput = toolsInput !== undefined && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput); @@ -509,7 +542,8 @@ export async function getCodeQLSource( `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`, ); } - toolsInput = await getNightlyToolsUrl(logger); + bundle = await getNightlyBundle(rawLanguages, variant, features, logger); + toolsInput = bundle.url; } /** @@ -738,12 +772,42 @@ export async function getCodeQLSource( ? "zstd" : "gzip"; - url = await getCodeQLBundleDownloadURL( - tagName!, - apiDetails, - compressionMethod, + const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( + { + rawLanguages, + cliVersion, + compressionMethod, + platform: getBundlePlatform(), + variant, + }, + features, logger, ); + + const resolveBundleURL = (language?: BuiltInLanguage) => + getCodeQLBundleDownloadURL( + tagName!, + apiDetails, + getCodeQLBundleName(compressionMethod, language), + logger, + ); + + if (perLanguageBundleLanguage !== undefined) { + logger.info( + `Downloading the ${perLanguageBundleLanguage} CodeQL bundle, since ${perLanguageBundleLanguage} ` + + "is the only language being analyzed.", + ); + url = await resolveBundleURL(perLanguageBundleLanguage); + bundle = { + kind: "per-language", + url, + language: perLanguageBundleLanguage, + combinedBundleURL: await resolveBundleURL(), + }; + } else { + url = await resolveBundleURL(); + bundle = { kind: "combined", url }; + } } else { const method = tar.inferCompressionMethod(url); if (method === undefined) { @@ -753,6 +817,20 @@ export async function getCodeQLSource( ); } compressionMethod = method; + + if (bundle === undefined) { + // Explicit per-language URLs must also stay out of the toolcache, but have no fallback. + const language = tryGetBundleLanguageFromUrl(url); + bundle = + language === undefined + ? { kind: "combined", url } + : { kind: "per-language", url, language }; + } + if (bundle.kind === "per-language") { + logger.info( + `${url} appears to be a CodeQL bundle that contains only ${bundle.language}.`, + ); + } } if (cliVersion) { @@ -761,7 +839,7 @@ export async function getCodeQLSource( logger.info(`Using CodeQL CLI sourced from ${url} .`); } return { - bundle: { kind: "combined", url }, + bundle, bundleVersion, cliVersion, compressionMethod, @@ -841,8 +919,11 @@ export const downloadCodeQL = async function ( writeToolcacheMarkerFile(toolcacheDestination, logger); } else { logger.debug( - "Could not cache CodeQL tools because we could not determine the bundle version from the " + - `URL ${codeqlURL}.`, + bundle.kind === "per-language" + ? "Not caching the CodeQL tools because they came from a bundle that contains only a " + + "single language." + : "Could not cache CodeQL tools because we could not determine the bundle version from the " + + `URL ${codeqlURL}.`, ); } @@ -860,7 +941,8 @@ function getToolcacheDestination( source: CodeQLDownloadSource, logger: Logger, ): string | undefined { - if (!source.bundleVersion) { + // Per-language bundles must not be stored in the toolcache. + if (source.bundle.kind !== "combined" || !source.bundleVersion) { return undefined; } @@ -1046,6 +1128,9 @@ export async function setupCodeQLBundle( /** * Performs eligible toolcache cleanup once, then downloads and extracts the resolved bundle. * + * If `source` refers to a bundle for a single language and that bundle turns out not to exist, this + * falls back to downloading the combined bundle. + * * @returns The extraction directory and download timings. */ export async function downloadCodeQLBundle( @@ -1058,14 +1143,60 @@ export async function downloadCodeQLBundle( codeqlFolder: string; statusReport: ToolsDownloadStatusReport; }> { + const { bundle } = source; + const { logger } = action; + await tryDeleteToolcacheBundles(action); - return await downloadCodeQL( - source, - apiDetails, - tarVersion, - tempDir, - action.logger, - ); + + try { + const result = await downloadCodeQL( + source, + apiDetails, + tarVersion, + tempDir, + logger, + ); + return bundle.kind === "combined" + ? result + : { + ...result, + statusReport: { + ...result.statusReport, + bundleLanguage: bundle.language, + }, + }; + } catch (e) { + if ( + bundle.kind !== "per-language" || + bundle.combinedBundleURL === undefined || + util.asHTTPError(e)?.status !== 404 + ) { + throw e; + } + logger.warning( + `No ${bundle.language} CodeQL bundle was found at ${bundle.url}, so ` + + "falling back to the bundle that contains all languages. This analysis will still " + + "produce correct results, but will take longer to set up.", + ); + + const result = await downloadCodeQL( + { + ...source, + bundle: { kind: "combined", url: bundle.combinedBundleURL }, + }, + apiDetails, + tarVersion, + tempDir, + logger, + ); + return { + ...result, + statusReport: { + ...result.statusReport, + perLanguageBundleFallback: true, + }, + }; + } } async function useZstdBundle( @@ -1084,10 +1215,13 @@ function getTempExtractionDir(tempDir: string) { return path.join(tempDir, uuidV4()); } -/** - * Get the URL of the latest nightly CodeQL bundle. - */ -async function getNightlyToolsUrl(logger: Logger) { +/** Selects a bundle from the latest nightly, with a same-release fallback when applicable. */ +async function getNightlyBundle( + rawLanguages: string[] | undefined, + variant: util.GitHubVariant, + features: FeatureEnablement, + logger: Logger, +): Promise { const zstdAvailability = await tar.isZstdAvailable(logger); // The nightly is guaranteed to have a zstd bundle const compressionMethod = (await useZstdBundle( @@ -1097,6 +1231,19 @@ async function getNightlyToolsUrl(logger: Logger) { ? "zstd" : "gzip"; + const language = await getPerLanguageBundleLanguage( + { + rawLanguages, + cliVersion: undefined, + compressionMethod, + platform: getBundlePlatform(), + variant, + isNightly: true, + }, + features, + logger, + ); + try { // Since nightlies are prereleases, we can't just download the latest release // on the repository. So instead we need to find the latest pre-release @@ -1112,7 +1259,17 @@ async function getNightlyToolsUrl(logger: Logger) { if (!latestRelease) { throw new Error("Could not find the latest nightly release."); } - return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`; + const assetUrl = (name: string) => + `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`; + const url = assetUrl(getCodeQLBundleName(compressionMethod, language)); + return language === undefined + ? { kind: "combined", url } + : { + kind: "per-language", + url, + language, + combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod)), + }; } catch (e) { throw new Error( `Failed to retrieve the latest nightly release: ${util.wrapError(e)}`, diff --git a/src/status-report.ts b/src/status-report.ts index a2acd631d6..820b1c2109 100644 --- a/src/status-report.ts +++ b/src/status-report.ts @@ -645,6 +645,13 @@ export interface InitToolsDownloadFields { * Whether the relevant tools dotcom feature flags have been misconfigured. * Only populated if we attempt to determine the default version based on the dotcom feature flags. */ tools_feature_flags_valid?: boolean; + /** The language of the single-language bundle that was downloaded, if any. */ + tools_bundle_language?: string; + /** + * Whether we tried to download a single-language bundle, but it did not exist and we fell back to + * the combined bundle. + */ + tools_per_language_bundle_fallback?: boolean; } /** diff --git a/src/tools-download.ts b/src/tools-download.ts index 222a18cd91..f7b0a708ce 100644 --- a/src/tools-download.ts +++ b/src/tools-download.ts @@ -54,6 +54,13 @@ export type ToolsDownloadStatusReport = { * spent on a streaming attempt that failed and fell back to downloading before extracting. */ totalDurationMs: number; + /** The language of the single-language bundle that was downloaded, if any. */ + bundleLanguage?: string; + /** + * Whether we tried to download a single-language bundle, but it did not exist and we fell back to + * the combined bundle. + */ + perLanguageBundleFallback?: boolean; }; export async function downloadAndExtract( From 59ce3a25ba97f6eccde6eb1e3c559d7b410ff597 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 15 Sep 2026 19:57:17 +0100 Subject: [PATCH 04/20] Include failed bundle attempts in fallback timing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 23 +++++++++++++---------- src/setup-codeql.test.ts | 28 ++++++++++++++++++++++++---- src/setup-codeql.ts | 3 +++ 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index 88ad90dcfa..01091d39a7 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -4305,7 +4305,7 @@ var require_util2 = __commonJS({ var { redirectStatusSet, referrerPolicySet: referrerPolicyTokens, badPortsSet } = require_constants3(); var { getGlobalOrigin } = require_global(); var { collectASequenceOfCodePoints, collectAnHTTPQuotedString, removeChars, parseMIMEType } = require_data_url(); - var { performance: performance6 } = require("node:perf_hooks"); + var { performance: performance7 } = require("node:perf_hooks"); var { isBlobLike, ReadableStreamFrom, isValidHTTPToken, normalizedMethodRecordsBase } = require_util(); var assert = require("node:assert"); var { isUint8Array } = require("node:util/types"); @@ -4464,7 +4464,7 @@ var require_util2 = __commonJS({ }; } function coarsenedSharedCurrentTime(crossOriginIsolatedCapability) { - return coarsenTime(performance6.now(), crossOriginIsolatedCapability); + return coarsenTime(performance7.now(), crossOriginIsolatedCapability); } function createOpaqueTimingInfo(timingInfo) { return { @@ -142119,7 +142119,7 @@ module.exports = __toCommonJS(entry_points_exports); // src/analyze-action.ts var fs23 = __toESM(require("fs")); var import_path5 = __toESM(require("path")); -var import_perf_hooks4 = require("perf_hooks"); +var import_perf_hooks5 = require("perf_hooks"); var core17 = __toESM(require_core()); // src/action-common.ts @@ -148611,7 +148611,7 @@ var SarifScanOrder = [ // src/analyze.ts var fs17 = __toESM(require("fs")); var path16 = __toESM(require("path")); -var import_perf_hooks3 = require("perf_hooks"); +var import_perf_hooks4 = require("perf_hooks"); var io5 = __toESM(require_io()); // src/autobuild.ts @@ -151194,6 +151194,7 @@ async function logGeneratedFilesTelemetry(config, duration, generatedFilesCount) // src/setup-codeql.ts var fs14 = __toESM(require("fs")); var path13 = __toESM(require("path")); +var import_perf_hooks3 = require("perf_hooks"); var core12 = __toESM(require_core()); var toolcache3 = __toESM(require_tool_cache()); var import_fast_deep_equal = __toESM(require_fast_deep_equal()); @@ -152632,6 +152633,7 @@ async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, temp const { bundle } = source; const { logger } = action; await tryDeleteToolcacheBundles(action); + const startTime = import_perf_hooks3.performance.now(); try { const result = await downloadCodeQL( source, @@ -152668,6 +152670,7 @@ async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, temp ...result, statusReport: { ...result.statusReport, + totalDurationMs: Math.round(import_perf_hooks3.performance.now() - startTime), perLanguageBundleFallback: true } }; @@ -153834,10 +153837,10 @@ function dbIsFinalized(config, language, logger) { } } async function finalizeDatabaseCreation(codeql, features, config, threadsFlag, memoryFlag, logger) { - const extractionStart = import_perf_hooks3.performance.now(); + const extractionStart = import_perf_hooks4.performance.now(); await runExtraction(codeql, features, config, logger); - const extractionTime = import_perf_hooks3.performance.now() - extractionStart; - const trapImportStart = import_perf_hooks3.performance.now(); + const extractionTime = import_perf_hooks4.performance.now() - extractionStart; + const trapImportStart = import_perf_hooks4.performance.now(); for (const language of config.languages) { if (dbIsFinalized(config, language, logger)) { logger.info( @@ -153854,7 +153857,7 @@ async function finalizeDatabaseCreation(codeql, features, config, threadsFlag, m logger.endGroup(); } } - const trapImportTime = import_perf_hooks3.performance.now() - trapImportStart; + const trapImportTime = import_perf_hooks4.performance.now() - trapImportStart; return { scanned_language_extraction_duration_ms: Math.round(extractionTime), trap_import_duration_ms: Math.round(trapImportTime) @@ -156661,9 +156664,9 @@ async function run({ startedAt, logger }) { features, logger ); - const trapCacheUploadStartTime = import_perf_hooks4.performance.now(); + const trapCacheUploadStartTime = import_perf_hooks5.performance.now(); didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger); - trapCacheUploadTime = import_perf_hooks4.performance.now() - trapCacheUploadStartTime; + trapCacheUploadTime = import_perf_hooks5.performance.now() - trapCacheUploadStartTime; trapCacheCleanupTelemetry = await cleanupTrapCaches( config, features, diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index f7caf575f3..5e69ddaaa7 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; +import { performance } from "perf_hooks"; import * as github from "@actions/github"; import * as toolcache from "@actions/tool-cache"; @@ -1223,10 +1224,23 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) { delete process.env[EnvVar.HAS_SET_UP_CODEQL]; const downloadSpy = sinon.spy(setupCodeql, "downloadCodeQL"); - const extractStub = stubDownloadAndExtract(); - if (bundle === "fallback") { - extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404)); - } + let elapsedMs = 1000; + sinon.stub(performance, "now").callsFake(() => elapsedMs); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .callsFake(async (_url, _compressionMethod, dest) => { + if (bundle === "fallback" && extractStub.callCount === 1) { + elapsedMs += 700.2; + throw new HTTPError("Not Found", 404); + } + elapsedMs += 300.2; + fs.mkdirSync(dest, { recursive: true }); + return { + downloadDurationMs: 200, + extractionDurationMs: 100, + totalDurationMs: 300, + }; + }); const addDiagnostic = sinon.stub(diagnostics, "addNoLanguageDiagnostic"); const features = createFeatures([ Feature.PerLanguageBundles, @@ -1255,6 +1269,12 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) { bundle === "combined" ? "combined" : "per-language", ); t.is(result.codeqlFolder, extractStub.lastCall.args[2]); + t.is( + result.toolsDownloadStatusReport?.totalDurationMs, + bundle === "fallback" ? 1000 : 300, + ); + t.is(result.toolsDownloadStatusReport?.downloadDurationMs, 200); + t.is(result.toolsDownloadStatusReport?.extractionDurationMs, 100); t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); t.is(downloadSpy.callCount, extractStub.callCount); t.is( diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 85639ff93b..27090ffdd3 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import { OutgoingHttpHeaders } from "http"; import * as path from "path"; +import { performance } from "perf_hooks"; import * as core from "@actions/core"; import * as toolcache from "@actions/tool-cache"; @@ -1148,6 +1149,7 @@ export async function downloadCodeQLBundle( await tryDeleteToolcacheBundles(action); + const startTime = performance.now(); try { const result = await downloadCodeQL( source, @@ -1193,6 +1195,7 @@ export async function downloadCodeQLBundle( ...result, statusReport: { ...result.statusReport, + totalDurationMs: Math.round(performance.now() - startTime), perLanguageBundleFallback: true, }, }; From 2d47caf1235ac0fd1400d51b879c3c468927c125 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 16 Sep 2026 17:25:24 +0100 Subject: [PATCH 05/20] Isolate per-language bundle eligibility state Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 48 +++++++------- src/actions-util.ts | 4 +- src/per-language-bundles.test.ts | 105 +++++++++++++++++++++---------- src/per-language-bundles.ts | 21 ++++--- src/setup-codeql.ts | 33 +++++----- 5 files changed, 129 insertions(+), 82 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index 01091d39a7..3467839285 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -151525,7 +151525,11 @@ var PER_LANGUAGE_BUNDLE_PLATFORMS = { ["rust" /* rust */]: "linux64", ["swift" /* swift */]: "osx64" }; -async function getPerLanguageBundleLanguage(options, features, logger) { +async function getPerLanguageBundleLanguage({ + env, + features, + logger +}, options) { const { rawLanguages, cliVersion: cliVersion2, @@ -151538,6 +151542,9 @@ async function getPerLanguageBundleLanguage(options, features, logger) { logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); return void 0; }; + if (!await features.getValue("per_language_bundles" /* PerLanguageBundles */)) { + return explain(`the ${"per_language_bundles" /* PerLanguageBundles */} feature is disabled`); + } if (rawLanguages?.length !== 1) { return explain( `exactly one language must be requested via the 'languages' input, but ${rawLanguages?.length ?? 0} were` @@ -151553,7 +151560,7 @@ async function getPerLanguageBundleLanguage(options, features, logger) { if (variant !== "GitHub.com" /* DOTCOM */) { return explain(`we are running against ${variant}`); } - if (!isGitHubHostedRunner()) { + if (!isGitHubHostedRunner(env)) { return explain("the job is not running on a GitHub-hosted runner"); } if (!isNightly) { @@ -151575,9 +151582,6 @@ async function getPerLanguageBundleLanguage(options, features, logger) { `the ${language} bundle is only published for ${supportedPlatform}, but this job is running on ${platform2 ?? "an unknown platform"}` ); } - if (!await features.getValue("per_language_bundles" /* PerLanguageBundles */)) { - return explain(`the ${"per_language_bundles" /* PerLanguageBundles */} feature is disabled`); - } return language; } @@ -152254,7 +152258,11 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.` ); } - bundle = await getNightlyBundle(rawLanguages, variant, features, logger); + bundle = await getNightlyBundle( + { env: getEnv(), features, logger }, + rawLanguages, + variant + ); toolsInput = bundle.url; } const forceShippedTools = toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput); @@ -152410,15 +152418,14 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO if (!url2) { compressionMethod = cliVersion2 !== void 0 && await useZstdBundle(cliVersion2, tarSupportsZstd) ? "zstd" : "gzip"; const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( + { env: getEnv(), features, logger }, { rawLanguages, cliVersion: cliVersion2, compressionMethod, platform: getBundlePlatform(), variant - }, - features, - logger + } ); const resolveBundleURL = (language) => getCodeQLBundleDownloadURL( tagName, @@ -152685,24 +152692,21 @@ async function useZstdBundle(cliVersion2, tarSupportsZstd) { function getTempExtractionDir(tempDir) { return path13.join(tempDir, v4_default()); } -async function getNightlyBundle(rawLanguages, variant, features, logger) { +async function getNightlyBundle(action, rawLanguages, variant) { + const { logger } = action; const zstdAvailability = await isZstdAvailable(logger); const compressionMethod = await useZstdBundle( CODEQL_VERSION_ZSTD_BUNDLE, zstdAvailability.available ) ? "zstd" : "gzip"; - const language = await getPerLanguageBundleLanguage( - { - rawLanguages, - cliVersion: void 0, - compressionMethod, - platform: getBundlePlatform(), - variant, - isNightly: true - }, - features, - logger - ); + const language = await getPerLanguageBundleLanguage(action, { + rawLanguages, + cliVersion: void 0, + compressionMethod, + platform: getBundlePlatform(), + variant, + isNightly: true + }); try { const release2 = await getApiClient().rest.repos.listReleases({ owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER, diff --git a/src/actions-util.ts b/src/actions-util.ts index eb7d92b517..677bb04b1b 100644 --- a/src/actions-util.ts +++ b/src/actions-util.ts @@ -7,7 +7,7 @@ import * as github from "@actions/github"; import * as io from "@actions/io"; import type { Config } from "./config-utils"; -import { Env, EnvVar, ActionsEnvVars } from "./environment"; +import { Env, EnvVar, ActionsEnvVars, ReadOnlyEnv } from "./environment"; import { Logger } from "./logging"; import { doesDirectoryExist, @@ -292,7 +292,7 @@ export function isSelfHostedRunner(env: Env = getEnv()) { * that are configured to resemble hosted ones, such as those that mount a persistent volume at * `/opt/hostedtoolcache`. */ -export function isGitHubHostedRunner(env: Env = getEnv()) { +export function isGitHubHostedRunner(env: ReadOnlyEnv = getEnv()) { return env.getOptional(ActionsEnvVars.RUNNER_ENVIRONMENT) === "github-hosted"; } diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts index 8242bd0144..5994246214 100644 --- a/src/per-language-bundles.test.ts +++ b/src/per-language-bundles.test.ts @@ -1,20 +1,22 @@ import test from "ava"; -import { ActionsEnvVars } from "./environment"; +import { ActionsEnvVars, ReadOnlyEnv } from "./environment"; import { Feature } from "./feature-flags"; import { BuiltInLanguage } from "./languages"; -import { getRunnerLogger } from "./logging"; import { getPerLanguageBundleLanguage, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, PerLanguageBundleOptions, tryGetBundleLanguageFromUrl, } from "./per-language-bundles"; -import { createFeatures, setupTests } from "./testing-utils"; +import { + createFeatures, + getRecordingLogger, + getTestEnv, + LoggedMessage, +} from "./testing-utils"; import { GitHubVariant } from "./util"; -setupTests(test); - /** Options for which we would use a per-language bundle. */ const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = { rawLanguages: ["java"], @@ -28,19 +30,21 @@ const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = { async function checkEligibility( overrides: Partial, enabledFeatures: Feature[] = [Feature.PerLanguageBundles], + env: ReadOnlyEnv = getTestEnv({ + [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted", + }), ) { return getPerLanguageBundleLanguage( + { + env, + features: createFeatures(enabledFeatures), + logger: getRecordingLogger([], { logToConsole: false }), + }, { ...ELIGIBLE_OPTIONS, ...overrides }, - createFeatures(enabledFeatures), - getRunnerLogger(true), ); } -test.beforeEach(() => { - process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; -}); - -test.serial("uses Linux bundles for non-Swift languages", async (t) => { +test("getPerLanguageBundleLanguage selects Linux bundles for non-Swift languages", async (t) => { for (const language of Object.values(BuiltInLanguage)) { if (language === BuiltInLanguage.swift) { continue; @@ -49,14 +53,14 @@ test.serial("uses Linux bundles for non-Swift languages", async (t) => { } }); -test.serial("normalizes an alias before selecting a bundle", async (t) => { +test("getPerLanguageBundleLanguage normalizes aliases before selecting a bundle", async (t) => { t.is( await checkEligibility({ rawLanguages: ["java-kotlin"] }), BuiltInLanguage.java, ); }); -test.serial("uses the macOS bundle for Swift", async (t) => { +test("getPerLanguageBundleLanguage selects the macOS bundle for Swift", async (t) => { t.is( await checkEligibility({ rawLanguages: ["swift"], platform: "osx64" }), BuiltInLanguage.swift, @@ -68,7 +72,7 @@ test.serial("uses the macOS bundle for Swift", async (t) => { ); }); -test.serial("only publishes non-Swift languages for Linux", async (t) => { +test("getPerLanguageBundleLanguage rejects unsupported platforms", async (t) => { t.is(await checkEligibility({ platform: "osx64" }), undefined); t.is(await checkEligibility({ platform: "win64" }), undefined); // We do not publish per-language bundles for Linux Arm64 either. @@ -76,21 +80,21 @@ test.serial("only publishes non-Swift languages for Linux", async (t) => { t.is(await checkEligibility({ platform: undefined }), undefined); }); -test.serial("requires exactly one language", async (t) => { +test("getPerLanguageBundleLanguage requires exactly one language", async (t) => { t.is(await checkEligibility({ rawLanguages: undefined }), undefined); t.is(await checkEligibility({ rawLanguages: [] }), undefined); t.is(await checkEligibility({ rawLanguages: ["java", "python"] }), undefined); }); -test.serial("requires a language that CodeQL knows about", async (t) => { +test("getPerLanguageBundleLanguage requires a known language", async (t) => { t.is(await checkEligibility({ rawLanguages: ["cobol"] }), undefined); }); -test.serial("requires a zstd bundle", async (t) => { +test("getPerLanguageBundleLanguage requires a zstd bundle", async (t) => { t.is(await checkEligibility({ compressionMethod: "gzip" }), undefined); }); -test.serial("requires GitHub.com", async (t) => { +test("getPerLanguageBundleLanguage requires GitHub.com", async (t) => { // Other products resolve the combined bundle against their own instance, so asking for a // per-language bundle they do not mirror would move the download off that instance. for (const variant of [GitHubVariant.GHES, GitHubVariant.GHEC_DR]) { @@ -98,30 +102,61 @@ test.serial("requires GitHub.com", async (t) => { } }); -test.serial("requires a GitHub-hosted runner", async (t) => { +test("getPerLanguageBundleLanguage requires a GitHub-hosted runner", async (t) => { // A self-hosted runner may have a toolcache that persists between jobs, which is worth more than // a smaller download. - process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; - t.is(await checkEligibility({}), undefined); + t.is( + await checkEligibility( + {}, + [Feature.PerLanguageBundles], + getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "self-hosted" }), + ), + undefined, + ); // Self-hosted runners are routinely configured to look like hosted ones, for example by mounting // a persistent volume at `/opt/hostedtoolcache`, so we require the service to tell us explicitly. - delete process.env[ActionsEnvVars.RUNNER_ENVIRONMENT]; - process.env["RUNNER_TOOL_CACHE"] = "/opt/hostedtoolcache"; - t.is(await checkEligibility({}), undefined); + t.is( + await checkEligibility( + {}, + [Feature.PerLanguageBundles], + getTestEnv({ RUNNER_TOOL_CACHE: "/opt/hostedtoolcache" }), + ), + undefined, + ); }); -test.serial("requires a new enough CLI version", async (t) => { +test("getPerLanguageBundleLanguage requires a supported release version", async (t) => { t.is(await checkEligibility({ cliVersion: undefined }), undefined); t.is(await checkEligibility({ cliVersion: "2.27.0" }), undefined); t.is(await checkEligibility({ cliVersion: "2.27.1" }), BuiltInLanguage.java); }); -test.serial("requires the feature flag", async (t) => { +test("getPerLanguageBundleLanguage requires the feature flag", async (t) => { t.is(await checkEligibility({}, []), undefined); }); -test.serial("nightlies skip only the release version check", async (t) => { +test("getPerLanguageBundleLanguage explains a disabled feature before checking eligibility", async (t) => { + const messages: LoggedMessage[] = []; + const language = await getPerLanguageBundleLanguage( + { + env: getTestEnv(), + features: createFeatures([]), + logger: getRecordingLogger(messages, { logToConsole: false }), + }, + { ...ELIGIBLE_OPTIONS, rawLanguages: undefined, cliVersion: undefined }, + ); + + t.is(language, undefined); + t.deepEqual( + messages.map((message) => message.message), + [ + "Not using a per-language CodeQL bundle since the per_language_bundles feature is disabled.", + ], + ); +}); + +test("getPerLanguageBundleLanguage skips only the release version check for nightlies", async (t) => { const nightly = { isNightly: true, cliVersion: undefined }; t.is(await checkEligibility(nightly), BuiltInLanguage.java); @@ -136,11 +171,17 @@ test.serial("nightlies skip only the release version check", async (t) => { t.is(await checkEligibility({ ...nightly, ...overrides }), undefined); } t.is(await checkEligibility(nightly, []), undefined); - process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; - t.is(await checkEligibility(nightly), undefined); + t.is( + await checkEligibility( + nightly, + [Feature.PerLanguageBundles], + getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "self-hosted" }), + ), + undefined, + ); }); -test.serial("recognizes a per-language bundle from its URL", (t) => { +test("tryGetBundleLanguageFromUrl recognizes per-language bundle URLs", (t) => { const url = (name: string) => `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; @@ -165,7 +206,7 @@ test.serial("recognizes a per-language bundle from its URL", (t) => { ); }); -test.serial("does not mistake other bundles for per-language ones", (t) => { +test("tryGetBundleLanguageFromUrl rejects other bundle URLs", (t) => { const url = (name: string) => `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts index 1b63e4f100..3ffa58d8fd 100644 --- a/src/per-language-bundles.ts +++ b/src/per-language-bundles.ts @@ -1,9 +1,9 @@ import * as semver from "semver"; +import { ActionState } from "./action-common"; import { isGitHubHostedRunner } from "./actions-util"; -import { Feature, FeatureEnablement } from "./feature-flags"; +import { Feature } from "./feature-flags"; import { BuiltInLanguage, parseBuiltInLanguage } from "./languages"; -import { Logger } from "./logging"; import * as tar from "./tar"; import { GitHubVariant } from "./util"; @@ -61,9 +61,12 @@ export interface PerLanguageBundleOptions { /** Returns the eligible bundle language, or undefined for the combined bundle. */ export async function getPerLanguageBundleLanguage( + { + env, + features, + logger, + }: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, options: PerLanguageBundleOptions, - features: FeatureEnablement, - logger: Logger, ): Promise { const { rawLanguages, @@ -79,6 +82,10 @@ export async function getPerLanguageBundleLanguage( return undefined; }; + if (!(await features.getValue(Feature.PerLanguageBundles))) { + return explain(`the ${Feature.PerLanguageBundles} feature is disabled`); + } + if (rawLanguages?.length !== 1) { return explain( `exactly one language must be requested via the 'languages' input, but ${ @@ -103,7 +110,7 @@ export async function getPerLanguageBundleLanguage( return explain(`we are running against ${variant}`); } - if (!isGitHubHostedRunner()) { + if (!isGitHubHostedRunner(env)) { // Per-language installs stay out of the toolcache; self-hosted runners should retain // the reusable combined bundle instead. return explain("the job is not running on a GitHub-hosted runner"); @@ -134,9 +141,5 @@ export async function getPerLanguageBundleLanguage( ); } - if (!(await features.getValue(Feature.PerLanguageBundles))) { - return explain(`the ${Feature.PerLanguageBundles} feature is disabled`); - } - return language; } diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 27090ffdd3..48c417f6da 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -543,7 +543,11 @@ export async function getCodeQLSource( `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`, ); } - bundle = await getNightlyBundle(rawLanguages, variant, features, logger); + bundle = await getNightlyBundle( + { env: getEnv(), features, logger }, + rawLanguages, + variant, + ); toolsInput = bundle.url; } @@ -774,6 +778,7 @@ export async function getCodeQLSource( : "gzip"; const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( + { env: getEnv(), features, logger }, { rawLanguages, cliVersion, @@ -781,8 +786,6 @@ export async function getCodeQLSource( platform: getBundlePlatform(), variant, }, - features, - logger, ); const resolveBundleURL = (language?: BuiltInLanguage) => @@ -1220,11 +1223,11 @@ function getTempExtractionDir(tempDir: string) { /** Selects a bundle from the latest nightly, with a same-release fallback when applicable. */ async function getNightlyBundle( + action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, rawLanguages: string[] | undefined, variant: util.GitHubVariant, - features: FeatureEnablement, - logger: Logger, ): Promise { + const { logger } = action; const zstdAvailability = await tar.isZstdAvailable(logger); // The nightly is guaranteed to have a zstd bundle const compressionMethod = (await useZstdBundle( @@ -1234,18 +1237,14 @@ async function getNightlyBundle( ? "zstd" : "gzip"; - const language = await getPerLanguageBundleLanguage( - { - rawLanguages, - cliVersion: undefined, - compressionMethod, - platform: getBundlePlatform(), - variant, - isNightly: true, - }, - features, - logger, - ); + const language = await getPerLanguageBundleLanguage(action, { + rawLanguages, + cliVersion: undefined, + compressionMethod, + platform: getBundlePlatform(), + variant, + isNightly: true, + }); try { // Since nightlies are prereleases, we can't just download the latest release From dfb9bf52c9208b8ac364b20962cab8a1caf07cea Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 16 Sep 2026 17:28:00 +0100 Subject: [PATCH 06/20] Share CodeQL bundle platform definitions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 61 +++++++++++++++++--------------- src/bundle-platform.test.ts | 18 ++++++++++ src/bundle-platform.ts | 26 ++++++++++++++ src/per-language-bundles.test.ts | 44 ++++++++++------------- src/per-language-bundles.ts | 48 +++++++++++++------------ src/setup-codeql.ts | 15 +------- src/testing-utils.ts | 12 ++----- 7 files changed, 124 insertions(+), 100 deletions(-) create mode 100644 src/bundle-platform.test.ts create mode 100644 src/bundle-platform.ts diff --git a/lib/entry-points.js b/lib/entry-points.js index 3467839285..77e1764cd1 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -151200,6 +151200,20 @@ var toolcache3 = __toESM(require_tool_cache()); var import_fast_deep_equal = __toESM(require_fast_deep_equal()); var semver10 = __toESM(require_semver2()); +// src/bundle-platform.ts +function getBundlePlatform(platform2 = process.platform, arch2 = process.arch) { + switch (platform2) { + case "win32": + return "win64" /* Win64 */; + case "linux": + return arch2 === "arm64" ? "linux-arm64" /* LinuxArm64 */ : "linux64" /* Linux64 */; + case "darwin": + return "osx64" /* Osx64 */; + default: + return void 0; + } +} + // src/overlay/caching.ts var fs11 = __toESM(require("fs")); var actionsCache3 = __toESM(require_cache4()); @@ -151513,17 +151527,21 @@ function tryGetBundleLanguageFromUrl(url2) { const match2 = assetName.match(PER_LANGUAGE_BUNDLE_NAME); return match2 ? parseBuiltInLanguage(match2[1]) : void 0; } -var PER_LANGUAGE_BUNDLE_PLATFORMS = { - ["actions" /* actions */]: "linux64", - ["cpp" /* cpp */]: "linux64", - ["csharp" /* csharp */]: "linux64", - ["go" /* go */]: "linux64", - ["java" /* java */]: "linux64", - ["javascript" /* javascript */]: "linux64", - ["python" /* python */]: "linux64", - ["ruby" /* ruby */]: "linux64", - ["rust" /* rust */]: "linux64", - ["swift" /* swift */]: "osx64" +var PER_LANGUAGE_BUNDLE_LANGUAGES = { + ["linux64" /* Linux64 */]: /* @__PURE__ */ new Set([ + "actions" /* actions */, + "cpp" /* cpp */, + "csharp" /* csharp */, + "go" /* go */, + "java" /* java */, + "javascript" /* javascript */, + "python" /* python */, + "ruby" /* ruby */, + "rust" /* rust */ + ]), + ["linux-arm64" /* LinuxArm64 */]: /* @__PURE__ */ new Set(), + ["osx64" /* Osx64 */]: /* @__PURE__ */ new Set(["swift" /* swift */]), + ["win64" /* Win64 */]: /* @__PURE__ */ new Set() }; async function getPerLanguageBundleLanguage({ env, @@ -151573,13 +151591,10 @@ async function getPerLanguageBundleLanguage({ ); } } - const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language]; - if (supportedPlatform === void 0) { - return explain(`no per-language bundle is published for ${language}`); - } - if (supportedPlatform !== platform2) { + const supportedLanguages = platform2 === void 0 ? void 0 : PER_LANGUAGE_BUNDLE_LANGUAGES[platform2]; + if (!supportedLanguages?.has(language)) { return explain( - `the ${language} bundle is only published for ${supportedPlatform}, but this job is running on ${platform2 ?? "an unknown platform"}` + `no per-language bundle is published for ${language} on ${platform2 ?? "an unknown platform"}` ); } return language; @@ -151992,18 +152007,6 @@ function getCodeQLBundleExtension(compressionMethod) { assertNever(compressionMethod); } } -function getBundlePlatform() { - switch (process.platform) { - case "win32": - return "win64"; - case "linux": - return process.arch === "arm64" ? "linux-arm64" : "linux64"; - case "darwin": - return "osx64"; - default: - return void 0; - } -} function getCodeQLBundleName(compressionMethod, language) { const extension = getCodeQLBundleExtension(compressionMethod); const platform2 = getBundlePlatform(); diff --git a/src/bundle-platform.test.ts b/src/bundle-platform.test.ts new file mode 100644 index 0000000000..73508ca236 --- /dev/null +++ b/src/bundle-platform.test.ts @@ -0,0 +1,18 @@ +import test from "ava"; + +import { BundlePlatform, getBundlePlatform } from "./bundle-platform"; + +for (const [platform, arch, expected] of [ + ["linux", "x64", BundlePlatform.Linux64], + ["linux", "arm64", BundlePlatform.LinuxArm64], + ["linux", "ia32", BundlePlatform.Linux64], + ["darwin", "x64", BundlePlatform.Osx64], + ["darwin", "arm64", BundlePlatform.Osx64], + ["win32", "x64", BundlePlatform.Win64], + ["win32", "arm64", BundlePlatform.Win64], + ["freebsd", "x64", undefined], +] as const) { + test(`getBundlePlatform maps ${platform}/${arch} to ${expected ?? "an all-platform bundle"}`, (t) => { + t.is(getBundlePlatform(platform, arch), expected); + }); +} diff --git a/src/bundle-platform.ts b/src/bundle-platform.ts new file mode 100644 index 0000000000..1cc085d6ab --- /dev/null +++ b/src/bundle-platform.ts @@ -0,0 +1,26 @@ +/** Platform identifiers used in CodeQL bundle asset names. */ +export enum BundlePlatform { + Linux64 = "linux64", + LinuxArm64 = "linux-arm64", + Osx64 = "osx64", + Win64 = "win64", +} + +/** Returns the bundle platform, or undefined when an all-platform bundle is required. */ +export function getBundlePlatform( + platform: NodeJS.Platform = process.platform, + arch: NodeJS.Architecture = process.arch, +): BundlePlatform | undefined { + switch (platform) { + case "win32": + return BundlePlatform.Win64; + case "linux": + return arch === "arm64" + ? BundlePlatform.LinuxArm64 + : BundlePlatform.Linux64; + case "darwin": + return BundlePlatform.Osx64; + default: + return undefined; + } +} diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts index 5994246214..d192d43c6d 100644 --- a/src/per-language-bundles.test.ts +++ b/src/per-language-bundles.test.ts @@ -1,5 +1,6 @@ import test from "ava"; +import { BundlePlatform } from "./bundle-platform"; import { ActionsEnvVars, ReadOnlyEnv } from "./environment"; import { Feature } from "./feature-flags"; import { BuiltInLanguage } from "./languages"; @@ -23,7 +24,7 @@ const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = { // Any version at least as new as the minimum will do. cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, compressionMethod: "zstd", - platform: "linux64", + platform: BundlePlatform.Linux64, variant: GitHubVariant.DOTCOM, }; @@ -44,14 +45,21 @@ async function checkEligibility( ); } -test("getPerLanguageBundleLanguage selects Linux bundles for non-Swift languages", async (t) => { - for (const language of Object.values(BuiltInLanguage)) { - if (language === BuiltInLanguage.swift) { - continue; +for (const platform of Object.values(BundlePlatform)) { + test(`getPerLanguageBundleLanguage selects only supported languages on ${platform}`, async (t) => { + for (const language of Object.values(BuiltInLanguage)) { + const supported = + language === BuiltInLanguage.swift + ? platform === BundlePlatform.Osx64 + : platform === BundlePlatform.Linux64; + t.is( + await checkEligibility({ rawLanguages: [language], platform }), + supported ? language : undefined, + language, + ); } - t.is(await checkEligibility({ rawLanguages: [language] }), language); - } -}); + }); +} test("getPerLanguageBundleLanguage normalizes aliases before selecting a bundle", async (t) => { t.is( @@ -60,23 +68,7 @@ test("getPerLanguageBundleLanguage normalizes aliases before selecting a bundle" ); }); -test("getPerLanguageBundleLanguage selects the macOS bundle for Swift", async (t) => { - t.is( - await checkEligibility({ rawLanguages: ["swift"], platform: "osx64" }), - BuiltInLanguage.swift, - ); - // Swift is only published for macOS. - t.is( - await checkEligibility({ rawLanguages: ["swift"], platform: "linux64" }), - undefined, - ); -}); - -test("getPerLanguageBundleLanguage rejects unsupported platforms", async (t) => { - t.is(await checkEligibility({ platform: "osx64" }), undefined); - t.is(await checkEligibility({ platform: "win64" }), undefined); - // We do not publish per-language bundles for Linux Arm64 either. - t.is(await checkEligibility({ platform: "linux-arm64" }), undefined); +test("getPerLanguageBundleLanguage rejects unknown platforms", async (t) => { t.is(await checkEligibility({ platform: undefined }), undefined); }); @@ -164,7 +156,7 @@ test("getPerLanguageBundleLanguage skips only the release version check for nigh { rawLanguages: undefined }, { rawLanguages: ["java", "python"] }, { compressionMethod: "gzip" as const }, - { platform: "osx64" }, + { platform: BundlePlatform.Osx64 }, { variant: GitHubVariant.GHES }, { variant: GitHubVariant.GHEC_DR }, ]) { diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts index 3ffa58d8fd..fee52d220b 100644 --- a/src/per-language-bundles.ts +++ b/src/per-language-bundles.ts @@ -2,6 +2,7 @@ import * as semver from "semver"; import { ActionState } from "./action-common"; import { isGitHubHostedRunner } from "./actions-util"; +import { BundlePlatform } from "./bundle-platform"; import { Feature } from "./feature-flags"; import { BuiltInLanguage, parseBuiltInLanguage } from "./languages"; import * as tar from "./tar"; @@ -30,20 +31,24 @@ export function tryGetBundleLanguageFromUrl( return match ? parseBuiltInLanguage(match[1]) : undefined; } -/** Published platform for each language; absent entries are ineligible. */ -const PER_LANGUAGE_BUNDLE_PLATFORMS: Readonly< - Partial> +/** Languages with per-language bundles published for each platform. */ +const PER_LANGUAGE_BUNDLE_LANGUAGES: Readonly< + Record> > = { - [BuiltInLanguage.actions]: "linux64", - [BuiltInLanguage.cpp]: "linux64", - [BuiltInLanguage.csharp]: "linux64", - [BuiltInLanguage.go]: "linux64", - [BuiltInLanguage.java]: "linux64", - [BuiltInLanguage.javascript]: "linux64", - [BuiltInLanguage.python]: "linux64", - [BuiltInLanguage.ruby]: "linux64", - [BuiltInLanguage.rust]: "linux64", - [BuiltInLanguage.swift]: "osx64", + [BundlePlatform.Linux64]: new Set([ + BuiltInLanguage.actions, + BuiltInLanguage.cpp, + BuiltInLanguage.csharp, + BuiltInLanguage.go, + BuiltInLanguage.java, + BuiltInLanguage.javascript, + BuiltInLanguage.python, + BuiltInLanguage.ruby, + BuiltInLanguage.rust, + ]), + [BundlePlatform.LinuxArm64]: new Set(), + [BundlePlatform.Osx64]: new Set([BuiltInLanguage.swift]), + [BundlePlatform.Win64]: new Set(), }; /** Inputs that determine whether we may download a per-language bundle. */ @@ -53,8 +58,8 @@ export interface PerLanguageBundleOptions { /** CLI version, if known. Ignored for nightly bundles. */ cliVersion: string | undefined; compressionMethod: tar.CompressionMethod; - /** Bundle platform identifier, such as linux64. */ - platform: string | undefined; + /** Platform for which the bundle is requested. */ + platform: BundlePlatform | undefined; variant: GitHubVariant; isNightly?: boolean; } @@ -130,14 +135,13 @@ export async function getPerLanguageBundleLanguage( } } - const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language]; - if (supportedPlatform === undefined) { - return explain(`no per-language bundle is published for ${language}`); - } - if (supportedPlatform !== platform) { + const supportedLanguages = + platform === undefined + ? undefined + : PER_LANGUAGE_BUNDLE_LANGUAGES[platform]; + if (!supportedLanguages?.has(language)) { return explain( - `the ${language} bundle is only published for ${supportedPlatform}, but this job is ` + - `running on ${platform ?? "an unknown platform"}`, + `no per-language bundle is published for ${language} on ${platform ?? "an unknown platform"}`, ); } diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 48c417f6da..a3c9f8ab98 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -17,6 +17,7 @@ import { isRunningLocalAction, } from "./actions-util"; import * as api from "./api-client"; +import { getBundlePlatform } from "./bundle-platform"; import * as defaults from "./defaults.json"; import { addNoLanguageDiagnostic, @@ -78,20 +79,6 @@ function getCodeQLBundleExtension( } } -/** Returns the platform component of the CodeQL bundle name for the current platform. */ -export function getBundlePlatform(): string | undefined { - switch (process.platform) { - case "win32": - return "win64"; - case "linux": - return process.arch === "arm64" ? "linux-arm64" : "linux64"; - case "darwin": - return "osx64"; - default: - return undefined; - } -} - /** * Returns the name of the CodeQL bundle asset to download. * diff --git a/src/testing-utils.ts b/src/testing-utils.ts index 7d33589ec6..f3456b7257 100644 --- a/src/testing-utils.ts +++ b/src/testing-utils.ts @@ -17,6 +17,7 @@ import { ActionsEnv, getActionVersion } from "./actions-util"; import { AnalysisKind } from "./analyses"; import * as apiClient from "./api-client"; import { GitHubApiDetails } from "./api-client"; +import { getBundlePlatform } from "./bundle-platform"; import { CachingKind } from "./caching-utils"; import { resetCachedCodeQlVersion } from "./cli/output-cache"; import type { VersionInfo } from "./cli/types"; @@ -933,21 +934,14 @@ export function mockBundleDownloadApi({ platformSpecific?: boolean; tagName: string; }): string { - const platform = - process.platform === "win32" - ? "win64" - : process.platform === "linux" - ? process.arch === "arm64" - ? "linux-arm64" - : "linux64" - : "osx64"; + const platform = platformSpecific ? getBundlePlatform() : undefined; const baseUrl = apiDetails?.url ?? "https://example.com"; const bundleUrls = ["tar.gz", "tar.zst"].map((extension) => { const relativeUrl = apiDetails ? `/${repo}/releases/download/${tagName}/codeql-bundle${ - platformSpecific ? `-${platform}` : "" + platform !== undefined ? `-${platform}` : "" }.${extension}` : `/download/${tagName}/codeql-bundle.${extension}`; From f536ef48b7b0ec4ba227ba93acd1b9974dcb604a Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 16 Sep 2026 17:30:50 +0100 Subject: [PATCH 07/20] Centralize CodeQL download telemetry fields Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 79 ++++++++++-------------- src/init-action.ts | 31 ++-------- src/setup-codeql-action.ts | 31 ++-------- src/setup-codeql.test.ts | 4 ++ src/setup-codeql.ts | 16 ++--- src/status-report.ts | 43 ++++++++++--- src/tools-download-status-report.test.ts | 76 +++++++++++++++++++++++ src/tools-download.ts | 4 +- 8 files changed, 165 insertions(+), 119 deletions(-) create mode 100644 src/tools-download-status-report.test.ts diff --git a/lib/entry-points.js b/lib/entry-points.js index 77e1764cd1..d18a5d66a0 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -147621,6 +147621,28 @@ async function sendStatusReport(statusReport) { ); } } +function createInitToolsDownloadFields(report, toolsFeatureFlagsValid) { + const fields = {}; + if (report?.downloadDurationMs !== void 0) { + fields.tools_download_duration_ms = report.downloadDurationMs; + } + if (report?.extractionDurationMs !== void 0) { + fields.tools_extraction_duration_ms = report.extractionDurationMs; + } + if (report?.totalDurationMs !== void 0) { + fields.tools_total_duration_ms = report.totalDurationMs; + } + if (report?.bundleLanguage !== void 0) { + fields.tools_bundle_language = report.bundleLanguage; + } + if (report?.perLanguageBundleFallback !== void 0) { + fields.tools_per_language_bundle_fallback = report.perLanguageBundleFallback; + } + if (toolsFeatureFlagsValid !== void 0) { + fields.tools_feature_flags_valid = toolsFeatureFlagsValid; + } + return fields; +} async function createInitWithConfigStatusReport(config, initStatusReport, configFile, totalCacheSize, overlayBaseDatabaseStats, dependencyCachingResults) { const languages = config.languages.join(","); const paths = (config.originalUserInput.paths || []).join(","); @@ -152532,7 +152554,7 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log } return { codeqlFolder: extractedBundlePath, - statusReport + statusReport: bundle.kind === "per-language" ? { ...statusReport, bundleLanguage: bundle.language } : statusReport }; }; function getToolcacheDestination(source, logger) { @@ -152645,20 +152667,13 @@ async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, temp await tryDeleteToolcacheBundles(action); const startTime = import_perf_hooks3.performance.now(); try { - const result = await downloadCodeQL( + return await downloadCodeQL( source, apiDetails, tarVersion, tempDir, logger ); - return bundle.kind === "combined" ? result : { - ...result, - statusReport: { - ...result.statusReport, - bundleLanguage: bundle.language - } - }; } catch (e) { if (bundle.kind !== "per-language" || bundle.combinedBundleURL === void 0 || asHTTPError(e)?.status !== 404) { throw e; @@ -162161,25 +162176,10 @@ async function sendCompletedStatusReport2(startedAt, config, configFile, toolsIn if (toolsInput !== void 0) { initStatusReport.computed_inputs.tools = toolsInput; } - const initToolsDownloadFields = {}; - if (toolsDownloadStatusReport?.downloadDurationMs !== void 0) { - initToolsDownloadFields.tools_download_duration_ms = toolsDownloadStatusReport.downloadDurationMs; - } - if (toolsDownloadStatusReport?.extractionDurationMs !== void 0) { - initToolsDownloadFields.tools_extraction_duration_ms = toolsDownloadStatusReport.extractionDurationMs; - } - if (toolsDownloadStatusReport?.totalDurationMs !== void 0) { - initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; - } - if (toolsDownloadStatusReport?.bundleLanguage !== void 0) { - initToolsDownloadFields.tools_bundle_language = toolsDownloadStatusReport.bundleLanguage; - } - if (toolsDownloadStatusReport?.perLanguageBundleFallback !== void 0) { - initToolsDownloadFields.tools_per_language_bundle_fallback = toolsDownloadStatusReport.perLanguageBundleFallback; - } - if (toolsFeatureFlagsValid !== void 0) { - initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } + const initToolsDownloadFields = createInitToolsDownloadFields( + toolsDownloadStatusReport, + toolsFeatureFlagsValid + ); if (config !== void 0) { const initWithConfigStatusReport = await createInitWithConfigStatusReport( config, @@ -163215,25 +163215,10 @@ async function sendCompletedStatusReport3(startedAt, toolsInput, toolsDownloadSt if (toolsInput !== void 0) { initStatusReport.computed_inputs.tools = toolsInput; } - const initToolsDownloadFields = {}; - if (toolsDownloadStatusReport?.downloadDurationMs !== void 0) { - initToolsDownloadFields.tools_download_duration_ms = toolsDownloadStatusReport.downloadDurationMs; - } - if (toolsDownloadStatusReport?.extractionDurationMs !== void 0) { - initToolsDownloadFields.tools_extraction_duration_ms = toolsDownloadStatusReport.extractionDurationMs; - } - if (toolsDownloadStatusReport?.totalDurationMs !== void 0) { - initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs; - } - if (toolsDownloadStatusReport?.bundleLanguage !== void 0) { - initToolsDownloadFields.tools_bundle_language = toolsDownloadStatusReport.bundleLanguage; - } - if (toolsDownloadStatusReport?.perLanguageBundleFallback !== void 0) { - initToolsDownloadFields.tools_per_language_bundle_fallback = toolsDownloadStatusReport.perLanguageBundleFallback; - } - if (toolsFeatureFlagsValid !== void 0) { - initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } + const initToolsDownloadFields = createInitToolsDownloadFields( + toolsDownloadStatusReport, + toolsFeatureFlagsValid + ); await sendStatusReport({ ...initStatusReport, ...initToolsDownloadFields }); } async function run6(actionState) { diff --git a/src/init-action.ts b/src/init-action.ts index dd576548dc..6cf50b8c2f 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -63,9 +63,9 @@ import { ToolsSource } from "./setup-codeql"; import { ActionName, InitStatusReport, - InitToolsDownloadFields, InitWithConfigStatusReport, createInitWithConfigStatusReport, + createInitToolsDownloadFields, createStatusReportBase, getActionsStatus, sendStatusReport, @@ -168,31 +168,10 @@ async function sendCompletedStatusReport( initStatusReport.computed_inputs.tools = toolsInput; } - const initToolsDownloadFields: InitToolsDownloadFields = {}; - - if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) { - initToolsDownloadFields.tools_download_duration_ms = - toolsDownloadStatusReport.downloadDurationMs; - } - if (toolsDownloadStatusReport?.extractionDurationMs !== undefined) { - initToolsDownloadFields.tools_extraction_duration_ms = - toolsDownloadStatusReport.extractionDurationMs; - } - if (toolsDownloadStatusReport?.totalDurationMs !== undefined) { - initToolsDownloadFields.tools_total_duration_ms = - toolsDownloadStatusReport.totalDurationMs; - } - if (toolsDownloadStatusReport?.bundleLanguage !== undefined) { - initToolsDownloadFields.tools_bundle_language = - toolsDownloadStatusReport.bundleLanguage; - } - if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) { - initToolsDownloadFields.tools_per_language_bundle_fallback = - toolsDownloadStatusReport.perLanguageBundleFallback; - } - if (toolsFeatureFlagsValid !== undefined) { - initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } + const initToolsDownloadFields = createInitToolsDownloadFields( + toolsDownloadStatusReport, + toolsFeatureFlagsValid, + ); if (config !== undefined) { // Append fields that are dependent on `config` diff --git a/src/setup-codeql-action.ts b/src/setup-codeql-action.ts index 3c2a191e7b..e78ee7ec9f 100644 --- a/src/setup-codeql-action.ts +++ b/src/setup-codeql-action.ts @@ -22,7 +22,7 @@ import { ToolsSource } from "./setup-codeql"; import { ActionName, InitStatusReport, - InitToolsDownloadFields, + createInitToolsDownloadFields, createStatusReportBase, getActionsStatus, sendStatusReport, @@ -79,31 +79,10 @@ async function sendCompletedStatusReport( initStatusReport.computed_inputs.tools = toolsInput; } - const initToolsDownloadFields: InitToolsDownloadFields = {}; - - if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) { - initToolsDownloadFields.tools_download_duration_ms = - toolsDownloadStatusReport.downloadDurationMs; - } - if (toolsDownloadStatusReport?.extractionDurationMs !== undefined) { - initToolsDownloadFields.tools_extraction_duration_ms = - toolsDownloadStatusReport.extractionDurationMs; - } - if (toolsDownloadStatusReport?.totalDurationMs !== undefined) { - initToolsDownloadFields.tools_total_duration_ms = - toolsDownloadStatusReport.totalDurationMs; - } - if (toolsDownloadStatusReport?.bundleLanguage !== undefined) { - initToolsDownloadFields.tools_bundle_language = - toolsDownloadStatusReport.bundleLanguage; - } - if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) { - initToolsDownloadFields.tools_per_language_bundle_fallback = - toolsDownloadStatusReport.perLanguageBundleFallback; - } - if (toolsFeatureFlagsValid !== undefined) { - initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } + const initToolsDownloadFields = createInitToolsDownloadFields( + toolsDownloadStatusReport, + toolsFeatureFlagsValid, + ); await sendStatusReport({ ...initStatusReport, ...initToolsDownloadFields }); } diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index 5e69ddaaa7..9a87a50276 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -1275,6 +1275,10 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) { ); t.is(result.toolsDownloadStatusReport?.downloadDurationMs, 200); t.is(result.toolsDownloadStatusReport?.extractionDurationMs, 100); + t.is( + (await downloadSpy.lastCall.returnValue).statusReport.bundleLanguage, + bundle === "per-language" ? BuiltInLanguage.javascript : undefined, + ); t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); t.is(downloadSpy.callCount, extractStub.callCount); t.is( diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index a3c9f8ab98..744f78ed49 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -920,7 +920,10 @@ export const downloadCodeQL = async function ( return { codeqlFolder: extractedBundlePath, - statusReport, + statusReport: + bundle.kind === "per-language" + ? { ...statusReport, bundleLanguage: bundle.language } + : statusReport, }; }; @@ -1141,22 +1144,13 @@ export async function downloadCodeQLBundle( const startTime = performance.now(); try { - const result = await downloadCodeQL( + return await downloadCodeQL( source, apiDetails, tarVersion, tempDir, logger, ); - return bundle.kind === "combined" - ? result - : { - ...result, - statusReport: { - ...result.statusReport, - bundleLanguage: bundle.language, - }, - }; } catch (e) { if ( bundle.kind !== "per-language" || diff --git a/src/status-report.ts b/src/status-report.ts index 820b1c2109..f6e6e16ca5 100644 --- a/src/status-report.ts +++ b/src/status-report.ts @@ -31,6 +31,7 @@ import type { OverlayBaseDatabaseDownloadStats } from "./overlay/caching"; import { getRepositoryNwo } from "./repository"; import type { ToolsSource } from "./setup-codeql"; import { registryBaseSchema } from "./start-proxy/types"; +import type { ToolsDownloadStatusReport } from "./tools-download"; import { ConfigurationError, getRequiredEnvParam, @@ -630,28 +631,56 @@ export interface InitToolsDownloadFields { * Time taken to download the bundle, in milliseconds. Not populated when the bundle is downloaded * and extracted concurrently. */ - tools_download_duration_ms?: number; + tools_download_duration_ms?: ToolsDownloadStatusReport["downloadDurationMs"]; /** * Time taken to extract the bundle, in milliseconds. Not populated when the bundle is downloaded * and extracted concurrently. */ - tools_extraction_duration_ms?: number; + tools_extraction_duration_ms?: ToolsDownloadStatusReport["extractionDurationMs"]; /** - * Total time taken to make the bundle available on disk, in milliseconds. This includes any time - * spent on a streaming attempt that failed and fell back to downloading before extracting. + * Total time taken to make the bundle available on disk, including failed download attempts + * before a fallback, in milliseconds. */ - tools_total_duration_ms?: number; + tools_total_duration_ms?: ToolsDownloadStatusReport["totalDurationMs"]; /** * Whether the relevant tools dotcom feature flags have been misconfigured. * Only populated if we attempt to determine the default version based on the dotcom feature flags. */ tools_feature_flags_valid?: boolean; /** The language of the single-language bundle that was downloaded, if any. */ - tools_bundle_language?: string; + tools_bundle_language?: ToolsDownloadStatusReport["bundleLanguage"]; /** * Whether we tried to download a single-language bundle, but it did not exist and we fell back to * the combined bundle. */ - tools_per_language_bundle_fallback?: boolean; + tools_per_language_bundle_fallback?: ToolsDownloadStatusReport["perLanguageBundleFallback"]; +} + +/** Converts download results to telemetry fields shared by the init and setup-codeql Actions. */ +export function createInitToolsDownloadFields( + report: ToolsDownloadStatusReport | undefined, + toolsFeatureFlagsValid: boolean | undefined, +): InitToolsDownloadFields { + const fields: InitToolsDownloadFields = {}; + if (report?.downloadDurationMs !== undefined) { + fields.tools_download_duration_ms = report.downloadDurationMs; + } + if (report?.extractionDurationMs !== undefined) { + fields.tools_extraction_duration_ms = report.extractionDurationMs; + } + if (report?.totalDurationMs !== undefined) { + fields.tools_total_duration_ms = report.totalDurationMs; + } + if (report?.bundleLanguage !== undefined) { + fields.tools_bundle_language = report.bundleLanguage; + } + if (report?.perLanguageBundleFallback !== undefined) { + fields.tools_per_language_bundle_fallback = + report.perLanguageBundleFallback; + } + if (toolsFeatureFlagsValid !== undefined) { + fields.tools_feature_flags_valid = toolsFeatureFlagsValid; + } + return fields; } /** diff --git a/src/tools-download-status-report.test.ts b/src/tools-download-status-report.test.ts new file mode 100644 index 0000000000..36cd318b07 --- /dev/null +++ b/src/tools-download-status-report.test.ts @@ -0,0 +1,76 @@ +import test from "ava"; + +import { BuiltInLanguage } from "./languages"; +import { createInitToolsDownloadFields } from "./status-report"; + +test("createInitToolsDownloadFields omits absent download data", (t) => { + t.deepEqual(createInitToolsDownloadFields(undefined, undefined), {}); +}); + +test("createInitToolsDownloadFields reports feature flags without a download", (t) => { + t.deepEqual(createInitToolsDownloadFields(undefined, false), { + tools_feature_flags_valid: false, + }); +}); + +test("createInitToolsDownloadFields reports only the total for a streaming download", (t) => { + t.deepEqual( + createInitToolsDownloadFields({ totalDurationMs: 300 }, undefined), + { tools_total_duration_ms: 300 }, + ); +}); + +test("createInitToolsDownloadFields preserves per-language metadata", (t) => { + t.deepEqual( + createInitToolsDownloadFields( + { totalDurationMs: 300, bundleLanguage: BuiltInLanguage.java }, + true, + ), + { + tools_total_duration_ms: 300, + tools_bundle_language: BuiltInLanguage.java, + tools_feature_flags_valid: true, + }, + ); +}); + +test("createInitToolsDownloadFields preserves fallback and per-attempt timings", (t) => { + t.deepEqual( + createInitToolsDownloadFields( + { + downloadDurationMs: 200, + extractionDurationMs: 100, + totalDurationMs: 1000, + perLanguageBundleFallback: true, + }, + undefined, + ), + { + tools_download_duration_ms: 200, + tools_extraction_duration_ms: 100, + tools_total_duration_ms: 1000, + tools_per_language_bundle_fallback: true, + }, + ); +}); + +test("createInitToolsDownloadFields preserves zero durations and false flags", (t) => { + t.deepEqual( + createInitToolsDownloadFields( + { + downloadDurationMs: 0, + extractionDurationMs: 0, + totalDurationMs: 0, + perLanguageBundleFallback: false, + }, + false, + ), + { + tools_download_duration_ms: 0, + tools_extraction_duration_ms: 0, + tools_total_duration_ms: 0, + tools_per_language_bundle_fallback: false, + tools_feature_flags_valid: false, + }, + ); +}); diff --git a/src/tools-download.ts b/src/tools-download.ts index f7b0a708ce..5494ab30f1 100644 --- a/src/tools-download.ts +++ b/src/tools-download.ts @@ -50,8 +50,8 @@ export type ToolsDownloadStatusReport = { */ extractionDurationMs?: number; /** - * Total time taken to make the bundle available on disk, in milliseconds. This includes any time - * spent on a streaming attempt that failed and fell back to downloading before extracting. + * Total time taken to make the bundle available on disk, including failed download attempts + * before a fallback, in milliseconds. */ totalDurationMs: number; /** The language of the single-language bundle that was downloaded, if any. */ From 2f552a99f36b59f3c6891d0435f3afa00df5a352 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 16 Sep 2026 17:36:46 +0100 Subject: [PATCH 08/20] Clarify bundle resolution and latest-nightly selection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/__bundle-toolcache.yml | 1 + lib/entry-points.js | 33 +++-- pr-checks/checks/bundle-toolcache.yml | 1 + src/per-language-bundles.test.ts | 4 +- src/per-language-bundles.ts | 22 ++-- src/setup-codeql.test.ts | 161 +++++++++++++++++++---- src/setup-codeql.ts | 46 ++++--- 7 files changed, 203 insertions(+), 65 deletions(-) diff --git a/.github/workflows/__bundle-toolcache.yml b/.github/workflows/__bundle-toolcache.yml index d12aeb6e78..0055f94705 100644 --- a/.github/workflows/__bundle-toolcache.yml +++ b/.github/workflows/__bundle-toolcache.yml @@ -80,6 +80,7 @@ jobs: - id: init uses: ./../action/init with: + # Request multiple languages so this check uses the combined bundle. languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - uses: ./../action/analyze diff --git a/lib/entry-points.js b/lib/entry-points.js index d18a5d66a0..0dc581dc56 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -151576,7 +151576,7 @@ async function getPerLanguageBundleLanguage({ compressionMethod, platform: platform2, variant, - isNightly + isLatestNightly } = options; const explain = (reason) => { logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); @@ -151595,7 +151595,7 @@ async function getPerLanguageBundleLanguage({ return explain(`'${rawLanguages[0]}' is not a known CodeQL language`); } if (compressionMethod !== "zstd") { - return explain(`the bundle would be downloaded as ${compressionMethod}`); + return explain(`the bundle would be downloaded as '${compressionMethod}'`); } if (variant !== "GitHub.com" /* DOTCOM */) { return explain(`we are running against ${variant}`); @@ -151603,13 +151603,13 @@ async function getPerLanguageBundleLanguage({ if (!isGitHubHostedRunner(env)) { return explain("the job is not running on a GitHub-hosted runner"); } - if (!isNightly) { + if (!isLatestNightly) { if (cliVersion2 === void 0) { - return explain("the CLI version of the bundle is unknown"); + return explain("the requested CLI version is unknown"); } if (!semver7.gte(cliVersion2, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { return explain( - `CodeQL ${cliVersion2} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the first version that publishes per-language bundles` + `the requested CodeQL version ${cliVersion2} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the first version for which per-language bundles are published` ); } } @@ -152283,7 +152283,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.` ); } - bundle = await getNightlyBundle( + bundle = await getLatestNightlyBundle( { env: getEnv(), features, logger }, rawLanguages, variant @@ -152441,6 +152441,12 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO } let compressionMethod; if (!url2) { + const bundleTagName = tagName; + if (bundleTagName === void 0) { + throw new Error( + "Could not determine a release tag for the requested CodeQL bundle." + ); + } compressionMethod = cliVersion2 !== void 0 && await useZstdBundle(cliVersion2, tarSupportsZstd) ? "zstd" : "gzip"; const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( { env: getEnv(), features, logger }, @@ -152453,24 +152459,25 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO } ); const resolveBundleURL = (language) => getCodeQLBundleDownloadURL( - tagName, + bundleTagName, apiDetails, getCodeQLBundleName(compressionMethod, language), logger ); + const combinedBundleURL = await resolveBundleURL(); if (perLanguageBundleLanguage !== void 0) { logger.info( - `Downloading the ${perLanguageBundleLanguage} CodeQL bundle, since ${perLanguageBundleLanguage} is the only language being analyzed.` + `Selected the per-language CodeQL bundle for '${perLanguageBundleLanguage}'.` ); url2 = await resolveBundleURL(perLanguageBundleLanguage); bundle = { kind: "per-language", url: url2, language: perLanguageBundleLanguage, - combinedBundleURL: await resolveBundleURL() + combinedBundleURL }; } else { - url2 = await resolveBundleURL(); + url2 = combinedBundleURL; bundle = { kind: "combined", url: url2 }; } } else { @@ -152679,7 +152686,7 @@ async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, temp throw e; } logger.warning( - `No ${bundle.language} CodeQL bundle was found at ${bundle.url}, so falling back to the bundle that contains all languages. This analysis will still produce correct results, but will take longer to set up.` + `No per-language CodeQL bundle for '${bundle.language}' was found at ${bundle.url}, so falling back to the bundle that contains all languages. This analysis will still produce correct results, but will take longer to set up.` ); const result = await downloadCodeQL( { @@ -152710,7 +152717,7 @@ async function useZstdBundle(cliVersion2, tarSupportsZstd) { function getTempExtractionDir(tempDir) { return path13.join(tempDir, v4_default()); } -async function getNightlyBundle(action, rawLanguages, variant) { +async function getLatestNightlyBundle(action, rawLanguages, variant) { const { logger } = action; const zstdAvailability = await isZstdAvailable(logger); const compressionMethod = await useZstdBundle( @@ -152723,7 +152730,7 @@ async function getNightlyBundle(action, rawLanguages, variant) { compressionMethod, platform: getBundlePlatform(), variant, - isNightly: true + isLatestNightly: true }); try { const release2 = await getApiClient().rest.repos.listReleases({ diff --git a/pr-checks/checks/bundle-toolcache.yml b/pr-checks/checks/bundle-toolcache.yml index efa1a4d76f..f74c6af75c 100644 --- a/pr-checks/checks/bundle-toolcache.yml +++ b/pr-checks/checks/bundle-toolcache.yml @@ -30,6 +30,7 @@ steps: - id: init uses: ./../action/init with: + # Request multiple languages so this check uses the combined bundle. languages: javascript,python tools: ${{ steps.prepare-test.outputs.tools-url }} - uses: ./../action/analyze diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts index d192d43c6d..ea8315001a 100644 --- a/src/per-language-bundles.test.ts +++ b/src/per-language-bundles.test.ts @@ -148,8 +148,8 @@ test("getPerLanguageBundleLanguage explains a disabled feature before checking e ); }); -test("getPerLanguageBundleLanguage skips only the release version check for nightlies", async (t) => { - const nightly = { isNightly: true, cliVersion: undefined }; +test("getPerLanguageBundleLanguage skips only the release version check for the latest nightly", async (t) => { + const nightly = { isLatestNightly: true, cliVersion: undefined }; t.is(await checkEligibility(nightly), BuiltInLanguage.java); for (const overrides of [ diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts index fee52d220b..20720636fc 100644 --- a/src/per-language-bundles.ts +++ b/src/per-language-bundles.ts @@ -14,7 +14,7 @@ export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; const PER_LANGUAGE_BUNDLE_NAME = /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; -/** Identifies per-language tools URLs that must not populate the toolcache. */ +/** Returns the language in a per-language tools URL, or undefined for other URLs. */ export function tryGetBundleLanguageFromUrl( url: string, ): BuiltInLanguage | undefined { @@ -55,13 +55,14 @@ const PER_LANGUAGE_BUNDLE_LANGUAGES: Readonly< export interface PerLanguageBundleOptions { /** Explicit input only: autodetection needs a CLI instance. */ rawLanguages: string[] | undefined; - /** CLI version, if known. Ignored for nightly bundles. */ + /** Requested CLI version, if known. Ignored when requesting the latest nightly. */ cliVersion: string | undefined; compressionMethod: tar.CompressionMethod; /** Platform for which the bundle is requested. */ platform: BundlePlatform | undefined; variant: GitHubVariant; - isNightly?: boolean; + /** Whether the Action is selecting the latest nightly rather than a release version. */ + isLatestNightly?: boolean; } /** Returns the eligible bundle language, or undefined for the combined bundle. */ @@ -79,7 +80,7 @@ export async function getPerLanguageBundleLanguage( compressionMethod, platform, variant, - isNightly, + isLatestNightly, } = options; const explain = (reason: string) => { @@ -106,7 +107,7 @@ export async function getPerLanguageBundleLanguage( if (compressionMethod !== "zstd") { // Per-language bundles are only published as zstd archives. - return explain(`the bundle would be downloaded as ${compressionMethod}`); + return explain(`the bundle would be downloaded as '${compressionMethod}'`); } if (variant !== GitHubVariant.DOTCOM) { @@ -121,16 +122,17 @@ export async function getPerLanguageBundleLanguage( return explain("the job is not running on a GitHub-hosted runner"); } - // Nightly tags contain dates rather than comparable CLI versions. - if (!isNightly) { + // Check whether per-language bundles are published for the requested CLI version. + // Skip this for the latest nightly, whose tag contains a date rather than a CLI version. + if (!isLatestNightly) { if (cliVersion === undefined) { - return explain("the CLI version of the bundle is unknown"); + return explain("the requested CLI version is unknown"); } if (!semver.gte(cliVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { return explain( - `CodeQL ${cliVersion} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the ` + - "first version that publishes per-language bundles", + `the requested CodeQL version ${cliVersion} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the ` + + "first version for which per-language bundles are published", ); } } diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index 9a87a50276..3f440894d7 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -58,6 +58,7 @@ function stubDownloadAndExtract() { }); } +/** Models a hosted Linux runner with zstd and the latest nightly release. */ function stubHostedNightly(tagName: string) { sinon.stub(process, "platform").value("linux"); sinon.stub(process, "arch").value("x64"); @@ -66,15 +67,25 @@ function stubHostedNightly(tagName: string) { available: true, foundZstdBinary: true, }); - const client = github.getOctokit("123", { - request: { - fetch: async () => + const fetchRelease = sinon + .stub, ReturnType>() + .rejects(new Error("Unexpected API request in nightly bundle test")); + fetchRelease + .withArgs( + "https://api.github.com/repos/dsp-testing/codeql-cli-nightlies/releases?per_page=1&page=1&prerelease=true", + sinon.match({ method: "GET" }), + ) + .callsFake( + async () => new Response(JSON.stringify([{ tag_name: tagName }]), { headers: { "content-type": "application/json" }, }), - }, + ); + const client = github.getOctokit("123", { + request: { fetch: fetchRelease }, }); sinon.stub(api, "getApiClient").value(() => client); + return fetchRelease; } test.serial("parse codeql bundle url version", (t) => { @@ -600,11 +611,11 @@ test.serial( for (const toolsInput of ["nightly", "nightly-latest"]) { test.serial( - `getCodeQLSource selects a per-language bundle for tools == ${toolsInput}`, + `getCodeQLSource selects the latest per-language nightly for tools == ${toolsInput}`, async (t) => { const expectedTag = "codeql-bundle-30260213"; const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; - stubHostedNightly(expectedTag); + const latestNightlyRequest = stubHostedNightly(expectedTag); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); @@ -633,13 +644,14 @@ for (const toolsInput of ["nightly", "nightly-latest"]) { compressionMethod: "zstd", toolsVersion: "0.0.0-30260213", } satisfies setupCodeql.CodeQLDownloadSource); + t.true(latestNightlyRequest.calledOnce); }); }, ); } test.serial( - "getCodeQLSource downloads the combined nightly bundle when not eligible", + "getCodeQLSource downloads a combined nightly bundle when per-language selection is ineligible", async (t) => { const expectedTag = "codeql-bundle-30260213"; stubHostedNightly(expectedTag); @@ -647,7 +659,9 @@ test.serial( await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir); for (const { languages, features } of [ + // The per-language feature is disabled. { languages: ["java"], features: createFeatures([]) }, + // More than one language requires a combined bundle. { languages: ["java", "python"], features: createFeatures([Feature.PerLanguageBundles]), @@ -679,11 +693,11 @@ test.serial( for (const perLanguageBundles of [false, true]) { test.serial( - `getCodeQLSource uses a ${perLanguageBundles ? "per-language" : "combined"} bundle for a forced nightly`, + `getCodeQLSource uses the latest ${perLanguageBundles ? "per-language" : "combined"} bundle for a forced nightly`, async (t) => { const expectedTag = "codeql-bundle-30260213"; const baseURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}`; - stubHostedNightly(expectedTag); + const latestNightlyRequest = stubHostedNightly(expectedTag); await withTmpDir(async (tmpDir) => { setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" }); @@ -704,6 +718,7 @@ for (const perLanguageBundles of [false, true]) { ); t.is(source.sourceType, "download"); + t.true(latestNightlyRequest.calledOnce); if (source.sourceType === "download") { const combinedURL = `${baseURL}/codeql-bundle-linux64.tar.zst`; t.deepEqual( @@ -723,6 +738,105 @@ for (const perLanguageBundles of [false, true]) { ); } +for (const date of ["20200101", "30260213"]) { + for (const bundle of ["combined", "per-language"] as const) { + test.serial( + `getCodeQLSource preserves an explicit ${bundle} nightly URL for ${date}`, + async (t) => { + const latestNightlyRequest = stubHostedNightly( + "codeql-bundle-30260213", + ); + const asset = + bundle === "combined" + ? "codeql-bundle-linux64.tar.zst" + : "codeql-bundle-java-linux64.tar.zst"; + const url = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/codeql-bundle-${date}/${asset}`; + const features = createFeatures([Feature.PerLanguageBundles]); + const logger = getRecordingLogger([], { logToConsole: false }); + const error = new HTTPError("Not Found", 404); + const extractStub = sinon + .stub(toolsDownload, "downloadAndExtract") + .rejects(error); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const source = await setupCodeql.getCodeQLSource( + url, + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, + features, + logger, + ); + t.deepEqual(source, { + sourceType: "download", + bundle: + bundle === "combined" + ? { kind: "combined", url } + : { kind: "per-language", url, language: BuiltInLanguage.java }, + bundleVersion: date, + cliVersion: undefined, + compressionMethod: "zstd", + toolsVersion: `0.0.0-${date}`, + } satisfies setupCodeql.CodeQLDownloadSource); + + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + url, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, + features, + logger, + ), + { is: error }, + ); + t.true(extractStub.calledOnce); + t.is(extractStub.firstCall.args[0], url); + t.true(latestNightlyRequest.notCalled); + }); + }, + ); + } +} + +test.serial( + "getCodeQLSource reports a missing release tag when a toolcache entry disappears", + async (t) => { + sinon + .stub(toolcache, "findAllVersions") + .returns([MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION]); + sinon.stub(toolcache, "find").returns(""); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" }); + await t.throwsAsync( + setupCodeql.getCodeQLSource( + "toolcache", + SAMPLE_DEFAULT_CLI_VERSION, + ["java"], + false, + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, + createFeatures([]), + getRunnerLogger(true), + ), + { + message: + "Could not determine a release tag for the requested CodeQL bundle.", + }, + ); + }); + }, +); + test.serial( "getCodeQLSource correctly returns latest version from toolcache when tools == toolcache", async (t) => { @@ -1016,18 +1130,21 @@ const PER_LANGUAGE_CLI_VERSION = { ], }; -test.serial("getCodeQLBundleName names the per-language bundle", (t) => { - sinon.stub(process, "platform").value("linux"); - sinon.stub(process, "arch").value("x64"); - t.is( - setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.java), - "codeql-bundle-java-linux64.tar.zst", - ); - t.is( - setupCodeql.getCodeQLBundleName("zstd"), - "codeql-bundle-linux64.tar.zst", - ); -}); +test.serial( + "getCodeQLBundleName returns a per-language bundle name only when a language is specified", + (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + t.is( + setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.java), + "codeql-bundle-java-linux64.tar.zst", + ); + t.is( + setupCodeql.getCodeQLBundleName("zstd"), + "codeql-bundle-linux64.tar.zst", + ); + }, +); test.serial("getCodeQLBundleName names the Swift bundle for macOS", (t) => { sinon.stub(process, "platform").value("darwin"); @@ -1313,7 +1430,7 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) { bundle: { kind: "combined", url: combinedURL }, }); checkExpectedLogMessages(t, loggedMessages, [ - `No javascript CodeQL bundle was found at ${perLanguageURL}`, + `No per-language CodeQL bundle for 'javascript' was found at ${perLanguageURL}`, ]); } if (bundle === "per-language") { diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 744f78ed49..36d10cd42e 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -83,8 +83,7 @@ function getCodeQLBundleExtension( * Returns the name of the CodeQL bundle asset to download. * * @param compressionMethod The compression method of the bundle. - * @param language If provided, the name of the bundle that contains only this language, rather than - * the name of the combined bundle that contains every language. + * @param language Optional language for a per-language bundle. If omitted, returns a combined bundle name. */ export function getCodeQLBundleName( compressionMethod: tar.CompressionMethod, @@ -247,7 +246,7 @@ export interface CodeQLDownloadSource { compressionMethod: tar.CompressionMethod; /** Bundle version of the tools, if known. */ bundleVersion?: string; - /** CLI version of the tools, if known. */ + /** Requested CLI version, if known. */ cliVersion?: string; /** Resolved version for telemetry, independent of whether the bundle can be cached. */ toolsVersion: string; @@ -476,7 +475,7 @@ export async function getCodeQLSource( }; } - /** CLI version number, for example 2.12.6. */ + /** Requested CLI version number, for example 2.12.6. */ let cliVersion: string | undefined; /** Tag name of the CodeQL bundle, for example `codeql-bundle-20230120`. */ let tagName: string | undefined; @@ -530,7 +529,7 @@ export async function getCodeQLSource( `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`, ); } - bundle = await getNightlyBundle( + bundle = await getLatestNightlyBundle( { env: getEnv(), features, logger }, rawLanguages, variant, @@ -758,6 +757,13 @@ export async function getCodeQLSource( let compressionMethod: tar.CompressionMethod; if (!url) { + const bundleTagName = tagName; + if (bundleTagName === undefined) { + throw new Error( + "Could not determine a release tag for the requested CodeQL bundle.", + ); + } + compressionMethod = cliVersion !== undefined && (await useZstdBundle(cliVersion, tarSupportsZstd)) @@ -775,28 +781,29 @@ export async function getCodeQLSource( }, ); + // Resolve both bundle variants against the same release and repository lookup order. const resolveBundleURL = (language?: BuiltInLanguage) => getCodeQLBundleDownloadURL( - tagName!, + bundleTagName, apiDetails, getCodeQLBundleName(compressionMethod, language), logger, ); + const combinedBundleURL = await resolveBundleURL(); if (perLanguageBundleLanguage !== undefined) { logger.info( - `Downloading the ${perLanguageBundleLanguage} CodeQL bundle, since ${perLanguageBundleLanguage} ` + - "is the only language being analyzed.", + `Selected the per-language CodeQL bundle for '${perLanguageBundleLanguage}'.`, ); url = await resolveBundleURL(perLanguageBundleLanguage); bundle = { kind: "per-language", url, language: perLanguageBundleLanguage, - combinedBundleURL: await resolveBundleURL(), + combinedBundleURL, }; } else { - url = await resolveBundleURL(); + url = combinedBundleURL; bundle = { kind: "combined", url }; } } else { @@ -928,8 +935,8 @@ export const downloadCodeQL = async function ( }; /** - * Returns the canonical toolcache directory for a resolved download, or `undefined` if its bundle - * version is unknown. + * Returns the canonical toolcache directory for a combined bundle with a known version. + * Returns undefined for per-language bundles or unknown versions. */ function getToolcacheDestination( source: CodeQLDownloadSource, @@ -1122,8 +1129,8 @@ export async function setupCodeQLBundle( /** * Performs eligible toolcache cleanup once, then downloads and extracts the resolved bundle. * - * If `source` refers to a bundle for a single language and that bundle turns out not to exist, this - * falls back to downloading the combined bundle. + * If an automatically selected per-language bundle is missing, downloads the combined bundle + * from the same release instead. Explicit bundle URLs are not substituted. * * @returns The extraction directory and download timings. */ @@ -1160,7 +1167,7 @@ export async function downloadCodeQLBundle( throw e; } logger.warning( - `No ${bundle.language} CodeQL bundle was found at ${bundle.url}, so ` + + `No per-language CodeQL bundle for '${bundle.language}' was found at ${bundle.url}, so ` + "falling back to the bundle that contains all languages. This analysis will still " + "produce correct results, but will take longer to set up.", ); @@ -1202,8 +1209,11 @@ function getTempExtractionDir(tempDir: string) { return path.join(tempDir, uuidV4()); } -/** Selects a bundle from the latest nightly, with a same-release fallback when applicable. */ -async function getNightlyBundle( +/** + * Selects a bundle from the latest nightly release, preferring a per-language bundle when eligible. + * Records the combined bundle URL from that release for use if the selected asset is missing. + */ +async function getLatestNightlyBundle( action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, rawLanguages: string[] | undefined, variant: util.GitHubVariant, @@ -1224,7 +1234,7 @@ async function getNightlyBundle( compressionMethod, platform: getBundlePlatform(), variant, - isNightly: true, + isLatestNightly: true, }); try { From bd2ddba96c5dd69626880a515d338955f4a855c6 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 16 Sep 2026 19:36:07 +0100 Subject: [PATCH 09/20] Extract explicit CodeQL bundle URL classification Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 56 ++++++++++++++++---------------- src/codeql-bundle.test.ts | 56 ++++++++++++++++++++++++++++++++ src/codeql-bundle.ts | 33 +++++++++++++++++++ src/per-language-bundles.test.ts | 49 ---------------------------- src/per-language-bundles.ts | 20 ------------ src/setup-codeql.ts | 26 ++------------- 6 files changed, 120 insertions(+), 120 deletions(-) create mode 100644 src/codeql-bundle.test.ts create mode 100644 src/codeql-bundle.ts diff --git a/lib/entry-points.js b/lib/entry-points.js index 0dc581dc56..0a0941fe96 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -149917,18 +149917,18 @@ var builtin_default = { }; // src/languages/index.ts -var BuiltInLanguage = /* @__PURE__ */ ((BuiltInLanguage3) => { - BuiltInLanguage3["actions"] = "actions"; - BuiltInLanguage3["cpp"] = "cpp"; - BuiltInLanguage3["csharp"] = "csharp"; - BuiltInLanguage3["go"] = "go"; - BuiltInLanguage3["java"] = "java"; - BuiltInLanguage3["javascript"] = "javascript"; - BuiltInLanguage3["python"] = "python"; - BuiltInLanguage3["ruby"] = "ruby"; - BuiltInLanguage3["rust"] = "rust"; - BuiltInLanguage3["swift"] = "swift"; - return BuiltInLanguage3; +var BuiltInLanguage = /* @__PURE__ */ ((BuiltInLanguage4) => { + BuiltInLanguage4["actions"] = "actions"; + BuiltInLanguage4["cpp"] = "cpp"; + BuiltInLanguage4["csharp"] = "csharp"; + BuiltInLanguage4["go"] = "go"; + BuiltInLanguage4["java"] = "java"; + BuiltInLanguage4["javascript"] = "javascript"; + BuiltInLanguage4["python"] = "python"; + BuiltInLanguage4["ruby"] = "ruby"; + BuiltInLanguage4["rust"] = "rust"; + BuiltInLanguage4["swift"] = "swift"; + return BuiltInLanguage4; })(BuiltInLanguage || {}); var builtInLanguageSet = new Set(builtin_default.languages); function isBuiltInLanguage(language) { @@ -151236,6 +151236,21 @@ function getBundlePlatform(platform2 = process.platform, arch2 = process.arch) { } } +// src/codeql-bundle.ts +var PER_LANGUAGE_BUNDLE_NAME = /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; +function getCodeQLBundleFromUrl(url2) { + let assetName; + try { + const pathname = new URL(url2).pathname; + assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); + } catch { + return { kind: "combined", url: url2 }; + } + const match2 = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + const language = match2 ? parseBuiltInLanguage(match2[1]) : void 0; + return language === void 0 ? { kind: "combined", url: url2 } : { kind: "per-language", url: url2, language }; +} + // src/overlay/caching.ts var fs11 = __toESM(require("fs")); var actionsCache3 = __toESM(require_cache4()); @@ -151537,18 +151552,6 @@ async function getCodeQlVersionsForOverlayBaseDatabases(rawLanguages, logger) { // src/per-language-bundles.ts var semver7 = __toESM(require_semver2()); var MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; -var PER_LANGUAGE_BUNDLE_NAME = /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; -function tryGetBundleLanguageFromUrl(url2) { - let assetName; - try { - const pathname = new URL(url2).pathname; - assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); - } catch { - return void 0; - } - const match2 = assetName.match(PER_LANGUAGE_BUNDLE_NAME); - return match2 ? parseBuiltInLanguage(match2[1]) : void 0; -} var PER_LANGUAGE_BUNDLE_LANGUAGES = { ["linux64" /* Linux64 */]: /* @__PURE__ */ new Set([ "actions" /* actions */, @@ -152488,10 +152491,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO ); } compressionMethod = method; - if (bundle === void 0) { - const language = tryGetBundleLanguageFromUrl(url2); - bundle = language === void 0 ? { kind: "combined", url: url2 } : { kind: "per-language", url: url2, language }; - } + bundle ??= getCodeQLBundleFromUrl(url2); if (bundle.kind === "per-language") { logger.info( `${url2} appears to be a CodeQL bundle that contains only ${bundle.language}.` diff --git a/src/codeql-bundle.test.ts b/src/codeql-bundle.test.ts new file mode 100644 index 0000000000..1014043eab --- /dev/null +++ b/src/codeql-bundle.test.ts @@ -0,0 +1,56 @@ +import test from "ava"; + +import { getCodeQLBundleFromUrl } from "./codeql-bundle"; +import { BuiltInLanguage } from "./languages"; + +for (const [assetName, language] of [ + ["codeql-bundle-java-linux64.tar.zst", BuiltInLanguage.java], + ["codeql-bundle-swift-osx64.tar.zst", BuiltInLanguage.swift], + // Recognize unpublished language/platform combinations to keep them out of the toolcache. + ["codeql-bundle-csharp-win64.tar.gz", BuiltInLanguage.csharp], + ["codeql-bundle-java-kotlin-linux64.tar.zst", BuiltInLanguage.java], + ["codeql-bundle-%70ython-linux64.tar.zst", BuiltInLanguage.python], +] as const) { + test(`getCodeQLBundleFromUrl identifies ${assetName} without adding a fallback`, (t) => { + const url = `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${assetName}`; + t.deepEqual(getCodeQLBundleFromUrl(url), { + kind: "per-language", + url, + language, + }); + }); +} + +test("getCodeQLBundleFromUrl preserves encoding, query parameters and fragments", (t) => { + const url = + "https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/codeql-bundle-%70ython-linux64.tar.zst?download=1#asset"; + t.deepEqual(getCodeQLBundleFromUrl(url), { + kind: "per-language", + url, + language: BuiltInLanguage.python, + }); +}); + +test("getCodeQLBundleFromUrl treats unrecognized assets as combined bundles", (t) => { + for (const name of [ + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-osx64.tar.gz", + "codeql-bundle-win64.tar.zst", + // The all-platform bundle. + "codeql-bundle.tar.gz", + // A platform we do not publish per-language bundles for, whose name also contains a hyphen. + "codeql-bundle-linux-arm64.tar.zst", + // Not a language we know about. + "codeql-bundle-cobol-linux64.tar.zst", + // A name we cannot decode must not be mistaken for a language either. + "codeql-bundle-%zz-linux64.tar.zst", + ]) { + const url = `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; + t.deepEqual(getCodeQLBundleFromUrl(url), { kind: "combined", url }); + } +}); + +test("getCodeQLBundleFromUrl preserves URLs it cannot parse", (t) => { + const url = "not a url"; + t.deepEqual(getCodeQLBundleFromUrl(url), { kind: "combined", url }); +}); diff --git a/src/codeql-bundle.ts b/src/codeql-bundle.ts new file mode 100644 index 0000000000..4e9f5d5de1 --- /dev/null +++ b/src/codeql-bundle.ts @@ -0,0 +1,33 @@ +import { BuiltInLanguage, parseBuiltInLanguage } from "./languages"; + +/** Describes the contents and location of a downloadable CodeQL bundle. */ +export type CodeQLBundle = + | { kind: "combined"; url: string } + | { + kind: "per-language"; + url: string; + language: BuiltInLanguage; + /** Only set when the Action selected the bundle, allowing a same-version fallback. */ + combinedBundleURL?: string; + }; + +const PER_LANGUAGE_BUNDLE_NAME = + /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; + +/** Classifies an explicit tools URL without changing it or adding a fallback. */ +export function getCodeQLBundleFromUrl(url: string): CodeQLBundle { + let assetName: string; + try { + const pathname = new URL(url).pathname; + // URL-encoded names must not bypass the toolcache safeguard. + assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); + } catch { + return { kind: "combined", url }; + } + + const match = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + const language = match ? parseBuiltInLanguage(match[1]) : undefined; + return language === undefined + ? { kind: "combined", url } + : { kind: "per-language", url, language }; +} diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts index ea8315001a..a755bba87f 100644 --- a/src/per-language-bundles.test.ts +++ b/src/per-language-bundles.test.ts @@ -8,7 +8,6 @@ import { getPerLanguageBundleLanguage, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, PerLanguageBundleOptions, - tryGetBundleLanguageFromUrl, } from "./per-language-bundles"; import { createFeatures, @@ -172,51 +171,3 @@ test("getPerLanguageBundleLanguage skips only the release version check for the undefined, ); }); - -test("tryGetBundleLanguageFromUrl recognizes per-language bundle URLs", (t) => { - const url = (name: string) => - `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; - - t.is( - tryGetBundleLanguageFromUrl(url("codeql-bundle-java-linux64.tar.zst")), - BuiltInLanguage.java, - ); - t.is( - tryGetBundleLanguageFromUrl(url("codeql-bundle-swift-osx64.tar.zst")), - BuiltInLanguage.swift, - ); - // We do not publish these, but should still recognize them if we ever do. - t.is( - tryGetBundleLanguageFromUrl(url("codeql-bundle-csharp-win64.tar.gz")), - BuiltInLanguage.csharp, - ); - // A percent-encoded name resolves to the same asset, so it must not let a bundle that contains a - // single language pass for one that contains them all and end up in the toolcache. - t.is( - tryGetBundleLanguageFromUrl(url("codeql-bundle-%70ython-linux64.tar.zst")), - BuiltInLanguage.python, - ); -}); - -test("tryGetBundleLanguageFromUrl rejects other bundle URLs", (t) => { - const url = (name: string) => - `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; - - for (const name of [ - "codeql-bundle-linux64.tar.zst", - "codeql-bundle-osx64.tar.gz", - "codeql-bundle-win64.tar.zst", - // The all-platform bundle. - "codeql-bundle.tar.gz", - // A platform we do not publish per-language bundles for, whose name also contains a hyphen. - "codeql-bundle-linux-arm64.tar.zst", - // Not a language we know about. - "codeql-bundle-cobol-linux64.tar.zst", - // A name we cannot decode must not be mistaken for a language either. - "codeql-bundle-%zz-linux64.tar.zst", - ]) { - t.is(tryGetBundleLanguageFromUrl(url(name)), undefined, name); - } - - t.is(tryGetBundleLanguageFromUrl("not a url"), undefined); -}); diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts index 20720636fc..9c07a68028 100644 --- a/src/per-language-bundles.ts +++ b/src/per-language-bundles.ts @@ -11,26 +11,6 @@ import { GitHubVariant } from "./util"; /** Minimum CLI version for selecting a per-language release bundle. */ export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; -const PER_LANGUAGE_BUNDLE_NAME = - /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; - -/** Returns the language in a per-language tools URL, or undefined for other URLs. */ -export function tryGetBundleLanguageFromUrl( - url: string, -): BuiltInLanguage | undefined { - let assetName: string; - try { - const pathname = new URL(url).pathname; - // URL-encoded names must not bypass the toolcache safeguard. - assetName = decodeURIComponent(pathname.split("/").pop() ?? ""); - } catch { - return undefined; - } - - const match = assetName.match(PER_LANGUAGE_BUNDLE_NAME); - return match ? parseBuiltInLanguage(match[1]) : undefined; -} - /** Languages with per-language bundles published for each platform. */ const PER_LANGUAGE_BUNDLE_LANGUAGES: Readonly< Record> diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 36d10cd42e..ba51d0b364 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -18,6 +18,7 @@ import { } from "./actions-util"; import * as api from "./api-client"; import { getBundlePlatform } from "./bundle-platform"; +import { CodeQLBundle, getCodeQLBundleFromUrl } from "./codeql-bundle"; import * as defaults from "./defaults.json"; import { addNoLanguageDiagnostic, @@ -35,10 +36,7 @@ import { import { BuiltInLanguage } from "./languages"; import { Logger } from "./logging"; import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching"; -import { - getPerLanguageBundleLanguage, - tryGetBundleLanguageFromUrl, -} from "./per-language-bundles"; +import { getPerLanguageBundleLanguage } from "./per-language-bundles"; import * as tar from "./tar"; import { deleteToolcacheBundles, @@ -225,17 +223,6 @@ export function convertToSemVer(version: string, logger: Logger): string { return s; } -/** Describes the contents and location of a downloadable CodeQL bundle. */ -type CodeQLBundle = - | { kind: "combined"; url: string } - | { - kind: "per-language"; - url: string; - language: BuiltInLanguage; - /** Only set when the Action selected the bundle, allowing a same-version fallback. */ - combinedBundleURL?: string; - }; - /** A resolved download, including its bundle identity and version. */ export interface CodeQLDownloadSource { /** Distinguishes downloads from local archives and cached installations. */ @@ -816,14 +803,7 @@ export async function getCodeQLSource( } compressionMethod = method; - if (bundle === undefined) { - // Explicit per-language URLs must also stay out of the toolcache, but have no fallback. - const language = tryGetBundleLanguageFromUrl(url); - bundle = - language === undefined - ? { kind: "combined", url } - : { kind: "per-language", url, language }; - } + bundle ??= getCodeQLBundleFromUrl(url); if (bundle.kind === "per-language") { logger.info( `${url} appears to be a CodeQL bundle that contains only ${bundle.language}.`, From 89606bbad18e5ed1b53cbf2167003a5006156e99 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 16 Sep 2026 19:43:05 +0100 Subject: [PATCH 10/20] Return toolcache rejection reasons with Result Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 37 ++++++++++++++++------------ src/setup-codeql.test.ts | 17 ++++++++++++- src/setup-codeql.ts | 52 +++++++++++++++++++++------------------- 3 files changed, 66 insertions(+), 40 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index 0a0941fe96..fc50bae4ff 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -152541,8 +152541,8 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log codeqlURL ); } - const toolcacheDestination = getToolcacheDestination(source, logger); - const extractedBundlePath = toolcacheDestination ?? getTempExtractionDir(tempDir); + const toolcacheDestination = getToolcacheDestination({ logger }, source); + const extractedBundlePath = toolcacheDestination.isSuccess() ? toolcacheDestination.value : getTempExtractionDir(tempDir); const statusReport = await downloadAndExtract( codeqlURL, compressionMethod, @@ -152552,27 +152552,34 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log tarVersion, logger ); - if (toolcacheDestination) { - writeToolcacheMarkerFile(toolcacheDestination, logger); + if (toolcacheDestination.isSuccess()) { + writeToolcacheMarkerFile(toolcacheDestination.value, logger); } else { - logger.debug( - bundle.kind === "per-language" ? "Not caching the CodeQL tools because they came from a bundle that contains only a single language." : `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${codeqlURL}.` - ); + logger.debug(toolcacheDestination.value); } return { codeqlFolder: extractedBundlePath, statusReport: bundle.kind === "per-language" ? { ...statusReport, bundleLanguage: bundle.language } : statusReport }; }; -function getToolcacheDestination(source, logger) { - if (source.bundle.kind !== "combined" || !source.bundleVersion) { - return void 0; +function getToolcacheDestination({ logger }, source) { + if (source.bundle.kind !== "combined") { + return new Failure( + "Not caching the CodeQL tools because they came from a bundle that contains only a single language." + ); } - return getToolcacheDirectory( - getCanonicalToolcacheVersion( - source.cliVersion, - source.bundleVersion, - logger + if (!source.bundleVersion) { + return new Failure( + `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${source.bundle.url}.` + ); + } + return new Success( + getToolcacheDirectory( + getCanonicalToolcacheVersion( + source.cliVersion, + source.bundleVersion, + logger + ) ) ); } diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index 3f440894d7..c32c22b64a 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -25,6 +25,7 @@ import { SAMPLE_DEFAULT_CLI_VERSION, SAMPLE_DOTCOM_API_DETAILS, checkExpectedLogMessages, + checkUnexpectedLogMessages, createFeatures, createTestConfig, getRecordingLogger, @@ -331,6 +332,10 @@ test.serial( downloadDurationMs: 200, totalDurationMs: 300, }); + checkUnexpectedLogMessages(t, loggedMessages, [ + "Not caching the CodeQL tools", + "Could not cache CodeQL tools", + ]); // Ensure message logging CodeQL CLI version was present in user logs. const expected_message: string = `Using CodeQL CLI version ${LINKED_CLI_VERSION.cliVersion}`; @@ -563,6 +568,9 @@ for (const bundlePath of [ t.deepEqual(toolcache.findAllVersions("CodeQL"), []); checkExpectedLogMessages(t, messages, [ `Using CodeQL CLI sourced from ${url}`, + bundlePath === "codeql-bundle-ruby-linux64.tar.zst" + ? "Not caching the CodeQL tools because they came from a bundle that contains only a single language." + : `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${url}.`, ]); }); }, @@ -1473,6 +1481,7 @@ for (const asset of [ `setupCodeQLBundle keeps explicitly requested ${asset} out of the toolcache`, async (t) => { const extractStub = stubDownloadAndExtract(); + const messages: LoggedMessage[] = []; const url = `https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/${asset}`; process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted"; @@ -1487,7 +1496,7 @@ for (const asset of [ undefined, // rawLanguages false, // useOverlayAwareDefaultCliVersion createFeatures([]), - getRunnerLogger(true), + getRecordingLogger(messages), ); t.true(extractStub.calledOnce); @@ -1500,6 +1509,12 @@ for (const asset of [ t.is(path.dirname(result.codeqlFolder), tmpDir); t.deepEqual(toolcache.findAllVersions("CodeQL"), []); t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + checkExpectedLogMessages(t, messages, [ + "Not caching the CodeQL tools because they came from a bundle that contains only a single language.", + ]); + checkUnexpectedLogMessages(t, messages, [ + "Could not cache CodeQL tools because we could not determine the bundle version", + ]); }); }, ); diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index ba51d0b364..4efb0dc2e1 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -879,9 +879,10 @@ export const downloadCodeQL = async function ( ); } - const toolcacheDestination = getToolcacheDestination(source, logger); - const extractedBundlePath = - toolcacheDestination ?? getTempExtractionDir(tempDir); + const toolcacheDestination = getToolcacheDestination({ logger }, source); + const extractedBundlePath = toolcacheDestination.isSuccess() + ? toolcacheDestination.value + : getTempExtractionDir(tempDir); const statusReport = await downloadAndExtract( codeqlURL, @@ -893,16 +894,10 @@ export const downloadCodeQL = async function ( logger, ); - if (toolcacheDestination) { - writeToolcacheMarkerFile(toolcacheDestination, logger); + if (toolcacheDestination.isSuccess()) { + writeToolcacheMarkerFile(toolcacheDestination.value, logger); } else { - logger.debug( - bundle.kind === "per-language" - ? "Not caching the CodeQL tools because they came from a bundle that contains only a " + - "single language." - : "Could not cache CodeQL tools because we could not determine the bundle version from the " + - `URL ${codeqlURL}.`, - ); + logger.debug(toolcacheDestination.value); } return { @@ -915,23 +910,32 @@ export const downloadCodeQL = async function ( }; /** - * Returns the canonical toolcache directory for a combined bundle with a known version. - * Returns undefined for per-language bundles or unknown versions. + * Returns the canonical toolcache directory, or the reason the bundle cannot be cached. */ function getToolcacheDestination( + { logger }: ActionState<["Logger"]>, source: CodeQLDownloadSource, - logger: Logger, -): string | undefined { - // Per-language bundles must not be stored in the toolcache. - if (source.bundle.kind !== "combined" || !source.bundleVersion) { - return undefined; +): util.Result { + if (source.bundle.kind !== "combined") { + return new util.Failure( + "Not caching the CodeQL tools because they came from a bundle that contains only a " + + "single language.", + ); + } + if (!source.bundleVersion) { + return new util.Failure( + "Could not cache CodeQL tools because we could not determine the bundle version from the " + + `URL ${source.bundle.url}.`, + ); } - return getToolcacheDirectory( - getCanonicalToolcacheVersion( - source.cliVersion, - source.bundleVersion, - logger, + return new util.Success( + getToolcacheDirectory( + getCanonicalToolcacheVersion( + source.cliVersion, + source.bundleVersion, + logger, + ), ), ); } From f4fa111630c809cc10991d9761d73c2f782230d6 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 16 Sep 2026 19:51:23 +0100 Subject: [PATCH 11/20] Share elapsed-time rounding for bundle downloads Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 56 +++++++++++++++++++++++-------------------- src/setup-codeql.ts | 2 +- src/tools-download.ts | 9 +++---- src/util.test.ts | 21 ++++++++++++++++ src/util.ts | 6 +++++ 5 files changed, 63 insertions(+), 31 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index fc50bae4ff..cc4cf5f24e 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -4305,7 +4305,7 @@ var require_util2 = __commonJS({ var { redirectStatusSet, referrerPolicySet: referrerPolicyTokens, badPortsSet } = require_constants3(); var { getGlobalOrigin } = require_global(); var { collectASequenceOfCodePoints, collectAnHTTPQuotedString, removeChars, parseMIMEType } = require_data_url(); - var { performance: performance7 } = require("node:perf_hooks"); + var { performance: performance8 } = require("node:perf_hooks"); var { isBlobLike, ReadableStreamFrom, isValidHTTPToken, normalizedMethodRecordsBase } = require_util(); var assert = require("node:assert"); var { isUint8Array } = require("node:util/types"); @@ -4464,7 +4464,7 @@ var require_util2 = __commonJS({ }; } function coarsenedSharedCurrentTime(crossOriginIsolatedCapability) { - return coarsenTime(performance7.now(), crossOriginIsolatedCapability); + return coarsenTime(performance8.now(), crossOriginIsolatedCapability); } function createOpaqueTimingInfo(timingInfo) { return { @@ -142119,7 +142119,7 @@ module.exports = __toCommonJS(entry_points_exports); // src/analyze-action.ts var fs23 = __toESM(require("fs")); var import_path5 = __toESM(require("path")); -var import_perf_hooks5 = require("perf_hooks"); +var import_perf_hooks6 = require("perf_hooks"); var core17 = __toESM(require_core()); // src/action-common.ts @@ -142201,6 +142201,7 @@ var fs = __toESM(require("fs")); var fsPromises = __toESM(require("fs/promises")); var os = __toESM(require("os")); var path = __toESM(require("path")); +var import_perf_hooks = require("perf_hooks"); var core2 = __toESM(require_core()); var io = __toESM(require_io()); @@ -145892,6 +145893,9 @@ async function bundleDb(config, language, codeql, dbName, { includeDiagnostics } ); return databaseBundlePath; } +function durationMsSince(startTime) { + return Math.round(import_perf_hooks.performance.now() - startTime); +} async function delay(milliseconds, opts) { const { allowProcessExit } = opts || {}; return new Promise((resolve14) => { @@ -148633,7 +148637,7 @@ var SarifScanOrder = [ // src/analyze.ts var fs17 = __toESM(require("fs")); var path16 = __toESM(require("path")); -var import_perf_hooks4 = require("perf_hooks"); +var import_perf_hooks5 = require("perf_hooks"); var io5 = __toESM(require_io()); // src/autobuild.ts @@ -148897,7 +148901,7 @@ function wrapCliConfigurationError(cliError) { // src/config-utils.ts var fs10 = __toESM(require("fs")); var path11 = __toESM(require("path")); -var import_perf_hooks = require("perf_hooks"); +var import_perf_hooks2 = require("perf_hooks"); var core10 = __toESM(require_core()); // src/caching-utils.ts @@ -150523,9 +150527,9 @@ async function initActionState({ }; } async function downloadCacheWithTime(codeQL, languages, logger) { - const start = import_perf_hooks.performance.now(); + const start = import_perf_hooks2.performance.now(); const trapCaches = await downloadTrapCaches(codeQL, languages, logger); - const trapCacheDownloadTime = import_perf_hooks.performance.now() - start; + const trapCacheDownloadTime = import_perf_hooks2.performance.now() - start; return { trapCaches, trapCacheDownloadTime }; } async function loadUserConfig(actionState, configFile, workspacePath, apiDetails, tempDir) { @@ -150929,10 +150933,10 @@ async function initConfig(actionState, inputs) { } if (await features.getValue("ignore_generated_files" /* IgnoreGeneratedFiles */) && isDynamicWorkflow()) { try { - const generatedFilesCheckStartedAt = import_perf_hooks.performance.now(); + const generatedFilesCheckStartedAt = import_perf_hooks2.performance.now(); const generatedFiles = await getGeneratedFiles(inputs.sourceRoot); const generatedFilesDuration = Math.round( - import_perf_hooks.performance.now() - generatedFilesCheckStartedAt + import_perf_hooks2.performance.now() - generatedFilesCheckStartedAt ); if (generatedFiles.length > 0) { config.computedConfig["paths-ignore"] ??= []; @@ -151216,7 +151220,7 @@ async function logGeneratedFilesTelemetry(config, duration, generatedFilesCount) // src/setup-codeql.ts var fs14 = __toESM(require("fs")); var path13 = __toESM(require("path")); -var import_perf_hooks3 = require("perf_hooks"); +var import_perf_hooks4 = require("perf_hooks"); var core12 = __toESM(require_core()); var toolcache3 = __toESM(require_tool_cache()); var import_fast_deep_equal = __toESM(require_fast_deep_equal()); @@ -151789,7 +151793,7 @@ function inferCompressionMethod(tarPath) { var fs13 = __toESM(require("fs")); var os4 = __toESM(require("os")); var path12 = __toESM(require("path")); -var import_perf_hooks2 = require("perf_hooks"); +var import_perf_hooks3 = require("perf_hooks"); var core11 = __toESM(require_core()); var import_http_client = __toESM(require_lib()); var toolcache2 = __toESM(require_tool_cache()); @@ -151802,7 +151806,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat logger.info( `Downloading CodeQL tools from ${codeqlURL} . This may take a while.` ); - const startTime = import_perf_hooks2.performance.now(); + const startTime = import_perf_hooks3.performance.now(); try { if (compressionMethod === "zstd" && process.platform === "linux") { logger.info(`Streaming the extraction of the CodeQL bundle.`); @@ -151814,7 +151818,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat tarVersion, logger ); - const totalDurationMs = Math.round(import_perf_hooks2.performance.now() - startTime); + const totalDurationMs = durationMsSince(startTime); logger.info( `Finished downloading and extracting CodeQL bundle to ${dest} (${formatDuration( totalDurationMs @@ -151832,14 +151836,14 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat ); core11.warning(`Falling back to downloading the bundle before extracting.`); } - const toolsDownloadStart = import_perf_hooks2.performance.now(); + const toolsDownloadStart = import_perf_hooks3.performance.now(); const archivedBundlePath = await toolcache2.downloadTool( codeqlURL, void 0, authorization, headers ); - const downloadDurationMs = Math.round(import_perf_hooks2.performance.now() - toolsDownloadStart); + const downloadDurationMs = durationMsSince(toolsDownloadStart); logger.info( `Finished downloading CodeQL bundle to ${archivedBundlePath} (${formatDuration( downloadDurationMs @@ -151848,7 +151852,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat let extractionDurationMs; try { logger.info("Extracting CodeQL bundle."); - const extractionStart = import_perf_hooks2.performance.now(); + const extractionStart = import_perf_hooks3.performance.now(); await extract( archivedBundlePath, dest, @@ -151856,7 +151860,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat tarVersion, logger ); - extractionDurationMs = Math.round(import_perf_hooks2.performance.now() - extractionStart); + extractionDurationMs = durationMsSince(extractionStart); logger.info( `Finished extracting CodeQL bundle to ${dest} (${formatDuration( extractionDurationMs @@ -151868,7 +151872,7 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat return { downloadDurationMs, extractionDurationMs, - totalDurationMs: Math.round(import_perf_hooks2.performance.now() - startTime) + totalDurationMs: durationMsSince(startTime) }; } async function downloadAndExtractZstdWithStreaming(codeqlURL, dest, authorization, headers, tarVersion, logger) { @@ -152679,7 +152683,7 @@ async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, temp const { bundle } = source; const { logger } = action; await tryDeleteToolcacheBundles(action); - const startTime = import_perf_hooks3.performance.now(); + const startTime = import_perf_hooks4.performance.now(); try { return await downloadCodeQL( source, @@ -152709,7 +152713,7 @@ async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, temp ...result, statusReport: { ...result.statusReport, - totalDurationMs: Math.round(import_perf_hooks3.performance.now() - startTime), + totalDurationMs: durationMsSince(startTime), perLanguageBundleFallback: true } }; @@ -153873,10 +153877,10 @@ function dbIsFinalized(config, language, logger) { } } async function finalizeDatabaseCreation(codeql, features, config, threadsFlag, memoryFlag, logger) { - const extractionStart = import_perf_hooks4.performance.now(); + const extractionStart = import_perf_hooks5.performance.now(); await runExtraction(codeql, features, config, logger); - const extractionTime = import_perf_hooks4.performance.now() - extractionStart; - const trapImportStart = import_perf_hooks4.performance.now(); + const extractionTime = import_perf_hooks5.performance.now() - extractionStart; + const trapImportStart = import_perf_hooks5.performance.now(); for (const language of config.languages) { if (dbIsFinalized(config, language, logger)) { logger.info( @@ -153893,7 +153897,7 @@ async function finalizeDatabaseCreation(codeql, features, config, threadsFlag, m logger.endGroup(); } } - const trapImportTime = import_perf_hooks4.performance.now() - trapImportStart; + const trapImportTime = import_perf_hooks5.performance.now() - trapImportStart; return { scanned_language_extraction_duration_ms: Math.round(extractionTime), trap_import_duration_ms: Math.round(trapImportTime) @@ -156700,9 +156704,9 @@ async function run({ startedAt, logger }) { features, logger ); - const trapCacheUploadStartTime = import_perf_hooks5.performance.now(); + const trapCacheUploadStartTime = import_perf_hooks6.performance.now(); didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger); - trapCacheUploadTime = import_perf_hooks5.performance.now() - trapCacheUploadStartTime; + trapCacheUploadTime = import_perf_hooks6.performance.now() - trapCacheUploadStartTime; trapCacheCleanupTelemetry = await cleanupTrapCaches( config, features, diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 4efb0dc2e1..cadbcb3993 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -1170,7 +1170,7 @@ export async function downloadCodeQLBundle( ...result, statusReport: { ...result.statusReport, - totalDurationMs: Math.round(performance.now() - startTime), + totalDurationMs: util.durationMsSince(startTime), perLanguageBundleFallback: true, }, }; diff --git a/src/tools-download.ts b/src/tools-download.ts index 5494ab30f1..61501557c5 100644 --- a/src/tools-download.ts +++ b/src/tools-download.ts @@ -17,6 +17,7 @@ import * as tar from "./tar"; import { asHTTPError, cleanUpPath, + durationMsSince, getErrorMessage, getRequiredEnvParam, HTTPError, @@ -91,7 +92,7 @@ export async function downloadAndExtract( logger, ); - const totalDurationMs = Math.round(performance.now() - startTime); + const totalDurationMs = durationMsSince(startTime); logger.info( `Finished downloading and extracting CodeQL bundle to ${dest} (${formatDuration( totalDurationMs, @@ -124,7 +125,7 @@ export async function downloadAndExtract( authorization, headers, ); - const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart); + const downloadDurationMs = durationMsSince(toolsDownloadStart); logger.info( `Finished downloading CodeQL bundle to ${archivedBundlePath} (${formatDuration( @@ -144,7 +145,7 @@ export async function downloadAndExtract( tarVersion, logger, ); - extractionDurationMs = Math.round(performance.now() - extractionStart); + extractionDurationMs = durationMsSince(extractionStart); logger.info( `Finished extracting CodeQL bundle to ${dest} (${formatDuration( extractionDurationMs, @@ -157,7 +158,7 @@ export async function downloadAndExtract( return { downloadDurationMs, extractionDurationMs, - totalDurationMs: Math.round(performance.now() - startTime), + totalDurationMs: durationMsSince(startTime), }; } diff --git a/src/util.test.ts b/src/util.test.ts index cca457cbe6..074310279f 100644 --- a/src/util.test.ts +++ b/src/util.test.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as os from "os"; import path from "path"; +import { performance } from "perf_hooks"; import * as core from "@actions/core"; import test from "ava"; @@ -508,6 +509,26 @@ test("joinAtMost - truncates list if array is > than limit", (t) => { t.false(result.includes("test6")); }); +test.serial( + "durationMsSince rounds elapsed milliseconds rather than the timestamps", + (t) => { + const startTime = 1000.25; + const now = sinon.stub(performance, "now"); + for (const [endTime, expected] of [ + [1000.25, 0], + [1000.74, 0], + [1000.75, 1], + [1001.74, 1], + [1001.75, 2], + [2000.74, 1000], + [2000.75, 1001], + ]) { + now.returns(endTime); + t.is(util.durationMsSince(startTime), expected); + } + }, +); + test("Success creates a success result", (t) => { const result = new util.Success("test value"); t.true(result.isSuccess()); diff --git a/src/util.ts b/src/util.ts index 49fe924f66..456cd7c3d2 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2,6 +2,7 @@ import * as fs from "fs"; import * as fsPromises from "fs/promises"; import * as os from "os"; import * as path from "path"; +import { performance } from "perf_hooks"; import * as core from "@actions/core"; import * as io from "@actions/io"; @@ -681,6 +682,11 @@ export async function bundleDb( return databaseBundlePath; } +/** Returns the elapsed milliseconds, rounded, since a `performance.now()` timestamp. */ +export function durationMsSince(startTime: number): number { + return Math.round(performance.now() - startTime); +} + /** * @param milliseconds time to delay * @param opts options From 69f47159b10c44e939a256f9fc73acc39300aaf9 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 16 Sep 2026 19:55:13 +0100 Subject: [PATCH 12/20] Use Result.orElse for bundle extraction paths Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 4 +++- src/setup-codeql.ts | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index cc4cf5f24e..d23dcaac35 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -152546,7 +152546,9 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log ); } const toolcacheDestination = getToolcacheDestination({ logger }, source); - const extractedBundlePath = toolcacheDestination.isSuccess() ? toolcacheDestination.value : getTempExtractionDir(tempDir); + const extractedBundlePath = toolcacheDestination.orElse( + getTempExtractionDir(tempDir) + ); const statusReport = await downloadAndExtract( codeqlURL, compressionMethod, diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index cadbcb3993..c476dd885e 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -880,9 +880,9 @@ export const downloadCodeQL = async function ( } const toolcacheDestination = getToolcacheDestination({ logger }, source); - const extractedBundlePath = toolcacheDestination.isSuccess() - ? toolcacheDestination.value - : getTempExtractionDir(tempDir); + const extractedBundlePath = toolcacheDestination.orElse( + getTempExtractionDir(tempDir), + ); const statusReport = await downloadAndExtract( codeqlURL, From 28b8f598f5032f2f254105608a3df22bab2b8ae4 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 17 Sep 2026 19:00:38 +0100 Subject: [PATCH 13/20] Use shared test state for bundle eligibility Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/per-language-bundles.test.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts index a755bba87f..105953564f 100644 --- a/src/per-language-bundles.test.ts +++ b/src/per-language-bundles.test.ts @@ -1,7 +1,7 @@ import test from "ava"; import { BundlePlatform } from "./bundle-platform"; -import { ActionsEnvVars, ReadOnlyEnv } from "./environment"; +import { ActionsEnvVars, Env } from "./environment"; import { Feature } from "./feature-flags"; import { BuiltInLanguage } from "./languages"; import { @@ -13,6 +13,7 @@ import { createFeatures, getRecordingLogger, getTestEnv, + initAllState, LoggedMessage, } from "./testing-utils"; import { GitHubVariant } from "./util"; @@ -30,16 +31,16 @@ const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = { async function checkEligibility( overrides: Partial, enabledFeatures: Feature[] = [Feature.PerLanguageBundles], - env: ReadOnlyEnv = getTestEnv({ + env: Env = getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted", }), ) { return getPerLanguageBundleLanguage( - { + initAllState({ env, features: createFeatures(enabledFeatures), logger: getRecordingLogger([], { logToConsole: false }), - }, + }), { ...ELIGIBLE_OPTIONS, ...overrides }, ); } @@ -130,11 +131,11 @@ test("getPerLanguageBundleLanguage requires the feature flag", async (t) => { test("getPerLanguageBundleLanguage explains a disabled feature before checking eligibility", async (t) => { const messages: LoggedMessage[] = []; const language = await getPerLanguageBundleLanguage( - { + initAllState({ env: getTestEnv(), features: createFeatures([]), logger: getRecordingLogger(messages, { logToConsole: false }), - }, + }), { ...ELIGIBLE_OPTIONS, rawLanguages: undefined, cliVersion: undefined }, ); From ed3a24ccbc90d2bc224bd6631358a809766a0a5a Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 17 Sep 2026 19:01:31 +0100 Subject: [PATCH 14/20] Group bundle eligibility test state overrides Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/per-language-bundles.test.ts | 39 +++++++++++++++++--------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts index 105953564f..76f347edfa 100644 --- a/src/per-language-bundles.test.ts +++ b/src/per-language-bundles.test.ts @@ -1,7 +1,7 @@ import test from "ava"; import { BundlePlatform } from "./bundle-platform"; -import { ActionsEnvVars, Env } from "./environment"; +import { ActionsEnvVars } from "./environment"; import { Feature } from "./feature-flags"; import { BuiltInLanguage } from "./languages"; import { @@ -30,16 +30,16 @@ const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = { async function checkEligibility( overrides: Partial, - enabledFeatures: Feature[] = [Feature.PerLanguageBundles], - env: Env = getTestEnv({ - [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted", - }), + stateOverrides: Partial> = {}, ) { return getPerLanguageBundleLanguage( initAllState({ - env, - features: createFeatures(enabledFeatures), + env: getTestEnv({ + [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted", + }), + features: createFeatures([Feature.PerLanguageBundles]), logger: getRecordingLogger([], { logToConsole: false }), + ...stateOverrides, }), { ...ELIGIBLE_OPTIONS, ...overrides }, ); @@ -100,8 +100,9 @@ test("getPerLanguageBundleLanguage requires a GitHub-hosted runner", async (t) = t.is( await checkEligibility( {}, - [Feature.PerLanguageBundles], - getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "self-hosted" }), + { + env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "self-hosted" }), + }, ), undefined, ); @@ -111,8 +112,9 @@ test("getPerLanguageBundleLanguage requires a GitHub-hosted runner", async (t) = t.is( await checkEligibility( {}, - [Feature.PerLanguageBundles], - getTestEnv({ RUNNER_TOOL_CACHE: "/opt/hostedtoolcache" }), + { + env: getTestEnv({ RUNNER_TOOL_CACHE: "/opt/hostedtoolcache" }), + }, ), undefined, ); @@ -125,7 +127,7 @@ test("getPerLanguageBundleLanguage requires a supported release version", async }); test("getPerLanguageBundleLanguage requires the feature flag", async (t) => { - t.is(await checkEligibility({}, []), undefined); + t.is(await checkEligibility({}, { features: createFeatures([]) }), undefined); }); test("getPerLanguageBundleLanguage explains a disabled feature before checking eligibility", async (t) => { @@ -162,13 +164,14 @@ test("getPerLanguageBundleLanguage skips only the release version check for the ]) { t.is(await checkEligibility({ ...nightly, ...overrides }), undefined); } - t.is(await checkEligibility(nightly, []), undefined); t.is( - await checkEligibility( - nightly, - [Feature.PerLanguageBundles], - getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "self-hosted" }), - ), + await checkEligibility(nightly, { features: createFeatures([]) }), + undefined, + ); + t.is( + await checkEligibility(nightly, { + env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "self-hosted" }), + }), undefined, ); }); From 549d498da392f61aadfc0416f08ed43ae7397a2f Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 17 Sep 2026 19:02:12 +0100 Subject: [PATCH 15/20] Simplify per-language platform eligibility checks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 3 +-- src/per-language-bundles.ts | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index d23dcaac35..1ccf9ee0bb 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -151620,8 +151620,7 @@ async function getPerLanguageBundleLanguage({ ); } } - const supportedLanguages = platform2 === void 0 ? void 0 : PER_LANGUAGE_BUNDLE_LANGUAGES[platform2]; - if (!supportedLanguages?.has(language)) { + if (platform2 === void 0 || !PER_LANGUAGE_BUNDLE_LANGUAGES[platform2].has(language)) { return explain( `no per-language bundle is published for ${language} on ${platform2 ?? "an unknown platform"}` ); diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts index 9c07a68028..567c365ff8 100644 --- a/src/per-language-bundles.ts +++ b/src/per-language-bundles.ts @@ -117,11 +117,10 @@ export async function getPerLanguageBundleLanguage( } } - const supportedLanguages = - platform === undefined - ? undefined - : PER_LANGUAGE_BUNDLE_LANGUAGES[platform]; - if (!supportedLanguages?.has(language)) { + if ( + platform === undefined || + !PER_LANGUAGE_BUNDLE_LANGUAGES[platform].has(language) + ) { return explain( `no per-language bundle is published for ${language} on ${platform ?? "an unknown platform"}`, ); From ead1f7d93f7fea11d3cf483d696b783b3f686607 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 17 Sep 2026 19:03:27 +0100 Subject: [PATCH 16/20] Rename the platform module Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 30 ++++++++++--------- src/per-language-bundles.test.ts | 2 +- src/per-language-bundles.ts | 2 +- ...ndle-platform.test.ts => platform.test.ts} | 2 +- src/{bundle-platform.ts => platform.ts} | 0 src/setup-codeql.ts | 2 +- src/testing-utils.ts | 2 +- 7 files changed, 21 insertions(+), 19 deletions(-) rename src/{bundle-platform.test.ts => platform.test.ts} (89%) rename src/{bundle-platform.ts => platform.ts} (100%) diff --git a/lib/entry-points.js b/lib/entry-points.js index 1ccf9ee0bb..ada8ee4ee4 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -151226,20 +151226,6 @@ var toolcache3 = __toESM(require_tool_cache()); var import_fast_deep_equal = __toESM(require_fast_deep_equal()); var semver10 = __toESM(require_semver2()); -// src/bundle-platform.ts -function getBundlePlatform(platform2 = process.platform, arch2 = process.arch) { - switch (platform2) { - case "win32": - return "win64" /* Win64 */; - case "linux": - return arch2 === "arm64" ? "linux-arm64" /* LinuxArm64 */ : "linux64" /* Linux64 */; - case "darwin": - return "osx64" /* Osx64 */; - default: - return void 0; - } -} - // src/codeql-bundle.ts var PER_LANGUAGE_BUNDLE_NAME = /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; function getCodeQLBundleFromUrl(url2) { @@ -151555,6 +151541,22 @@ async function getCodeQlVersionsForOverlayBaseDatabases(rawLanguages, logger) { // src/per-language-bundles.ts var semver7 = __toESM(require_semver2()); + +// src/platform.ts +function getBundlePlatform(platform2 = process.platform, arch2 = process.arch) { + switch (platform2) { + case "win32": + return "win64" /* Win64 */; + case "linux": + return arch2 === "arm64" ? "linux-arm64" /* LinuxArm64 */ : "linux64" /* Linux64 */; + case "darwin": + return "osx64" /* Osx64 */; + default: + return void 0; + } +} + +// src/per-language-bundles.ts var MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; var PER_LANGUAGE_BUNDLE_LANGUAGES = { ["linux64" /* Linux64 */]: /* @__PURE__ */ new Set([ diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts index 76f347edfa..b8f48512fe 100644 --- a/src/per-language-bundles.test.ts +++ b/src/per-language-bundles.test.ts @@ -1,6 +1,5 @@ import test from "ava"; -import { BundlePlatform } from "./bundle-platform"; import { ActionsEnvVars } from "./environment"; import { Feature } from "./feature-flags"; import { BuiltInLanguage } from "./languages"; @@ -9,6 +8,7 @@ import { MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, PerLanguageBundleOptions, } from "./per-language-bundles"; +import { BundlePlatform } from "./platform"; import { createFeatures, getRecordingLogger, diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts index 567c365ff8..ea715aeb1c 100644 --- a/src/per-language-bundles.ts +++ b/src/per-language-bundles.ts @@ -2,9 +2,9 @@ import * as semver from "semver"; import { ActionState } from "./action-common"; import { isGitHubHostedRunner } from "./actions-util"; -import { BundlePlatform } from "./bundle-platform"; import { Feature } from "./feature-flags"; import { BuiltInLanguage, parseBuiltInLanguage } from "./languages"; +import { BundlePlatform } from "./platform"; import * as tar from "./tar"; import { GitHubVariant } from "./util"; diff --git a/src/bundle-platform.test.ts b/src/platform.test.ts similarity index 89% rename from src/bundle-platform.test.ts rename to src/platform.test.ts index 73508ca236..c8a6a7955d 100644 --- a/src/bundle-platform.test.ts +++ b/src/platform.test.ts @@ -1,6 +1,6 @@ import test from "ava"; -import { BundlePlatform, getBundlePlatform } from "./bundle-platform"; +import { BundlePlatform, getBundlePlatform } from "./platform"; for (const [platform, arch, expected] of [ ["linux", "x64", BundlePlatform.Linux64], diff --git a/src/bundle-platform.ts b/src/platform.ts similarity index 100% rename from src/bundle-platform.ts rename to src/platform.ts diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index c476dd885e..60f68a37aa 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -17,7 +17,6 @@ import { isRunningLocalAction, } from "./actions-util"; import * as api from "./api-client"; -import { getBundlePlatform } from "./bundle-platform"; import { CodeQLBundle, getCodeQLBundleFromUrl } from "./codeql-bundle"; import * as defaults from "./defaults.json"; import { @@ -37,6 +36,7 @@ import { BuiltInLanguage } from "./languages"; import { Logger } from "./logging"; import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching"; import { getPerLanguageBundleLanguage } from "./per-language-bundles"; +import { getBundlePlatform } from "./platform"; import * as tar from "./tar"; import { deleteToolcacheBundles, diff --git a/src/testing-utils.ts b/src/testing-utils.ts index f3456b7257..f15cee2e71 100644 --- a/src/testing-utils.ts +++ b/src/testing-utils.ts @@ -17,7 +17,6 @@ import { ActionsEnv, getActionVersion } from "./actions-util"; import { AnalysisKind } from "./analyses"; import * as apiClient from "./api-client"; import { GitHubApiDetails } from "./api-client"; -import { getBundlePlatform } from "./bundle-platform"; import { CachingKind } from "./caching-utils"; import { resetCachedCodeQlVersion } from "./cli/output-cache"; import type { VersionInfo } from "./cli/types"; @@ -33,6 +32,7 @@ import { } from "./feature-flags"; import { Logger } from "./logging"; import { OverlayDatabaseMode } from "./overlay/overlay-database-mode"; +import { getBundlePlatform } from "./platform"; import { ActionName } from "./status-report"; import { DEFAULT_DEBUG_ARTIFACT_NAME, From 79fe3a1270f5a101a20367147a05eb6d8ed533af Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 17 Sep 2026 19:04:38 +0100 Subject: [PATCH 17/20] Move download telemetry into the status-report directory Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 46 ++++++++------- src/init-action.ts | 2 +- src/setup-codeql-action.ts | 2 +- src/status-report.ts | 59 ------------------- .../tools-download.test.ts} | 5 +- src/status-report/tools-download.ts | 59 +++++++++++++++++++ 6 files changed, 88 insertions(+), 85 deletions(-) rename src/{tools-download-status-report.test.ts => status-report/tools-download.test.ts} (94%) create mode 100644 src/status-report/tools-download.ts diff --git a/lib/entry-points.js b/lib/entry-points.js index ada8ee4ee4..53532f2f75 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -147625,28 +147625,6 @@ async function sendStatusReport(statusReport) { ); } } -function createInitToolsDownloadFields(report, toolsFeatureFlagsValid) { - const fields = {}; - if (report?.downloadDurationMs !== void 0) { - fields.tools_download_duration_ms = report.downloadDurationMs; - } - if (report?.extractionDurationMs !== void 0) { - fields.tools_extraction_duration_ms = report.extractionDurationMs; - } - if (report?.totalDurationMs !== void 0) { - fields.tools_total_duration_ms = report.totalDurationMs; - } - if (report?.bundleLanguage !== void 0) { - fields.tools_bundle_language = report.bundleLanguage; - } - if (report?.perLanguageBundleFallback !== void 0) { - fields.tools_per_language_bundle_fallback = report.perLanguageBundleFallback; - } - if (toolsFeatureFlagsValid !== void 0) { - fields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } - return fields; -} async function createInitWithConfigStatusReport(config, initStatusReport, configFile, totalCacheSize, overlayBaseDatabaseStats, dependencyCachingResults) { const languages = config.languages.join(","); const paths = (config.originalUserInput.paths || []).join(","); @@ -161882,6 +161860,30 @@ async function getToolsInput(action, repositoryProperties) { return void 0; } +// src/status-report/tools-download.ts +function createInitToolsDownloadFields(report, toolsFeatureFlagsValid) { + const fields = {}; + if (report?.downloadDurationMs !== void 0) { + fields.tools_download_duration_ms = report.downloadDurationMs; + } + if (report?.extractionDurationMs !== void 0) { + fields.tools_extraction_duration_ms = report.extractionDurationMs; + } + if (report?.totalDurationMs !== void 0) { + fields.tools_total_duration_ms = report.totalDurationMs; + } + if (report?.bundleLanguage !== void 0) { + fields.tools_bundle_language = report.bundleLanguage; + } + if (report?.perLanguageBundleFallback !== void 0) { + fields.tools_per_language_bundle_fallback = report.perLanguageBundleFallback; + } + if (toolsFeatureFlagsValid !== void 0) { + fields.tools_feature_flags_valid = toolsFeatureFlagsValid; + } + return fields; +} + // src/workflow.ts var fs28 = __toESM(require("fs")); var path24 = __toESM(require("path")); diff --git a/src/init-action.ts b/src/init-action.ts index 6cf50b8c2f..79c509a5be 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -65,11 +65,11 @@ import { InitStatusReport, InitWithConfigStatusReport, createInitWithConfigStatusReport, - createInitToolsDownloadFields, createStatusReportBase, getActionsStatus, sendStatusReport, } from "./status-report"; +import { createInitToolsDownloadFields } from "./status-report/tools-download"; import { ToolsDownloadStatusReport } from "./tools-download"; import { ToolsFeature } from "./tools-features"; import { getCombinedTracerConfig } from "./tracer-config"; diff --git a/src/setup-codeql-action.ts b/src/setup-codeql-action.ts index e78ee7ec9f..4bd53e517f 100644 --- a/src/setup-codeql-action.ts +++ b/src/setup-codeql-action.ts @@ -22,11 +22,11 @@ import { ToolsSource } from "./setup-codeql"; import { ActionName, InitStatusReport, - createInitToolsDownloadFields, createStatusReportBase, getActionsStatus, sendStatusReport, } from "./status-report"; +import { createInitToolsDownloadFields } from "./status-report/tools-download"; import { ToolsDownloadStatusReport } from "./tools-download"; import { checkDiskUsage, diff --git a/src/status-report.ts b/src/status-report.ts index f6e6e16ca5..c392b51922 100644 --- a/src/status-report.ts +++ b/src/status-report.ts @@ -31,7 +31,6 @@ import type { OverlayBaseDatabaseDownloadStats } from "./overlay/caching"; import { getRepositoryNwo } from "./repository"; import type { ToolsSource } from "./setup-codeql"; import { registryBaseSchema } from "./start-proxy/types"; -import type { ToolsDownloadStatusReport } from "./tools-download"; import { ConfigurationError, getRequiredEnvParam, @@ -625,64 +624,6 @@ export interface InitWithConfigStatusReport extends InitStatusReport { config_file: string; } -/** Fields of the init status report populated when the tools source is `download`. */ -export interface InitToolsDownloadFields { - /** - * Time taken to download the bundle, in milliseconds. Not populated when the bundle is downloaded - * and extracted concurrently. - */ - tools_download_duration_ms?: ToolsDownloadStatusReport["downloadDurationMs"]; - /** - * Time taken to extract the bundle, in milliseconds. Not populated when the bundle is downloaded - * and extracted concurrently. - */ - tools_extraction_duration_ms?: ToolsDownloadStatusReport["extractionDurationMs"]; - /** - * Total time taken to make the bundle available on disk, including failed download attempts - * before a fallback, in milliseconds. - */ - tools_total_duration_ms?: ToolsDownloadStatusReport["totalDurationMs"]; - /** - * Whether the relevant tools dotcom feature flags have been misconfigured. - * Only populated if we attempt to determine the default version based on the dotcom feature flags. */ - tools_feature_flags_valid?: boolean; - /** The language of the single-language bundle that was downloaded, if any. */ - tools_bundle_language?: ToolsDownloadStatusReport["bundleLanguage"]; - /** - * Whether we tried to download a single-language bundle, but it did not exist and we fell back to - * the combined bundle. - */ - tools_per_language_bundle_fallback?: ToolsDownloadStatusReport["perLanguageBundleFallback"]; -} - -/** Converts download results to telemetry fields shared by the init and setup-codeql Actions. */ -export function createInitToolsDownloadFields( - report: ToolsDownloadStatusReport | undefined, - toolsFeatureFlagsValid: boolean | undefined, -): InitToolsDownloadFields { - const fields: InitToolsDownloadFields = {}; - if (report?.downloadDurationMs !== undefined) { - fields.tools_download_duration_ms = report.downloadDurationMs; - } - if (report?.extractionDurationMs !== undefined) { - fields.tools_extraction_duration_ms = report.extractionDurationMs; - } - if (report?.totalDurationMs !== undefined) { - fields.tools_total_duration_ms = report.totalDurationMs; - } - if (report?.bundleLanguage !== undefined) { - fields.tools_bundle_language = report.bundleLanguage; - } - if (report?.perLanguageBundleFallback !== undefined) { - fields.tools_per_language_bundle_fallback = - report.perLanguageBundleFallback; - } - if (toolsFeatureFlagsValid !== undefined) { - fields.tools_feature_flags_valid = toolsFeatureFlagsValid; - } - return fields; -} - /** * Composes a `InitWithConfigStatusReport` from the given values. * diff --git a/src/tools-download-status-report.test.ts b/src/status-report/tools-download.test.ts similarity index 94% rename from src/tools-download-status-report.test.ts rename to src/status-report/tools-download.test.ts index 36cd318b07..9b39242175 100644 --- a/src/tools-download-status-report.test.ts +++ b/src/status-report/tools-download.test.ts @@ -1,7 +1,8 @@ import test from "ava"; -import { BuiltInLanguage } from "./languages"; -import { createInitToolsDownloadFields } from "./status-report"; +import { BuiltInLanguage } from "../languages"; + +import { createInitToolsDownloadFields } from "./tools-download"; test("createInitToolsDownloadFields omits absent download data", (t) => { t.deepEqual(createInitToolsDownloadFields(undefined, undefined), {}); diff --git a/src/status-report/tools-download.ts b/src/status-report/tools-download.ts new file mode 100644 index 0000000000..d9698ffea8 --- /dev/null +++ b/src/status-report/tools-download.ts @@ -0,0 +1,59 @@ +import type { ToolsDownloadStatusReport } from "../tools-download"; + +/** Fields of the init status report populated when the tools source is `download`. */ +export interface InitToolsDownloadFields { + /** + * Time taken to download the bundle, in milliseconds. Not populated when the bundle is downloaded + * and extracted concurrently. + */ + tools_download_duration_ms?: ToolsDownloadStatusReport["downloadDurationMs"]; + /** + * Time taken to extract the bundle, in milliseconds. Not populated when the bundle is downloaded + * and extracted concurrently. + */ + tools_extraction_duration_ms?: ToolsDownloadStatusReport["extractionDurationMs"]; + /** + * Total time taken to make the bundle available on disk, including failed download attempts + * before a fallback, in milliseconds. + */ + tools_total_duration_ms?: ToolsDownloadStatusReport["totalDurationMs"]; + /** + * Whether the relevant tools dotcom feature flags have been misconfigured. + * Only populated if we attempt to determine the default version based on the dotcom feature flags. */ + tools_feature_flags_valid?: boolean; + /** The language of the single-language bundle that was downloaded, if any. */ + tools_bundle_language?: ToolsDownloadStatusReport["bundleLanguage"]; + /** + * Whether we tried to download a single-language bundle, but it did not exist and we fell back to + * the combined bundle. + */ + tools_per_language_bundle_fallback?: ToolsDownloadStatusReport["perLanguageBundleFallback"]; +} + +/** Converts download results to telemetry fields shared by the init and setup-codeql Actions. */ +export function createInitToolsDownloadFields( + report: ToolsDownloadStatusReport | undefined, + toolsFeatureFlagsValid: boolean | undefined, +): InitToolsDownloadFields { + const fields: InitToolsDownloadFields = {}; + if (report?.downloadDurationMs !== undefined) { + fields.tools_download_duration_ms = report.downloadDurationMs; + } + if (report?.extractionDurationMs !== undefined) { + fields.tools_extraction_duration_ms = report.extractionDurationMs; + } + if (report?.totalDurationMs !== undefined) { + fields.tools_total_duration_ms = report.totalDurationMs; + } + if (report?.bundleLanguage !== undefined) { + fields.tools_bundle_language = report.bundleLanguage; + } + if (report?.perLanguageBundleFallback !== undefined) { + fields.tools_per_language_bundle_fallback = + report.perLanguageBundleFallback; + } + if (toolsFeatureFlagsValid !== undefined) { + fields.tools_feature_flags_valid = toolsFeatureFlagsValid; + } + return fields; +} From ecec9b5a3756247bd2bfec7da1b6f7bb3eb92d46 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 17 Sep 2026 19:06:17 +0100 Subject: [PATCH 18/20] Share per-language telemetry fields without renaming Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- lib/entry-points.js | 15 ++++------ src/setup-codeql.test.ts | 17 +++++++----- src/setup-codeql.ts | 7 +++-- src/status-report/tools-download.test.ts | 9 ++++-- src/status-report/tools-download.ts | 35 +++++++++++------------- src/tools-download.ts | 9 ++---- 6 files changed, 45 insertions(+), 47 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index 53532f2f75..f8a7d6e76a 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -152544,7 +152544,10 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log } return { codeqlFolder: extractedBundlePath, - statusReport: bundle.kind === "per-language" ? { ...statusReport, bundleLanguage: bundle.language } : statusReport + statusReport: bundle.kind === "per-language" ? { + ...statusReport, + perLanguage: { tools_bundle_language: bundle.language } + } : statusReport }; }; function getToolcacheDestination({ logger }, source) { @@ -152695,7 +152698,7 @@ async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, temp statusReport: { ...result.statusReport, totalDurationMs: durationMsSince(startTime), - perLanguageBundleFallback: true + perLanguage: { tools_per_language_bundle_fallback: true } } }; } @@ -161862,7 +161865,7 @@ async function getToolsInput(action, repositoryProperties) { // src/status-report/tools-download.ts function createInitToolsDownloadFields(report, toolsFeatureFlagsValid) { - const fields = {}; + const fields = { ...report?.perLanguage }; if (report?.downloadDurationMs !== void 0) { fields.tools_download_duration_ms = report.downloadDurationMs; } @@ -161872,12 +161875,6 @@ function createInitToolsDownloadFields(report, toolsFeatureFlagsValid) { if (report?.totalDurationMs !== void 0) { fields.tools_total_duration_ms = report.totalDurationMs; } - if (report?.bundleLanguage !== void 0) { - fields.tools_bundle_language = report.bundleLanguage; - } - if (report?.perLanguageBundleFallback !== void 0) { - fields.tools_per_language_bundle_fallback = report.perLanguageBundleFallback; - } if (toolsFeatureFlagsValid !== void 0) { fields.tools_feature_flags_valid = toolsFeatureFlagsValid; } diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index c32c22b64a..9346c30e6a 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -557,7 +557,7 @@ for (const bundlePath of [ t.is(result.toolsVersion, "unknown"); t.is(result.toolsSource, setupCodeql.ToolsSource.Download); t.is( - result.toolsDownloadStatusReport?.bundleLanguage, + result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, bundlePath === "codeql-bundle-ruby-linux64.tar.zst" ? BuiltInLanguage.ruby : undefined, @@ -1309,11 +1309,12 @@ for (const fallback of [false, true]) { t.is(extractStub.lastCall.args[3], "token token"); t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION); t.is( - result.toolsDownloadStatusReport?.bundleLanguage, + result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, fallback ? undefined : BuiltInLanguage.java, ); t.is( - result.toolsDownloadStatusReport?.perLanguageBundleFallback, + result.toolsDownloadStatusReport?.perLanguage + ?.tools_per_language_bundle_fallback, fallback ? true : undefined, ); if (fallback) { @@ -1401,7 +1402,8 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) { t.is(result.toolsDownloadStatusReport?.downloadDurationMs, 200); t.is(result.toolsDownloadStatusReport?.extractionDurationMs, 100); t.is( - (await downloadSpy.lastCall.returnValue).statusReport.bundleLanguage, + (await downloadSpy.lastCall.returnValue).statusReport.perLanguage + ?.tools_bundle_language, bundle === "per-language" ? BuiltInLanguage.javascript : undefined, ); t.is(extractStub.callCount, bundle === "fallback" ? 2 : 1); @@ -1415,11 +1417,12 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) { bundle === "per-language" ? perLanguageURL : combinedURL, ); t.is( - result.toolsDownloadStatusReport?.bundleLanguage, + result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, bundle === "per-language" ? BuiltInLanguage.javascript : undefined, ); t.is( - result.toolsDownloadStatusReport?.perLanguageBundleFallback, + result.toolsDownloadStatusReport?.perLanguage + ?.tools_per_language_bundle_fallback, bundle === "fallback" ? true : undefined, ); t.is( @@ -1503,7 +1506,7 @@ for (const asset of [ t.is(extractStub.firstCall.args[0], url); t.is(result.toolsVersion, "9.9.9"); t.is( - result.toolsDownloadStatusReport?.bundleLanguage, + result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, BuiltInLanguage.ruby, ); t.is(path.dirname(result.codeqlFolder), tmpDir); diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 60f68a37aa..cf4050987c 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -904,7 +904,10 @@ export const downloadCodeQL = async function ( codeqlFolder: extractedBundlePath, statusReport: bundle.kind === "per-language" - ? { ...statusReport, bundleLanguage: bundle.language } + ? { + ...statusReport, + perLanguage: { tools_bundle_language: bundle.language }, + } : statusReport, }; }; @@ -1171,7 +1174,7 @@ export async function downloadCodeQLBundle( statusReport: { ...result.statusReport, totalDurationMs: util.durationMsSince(startTime), - perLanguageBundleFallback: true, + perLanguage: { tools_per_language_bundle_fallback: true }, }, }; } diff --git a/src/status-report/tools-download.test.ts b/src/status-report/tools-download.test.ts index 9b39242175..856bf882f6 100644 --- a/src/status-report/tools-download.test.ts +++ b/src/status-report/tools-download.test.ts @@ -24,7 +24,10 @@ test("createInitToolsDownloadFields reports only the total for a streaming downl test("createInitToolsDownloadFields preserves per-language metadata", (t) => { t.deepEqual( createInitToolsDownloadFields( - { totalDurationMs: 300, bundleLanguage: BuiltInLanguage.java }, + { + totalDurationMs: 300, + perLanguage: { tools_bundle_language: BuiltInLanguage.java }, + }, true, ), { @@ -42,7 +45,7 @@ test("createInitToolsDownloadFields preserves fallback and per-attempt timings", downloadDurationMs: 200, extractionDurationMs: 100, totalDurationMs: 1000, - perLanguageBundleFallback: true, + perLanguage: { tools_per_language_bundle_fallback: true }, }, undefined, ), @@ -62,7 +65,7 @@ test("createInitToolsDownloadFields preserves zero durations and false flags", ( downloadDurationMs: 0, extractionDurationMs: 0, totalDurationMs: 0, - perLanguageBundleFallback: false, + perLanguage: { tools_per_language_bundle_fallback: false }, }, false, ), diff --git a/src/status-report/tools-download.ts b/src/status-report/tools-download.ts index d9698ffea8..a5f7dffbf8 100644 --- a/src/status-report/tools-download.ts +++ b/src/status-report/tools-download.ts @@ -1,33 +1,37 @@ import type { ToolsDownloadStatusReport } from "../tools-download"; +/** Telemetry describing per-language bundle downloads. */ +export interface PerLanguageToolsStatusReport { + /** The language of the single-language bundle that was downloaded, if any. */ + tools_bundle_language?: string; + /** + * Whether we tried to download a single-language bundle, but it did not exist and we fell back to + * the combined bundle. + */ + tools_per_language_bundle_fallback?: boolean; +} + /** Fields of the init status report populated when the tools source is `download`. */ -export interface InitToolsDownloadFields { +export interface InitToolsDownloadFields extends PerLanguageToolsStatusReport { /** * Time taken to download the bundle, in milliseconds. Not populated when the bundle is downloaded * and extracted concurrently. */ - tools_download_duration_ms?: ToolsDownloadStatusReport["downloadDurationMs"]; + tools_download_duration_ms?: number; /** * Time taken to extract the bundle, in milliseconds. Not populated when the bundle is downloaded * and extracted concurrently. */ - tools_extraction_duration_ms?: ToolsDownloadStatusReport["extractionDurationMs"]; + tools_extraction_duration_ms?: number; /** * Total time taken to make the bundle available on disk, including failed download attempts * before a fallback, in milliseconds. */ - tools_total_duration_ms?: ToolsDownloadStatusReport["totalDurationMs"]; + tools_total_duration_ms?: number; /** * Whether the relevant tools dotcom feature flags have been misconfigured. * Only populated if we attempt to determine the default version based on the dotcom feature flags. */ tools_feature_flags_valid?: boolean; - /** The language of the single-language bundle that was downloaded, if any. */ - tools_bundle_language?: ToolsDownloadStatusReport["bundleLanguage"]; - /** - * Whether we tried to download a single-language bundle, but it did not exist and we fell back to - * the combined bundle. - */ - tools_per_language_bundle_fallback?: ToolsDownloadStatusReport["perLanguageBundleFallback"]; } /** Converts download results to telemetry fields shared by the init and setup-codeql Actions. */ @@ -35,7 +39,7 @@ export function createInitToolsDownloadFields( report: ToolsDownloadStatusReport | undefined, toolsFeatureFlagsValid: boolean | undefined, ): InitToolsDownloadFields { - const fields: InitToolsDownloadFields = {}; + const fields: InitToolsDownloadFields = { ...report?.perLanguage }; if (report?.downloadDurationMs !== undefined) { fields.tools_download_duration_ms = report.downloadDurationMs; } @@ -45,13 +49,6 @@ export function createInitToolsDownloadFields( if (report?.totalDurationMs !== undefined) { fields.tools_total_duration_ms = report.totalDurationMs; } - if (report?.bundleLanguage !== undefined) { - fields.tools_bundle_language = report.bundleLanguage; - } - if (report?.perLanguageBundleFallback !== undefined) { - fields.tools_per_language_bundle_fallback = - report.perLanguageBundleFallback; - } if (toolsFeatureFlagsValid !== undefined) { fields.tools_feature_flags_valid = toolsFeatureFlagsValid; } diff --git a/src/tools-download.ts b/src/tools-download.ts index 61501557c5..a92dad0acf 100644 --- a/src/tools-download.ts +++ b/src/tools-download.ts @@ -13,6 +13,7 @@ import * as semver from "semver"; import { ActionState } from "./action-common"; import { ActionsEnvVars, getEnv, ReadOnlyEnv } from "./environment"; import { formatDuration, Logger } from "./logging"; +import type { PerLanguageToolsStatusReport } from "./status-report/tools-download"; import * as tar from "./tar"; import { asHTTPError, @@ -55,13 +56,7 @@ export type ToolsDownloadStatusReport = { * before a fallback, in milliseconds. */ totalDurationMs: number; - /** The language of the single-language bundle that was downloaded, if any. */ - bundleLanguage?: string; - /** - * Whether we tried to download a single-language bundle, but it did not exist and we fell back to - * the combined bundle. - */ - perLanguageBundleFallback?: boolean; + perLanguage?: PerLanguageToolsStatusReport; }; export async function downloadAndExtract( From f18f3536f13ef44ab98c9ef15f8aa05c7f6ac4ae Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 17 Sep 2026 19:07:06 +0100 Subject: [PATCH 19/20] Describe the bundle URL resolver Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/setup-codeql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index cf4050987c..648435d805 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -768,7 +768,7 @@ export async function getCodeQLSource( }, ); - // Resolve both bundle variants against the same release and repository lookup order. + // Resolves the combined or per-language bundle URL for the requested release. const resolveBundleURL = (language?: BuiltInLanguage) => getCodeQLBundleDownloadURL( bundleTagName, From 07fa87d33359d182be54e4da4bf41664595e3042 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Thu, 17 Sep 2026 19:08:05 +0100 Subject: [PATCH 20/20] Clarify the latest-nightly eligibility exception Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/per-language-bundles.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts index ea715aeb1c..f4e46403db 100644 --- a/src/per-language-bundles.ts +++ b/src/per-language-bundles.ts @@ -103,7 +103,7 @@ export async function getPerLanguageBundleLanguage( } // Check whether per-language bundles are published for the requested CLI version. - // Skip this for the latest nightly, whose tag contains a date rather than a CLI version. + // Latest-nightly selection skips this release-version check, but not the other eligibility checks. if (!isLatestNightly) { if (cliVersion === undefined) { return explain("the requested CLI version is unknown");