Merge pull request #4516 from github/cklin/fix-cli-pack-test

Pin CLI integration test pack fixtures

Author: cklin

extensions/ql-vscode/test/vscode-tests/cli-integration/pack-fixtures.ts

@@ -0,0 +1,84 @@
+import type { CodeQLCliServer } from "../../../src/codeql-cli/cli";
+
+// Most of these versions were published by CLI 2.26.2 and are the newest
+// versions whose manifests can be read by every supported CLI. Older test
+// fixtures use their latest known compatible releases.
+const COMPATIBLE_PACK_VERSIONS: Readonly<Record<string, string>> = {
+  "codeql/actions-queries": "0.6.32",
+  "codeql/concepts": "0.0.28",
+  "codeql/cpp-queries": "1.8.0",
+  "codeql/csharp-queries": "1.9.0",
+  "codeql/csharp-solorigate-queries": "1.0.1",
+  "codeql/dataflow": "2.1.10",
+  "codeql/go-queries": "1.6.7",
+  "codeql/java-queries": "1.11.7",
+  "codeql/javascript-all": "2.8.2",
+  "codeql/javascript-queries": "2.4.2",
+  "codeql/mad": "1.0.54",
+  "codeql/python-queries": "1.8.7",
+  "codeql/regex": "1.0.54",
+  "codeql/ruby-queries": "1.6.7",
+  "codeql/rust-queries": "0.1.39",
+  "codeql/ssa": "2.0.30",
+  "codeql/threat-models": "1.0.54",
+  "codeql/tutorial": "1.0.54",
+  "codeql/typetracking": "2.0.38",
+  "codeql/util": "2.0.41",
+  "codeql/xml": "1.0.54",
+  "codeql/yaml": "1.0.54",
+};
+
+export const AUTHENTICATION_TEST_PACK = "codeql/tutorial@0.0.11";
+
+export const QUICK_QUERY_PACK_DEPENDENCIES = [
+  "codeql/concepts",
+  "codeql/dataflow",
+  "codeql/javascript-all",
+  "codeql/mad",
+  "codeql/regex",
+  "codeql/ssa",
+  "codeql/threat-models",
+  "codeql/tutorial",
+  "codeql/typetracking",
+  "codeql/util",
+  "codeql/xml",
+  "codeql/yaml",
+] as const;
+
+export function getCompatiblePackVersion(packName: string): string {
+  const version = COMPATIBLE_PACK_VERSIONS[packName];
+  if (!version) {
+    throw new Error(
+      `No compatible CLI test version configured for ${packName}`,
+    );
+  }
+  return version;
+}
+
+export function getCompatiblePackSpec(packName: string): string {
+  return `${packName}@${getCompatiblePackVersion(packName)}`;
+}
+
+export function getCompatiblePackLock(packNames: readonly string[]) {
+  return {
+    lockVersion: "1.0.0",
+    dependencies: Object.fromEntries(
+      packNames.map((packName) => [
+        packName,
+        { version: getCompatiblePackVersion(packName) },
+      ]),
+    ),
+    compiled: false,
+  };
+}
+
+export function useCompatiblePackDownloads(
+  cli: CodeQLCliServer,
+): jest.SpiedFunction<CodeQLCliServer["packDownload"]> {
+  const packDownload = cli.packDownload.bind(cli);
+  return jest
+    .spyOn(cli, "packDownload")
+    .mockImplementation((packs, token) =>
+      packDownload(packs.map(getCompatiblePackSpec), token),
+    );
+}

extensions/ql-vscode/test/vscode-tests/cli-integration/packaging/packaging.test.ts

@@ -18,6 +18,8 @@ import type {
 import * as workspaceFolders from "../../../../src/common/vscode/workspace-folders";
 import { getOnDiskWorkspaceFolders } from "../../../../src/common/vscode/workspace-folders";
 import { pathsEqual } from "../../../../src/common/files";
+import { PACKS_BY_QUERY_LANGUAGE } from "../../../../src/common/query-language";
+import { useCompatiblePackDownloads } from "../pack-fixtures";
 
 describe("Packaging commands", () => {
   let cli: CodeQLCliServer;
@@ -50,11 +52,15 @@ describe("Packaging commands", () => {
   });
 
   it("should download all core query packs", async () => {
+    const packDownloadSpy = useCompatiblePackDownloads(cli);
     quickPickSpy.mockResolvedValue(
       mockedQuickPickItem("Download all core query packs"),
     );
 
     await handleDownloadPacks(cli, progress);
+    expect(packDownloadSpy).toHaveBeenCalledWith(
+      Object.values(PACKS_BY_QUERY_LANGUAGE).flat(),
+    );
     expect(showAndLogExceptionWithTelemetrySpy).not.toHaveBeenCalled();
     expect(showAndLogInformationMessageSpy).toHaveBeenCalledWith(
       expect.anything(),
@@ -63,12 +69,16 @@ describe("Packaging commands", () => {
   });
 
   it("should download valid user-specified pack", async () => {
+    const packDownloadSpy = useCompatiblePackDownloads(cli);
     quickPickSpy.mockResolvedValue(
       mockedQuickPickItem("Download custom specified pack"),
     );
     inputBoxSpy.mockResolvedValue("codeql/csharp-solorigate-queries");
 
     await handleDownloadPacks(cli, progress);
+    expect(packDownloadSpy).toHaveBeenCalledWith([
+      "codeql/csharp-solorigate-queries",
+    ]);
     expect(showAndLogExceptionWithTelemetrySpy).not.toHaveBeenCalled();
     expect(showAndLogInformationMessageSpy).toHaveBeenCalledWith(
       expect.anything(),

extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts

@@ -38,6 +38,12 @@ import type {
 import type { ProgressCallback } from "../../../src/common/vscode/progress";
 import { withDebugController } from "./debugger/debug-controller";
 import { getDataFolderFilePath } from "./utils";
+import {
+  getCompatiblePackLock,
+  getCompatiblePackSpec,
+  getCompatiblePackVersion,
+  QUICK_QUERY_PACK_DEPENDENCIES,
+} from "./pack-fixtures";
 
 const simpleQueryPath = getDataFolderFilePath("debugger/simple-query.ql");
 
@@ -241,6 +247,16 @@ describeWithCodeQL()("Queries", () => {
 
   describe("quick query", () => {
     it("should create a quick query", async () => {
+      jest.spyOn(cli, "packInstall").mockImplementation(async () => {
+        await cli.packDownload(
+          QUICK_QUERY_PACK_DEPENDENCIES.map(getCompatiblePackSpec),
+        );
+        writeFileSync(
+          qlpackLockFile,
+          dump(getCompatiblePackLock(QUICK_QUERY_PACK_DEPENDENCIES)),
+        );
+      });
+
       await queryServerCommandManager.execute("codeQL.quickQuery");
 
       // should have created the quick query file and query pack file
@@ -256,8 +272,8 @@ describeWithCodeQL()("Queries", () => {
         ? qlpackLockFile
         : oldQlpackLockFile;
       const qlpackLock: any = await load(readFileSync(packFileToUse, "utf8"));
-      expect(!!qlpackLock.dependencies["codeql/javascript-all"].version).toBe(
-        true,
+      expect(qlpackLock.dependencies["codeql/javascript-all"].version).toBe(
+        getCompatiblePackVersion("codeql/javascript-all"),
       );
     });
 

extensions/ql-vscode/test/vscode-tests/cli-integration/run-cli.test.ts

@@ -17,6 +17,7 @@ import { getActivatedExtension } from "../global.helper";
 import type { BaseLogger } from "../../../src/common/logging";
 import { getQlPackForDbscheme } from "../../../src/databases/qlpack";
 import { languageToDbScheme } from "../../../src/common/query-language";
+import { AUTHENTICATION_TEST_PACK } from "./pack-fixtures";
 
 /**
  * Perform proper integration tests by running the CLI
@@ -140,7 +141,7 @@ describe("Use cli", () => {
           .spyOn(authentication, "getSession")
           .mockResolvedValue(undefined);
 
-        await cli.packDownload(["codeql/tutorial@0.0.11"]);
+        await cli.packDownload([AUTHENTICATION_TEST_PACK]);
         expect(getSession).toHaveBeenCalledTimes(1);
         expect(getSession).toHaveBeenCalledWith(
           "github",
@@ -167,7 +168,7 @@ describe("Use cli", () => {
             scopes: ["read:packages"],
           });
 
-        await cli.packDownload(["codeql/tutorial@0.0.11"]);
+        await cli.packDownload([AUTHENTICATION_TEST_PACK]);
         expect(getSession).toHaveBeenCalledTimes(2);
         expect(getSession).toHaveBeenCalledWith(
           "github",

extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/code-scanning-pack.test.ts

@@ -5,6 +5,7 @@ import { QueryLanguage } from "../../../../src/common/query-language";
 import { ExtensionApp } from "../../../../src/common/vscode/extension-app";
 import { resolveCodeScanningQueryPack } from "../../../../src/variant-analysis/code-scanning-pack";
 import { getActivatedExtension } from "../../global.helper";
+import { useCompatiblePackDownloads } from "../pack-fixtures";
 
 describe("Code Scanning pack", () => {
   let cli: CodeQLCliServer;
@@ -17,12 +18,18 @@ describe("Code Scanning pack", () => {
   });
 
   it("should download pack for correct language and identify problem queries", async () => {
+    const packDownloadSpy = useCompatiblePackDownloads(cli);
+
     const pack = await resolveCodeScanningQueryPack(
       app.logger,
       cli,
       QueryLanguage.Javascript,
       new CancellationTokenSource().token,
     );
+    expect(packDownloadSpy).toHaveBeenCalledWith(
+      ["codeql/javascript-queries"],
+      expect.anything(),
+    );
     // Should include queries. Just check that at least one known query exists.
     // It doesn't particularly matter which query we check for.
     expect(