Remote queries: Handle nested queries

This change allows remote queries to run a query from a directory that
is not in the root of the qlpack.

The change is the following:

1. walk up the directory hierarchy to check for a non-local qlpack.yml
2. Copy over the files as before, but keep track of the relative
   location of the query compared to the location of the qlpack.yml.
3. Change the defaultSuite of the qlpack.yml so that _only_ this query
   is run as part of the default query.

Also, this adds a new integration test to ensure the nested query is
packaged appropriately.

Author: aeisenberg

extensions/ql-vscode/src/run-remote-query.ts

@@ -11,13 +11,19 @@ import { getRemoteControllerRepo, getRemoteRepositoryLists, setRemoteControllerR
 import { tmpDir } from './run-queries';
 import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { OctokitResponse } from '@octokit/types/dist-types';
-
 interface Config {
   repositories: string[];
   ref?: string;
   language?: string;
 }
 
+interface QlPack {
+  name: string;
+  version: string;
+  dependencies: { [key: string]: string };
+  defaultSuite?: Record<string, unknown>;
+  defaultSuiteFile?: Record<string, unknown>;
+}
 interface RepoListQuickPickItem extends QuickPickItem {
   repoList: string[];
 }
@@ -89,13 +95,9 @@ async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: stri
   base64Pack: string,
   language: string
 }> {
-  const originalPackRoot = path.dirname(queryFile);
-  // TODO this assumes that the qlpack.yml is in the same directory as the query file, but in reality,
-  // the file could be in a parent directory.
-  const targetQueryFileName = path.join(queryPackDir, path.basename(queryFile));
-
-  // the server is expecting the query file to be named `query.ql`. Rename it here.
-  const renamedQueryFile = path.join(queryPackDir, 'query.ql');
+  const originalPackRoot = await findPackRoot(queryFile);
+  const packRelativePath = path.relative(originalPackRoot, queryFile);
+  const targetQueryFileName = path.join(queryPackDir, packRelativePath);
 
   let language: string | undefined;
   if (await fs.pathExists(path.join(originalPackRoot, 'qlpack.yml'))) {
@@ -138,8 +140,6 @@ async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: stri
 
     // copy only the query file to the query pack directory
     // and generate a synthetic query pack
-    // TODO this has a limitation that query packs inside of a workspace will not resolve its peer dependencies.
-    // Something to work on later. For now, we will only support query packs that are not in a workspace.
     void logger.log(`Copying ${queryFile} to ${queryPackDir}`);
     await fs.copy(queryFile, targetQueryFileName);
     void logger.log('Generating synthetic query pack');
@@ -156,7 +156,8 @@ async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: stri
     throw new UserCancellationException('Could not determine language.');
   }
 
-  await fs.rename(targetQueryFileName, renamedQueryFile);
+  // fix the default suite of the query pack dir
+  await fixDefaultSuite(queryPackDir, packRelativePath);
 
   const bundlePath = await getPackedBundlePath(queryPackDir);
   void logger.log(`Compiling and bundling query pack from ${queryPackDir} to ${bundlePath}. (This may take a while.)`);
@@ -189,6 +190,21 @@ async function ensureQueryPackName(queryPackDir: string) {
   }
 }
 
+async function findPackRoot(queryFile: string): Promise<string> {
+  // recursively find the directory containing qlpack.yml
+  let dir = path.dirname(queryFile);
+  while (!(await fs.pathExists(path.join(dir, 'qlpack.yml')))) {
+    dir = path.dirname(dir);
+    if (dir === '/') {
+      // there is no qlpack.yml in this direcory or any parent directory.
+      // just use the query file's directory as the pack root.
+      return path.dirname(queryFile);
+    }
+  }
+
+  return dir;
+}
+
 async function createRemoteQueriesTempDirectory() {
   const remoteQueryDir = await tmp.dir({ dir: tmpDir.name, unsafeCleanup: true });
   const queryPackDir = path.join(remoteQueryDir.path, 'query-pack');
@@ -413,3 +429,23 @@ export async function attemptRerun(
     void showAndLogErrorMessage(error);
   }
 }
+
+/**
+ * Updates the default suite of the query pack. This is used to ensure
+ * only the specified query is run.
+ *
+ * @param queryPackDir The directory containing the query pack
+ * @param packRelativePath The relative path to the query pack from the root of the query pack
+ */
+async function fixDefaultSuite(queryPackDir: string, packRelativePath: string): Promise<void> {
+  const packPath = path.join(queryPackDir, 'qlpack.yml');
+  const qlpack = (await yaml.safeLoad(await fs.readFile(packPath, 'utf8'))) as QlPack;
+  delete qlpack.defaultSuite;
+  delete qlpack.defaultSuiteFile;
+
+  qlpack.defaultSuite = {
+    description: 'Query suite for remote query',
+    query: packRelativePath
+  };
+  await fs.writeFile(packPath, yaml.safeDump(qlpack));
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/not-in-pack.ql

@@ -0,0 +1,2 @@
+// This file should not be included the remote query pack.
+select 1

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/otherfolder/lib.qll

@@ -0,0 +1,3 @@
+int number() {
+  result = 1
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/qlpack.yml

@@ -0,0 +1,4 @@
+name: github/remote-query-pack
+version: 0.0.0
+dependencies:
+  codeql/javascript-all: '*'

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack-nested/subfolder/in-pack.ql

@@ -0,0 +1,4 @@
+import javascript
+import otherfolder.lib
+
+select number()

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/qlpack.yml

@@ -1,5 +1,4 @@
 name: github/remote-query-pack
 version: 0.0.0
-extractor: javascript
 dependencies:
   codeql/javascript-all: '*'

extensions/ql-vscode/src/vscode-tests/cli-integration/run-remote-query.test.ts

@@ -13,7 +13,7 @@ import { CodeQLExtensionInterface } from '../../extension';
 import { setRemoteControllerRepo, setRemoteRepositoryLists } from '../../config';
 import { UserCancellationException } from '../../commandRunner';
 
-describe('Remote queries', function() {
+describe.only('Remote queries', function() {
   const baseDir = path.join(__dirname, '../../../src/vscode-tests/cli-integration');
 
   let sandbox: sinon.SinonSandbox;
@@ -80,8 +80,7 @@ describe('Remote queries', function() {
     const queryPackDir = path.join(queryPackRootDir, 'query-pack');
     printDirectoryContents(queryPackDir);
 
-    // in-pack.ql renamed to query.ql
-    expect(fs.existsSync(path.join(queryPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
     expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.true;
     expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
 
@@ -96,7 +95,7 @@ describe('Remote queries', function() {
     const compiledPackDir = path.join(queryPackDir, '.codeql/pack/github/remote-query-pack/0.0.0/');
     printDirectoryContents(compiledPackDir);
 
-    expect(fs.existsSync(path.join(compiledPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
     expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.true;
     expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
     // depending on the cli version, we should have one of these files
@@ -105,6 +104,7 @@ describe('Remote queries', function() {
       fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
     ).to.be.true;
     expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
+    verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', 'github/remote-query-pack', '0.0.0');
 
     // dependencies
     const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
@@ -129,8 +129,8 @@ describe('Remote queries', function() {
 
     const queryPackDir = path.join(queryPackRootDir, 'query-pack');
     printDirectoryContents(queryPackDir);
-    // in-pack.ql renamed to query.ql
-    expect(fs.existsSync(path.join(queryPackDir, 'query.ql'))).to.be.true;
+
+    expect(fs.existsSync(path.join(queryPackDir, 'in-pack.ql'))).to.be.true;
     expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
     // depending on the cli version, we should have one of these files
     expect(
@@ -143,8 +143,10 @@ describe('Remote queries', function() {
     // the compiled pack
     const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
     printDirectoryContents(compiledPackDir);
-    expect(fs.existsSync(path.join(compiledPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'in-pack.ql'))).to.be.true;
     expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
+    verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'in-pack.ql', 'codeql-remote/query', '0.0.0');
+
     // depending on the cli version, we should have one of these files
     expect(
       fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
@@ -165,6 +167,60 @@ describe('Remote queries', function() {
     expect(packNames).to.deep.equal(['javascript-all', 'javascript-upgrades']);
   });
 
+  it('should run a remote query that is nested inside a qlpack', async () => {
+    const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
+
+    const queryPackRootDir = (await runRemoteQuery(cli, credentials, fileUri, true, progress, token))!;
+
+    // to retrieve the list of repositories
+    expect(showQuickPickSpy).to.have.been.calledOnce;
+
+    // check a few files that we know should exist and others that we know should not
+
+    // the tarball to deliver to the server
+    printDirectoryContents(queryPackRootDir);
+    expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
+
+    const queryPackDir = path.join(queryPackRootDir, 'query-pack');
+    printDirectoryContents(queryPackDir);
+
+    expect(fs.existsSync(path.join(queryPackDir, 'subfolder/in-pack.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'otherfolder/lib.qll'))).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
+
+    // the compiled pack
+    const compiledPackDir = path.join(queryPackDir, '.codeql/pack/github/remote-query-pack/0.0.0/');
+    printDirectoryContents(compiledPackDir);
+    expect(fs.existsSync(path.join(compiledPackDir, 'otherfolder/lib.qll'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'subfolder/in-pack.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
+    verifyQlPack(path.join(compiledPackDir, 'qlpack.yml'), 'subfolder/in-pack.ql', 'github/remote-query-pack', '0.0.0');
+
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
+    // should have generated a correct qlpack file
+    const qlpackContents: any = yaml.safeLoad(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
+    expect(qlpackContents.name).to.equal('github/remote-query-pack');
+    expect(qlpackContents.version).to.equal('0.0.0');
+    expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
+
+    // dependencies
+    const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
+    printDirectoryContents(libraryDir);
+    const packNames = fs.readdirSync(libraryDir).sort();
+    expect(packNames).to.deep.equal(['javascript-all', 'javascript-upgrades']);
+  });
+
   it('should cancel a run before uploading', async () => {
     const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
 
@@ -180,15 +236,37 @@ describe('Remote queries', function() {
     }
   });
 
+  function verifyQlPack(qlpackPath: string, queryPath: string, packName: string, packVersion: string) {
+    const qlPack = yaml.safeLoad(fs.readFileSync(qlpackPath, 'utf8'));
+
+    // don't check the build metadata since it is variable
+    delete (qlPack as any).buildMetadata;
+
+    expect(qlPack).to.deep.equal({
+      name: packName,
+      version: packVersion,
+      dependencies: {
+        'codeql/javascript-all': '*',
+      },
+      library: false,
+      defaultSuite: [{
+        description: 'Query suite for remote query',
+        query: queryPath,
+      }]
+    });
+  }
+
   function getFile(file: string): Uri {
     return Uri.file(path.join(baseDir, file));
   }
 
   function printDirectoryContents(dir: string) {
-    console.log(`DIR ${dir}`);
-    if (!fs.existsSync(dir)) {
-      console.log(`DIR ${dir} does not exist`);
-    }
-    fs.readdirSync(dir).sort().forEach(f => console.log(`  ${f}`));
+    dir;
+    // uncomment to debug
+    // console.log(`DIR ${dir}`);
+    // if (!fs.existsSync(dir)) {
+    //   console.log(`DIR ${dir} does not exist`);
+    // }
+    // fs.readdirSync(dir).sort().forEach(f => console.log(`  ${f}`));
   }
 });