Merge pull request #1111 from github/aeisenberg/graph-viewer

More work on the graph viewer

Author: aeisenberg

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '14.14.0'
+          node-version: '16.13.0'
 
       - name: Install dependencies
         working-directory: extensions/ql-vscode
@@ -82,7 +82,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '14.14.0'
+          node-version: '16.13.0'
 
       - name: Install dependencies
         working-directory: extensions/ql-vscode
@@ -147,7 +147,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '14.14.0'
+          node-version: '16.13.0'
 
       - name: Install dependencies
         working-directory: extensions/ql-vscode

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '10.18.1'
+          node-version: '16.13.0'
 
       - name: Install dependencies
         run: |

extensions/ql-vscode/package-lock.json

@@ -48,6 +48,7 @@
     "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQL.viewAst",
+    "onCommand:codeQL.viewCfg",
     "onCommand:codeQL.openReferencedFile",
     "onCommand:codeQL.previewQueryHelp",
     "onCommand:codeQL.chooseDatabaseFolder",
@@ -374,6 +375,10 @@
         "command": "codeQL.viewAst",
         "title": "CodeQL: View AST"
       },
+      {
+        "command": "codeQL.viewCfg",
+        "title": "CodeQL: View CFG"
+      },
       {
         "command": "codeQL.upgradeCurrentDatabase",
         "title": "CodeQL: Upgrade Current Database"
@@ -743,6 +748,11 @@
           "group": "9_qlCommands",
           "when": "resourceScheme == codeql-zip-archive && !explorerResourceIsFolder && !listMultiSelection"
         },
+        {
+          "command": "codeQL.viewCfg",
+          "group": "9_qlCommands",
+          "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
+        },
         {
           "command": "codeQL.runQueries",
           "group": "9_qlCommands",
@@ -804,6 +814,10 @@
           "command": "codeQL.viewAst",
           "when": "resourceScheme == codeql-zip-archive"
         },
+        {
+          "command": "codeQL.viewCfg",
+          "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
@@ -950,6 +964,10 @@
           "command": "codeQL.viewAst",
           "when": "resourceScheme == codeql-zip-archive"
         },
+        {
+          "command": "codeQL.viewCfg",
+          "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
+        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
@@ -1023,6 +1041,8 @@
     "@primer/react": "^34.3.0",
     "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
+    "d3": "^6.3.1",
+    "d3-graphviz": "^2.6.1",
     "fs-extra": "^9.0.1",
     "glob-promise": "^3.4.0",
     "js-yaml": "^3.14.0",
@@ -1054,6 +1074,8 @@
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
     "@types/del": "^4.0.0",
+    "@types/d3": "^6.2.0",
+    "@types/d3-graphviz": "^2.6.6",
     "@types/fs-extra": "^9.0.6",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",

extensions/ql-vscode/package.json

@@ -0,0 +1,15 @@
+/**
+ * The d3 library is designed to work in both the browser and
+ * node. Consequently their typings files refer to both node
+ * types like `Buffer` (which don't exist in the browser), and browser
+ * types like `Blob` (which don't exist in node). Instead of sticking
+ * all of `dom` in `compilerOptions.lib`, it suffices just to put in a
+ * stub definition of the affected types so that compilation
+ * succeeds.
+ */
+
+declare type RequestInit = Record<string, unknown>;
+declare type ElementTagNameMap = any;
+declare type NodeListOf<T> = Record<string, T>;
+declare type Node = Record<string, unknown>;
+declare type XMLDocument = Record<string, unknown>;

extensions/ql-vscode/src/additional-typings.d.ts

@@ -1,5 +1,6 @@
 import * as cpp from 'child-process-promise';
 import * as child_process from 'child_process';
+import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
 import { SemVer } from 'semver';
@@ -17,7 +18,7 @@ import { QueryMetadata, SortDirection } from './pure/interface-types';
 import { Logger, ProgressReporter } from './logging';
 import { CompilationMessage } from './pure/messages';
 import { sarifParser } from './sarif-parser';
-import { dbSchemeToLanguage } from './helpers';
+import { dbSchemeToLanguage, walkDirectory } from './helpers';
 
 /**
  * The version of the SARIF format that we are using.
@@ -687,20 +688,13 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
   }
 
-  async runInterpretCommand(format: string, metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo) {
+  async runInterpretCommand(format: string, additonalArgs: string[], metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo) {
     const args = [
       '--output', interpretedResultsPath,
       '--format', format,
       // Forward all of the query metadata.
       ...Object.entries(metadata).map(([key, value]) => `-t=${key}=${value}`)
-    ];
-    if (format == SARIF_FORMAT) {
-      // TODO: This flag means that we don't group interpreted results
-      // by primary location. We may want to revisit whether we call
-      // interpretation with and without this flag, or do some
-      // grouping client-side.
-      args.push('--no-group-results');
-    }
+    ].concat(additonalArgs);
     if (sourceInfo !== undefined) {
       args.push(
         '--source-archive', sourceInfo.sourceArchive,
@@ -722,13 +716,47 @@ export class CodeQLCliServer implements Disposable {
     await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, 'Interpreting query results');
   }
 
-  async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
-    await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
+  async interpretBqrsSarif(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+    const additionalArgs = [
+      // TODO: This flag means that we don't group interpreted results
+      // by primary location. We may want to revisit whether we call
+      // interpretation with and without this flag, or do some
+      // grouping client-side.
+      '--no-group-results'
+    ];
+
+    await this.runInterpretCommand(SARIF_FORMAT, additionalArgs, metadata, resultsPath, interpretedResultsPath, sourceInfo);
     return await sarifParser(interpretedResultsPath);
   }
 
+  // Warning: this function is untenable for large dot files,
+  async readDotFiles(dir: string): Promise<string[]> {
+    const dotFiles: Promise<string>[] = [];
+    for await (const file of walkDirectory(dir)) {
+      if (file.endsWith('.dot')) {
+        dotFiles.push(fs.readFile(file, 'utf8'));
+      }
+    }
+    return Promise.all(dotFiles);
+  }
+
+  async interpretBqrsGraph(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<string[]> {
+    const additionalArgs = sourceInfo
+      ? ['--dot-location-url-format', 'file://' + sourceInfo.sourceLocationPrefix + '{path}:{start:line}:{start:column}:{end:line}:{end:column}']
+      : [];
+
+    await this.runInterpretCommand('dot', additionalArgs, metadata, resultsPath, interpretedResultsPath, sourceInfo);
+
+    try {
+      const dot = await this.readDotFiles(interpretedResultsPath);
+      return dot;
+    } catch (err) {
+      throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`);
+    }
+  }
+
   async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
-    await this.runInterpretCommand(CSV_FORMAT, metadata, resultsPath, csvPath, sourceInfo);
+    await this.runInterpretCommand(CSV_FORMAT, [], metadata, resultsPath, csvPath, sourceInfo);
   }
 
   async sortBqrs(resultsPath: string, sortedResultsPath: string, resultSet: string, sortKeys: number[], sortDirections: SortDirection[]): Promise<void> {
@@ -1224,9 +1252,9 @@ export class CliVersionConstraint {
 
   /**
    * CLI version where the `--evaluator-log` and related options to the query server were introduced,
-   * on a per-query server basis. 
+   * on a per-query server basis.
    */
-   public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer('2.8.2');
+  public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer('2.8.2');
 
   constructor(private readonly cli: CodeQLCliServer) {
     /**/

extensions/ql-vscode/src/cli.ts

@@ -95,8 +95,8 @@ const CUSTOM_LOG_DIRECTORY_SETTING = new Setting('customLogDirectory', RUNNING_Q
 
 /** When these settings change, the running query server should be restarted. */
 const QUERY_SERVER_RESTARTING_SETTINGS = [
-  NUMBER_OF_THREADS_SETTING, SAVE_CACHE_SETTING, CACHE_SIZE_SETTING, MEMORY_SETTING, 
-  DEBUG_SETTING, CUSTOM_LOG_DIRECTORY_SETTING, 
+  NUMBER_OF_THREADS_SETTING, SAVE_CACHE_SETTING, CACHE_SIZE_SETTING, MEMORY_SETTING,
+  DEBUG_SETTING, CUSTOM_LOG_DIRECTORY_SETTING,
 ];
 
 export interface QueryServerConfig {

extensions/ql-vscode/src/config.ts

@@ -2,6 +2,7 @@ export enum KeyType {
   DefinitionQuery = 'DefinitionQuery',
   ReferenceQuery = 'ReferenceQuery',
   PrintAstQuery = 'PrintAstQuery',
+  PrintCfgQuery = 'PrintCfgQuery',
 }
 
 export function tagOfKeyType(keyType: KeyType): string {
@@ -12,6 +13,8 @@ export function tagOfKeyType(keyType: KeyType): string {
       return 'ide-contextual-queries/local-references';
     case KeyType.PrintAstQuery:
       return 'ide-contextual-queries/print-ast';
+    case KeyType.PrintCfgQuery:
+      return 'ide-contextual-queries/print-cfg';
   }
 }
 
@@ -23,6 +26,8 @@ export function nameOfKeyType(keyType: KeyType): string {
       return 'references';
     case KeyType.PrintAstQuery:
       return 'print AST';
+    case KeyType.PrintCfgQuery:
+      return 'print CFG';
   }
 }
 
@@ -32,6 +37,7 @@ export function kindOfKeyType(keyType: KeyType): string {
     case KeyType.ReferenceQuery:
       return 'definitions';
     case KeyType.PrintAstQuery:
+    case KeyType.PrintCfgQuery:
       return 'graph';
   }
 }

extensions/ql-vscode/src/contextual/keyType.ts

@@ -230,3 +230,62 @@ export class TemplatePrintAstProvider {
     };
   }
 }
+
+export class TemplatePrintCfgProvider {
+  private cache: CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private dbm: DatabaseManager,
+  ) {
+    this.cache = new CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>(this.getCfgUri.bind(this));
+  }
+
+  async provideCfgUri(document?: TextDocument): Promise<[Uri, messages.TemplateDefinitions] | undefined> {
+    if (!document) {
+      return;
+    }
+    return await this.cache.get(document.uri.toString());
+  }
+
+  private async getCfgUri(uriString: string): Promise<[Uri, messages.TemplateDefinitions]> {
+    const uri = Uri.parse(uriString, true);
+    if (uri.scheme !== zipArchiveScheme) {
+      throw new Error('CFG Viewing is only available for databases with zipped source archives.');
+    }
+
+    const zippedArchive = decodeSourceArchiveUri(uri);
+    const sourceArchiveUri = encodeArchiveBasePath(zippedArchive.sourceArchiveZipPath);
+    const db = this.dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
+
+    if (!db) {
+      throw new Error('Can\'t infer database from the provided source.');
+    }
+
+    const qlpack = await qlpackOfDatabase(this.cli, db);
+    if (!qlpack) {
+      throw new Error('Can\'t infer qlpack from database source archive.');
+    }
+    const queries = await resolveQueries(this.cli, qlpack, KeyType.PrintCfgQuery);
+    if (queries.length > 1) {
+      throw new Error(`Found multiple Print CFG queries. Can't continue. Make sure there is exacly one query with the tag ${KeyType.PrintCfgQuery}`);
+    }
+    if (queries.length === 0) {
+      throw new Error(`Did not find any Print CFG queries. Can't continue. Make sure there is exacly one query with the tag ${KeyType.PrintCfgQuery}`);
+    }
+
+    const queryUri = Uri.file(queries[0]);
+
+    const templates: messages.TemplateDefinitions = {
+      [TEMPLATE_NAME]: {
+        values: {
+          tuples: [[{
+            stringValue: zippedArchive.pathWithinSourceArchive
+          }]]
+        }
+      }
+    };
+
+    return [queryUri, templates];
+  }
+}

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -42,7 +42,8 @@ import { DatabaseUI } from './databases-ui';
 import {
   TemplateQueryDefinitionProvider,
   TemplateQueryReferenceProvider,
-  TemplatePrintAstProvider
+  TemplatePrintAstProvider,
+  TemplatePrintCfgProvider
 } from './contextual/templateProvider';
 import {
   DEFAULT_DISTRIBUTION_VERSION_RANGE,
@@ -1047,15 +1048,16 @@ async function activateWithInstalledDistribution(
   );
 
   const astViewer = new AstViewer();
-  const templateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm, contextualQueryStorageDir);
+  const printAstTemplateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm, contextualQueryStorageDir);
+  const cfgTemplateProvider = new TemplatePrintCfgProvider(cliServer, dbm);
 
   ctx.subscriptions.push(astViewer);
   ctx.subscriptions.push(commandRunnerWithProgress('codeQL.viewAst', async (
     progress: ProgressCallback,
     token: CancellationToken,
     selectedFile: Uri
   ) => {
-    const ast = await templateProvider.provideAst(
+    const ast = await printAstTemplateProvider.provideAst(
       progress,
       token,
       selectedFile ?? window.activeTextEditor?.document.uri,
@@ -1068,6 +1070,25 @@ async function activateWithInstalledDistribution(
     title: 'Calculate AST'
   }));
 
+  ctx.subscriptions.push(
+    commandRunnerWithProgress(
+      'codeQL.viewCfg',
+      async (
+        progress: ProgressCallback,
+        token: CancellationToken
+      ) => {
+        const res = await cfgTemplateProvider.provideCfgUri(window.activeTextEditor?.document);
+        if (res) {
+          await compileAndRunQuery(false, res[0], progress, token, undefined);
+        }
+      },
+      {
+        title: 'Calculating Control Flow Graph',
+        cancellable: true
+      }
+    )
+  );
+
   await commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
 
   void logger.log('Successfully finished extension initialization.');