Implement download behaviour in remote queries view (#1046)

Author: charisk

extensions/ql-vscode/src/pure/interface-types.ts

@@ -1,4 +1,5 @@
 import * as sarif from 'sarif';
+import { DownloadLink } from '../remote-queries/download-link';
 import { RemoteQueryResult } from '../remote-queries/shared/remote-query-result';
 import { RawResultSet, ResultRow, ResultSetSchema, Column, ResolvableLocationValue } from './bqrs-cli-types';
 
@@ -375,7 +376,8 @@ export type FromRemoteQueriesMessage =
   | RemoteQueryLoadedMessage
   | RemoteQueryErrorMessage
   | OpenFileMsg
-  | OpenVirtualFileMsg;
+  | OpenVirtualFileMsg
+  | RemoteQueryDownloadLinkClickedMessage;
 
 export type ToRemoteQueriesMessage =
   | SetRemoteQueryResultMessage;
@@ -393,3 +395,8 @@ export interface RemoteQueryErrorMessage {
   t: 'remoteQueryError';
   error: string;
 }
+
+export interface RemoteQueryDownloadLinkClickedMessage {
+  t: 'remoteQueryDownloadLinkClicked';
+  downloadLink: DownloadLink;
+}

extensions/ql-vscode/src/remote-queries/download-link.ts

@@ -0,0 +1,20 @@
+/**
+ * Represents a link to an artifact to be downloaded. 
+ */
+export interface DownloadLink {
+  /**
+   *  A unique id of the artifact being downloaded. 
+   */
+  id: string;
+
+  /**
+   * The URL path to use against the GitHub API to download the
+   * linked artifact. 
+   */
+  urlPath: string;
+
+  /**
+   * An optional path to follow inside the downloaded archive containing the artifact.
+   */
+  innerFilePath?: string;
+}

extensions/ql-vscode/src/remote-queries/gh-actions-api-client.ts

@@ -6,38 +6,91 @@ import { Credentials } from '../authentication';
 import { logger } from '../logging';
 import { tmpDir } from '../run-queries';
 import { RemoteQueryWorkflowResult } from './remote-query-workflow-result';
+import { DownloadLink } from './download-link';
+import { RemoteQuery } from './remote-query';
+import { RemoteQueryResultIndex, RemoteQueryResultIndexItem } from './remote-query-result-index';
 
-export interface ResultIndexItem {
+interface ApiResultIndexItem {
   nwo: string;
   id: string;
   results_count: number;
   bqrs_file_size: number;
   sarif_file_size?: number;
 }
 
+export async function getRemoteQueryIndex(
+  credentials: Credentials,
+  remoteQuery: RemoteQuery
+): Promise<RemoteQueryResultIndex | undefined> {
+  const controllerRepo = remoteQuery.controllerRepository;
+  const owner = controllerRepo.owner;
+  const repoName = controllerRepo.name;
+  const workflowRunId = remoteQuery.actionsWorkflowRunId;
+
+  const workflowUri = `https://github.com/${owner}/${repoName}/actions/runs/${workflowRunId}`;
+  const artifactsUrlPath = `/repos/${owner}/${repoName}/actions/artifacts`;
+
+  const artifactList = await listWorkflowRunArtifacts(credentials, owner, repoName, workflowRunId);
+  const resultIndexArtifactId = getArtifactIDfromName('result-index', workflowUri, artifactList);
+  const resultIndexItems = await getResultIndexItems(credentials, owner, repoName, resultIndexArtifactId);
+
+  const allResultsArtifactId = getArtifactIDfromName('all-results', workflowUri, artifactList);
+
+  const items = resultIndexItems.map(item => {
+    const artifactId = getArtifactIDfromName(item.id, workflowUri, artifactList);
+
+    return {
+      id: item.id.toString(),
+      artifactId: artifactId,
+      nwo: item.nwo,
+      resultCount: item.results_count,
+      bqrsFileSize: item.bqrs_file_size,
+      sarifFileSize: item.sarif_file_size,
+    } as RemoteQueryResultIndexItem;
+  });
+
+  return {
+    allResultsArtifactId,
+    artifactsUrlPath,
+    items,
+  };
+}
+
+export async function downloadArtifactFromLink(
+  credentials: Credentials,
+  downloadLink: DownloadLink
+): Promise<string> {
+  const octokit = await credentials.getOctokit();
+
+  // Download the zipped artifact.
+  const response = await octokit.request(`GET ${downloadLink.urlPath}/zip`, {});
+
+  const zipFilePath = path.join(tmpDir.name, `${downloadLink.id}.zip`);
+  await saveFile(`${zipFilePath}`, response.data as ArrayBuffer);
+
+  // Extract the zipped artifact.
+  const extractedPath = path.join(tmpDir.name, downloadLink.id);
+  await unzipFile(zipFilePath, extractedPath);
+
+  return downloadLink.innerFilePath
+    ? path.join(extractedPath, downloadLink.innerFilePath)
+    : extractedPath;
+}
+
 /**
- * Gets the result index file for a given remote queries run.
+ * Downloads the result index artifact and extracts the result index items.
  * @param credentials Credentials for authenticating to the GitHub API.
  * @param owner
  * @param repo
  * @param workflowRunId The ID of the workflow run to get the result index for.
  * @returns An object containing the result index.
  */
-export async function getResultIndex(
+async function getResultIndexItems(
   credentials: Credentials,
   owner: string,
   repo: string,
-  workflowRunId: number
-): Promise<ResultIndexItem[]> {
-  const artifactList = await listWorkflowRunArtifacts(credentials, owner, repo, workflowRunId);
-  const artifactId = getArtifactIDfromName('result-index', artifactList);
-  if (!artifactId) {
-    void showAndLogWarningMessage(
-      `Could not find a result index for the [specified workflow](https://github.com/${owner}/${repo}/actions/runs/${workflowRunId}).
-      Please check whether the workflow run has successfully completed.`
-    );
-    return [];
-  }
+  artifactId: number
+): Promise<ApiResultIndexItem[]> {
   const artifactPath = await downloadArtifact(credentials, owner, repo, artifactId);
   const indexFilePath = path.join(artifactPath, 'index.json');
   if (!(await fs.pathExists(indexFilePath))) {
@@ -115,8 +168,20 @@ async function listWorkflowRunArtifacts(
  * @param artifacts An array of artifact details (from the "list workflow run artifacts" API response).
  * @returns The artifact ID corresponding to the given artifact name.
  */
-function getArtifactIDfromName(artifactName: string, artifacts: Array<{ id: number, name: string }>): number | undefined {
+function getArtifactIDfromName(
+  artifactName: string,
+  workflowUri: string,
+  artifacts: Array<{ id: number, name: string }>
+): number {
   const artifact = artifacts.find(a => a.name === artifactName);
+
+  if (!artifact) {
+    const errorMessage =
+      `Could not find artifact with name ${artifactName} in workflow ${workflowUri}.
+      Please check whether the workflow run has successfully completed.`;
+    throw Error(errorMessage);
+  }
+
   return artifact?.id;
 }
 
@@ -142,19 +207,22 @@ async function downloadArtifact(
     archive_format: 'zip',
   });
   const artifactPath = path.join(tmpDir.name, `${artifactId}`);
-  void logger.log(`Downloading artifact to ${artifactPath}.zip`);
-  await fs.writeFile(
-    `${artifactPath}.zip`,
-    Buffer.from(response.data as ArrayBuffer)
-  );
-
-  void logger.log(`Extracting artifact to ${artifactPath}`);
-  await (
-    await unzipper.Open.file(`${artifactPath}.zip`)
-  ).extract({ path: artifactPath });
+  await saveFile(`${artifactPath}.zip`, response.data as ArrayBuffer);
+  await unzipFile(`${artifactPath}.zip`, artifactPath);
   return artifactPath;
 }
 
+async function saveFile(filePath: string, data: ArrayBuffer): Promise<void> {
+  void logger.log(`Saving file to ${filePath}`);
+  await fs.writeFile(filePath, Buffer.from(data));
+}
+
+async function unzipFile(sourcePath: string, destinationPath: string) {
+  void logger.log(`Unzipping file to ${destinationPath}`);
+  const file = await unzipper.Open.file(sourcePath);
+  await file.extract({ path: destinationPath });
+}
+
 function getWorkflowError(conclusion: string | null): string {
   if (!conclusion) {
     return 'Workflow finished without a conclusion';

extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts

@@ -7,11 +7,14 @@ import {
   workspace,
 } from 'vscode';
 import * as path from 'path';
+import * as vscode from 'vscode';
+import * as fs from 'fs-extra';
 
 import { tmpDir } from '../run-queries';
 import {
   ToRemoteQueriesMessage,
   FromRemoteQueriesMessage,
+  RemoteQueryDownloadLinkClickedMessage,
 } from '../pure/interface-types';
 import { Logger } from '../logging';
 import { getHtmlForWebview } from '../interface-utils';
@@ -20,7 +23,9 @@ import { AnalysisResult, RemoteQueryResult } from './remote-query-result';
 import { RemoteQuery } from './remote-query';
 import { RemoteQueryResult as RemoteQueryResultViewModel } from './shared/remote-query-result';
 import { AnalysisResult as AnalysisResultViewModel } from './shared/remote-query-result';
-import { showAndLogWarningMessage } from '../helpers';
+import { downloadArtifactFromLink } from './gh-actions-api-client';
+import { Credentials } from '../authentication';
+import { showAndLogWarningMessage, showInformationMessageWithAction } from '../helpers';
 import { URLSearchParams } from 'url';
 import { SHOW_QUERY_TEXT_MSG } from '../query-history';
 
@@ -73,7 +78,7 @@ export class RemoteQueriesInterfaceManager {
       totalResultCount: totalResultCount,
       executionTimestamp: this.formatDate(query.executionStartTime),
       executionDuration: executionDuration,
-      downloadLink: queryResult.allResultsDownloadUri,
+      downloadLink: queryResult.allResultsDownloadLink,
       results: analysisResults
     };
   }
@@ -180,11 +185,32 @@ export class RemoteQueriesInterfaceManager {
       case 'openVirtualFile':
         await this.openVirtualFile(msg.queryText);
         break;
+      case 'remoteQueryDownloadLinkClicked':
+        await this.handleDownloadLinkClicked(msg);
+        break;
       default:
         assertNever(msg);
     }
   }
 
+  private async handleDownloadLinkClicked(msg: RemoteQueryDownloadLinkClickedMessage): Promise<void> {
+    const credentials = await Credentials.initialize(this.ctx);
+
+    const filePath = await downloadArtifactFromLink(credentials, msg.downloadLink);
+    const isDir = (await fs.stat(filePath)).isDirectory();
+    const message = `Result file saved at ${filePath}`;
+    if (isDir) {
+      await vscode.window.showInformationMessage(message);
+    }
+    else {
+      const shouldOpenResults = await showInformationMessageWithAction(message, 'Open');
+      if (shouldOpenResults) {
+        const textDocument = await vscode.workspace.openTextDocument(filePath);
+        await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
+      }
+    }
+  }
+
   private postMessage(msg: ToRemoteQueriesMessage): Thenable<boolean> {
     return this.getPanel().webview.postMessage(msg);
   }
@@ -245,9 +271,8 @@ export class RemoteQueriesInterfaceManager {
     return sortedAnalysisResults.map((analysisResult) => ({
       nwo: analysisResult.nwo,
       resultCount: analysisResult.resultCount,
-      downloadLink: analysisResult.downloadUri,
+      downloadLink: analysisResult.downloadLink,
       fileSize: this.formatFileSize(analysisResult.fileSizeInBytes)
     }));
   }
 }
-

extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts

@@ -5,11 +5,13 @@ import { ProgressCallback } from '../commandRunner';
 import { showAndLogErrorMessage, showInformationMessageWithAction } from '../helpers';
 import { Logger } from '../logging';
 import { runRemoteQuery } from './run-remote-query';
-import { getResultIndex, ResultIndexItem } from './gh-actions-api-client';
 import { RemoteQueriesInterfaceManager } from './remote-queries-interface';
 import { RemoteQuery } from './remote-query';
 import { RemoteQueriesMonitor } from './remote-queries-monitor';
+import { getRemoteQueryIndex } from './gh-actions-api-client';
+import { RemoteQueryResultIndex } from './remote-query-result-index';
 import { RemoteQueryResult } from './remote-query-result';
+import { DownloadLink } from './download-link';
 
 export class RemoteQueriesManager {
   private readonly remoteQueriesMonitor: RemoteQueriesMonitor;
@@ -52,14 +54,19 @@ export class RemoteQueriesManager {
     const executionEndTime = new Date();
 
     if (queryResult.status === 'CompletedSuccessfully') {
-      const resultIndexItems = await this.downloadResultIndex(credentials, query);
+      const resultIndex = await getRemoteQueryIndex(credentials, query);
+      if (!resultIndex) {
+        void showAndLogErrorMessage(`There was an issue retrieving the result for the query ${query.queryName}`);
+        return;
+      }
+
+      const queryResult = this.mapQueryResult(executionEndTime, resultIndex);
 
-      const totalResultCount = resultIndexItems.reduce((acc, cur) => acc + cur.results_count, 0);
+      const totalResultCount = queryResult.analysisResults.reduce((acc, cur) => acc + cur.resultCount, 0);
       const message = `Query "${query.queryName}" run on ${query.repositories.length} repositories and returned ${totalResultCount} results`;
 
       const shouldOpenView = await showInformationMessageWithAction(message, 'View');
       if (shouldOpenView) {
-        const queryResult = this.mapQueryResult(executionEndTime, resultIndexItems);
         const rqim = new RemoteQueriesInterfaceManager(this.ctx, this.logger);
         await rqim.showResults(query, queryResult);
       }
@@ -71,31 +78,25 @@ export class RemoteQueriesManager {
     }
   }
 
-  private async downloadResultIndex(credentials: Credentials, query: RemoteQuery) {
-    return await getResultIndex(
-      credentials,
-      query.controllerRepository.owner,
-      query.controllerRepository.name,
-      query.actionsWorkflowRunId);
-  }
-
-  private mapQueryResult(executionEndTime: Date, resultindexItems: ResultIndexItem[]): RemoteQueryResult {
-    // Example URIs are used for now, but a solution for downloading the results will soon be implemented.
-    const allResultsDownloadUri = 'www.example.com';
-    const analysisDownloadUri = 'www.example.com';
-
-    const analysisResults = resultindexItems.map(ri => ({
-      nwo: ri.nwo,
-      resultCount: ri.results_count,
-      downloadUri: analysisDownloadUri,
-      fileSizeInBytes: ri.sarif_file_size || ri.bqrs_file_size,
-    })
-    );
+  private mapQueryResult(executionEndTime: Date, resultIndex: RemoteQueryResultIndex): RemoteQueryResult {
+    const analysisResults = resultIndex.items.map(item => ({
+      nwo: item.nwo,
+      resultCount: item.resultCount,
+      fileSizeInBytes: item.sarifFileSize ? item.sarifFileSize : item.bqrsFileSize,
+      downloadLink: {
+        id: item.artifactId.toString(),
+        urlPath: `${resultIndex.artifactsUrlPath}/${item.artifactId}`,
+        innerFilePath: item.sarifFileSize ? 'results.sarif' : 'results.bqrs'
+      } as DownloadLink
+    }));
 
     return {
       executionEndTime,
       analysisResults,
-      allResultsDownloadUri,
+      allResultsDownloadLink: {
+        id: resultIndex.allResultsArtifactId.toString(),
+        urlPath: `${resultIndex.artifactsUrlPath}/${resultIndex.allResultsArtifactId}`
+      }
     };
   }
 }

extensions/ql-vscode/src/remote-queries/remote-query-result-index.ts

@@ -0,0 +1,14 @@
+export interface RemoteQueryResultIndex {
+  artifactsUrlPath: string;
+  allResultsArtifactId: number;
+  items: RemoteQueryResultIndexItem[];
+}
+
+export interface RemoteQueryResultIndexItem {
+  id: string;
+  artifactId: number;
+  nwo: string;
+  resultCount: number;
+  bqrsFileSize: number;
+  sarifFileSize?: number;
+}