|
| 1 | +/** |
| 2 | + * This scripts helps after adding a new field in the GitHub API. You will |
| 3 | + * need to modify this script to add the new field to the scenarios. This |
| 4 | + * is just a template and should not be used as-is since it has already been |
| 5 | + * applied. |
| 6 | + * |
| 7 | + * Depending on the actual implementation of the script, you might run into |
| 8 | + * rate limits. If that happens, you can set a `GITHUB_TOKEN` environment |
| 9 | + * variable. For example, use: ``export GITHUB_TOKEN=`gh auth token```. |
| 10 | + * |
| 11 | + * Usage: npx ts-node scripts/add-fields-to-scenarios.ts |
| 12 | + */ |
| 13 | + |
| 14 | +import * as fs from 'fs-extra'; |
| 15 | +import * as path from 'path'; |
| 16 | + |
| 17 | +import { Octokit, type RestEndpointMethodTypes } from '@octokit/rest'; |
| 18 | +import { throttling } from '@octokit/plugin-throttling'; |
| 19 | + |
| 20 | +import { getFiles } from './util/files'; |
| 21 | +import type { GitHubApiRequest } from '../src/mocks/gh-api-request'; |
| 22 | +import { isGetVariantAnalysisRequest } from '../src/mocks/gh-api-request'; |
| 23 | +import { VariantAnalysis } from '../src/remote-queries/gh-api/variant-analysis'; |
| 24 | +import { RepositoryWithMetadata } from '../src/remote-queries/gh-api/repository'; |
| 25 | + |
| 26 | +const extensionDirectory = path.resolve(__dirname, '..'); |
| 27 | +const scenariosDirectory = path.resolve(extensionDirectory, 'src/mocks/scenarios'); |
| 28 | + |
| 29 | +// Make sure we don't run into rate limits by automatically waiting until we can |
| 30 | +// make another request. |
| 31 | +const MyOctokit = Octokit.plugin(throttling); |
| 32 | + |
| 33 | +const auth = process.env.GITHUB_TOKEN; |
| 34 | + |
| 35 | +const octokit = new MyOctokit({ |
| 36 | + auth, |
| 37 | + throttle: { |
| 38 | + onRateLimit: (retryAfter: number, options: any, octokit: Octokit): boolean => { |
| 39 | + octokit.log.warn( |
| 40 | + `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!` |
| 41 | + ); |
| 42 | + |
| 43 | + return true; |
| 44 | + }, |
| 45 | + onSecondaryRateLimit: (_retryAfter: number, options: any, octokit: Octokit): void => { |
| 46 | + octokit.log.warn( |
| 47 | + `SecondaryRateLimit detected for request ${options.method} ${options.url}` |
| 48 | + ); |
| 49 | + }, |
| 50 | + } |
| 51 | +}); |
| 52 | +const repositories = new Map<number, RestEndpointMethodTypes['repos']['get']['response']['data']>(); |
| 53 | + |
| 54 | +async function addFieldsToRepository(repository: RepositoryWithMetadata) { |
| 55 | + if (!repositories.has(repository.id)) { |
| 56 | + const [owner, repo] = repository.full_name.split('/'); |
| 57 | + |
| 58 | + const apiRepository = await octokit.repos.get({ |
| 59 | + owner, |
| 60 | + repo, |
| 61 | + }); |
| 62 | + |
| 63 | + repositories.set(repository.id, apiRepository.data); |
| 64 | + } |
| 65 | + |
| 66 | + const apiRepository = repositories.get(repository.id)!; |
| 67 | + |
| 68 | + repository.stargazers_count = apiRepository.stargazers_count; |
| 69 | + repository.updated_at = apiRepository.updated_at; |
| 70 | +} |
| 71 | + |
| 72 | +async function addFieldsToScenarios() { |
| 73 | + if (!(await fs.pathExists(scenariosDirectory))) { |
| 74 | + console.error('Scenarios directory does not exist: ' + scenariosDirectory); |
| 75 | + return; |
| 76 | + } |
| 77 | + |
| 78 | + for await (const file of getFiles(scenariosDirectory)) { |
| 79 | + if (!file.endsWith('.json')) { |
| 80 | + continue; |
| 81 | + } |
| 82 | + |
| 83 | + const data: GitHubApiRequest = await fs.readJson(file); |
| 84 | + |
| 85 | + if (!isGetVariantAnalysisRequest(data)) { |
| 86 | + continue; |
| 87 | + } |
| 88 | + |
| 89 | + if (!data.response.body || !('controller_repo' in data.response.body)) { |
| 90 | + continue; |
| 91 | + } |
| 92 | + |
| 93 | + console.log(`Adding fields to '${path.relative(scenariosDirectory, file)}'`); |
| 94 | + |
| 95 | + const variantAnalysis = data.response.body as VariantAnalysis; |
| 96 | + |
| 97 | + if (variantAnalysis.scanned_repositories) { |
| 98 | + for (const item of variantAnalysis.scanned_repositories) { |
| 99 | + await addFieldsToRepository(item.repository); |
| 100 | + } |
| 101 | + } |
| 102 | + |
| 103 | + if (variantAnalysis.skipped_repositories?.access_mismatch_repos) { |
| 104 | + for (const item of variantAnalysis.skipped_repositories.access_mismatch_repos.repositories) { |
| 105 | + await addFieldsToRepository(item); |
| 106 | + } |
| 107 | + } |
| 108 | + |
| 109 | + if (variantAnalysis.skipped_repositories?.no_codeql_db_repos) { |
| 110 | + for (const item of variantAnalysis.skipped_repositories.no_codeql_db_repos.repositories) { |
| 111 | + await addFieldsToRepository(item); |
| 112 | + } |
| 113 | + } |
| 114 | + |
| 115 | + if (variantAnalysis.skipped_repositories?.over_limit_repos) { |
| 116 | + for (const item of variantAnalysis.skipped_repositories.over_limit_repos.repositories) { |
| 117 | + await addFieldsToRepository(item); |
| 118 | + } |
| 119 | + } |
| 120 | + |
| 121 | + await fs.writeJson(file, data, { spaces: 2 }); |
| 122 | + } |
| 123 | +} |
| 124 | + |
| 125 | +addFieldsToScenarios().catch(e => { |
| 126 | + console.error(e); |
| 127 | + process.exit(2); |
| 128 | +}); |
0 commit comments