self-test

Author: hesreallyhim

.github/workflows/self-test.yml

@@ -10,6 +10,54 @@ on:
         description: "Fail jobs on assertion mismatch"
         type: boolean
         default: false
+      run_baseline:
+        description: "Run scenario: Baseline — zero calls"
+        type: boolean
+        default: true
+      run_core_5:
+        description: "Run scenario: 5 core GET calls"
+        type: boolean
+        default: true
+      run_core_10:
+        description: "Run scenario: 10 core GET calls"
+        type: boolean
+        default: true
+      run_search_2:
+        description: "Run scenario: 2 search calls"
+        type: boolean
+        default: true
+      run_code_search_2:
+        description: "Run scenario: 2 code_search calls"
+        type: boolean
+        default: true
+      run_graphql_2:
+        description: "Run scenario: 2 GraphQL calls"
+        type: boolean
+        default: true
+      run_mixed_small:
+        description: "Run scenario: Mixed — 2 core + 2 search + 2 graphql"
+        type: boolean
+        default: true
+      run_core_burst:
+        description: "Run scenario: 5 core GET calls — burst (no sleep)"
+        type: boolean
+        default: true
+      run_search_window_cross:
+        description: "Run scenario: 2 search calls — 65s apart (window crossing)"
+        type: boolean
+        default: true
+      run_idle_check:
+        description: "Run scenario: Idle check — no calls, 120s observation"
+        type: boolean
+        default: true
+      run_core_then_search:
+        description: "Run scenario: 3 core + 2 search (sequential)"
+        type: boolean
+        default: true
+      run_graphql_5:
+        description: "Run scenario: 5 GraphQL calls"
+        type: boolean
+        default: true
 
 concurrency:
   group: self-test
@@ -18,6 +66,7 @@ concurrency:
 jobs:
   baseline:
     runs-on: ubuntu-latest
+    if: ${{ inputs.run_baseline }}
     steps:
       - uses: actions/checkout@v4
 
@@ -53,7 +102,7 @@ jobs:
   core-5:
     runs-on: ubuntu-latest
     needs: [baseline-diag, baseline]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_core_5 }}
     steps:
       - uses: actions/checkout@v4
 
@@ -180,7 +229,7 @@ jobs:
   core-10:
     runs-on: ubuntu-latest
     needs: [core-5-diag, core-5]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_core_10 }}
     steps:
       - uses: actions/checkout@v4
 
@@ -307,7 +356,7 @@ jobs:
   search-2:
     runs-on: ubuntu-latest
     needs: [core-10-diag, core-10]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_search_2 }}
     steps:
       - uses: actions/checkout@v4
 
@@ -434,7 +483,7 @@ jobs:
   code-search-2:
     runs-on: ubuntu-latest
     needs: [search-2-diag, search-2]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_code_search_2 }}
     steps:
       - uses: actions/checkout@v4
 
@@ -501,7 +550,7 @@ jobs:
               state = json.load(f)
           buckets = state.get('buckets', {})
           expectations = {
-              'code_search': {'delta': 2, 'max_windows': 0}
+              'code_search': {'delta': 2, 'min_windows': 1}
           }
           results = []  # list of (bucket, check_name, passed, detail)
           for bucket_name, expect in expectations.items():
@@ -561,7 +610,7 @@ jobs:
   graphql-2:
     runs-on: ubuntu-latest
     needs: [code-search-2-diag, code-search-2]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_graphql_2 }}
     steps:
       - uses: actions/checkout@v4
 
@@ -690,7 +739,7 @@ jobs:
   mixed-small:
     runs-on: ubuntu-latest
     needs: [graphql-2-diag, graphql-2]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_mixed_small }}
     steps:
       - uses: actions/checkout@v4
 
@@ -843,7 +892,7 @@ jobs:
   core-burst:
     runs-on: ubuntu-latest
     needs: [mixed-small-diag, mixed-small]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_core_burst }}
     steps:
       - uses: actions/checkout@v4
 
@@ -969,7 +1018,7 @@ jobs:
   search-window-cross:
     runs-on: ubuntu-latest
     needs: [core-burst-diag, core-burst]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_search_window_cross }}
     steps:
       - uses: actions/checkout@v4
 
@@ -1096,7 +1145,7 @@ jobs:
   idle-check:
     runs-on: ubuntu-latest
     needs: [search-window-cross-diag, search-window-cross]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_idle_check }}
     steps:
       - uses: actions/checkout@v4
 
@@ -1132,7 +1181,7 @@ jobs:
   core-then-search:
     runs-on: ubuntu-latest
     needs: [idle-check-diag, idle-check]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_core_then_search }}
     steps:
       - uses: actions/checkout@v4
 
@@ -1271,7 +1320,7 @@ jobs:
   graphql-5:
     runs-on: ubuntu-latest
     needs: [core-then-search-diag, core-then-search]
-    if: ${{ always() }}
+    if: ${{ always() && inputs.run_graphql_5 }}
     steps:
       - uses: actions/checkout@v4
 

scripts/generate-self-test.ts

@@ -20,6 +20,10 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 const outputPath = join(__dirname, "..", ".github", "workflows", "self-test.yml");
 const ARTIFACT_PREFIX = "github-api-usage-monitor";
 
+function scenarioInputName(id: string): string {
+  return `run_${id.replace(/-/g, "_")}`;
+}
+
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -201,15 +205,19 @@ function generateJob(
   previousScenarioId: string | null
 ): string {
   const lines: string[] = [];
+  const runInput = scenarioInputName(scenario.id);
 
   lines.push(`${scenario.id}:`);
   lines.push(`  runs-on: ubuntu-latest`);
+  if (previousId === null) {
+    lines.push(`  if: \${{ inputs.${runInput} }}`);
+  }
   if (previousId !== null) {
     const needs = previousScenarioId
       ? `[${previousId}, ${previousScenarioId}]`
       : `[${previousId}]`;
     lines.push(`  needs: ${needs}`);
-    lines.push(`  if: \${{ always() }}`);
+    lines.push(`  if: \${{ always() && inputs.${runInput} }}`);
   }
   lines.push(`  steps:`);
   lines.push(`    - uses: actions/checkout@v4`);
@@ -307,18 +315,30 @@ function generateDiagnosticsJob(scenario: Scenario): string {
 // ---------------------------------------------------------------------------
 
 function generateWorkflow(): string {
+  const inputLines: string[] = [
+    "  workflow_dispatch:",
+    "    inputs:",
+    "      strict_validation:",
+    "        description: \"Fail jobs on assertion mismatch\"",
+    "        type: boolean",
+    "        default: false",
+  ];
+
+  for (const scenario of SCENARIOS) {
+    const inputName = scenarioInputName(scenario.id);
+    inputLines.push(`      ${inputName}:`);
+    inputLines.push(`        description: "Run scenario: ${yamlEscape(scenario.name)}"`);
+    inputLines.push(`        type: boolean`);
+    inputLines.push(`        default: true`);
+  }
+
   const header = `# AUTO-GENERATED by scripts/generate-self-test.ts — do not edit manually
 # Regenerate: npx tsx scripts/generate-self-test.ts
 
 name: Self-Test
 
 on:
-  workflow_dispatch:
-    inputs:
-      strict_validation:
-        description: "Fail jobs on assertion mismatch"
-        type: boolean
-        default: false
+${inputLines.join("\n")}
 
 concurrency:
   group: self-test

scripts/scenarios.ts

@@ -153,7 +153,7 @@ export const SCENARIOS: Scenario[] = [
     inter_call_sleep_s: 3,
     poll_duration_s: 90,
     expected: {
-      code_search: { total_used_delta: 2, windows_crossed_max: 0 },
+      code_search: { total_used_delta: 2, windows_crossed_min: 1 },
     },
   },