Skip to content

Commit c71e440

Browse files
E2E: Add perf/VRAM regression harness.
- test/e2e/perf-regression.js: run a WebGPU example and capture JS heap, GC, frame timing, and WebGPU resource deltas (buffers, textures, bind groups, pipelines, submits/render passes/compute passes, uncaptured errors). - test/e2e/perf-regression-compare.js: diff two runs and print a clean A/B table. - Self-contained — relies only on puppeteer plus the existing utils/server.js. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent d050d8e commit c71e440

4 files changed

Lines changed: 651 additions & 0 deletions

File tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ npm-debug.log
1111

1212
test/e2e/chromium
1313
test/e2e/output-screenshots
14+
test/e2e/perf-regression-*.json
1415
test/treeshake/index.bundle.js
1516
test/treeshake/index.bundle.min.js
1617
test/treeshake/index.webgpu.bundle.js

test/e2e/README.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,22 @@ Merge only those commits that pass the tests, otherwise all next commits will al
3939

4040
### Status
4141
97% examples are covered with tests. Check exception list for more information.
42+
43+
### Perf & VRAM regression
44+
45+
`perf-regression.js` measures JS heap, GC, frame timing, and WebGPU resource deltas for a given example. Useful for catching VRAM leaks, GC churn increases, or frame-time regressions introduced by a renderer change. The WebGPU device is wrapped in a page-side interceptor; no external tooling required.
46+
47+
```shell
48+
# capture a baseline on one branch
49+
node test/e2e/perf-regression.js baseline webgpu_backdrop_water
50+
51+
# switch branches, capture the candidate
52+
node test/e2e/perf-regression.js candidate webgpu_backdrop_water
53+
54+
# compare
55+
node test/e2e/perf-regression-compare.js baseline candidate webgpu_backdrop_water
56+
```
57+
58+
Each run produces `test/e2e/perf-regression-<label>-<example>.json`. Optional args: `durationMs` (default 10000), `warmupMs` (default 3000).
59+
60+
Single-run numbers have measurement noise — for regression signals below ~5%, average 3 runs per branch (different `<label>` each run, then average the fields manually). Frame `p50/p95/p99/max`, JS heap `mean`/`max`, `gc.totalFreedBytes`, and all WebGPU resource deltas are the most repeatable signals.
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// Compare two perf-regression JSON runs and print a clean A/B table.
2+
//
3+
// Usage:
4+
// node test/e2e/perf-regression-compare.js <baseline-label> <candidate-label> <example>
5+
//
6+
// Reads:
7+
// test/e2e/perf-regression-<baseline-label>-<example>.json
8+
// test/e2e/perf-regression-<candidate-label>-<example>.json
9+
10+
import * as fs from 'node:fs/promises';
11+
12+
const [ , , baselineLabel, candidateLabel, example ] = process.argv;
13+
14+
if ( ! baselineLabel || ! candidateLabel || ! example ) {
15+
16+
console.error( 'usage: node test/e2e/perf-regression-compare.js <baseline-label> <candidate-label> <example>' );
17+
process.exit( 1 );
18+
19+
}
20+
21+
const baseline = JSON.parse( await fs.readFile( `test/e2e/perf-regression-${ baselineLabel }-${ example }.json`, 'utf8' ) );
22+
const candidate = JSON.parse( await fs.readFile( `test/e2e/perf-regression-${ candidateLabel }-${ example }.json`, 'utf8' ) );
23+
24+
const mb = v => v / 1024 / 1024;
25+
const pad = ( s, n ) => String( s ).padStart( n );
26+
27+
function fmtPct( base, cand ) {
28+
29+
if ( base === cand ) return '·';
30+
if ( base === 0 ) return 'new';
31+
return ( ( cand - base ) / Math.abs( base ) * 100 ).toFixed( 1 ) + '%';
32+
33+
}
34+
35+
function fmt( v, kind ) {
36+
37+
if ( typeof v !== 'number' || ! Number.isFinite( v ) ) return String( v );
38+
if ( kind === 'bytes-mb' ) return mb( v ).toFixed( 2 );
39+
if ( kind === 'ms' ) return v.toFixed( 2 );
40+
if ( kind === 'int' ) return String( Math.round( v ) );
41+
return v.toFixed( 2 );
42+
43+
}
44+
45+
function arrow( base, cand ) {
46+
47+
if ( base === cand ) return '·';
48+
return cand < base ? '▼' : '▲';
49+
50+
}
51+
52+
function row( name, path, kind = 'num' ) {
53+
54+
const b = path.split( '.' ).reduce( ( a, k ) => a && a[ k ], baseline );
55+
const c = path.split( '.' ).reduce( ( a, k ) => a && a[ k ], candidate );
56+
const delta = c - b;
57+
const deltaStr = ( delta >= 0 ? '+' : '' ) + fmt( delta, kind );
58+
console.log(
59+
pad( name, 36 ), ' | ',
60+
pad( fmt( b, kind ), 12 ), ' | ',
61+
pad( fmt( c, kind ), 12 ), ' | ',
62+
pad( deltaStr + ' ' + arrow( b, c ), 14 ), ' | ',
63+
pad( fmtPct( b, c ), 8 )
64+
);
65+
66+
}
67+
68+
function section( title ) {
69+
70+
console.log( '\n-- ' + title + ' --' );
71+
72+
}
73+
74+
console.log( `\nperf-regression: ${ example }` );
75+
console.log( ` baseline: ${ baselineLabel }` );
76+
console.log( ` candidate: ${ candidateLabel }` );
77+
console.log( ` duration: ${ baseline.durationMs } ms (baseline), ${ candidate.durationMs } ms (candidate)\n` );
78+
79+
console.log( pad( 'metric', 36 ), ' | ', pad( baselineLabel, 12 ), ' | ', pad( candidateLabel, 12 ), ' | ', pad( 'Δ', 14 ), ' | ', pad( 'Δ %', 8 ) );
80+
console.log( '-'.repeat( 36 ), '-+-', '-'.repeat( 12 ), '-+-', '-'.repeat( 12 ), '-+-', '-'.repeat( 14 ), '-+-', '-'.repeat( 8 ) );
81+
82+
section( 'Frame timing' );
83+
row( 'fps (rAF)', 'fps' );
84+
row( 'mean (ms)', 'frameTimeMs.mean', 'ms' );
85+
row( 'p50 (ms)', 'frameTimeMs.p50', 'ms' );
86+
row( 'p95 (ms)', 'frameTimeMs.p95', 'ms' );
87+
row( 'p99 (ms)', 'frameTimeMs.p99', 'ms' );
88+
row( 'max (ms)', 'frameTimeMs.max', 'ms' );
89+
90+
section( 'JS heap' );
91+
row( 'min (MB)', 'jsHeapBytes.min', 'bytes-mb' );
92+
row( 'mean (MB)', 'jsHeapBytes.mean', 'bytes-mb' );
93+
row( 'max (MB)', 'jsHeapBytes.max', 'bytes-mb' );
94+
row( 'growth (MB)', 'jsHeapBytes.growth', 'bytes-mb' );
95+
96+
section( 'GC' );
97+
row( 'events', 'gc.events', 'int' );
98+
row( 'events/s', 'gc.eventsPerSec' );
99+
row( 'bytes freed (MB)', 'gc.totalFreedBytes', 'bytes-mb' );
100+
101+
section( 'WebGPU VRAM (estimated)' );
102+
row( 'buffers+textures before (MB)', 'webgpu.estimatedVRAMBefore', 'bytes-mb' );
103+
row( 'buffers+textures after (MB)', 'webgpu.estimatedVRAMAfter', 'bytes-mb' );
104+
row( 'Δ over window (MB)', 'webgpu.estimatedVRAMDelta', 'bytes-mb' );
105+
106+
section( 'WebGPU resources (delta over window)' );
107+
row( 'live buffers Δ', 'webgpu.liveBuffersDelta', 'int' );
108+
row( 'live textures Δ', 'webgpu.liveTexturesDelta', 'int' );
109+
row( 'bind groups (cumulative) Δ', 'webgpu.bindGroupsTotalDelta', 'int' );
110+
row( 'samplers Δ', 'webgpu.samplersDelta', 'int' );
111+
112+
section( 'WebGPU resources (totals)' );
113+
row( 'shader modules', 'webgpu.shaderModules', 'int' );
114+
row( 'render pipelines', 'webgpu.renderPipelines', 'int' );
115+
row( 'compute pipelines', 'webgpu.computePipelines', 'int' );
116+
row( 'live buffers after', 'webgpu.liveBuffersAfter', 'int' );
117+
row( 'live textures after', 'webgpu.liveTexturesAfter', 'int' );
118+
119+
section( 'WebGPU command rate (per frame)' );
120+
row( 'submits/frame', 'webgpu.cmdSubmitsPerFrame' );
121+
row( 'render passes/frame', 'webgpu.renderPassesPerFrame' );
122+
row( 'compute passes/frame', 'webgpu.computePassesPerFrame' );
123+
124+
section( 'Errors' );
125+
row( 'uncaptured WebGPU errors', 'webgpu.errors', 'int' );
126+
127+
console.log();

0 commit comments

Comments
 (0)