perf(@angular/build): selectively compress large entries in SQLite cache store

Transformed JavaScript files from node_modules stored in the persistent cache often range from tens to hundreds of kilobytes. In SQLite, rows exceeding the 4KB page size spill into overflow pages, resulting in repeated pointer traversals and increased disk I/O.

Entries with serialized payloads of 32 KB or greater are now selectively compressed asynchronously using raw DEFLATE at level 1 and marked with a single-byte prefix. Performing compression asynchronously offloads CPU-intensive compression work to background worker threads, avoiding blocking the main JavaScript event loop during parallel file compilation. Smaller payloads and payloads that do not benefit from compression are stored raw with zero copy overhead. Because V8 serialized streams always begin with version byte 0xFF, existing uncompressed cache entries remain fully backward-compatible.

This reduces the disk footprint of large modules by 70 to 80 percent and drastically cuts SQLite overflow page allocations while preserving microsecond read performance for small entries.

Author: clydin

packages/angular/build/src/tools/esbuild/sqlite-cache-store.ts

@@ -6,12 +6,32 @@
  * found in the LICENSE file at https://angular.dev/license
  */
 
+import { Buffer } from 'node:buffer';
 import { mkdirSync, rmSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { DatabaseSync, StatementSync } from 'node:sqlite';
+import { promisify } from 'node:util';
 import { deserialize, serialize } from 'node:v8';
+import { deflateRaw, inflateRawSync } from 'node:zlib';
 import { Cache, PersistentCacheStore } from './cache';
 
+const deflateRawAsync = promisify(deflateRaw);
+
+/**
+ * Minimum serialized payload size (in bytes) required to attempt compression.
+ * Smaller payloads generally do not achieve meaningful compression ratios, while larger payloads
+ * (such as transformed JavaScript modules from node_modules) achieve 70-80% size reduction
+ * and drastically reduce SQLite overflow pages.
+ */
+const COMPRESSION_THRESHOLD = 32 * 1024; // 32 KB
+
+/**
+ * Prefix byte used to indicate a compressed payload.
+ * V8 serialization streams always begin with the version tag byte 0xFF, so 0x01 never collides
+ * with uncompressed payloads.
+ */
+const COMPRESSION_FLAG_DEFLATE = 0x01;
+
 /**
  * Common SQLite primary result codes.
  * @see https://www.sqlite.org/rescode.html
@@ -213,7 +233,11 @@ export class SqliteCacheStore implements PersistentCacheStore<unknown> {
 
         if (row.value instanceof Uint8Array) {
           try {
-            return deserialize(row.value);
+            const raw = row.value;
+            const data =
+              raw[0] === COMPRESSION_FLAG_DEFLATE ? inflateRawSync(raw.subarray(1)) : raw;
+
+            return deserialize(data);
           } catch {
             // Treat corrupt or unparseable cached payloads as a cache miss.
           }
@@ -245,7 +269,20 @@ export class SqliteCacheStore implements PersistentCacheStore<unknown> {
 
     try {
       this.#pendingAccessedKeys.delete(key);
-      this.#setStmt?.run(key, serialize(value));
+
+      const serialized = serialize(value);
+      let payload: Uint8Array = serialized;
+
+      if (serialized.byteLength >= COMPRESSION_THRESHOLD) {
+        const compressed = await deflateRawAsync(serialized, { level: 1 });
+        if (compressed.byteLength + 1 < serialized.byteLength) {
+          payload = Buffer.allocUnsafe(compressed.byteLength + 1);
+          payload[0] = COMPRESSION_FLAG_DEFLATE;
+          payload.set(compressed, 1);
+        }
+      }
+
+      this.#setStmt?.run(key, payload);
     } catch {
       // Writing to cache is non-fatal and should not fail the build.
     }

packages/angular/build/src/tools/esbuild/sqlite-cache-store_spec.ts

@@ -224,6 +224,130 @@ describe('SqliteCacheStore', () => {
     expect(row.last_accessed).toBeGreaterThan(pastTimestamp);
   });
 
+  it('should selectively compress entries that exceed the 32KB threshold', async () => {
+    const largeContent =
+      'export const testFunction = () => { console.log("hello world"); };\n'.repeat(1000);
+    await store.set('large-key', largeContent);
+    store.close();
+
+    const { DatabaseSync } = await import('node:sqlite');
+    const directDb = new DatabaseSync(cachePath);
+    const row = directDb.prepare('SELECT value FROM cache WHERE key = ?').get('large-key') as {
+      value: Uint8Array;
+    };
+    directDb.close();
+
+    // Verify format flag is 0x01 (compressed) and compressed size is drastically smaller
+    expect(row.value[0]).toBe(0x01);
+    expect(row.value.byteLength).toBeLessThan(largeContent.length / 5);
+
+    const reopenedStore = new SqliteCacheStore(cachePath);
+    try {
+      const result = await reopenedStore.get('large-key');
+      expect(result).toBe(largeContent);
+    } finally {
+      reopenedStore.close();
+    }
+  });
+
+  it('should not compress entries below the 32KB threshold', async () => {
+    const smallContent = 'export const small = true;';
+    await store.set('small-key', smallContent);
+    store.close();
+
+    const { DatabaseSync } = await import('node:sqlite');
+    const directDb = new DatabaseSync(cachePath);
+    const row = directDb.prepare('SELECT value FROM cache WHERE key = ?').get('small-key') as {
+      value: Uint8Array;
+    };
+    directDb.close();
+
+    // Verify entry is uncompressed (starts with V8 serialization header 0xFF, not 0x01)
+    expect(row.value[0]).not.toBe(0x01);
+
+    const reopenedStore = new SqliteCacheStore(cachePath);
+    try {
+      const result = await reopenedStore.get('small-key');
+      expect(result).toBe(smallContent);
+    } finally {
+      reopenedStore.close();
+    }
+  });
+
+  it('should not compress entries exceeding the 32KB threshold if compression does not reduce size', async () => {
+    const { randomBytes } = await import('node:crypto');
+    // High-entropy random bytes cannot be compressed by DEFLATE
+    const incompressibleData = randomBytes(48 * 1024);
+    await store.set('incompressible-key', incompressibleData);
+    store.close();
+
+    const { DatabaseSync } = await import('node:sqlite');
+    const directDb = new DatabaseSync(cachePath);
+    const row = directDb
+      .prepare('SELECT value FROM cache WHERE key = ?')
+      .get('incompressible-key') as {
+      value: Uint8Array;
+    };
+    directDb.close();
+
+    // Verify entry is stored uncompressed (starts with V8 serialization header 0xFF, not 0x01)
+    expect(row.value[0]).not.toBe(0x01);
+
+    const reopenedStore = new SqliteCacheStore(cachePath);
+    try {
+      const result = await reopenedStore.get('incompressible-key');
+      expect(result).toEqual(incompressibleData);
+    } finally {
+      reopenedStore.close();
+    }
+  });
+
+  it('should seamlessly read legacy uncompressed entries without format prefix', async () => {
+    const { serialize } = await import('node:v8');
+    const { DatabaseSync } = await import('node:sqlite');
+
+    // Initialize database
+    await store.set('init-key', 'init-val');
+    store.close();
+
+    // Insert a legacy entry directly serialized with v8 without a 1-byte format prefix
+    const directDb = new DatabaseSync(cachePath);
+    const legacyPayload = serialize('legacy-content');
+    directDb
+      .prepare('INSERT INTO cache (key, value, last_accessed) VALUES (?, ?, unixepoch())')
+      .run('legacy-key', legacyPayload);
+    directDb.close();
+
+    const reopenedStore = new SqliteCacheStore(cachePath);
+    try {
+      const result = await reopenedStore.get('legacy-key');
+      expect(result).toBe('legacy-content');
+    } finally {
+      reopenedStore.close();
+    }
+  });
+
+  it('should treat a corrupted compressed payload as a cache miss', async () => {
+    // Initialize database
+    await store.set('init-key', 'init-val');
+    store.close();
+
+    // Insert an entry with 0x01 compression flag followed by garbage
+    const { DatabaseSync } = await import('node:sqlite');
+    const directDb = new DatabaseSync(cachePath);
+    directDb
+      .prepare('INSERT INTO cache (key, value, last_accessed) VALUES (?, ?, unixepoch())')
+      .run('corrupt-compressed-key', new Uint8Array([0x01, 0xde, 0xad, 0xbe, 0xef]));
+    directDb.close();
+
+    const reopenedStore = new SqliteCacheStore(cachePath);
+    try {
+      expect(await reopenedStore.get('corrupt-compressed-key')).toBeUndefined();
+    } finally {
+      reopenedStore.close();
+    }
+  });
+
   it('should treat a non-binary payload as a cache miss', async () => {
     await store.set('text-key', 'value');
     store.close();