feat(cli): cache processed data chunks, recover from cache (#432)

mathio · web-flow · commit cd836e45cf81 · 2025-01-29T16:56:07.000+01:00
diff --git a/.changeset/clean-swans-whisper.md b/.changeset/clean-swans-whisper.md
@@ -0,0 +1,5 @@
+---
+"lingo.dev": minor
+---
+
+cache processed data chunks, recover from cache
diff --git a/packages/cli/src/cli/cli/i18n.ts b/packages/cli/src/cli/cli/i18n.ts
@@ -15,6 +15,7 @@ import chalk from "chalk";
 import { createTwoFilesPatch } from "diff";
 import inquirer from "inquirer";
 import externalEditor from "external-editor";
+import { cacheChunk, deleteCache, getNormalizedCache } from "../utils/cache";
 
 export default new Command()
   .command("i18n")
@@ -90,6 +91,54 @@ export default new Command()
         ora.succeed("i18n.lock loaded");
       }
 
+      // recover cache if exists
+      const cache = getNormalizedCache();
+      if (cache) {
+        console.log();
+        ora.succeed(`Cache loaded. Attempting recovery...`);
+        const cacheOra = Ora({ indent: 2 });
+
+        for (const bucket of buckets) {
+          cacheOra.info(`Processing bucket: ${bucket.type}`);
+          for (const bucketConfig of bucket.config) {
+            const bucketOra = ora.info(`Processing path: ${bucketConfig.pathPattern}`);
+
+            const sourceLocale = resolveOverridenLocale(i18nConfig!.locale.source, bucketConfig.delimiter);
+            const bucketLoader = createBucketLoader(bucket.type, bucketConfig.pathPattern);
+            bucketLoader.setDefaultLocale(sourceLocale);
+            await bucketLoader.init();
+            const sourceData = await bucketLoader.pull(sourceLocale);
+            const cachedSourceData: Record<string, string> = {};
+
+            for (const targetLocale in cache) {
+              const targetData = await bucketLoader.pull(targetLocale);
+
+              for (const key in cache[targetLocale]) {
+                const { source, result } = cache[targetLocale][key];
+
+                if (sourceData[key] === source && targetData[key] !== result) {
+                  targetData[key] = result;
+                  cachedSourceData[key] = source;
+                }
+              }
+
+              await bucketLoader.push(targetLocale, targetData);
+              lockfileHelper.registerPartialSourceData(bucketConfig.pathPattern, cachedSourceData);
+
+              bucketOra.succeed(
+                `[${sourceLocale} -> ${targetLocale}] Recovered ${Object.keys(cachedSourceData).length} entries from cache`,
+              );
+            }
+          }
+        }
+        deleteCache();
+        if (flags.verbose) {
+          cacheOra.info("Cache file deleted.");
+        }
+      } else if (flags.verbose) {
+        ora.info("Cache file not found. Skipping recovery.");
+      }
+
       if (flags.frozen) {
         ora.start("Checking for lockfile updates...");
         let requiresUpdate = false;
@@ -174,8 +223,18 @@ export default new Command()
                     targetLocale,
                     targetData,
                   },
-                  (progress) => {
-                    bucketOra.text = `[${sourceLocale} -> ${targetLocale}] [${Object.keys(processableData).length} entries] (${progress}%) AI localization in progress...`;
+                  (progress, sourceChunk, processedChunk) => {
+                    cacheChunk(targetLocale, sourceChunk, processedChunk);
+
+                    const progressLog = `[${sourceLocale} -> ${targetLocale}] [${Object.keys(processableData).length} entries] (${progress}%) AI localization in progress...`;
+                    if (flags.verbose) {
+                      ora.info(progressLog);
+                      ora.info(
+                        `Caching chunk ${JSON.stringify(sourceChunk, null, 2)} -> ${JSON.stringify(processedChunk, null, 2)}`,
+                      );
+                    } else {
+                      ora.text = progressLog;
+                    }
                   },
                 );
 
@@ -236,6 +295,10 @@ export default new Command()
       console.log();
       if (!hasErrors) {
         ora.succeed("Localization completed.");
+        deleteCache();
+        if (flags.verbose) {
+          ora.info("Cache file deleted.");
+        }
       } else {
         ora.warn("Localization completed with errors.");
       }
@@ -296,7 +359,11 @@ function createLocalizationEngineConnection(params: { apiKey: string; apiUrl: st
         targetLocale: string;
         targetData: Record<string, any>;
       },
-      onProgress: (progress: number) => void,
+      onProgress: (
+        progress: number,
+        sourceChunk: Record<string, string>,
+        processedChunk: Record<string, string>,
+      ) => void,
     ) => {
       return retryWithExponentialBackoff(
         () =>
diff --git a/packages/cli/src/cli/utils/cache.ts b/packages/cli/src/cli/utils/cache.ts
@@ -0,0 +1,104 @@
+import path from "path";
+import fs from "fs";
+
+interface CacheRow {
+  targetLocale: string;
+  key: string;
+  source: string;
+  processed: string;
+}
+
+interface NormalizedCacheItem {
+  source: string;
+  result: string;
+}
+
+type NormalizedCache = Record<string, NormalizedCacheItem>;
+
+interface NormalizedLocaleCache {
+  [targetLocale: string]: NormalizedCache;
+}
+
+export const cacheChunk = (
+  targetLocale: string,
+  sourceChunk: Record<string, string>,
+  processedChunk: Record<string, string>,
+) => {
+  const rows = Object.entries(sourceChunk).map(([key, source]) => ({
+    targetLocale,
+    key,
+    source,
+    processed: processedChunk[key],
+  }));
+  _appendToCache(rows);
+};
+
+export function getNormalizedCache() {
+  const rows = _loadCache();
+  if (!rows.length) {
+    return null;
+  }
+
+  const normalized: NormalizedLocaleCache = {};
+
+  for (const row of rows) {
+    if (!normalized[row.targetLocale]) {
+      normalized[row.targetLocale] = {};
+    }
+
+    normalized[row.targetLocale][row.key] = {
+      source: row.source,
+      result: row.processed,
+    };
+  }
+
+  return normalized;
+}
+
+export function deleteCache() {
+  const cacheFilePath = _getCacheFilePath();
+  try {
+    fs.unlinkSync(cacheFilePath);
+  } catch (e) {
+    // file might not exist
+  }
+}
+
+function _loadCache() {
+  const cacheFilePath = _getCacheFilePath();
+  if (!fs.existsSync(cacheFilePath)) {
+    return [];
+  }
+  const content = fs.readFileSync(cacheFilePath, "utf-8");
+  const result = _parseJSONLines(content);
+  return result;
+}
+
+function _appendToCache(rows: CacheRow[]) {
+  const cacheFilePath = _getCacheFilePath();
+  const lines = _buildJSONLines(rows);
+  fs.appendFileSync(cacheFilePath, lines);
+}
+
+function _getCacheFilePath() {
+  return path.join(process.cwd(), "i18n.cache");
+}
+
+function _buildJSONLines(rows: CacheRow[]) {
+  return rows.map((row) => JSON.stringify(row)).join("\n") + "\n";
+}
+
+function _parseJSONLines(lines: string) {
+  return lines
+    .split("\n")
+    .map(_tryParseJSON)
+    .filter((line) => line !== null);
+}
+
+function _tryParseJSON(line: string) {
+  try {
+    return JSON.parse(line);
+  } catch (e) {
+    return null;
+  }
+}
diff --git a/packages/cli/src/cli/utils/lockfile.ts b/packages/cli/src/cli/utils/lockfile.ts
@@ -21,6 +21,16 @@ export function createLockfileHelper() {
 
       _saveLockfile(lockfile);
     },
+    registerPartialSourceData: (pathPattern: string, partialSourceData: Record<string, any>) => {
+      const lockfile = _loadLockfile();
+
+      const sectionKey = MD5(pathPattern);
+      const sectionChecksums = _.mapValues(partialSourceData, (value) => MD5(value));
+
+      lockfile.checksums[sectionKey] = _.merge({}, lockfile.checksums[sectionKey] ?? {}, sectionChecksums);
+
+      _saveLockfile(lockfile);
+    },
     extractUpdatedData: (pathPattern: string, sourceData: Record<string, any>) => {
       const lockfile = _loadLockfile();
 
diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts
@@ -48,7 +48,11 @@ export class ReplexicaEngine {
     payload: Z.infer<typeof payloadSchema>,
     params: Z.infer<typeof localizationParamsSchema>,
     reference?: Z.infer<typeof referenceSchema>,
-    progressCallback?: (progress: number) => void,
+    progressCallback?: (
+      progress: number,
+      sourceChunk: Record<string, string>,
+      processedChunk: Record<string, string>,
+    ) => void,
   ): Promise<Record<string, string>> {
     const finalPayload = payloadSchema.parse(payload);
     const finalParams = localizationParamsSchema.parse(params);
@@ -61,17 +65,18 @@ export class ReplexicaEngine {
       const chunk = chunkedPayload[i];
       const percentageCompleted = Math.round(((i + 1) / chunkedPayload.length) * 100);
 
-      if (progressCallback) {
-        progressCallback(percentageCompleted);
-      }
-
       const processedPayloadChunk = await this.localizeChunk(
         finalParams.sourceLocale,
         finalParams.targetLocale,
         { data: chunk, reference },
         workflowId,
         params.fast || false,
       );
+
+      if (progressCallback) {
+        progressCallback(percentageCompleted, chunk, processedPayloadChunk);
+      }
+
       processedPayloadChunks.push(processedPayloadChunk);
     }
 
@@ -187,7 +192,11 @@ export class ReplexicaEngine {
   async localizeObject(
     obj: Record<string, any>,
     params: Z.infer<typeof localizationParamsSchema>,
-    progressCallback?: (progress: number) => void,
+    progressCallback?: (
+      progress: number,
+      sourceChunk: Record<string, string>,
+      processedChunk: Record<string, string>,
+    ) => void,
   ): Promise<Record<string, any>> {
     return this._localizeRaw(obj, params, undefined, progressCallback);
   }

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +"lingo.dev": minor
 +---
++
 +cache processed data chunks, recover from cache