Batch delete resources

Author: TheodoreSpeaks

apps/sim/background/cleanup-soft-deletes.ts

@@ -56,26 +56,36 @@ async function cleanupTable(
 
   while (hasMore && batchesProcessed < MAX_BATCHES_PER_TABLE) {
     try {
-      const deleted = await db
-        .delete(tableDef)
+      // SELECT with LIMIT first to avoid unbounded DELETE
+      const batch = await db
+        .select({ id: sql<string>`id` })
+        .from(tableDef)
         .where(
           and(
             inArray(workspaceIdCol, workspaceIds),
             isNotNull(softDeleteCol),
             lt(softDeleteCol, retentionDate)
           )
         )
+        .limit(BATCH_SIZE)
+
+      if (batch.length === 0) {
+        logger.info(`[${tableName}] No expired soft-deleted rows found`)
+        hasMore = false
+        break
+      }
+
+      const ids = batch.map((r) => r.id)
+      const deleted = await db
+        .delete(tableDef)
+        .where(inArray(sql`id`, ids))
         .returning({ id: sql`id` })
 
       result.deleted += deleted.length
-      hasMore = deleted.length === BATCH_SIZE
+      hasMore = batch.length === BATCH_SIZE
       batchesProcessed++
 
-      if (deleted.length > 0) {
-        logger.info(`[${tableName}] Batch ${batchesProcessed}: deleted ${deleted.length} rows`)
-      } else {
-        logger.info(`[${tableName}] No expired soft-deleted rows found`)
-      }
+      logger.info(`[${tableName}] Batch ${batchesProcessed}: deleted ${deleted.length} rows`)
     } catch (error) {
       result.failed++
       logger.error(`[${tableName}] Batch delete failed:`, { error })

apps/sim/background/cleanup-tasks.ts

@@ -33,7 +33,7 @@ interface TableCleanupResult {
 async function cleanupTable(
   tableDef: PgTable,
   workspaceIdCol: PgColumn,
-  createdAtCol: PgColumn,
+  timestampCol: PgColumn,
   workspaceIds: string[],
   retentionDate: Date,
   tableName: string
@@ -46,20 +46,30 @@ async function cleanupTable(
 
   while (hasMore && batchesProcessed < MAX_BATCHES_PER_TABLE) {
     try {
+      // SELECT with LIMIT first to avoid unbounded DELETE
+      const batch = await db
+        .select({ id: sql<string>`id` })
+        .from(tableDef)
+        .where(and(inArray(workspaceIdCol, workspaceIds), lt(timestampCol, retentionDate)))
+        .limit(BATCH_SIZE)
+
+      if (batch.length === 0) {
+        logger.info(`[${tableName}] No expired rows found`)
+        hasMore = false
+        break
+      }
+
+      const ids = batch.map((r) => r.id)
       const deleted = await db
         .delete(tableDef)
-        .where(and(inArray(workspaceIdCol, workspaceIds), lt(createdAtCol, retentionDate)))
+        .where(inArray(sql`id`, ids))
         .returning({ id: sql`id` })
 
       result.deleted += deleted.length
-      hasMore = deleted.length === BATCH_SIZE
+      hasMore = batch.length === BATCH_SIZE
       batchesProcessed++
 
-      if (deleted.length > 0) {
-        logger.info(`[${tableName}] Batch ${batchesProcessed}: deleted ${deleted.length} rows`)
-      } else {
-        logger.info(`[${tableName}] No expired rows found`)
-      }
+      logger.info(`[${tableName}] Batch ${batchesProcessed}: deleted ${deleted.length} rows`)
     } catch (error) {
       result.failed++
       logger.error(`[${tableName}] Batch delete failed:`, { error })