add bucket attachment cleanup

Author: TheodoreSpeaks

apps/sim/background/cleanup-soft-deletes.ts

@@ -1,6 +1,7 @@
 import { db } from '@sim/db'
 import {
   a2aAgent,
+  copilotChats,
   knowledgeBase,
   mcpServers,
   memory,
@@ -21,6 +22,7 @@ import {
   getRetentionDefaultHours,
   resolveTierWorkspaceIds,
 } from '@/lib/billing/cleanup-dispatcher'
+import { prepareChatCleanup } from '@/lib/cleanup/chat-cleanup'
 import { isUsingCloudStorage, StorageService } from '@/lib/uploads'
 
 const logger = createLogger('CleanupSoftDeletes')
@@ -181,6 +183,33 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise
     `[${tierLabel}] Processing ${workspaceIds.length} workspaces, cutoff: ${retentionDate.toISOString()}`
   )
 
+  // Find chats linked to workflows that are about to be cascade-deleted
+  const doomedWorkflows = await db
+    .select({ id: workflow.id })
+    .from(workflow)
+    .where(
+      and(
+        inArray(workflow.workspaceId, workspaceIds),
+        isNotNull(workflow.archivedAt),
+        lt(workflow.archivedAt, retentionDate)
+      )
+    )
+
+  const doomedWorkflowIds = doomedWorkflows.map((w) => w.id)
+  let chatCleanup: { execute: () => Promise<void> } | null = null
+
+  if (doomedWorkflowIds.length > 0) {
+    const doomedChats = await db
+      .select({ id: copilotChats.id })
+      .from(copilotChats)
+      .where(inArray(copilotChats.workflowId, doomedWorkflowIds))
+
+    const doomedChatIds = doomedChats.map((c) => c.id)
+    if (doomedChatIds.length > 0) {
+      chatCleanup = await prepareChatCleanup(doomedChatIds, tierLabel)
+    }
+  }
+
   const fileStats = await cleanupWorkspaceFileStorage(workspaceIds, retentionDate)
 
   let totalDeleted = 0
@@ -200,6 +229,11 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise
     `[${tierLabel}] Complete: ${totalDeleted} rows deleted, ${fileStats.filesDeleted} files cleaned`
   )
 
+  // Clean up copilot backend + chat storage files after DB rows are gone
+  if (chatCleanup) {
+    await chatCleanup.execute()
+  }
+
   const timeElapsed = (Date.now() - startTime) / 1000
   logger.info(`[${tierLabel}] Job completed in ${timeElapsed.toFixed(2)}s`)
 }

apps/sim/background/cleanup-tasks.ts

@@ -17,8 +17,7 @@ import {
   getRetentionDefaultHours,
   resolveTierWorkspaceIds,
 } from '@/lib/billing/cleanup-dispatcher'
-import { SIM_AGENT_API_URL } from '@/lib/copilot/constants'
-import { env } from '@/lib/core/config/env'
+import { prepareChatCleanup } from '@/lib/cleanup/chat-cleanup'
 
 const logger = createLogger('CleanupTasks')
 
@@ -87,7 +86,7 @@ async function cleanupRunChildren(
   const runIds = await db
     .select({ id: copilotRuns.id })
     .from(copilotRuns)
-    .where(and(inArray(copilotRuns.workspaceId, workspaceIds), lt(copilotRuns.createdAt, retentionDate)))
+    .where(and(inArray(copilotRuns.workspaceId, workspaceIds), lt(copilotRuns.updatedAt, retentionDate)))
     .limit(BATCH_SIZE * MAX_BATCHES_PER_TABLE)
 
   if (runIds.length === 0) {
@@ -139,61 +138,6 @@ async function cleanupRunChildren(
   return results
 }
 
-const COPILOT_CLEANUP_BATCH_SIZE = 1000
-
-/**
- * Call the copilot backend to delete chat data (memory_files, checkpoints, task_chains, etc.)
- * before we delete the Sim DB rows. Chunked at 1000 per request.
- */
-async function cleanupCopilotBackend(
-  chatIds: string[],
-  tierLabel: string
-): Promise<{ deleted: number; failed: number }> {
-  const stats = { deleted: 0, failed: 0 }
-
-  if (chatIds.length === 0 || !env.COPILOT_API_KEY) {
-    if (!env.COPILOT_API_KEY) {
-      logger.warn(`[${tierLabel}] COPILOT_API_KEY not set, skipping copilot backend cleanup`)
-    }
-    return stats
-  }
-
-  for (let i = 0; i < chatIds.length; i += COPILOT_CLEANUP_BATCH_SIZE) {
-    const chunk = chatIds.slice(i, i + COPILOT_CLEANUP_BATCH_SIZE)
-    try {
-      const response = await fetch(`${SIM_AGENT_API_URL}/api/tasks/cleanup`, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          'x-api-key': env.COPILOT_API_KEY,
-        },
-        body: JSON.stringify({ chatIds: chunk }),
-      })
-
-      if (!response.ok) {
-        const errorBody = await response.text().catch(() => '')
-        logger.error(`[${tierLabel}] Copilot backend cleanup failed: ${response.status}`, {
-          errorBody,
-          chatCount: chunk.length,
-        })
-        stats.failed += chunk.length
-        continue
-      }
-
-      const result = await response.json()
-      stats.deleted += result.deleted ?? 0
-      logger.info(
-        `[${tierLabel}] Copilot backend cleanup: ${result.deleted} chats deleted (batch ${Math.floor(i / COPILOT_CLEANUP_BATCH_SIZE) + 1})`
-      )
-    } catch (error) {
-      stats.failed += chunk.length
-      logger.error(`[${tierLabel}] Copilot backend cleanup request failed:`, { error })
-    }
-  }
-
-  return stats
-}
-
 async function resolvePayload(payload: CleanupJobPayload): Promise<{
   workspaceIds: string[]
   retentionHours: number
@@ -252,13 +196,16 @@ export async function runCleanupTasks(payload: CleanupJobPayload): Promise<void>
       .where(
         and(
           inArray(copilotChats.workspaceId, workspaceIds),
-          lt(copilotChats.createdAt, retentionDate)
+          lt(copilotChats.updatedAt, retentionDate)
         )
       )
       .limit(BATCH_SIZE * MAX_BATCHES_PER_TABLE)
 
     const doomedChatIds = doomedChats.map((c) => c.id)
 
+    // Prepare chat cleanup (collect file keys + copilot backend call) BEFORE DB deletion
+    const chatCleanup = await prepareChatCleanup(doomedChatIds, tierLabel)
+
     // Delete run children first (checkpoints, tool calls) since they reference runs
     const runChildResults = await cleanupRunChildren(workspaceIds, retentionDate, tierLabel)
     for (const r of runChildResults) {
@@ -278,7 +225,7 @@ export async function runCleanupTasks(payload: CleanupJobPayload): Promise<void>
         .where(
           and(
             inArray(copilotChats.workspaceId, workspaceIds),
-            lt(copilotChats.createdAt, retentionDate)
+            lt(copilotChats.updatedAt, retentionDate)
           )
         )
         .limit(BATCH_SIZE * MAX_BATCHES_PER_TABLE)
@@ -287,7 +234,7 @@ export async function runCleanupTasks(payload: CleanupJobPayload): Promise<void>
         const deleted = await db
           .delete(copilotFeedback)
           .where(inArray(copilotFeedback.chatId, chatIds.map((c) => c.id)))
-          .returning({ id: sql`id` })
+          .returning({ id: copilotFeedback.feedbackId })
         feedbackResult.deleted = deleted.length
         logger.info(`[${feedbackResult.table}] Deleted ${deleted.length} rows`)
       } else {
@@ -302,7 +249,7 @@ export async function runCleanupTasks(payload: CleanupJobPayload): Promise<void>
     const runsResult = await cleanupTable(
       copilotRuns,
       copilotRuns.workspaceId,
-      copilotRuns.createdAt,
+      copilotRuns.updatedAt,
       workspaceIds,
       retentionDate,
       `${tierLabel}/copilotRuns`
@@ -312,7 +259,7 @@ export async function runCleanupTasks(payload: CleanupJobPayload): Promise<void>
     const chatsResult = await cleanupTable(
       copilotChats,
       copilotChats.workspaceId,
-      copilotChats.createdAt,
+      copilotChats.updatedAt,
       workspaceIds,
       retentionDate,
       `${tierLabel}/copilotChats`
@@ -336,13 +283,8 @@ export async function runCleanupTasks(payload: CleanupJobPayload): Promise<void>
 
     logger.info(`[${tierLabel}] Complete: ${totalDeleted} total rows deleted`)
 
-    // Clean up copilot backend after Sim DB rows are gone (chat no longer accessible)
-    if (doomedChatIds.length > 0) {
-      const copilotResult = await cleanupCopilotBackend(doomedChatIds, tierLabel)
-      logger.info(
-        `[${tierLabel}] Copilot backend: ${copilotResult.deleted} deleted, ${copilotResult.failed} failed`
-      )
-    }
+    // Clean up copilot backend + storage files after DB rows are gone
+    await chatCleanup.execute()
 
   const timeElapsed = (Date.now() - startTime) / 1000
   logger.info(`Task cleanup completed in ${timeElapsed.toFixed(2)}s`)