fix(store): persistent WAL + version-gated migrations to stop multi-process SQLite corruption

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/jackc/pgx/v5 v5.7.6
 	github.com/mark3labs/mcp-go v0.44.0
 	golang.org/x/net v0.52.0
+	golang.org/x/sys v0.43.0
 	modernc.org/sqlite v1.45.0
 )
 
@@ -59,7 +60,6 @@ require (
 	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
 	golang.org/x/mod v0.34.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/sys v0.43.0 // indirect
 	golang.org/x/text v0.36.0 // indirect
 	golang.org/x/tools v0.43.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

internal/store/generation_fence.go

@@ -80,8 +80,15 @@ func ensureDatabaseFile(path string) error {
 }
 
 var openDB = func(dbPath string, generation *databaseGeneration) (*sql.DB, error) {
-	d := &generationDriver{Driver: &sqlite.Driver{}, generation: generation}
-	return sql.OpenDB(generationConnector{driver: d, name: dbPath}), nil
+	sqliteDriver := &sqlite.Driver{}
+	sqliteDriver.RegisterConnectionHook(func(conn sqlite.ExecQuerierContext, _ string) error {
+		if fc, ok := conn.(sqlite.FileControl); ok {
+			_, _ = fc.FileControlPersistWAL("main", 1)
+		}
+		return nil
+	})
+	d := &generationDriver{Driver: sqliteDriver, generation: generation}
+	return sql.OpenDB(generationConnector{driver: d, name: storeDSN(dbPath)}), nil
 }
 
 type generationConnector struct {
@@ -267,6 +274,28 @@ func (c generationConn) CheckNamedValue(value *driver.NamedValue) error {
 	return driver.ErrSkip
 }
 
+// FileControlPersistWAL forwards modernc's optional FileControl interface
+// through the generation fence. database/sql exposes this wrapped connection
+// to primeConnection via Conn.Raw, so omitting it would make persistent WAL
+// unavailable whenever generation fencing is enabled.
+func (c generationConn) FileControlPersistWAL(dbName string, mode int) (int, error) {
+	if err := c.generation.check(); err != nil {
+		return 0, err
+	}
+	fc, ok := c.Conn.(sqlite.FileControl)
+	if !ok {
+		return 0, errors.New("database connection does not implement sqlite.FileControl")
+	}
+	result, err := fc.FileControlPersistWAL(dbName, mode)
+	if err != nil {
+		return 0, err
+	}
+	if err := c.generation.check(); err != nil {
+		return 0, err
+	}
+	return result, nil
+}
+
 type generationStmt struct {
 	driver.Stmt
 	generation *databaseGeneration

internal/store/generation_fence_test.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+
+	sqlite "modernc.org/sqlite"
 )
 
 func TestDatabaseGeneration(t *testing.T) {
@@ -110,6 +112,24 @@ func TestGenerationFenceRejectsUnsafeOperations(t *testing.T) {
 	})
 }
 
+func TestGenerationConnExposesFileControlForPrimeConnection(t *testing.T) {
+	generation, _ := newTestDatabaseGeneration(t, false, false)
+	base := &testFenceFileControlConn{}
+	conn := generationConn{Conn: base, generation: generation}
+
+	fc, ok := any(conn).(sqlite.FileControl)
+	if !ok {
+		t.Fatal("generation connection does not expose sqlite.FileControl")
+	}
+	mode, err := fc.FileControlPersistWAL("main", 1)
+	if err != nil {
+		t.Fatalf("FileControlPersistWAL: %v", err)
+	}
+	if mode != 1 || base.dbName != "main" || base.mode != 1 {
+		t.Fatalf("persist WAL = (%d, %q, %d), want (1, main, 1)", mode, base.dbName, base.mode)
+	}
+}
+
 func TestNewRejectsGenerationChangedBeforeSQLiteOpens(t *testing.T) {
 	original := openDB
 	t.Cleanup(func() { openDB = original })
@@ -209,6 +229,17 @@ type testFenceConn struct {
 	rows *testFenceRows
 }
 
+type testFenceFileControlConn struct {
+	testFenceConn
+	dbName string
+	mode   int
+}
+
+func (c *testFenceFileControlConn) FileControlPersistWAL(dbName string, mode int) (int, error) {
+	c.dbName, c.mode = dbName, mode
+	return mode, nil
+}
+
 func (c *testFenceConn) Prepare(string) (driver.Stmt, error) { return testFenceStmt{}, nil }
 func (c *testFenceConn) Close() error                        { return nil }
 func (c *testFenceConn) Begin() (driver.Tx, error)           { return testFenceTx{}, nil }

internal/store/migration_lock.go

@@ -0,0 +1,68 @@
+package store
+
+import (
+	"fmt"
+	"os"
+	"time"
+)
+
+// migrationLockTimeout bounds how long a process waits for the migration
+// lock. A hung holder must produce a loud, actionable error instead of
+// silently blocking every engram process on the machine forever. It is a
+// variable (not a constant) so tests can shorten the timeout path.
+var migrationLockTimeout = 60 * time.Second
+
+// acquireMigrationLock takes an exclusive advisory lock on path and returns
+// a function that releases it. It serializes whole processes around the
+// migration suite and the startup repair so that the destructive
+// check-then-act rebuilds inside migrate() can never run twice concurrently
+// against the same database.
+//
+// Acquisition is non-blocking with a bounded growing backoff (up to
+// migrationLockTimeout total) rather than a blocking lock: a stuck holder
+// then surfaces as a clear error naming the lock file instead of a silent
+// machine-wide hang.
+//
+// The lock file is deliberately left in place after unlock: unlinking it
+// would open a race where a third process re-creates the path and locks a
+// different inode/file object, defeating the exclusion.
+func acquireMigrationLock(path string) (func(), error) {
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)
+	if err != nil {
+		return nil, fmt.Errorf("open migration lock file %s: %w", path, err)
+	}
+
+	deadline := time.Now().Add(migrationLockTimeout)
+	backoff := 10 * time.Millisecond
+	for {
+		acquired, err := tryLockMigrationFile(f)
+		if err != nil {
+			_ = f.Close()
+			return nil, fmt.Errorf("lock migration lock file %s: %w", path, err)
+		}
+		if acquired {
+			return func() {
+				_ = unlockMigrationFile(f)
+				_ = f.Close()
+			}, nil
+		}
+		if time.Now().After(deadline) {
+			_ = f.Close()
+			return nil, fmt.Errorf(
+				"timed out after %s waiting for migration lock %s — another engram process appears to be holding it; check for a stuck engram process (and terminate it) before retrying",
+				migrationLockTimeout, path,
+			)
+		}
+		time.Sleep(backoff)
+		// Grow the poll interval but cap it low: healthy holders release the
+		// lock within milliseconds (the startup repair fast path is read-only),
+		// and every engram subcommand acquires this lock, so an aggressive cap
+		// keeps contended cold starts snappy.
+		if backoff < 100*time.Millisecond {
+			backoff *= 2
+			if backoff > 100*time.Millisecond {
+				backoff = 100 * time.Millisecond
+			}
+		}
+	}
+}

internal/store/migration_lock_unix.go

@@ -0,0 +1,30 @@
+//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd
+
+package store
+
+import (
+	"errors"
+	"os"
+	"syscall"
+)
+
+// tryLockMigrationFile attempts a non-blocking exclusive flock(2) on f.
+// It reports (false, nil) when another process (or file description) holds
+// the lock, so the caller can retry with backoff.
+func tryLockMigrationFile(f *os.File) (bool, error) {
+	err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+	if err == nil {
+		return true, nil
+	}
+	// EWOULDBLOCK/EAGAIN: lock is held elsewhere. EINTR: interrupted by a
+	// signal. Both are retryable, not failures.
+	if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR) {
+		return false, nil
+	}
+	return false, err
+}
+
+// unlockMigrationFile releases the flock taken by tryLockMigrationFile.
+func unlockMigrationFile(f *os.File) error {
+	return syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+}

internal/store/migration_lock_windows.go

@@ -0,0 +1,34 @@
+//go:build windows
+
+package store
+
+import (
+	"errors"
+	"os"
+
+	"golang.org/x/sys/windows"
+)
+
+// tryLockMigrationFile attempts a non-blocking exclusive LockFileEx on f.
+// It reports (false, nil) when another process holds the lock, so the
+// caller can retry with backoff.
+func tryLockMigrationFile(f *os.File) (bool, error) {
+	ol := new(windows.Overlapped)
+	err := windows.LockFileEx(
+		windows.Handle(f.Fd()),
+		windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY,
+		0, 1, 0, ol,
+	)
+	if err == nil {
+		return true, nil
+	}
+	if errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
+		return false, nil
+	}
+	return false, err
+}
+
+// unlockMigrationFile releases the lock taken by tryLockMigrationFile.
+func unlockMigrationFile(f *os.File) error {
+	return windows.UnlockFileEx(windows.Handle(f.Fd()), 0, 1, 0, new(windows.Overlapped))
+}