feat: implement STCS compaction

.seqbench/baseline.env

@@ -1,8 +1,20 @@
 GOGC=100
 
-SEQDB_STORAGE_FRAC_SIZE=16MiB
+SEQDB_STORAGE_FRAC_SIZE=1MiB
 SEQDB_STORAGE_TOTAL_SIZE=10GiB
 
+SEQDB_COMPACTION_ENABLED=true
+SEQDB_COMPACTION_WORKERS=4
+SEQDB_COMPACTION_TIME_WINDOW=1h
+SEQDB_COMPACTION_TICK_INTERVAL=1s
+
+SEQDB_COMPACTION_STCS_MERGE_TRIGGER=4
+SEQDB_COMPACTION_STCS_MERGE_FAN_IN=8
+SEQDB_COMPACTION_STCS_MERGE_FAN_OUT_SIZE=256MiB
+
+SEQDB_COMPACTION_STCS_BUCKET_LOWERBOUND=0.5
+SEQDB_COMPACTION_STCS_BUCKET_UPPERBOUND=1.5
+
 SEQDB_LIMITS_QUERY_RATE=1024
 SEQDB_LIMITS_SEARCH_REQUESTS=1024
 SEQDB_LIMITS_BULK_REQUESTS=128

.seqbench/comparison.env

@@ -1,8 +1,20 @@
 GOGC=100
 
-SEQDB_STORAGE_FRAC_SIZE=16MiB
+SEQDB_STORAGE_FRAC_SIZE=1MiB
 SEQDB_STORAGE_TOTAL_SIZE=10GiB
 
+SEQDB_COMPACTION_ENABLED=true
+SEQDB_COMPACTION_WORKERS=4
+SEQDB_COMPACTION_TIME_WINDOW=1h
+SEQDB_COMPACTION_TICK_INTERVAL=1s
+
+SEQDB_COMPACTION_STCS_MERGE_TRIGGER=4
+SEQDB_COMPACTION_STCS_MERGE_FAN_IN=8
+SEQDB_COMPACTION_STCS_MERGE_FAN_OUT_SIZE=256MiB
+
+SEQDB_COMPACTION_STCS_BUCKET_LOWERBOUND=0.5
+SEQDB_COMPACTION_STCS_BUCKET_UPPERBOUND=1.5
+
 SEQDB_LIMITS_QUERY_RATE=1024
 SEQDB_LIMITS_SEARCH_REQUESTS=1024
 SEQDB_LIMITS_BULK_REQUESTS=128

.seqbench/continuous.env

@@ -2,9 +2,21 @@ GOGC=100
 
 SEQDB_RESOURCES_SKIP_FSYNC=true
 
-SEQDB_STORAGE_FRAC_SIZE=16MiB
+SEQDB_STORAGE_FRAC_SIZE=1MiB
 SEQDB_STORAGE_TOTAL_SIZE=10GiB
 
+SEQDB_COMPACTION_ENABLED=true
+SEQDB_COMPACTION_WORKERS=4
+SEQDB_COMPACTION_TIME_WINDOW=1h
+SEQDB_COMPACTION_TICK_INTERVAL=1s
+
+SEQDB_COMPACTION_STCS_MERGE_TRIGGER=4
+SEQDB_COMPACTION_STCS_MERGE_FAN_IN=8
+SEQDB_COMPACTION_STCS_MERGE_FAN_OUT_SIZE=256MiB
+
+SEQDB_COMPACTION_STCS_BUCKET_LOWERBOUND=0.5
+SEQDB_COMPACTION_STCS_BUCKET_UPPERBOUND=1.5
+
 SEQDB_LIMITS_QUERY_RATE=1024
 SEQDB_LIMITS_SEARCH_REQUESTS=1024
 SEQDB_LIMITS_BULK_REQUESTS=128

cmd/seq-db/seq-db.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/ozontech/seq-db/asyncsearcher"
 	"github.com/ozontech/seq-db/buildinfo"
+	"github.com/ozontech/seq-db/compaction"
 	"github.com/ozontech/seq-db/config"
 	"github.com/ozontech/seq-db/consts"
 	"github.com/ozontech/seq-db/frac"
@@ -289,6 +290,7 @@ func startStore(
 			OffloadingRetention:  cfg.Offloading.Retention,
 			OffloadingRetryDelay: cfg.Offloading.RetryDelay,
 			OffloadingQueueSize:  uint64(float64(cfg.Storage.TotalSize) * cfg.Offloading.QueueSizePercent / 100),
+			CompactionEnabled:    cfg.Compaction.Enabled,
 		},
 		API: storeapi.APIConfig{
 			StoreMode: configMode,
@@ -324,6 +326,20 @@ func startStore(
 			Workers:        cfg.SkipMaskManager.Workers,
 			CacheSizeLimit: uint64(cfg.SkipMaskManager.CacheSize),
 		},
+		Compaction: compaction.Config{
+			Enabled: cfg.Compaction.Enabled,
+
+			MergeTrigger:    cfg.Compaction.STCS.MergeTrigger,
+			MergeFanIn:      cfg.Compaction.STCS.MergeFanIn,
+			MergeFanOutSize: uint64(cfg.Compaction.STCS.MergeFanOutSize),
+
+			BucketLowerbound: cfg.Compaction.STCS.BucketLowerbound,
+			BucketUpperbound: cfg.Compaction.STCS.BucketUpperbound,
+
+			Workers:      cfg.Compaction.Workers,
+			TimeWindow:   cfg.Compaction.TimeWindow,
+			TickInterval: cfg.Compaction.TickInterval,
+		},
 	}
 
 	s3cli := initS3Client(cfg)

compaction/executor.go

@@ -0,0 +1,76 @@
+package compaction
+
+import (
+	"sync"
+	"time"
+
+	"go.uber.org/zap"
+
+	"github.com/ozontech/seq-db/frac"
+	"github.com/ozontech/seq-db/frac/common"
+	"github.com/ozontech/seq-db/frac/sealed"
+	"github.com/ozontech/seq-db/logger"
+)
+
+type Executor struct {
+	params common.SealParams
+
+	workers int
+	wg      sync.WaitGroup
+
+	p *planner
+}
+
+func NewExecutor(workers int, params common.SealParams, p *planner) *Executor {
+	e := Executor{params: params, workers: workers, p: p}
+	e.init()
+	return &e
+}
+
+func (e *Executor) Stop() {
+	e.p.stop()
+	e.wg.Wait()
+}
+
+func (e *Executor) init() {
+	for range e.workers {
+		e.wg.Go(func() {
+			for t := range e.p.tasks {
+				start := time.Now()
+
+				result, err := e.compact(t)
+				compactionDurationSeconds.
+					WithLabelValues(t.bucketSize).
+					Observe(time.Since(start).Seconds())
+
+				t.onComplete(result, err)
+			}
+		})
+	}
+}
+
+func (e *Executor) compact(t task) (*sealed.PreloadedData, error) {
+	var (
+		names []string
+		srcs  []Source
+	)
+
+	for _, f := range t.snapshot.Fractions() {
+		names = append(names, f.Info().Name())
+		srcs = append(srcs, frac.NewSealedSource(f))
+
+		compactionBytesTotal.
+			WithLabelValues(t.bucketSize).
+			Add(float64(f.Info().IndexOnDisk))
+	}
+
+	logger.Info(
+		"compacting fractions",
+		zap.Time("bin", t.bin),
+		zap.Strings("names", names),
+		zap.String("bucket_size", t.bucketSize),
+	)
+
+	preloaded, err := Merge(t.filename, e.params, srcs...)
+	return preloaded, err
+}

compaction/merge.go

@@ -11,21 +11,21 @@ import (
 )
 
 func Merge(filename string, params common.SealParams, srcs ...Source) (*sealed.PreloadedData, error) {
-	writer := indexwriter.New(params)
+	w := indexwriter.New(params)
 	src := NewMergeSource(filename, srcs)
 
 	if err := createAndWrite(
 		filename+consts.OffsetsTmpFileSuffix,
 		filename+consts.OffsetsFileSuffix,
-		func(f *os.File) error { return writer.WriteOffsetsFile(f, src) },
+		func(f *os.File) error { return w.WriteOffsetsFile(f, src) },
 	); err != nil {
 		return nil, err
 	}
 
 	if err := createAndWrite(
 		filename+consts.IDTmpFileSuffix,
 		filename+consts.IDFileSuffix,
-		func(f *os.File) error { return writer.WriteIDFile(f, src) },
+		func(f *os.File) error { return w.WriteIDFile(f, src) },
 	); err != nil {
 		return nil, err
 	}
@@ -35,15 +35,15 @@ func Merge(filename string, params common.SealParams, srcs ...Source) (*sealed.P
 		filename+consts.TokenFileSuffix,
 		filename+consts.LIDTmpFileSuffix,
 		filename+consts.LIDFileSuffix,
-		func(tf, lf *os.File) error { return writer.WriteTokenTriplet(tf, lf, src) },
+		func(tf, lf *os.File) error { return w.WriteTokenTriplet(tf, lf, src) },
 	); err != nil {
 		return nil, err
 	}
 
 	if err := createAndWrite(
 		filename+consts.InfoTmpFileSuffix,
 		filename+consts.InfoFileSuffix,
-		func(f *os.File) error { return writer.WriteInfoFile(f, src) },
+		func(f *os.File) error { return w.WriteInfoFile(f, src) },
 	); err != nil {
 		return nil, err
 	}
@@ -69,13 +69,13 @@ func Merge(filename string, params common.SealParams, srcs ...Source) (*sealed.P
 		info.IndexOnDisk += uint64(st.Size())
 	}
 
-	lidsTable := writer.LIDsTable()
+	lidsTable := w.LIDsTable()
 	preloaded := &sealed.PreloadedData{
 		Info:       info,
-		TokenTable: writer.TokenTable(),
+		TokenTable: w.TokenTable(),
 		BlocksData: sealed.BlocksData{
 			LIDsTable:     &lidsTable,
-			IDsTable:      writer.IDsTable(),
+			IDsTable:      w.IDsTable(),
 			BlocksOffsets: src.BlockOffsets(),
 		},
 	}

compaction/metrics.go

@@ -0,0 +1,46 @@
+package compaction
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/ozontech/seq-db/metric"
+)
+
+var (
+	compactionSkipped = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: "seq_db_store",
+		Subsystem: "compaction",
+		Name:      "skipped_total",
+		Help:      "Tick-triggered tasks dropped because all workers were busy or no candidates were found",
+	})
+
+	compactionBins = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: "seq_db_store",
+		Subsystem: "compaction",
+		Name:      "bins",
+		Help:      "Number of active time-bins considered for compaction",
+	})
+
+	compactionDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "seq_db_store",
+		Subsystem: "compaction",
+		Name:      "duration_seconds",
+		Help:      "Time spent executing a single compaction",
+		Buckets:   metric.SecondsBuckets,
+	}, []string{"bucket"})
+
+	compactionBytesTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "seq_db_store",
+		Subsystem: "compaction",
+		Name:      "bytes_total",
+		Help:      "Total index bytes merged across all compactions",
+	}, []string{"bucket"})
+
+	compactionResultTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "seq_db_store",
+		Subsystem: "compaction",
+		Name:      "result_total",
+		Help:      "Compaction outcomes by result (success, empty, error)",
+	}, []string{"bucket", "result"})
+)