diff --git a/.gitignore b/.gitignore index 2946e4e..1c1219a 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,6 @@ third-party/folly/ output .pre-commit-config.yaml + +# Purger tests read MANIFEST- objects into the working directory +/MANIFEST-* diff --git a/CMakeLists.txt b/CMakeLists.txt index 44caa06..d8145e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -976,11 +976,12 @@ set(SOURCES cloud/cloud_file_system_impl.cc cloud/cloud_log_controller.cc cloud/manifest_reader.cc - # cloud/purge.cc - cloud/improved_purger.cc + cloud/eloq_purger.cc + cloud/file_number_guard.cc cloud/cloud_manifest.cc cloud/cloud_scheduler.cc cloud/cloud_storage_provider.cc + cloud/cloud_file_cache.cc cloud/cloud_file_deletion_scheduler.cc db/db_impl/replication_codec.cc) @@ -1318,6 +1319,9 @@ if(WITH_TESTS) cloud/db_cloud_test.cc cloud/cloud_manifest_test.cc cloud/cloud_scheduler_test.cc + cloud/eloq_purger_test.cc + cloud/eloq_purger_integration_test.cc + cloud/file_number_guard_test.cc cloud/replication_test.cc cache/tiered_secondary_cache_test.cc db/blob/blob_counting_iterator_test.cc diff --git a/Makefile b/Makefile index 235a7e4..a3c71c2 100644 --- a/Makefile +++ b/Makefile @@ -1901,6 +1901,15 @@ cloud_manifest_test: cloud/cloud_manifest_test.o $(TEST_LIBRARY) $(LIBRARY) cloud_scheduler_test: cloud/cloud_scheduler_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) +eloq_purger_test: cloud/eloq_purger_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +file_number_guard_test: cloud/file_number_guard_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +eloq_purger_integration_test: cloud/eloq_purger_integration_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + iostats_context_test: $(OBJ_DIR)/monitoring/iostats_context_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) diff --git a/cloud/aws/aws_s3.cc b/cloud/aws/aws_s3.cc index e317f83..b7b4c92 100644 --- a/cloud/aws/aws_s3.cc +++ b/cloud/aws/aws_s3.cc @@ -18,9 +18,13 @@ #include #include #include +#include #include #include #include +#include +#include +#include #include #include #include @@ -184,6 +188,15 @@ class AwsS3ClientWrapper { return outcome; } + Aws::S3::Model::DeleteObjectsOutcome DeleteCloudObjects( + const Aws::S3::Model::DeleteObjectsRequest& request) { + CloudRequestCallbackGuard t(cloud_request_callback_.get(), + CloudRequestOpType::kDeleteOp); + auto outcome = client_->DeleteObjects(request); + t.SetSuccess(outcome.IsSuccess()); + return outcome; + } + Aws::S3::Model::CopyObjectOutcome CopyCloudObject( const Aws::S3::Model::CopyObjectRequest& request) { CloudRequestCallbackGuard t(cloud_request_callback_.get(), @@ -392,6 +405,10 @@ class S3StorageProvider : public CloudStorageProviderImpl { const std::string& object_path) override; IOStatus DeleteCloudObject(const std::string& bucket_name, const std::string& object_path) override; + IOStatus DeleteCloudObjects(const std::string& bucket_name, + const std::vector& object_paths, + size_t* deleted_count, + size_t* failed_count) override; IOStatus ListCloudObjects(const std::string& bucket_name, const std::string& object_path, std::vector* result) override; @@ -634,6 +651,73 @@ IOStatus S3StorageProvider::DeleteCloudObject(const std::string& bucket_name, return st; } +IOStatus S3StorageProvider::DeleteCloudObjects( + const std::string& bucket_name, + const std::vector& object_paths, size_t* deleted_count, + size_t* failed_count) { + assert(deleted_count != nullptr && failed_count != nullptr); + IOStatus first_error; + // S3 DeleteObjects accepts at most 1000 keys per request. A key that does + // not exist is reported as deleted, matching the interface contract. + constexpr size_t kMaxKeysPerBatch = 1000; + + for (size_t begin = 0; begin < object_paths.size(); + begin += kMaxKeysPerBatch) { + size_t end = begin + kMaxKeysPerBatch; + if (end > object_paths.size()) { + end = object_paths.size(); + } + size_t batch_size = end - begin; + + Aws::S3::Model::Delete to_delete; + for (size_t i = begin; i < end; ++i) { + to_delete.AddObjects( + Aws::S3::Model::ObjectIdentifier().WithKey( + ToAwsString(object_paths[i]))); + } + // Quiet mode: the response lists only the keys that failed. + to_delete.SetQuiet(true); + + Aws::S3::Model::DeleteObjectsRequest request; + request.SetBucket(ToAwsString(bucket_name)); + request.SetDelete(std::move(to_delete)); + + auto outcome = s3client_->DeleteCloudObjects(request); + if (!outcome.IsSuccess()) { + const Aws::Client::AWSError& error = + outcome.GetError(); + std::string errmsg(error.GetMessage().c_str()); + Log(InfoLogLevel::ERROR_LEVEL, cfs_->GetLogger(), + "[s3] DeleteObjects batch of %zu keys failed in bucket %s: %s", + batch_size, bucket_name.c_str(), errmsg.c_str()); + *failed_count += batch_size; + if (first_error.ok()) { + first_error = IOStatus::IOError(bucket_name, errmsg.c_str()); + } + continue; + } + + const auto& errors = outcome.GetResult().GetErrors(); + for (const auto& error : errors) { + Log(InfoLogLevel::ERROR_LEVEL, cfs_->GetLogger(), + "[s3] DeleteObjects failed to delete %s/%s: %s %s", + bucket_name.c_str(), error.GetKey().c_str(), + error.GetCode().c_str(), error.GetMessage().c_str()); + if (first_error.ok()) { + first_error = IOStatus::IOError(std::string(error.GetKey().c_str()), + std::string(error.GetMessage().c_str())); + } + } + *failed_count += errors.size(); + *deleted_count += batch_size - errors.size(); + + Log(InfoLogLevel::INFO_LEVEL, cfs_->GetLogger(), + "[s3] DeleteObjects deleted %zu of %zu keys from bucket %s", + batch_size - errors.size(), batch_size, bucket_name.c_str()); + } + return first_error; +} + // // Appends the names of all children of the specified path from S3 // into the result set. diff --git a/cloud/cloud_file_system.cc b/cloud/cloud_file_system.cc index a1714fa..6d7cad3 100644 --- a/cloud/cloud_file_system.cc +++ b/cloud/cloud_file_system.cc @@ -64,6 +64,12 @@ void CloudFileSystemOptions::Dump(Logger* log) const { create_bucket_if_missing ? "true" : "false"); Header(log, " COptions.run_purger: %s", run_purger ? "true" : "false"); + Header(log, " COptions.publish_file_number_guard: %s", + publish_file_number_guard ? "true" : "false"); + Header(log, " COptions.guard_publish_interval: %llu ms", + static_cast(guard_publish_interval.count())); + Header(log, " COptions.guard_entry_duration: %llu ms", + static_cast(guard_entry_duration.count())); Header(log, " COptions.resync_on_open: %s", resync_on_open ? "true" : "false"); Header(log, " COptions.skip_dbid_verification: %s", @@ -302,6 +308,31 @@ int offset_of(T1 CloudFileSystemOptions::*member) { return int(size_t(&(dummy_ceo_options.*member)) - size_t(&dummy_ceo_options)); } +static OptionTypeInfo MillisecondsOption(int offset) { + return {offset, + OptionType::kInt64T, + OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, void* addr) { + *static_cast(addr) = + std::chrono::milliseconds(ParseInt64(value)); + return Status::OK(); + }, + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const void* addr, std::string* value) { + *value = std::to_string( + static_cast(addr)->count()); + return Status::OK(); + }, + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const void* addr1, const void* addr2, + std::string* /*mismatch*/) { + return *static_cast(addr1) == + *static_cast(addr2); + }}; +} + const std::unordered_map CloudFileSystemOptions::cloud_fs_option_type_info = { {"keep_local_sst_files", @@ -335,6 +366,15 @@ const std::unordered_map {"purger_periodicity_ms", {offset_of(&CloudFileSystemOptions::purger_periodicity_millis), OptionType::kUInt64T}}, + {"publish_file_number_guard", + {offset_of(&CloudFileSystemOptions::publish_file_number_guard), + OptionType::kBoolean}}, + {"guard_publish_interval_ms", + MillisecondsOption( + offset_of(&CloudFileSystemOptions::guard_publish_interval))}, + {"guard_entry_duration_ms", + MillisecondsOption( + offset_of(&CloudFileSystemOptions::guard_entry_duration))}, {"provider", {offset_of(&CloudFileSystemOptions::storage_provider), diff --git a/cloud/cloud_file_system_impl.cc b/cloud/cloud_file_system_impl.cc index 93ec750..be4ad82 100644 --- a/cloud/cloud_file_system_impl.cc +++ b/cloud/cloud_file_system_impl.cc @@ -53,6 +53,9 @@ CloudFileSystemImpl::~CloudFileSystemImpl() { if (cloud_fs_options.cloud_log_controller) { cloud_fs_options.cloud_log_controller->StopTailingStream(); } + // Cancel the file number guard's scheduled publish before the members its + // callback reaches through (storage provider, cloud manifest) go away. + StopFileNumberGuard(); StopPurger(); FileCachePurge(); cloud_fs_options.cloud_log_controller.reset(); @@ -2343,6 +2346,12 @@ Status CloudFileSystemImpl::CheckValidity() const { cloud_fs_options.dest_bucket.GetObjectPath().empty()) { return Status::InvalidArgument( "Must specify both dest bucket name and path"); + } else if (cloud_fs_options.guard_publish_interval.count() <= 0) { + return Status::InvalidArgument( + "guard_publish_interval_ms must be greater than zero"); + } else if (cloud_fs_options.guard_entry_duration.count() <= 0) { + return Status::InvalidArgument( + "guard_entry_duration_ms must be greater than zero"); } else if (!cloud_fs_options.storage_provider) { return Status::InvalidArgument( "Cloud environment requires a storage provider"); diff --git a/cloud/cloud_storage_provider.cc b/cloud/cloud_storage_provider.cc index 6eaeb5f..7f78a2e 100644 --- a/cloud/cloud_storage_provider.cc +++ b/cloud/cloud_storage_provider.cc @@ -5,6 +5,7 @@ #include +#include "cloud/file_number_guard.h" #include "cloud/filename.h" #include "file/filename.h" #include "rocksdb/cloud/cloud_file_system.h" @@ -113,7 +114,6 @@ CloudStorageWritableFileImpl::CloudStorageWritableFileImpl( auto fname_no_epoch = RemoveEpoch(fname_); // Is this a manifest file? is_manifest_ = IsManifestFile(fname_no_epoch); - assert(IsSstFile(fname_no_epoch) || is_manifest_); Log(InfoLogLevel::DEBUG_LEVEL, cfs_->GetLogger(), "[%s] CloudWritableFile bucket %s opened local file %s " @@ -176,7 +176,37 @@ IOStatus CloudStorageWritableFileImpl::Close(const IOOptions& opts, local_file_.reset(); if (!is_manifest_) { - status_ = cfs_->CopyLocalFileToDest(fname_, cloud_fname_); + auto* cfs_impl = dynamic_cast(cfs_); + auto publisher = + cfs_impl == nullptr ? nullptr : cfs_impl->GetFileNumberGuardPublisher(); + if (publisher && !publisher->CoversFile(fname_)) { + // Written outside the DB's own SST directories (a checkpoint copy, a + // column family export): not a DB-visible SST. No flush/compaction job + // registers it and the purger never evaluates it as this DB's live or + // obsolete file, so the guard neither protects nor blocks it. + Log(InfoLogLevel::DEBUG_LEVEL, cfs_->GetLogger(), + "[%s] CloudWritableFile %s is outside the DB directories; skipping " + "file number guard", + Name(), fname_.c_str()); + publisher.reset(); + } + uint64_t file_number = 0; + FileType file_type; + const std::string logical_name = basename(RemoveEpoch(fname_)); + const bool parsed = ParseFileName(logical_name, &file_number, &file_type); + if (publisher && !parsed && IsSstFile(logical_name)) { + status_ = IOStatus::InvalidArgument("cannot parse SST file number", + logical_name); + return status_; + } + if (publisher && parsed && file_type == kTableFile) { + Status protection = publisher->ProtectFileUpload(file_number, [&] { + return cfs_->CopyLocalFileToDest(fname_, cloud_fname_); + }); + status_ = status_to_io_status(std::move(protection)); + } else { + status_ = cfs_->CopyLocalFileToDest(fname_, cloud_fname_); + } if (!status_.ok()) { Log(InfoLogLevel::ERROR_LEVEL, cfs_->GetLogger(), "[%s] CloudWritableFile closing PutObject failed on local file %s", @@ -266,6 +296,26 @@ IOStatus CloudStorageWritableFileImpl::Sync(const IOOptions& opts, CloudStorageProvider::~CloudStorageProvider() {} +IOStatus CloudStorageProvider::DeleteCloudObjects( + const std::string& bucket_name, + const std::vector& object_paths, size_t* deleted_count, + size_t* failed_count) { + assert(deleted_count != nullptr && failed_count != nullptr); + IOStatus first_error; + for (const auto& object_path : object_paths) { + auto st = DeleteCloudObject(bucket_name, object_path); + if (st.ok() || st.IsNotFound()) { + ++(*deleted_count); + } else { + ++(*failed_count); + if (first_error.ok()) { + first_error = st; + } + } + } + return first_error; +} + Status CloudStorageProvider::CreateFromString( const ConfigOptions& /*config_options*/, const std::string& id, std::shared_ptr* provider) { diff --git a/cloud/db_cloud_impl.cc b/cloud/db_cloud_impl.cc index 87c6836..806f3cf 100644 --- a/cloud/db_cloud_impl.cc +++ b/cloud/db_cloud_impl.cc @@ -12,6 +12,7 @@ #include "cache/lru_cache.h" #include "cloud/cloud_manifest.h" #include "cloud/db_cloud_impl.h" +#include "cloud/file_number_guard.h" #include "cloud/filename.h" #include "cloud/manifest_reader.h" #include "env/composite_env_wrapper.h" @@ -60,10 +61,19 @@ class ConstantSizeSstFileManager : public SstFileManagerImpl { }; } // namespace -DBCloudImpl::DBCloudImpl(DB* db, std::unique_ptr local_env) - : DBCloud(db), cfs_(nullptr), local_env_(std::move(local_env)) {} +DBCloudImpl::DBCloudImpl( + DB *db, std::unique_ptr local_env, CloudFileSystemImpl *cfs, + std::shared_ptr guard_publisher) + : DBCloud(db), cfs_(cfs), local_env_(std::move(local_env)), + guard_publisher_(std::move(guard_publisher)) {} DBCloudImpl::~DBCloudImpl() { + if (cfs_ != nullptr && guard_publisher_) { + // Keep this DB's stopped gate installed so SST Close calls during wrapped + // DB destruction fail instead of bypassing protection. A later Open may + // already own the CFS, in which case its publisher must remain untouched. + cfs_->StopFileNumberGuardPublisher(guard_publisher_); + } warm_up_is_running_.store(false, std::memory_order_release); for (auto& thd : warm_up_threads_) { if (thd.joinable()) { @@ -117,6 +127,37 @@ Status DBCloud::Open(const Options& opt, const std::string& local_dbname, if (!cfs->GetLogger()) { cfs->SetLogger(options.info_log); } + + // Protocol enforcement for buckets whose deletion is delegated to the + // purger: every writable DBCloud must publish the file number guard, or the + // purger has no way to tell an in-flight upload from garbage. Checked here + // rather than in CheckValidity because a read-only open creates no SSTs and + // needs no guard. + const auto &guard_check_opts = cfs->GetCloudFileSystemOptions(); + if (!read_only && cfs->HasDestBucket() && + (guard_check_opts.disable_cloud_file_deletion || + guard_check_opts.run_purger) && + !guard_check_opts.publish_file_number_guard) { + return Status::InvalidArgument( + "publish_file_number_guard must be enabled for a writable DBCloud " + "when cloud file deletion is delegated to the purger " + "(disable_cloud_file_deletion or run_purger)"); + } + + // The guard's safety argument requires that a completed flush/compaction + // implies its MANIFEST is durable in the cloud. The MANIFEST only reaches + // cloud storage on Sync(), which disable_manifest_sync skips -- the guard + // would then rise to UINT64_MAX while the cloud MANIFEST still omits + // committed files, and the purger would delete them. + if (!read_only && cfs->HasDestBucket() && + guard_check_opts.publish_file_number_guard && + options.disable_manifest_sync) { + return Status::InvalidArgument( + "disable_manifest_sync is incompatible with " + "publish_file_number_guard: committed SSTs could be purged because " + "the cloud MANIFEST would not reference them"); + } + // Use a constant sized SST File Manager if necesary. // NOTE: if user already passes in an SST File Manager, we will respect user's // SST File Manager instead. @@ -207,6 +248,46 @@ Status DBCloud::Open(const Options& opt, const std::string& local_dbname, // uploaded to S3 for every update, so always enable rolling of Manifest file options.max_manifest_file_size = DBCloudImpl::max_manifest_file_size; + std::shared_ptr publisher; + CloudFileSystemImpl *guard_cfs = nullptr; + // Install a stopped-safe gate immediately before DB::Open. Recovery table + // creation is covered by the sentinel and listener, while the scheduler + // stays off until the database is fully committed in cloud storage. + if (!read_only && cfs->HasDestBucket() && + cfs->GetCloudFileSystemOptions().publish_file_number_guard) { + guard_cfs = dynamic_cast(cfs); + if (guard_cfs == nullptr) { + return Status::InvalidArgument( + "publish_file_number_guard requires CloudFileSystemImpl"); + } + // Every directory in which this DB may place its own SSTs. Files the + // cloud file system sees outside these (checkpoint copies, exports) are + // not DB-visible SSTs and are not gated. + std::vector guard_dirs; + guard_dirs.push_back(local_dbname); + for (const auto& db_path : options.db_paths) { + guard_dirs.push_back(db_path.path); + } + for (const auto& cf : column_families) { + for (const auto& cf_path : cf.options.cf_paths) { + guard_dirs.push_back(cf_path.path); + } + } + publisher = std::make_shared( + guard_cfs, cfs->GetCloudFileSystemOptions().guard_publish_interval, + cfs->GetCloudFileSystemOptions().guard_entry_duration, + std::move(guard_dirs)); + st = guard_cfs->InstallFileNumberGuardPublisher(publisher); + if (!st.ok()) { + Log(InfoLogLevel::ERROR_LEVEL, options.info_log, + "Failed to publish file number guard sentinel: %s", + st.ToString().c_str()); + return st; + } + options.listeners.push_back( + std::make_shared(publisher)); + } + DB* db = nullptr; std::string dbid; if (read_only) { @@ -216,7 +297,15 @@ Status DBCloud::Open(const Options& opt, const std::string& local_dbname, st = DB::Open(options, local_dbname, column_families, handles, &db); } - if (new_db && st.ok() && cfs->HasDestBucket() && + if (!st.ok()) { + if (publisher) { + publisher->Stop(); + guard_cfs->RemoveFileNumberGuardPublisher(publisher); + } + return st; + } + + if (new_db && cfs->HasDestBucket() && cfs->GetCloudFileSystemOptions().roll_cloud_manifest_on_open) { // This is a new database, upload the CLOUDMANIFEST after all MANIFEST file // was already uploaded. It is at this point we consider the database @@ -225,6 +314,31 @@ Status DBCloud::Open(const Options& opt, const std::string& local_dbname, local_dbname, cfs->GetCloudFileSystemOptions().new_cookie_on_open); } + if (!st.ok()) { + if (publisher) { + // Keep the stopped gate installed until the base DB has closed every + // writable file; only then remove this exact publisher. + publisher->Stop(); + } + for (auto *handle : *handles) { + delete handle; + } + handles->clear(); + delete db; + db = nullptr; + if (publisher) { + guard_cfs->RemoveFileNumberGuardPublisher(publisher); + } + return st; + } + + if (publisher) { + // Best effort: the known 0 sentinel remains safe if this initial refresh + // fails, and the scheduler will retry unknown remote state. + publisher->PeriodicPublish(); + publisher->Start(); + } + // now that the database is opened, all file sizes have been verified and we // no longer need to verify file sizes for each file that we open. Note that // this might have a data race with background compaction, but it's not a big @@ -234,11 +348,11 @@ Status DBCloud::Open(const Options& opt, const std::string& local_dbname, false; } - if (st.ok()) { - DBCloudImpl* cloud = new DBCloudImpl(db, std::move(local_env)); - *dbptr = cloud; - db->GetDbIdentity(dbid); - } + DBCloudImpl *cloud = + new DBCloudImpl(db, std::move(local_env), + dynamic_cast(cfs), publisher); + *dbptr = cloud; + db->GetDbIdentity(dbid); Log(InfoLogLevel::INFO_LEVEL, options.info_log, "Opened cloud db with local dir %s dbid %s. %s", local_dbname.c_str(), dbid.c_str(), st.ToString().c_str()); diff --git a/cloud/db_cloud_impl.h b/cloud/db_cloud_impl.h index 96af555..8fece6e 100644 --- a/cloud/db_cloud_impl.h +++ b/cloud/db_cloud_impl.h @@ -16,6 +16,8 @@ namespace ROCKSDB_NAMESPACE { class Env; +class CloudFileSystemImpl; +class FileNumberGuardPublisher; // // All writes to this DB can be configured to be persisted @@ -36,8 +38,9 @@ class DBCloudImpl : public DBCloud { Status GetCurrentEpoch(std::string *epoch) const override; protected: - // The CloudFileSystem used by this open instance. - CloudFileSystem* cfs_; + // Non-owning; the environment keeps the CloudFileSystem alive through DB + // teardown. + CloudFileSystemImpl* cfs_; private: Status DoCheckpointToCloud(const BucketOptions& destination, @@ -46,9 +49,13 @@ class DBCloudImpl : public DBCloud { // Maximum manifest file size static const uint64_t max_manifest_file_size = 4 * 1024L * 1024L; - DBCloudImpl(DB* db, std::unique_ptr local_env); + DBCloudImpl(DB *db, std::unique_ptr local_env, CloudFileSystemImpl *cfs, + std::shared_ptr guard_publisher); std::unique_ptr local_env_; + // Identifies the publisher installed by this DB instance. Teardown must not + // stop a publisher installed by a later DBCloud::Open on the same CFS. + std::shared_ptr guard_publisher_; std::atomic warm_up_is_running_{false}; std::vector warm_up_threads_; diff --git a/cloud/eloq_purger.cc b/cloud/eloq_purger.cc index 4bdeb50..41a54b9 100644 --- a/cloud/eloq_purger.cc +++ b/cloud/eloq_purger.cc @@ -41,6 +41,7 @@ #include "cloud/cloud_manifest.h" #include "cloud/eloq_purger.h" +#include "cloud/file_number_guard.h" #include "cloud/filename.h" #include "cloud/manifest_reader.h" #include "file/filename.h" @@ -51,14 +52,60 @@ namespace ROCKSDB_NAMESPACE { +namespace { + +bool HasReachedAge(uint64_t now, uint64_t mtime, uint64_t threshold) { + return now > mtime && now - mtime >= threshold; +} + +// Largest control object we will read. These objects hold a single decimal +// uint64 (20 digits max); anything larger is malformed by definition and must +// not be slurped into memory. +constexpr size_t kMaxControlObjectBytes = 64; + +// Strict decimal uint64 parse for control objects that gate deletion. +// +// std::stoull is unusable here: it skips leading whitespace, accepts a sign +// (so "-1" yields UINT64_MAX -- a threshold that authorizes deleting every +// non-live SST of a live epoch), and ignores trailing garbage. Require a +// non-empty run of ASCII digits, full consumption, and no overflow. A single +// trailing newline is tolerated because writers commonly append one. +bool ParseStrictUint64(const std::string &input, uint64_t *value) { + size_t end = input.size(); + while (end > 0 && (input[end - 1] == '\n' || input[end - 1] == '\r')) { + --end; + } + if (end == 0 || end > 20) { + return false; + } + uint64_t parsed = 0; + for (size_t i = 0; i < end; ++i) { + const char c = input[i]; + if (c < '0' || c > '9') { + return false; + } + const uint64_t digit = static_cast(c - '0'); + if (parsed > (std::numeric_limits::max() - digit) / 10) { + return false; // overflow + } + parsed = parsed * 10 + digit; + } + *value = parsed; + return true; +} + +} // namespace + S3FileNumberReader::S3FileNumberReader(const std::string &bucket_name, const std::string &s3_object_path, const std::string &epoch, - CloudFileSystemImpl *cfs) + CloudFileSystemImpl *cfs, + bool require_guard_marker) : bucket_name_(bucket_name), s3_object_path_(s3_object_path), epoch_(epoch), - cfs_(cfs) {} + cfs_(cfs), + require_guard_marker_(require_guard_marker) {} Status S3FileNumberReader::ReadSmallestFileNumber(uint64_t *file_number) { std::string object_key = GetS3ObjectKey(); @@ -88,16 +135,42 @@ Status S3FileNumberReader::ReadSmallestFileNumber(uint64_t *file_number) { bucket_name_, object_key, temp_file_path); if (!s.ok()) { + std::remove(temp_file_path.c_str()); Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, "Failed to read smallest file number from S3: %s, object_key: %s, ", s.ToString().c_str(), object_key.c_str()); - // For snapshots and branching, the smallest file number object might not - // exist yet. In that case, we read the max file number from the MANIFEST - // file as the smallest file number. + if (!s.IsNotFound()) { + // Transient/unknown failure. Do NOT fall back to the MANIFEST-derived + // number: that is a high watermark of allocated file numbers and can + // exceed in-flight compaction outputs, so substituting it for the + // published low watermark risks deleting a file that is about to be + // committed. Propagate the error so the purge cycle aborts. + *file_number = std::numeric_limits::min(); + return Status::IOError(s.ToString()); + } + if (require_guard_marker_) { + // The protocol requires every writable DBCloud on a purged bucket to + // publish this marker before it can upload an SST. Absence therefore + // means an unguarded or out-of-date writer may be running, and we + // cannot tell it apart from a writer-less epoch. Falling back to the + // MANIFEST high watermark here is what allows an in-flight compaction + // output to be deleted, so refuse instead. + Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, + "[pg] No file number guard marker for epoch %s (%s); aborting purge " + "cycle. Every writable DBCloud on this bucket must set " + "publish_file_number_guard.", + epoch_.c_str(), object_key.c_str()); + *file_number = std::numeric_limits::min(); + return Status::NotFound("Missing file number guard marker", object_key); + } + // Legacy (staged-rollout) behavior: treat marker absence as a + // snapshot/branch epoch with no active writer and derive a threshold from + // its MANIFEST. Unsafe if an unguarded writer is in fact active. uint64_t manifest_max_file_number = 0; const std::string manifest_file_name = ManifestFileWithEpoch(epoch_); Status status = ManifestReader::GetMaxFileNumberFromManifest( - cfs_, manifest_file_name, &manifest_max_file_number); + cfs_, manifest_file_name, &manifest_max_file_number, + kManifestScanStrictMode); if (status.ok()) { if (manifest_max_file_number > 0) { manifest_max_file_number -= 1; @@ -117,10 +190,7 @@ Status S3FileNumberReader::ReadSmallestFileNumber(uint64_t *file_number) { } *file_number = std::numeric_limits::min(); - if (s.IsNotFound()) { - return Status::NotFound("Smallest file number object not found"); - } - return Status::IOError(s.ToString()); + return Status::NotFound("Smallest file number object not found"); } // Read the content of the temp file @@ -135,8 +205,12 @@ Status S3FileNumberReader::ReadSmallestFileNumber(uint64_t *file_number) { return Status::IOError("Failed to open temp file"); } - std::string content((std::istreambuf_iterator(temp_file)), - std::istreambuf_iterator()); + // Bounded read: this object holds a single decimal uint64. + std::string content; + content.resize(kMaxControlObjectBytes + 1); + temp_file.read(&content[0], static_cast(content.size())); + content.resize(static_cast(temp_file.gcount())); + const bool oversized = content.size() > kMaxControlObjectBytes; temp_file.close(); // Remove the temp file @@ -145,41 +219,48 @@ Status S3FileNumberReader::ReadSmallestFileNumber(uint64_t *file_number) { "Warning: Failed to remove temp file %s", temp_file_path.c_str()); } - try { - *file_number = std::stoull(content); - Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, - "Read smallest file number from S3: %llu, object_key: %s", - static_cast(*file_number), object_key.c_str()); - return Status::OK(); - } catch (const std::exception &e) { + uint64_t parsed = 0; + if (oversized || !ParseStrictUint64(content, &parsed)) { + // A malformed guard object cannot be interpreted safely: a permissive + // parse of "-1" would yield UINT64_MAX and authorize deleting every + // non-live SST of this epoch. Fail the cycle instead. Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, - "Failed to parse smallest file number from S3 content: '%s', " - "returning UINT64_MIN", - content.c_str()); + "[pg] Malformed smallest file number object %s (%zu bytes): '%s'; " + "aborting purge cycle", + object_key.c_str(), content.size(), + oversized ? "" : content.c_str()); *file_number = std::numeric_limits::min(); - return Status::Corruption("Failed to parse smallest file number: %s", - e.what()); + return Status::Corruption("Malformed smallest file number object", + object_key); } + + *file_number = parsed; + Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, + "Read smallest file number from S3: %llu, object_key: %s", + static_cast(*file_number), object_key.c_str()); + return Status::OK(); } std::string S3FileNumberReader::GetS3ObjectKey() const { - std::ostringstream oss; - oss << s3_object_path_; - if (!s3_object_path_.empty() && s3_object_path_.back() != '/') { - oss << "/"; - } - oss << "smallest_new_file_number-" << epoch_; - return oss.str(); + // Shared with the writer (FileNumberGuardPublisher): one definition of the + // guard object's key for both sides of the protocol. + return SmallestFileNumberObjectKey(s3_object_path_, epoch_); } EloqPurger::EloqPurger(CloudFileSystemImpl *cfs, const std::string &bucket_name, const std::string &object_path, bool dry_run, - uint64_t cloudmanifest_retention_ms) + uint64_t cloudmanifest_retention_ms, + uint64_t dead_epoch_file_age_ms, + uint64_t max_deletions_per_cycle, + bool require_guard_marker) : cfs_(cfs), bucket_name_(bucket_name), object_path_(object_path), dry_run_(dry_run), - cloudmanifest_retention_ms_(cloudmanifest_retention_ms) {} + cloudmanifest_retention_ms_(cloudmanifest_retention_ms), + dead_epoch_file_age_ms_(dead_epoch_file_age_ms), + max_deletions_per_cycle_(max_deletions_per_cycle), + require_guard_marker_(require_guard_marker) {} bool EloqPurger::RunSinglePurgeCycle() { PurgerCycleState state; @@ -230,21 +311,50 @@ bool EloqPurger::RunSinglePurgeCycle() { return false; } - // Enhanced selection with file number checking + // Read the S3-derived clock once; it serves every age-based decision this + // cycle (dead-epoch reclamation and CLOUDMANIFEST retention). + if (!GetS3CurrentTime(&state.s3_current_time).ok()) { + Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, + "[pg] Failed to get S3 current time, aborting purge cycle"); + return false; + } + + // The cap consumes a deterministic SST -> MANIFEST -> CLOUDMANIFEST -> + // marker prefix. Metadata can wait behind an SST backlog, but the remainder + // is re-selected and converges on later cycles. SelectObsoleteSSTFilesWithThreshold(state.all_files, state.live_file_names, state.file_number_thresholds, + state.s3_current_time, &state.obsolete_files); // Select obsolete manifest files SelectObsoleteManifestFiles(state.all_files, state.current_epoch_manifest_files, - &state.obsolete_files); + state.s3_current_time, &state.obsolete_files); + + // Select obsolete CLOUDMANIFEST files. A malformed CLOUDMANIFEST name + // aborts the cycle: we cannot reason about generational lineage from a name + // we cannot parse, and guessing risks retiring the authoritative one. + if (!SelectObsoleteCloudManifestFiles( + state.all_files, state.cloudmanifests, + state.current_epoch_manifest_files, state.s3_current_time, + &state.obsolete_files) + .ok()) { + Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, + "[pg] Failed to select obsolete CLOUDMANIFEST files, aborting purge " + "cycle"); + return false; + } - // Select obsolete CLOUDMANIFEST files - SelectObsoleteCloudManifestFiles(state.all_files, state.cloudmanifests, - state.current_epoch_manifest_files, - &state.obsolete_files); + // Select smallest_new_file_number markers of dead epochs + SelectObsoleteFileNumberMarkers(state.all_files, + state.file_number_thresholds, + state.s3_current_time, + &state.obsolete_files); + Status deletion_status; + size_t deleted = 0; + size_t failures = 0; if (dry_run_) { Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, "[pg] DRY RUN: Would delete %zu files", state.obsolete_files.size()); @@ -253,17 +363,20 @@ bool EloqPurger::RunSinglePurgeCycle() { "[pg] DRY RUN: Would delete %s", file.c_str()); } } else { - DeleteObsoleteFiles(state.obsolete_files); + deletion_status = + DeleteObsoleteFiles(state.obsolete_files, &deleted, &failures); } Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, "[pg] Purge cycle summary: total_files=%zu manifests=%zu " - "live_files=%zu obsolete_selected=%zu thresholds_loaded=%zu", + "live_files=%zu obsolete_selected=%zu deleted=%zu failed=%zu " + "thresholds_loaded=%zu", state.all_files.size(), state.cloudmanifests.size(), - state.live_file_names.size(), state.obsolete_files.size(), + state.live_file_names.size(), state.obsolete_files.size(), deleted, + failures, state.file_number_thresholds.size()); - return true; + return deletion_status.ok(); } Status EloqPurger::ListAllFiles(PurgerAllFiles *all_files) { @@ -371,8 +484,12 @@ Status EloqPurger::CollectLiveFiles( (*current_epoch_manifest_infos)[current_epoch] = manifest_file_info; + // Absolute consistency: a tolerated corrupt tail would silently drop + // live-file additions from this set, and those files would then be + // selected for deletion. s = manifest_reader->GetLiveFiles(object_path_, current_epoch, - &live_file_numbers); + &live_file_numbers, + kManifestScanStrictMode); if (!s.ok()) { Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, "[pg] Failed to get live files from cloud manifest file %s: %s", @@ -403,7 +520,7 @@ Status EloqPurger::LoadFileNumberThresholds( // Create S3 file number updater to read threshold auto s3_updater = std::make_unique( - bucket_name_, object_path_, epoch, cfs_); + bucket_name_, object_path_, epoch, cfs_, require_guard_marker_); uint64_t threshold; Status s = s3_updater->ReadSmallestFileNumber(&threshold); @@ -426,7 +543,7 @@ Status EloqPurger::LoadFileNumberThresholds( void EloqPurger::SelectObsoleteSSTFilesWithThreshold( const PurgerAllFiles &all_files, const PurgerLiveFileSet &live_files, - const PurgerFileNumberThresholds &thresholds, + const PurgerFileNumberThresholds &thresholds, uint64_t s3_current_time, std::vector *obsolete_files) { for (const auto &candidate : all_files) { const std::string &candidate_file_path = candidate.first; @@ -447,6 +564,7 @@ void EloqPurger::SelectObsoleteSSTFilesWithThreshold( auto threshold_it = thresholds.find(candidate_epoch); if (threshold_it != thresholds.end()) { uint64_t threshold = threshold_it->second; + // Zero is the intentional full-block sentinel for a live epoch. if (threshold != std::numeric_limits::min()) { // Extract file number from candidate file name uint64_t file_number = 0; @@ -472,10 +590,30 @@ void EloqPurger::SelectObsoleteSSTFilesWithThreshold( } } } else { - Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, - "[pg] threshold is 0 for epoch %s, purge is blocked intentionally. " - "%s is skipped.", - candidate_epoch.c_str(), candidate_file_path.c_str()); + // No threshold entry means no loaded CLOUDMANIFEST claims this epoch + // as its current epoch: the epoch is dead, no writer can ever add + // files to it, so a non-live file is garbage. The age guard covers + // the one race this reasoning misses: a node mid-open (or + // mid-branch-creation) whose SSTs were uploaded before our listing + // but whose CLOUDMANIFEST landed after it. + uint64_t file_mtime = candidate.second.modification_time; + if (HasReachedAge(s3_current_time, file_mtime, dead_epoch_file_age_ms_)) { + obsolete_files->push_back(candidate_file_path); + Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, + "[pg] File %s selected for deletion (dead epoch %s, " + "file_mtime=%llu, s3_current_time=%llu)", + candidate_file_path.c_str(), candidate_epoch.c_str(), + static_cast(file_mtime), + static_cast(s3_current_time)); + } else { + Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, + "[pg] Keeping file %s: epoch %s has no threshold and file is " + "younger than %llu ms (file_mtime=%llu, s3_current_time=%llu)", + candidate_file_path.c_str(), candidate_epoch.c_str(), + static_cast(dead_epoch_file_age_ms_), + static_cast(file_mtime), + static_cast(s3_current_time)); + } } } } @@ -483,7 +621,7 @@ void EloqPurger::SelectObsoleteSSTFilesWithThreshold( void EloqPurger::SelectObsoleteManifestFiles( const PurgerAllFiles &all_files, const PurgerEpochManifestMap ¤t_epoch_manifest_infos, - std::vector *obsolete_files) { + uint64_t s3_current_time, std::vector *obsolete_files) { for (const auto &candidate : all_files) { const std::string &candidate_file_path = candidate.first; @@ -500,10 +638,31 @@ void EloqPurger::SelectObsoleteManifestFiles( continue; } - obsolete_files->push_back(candidate_file_path); - Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, - "[pg] Manifest file %s selected for deletion", - candidate_file_path.c_str()); + // A node rolling a new epoch uploads MANIFEST- BEFORE the + // CLOUDMANIFEST that makes the epoch current, so a purge cycle that + // lists between the two uploads sees a MANIFEST with no protecting + // CLOUDMANIFEST. The age guard keeps such a fresh MANIFEST until the + // open either commits (epoch becomes current) or is abandoned. + uint64_t manifest_mtime = candidate.second.modification_time; + if (HasReachedAge(s3_current_time, manifest_mtime, + dead_epoch_file_age_ms_)) { + obsolete_files->push_back(candidate_file_path); + Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, + "[pg] Manifest file %s selected for deletion (dead epoch %s, " + "manifest_mtime=%llu, s3_current_time=%llu)", + candidate_file_path.c_str(), candidate_epoch.c_str(), + static_cast(manifest_mtime), + static_cast(s3_current_time)); + } else { + Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, + "[pg] Keeping manifest file %s: epoch %s is not current but the " + "manifest is younger than %llu ms (manifest_mtime=%llu, " + "s3_current_time=%llu)", + candidate_file_path.c_str(), candidate_epoch.c_str(), + static_cast(dead_epoch_file_age_ms_), + static_cast(manifest_mtime), + static_cast(s3_current_time)); + } } } @@ -559,7 +718,6 @@ Status EloqPurger::GetS3CurrentTime(uint64_t *current_time) { return Status::IOError(s.ToString()); } - // Get the metadata to read the timestamp CloudObjectInformation file_info; s = cfs_->GetStorageProvider()->GetCloudObjectMetadata( bucket_name_, temp_s3_path, &file_info); @@ -599,23 +757,26 @@ Status EloqPurger::GetS3CurrentTime(uint64_t *current_time) { return Status::OK(); } -void EloqPurger::SelectObsoleteCloudManifestFiles( +Status EloqPurger::SelectObsoleteCloudManifestFiles( const PurgerAllFiles &all_files, const PurgerCloudManifestMap &cloudmanifests, const PurgerEpochManifestMap ¤t_epoch_manifest_infos, - std::vector *obsolete_files) { + uint64_t s3_current_time, std::vector *obsolete_files) { // Struct to represent CLOUDMANIFEST file information struct CloudManifestFileInfo { uint64_t term; std::string file_path; - uint64_t current_manifest_timestamp; + // The CLOUDMANIFEST object's own S3 mtime. A CLOUDMANIFEST is written + // once, when its generation starts, so the max-term entry's mtime is + // the moment the older terms in the group were superseded. + uint64_t cloudmanifest_timestamp; std::string epoch; CloudManifestFileInfo(uint64_t t, const std::string &path, - uint64_t manifest_ts, const std::string &ep) + uint64_t cloudmanifest_ts, const std::string &ep) : term(t), file_path(path), - current_manifest_timestamp(manifest_ts), + cloudmanifest_timestamp(cloudmanifest_ts), epoch(ep) {} }; @@ -645,7 +806,8 @@ void EloqPurger::SelectObsoleteCloudManifestFiles( std::string current_epoch = manifest_it->second->GetCurrentEpoch(); - // Look up the current manifest file timestamp for this epoch + // Preserve the current-epoch manifest lookup: missing info means this + // CLOUDMANIFEST is not safe to consider for deletion this cycle. auto manifest_info_it = current_epoch_manifest_infos.find(current_epoch); if (manifest_info_it == current_epoch_manifest_infos.end()) { Log(InfoLogLevel::WARN_LEVEL, cfs_->info_log_, @@ -655,9 +817,6 @@ void EloqPurger::SelectObsoleteCloudManifestFiles( continue; } - uint64_t current_manifest_timestamp = - manifest_info_it->second.modification_time; - // Extract the part after "CLOUDMANIFEST-" std::string remainder = candidate_file_path.substr(prefix.length()); @@ -677,41 +836,36 @@ void EloqPurger::SelectObsoleteCloudManifestFiles( term_str = remainder.substr(last_dash + 1); } - // Validate that term is a number + // Validate that term is a number. A permissive parse is dangerous in + // both directions: "9999x" would read as 9999 and could make a bogus + // object the group's max term, retiring the authoritative CLOUDMANIFEST; + // silently skipping an unparseable name hides an object whose lineage we + // cannot reason about. Abort the cycle instead. uint64_t term = 0; - try { - term = std::stoull(term_str); - } catch (const std::exception &e) { - // Not a valid pattern, skip this file - Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, - "[pg] Skipping CLOUDMANIFEST file %s (invalid term: %s)", + if (!ParseStrictUint64(term_str, &term)) { + Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, + "[pg] CLOUDMANIFEST file %s has a malformed term '%s'; aborting " + "purge cycle", candidate_file_path.c_str(), term_str.c_str()); - continue; + return Status::Corruption("Malformed CLOUDMANIFEST term", + candidate_file_path); } // Group by postfix - grouped_manifests[postfix].emplace_back( - term, candidate_file_path, current_manifest_timestamp, current_epoch); + grouped_manifests[postfix].emplace_back(term, candidate_file_path, + candidate.second.modification_time, + current_epoch); Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, "[pg] Found CLOUDMANIFEST file %s with postfix='%s', term=%llu, " - "manifest_timestamp=%llu, epoch=%s", + "cloudmanifest_timestamp=%llu, epoch=%s", candidate_file_path.c_str(), postfix.c_str(), static_cast(term), - static_cast(current_manifest_timestamp), + static_cast(candidate.second.modification_time), current_epoch.c_str()); } - // Get current timestamp from S3 - uint64_t current_time = 0; - Status time_status = GetS3CurrentTime(¤t_time); - if (!time_status.ok()) { - Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, - "[pg] Failed to get S3 current time, aborting CLOUDMANIFEST cleanup: " - "%s", - time_status.ToString().c_str()); - return; - } + const uint64_t current_time = s3_current_time; // Use configurable retention time (in milliseconds) const uint64_t retention_threshold_ms = cloudmanifest_retention_ms_; @@ -733,10 +887,20 @@ void EloqPurger::SelectObsoleteCloudManifestFiles( }); uint64_t max_term = max_it->term; + // A CLOUDMANIFEST is written once, at generation start, so the max-term + // entry's own mtime is the moment every older term in this group was + // superseded. Retention is measured from that supersession, NOT from the + // old generation's MANIFEST mtime: the latter is a last-write time, and + // a write-idle (e.g. read-only) generation would look expired the + // instant it is superseded, stripping a still-running old primary of + // live-file and threshold protection with no grace period. + uint64_t supersession_time = max_it->cloudmanifest_timestamp; Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, - "[pg] CLOUDMANIFEST group postfix='%s': largest term=%llu", - postfix.c_str(), static_cast(max_term)); + "[pg] CLOUDMANIFEST group postfix='%s': largest term=%llu, " + "superseded older terms at %llu", + postfix.c_str(), static_cast(max_term), + static_cast(supersession_time)); // Check each file in the group for (const auto &file_info : files) { @@ -748,73 +912,121 @@ void EloqPurger::SelectObsoleteCloudManifestFiles( continue; } - // Compare current MANIFEST file timestamp with S3 current time - // If MANIFEST timestamp is earlier than S3 current time by the retention - // threshold or more, delete the CLOUDMANIFEST - if (current_time > file_info.current_manifest_timestamp && - (current_time - file_info.current_manifest_timestamp) >= - retention_threshold_ms) { + // Delete the superseded CLOUDMANIFEST once the successor has existed + // for the full retention window, giving the superseded generation a + // guaranteed grace period regardless of how long it had been + // write-idle before the failover. + if (HasReachedAge(current_time, supersession_time, + retention_threshold_ms)) { obsolete_files->push_back(file_info.file_path); Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, "[pg] CLOUDMANIFEST file %s selected for deletion " - "(term=%llu, manifest_timestamp=%llu, s3_current_time=%llu, " - "epoch=%s, time_diff=%llu ms)", + "(term=%llu, supersession_time=%llu, s3_current_time=%llu, " + "epoch=%s, superseded_for=%llu ms)", file_info.file_path.c_str(), static_cast(file_info.term), - static_cast( - file_info.current_manifest_timestamp), + static_cast(supersession_time), static_cast(current_time), file_info.epoch.c_str(), - static_cast( - current_time - file_info.current_manifest_timestamp)); + static_cast(current_time - supersession_time)); } else { - uint64_t time_diff = - (current_time > file_info.current_manifest_timestamp) - ? (current_time - file_info.current_manifest_timestamp) - : 0; + uint64_t time_diff = (current_time > supersession_time) + ? (current_time - supersession_time) + : 0; Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, "[pg] Keeping CLOUDMANIFEST file %s " - "(term=%llu, manifest_timestamp=%llu, s3_current_time=%llu, " - "epoch=%s, time_diff=%llu ms < retention threshold %llu ms)", + "(term=%llu, supersession_time=%llu, s3_current_time=%llu, " + "epoch=%s, superseded_for=%llu ms < retention threshold %llu " + "ms)", file_info.file_path.c_str(), static_cast(file_info.term), - static_cast( - file_info.current_manifest_timestamp), + static_cast(supersession_time), static_cast(current_time), file_info.epoch.c_str(), static_cast(time_diff), static_cast(retention_threshold_ms)); } } } + return Status::OK(); } -void EloqPurger::DeleteObsoleteFiles( - const std::vector &obsolete_files) { - size_t deleted = 0; - size_t failures = 0; +void EloqPurger::SelectObsoleteFileNumberMarkers( + const PurgerAllFiles &all_files, + const PurgerFileNumberThresholds &thresholds, uint64_t s3_current_time, + std::vector *obsolete_files) { + // The writer publishes one smallest_new_file_number- object per + // epoch it writes in. Markers of living epochs (present in `thresholds`, + // which holds one entry per loaded CLOUDMANIFEST's current epoch) are in + // use; markers of dead epochs are garbage. The age guard covers a node + // mid-open that published its marker before its CLOUDMANIFEST landed. + const std::string marker_prefix = kSmallestFileNumberFilePrefix; + for (const auto &candidate : all_files) { + const std::string &candidate_file_path = candidate.first; - for (const auto &file_to_delete : obsolete_files) { - std::string file_path = object_path_ + "/" + file_to_delete; - Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, - "[pg] Deleting obsolete file %s from destination bucket", - file_to_delete.c_str()); + if (candidate_file_path.compare(0, marker_prefix.size(), marker_prefix) != + 0) { + continue; + } - IOStatus s = - cfs_->GetStorageProvider()->DeleteCloudObject(bucket_name_, file_path); - if (!s.ok()) { - ++failures; - Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, - "[pg] Failed to delete obsolete file %s: %s", file_path.c_str(), - s.ToString().c_str()); + std::string epoch = candidate_file_path.substr(marker_prefix.size()); + if (thresholds.find(epoch) != thresholds.end()) { + continue; // living epoch, marker still in use + } + + uint64_t marker_mtime = candidate.second.modification_time; + if (HasReachedAge(s3_current_time, marker_mtime, dead_epoch_file_age_ms_)) { + obsolete_files->push_back(candidate_file_path); + Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, + "[pg] File number marker %s selected for deletion (dead epoch %s)", + candidate_file_path.c_str(), epoch.c_str()); } else { - ++deleted; + Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, + "[pg] Keeping file number marker %s: epoch %s has no threshold but " + "marker is younger than %llu ms", + candidate_file_path.c_str(), epoch.c_str(), + static_cast(dead_epoch_file_age_ms_)); } } +} + +Status EloqPurger::DeleteObsoleteFiles( + const std::vector &obsolete_files, size_t *deleted, + size_t *failures) { + + size_t to_delete = obsolete_files.size(); + if (max_deletions_per_cycle_ > 0 && to_delete > max_deletions_per_cycle_) { + to_delete = max_deletions_per_cycle_; + Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, + "[pg] Deletion cap reached: deleting %zu of %zu selected files this " + "cycle; the remainder will be re-selected next cycle", + to_delete, obsolete_files.size()); + } + + std::vector paths_to_delete; + paths_to_delete.reserve(to_delete); + for (size_t i = 0; i < to_delete; ++i) { + Log(InfoLogLevel::INFO_LEVEL, cfs_->info_log_, + "[pg] Deleting obsolete file %s from destination bucket", + obsolete_files[i].c_str()); + paths_to_delete.push_back(object_path_ + "/" + obsolete_files[i]); + } + + IOStatus s = cfs_->GetStorageProvider()->DeleteCloudObjects( + bucket_name_, paths_to_delete, deleted, failures); + if (!s.ok()) { + Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, + "[pg] Obsolete deletion failed: selected=%zu requested=%zu " + "deleted=%zu failures=%zu: %s", + obsolete_files.size(), to_delete, *deleted, *failures, + s.ToString().c_str()); + return s; + } Log(InfoLogLevel::DEBUG_LEVEL, cfs_->info_log_, - "[pg] Obsolete deletion summary: requested=%zu deleted=%zu " - "failures=%zu", - obsolete_files.size(), deleted, failures); + "[pg] Obsolete deletion summary: selected=%zu requested=%zu " + "deleted=%zu failures=%zu", + obsolete_files.size(), to_delete, *deleted, *failures); + return Status::OK(); } // ------------- Main purger thread ------------- // diff --git a/cloud/eloq_purger.h b/cloud/eloq_purger.h index 1e4a06c..9eb8d8d 100644 --- a/cloud/eloq_purger.h +++ b/cloud/eloq_purger.h @@ -47,9 +47,13 @@ bool PrerequisitesMet(const CloudFileSystemImpl &cfs); */ class S3FileNumberReader { public: + // When require_guard_marker is true, a missing marker is an error rather + // than a cue to fall back to the MANIFEST-derived high watermark. See + // ReadSmallestFileNumber. S3FileNumberReader(const std::string &bucket_name, const std::string &s3_object_path, - const std::string &epoch, CloudFileSystemImpl *cfs); + const std::string &epoch, CloudFileSystemImpl *cfs, + bool require_guard_marker = true); ~S3FileNumberReader() = default; @@ -64,6 +68,7 @@ class S3FileNumberReader { std::string s3_object_path_; std::string epoch_; CloudFileSystemImpl *cfs_; + bool require_guard_marker_; std::string GetS3ObjectKey() const; }; @@ -93,23 +98,62 @@ class EloqPurger { PurgerFileNumberThresholds file_number_thresholds; // NEW: epoch -> min file number std::vector obsolete_files; + uint64_t s3_current_time = 0; // S3-derived clock (ms), read once per cycle }; EloqPurger(CloudFileSystemImpl *cfs, const std::string &bucket_name, const std::string &object_path, bool dry_run, - uint64_t cloudmanifest_retention_ms = 3600 * 1000); + uint64_t cloudmanifest_retention_ms = 3600 * 1000, + uint64_t dead_epoch_file_age_ms = 3600 * 1000, + uint64_t max_deletions_per_cycle = 10000, + bool require_guard_marker = true); /** * @brief Run a single purge cycle with improved file number checking */ bool RunSinglePurgeCycle(); + // In-memory selection helpers (no cloud IO). They append decisions and emit + // diagnostic logs; public so unit tests can exercise them directly. + void SelectObsoleteSSTFilesWithThreshold( + const PurgerAllFiles &all_files, const PurgerLiveFileSet &live_files, + const PurgerFileNumberThresholds &thresholds, uint64_t s3_current_time, + std::vector *obsolete_files); + void SelectObsoleteManifestFiles( + const PurgerAllFiles &all_files, + const PurgerEpochManifestMap ¤t_epoch_manifest_infos, + uint64_t s3_current_time, std::vector *obsolete_files); + Status SelectObsoleteCloudManifestFiles( + const PurgerAllFiles &all_files, + const PurgerCloudManifestMap &cloudmanifests, + const PurgerEpochManifestMap ¤t_epoch_manifest_infos, + uint64_t s3_current_time, std::vector *obsolete_files); + void SelectObsoleteFileNumberMarkers( + const PurgerAllFiles &all_files, + const PurgerFileNumberThresholds &thresholds, uint64_t s3_current_time, + std::vector *obsolete_files); + private: CloudFileSystemImpl *cfs_; std::string bucket_name_; std::string object_path_; bool dry_run_; uint64_t cloudmanifest_retention_ms_; + // Minimum object age before a non-live file in a dead epoch (an epoch that + // is not any loaded CLOUDMANIFEST's current epoch) may be deleted. Covers + // the race with a node mid-open/mid-branch-creation whose files were + // uploaded before our listing but whose CLOUDMANIFEST landed after it. + uint64_t dead_epoch_file_age_ms_; + // Upper bound on deletions per cycle; the remainder is re-selected next + // cycle. 0 means unlimited. + uint64_t max_deletions_per_cycle_; + // Every writable DBCloud that shares a purged bucket must publish a + // smallest_new_file_number marker for its epoch (a 0 sentinel is written + // before recovery can flush). With this set, a missing marker for a live + // epoch means the protocol is not being honored -- an unguarded or + // out-of-date writer -- and the cycle aborts rather than falling back to a + // threshold that is unsafe for an active writer. + bool require_guard_marker_; Status ListAllFiles(PurgerAllFiles *all_files); Status ListCloudManifests(std::vector *cloud_manifest_files); @@ -121,21 +165,9 @@ class EloqPurger { PurgerEpochManifestMap *current_epoch_manifest_infos); Status LoadFileNumberThresholds(const PurgerCloudManifestMap &cloudmanifests, PurgerFileNumberThresholds *thresholds); - void SelectObsoleteSSTFilesWithThreshold( - const PurgerAllFiles &all_files, const PurgerLiveFileSet &live_files, - const PurgerFileNumberThresholds &thresholds, - std::vector *obsolete_files); - void SelectObsoleteManifestFiles( - const PurgerAllFiles &all_files, - const PurgerEpochManifestMap ¤t_epoch_manifest_infos, - std::vector *obsolete_files); - void SelectObsoleteCloudManifestFiles( - const PurgerAllFiles &all_files, - const PurgerCloudManifestMap &cloudmanifests, - const PurgerEpochManifestMap ¤t_epoch_manifest_infos, - std::vector *obsolete_files); Status GetS3CurrentTime(uint64_t *current_time); - void DeleteObsoleteFiles(const std::vector &obsolete_files); + Status DeleteObsoleteFiles(const std::vector &obsolete_files, + size_t *deleted, size_t *failures); }; } // namespace ROCKSDB_NAMESPACE diff --git a/cloud/eloq_purger_command.cc b/cloud/eloq_purger_command.cc index b4243c8..ba528ca 100644 --- a/cloud/eloq_purger_command.cc +++ b/cloud/eloq_purger_command.cc @@ -88,6 +88,19 @@ DEFINE_string(aws_secret_key, "", "AWS Secret Access Key"); DEFINE_uint64(cloudmanifest_retention_ms, 3600 * 1000, "Time threshold in milliseconds for CLOUDMANIFEST file retention " "(default: 3600000 ms = 1 hour)"); +DEFINE_uint64(dead_epoch_file_age_ms, 3600 * 1000, + "Minimum S3 object age in milliseconds before a non-live file " + "in a dead epoch (an epoch no CLOUDMANIFEST claims as current) " + "is deleted (default: 3600000 ms = 1 hour)"); +DEFINE_bool(require_guard_marker, true, + "Abort the purge cycle when a live epoch has no " + "smallest_new_file_number marker. Disable only for a staged " + "rollout in which some writers do not yet publish the guard; " + "doing so allows deleting an in-flight upload."); +DEFINE_uint64(max_deletions_per_cycle, 10000, + "Maximum number of objects deleted per purge cycle; the " + "remainder is re-selected next cycle. 0 means unlimited " + "(default: 10000)"); /** * @brief Parse URL into bucket and object path components @@ -223,6 +236,10 @@ int main(int argc, char **argv) { "actually delete files\n"; std::cerr << " --aws_region=ap-northeast-1 AWS region\n"; std::cerr << " --cloudmanifest_retention_ms=3600000 CLOUDMANIFEST retention time in milliseconds\n"; + std::cerr << " --dead_epoch_file_age_ms=3600000 Minimum age before " + "deleting a non-live file in a dead epoch\n"; + std::cerr << " --max_deletions_per_cycle=10000 Maximum objects " + "deleted per cycle; 0 means unlimited\n"; return 1; } @@ -328,7 +345,10 @@ int main(int argc, char **argv) { // Create and run improved purger ROCKSDB_NAMESPACE::EloqPurger purger(cfs_impl, bucket_name, object_path, FLAGS_dry_run, - FLAGS_cloudmanifest_retention_ms); + FLAGS_cloudmanifest_retention_ms, + FLAGS_dead_epoch_file_age_ms, + FLAGS_max_deletions_per_cycle, + FLAGS_require_guard_marker); if (!ROCKSDB_NAMESPACE::PrerequisitesMet(*cfs_impl)) { std::cerr << "Error: Prerequisites for purger not met" << std::endl; diff --git a/cloud/eloq_purger_integration_test.cc b/cloud/eloq_purger_integration_test.cc new file mode 100644 index 0000000..d6a8396 --- /dev/null +++ b/cloud/eloq_purger_integration_test.cc @@ -0,0 +1,839 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ + +// End-to-end integration test for the purger against a real S3-compatible +// endpoint (Minio). It creates a real DBCloud, restarts it to mint dead +// epochs, compacts to create garbage, runs real purge cycles, and then +// reopens the database and reads every key back. +// +// The test is skipped unless these environment variables are set: +// ELOQ_PURGER_TEST_S3_ENDPOINT e.g. http://127.0.0.1:9900 +// ELOQ_PURGER_TEST_ACCESS_KEY +// ELOQ_PURGER_TEST_SECRET_KEY +// Optional: +// ELOQ_PURGER_TEST_S3_BUCKET default: eloq-purger-it + +#ifndef USE_AWS + +#include +int main() { + fprintf(stderr, + "SKIPPED: eloq_purger_integration_test requires USE_AWS=1\n"); + return 0; +} + +#else + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "cloud/cloud_manifest.h" +#include "cloud/eloq_purger.h" +#include "cloud/file_number_guard.h" +#include "cloud/filename.h" +#include "cloud/manifest_reader.h" +#include "file/filename.h" +#include "file/sequence_file_reader.h" +#include "rocksdb/cloud/cloud_file_system.h" +#include "rocksdb/cloud/cloud_file_system_impl.h" +#include "rocksdb/cloud/db_cloud.h" +#include "rocksdb/metadata.h" +#include "rocksdb/options.h" +#include "rocksdb/utilities/checkpoint.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/stderr_logger.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +std::string GetEnvOr(const char *name, const std::string &def) { + const char *v = getenv(name); + return (v != nullptr && *v != '\0') ? std::string(v) : def; +} + +// Path-style S3 client against a custom endpoint. Minio does not support +// virtual-host addressing without wildcard DNS, so unlike +// eloq_purger_command's factory this one forces path-style. +S3ClientFactory BuildPathStyleS3ClientFactory(const std::string &endpoint, + bool https) { + return [endpoint, https]( + const std::shared_ptr + &credentialsProvider, + const Aws::Client::ClientConfiguration &baseConfig) + -> std::shared_ptr { + Aws::Client::ClientConfiguration config = baseConfig; + config.endpointOverride = endpoint; + config.scheme = https ? Aws::Http::Scheme::HTTPS : Aws::Http::Scheme::HTTP; + if (credentialsProvider) { + return std::make_shared( + credentialsProvider, config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + false /* useVirtualAddressing: path-style for Minio */); + } + return std::make_shared(config); + }; +} + +} // namespace + +class EloqPurgerIntegrationTest : public testing::Test { + public: + EloqPurgerIntegrationTest() { + endpoint_url_ = GetEnvOr("ELOQ_PURGER_TEST_S3_ENDPOINT", ""); + access_key_ = GetEnvOr("ELOQ_PURGER_TEST_ACCESS_KEY", ""); + secret_key_ = GetEnvOr("ELOQ_PURGER_TEST_SECRET_KEY", ""); + bucket_ = GetEnvOr("ELOQ_PURGER_TEST_S3_BUCKET", "eloq-purger-it"); + + // Strip the scheme; the client factory carries it separately. + https_ = endpoint_url_.rfind("https://", 0) == 0; + endpoint_ = endpoint_url_; + auto scheme_pos = endpoint_.find("://"); + if (scheme_pos != std::string::npos) { + endpoint_ = endpoint_.substr(scheme_pos + 3); + } + + // Unique object path per run so reruns never collide. The local dbpath + // must be unique alongside it: a leftover local dir from a previous run + // records that run's dest path, and rocksdb-cloud refuses to pair it + // with a different one ("NeedsReinitialization: bad dest path"). + object_path_ = "purger-it-" + std::to_string(getpid()) + "-" + + std::to_string(Env::Default()->NowMicros()); + local_dbpath_ = test::TmpDir() + "/" + object_path_; + } + + bool Configured() const { + return !endpoint_url_.empty() && !access_key_.empty() && + !secret_key_.empty(); + } + + protected: + // One "process lifetime" of the database or the purger: a cloud FS plus + // the composite Env the DB runs on. + struct CloudSession { + std::shared_ptr cloud_fs; + std::unique_ptr env; + CloudFileSystemImpl *cfs_impl = nullptr; + }; + + CloudFileSystemOptions MakeCfsOptions() const { + CloudFileSystemOptions cfs_options; + cfs_options.publish_file_number_guard = guard_enabled_; + cfs_options.guard_publish_interval = guard_publish_interval_; + cfs_options.guard_entry_duration = guard_entry_duration_; + cfs_options.credentials.InitializeSimple(access_key_, secret_key_); + cfs_options.src_bucket.SetBucketName(bucket_); + cfs_options.src_bucket.SetBucketPrefix(""); + cfs_options.src_bucket.SetObjectPath(object_path_); + cfs_options.src_bucket.SetRegion("us-east-1"); + cfs_options.dest_bucket.SetBucketName(bucket_); + cfs_options.dest_bucket.SetBucketPrefix(""); + cfs_options.dest_bucket.SetObjectPath(object_path_); + cfs_options.dest_bucket.SetRegion("us-east-1"); + cfs_options.s3_client_factory = + BuildPathStyleS3ClientFactory(endpoint_, https_); + cfs_options.use_aws_transfer_manager = false; + return cfs_options; + } + + Status OpenSession(CloudSession *session, bool verbose_logging = false) { + CloudFileSystem *cfs = nullptr; + Status s = CloudFileSystemEnv::NewAwsFileSystem( + FileSystem::Default(), MakeCfsOptions(), nullptr /*logger*/, &cfs); + if (!s.ok()) { + return s; + } + session->cloud_fs.reset(cfs); + session->cfs_impl = dynamic_cast(cfs); + session->env = + CloudFileSystemEnv::NewCompositeEnv(Env::Default(), session->cloud_fs); + if (verbose_logging && getenv("ELOQ_PURGER_TEST_VERBOSE") != nullptr) { + session->cfs_impl->info_log_ = + std::make_shared(InfoLogLevel::INFO_LEVEL); + } + return Status::OK(); + } + + // Opens the DB, hands it to `body`, closes it again. Each call is one + // database generation: reopening rolls a new epoch. + void WithDb(const std::function &body) { + CloudSession session; + ASSERT_OK(OpenSession(&session)); + + Options options; + options.env = session.env.get(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + + DBCloud *db = nullptr; + ASSERT_OK(DBCloud::Open(options, local_dbpath_, + "" /*persistent_cache_path*/, 0, &db)); + body(db); + ASSERT_OK(db->Flush(FlushOptions())); + delete db; + } + + std::vector ListObjects(CloudSession *session) { + std::vector names; + IOStatus s = session->cfs_impl->GetStorageProvider()->ListCloudObjects( + bucket_, object_path_, &names); + EXPECT_TRUE(s.ok()) << s.ToString(); + return names; + } + + static size_t CountWithPrefix(const std::vector &names, + const std::string &prefix) { + size_t n = 0; + for (const auto &name : names) { + if (name.rfind(prefix, 0) == 0) { + ++n; + } + } + return n; + } + + static size_t CountSst(const std::vector &names) { + size_t n = 0; + for (const auto &name : names) { + if (name.find(".sst-") != std::string::npos) { + ++n; + } + } + return n; + } + + // Reads the smallest_new_file_number- guard object; empty string + // when absent or unreadable. + std::string ReadGuardMarker(CloudSession *session, + const std::string &epoch) { + std::string key = SmallestFileNumberObjectKey(object_path_, epoch); + std::string local = test::TmpDir() + "/guard_marker_" + + std::to_string(Env::Default()->NowMicros()); + IOStatus s = session->cfs_impl->GetStorageProvider()->GetCloudObject( + bucket_, key, local); + if (!s.ok()) { + return ""; + } + std::string content; + ReadFileToString(Env::Default(), local, &content); + Env::Default()->DeleteFile(local); + return content; + } + + std::string endpoint_url_; + std::string endpoint_; + bool https_ = false; + std::string access_key_; + std::string secret_key_; + std::string bucket_; + std::string object_path_; + std::string local_dbpath_; + bool guard_enabled_ = false; + std::chrono::milliseconds guard_publish_interval_{std::chrono::seconds(30)}; + std::chrono::milliseconds guard_entry_duration_{std::chrono::seconds(15)}; +}; + +TEST_F(EloqPurgerIntegrationTest, PurgeEndToEnd) { + if (!Configured()) { + ROCKSDB_GTEST_SKIP( + "set ELOQ_PURGER_TEST_S3_ENDPOINT, ELOQ_PURGER_TEST_ACCESS_KEY, " + "ELOQ_PURGER_TEST_SECRET_KEY to run this test"); + return; + } + + // Purging requires every writable DBCloud to publish the file number guard; + // without a marker the cycle now fails closed. Model the supported + // deployment. + guard_enabled_ = true; + guard_publish_interval_ = std::chrono::seconds(3); + guard_entry_duration_ = std::chrono::seconds(2); + + WriteOptions wopt; + wopt.disableWAL = true; + + auto Key = [](int i) { + char buf[16]; + snprintf(buf, sizeof(buf), "key%04d", i); + return std::string(buf); + }; + + // Generation 1: 200 keys across several SSTs. + WithDb([&](DBCloud *db) { + for (int i = 0; i < 200; i++) { + ASSERT_OK(db->Put(wopt, Key(i), "v1-" + Key(i))); + if ((i + 1) % 50 == 0) { + ASSERT_OK(db->Flush(FlushOptions())); + } + } + }); + + // Generation 2 (epoch rolled; generation 1's epoch is now history): + // overwrite half the keys, add new ones, then compact everything. The + // compaction obsoletes the old-epoch SSTs -- exactly the files the + // dead-epoch leak used to strand forever. + WithDb([&](DBCloud *db) { + for (int i = 0; i < 100; i++) { + ASSERT_OK(db->Put(wopt, Key(i), "v2-" + Key(i))); + } + for (int i = 200; i < 300; i++) { + ASSERT_OK(db->Put(wopt, Key(i), "v2-" + Key(i))); + } + ASSERT_OK(db->Flush(FlushOptions())); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + }); + + // Generation 3: open/close only, so generation 2's epoch also goes dead + // while its compaction output stays live. + WithDb([&](DBCloud *) {}); + + // Snapshot the bucket before purging. + CloudSession purge_session; + ASSERT_OK(OpenSession(&purge_session)); + auto before = ListObjects(&purge_session); + size_t sst_before = CountSst(before); + size_t manifest_before = CountWithPrefix(before, "MANIFEST-"); + ASSERT_GT(sst_before, 0u); + // Three generations => at least three MANIFEST- objects. + ASSERT_GE(manifest_before, 3u); + + // Let everything age past the (shortened) guards, then purge. Two cycles, + // mirroring production where selection is recomputed per cycle. + const uint64_t kGuardMs = 2000; + std::this_thread::sleep_for(std::chrono::milliseconds(3000)); + EloqPurger purger(purge_session.cfs_impl, bucket_, object_path_, + false /*dry_run*/, kGuardMs /*cloudmanifest_retention_ms*/, + kGuardMs /*dead_epoch_file_age_ms*/, + 100000 /*max_deletions_per_cycle*/); + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + + auto after = ListObjects(&purge_session); + size_t sst_after = CountSst(after); + size_t manifest_after = CountWithPrefix(after, "MANIFEST-"); + fprintf(stderr, + "purge result: objects %zu -> %zu, sst %zu -> %zu, " + "manifests %zu -> %zu\n", + before.size(), after.size(), sst_before, sst_after, manifest_before, + manifest_after); + + // Garbage was actually collected... + ASSERT_LT(sst_after, sst_before); + // ...dead-epoch MANIFESTs are gone, the live generation's remains... + ASSERT_EQ(manifest_after, 1u); + // ...and the CLOUDMANIFEST survived. + ASSERT_EQ(CountWithPrefix(after, "CLOUDMANIFEST"), 1u); + + // The decisive check: reopen the database and read every key back. + WithDb([&](DBCloud *db) { + std::string value; + for (int i = 0; i < 100; i++) { + ASSERT_OK(db->Get(ReadOptions(), Key(i), &value)) << Key(i); + ASSERT_EQ(value, "v2-" + Key(i)); + } + for (int i = 100; i < 200; i++) { + ASSERT_OK(db->Get(ReadOptions(), Key(i), &value)) << Key(i); + ASSERT_EQ(value, "v1-" + Key(i)); + } + for (int i = 200; i < 300; i++) { + ASSERT_OK(db->Get(ReadOptions(), Key(i), &value)) << Key(i); + ASSERT_EQ(value, "v2-" + Key(i)); + } + }); + + // Best-effort cleanup of the run's unique object path. + purge_session.cfs_impl->GetStorageProvider()->EmptyBucket(bucket_, + object_path_); +} + +// End-to-end test of the smallest_new_file_number guard: the open-time +// sentinel is replaced before Open returns, flush publishes downward before +// its SST reaches the cloud, and the purger honors the threshold. +TEST_F(EloqPurgerIntegrationTest, FileNumberGuardEndToEnd) { + if (!Configured()) { + ROCKSDB_GTEST_SKIP( + "set ELOQ_PURGER_TEST_S3_ENDPOINT, ELOQ_PURGER_TEST_ACCESS_KEY, " + "ELOQ_PURGER_TEST_SECRET_KEY to run this test"); + return; + } + + guard_enabled_ = true; + // Keep the background refresh and idle-decay loops reasonably quick. + guard_publish_interval_ = std::chrono::seconds(3); + guard_entry_duration_ = std::chrono::seconds(2); + const std::string kMaxStr = + std::to_string(std::numeric_limits::max()); + + WriteOptions wopt; + wopt.disableWAL = true; + + CloudSession db_session; + ASSERT_OK(OpenSession(&db_session)); + Options options; + options.env = db_session.env.get(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + + DBCloud *db = nullptr; + ASSERT_OK( + DBCloud::Open(options, local_dbpath_, "" /*persistent_cache*/, 0, &db)); + std::string epoch = + db_session.cfs_impl->GetCloudManifest()->GetCurrentEpoch(); + ASSERT_FALSE(epoch.empty()); + + // A separate session for observing the bucket (like a standalone purger). + CloudSession observe_session; + ASSERT_OK(OpenSession(&observe_session, /*verbose_logging=*/true)); + + // (1) Open replaces its temporary 0 sentinel with the idle watermark before + // returning, so the epoch does not remain unnecessarily blocked. + ASSERT_EQ(ReadGuardMarker(&observe_session, epoch), kMaxStr); + + // (2) Flush: the downward publish lands a real value <= the flushed SST's + // file number, synchronously within the flush. + ASSERT_OK(db->Put(wopt, "k1", "v1")); + ASSERT_OK(db->Flush(FlushOptions())); + { + std::string marker = ReadGuardMarker(&observe_session, epoch); + ASSERT_FALSE(marker.empty()); + uint64_t marker_value = std::stoull(marker); + ASSERT_GT(marker_value, 0u); + std::vector files; + db->GetLiveFilesMetaData(&files); + ASSERT_EQ(files.size(), 1u); + uint64_t sst_number = std::stoull(files[0].name.substr(1)); // "/000123.sst" + ASSERT_LE(marker_value, sst_number); + ASSERT_LT(marker_value, std::numeric_limits::max()); + } + + // (3) Idle: past linger + one publish interval, the marker decays to + // UINT64_MAX. + for (int i = 0; i < 20 && ReadGuardMarker(&observe_session, epoch) != kMaxStr; + i++) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + ASSERT_EQ(ReadGuardMarker(&observe_session, epoch), kMaxStr); + + // (4) A new flush after idle drops the marker BEFORE the flushed SST + // appears in cloud storage: capture the bucket listing at downward-publish + // time via the sync point, then check the new SST wasn't there yet. + std::vector listing_at_publish; + std::atomic capture_armed{false}; + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuard::PutSmallestFileNumberObject:Status", [&](void *) { + bool expected = true; + if (capture_armed.compare_exchange_strong(expected, false)) { + listing_at_publish = ListObjects(&observe_session); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::vector before_files; + db->GetLiveFilesMetaData(&before_files); + capture_armed = true; + ASSERT_OK(db->Put(wopt, "k2", "v2")); + ASSERT_OK(db->Flush(FlushOptions())); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_FALSE(capture_armed.load()); // the downward publish fired + + std::vector after_files; + db->GetLiveFilesMetaData(&after_files); + ASSERT_EQ(after_files.size(), before_files.size() + 1); + // Find the new SST's cloud object name and assert it was absent from the + // listing taken at publish time. + std::string new_sst_cloud_name; + for (const auto &f : after_files) { + bool existed = false; + for (const auto &b : before_files) { + if (f.name == b.name) { + existed = true; + break; + } + } + if (!existed) { + new_sst_cloud_name = + db_session.cfs_impl->RemapFilename(f.name.substr(1)); + } + } + ASSERT_FALSE(new_sst_cloud_name.empty()); + for (const auto &name : listing_at_publish) { + ASSERT_NE(name, new_sst_cloud_name); + } + + // (5) Purger honors the threshold. Create current-epoch garbage, then pin + // the marker low with a synthetic in-flight job and verify a purge cycle + // deletes nothing (file_number >= threshold is never deleted); release the + // pin, let the marker decay to MAX, and verify the garbage then goes while + // every live file survives. + for (int i = 0; i < 4; i++) { + // Overlapping key ranges in every SST, so CompactRange must rewrite the + // inputs (a trivial move of non-overlapping files would create no + // garbage). + ASSERT_OK(db->Put(wopt, "a", "v" + std::to_string(i))); + ASSERT_OK(db->Put(wopt, "z", "v" + std::to_string(i))); + ASSERT_OK(db->Put(wopt, "bulk" + std::to_string(i), "v")); + ASSERT_OK(db->Flush(FlushOptions())); + } + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + auto publisher = db_session.cfs_impl->GetFileNumberGuardPublisher(); + ASSERT_TRUE(publisher != nullptr); + publisher->OnJobBegin(1, /*thread*/ 424242, /*job*/ 1); + publisher->PeriodicPublish(); + ASSERT_EQ(ReadGuardMarker(&observe_session, epoch), "1"); + + EloqPurger purger(observe_session.cfs_impl, bucket_, object_path_, + false /*dry_run*/, 3600 * 1000 /*retention*/, + 1 /*dead_epoch_file_age_ms*/, 100000 /*cap*/); + size_t sst_before = CountSst(ListObjects(&observe_session)); + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + // Threshold 1: every non-live SST has file_number >= 1, so nothing may be + // deleted. + ASSERT_EQ(CountSst(ListObjects(&observe_session)), sst_before); + + // Release the pin and wait for the marker to decay to MAX. + publisher->OnJobEnd(424242, 1); + for (int i = 0; i < 20 && ReadGuardMarker(&observe_session, epoch) != kMaxStr; + i++) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + ASSERT_EQ(ReadGuardMarker(&observe_session, epoch), kMaxStr); + + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + size_t sst_after = CountSst(ListObjects(&observe_session)); + ASSERT_LT(sst_after, sst_before); + + // Every file the manifest references must have survived: read everything + // back through the still-open DB. + std::string value; + ASSERT_OK(db->Get(ReadOptions(), "k1", &value)); + ASSERT_EQ(value, "v1"); + ASSERT_OK(db->Get(ReadOptions(), "k2", &value)); + ASSERT_EQ(value, "v2"); + for (int i = 0; i < 4; i++) { + ASSERT_OK(db->Get(ReadOptions(), "bulk" + std::to_string(i), &value)); + } + + delete db; + observe_session.cfs_impl->GetStorageProvider()->EmptyBucket(bucket_, + object_path_); +} + +// RollNewBranch creates a frozen backup in the database's object path: it +// copies the current MANIFEST into a new epoch and publishes a CLOUDMANIFEST +// under the supplied branch cookie. The purger must include every such branch +// when computing the union of live SSTs. +TEST_F(EloqPurgerIntegrationTest, RolledBranchesProtectReferencedSsts) { + if (!Configured()) { + ROCKSDB_GTEST_SKIP( + "set ELOQ_PURGER_TEST_S3_ENDPOINT, ELOQ_PURGER_TEST_ACCESS_KEY, " + "ELOQ_PURGER_TEST_SECRET_KEY to run this test"); + return; + } + + guard_enabled_ = true; + guard_publish_interval_ = std::chrono::seconds(3); + guard_entry_duration_ = std::chrono::seconds(2); + + CloudSession db_session; + ASSERT_OK(OpenSession(&db_session)); + Options options; + options.env = db_session.env.get(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + + DBCloud *db = nullptr; + ASSERT_OK( + DBCloud::Open(options, local_dbpath_, "" /*persistent_cache*/, 0, &db)); + + WriteOptions wopt; + wopt.disableWAL = true; + auto key = [](int i) { + char buf[16]; + snprintf(buf, sizeof(buf), "backup-key-%04d", i); + return std::string(buf); + }; + auto write_generation = [&](int generation) { + for (int i = 0; i < 200; ++i) { + ASSERT_OK(db->Put(wopt, key(i), + "generation-" + std::to_string(generation))); + if ((i + 1) % 50 == 0) { + ASSERT_OK(db->Flush(FlushOptions())); + } + } + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + }; + + // Read the CLOUDMANIFEST produced by RollNewBranch, scan its new-epoch + // MANIFEST, and map every live file number using that branch's epoch map. + auto read_branch_ssts = [&](const std::string &branch_cookie) { + const std::string cloud_manifest_name = + MakeCloudManifestFile(object_path_, branch_cookie); + std::unique_ptr cloud_manifest_file; + IOStatus status = db_session.cfs_impl->NewSequentialFileCloud( + bucket_, cloud_manifest_name, FileOptions(), &cloud_manifest_file, + nullptr /*dbg*/); + EXPECT_OK(status); + if (!status.ok()) { + return std::set{}; + } + + std::unique_ptr branch_manifest; + status = CloudManifest::LoadFromLog( + std::make_unique( + std::move(cloud_manifest_file), cloud_manifest_name), + &branch_manifest); + EXPECT_OK(status); + if (!status.ok()) { + return std::set{}; + } + + std::set file_numbers; + ManifestReader reader(db_session.cfs_impl->info_log_, db_session.cfs_impl, + bucket_); + status = reader.GetLiveFiles(object_path_, branch_manifest->GetCurrentEpoch(), + &file_numbers, kManifestScanStrictMode); + EXPECT_OK(status); + if (!status.ok()) { + return std::set{}; + } + + std::set result; + for (uint64_t number : file_numbers) { + result.insert(db_session.cfs_impl->RemapFilenameWithCloudManifest( + MakeTableFileName(number), branch_manifest.get())); + } + return result; + }; + + // Each backup has an independent postfix and term, matching the purger's + // CLOUDMANIFEST cookie grouping contract. + const std::string branch1_cookie = "backup-one-1"; + const std::string branch2_cookie = "backup-two-1"; + + write_generation(1); + ASSERT_OK(db_session.cfs_impl->RollNewBranch(local_dbpath_, branch1_cookie)); + const std::set backup1_ssts = + read_branch_ssts(branch1_cookie); + ASSERT_FALSE(backup1_ssts.empty()); + + write_generation(2); + ASSERT_OK(db_session.cfs_impl->RollNewBranch(local_dbpath_, branch2_cookie)); + const std::set backup2_ssts = + read_branch_ssts(branch2_cookie); + ASSERT_FALSE(backup2_ssts.empty()); + + // Advance the primary again so each backup has a chance to be the sole + // remaining reference to one or more older SSTs. + write_generation(3); + std::vector current_files; + db->GetLiveFilesMetaData(¤t_files); + std::set current_ssts; + for (const auto &file : current_files) { + current_ssts.insert(db_session.cfs_impl->RemapFilename(file.name.substr(1))); + } + + std::set all_backup_ssts = backup1_ssts; + all_backup_ssts.insert(backup2_ssts.begin(), backup2_ssts.end()); + std::set backup_only_ssts; + for (const auto &file : all_backup_ssts) { + if (current_ssts.count(file) == 0) { + backup_only_ssts.insert(file); + } + } + ASSERT_FALSE(backup_only_ssts.empty()) + << "test did not create an SST referenced only by a backup"; + + auto provider = db_session.cfs_impl->GetStorageProvider(); + for (const auto &file : all_backup_ssts) { + ASSERT_OK(provider->ExistsCloudObject(bucket_, object_path_ + "/" + file)); + } + + // Wait until no primary flush/compaction is protected by the guard. This + // makes every non-current SST eligible unless a branch references it. + const std::string epoch = + db_session.cfs_impl->GetCloudManifest()->GetCurrentEpoch(); + const std::string max_marker = + std::to_string(std::numeric_limits::max()); + CloudSession purge_session; + ASSERT_OK(OpenSession(&purge_session)); + for (int i = 0; i < 20 && + ReadGuardMarker(&purge_session, epoch) != max_marker; + ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + ASSERT_EQ(ReadGuardMarker(&purge_session, epoch), max_marker); + + const size_t sst_count_before = CountSst(ListObjects(&purge_session)); + // RollNewBranch creates no writer and therefore no guard marker. The legacy + // fallback is safe for these immutable branch epochs and is required for the + // cycle to proceed; the writable primary still has its published marker. + EloqPurger purger(purge_session.cfs_impl, bucket_, object_path_, + false /*dry_run*/, 3600 * 1000 /*retention_ms*/, + 1 /*dead_epoch_file_age_ms*/, 100000 /*cap*/, + false /*require_guard_marker*/); + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + const size_t sst_count_after = CountSst(ListObjects(&purge_session)); + ASSERT_LT(sst_count_after, sst_count_before) + << "purge did not reclaim any unreferenced SSTs"; + + bool all_backup_ssts_survived = true; + for (const auto &file : all_backup_ssts) { + IOStatus exists = + provider->ExistsCloudObject(bucket_, object_path_ + "/" + file); + EXPECT_OK(exists) << "purger deleted backup-referenced SST " << file; + all_backup_ssts_survived = all_backup_ssts_survived && exists.ok(); + } + + // Verify the active database too, then clean up the unique object path even + // when an EXPECT above detects a missing branch SST. + std::string value; + for (int i = 0; i < 200; ++i) { + ASSERT_OK(db->Get(ReadOptions(), key(i), &value)); + ASSERT_EQ(value, "generation-3"); + } + delete db; + provider->EmptyBucket(bucket_, object_path_).PermitUncheckedError(); + ASSERT_TRUE(all_backup_ssts_survived); +} + +// Checkpoint copies SSTs through the cloud file system (LinkFile is +// NotSupported for bucket-backed file systems, so CreateCheckpoint always +// falls back to copying). Those copies land outside the DB directory and no +// flush/compaction job registers them, so gating them would fail the +// checkpoint outright. +// +// This test asserts only that the file number guard does not change +// checkpoint behavior. It does NOT assert that checkpointing a cloud +// database is useful: a cloud checkpoint currently contains no SST files at +// all, with or without the guard, because the copies are treated as this +// DB's own cloud SSTs -- uploaded to destname() (the DB's own object path) +// and then deleted locally when keep_local_sst_files is false. That is +// pre-existing behavior and out of scope here; EloqData creates backups and +// branches through a separate path rather than the Checkpoint API. +TEST_F(EloqPurgerIntegrationTest, GuardDoesNotAffectCheckpoint) { + if (!Configured()) { + ROCKSDB_GTEST_SKIP( + "set ELOQ_PURGER_TEST_S3_ENDPOINT, ELOQ_PURGER_TEST_ACCESS_KEY, " + "ELOQ_PURGER_TEST_SECRET_KEY to run this test"); + return; + } + + guard_publish_interval_ = std::chrono::seconds(3); + guard_entry_duration_ = std::chrono::seconds(2); + + // Runs an identical write + checkpoint flow and reports the checkpoint's + // status and the names it produced, so the guard's effect can be isolated. + auto run_checkpoint = [&](bool guard_enabled, Status *status, + std::vector *children) { + guard_enabled_ = guard_enabled; + const std::string db_path = + local_dbpath_ + (guard_enabled ? "_guarded" : "_plain"); + const std::string checkpoint_dir = db_path + "_checkpoint"; + + CloudSession session; + ASSERT_OK(OpenSession(&session)); + Options options; + options.env = session.env.get(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + + WriteOptions wopt; + wopt.disableWAL = true; + + DBCloud *db = nullptr; + ASSERT_OK(DBCloud::Open(options, db_path, "" /*persistent_cache*/, 0, &db)); + for (int i = 0; i < 20; i++) { + ASSERT_OK(db->Put(wopt, "ck" + std::to_string(i), "v")); + } + ASSERT_OK(db->Flush(FlushOptions())); + + Checkpoint *checkpoint = nullptr; + ASSERT_OK(Checkpoint::Create(db, &checkpoint)); + std::unique_ptr checkpoint_guard(checkpoint); + *status = checkpoint->CreateCheckpoint(checkpoint_dir); + children->clear(); + if (status->ok()) { + std::vector names; + Env::Default()->GetChildren(checkpoint_dir, &names).PermitUncheckedError(); + // Compare file kinds, not exact names: MANIFEST names embed the epoch, + // which necessarily differs between two separate databases. + for (const auto &name : names) { + if (name == "." || name == "..") { + continue; + } else if (name.find(".sst") != std::string::npos) { + children->push_back("SST"); + } else if (name.rfind("MANIFEST", 0) == 0) { + children->push_back("MANIFEST"); + } else if (name.rfind("OPTIONS", 0) == 0) { + children->push_back("OPTIONS"); + } else { + children->push_back(name); + } + } + std::sort(children->begin(), children->end()); + } + + delete db; + Env::Default()->DeleteDir(checkpoint_dir); + session.cfs_impl->GetStorageProvider()->EmptyBucket(bucket_, object_path_); + }; + + Status plain_status; + std::vector plain_children; + run_checkpoint(false, &plain_status, &plain_children); + + Status guarded_status; + std::vector guarded_children; + run_checkpoint(true, &guarded_status, &guarded_children); + + // The guard must not change checkpoint behavior. Before the DB-directory + // exemption the guarded run failed with "no live file number registration", + // because checkpoint copies are not registered by any flush/compaction job. + ASSERT_OK(guarded_status) << guarded_status.ToString(); + ASSERT_EQ(guarded_status.ok(), plain_status.ok()); + ASSERT_EQ(guarded_children, plain_children); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char **argv) { + Aws::SDKOptions aws_options; + Aws::InitAPI(aws_options); + ::testing::InitGoogleTest(&argc, argv); + int ret = RUN_ALL_TESTS(); + Aws::ShutdownAPI(aws_options); + return ret; +} + +#endif // USE_AWS diff --git a/cloud/eloq_purger_test.cc b/cloud/eloq_purger_test.cc new file mode 100644 index 0000000..ecfa8fc --- /dev/null +++ b/cloud/eloq_purger_test.cc @@ -0,0 +1,1119 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ + +#include "cloud/eloq_purger.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cloud/cloud_manifest.h" +#include "cloud/manifest_reader.h" +#include "db/log_writer.h" +#include "db/version_edit.h" +#include "file/writable_file_writer.h" +#include "rocksdb/cloud/cloud_file_system.h" +#include "rocksdb/cloud/cloud_storage_provider.h" +#include "rocksdb/cloud/cloud_storage_provider_impl.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +constexpr uint64_t kHourMs = 3600ULL * 1000; +// The S3-derived clock reading for all tests. Absolute value is arbitrary; +// only differences against object mtimes matter. +constexpr uint64_t kNow = 1000ULL * kHourMs; + +CloudObjectInformation MakeInfo(uint64_t mtime) { + CloudObjectInformation info; + info.size = 0; + info.modification_time = mtime; + return info; +} + +class StringCloudReadableFile : public CloudStorageReadableFileImpl { +public: + StringCloudReadableFile(const std::string &name, std::string contents) + : CloudStorageReadableFileImpl(nullptr, "test-bucket", name, + contents.size()), + contents_(std::move(contents)) {} + +protected: + IOStatus DoCloudRead(uint64_t offset, size_t n, const IOOptions &, + char *scratch, uint64_t *bytes_read, + IODebugContext *) const override { + const size_t available = contents_.size() - static_cast(offset); + const size_t to_read = std::min(n, available); + std::memcpy(scratch, contents_.data() + offset, to_read); + *bytes_read = to_read; + return IOStatus::OK(); + } + +private: + std::string contents_; +}; + +class RecordingPurgerStorageProvider : public CloudStorageProvider { + public: + using PurgerAllFiles = EloqPurger::PurgerAllFiles; + + explicit RecordingPurgerStorageProvider(PurgerAllFiles files) + : files_(std::move(files)) {} + + const char *Name() const override { return "recording-purger"; } + + void SetDeleteStatus(const std::string &path, IOStatus status) { + delete_statuses_[path] = std::move(status); + } + + void SetGetStatus(const std::string &path, IOStatus status) { + get_statuses_[path] = std::move(status); + } + + void SetObjectContents(const std::string &path, std::string contents) { + object_contents_[path] = std::move(contents); + } + + void SetListAllStatus(IOStatus status) { + list_all_status_ = std::move(status); + } + + void SetListPrefixStatus(IOStatus status) { + list_prefix_status_ = std::move(status); + } + + void SetPutStatus(IOStatus status) { put_status_ = std::move(status); } + + void SetClockMetadataStatus(IOStatus status) { + clock_metadata_status_ = std::move(status); + } + + void SetMetadataStatus(const std::string &path, IOStatus status) { + metadata_statuses_[path] = std::move(status); + } + + const std::vector &delete_attempts() const { + return delete_attempts_; + } + + IOStatus DeleteCloudObject(const std::string & /*bucket_name*/, + const std::string &object_path) override { + if (clock_objects_.erase(object_path) != 0) { + return IOStatus::OK(); + } + + delete_attempts_.push_back(object_path); + auto status_it = delete_statuses_.find(object_path); + IOStatus status = status_it == delete_statuses_.end() ? IOStatus::OK() + : status_it->second; + if (status.ok() || status.IsNotFound()) { + const std::string prefix = object_path_ + "/"; + const std::string relative_path = + object_path.compare(0, prefix.size(), prefix) == 0 + ? object_path.substr(prefix.size()) + : object_path; + files_.erase(std::remove_if(files_.begin(), files_.end(), + [&](const auto &file) { + return file.first == relative_path; + }), + files_.end()); + } + return status; + } + + IOStatus ListCloudObjects(const std::string & /*bucket_name*/, + const std::string &object_path, + PurgerAllFiles *files) override { + if (!list_all_status_.ok()) { + return list_all_status_; + } + object_path_ = object_path; + *files = files_; + return IOStatus::OK(); + } + + IOStatus ListCloudObjectsWithPrefix( + const std::string & /*bucket_name*/, const std::string & /*object_path*/, + const std::string &object_prefix, + std::vector *paths) override { + if (!list_prefix_status_.ok()) { + return list_prefix_status_; + } + for (const auto &file : files_) { + if (file.first.compare(0, object_prefix.size(), object_prefix) == 0) { + paths->push_back(file.first); + } + } + return IOStatus::OK(); + } + + IOStatus PutCloudObject(const std::string & /*local_path*/, + const std::string & /*bucket_name*/, + const std::string &object_path) override { + if (!put_status_.ok()) { + return put_status_; + } + clock_objects_.insert(object_path); + return IOStatus::OK(); + } + + IOStatus GetCloudObjectModificationTime(const std::string & /*bucket_name*/, + const std::string & /*object_path*/, + uint64_t * /*time*/) override { + return NotSupported(); + } + + IOStatus GetCloudObjectMetadata(const std::string & /*bucket_name*/, + const std::string &object_path, + CloudObjectInformation *info) override { + auto status = metadata_statuses_.find(object_path); + if (status != metadata_statuses_.end()) { + return status->second; + } + if (clock_objects_.find(object_path) != clock_objects_.end()) { + if (!clock_metadata_status_.ok()) { + return clock_metadata_status_; + } + *info = MakeInfo(kNow); + return IOStatus::OK(); + } + + const std::string prefix = object_path_ + "/"; + const std::string relative_path = + object_path.compare(0, prefix.size(), prefix) == 0 + ? object_path.substr(prefix.size()) + : object_path; + for (const auto &file : files_) { + if (file.first == relative_path) { + *info = file.second; + return IOStatus::OK(); + } + } + return IOStatus::NotFound(object_path); + } + + IOStatus CreateBucket(const std::string &) override { return NotSupported(); } + IOStatus ExistsBucket(const std::string &) override { return NotSupported(); } + IOStatus EmptyBucket(const std::string &, const std::string &) override { + return NotSupported(); + } + IOStatus ListCloudObjects(const std::string &, const std::string &, + std::vector *) override { + return NotSupported(); + } + IOStatus ExistsCloudObject(const std::string &, + const std::string &) override { + return NotSupported(); + } + IOStatus GetCloudObjectSize(const std::string &, const std::string &, + uint64_t *) override { + return NotSupported(); + } + IOStatus CopyCloudObject(const std::string &, const std::string &, + const std::string &, const std::string &) override { + return NotSupported(); + } + IOStatus GetCloudObject(const std::string &, const std::string &object_path, + const std::string &local_path) override { + auto status = get_statuses_.find(object_path); + if (status != get_statuses_.end()) { + return status->second; + } + auto contents = object_contents_.find(object_path); + if (contents == object_contents_.end()) { + return NotSupported(); + } + std::ofstream out(local_path, std::ios::binary | std::ios::trunc); + if (!out.is_open()) { + return IOStatus::IOError(local_path); + } + out.write(contents->second.data(), + static_cast(contents->second.size())); + out.close(); + return IOStatus::OK(); + } + IOStatus PutCloudObjectMetadata( + const std::string &, const std::string &, + const std::unordered_map &) override { + return NotSupported(); + } + IOStatus NewCloudWritableFile(const std::string &, const std::string &, + const std::string &, const FileOptions &, + std::unique_ptr *, + IODebugContext *) override { + return NotSupported(); + } + IOStatus + NewCloudReadableFile(const std::string &, const std::string &object_path, + const FileOptions &, + std::unique_ptr *result, + IODebugContext *) override { + auto status = get_statuses_.find(object_path); + if (status != get_statuses_.end()) { + return status->second; + } + auto contents = object_contents_.find(object_path); + if (contents == object_contents_.end()) { + return IOStatus::NotFound(object_path); + } + result->reset(new StringCloudReadableFile(object_path, contents->second)); + return IOStatus::OK(); + } + + private: + static IOStatus NotSupported() { + return IOStatus::NotSupported("RecordingPurgerStorageProvider"); + } + + PurgerAllFiles files_; + std::string object_path_; + std::unordered_map delete_statuses_; + std::unordered_map get_statuses_; + std::unordered_map metadata_statuses_; + std::unordered_map object_contents_; + std::unordered_set clock_objects_; + std::vector delete_attempts_; + IOStatus list_all_status_; + IOStatus list_prefix_status_; + IOStatus put_status_; + IOStatus clock_metadata_status_; +}; + +class RecordingLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char *format, va_list ap) override { + char buffer[2048]; + vsnprintf(buffer, sizeof(buffer), format, ap); + log_.append(buffer).push_back('\n'); + } + + const std::string &log() const { return log_; } + + private: + std::string log_; +}; + +std::unique_ptr MakeCloudFileSystem( + const std::shared_ptr &provider, + const std::shared_ptr &logger = nullptr) { + CloudFileSystemOptions opts; + opts.storage_provider = provider; + opts.dest_bucket.SetBucketName("test-bucket"); + opts.dest_bucket.SetObjectPath("dbpath"); + opts.cloud_file_deletion_delay = std::nullopt; + return std::make_unique(opts, FileSystem::Default(), + logger); +} + +std::string ReadTestFileAndRemove(const std::string &path) { + std::ifstream in(path, std::ios::binary); + std::string contents((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + in.close(); + EXPECT_EQ(std::remove(path.c_str()), 0); + return contents; +} + +// Serialized CLOUDMANIFEST bytes. `epoch_transitions` uses the same exclusive +// file-number boundaries as CloudManifest::AddEpoch, allowing cycle tests to +// verify that live files from earlier epochs are remapped and preserved. +std::string MakeCloudManifestContents( + const std::string &initial_epoch, + const std::vector> &epoch_transitions = + {}) { + std::unique_ptr manifest; + EXPECT_OK(CloudManifest::CreateForEmptyDatabase(initial_epoch, &manifest)); + for (const auto &transition : epoch_transitions) { + EXPECT_TRUE(manifest->AddEpoch(transition.first, transition.second)); + } + static uint64_t serial = 0; + const std::string path = test::TmpDir() + "/purger_test_cloud_manifest_" + + std::to_string(Env::Default()->NowMicros()) + "_" + + std::to_string(++serial); + std::unique_ptr writer; + EXPECT_OK(WritableFileWriter::Create(FileSystem::Default(), path, + FileOptions(), &writer, nullptr)); + EXPECT_OK(manifest->WriteToLog(std::move(writer))); + return ReadTestFileAndRemove(path); +} + +// Serialized RocksDB MANIFEST bytes containing one add record for every live +// file number. The purger reads these exact bytes through CloudFileSystemImpl, +// so cycle tests cover decoding and CloudManifest epoch remapping rather than +// injecting a precomputed live-file set. +std::string MakeManifestContents(const std::vector &live_files) { + static uint64_t serial = 0; + const std::string path = test::TmpDir() + "/purger_test_manifest_" + + std::to_string(Env::Default()->NowMicros()) + "_" + + std::to_string(++serial); + std::unique_ptr writer; + EXPECT_OK(WritableFileWriter::Create(FileSystem::Default(), path, + FileOptions(), &writer, nullptr)); + { + log::Writer log_writer(std::move(writer), 0, false); + for (uint64_t file_number : live_files) { + VersionEdit edit; + edit.AddFile( + 0 /*level*/, file_number, 0 /*path_id*/, 1 /*file_size*/, + InternalKey("a", 1, kTypeValue), InternalKey("z", 1, kTypeValue), + 1 /*smallest_seqno*/, 1 /*largest_seqno*/, + false /*marked_for_compaction*/, Temperature::kUnknown, + kInvalidBlobFileNumber, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, file_number /*epoch_number*/, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, + 0 /*compensated_range_deletion_size*/, 0 /*tail_size*/, + true /*user_defined_timestamps_persisted*/); + std::string record; + EXPECT_TRUE(edit.EncodeTo(&record, 0 /*timestamp_size*/)); + EXPECT_OK(log_writer.AddRecord(WriteOptions(), record)); + } + EXPECT_OK(log_writer.file()->Sync(IOOptions(), false)); + } + return ReadTestFileAndRemove(path); +} + +std::shared_ptr MakeValidCycleProvider() { + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{ + {"CLOUDMANIFEST-db-1", MakeInfo(kNow - 2 * kHourMs)}, + {"MANIFEST-epochA", MakeInfo(kNow - 2 * kHourMs)}, + {"000001.sst-epochA", MakeInfo(kNow - 2 * kHourMs)}, + {"000002.sst-epochA", MakeInfo(kNow - 2 * kHourMs)}, + {"smallest_new_file_number-epochA", + MakeInfo(kNow - 2 * kHourMs)}, + }); + provider->SetObjectContents("dbpath/CLOUDMANIFEST-db-1", + MakeCloudManifestContents("epochA")); + provider->SetObjectContents("dbpath/MANIFEST-epochA", + MakeManifestContents({1})); + provider->SetObjectContents("dbpath/smallest_new_file_number-epochA", "10"); + return provider; +} + +} // namespace + +class EloqPurgerTest : public testing::Test { + public: + EloqPurgerTest() + : cfs_(TestOptions(), FileSystem::Default(), nullptr /*logger*/), + purger_(&cfs_, "test-bucket", "dbpath", true /*dry_run*/, + kHourMs /*cloudmanifest_retention_ms*/, + kHourMs /*dead_epoch_file_age_ms*/, + 10000 /*max_deletions_per_cycle*/) {} + + protected: + static CloudFileSystemOptions TestOptions() { + CloudFileSystemOptions opts; + // Keep the constructor from spinning up the file-deletion scheduler. + opts.cloud_file_deletion_delay = std::nullopt; + return opts; + } + + CloudFileSystemImpl cfs_; + EloqPurger purger_; +}; + +// A living epoch (one with a threshold entry) must keep the pre-existing +// watermark semantics: non-live files below the watermark are deleted, files +// at or above it are spared regardless of age, live files are untouchable. +TEST_F(EloqPurgerTest, LivingEpochThresholdSemanticsUnchanged) { + EloqPurger::PurgerFileNumberThresholds thresholds{{"epochA", 100}}; + EloqPurger::PurgerLiveFileSet live{"000042.sst-epochA"}; + EloqPurger::PurgerAllFiles all_files{ + // Live: kept even though it is below the watermark and old. + {"000042.sst-epochA", MakeInfo(kNow - 10 * kHourMs)}, + // Non-live, below watermark: deleted even though it is brand new -- + // the watermark, not age, is the guard for living epochs. + {"000050.sst-epochA", MakeInfo(kNow)}, + // Non-live, at/above watermark: possibly still in flight, kept. + {"000150.sst-epochA", MakeInfo(kNow - 10 * kHourMs)}, + // Not an SST: ignored by this selector. + {"MANIFEST-epochA", MakeInfo(kNow - 10 * kHourMs)}, + }; + + std::vector obsolete; + purger_.SelectObsoleteSSTFilesWithThreshold(all_files, live, thresholds, + kNow, &obsolete); + ASSERT_EQ(obsolete, std::vector{"000050.sst-epochA"}); +} + +// A published watermark of UINT64_MIN means "unknown": nothing in that epoch +// may be deleted. +TEST_F(EloqPurgerTest, UnknownThresholdBlocksLivingEpoch) { + EloqPurger::PurgerFileNumberThresholds thresholds{ + {"epochA", std::numeric_limits::min()}}; + EloqPurger::PurgerAllFiles all_files{ + {"000050.sst-epochA", MakeInfo(kNow - 10 * kHourMs)}, + }; + + std::vector obsolete; + purger_.SelectObsoleteSSTFilesWithThreshold(all_files, {}, thresholds, kNow, + &obsolete); + ASSERT_TRUE(obsolete.empty()); +} + +// Regression test for the dead-epoch leak: a non-live SST whose epoch is no +// loaded CLOUDMANIFEST's current epoch used to be blocked forever; it must +// now be reclaimed once older than the age guard, while young files (a node +// mid-open) and live files stay protected. +TEST_F(EloqPurgerTest, DeadEpochSstReclaimedOnceOldEnough) { + EloqPurger::PurgerFileNumberThresholds thresholds{{"epochA", 100}}; + EloqPurger::PurgerLiveFileSet live{"000010.sst-epochDead"}; + EloqPurger::PurgerAllFiles all_files{ + // Dead epoch, older than the age guard: reclaimed. + {"000200.sst-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + // Dead epoch but younger than the age guard: kept (mid-open race). + {"000201.sst-epochDead", MakeInfo(kNow - kHourMs + 1)}, + // Live files are protected regardless of epoch. + {"000010.sst-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + }; + + std::vector obsolete; + purger_.SelectObsoleteSSTFilesWithThreshold(all_files, live, thresholds, + kNow, &obsolete); + ASSERT_EQ(obsolete, std::vector{"000200.sst-epochDead"}); +} + +// MANIFEST selection: current-epoch manifests are untouchable, dead-epoch +// manifests are reclaimed only once old enough. The young-manifest case is +// the open-sequencing race: MANIFEST- is uploaded BEFORE the +// CLOUDMANIFEST that makes the epoch current. +TEST_F(EloqPurgerTest, ManifestAgeGuard) { + EloqPurger::PurgerEpochManifestMap current_epochs{ + {"epochA", MakeInfo(kNow)}}; + EloqPurger::PurgerAllFiles all_files{ + // Current epoch: kept no matter how old. + {"MANIFEST-epochA", MakeInfo(kNow - 10 * kHourMs)}, + // Dead epoch, old: reclaimed. + {"MANIFEST-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + // Unknown epoch but freshly uploaded: a node mid-open, kept. + {"MANIFEST-epochOpening", MakeInfo(kNow - kHourMs / 2)}, + // Not a manifest: ignored by this selector. + {"000200.sst-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + }; + + std::vector obsolete; + purger_.SelectObsoleteManifestFiles(all_files, current_epochs, kNow, + &obsolete); + ASSERT_EQ(obsolete, std::vector{"MANIFEST-epochDead"}); +} + +// Regression test for the read-only-zombie bug: retention must be measured +// from the moment the old term was superseded (the successor CLOUDMANIFEST's +// own mtime), not from the old generation's MANIFEST mtime. A generation that +// was write-idle for a day before failover must still get the full grace +// window after failover. +TEST_F(EloqPurgerTest, SupersededCloudManifestKeptUntilSuccessorAges) { + EloqPurger::PurgerCloudManifestMap cloudmanifests; + std::unique_ptr old_cm; + std::unique_ptr new_cm; + ASSERT_OK(CloudManifest::CreateForEmptyDatabase("epochOld", &old_cm)); + ASSERT_OK(CloudManifest::CreateForEmptyDatabase("epochNew", &new_cm)); + cloudmanifests["CLOUDMANIFEST-db-1"] = std::move(old_cm); + cloudmanifests["CLOUDMANIFEST-db-2"] = std::move(new_cm); + + EloqPurger::PurgerEpochManifestMap current_epochs{ + // The old generation served read-only traffic: last write a day ago. + {"epochOld", MakeInfo(kNow - 24 * kHourMs)}, + {"epochNew", MakeInfo(kNow)}, + }; + + // Successor born 30 minutes ago: within retention, keep the old term even + // though its MANIFEST heartbeat has been silent for a day. + { + EloqPurger::PurgerAllFiles all_files{ + {"CLOUDMANIFEST-db-1", MakeInfo(kNow - 25 * kHourMs)}, + {"CLOUDMANIFEST-db-2", MakeInfo(kNow - kHourMs / 2)}, + }; + std::vector obsolete; + purger_.SelectObsoleteCloudManifestFiles(all_files, cloudmanifests, + current_epochs, kNow, &obsolete); + ASSERT_TRUE(obsolete.empty()); + } + + // Successor born two hours ago: retention expired, the old term goes and + // the max term stays. + { + EloqPurger::PurgerAllFiles all_files{ + {"CLOUDMANIFEST-db-1", MakeInfo(kNow - 25 * kHourMs)}, + {"CLOUDMANIFEST-db-2", MakeInfo(kNow - 2 * kHourMs)}, + }; + std::vector obsolete; + purger_.SelectObsoleteCloudManifestFiles(all_files, cloudmanifests, + current_epochs, kNow, &obsolete); + ASSERT_EQ(obsolete, std::vector{"CLOUDMANIFEST-db-1"}); + } +} + +// smallest_new_file_number markers: kept while their epoch is living or the +// marker is young (a node mid-open may publish the marker before its +// CLOUDMANIFEST lands), reclaimed once the epoch is dead and the marker old. +TEST_F(EloqPurgerTest, DeadEpochMarkersReclaimed) { + EloqPurger::PurgerFileNumberThresholds thresholds{{"epochA", 100}}; + EloqPurger::PurgerAllFiles all_files{ + // Living epoch: marker in use, kept. + {"smallest_new_file_number-epochA", MakeInfo(kNow - 10 * kHourMs)}, + // Dead epoch, old: reclaimed. + {"smallest_new_file_number-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + // Unknown epoch but young: kept. + {"smallest_new_file_number-epochOpening", MakeInfo(kNow - kHourMs / 2)}, + // Not a marker: ignored by this selector. + {"000050.sst-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + }; + + std::vector obsolete; + purger_.SelectObsoleteFileNumberMarkers(all_files, thresholds, kNow, + &obsolete); + ASSERT_EQ(obsolete, + std::vector{"smallest_new_file_number-epochDead"}); +} + +TEST_F(EloqPurgerTest, FutureTimestampsAreRetainedByEveryAgeGuard) { + EloqPurger::PurgerCloudManifestMap cloudmanifests; + std::unique_ptr old_cm; + std::unique_ptr new_cm; + ASSERT_OK(CloudManifest::CreateForEmptyDatabase("epochOld", &old_cm)); + ASSERT_OK(CloudManifest::CreateForEmptyDatabase("epochNew", &new_cm)); + cloudmanifests["CLOUDMANIFEST-db-1"] = std::move(old_cm); + cloudmanifests["CLOUDMANIFEST-db-2"] = std::move(new_cm); + + EloqPurger::PurgerAllFiles all_files{ + {"000001.sst-epochDead", MakeInfo(kNow + 1)}, + {"MANIFEST-epochDead", MakeInfo(kNow + 1)}, + {"CLOUDMANIFEST-db-1", MakeInfo(kNow - 2 * kHourMs)}, + {"CLOUDMANIFEST-db-2", MakeInfo(kNow + 1)}, + {"smallest_new_file_number-epochDead", MakeInfo(kNow + 1)}, + }; + EloqPurger::PurgerEpochManifestMap current_epochs{ + {"epochOld", MakeInfo(kNow)}, {"epochNew", MakeInfo(kNow)}}; + + std::vector obsolete; + purger_.SelectObsoleteSSTFilesWithThreshold(all_files, {}, {}, kNow, + &obsolete); + purger_.SelectObsoleteManifestFiles(all_files, current_epochs, kNow, + &obsolete); + purger_.SelectObsoleteCloudManifestFiles(all_files, cloudmanifests, + current_epochs, kNow, &obsolete); + purger_.SelectObsoleteFileNumberMarkers(all_files, {}, kNow, &obsolete); + ASSERT_TRUE(obsolete.empty()); +} + +TEST_F(EloqPurgerTest, ZeroThresholdBlocksLivingEpochSstDeletion) { + EloqPurger::PurgerAllFiles all_files{ + {"000001.sst-epochA", MakeInfo(kNow - 10 * kHourMs)}, + {"000002.sst-epochA", MakeInfo(kNow - 10 * kHourMs)}, + }; + + std::vector obsolete; + purger_.SelectObsoleteSSTFilesWithThreshold(all_files, {}, {{"epochA", 0}}, + kNow, &obsolete); + ASSERT_TRUE(obsolete.empty()); +} + +TEST(EloqPurgerCycleTest, NotFoundDeletionCountsAsSuccess) { + const std::string file = "000001.sst-epochDead"; + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{{file, MakeInfo(kNow - 2 * kHourMs)}}); + provider->SetDeleteStatus("dbpath/" + file, + IOStatus::NotFound("already deleted")); + auto cfs = MakeCloudFileSystem(provider); + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", false /*dry_run*/, + kHourMs, kHourMs, 10000); + + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + ASSERT_EQ(provider->delete_attempts(), + std::vector{"dbpath/" + file}); +} + +TEST(EloqPurgerCycleTest, GuardReadIoErrorFailsClosed) { + const std::string cloud_manifest_name = "CLOUDMANIFEST-db-1"; + const std::string sst_name = "000001.sst-epochA"; + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{ + {cloud_manifest_name, MakeInfo(kNow - 2 * kHourMs)}, + {sst_name, MakeInfo(kNow - 2 * kHourMs)}}); + + std::unique_ptr manifest; + ASSERT_OK(CloudManifest::CreateForEmptyDatabase("epochA", &manifest)); + const std::string manifest_path = test::TmpDir() + + "/purger_guard_read_error_cloud_manifest_" + + std::to_string(Env::Default()->NowMicros()); + std::unique_ptr writer; + ASSERT_OK(WritableFileWriter::Create(FileSystem::Default(), manifest_path, + FileOptions(), &writer, nullptr)); + ASSERT_OK(manifest->WriteToLog(std::move(writer))); + std::ifstream manifest_file(manifest_path, std::ios::binary); + const std::string manifest_contents( + (std::istreambuf_iterator(manifest_file)), + std::istreambuf_iterator()); + ASSERT_FALSE(manifest_contents.empty()); + ASSERT_EQ(std::remove(manifest_path.c_str()), 0); + provider->SetObjectContents("dbpath/" + cloud_manifest_name, + manifest_contents); + provider->SetGetStatus("dbpath/smallest_new_file_number-epochA", + IOStatus::IOError("injected guard read failure")); + auto cfs = MakeCloudFileSystem(provider); + S3FileNumberReader reader("test-bucket", "dbpath", "epochA", cfs.get()); + + uint64_t threshold = std::numeric_limits::max(); + ASSERT_TRUE(reader.ReadSmallestFileNumber(&threshold).IsIOError()); + ASSERT_EQ(threshold, std::numeric_limits::min()); + + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", false /*dry_run*/, + kHourMs, kHourMs, 10000); + ASSERT_FALSE(purger.RunSinglePurgeCycle()); + ASSERT_TRUE(provider->delete_attempts().empty()); +} + +TEST(EloqPurgerCycleTest, DeletionCapConsumesDeterministicPrefixThenConverges) { + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{ + {"smallest_new_file_number-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + {"000001.sst-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + {"MANIFEST-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + {"000002.sst-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + }); + auto cfs = MakeCloudFileSystem(provider); + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", false /*dry_run*/, + kHourMs, kHourMs, 2 /*max_deletions_per_cycle*/); + + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + ASSERT_EQ(provider->delete_attempts(), + std::vector({"dbpath/000001.sst-epochDead", + "dbpath/000002.sst-epochDead"})); + + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + ASSERT_EQ(provider->delete_attempts(), + std::vector( + {"dbpath/000001.sst-epochDead", "dbpath/000002.sst-epochDead", + "dbpath/MANIFEST-epochDead", + "dbpath/smallest_new_file_number-epochDead"})); +} + +TEST(EloqPurgerCycleTest, BulkDeleteFailureFailsCycle) { + const std::string file = "000001.sst-epochDead"; + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{{file, MakeInfo(kNow - 2 * kHourMs)}}); + provider->SetDeleteStatus("dbpath/" + file, + IOStatus::IOError("injected delete failure")); + auto logger = std::make_shared(); + auto cfs = MakeCloudFileSystem(provider, logger); + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", false /*dry_run*/, + kHourMs, kHourMs, 10000); + + ASSERT_FALSE(purger.RunSinglePurgeCycle()); + ASSERT_EQ(provider->delete_attempts(), + std::vector{"dbpath/" + file}); + ASSERT_NE(logger->log().find( + "obsolete_selected=1 deleted=0 failed=1"), + std::string::npos); +} + +// Exhaust the decision boundary for a living epoch. This is the central safety +// property of the guard protocol: being live dominates every other signal, and +// an unreferenced file is eligible only when it is strictly below a nonzero +// published watermark. +TEST_F(EloqPurgerTest, LivingEpochSelectionSafetyMatrix) { + for (uint64_t threshold : {uint64_t{0}, uint64_t{1}, uint64_t{10}}) { + for (uint64_t file_number : + {uint64_t{1}, uint64_t{9}, uint64_t{10}, uint64_t{11}}) { + for (bool is_live : {false, true}) { + char name_buffer[64]; + snprintf(name_buffer, sizeof(name_buffer), "%06llu.sst-epochA", + static_cast(file_number)); + const std::string name(name_buffer); + const EloqPurger::PurgerAllFiles all_files{ + {name, MakeInfo(kNow - 10 * kHourMs)}}; + EloqPurger::PurgerLiveFileSet live_files; + if (is_live) { + live_files.insert(name); + } + + std::vector obsolete; + purger_.SelectObsoleteSSTFilesWithThreshold( + all_files, live_files, {{"epochA", threshold}}, kNow, &obsolete); + + const bool expected_delete = + !is_live && threshold != 0 && file_number < threshold; + SCOPED_TRACE(testing::Message() + << "threshold=" << threshold + << " file_number=" << file_number + << " live=" << is_live); + ASSERT_EQ(!obsolete.empty(), expected_delete); + if (expected_delete) { + ASSERT_EQ(obsolete, std::vector{name}); + } + } + } + } +} + +TEST_F(EloqPurgerTest, AgeGuardsUseInclusiveBoundaryAndRejectYoungerObjects) { + const uint64_t boundary = kNow - kHourMs; + const EloqPurger::PurgerAllFiles all_files{ + {"000001.sst-epochDead", MakeInfo(boundary)}, + {"000002.sst-epochDead", MakeInfo(boundary + 1)}, + {"MANIFEST-epochDeadBoundary", MakeInfo(boundary)}, + {"MANIFEST-epochDeadYoung", MakeInfo(boundary + 1)}, + {"smallest_new_file_number-epochDeadBoundary", MakeInfo(boundary)}, + {"smallest_new_file_number-epochDeadYoung", MakeInfo(boundary + 1)}, + }; + + std::vector obsolete; + purger_.SelectObsoleteSSTFilesWithThreshold(all_files, {}, {}, kNow, + &obsolete); + purger_.SelectObsoleteManifestFiles(all_files, {}, kNow, &obsolete); + purger_.SelectObsoleteFileNumberMarkers(all_files, {}, kNow, &obsolete); + ASSERT_EQ(obsolete, + std::vector({"000001.sst-epochDead", + "MANIFEST-epochDeadBoundary", + "smallest_new_file_number-epochDeadBoundary"})); +} + +// Full cycle over real serialized metadata. It simultaneously checks: +// * live files from both current and historical epochs survive; +// * current-epoch garbage below the guard is reclaimed; +// * an in-flight boundary file (number == guard) survives; +// * dead-epoch garbage obeys its age delay; and +// * current metadata and unrelated objects are never selected. +TEST(EloqPurgerCycleTest, DeletesOnlyObjectsProvenUnreachable) { + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{ + {"CLOUDMANIFEST-db-2", MakeInfo(kNow - 2 * kHourMs)}, + {"MANIFEST-epochNew", MakeInfo(kNow - 2 * kHourMs)}, + {"000050.sst-epochOld", MakeInfo(kNow - 10 * kHourMs)}, + {"000060.sst-epochOld", MakeInfo(kNow - 10 * kHourMs)}, + {"000150.sst-epochNew", MakeInfo(kNow - 10 * kHourMs)}, + {"000130.sst-epochNew", MakeInfo(kNow - 10 * kHourMs)}, + {"000140.sst-epochNew", MakeInfo(kNow - 10 * kHourMs)}, + {"000007.sst-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + {"000008.sst-epochOpening", MakeInfo(kNow - kHourMs + 1)}, + {"MANIFEST-epochDead", MakeInfo(kNow - 2 * kHourMs)}, + {"smallest_new_file_number-epochNew", + MakeInfo(kNow - 2 * kHourMs)}, + {"smallest_new_file_number-epochDead", + MakeInfo(kNow - 2 * kHourMs)}, + {"OPTIONS-000001", MakeInfo(kNow - 10 * kHourMs)}, + }); + provider->SetObjectContents( + "dbpath/CLOUDMANIFEST-db-2", + MakeCloudManifestContents("epochOld", {{100, "epochNew"}})); + provider->SetObjectContents("dbpath/MANIFEST-epochNew", + MakeManifestContents({50, 150})); + provider->SetObjectContents("dbpath/smallest_new_file_number-epochNew", + "140"); + auto cfs = MakeCloudFileSystem(provider); + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", false /*dry_run*/, + kHourMs, kHourMs, 10000); + + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + ASSERT_EQ(provider->delete_attempts(), + std::vector({ + "dbpath/000060.sst-epochOld", + "dbpath/000130.sst-epochNew", + "dbpath/000007.sst-epochDead", + "dbpath/MANIFEST-epochDead", + "dbpath/smallest_new_file_number-epochDead", + })); +} + +// Failover is deliberately two-phase. During the cycle that retires the old +// CLOUDMANIFEST, its guard and MANIFEST are still loaded and must protect old +// epoch files. Only a subsequent fresh cycle may classify that epoch as dead. +TEST(EloqPurgerCycleTest, FailoverRetiresProtectionBeforeReclaimingOldEpoch) { + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{ + {"CLOUDMANIFEST-db-1", MakeInfo(kNow - 10 * kHourMs)}, + {"CLOUDMANIFEST-db-2", MakeInfo(kNow - 2 * kHourMs)}, + {"MANIFEST-epochOld", MakeInfo(kNow - 10 * kHourMs)}, + {"MANIFEST-epochNew", MakeInfo(kNow - 2 * kHourMs)}, + {"000040.sst-epochOld", MakeInfo(kNow - 10 * kHourMs)}, + {"000060.sst-epochOld", MakeInfo(kNow - 10 * kHourMs)}, + {"smallest_new_file_number-epochOld", + MakeInfo(kNow - 10 * kHourMs)}, + {"smallest_new_file_number-epochNew", + MakeInfo(kNow - 2 * kHourMs)}, + }); + provider->SetObjectContents("dbpath/CLOUDMANIFEST-db-1", + MakeCloudManifestContents("epochOld")); + provider->SetObjectContents( + "dbpath/CLOUDMANIFEST-db-2", + MakeCloudManifestContents("epochOld", {{100, "epochNew"}})); + provider->SetObjectContents("dbpath/MANIFEST-epochOld", + MakeManifestContents({40})); + provider->SetObjectContents("dbpath/MANIFEST-epochNew", + MakeManifestContents({40})); + provider->SetObjectContents("dbpath/smallest_new_file_number-epochOld", + "50"); + provider->SetObjectContents("dbpath/smallest_new_file_number-epochNew", + "100"); + auto cfs = MakeCloudFileSystem(provider); + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", false /*dry_run*/, + kHourMs, kHourMs, 10000); + + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + ASSERT_EQ(provider->delete_attempts(), + std::vector{"dbpath/CLOUDMANIFEST-db-1"}); + + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + ASSERT_EQ(provider->delete_attempts(), + std::vector({ + "dbpath/CLOUDMANIFEST-db-1", + "dbpath/000060.sst-epochOld", + "dbpath/MANIFEST-epochOld", + "dbpath/smallest_new_file_number-epochOld", + })); +} + +// Every observation required to prove unreachability happens before the first +// delete. Inject failures at each phase and require the entire cycle to be +// side-effect free. +TEST(EloqPurgerCycleTest, EveryPreDeletionFailureFailsClosed) { + const std::vector phases = { + "list all", "list cloud manifests", "read cloud manifest", + "corrupt cloud manifest", "read guard", "corrupt guard", + "manifest metadata", "read manifest", "upload clock", + "clock metadata", + }; + + for (size_t i = 0; i < phases.size(); ++i) { + auto provider = MakeValidCycleProvider(); + switch (i) { + case 0: + provider->SetListAllStatus(IOStatus::IOError("injected list failure")); + break; + case 1: + provider->SetListPrefixStatus( + IOStatus::IOError("injected prefix-list failure")); + break; + case 2: + provider->SetGetStatus("dbpath/CLOUDMANIFEST-db-1", + IOStatus::IOError("injected CM read failure")); + break; + case 3: + provider->SetObjectContents("dbpath/CLOUDMANIFEST-db-1", "corrupt"); + break; + case 4: + provider->SetGetStatus( + "dbpath/smallest_new_file_number-epochA", + IOStatus::IOError("injected guard read failure")); + break; + case 5: + provider->SetObjectContents( + "dbpath/smallest_new_file_number-epochA", "-1"); + break; + case 6: + provider->SetMetadataStatus( + "dbpath/MANIFEST-epochA", + IOStatus::IOError("injected manifest metadata failure")); + break; + case 7: + provider->SetGetStatus( + "dbpath/MANIFEST-epochA", + IOStatus::IOError("injected manifest read failure")); + break; + case 8: + provider->SetPutStatus(IOStatus::IOError("injected clock put failure")); + break; + case 9: + provider->SetClockMetadataStatus( + IOStatus::IOError("injected clock metadata failure")); + break; + } + + auto cfs = MakeCloudFileSystem(provider); + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", false /*dry_run*/, + kHourMs, kHourMs, 10000); + SCOPED_TRACE(phases[i]); + ASSERT_FALSE(purger.RunSinglePurgeCycle()); + ASSERT_TRUE(provider->delete_attempts().empty()); + } +} + +TEST(EloqPurgerCycleTest, DryRunExecutesProofButNeverDeletes) { + auto provider = MakeValidCycleProvider(); + auto cfs = MakeCloudFileSystem(provider); + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", true /*dry_run*/, + kHourMs, kHourMs, 10000); + + ASSERT_TRUE(purger.RunSinglePurgeCycle()); + ASSERT_TRUE(provider->delete_attempts().empty()); +} + +// ---- Strict parsing of numeric control objects (guard marker) ---- + +// std::stoull would accept all of these; a permissive parse of "-1" yields +// UINT64_MAX, a threshold that authorizes deleting every non-live SST of a +// live epoch. +TEST(EloqPurgerGuardParseTest, RejectsMalformedMarkerContents) { + const std::string key = "dbpath/smallest_new_file_number-epochA"; + for (const std::string &bad : + {std::string("-1"), std::string("+1"), std::string(" 1"), + std::string("1 "), std::string("12abc"), std::string("0x10"), + std::string(""), std::string("\n"), std::string("1.5"), + std::string("99999999999999999999999"), + std::string(70, '7') /* oversized object */}) { + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{}); + provider->SetObjectContents(key, bad); + auto cfs = MakeCloudFileSystem(provider); + S3FileNumberReader reader("test-bucket", "dbpath", "epochA", cfs.get()); + + uint64_t threshold = 12345; + Status s = reader.ReadSmallestFileNumber(&threshold); + ASSERT_TRUE(s.IsCorruption()) << "input '" << bad << "' -> " << s.ToString(); + ASSERT_EQ(threshold, std::numeric_limits::min()); + } +} + +TEST(EloqPurgerGuardParseTest, AcceptsWellFormedMarkerContents) { + const std::string key = "dbpath/smallest_new_file_number-epochA"; + const std::vector> cases = { + {"0", 0}, + {"123", 123}, + {"123\n", 123}, + {"123\r\n", 123}, + {"18446744073709551615", std::numeric_limits::max()}, + }; + for (const auto &[content, expected] : cases) { + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{}); + provider->SetObjectContents(key, content); + auto cfs = MakeCloudFileSystem(provider); + S3FileNumberReader reader("test-bucket", "dbpath", "epochA", cfs.get()); + + uint64_t threshold = 0; + ASSERT_OK(reader.ReadSmallestFileNumber(&threshold)) << content; + ASSERT_EQ(threshold, expected) << content; + } +} + +// ---- Missing marker must fail closed ---- + +TEST(EloqPurgerCycleTest, MissingGuardMarkerAbortsCycle) { + const std::string cloud_manifest_name = "CLOUDMANIFEST-db-1"; + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{ + {cloud_manifest_name, MakeInfo(kNow - 2 * kHourMs)}, + {"000001.sst-epochA", MakeInfo(kNow - 2 * kHourMs)}}); + provider->SetObjectContents("dbpath/" + cloud_manifest_name, + MakeCloudManifestContents("epochA")); + // No smallest_new_file_number-epochA object at all. + provider->SetGetStatus("dbpath/smallest_new_file_number-epochA", + IOStatus::NotFound()); + auto cfs = MakeCloudFileSystem(provider); + + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", false /*dry_run*/, + kHourMs, kHourMs, 10000, true /*require_guard_marker*/); + ASSERT_FALSE(purger.RunSinglePurgeCycle()); + ASSERT_TRUE(provider->delete_attempts().empty()); +} + +// ---- Malformed CLOUDMANIFEST term must fail closed ---- + +// A permissive parse of "1x" reads as 1; a bogus high term could make the +// authoritative CLOUDMANIFEST look superseded. +TEST(EloqPurgerCycleTest, MalformedCloudManifestTermAbortsCycle) { + const std::string bad_name = "CLOUDMANIFEST-db-1x"; + auto provider = std::make_shared( + EloqPurger::PurgerAllFiles{{bad_name, MakeInfo(kNow - 2 * kHourMs)}, + {"MANIFEST-epochA", + MakeInfo(kNow - 2 * kHourMs)}, + {"000001.sst-epochDead", + MakeInfo(kNow - 2 * kHourMs)}}); + provider->SetObjectContents("dbpath/" + bad_name, + MakeCloudManifestContents("epochA")); + provider->SetObjectContents("dbpath/MANIFEST-epochA", + MakeManifestContents({})); + provider->SetObjectContents("dbpath/smallest_new_file_number-epochA", "5"); + auto cfs = MakeCloudFileSystem(provider); + + EloqPurger purger(cfs.get(), "test-bucket", "dbpath", false /*dry_run*/, + kHourMs, kHourMs, 10000); + ASSERT_FALSE(purger.RunSinglePurgeCycle()); + ASSERT_TRUE(provider->delete_attempts().empty()); +} + +// ---- MANIFEST corruption must fail closed for the purger ---- + +// A torn tail is tolerated by the DB-open reader (so recovery can proceed) +// but must be an error for a scanner that deletes data: a skipped tail +// containing a file addition would make a live SST look unreferenced. +TEST(EloqPurgerManifestScanTest, StrictModeRejectsTornManifestTail) { + const std::string path = test::TmpDir() + "/purger_torn_manifest_" + + std::to_string(Env::Default()->NowMicros()); + { + std::unique_ptr writer; + ASSERT_OK(WritableFileWriter::Create(FileSystem::Default(), path, + FileOptions(), &writer, nullptr)); + log::Writer log_writer(std::move(writer), 0, false); + VersionEdit edit; + edit.SetNextFile(42); + std::string record; + ASSERT_TRUE(edit.EncodeTo(&record)); + ASSERT_OK(log_writer.AddRecord(WriteOptions(), record)); + ASSERT_OK(log_writer.file()->Sync(IOOptions(), false)); + } + // Append a torn record: a partial header is exactly what a truncated + // upload or a corrupted tail looks like. + { + std::ofstream out(path, std::ios::binary | std::ios::app); + const std::string garbage(5, '\xab'); + out.write(garbage.data(), static_cast(garbage.size())); + } + + uint64_t max_file_number = 0; + // DB-open behavior: tolerate the tail so recovery can proceed. + ASSERT_OK(ManifestReader::GetMaxFileNumberFromManifest( + FileSystem::Default().get(), path, &max_file_number, + kManifestScanDefaultMode)); + ASSERT_EQ(max_file_number, 42u); + + // Purger behavior: refuse. + max_file_number = 0; + Status strict = ManifestReader::GetMaxFileNumberFromManifest( + FileSystem::Default().get(), path, &max_file_number, + kManifestScanStrictMode); + ASSERT_FALSE(strict.ok()) << strict.ToString(); + + ASSERT_EQ(std::remove(path.c_str()), 0); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/cloud/file_number_guard.cc b/cloud/file_number_guard.cc new file mode 100644 index 0000000..5057858 --- /dev/null +++ b/cloud/file_number_guard.cc @@ -0,0 +1,655 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ + +#include "cloud/file_number_guard.h" + +#include +#include + +#include +#include +#include +#include + +#include "cloud/cloud_manifest.h" +#include "cloud/cloud_scheduler.h" +#include "cloud/filename.h" +#include "file/filename.h" +#include "rocksdb/cloud/cloud_file_system_impl.h" +#include "rocksdb/cloud/cloud_storage_provider.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { + +std::string SmallestFileNumberObjectKey(const std::string &object_path, + const std::string &epoch) { + std::ostringstream oss; + oss << object_path; + if (!object_path.empty() && object_path.back() != '/') { + oss << "/"; + } + oss << kSmallestFileNumberFilePrefix << epoch; + return oss.str(); +} + +// One-shot PUT of the guard object. Shared by the publisher and by +// CloudFileSystemImpl::BlockPurger when no publisher is registered. Callers +// that care about PUT ordering must serialize calls themselves (the +// publisher's publish_mutex_ does this). +IOStatus PutSmallestFileNumberObject(CloudFileSystemImpl *cfs, uint64_t value, + const std::string &epoch) { + if (epoch.empty()) { + // Publishing to "smallest_new_file_number-" (empty epoch) would create a + // malformed guard object no reader ever consults. Should be impossible + // now that the epoch is sourced from the cloud manifest, but keep the + // guard. + Log(InfoLogLevel::ERROR_LEVEL, cfs->info_log_, + "[fng] Refusing to publish smallest file number: epoch is empty"); + return IOStatus::InvalidArgument("empty epoch for file number guard"); + } + + std::string content = std::to_string(value); + std::string object_key = + SmallestFileNumberObjectKey(cfs->GetDestObjectPath(), epoch); + + char tmp_template[] = "/tmp/smallest_file_number_upload_XXXXXX"; + int fd = mkstemp(tmp_template); + if (fd == -1) { + Log(InfoLogLevel::ERROR_LEVEL, cfs->info_log_, + "[fng] Failed to create temp file for publishing smallest file " + "number: %s", + strerror(errno)); + return IOStatus::IOError("Failed to create temp file"); + } + std::string temp_file_path = tmp_template; + if (fchmod(fd, S_IRUSR | S_IWUSR) != 0) { + Log(InfoLogLevel::WARN_LEVEL, cfs->info_log_, + "[fng] Failed to set restricted permissions on temp file: %s", + strerror(errno)); + } + ssize_t written = write(fd, content.c_str(), content.size()); + close(fd); + if (written < 0 || static_cast(written) != content.size()) { + Log(InfoLogLevel::ERROR_LEVEL, cfs->info_log_, + "[fng] Failed to write temp file %s: %s", temp_file_path.c_str(), + strerror(errno)); + std::remove(temp_file_path.c_str()); + return IOStatus::IOError("Failed to write temp file"); + } + + IOStatus st = cfs->GetStorageProvider()->PutCloudObject( + temp_file_path, cfs->GetDestBucketName(), object_key); + TEST_SYNC_POINT_CALLBACK( + "FileNumberGuard::PutSmallestFileNumberObject:Status", &st); + + if (std::remove(temp_file_path.c_str()) != 0) { + Log(InfoLogLevel::WARN_LEVEL, cfs->info_log_, + "[fng] Failed to remove temp file %s", temp_file_path.c_str()); + } + + if (st.ok()) { + Log(InfoLogLevel::INFO_LEVEL, cfs->info_log_, + "[fng] Published smallest file number %llu for epoch %s (%s)", + static_cast(value), epoch.c_str(), + object_key.c_str()); + } else { + Log(InfoLogLevel::ERROR_LEVEL, cfs->info_log_, + "[fng] Failed to publish smallest file number %llu for epoch %s: %s", + static_cast(value), epoch.c_str(), + st.ToString().c_str()); + } + return st; +} + +// ---------------- FileNumberSlidingWindow ---------------- + +void FileNumberSlidingWindow::Add(uint64_t file_number, uint64_t thread_id, + int job_id) { + // emplace keeps an existing entry: for multi-CF jobs firing several begin + // events, the earliest (smallest, hence safest) snapshot wins. + entries_.emplace(std::make_pair(thread_id, job_id), Entry{file_number, {}}); +} + +void FileNumberSlidingWindow::MarkRemoved(uint64_t thread_id, int job_id, + Clock::time_point now) { + auto it = entries_.find(std::make_pair(thread_id, job_id)); + if (it != entries_.end()) { + it->second.removed = true; + it->second.removed_at = now; + } +} + +uint64_t FileNumberSlidingWindow::SmallestFileNumber(Clock::time_point now) { + uint64_t smallest = std::numeric_limits::max(); + for (auto it = entries_.begin(); it != entries_.end();) { + if (it->second.removed && now - it->second.removed_at >= entry_duration_) { + it = entries_.erase(it); + continue; + } + if (it->second.file_number < smallest) { + smallest = it->second.file_number; + } + ++it; + } + return smallest; +} + +// ---------------- FileNumberGuardPublisher ---------------- + +namespace { + +std::vector NormalizeDirectories( + std::vector directories) { + for (auto &dir : directories) { + dir = rtrim_if(std::move(dir), '/'); + } + return directories; +} + +} // namespace + +FileNumberGuardPublisher::FileNumberGuardPublisher( + CloudFileSystemImpl *cfs, std::chrono::milliseconds publish_interval, + std::chrono::milliseconds entry_duration, + std::vector db_directories) + : cfs_(cfs), + publish_interval_(publish_interval), + db_directories_(NormalizeDirectories(std::move(db_directories))), + window_(entry_duration), + scheduler_(CloudScheduler::Get()) {} + +bool FileNumberGuardPublisher::CoversFile( + const std::string &local_path) const { + if (db_directories_.empty()) { + // Unknown layout: gate everything rather than risk an unprotected + // upload. + return true; + } + const std::string dir = rtrim_if(dirname(local_path), '/'); + for (const auto &db_dir : db_directories_) { + if (dir == db_dir) { + return true; + } + } + return false; +} + +FileNumberGuardPublisher::~FileNumberGuardPublisher() { Stop(); } + +void FileNumberGuardPublisher::Start() { + std::lock_guard lk(stop_mutex_); + if (stopped_ || job_handle_ >= 0) { + return; + } + auto interval = std::chrono::duration_cast( + publish_interval_); + job_handle_ = scheduler_->ScheduleRecurringJob( + interval, interval, [this](void *) { PeriodicPublish(); }, nullptr); +} + +void FileNumberGuardPublisher::Stop() { + long handle = -1; + bool notify = false; + { + std::lock_guard lk(stop_mutex_); + if (!stopped_) { + stopped_ = true; + notify = true; + handle = job_handle_; + job_handle_ = -1; + } + } + if (notify) { + TEST_SYNC_POINT("FileNumberGuardPublisher::Stop:Stopped"); + stop_cv_.notify_all(); + } + if (handle >= 0) { + // Waits for a currently running periodic callback to finish, so after + // this returns no callback can touch this object. + scheduler_->CancelJob(handle); + } + + // ProtectFileUpload holds this gate across both guard publication and the + // SST upload. Since stopped_ was set first, a protection that has not yet + // crossed its final stopped_ check cannot start an upload; one that has + // crossed it completes before this lock is acquired. + std::lock_guard publish_lk(publish_mutex_); +} + +std::string FileNumberGuardPublisher::CurrentEpoch() const { + auto *manifest = cfs_->GetCloudManifest(); + if (manifest == nullptr) { + return ""; + } + return manifest->GetCurrentEpoch(); +} + +Status FileNumberGuardPublisher::PutValue(uint64_t value, + const std::string &epoch) { + return PutSmallestFileNumberObject(cfs_, value, epoch); +} + +void FileNumberGuardPublisher::OnJobBegin(uint64_t file_number, + uint64_t thread_id, int job_id) { + std::lock_guard lk(state_mutex_); + window_.Add(file_number, thread_id, job_id); +} + +Status FileNumberGuardPublisher::ProtectFileUpload( + uint64_t file_number, const std::function &upload) { + { + std::lock_guard lk(stop_mutex_); + if (stopped_) { + return Status::ShutdownInProgress("file number guard stopped"); + } + } + + std::lock_guard publish_lk(publish_mutex_); + { + std::lock_guard lk(stop_mutex_); + if (stopped_) { + return Status::ShutdownInProgress("file number guard stopped"); + } + } + + uint64_t desired; + bool needs_publish; + { + std::lock_guard lk(state_mutex_); + desired = window_.SmallestFileNumber(); + if (desired == std::numeric_limits::max()) { + return Status::InvalidArgument("no live file number registration"); + } + if (desired > file_number) { + return Status::InvalidArgument( + "live file number minimum exceeds SST file number"); + } + needs_publish = + !sentinel_active_ && (remote_unknown_ || desired < last_published_); + } + + if (needs_publish) { + std::string epoch = CurrentEpoch(); + if (epoch.empty()) { + Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, + "[fng] Cannot protect SST %llu: epoch unavailable", + static_cast(file_number)); + return Status::InvalidArgument("empty epoch for file number guard"); + } + + Status st; + auto backoff = std::chrono::milliseconds(100); + const auto max_backoff = std::chrono::milliseconds(2000); + while (true) { + { + std::lock_guard lk(stop_mutex_); + if (stopped_) { + return Status::ShutdownInProgress("file number guard stopped"); + } + } + st = PutValue(desired, epoch); + if (st.ok()) { + std::lock_guard lk(state_mutex_); + last_published_ = desired; + sentinel_active_ = false; + remote_unknown_ = false; + break; + } + MarkRemoteUnknown(); + Log(InfoLogLevel::WARN_LEVEL, cfs_->info_log_, + "[fng] Downward publish of %llu for SST %llu failed (%s); retrying " + "in %lld ms", + static_cast(desired), + static_cast(file_number), st.ToString().c_str(), + static_cast(backoff.count())); + std::unique_lock lk(stop_mutex_); + if (stop_cv_.wait_for(lk, backoff, [this] { return stopped_; })) { + return Status::ShutdownInProgress("file number guard stopped"); + } + backoff = std::min(backoff * 2, max_backoff); + } + } + + { + // This check is the upload/Stop linearization point. publish_mutex_ stays + // held through upload, so Stop either sets stopped_ before this check or + // waits for the callback to finish. + std::lock_guard lk(stop_mutex_); + if (stopped_) { + return Status::ShutdownInProgress("file number guard stopped"); + } + } + if (upload) { + return upload(); + } + return Status::OK(); +} + +void FileNumberGuardPublisher::OnJobEnd(uint64_t thread_id, int job_id) { + std::lock_guard lk(state_mutex_); + window_.MarkRemoved(thread_id, job_id); +} + +Status FileNumberGuardPublisher::BlockPurger() { + std::string epoch = CurrentEpoch(); + if (epoch.empty()) { + Log(InfoLogLevel::ERROR_LEVEL, cfs_->info_log_, + "[fng] Cannot block purger: epoch unavailable"); + return Status::InvalidArgument("empty epoch for file number guard"); + } + std::lock_guard publish_lk(publish_mutex_); + Status st = PutValue(0, epoch); + if (st.ok()) { + std::lock_guard lk(state_mutex_); + sentinel_active_ = true; + remote_unknown_ = false; + } else { + MarkRemoteUnknown(); + } + return st; +} + +void FileNumberGuardPublisher::PeriodicPublish() { + { + std::lock_guard lk(stop_mutex_); + if (stopped_) { + return; + } + } + + uint64_t desired; + { + std::lock_guard lk(state_mutex_); + desired = window_.SmallestFileNumber(); + if (desired == last_published_ && !sentinel_active_ && !remote_unknown_) { + return; + } + } + + std::string epoch = CurrentEpoch(); + if (epoch.empty()) { + Log(InfoLogLevel::WARN_LEVEL, cfs_->info_log_, + "[fng] Skipping periodic publish: epoch unavailable"); + return; + } + + // try_lock: if a downward publish is in flight (possibly retrying with + // backoff), skip this tick instead of stalling the shared scheduler + // thread behind S3 latency. + std::unique_lock publish_lk(publish_mutex_, std::try_to_lock); + if (!publish_lk.owns_lock()) { + return; + } + + { + std::lock_guard lk(stop_mutex_); + if (stopped_) { + return; + } + } + + { + // Staleness re-check after acquiring the publish mutex: a downward + // publish that landed while we waited must not be overwritten with our + // older, higher value. + std::lock_guard lk(state_mutex_); + desired = window_.SmallestFileNumber(); + if (desired == last_published_ && !sentinel_active_ && !remote_unknown_) { + return; + } + } + + Status st = PutValue(desired, epoch); + if (st.ok()) { + std::lock_guard lk(state_mutex_); + last_published_ = desired; + sentinel_active_ = false; + remote_unknown_ = false; + } else { + MarkRemoteUnknown(); + } +} + +void FileNumberGuardPublisher::MarkRemoteUnknown() { + std::lock_guard lk(state_mutex_); + last_published_ = std::numeric_limits::max(); + sentinel_active_ = false; + remote_unknown_ = true; +} + +uint64_t FileNumberGuardPublisher::TEST_LastPublished() { + std::lock_guard lk(state_mutex_); + return last_published_; +} + +// ---------------- FileNumberGuardListener ---------------- + +void FileNumberGuardListener::JobBegin(uint64_t file_number, uint64_t thread_id, + int job_id) { + if (!publisher_) { + return; + } + assert(job_id >= 0); + publisher_->OnJobBegin(file_number, thread_id, job_id); +} + +void FileNumberGuardListener::OnFlushBegin(DB *db, const FlushJobInfo &info) { + if (!publisher_) { + return; + } + uint64_t file_number = info.file_number; + if (file_number == 0) { + if (db == nullptr) { + return; + } + // Atomic flush notifies before allocating its output numbers, so every + // output from that job is greater than this conservative snapshot. + file_number = db->GetNextFileNumber() - 1; + } + JobBegin(file_number, info.thread_id, info.job_id); +} + +void FileNumberGuardListener::OnFlushFinished(DB * /*db*/, + const FlushJobEndInfo &info) { + if (publisher_) { + publisher_->OnJobEnd(info.thread_id, info.job_id); + } +} + +void FileNumberGuardListener::OnCompactionBegin(DB *db, + const CompactionJobInfo &info) { + if (db != nullptr) { + // Compaction output numbers are allocated after this callback. + JobBegin(db->GetNextFileNumber() - 1, info.thread_id, info.job_id); + } +} + +void FileNumberGuardListener::OnCompactionCompleted( + DB * /*db*/, const CompactionJobInfo &info) { + if (publisher_) { + publisher_->OnJobEnd(info.thread_id, info.job_id); + } +} + +void FileNumberGuardListener::OnExternalFileIngestionStarted( + DB * /*db*/, uint64_t file_number) { + if (publisher_) { + publisher_->OnJobBegin(file_number, file_number, kExternalFileJobId); + } +} + +void FileNumberGuardListener::OnExternalFileIngestionFinished( + DB * /*db*/, uint64_t file_number) { + if (publisher_) { + publisher_->OnJobEnd(file_number, kExternalFileJobId); + } +} + +void FileNumberGuardListener::OnTableFileCreationStarted( + const TableFileCreationBriefInfo &info) { + if (!publisher_ || info.reason != TableFileCreationReason::kRecovery) { + return; + } + + uint64_t file_number = 0; + FileType file_type; + const bool parsed = + ParseFileName(basename(info.file_path), &file_number, &file_type); + assert(parsed && file_type == kTableFile); + if (!parsed || file_type != kTableFile) { + return; + } + + const uint64_t thread_id = Env::Default()->GetThreadID(); + const RecoveryKey key(info.db_name, info.cf_name, info.job_id); + { + std::lock_guard lock(recovery_mutex_); + const bool inserted = + recovery_files_.emplace(key, RecoveryEntry{file_number, thread_id}) + .second; + assert(inserted); + if (!inserted) { + return; + } + } + publisher_->OnJobBegin(file_number, file_number, kRecoveryJobId); +} + +void FileNumberGuardListener::OnTableFileCreated( + const TableFileCreationInfo &info) { + if (!publisher_ || info.reason != TableFileCreationReason::kRecovery) { + return; + } + + const RecoveryKey key(info.db_name, info.cf_name, info.job_id); + uint64_t file_number = 0; + { + std::lock_guard lock(recovery_mutex_); + auto it = recovery_files_.find(key); + assert(it != recovery_files_.end()); + if (it == recovery_files_.end()) { + return; + } + assert(it->second.thread_id == Env::Default()->GetThreadID()); + file_number = it->second.file_number; + recovery_files_.erase(it); + } + publisher_->OnJobEnd(file_number, kRecoveryJobId); +} + +// ---------------- CloudFileSystemImpl glue ---------------- + +void CloudFileSystemImpl::SetFileNumberGuardPublisher( + std::shared_ptr publisher) { + std::lock_guard install_lk(file_number_guard_install_mutex_); + auto previous = std::atomic_load(&file_number_guard_); + if (previous) { + previous->Stop(); + } + std::atomic_store(&file_number_guard_, std::move(publisher)); +} + +Status CloudFileSystemImpl::InstallFileNumberGuardPublisher( + const std::shared_ptr &publisher) { + if (!publisher) { + return Status::InvalidArgument("null file number guard publisher"); + } + std::lock_guard install_lk(file_number_guard_install_mutex_); + auto previous = std::atomic_load(&file_number_guard_); + if (previous) { + previous->Stop(); + } + + Status status = publisher->BlockPurger(); + if (!status.ok()) { + // Leave the old stopped publisher installed. Until a later successful + // replacement, uploads then fail closed instead of bypassing the guard. + return status; + } + std::atomic_store(&file_number_guard_, publisher); + return Status::OK(); +} + +bool CloudFileSystemImpl::RemoveFileNumberGuardPublisher( + const std::shared_ptr &expected) { + if (!expected) { + return false; + } + std::lock_guard install_lk(file_number_guard_install_mutex_); + auto current = std::atomic_load(&file_number_guard_); + if (current != expected) { + return false; + } + expected->Stop(); + std::atomic_store(&file_number_guard_, + std::shared_ptr()); + return true; +} + +bool CloudFileSystemImpl::StopFileNumberGuardPublisher( + const std::shared_ptr &expected) { + if (!expected) { + return false; + } + std::lock_guard install_lk(file_number_guard_install_mutex_); + if (std::atomic_load(&file_number_guard_) != expected) { + return false; + } + expected->Stop(); + return true; +} + +std::shared_ptr +CloudFileSystemImpl::GetFileNumberGuardPublisher() const { + return std::atomic_load(&file_number_guard_); +} + +void CloudFileSystemImpl::StopFileNumberGuard() { + std::lock_guard install_lk(file_number_guard_install_mutex_); + auto publisher = std::atomic_load(&file_number_guard_); + if (publisher) { + publisher->Stop(); + std::atomic_store(&file_number_guard_, + std::shared_ptr()); + } +} + +Status CloudFileSystemImpl::BlockPurger() { + std::lock_guard install_lk(file_number_guard_install_mutex_); + auto publisher = std::atomic_load(&file_number_guard_); + if (publisher) { + return publisher->BlockPurger(); + } + // No publisher is installed: write a one-shot sentinel that remains until + // an external writer replaces it. The embedder owns that lifetime because + // there is no local scheduler to publish a finite threshold. + auto *manifest = GetCloudManifest(); + if (manifest == nullptr) { + return Status::InvalidArgument( + "BlockPurger requires a loaded cloud manifest"); + } + return PutSmallestFileNumberObject(this, 0, manifest->GetCurrentEpoch()); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/cloud/file_number_guard.h b/cloud/file_number_guard.h new file mode 100644 index 0000000..eaddd0c --- /dev/null +++ b/cloud/file_number_guard.h @@ -0,0 +1,256 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rocksdb/listener.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class CloudFileSystemImpl; +class CloudScheduler; + +// Basename prefix of the guard object; the full basename is +// . Exposed so the purger's marker selector matches listing +// entries with the same definition the key builder uses. +inline constexpr char kSmallestFileNumberFilePrefix[] = + "smallest_new_file_number-"; + +// The single source of truth for the purger guard object's S3 key. Both the +// writer (FileNumberGuardPublisher) and the reader (S3FileNumberReader in +// eloq_purger.cc) build the key through this function; the format is part of +// the on-cloud protocol and must not change: +// /smallest_new_file_number- +// The object's value is an ASCII uint64. Semantics: no in-flight job on the +// epoch's writer node can create an SST with a file number below this value. +// 0 means "purging blocked for this epoch"; UINT64_MAX means "nothing in +// flight". +std::string SmallestFileNumberObjectKey(const std::string &object_path, + const std::string &epoch); + +// Tracks the file-number snapshots of in-flight flush/compaction jobs, keyed +// by (thread_id, job_id). A completed job's entry lingers for entry_duration +// before it stops contributing to the minimum, giving the job's MANIFEST +// update time to reach the cloud. Pure in-memory bookkeeping: no locking +// (the owner locks) and no I/O, so it is unit-testable in isolation. +class FileNumberSlidingWindow { + public: + using Clock = std::chrono::steady_clock; + + explicit FileNumberSlidingWindow(std::chrono::milliseconds entry_duration) + : entry_duration_(entry_duration) {} + + // Registers a job's begin snapshot. Re-adding the same (thread_id, job_id) + // keeps the existing (earlier, hence smaller and safer) entry. + void Add(uint64_t file_number, uint64_t thread_id, int job_id); + + // Marks a job complete. The entry keeps contributing to the minimum until + // it has lingered for entry_duration past `now`. + void MarkRemoved(uint64_t thread_id, int job_id, + Clock::time_point now = Clock::now()); + + // Smallest file number among live and still-lingering entries; expired + // entries are dropped as a side effect. UINT64_MAX when nothing remains. + uint64_t SmallestFileNumber(Clock::time_point now = Clock::now()); + + size_t Size() const { return entries_.size(); } + + private: + struct Entry { + uint64_t file_number; + Clock::time_point removed_at; + bool removed = false; + }; + + std::map, Entry> entries_; + std::chrono::milliseconds entry_duration_; +}; + +// Publishes the smallest_new_file_number- guard object for the epoch +// this node is writing. The epoch is always read from the cloud manifest at +// publish time, never injected externally. +// +// Locking design (load-bearing, see the purger safety argument): +// - state_mutex_ guards the window and remote-publication state; critical +// sections are O(microseconds) and never perform I/O. +// - publish_mutex_ serializes every S3 PUT. Two concurrent PUTs could land +// out of order and reinstate a dangerously high threshold; the publish +// mutex plus the post-acquire staleness re-check make that impossible. +// - ProtectFileUpload serializes a required downward PUT and the protected +// SST upload callback. Stop() first marks the publisher stopped, then waits +// on the same publish gate. Thus Stop wins before the callback or waits for +// an upload that already crossed the stopped_ check. A successful PUT +// records its value; an ambiguous failure resets the state to unknown/MAX. +// +// cfs_ is non-owning. CloudFileSystemImpl owns an installed publisher and +// synchronously stops it before destroying the manifest and storage-provider +// members reached by publisher callbacks. +class FileNumberGuardPublisher { + public: + // db_directories lists every directory in which this DB may legitimately + // place its own SST files: the DB directory itself plus any configured + // db_paths/cf_paths. Files written elsewhere through the cloud file system + // (checkpoints, column family exports) are not DB-visible SSTs -- no + // flush/compaction job registers them and the purger never evaluates them + // as this DB's live or obsolete files -- so they are not gated. An empty + // list gates every SST, which is the conservative default. + FileNumberGuardPublisher(CloudFileSystemImpl *cfs, + std::chrono::milliseconds publish_interval, + std::chrono::milliseconds entry_duration, + std::vector db_directories = {}); + ~FileNumberGuardPublisher(); + + FileNumberGuardPublisher(const FileNumberGuardPublisher &) = delete; + FileNumberGuardPublisher &operator=(const FileNumberGuardPublisher &) = + delete; + + // Schedules the periodic publish job. Idempotent. + void Start(); + + // Cancels the periodic job, unblocks any in-progress downward retry loop, + // and waits for any protected SST upload already in progress. Idempotent; + // called from the destructor. + void Stop(); + + // Registers a flush/compaction job in the live window. This bookkeeping-only + // listener path performs no remote I/O. + void OnJobBegin(uint64_t file_number, uint64_t thread_id, int job_id); + + // The job completed; its entry starts lingering. + void OnJobEnd(uint64_t thread_id, int job_id); + + // Ensures the current live-window minimum is published at or below the SST + // file number, then invokes upload while holding the serialized publish + // gate. A required downward PUT retries until it succeeds or Stop() + // interrupts it. Tests may omit upload to exercise protection alone. + Status ProtectFileUpload( + uint64_t file_number, + const std::function &upload = std::function()); + + // Publishes the 0 sentinel ("purging blocked") for the current epoch. + // Does not advance last_published_. Protected SST uploads preserve the + // known-safe sentinel; the next periodic pass replaces it with the current + // live-window minimum. Called at DB open before recovery can flush, and by + // the embedder around leader transfer. + Status BlockPurger(); + + // True when local_path lies in one of this DB's own SST directories, i.e. + // the file is a DB-visible SST that must be protected before upload. + // Always true when no directories were configured. + bool CoversFile(const std::string &local_path) const; + + // The periodic publish body: computes the window minimum (UINT64_MAX when + // idle) and publishes it if it differs from the last published value or if + // the remote value is a sentinel/unknown after an ambiguous PUT failure. + // Public so tests can drive it without waiting for the timer. + void PeriodicPublish(); + + uint64_t TEST_LastPublished(); + + private: + // Serialized by publish_mutex_ (caller must hold it). Refuses to publish + // when the epoch is empty. + Status PutValue(uint64_t value, const std::string &epoch); + + // Current epoch from the cloud manifest; empty when unavailable. + std::string CurrentEpoch() const; + + void MarkRemoteUnknown(); + + CloudFileSystemImpl *cfs_; + const std::chrono::milliseconds publish_interval_; + // Normalized (no trailing separator) directories holding this DB's SSTs. + // Immutable after construction, so it needs no lock. + const std::vector db_directories_; + + std::mutex state_mutex_; + FileNumberSlidingWindow window_; + uint64_t last_published_ = std::numeric_limits::max(); + // The remote object is known to contain the safe 0 sentinel. Protected SST + // uploads preserve it until PeriodicPublish installs a real watermark. + bool sentinel_active_ = false; + // A PUT reported failure and may nevertheless have committed remotely. + // No comparison against last_published_ is safe until a real PUT succeeds. + bool remote_unknown_ = false; + + std::mutex publish_mutex_; + + std::mutex stop_mutex_; + std::condition_variable stop_cv_; + bool stopped_ = false; + + std::shared_ptr scheduler_; + long job_handle_ = -1; +}; + +// EventListener bridging RocksDB job events to the publisher. Registered +// automatically by DBCloudImpl::Open when +// CloudFileSystemOptions::publish_file_number_guard is set. +class FileNumberGuardListener : public EventListener { + public: + explicit FileNumberGuardListener( + std::shared_ptr publisher) + : publisher_(std::move(publisher)) {} + + const char *Name() const override { return "FileNumberGuardListener"; } + + void OnFlushBegin(DB *db, const FlushJobInfo &info) override; + void OnFlushFinished(DB *db, const FlushJobEndInfo &info) override; + void OnCompactionBegin(DB *db, const CompactionJobInfo &info) override; + void OnCompactionCompleted(DB *db, const CompactionJobInfo &info) override; + void OnExternalFileIngestionStarted(DB *db, uint64_t file_number) override; + void OnExternalFileIngestionFinished(DB *db, uint64_t file_number) override; + void OnTableFileCreationStarted(const TableFileCreationBriefInfo &) override; + void OnTableFileCreated(const TableFileCreationInfo &info) override; + +private: + void JobBegin(uint64_t file_number, uint64_t thread_id, int job_id); + + static constexpr int kExternalFileJobId = -1; + static constexpr int kRecoveryJobId = -2; + + struct RecoveryEntry { + uint64_t file_number; + uint64_t thread_id; + }; + using RecoveryKey = std::tuple; + + std::shared_ptr publisher_; + std::mutex recovery_mutex_; + std::map recovery_files_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/cloud/file_number_guard_test.cc b/cloud/file_number_guard_test.cc new file mode 100644 index 0000000..118dd5d --- /dev/null +++ b/cloud/file_number_guard_test.cc @@ -0,0 +1,1242 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ + +#include "cloud/file_number_guard.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cloud/cloud_manifest.h" +#include "file/writable_file_writer.h" +#include "rocksdb/cloud/cloud_file_system.h" +#include "rocksdb/cloud/cloud_file_system_impl.h" +#include "rocksdb/cloud/cloud_storage_provider.h" +#include "rocksdb/cloud/db_cloud.h" +#include "rocksdb/cloud/cloud_storage_provider_impl.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/utilities/options_type.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +constexpr uint64_t kMax = std::numeric_limits::max(); +constexpr auto kAsyncWaitTimeout = std::chrono::seconds(5); + +// Records every PutCloudObject; everything else is unsupported. The +// publisher only ever needs PUT. +class RecordingStorageProvider : public CloudStorageProvider { + public: + const char *Name() const override { return "recording"; } + + void SetPutStatus(const std::string &object_path, IOStatus status) { + std::lock_guard lk(mu_); + put_statuses_[object_path] = std::move(status); + } + + IOStatus PutCloudObject(const std::string &local_path, + const std::string & /*bucket_name*/, + const std::string &object_path) override { + TEST_SYNC_POINT_CALLBACK("FileNumberGuardTest::PutCloudObject", + const_cast(&object_path)); + std::ifstream f(local_path); + std::string content((std::istreambuf_iterator(f)), + std::istreambuf_iterator()); + std::lock_guard lk(mu_); + ++put_count_; + auto status = put_statuses_.find(object_path); + if (status != put_statuses_.end() && !status->second.ok()) { + return status->second; + } + objects_[object_path] = content; + return IOStatus::OK(); + } + + int PutCount() { + std::lock_guard lk(mu_); + return put_count_; + } + + std::string Content(const std::string &key) { + std::lock_guard lk(mu_); + auto it = objects_.find(key); + return it == objects_.end() ? "" : it->second; + } + + bool HasObject(const std::string &key) { + std::lock_guard lk(mu_); + return objects_.find(key) != objects_.end(); + } + + // -- everything below is unused by the publisher -- + IOStatus CreateBucket(const std::string &) override { return NotSup(); } + IOStatus ExistsBucket(const std::string &) override { return NotSup(); } + IOStatus EmptyBucket(const std::string &, const std::string &) override { + return NotSup(); + } + IOStatus DeleteCloudObject(const std::string &, + const std::string &) override { + return NotSup(); + } + IOStatus ListCloudObjects(const std::string &, const std::string &, + std::vector *) override { + return NotSup(); + } + IOStatus ListCloudObjects( + const std::string &, const std::string &, + std::vector> *) override { + return NotSup(); + } + IOStatus ListCloudObjectsWithPrefix(const std::string &, const std::string &, + const std::string &, + std::vector *) override { + return NotSup(); + } + IOStatus ExistsCloudObject(const std::string &, + const std::string &) override { + return NotSup(); + } + IOStatus GetCloudObjectSize(const std::string &, const std::string &, + uint64_t *) override { + return NotSup(); + } + IOStatus GetCloudObjectModificationTime(const std::string &, + const std::string &, + uint64_t *) override { + return NotSup(); + } + IOStatus GetCloudObjectMetadata(const std::string &, const std::string &, + CloudObjectInformation *) override { + return NotSup(); + } + IOStatus CopyCloudObject(const std::string &, const std::string &, + const std::string &, const std::string &) override { + return NotSup(); + } + IOStatus GetCloudObject(const std::string &, const std::string &, + const std::string &) override { + return NotSup(); + } + IOStatus PutCloudObjectMetadata( + const std::string &, const std::string &, + const std::unordered_map &) override { + return NotSup(); + } + IOStatus NewCloudWritableFile( + const std::string &, const std::string &, const std::string &, + const FileOptions &, std::unique_ptr *, + IODebugContext *) override { + return NotSup(); + } + IOStatus NewCloudReadableFile(const std::string &, const std::string &, + const FileOptions &, + std::unique_ptr *, + IODebugContext *) override { + return NotSup(); + } + + private: + static IOStatus NotSup() { + return IOStatus::NotSupported("RecordingStorageProvider"); + } + + std::mutex mu_; + int put_count_ = 0; + std::unordered_map put_statuses_; + std::unordered_map objects_; +}; + +} // namespace + +// ---------------- Window unit tests ---------------- + +TEST(FileNumberSlidingWindowTest, MinAddRemoveLingerExpiry) { + using Clock = FileNumberSlidingWindow::Clock; + FileNumberSlidingWindow w(std::chrono::milliseconds(1000)); + auto t0 = Clock::now(); + + // Empty window == UINT64_MAX. + ASSERT_EQ(w.SmallestFileNumber(t0), kMax); + + w.Add(100, /*thread*/ 1, /*job*/ 1); + w.Add(50, 2, 2); + ASSERT_EQ(w.SmallestFileNumber(t0), 50u); + + // Re-adding the same job keeps the original entry. + w.Add(10, 1, 1); + ASSERT_EQ(w.SmallestFileNumber(t0), 50u); + + // A removed entry lingers for entry_duration... + w.MarkRemoved(2, 2, t0); + ASSERT_EQ(w.SmallestFileNumber(t0 + std::chrono::milliseconds(500)), 50u); + // ...and stops contributing once expired. + ASSERT_EQ(w.SmallestFileNumber(t0 + std::chrono::milliseconds(1500)), 100u); + + // Removing an unknown job is a no-op. + w.MarkRemoved(9, 9, t0); + ASSERT_EQ(w.SmallestFileNumber(t0 + std::chrono::milliseconds(1500)), 100u); + + auto t1 = t0 + std::chrono::milliseconds(1500); + w.MarkRemoved(1, 1, t1); + ASSERT_EQ(w.SmallestFileNumber(t1 + std::chrono::milliseconds(900)), 100u); + ASSERT_EQ(w.SmallestFileNumber(t1 + std::chrono::milliseconds(1100)), kMax); + ASSERT_EQ(w.Size(), 0u); +} + +// ---------------- Key format ---------------- + +TEST(FileNumberGuardKeyTest, KeyFormatIsStable) { + // This format is the on-cloud protocol shared with pre-existing external + // writers; it must remain byte-identical. + ASSERT_EQ(SmallestFileNumberObjectKey("db/path", "abc123"), + "db/path/smallest_new_file_number-abc123"); + ASSERT_EQ(SmallestFileNumberObjectKey("db/path/", "abc123"), + "db/path/smallest_new_file_number-abc123"); + ASSERT_EQ(SmallestFileNumberObjectKey("", "e"), + "smallest_new_file_number-e"); + ASSERT_EQ(std::string(kSmallestFileNumberFilePrefix), + "smallest_new_file_number-"); +} + +// ---------------- Cloud file system option tests ---------------- + +TEST(CloudFileSystemGuardOptionsTest, DurationsRoundTripInMilliseconds) { + ConfigOptions config_options; + CloudFileSystemOptions options; + + ASSERT_OK(options.Configure( + config_options, + "guard_publish_interval_ms=1234;guard_entry_duration_ms=5678")); + ASSERT_EQ(options.guard_publish_interval, std::chrono::milliseconds(1234)); + ASSERT_EQ(options.guard_entry_duration, std::chrono::milliseconds(5678)); + + std::string serialized; + ASSERT_OK(options.Serialize(config_options, &serialized)); + CloudFileSystemOptions copy; + ASSERT_OK(copy.Configure(config_options, serialized)); + ASSERT_EQ(copy.guard_publish_interval, std::chrono::milliseconds(1234)); + ASSERT_EQ(copy.guard_entry_duration, std::chrono::milliseconds(5678)); +} + +TEST(CloudFileSystemGuardOptionsTest, DurationsParticipateInEquality) { + ConfigOptions config_options; + CloudFileSystemOptions expected; + CloudFileSystemOptions actual; + std::string mismatch; + + actual.guard_publish_interval = std::chrono::milliseconds(1); + ASSERT_FALSE(OptionTypeInfo::TypesAreEqual( + config_options, CloudFileSystemOptions::cloud_fs_option_type_info, + &expected, &actual, &mismatch)); + ASSERT_EQ(mismatch, "guard_publish_interval_ms"); + + actual = expected; + actual.guard_entry_duration = std::chrono::milliseconds(1); + mismatch.clear(); + ASSERT_FALSE(OptionTypeInfo::TypesAreEqual( + config_options, CloudFileSystemOptions::cloud_fs_option_type_info, + &expected, &actual, &mismatch)); + ASSERT_EQ(mismatch, "guard_entry_duration_ms"); +} + +TEST(CloudFileSystemGuardOptionsTest, DurationsMustBePositive) { + auto validate = [](std::chrono::milliseconds publish_interval, + std::chrono::milliseconds entry_duration) { + CloudFileSystemOptions options; + options.storage_provider = std::make_shared(); + options.guard_publish_interval = publish_interval; + options.guard_entry_duration = entry_duration; + CloudFileSystemImpl cfs(options, FileSystem::Default(), nullptr); + return cfs.ValidateOptions(DBOptions(), ColumnFamilyOptions()); + }; + + for (int value : {0, -1}) { + Status status = validate(std::chrono::milliseconds(value), + std::chrono::milliseconds(1)); + ASSERT_TRUE(status.IsInvalidArgument()) << status.ToString(); + ASSERT_NE(status.ToString().find("guard_publish_interval_ms"), + std::string::npos); + + status = validate(std::chrono::milliseconds(1), + std::chrono::milliseconds(value)); + ASSERT_TRUE(status.IsInvalidArgument()) << status.ToString(); + ASSERT_NE(status.ToString().find("guard_entry_duration_ms"), + std::string::npos); + } + + ASSERT_OK(validate(std::chrono::milliseconds(1), + std::chrono::milliseconds(1))); +} + +// ---------------- Publisher tests ---------------- + +class FileNumberGuardTest : public testing::Test { + public: + FileNumberGuardTest() { + tmp_dir_ = test::TmpDir() + "/file_number_guard_test_" + + std::to_string(Env::Default()->NowMicros()); + Env::Default()->CreateDirIfMissing(tmp_dir_); + provider_ = std::make_shared(); + + CloudFileSystemOptions opts; + opts.storage_provider = provider_; + opts.dest_bucket.SetBucketName("guard-bucket"); + opts.dest_bucket.SetBucketPrefix(""); + opts.dest_bucket.SetObjectPath("dbpath"); + opts.cloud_file_deletion_delay = std::nullopt; + cfs_ = std::make_unique(opts, FileSystem::Default(), + nullptr); + } + + ~FileNumberGuardTest() override { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } + + void LoadManifestWithEpoch(const std::string &epoch) { + std::unique_ptr manifest; + ASSERT_OK(CloudManifest::CreateForEmptyDatabase(epoch, &manifest)); + std::unique_ptr writer; + ASSERT_OK(WritableFileWriter::Create(FileSystem::Default(), + tmp_dir_ + "/CLOUDMANIFEST", + FileOptions(), &writer, nullptr)); + ASSERT_OK(manifest->WriteToLog(std::move(writer))); + ASSERT_OK(cfs_->LoadLocalCloudManifest(tmp_dir_)); + } + + static const std::string &GuardKey() { + static const std::string key = + SmallestFileNumberObjectKey("dbpath", "epoch1"); + return key; + } + + std::string tmp_dir_; + std::shared_ptr provider_; + std::unique_ptr cfs_; +}; + +TEST_F(FileNumberGuardTest, JobRegistrationDoesNotPublish) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + + pub.OnJobBegin(42, 1, 1); + ASSERT_EQ(provider_->PutCount(), 0); + ASSERT_EQ(pub.TEST_LastPublished(), kMax); +} + +TEST_F(FileNumberGuardTest, SentinelDoesNotAdvanceWatermark) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + + ASSERT_OK(pub.BlockPurger()); + ASSERT_EQ(provider_->Content(GuardKey()), "0"); + // The sentinel must not advance the watermark: a later real value + // (necessarily > 0) still has to be published. + ASSERT_EQ(pub.TEST_LastPublished(), kMax); + + // Protected uploads keep the known-safe sentinel in place until the + // post-open periodic publish replaces it. + pub.OnJobBegin(42, 1, 1); + ASSERT_EQ(provider_->Content(GuardKey()), "0"); + bool uploaded = false; + ASSERT_OK(pub.ProtectFileUpload(42, [&] { + uploaded = true; + return Status::OK(); + })); + ASSERT_TRUE(uploaded); + ASSERT_EQ(provider_->Content(GuardKey()), "0"); + ASSERT_EQ(pub.TEST_LastPublished(), kMax); + + pub.PeriodicPublish(); + ASSERT_EQ(provider_->Content(GuardKey()), "42"); + ASSERT_EQ(pub.TEST_LastPublished(), 42u); +} + +TEST_F(FileNumberGuardTest, SentinelOverwrittenByNextPeriodicPublish) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + + ASSERT_OK(pub.BlockPurger()); + ASSERT_EQ(provider_->Content(GuardKey()), "0"); + // Idle: the next periodic publish must replace the sentinel with + // UINT64_MAX even though the window minimum (MAX) equals last_published_. + pub.PeriodicPublish(); + ASSERT_EQ(provider_->Content(GuardKey()), std::to_string(kMax)); + // And once refreshed, further idle ticks are no-ops. + int puts = provider_->PutCount(); + pub.PeriodicPublish(); + ASSERT_EQ(provider_->PutCount(), puts); +} + +TEST_F(FileNumberGuardTest, DownwardPublishTriggerCondition) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + + pub.OnJobBegin(100, 1, 1); + ASSERT_EQ(provider_->PutCount(), 0); + ASSERT_OK(pub.ProtectFileUpload(100)); + int puts_after_first = provider_->PutCount(); + ASSERT_EQ(provider_->Content(GuardKey()), "100"); + + // A job at or above the watermark publishes nothing. + pub.OnJobBegin(200, 2, 2); + pub.OnJobBegin(100, 3, 3); + ASSERT_EQ(provider_->PutCount(), puts_after_first); + + // A job below the watermark publishes synchronously. + pub.OnJobBegin(50, 4, 4); + ASSERT_OK(pub.ProtectFileUpload(50)); + ASSERT_EQ(provider_->Content(GuardKey()), "50"); + ASSERT_EQ(pub.TEST_LastPublished(), 50u); +} + +TEST_F(FileNumberGuardTest, ProtectPublishesLiveWindowMinimum) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + + pub.OnJobBegin(100, 1, 1); + pub.OnJobBegin(200, 2, 2); + ASSERT_OK(pub.ProtectFileUpload(200)); + + ASSERT_EQ(provider_->Content(GuardKey()), "100"); + ASSERT_EQ(pub.TEST_LastPublished(), 100u); +} + +TEST_F(FileNumberGuardTest, ProtectWaitsForInFlightUpwardPublish) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::milliseconds(0)); + + pub.OnJobBegin(50, 1, 1); + ASSERT_OK(pub.ProtectFileUpload(50)); + pub.OnJobEnd(1, 1); + pub.OnJobBegin(100, 2, 2); + + std::mutex block_mutex; + std::condition_variable block_cv; + bool upward_put_reached = false; + bool release_upward_put = false; + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuard::PutSmallestFileNumberObject:Status", [&](void*) { + if (provider_->Content(GuardKey()) != "100") { + return; + } + std::unique_lock lock(block_mutex); + upward_put_reached = true; + block_cv.notify_all(); + block_cv.wait(lock, [&] { return release_upward_put; }); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::thread periodic([&] { pub.PeriodicPublish(); }); + bool reached = false; + { + std::unique_lock lock(block_mutex); + reached = block_cv.wait_for(lock, kAsyncWaitTimeout, + [&] { return upward_put_reached; }); + } + + pub.OnJobBegin(60, 3, 3); + auto protection = std::async(std::launch::async, + [&] { return pub.ProtectFileUpload(60); }); + const auto state = protection.wait_for(std::chrono::milliseconds(100)); + + { + std::lock_guard lock(block_mutex); + release_upward_put = true; + } + block_cv.notify_all(); + periodic.join(); + Status protection_status = protection.get(); + + EXPECT_TRUE(reached); + ASSERT_EQ(state, std::future_status::timeout); + ASSERT_OK(protection_status); + ASSERT_EQ(provider_->Content(GuardKey()), "60"); + ASSERT_EQ(pub.TEST_LastPublished(), 60u); +} + +TEST_F(FileNumberGuardTest, ProtectRejectsMissingOrTooHighWindow) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + + ASSERT_TRUE(pub.ProtectFileUpload(100).IsInvalidArgument()); + pub.OnJobBegin(200, 1, 1); + ASSERT_TRUE(pub.ProtectFileUpload(100).IsInvalidArgument()); + ASSERT_EQ(provider_->PutCount(), 0); +} + +TEST_F(FileNumberGuardTest, PeriodicPublishesWindowMinThenMaxWhenIdle) { + LoadManifestWithEpoch("epoch1"); + // Short entry duration so completed entries expire quickly. + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::milliseconds(50)); + + pub.OnJobBegin(100, 1, 1); + ASSERT_OK(pub.ProtectFileUpload(100)); // publishes 100 downward + pub.OnJobBegin(200, 2, 2); // no publish + + // Window min is 100 == last published: periodic publish is a no-op. + int puts = provider_->PutCount(); + pub.PeriodicPublish(); + ASSERT_EQ(provider_->PutCount(), puts); + + // Job 1 completes; entry lingers, so min is still 100. + pub.OnJobEnd(1, 1); + pub.PeriodicPublish(); + ASSERT_EQ(provider_->PutCount(), puts); + + // After the linger expires, the min rises to job 2's snapshot. + std::this_thread::sleep_for(std::chrono::milliseconds(80)); + pub.PeriodicPublish(); + ASSERT_EQ(provider_->Content(GuardKey()), "200"); + + // All jobs done and expired: idle publishes UINT64_MAX. + pub.OnJobEnd(2, 2); + std::this_thread::sleep_for(std::chrono::milliseconds(80)); + pub.PeriodicPublish(); + ASSERT_EQ(provider_->Content(GuardKey()), std::to_string(kMax)); + ASSERT_EQ(pub.TEST_LastPublished(), kMax); +} + +TEST_F(FileNumberGuardTest, + AmbiguousPeriodicFailureRepairsFiniteAndRetriesIdle) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::milliseconds(0)); + + pub.OnJobBegin(100, 1, 1); + ASSERT_OK(pub.ProtectFileUpload(100)); + pub.OnJobEnd(1, 1); + pub.OnJobBegin(200, 2, 2); + + std::atomic fail_next{true}; + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuard::PutSmallestFileNumberObject:Status", [&](void *arg) { + if (fail_next.exchange(false)) { + *static_cast(arg) = + IOStatus::IOError("injected post-store failure"); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // The provider stored 200, but the caller cannot know that after the error. + pub.PeriodicPublish(); + ASSERT_EQ(provider_->Content(GuardKey()), "200"); + ASSERT_EQ(pub.TEST_LastPublished(), kMax); + + // Unknown remote state forces a finite repair even though MAX would + // otherwise suppress a downward comparison. + pub.OnJobBegin(150, 3, 3); + pub.PeriodicPublish(); + ASSERT_EQ(provider_->Content(GuardKey()), "150"); + ASSERT_EQ(pub.TEST_LastPublished(), 150u); + + pub.OnJobEnd(2, 2); + pub.OnJobEnd(3, 3); + fail_next = true; + pub.PeriodicPublish(); + ASSERT_EQ(provider_->Content(GuardKey()), std::to_string(kMax)); + int puts_after_ambiguous_idle = provider_->PutCount(); + ASSERT_EQ(pub.TEST_LastPublished(), kMax); + + // desired == last == MAX must still retry while the remote state is + // unknown. + pub.PeriodicPublish(); + ASSERT_EQ(provider_->PutCount(), puts_after_ambiguous_idle + 1); + ASSERT_EQ(pub.TEST_LastPublished(), kMax); +} + +TEST_F(FileNumberGuardTest, ListenerSeparatesIngestionAndRecoveryDomains) { + LoadManifestWithEpoch("epoch1"); + auto pub = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::milliseconds(0)); + FileNumberGuardListener listener(pub); + + listener.OnExternalFileIngestionStarted(nullptr, 77); + TableFileCreationBriefInfo started; + started.db_name = "db"; + started.cf_name = "default"; + started.file_path = "/tmp/000077.sst"; + started.job_id = 9; + started.reason = TableFileCreationReason::kRecovery; + listener.OnTableFileCreationStarted(started); + + listener.OnExternalFileIngestionFinished(nullptr, 77); + ASSERT_OK(pub->ProtectFileUpload(77)); + + TableFileCreationInfo finished; + finished.db_name = started.db_name; + finished.cf_name = started.cf_name; + finished.file_path = "(nil)"; + finished.job_id = started.job_id; + finished.reason = started.reason; + finished.status = Status::Aborted("empty recovery output"); + listener.OnTableFileCreated(finished); + finished.status.PermitUncheckedError(); + + ASSERT_TRUE(pub->ProtectFileUpload(77).IsInvalidArgument()); + + started.file_path = "/tmp/000078.sst"; + started.job_id = 10; + listener.OnTableFileCreationStarted(started); + ASSERT_OK(pub->ProtectFileUpload(78)); + + finished.file_path = started.file_path; + finished.job_id = started.job_id; + finished.status = Status::OK(); + listener.OnTableFileCreated(finished); + ASSERT_TRUE(pub->ProtectFileUpload(78).IsInvalidArgument()); +} + +TEST_F(FileNumberGuardTest, ConditionalPublisherRemovalPreservesReplacement) { + LoadManifestWithEpoch("epoch1"); + auto old = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + auto replacement = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + cfs_->SetFileNumberGuardPublisher(old); + cfs_->SetFileNumberGuardPublisher(replacement); + + ASSERT_TRUE(old->ProtectFileUpload(1).IsShutdownInProgress()); + ASSERT_FALSE(cfs_->StopFileNumberGuardPublisher(old)); + ASSERT_FALSE(cfs_->RemoveFileNumberGuardPublisher(old)); + ASSERT_EQ(cfs_->GetFileNumberGuardPublisher(), replacement); + ASSERT_TRUE(cfs_->StopFileNumberGuardPublisher(replacement)); + ASSERT_EQ(cfs_->GetFileNumberGuardPublisher(), replacement); + ASSERT_TRUE(replacement->ProtectFileUpload(1).IsShutdownInProgress()); + ASSERT_TRUE(cfs_->RemoveFileNumberGuardPublisher(replacement)); + ASSERT_EQ(cfs_->GetFileNumberGuardPublisher(), nullptr); +} + +TEST_F(FileNumberGuardTest, InstallWaitsForOldPublisherBeforeSentinel) { + LoadManifestWithEpoch("epoch1"); + auto old = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::milliseconds(0)); + old->OnJobBegin(10, 1, 1); + cfs_->SetFileNumberGuardPublisher(old); + ASSERT_OK(old->ProtectFileUpload(10)); + old->OnJobEnd(1, 1); + + std::mutex mutex; + std::condition_variable cv; + bool old_put_reached = false; + bool release_old_put = false; + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuard::PutSmallestFileNumberObject:Status", [&](void *) { + if (provider_->Content(GuardKey()) != std::to_string(kMax)) { + return; + } + std::unique_lock lock(mutex); + old_put_reached = true; + cv.notify_all(); + cv.wait(lock, [&] { return release_old_put; }); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + auto old_publish = + std::async(std::launch::async, [&] { old->PeriodicPublish(); }); + { + std::unique_lock lock(mutex); + const bool reached = + cv.wait_for(lock, kAsyncWaitTimeout, [&] { return old_put_reached; }); + EXPECT_TRUE(reached); + } + + auto replacement = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + auto install = std::async(std::launch::async, [&] { + return cfs_->InstallFileNumberGuardPublisher(replacement); + }); + const auto install_state = install.wait_for(std::chrono::milliseconds(100)); + EXPECT_EQ(install_state, std::future_status::timeout); + + { + std::lock_guard lock(mutex); + release_old_put = true; + } + cv.notify_all(); + old_publish.get(); + ASSERT_OK(install.get()); + + ASSERT_EQ(cfs_->GetFileNumberGuardPublisher(), replacement); + ASSERT_EQ(provider_->Content(GuardKey()), "0"); + ASSERT_TRUE(old->ProtectFileUpload(10).IsShutdownInProgress()); +} + +TEST_F(FileNumberGuardTest, FailedInstallKeepsOldPublisherFailClosed) { + LoadManifestWithEpoch("epoch1"); + auto old = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + auto replacement = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + cfs_->SetFileNumberGuardPublisher(old); + provider_->SetPutStatus(GuardKey(), IOStatus::IOError("sentinel failure")); + + ASSERT_NOK(cfs_->InstallFileNumberGuardPublisher(replacement)); + ASSERT_EQ(cfs_->GetFileNumberGuardPublisher(), old); + ASSERT_TRUE(old->ProtectFileUpload(1).IsShutdownInProgress()); +} + +TEST_F(FileNumberGuardTest, ConcurrentInstallsPublishSentinelsInOrder) { + LoadManifestWithEpoch("epoch1"); + auto first = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + auto second = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + + std::mutex mutex; + std::condition_variable cv; + bool block_first_sentinel = true; + bool first_sentinel_reached = false; + bool release_first_sentinel = false; + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuard::PutSmallestFileNumberObject:Status", [&](void *) { + std::unique_lock lock(mutex); + if (!block_first_sentinel || provider_->Content(GuardKey()) != "0") { + return; + } + block_first_sentinel = false; + first_sentinel_reached = true; + cv.notify_all(); + cv.wait(lock, [&] { return release_first_sentinel; }); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + auto first_install = std::async(std::launch::async, [&] { + return cfs_->InstallFileNumberGuardPublisher(first); + }); + { + std::unique_lock lock(mutex); + const bool reached = cv.wait_for(lock, kAsyncWaitTimeout, + [&] { return first_sentinel_reached; }); + EXPECT_TRUE(reached); + } + auto second_install = std::async(std::launch::async, [&] { + return cfs_->InstallFileNumberGuardPublisher(second); + }); + const auto second_install_state = + second_install.wait_for(std::chrono::milliseconds(100)); + EXPECT_EQ(second_install_state, std::future_status::timeout); + + { + std::lock_guard lock(mutex); + release_first_sentinel = true; + } + cv.notify_all(); + ASSERT_OK(first_install.get()); + ASSERT_OK(second_install.get()); + ASSERT_EQ(provider_->PutCount(), 2); + + ASSERT_EQ(cfs_->GetFileNumberGuardPublisher(), second); + ASSERT_EQ(provider_->Content(GuardKey()), "0"); + ASSERT_TRUE(first->ProtectFileUpload(1).IsShutdownInProgress()); +} + +// Failure injection: the downward PUT fails; the publisher must retry and +// must not advance the watermark past the failure. +TEST_F(FileNumberGuardTest, DownwardPublishFailureRetriesWithoutAdvancing) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + + std::atomic failures_left{2}; + std::atomic attempts{0}; + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuard::PutSmallestFileNumberObject:Status", [&](void *arg) { + ++attempts; + if (failures_left.fetch_sub(1) > 0) { + *static_cast(arg) = + IOStatus::IOError("injected PUT failure"); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Run the downward publish on a separate thread: it blocks (retrying) + // until the PUT succeeds. + pub.OnJobBegin(10, 1, 1); + Status job_status; + std::thread job([&] { job_status = pub.ProtectFileUpload(10); }); + + // While failures are being injected, the watermark must not advance. + while (failures_left.load() > 0) { + ASSERT_EQ(pub.TEST_LastPublished(), kMax); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + job.join(); + + ASSERT_OK(job_status); + ASSERT_GE(attempts.load(), 3); // two failures + at least one success + ASSERT_EQ(pub.TEST_LastPublished(), 10u); + ASSERT_EQ(provider_->Content(GuardKey()), "10"); +} + +TEST_F(FileNumberGuardTest, StopUnblocksFailingDownwardPublish) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuard::PutSmallestFileNumberObject:Status", [&](void *arg) { + *static_cast(arg) = + IOStatus::IOError("injected PUT failure"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::atomic returned{false}; + Status job_status; + pub.OnJobBegin(10, 1, 1); + std::thread job([&] { + job_status = pub.ProtectFileUpload(10); + returned = true; + }); + + // The job stays blocked while the PUT keeps failing. + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + ASSERT_FALSE(returned.load()); + ASSERT_EQ(pub.TEST_LastPublished(), kMax); + + pub.Stop(); + job.join(); + ASSERT_TRUE(returned.load()); + // The failure is reported, and the watermark was never advanced past it. + ASSERT_FALSE(job_status.ok()); + ASSERT_EQ(pub.TEST_LastPublished(), kMax); +} + +TEST_F(FileNumberGuardTest, EmptyEpochPublishRefused) { + // A manifest whose current epoch is empty: publishing would create the + // malformed key "smallest_new_file_number-". + cfs_->TEST_InitEmptyCloudManifest(); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + + pub.OnJobBegin(10, 1, 1); + ASSERT_TRUE(pub.ProtectFileUpload(10).IsInvalidArgument()); + ASSERT_TRUE(pub.BlockPurger().IsInvalidArgument()); + pub.PeriodicPublish(); + ASSERT_EQ(provider_->PutCount(), 0); +} + +TEST_F(FileNumberGuardTest, BlockPurgerOnFileSystemWithoutPublisher) { + LoadManifestWithEpoch("epoch1"); + // No publisher registered: BlockPurger takes the direct one-shot path. + ASSERT_OK(cfs_->BlockPurger()); + ASSERT_EQ(provider_->Content(GuardKey()), "0"); +} + +// Files written outside the DB's own SST directories (a Checkpoint copy, a +// column family export) are not DB-visible SSTs: they must not be gated, or +// the copy would fail with "no live file number registration" because no +// flush/compaction job ever registers them. +TEST_F(FileNumberGuardTest, CoversFileOnlyInsideDbDirectories) { + LoadManifestWithEpoch("epoch1"); + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15), + {"/data/db", "/data/extra_path/"}); + + // The DB directory itself, and a configured db_path (trailing separator + // normalized on both sides). + ASSERT_TRUE(pub.CoversFile("/data/db/000123.sst-epoch1")); + ASSERT_TRUE(pub.CoversFile("/data/extra_path/000124.sst-epoch1")); + + // Checkpoint copies live under their own directory tree. + ASSERT_FALSE(pub.CoversFile("/data/ckpt/private/1/000123.sst-epoch1")); + ASSERT_FALSE(pub.CoversFile("/data/export/000123.sst-epoch1")); + // A sibling directory sharing a prefix must not be treated as inside. + ASSERT_FALSE(pub.CoversFile("/data/db_backup/000123.sst-epoch1")); + // Nested subdirectories of the DB directory are not SST locations either. + ASSERT_FALSE(pub.CoversFile("/data/db/sub/000123.sst-epoch1")); +} + +TEST_F(FileNumberGuardTest, CoversFileGatesEverythingWhenUnconfigured) { + LoadManifestWithEpoch("epoch1"); + // No directories configured: fail safe by gating every SST rather than + // risking an unprotected upload. + FileNumberGuardPublisher pub(cfs_.get(), std::chrono::seconds(30), + std::chrono::seconds(15)); + ASSERT_TRUE(pub.CoversFile("/anywhere/000123.sst-epoch1")); +} + +// RepairDB rebuilds SSTs without the guard listener (that is registered by +// DBCloud::Open on its own options copy), so its uploads would bypass the +// window and reach the cloud unprotected. It must refuse rather than write +// files the purger may delete. +TEST_F(FileNumberGuardTest, RepairDBRefusedWhenGuardEnabled) { + LoadManifestWithEpoch("epoch1"); + auto env = CloudFileSystemEnv::NewCompositeEnv( + Env::Default(), std::shared_ptr(cfs_.get(), [](FileSystem*) { + })); + + Options options; + options.env = env.get(); + + cfs_->GetMutableCloudFileSystemOptions().publish_file_number_guard = true; + Status guarded = RepairDB(tmp_dir_, options); + ASSERT_TRUE(guarded.IsNotSupported()) << guarded.ToString(); + + // With the guard disabled the check must not fire; repair proceeds far + // enough to fail on its own terms (this directory is not a database), which + // is any status other than the guard's NotSupported message. + cfs_->GetMutableCloudFileSystemOptions().publish_file_number_guard = false; + Status unguarded = RepairDB(tmp_dir_, options); + ASSERT_EQ(unguarded.ToString().find("publish_file_number_guard"), + std::string::npos) + << unguarded.ToString(); +} + +TEST_F(FileNumberGuardTest, StartStopSchedulerSmoke) { + LoadManifestWithEpoch("epoch1"); + auto pub = std::make_shared( + cfs_.get(), std::chrono::milliseconds(20), + std::chrono::milliseconds(10)); + pub->Start(); + pub->OnJobBegin(7, 1, 1); + ASSERT_OK(pub->ProtectFileUpload(7)); + pub->OnJobEnd(1, 1); + const int puts_before_periodic = provider_->PutCount(); + const auto deadline = std::chrono::steady_clock::now() + kAsyncWaitTimeout; + while (provider_->PutCount() == puts_before_periodic && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + EXPECT_GT(provider_->PutCount(), puts_before_periodic); + EXPECT_EQ(provider_->Content(GuardKey()), std::to_string(kMax)); + pub->Stop(); + int puts_at_stop = provider_->PutCount(); + std::this_thread::sleep_for(std::chrono::milliseconds(60)); + // No publishes after Stop. + ASSERT_EQ(provider_->PutCount(), puts_at_stop); +} + +TEST_F(FileNumberGuardTest, SstUploadWithoutPublisherPassesThrough) { + const std::string local_path = tmp_dir_ + "/000123.sst-epoch1"; + const std::string object_path = "dbpath/000123.sst-epoch1"; + CloudStorageWritableFileImpl file(cfs_.get(), local_path, "guard-bucket", + object_path, FileOptions()); + ASSERT_OK(file.status()); + ASSERT_OK(file.Append(Slice("sst contents"), IOOptions(), nullptr)); + + ASSERT_OK(file.Close(IOOptions(), nullptr)); + ASSERT_TRUE(provider_->HasObject(object_path)); +} + +TEST_F(FileNumberGuardTest, StoppedPublisherRejectsSstUpload) { + LoadManifestWithEpoch("epoch1"); + auto pub = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + pub->OnJobBegin(123, 1, 1); + cfs_->SetFileNumberGuardPublisher(pub); + pub->Stop(); + + const std::string local_path = tmp_dir_ + "/000123.sst-epoch1"; + const std::string object_path = "dbpath/000123.sst-epoch1"; + CloudStorageWritableFileImpl file(cfs_.get(), local_path, "guard-bucket", + object_path, FileOptions()); + ASSERT_OK(file.status()); + ASSERT_OK(file.Append(Slice("sst contents"), IOOptions(), nullptr)); + + ASSERT_NOK(file.Close(IOOptions(), nullptr)); + ASSERT_FALSE(provider_->HasObject(object_path)); +} + +TEST_F(FileNumberGuardTest, StopDuringGuardPublishPreventsSstUpload) { + LoadManifestWithEpoch("epoch1"); + auto pub = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + pub->OnJobBegin(123, 1, 1); + cfs_->SetFileNumberGuardPublisher(pub); + + std::mutex mutex; + std::condition_variable cv; + bool guard_put_reached = false; + bool release_guard_put = false; + bool guard_put_wait_timed_out = false; + bool stop_reached = false; + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuard::PutSmallestFileNumberObject:Status", [&](void *) { + std::unique_lock lock(mutex); + guard_put_reached = true; + cv.notify_all(); + guard_put_wait_timed_out = !cv.wait_for( + lock, kAsyncWaitTimeout, [&] { return release_guard_put; }); + }); + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuardPublisher::Stop:Stopped", [&](void *) { + std::lock_guard lock(mutex); + stop_reached = true; + cv.notify_all(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + const std::string local_path = tmp_dir_ + "/000123.sst-epoch1"; + const std::string object_path = "dbpath/000123.sst-epoch1"; + CloudStorageWritableFileImpl file(cfs_.get(), local_path, "guard-bucket", + object_path, FileOptions()); + ASSERT_OK(file.status()); + ASSERT_OK(file.Append(Slice("sst contents"), IOOptions(), nullptr)); + + auto close = std::async(std::launch::async, + [&] { return file.Close(IOOptions(), nullptr); }); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, kAsyncWaitTimeout, + [&] { return guard_put_reached; })); + } + auto stop = std::async(std::launch::async, [&] { pub->Stop(); }); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, kAsyncWaitTimeout, + [&] { return stop_reached; })); + } + ASSERT_EQ(stop.wait_for(std::chrono::milliseconds(0)), + std::future_status::timeout); + + { + std::lock_guard lock(mutex); + release_guard_put = true; + } + cv.notify_all(); + const IOStatus close_status = close.get(); + stop.get(); + + ASSERT_FALSE(guard_put_wait_timed_out); + ASSERT_NOK(close_status); + ASSERT_FALSE(provider_->HasObject(object_path)); +} + +TEST_F(FileNumberGuardTest, StopWaitsForProtectedSstUpload) { + LoadManifestWithEpoch("epoch1"); + auto pub = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + pub->OnJobBegin(123, 1, 1); + cfs_->SetFileNumberGuardPublisher(pub); + + const std::string local_path = tmp_dir_ + "/000123.sst-epoch1"; + const std::string object_path = "dbpath/000123.sst-epoch1"; + std::mutex mutex; + std::condition_variable cv; + bool upload_reached = false; + bool release_upload = false; + bool upload_wait_timed_out = false; + bool stop_reached = false; + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuardTest::PutCloudObject", [&](void *arg) { + if (*static_cast(arg) != object_path) { + return; + } + std::unique_lock lock(mutex); + upload_reached = true; + cv.notify_all(); + upload_wait_timed_out = !cv.wait_for( + lock, kAsyncWaitTimeout, [&] { return release_upload; }); + }); + SyncPoint::GetInstance()->SetCallBack( + "FileNumberGuardPublisher::Stop:Stopped", [&](void *) { + std::lock_guard lock(mutex); + stop_reached = true; + cv.notify_all(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + CloudStorageWritableFileImpl file(cfs_.get(), local_path, "guard-bucket", + object_path, FileOptions()); + ASSERT_OK(file.status()); + ASSERT_OK(file.Append(Slice("sst contents"), IOOptions(), nullptr)); + + auto close = std::async(std::launch::async, + [&] { return file.Close(IOOptions(), nullptr); }); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, kAsyncWaitTimeout, + [&] { return upload_reached; })); + } + auto stop = std::async(std::launch::async, [&] { pub->Stop(); }); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, kAsyncWaitTimeout, + [&] { return stop_reached; })); + } + ASSERT_EQ(stop.wait_for(std::chrono::milliseconds(0)), + std::future_status::timeout); + + { + std::lock_guard lock(mutex); + release_upload = true; + } + cv.notify_all(); + ASSERT_OK(close.get()); + stop.get(); + + ASSERT_FALSE(upload_wait_timed_out); + ASSERT_TRUE(provider_->HasObject(object_path)); +} + +TEST_F(FileNumberGuardTest, GuardEnabledIdentityUploadPassesThrough) { + LoadManifestWithEpoch("epoch1"); + auto pub = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + cfs_->SetFileNumberGuardPublisher(pub); + + const std::string local_path = tmp_dir_ + "/IDENTITY"; + const std::string object_path = "dbpath/IDENTITY"; + CloudStorageWritableFileImpl file(cfs_.get(), local_path, "guard-bucket", + object_path, FileOptions()); + ASSERT_OK(file.status()); + ASSERT_OK(file.Append(Slice("db-id"), IOOptions(), nullptr)); + + ASSERT_OK(file.Close(IOOptions(), nullptr)); + ASSERT_TRUE(provider_->HasObject(object_path)); + ASSERT_EQ(provider_->PutCount(), 1); +} + +TEST_F(FileNumberGuardTest, MalformedFileNumberSstUploadFailsClosed) { + LoadManifestWithEpoch("epoch1"); + auto pub = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + cfs_->SetFileNumberGuardPublisher(pub); + + const std::string local_path = tmp_dir_ + "/not-a-number.sst-epoch1"; + const std::string object_path = "dbpath/not-a-number.sst-epoch1"; + CloudStorageWritableFileImpl file(cfs_.get(), local_path, "guard-bucket", + object_path, FileOptions()); + ASSERT_OK(file.status()); + ASSERT_OK(file.Append(Slice("sst contents"), IOOptions(), nullptr)); + + const IOStatus status = file.Close(IOOptions(), nullptr); + ASSERT_TRUE(status.IsInvalidArgument()) << status.ToString(); + ASSERT_FALSE(provider_->HasObject(object_path)); + ASSERT_EQ(provider_->PutCount(), 0); +} + +TEST_F(FileNumberGuardTest, ProtectedSstUploadFailurePropagates) { + LoadManifestWithEpoch("epoch1"); + auto pub = std::make_shared( + cfs_.get(), std::chrono::seconds(30), std::chrono::seconds(15)); + pub->OnJobBegin(123, 1, 1); + cfs_->SetFileNumberGuardPublisher(pub); + + const std::string local_path = tmp_dir_ + "/000123.sst-epoch1"; + const std::string object_path = "dbpath/000123.sst-epoch1"; + provider_->SetPutStatus(object_path, + IOStatus::IOError("injected SST upload failure")); + CloudStorageWritableFileImpl file(cfs_.get(), local_path, "guard-bucket", + object_path, FileOptions()); + ASSERT_OK(file.status()); + ASSERT_OK(file.Append(Slice("sst contents"), IOOptions(), nullptr)); + + const IOStatus status = file.Close(IOOptions(), nullptr); + ASSERT_TRUE(status.IsIOError()) << status.ToString(); + ASSERT_FALSE(provider_->HasObject(object_path)); + ASSERT_EQ(provider_->Content(GuardKey()), "123"); +} + + +// A writable DBCloud on a bucket whose deletion is delegated to the purger +// must publish the guard; otherwise the purger cannot tell an in-flight +// upload from garbage and its marker-absence fallback deletes live data. +TEST_F(FileNumberGuardTest, OpenRejectsPurgedBucketWithoutGuard) { + auto env = CloudFileSystemEnv::NewCompositeEnv( + Env::Default(), + std::shared_ptr(cfs_.get(), [](FileSystem *) {})); + Options options; + options.env = env.get(); + options.create_if_missing = true; + + auto &mutable_opts = cfs_->GetMutableCloudFileSystemOptions(); + mutable_opts.disable_cloud_file_deletion = true; + mutable_opts.publish_file_number_guard = false; + + DBCloud *db = nullptr; + Status s = DBCloud::Open(options, tmp_dir_ + "/guard_required", "", 0, &db); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); + ASSERT_NE(s.ToString().find("publish_file_number_guard"), std::string::npos) + << s.ToString(); + ASSERT_EQ(db, nullptr); + + // A read-only open creates no SSTs, so it needs no guard and must not be + // rejected by this check. + Status read_only = + DBCloud::Open(options, tmp_dir_ + "/guard_required", "", 0, &db, true); + ASSERT_EQ(read_only.ToString().find("publish_file_number_guard"), + std::string::npos) + << read_only.ToString(); +} + +// The guard assumes a completed job's MANIFEST is durable in the cloud. +// disable_manifest_sync removes the Sync() that uploads it, so the guard +// could rise to UINT64_MAX while the cloud MANIFEST still omits committed +// files. +TEST_F(FileNumberGuardTest, OpenRejectsDisableManifestSyncWithGuard) { + auto env = CloudFileSystemEnv::NewCompositeEnv( + Env::Default(), + std::shared_ptr(cfs_.get(), [](FileSystem *) {})); + Options options; + options.env = env.get(); + options.create_if_missing = true; + options.disable_manifest_sync = true; + + cfs_->GetMutableCloudFileSystemOptions().publish_file_number_guard = true; + + DBCloud *db = nullptr; + Status s = DBCloud::Open(options, tmp_dir_ + "/manifest_sync", "", 0, &db); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); + ASSERT_NE(s.ToString().find("disable_manifest_sync"), std::string::npos) + << s.ToString(); + ASSERT_EQ(db, nullptr); + + // Without the guard the option is the caller's own risk, and this check + // must not fire. + cfs_->GetMutableCloudFileSystemOptions().publish_file_number_guard = false; + Status unguarded = + DBCloud::Open(options, tmp_dir_ + "/manifest_sync", "", 0, &db); + ASSERT_EQ(unguarded.ToString().find("disable_manifest_sync"), + std::string::npos) + << unguarded.ToString(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/cloud/gcp/gcp_cs.cc b/cloud/gcp/gcp_cs.cc index 90cad24..7a68632 100644 --- a/cloud/gcp/gcp_cs.cc +++ b/cloud/gcp/gcp_cs.cc @@ -452,11 +452,48 @@ IOStatus GcsStorageProvider::EmptyBucket(std::string const& bucket_name, return IOStatus::OK(); } -IOStatus GcsStorageProvider::ListCloudObjects(const std::string& /*bucket_name*/, - const std::string& /*object_path*/, - std::vector>* /*result*/) { - return IOStatus::NotSupported("GcsStorageProvider::ListCloudObjects"); +IOStatus GcsStorageProvider::ListCloudObjects( + const std::string& bucket_name, const std::string& object_path, + std::vector>* result) { + auto prefix = ensure_ends_with_pathsep(normalzie_object_path(object_path)); + auto objects = gcs_client_->ListCloudObjects( + bucket_name, prefix, + cfs_->GetCloudFileSystemOptions().number_objects_listed_in_one_iteration); + if (!objects.ok()) { + std::string errmsg(objects.status().message()); + if (IsNotFound(objects.status())) { + return IOStatus::NotFound(object_path, errmsg.c_str()); + } + return IOStatus::IOError(object_path, errmsg.c_str()); + } + + for (const auto& object : objects.value()) { + if (!object.ok()) { + std::string errmsg(object.status().message()); + if (IsNotFound(object.status())) { + return IOStatus::NotFound(object_path, errmsg.c_str()); + } + return IOStatus::IOError(object_path, errmsg.c_str()); + } + const auto& metadata = object.value(); + const std::string& name = metadata.name(); + if (name.find(prefix) != 0) { + return IOStatus::IOError("Unexpected result from Gcs: " + name); + } + + CloudObjectInformation info; + info.size = metadata.size(); + info.modification_time = + std::chrono::duration_cast( + metadata.updated().time_since_epoch()) + .count(); + info.content_hash = metadata.etag(); + for (const auto& entry : metadata.metadata()) { + info.metadata.emplace(entry.first, entry.second); + } + result->emplace_back(name.substr(prefix.size()), std::move(info)); + } + return IOStatus::OK(); } IOStatus GcsStorageProvider::DeleteCloudObject(std::string const& bucket_name, diff --git a/cloud/manifest_reader.cc b/cloud/manifest_reader.cc index a148485..0bc768f 100644 --- a/cloud/manifest_reader.cc +++ b/cloud/manifest_reader.cc @@ -75,8 +75,8 @@ IOStatus LocalManifestReader::GetManifestLiveFiles( } IOStatus LocalManifestReader::GetLiveFilesFromFileReader( - std::unique_ptr file_reader, - std::set *list) const { + std::unique_ptr file_reader, std::set *list, + WALRecoveryMode recovery_mode) const { Status s; // create a callback that gets invoked whil looping through the log records VersionSet::LogReporter reporter; @@ -93,7 +93,7 @@ IOStatus LocalManifestReader::GetLiveFilesFromFileReader( std::unordered_set>> cf_live_files; - while (reader.ReadRecord(&record, &scratch) && s.ok()) { + while (reader.ReadRecord(&record, &scratch, recovery_mode) && s.ok()) { VersionEdit edit; s = edit.DecodeFrom(record); if (!s.ok()) { @@ -152,7 +152,8 @@ ManifestReader::ManifestReader(std::shared_ptr info_log, // cloud_manifest object // IOStatus ManifestReader::GetLiveFiles(const std::string &bucket_path, - std::set *list) const { + std::set *list, + WALRecoveryMode recovery_mode) const { IOStatus s; std::unique_ptr cloud_manifest; const FileOptions file_opts; @@ -189,12 +190,14 @@ IOStatus ManifestReader::GetLiveFiles(const std::string &bucket_path, file_reader.reset(new SequentialFileReader(std::move(file), manifestFile)); } - return GetLiveFilesFromFileReader(std::move(file_reader), list); + return GetLiveFilesFromFileReader(std::move(file_reader), list, + recovery_mode); } IOStatus ManifestReader::GetLiveFiles(const std::string &bucket_path, const std::string &epoch, - std::set *list) const { + std::set *list, + WALRecoveryMode recovery_mode) const { auto manifestFile = ManifestFileWithEpoch(bucket_path, epoch); IOStatus s; @@ -213,12 +216,13 @@ IOStatus ManifestReader::GetLiveFiles(const std::string &bucket_path, file_reader.reset(new SequentialFileReader(std::move(file), manifestFile)); } - return GetLiveFilesFromFileReader(std::move(file_reader), list); + return GetLiveFilesFromFileReader(std::move(file_reader), list, + recovery_mode); } -IOStatus ManifestReader::GetMaxFileNumberFromManifest(FileSystem *fs, - const std::string &fname, - uint64_t *maxFileNumber) { +IOStatus ManifestReader::GetMaxFileNumberFromManifest( + FileSystem *fs, const std::string &fname, uint64_t *maxFileNumber, + WALRecoveryMode recovery_mode) { // We check if the file exists to return IsNotFound() error status if it does // (NewSequentialFile) doesn't have the same behavior on file not existing -- // it returns IOError instead. @@ -245,7 +249,7 @@ IOStatus ManifestReader::GetMaxFileNumberFromManifest(FileSystem *fs, std::string scratch; *maxFileNumber = 0; - while (reader.ReadRecord(&record, &scratch) && s.ok()) { + while (reader.ReadRecord(&record, &scratch, recovery_mode) && s.ok()) { VersionEdit edit; s = status_to_io_status(edit.DecodeFrom(record)); if (!s.ok()) { diff --git a/cloud/manifest_reader.h b/cloud/manifest_reader.h index ac9a559..53cfe00 100644 --- a/cloud/manifest_reader.h +++ b/cloud/manifest_reader.h @@ -7,9 +7,22 @@ #include #include "rocksdb/io_status.h" +#include "rocksdb/options.h" namespace ROCKSDB_NAMESPACE { +// Recovery mode for MANIFEST scans. +// +// kTolerateCorruptedTailRecords is right for DB open: a torn tail record is +// skipped so recovery can proceed. It is WRONG for a destructive scanner -- +// a skipped tail containing a file addition makes a live SST look +// unreferenced, and the purger would delete it. Callers that delete data +// must pass kAbsoluteConsistency so any corruption surfaces as an error. +constexpr WALRecoveryMode kManifestScanDefaultMode = + WALRecoveryMode::kTolerateCorruptedTailRecords; +constexpr WALRecoveryMode kManifestScanStrictMode = + WALRecoveryMode::kAbsoluteConsistency; + class CloudFileSystem; class FileSystem; class Logger; @@ -43,7 +56,8 @@ class LocalManifestReader { // file_reader IOStatus GetLiveFilesFromFileReader( std::unique_ptr file_reader, - std::set* list) const; + std::set* list, + WALRecoveryMode recovery_mode = kManifestScanDefaultMode) const; std::shared_ptr info_log_; CloudFileSystem* cfs_; @@ -61,16 +75,18 @@ class ManifestReader : public LocalManifestReader { // It will read from CLOUDMANIFEST and MANIFEST file in s3 directly // TODO(wei): remove this function. Reading from s3 directly is very slow for // large MANIFEST file - IOStatus GetLiveFiles(const std::string& bucket_path, - std::set* list) const; - - IOStatus GetLiveFiles(const std::string& bucket_path, - const std::string& epoch, - std::set* list) const; - - static IOStatus GetMaxFileNumberFromManifest(FileSystem* fs, - const std::string& fname, - uint64_t* maxFileNumber); + IOStatus GetLiveFiles( + const std::string& bucket_path, std::set* list, + WALRecoveryMode recovery_mode = kManifestScanDefaultMode) const; + + IOStatus GetLiveFiles( + const std::string& bucket_path, const std::string& epoch, + std::set* list, + WALRecoveryMode recovery_mode = kManifestScanDefaultMode) const; + + static IOStatus GetMaxFileNumberFromManifest( + FileSystem* fs, const std::string& fname, uint64_t* maxFileNumber, + WALRecoveryMode recovery_mode = kManifestScanDefaultMode); private: std::string bucket_prefix_; diff --git a/db/builder.cc b/db/builder.cc index a3c15ad..2326953 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -130,10 +130,15 @@ Status BuildTable( ioptions.compaction_filter_factory->CreateCompactionFilter(context); if (compaction_filter != nullptr && !compaction_filter->IgnoreSnapshots()) { - s.PermitUncheckedError(); - return Status::NotSupported( + s = Status::NotSupported( "CompactionFilter::IgnoreSnapshots() = false is not supported " "anymore."); + EventHelpers::LogAndNotifyTableFileCreationFinished( + event_logger, ioptions.listeners, dbname, + tboptions.column_family_name, "(nil)", job_id, meta->fd, + kInvalidBlobFileNumber, tp, tboptions.reason, s, file_checksum, + file_checksum_func_name); + return s; } } diff --git a/db/db_compaction_filter_test.cc b/db/db_compaction_filter_test.cc index d84e24c..69ddd16 100644 --- a/db/db_compaction_filter_test.cc +++ b/db/db_compaction_filter_test.cc @@ -970,14 +970,43 @@ TEST_F(DBTestCompactionFilter, IgnoreSnapshotsFalseDuringFlush) { } TEST_F(DBTestCompactionFilter, IgnoreSnapshotsFalseRecovery) { + class RecoveryCreationListener : public EventListener { + public: + void OnTableFileCreationStarted( + const TableFileCreationBriefInfo &info) override { + if (info.reason == TableFileCreationReason::kRecovery) { + ++started; + } + } + + void OnTableFileCreated(const TableFileCreationInfo &info) override { + if (info.reason == TableFileCreationReason::kRecovery) { + ++finished; + file_path = info.file_path; + status = info.status; + } + } + + int started = 0; + int finished = 0; + std::string file_path; + Status status; + }; + + auto listener = std::make_shared(); Options options = CurrentOptions(); options.compaction_filter_factory = std::make_shared( TableFileCreationReason::kRecovery); + options.listeners.emplace_back(listener); Reopen(options); ASSERT_OK(Put("a", "v10")); ASSERT_TRUE(TryReopen(options).IsNotSupported()); + ASSERT_EQ(listener->started, 1); + ASSERT_EQ(listener->finished, 1); + ASSERT_EQ(listener->file_path, "(nil)"); + ASSERT_TRUE(listener->status.IsNotSupported()); } TEST_F(DBTestCompactionFilter, DropKeyWithSingleDelete) { diff --git a/db/db_flush_test.cc b/db/db_flush_test.cc index 8df111e..6ef9c63 100644 --- a/db/db_flush_test.cc +++ b/db/db_flush_test.cc @@ -7,8 +7,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include #include #include +#include #include "db/db_impl/db_impl.h" #include "db/db_test_util.h" @@ -48,6 +50,184 @@ class DBAtomicFlushTest : public DBFlushTest, DBAtomicFlushTest() : DBFlushTest() {} }; +TEST_F(DBFlushTest, FlushBeginReportsAllocatedFileNumber) { + class RecordingListener : public EventListener { + public: + void OnFlushBegin(DB* /*db*/, const FlushJobInfo& info) override { + ++begin_count; + file_number = info.file_number; + file_path = info.file_path; + } + + std::atomic begin_count{0}; + uint64_t file_number = 0; + std::string file_path; + }; + + auto listener = std::make_shared(); + Options options = CurrentOptions(); + options.listeners.push_back(listener); + Reopen(options); + + ASSERT_OK(Put("key", "value")); + ASSERT_OK(Flush()); + + ASSERT_EQ(listener->begin_count.load(), 1); + ASSERT_NE(listener->file_number, 0); + ASSERT_EQ(TableFileNameToNumber(listener->file_path), listener->file_number); +} + +class RecordingFlushLifecycleListener : public EventListener { + public: + void OnFlushBegin(DB* /*db*/, const FlushJobInfo& info) override { + ++begin_count; + flush_reason = static_cast(info.flush_reason); + } + + void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& /*info*/) override { + ++completed_count; + } + + void OnFlushFinished(DB* /*db*/, const FlushJobEndInfo& info) override { + std::lock_guard lock(mutex_); + ++finished_count; + finished_column_families.emplace_back(info.cf_id, info.cf_name); + status_ = info.status; + switched_to_mempurge_ = info.switched_to_mempurge; + } + + std::vector> finished_cfs() { + std::lock_guard lock(mutex_); + auto result = finished_column_families; + std::sort(result.begin(), result.end()); + return result; + } + + Status status() { + std::lock_guard lock(mutex_); + return status_; + } + + bool switched_to_mempurge() { + std::lock_guard lock(mutex_); + return switched_to_mempurge_; + } + + std::atomic begin_count{0}; + std::atomic completed_count{0}; + std::atomic finished_count{0}; + std::atomic flush_reason{-1}; + + private: + std::mutex mutex_; + std::vector> finished_column_families; + Status status_; + bool switched_to_mempurge_ = false; +}; + +TEST_F(DBFlushTest, FlushFinishedReportsSuccess) { + auto listener = std::make_shared(); + Options options = CurrentOptions(); + options.listeners.push_back(listener); + Reopen(options); + + ASSERT_OK(Put("key", "value")); + ASSERT_OK(Flush()); + + ASSERT_EQ(listener->begin_count.load(), 1); + ASSERT_EQ(listener->completed_count.load(), 1); + ASSERT_EQ(listener->finished_count.load(), 1); + ASSERT_EQ(listener->finished_cfs(), + (std::vector>{{0, "default"}})); + ASSERT_OK(listener->status()); + ASSERT_FALSE(listener->switched_to_mempurge()); +} + +TEST_F(DBFlushTest, FlushFinishedReportsFailure) { + auto listener = std::make_shared(); + Options options = CurrentOptions(); + options.listeners.push_back(listener); + Reopen(options); + + SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:s", [](void* arg) { + *static_cast(arg) = Status::IOError("injected flush failure"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put("key", "value")); + ASSERT_NOK(Flush()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_EQ(listener->begin_count.load(), 1); + ASSERT_EQ(listener->completed_count.load(), 0); + ASSERT_EQ(listener->finished_count.load(), 1); + ASSERT_NOK(listener->status()); + ASSERT_FALSE(listener->switched_to_mempurge()); +} + +TEST_F(DBFlushTest, FlushFinishedReportsMempurge) { + auto listener = std::make_shared(); + Options options = CurrentOptions(); + options.atomic_flush = false; + options.allow_concurrent_memtable_write = true; + options.disable_auto_compactions = true; + options.inplace_update_support = false; + options.write_buffer_size = 1 << 20; + options.experimental_mempurge_threshold = 100.0; + options.listeners.push_back(listener); + Reopen(options); + + std::atomic mempurge_count{0}; + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkFlush:done", + "DBFlushTest::FlushFinishedReportsMempurge:Done"}}); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:MemPurgeSuccessful", + [&](void* /*arg*/) { ++mempurge_count; }); + SyncPoint::GetInstance()->EnableProcessing(); + + const std::string value(10 * 1024, 'v'); + for (int i = 0; i < 60; ++i) { + for (int key = 0; key < 2; ++key) { + ASSERT_OK(Put("key" + std::to_string(key), value)); + } + } + TEST_SYNC_POINT("DBFlushTest::FlushFinishedReportsMempurge:Done"); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_EQ(listener->flush_reason.load(), + static_cast(FlushReason::kWriteBufferFull)); + ASSERT_GE(mempurge_count.load(), 1); + ASSERT_EQ(listener->begin_count.load(), listener->finished_count.load()); + ASSERT_EQ(listener->completed_count.load(), 0); + ASSERT_OK(listener->status()); + ASSERT_TRUE(listener->switched_to_mempurge()); +} + +TEST_F(DBFlushTest, AtomicFlushFinishesEachColumnFamily) { + auto listener = std::make_shared(); + Options options = CurrentOptions(); + options.atomic_flush = true; + options.listeners.push_back(listener); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(0, "key", "value")); + ASSERT_OK(Put(1, "key", "value")); + ASSERT_OK(db_->Flush(FlushOptions(), handles_)); + + ASSERT_EQ(listener->begin_count.load(), 2); + ASSERT_EQ(listener->completed_count.load(), 2); + ASSERT_EQ(listener->finished_count.load(), 2); + ASSERT_EQ(listener->finished_cfs(), + (std::vector>{{0, "default"}, + {1, "pikachu"}})); + ASSERT_OK(listener->status()); +} + // We had issue when two background threads trying to flush at the same time, // only one of them get committed. The test verifies the issue is fixed. TEST_F(DBFlushTest, FlushWhileWritingManifest) { diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index fff0fd1..80d43c5 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -6344,10 +6344,20 @@ Status DBImpl::IngestExternalFiles( static_cast(args[0].column_family)->cfd(), total, pending_output_elem, &next_file_number); if (!status.ok()) { - InstrumentedMutexLock l(&mutex_); - ReleaseFileNumberFromPendingOutputs(pending_output_elem); + const bool file_numbers_reserved = pending_output_elem != nullptr; + if (file_numbers_reserved) { + NotifyOnExternalFileIngestionStarted(next_file_number, total); + } + { + InstrumentedMutexLock l(&mutex_); + ReleaseFileNumberFromPendingOutputs(pending_output_elem); + } + if (file_numbers_reserved) { + NotifyOnExternalFileIngestionFinished(next_file_number, total); + } return status; } + NotifyOnExternalFileIngestionStarted(next_file_number, total); std::vector ingestion_jobs; for (const auto& arg : args) { @@ -6394,8 +6404,11 @@ Status DBImpl::IngestExternalFiles( for (size_t i = 0; i != num_cfs; ++i) { ingestion_jobs[i].Cleanup(status); } - InstrumentedMutexLock l(&mutex_); - ReleaseFileNumberFromPendingOutputs(pending_output_elem); + { + InstrumentedMutexLock l(&mutex_); + ReleaseFileNumberFromPendingOutputs(pending_output_elem); + } + NotifyOnExternalFileIngestionFinished(next_file_number, total); return status; } @@ -6602,6 +6615,7 @@ Status DBImpl::IngestExternalFiles( // intended for atomicity. ingestion_jobs[i].Cleanup(status); } + NotifyOnExternalFileIngestionFinished(next_file_number, total); if (status.ok()) { for (size_t i = 0; i != num_cfs; ++i) { auto* cfd = @@ -6656,6 +6670,7 @@ Status DBImpl::CreateColumnFamilyWithImport( SuperVersionContext dummy_sv_ctx(/* create_superversion */ true); VersionEdit dummy_edit; uint64_t next_file_number = 0; + bool file_numbers_reserved = false; std::unique_ptr::iterator> pending_output_elem; { // Lock db mutex @@ -6675,6 +6690,7 @@ Status DBImpl::CreateColumnFamilyWithImport( // and this will overwrite the external file. To protect the external // file, we have to make sure the file number will never being reused. next_file_number = versions_->FetchAddFileNumber(total_file_num); + file_numbers_reserved = true; auto cf_options = cfd->GetLatestMutableCFOptions(); status = versions_->LogAndApply(cfd, *cf_options, read_options, write_options, @@ -6685,6 +6701,9 @@ Status DBImpl::CreateColumnFamilyWithImport( } } dummy_sv_ctx.Clean(); + if (file_numbers_reserved) { + NotifyOnExternalFileIngestionStarted(next_file_number, total_file_num); + } if (status.ok()) { SuperVersion* sv = cfd->GetReferencedSuperVersion(this); @@ -6744,6 +6763,9 @@ Status DBImpl::CreateColumnFamilyWithImport( } import_job.Cleanup(status); + if (file_numbers_reserved) { + NotifyOnExternalFileIngestionFinished(next_file_number, total_file_num); + } if (!status.ok()) { Status temp_s = DropColumnFamily(*handle); if (!temp_s.ok()) { @@ -7035,6 +7057,24 @@ void DBImpl::NotifyOnExternalFileIngested( } } +void DBImpl::NotifyOnExternalFileIngestionStarted(uint64_t first_file_number, + size_t file_count) { + for (size_t i = 0; i < file_count; ++i) { + for (const auto& listener : immutable_db_options_.listeners) { + listener->OnExternalFileIngestionStarted(this, first_file_number + i); + } + } +} + +void DBImpl::NotifyOnExternalFileIngestionFinished(uint64_t first_file_number, + size_t file_count) { + for (size_t i = 0; i < file_count; ++i) { + for (const auto& listener : immutable_db_options_.listeners) { + listener->OnExternalFileIngestionFinished(this, first_file_number + i); + } + } +} + Status DBImpl::StartTrace(const TraceOptions& trace_options, std::unique_ptr&& trace_writer) { InstrumentedMutexLock lock(&trace_mutex_); diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index dd0482a..617a87f 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -1472,12 +1472,16 @@ class DBImpl : public DB { void NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta, const MutableCFOptions& mutable_cf_options, - int job_id, FlushReason flush_reason); + int job_id, FlushReason flush_reason, + uint64_t file_number); void NotifyOnFlushCompleted( ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options, std::list>* flush_jobs_info); + void NotifyOnFlushFinished(ColumnFamilyData *cfd, int job_id, + const Status &status, bool switched_to_mempurge); + void NotifyOnCompactionBegin(ColumnFamilyData* cfd, Compaction* c, const Status& st, const CompactionJobStats& job_stats, int job_id); @@ -1491,6 +1495,10 @@ class DBImpl : public DB { void NotifyOnExternalFileIngested( ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job); + void NotifyOnExternalFileIngestionStarted(uint64_t first_file_number, + size_t file_count); + void NotifyOnExternalFileIngestionFinished(uint64_t first_file_number, + size_t file_count); Status FlushAllColumnFamilies(const FlushOptions& flush_options, FlushReason flush_reason); diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index 1db619f..691ade3 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -335,7 +335,7 @@ Status DBImpl::FlushMemTableToOutputFile( // may temporarily unlock and lock the mutex. NotifyOnFlushBegin(cfd, &file_meta, mutable_cf_options, job_context->job_id, - flush_reason); + flush_reason, flush_job.GetFileNumber()); bool switched_to_mempurge = false; // Within flush_job.Run, rocksdb may call event listener to notify @@ -441,6 +441,7 @@ Status DBImpl::FlushMemTableToOutputFile( } } } + NotifyOnFlushFinished(cfd, job_context->job_id, s, switched_to_mempurge); TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:Finish"); return s; } @@ -566,7 +567,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( // may temporarily unlock and lock the mutex. FlushReason flush_reason = bg_flush_args[i].flush_reason_; NotifyOnFlushBegin(cfds[i], &file_meta[i], mutable_cf_options, - job_context->job_id, flush_reason); + job_context->job_id, flush_reason, 0 /* file_number */); } if (logfile_number_ > 0) { @@ -936,12 +937,18 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( } } + for (int i = 0; i != num_cfs; ++i) { + NotifyOnFlushFinished(cfds[i], job_context->job_id, s, + switched_to_mempurge[i]); + } + return s; } void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta, const MutableCFOptions& mutable_cf_options, - int job_id, FlushReason flush_reason) { + int job_id, FlushReason flush_reason, + uint64_t file_number) { if (immutable_db_options_.listeners.size() == 0U) { return; } @@ -963,7 +970,6 @@ void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta, info.cf_name = cfd->GetName(); // TODO(yhchiang): make db_paths dynamic in case flush does not // go to L0 in the future. - const uint64_t file_number = file_meta->fd.GetNumber(); info.file_path = MakeTableFileName(cfd->ioptions()->cf_paths[0].path, file_number); info.file_number = file_number; @@ -1019,6 +1025,26 @@ void DBImpl::NotifyOnFlushCompleted( // flush process. } +void DBImpl::NotifyOnFlushFinished(ColumnFamilyData *cfd, int job_id, + const Status &status, + bool switched_to_mempurge) { + if (immutable_db_options_.listeners.empty()) { + return; + } + mutex_.AssertHeld(); + const uint32_t cf_id = cfd->GetID(); + const std::string cf_name = cfd->GetName(); + mutex_.Unlock(); + { + FlushJobEndInfo info{cf_id, cf_name, env_->GetThreadID(), + job_id, status, switched_to_mempurge}; + for (const auto& listener : immutable_db_options_.listeners) { + listener->OnFlushFinished(this, info); + } + } + mutex_.Lock(); +} + Status DBImpl::CompactRange(const CompactRangeOptions& options, ColumnFamilyHandle* column_family, const Slice* begin_without_ts, diff --git a/db/external_sst_file_test.cc b/db/external_sst_file_test.cc index 64f8a03..738febb 100644 --- a/db/external_sst_file_test.cc +++ b/db/external_sst_file_test.cc @@ -37,10 +37,37 @@ class ExternalSSTTestFS : public FileSystemWrapper { return target()->LinkFile(s, t, options, dbg); } + IOStatus NewWritableFile(const std::string& f, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + IOStatus status = + target()->NewWritableFile(f, file_opts, result, dbg); + if (status.ok() && fail_close_) { + class CloseFailingWritableFile : public FSWritableFileOwnerWrapper { + public: + explicit CloseFailingWritableFile( + std::unique_ptr&& target) + : FSWritableFileOwnerWrapper(std::move(target)) {} + + IOStatus Close(const IOOptions& options, + IODebugContext* dbg) override { + IOStatus status = target()->Close(options, dbg); + return status.ok() ? IOStatus::IOError("injected close failure") + : status; + } + }; + result->reset(new CloseFailingWritableFile(std::move(*result))); + } + return status; + } + void set_fail_link(bool fail_link) { fail_link_ = fail_link; } + void set_fail_close(bool fail_close) { fail_close_ = fail_close; } private: bool fail_link_; + bool fail_close_ = false; }; class ExternalSSTFileTestBase : public DBTestBase { @@ -2271,16 +2298,84 @@ TEST_P(ExternSSTFileLinkFailFallbackTest, LinkFailFallBackExternalSst) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } +TEST_P(ExternSSTFileLinkFailFallbackTest, CopyCloseFailureIsPropagated) { + fs_->set_fail_link(true); + DestroyAndReopen(options_); + + const std::string file_path = sst_files_dir_ + "close_error.sst"; + SstFileWriter writer(EnvOptions(), options_); + ASSERT_OK(writer.Open(file_path)); + ASSERT_OK(writer.Put("key", "value")); + ASSERT_OK(writer.Finish()); + + fs_->set_fail_close(true); + IngestExternalFileOptions ingest_options; + ingest_options.move_files = true; + ingest_options.failed_move_fall_back_to_copy = true; + Status status = db_->IngestExternalFile({file_path}, ingest_options); + fs_->set_fail_close(false); + + ASSERT_TRUE(status.IsIOError()); + ASSERT_NE(status.ToString().find("injected close failure"), + std::string::npos); + + std::vector files; + ASSERT_OK(env_->GetChildren(dbname_, &files)); + for (const auto& file : files) { + uint64_t number; + FileType type; + ASSERT_FALSE(ParseFileName(file, &number, &type) && type == kTableFile) + << "partial destination left behind: " << file; + } +} + class TestIngestExternalFileListener : public EventListener { public: + void OnExternalFileIngestionStarted(DB* /*db*/, + uint64_t file_number) override { + started_file_numbers.push_back(file_number); + } + + void OnExternalFileIngestionFinished(DB* /*db*/, + uint64_t file_number) override { + finished_file_numbers.push_back(file_number); + } + void OnExternalFileIngested(DB* /*db*/, const ExternalFileIngestionInfo& info) override { ingested_files.push_back(info); } + std::vector started_file_numbers; + std::vector finished_file_numbers; std::vector ingested_files; }; +TEST_P(ExternalSSTFileTest, ReservationManifestFailureBalancesLifecycle) { + auto fault_injection_env = std::make_unique(env_); + Options options = CurrentOptions(); + options.env = fault_injection_env.get(); + auto listener = std::make_shared(); + options.listeners.emplace_back(listener); + DestroyAndReopen(options); + + std::vector> data{{"key", "value"}}; + std::string file_path; + ASSERT_OK(GenerateOneExternalFile( + options, db_->DefaultColumnFamily(), data, -1, false /*sort_data*/, + &file_path, nullptr /*true_data*/)); + + fault_injection_env->SetFilesystemActive(false); + Status status = db_->IngestExternalFile({file_path}, + IngestExternalFileOptions()); + fault_injection_env->SetFilesystemActive(true); + + ASSERT_NOK(status); + ASSERT_EQ(listener->started_file_numbers.size(), 1); + ASSERT_EQ(listener->finished_file_numbers, listener->started_file_numbers); + Close(); +} + TEST_P(ExternalSSTFileTest, IngestionListener) { Options options = CurrentOptions(); TestIngestExternalFileListener* listener = @@ -2580,6 +2675,8 @@ TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_Success) { new FaultInjectionTestEnv(env_)); Options options = CurrentOptions(); options.env = fault_injection_env.get(); + auto listener = std::make_shared(); + options.listeners.emplace_back(listener); CreateAndReopenWithCF({"pikachu", "eevee"}, options); // Exercise different situations in different column families: two are empty @@ -2612,6 +2709,12 @@ TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_Success) { column_families.size()); ASSERT_OK(GenerateAndAddExternalFiles(options, column_families, ifos, data, -1, true, true_data)); + ASSERT_EQ(listener->started_file_numbers.size(), 3); + ASSERT_EQ(listener->finished_file_numbers, listener->started_file_numbers); + for (size_t i = 1; i < listener->started_file_numbers.size(); ++i) { + ASSERT_EQ(listener->started_file_numbers[i], + listener->started_file_numbers[i - 1] + 1); + } Close(); ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu", "eevee"}, options); @@ -2757,6 +2860,8 @@ TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_PrepareFail) { new FaultInjectionTestEnv(env_)); Options options = CurrentOptions(); options.env = fault_injection_env.get(); + auto listener = std::make_shared(); + options.listeners.emplace_back(listener); SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->LoadDependency({ @@ -2805,6 +2910,9 @@ TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_PrepareFail) { "1"); ingest_thread.join(); + ASSERT_EQ(listener->started_file_numbers.size(), 3); + ASSERT_EQ(listener->finished_file_numbers, listener->started_file_numbers); + fault_injection_env->SetFilesystemActive(true); Close(); ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu", "eevee"}, @@ -2827,6 +2935,8 @@ TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_CommitFail) { new FaultInjectionTestEnv(env_)); Options options = CurrentOptions(); options.env = fault_injection_env.get(); + auto listener = std::make_shared(); + options.listeners.emplace_back(listener); SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->LoadDependency({ @@ -2874,6 +2984,9 @@ TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_CommitFail) { "1"); ingest_thread.join(); + ASSERT_EQ(listener->started_file_numbers.size(), 3); + ASSERT_EQ(listener->finished_file_numbers, listener->started_file_numbers); + fault_injection_env->SetFilesystemActive(true); Close(); ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu", "eevee"}, diff --git a/db/flush_job.h b/db/flush_job.h index 337e9cd..ccf1c9d 100644 --- a/db/flush_job.h +++ b/db/flush_job.h @@ -93,6 +93,10 @@ class FlushJob { void Cancel(); const autovector& GetMemTables() const { return mems_; } + // Returns the output file number allocated by PickMemTable(), or zero when + // no output was allocated. + uint64_t GetFileNumber() const { return meta_.fd.GetNumber(); } + std::list>* GetCommittedFlushJobsInfo() { return &committed_flush_jobs_info_; } diff --git a/db/import_column_family_test.cc b/db/import_column_family_test.cc index 89586bc..9506abd 100644 --- a/db/import_column_family_test.cc +++ b/db/import_column_family_test.cc @@ -16,6 +16,22 @@ namespace ROCKSDB_NAMESPACE { +class ImportFileLifecycleListener : public EventListener { + public: + void OnExternalFileIngestionStarted(DB* /*db*/, + uint64_t file_number) override { + started_file_numbers.push_back(file_number); + } + + void OnExternalFileIngestionFinished(DB* /*db*/, + uint64_t file_number) override { + finished_file_numbers.push_back(file_number); + } + + std::vector started_file_numbers; + std::vector finished_file_numbers; +}; + class ImportColumnFamilyTest : public DBTestBase { public: ImportColumnFamilyTest() @@ -157,6 +173,8 @@ TEST_F(ImportColumnFamilyTest, ImportSSTFileWriterFiles) { TEST_F(ImportColumnFamilyTest, ImportSSTFileWriterFilesWithOverlap) { Options options = CurrentOptions(); + auto listener = std::make_shared(); + options.listeners.emplace_back(listener); CreateAndReopenWithCF({"koko"}, options); SstFileWriter sfw_cf1(EnvOptions(), options, handles_[1]); @@ -234,6 +252,12 @@ TEST_F(ImportColumnFamilyTest, ImportSSTFileWriterFilesWithOverlap) { ASSERT_OK(db_->CreateColumnFamilyWithImport( options, "toto", ImportColumnFamilyOptions(), metadata, &import_cfh_)); ASSERT_NE(import_cfh_, nullptr); + ASSERT_EQ(listener->started_file_numbers.size(), metadata.files.size()); + ASSERT_EQ(listener->finished_file_numbers, listener->started_file_numbers); + for (size_t i = 1; i < listener->started_file_numbers.size(); ++i) { + ASSERT_EQ(listener->started_file_numbers[i], + listener->started_file_numbers[i - 1] + 1); + } for (int i = 0; i < 100; i++) { std::string value; @@ -637,6 +661,8 @@ TEST_F(ImportColumnFamilyTest, LevelFilesOverlappingAtEndpoints) { TEST_F(ImportColumnFamilyTest, ImportColumnFamilyNegativeTest) { Options options = CurrentOptions(); + auto listener = std::make_shared(); + options.listeners.emplace_back(listener); CreateAndReopenWithCF({"koko"}, options); { @@ -748,6 +774,8 @@ TEST_F(ImportColumnFamilyTest, ImportColumnFamilyNegativeTest) { metadata, &import_cfh_)); ASSERT_NE(import_cfh_, nullptr); } + ASSERT_EQ(listener->started_file_numbers.size(), 5); + ASSERT_EQ(listener->finished_file_numbers, listener->started_file_numbers); } TEST_F(ImportColumnFamilyTest, ImportMultiColumnFamilyTest) { @@ -888,4 +916,3 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } - diff --git a/db/repair.cc b/db/repair.cc index 4fe8b47..d1d906f 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -76,6 +76,7 @@ #include "file/writable_file_writer.h" #include "logging/logging.h" #include "options/cf_options.h" +#include "rocksdb/cloud/cloud_file_system.h" #include "rocksdb/comparator.h" #include "rocksdb/db.h" #include "rocksdb/env.h" @@ -809,10 +810,44 @@ Status GetDefaultCFOptions( } } // anonymous namespace +namespace { + +// RepairDB rebuilds SSTs through BuildTable, which uploads them through the +// cloud writable file. Those uploads are only protected from the purger when +// the FileNumberGuardListener is registered, and that registration happens in +// DBCloud::Open on its own copy of the options -- a standalone RepairDB call +// has no listener, so the publisher (if any) never learns about repair's +// output and the files would be uploaded unprotected. Refuse instead of +// writing files the purger may delete. +Status CheckFileNumberGuardCompatibility(const DBOptions& db_options) { + if (db_options.env == nullptr) { + return Status::OK(); + } + const auto& fs = db_options.env->GetFileSystem(); + if (fs == nullptr) { + return Status::OK(); + } + const auto* cfs = fs->CheckedCast(); + if (cfs != nullptr && + cfs->GetCloudFileSystemOptions().publish_file_number_guard) { + return Status::NotSupported( + "RepairDB is not supported when publish_file_number_guard is enabled: " + "repaired SST uploads would not be protected from the purger"); + } + return Status::OK(); +} + +} // anonymous namespace + Status RepairDB(const std::string& dbname, const DBOptions& db_options, const std::vector& column_families) { + Status status = CheckFileNumberGuardCompatibility(db_options); + if (!status.ok()) { + return status; + } + ColumnFamilyOptions default_cf_opts; - Status status = GetDefaultCFOptions(column_families, &default_cf_opts); + status = GetDefaultCFOptions(column_families, &default_cf_opts); if (!status.ok()) { return status; } @@ -830,8 +865,13 @@ Status RepairDB(const std::string& dbname, const DBOptions& db_options, Status RepairDB(const std::string& dbname, const DBOptions& db_options, const std::vector& column_families, const ColumnFamilyOptions& unknown_cf_opts) { + Status status = CheckFileNumberGuardCompatibility(db_options); + if (!status.ok()) { + return status; + } + ColumnFamilyOptions default_cf_opts; - Status status = GetDefaultCFOptions(column_families, &default_cf_opts); + status = GetDefaultCFOptions(column_families, &default_cf_opts); if (!status.ok()) { return status; } @@ -850,10 +890,15 @@ Status RepairDB(const std::string& dbname, const Options& options) { DBOptions db_options(opts); ColumnFamilyOptions cf_options(opts); + Status status = CheckFileNumberGuardCompatibility(db_options); + if (!status.ok()) { + return status; + } + Repairer repairer(dbname, db_options, {}, cf_options /* default_cf_opts */, cf_options /* unknown_cf_opts */, true /* create_unknown_cfs */); - Status status = repairer.Run(); + status = repairer.Run(); if (status.ok()) { status = repairer.Close(); } diff --git a/file/file_util.cc b/file/file_util.cc index cd63642..3f1819b 100644 --- a/file/file_util.cc +++ b/file/file_util.cc @@ -92,8 +92,17 @@ IOStatus CopyFile(FileSystem* fs, const std::string& source, new WritableFileWriter(std::move(destfile), destination, options)); } - return CopyFile(fs, source, src_temp_hint, dest_writer, size, use_fsync, + io_s = CopyFile(fs, source, src_temp_hint, dest_writer, size, use_fsync, io_tracer); + if (io_s.ok()) { + io_s = dest_writer->Close(IOOptions()); + } else { + dest_writer->Close(IOOptions()).PermitUncheckedError(); + } + if (!io_s.ok()) { + fs->DeleteFile(destination, IOOptions(), nullptr).PermitUncheckedError(); + } + return io_s; } // Utility function to create a file with the provided contents diff --git a/include/rocksdb/cloud/cloud_file_system.h b/include/rocksdb/cloud/cloud_file_system.h index 597989c..c0e7ff1 100644 --- a/include/rocksdb/cloud/cloud_file_system.h +++ b/include/rocksdb/cloud/cloud_file_system.h @@ -446,6 +446,31 @@ class CloudFileSystemOptions { // Default: false bool disable_cloud_file_deletion; + // Publish the per-epoch smallest_new_file_number- guard object that + // protects in-flight flush/compaction uploads from the standalone purger + // (see cloud/eloq_purger.cc). When enabled, DBCloud::Open registers an + // event listener that tracks in-flight jobs and publishes the low + // watermark, and writes a 0 sentinel ("purging blocked") for the epoch + // before recovery can flush. + // + // Enable this on writer nodes in deployments where cloud file deletion is + // delegated to the purger (i.e. disable_cloud_file_deletion=true and a + // purger runs against the bucket). Default false: no guard is published, + // matching the behavior before this option existed (external embedders may + // publish the guard themselves). + bool publish_file_number_guard{false}; + + // Interval of the periodic (upward) publish of the file number guard, + // configured as guard_publish_interval_ms. Must be greater than zero. + // Default: 30 seconds. + std::chrono::milliseconds guard_publish_interval{std::chrono::seconds(30)}; + + // How long a completed job's entry keeps holding the guard down before it + // expires, giving the job's MANIFEST update time to reach the cloud. + // Configured as guard_entry_duration_ms and must be greater than zero. + // Default: 15 seconds. + std::chrono::milliseconds guard_entry_duration{std::chrono::seconds(15)}; + // Type info map for this class. static const std::unordered_map cloud_fs_option_type_info; @@ -544,6 +569,9 @@ struct CloudManifestDelta { class CloudFileSystem : public FileSystem { public: static const char* kCloud() { return "cloud"; } + // Lets callers reach a CloudFileSystem through Customizable::CheckedCast + // without requiring RTTI. + static const char* kClassName() { return kCloud(); } static const char* kAws() { return "aws"; } static char const* kGcp() { return "gcp"; } @@ -704,8 +732,18 @@ class CloudFileSystem : public FileSystem { // REQUIRES: stop writes, stop compactions virtual IOStatus BackupCloudManifest(const std::string& dest_folder, std::vector &backup_files) = 0; - virtual Logger* GetLogger() const = 0; + virtual Logger *GetLogger() const = 0; virtual void SetLogger(std::shared_ptr) = 0; + + // Publishes the 0 sentinel for the current epoch's + // smallest_new_file_number guard object, blocking the standalone purger + // for this epoch until the next periodic publish. Embedding storage layers + // call this around leader transfer. + // + // REQUIRES: CloudManifest loaded + virtual Status BlockPurger() { + return Status::NotSupported("BlockPurger not supported"); + } }; // diff --git a/include/rocksdb/cloud/cloud_file_system_impl.h b/include/rocksdb/cloud/cloud_file_system_impl.h index 763d39a..3212216 100644 --- a/include/rocksdb/cloud/cloud_file_system_impl.h +++ b/include/rocksdb/cloud/cloud_file_system_impl.h @@ -14,6 +14,7 @@ namespace ROCKSDB_NAMESPACE { class CloudManifest; +class FileNumberGuardPublisher; class CloudScheduler; class CloudStorageReadableFile; class ObjectLibrary; @@ -395,12 +396,36 @@ class CloudFileSystemImpl : public CloudFileSystem { void Purger(); void StopPurger(); + // Publisher of the smallest_new_file_number guard object (see + // cloud/file_number_guard.h). Set by DBCloudImpl::Open when + // CloudFileSystemOptions::publish_file_number_guard is enabled. Readers use + // atomic_load; publisher lifecycle changes are serialized below. + std::shared_ptr file_number_guard_; + // Serializes publisher replacement with the sentinel PUT. The old + // publisher remains installed but stopped until the replacement's 0 + // sentinel is durable, so SST uploads fail closed throughout handoff. + std::mutex file_number_guard_install_mutex_; + // Delete all local files that are invisible IOStatus DeleteLocalInvisibleFiles( const std::string& dbname, const std::vector& active_cookies) override; public: + // File number guard (defined in cloud/file_number_guard.cc). + void SetFileNumberGuardPublisher( + std::shared_ptr publisher); + Status InstallFileNumberGuardPublisher( + const std::shared_ptr &publisher); + bool RemoveFileNumberGuardPublisher( + const std::shared_ptr &expected); + bool StopFileNumberGuardPublisher( + const std::shared_ptr &expected); + std::shared_ptr GetFileNumberGuardPublisher() + const; + void StopFileNumberGuard(); + Status BlockPurger() override; + // returns the options used to create this object const CloudFileSystemOptions& GetCloudFileSystemOptions() const override { return cloud_fs_options; diff --git a/include/rocksdb/cloud/cloud_storage_provider.h b/include/rocksdb/cloud/cloud_storage_provider.h index ad58382..d0720b8 100644 --- a/include/rocksdb/cloud/cloud_storage_provider.h +++ b/include/rocksdb/cloud/cloud_storage_provider.h @@ -148,5 +148,19 @@ class CloudStorageProvider : public Configurable { const FileOptions& options, std::unique_ptr* result, IODebugContext* dbg) = 0; + + // Delete the specified objects from the specified cloud bucket. Attempts + // every object even when some deletions fail; an object that does not + // exist counts as deleted. Adds the number of objects deleted and failed + // to *deleted_count and *failed_count (both must be non-null). Returns OK + // when every object was deleted or already absent, otherwise the first + // failure. + // + // The base implementation deletes serially via DeleteCloudObject. + // Providers with a native bulk-delete API may override it. + virtual IOStatus + DeleteCloudObjects(const std::string &bucket_name, + const std::vector &object_paths, + size_t *deleted_count, size_t *failed_count); }; } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index 452ae54..c0b1113 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -369,6 +369,27 @@ struct FlushJobInfo { std::vector blob_file_addition_infos; }; +// Information about the terminal outcome of a flush attempt. Unlike +// FlushJobInfo, this is reported for failed flushes and successful mempurges +// that produce no SST. +struct FlushJobEndInfo { + ~FlushJobEndInfo() { status.PermitUncheckedError(); } + + // The id of the column family. + uint32_t cf_id; + // The name of the column family. + std::string cf_name; + // The id of the thread that ran the flush job. + uint64_t thread_id; + // The job id, which is unique in the same thread. + int job_id; + // The final status after the flush result was installed or rolled back. + Status status; + // True when the job completed by replacing its input with a mempurged + // memtable instead of creating an SST. + bool switched_to_mempurge; +}; + struct CompactionFileInfo { // The level of the file. int level; @@ -842,6 +863,24 @@ class EventListener : public Customizable { virtual void OnIOError(const IOErrorInfo& /*info*/) {} ~EventListener() override {} + + // Called once for each exact file number reserved for an external file + // ingestion or column family import, before file preparation starts. + virtual void OnExternalFileIngestionStarted(DB* /*db*/, + uint64_t /*file_number*/) {} + + // Called once after cleanup for every file number reported to + // OnExternalFileIngestionStarted(), whether the operation succeeds or fails. + virtual void OnExternalFileIngestionFinished(DB* /*db*/, + uint64_t /*file_number*/) {} + + // Called exactly once after every flush attempt whose OnFlushBegin callback + // was issued, after its final status is known. This includes failures and + // successful mempurges; OnFlushCompleted remains success-only and is not + // called for mempurges because they produce no SST. + virtual void OnFlushFinished(DB * /*db*/, + const FlushJobEndInfo & /*flush_job_end_info*/) { + } }; diff --git a/src.mk b/src.mk index 2122c29..6bbe06f 100644 --- a/src.mk +++ b/src.mk @@ -27,6 +27,7 @@ LIB_SOURCES = \ cloud/cloud_log_controller.cc \ cloud/manifest_reader.cc \ cloud/eloq_purger.cc \ + cloud/file_number_guard.cc \ cloud/cloud_manifest.cc \ cloud/cloud_scheduler.cc \ cloud/cloud_storage_provider.cc \ @@ -466,6 +467,9 @@ TEST_MAIN_SOURCES = \ cloud/gcp/gcp_db_cloud_test.cc \ cloud/cloud_manifest_test.cc \ cloud/cloud_scheduler_test.cc \ + cloud/eloq_purger_test.cc \ + cloud/eloq_purger_integration_test.cc \ + cloud/file_number_guard_test.cc \ cloud/replication_test.cc \ cache/compressed_secondary_cache_test.cc \ cache/lru_cache_test.cc \ diff --git a/util/stderr_logger.h b/util/stderr_logger.h index 6a3fe3b..f0639d1 100644 --- a/util/stderr_logger.h +++ b/util/stderr_logger.h @@ -17,7 +17,7 @@ namespace ROCKSDB_NAMESPACE { class StderrLogger : public Logger { public: explicit StderrLogger(const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL) - : Logger(log_level), log_prefix(nullptr) {} + : Logger(log_level), log_prefix(nullptr), log_prefix_len(0) {} explicit StderrLogger(const InfoLogLevel log_level, const std::string prefix) : Logger(log_level), log_prefix(strdup(prefix.c_str())),