diff --git a/bindings/cpp/include/svs/runtime/vamana_index.h b/bindings/cpp/include/svs/runtime/vamana_index.h index 180daf55c..0f11edec6 100644 --- a/bindings/cpp/include/svs/runtime/vamana_index.h +++ b/bindings/cpp/include/svs/runtime/vamana_index.h @@ -50,6 +50,14 @@ struct VamanaSearchParameters { }; } // namespace detail +struct MemoryBreakdown { + size_t graph_bytes = 0; + size_t data_bytes = 0; + size_t metadata_bytes = 0; + + size_t total() const { return graph_bytes + data_bytes + metadata_bytes; } +}; + // Abstract interface for Vamana-based indices. struct SVS_RUNTIME_API VamanaIndex { virtual ~VamanaIndex(); @@ -90,6 +98,12 @@ struct SVS_RUNTIME_API VamanaIndex { // Reconstruct `n` vectors by ID into `output` buffer (n * dim floats). virtual Status reconstruct_at(size_t n, const size_t* ids, float* output) noexcept = 0; + // Return the index memory usage in bytes. + virtual size_t get_memory_usage() const noexcept = 0; + + // Return the bytes allocated by each index component. + virtual Status get_memory_breakdown(MemoryBreakdown* out) const noexcept = 0; + // Utility function to check storage kind support static Status check_storage_kind(StorageKind storage_kind) noexcept; diff --git a/bindings/cpp/src/dynamic_vamana_index.cpp b/bindings/cpp/src/dynamic_vamana_index.cpp index 47366481c..0e807bd98 100644 --- a/bindings/cpp/src/dynamic_vamana_index.cpp +++ b/bindings/cpp/src/dynamic_vamana_index.cpp @@ -65,6 +65,17 @@ struct DynamicVamanaIndexManagerBase : public DynamicVamanaIndex { size_t blocksize_bytes() const noexcept override { return impl_->blocksize_bytes(); } + size_t get_memory_usage() const noexcept override { return impl_->get_memory_usage(); } + + Status get_memory_breakdown(MemoryBreakdown* out) const noexcept override { + if (out == nullptr) { + return Status( + ErrorCode::INVALID_ARGUMENT, "memory breakdown output must not be null" + ); + } + return runtime_error_wrapper([&] { impl_->get_memory_breakdown(*out); }); + } + Status remove_selected(size_t* num_removed, const IDFilter& selector) noexcept override { return runtime_error_wrapper([&] { @@ -160,11 +171,13 @@ Status DynamicVamanaIndex::check_params( constexpr static size_t kMaxBlockSizeExp = 30; // 1GB constexpr static size_t kMinBlockSizeExp = 12; // 4KB - if (dynamic_index_params.blocksize_exp > kMaxBlockSizeExp) + if (dynamic_index_params.blocksize_exp > kMaxBlockSizeExp) { return Status(ErrorCode::INVALID_ARGUMENT, "Blocksize is too large"); + } - if (dynamic_index_params.blocksize_exp < kMinBlockSizeExp) + if (dynamic_index_params.blocksize_exp < kMinBlockSizeExp) { return Status(ErrorCode::INVALID_ARGUMENT, "Blocksize is too small"); + } return Status_Ok; } @@ -202,8 +215,9 @@ Status DynamicVamanaIndex::build( *index = nullptr; auto status = DynamicVamanaIndex::check_params(dynamic_index_params); - if (!status.ok()) + if (!status.ok()) { return status; + } return runtime_error_wrapper([&] { auto impl = std::make_unique( @@ -293,8 +307,9 @@ Status DynamicVamanaIndexLeanVec::build( *index = nullptr; auto status = DynamicVamanaIndex::check_params(dynamic_index_params); - if (!status.ok()) + if (!status.ok()) { return status; + } return runtime_error_wrapper([&] { auto impl = std::make_unique( @@ -325,8 +340,9 @@ Status DynamicVamanaIndexLeanVec::build( *index = nullptr; auto status = DynamicVamanaIndex::check_params(dynamic_index_params); - if (!status.ok()) + if (!status.ok()) { return status; + } return runtime_error_wrapper([&] { auto training_data_impl = diff --git a/bindings/cpp/src/dynamic_vamana_index_impl.h b/bindings/cpp/src/dynamic_vamana_index_impl.h index 50cb20932..f89e1bb59 100644 --- a/bindings/cpp/src/dynamic_vamana_index_impl.h +++ b/bindings/cpp/src/dynamic_vamana_index_impl.h @@ -69,6 +69,22 @@ class DynamicVamanaIndexImpl { size_t size() const { return impl_ ? impl_->size() : 0; } + size_t get_memory_usage() const { + return impl_ ? impl_->get_memory_breakdown().total() : 0; + } + + void get_memory_breakdown(MemoryBreakdown& out) const { + if (!impl_) { + out = MemoryBreakdown{}; + return; + } + + auto breakdown = impl_->get_memory_breakdown(); + out.graph_bytes = breakdown.graph_bytes; + out.data_bytes = breakdown.data_bytes; + out.metadata_bytes = breakdown.metadata_bytes; + } + size_t blocksize_bytes() const { return 1u << dynamic_index_params_.blocksize_exp; } size_t dimensions() const { return dim_; } diff --git a/bindings/cpp/src/vamana_index.cpp b/bindings/cpp/src/vamana_index.cpp index 195164c50..e3e5be589 100644 --- a/bindings/cpp/src/vamana_index.cpp +++ b/bindings/cpp/src/vamana_index.cpp @@ -104,6 +104,17 @@ struct VamanaIndexManagerBase : public VamanaIndex { impl_->reconstruct_at(dst, id_span); }); } + + size_t get_memory_usage() const noexcept override { return impl_->get_memory_usage(); } + + Status get_memory_breakdown(MemoryBreakdown* out) const noexcept override { + if (out == nullptr) { + return Status( + ErrorCode::INVALID_ARGUMENT, "memory breakdown output must not be null" + ); + } + return runtime_error_wrapper([&] { impl_->get_memory_breakdown(*out); }); + } }; } // namespace diff --git a/bindings/cpp/src/vamana_index_impl.h b/bindings/cpp/src/vamana_index_impl.h index b145b7f11..21a56abfc 100644 --- a/bindings/cpp/src/vamana_index_impl.h +++ b/bindings/cpp/src/vamana_index_impl.h @@ -74,6 +74,22 @@ class VamanaIndexImpl { size_t size() const { return impl_ ? get_impl()->size() : 0; } + size_t get_memory_usage() const { + return impl_ ? get_impl()->get_memory_breakdown().total() : 0; + } + + void get_memory_breakdown(MemoryBreakdown& out) const { + if (!impl_) { + out = MemoryBreakdown{}; + return; + } + + auto breakdown = get_impl()->get_memory_breakdown(); + out.graph_bytes = breakdown.graph_bytes; + out.data_bytes = breakdown.data_bytes; + out.metadata_bytes = breakdown.metadata_bytes; + } + size_t dimensions() const { return dim_; } MetricType metric_type() const { return metric_type_; } diff --git a/bindings/cpp/tests/runtime_test.cpp b/bindings/cpp/tests/runtime_test.cpp index f0972b7f0..4ebc25df7 100644 --- a/bindings/cpp/tests/runtime_test.cpp +++ b/bindings/cpp/tests/runtime_test.cpp @@ -1483,3 +1483,130 @@ CATCH_TEST_CASE("ReconstructAtStatic", "[runtime][static_vamana]") { svs::runtime::v0::VamanaIndex::destroy(index); } + +CATCH_TEST_CASE("GetMemoryUsageDynamic", "[runtime][memory]") { + const auto& test_data = get_test_data(); + svs::runtime::v0::DynamicVamanaIndex* index = nullptr; + svs::runtime::v0::VamanaIndex::BuildParams build_params{64}; + svs::runtime::v0::Status status = svs::runtime::v0::DynamicVamanaIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + build_params + ); + if (!svs::runtime::v0::DynamicVamanaIndex::check_storage_kind( + svs::runtime::v0::StorageKind::FP32 + ) + .ok()) { + CATCH_REQUIRE(!status.ok()); + CATCH_SKIP("Storage kind is not supported, skipping test."); + } + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + status = index->add(test_n, labels.data(), test_data.data()); + CATCH_REQUIRE(status.ok()); + + // After adding points, the index must report non-zero memory usage and the + // component breakdown must sum to the same value. + const auto dynamic_usage = index->get_memory_usage(); + CATCH_REQUIRE(dynamic_usage > 0); + svs::runtime::v0::MemoryBreakdown dynamic_breakdown{}; + status = index->get_memory_breakdown(&dynamic_breakdown); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(dynamic_breakdown.graph_bytes > 0); + CATCH_REQUIRE(dynamic_breakdown.data_bytes > 0); + CATCH_REQUIRE(dynamic_breakdown.metadata_bytes > 0); + CATCH_REQUIRE(dynamic_breakdown.total() == dynamic_usage); + status = index->get_memory_breakdown(nullptr); + CATCH_REQUIRE(!status.ok()); + CATCH_REQUIRE(status.code == svs::runtime::v0::ErrorCode::INVALID_ARGUMENT); + + svs::runtime::v0::DynamicVamanaIndex::destroy(index); + + // A larger index (more points) should use at least as much memory as a + // smaller one built with the same storage kind / parameters. + svs::runtime::v0::DynamicVamanaIndex* small_index = nullptr; + status = svs::runtime::v0::DynamicVamanaIndex::build( + &small_index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(small_index != nullptr); + + const size_t small_n = test_n / 2; + std::vector small_labels(small_n); + std::iota(small_labels.begin(), small_labels.end(), 0); + status = small_index->add(small_n, small_labels.data(), test_data.data()); + CATCH_REQUIRE(status.ok()); + const size_t small_usage = small_index->get_memory_usage(); + CATCH_REQUIRE(small_usage > 0); + + svs::runtime::v0::DynamicVamanaIndex* large_index = nullptr; + status = svs::runtime::v0::DynamicVamanaIndex::build( + &large_index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(large_index != nullptr); + std::vector large_labels(test_n); + std::iota(large_labels.begin(), large_labels.end(), 0); + status = large_index->add(test_n, large_labels.data(), test_data.data()); + CATCH_REQUIRE(status.ok()); + const size_t large_usage = large_index->get_memory_usage(); + CATCH_REQUIRE(large_usage > 0); + + CATCH_REQUIRE(large_usage >= small_usage); + + svs::runtime::v0::DynamicVamanaIndex::destroy(small_index); + svs::runtime::v0::DynamicVamanaIndex::destroy(large_index); +} + +CATCH_TEST_CASE("GetMemoryUsageStatic", "[runtime][static_vamana][memory]") { + const auto& test_data = get_test_data(); + svs::runtime::v0::VamanaIndex* index = nullptr; + svs::runtime::v0::VamanaIndex::BuildParams build_params{64}; + svs::runtime::v0::Status status = svs::runtime::v0::VamanaIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + build_params + ); + if (!svs::runtime::v0::VamanaIndex::check_storage_kind( + svs::runtime::v0::StorageKind::FP32 + ) + .ok()) { + CATCH_REQUIRE(!status.ok()); + CATCH_SKIP("Storage kind is not supported, skipping test."); + } + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + status = index->add(test_n, test_data.data()); + CATCH_REQUIRE(status.ok()); + + // After adding points, the static index must report non-zero memory usage and the + // component breakdown must sum to the same value. + const auto static_usage = index->get_memory_usage(); + CATCH_REQUIRE(static_usage > 0); + svs::runtime::v0::MemoryBreakdown static_breakdown{}; + status = index->get_memory_breakdown(&static_breakdown); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(static_breakdown.graph_bytes > 0); + CATCH_REQUIRE(static_breakdown.data_bytes > 0); + CATCH_REQUIRE(static_breakdown.metadata_bytes > 0); + CATCH_REQUIRE(static_breakdown.total() == static_usage); + + svs::runtime::v0::VamanaIndex::destroy(index); +} diff --git a/include/svs/core/data.h b/include/svs/core/data.h index ba8d85c15..386b6b109 100644 --- a/include/svs/core/data.h +++ b/include/svs/core/data.h @@ -153,6 +153,21 @@ class VectorDataLoader { // Matching rule for uncompressed data. namespace data::detail { + +/// @brief Return the number of bytes allocated for the backing storage of ``dataset``. +/// +/// Capacity-based accounting (the bytes the containers have reserved) is used whenever the +/// dataset exposes a ``capacity()`` accessor (e.g. flat and blocked ``SimpleData``), so +/// that block over-allocation is reflected. Datasets that do not expose ``capacity()`` +/// fall back to the number of live elements. +template size_t dataset_allocated_bytes(const Dataset& dataset) { + if constexpr (requires(const Dataset& d) { d.capacity(); }) { + return dataset.capacity() * dataset.element_size(); + } else { + return dataset.size() * dataset.element_size(); + } +} + template int64_t check_match(svs::DataType type, size_t dims) { // If the types don't match - then there is no match. if (type != svs::datatype_v) { diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 5f0ce7c16..5078a1702 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -321,6 +321,32 @@ class MutableVamanaIndex { /// @brief Get the ``graph_max_degree`` used while mutating the graph. size_t get_graph_max_degree() const { return graph_.max_degree(); } + /// @brief Return the bytes allocated by each index component. + /// + /// Reports the capacity-based bytes reserved by the graph adjacency lists, the vector + /// data, and the dynamic metadata (per-slot status, entry-point list, and the + /// external/internal ID translation maps). Capacity-based accounting includes the + /// block over-allocation so integrators can report the true memory footprint. + MemoryBreakdown get_memory_breakdown() const { + MemoryBreakdown usage{}; + usage.graph_bytes = svs::data::detail::dataset_allocated_bytes(graph_.get_data()); + usage.data_bytes = svs::data::detail::dataset_allocated_bytes(data_); + + size_t metadata_bytes = status_.capacity() * sizeof(SlotMetadata); + metadata_bytes += + entry_point_.capacity() * sizeof(typename entry_point_type::value_type); + // The IDTranslator holds two tsl::robin_map instances (external->internal and + // internal->external), neither of which exposes its allocated byte count. We + // approximate the storage as the id pair held in each of the two directions. This + // ignores the maps' load-factor slack and control bytes, so it is an estimate of + // the hash-map overhead that is accurate to within a few percent. + metadata_bytes += 2 * translator_.size() * + (sizeof(IDTranslator::external_id_type) + + sizeof(IDTranslator::internal_id_type)); + usage.metadata_bytes = metadata_bytes; + return usage; + } + /// @brief Get the max candidate pool size used while mutating the graph. size_t get_max_candidates() const { return max_candidates_; } /// @brief Set the max candidate pool size to be used while mutating the graph. diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index 70a921353..239054fd5 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -177,6 +177,14 @@ struct VamanaIndexParameters { operator==(const VamanaIndexParameters&, const VamanaIndexParameters&) = default; }; +struct MemoryBreakdown { + size_t graph_bytes = 0; + size_t data_bytes = 0; + size_t metadata_bytes = 0; + + size_t total() const { return graph_bytes + data_bytes + metadata_bytes; } +}; + /// /// @brief Search scratchspace used by the Vamana index. /// @@ -744,6 +752,21 @@ class VamanaIndex { /// @brief Get the ``graph_max_degree`` that was used for graph construction. size_t get_graph_max_degree() const { return graph_.max_degree(); } + /// @brief Return the bytes allocated by each index component. + /// + /// Reports the capacity-based bytes reserved by the graph adjacency lists, the vector + /// data, and the entry-point list (the static index has no slot-status or + /// ID-translation metadata). Capacity-based accounting includes the block + /// over-allocation so integrators can report the true memory footprint. + MemoryBreakdown get_memory_breakdown() const { + MemoryBreakdown usage{}; + usage.graph_bytes = svs::data::detail::dataset_allocated_bytes(graph_.get_data()); + usage.data_bytes = svs::data::detail::dataset_allocated_bytes(data_); + usage.metadata_bytes = + entry_point_.capacity() * sizeof(typename entry_point_type::value_type); + return usage; + } + /// @brief Get the max candidate pool size that was used for graph construction. size_t get_max_candidates() const { return build_parameters_.max_candidate_pool_size; } /// @brief Set the max candidate pool size to be used for graph construction. diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 045077387..6d20476b3 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -198,6 +198,11 @@ class DynamicVamana : public manager::IndexManager { /// @copydoc svs::index::vamana::MutableVamanaIndex::get_graph_max_degree size_t get_graph_max_degree() const { return impl_->get_graph_max_degree(); } + /// @copydoc svs::index::vamana::MutableVamanaIndex::get_memory_breakdown + svs::index::vamana::MemoryBreakdown get_memory_breakdown() const { + return impl_->get_memory_breakdown(); + } + /// @copydoc svs::index::vamana::MutableVamanaIndex::set_construction_window_size size_t get_construction_window_size() const { return impl_->get_construction_window_size(); diff --git a/include/svs/orchestrators/vamana.h b/include/svs/orchestrators/vamana.h index c4c4422b6..6b141ed0c 100644 --- a/include/svs/orchestrators/vamana.h +++ b/include/svs/orchestrators/vamana.h @@ -50,6 +50,8 @@ class VamanaInterface { virtual size_t get_graph_max_degree() const = 0; + virtual svs::index::vamana::MemoryBreakdown get_memory_breakdown() const = 0; + virtual void set_construction_window_size(size_t window_size) = 0; virtual size_t get_construction_window_size() const = 0; @@ -127,6 +129,10 @@ class VamanaImpl : public manager::ManagerImpl { size_t get_graph_max_degree() const override { return impl().get_graph_max_degree(); } + svs::index::vamana::MemoryBreakdown get_memory_breakdown() const override { + return impl().get_memory_breakdown(); + } + void set_construction_window_size(size_t window_size) override { impl().set_construction_window_size(window_size); } @@ -321,6 +327,11 @@ class Vamana : public manager::IndexManager { /// @copydoc svs::index::vamana::VamanaIndex::get_graph_max_degree size_t get_graph_max_degree() const { return impl_->get_graph_max_degree(); } + /// @copydoc svs::index::vamana::VamanaIndex::get_memory_breakdown + svs::index::vamana::MemoryBreakdown get_memory_breakdown() const { + return impl_->get_memory_breakdown(); + } + /// @copydoc svs::index::vamana::VamanaIndex::set_construction_window_size size_t get_construction_window_size() const { return impl_->get_construction_window_size(); diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index 798a584c2..7f64ce659 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -364,3 +364,43 @@ CATCH_TEST_CASE( } } } + +CATCH_TEST_CASE("MutableVamana Index Memory Usage", "[graph_index][dynamic_index]") { + const size_t num_threads = 2; + using Distance = svs::distance::DistanceL2; + + auto data = test_dataset::data_blocked_f32(); + const size_t data_size = data.size(); + // Expected data bytes are capacity-based; capture them before the dataset is moved + // into the index so the test can pin the exact value. + const size_t expected_data_bytes = data.capacity() * data.element_size(); + std::vector indices(data_size); + std::iota(indices.begin(), indices.end(), 0); + + svs::index::vamana::VamanaBuildParameters parameters{1.2, 64, 10, 20, 10, true}; + auto index = svs::index::vamana::MutableVamanaIndex( + parameters, std::move(data), indices, Distance(), num_threads + ); + + const size_t expected_graph_bytes = index.view_graph().get_data().capacity() * + index.view_graph().get_data().element_size(); + using Index = decltype(index); + const size_t expected_metadata_bytes = + data_size * sizeof(svs::index::vamana::SlotMetadata) + + sizeof(typename Index::internal_id_type) + + 2 * indices.size() * + (sizeof(typename Index::external_id_type) + + sizeof(typename Index::internal_id_type)); + const size_t expected_total_bytes = + expected_data_bytes + expected_graph_bytes + expected_metadata_bytes; + + // Dynamic get_memory_usage() should exactly match the capacity-based graph and data + // bytes plus the deterministic metadata implied by the input ids. + const auto breakdown = index.get_memory_breakdown(); + CATCH_REQUIRE(breakdown.graph_bytes == expected_graph_bytes); + CATCH_REQUIRE(breakdown.data_bytes == expected_data_bytes); + CATCH_REQUIRE(breakdown.metadata_bytes == expected_metadata_bytes); + CATCH_REQUIRE(breakdown.total() == expected_total_bytes); + const size_t usage = index.get_memory_breakdown().total(); + CATCH_REQUIRE(usage == expected_total_bytes); +} diff --git a/tests/svs/index/vamana/index.cpp b/tests/svs/index/vamana/index.cpp index c63bcefe3..284bc68d2 100644 --- a/tests/svs/index/vamana/index.cpp +++ b/tests/svs/index/vamana/index.cpp @@ -186,6 +186,51 @@ CATCH_TEST_CASE("Static VamanaIndex Per-Index Logging", "[logging]") { CATCH_REQUIRE(captured_logs[2].find("Batch Size:") != std::string::npos); } +CATCH_TEST_CASE("Vamana Index Memory Usage", "[vamana][index]") { + const size_t N = 128; + using Eltype = float; + const size_t graph_max_degree = 64; + auto data = svs::data::SimpleData::load(test_dataset::data_svs_file()); + + auto graph = svs::graphs::SimpleGraph(data.size(), graph_max_degree); + svs::distance::DistanceL2 distance_function; + uint32_t entry_point = 0; + auto threadpool = svs::threads::DefaultThreadPool(1); + + // Compute the expected allocated bytes directly from each container's own + // capacity()/element_size() before they are moved into the index, so the test pins + // the exact value rather than a lower bound. + const size_t expected_data_bytes = data.capacity() * data.element_size(); + const size_t expected_graph_bytes = + graph.get_data().capacity() * graph.get_data().element_size(); + + svs::index::vamana::VamanaBuildParameters buildParams( + 1.2, graph_max_degree, 10, 20, 10, true + ); + svs::index::vamana::VamanaIndex index( + buildParams, + std::move(graph), + std::move(data), + entry_point, + distance_function, + std::move(threadpool) + ); + + const size_t expected_metadata_bytes = sizeof(uint32_t); + const size_t expected_total_bytes = + expected_data_bytes + expected_graph_bytes + expected_metadata_bytes; + + // Static get_memory_usage() should exactly match the capacity-based bytes implied by + // the input graph, input data, and one entry-point id. + const auto breakdown = index.get_memory_breakdown(); + CATCH_REQUIRE(breakdown.graph_bytes == expected_graph_bytes); + CATCH_REQUIRE(breakdown.data_bytes == expected_data_bytes); + CATCH_REQUIRE(breakdown.metadata_bytes == expected_metadata_bytes); + CATCH_REQUIRE(breakdown.total() == expected_total_bytes); + const size_t usage = index.get_memory_breakdown().total(); + CATCH_REQUIRE(usage == expected_total_bytes); +} + CATCH_TEST_CASE("Vamana Index Save and Load", "[vamana][index][saveload]") { const size_t N = 128; using Eltype = float; diff --git a/tests/svs/orchestrators/dynamic_vamana.cpp b/tests/svs/orchestrators/dynamic_vamana.cpp index b35234eb6..10951e7ca 100644 --- a/tests/svs/orchestrators/dynamic_vamana.cpp +++ b/tests/svs/orchestrators/dynamic_vamana.cpp @@ -118,3 +118,47 @@ CATCH_TEST_CASE("DynamicVamana Build", "[managers][dynamic_vamana][build]") { } } } + +CATCH_TEST_CASE("DynamicVamana Memory Usage", "[managers][dynamic_vamana]") { + auto distance = svs::distance::DistanceL2(); + auto expected_result = test_dataset::vamana::expected_build_results( + distance, svsbenchmark::Uncompressed(svs::DataType::float32) + ); + auto build_params = expected_result.build_parameters_.value(); + size_t num_threads = 2; + + auto data = svs::data::SimpleData::load(test_dataset::data_svs_file()); + const size_t n = data.size(); + const size_t half = n / 2; + CATCH_REQUIRE(half > 0); + CATCH_REQUIRE(n - half > 0); + + // Build the index over the first half of the dataset (external IDs 0 .. half-1). + auto first_data = svs::data::SimpleData(half, data.dimensions()); + for (size_t i = 0; i < half; ++i) { + first_data.set_datum(i, data.get_datum(i)); + } + std::vector first_ids(half); + std::iota(first_ids.begin(), first_ids.end(), 0); + + svs::DynamicVamana index = svs::DynamicVamana::build( + build_params, std::move(first_data), first_ids, distance, num_threads + ); + + const size_t usage_before = index.get_memory_breakdown().total(); + CATCH_REQUIRE(usage_before > 0); + + // Add the second half of the dataset (external IDs half .. n-1). + const size_t rest = n - half; + auto second_data = svs::data::SimpleData(rest, data.dimensions()); + for (size_t i = 0; i < rest; ++i) { + second_data.set_datum(i, data.get_datum(half + i)); + } + std::vector second_ids(rest); + std::iota(second_ids.begin(), second_ids.end(), half); + index.add_points(second_data.cview(), second_ids); + + // Adding points must increase the reported allocation. + const size_t usage_after = index.get_memory_breakdown().total(); + CATCH_REQUIRE(usage_after > usage_before); +} diff --git a/tests/svs/orchestrators/vamana.cpp b/tests/svs/orchestrators/vamana.cpp index 18efb4ac5..fdab19de2 100644 --- a/tests/svs/orchestrators/vamana.cpp +++ b/tests/svs/orchestrators/vamana.cpp @@ -107,3 +107,39 @@ CATCH_TEST_CASE("Vamana Build", "[managers][vamana][build]") { } } } + +CATCH_TEST_CASE("Vamana Memory Usage", "[managers][vamana]") { + auto distance = svs::distance::DistanceL2(); + auto expected_result = test_dataset::vamana::expected_build_results( + distance, svsbenchmark::Uncompressed(svs::DataType::float32) + ); + auto build_params = expected_result.build_parameters_.value(); + size_t num_threads = 2; + + auto data = svs::data::SimpleData::load(test_dataset::data_svs_file()); + const size_t full_size = data.size(); + + // Build a full index and assert the reported allocation is non-zero. + svs::Vamana full = svs::Vamana::build( + build_params, + svs::data::SimpleData::load(test_dataset::data_svs_file()), + distance, + num_threads + ); + const size_t full_usage = full.get_memory_breakdown().total(); + CATCH_REQUIRE(full_usage > 0); + + // Monotonicity: an index built over fewer vectors must allocate fewer bytes. + const size_t half_size = full_size / 2; + CATCH_REQUIRE(half_size > 0); + auto half_data = svs::data::SimpleData(half_size, data.dimensions()); + for (size_t i = 0; i < half_size; ++i) { + half_data.set_datum(i, data.get_datum(i)); + } + svs::Vamana half = svs::Vamana::build( + build_params, std::move(half_data), distance, num_threads + ); + const size_t half_usage = half.get_memory_breakdown().total(); + CATCH_REQUIRE(half_usage > 0); + CATCH_REQUIRE(full_usage > half_usage); +}