From 3ab89d7c9e7ba6e8f73b285785b01d66c80abd50 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Thu, 12 Nov 2026 15:34:42 +0200 Subject: [PATCH] HNSW: allocate incoming edge lists lazily in one compact block Every element level used to own a heap allocated vecsim_stl::vector for its incoming unidirectional edges, even when the list was empty (87% of level 0 lists on DBpedia 100k, M=16). Each level now holds a pointer to a single allocation (size, capacity, ids) that stays null while the list is empty. Growth and shrink happen at the same points and to the same capacities as before, so the graph and the serialized format are unchanged. On DBpedia 100k x 768, M=16, efC=100 the allocator reports 75.4 bytes less per vector (3390.97 to 3315.60), with identical links, incoming lists, top-10 query results and recall. Index files saved before this change still load. EstimateElementSize keeps its per-block meaning; its comment now lists what a stored vector allocates beyond it. Add a diagnostic test (HNSWMemoryDiagnostic.breakdown) that prints the struct sizes and, given a dataset via VECSIM_MEM_DIAG_NPY, a per-element memory breakdown and graph fingerprints. Co-Authored-By: Claude Opus 5.5 --- src/VecSim/algorithms/hnsw/graph_data.h | 114 +++++-- src/VecSim/algorithms/hnsw/hnsw.h | 33 +- .../algorithms/hnsw/hnsw_base_tests_friends.h | 1 + .../algorithms/hnsw/hnsw_serializer_impl.h | 12 +- .../hnsw/hnsw_single_tests_friends.h | 1 + src/VecSim/index_factories/hnsw_factory.cpp | 18 +- .../index_internals/bm_incoming_edges.h | 9 +- tests/unit/CMakeLists.txt | 2 +- tests/unit/test_allocator.cpp | 33 +- tests/unit/test_hnsw.cpp | 2 +- tests/unit/test_hnsw_mem_diag.cpp | 291 ++++++++++++++++++ 11 files changed, 437 insertions(+), 79 deletions(-) create mode 100644 tests/unit/test_hnsw_mem_diag.cpp diff --git a/src/VecSim/algorithms/hnsw/graph_data.h b/src/VecSim/algorithms/hnsw/graph_data.h index ece03ce20..630b7b10b 100644 --- a/src/VecSim/algorithms/hnsw/graph_data.h +++ b/src/VecSim/algorithms/hnsw/graph_data.h @@ -3,6 +3,8 @@ #include #include +#include +#include #include "VecSim/utils/vec_utils.h" // Amortized shrink thresholds for incoming edges vectors. @@ -14,9 +16,19 @@ using candidatesList = vecsim_stl::vector>; typedef uint16_t linkListSize; +// The unidirectional incoming edges of one element at one level, stored as a single allocation of +// this header followed by the ids. +struct IncomingEdges { + uint32_t size; + uint32_t capacity; + idType ids[]; +}; + struct ElementLevelData { - // A list of ids that are pointing to the node where each edge is *unidirectional* - vecsim_stl::vector *incomingUnidirectionalEdges; + // The ids of the elements that point to this one where the edge is *unidirectional*, or null + // while there are none. Owned by this level data and freed with `freeIncomingEdges`. It is + // guarded by the element's lock, like the links. + IncomingEdges *incomingUnidirectionalEdges; linkListSize numLinks; // Flexible array member - https://en.wikipedia.org/wiki/Flexible_array_member // Using this trick, we can have the links list as part of the ElementLevelData struct, and @@ -27,17 +39,21 @@ struct ElementLevelData { // all nesting structs. idType links[]; - explicit ElementLevelData(std::shared_ptr allocator) - : incomingUnidirectionalEdges(new(allocator) vecsim_stl::vector(allocator)), - numLinks(0) {} + ElementLevelData() : incomingUnidirectionalEdges(nullptr), numLinks(0) {} linkListSize getNumLinks() const { return this->numLinks; } idType getLinkAtPos(size_t pos) const { assert(pos < numLinks); return this->links[pos]; } - const vecsim_stl::vector &getIncomingEdges() const { - return *incomingUnidirectionalEdges; + std::span getIncomingEdges() const { + if (!incomingUnidirectionalEdges) { + return {}; + } + return {incomingUnidirectionalEdges->ids, incomingUnidirectionalEdges->size}; + } + size_t getIncomingEdgesCapacity() const { + return incomingUnidirectionalEdges ? incomingUnidirectionalEdges->capacity : 0; } std::vector copyLinks() { std::vector links_copy; @@ -72,27 +88,71 @@ struct ElementLevelData { assert(i < numLinks && "Corruption in HNSW index"); // node_id not found - error numLinks--; } - void newIncomingUnidirectionalEdge(idType node_id) { - this->incomingUnidirectionalEdges->push_back(node_id); + // Grows the capacity like `std::vector` does (doubling, starting at one). + void newIncomingUnidirectionalEdge(idType node_id, VecSimAllocator &allocator) { + size_t size = getIncomingEdges().size(); + if (size == getIncomingEdgesCapacity()) { + resizeIncomingEdges(std::max(1, 2 * size), allocator); + } + incomingUnidirectionalEdges->ids[incomingUnidirectionalEdges->size++] = node_id; } - bool removeIncomingUnidirectionalEdgeIfExists(idType node_id) { - bool result = this->incomingUnidirectionalEdges->remove(node_id); - - if (result) { - auto &vec = *this->incomingUnidirectionalEdges; - if (vec.capacity() > INCOMING_EDGES_SHRINK_RATIO * vec.size()) { - vec.shrink_to_fit(); - } + // Removes node_id by moving the last id into its place. Shrinks the allocation when it is more + // than INCOMING_EDGES_SHRINK_RATIO times larger than needed, and frees it when it empties. + bool removeIncomingUnidirectionalEdgeIfExists(idType node_id, VecSimAllocator &allocator) { + auto edges = getIncomingEdges(); + auto it = std::find(edges.begin(), edges.end(), node_id); + if (it == edges.end()) { + return false; } - - return result; + incomingUnidirectionalEdges->ids[it - edges.begin()] = edges.back(); + size_t size = --incomingUnidirectionalEdges->size; + if (getIncomingEdgesCapacity() > INCOMING_EDGES_SHRINK_RATIO * size) { + resizeIncomingEdges(size, allocator); + } + return true; } void swapNodeIdInIncomingEdges(idType id_before, idType id_after) { - auto it = std::find(this->incomingUnidirectionalEdges->begin(), - this->incomingUnidirectionalEdges->end(), id_before); + auto edges = getIncomingEdges(); + auto it = std::find(edges.begin(), edges.end(), id_before); // This should always succeed - assert(it != this->incomingUnidirectionalEdges->end()); - *it = id_after; + assert(it != edges.end()); + incomingUnidirectionalEdges->ids[it - edges.begin()] = id_after; + } + // Makes room for at least `capacity` incoming edges in one allocation. + void reserveIncomingEdges(size_t capacity, VecSimAllocator &allocator) { + if (capacity > getIncomingEdgesCapacity()) { + resizeIncomingEdges(capacity, allocator); + } + } + void shrinkIncomingEdgesToFit(VecSimAllocator &allocator) { + if (getIncomingEdgesCapacity() > getIncomingEdges().size()) { + resizeIncomingEdges(getIncomingEdges().size(), allocator); + } + } + void freeIncomingEdges(VecSimAllocator &allocator) { + allocator.free_allocation(incomingUnidirectionalEdges); + incomingUnidirectionalEdges = nullptr; + } + +private: + // Sets the capacity to `capacity` (at least the current size), keeping the ids. + void resizeIncomingEdges(size_t capacity, VecSimAllocator &allocator) { + assert(capacity >= getIncomingEdges().size()); + if (capacity == 0) { + freeIncomingEdges(allocator); + return; + } + bool was_empty = incomingUnidirectionalEdges == nullptr; + auto *resized = static_cast(allocator.reallocate( + incomingUnidirectionalEdges, sizeof(IncomingEdges) + capacity * sizeof(idType))); + if (resized == nullptr) { + throw std::runtime_error("VecSim index low memory error"); + } + if (was_empty) { + resized->size = 0; + } + resized->capacity = capacity; + incomingUnidirectionalEdges = resized; } }; @@ -103,24 +163,24 @@ struct ElementGraphData { ElementGraphData(size_t maxLevel, size_t high_level_size, std::shared_ptr allocator) - : toplevel(maxLevel), others(nullptr), level0(allocator) { + : toplevel(maxLevel), others(nullptr), level0() { if (toplevel > 0) { others = (ElementLevelData *)allocator->callocate(high_level_size * toplevel); if (others == nullptr) { throw std::runtime_error("VecSim index low memory error"); } for (size_t i = 0; i < maxLevel; i++) { - new ((char *)others + i * high_level_size) ElementLevelData(allocator); + new ((char *)others + i * high_level_size) ElementLevelData(); } } } ~ElementGraphData() = delete; // should be destroyed using `destroy' void destroy(size_t levelDataSize, std::shared_ptr allocator) { - delete this->level0.incomingUnidirectionalEdges; + this->level0.freeIncomingEdges(*allocator); ElementLevelData *cur_ld = this->others; for (size_t i = 0; i < this->toplevel; i++) { - delete cur_ld->incomingUnidirectionalEdges; + cur_ld->freeIncomingEdges(*allocator); cur_ld = reinterpret_cast(reinterpret_cast(cur_ld) + levelDataSize); } diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h index 11b4247ef..2f7d96778 100644 --- a/src/VecSim/algorithms/hnsw/hnsw.h +++ b/src/VecSim/algorithms/hnsw/hnsw.h @@ -1058,7 +1058,7 @@ void HNSWIndex::revisitNeighborConnections( neighbor_level.setLinkAtPos(neighbour_neighbours_idx++, new_node_id); } else { // unidirectional connection - put the new node in the neighbour's incoming edges. - neighbor_level.newIncomingUnidirectionalEdge(new_node_id); + neighbor_level.newIncomingUnidirectionalEdge(new_node_id, *this->allocator); } } // Done updating the neighbor's neighbors. @@ -1207,8 +1207,9 @@ void HNSWIndex::repairConnectionsForDeletion( // otherwise, edge turned from bidirectional to one directional, // and it should be saved in the neighbor's incoming edges. auto &node_level_data = getElementLevelData(node_id, level); - if (!node_level_data.removeIncomingUnidirectionalEdgeIfExists(neighbour_id)) { - neighbor_level.newIncomingUnidirectionalEdge(node_id); + if (!node_level_data.removeIncomingUnidirectionalEdgeIfExists(neighbour_id, + *this->allocator)) { + neighbor_level.newIncomingUnidirectionalEdge(node_id, *this->allocator); } } } @@ -1229,7 +1230,8 @@ void HNSWIndex::repairConnectionsForDeletion( if (node_level.getLinkAtPos(j) == neighbour_id) { // Swap the last element with the current one (equivalent to removing the // neighbor from the list) - this should always succeed and return true. - bool res = neighbor_level.removeIncomingUnidirectionalEdgeIfExists(node_id); + bool res = neighbor_level.removeIncomingUnidirectionalEdgeIfExists( + node_id, *this->allocator); (void)res; assert(res && "The edge should be in the incoming unidirectional edges"); bidirectional_edge = true; @@ -1237,7 +1239,7 @@ void HNSWIndex::repairConnectionsForDeletion( } } if (!bidirectional_edge) { - node_level.newIncomingUnidirectionalEdge(neighbour_id); + node_level.newIncomingUnidirectionalEdge(neighbour_id, *this->allocator); } } } @@ -1612,8 +1614,8 @@ void HNSWIndex::mutuallyUpdateForRepairedNode( // we add it to the unidirectional edges set. Note: we assume that all updates occur // mutually and atomically, then can rely on this assumption. auto &chosen_node_level_data = getElementLevelData(chosen_id, level); - if (!node_level.removeIncomingUnidirectionalEdgeIfExists(chosen_id)) { - chosen_node_level_data.newIncomingUnidirectionalEdge(node_id); + if (!node_level.removeIncomingUnidirectionalEdgeIfExists(chosen_id, *this->allocator)) { + chosen_node_level_data.newIncomingUnidirectionalEdge(node_id, *this->allocator); } } // Done updating the node's neighbors. @@ -1741,8 +1743,8 @@ void HNSWIndex::mutuallyRemoveNeighborAtPos(ElementLevelData // neighbour's incoming edges set. Note: we assume that every update is performed atomically // mutually, so it should be sufficient to look at the removed node's incoming edges set // alone. - if (!removed_node_level.removeIncomingUnidirectionalEdgeIfExists(node_id)) { - node_level.newIncomingUnidirectionalEdge(removed_node); + if (!removed_node_level.removeIncomingUnidirectionalEdgeIfExists(node_id, *this->allocator)) { + node_level.newIncomingUnidirectionalEdge(removed_node, *this->allocator); } } @@ -1885,7 +1887,8 @@ void HNSWIndex::isolateDeletedElement(idType internalId) { } else if (points_to_other) { // Unidirectional out - the other side recorded it as an incoming edge. level_data.removeLink(other_id); - bool res = other.removeIncomingUnidirectionalEdgeIfExists(internalId); + bool res = + other.removeIncomingUnidirectionalEdgeIfExists(internalId, *this->allocator); (void)res; assert(res && "The edge should be in the incoming unidirectional edges"); } else if (other_points_here) { @@ -1894,7 +1897,8 @@ void HNSWIndex::isolateDeletedElement(idType internalId) { assert(isMarkedDeleted(other_id) && "a live element still points to a fully repaired deleted element"); other.removeLink(internalId); - bool res = level_data.removeIncomingUnidirectionalEdgeIfExists(other_id); + bool res = + level_data.removeIncomingUnidirectionalEdgeIfExists(other_id, *this->allocator); (void)res; assert(res && "The edge should be in the incoming unidirectional edges"); } @@ -1930,7 +1934,8 @@ void HNSWIndex::disposeElementData(idType internalId) { // . (We know we will get here and remove this deleted id permanently.) // However, upon asynchronous delete, this should always succeed since we do update // the incoming edges in the mutual update even for deleted elements. - bool res = neighbour.removeIncomingUnidirectionalEdgeIfExists(internalId); + bool res = + neighbour.removeIncomingUnidirectionalEdgeIfExists(internalId, *this->allocator); // Assert the logical condition of: is_marked_deleted(id) => res==True. (void)res; assert((!isMarkedDeleted(internalId) || res) && "The edge should be in the incoming " @@ -2013,8 +2018,8 @@ void HNSWIndex::repairConnectionsAndEntryPoint( // incoming edges. if (!bidirectional_edge) { // This should always return true (remove should succeed). - bool res = - neighbor_level.removeIncomingUnidirectionalEdgeIfExists(element_internal_id); + bool res = neighbor_level.removeIncomingUnidirectionalEdgeIfExists( + element_internal_id, *this->allocator); (void)res; assert(res && "The edge should be in the incoming unidirectional edges"); } diff --git a/src/VecSim/algorithms/hnsw/hnsw_base_tests_friends.h b/src/VecSim/algorithms/hnsw/hnsw_base_tests_friends.h index 37566aa9f..529766e73 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_base_tests_friends.h +++ b/src/VecSim/algorithms/hnsw/hnsw_base_tests_friends.h @@ -47,3 +47,4 @@ INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_updateVectorsTwiceBeforeIngesti INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_updateVectorsAsync_Test) INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_addRelabelAndUpdateTogetherUnderLoad_Test) INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_queryWhileUpdating_Test) +friend class HNSWMemoryDiagnostic_breakdown_Test; diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h index 2865c5d81..6733c2aef 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h @@ -117,7 +117,7 @@ HNSWIndexMetaData HNSWIndex::checkIntegrity() const { if (s.size() != cur.numLinks) { return res; } - incoming_edges_sets_sizes += cur.incomingUnidirectionalEdges->size(); + incoming_edges_sets_sizes += cur.getIncomingEdges().size(); } } if (num_deleted != this->numMarkedDeleted) { @@ -241,11 +241,11 @@ void HNSWIndex::restoreLevel(std::ifstream &input, ElementLe // Restore the incoming edges of the current element unsigned int size; readBinaryPOD(input, size); - data.incomingUnidirectionalEdges->reserve(size); + data.reserveIncomingEdges(size, *this->allocator); idType id = INVALID_ID; for (size_t i = 0; i < size; i++) { readBinaryPOD(input, id); - data.incomingUnidirectionalEdges->push_back(id); + data.newIncomingUnidirectionalEdge(id, *this->allocator); } } @@ -332,12 +332,12 @@ void HNSWIndex::saveLevel(std::ofstream &output, ElementLeve } // Save the incoming edges of the current element - unsigned int size = data.incomingUnidirectionalEdges->size(); + unsigned int size = data.getIncomingEdges().size(); writeBinaryPOD(output, size); - for (idType id : *data.incomingUnidirectionalEdges) { + for (idType id : data.getIncomingEdges()) { writeBinaryPOD(output, id); } // Shrink the incoming edges vector for integrity check - data.incomingUnidirectionalEdges->shrink_to_fit(); + data.shrinkIncomingEdgesToFit(*this->allocator); } diff --git a/src/VecSim/algorithms/hnsw/hnsw_single_tests_friends.h b/src/VecSim/algorithms/hnsw/hnsw_single_tests_friends.h index 9c27adead..eb8e5f03e 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_single_tests_friends.h +++ b/src/VecSim/algorithms/hnsw/hnsw_single_tests_friends.h @@ -19,3 +19,4 @@ friend class BF16HNSWTest_testSizeEstimation_Test; friend class BF16TieredTest_testSizeEstimation_Test; friend class FP16HNSWTest_testSizeEstimation_Test; friend class FP16TieredTest_testSizeEstimation_Test; +friend class HNSWMemoryDiagnostic_breakdown_Test; diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index 2d03fc233..bee70699e 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -291,12 +291,18 @@ size_t EstimateElementSize(const HNSWParams *params, bool with_mean) { size_t size_meta_data = sizeof(tag_t) + sizeof(ElementMetaData) + sizeof(vecsim_stl::one_byte_mutex) + size_label_lookup_entry; - /* Disclaimer: we are neglecting two additional factors that consume memory: - * 1. The overall bucket size in labels_lookup hash table is usually higher than the number of - * requested buckets (which is the index capacity), and it is auto selected according to the - * hashing policy and the max load factor. - * 2. The incoming edges that aren't bidirectional are stored in a dynamic array - * (vecsim_stl::vector) Those edges' memory *is omitted completely* from this estimation. + /* This is the memory one more element of index capacity takes, which is what growing the + * index by a block allocates per element. Storing a vector in that capacity allocates more, + * which this estimation leaves out: + * 1. Its node in the labels lookup hash table (a next pointer, the label and the id, plus the + * allocator header). + * 2. Its incoming unidirectional edges, one allocation per level that has any. Most elements + * have none at level 0; on average they take about M bytes per element. + * 3. Its links in the levels above level 0, if it has any (levelDataSize per level, plus the + * allocator header). + * Also, the bucket count of the labels lookup hash table is usually higher than the requested + * number of buckets (the index capacity), as it is selected by the hashing policy and the max + * load factor. */ return size_meta_data + size_total_data_per_element; } diff --git a/tests/benchmark/index_internals/bm_incoming_edges.h b/tests/benchmark/index_internals/bm_incoming_edges.h index ee0b31404..6509bcf9e 100644 --- a/tests/benchmark/index_internals/bm_incoming_edges.h +++ b/tests/benchmark/index_internals/bm_incoming_edges.h @@ -114,7 +114,7 @@ class BM_IncomingEdgesBase : public benchmark::Fixture { // Measures ghost memory across all incoming edges vectors in the HNSW graph. // Ghost memory = sum of (capacity - size) * sizeof(idType) for every - // incomingUnidirectionalEdges vector across all nodes and all levels. + // incoming-edge list across all nodes and all levels. // Reports scalar stats via benchmark state counters and prints distribution // details (top-10, percentiles) to stdout. void measure_ghost_memory(benchmark::State &state, int iteration = -1, @@ -133,9 +133,8 @@ class BM_IncomingEdgesBase : public benchmark::Fixture { auto *graph_data = hnsw_->getGraphDataByInternalId(id); for (size_t level = 0; level <= graph_data->toplevel; level++) { auto &level_data = hnsw_->getElementLevelData(graph_data, level); - auto &incoming = level_data.getIncomingEdges(); - size_t sz = incoming.size(); - size_t cap = incoming.capacity(); + size_t sz = level_data.getIncomingEdges().size(); + size_t cap = level_data.getIncomingEdgesCapacity(); total_used_bytes += sz * sizeof(idType); total_alloc_bytes += cap * sizeof(idType); @@ -201,7 +200,7 @@ class BM_IncomingEdgesBase : public benchmark::Fixture { auto *graph_data = hnsw_->getGraphDataByInternalId(id); for (size_t level = 0; level <= graph_data->toplevel; level++) { auto &level_data = hnsw_->getElementLevelData(graph_data, level); - level_data.incomingUnidirectionalEdges->shrink_to_fit(); + level_data.shrinkIncomingEdgesToFit(*hnsw_->getAllocator()); } } } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index fd8e18721..e6d1f43b2 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -34,7 +34,7 @@ else() include(${root}/cmake/x86_64InstructionFlags.cmake) endif() -add_executable(test_hnsw ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw.cpp test_hnsw_multi.cpp test_hnsw_tiered.cpp unit_test_utils.cpp) +add_executable(test_hnsw ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw.cpp test_hnsw_multi.cpp test_hnsw_tiered.cpp test_hnsw_mem_diag.cpp unit_test_utils.cpp) add_executable(test_hnsw_sq4 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_sq4.cpp test_sq4_distances.cpp test_sq4_components.cpp unit_test_utils.cpp) add_executable(test_hnsw_sq8 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_sq8.cpp unit_test_utils.cpp) add_executable(test_bruteforce ../utils/test_main_with_timeout.cpp test_bruteforce.cpp test_bruteforce_multi.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) diff --git a/tests/unit/test_allocator.cpp b/tests/unit/test_allocator.cpp index b5e4f76fb..edf2f4c00 100644 --- a/tests/unit/test_allocator.cpp +++ b/tests/unit/test_allocator.cpp @@ -424,11 +424,9 @@ TYPED_TEST(IndexAllocatorTest, testIncomingEdgesSet) { int allocation_delta = allocator->getAllocationSize() - before; size_t vec_max_level = hnswIndex->getGraphDataByInternalId(1)->toplevel; - // Expect the creation of an empty incoming edges set in every level (+ the allocator header - // overhead), and a single node in the labels' lookup hash table. - size_t expected_allocation_delta = - (vec_max_level + 1) * (sizeof(vecsim_stl::vector) + vecsimAllocationOverhead); - expected_allocation_delta += hashTableNodeSize; + // Expect a single node in the labels' lookup hash table. Empty incoming edges sets allocate + // nothing. + size_t expected_allocation_delta = hashTableNodeSize; // Account for allocating link lists for levels higher than 0, if exists. if (vec_max_level > 0) { @@ -462,16 +460,13 @@ TYPED_TEST(IndexAllocatorTest, testIncomingEdgesSet) { vec_max_level = hnswIndex->getGraphDataByInternalId(5)->toplevel; /* Compute the expected allocation delta: - * 1. empty incoming edges set in every level (+ allocator's header). - * 2. A node in the labels_lookup has table (+ allocator's header). If rehashing occurred, we + * 1. A node in the labels_lookup has table (+ allocator's header). If rehashing occurred, we * account also for the diff in the buckets size (each bucket has sizeof(size_t) overhead). - * 3. Account for allocating link lists for levels higher than 0, if exists. - * 4. Finally, expect an allocation of the data buffer in the incoming edges vector of vec1 due - * to the insertion, and the fact that vec1 will re-select its neighbours. + * 2. Account for allocating link lists for levels higher than 0, if exists. + * 3. Finally, expect the allocation of the incoming edges set of vec1 due to the insertion, + * and the fact that vec1 will re-select its neighbours. Empty sets allocate nothing. */ - expected_allocation_delta = - (vec_max_level + 1) * (sizeof(vecsim_stl::vector) + vecsimAllocationOverhead) + - hashTableNodeSize; + expected_allocation_delta = hashTableNodeSize; size_t buckets_diff = hnswIndex->labelLookup.bucket_count() - buckets_num_before; expected_allocation_delta += buckets_diff * sizeof(size_t); if (vec_max_level > 0) { @@ -479,11 +474,12 @@ TYPED_TEST(IndexAllocatorTest, testIncomingEdgesSet) { hnswIndex->levelDataSize * vec_max_level + vecsimAllocationOverhead; } - // Expect that the first element is pushed to the incoming edges vector of element 1 in level 0. - // Then, we account for the capacity of the buffer that is allocated for the vector data. + // Expect that the first element is pushed to the incoming edges set of element 1 in level 0, + // which allocates the set's header and room for its capacity. + ASSERT_EQ(hnswIndex->getElementLevelData(1, 0).getIncomingEdges().size(), 1); expected_allocation_delta += - hnswIndex->getElementLevelData(1, 0).incomingUnidirectionalEdges->capacity() * - sizeof(idType) + + sizeof(IncomingEdges) + + hnswIndex->getElementLevelData(1, 0).getIncomingEdgesCapacity() * sizeof(idType) + vecsimAllocationOverhead; ASSERT_EQ(allocation_delta, expected_allocation_delta); @@ -556,8 +552,7 @@ TYPED_TEST(IndexAllocatorTest, test_hnsw_reclaim_memory) { // Compute the expected memory allocation due to the last vector insertion. size_t vec_max_level = hnswIndex->getGraphDataByInternalId(block_size)->toplevel; - size_t last_vec_graph_data_mem = - (sizeof(vecsim_stl::vector) + vecsimAllocationOverhead) + hashTableNodeSize; + size_t last_vec_graph_data_mem = hashTableNodeSize; if (vec_max_level > 0) { last_vec_graph_data_mem += hnswIndex->levelDataSize * vec_max_level + vecsimAllocationOverhead; diff --git a/tests/unit/test_hnsw.cpp b/tests/unit/test_hnsw.cpp index cb687f032..ebe868139 100644 --- a/tests/unit/test_hnsw.cpp +++ b/tests/unit/test_hnsw.cpp @@ -2380,7 +2380,7 @@ TYPED_TEST(HNSWTest, isolateDeletedElementEdgeShapes) { auto *hnsw_index = this->CastToHNSW(index); ElementLevelData &isolated = hnsw_index->getElementLevelData(isolated_id, 0); isolated.removeLink(deleted_neighbour_id); - isolated.newIncomingUnidirectionalEdge(deleted_neighbour_id); + isolated.newIncomingUnidirectionalEdge(deleted_neighbour_id, *hnsw_index->getAllocator()); ASSERT_TRUE(hnsw_index->checkIntegrity().valid_state); hnsw_index->isolateDeletedElement(isolated_id); diff --git a/tests/unit/test_hnsw_mem_diag.cpp b/tests/unit/test_hnsw_mem_diag.cpp new file mode 100644 index 000000000..e6b5393f3 --- /dev/null +++ b/tests/unit/test_hnsw_mem_diag.cpp @@ -0,0 +1,291 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +// Diagnostic for the per-element memory of the in-RAM HNSW index that is not vector data. +// It always prints the real struct sizes. When VECSIM_MEM_DIAG_NPY points to a float32 .npy file +// (rows x dim, C order), it also builds an HNSW index over the first VECSIM_MEM_DIAG_N rows +// (default 100000) and prints the incoming-edge and link histograms, the allocator-reported index +// memory, a per-element breakdown, and a fingerprint of the graph (for before/after comparisons). +// Optional: VECSIM_MEM_DIAG_SAVE= saves the built index; VECSIM_MEM_DIAG_LOAD= loads an +// index from a file instead of building one (no allocator comparison then); +// VECSIM_MEM_DIAG_QUERIES / VECSIM_MEM_DIAG_TRUTH (float32 / int32 .npy) run top-10 queries at +// ef=100 and print a fingerprint of the results and the recall. + +#include "gtest/gtest.h" +#include "VecSim/vec_sim.h" +#include "VecSim/algorithms/hnsw/hnsw_single.h" +#include "VecSim/index_factories/hnsw_factory.h" +#include "unit_test_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// Minimal reader for a version 1 or 2 float32 .npy file. Returns the row count and fills dim. +template +std::vector LoadNpy(const char *path, const char *descr, size_t max_rows, size_t &rows, + size_t &dim) { + std::ifstream in(path, std::ios::binary); + if (!in) { + throw std::runtime_error(std::string("cannot open ") + path); + } + char magic[8]; + in.read(magic, 8); + if (std::memcmp(magic, "\x93NUMPY", 6) != 0) { + throw std::runtime_error("not an npy file"); + } + size_t header_len = 0; + if (magic[6] == 1) { + uint16_t len; + in.read(reinterpret_cast(&len), 2); + header_len = len; + } else { + uint32_t len; + in.read(reinterpret_cast(&len), 4); + header_len = len; + } + std::string header(header_len, '\0'); + in.read(header.data(), header_len); + if (header.find(descr) == std::string::npos || + header.find("'fortran_order': False") == std::string::npos) { + throw std::runtime_error(std::string("expected a C-order npy file of ") + descr); + } + auto shape_pos = header.find("'shape': ("); + size_t file_rows = 0; + std::sscanf(header.c_str() + shape_pos, "'shape': (%zu, %zu)", &file_rows, &dim); + rows = std::min(file_rows, max_rows); + std::vector data(rows * dim); + in.read(reinterpret_cast(data.data()), data.size() * sizeof(T)); + return data; +} + +uint64_t Mix(uint64_t h, uint64_t v) { + // FNV-1a over the 8 bytes of v. + for (int i = 0; i < 8; i++) { + h ^= (v >> (8 * i)) & 0xff; + h *= 1099511628211ULL; + } + return h; +} + +void PrintHistogram(const char *title, const std::map &hist, size_t total) { + printf("%s\n", title); + for (auto &[k, v] : hist) { + printf(" %4zu : %8zu (%5.2f%%)\n", k, v, 100.0 * v / total); + } +} + +} // namespace + +TEST(HNSWMemoryDiagnostic, breakdown) { + const size_t hdr = VecSimAllocator::getAllocationOverheadSize(); + printf("sizeof(ElementGraphData) = %zu\n", sizeof(ElementGraphData)); + printf("sizeof(ElementLevelData) = %zu\n", sizeof(ElementLevelData)); + printf("offsetof(ElementLevelData, links) = %zu\n", offsetof(ElementLevelData, links)); + printf("offsetof(ElementGraphData, level0) = %zu\n", offsetof(ElementGraphData, level0)); + printf("sizeof(IncomingEdges) = %zu\n", sizeof(IncomingEdges)); + printf("sizeof(std::vector) = %zu\n", + sizeof(std::vector>)); + printf("sizeof(VecsimBaseObject) = %zu\n", sizeof(VecsimBaseObject)); + printf("sizeof(ElementMetaData) = %zu\n", sizeof(ElementMetaData)); + printf("sizeof(one_byte_mutex) = %zu\n", sizeof(vecsim_stl::one_byte_mutex)); + printf("sizeof(tag_t) = %zu\n", sizeof(tag_t)); + printf("allocator header = %zu\n", hdr); + + const char *npy = std::getenv("VECSIM_MEM_DIAG_NPY"); + if (!npy) { + GTEST_SKIP() << "set VECSIM_MEM_DIAG_NPY to a float32 .npy file for the index part"; + } + size_t max_rows = 100000; + if (const char *n_env = std::getenv("VECSIM_MEM_DIAG_N")) { + max_rows = std::strtoull(n_env, nullptr, 10); + } + size_t n = 0, dim = 0; + std::vector data = LoadNpy(npy, "' *>(index); + ASSERT_NE(hnsw, nullptr); + + const size_t mem_empty = load_path ? 0 : index->getAllocationSize(); + if (load_path) { + hnsw->setEf(100); + printf("loaded %s\n", load_path); + } else { + for (size_t i = 0; i < n; i++) { + VecSimIndex_AddVector(index, data.data() + i * dim, i); + } + } + const size_t mem_full = index->getAllocationSize(); + ASSERT_EQ(VecSimIndex_IndexSize(index), n); + ASSERT_TRUE(hnsw->checkIntegrity().valid_state); + if (const char *save_path = std::getenv("VECSIM_MEM_DIAG_SAVE")) { + hnsw->saveIndex(save_path); + printf("saved %s\n", save_path); + } + + if (const char *q_path = std::getenv("VECSIM_MEM_DIAG_QUERIES")) { + size_t nq = 0, qdim = 0; + std::vector queries = LoadNpy(q_path, "' truth; + size_t tk = 0, tn = 0; + if (const char *t_path = std::getenv("VECSIM_MEM_DIAG_TRUTH")) { + truth = LoadNpy(t_path, "' links0_hist, in0_size_hist, in0_cap_hist, level_hist; + size_t in_objects = 0, in_nonempty = 0, in_elems = 0, in_cap = 0; + size_t in0_nonempty = 0, in0_elems = 0, in0_cap = 0; + size_t upper_levels = 0, upper_alloc = 0, links0_total = 0; + uint64_t links_fp = 1469598103934665603ULL, incoming_fp = 1469598103934665603ULL; + for (idType id = 0; id < n; id++) { + ElementGraphData *g = hnsw->getGraphDataByInternalId(id); + level_hist[g->toplevel]++; + if (g->toplevel > 0) { + upper_levels += g->toplevel; + upper_alloc += 1; + } + links_fp = Mix(links_fp, g->toplevel); + for (size_t l = 0; l <= g->toplevel; l++) { + ElementLevelData &ld = hnsw->getElementLevelData(g, l); + links_fp = Mix(links_fp, ld.getNumLinks()); + for (size_t j = 0; j < ld.getNumLinks(); j++) { + links_fp = Mix(links_fp, ld.getLinkAtPos(j)); + } + auto in = ld.getIncomingEdges(); + const size_t in_capacity = ld.getIncomingEdgesCapacity(); + std::vector sorted_in(in.begin(), in.end()); + std::sort(sorted_in.begin(), sorted_in.end()); + incoming_fp = Mix(incoming_fp, sorted_in.size()); + for (idType x : sorted_in) { + incoming_fp = Mix(incoming_fp, x); + } + in_objects++; + in_elems += in.size(); + in_cap += in_capacity; + in_nonempty += in_capacity > 0; + if (l == 0) { + links0_hist[ld.getNumLinks()]++; + links0_total += ld.getNumLinks(); + in0_size_hist[in.size()]++; + in0_cap_hist[in_capacity]++; + in0_elems += in.size(); + in0_cap += in_capacity; + in0_nonempty += in_capacity > 0; + } + } + } + + printf("\nindex: n=%zu, M=%zu, M0=%zu, elementGraphDataSize=%zu, levelDataSize=%zu\n", n, + hnsw->M, hnsw->M0, hnsw->elementGraphDataSize, hnsw->levelDataSize); + printf("graph fingerprint: links=%016llx incoming=%016llx\n", (unsigned long long)links_fp, + (unsigned long long)incoming_fp); + PrintHistogram("toplevel histogram:", level_hist, n); + PrintHistogram("level-0 link count histogram:", links0_hist, n); + printf("level-0 mean links: %.2f\n", (double)links0_total / n); + PrintHistogram("level-0 incoming-edge list size histogram:", in0_size_hist, n); + PrintHistogram("level-0 incoming-edge list capacity histogram:", in0_cap_hist, n); + printf("level-0 incoming lists: %zu, with a buffer %zu (%.2f%%), elements %zu, capacity %zu\n", + n, in0_nonempty, 100.0 * in0_nonempty / n, in0_elems, in0_cap); + printf("all-level incoming lists: %zu, with a buffer %zu, elements %zu, capacity %zu\n", + in_objects, in_nonempty, in_elems, in_cap); + + // Per-element breakdown from the structures, compared with the allocator. + const size_t blocks = hnsw->graphDataBlocks.size(); + const size_t cap = hnsw->maxElements; + const double N = (double)n; + struct Row { + const char *name; + double bytes; + }; + std::vector rows = { + {"graph record blocks (capacity x elementGraphDataSize)", + (double)cap * hnsw->elementGraphDataSize + blocks * hdr}, + {"incoming-edge lists (header + ids + allocator header)", + (double)in_nonempty * (sizeof(IncomingEdges) + hdr) + in_cap * sizeof(idType)}, + {"upper-level records", (double)upper_levels * hnsw->levelDataSize + upper_alloc * hdr}, + {"vector blocks (capacity x storedDataSize)", + (double)cap * hnsw->getStoredDataSize() + blocks * hdr}, + {"idToMetaData (capacity)", + (double)hnsw->idToMetaData.capacity() * sizeof(ElementMetaData)}, + {"elementLocks (capacity)", + (double)hnsw->elementLocks.capacity() * sizeof(vecsim_stl::one_byte_mutex)}, + }; + // Label map: rebuild an identical map on a fresh allocator and measure it. + { + auto alloc = VecSimAllocator::newVecsimAllocator(); + size_t before = alloc->getAllocationSize(); + { + vecsim_stl::unordered_map m(alloc); + m.reserve(cap); + for (size_t i = 0; i < n; i++) { + m[i] = i; + } + printf("label map: size %zu, bucket_count %zu (index map bucket_count %zu)\n", m.size(), + m.bucket_count(), hnsw->labelLookup.bucket_count()); + rows.push_back({"label map (replica)", (double)(alloc->getAllocationSize() - before)}); + } + } + double sum = 0; + printf("\n%-55s %12s %10s\n", "component", "bytes", "B/elem"); + for (auto &r : rows) { + printf("%-55s %12.0f %10.2f\n", r.name, r.bytes, r.bytes / N); + sum += r.bytes; + } + const double delta = (double)mem_full - mem_empty; + printf("%-55s %12.0f %10.2f\n", "sum of components", sum, sum / N); + printf("%-55s %12.0f %10.2f\n", "allocator: full - empty", delta, delta / N); + printf("%-55s %12zu %10.2f\n", "allocator: full", mem_full, mem_full / N); + printf("%-55s %12.0f %10.2f\n", "unexplained (full - empty - sum)", delta - sum, + (delta - sum) / N); + + VecSimIndex_Free(index); +}