Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 87 additions & 27 deletions src/VecSim/algorithms/hnsw/graph_data.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

#include <cassert>
#include <algorithm>
#include <span>
#include <stdexcept>
#include "VecSim/utils/vec_utils.h"

// Amortized shrink thresholds for incoming edges vectors.
Expand All @@ -14,9 +16,19 @@ using candidatesList = vecsim_stl::vector<std::pair<DistType, idType>>;

typedef uint16_t linkListSize;

// The unidirectional incoming edges of one element at one level, stored as a single allocation of
// this header followed by the ids.
struct IncomingEdges {
uint32_t size;
uint32_t capacity;
idType ids[];
};

struct ElementLevelData {
// A list of ids that are pointing to the node where each edge is *unidirectional*
vecsim_stl::vector<idType> *incomingUnidirectionalEdges;
// The ids of the elements that point to this one where the edge is *unidirectional*, or null
// while there are none. Owned by this level data and freed with `freeIncomingEdges`. It is
// guarded by the element's lock, like the links.
IncomingEdges *incomingUnidirectionalEdges;
linkListSize numLinks;
// Flexible array member - https://en.wikipedia.org/wiki/Flexible_array_member
// Using this trick, we can have the links list as part of the ElementLevelData struct, and
Expand All @@ -27,17 +39,21 @@ struct ElementLevelData {
// all nesting structs.
idType links[];

explicit ElementLevelData(std::shared_ptr<VecSimAllocator> allocator)
: incomingUnidirectionalEdges(new(allocator) vecsim_stl::vector<idType>(allocator)),
numLinks(0) {}
ElementLevelData() : incomingUnidirectionalEdges(nullptr), numLinks(0) {}

linkListSize getNumLinks() const { return this->numLinks; }
idType getLinkAtPos(size_t pos) const {
assert(pos < numLinks);
return this->links[pos];
}
const vecsim_stl::vector<idType> &getIncomingEdges() const {
return *incomingUnidirectionalEdges;
std::span<const idType> getIncomingEdges() const {
if (!incomingUnidirectionalEdges) {
return {};
}
return {incomingUnidirectionalEdges->ids, incomingUnidirectionalEdges->size};
}
size_t getIncomingEdgesCapacity() const {
return incomingUnidirectionalEdges ? incomingUnidirectionalEdges->capacity : 0;
}
std::vector<idType> copyLinks() {
std::vector<idType> links_copy;
Expand Down Expand Up @@ -72,27 +88,71 @@ struct ElementLevelData {
assert(i < numLinks && "Corruption in HNSW index"); // node_id not found - error
numLinks--;
}
void newIncomingUnidirectionalEdge(idType node_id) {
this->incomingUnidirectionalEdges->push_back(node_id);
// Grows the capacity like `std::vector` does (doubling, starting at one).
void newIncomingUnidirectionalEdge(idType node_id, VecSimAllocator &allocator) {
size_t size = getIncomingEdges().size();
if (size == getIncomingEdgesCapacity()) {
resizeIncomingEdges(std::max<size_t>(1, 2 * size), allocator);
}
incomingUnidirectionalEdges->ids[incomingUnidirectionalEdges->size++] = node_id;
}
bool removeIncomingUnidirectionalEdgeIfExists(idType node_id) {
bool result = this->incomingUnidirectionalEdges->remove(node_id);

if (result) {
auto &vec = *this->incomingUnidirectionalEdges;
if (vec.capacity() > INCOMING_EDGES_SHRINK_RATIO * vec.size()) {
vec.shrink_to_fit();
}
// Removes node_id by moving the last id into its place. Shrinks the allocation when it is more
// than INCOMING_EDGES_SHRINK_RATIO times larger than needed, and frees it when it empties.
bool removeIncomingUnidirectionalEdgeIfExists(idType node_id, VecSimAllocator &allocator) {
auto edges = getIncomingEdges();
auto it = std::find(edges.begin(), edges.end(), node_id);
if (it == edges.end()) {
return false;
}

return result;
incomingUnidirectionalEdges->ids[it - edges.begin()] = edges.back();
size_t size = --incomingUnidirectionalEdges->size;
if (getIncomingEdgesCapacity() > INCOMING_EDGES_SHRINK_RATIO * size) {
resizeIncomingEdges(size, allocator);
}
return true;
}
void swapNodeIdInIncomingEdges(idType id_before, idType id_after) {
auto it = std::find(this->incomingUnidirectionalEdges->begin(),
this->incomingUnidirectionalEdges->end(), id_before);
auto edges = getIncomingEdges();
auto it = std::find(edges.begin(), edges.end(), id_before);
// This should always succeed
assert(it != this->incomingUnidirectionalEdges->end());
*it = id_after;
assert(it != edges.end());
incomingUnidirectionalEdges->ids[it - edges.begin()] = id_after;
}
// Makes room for at least `capacity` incoming edges in one allocation.
void reserveIncomingEdges(size_t capacity, VecSimAllocator &allocator) {
if (capacity > getIncomingEdgesCapacity()) {
resizeIncomingEdges(capacity, allocator);
}
}
void shrinkIncomingEdgesToFit(VecSimAllocator &allocator) {
if (getIncomingEdgesCapacity() > getIncomingEdges().size()) {
resizeIncomingEdges(getIncomingEdges().size(), allocator);
}
}
void freeIncomingEdges(VecSimAllocator &allocator) {
allocator.free_allocation(incomingUnidirectionalEdges);
incomingUnidirectionalEdges = nullptr;
}

private:
// Sets the capacity to `capacity` (at least the current size), keeping the ids.
void resizeIncomingEdges(size_t capacity, VecSimAllocator &allocator) {
assert(capacity >= getIncomingEdges().size());
if (capacity == 0) {
freeIncomingEdges(allocator);
return;
}
bool was_empty = incomingUnidirectionalEdges == nullptr;
auto *resized = static_cast<IncomingEdges *>(allocator.reallocate(
incomingUnidirectionalEdges, sizeof(IncomingEdges) + capacity * sizeof(idType)));
if (resized == nullptr) {
throw std::runtime_error("VecSim index low memory error");
}
if (was_empty) {
resized->size = 0;
}
resized->capacity = capacity;
incomingUnidirectionalEdges = resized;
}
};

Expand All @@ -103,24 +163,24 @@ struct ElementGraphData {

ElementGraphData(size_t maxLevel, size_t high_level_size,
std::shared_ptr<VecSimAllocator> allocator)
: toplevel(maxLevel), others(nullptr), level0(allocator) {
: toplevel(maxLevel), others(nullptr), level0() {
if (toplevel > 0) {
others = (ElementLevelData *)allocator->callocate(high_level_size * toplevel);
if (others == nullptr) {
throw std::runtime_error("VecSim index low memory error");
}
for (size_t i = 0; i < maxLevel; i++) {
new ((char *)others + i * high_level_size) ElementLevelData(allocator);
new ((char *)others + i * high_level_size) ElementLevelData();
}
}
}
~ElementGraphData() = delete; // should be destroyed using `destroy'

void destroy(size_t levelDataSize, std::shared_ptr<VecSimAllocator> allocator) {
delete this->level0.incomingUnidirectionalEdges;
this->level0.freeIncomingEdges(*allocator);
ElementLevelData *cur_ld = this->others;
for (size_t i = 0; i < this->toplevel; i++) {
delete cur_ld->incomingUnidirectionalEdges;
cur_ld->freeIncomingEdges(*allocator);
cur_ld = reinterpret_cast<ElementLevelData *>(reinterpret_cast<char *>(cur_ld) +
levelDataSize);
}
Expand Down
33 changes: 19 additions & 14 deletions src/VecSim/algorithms/hnsw/hnsw.h
Original file line number Diff line number Diff line change
Expand Up @@ -1058,7 +1058,7 @@ void HNSWIndex<DataType, DistType>::revisitNeighborConnections(
neighbor_level.setLinkAtPos(neighbour_neighbours_idx++, new_node_id);
} else {
// unidirectional connection - put the new node in the neighbour's incoming edges.
neighbor_level.newIncomingUnidirectionalEdge(new_node_id);
neighbor_level.newIncomingUnidirectionalEdge(new_node_id, *this->allocator);
}
}
// Done updating the neighbor's neighbors.
Expand Down Expand Up @@ -1207,8 +1207,9 @@ void HNSWIndex<DataType, DistType>::repairConnectionsForDeletion(
// otherwise, edge turned from bidirectional to one directional,
// and it should be saved in the neighbor's incoming edges.
auto &node_level_data = getElementLevelData(node_id, level);
if (!node_level_data.removeIncomingUnidirectionalEdgeIfExists(neighbour_id)) {
neighbor_level.newIncomingUnidirectionalEdge(node_id);
if (!node_level_data.removeIncomingUnidirectionalEdgeIfExists(neighbour_id,
*this->allocator)) {
neighbor_level.newIncomingUnidirectionalEdge(node_id, *this->allocator);
}
}
}
Expand All @@ -1229,15 +1230,16 @@ void HNSWIndex<DataType, DistType>::repairConnectionsForDeletion(
if (node_level.getLinkAtPos(j) == neighbour_id) {
// Swap the last element with the current one (equivalent to removing the
// neighbor from the list) - this should always succeed and return true.
bool res = neighbor_level.removeIncomingUnidirectionalEdgeIfExists(node_id);
bool res = neighbor_level.removeIncomingUnidirectionalEdgeIfExists(
node_id, *this->allocator);
(void)res;
assert(res && "The edge should be in the incoming unidirectional edges");
bidirectional_edge = true;
break;
}
}
if (!bidirectional_edge) {
node_level.newIncomingUnidirectionalEdge(neighbour_id);
node_level.newIncomingUnidirectionalEdge(neighbour_id, *this->allocator);
}
}
}
Expand Down Expand Up @@ -1612,8 +1614,8 @@ void HNSWIndex<DataType, DistType>::mutuallyUpdateForRepairedNode(
// we add it to the unidirectional edges set. Note: we assume that all updates occur
// mutually and atomically, then can rely on this assumption.
auto &chosen_node_level_data = getElementLevelData(chosen_id, level);
if (!node_level.removeIncomingUnidirectionalEdgeIfExists(chosen_id)) {
chosen_node_level_data.newIncomingUnidirectionalEdge(node_id);
if (!node_level.removeIncomingUnidirectionalEdgeIfExists(chosen_id, *this->allocator)) {
chosen_node_level_data.newIncomingUnidirectionalEdge(node_id, *this->allocator);
}
}
// Done updating the node's neighbors.
Expand Down Expand Up @@ -1741,8 +1743,8 @@ void HNSWIndex<DataType, DistType>::mutuallyRemoveNeighborAtPos(ElementLevelData
// neighbour's incoming edges set. Note: we assume that every update is performed atomically
// mutually, so it should be sufficient to look at the removed node's incoming edges set
// alone.
if (!removed_node_level.removeIncomingUnidirectionalEdgeIfExists(node_id)) {
node_level.newIncomingUnidirectionalEdge(removed_node);
if (!removed_node_level.removeIncomingUnidirectionalEdgeIfExists(node_id, *this->allocator)) {
node_level.newIncomingUnidirectionalEdge(removed_node, *this->allocator);
}
}

Expand Down Expand Up @@ -1885,7 +1887,8 @@ void HNSWIndex<DataType, DistType>::isolateDeletedElement(idType internalId) {
} else if (points_to_other) {
// Unidirectional out - the other side recorded it as an incoming edge.
level_data.removeLink(other_id);
bool res = other.removeIncomingUnidirectionalEdgeIfExists(internalId);
bool res =
other.removeIncomingUnidirectionalEdgeIfExists(internalId, *this->allocator);
(void)res;
assert(res && "The edge should be in the incoming unidirectional edges");
} else if (other_points_here) {
Expand All @@ -1894,7 +1897,8 @@ void HNSWIndex<DataType, DistType>::isolateDeletedElement(idType internalId) {
assert(isMarkedDeleted(other_id) &&
"a live element still points to a fully repaired deleted element");
other.removeLink(internalId);
bool res = level_data.removeIncomingUnidirectionalEdgeIfExists(other_id);
bool res =
level_data.removeIncomingUnidirectionalEdgeIfExists(other_id, *this->allocator);
(void)res;
assert(res && "The edge should be in the incoming unidirectional edges");
}
Expand Down Expand Up @@ -1930,7 +1934,8 @@ void HNSWIndex<DataType, DistType>::disposeElementData(idType internalId) {
// . (We know we will get here and remove this deleted id permanently.)
// However, upon asynchronous delete, this should always succeed since we do update
// the incoming edges in the mutual update even for deleted elements.
bool res = neighbour.removeIncomingUnidirectionalEdgeIfExists(internalId);
bool res =
neighbour.removeIncomingUnidirectionalEdgeIfExists(internalId, *this->allocator);
// Assert the logical condition of: is_marked_deleted(id) => res==True.
(void)res;
assert((!isMarkedDeleted(internalId) || res) && "The edge should be in the incoming "
Expand Down Expand Up @@ -2013,8 +2018,8 @@ void HNSWIndex<DataType, DistType>::repairConnectionsAndEntryPoint(
// incoming edges.
if (!bidirectional_edge) {
// This should always return true (remove should succeed).
bool res =
neighbor_level.removeIncomingUnidirectionalEdgeIfExists(element_internal_id);
bool res = neighbor_level.removeIncomingUnidirectionalEdgeIfExists(
element_internal_id, *this->allocator);
(void)res;
assert(res && "The edge should be in the incoming unidirectional edges");
}
Expand Down
1 change: 1 addition & 0 deletions src/VecSim/algorithms/hnsw/hnsw_base_tests_friends.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,4 @@ INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_updateVectorsTwiceBeforeIngesti
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_updateVectorsAsync_Test)
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_addRelabelAndUpdateTogetherUnderLoad_Test)
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_queryWhileUpdating_Test)
friend class HNSWMemoryDiagnostic_breakdown_Test;
12 changes: 6 additions & 6 deletions src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ HNSWIndexMetaData HNSWIndex<DataType, DistType>::checkIntegrity() const {
if (s.size() != cur.numLinks) {
return res;
}
incoming_edges_sets_sizes += cur.incomingUnidirectionalEdges->size();
incoming_edges_sets_sizes += cur.getIncomingEdges().size();
}
}
if (num_deleted != this->numMarkedDeleted) {
Expand Down Expand Up @@ -241,11 +241,11 @@ void HNSWIndex<DataType, DistType>::restoreLevel(std::ifstream &input, ElementLe
// Restore the incoming edges of the current element
unsigned int size;
readBinaryPOD(input, size);
data.incomingUnidirectionalEdges->reserve(size);
data.reserveIncomingEdges(size, *this->allocator);
idType id = INVALID_ID;
for (size_t i = 0; i < size; i++) {
readBinaryPOD(input, id);
data.incomingUnidirectionalEdges->push_back(id);
data.newIncomingUnidirectionalEdge(id, *this->allocator);
}
}

Expand Down Expand Up @@ -332,12 +332,12 @@ void HNSWIndex<DataType, DistType>::saveLevel(std::ofstream &output, ElementLeve
}

// Save the incoming edges of the current element
unsigned int size = data.incomingUnidirectionalEdges->size();
unsigned int size = data.getIncomingEdges().size();
writeBinaryPOD(output, size);
for (idType id : *data.incomingUnidirectionalEdges) {
for (idType id : data.getIncomingEdges()) {
writeBinaryPOD(output, id);
}

// Shrink the incoming edges vector for integrity check
data.incomingUnidirectionalEdges->shrink_to_fit();
data.shrinkIncomingEdgesToFit(*this->allocator);
}
1 change: 1 addition & 0 deletions src/VecSim/algorithms/hnsw/hnsw_single_tests_friends.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ friend class BF16HNSWTest_testSizeEstimation_Test;
friend class BF16TieredTest_testSizeEstimation_Test;
friend class FP16HNSWTest_testSizeEstimation_Test;
friend class FP16TieredTest_testSizeEstimation_Test;
friend class HNSWMemoryDiagnostic_breakdown_Test;
18 changes: 12 additions & 6 deletions src/VecSim/index_factories/hnsw_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -291,12 +291,18 @@ size_t EstimateElementSize(const HNSWParams *params, bool with_mean) {
size_t size_meta_data = sizeof(tag_t) + sizeof(ElementMetaData) +
sizeof(vecsim_stl::one_byte_mutex) + size_label_lookup_entry;

/* Disclaimer: we are neglecting two additional factors that consume memory:
* 1. The overall bucket size in labels_lookup hash table is usually higher than the number of
* requested buckets (which is the index capacity), and it is auto selected according to the
* hashing policy and the max load factor.
* 2. The incoming edges that aren't bidirectional are stored in a dynamic array
* (vecsim_stl::vector) Those edges' memory *is omitted completely* from this estimation.
/* This is the memory one more element of index capacity takes, which is what growing the
* index by a block allocates per element. Storing a vector in that capacity allocates more,
* which this estimation leaves out:
* 1. Its node in the labels lookup hash table (a next pointer, the label and the id, plus the
* allocator header).
* 2. Its incoming unidirectional edges, one allocation per level that has any. Most elements
* have none at level 0; on average they take about M bytes per element.
* 3. Its links in the levels above level 0, if it has any (levelDataSize per level, plus the
* allocator header).
* Also, the bucket count of the labels lookup hash table is usually higher than the requested
* number of buckets (the index capacity), as it is selected by the hashing policy and the max
* load factor.
*/
return size_meta_data + size_total_data_per_element;
}
Expand Down
9 changes: 4 additions & 5 deletions tests/benchmark/index_internals/bm_incoming_edges.h
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ class BM_IncomingEdgesBase : public benchmark::Fixture {

// Measures ghost memory across all incoming edges vectors in the HNSW graph.
// Ghost memory = sum of (capacity - size) * sizeof(idType) for every
// incomingUnidirectionalEdges vector across all nodes and all levels.
// incoming-edge list across all nodes and all levels.
// Reports scalar stats via benchmark state counters and prints distribution
// details (top-10, percentiles) to stdout.
void measure_ghost_memory(benchmark::State &state, int iteration = -1,
Expand All @@ -133,9 +133,8 @@ class BM_IncomingEdgesBase : public benchmark::Fixture {
auto *graph_data = hnsw_->getGraphDataByInternalId(id);
for (size_t level = 0; level <= graph_data->toplevel; level++) {
auto &level_data = hnsw_->getElementLevelData(graph_data, level);
auto &incoming = level_data.getIncomingEdges();
size_t sz = incoming.size();
size_t cap = incoming.capacity();
size_t sz = level_data.getIncomingEdges().size();
size_t cap = level_data.getIncomingEdgesCapacity();

total_used_bytes += sz * sizeof(idType);
total_alloc_bytes += cap * sizeof(idType);
Expand Down Expand Up @@ -201,7 +200,7 @@ class BM_IncomingEdgesBase : public benchmark::Fixture {
auto *graph_data = hnsw_->getGraphDataByInternalId(id);
for (size_t level = 0; level <= graph_data->toplevel; level++) {
auto &level_data = hnsw_->getElementLevelData(graph_data, level);
level_data.incomingUnidirectionalEdges->shrink_to_fit();
level_data.shrinkIncomingEdgesToFit(*hnsw_->getAllocator());
}
}
}
Expand Down
Loading
Loading