diff --git a/docs/navix-poc.md b/docs/navix-poc.md index e0aeed263..002299b6f 100644 --- a/docs/navix-poc.md +++ b/docs/navix-poc.md @@ -49,3 +49,34 @@ unquantized, single-value FLOAT32 RAM HNSW. `test_navix` covers rejected seeds/branches, shared second-hop nodes, equal-distance batch continuation, strategy selection, timeout/reset, frontend filtering/reset, cached NO/MAYBE decisions, reset invalidation, isolation between query iterators, and backend growth. + +## Traversal experiments + +The default and `forceBlind` behavior are unchanged. Three opt-in controls can be +combined, except that `bridgeRejected` takes precedence over the two-hop controls: + +- `skipScanned`: skip second-hop adjacency lists already processed completely by this + iterator, including lists processed as direct neighbors. Partial lists remain eligible. + An additional byte per encountered internal-ID slot is retained across batches and + cleared on reset. As with visited nodes, completed lists are not reconsidered for + concurrent graph changes; compare these policies on a fully built, stable graph. +- `secondHopBudget`: maximum second-hop neighbor inspections per expansion (zero is + unlimited). Already-visited and rejected neighbors consume the budget too. This bounds + inspections, not total query work. Budget stops can reduce recall or exhaust the frontier; + there is no automatic retry/fallback. Partial lists restart from their beginning on a + later visit, so small budgets can repeatedly scan the same prefix. +- `bridgeRejected`: enqueue every direct neighbor not yet visited, including rejected nodes, + and disable second-hop expansion. Only MAYBE nodes enter result heaps and affect the + result bound. This uses one-hop graph navigation with filtered result admission; + it can cross longer rejected chains, but does not guarantee exact recall or recovery + from genuinely disconnected graph components. With no matches it may scan the whole + reachable graph, subject to the query timeout. The existing NaviX stopping rule is + retained: distance ties at a full result bound wait for the next batch (`>=`), unlike + the strict `>` comparison in ordinary HNSW. This can defer a rejected bridge and a closer + match behind it when the consumer stops after the first batch. + +New counters report skipped complete second-hop lists, expansions stopped by the +inspection budget, and bridge-mode expansions. A budget exhaustion is counted only if +an additional inspection was prevented. The parameter/stat struct additions change the +C ABI; rebuild consumers with the matching headers and library. The flat frontend still +uses its existing callback filter and has no hop policy. diff --git a/src/VecSim/algorithms/hnsw/hnsw_batch_iterator.h b/src/VecSim/algorithms/hnsw/hnsw_batch_iterator.h index fcbef823c..034d2f38d 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_batch_iterator.h +++ b/src/VecSim/algorithms/hnsw/hnsw_batch_iterator.h @@ -35,12 +35,14 @@ class HNSW_BatchIterator : public VecSimBatchIterator { VecSimNavixParams navix{}; enum class NavixMatch : uint8_t { Unknown, No, Maybe }; vecsim_stl::vector navix_filter_cache; + vecsim_stl::vector navix_scanned; vecsim_stl::vector navix_direct, navix_second; vecsim_stl::vector navix_matches; vecsim_stl::vector> navix_branches; bool navixMayMatch(idType id); DistType navixDistance(idType id); void navixLinks(idType id, vecsim_stl::vector &out); + void navixMarkScanned(idType id); VecSimQueryReply_Code expandNavix(idType id); VecSimQueryReply_Code scanNavix(candidatesLabelsMaxHeap *top_candidates); @@ -86,9 +88,9 @@ HNSW_BatchIterator::HNSW_BatchIterator( : VecSimBatchIterator(query_vector, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)), index(index), depleted(false), navix_filter_cache(this->allocator), - navix_direct(this->allocator), navix_second(this->allocator), navix_matches(this->allocator), - navix_branches(this->allocator), top_candidates_extras(this->allocator), - candidates(this->allocator) { + navix_scanned(this->allocator), navix_direct(this->allocator), navix_second(this->allocator), + navix_matches(this->allocator), navix_branches(this->allocator), + top_candidates_extras(this->allocator), candidates(this->allocator) { if (queryParams) navix = queryParams->navix; @@ -147,9 +149,33 @@ void HNSW_BatchIterator::navixLinks(idType id, out.resize(count); } +template +void HNSW_BatchIterator::navixMarkScanned(idType id) { + if (!navix.skipScanned) + return; + if (id >= navix_scanned.size()) + navix_scanned.resize(size_t(id) + 1, 0); + navix_scanned[id] = 1; +} + template VecSimQueryReply_Code HNSW_BatchIterator::expandNavix(idType id) { navixLinks(id, navix_direct); + if (navix.bridgeRejected) { + if (navix.stats) { + ++navix.stats->expandedNodes; + ++navix.stats->bridgeExpansions; + } + for (idType neighbor : navix_direct) { + if (VECSIM_TIMEOUT(this->getTimeoutCtx())) + return VecSim_QueryReply_TimedOut; + if (hasVisitedNode(neighbor)) + continue; + visitNode(neighbor); + candidates.emplace(navixDistance(neighbor), neighbor); + } + return VecSim_QueryReply_OK; + } navix_matches.resize(navix_direct.size()); size_t passing = 0; for (size_t i = 0; i < navix_direct.size(); ++i) { @@ -189,28 +215,45 @@ VecSimQueryReply_Code HNSW_BatchIterator::expandNavix(idType ++admitted; } } + navixMarkScanned(id); if (onehop || admitted >= degree) return VecSim_QueryReply_OK; if (directed) std::sort(navix_branches.begin(), navix_branches.end()); + size_t inspected = 0; for (size_t i = 0; i < navix_direct.size() && admitted < degree; ++i) { if (VECSIM_TIMEOUT(this->getTimeoutCtx())) return VecSim_QueryReply_TimedOut; const idType branch = directed ? navix_branches[i].second : navix_direct[i]; + if (navix.skipScanned && branch < navix_scanned.size() && navix_scanned[branch]) { + if (navix.stats) + ++navix.stats->secondHopSkipped; + continue; + } navixLinks(branch, navix_second); - for (idType neighbor : navix_second) { + size_t j = 0; + for (; j < navix_second.size() && admitted < degree; ++j) { if (VECSIM_TIMEOUT(this->getTimeoutCtx())) return VecSim_QueryReply_TimedOut; + if (navix.secondHopBudget && inspected == navix.secondHopBudget) { + if (navix.stats) + ++navix.stats->secondHopBudgetExhausted; + return VecSim_QueryReply_OK; + } + ++inspected; if (navix.stats) ++navix.stats->secondHopVisits; + const idType neighbor = navix_second[j]; if (hasVisitedNode(neighbor) || !navixMayMatch(neighbor)) continue; visitNode(neighbor); candidates.emplace(navixDistance(neighbor), neighbor); - if (++admitted == degree) - break; + ++admitted; } + // A budget, admission limit, or timeout must not mark a partial list complete. + if (j == navix_second.size()) + navixMarkScanned(branch); } return VecSim_QueryReply_OK; } @@ -408,6 +451,7 @@ void HNSW_BatchIterator::reset() { this->resetResultsCount(); this->depleted = false; navix_filter_cache.clear(); + navix_scanned.clear(); // Reset the visited nodes handler. this->visited_tag = this->visited_list->getFreshTag(); diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 4800e1d64..cefd763a4 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -380,6 +380,9 @@ typedef struct { size_t directedExpansions; size_t secondHopVisits; size_t frontendCandidates; + size_t secondHopSkipped; + size_t secondHopBudgetExhausted; + size_t bridgeExpansions; } VecSimNavixStats; typedef struct { @@ -387,6 +390,9 @@ typedef struct { void *context; VecSimNavixStats *stats; bool forceBlind; + bool skipScanned; + size_t secondHopBudget; // Neighbor inspections per expansion; zero means unlimited. + bool bridgeRejected; // One-hop traversal through all nodes, filtered result admission. } VecSimNavixParams; /** diff --git a/tests/unit/test_navix.cpp b/tests/unit/test_navix.cpp index 9788022e7..44648396a 100644 --- a/tests/unit/test_navix.cpp +++ b/tests/unit/test_navix.cpp @@ -240,3 +240,82 @@ TEST(Navix, FlatFrontendFiltersAndResets) { VecSimBatchIterator_Free(it); VecSimIndex_Free(index); } + +TEST(Navix, SkipsCompletedNeighborhoodsAcrossBatchesAndResets) { + size_t baseline_visits = 0; + for (bool skip : {false, true}) { + Fixture f; + f.sparseGraph(); + f.params.navix.skipScanned = skip; + auto *it = f.iterator(); + EXPECT_EQ(f.drain(it), f.allowed); + if (!skip) { + baseline_visits = f.stats.secondHopVisits; + } else { + EXPECT_LT(f.stats.secondHopVisits, baseline_visits); + EXPECT_GT(f.stats.secondHopSkipped, 0u); + const auto visits = f.stats.secondHopVisits; + VecSimBatchIterator_Reset(it); + EXPECT_EQ(f.drain(it), f.allowed); + EXPECT_EQ(f.stats.secondHopVisits, 2 * visits); + } + VecSimBatchIterator_Free(it); + } +} + +TEST(Navix, AdmissionLimitDoesNotCompletePartialNeighborhood) { + Fixture f; + f.params.navix.forceBlind = true; + f.params.navix.skipScanned = true; + f.edges(0, {1, 2, 3, 4, 5, 6, 7, 8}); + f.edges(1, {9, 10}); + f.edges(2, {1}); + for (size_t i = 2; i <= 10; ++i) + f.accept(i); + auto *it = f.iterator(); + EXPECT_EQ(f.drain(it), f.allowed); + VecSimBatchIterator_Free(it); +} + +TEST(Navix, SecondHopBudgetBoundsInspectionsAndLeavesPartialListsEligible) { + Fixture f; + f.params.navix.forceBlind = true; + f.params.navix.skipScanned = true; + f.params.navix.secondHopBudget = 2; + f.edges(0, {1, 2}); + f.edges(1, {3, 4, 5}); + f.edges(2, {1}); + f.accept(2); + f.accept(4); + auto *it = f.iterator(); + EXPECT_EQ(f.drain(it), f.allowed); + // Both expansions reach the still-partial list through rejected node 1. + EXPECT_EQ(f.stats.secondHopBudgetExhausted, 2u); + EXPECT_EQ(f.stats.secondHopVisits, 4u); + EXPECT_LE(f.stats.secondHopVisits, f.stats.expandedNodes * f.params.navix.secondHopBudget); + VecSimBatchIterator_Reset(it); + EXPECT_EQ(f.drain(it), f.allowed); + VecSimBatchIterator_Free(it); +} + +TEST(Navix, OneHopBridgesRejectedChainsWithoutReturningThem) { + for (bool matches : {false, true}) { + Fixture f; + f.params.navix.bridgeRejected = true; + f.edges(0, {1}); + f.edges(1, {2, 0}); + f.edges(2, {3, 1}); + f.edges(3, {4, 2}); + if (matches) { + f.accept(3); + f.accept(4); + } + auto *it = f.iterator(); + EXPECT_EQ(f.drain(it), f.allowed); + EXPECT_GT(f.stats.bridgeExpansions, 0u); + EXPECT_EQ(f.stats.secondHopVisits, 0u); + VecSimBatchIterator_Reset(it); + EXPECT_EQ(f.drain(it), f.allowed); + VecSimBatchIterator_Free(it); + } +}