From d84131bbedfcbf6393a03709e381184bea54b6af Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 30 Oct 2026 02:59:22 -0700 Subject: [PATCH 01/68] feat(distance): add quantized EMD and Chamfer distances for token-set vectors Distances for multi-vector retrieval in the style of GEM (arXiv:2603.20336) over a document's token set stored as one fixed-width vector of centroid ids ([n, ids..., padding]): - QEmd: exact earth mover's distance over a centroid-pair cost table, solved with POT's network simplex (vendored unmodified in third-party/pot). - QChamfer: quantized Chamfer distance with both sides quantized. - QChamferExactQuery: raw float query tokens against document centroids, with the per-query dot table built in fix_argument. The kernels are instantiated per ISA in multi-arch/x86 like L2 and IP. pre-commit skips the vendored headers so they stay byte-identical. Co-Authored-By: Claude Opus 5.5 --- .pre-commit-config.yaml | 3 + include/svs/core/distance/gem.h | 678 +++++++ include/svs/multi-arch/x86/avx2.cpp | 56 + include/svs/multi-arch/x86/avx512.cpp | 62 + include/svs/third-party/pot/README.md | 45 + include/svs/third-party/pot/core.h | 103 ++ .../svs/third-party/pot/full_bipartitegraph.h | 212 +++ .../third-party/pot/network_simplex_simple.h | 1552 +++++++++++++++++ 8 files changed, 2711 insertions(+) create mode 100644 include/svs/core/distance/gem.h create mode 100644 include/svs/third-party/pot/README.md create mode 100644 include/svs/third-party/pot/core.h create mode 100644 include/svs/third-party/pot/full_bipartitegraph.h create mode 100644 include/svs/third-party/pot/network_simplex_simple.h diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index baf697990..5886baccc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Vendored headers must stay byte-identical to upstream; include/svs/third-party/pot/README.md lists their hashes. +exclude: ^include/svs/third-party/pot/ + repos: - repo: https://github.com/pre-commit/mirrors-clang-format rev: "ea59a72ffc9a1ce6b79b02a8076d031aa7ea7805" # v15.0.7 diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h new file mode 100644 index 000000000..1937d74ed --- /dev/null +++ b/include/svs/core/distance/gem.h @@ -0,0 +1,678 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Vendored POT 0.9.5 network simplex solver (include/svs/third-party/pot/); not +// warning-clean under -Werror, so only the warnings it actually trips are silenced. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wreorder" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "svs/third-party/pot/network_simplex_simple.h" +#pragma GCC diagnostic pop + +#include "svs/core/distance/distance_core.h" +#include "svs/lib/avx_detection.h" +#include "svs/lib/preprocessor.h" +#include "svs/lib/threads.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__x86_64__) +#include +#endif + +namespace svs::distance { + +// Same node-index type POT's EMD_wrap instantiates the solver with (ot/lp/EMD.h). +using gem_node_id_type = unsigned int; + +/// Ground distance between two unit centroids the from_centroids() table is built from: +/// Cos is `max(0, 1 - )` (GEM Eq. 11), L2 is `sqrt(max(0, 2 - 2))`. +enum class Ground { Cos, L2 }; + +inline Ground parse_ground(std::string_view s) { + if (s == "cos") { + return Ground::Cos; + } + if (s == "l2") { + return Ground::L2; + } + throw std::invalid_argument( + "parse_ground: unsupported ground distance '" + std::string(s) + + "'; expected cos or l2" + ); +} + +/// Owns a dense K x K matrix of pairwise centroid distances, row-major. Built either by +/// loading a precomputed file or in-process from centroids (see from_centroids()). +struct CentroidTable { + size_t K; + std::unique_ptr data; + + CentroidTable(size_t k, std::vector table) + : K{k} + , data{std::make_unique(k * k)} { + if (table.size() != K * K) { + throw std::invalid_argument("CentroidTable: data size does not match K*K"); + } + std::copy(table.begin(), table.end(), data.get()); + } + + // The file has no header, so K comes from table.json via the caller. + static CentroidTable load(const std::filesystem::path& path, size_t k) { + auto expected_bytes = k * k * sizeof(float); + std::ifstream in(path, std::ios::binary); + if (!in) { + throw std::runtime_error("CentroidTable::load: cannot open " + path.string()); + } + in.seekg(0, std::ios::end); + auto actual_bytes = static_cast(in.tellg()); + if (actual_bytes != expected_bytes) { + throw std::runtime_error( + "CentroidTable::load: " + path.string() + " has " + + std::to_string(actual_bytes) + " bytes, expected " + + std::to_string(expected_bytes) + " for K=" + std::to_string(k) + ); + } + in.seekg(0, std::ios::beg); + // Uninitialized: the read below fills every element, so zero-filling first is + // wasted bandwidth on a buffer that can be hundreds of GiB. + CentroidTable result{k, uninitialized_tag{}}; + in.read( + reinterpret_cast(result.data.get()), + static_cast(expected_bytes) + ); + if (!in) { + throw std::runtime_error("CentroidTable::load: short read on " + path.string()); + } + return result; + } + + /// Computes T[i][j] = ground(c_i, c_j) from K row-major `dim`-wide centroids, + /// T[i][i] = 0. Parallelized over row blocks on `num_threads` threads; each thread's + /// first write to its row range is also its NUMA first-touch. + static CentroidTable from_centroids( + const float* centroids, size_t k, size_t dim, Ground ground, size_t num_threads + ) { + CentroidTable result{k, uninitialized_tag{}}; + float* out = result.data.get(); + svs::threads::DefaultThreadPool pool(num_threads); + + // Column block width: a block's centroid rows (col_block * dim * 4 bytes, 128 KiB + // at dim=128) stay resident in L2 while every row a thread owns sweeps over it. + constexpr size_t col_block = 256; + + svs::threads::parallel_for( + pool, + svs::threads::StaticPartition(k), + [&](const auto& row_ids, uint64_t /*tid*/) { + for (size_t i : row_ids) { + const float* ci = centroids + i * dim; + for (size_t jb = 0; jb < k; jb += col_block) { + size_t jmax = std::min(jb + col_block, k); + for (size_t j = jb; j < jmax; ++j) { + if (i == j) { + out[i * k + j] = 0.0F; + continue; + } + const float* cj = centroids + j * dim; + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += ci[d] * cj[d]; + } + out[i * k + j] = + ground == Ground::Cos + ? std::max(0.0F, 1.0F - dot) + : std::sqrt(std::max(0.0F, 2.0F - 2.0F * dot)); + } + } + } + } + ); + return result; + } + + float operator()(uint32_t i, uint32_t j) const { + return data[static_cast(i) * K + j]; + } + + private: + struct uninitialized_tag {}; + + // new[] with no parens default-initializes float (indeterminate value, no zero + // fill); ownership passes to unique_ptr immediately, so this stays exception-safe. + CentroidTable(size_t k, uninitialized_tag) + : K{k} + , data{std::make_unique_for_overwrite(k * k)} {} +}; + +/// Owns the raw K x dim fine-centroid vectors (row-major, not a pairwise table). Used by +/// QChamferExactQuery to build a per-query (K x query_tokens) dot table against the exact +/// float query, mirroring GEM's score_fine_centers (src/search/routing.cpp:55-61), which +/// never quantizes the query side. +struct FineCentroids { + size_t K; + size_t dim; + std::unique_ptr data; + + FineCentroids(size_t k, size_t dim_, const float* flat) + : K{k} + , dim{dim_} + , data{std::make_unique(k * dim_)} { + std::copy(flat, flat + k * dim_, data.get()); + } + + // Raw file of K rows of `dim` floats with no header, so K comes from the file size. + static FineCentroids load(const std::filesystem::path& path, size_t dim) { + std::ifstream in(path, std::ios::binary); + if (!in) { + throw std::runtime_error("FineCentroids::load: cannot open " + path.string()); + } + in.seekg(0, std::ios::end); + auto bytes = static_cast(in.tellg()); + auto row_bytes = dim * sizeof(float); + if (bytes % row_bytes != 0) { + throw std::runtime_error( + "FineCentroids::load: " + path.string() + " has " + std::to_string(bytes) + + " bytes, not a multiple of dim=" + std::to_string(dim) + " * 4 bytes" + ); + } + auto k = bytes / row_bytes; + std::vector flat(k * dim); + in.seekg(0, std::ios::beg); + in.read(reinterpret_cast(flat.data()), static_cast(bytes)); + if (!in) { + throw std::runtime_error("FineCentroids::load: short read on " + path.string()); + } + return FineCentroids{k, dim, flat.data()}; + } +}; + +namespace gem_detail { + +// Token-set span layout: [0] = token count n, [1..n] = centroid ids, rest unused padding. +inline size_t set_size(std::span s) { return static_cast(s[0]); } +inline const float* set_ids(std::span s) { return s.data() + 1; } + +// Raw-query span layout (QChamferExactQuery): [0] = token count n, [1..1+n*dim) = n +// raw fp32 token embeddings row-major, rest unused padding. +inline size_t vector_set_size(std::span s) { + return static_cast(s[0]); +} +inline const float* vector_set_tokens(std::span s) { return s.data() + 1; } + +// Thread-local because the functors are shared across threads (implicit_broadcast). +struct MergeScratch { + std::vector ids; + std::vector weights; +}; + +// Per-token mass is 1/n (GEM Eq. 12); tokens sharing a centroid merge into one weighted +// point. +inline size_t merge_duplicates(const float* ids_f, size_t n, MergeScratch& scratch) { + scratch.ids.resize(n); + for (size_t i = 0; i < n; ++i) { + scratch.ids[i] = static_cast(ids_f[i]); + } + std::sort(scratch.ids.begin(), scratch.ids.end()); + scratch.weights.clear(); + size_t unique_count = 0; + size_t i = 0; + while (i < n) { + size_t j = i; + while (j < n && scratch.ids[j] == scratch.ids[i]) { + ++j; + } + scratch.ids[unique_count] = scratch.ids[i]; + scratch.weights.push_back(static_cast(j - i) / static_cast(n)); + ++unique_count; + i = j; + } + scratch.ids.resize(unique_count); + return unique_count; +} + +// Shared by QChamferExactImpl's generic and per-ISA compute_decoded specializations +// (GEM's code_distance, src/distance/maxsim.cpp:9-45) so the reduction is written once. +inline float qchamfer_exact_reduce( + const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc +) { + thread_local std::vector maxima; + maxima.assign(n_query, -std::numeric_limits::infinity()); + for (size_t j = 0; j < n_doc; ++j) { + const float* row = table + static_cast(doc_ids[j]) * n_query; + for (size_t i = 0; i < n_query; ++i) { + maxima[i] = std::max(maxima[i], row[i]); + } + } + double total = 0.0; + for (size_t i = 0; i < n_query; ++i) { + total += 1.0 - static_cast(maxima[i]); + } + return static_cast(total / static_cast(n_query)); +} + +// Casts a token-set's float centroid ids to uint32_t with no merge; qCH counts duplicates. +inline void decode_ids(const float* ids_f, size_t n, std::vector& out) { + out.resize(n); + for (size_t i = 0; i < n; ++i) { + out[i] = static_cast(ids_f[i]); + } +} + +} // namespace gem_detail + +// Instantiated per ISA in svs/multi-arch/x86; without SVS_NOINLINE the caller inlines the +// body at baseline flags. The bodies are scalar, so only compiler auto-vectorization +// differs. compute_decoded takes already-decoded (deduplicated, weighted) centroid ids so +// gem_time can benchmark the solve separately from the span decode. +template struct QEmdImpl { + SVS_NOINLINE static float compute_decoded( + const CentroidTable& table, + const uint32_t* ids_a, + const double* w_a, + size_t ua, + const uint32_t* ids_b, + const double* w_b, + size_t ub + ) { + if (ua == 1 && ub == 1) { + return table(ids_a[0], ids_b[0]); + } + + // Mirrors POT's EMD_wrap (ot/lp/EMD_wrapper.cpp); all weights are already nonzero. + using Digraph = lemon::FullBipartiteDigraph; + Digraph di(static_cast(ua), static_cast(ub)); + lemon::NetworkSimplexSimple net( + di, + true, + static_cast(ua + ub), + static_cast(ua) * static_cast(ub), + std::numeric_limits::max() + ); + + thread_local std::vector neg_wb; + neg_wb.resize(ub); + for (size_t j = 0; j < ub; ++j) { + neg_wb[j] = -w_b[j]; + } + net.supplyMap(w_a, static_cast(ua), neg_wb.data(), static_cast(ub)); + + int64_t idarc = 0; + for (size_t i = 0; i < ua; ++i) { + for (size_t j = 0; j < ub; ++j) { + net.setCost(di.arcFromId(idarc), table(ids_a[i], ids_b[j])); + ++idarc; + } + } + + if (net.run() != decltype(net)::OPTIMAL) { + throw std::runtime_error( + "QEmd: network simplex did not reach an optimal solution" + ); + } + + double cost = 0.0; + Digraph::Arc arc; + for (di.first(arc); arc != INVALID; di.next(arc)) { + auto src = static_cast(di.source(arc)); + auto dst = static_cast(di.target(arc)) - ua; + cost += net.flow(arc) * table(ids_a[src], ids_b[dst]); + } + return static_cast(cost); + } + + SVS_NOINLINE static float compute( + const CentroidTable& table, std::span a, std::span b + ) { + thread_local gem_detail::MergeScratch scratch_a; + thread_local gem_detail::MergeScratch scratch_b; + size_t ua = gem_detail::merge_duplicates( + gem_detail::set_ids(a), gem_detail::set_size(a), scratch_a + ); + size_t ub = gem_detail::merge_duplicates( + gem_detail::set_ids(b), gem_detail::set_size(b), scratch_b + ); + return compute_decoded( + table, + scratch_a.ids.data(), + scratch_a.weights.data(), + ua, + scratch_b.ids.data(), + scratch_b.weights.data(), + ub + ); + } +}; + +// Dispatched like QEmdImpl. compute_decoded takes already-decoded (cast, unmerged) centroid +// id arrays so gem_time can benchmark the min-sum loop separately from the span decode. +template struct QChamferImpl { + SVS_NOINLINE static float compute_decoded( + const CentroidTable& table, + const uint32_t* q, + size_t nq, + const uint32_t* d, + size_t nd + ) { + double total = 0.0; + for (size_t i = 0; i < nq; ++i) { + float best = std::numeric_limits::infinity(); + for (size_t j = 0; j < nd; ++j) { + best = std::min(best, table(q[i], d[j])); + } + total += best; + } + return static_cast(total); + } + + SVS_NOINLINE static float compute( + const CentroidTable& table, std::span query, std::span doc + ) { + thread_local std::vector qbuf; + thread_local std::vector dbuf; + gem_detail::decode_ids( + gem_detail::set_ids(query), gem_detail::set_size(query), qbuf + ); + gem_detail::decode_ids(gem_detail::set_ids(doc), gem_detail::set_size(doc), dbuf); + return compute_decoded(table, qbuf.data(), qbuf.size(), dbuf.data(), dbuf.size()); + } +}; + +// Dispatched like QChamferImpl. GEM never quantizes the query token side (src/search/ +// routing.cpp:55-61, src/distance/maxsim.cpp:9-45): build_table scores the raw float +// query tokens against every fine centroid (score_fine_centers), and compute_decoded +// reduces that table against the document's quantized centroid ids (code_distance). +template struct QChamferExactImpl { + // table[f * n_query + i] = dot(centroid_f, query_token_i), matching maxsim.cpp's + // Eigen::Map(table, query_tokens, fine_centers) column-major read of the + // same buffer (scores.col(f)[i] == table[f*n_query+i]). + SVS_NOINLINE static void build_table( + const float* centroids, + size_t fine_centers, + size_t dim, + const float* query_tokens, + size_t n_query, + float* table + ) { + for (size_t f = 0; f < fine_centers; ++f) { + const float* c = centroids + f * dim; + float* row = table + f * n_query; + for (size_t i = 0; i < n_query; ++i) { + const float* t = query_tokens + i * dim; + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += c[d] * t[d]; + } + row[i] = dot; + } + } + } + + // GEM's code_distance (src/distance/maxsim.cpp:9-45): for each query token, the max + // dot product to any of the document's (possibly repeated) centroid ids; final score + // is `1 - mean_q max_doc dot`, i.e. 1 minus the mean of those per-token maxima. + SVS_NOINLINE static float compute_decoded( + const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc + ) { + return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); + } +}; + +#if defined(__x86_64__) + +// build_table vectorizes over query tokens, not over `dim`: per output lane i, +// the d=0..dim-1 FMA order matches the scalar version exactly, so only the +// single-rounding FMA (vs. separate mul+add) can move the result off the scalar +// value, well under the 1e-6 relative tolerance this table is checked against. +// Transposing query_tokens into [dim][n_query] turns each centroid row into +// contiguous per-token accumulators with no horizontal reduction, at the cost of +// a one-time O(dim * n_query) transpose (query is a handful of KiB; centroids +// are tens to hundreds of MiB, so the transpose is noise against the table build +// it replaces the scalar dot loop in). +// Declared here, defined in avx512.cpp/avx2.cpp: the bodies use __m512/__m256 intrinsics, +// so each must only be compiled under its own -march flags, not wherever gem.h is included +// (e.g. gem_time.cpp, which carries neither). Defining them in-class would make them +// implicitly inline, and an explicit instantiation of an already-specialized class is a +// no-op in GCC, so no other TU would get a symbol to link against. +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +#if SVS_AVX512_F +template <> struct QChamferExactImpl { + SVS_NOINLINE static void build_table( + const float* centroids, + size_t fine_centers, + size_t dim, + const float* query_tokens, + size_t n_query, + float* table + ); + SVS_NOINLINE static float compute_decoded( + const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc + ); +}; +#endif + +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +SVS_VALIDATE_BOOL_ENV(SVS_AVX2) +#if !SVS_AVX512_F && SVS_AVX2 +template <> struct QChamferExactImpl { + SVS_NOINLINE static void build_table( + const float* centroids, + size_t fine_centers, + size_t dim, + const float* query_tokens, + size_t n_query, + float* table + ); + SVS_NOINLINE static float compute_decoded( + const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc + ); +}; +#endif + +#endif // defined(__x86_64__) + +#if defined(__x86_64__) +extern template struct QEmdImpl; +extern template struct QEmdImpl; +extern template struct QChamferImpl; +extern template struct QChamferImpl; +extern template struct QChamferExactImpl; +extern template struct QChamferExactImpl; +#endif + +/// Reports which ISA path QEmd/QChamfer dispatch to on this process, for diagnostics. +inline std::string_view gem_dispatch_isa() { + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + return "avx512"; + } + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + return "avx2"; + } + return "none"; +} + +/// Build-time distance: quantized Earth Mover's Distance (qEMD) between two token sets, +/// solved exactly as a balanced transportation problem over merged centroid weights. +/// Smaller is better. +class QEmd { + public: + using compare = std::less<>; + + static constexpr bool implicit_broadcast = true; + static constexpr std::string_view name = "gem_qemd"; + + explicit QEmd(std::shared_ptr table) + : table_{std::move(table)} {} + + template + float compute(std::span a, std::span b) const { + std::span da{a.data(), a.size()}; + std::span db{b.data(), b.size()}; + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { + return QEmdImpl::compute(*table_, da, db); + } + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { + return QEmdImpl::compute(*table_, da, db); + } + return QEmdImpl::compute(*table_, da, db); + } + + private: + std::shared_ptr table_; +}; + +inline constexpr bool operator==(const QEmd&, const QEmd&) { return true; } + +/// Search-time distance: quantized Chamfer distance (qCH, GEM Eq. 16). For every query +/// token (duplicates counted), takes the min centroid distance to any doc token, and +/// sums. Smaller is better. The query is the left-hand argument. +class QChamfer { + public: + using compare = std::less<>; + static constexpr bool implicit_broadcast = true; + static constexpr std::string_view name = "gem_qchamfer"; + + explicit QChamfer(std::shared_ptr table) + : table_{std::move(table)} {} + + template + float compute(std::span query, std::span doc) const { + std::span dq{query.data(), query.size()}; + std::span dd{doc.data(), doc.size()}; + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { + return QChamferImpl::compute(*table_, dq, dd); + } + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { + return QChamferImpl::compute(*table_, dq, dd); + } + return QChamferImpl::compute(*table_, dq, dd); + } + + private: + std::shared_ptr table_; +}; + +inline constexpr bool operator==(const QChamfer&, const QChamfer&) { return true; } + +/// Search-time distance: GEM's actual in-graph qCH (GEM_REFERENCE.md "search-time distance: +/// is the query quantized?" row). Unlike QChamfer, the query is never quantized: it stays +/// as raw float tokens, and fix_argument() builds a dense (K x query_tokens) dot table +/// against every fine centroid once per query (score_fine_centers, src/search/ +/// routing.cpp:55-61); compute() then reduces that table against the document's quantized +/// centroid ids (code_distance, src/distance/maxsim.cpp:9-45). Smaller is better. +/// +/// Holds a per-query table as ordinary (non-shared) state, so each search thread's +/// shallow_copy of this functor gets its own table; implicit_broadcast is false and +/// must_fix_argument is true to force a fix_argument() call before any compute(). +class QChamferExactQuery { + public: + using compare = std::less<>; + static constexpr bool implicit_broadcast = false; + static constexpr bool must_fix_argument = true; + static constexpr std::string_view name = "gem_qchamfer_exact_query"; + + explicit QChamferExactQuery(std::shared_ptr centroids) + : centroids_{std::move(centroids)} {} + + // Raw-query span layout (gem_detail::vector_set_size/vector_set_tokens): [0] = token + // count n, [1..1+n*dim) = n raw fp32 tokens row-major, rest padding. + template void fix_argument(std::span query) { + std::span dq{query.data(), query.size()}; + n_query_ = gem_detail::vector_set_size(dq); + const float* tokens = gem_detail::vector_set_tokens(dq); + table_.resize(centroids_->K * n_query_); + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { + QChamferExactImpl::build_table( + centroids_->data.get(), + centroids_->K, + centroids_->dim, + tokens, + n_query_, + table_.data() + ); + } else if (__builtin_expect( + svs::detail::avx_runtime_flags.is_avx2_supported(), 1 + )) { + QChamferExactImpl::build_table( + centroids_->data.get(), + centroids_->K, + centroids_->dim, + tokens, + n_query_, + table_.data() + ); + } else { + QChamferExactImpl::build_table( + centroids_->data.get(), + centroids_->K, + centroids_->dim, + tokens, + n_query_, + table_.data() + ); + } + } + + // `query` is unused: fix_argument() already consumed it into table_/n_query_, and + // every compute() call in a search shares that one query's table. + template + float + compute(std::span /*query*/, std::span doc) const { + std::span dd{doc.data(), doc.size()}; + thread_local std::vector dbuf; + gem_detail::decode_ids(gem_detail::set_ids(dd), gem_detail::set_size(dd), dbuf); + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { + return QChamferExactImpl::compute_decoded( + table_.data(), n_query_, dbuf.data(), dbuf.size() + ); + } + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { + return QChamferExactImpl::compute_decoded( + table_.data(), n_query_, dbuf.data(), dbuf.size() + ); + } + return QChamferExactImpl::compute_decoded( + table_.data(), n_query_, dbuf.data(), dbuf.size() + ); + } + + private: + std::shared_ptr centroids_; + std::vector table_; + size_t n_query_ = 0; +}; + +inline constexpr bool operator==(const QChamferExactQuery&, const QChamferExactQuery&) { + return true; +} + +} // namespace svs::distance diff --git a/include/svs/multi-arch/x86/avx2.cpp b/include/svs/multi-arch/x86/avx2.cpp index bff53ae10..da312e1c6 100644 --- a/include/svs/multi-arch/x86/avx2.cpp +++ b/include/svs/multi-arch/x86/avx2.cpp @@ -17,6 +17,7 @@ #if defined(__x86_64__) #include "svs/core/distance/cosine.h" #include "svs/core/distance/euclidean.h" +#include "svs/core/distance/gem.h" #include "svs/core/distance/inner_product.h" namespace svs::distance { @@ -52,6 +53,61 @@ DISTANCE_CS_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX2); DISTANCE_CS_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX2); DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); +template struct QEmdImpl; +template struct QChamferImpl; +template struct QChamferExactImpl; + +// See avx512.cpp's QChamferExactImpl::build_table for the accumulation-order and +// rounding rationale; this is the same scheme at AVX2's 8-wide registers. +void QChamferExactImpl::build_table( + const float* centroids, + size_t fine_centers, + size_t dim, + const float* query_tokens, + size_t n_query, + float* table +) { + constexpr size_t vl = 8; + thread_local std::vector qt; + qt.resize(dim * n_query); + for (size_t i = 0; i < n_query; ++i) { + const float* t = query_tokens + i * dim; + for (size_t d = 0; d < dim; ++d) { + qt[d * n_query + i] = t[d]; + } + } + + size_t vec_end = n_query - n_query % vl; + for (size_t f = 0; f < fine_centers; ++f) { + const float* c = centroids + f * dim; + float* row = table + f * n_query; + size_t ci = 0; + for (; ci < vec_end; ci += vl) { + __m256 acc = _mm256_setzero_ps(); + for (size_t d = 0; d < dim; ++d) { + __m256 cval = _mm256_set1_ps(c[d]); + __m256 qv = _mm256_loadu_ps(qt.data() + d * n_query + ci); + acc = _mm256_fmadd_ps(cval, qv, acc); + } + _mm256_storeu_ps(row + ci, acc); + } + for (; ci < n_query; ++ci) { + const float* t = query_tokens + ci * dim; + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += c[d] * t[d]; + } + row[ci] = dot; + } + } +} + +float QChamferExactImpl::compute_decoded( + const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc +) { + return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); +} + } // namespace svs::distance #endif diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index bee150d75..fbe941df9 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -17,6 +17,7 @@ #if defined(__x86_64__) #include "svs/core/distance/cosine.h" #include "svs/core/distance/euclidean.h" +#include "svs/core/distance/gem.h" #include "svs/core/distance/inner_product.h" namespace svs::distance { @@ -52,6 +53,67 @@ DISTANCE_CS_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512); DISTANCE_CS_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512); DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); +template struct QEmdImpl; +template struct QChamferImpl; +template struct QChamferExactImpl; + +// build_table vectorizes over query tokens, not over `dim`: per output lane i, the +// d=0..dim-1 FMA order matches the scalar version exactly, so only the single-rounding +// FMA (vs. separate mul+add) can move the result off the scalar value, well under the +// 1e-6 relative tolerance this table is checked against. Transposing query_tokens into +// [dim][n_query] turns each centroid row into contiguous per-token accumulators with no +// horizontal reduction, at the cost of a one-time O(dim * n_query) transpose (query is a +// handful of KiB; centroids are tens to hundreds of MiB, so the transpose is noise +// against the table build it replaces the scalar dot loop in). +void QChamferExactImpl::build_table( + const float* centroids, + size_t fine_centers, + size_t dim, + const float* query_tokens, + size_t n_query, + float* table +) { + constexpr size_t vl = 16; + thread_local std::vector qt; + qt.resize(dim * n_query); + for (size_t i = 0; i < n_query; ++i) { + const float* t = query_tokens + i * dim; + for (size_t d = 0; d < dim; ++d) { + qt[d * n_query + i] = t[d]; + } + } + + size_t vec_end = n_query - n_query % vl; + for (size_t f = 0; f < fine_centers; ++f) { + const float* c = centroids + f * dim; + float* row = table + f * n_query; + size_t ci = 0; + for (; ci < vec_end; ci += vl) { + __m512 acc = _mm512_setzero_ps(); + for (size_t d = 0; d < dim; ++d) { + __m512 cval = _mm512_set1_ps(c[d]); + __m512 qv = _mm512_loadu_ps(qt.data() + d * n_query + ci); + acc = _mm512_fmadd_ps(cval, qv, acc); + } + _mm512_storeu_ps(row + ci, acc); + } + for (; ci < n_query; ++ci) { + const float* t = query_tokens + ci * dim; + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += c[d] * t[d]; + } + row[ci] = dot; + } + } +} + +float QChamferExactImpl::compute_decoded( + const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc +) { + return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); +} + } // namespace svs::distance #endif diff --git a/include/svs/third-party/pot/README.md b/include/svs/third-party/pot/README.md new file mode 100644 index 000000000..30b30c8ee --- /dev/null +++ b/include/svs/third-party/pot/README.md @@ -0,0 +1,45 @@ +# Vendored POT network simplex solver + +Source: https://github.com/PythonOT/POT, tag `0.9.5`, directory `ot/lp/`. + +Fetched unmodified from `https://raw.githubusercontent.com/PythonOT/POT/0.9.5/ot/lp/`. + +SHA-256: + +``` +4264bfe6760fcdf9b4c591e0e1fac9b2d983ccf6874dffb4d9e314868b197c93 network_simplex_simple.h +c16d242d385b9ad5c5b720d5a9b8526cb141b753f93eded9035368d0b04e66ec full_bipartitegraph.h +359e9b1d580c1eb7213bfd961d57ea70f6713d364534ae343f6d79b90626d753 core.h +``` + +## Files + +- `network_simplex_simple.h` — the exact transport solver behind `svs::distance::QEmd`. +- `full_bipartitegraph.h` — local header `network_simplex_simple.h` includes. +- `core.h` — local header `full_bipartitegraph.h` includes. + +(`network_simplex_simple.h`'s other includes are all standard library headers.) + +## Licenses + +POT itself is MIT-licensed: "Copyright (c) 2016-2023 POT contributors". + +All three files are Nicolas Bonneel's 2013 adaptation of LEMON +(`full_graph.h` / `network_simplex.h`) into a lighter mass-transport solver. +Each carries LEMON's original notice: + +``` +Copyright (C) 2003-2010 +Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport +(Egervary Research Group on Combinatorial Optimization, EGRES). + +Permission to use, modify and distribute this software is granted +provided that this copyright notice appears in all copies. For +precise terms see the accompanying LICENSE file. + +This software is provided "AS IS" with no warranty of any kind, +express or implied, and with no claim as to its suitability for any +purpose. +``` + +This permission notice must stay in each vendored file; do not strip it when editing. diff --git a/include/svs/third-party/pot/core.h b/include/svs/third-party/pot/core.h new file mode 100644 index 000000000..04dddf721 --- /dev/null +++ b/include/svs/third-party/pot/core.h @@ -0,0 +1,103 @@ +/* -*- mode: C++; indent-tabs-mode: nil; -*- + * + * This file has been adapted by Nicolas Bonneel (2013), + * from full_graph.h from LEMON, a generic C++ optimization library, + * to make the other files independant from the rest of + * the original library. + * + * + **** Original file Copyright Notice : + * Copyright (C) 2003-2010 + * Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport + * (Egervary Research Group on Combinatorial Optimization, EGRES). + * + * Permission to use, modify and distribute this software is granted + * provided that this copyright notice appears in all copies. For + * precise terms see the accompanying LICENSE file. + * + * This software is provided "AS IS" with no warranty of any kind, + * express or implied, and with no claim as to its suitability for any + * purpose. + * + */ + +#ifndef LEMON_CORE_H +#define LEMON_CORE_H + +#include +#include + + +// Disable the following warnings when compiling with MSVC: +// C4250: 'class1' : inherits 'class2::member' via dominance +// C4355: 'this' : used in base member initializer list +// C4503: 'function' : decorated name length exceeded, name was truncated +// C4800: 'type' : forcing value to bool 'true' or 'false' (performance warning) +// C4996: 'function': was declared deprecated +#ifdef _MSC_VER +#pragma warning( disable : 4250 4355 4503 4800 4996 ) +#endif + +///\file +///\brief LEMON core utilities. +/// +///This header file contains core utilities for LEMON. +///It is automatically included by all graph types, therefore it usually +///do not have to be included directly. + +namespace lemon { + + /// \brief Dummy type to make it easier to create invalid iterators. + /// + /// Dummy type to make it easier to create invalid iterators. + /// See \ref INVALID for the usage. + struct Invalid { + public: + bool operator==(Invalid) { return true; } + bool operator!=(Invalid) { return false; } + bool operator< (Invalid) { return false; } + }; + + /// \brief Invalid iterators. + /// + /// \ref Invalid is a global type that converts to each iterator + /// in such a way that the value of the target iterator will be invalid. +#ifdef LEMON_ONLY_TEMPLATES + const Invalid INVALID = Invalid(); +#else + extern const Invalid INVALID; +#endif + + /// \addtogroup gutils + /// @{ + + ///Create convenience typedefs for the digraph types and iterators + + ///This \c \#define creates convenient type definitions for the following + ///types of \c Digraph: \c Node, \c NodeIt, \c Arc, \c ArcIt, \c InArcIt, + ///\c OutArcIt, \c BoolNodeMap, \c IntNodeMap, \c DoubleNodeMap, + ///\c BoolArcMap, \c IntArcMap, \c DoubleArcMap. + /// + ///\note If the graph type is a dependent type, ie. the graph type depend + ///on a template parameter, then use \c TEMPLATE_DIGRAPH_TYPEDEFS() + ///macro. +#define DIGRAPH_TYPEDEFS(Digraph) \ + typedef Digraph::Node Node; \ + typedef Digraph::Arc Arc; \ + + + ///Create convenience typedefs for the digraph types and iterators + + ///\see DIGRAPH_TYPEDEFS + /// + ///\note Use this macro, if the graph type is a dependent type, + ///ie. the graph type depend on a template parameter. +#define TEMPLATE_DIGRAPH_TYPEDEFS(Digraph) \ + typedef typename Digraph::Node Node; \ + typedef typename Digraph::Arc Arc; \ + + + +} //namespace lemon + +#endif diff --git a/include/svs/third-party/pot/full_bipartitegraph.h b/include/svs/third-party/pot/full_bipartitegraph.h new file mode 100644 index 000000000..713ccb593 --- /dev/null +++ b/include/svs/third-party/pot/full_bipartitegraph.h @@ -0,0 +1,212 @@ +/* -*- mode: C++; indent-tabs-mode: nil; -*- + * + * This file has been adapted by Nicolas Bonneel (2013), + * from full_graph.h from LEMON, a generic C++ optimization library, + * to implement a lightweight fully connected bipartite graph. A previous + * version of this file is used as part of the Displacement Interpolation + * project, + * Web: http://www.cs.ubc.ca/labs/imager/tr/2011/DisplacementInterpolation/ + * + * + **** Original file Copyright Notice : + * Copyright (C) 2003-2010 + * Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport + * (Egervary Research Group on Combinatorial Optimization, EGRES). + * + * Permission to use, modify and distribute this software is granted + * provided that this copyright notice appears in all copies. For + * precise terms see the accompanying LICENSE file. + * + * This software is provided "AS IS" with no warranty of any kind, + * express or implied, and with no claim as to its suitability for any + * purpose. + * + */ + +#pragma once + +#include "core.h" +#include + +///\ingroup graphs +///\file +///\brief FullBipartiteDigraph and FullBipartiteGraph classes. + + +namespace lemon { + + + class FullBipartiteDigraphBase { + public: + + typedef FullBipartiteDigraphBase Digraph; + + //class Node; + typedef int Node; + //class Arc; + typedef int64_t Arc; + + protected: + + int _node_num; + int64_t _arc_num; + + FullBipartiteDigraphBase() {} + + void construct(int n1, int n2) { _node_num = n1+n2; _arc_num = (int64_t)n1 * (int64_t)n2; _n1=n1; _n2=n2;} + + public: + + int _n1, _n2; + + + Node operator()(int ix) const { return Node(ix); } + static int index(const Node& node) { return node; } + + Arc arc(const Node& s, const Node& t) const { + if (s<_n1 && t>=_n1) + return Arc((int64_t)s * (int64_t)_n2 + (int64_t)(t-_n1) ); + else + return Arc(-1); + } + + int nodeNum() const { return _node_num; } + int64_t arcNum() const { return _arc_num; } + + int maxNodeId() const { return _node_num - 1; } + int64_t maxArcId() const { return _arc_num - 1; } + + Node source(Arc arc) const { return arc / _n2; } + Node target(Arc arc) const { return (arc % _n2) + _n1; } + + static int id(Node node) { return node; } + static int64_t id(Arc arc) { return arc; } + + static Node nodeFromId(int id) { return Node(id);} + static Arc arcFromId(int64_t id) { return Arc(id);} + + + Arc findArc(Node s, Node t, Arc prev = -1) const { + return prev == -1 ? arc(s, t) : -1; + } + + void first(Node& node) const { + node = _node_num - 1; + } + + static void next(Node& node) { + --node; + } + + void first(Arc& arc) const { + arc = _arc_num - 1; + } + + static void next(Arc& arc) { + --arc; + } + + void firstOut(Arc& arc, const Node& node) const { + if (node>=_n1) + arc = -1; + else + arc = (node + 1) * _n2 - 1; + } + + void nextOut(Arc& arc) const { + if (arc % _n2 == 0) arc = 0; + --arc; + } + + void firstIn(Arc& arc, const Node& node) const { + if (node<_n1) + arc = -1; + else + arc = _arc_num + node - _node_num; + } + + void nextIn(Arc& arc) const { + arc -= _n2; + if (arc < 0) arc = -1; + } + + }; + + /// \ingroup graphs + /// + /// \brief A directed full graph class. + /// + /// FullBipartiteDigraph is a simple and fast implementation of directed full + /// (complete) graphs. It contains an arc from each node to each node + /// (including a loop for each node), therefore the number of arcs + /// is the square of the number of nodes. + /// This class is completely static and it needs constant memory space. + /// Thus you can neither add nor delete nodes or arcs, however + /// the structure can be resized using resize(). + /// + /// This type fully conforms to the \ref concepts::Digraph "Digraph concept". + /// Most of its member functions and nested classes are documented + /// only in the concept class. + /// + /// This class provides constant time counting for nodes and arcs. + /// + /// \note FullBipartiteDigraph and FullBipartiteGraph classes are very similar, + /// but there are two differences. While this class conforms only + /// to the \ref concepts::Digraph "Digraph" concept, FullBipartiteGraph + /// conforms to the \ref concepts::Graph "Graph" concept, + /// moreover FullBipartiteGraph does not contain a loop for each + /// node as this class does. + /// + /// \sa FullBipartiteGraph + class FullBipartiteDigraph : public FullBipartiteDigraphBase { + typedef FullBipartiteDigraphBase Parent; + + public: + + /// \brief Default constructor. + /// + /// Default constructor. The number of nodes and arcs will be zero. + FullBipartiteDigraph() { construct(0,0); } + + /// \brief Constructor + /// + /// Constructor. + /// \param n The number of the nodes. + FullBipartiteDigraph(int n1, int n2) { construct(n1, n2); } + + + /// \brief Returns the node with the given index. + /// + /// Returns the node with the given index. Since this structure is + /// completely static, the nodes can be indexed with integers from + /// the range [0..nodeNum()-1]. + /// The index of a node is the same as its ID. + /// \sa index() + Node operator()(int ix) const { return Parent::operator()(ix); } + + /// \brief Returns the index of the given node. + /// + /// Returns the index of the given node. Since this structure is + /// completely static, the nodes can be indexed with integers from + /// the range [0..nodeNum()-1]. + /// The index of a node is the same as its ID. + /// \sa operator()() + static int index(const Node& node) { return Parent::index(node); } + + /// \brief Returns the arc connecting the given nodes. + /// + /// Returns the arc connecting the given nodes. + /*Arc arc(Node u, Node v) const { + return Parent::arc(u, v); + }*/ + + /// \brief Number of nodes. + int nodeNum() const { return Parent::nodeNum(); } + /// \brief Number of arcs. + int64_t arcNum() const { return Parent::arcNum(); } + }; + + + + +} //namespace lemon diff --git a/include/svs/third-party/pot/network_simplex_simple.h b/include/svs/third-party/pot/network_simplex_simple.h new file mode 100644 index 000000000..9612a8a24 --- /dev/null +++ b/include/svs/third-party/pot/network_simplex_simple.h @@ -0,0 +1,1552 @@ +/* -*- mode: C++; indent-tabs-mode: nil; -*- + * + * + * This file has been adapted by Nicolas Bonneel (2013), + * from network_simplex.h from LEMON, a generic C++ optimization library, + * to implement a lightweight network simplex for mass transport, more + * memory efficient that the original file. A previous version of this file + * is used as part of the Displacement Interpolation project, + * Web: http://www.cs.ubc.ca/labs/imager/tr/2011/DisplacementInterpolation/ + * + * + **** Original file Copyright Notice : + * + * Copyright (C) 2003-2010 + * Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport + * (Egervary Research Group on Combinatorial Optimization, EGRES). + * + * Permission to use, modify and distribute this software is granted + * provided that this copyright notice appears in all copies. For + * precise terms see the accompanying LICENSE file. + * + * This software is provided "AS IS" with no warranty of any kind, + * express or implied, and with no claim as to its suitability for any + * purpose. + * + */ + +#pragma once +#undef DEBUG_LVL +#define DEBUG_LVL 0 + +#if DEBUG_LVL>0 +#include +#endif + +#undef EPSILON +#undef _EPSILON +#undef MAX_DEBUG_ITER +#define EPSILON 2.2204460492503131e-15 +#define _EPSILON 1e-8 +#define MAX_DEBUG_ITER 100000 + + +/// \ingroup min_cost_flow_algs +/// +/// \file +/// \brief Network Simplex algorithm for finding a minimum cost flow. + +// if your compiler has troubles with stdext or hashmaps, just comment the following line to use a slower std::map instead +//#define HASHMAP + +#include +#include +#include +#include +#include +#ifdef HASHMAP +#include +#else +#include +#endif +#include +//#include "core.h" +//#include "lmath.h" + +//#include "sparse_array_n.h" +#include "full_bipartitegraph.h" + +#undef INVALIDNODE +#undef INVALID +#define INVALIDNODE -1 +#define INVALID (-1) + +namespace lemon { + + + template + class ProxyObject; + + template + class SparseValueVector + { + public: + SparseValueVector(size_t n=0) + { + } + void resize(size_t n=0){}; + T operator[](const size_t id) const + { +#ifdef HASHMAP + typename stdext::hash_map::const_iterator it = data.find(id); +#else + typename std::map::const_iterator it = data.find(id); +#endif + if (it==data.end()) + return 0; + else + return it->second; + } + + ProxyObject operator[](const size_t id) + { + return ProxyObject( this, id ); + } + + //private: +#ifdef HASHMAP + stdext::hash_map data; +#else + std::map data; +#endif + + }; + + template + class ProxyObject { + public: + ProxyObject( SparseValueVector *v, size_t idx ){_v=v; _idx=idx;}; + ProxyObject & operator=( const T &v ) { + // If we get here, we know that operator[] was called to perform a write access, + // so we can insert an item in the vector if needed + if (v!=0) + _v->data[_idx]=v; + return *this; + } + + operator T() { + // If we get here, we know that operator[] was called to perform a read access, + // so we can simply return the existing object +#ifdef HASHMAP + typename stdext::hash_map::iterator it = _v->data.find(_idx); +#else + typename std::map::iterator it = _v->data.find(_idx); +#endif + if (it==_v->data.end()) + return 0; + else + return it->second; + } + + void operator+=(T val) + { + if (val==0) return; +#ifdef HASHMAP + typename stdext::hash_map::iterator it = _v->data.find(_idx); +#else + typename std::map::iterator it = _v->data.find(_idx); +#endif + if (it==_v->data.end()) + _v->data[_idx] = val; + else + { + T sum = it->second + val; + if (sum==0) + _v->data.erase(it); + else + it->second = sum; + } + } + void operator-=(T val) + { + if (val==0) return; +#ifdef HASHMAP + typename stdext::hash_map::iterator it = _v->data.find(_idx); +#else + typename std::map::iterator it = _v->data.find(_idx); +#endif + if (it==_v->data.end()) + _v->data[_idx] = -val; + else + { + T sum = it->second - val; + if (sum==0) + _v->data.erase(it); + else + it->second = sum; + } + } + + SparseValueVector *_v; + size_t _idx; + }; + + + + /// \addtogroup min_cost_flow_algs + /// @{ + + /// \brief Implementation of the primal Network Simplex algorithm + /// for finding a \ref min_cost_flow "minimum cost flow". + /// + /// \ref NetworkSimplexSimple implements the primal Network Simplex algorithm + /// for finding a \ref min_cost_flow "minimum cost flow" + /// \ref amo93networkflows, \ref dantzig63linearprog, + /// \ref kellyoneill91netsimplex. + /// This algorithm is a highly efficient specialized version of the + /// linear programming simplex method directly for the minimum cost + /// flow problem. + /// + /// In general, %NetworkSimplexSimple is the fastest implementation available + /// in LEMON for this problem. + /// Moreover, it supports both directions of the supply/demand inequality + /// constraints. For more information, see \ref SupplyType. + /// + /// Most of the parameters of the problem (except for the digraph) + /// can be given using separate functions, and the algorithm can be + /// executed using the \ref run() function. If some parameters are not + /// specified, then default values will be used. + /// + /// \tparam GR The digraph type the algorithm runs on. + /// \tparam V The number type used for flow amounts, capacity bounds + /// and supply values in the algorithm. By default, it is \c int64_t. + /// \tparam C The number type used for costs and potentials in the + /// algorithm. By default, it is the same as \c V. + /// + /// \warning Both number types must be signed and all input data must + /// be integer. + /// + /// \note %NetworkSimplexSimple provides five different pivot rule + /// implementations, from which the most efficient one is used + /// by default. For more information, see \ref PivotRule. + template + class NetworkSimplexSimple + { + public: + + /// \brief Constructor. + /// + /// The constructor of the class. + /// + /// \param graph The digraph the algorithm runs on. + /// \param arc_mixing Indicate if the arcs have to be stored in a + /// mixed order in the internal data structure. + /// In special cases, it could lead to better overall performance, + /// but it is usually slower. Therefore it is disabled by default. + NetworkSimplexSimple(const GR& graph, bool arc_mixing, int nbnodes, ArcsType nb_arcs, uint64_t maxiters) : + _graph(graph), //_arc_id(graph), + _arc_mixing(arc_mixing), _init_nb_nodes(nbnodes), _init_nb_arcs(nb_arcs), + MAX(std::numeric_limits::max()), + INF(std::numeric_limits::has_infinity ? + std::numeric_limits::infinity() : MAX) + { + // Reset data structures + reset(); + max_iter = maxiters; + } + + /// The type of the flow amounts, capacity bounds and supply values + typedef V Value; + /// The type of the arc costs + typedef C Cost; + + public: + + /// \brief Problem type constants for the \c run() function. + /// + /// Enum type containing the problem type constants that can be + /// returned by the \ref run() function of the algorithm. + enum ProblemType { + /// The problem has no feasible solution (flow). + INFEASIBLE, + /// The problem has optimal solution (i.e. it is feasible and + /// bounded), and the algorithm has found optimal flow and node + /// potentials (primal and dual solutions). + OPTIMAL, + /// The objective function of the problem is unbounded, i.e. + /// there is a directed cycle having negative total cost and + /// infinite upper bound. + UNBOUNDED, + /// The maximum number of iteration has been reached + MAX_ITER_REACHED + }; + + /// \brief Constants for selecting the type of the supply constraints. + /// + /// Enum type containing constants for selecting the supply type, + /// i.e. the direction of the inequalities in the supply/demand + /// constraints of the \ref min_cost_flow "minimum cost flow problem". + /// + /// The default supply type is \c GEQ, the \c LEQ type can be + /// selected using \ref supplyType(). + /// The equality form is a special case of both supply types. + enum SupplyType { + /// This option means that there are "greater or equal" + /// supply/demand constraints in the definition of the problem. + GEQ, + /// This option means that there are "less or equal" + /// supply/demand constraints in the definition of the problem. + LEQ + }; + + + + private: + + uint64_t max_iter; + TEMPLATE_DIGRAPH_TYPEDEFS(GR); + + typedef std::vector IntVector; + typedef std::vector ArcVector; + typedef std::vector ValueVector; + typedef std::vector CostVector; + // typedef SparseValueVector CostVector; + typedef std::vector BoolVector; + // Note: vector is used instead of vector for efficiency reasons + + // State constants for arcs + enum ArcState { + STATE_UPPER = -1, + STATE_TREE = 0, + STATE_LOWER = 1 + }; + + typedef std::vector StateVector; + // Note: vector is used instead of vector for + // efficiency reasons + + private: + + // Data related to the underlying digraph + const GR &_graph; + int _node_num; + ArcsType _arc_num; + ArcsType _all_arc_num; + ArcsType _search_arc_num; + + // Parameters of the problem + SupplyType _stype; + Value _sum_supply; + + inline int _node_id(int n) const {return _node_num-n-1;} ; + +// IntArcMap _arc_id; + IntVector _source; // keep nodes as integers + IntVector _target; + bool _arc_mixing; + public: + // Node and arc data + CostVector _cost; + ValueVector _supply; + ValueVector _flow; + //SparseValueVector _flow; + CostVector _pi; + + + private: + // Data for storing the spanning tree structure + IntVector _parent; + ArcVector _pred; + IntVector _thread; + IntVector _rev_thread; + IntVector _succ_num; + IntVector _last_succ; + IntVector _dirty_revs; + BoolVector _forward; + StateVector _state; + ArcsType _root; + + // Temporary data used in the current pivot iteration + ArcsType in_arc, join, u_in, v_in, u_out, v_out; + ArcsType first, second, right, last; + ArcsType stem, par_stem, new_stem; + Value delta; + + const Value MAX; + + ArcsType mixingCoeff; + + public: + + /// \brief Constant for infinite upper bounds (capacities). + /// + /// Constant for infinite upper bounds (capacities). + /// It is \c std::numeric_limits::infinity() if available, + /// \c std::numeric_limits::max() otherwise. + const Value INF; + + private: + + // thank you to DVK and MizardX from StackOverflow for this function! + inline ArcsType sequence(ArcsType k) const { + ArcsType smallv = (k > num_total_big_subsequence_numbers) & 1; + + k -= num_total_big_subsequence_numbers * smallv; + ArcsType subsequence_length2 = subsequence_length- smallv; + ArcsType subsequence_num = (k / subsequence_length2) + num_big_subseqiences * smallv; + ArcsType subsequence_offset = (k % subsequence_length2) * mixingCoeff; + + return subsequence_offset + subsequence_num; + } + ArcsType subsequence_length; + ArcsType num_big_subseqiences; + ArcsType num_total_big_subsequence_numbers; + + inline ArcsType getArcID(const Arc &arc) const + { + //int n = _arc_num-arc._id-1; + ArcsType n = _arc_num-GR::id(arc)-1; + + //ArcsType a = mixingCoeff*(n%mixingCoeff) + n/mixingCoeff; + //ArcsType b = _arc_id[arc]; + if (_arc_mixing) + return sequence(n); + else + return n; + } + + // finally unused because too slow + inline ArcsType getSource(const ArcsType arc) const + { + //ArcsType a = _source[arc]; + //return a; + + ArcsType n = _arc_num-arc-1; + if (_arc_mixing) + n = mixingCoeff*(n%mixingCoeff) + n/mixingCoeff; + + ArcsType b; + if (n>=0) + b = _node_id(_graph.source(GR::arcFromId( n ) )); + else + { + n = arc+1-_arc_num; + if ( n<=_node_num) + b = _node_num; + else + if ( n>=_graph._n1) + b = _graph._n1; + else + b = _graph._n1-n; + } + + return b; + } + + + + // Implementation of the Block Search pivot rule + class BlockSearchPivotRule + { + private: + + // References to the NetworkSimplexSimple class + const IntVector &_source; + const IntVector &_target; + const CostVector &_cost; + const StateVector &_state; + const CostVector &_pi; + ArcsType &_in_arc; + ArcsType _search_arc_num; + + // Pivot rule data + ArcsType _block_size; + ArcsType _next_arc; + NetworkSimplexSimple &_ns; + + public: + + // Constructor + BlockSearchPivotRule(NetworkSimplexSimple &ns) : + _source(ns._source), _target(ns._target), + _cost(ns._cost), _state(ns._state), _pi(ns._pi), + _in_arc(ns.in_arc), _search_arc_num(ns._search_arc_num), + _next_arc(0),_ns(ns) + { + // The main parameters of the pivot rule + const double BLOCK_SIZE_FACTOR = 1.0; + const ArcsType MIN_BLOCK_SIZE = 10; + + _block_size = std::max(ArcsType(BLOCK_SIZE_FACTOR * std::sqrt(double(_search_arc_num))), MIN_BLOCK_SIZE); + } + + // Find next entering arc + bool findEnteringArc() { + Cost c, min = 0; + ArcsType e; + ArcsType cnt = _block_size; + double a; + for (e = _next_arc; e != _search_arc_num; ++e) { + c = _state[e] * (_cost[e] + _pi[_source[e]] - _pi[_target[e]]); + if (c < min) { + min = c; + _in_arc = e; + } + if (--cnt == 0) { + a=fabs(_pi[_source[_in_arc]])>fabs(_pi[_target[_in_arc]]) ? fabs(_pi[_source[_in_arc]]):fabs(_pi[_target[_in_arc]]); + a=a>fabs(_cost[_in_arc])?a:fabs(_cost[_in_arc]); + if (min < -EPSILON*a) goto search_end; + cnt = _block_size; + } + } + for (e = 0; e != _next_arc; ++e) { + c = _state[e] * (_cost[e] + _pi[_source[e]] - _pi[_target[e]]); + if (c < min) { + min = c; + _in_arc = e; + } + if (--cnt == 0) { + a=fabs(_pi[_source[_in_arc]])>fabs(_pi[_target[_in_arc]]) ? fabs(_pi[_source[_in_arc]]):fabs(_pi[_target[_in_arc]]); + a=a>fabs(_cost[_in_arc])?a:fabs(_cost[_in_arc]); + if (min < -EPSILON*a) goto search_end; + cnt = _block_size; + } + } + a=fabs(_pi[_source[_in_arc]])>fabs(_pi[_target[_in_arc]]) ? fabs(_pi[_source[_in_arc]]):fabs(_pi[_target[_in_arc]]); + a=a>fabs(_cost[_in_arc])?a:fabs(_cost[_in_arc]); + if (min >= -EPSILON*a) return false; + + search_end: + _next_arc = e; + return true; + } + + }; //class BlockSearchPivotRule + + + + public: + + + + int _init_nb_nodes; + ArcsType _init_nb_arcs; + + /// \name Parameters + /// The parameters of the algorithm can be specified using these + /// functions. + + /// @{ + + + /// \brief Set the costs of the arcs. + /// + /// This function sets the costs of the arcs. + /// If it is not used before calling \ref run(), the costs + /// will be set to \c 1 on all arcs. + /// + /// \param map An arc map storing the costs. + /// Its \c Value type must be convertible to the \c Cost type + /// of the algorithm. + /// + /// \return (*this) + template + NetworkSimplexSimple& costMap(const CostMap& map) { + Arc a; _graph.first(a); + for (; a != INVALID; _graph.next(a)) { + _cost[getArcID(a)] = map[a]; + } + return *this; + } + + + /// \brief Set the costs of one arc. + /// + /// This function sets the costs of one arcs. + /// Done for memory reasons + /// + /// \param arc An arc. + /// \param arc A cost + /// + /// \return (*this) + template + NetworkSimplexSimple& setCost(const Arc& arc, const Value cost) { + _cost[getArcID(arc)] = cost; + return *this; + } + + + /// \brief Set the supply values of the nodes. + /// + /// This function sets the supply values of the nodes. + /// If neither this function nor \ref stSupply() is used before + /// calling \ref run(), the supply of each node will be set to zero. + /// + /// \param map A node map storing the supply values. + /// Its \c Value type must be convertible to the \c Value type + /// of the algorithm. + /// + /// \return (*this) + template + NetworkSimplexSimple& supplyMap(const SupplyMap& map) { + Node n; _graph.first(n); + for (; n != INVALIDNODE; _graph.next(n)) { + _supply[_node_id(n)] = map[n]; + } + return *this; + } + template + NetworkSimplexSimple& supplyMap(const SupplyMap* map1, int n1, const SupplyMap* map2, int n2) { + Node n; _graph.first(n); + for (; n != INVALIDNODE; _graph.next(n)) { + if (n + NetworkSimplexSimple& supplyMapAll(SupplyMap val1, int n1, SupplyMap val2, int n2) { + Node n; _graph.first(n); + for (; n != INVALIDNODE; _graph.next(n)) { + if (n(*this) + NetworkSimplexSimple& stSupply(const Node& s, const Node& t, Value k) { + for (int i = 0; i != _node_num; ++i) { + _supply[i] = 0; + } + _supply[_node_id(s)] = k; + _supply[_node_id(t)] = -k; + return *this; + } + + /// \brief Set the type of the supply constraints. + /// + /// This function sets the type of the supply/demand constraints. + /// If it is not used before calling \ref run(), the \ref GEQ supply + /// type will be used. + /// + /// For more information, see \ref SupplyType. + /// + /// \return (*this) + NetworkSimplexSimple& supplyType(SupplyType supply_type) { + _stype = supply_type; + return *this; + } + + /// @} + + /// \name Execution Control + /// The algorithm can be executed using \ref run(). + + /// @{ + + /// \brief Run the algorithm. + /// + /// This function runs the algorithm. + /// The paramters can be specified using functions \ref lowerMap(), + /// \ref upperMap(), \ref costMap(), \ref supplyMap(), \ref stSupply(), + /// \ref supplyType(). + /// For example, + /// \code + /// NetworkSimplexSimple ns(graph); + /// ns.lowerMap(lower).upperMap(upper).costMap(cost) + /// .supplyMap(sup).run(); + /// \endcode + /// + /// This function can be called more than once. All the given parameters + /// are kept for the next call, unless \ref resetParams() or \ref reset() + /// is used, thus only the modified parameters have to be set again. + /// If the underlying digraph was also modified after the construction + /// of the class (or the last \ref reset() call), then the \ref reset() + /// function must be called. + /// + /// \param pivot_rule The pivot rule that will be used during the + /// algorithm. For more information, see \ref PivotRule. + /// + /// \return \c INFEASIBLE if no feasible flow exists, + /// \n \c OPTIMAL if the problem has optimal solution + /// (i.e. it is feasible and bounded), and the algorithm has found + /// optimal flow and node potentials (primal and dual solutions), + /// \n \c UNBOUNDED if the objective function of the problem is + /// unbounded, i.e. there is a directed cycle having negative total + /// cost and infinite upper bound. + /// + /// \see ProblemType, PivotRule + /// \see resetParams(), reset() + ProblemType run() { +#if DEBUG_LVL>0 + std::cout << "OPTIMAL = " << OPTIMAL << "\nINFEASIBLE = " << INFEASIBLE << "\nUNBOUNDED = " << UNBOUNDED << "\nMAX_ITER_REACHED" << MAX_ITER_REACHED << "\n" ; +#endif + + if (!init()) return INFEASIBLE; +#if DEBUG_LVL>0 + std::cout << "Init done, starting iterations\n"; +#endif + return start(); + } + + /// \brief Reset all the parameters that have been given before. + /// + /// This function resets all the paramaters that have been given + /// before using functions \ref lowerMap(), \ref upperMap(), + /// \ref costMap(), \ref supplyMap(), \ref stSupply(), \ref supplyType(). + /// + /// It is useful for multiple \ref run() calls. Basically, all the given + /// parameters are kept for the next \ref run() call, unless + /// \ref resetParams() or \ref reset() is used. + /// If the underlying digraph was also modified after the construction + /// of the class or the last \ref reset() call, then the \ref reset() + /// function must be used, otherwise \ref resetParams() is sufficient. + /// + /// For example, + /// \code + /// NetworkSimplexSimple ns(graph); + /// + /// // First run + /// ns.lowerMap(lower).upperMap(upper).costMap(cost) + /// .supplyMap(sup).run(); + /// + /// // Run again with modified cost map (resetParams() is not called, + /// // so only the cost map have to be set again) + /// cost[e] += 100; + /// ns.costMap(cost).run(); + /// + /// // Run again from scratch using resetParams() + /// // (the lower bounds will be set to zero on all arcs) + /// ns.resetParams(); + /// ns.upperMap(capacity).costMap(cost) + /// .supplyMap(sup).run(); + /// \endcode + /// + /// \return (*this) + /// + /// \see reset(), run() + NetworkSimplexSimple& resetParams() { + for (int i = 0; i != _node_num; ++i) { + _supply[i] = 0; + } + for (ArcsType i = 0; i != _arc_num; ++i) { + _cost[i] = 1; + } + _stype = GEQ; + return *this; + } + + + + int64_t divid (int64_t x, int64_t y) + { + return (x-x%y)/y; + } + + /// \brief Reset the internal data structures and all the parameters + /// that have been given before. + /// + /// This function resets the internal data structures and all the + /// paramaters that have been given before using functions \ref lowerMap(), + /// \ref upperMap(), \ref costMap(), \ref supplyMap(), \ref stSupply(), + /// \ref supplyType(). + /// + /// It is useful for multiple \ref run() calls. Basically, all the given + /// parameters are kept for the next \ref run() call, unless + /// \ref resetParams() or \ref reset() is used. + /// If the underlying digraph was also modified after the construction + /// of the class or the last \ref reset() call, then the \ref reset() + /// function must be used, otherwise \ref resetParams() is sufficient. + /// + /// See \ref resetParams() for examples. + /// + /// \return (*this) + /// + /// \see resetParams(), run() + NetworkSimplexSimple& reset() { + // Resize vectors + _node_num = _init_nb_nodes; + _arc_num = _init_nb_arcs; + int all_node_num = _node_num + 1; + ArcsType max_arc_num = _arc_num + 2 * _node_num; + + _source.resize(max_arc_num); + _target.resize(max_arc_num); + + _cost.resize(max_arc_num); + _supply.resize(all_node_num); + _flow.resize(max_arc_num); + _pi.resize(all_node_num); + + _parent.resize(all_node_num); + _pred.resize(all_node_num); + _forward.resize(all_node_num); + _thread.resize(all_node_num); + _rev_thread.resize(all_node_num); + _succ_num.resize(all_node_num); + _last_succ.resize(all_node_num); + _state.resize(max_arc_num); + + + //_arc_mixing=false; + if (_arc_mixing) { + // Store the arcs in a mixed order + const ArcsType k = std::max(ArcsType(std::sqrt(double(_arc_num))), ArcsType(10)); + mixingCoeff = k; + subsequence_length = _arc_num / mixingCoeff + 1; + num_big_subseqiences = _arc_num % mixingCoeff; + num_total_big_subsequence_numbers = subsequence_length * num_big_subseqiences; + + ArcsType i = 0, j = 0; + Arc a; _graph.first(a); + for (; a != INVALID; _graph.next(a)) { + _source[i] = _node_id(_graph.source(a)); + _target[i] = _node_id(_graph.target(a)); + //_arc_id[a] = i; + if ((i += k) >= _arc_num) i = ++j; + } + } else { + // Store the arcs in the original order + ArcsType i = 0; + Arc a; _graph.first(a); + for (; a != INVALID; _graph.next(a), ++i) { + _source[i] = _node_id(_graph.source(a)); + _target[i] = _node_id(_graph.target(a)); + //_arc_id[a] = i; + } + } + + // Reset parameters + resetParams(); + return *this; + } + + /// @} + + /// \name Query Functions + /// The results of the algorithm can be obtained using these + /// functions.\n + /// The \ref run() function must be called before using them. + + /// @{ + + /// \brief Return the total cost of the found flow. + /// + /// This function returns the total cost of the found flow. + /// Its complexity is O(e). + /// + /// \note The return type of the function can be specified as a + /// template parameter. For example, + /// \code + /// ns.totalCost(); + /// \endcode + /// It is useful if the total cost cannot be stored in the \c Cost + /// type of the algorithm, which is the default return type of the + /// function. + /// + /// \pre \ref run() must be called before using this function. + /*template + Number totalCost() const { + Number c = 0; + for (ArcIt a(_graph); a != INVALID; ++a) { + int64_t i = getArcID(a); + c += Number(_flow[i]) * Number(_cost[i]); + } + return c; + }*/ + + template + Number totalCost() const { + Number c = 0; + + /*#ifdef HASHMAP + typename stdext::hash_map::const_iterator it; + #else + typename std::map::const_iterator it; + #endif + for (it = _flow.data.begin(); it!=_flow.data.end(); ++it) + c += Number(it->second) * Number(_cost[it->first]); + return c;*/ + + for (ArcsType i=0; i<_flow.size(); i++) + c += _flow[i] * Number(_cost[i]); + return c; + + } + +#ifndef DOXYGEN + Cost totalCost() const { + return totalCost(); + } +#endif + + /// \brief Return the flow on the given arc. + /// + /// This function returns the flow on the given arc. + /// + /// \pre \ref run() must be called before using this function. + Value flow(const Arc& a) const { + return _flow[getArcID(a)]; + } + + /// \brief Return the flow map (the primal solution). + /// + /// This function copies the flow value on each arc into the given + /// map. The \c Value type of the algorithm must be convertible to + /// the \c Value type of the map. + /// + /// \pre \ref run() must be called before using this function. + template + void flowMap(FlowMap &map) const { + Arc a; _graph.first(a); + for (; a != INVALID; _graph.next(a)) { + map.set(a, _flow[getArcID(a)]); + } + } + + /// \brief Return the potential (dual value) of the given node. + /// + /// This function returns the potential (dual value) of the + /// given node. + /// + /// \pre \ref run() must be called before using this function. + Cost potential(const Node& n) const { + return _pi[_node_id(n)]; + } + + /// \brief Return the potential map (the dual solution). + /// + /// This function copies the potential (dual value) of each node + /// into the given map. + /// The \c Cost type of the algorithm must be convertible to the + /// \c Value type of the map. + /// + /// \pre \ref run() must be called before using this function. + template + void potentialMap(PotentialMap &map) const { + Node n; _graph.first(n); + for (; n != INVALID; _graph.next(n)) { + map.set(n, _pi[_node_id(n)]); + } + } + + /// @} + + private: + + // Initialize internal data structures + bool init() { + if (_node_num == 0) return false; + + // Check the sum of supply values + _sum_supply = 0; + for (int i = 0; i != _node_num; ++i) { + _sum_supply += _supply[i]; + } + if ( fabs(_sum_supply) > _EPSILON ) return false; + + _sum_supply = 0; + + // Initialize artifical cost + Cost ART_COST; + if (std::numeric_limits::is_exact) { + ART_COST = std::numeric_limits::max() / 2 + 1; + } else { + ART_COST = 0; + for (ArcsType i = 0; i != _arc_num; ++i) { + if (_cost[i] > ART_COST) ART_COST = _cost[i]; + } + ART_COST = (ART_COST + 1) * _node_num; + } + + // Initialize arc maps + for (ArcsType i = 0; i != _arc_num; ++i) { + //_flow[i] = 0; //by default, the sparse matrix is empty + _state[i] = STATE_LOWER; + } + + // Set data for the artificial root node + _root = _node_num; + _parent[_root] = -1; + _pred[_root] = -1; + _thread[_root] = 0; + _rev_thread[0] = _root; + _succ_num[_root] = _node_num + 1; + _last_succ[_root] = _root - 1; + _supply[_root] = -_sum_supply; + _pi[_root] = 0; + + // Add artificial arcs and initialize the spanning tree data structure + if (_sum_supply == 0) { + // EQ supply constraints + _search_arc_num = _arc_num; + _all_arc_num = _arc_num + _node_num; + for (ArcsType u = 0, e = _arc_num; u != _node_num; ++u, ++e) { + _parent[u] = _root; + _pred[u] = e; + _thread[u] = u + 1; + _rev_thread[u + 1] = u; + _succ_num[u] = 1; + _last_succ[u] = u; + _state[e] = STATE_TREE; + if (_supply[u] >= 0) { + _forward[u] = true; + _pi[u] = 0; + _source[e] = u; + _target[e] = _root; + _flow[e] = _supply[u]; + _cost[e] = 0; + } else { + _forward[u] = false; + _pi[u] = ART_COST; + _source[e] = _root; + _target[e] = u; + _flow[e] = -_supply[u]; + _cost[e] = ART_COST; + } + } + } + else if (_sum_supply > 0) { + // LEQ supply constraints + _search_arc_num = _arc_num + _node_num; + ArcsType f = _arc_num + _node_num; + for (ArcsType u = 0, e = _arc_num; u != _node_num; ++u, ++e) { + _parent[u] = _root; + _thread[u] = u + 1; + _rev_thread[u + 1] = u; + _succ_num[u] = 1; + _last_succ[u] = u; + if (_supply[u] >= 0) { + _forward[u] = true; + _pi[u] = 0; + _pred[u] = e; + _source[e] = u; + _target[e] = _root; + _flow[e] = _supply[u]; + _cost[e] = 0; + _state[e] = STATE_TREE; + } else { + _forward[u] = false; + _pi[u] = ART_COST; + _pred[u] = f; + _source[f] = _root; + _target[f] = u; + _flow[f] = -_supply[u]; + _cost[f] = ART_COST; + _state[f] = STATE_TREE; + _source[e] = u; + _target[e] = _root; + //_flow[e] = 0; //by default, the sparse matrix is empty + _cost[e] = 0; + _state[e] = STATE_LOWER; + ++f; + } + } + _all_arc_num = f; + } + else { + // GEQ supply constraints + _search_arc_num = _arc_num + _node_num; + ArcsType f = _arc_num + _node_num; + for (ArcsType u = 0, e = _arc_num; u != _node_num; ++u, ++e) { + _parent[u] = _root; + _thread[u] = u + 1; + _rev_thread[u + 1] = u; + _succ_num[u] = 1; + _last_succ[u] = u; + if (_supply[u] <= 0) { + _forward[u] = false; + _pi[u] = 0; + _pred[u] = e; + _source[e] = _root; + _target[e] = u; + _flow[e] = -_supply[u]; + _cost[e] = 0; + _state[e] = STATE_TREE; + } else { + _forward[u] = true; + _pi[u] = -ART_COST; + _pred[u] = f; + _source[f] = u; + _target[f] = _root; + _flow[f] = _supply[u]; + _state[f] = STATE_TREE; + _cost[f] = ART_COST; + _source[e] = _root; + _target[e] = u; + //_flow[e] = 0; //by default, the sparse matrix is empty + _cost[e] = 0; + _state[e] = STATE_LOWER; + ++f; + } + } + _all_arc_num = f; + } + + return true; + } + + // Find the join node + void findJoinNode() { + int u = _source[in_arc]; + int v = _target[in_arc]; + while (u != v) { + if (_succ_num[u] < _succ_num[v]) { + u = _parent[u]; + } else { + v = _parent[v]; + } + } + join = u; + } + + // Find the leaving arc of the cycle and returns true if the + // leaving arc is not the same as the entering arc + bool findLeavingArc() { + // Initialize first and second nodes according to the direction + // of the cycle + if (_state[in_arc] == STATE_LOWER) { + first = _source[in_arc]; + second = _target[in_arc]; + } else { + first = _target[in_arc]; + second = _source[in_arc]; + } + delta = INF; + char result = 0; + Value d; + ArcsType e; + + // Search the cycle along the path form the first node to the root + for (int u = first; u != join; u = _parent[u]) { + e = _pred[u]; + d = _forward[u] ? _flow[e] : INF ; + if (d < delta) { + delta = d; + u_out = u; + result = 1; + } + } + // Search the cycle along the path form the second node to the root + for (int u = second; u != join; u = _parent[u]) { + e = _pred[u]; + d = _forward[u] ? INF : _flow[e]; + if (d <= delta) { + delta = d; + u_out = u; + result = 2; + } + } + + if (result == 1) { + u_in = first; + v_in = second; + } else { + u_in = second; + v_in = first; + } + return result != 0; + } + + // Change _flow and _state vectors + void changeFlow(bool change) { + // Augment along the cycle + if (delta > 0) { + Value val = _state[in_arc] * delta; + _flow[in_arc] += val; + for (int u = _source[in_arc]; u != join; u = _parent[u]) { + _flow[_pred[u]] += _forward[u] ? -val : val; + } + for (int u = _target[in_arc]; u != join; u = _parent[u]) { + _flow[_pred[u]] += _forward[u] ? val : -val; + } + } + // Update the state of the entering and leaving arcs + if (change) { + _state[in_arc] = STATE_TREE; + _state[_pred[u_out]] = + (_flow[_pred[u_out]] == 0) ? STATE_LOWER : STATE_UPPER; + } else { + _state[in_arc] = -_state[in_arc]; + } + } + + // Update the tree structure + void updateTreeStructure() { + int u, w; + int old_rev_thread = _rev_thread[u_out]; + int old_succ_num = _succ_num[u_out]; + int old_last_succ = _last_succ[u_out]; + v_out = _parent[u_out]; + + u = _last_succ[u_in]; // the last successor of u_in + right = _thread[u]; // the node after it + + // Handle the case when old_rev_thread equals to v_in + // (it also means that join and v_out coincide) + if (old_rev_thread == v_in) { + last = _thread[_last_succ[u_out]]; + } else { + last = _thread[v_in]; + } + + // Update _thread and _parent along the stem nodes (i.e. the nodes + // between u_in and u_out, whose parent have to be changed) + _thread[v_in] = stem = u_in; + _dirty_revs.clear(); + _dirty_revs.push_back(v_in); + par_stem = v_in; + while (stem != u_out) { + // Insert the next stem node into the thread list + new_stem = _parent[stem]; + _thread[u] = new_stem; + _dirty_revs.push_back(u); + + // Remove the subtree of stem from the thread list + w = _rev_thread[stem]; + _thread[w] = right; + _rev_thread[right] = w; + + // Change the parent node and shift stem nodes + _parent[stem] = par_stem; + par_stem = stem; + stem = new_stem; + + // Update u and right + u = _last_succ[stem] == _last_succ[par_stem] ? + _rev_thread[par_stem] : _last_succ[stem]; + right = _thread[u]; + } + _parent[u_out] = par_stem; + _thread[u] = last; + _rev_thread[last] = u; + _last_succ[u_out] = u; + + // Remove the subtree of u_out from the thread list except for + // the case when old_rev_thread equals to v_in + // (it also means that join and v_out coincide) + if (old_rev_thread != v_in) { + _thread[old_rev_thread] = right; + _rev_thread[right] = old_rev_thread; + } + + // Update _rev_thread using the new _thread values + for (int i = 0; i != int(_dirty_revs.size()); ++i) { + int u = _dirty_revs[i]; + _rev_thread[_thread[u]] = u; + } + + // Update _pred, _forward, _last_succ and _succ_num for the + // stem nodes from u_out to u_in + int tmp_sc = 0, tmp_ls = _last_succ[u_out]; + u = u_out; + while (u != u_in) { + w = _parent[u]; + _pred[u] = _pred[w]; + _forward[u] = !_forward[w]; + tmp_sc += _succ_num[u] - _succ_num[w]; + _succ_num[u] = tmp_sc; + _last_succ[w] = tmp_ls; + u = w; + } + _pred[u_in] = in_arc; + _forward[u_in] = (u_in == _source[in_arc]); + _succ_num[u_in] = old_succ_num; + + // Set limits for updating _last_succ form v_in and v_out + // towards the root + int up_limit_in = -1; + int up_limit_out = -1; + if (_last_succ[join] == v_in) { + up_limit_out = join; + } else { + up_limit_in = join; + } + + // Update _last_succ from v_in towards the root + for (u = v_in; u != up_limit_in && _last_succ[u] == v_in; + u = _parent[u]) { + _last_succ[u] = _last_succ[u_out]; + } + // Update _last_succ from v_out towards the root + if (join != old_rev_thread && v_in != old_rev_thread) { + for (u = v_out; u != up_limit_out && _last_succ[u] == old_last_succ; + u = _parent[u]) { + _last_succ[u] = old_rev_thread; + } + } else { + for (u = v_out; u != up_limit_out && _last_succ[u] == old_last_succ; + u = _parent[u]) { + _last_succ[u] = _last_succ[u_out]; + } + } + + // Update _succ_num from v_in to join + for (u = v_in; u != join; u = _parent[u]) { + _succ_num[u] += old_succ_num; + } + // Update _succ_num from v_out to join + for (u = v_out; u != join; u = _parent[u]) { + _succ_num[u] -= old_succ_num; + } + } + + // Update potentials + void updatePotential() { + Cost sigma = _forward[u_in] ? + _pi[v_in] - _pi[u_in] - _cost[_pred[u_in]] : + _pi[v_in] - _pi[u_in] + _cost[_pred[u_in]]; + // Update potentials in the subtree, which has been moved + int end = _thread[_last_succ[u_in]]; + for (int u = u_in; u != end; u = _thread[u]) { + _pi[u] += sigma; + } + } + + // Heuristic initial pivots + bool initialPivots() { + Value curr, total = 0; + std::vector supply_nodes, demand_nodes; + Node u; _graph.first(u); + for (; u != INVALIDNODE; _graph.next(u)) { + curr = _supply[_node_id(u)]; + if (curr > 0) { + total += curr; + supply_nodes.push_back(u); + } + else if (curr < 0) { + demand_nodes.push_back(u); + } + } + if (_sum_supply > 0) total -= _sum_supply; + if (total <= 0) return true; + + ArcVector arc_vector; + if (_sum_supply >= 0) { + if (supply_nodes.size() == 1 && demand_nodes.size() == 1) { + // Perform a reverse graph search from the sink to the source + //typename GR::template NodeMap reached(_graph, false); + BoolVector reached(_node_num, false); + Node s = supply_nodes[0], t = demand_nodes[0]; + std::vector stack; + reached[t] = true; + stack.push_back(t); + while (!stack.empty()) { + Node u, v = stack.back(); + stack.pop_back(); + if (v == s) break; + Arc a; _graph.firstIn(a, v); + for (; a != INVALID; _graph.nextIn(a)) { + if (reached[u = _graph.source(a)]) continue; + ArcsType j = getArcID(a); + if (INF >= total) { + arc_vector.push_back(j); + reached[u] = true; + stack.push_back(u); + } + } + } + } else { + // Find the min. cost incomming arc for each demand node + for (int i = 0; i != demand_nodes.size(); ++i) { + Node v = demand_nodes[i]; + Cost c, min_cost = std::numeric_limits::max(); + Arc min_arc = INVALID; + Arc a; _graph.firstIn(a, v); + for (; a != INVALID; _graph.nextIn(a)) { + c = _cost[getArcID(a)]; + if (c < min_cost) { + min_cost = c; + min_arc = a; + } + } + if (min_arc != INVALID) { + arc_vector.push_back(getArcID(min_arc)); + } + } + } + } else { + // Find the min. cost outgoing arc for each supply node + for (int i = 0; i != int(supply_nodes.size()); ++i) { + Node u = supply_nodes[i]; + Cost c, min_cost = std::numeric_limits::max(); + Arc min_arc = INVALID; + Arc a; _graph.firstOut(a, u); + for (; a != INVALID; _graph.nextOut(a)) { + c = _cost[getArcID(a)]; + if (c < min_cost) { + min_cost = c; + min_arc = a; + } + } + if (min_arc != INVALID) { + arc_vector.push_back(getArcID(min_arc)); + } + } + } + + // Perform heuristic initial pivots + for (ArcsType i = 0; i != arc_vector.size(); ++i) { + in_arc = arc_vector[i]; + // l'erreur est probablement ici... + if (_state[in_arc] * (_cost[in_arc] + _pi[_source[in_arc]] - + _pi[_target[in_arc]]) >= 0) continue; + findJoinNode(); + bool change = findLeavingArc(); + if (delta >= MAX) return false; + changeFlow(change); + if (change) { + updateTreeStructure(); + updatePotential(); + } + } + return true; + } + + // Execute the algorithm + ProblemType start() { + return start(); + } + + template + ProblemType start() { + PivotRuleImpl pivot(*this); + ProblemType retVal = OPTIMAL; + + // Perform heuristic initial pivots + if (!initialPivots()) return UNBOUNDED; + + uint64_t iter_number = 0; + //pivot.setDantzig(true); + // Execute the Network Simplex algorithm + while (pivot.findEnteringArc()) { + if(max_iter > 0 && ++iter_number>=max_iter&&max_iter>0){ + // max iterations hit + retVal = MAX_ITER_REACHED; + break; + } +#if DEBUG_LVL>0 + if(iter_number>MAX_DEBUG_ITER) + break; + if(iter_number%1000==0||iter_number%1000==1){ + double curCost=totalCost(); + double sumFlow=0; + double a; + a= (fabs(_pi[_source[in_arc]])>=fabs(_pi[_target[in_arc]])) ? fabs(_pi[_source[in_arc]]) : fabs(_pi[_target[in_arc]]); + a=a>=fabs(_cost[in_arc])?a:fabs(_cost[in_arc]); + for (int64_t i=0; i<_flow.size(); i++) { + sumFlow+=_state[i]*_flow[i]; + } + std::cout << "Sum of the flow " << std::setprecision(20) << sumFlow << "\n" << iter_number << " iterations, current cost=" << curCost << "\nReduced cost=" << _state[in_arc] * (_cost[in_arc] + _pi[_source[in_arc]] -_pi[_target[in_arc]]) << "\nPrecision = "<< -EPSILON*(a) << "\n"; + std::cout << "Arc in = (" << _node_id(_source[in_arc]) << ", " << _node_id(_target[in_arc]) <<")\n"; + std::cout << "Supplies = (" << _supply[_source[in_arc]] << ", " << _supply[_target[in_arc]] << ")\n"; + std::cout << _cost[in_arc] << "\n"; + std::cout << _pi[_source[in_arc]] << "\n"; + std::cout << _pi[_target[in_arc]] << "\n"; + std::cout << a << "\n"; + } +#endif + + findJoinNode(); + bool change = findLeavingArc(); + if (delta >= MAX) return UNBOUNDED; + changeFlow(change); + if (change) { + updateTreeStructure(); + updatePotential(); + } +#if DEBUG_LVL>0 + else{ + std::cout << "No change\n"; + } +#endif +#if DEBUG_LVL>1 + std::cout << "Arc in = (" << _source[in_arc] << ", " << _target[in_arc] << ")\n"; +#endif + + } + + +#if DEBUG_LVL>0 + double curCost=totalCost(); + double sumFlow=0; + double a; + a= (fabs(_pi[_source[in_arc]])>=fabs(_pi[_target[in_arc]])) ? fabs(_pi[_source[in_arc]]) : fabs(_pi[_target[in_arc]]); + a=a>=fabs(_cost[in_arc])?a:fabs(_cost[in_arc]); + for (int64_t i=0; i<_flow.size(); i++) { + sumFlow+=_state[i]*_flow[i]; + } + + std::cout << "Sum of the flow " << std::setprecision(20) << sumFlow << "\n" << niter << " iterations, current cost=" << curCost << "\nReduced cost=" << _state[in_arc] * (_cost[in_arc] + _pi[_source[in_arc]] -_pi[_target[in_arc]]) << "\nPrecision = "<< -EPSILON*(a) << "\n"; + + std::cout << "Arc in = (" << _node_id(_source[in_arc]) << ", " << _node_id(_target[in_arc]) <<")\n"; + std::cout << "Supplies = (" << _supply[_source[in_arc]] << ", " << _supply[_target[in_arc]] << ")\n"; + +#endif + +#if DEBUG_LVL>1 + sumFlow=0; + for (int i=0; i<_flow.size(); i++) { + sumFlow+=_state[i]*_flow[i]; + if (_state[i]==STATE_TREE) { + std::cout << "Non zero value at (" << _node_num+1-_source[i] << ", " << _node_num+1-_target[i] << ")\n"; + } + } + std::cout << "Sum of the flow " << sumFlow << "\n"<< niter <<" iterations, current cost=" << totalCost() << "\n"; +#endif + // Check feasibility + if( retVal == OPTIMAL){ + for (ArcsType e = _search_arc_num; e != _all_arc_num; ++e) { + if (_flow[e] != 0){ + if (fabs(_flow[e]) > _EPSILON) // change of the original code following issue #126 + return INFEASIBLE; + else + _flow[e]=0; + + } + } + } + + // Shift potentials to meet the requirements of the GEQ/LEQ type + // optimality conditions + if (_sum_supply == 0) { + if (_stype == GEQ) { + Cost max_pot = -std::numeric_limits::max(); + for (ArcsType i = 0; i != _node_num; ++i) { + if (_pi[i] > max_pot) max_pot = _pi[i]; + } + if (max_pot > 0) { + for (ArcsType i = 0; i != _node_num; ++i) + _pi[i] -= max_pot; + } + } else { + Cost min_pot = std::numeric_limits::max(); + for (ArcsType i = 0; i != _node_num; ++i) { + if (_pi[i] < min_pot) min_pot = _pi[i]; + } + if (min_pot < 0) { + for (ArcsType i = 0; i != _node_num; ++i) + _pi[i] -= min_pot; + } + } + } + + return retVal; + } + + }; //class NetworkSimplexSimple + + ///@} + +} //namespace lemon From 62ae84e3fc1da0b12047cb57c3c2d8c80fa5884b Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 30 Oct 2026 02:59:23 -0700 Subject: [PATCH 02/68] feat(vamana): let the GEM distances drive Vamana builds DefaultBuildAdaptor default-constructs its distance instead of copying the one passed to the builder, which does not compile for QEmd and QChamfer (they hold their centroid table), and PruneStrategy has no primary template. Override both for these functors. Co-Authored-By: Claude Opus 5.5 --- include/svs/extensions/vamana/gem.h | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 include/svs/extensions/vamana/gem.h diff --git a/include/svs/extensions/vamana/gem.h b/include/svs/extensions/vamana/gem.h new file mode 100644 index 000000000..b17b8efc8 --- /dev/null +++ b/include/svs/extensions/vamana/gem.h @@ -0,0 +1,59 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// index/vamana depends on core/distance, not the reverse, so these overrides live here +// rather than in core/distance/gem.h. +#include "svs/core/distance/gem.h" +#include "svs/index/vamana/extensions.h" +#include "svs/index/vamana/prune.h" + +namespace svs::index::vamana { + +// qEMD is "smaller is better" like L2 (std::less<>), so share its MRNG pruning +// bookkeeping rather than IP/cosine's. +template <> struct PruneStrategy { + using type = ProgressivePruneStrategy; +}; + +// qCH is also std::less<>; the MVG ablation (--metric qch) builds the graph with QChamfer +// directly, so it needs the same pruning strategy as QEmd. +template <> struct PruneStrategy { + using type = ProgressivePruneStrategy; +}; + +namespace extensions { + +// DefaultBuildAdaptor default-constructs its distance_ member, which drops the +// CentroidTable svs::distance::QEmd holds; forward the real functor through aggregate-init +// instead. +inline DefaultBuildAdaptor svs_invoke( + svs::tag_t, const auto& /*data*/, const svs::distance::QEmd& distance +) { + return DefaultBuildAdaptor{distance}; +} + +// Same reason as the QEmd overload above: QChamfer also holds a CentroidTable that a +// default-constructed distance_ would drop. +inline DefaultBuildAdaptor svs_invoke( + svs::tag_t, const auto& /*data*/, const svs::distance::QChamfer& distance +) { + return DefaultBuildAdaptor{distance}; +} + +} // namespace extensions +} // namespace svs::index::vamana From 2fd1670f036dfd93200b059f59591d587dc0cb23 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 12 Nov 2026 08:42:32 -0800 Subject: [PATCH 03/68] feat(data): add vector-set dataset concept and non-owning CSR view A vector set is a variable-cardinality set of token vectors, as used by late-interaction retrieval. The concept comes first and the only model is a non-owning view over caller-held tokens and offsets; an owning container can follow behind the same concept. The view also models ImmutableMemoryDataset, so existing index code can traverse it. Co-Authored-By: Claude Fable 5.2 --- include/svs/concepts/vector_set.h | 99 +++++++++++++ include/svs/core/data/vector_set_view.h | 184 ++++++++++++++++++++++++ tests/svs/core/data/vector_set_view.cpp | 137 ++++++++++++++++++ 3 files changed, 420 insertions(+) create mode 100644 include/svs/concepts/vector_set.h create mode 100644 include/svs/core/data/vector_set_view.h create mode 100644 tests/svs/core/data/vector_set_view.cpp diff --git a/include/svs/concepts/vector_set.h b/include/svs/concepts/vector_set.h new file mode 100644 index 000000000..6755e5ab1 --- /dev/null +++ b/include/svs/concepts/vector_set.h @@ -0,0 +1,99 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/// +/// @ingroup concepts +/// @defgroup vector_set_concept_entry Concepts modeling late-interaction vector-set +/// datasets. +/// + +#include +#include +#include + +namespace svs { +namespace data { + +// clang-format off + +/// +/// @ingroup vector_set_concept_entry +/// @brief A variable-cardinality set of token vectors, as used by late-interaction +/// retrieval (e.g. one set per document, one token vector per word-piece embedding). +/// +/// @code{.cpp} +/// template +/// concept VectorSet = requires(const S& s, size_t i) { +/// // The element type of each token vector. +/// typename S::element_type; +/// +/// // The number of token vectors in the set. +/// { s.cardinality() } -> std::convertible_to; +/// +/// // The dimensionality shared by every token vector in the set. +/// { s.dimensions() } -> std::convertible_to; +/// +/// // The token vector at index ``i``, with ``0 <= i < cardinality()``. +/// { s.token(i) } -> std::convertible_to>; +/// }; +/// @endcode +/// +template +concept VectorSet = requires(const S& s, size_t i) { + typename S::element_type; + { s.cardinality() } -> std::convertible_to; + { s.dimensions() } -> std::convertible_to; + { s.token(i) } -> std::convertible_to>; +}; + +/// +/// @ingroup vector_set_concept_entry +/// @brief Compatibility interface for a dataset whose elements are vector sets. +/// +/// @code{.cpp} +/// template +/// concept ImmutableVectorSetDataset = requires(const D& d, size_t i) { +/// // The value returned by ``get_datum`` must itself model ``VectorSet``. +/// requires VectorSet; +/// +/// // The number of vector sets in the dataset. +/// { d.size() } -> std::convertible_to; +/// +/// // The dimensionality shared by every token vector in the dataset. +/// { d.dimensions() } -> std::convertible_to; +/// +/// // A constant handle to the vector set at index ``i``. +/// { d.get_datum(i) } -> std::convertible_to; +/// +/// // Performance optimization. Prefetch the vector set at index ``i``. +/// d.prefetch(i); +/// }; +/// @endcode +/// +template +concept ImmutableVectorSetDataset = requires(const D& d, size_t i) { + requires VectorSet; + { d.size() } -> std::convertible_to; + { d.dimensions() } -> std::convertible_to; + { d.get_datum(i) } -> std::convertible_to; + d.prefetch(i); +}; +// clang-format on + +} // namespace data +} // namespace svs diff --git a/include/svs/core/data/vector_set_view.h b/include/svs/core/data/vector_set_view.h new file mode 100644 index 000000000..15b6d7b57 --- /dev/null +++ b/include/svs/core/data/vector_set_view.h @@ -0,0 +1,184 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/vector_set.h" +#include "svs/lib/exception.h" +#include "svs/lib/misc.h" +#include "svs/lib/narrow.h" +#include "svs/lib/prefetch.h" + +// stl +#include +#include +#include +#include +#include + +namespace svs { +namespace data { + +/// +/// @brief Non-owning view of one vector set: a row-major, contiguous run of token +/// vectors sharing a dimensionality. +/// +template class ConstVectorSet { + public: + using element_type = T; + + ConstVectorSet(std::span data, size_t cardinality, size_t dimensions) + : data_{data} + , cardinality_{cardinality} + , dimensions_{dimensions} {} + + const T* data() const { return data_.data(); } + size_t cardinality() const { return cardinality_; } + constexpr size_t dimensions() const { + if constexpr (Extent != Dynamic) { + return Extent; + } else { + return dimensions_; + } + } + bool empty() const { return cardinality_ == 0; } + + std::span token(size_t i) const { + assert(i < cardinality_); + const T* base = data_.data() + i * dimensions(); + if constexpr (Extent != Dynamic) { + return std::span(base, Extent); + } else { + return std::span(base, dimensions_); + } + } + + private: + std::span data_; + size_t cardinality_; + size_t dimensions_; +}; + +static_assert(VectorSet>); + +/// +/// @brief Non-owning CSR-style view of a dataset whose elements are vector sets. +/// +/// ``offsets`` counts token vectors (not elements), has length ``size() + 1``, is +/// non-decreasing, and ``offsets[0] == 0``; ``offsets[i + 1] - offsets[i]`` is the +/// cardinality of set ``i``. +/// +template class ConstVectorSetView { + public: + using element_type = T; + using value_type = ConstVectorSet; + using const_value_type = ConstVectorSet; + + ConstVectorSetView( + std::span tokens, std::span offsets, size_t dimensions + ) + : tokens_{tokens} + , offsets_{offsets} + , dimensions_{dimensions} { + if (offsets_.empty()) { + throw ANNEXCEPTION("Vector set offsets must be non-empty, got an empty span"); + } + if (offsets_.front() != 0) { + throw ANNEXCEPTION( + "Vector set offsets must start at 0, got {}", offsets_.front() + ); + } + for (size_t i = 1; i < offsets_.size(); ++i) { + if (offsets_[i] < offsets_[i - 1]) { + throw ANNEXCEPTION( + "Vector set offsets must be non-decreasing, got {} after {}", + offsets_[i], + offsets_[i - 1] + ); + } + } + size_t total_tokens = lib::narrow(offsets_.back()); + if (dimensions_ != 0 && + total_tokens > std::numeric_limits::max() / dimensions_) { + throw ANNEXCEPTION( + "Token count {} times dimensions {} overflows size_t!", + total_tokens, + dimensions_ + ); + } + size_t expected_tokens = total_tokens * dimensions_; + if (expected_tokens != tokens_.size()) { + throw ANNEXCEPTION( + "Vector set offsets imply {} token elements but got {} token elements", + expected_tokens, + tokens_.size() + ); + } + if constexpr (Extent != Dynamic) { + if (dimensions_ != Extent) { + throw ANNEXCEPTION( + "Vector set view has static extent {} but got dimension {}", + Extent, + dimensions_ + ); + } + } + } + + size_t size() const { return offsets_.size() - 1; } + constexpr size_t dimensions() const { + if constexpr (Extent != Dynamic) { + return Extent; + } else { + return dimensions_; + } + } + size_t cardinality(size_t i) const { + assert(i < size()); + return lib::narrow(offsets_[i + 1] - offsets_[i]); + } + size_t total_tokens() const { return lib::narrow(offsets_.back()); } + + const_value_type get_datum(size_t i) const { + size_t set_cardinality = cardinality(i); + size_t begin = lib::narrow(offsets_[i]) * dimensions(); + size_t count = set_cardinality * dimensions(); + return ConstVectorSet( + tokens_.subspan(begin, count), set_cardinality, dimensions() + ); + } + + // Only the first token vector of the set is close enough to be worth prefetching; + // a large set still has to stream the rest from memory. + void prefetch(size_t i) const { + if (cardinality(i) == 0) { + return; + } + size_t begin = lib::narrow(offsets_[i]) * dimensions(); + lib::prefetch(tokens_.subspan(begin, dimensions())); + } + + private: + std::span tokens_; + std::span offsets_; + size_t dimensions_; +}; + +static_assert(ImmutableVectorSetDataset>); + +} // namespace data +} // namespace svs diff --git a/tests/svs/core/data/vector_set_view.cpp b/tests/svs/core/data/vector_set_view.cpp new file mode 100644 index 000000000..7fe651105 --- /dev/null +++ b/tests/svs/core/data/vector_set_view.cpp @@ -0,0 +1,137 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/core/data/vector_set_view.h" + +// other misc SVS stuff +#include "svs/concepts/vector_set.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include + +CATCH_TEST_CASE("Vector Set View", "[core][data][vector_set_view]") { + CATCH_SECTION("Construction and contents") { + // 3 sets with cardinalities 2, 0, 3 over dimension 2. + auto tokens = std::vector{ + 1.0f, + 2.0f, // set 0, token 0 + 3.0f, + 4.0f, // set 0, token 1 + 5.0f, + 6.0f, // set 2, token 0 + 7.0f, + 8.0f, // set 2, token 1 + 9.0f, + 10.0f // set 2, token 2 + }; + auto offsets = std::vector{0, 2, 2, 5}; + auto view = svs::data::ConstVectorSetView(tokens, offsets, 2); + + CATCH_REQUIRE(view.size() == 3); + CATCH_REQUIRE(view.dimensions() == 2); + CATCH_REQUIRE(view.total_tokens() == 5); + + CATCH_REQUIRE(view.cardinality(0) == 2); + CATCH_REQUIRE(view.cardinality(1) == 0); + CATCH_REQUIRE(view.cardinality(2) == 3); + + auto set0 = view.get_datum(0); + CATCH_REQUIRE(set0.cardinality() == 2); + CATCH_REQUIRE(set0.dimensions() == 2); + CATCH_REQUIRE(!set0.empty()); + CATCH_REQUIRE(set0.token(0)[0] == 1.0f); + CATCH_REQUIRE(set0.token(0)[1] == 2.0f); + CATCH_REQUIRE(set0.token(1)[0] == 3.0f); + CATCH_REQUIRE(set0.token(1)[1] == 4.0f); + + auto set1 = view.get_datum(1); + CATCH_REQUIRE(set1.cardinality() == 0); + CATCH_REQUIRE(set1.empty()); + + auto set2 = view.get_datum(2); + CATCH_REQUIRE(set2.cardinality() == 3); + CATCH_REQUIRE(set2.token(2)[0] == 9.0f); + CATCH_REQUIRE(set2.token(2)[1] == 10.0f); + + view.prefetch(0); + view.prefetch(1); + view.prefetch(2); + } + + CATCH_SECTION("Validation failures") { + auto tokens = std::vector{1.0f, 2.0f, 3.0f, 4.0f}; + + CATCH_REQUIRE_THROWS_AS( + svs::data::ConstVectorSetView( + std::span(tokens), std::span(), 2 + ), + svs::ANNException + ); + + auto bad_front = std::vector{1, 2}; + CATCH_REQUIRE_THROWS_AS( + svs::data::ConstVectorSetView(tokens, bad_front, 2), svs::ANNException + ); + + auto decreasing = std::vector{0, 2, 1}; + CATCH_REQUIRE_THROWS_AS( + svs::data::ConstVectorSetView(tokens, decreasing, 2), svs::ANNException + ); + + auto mismatched = std::vector{0, 1}; + CATCH_REQUIRE_THROWS_AS( + svs::data::ConstVectorSetView(tokens, mismatched, 2), svs::ANNException + ); + + auto overflowing = std::vector{0, uint64_t{1} << 63}; + CATCH_REQUIRE_THROWS_AS( + svs::data::ConstVectorSetView(std::span(), overflowing, 2), + svs::ANNException + ); + } + + CATCH_SECTION("Static extent") { + auto tokens = std::vector{1.0f, 2.0f, 3.0f, 4.0f}; + auto offsets = std::vector{0, 1, 2}; + + auto view = svs::data::ConstVectorSetView(tokens, offsets, 2); + CATCH_REQUIRE(view.dimensions() == 2); + CATCH_REQUIRE(view.get_datum(0).token(0)[0] == 1.0f); + + CATCH_REQUIRE_THROWS_AS( + (svs::data::ConstVectorSetView(tokens, offsets, 2)), svs::ANNException + ); + } + + CATCH_SECTION("Non-float element type") { + auto tokens = std::vector{1, 2, 3, 4, 5, 6}; + auto offsets = std::vector{0, 1, 3}; + auto view = svs::data::ConstVectorSetView(tokens, offsets, 2); + + CATCH_REQUIRE(view.size() == 2); + CATCH_REQUIRE(view.cardinality(0) == 1); + CATCH_REQUIRE(view.cardinality(1) == 2); + CATCH_REQUIRE(view.get_datum(1).token(1)[1] == 6); + + static_assert(svs::data::VectorSet); + static_assert(svs::data::ImmutableVectorSetDataset); + } +} From d558cbc9444f467479b5453a0f9c71bf47d28088 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 12 Nov 2026 08:42:34 -0800 Subject: [PATCH 04/68] feat(distance): add MaxSim score over vector sets Sum over query tokens of the maximum inner product over document tokens, built on the existing dispatched inner product so every supported element type pair works without new SIMD code. A tile-blocked kernel can replace the body of maxsim_batch behind the same signature. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/distance/maxsim.h | 85 +++++++++++++ tests/svs/core/distances/maxsim.cpp | 178 ++++++++++++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 include/svs/core/distance/maxsim.h create mode 100644 tests/svs/core/distances/maxsim.cpp diff --git a/include/svs/core/distance/maxsim.h b/include/svs/core/distance/maxsim.h new file mode 100644 index 000000000..2b38ec4f5 --- /dev/null +++ b/include/svs/core/distance/maxsim.h @@ -0,0 +1,85 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/vector_set.h" +#include "svs/core/distance/inner_product.h" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs::distance { + +/// Sum over query tokens of the max inner product over document tokens. Asymmetric; +/// larger is better; inputs are not normalised here. +struct MaxSim { + public: + using compare = std::greater<>; + + static constexpr bool implicit_broadcast = true; + static constexpr std::string_view name = "maxsim"; +}; + +inline constexpr bool operator==(const MaxSim&, const MaxSim&) { return true; } + +/// Empty document -> -infinity, strictly worse than any real score. Empty query and +/// dimension mismatch are caller errors, asserted only: a mismatch reads out of bounds. +template +float compute(MaxSim /*unused*/, const Q& query, const D& document) { + size_t n_query = query.cardinality(); + assert(n_query > 0); + size_t n_doc = document.cardinality(); + if (n_doc == 0) { + return -std::numeric_limits::infinity(); + } + assert(query.dimensions() == document.dimensions()); + + double total = 0.0; + for (size_t i = 0; i < n_query; ++i) { + auto q_token = query.token(i); + float best = -std::numeric_limits::infinity(); + for (size_t j = 0; j < n_doc; ++j) { + auto d_token = document.token(j); + float ip = IP::compute(q_token.data(), d_token.data(), q_token.size()); + best = std::max(best, ip); + } + total += best; + } + return static_cast(total); +} + +/// `get_datum(id)` must return a type modeling `svs::data::VectorSet`; `scores` must have +/// the same length as `ids`. +template +void maxsim_batch( + const Q& query, GetDatum&& get_datum, std::span ids, std::span scores +) { + assert(ids.size() == scores.size()); + for (size_t i = 0; i < ids.size(); ++i) { + scores[i] = compute(MaxSim{}, query, get_datum(ids[i])); + } +} + +} // namespace svs::distance diff --git a/tests/svs/core/distances/maxsim.cpp b/tests/svs/core/distances/maxsim.cpp new file mode 100644 index 000000000..580fe9558 --- /dev/null +++ b/tests/svs/core/distances/maxsim.cpp @@ -0,0 +1,178 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test +#include "svs/core/distance/maxsim.h" + +// svs +#include "svs/lib/float16.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stdlib +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// Minimal svs::data::VectorSet model over a vector of row-major tokens; local to this +// test so it does not depend on the ConstVectorSet view another agent is adding. +template struct TestVectorSet { + using element_type = T; + std::vector> tokens; + + size_t cardinality() const { return tokens.size(); } + size_t dimensions() const { return tokens.empty() ? 0 : tokens[0].size(); } + std::span token(size_t i) const { return std::span(tokens[i]); } +}; + +template double promote(T x) { return static_cast(x); } + +// Naive double-precision oracle: sum over query tokens of the max inner product to any +// document token. +template +double maxsim_reference(const TestVectorSet& query, const TestVectorSet& document) { + if (document.cardinality() == 0) { + return -std::numeric_limits::infinity(); + } + double total = 0.0; + for (size_t i = 0; i < query.cardinality(); ++i) { + double best = -std::numeric_limits::infinity(); + for (size_t j = 0; j < document.cardinality(); ++j) { + double dot = 0.0; + for (size_t d = 0; d < query.dimensions(); ++d) { + dot += promote(query.tokens[i][d]) * promote(document.tokens[j][d]); + } + best = std::max(best, dot); + } + total += best; + } + return total; +} + +TestVectorSet make_random_set(size_t cardinality, size_t dim, std::mt19937& rng) { + std::uniform_real_distribution dist(-1.0F, 1.0F); + TestVectorSet set; + set.tokens.resize(cardinality); + for (auto& token : set.tokens) { + token.resize(dim); + for (auto& v : token) { + v = dist(rng); + } + } + return set; +} + +constexpr double MAXSIM_EPSILON = 1e-4; + +} // namespace + +CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { + CATCH_SECTION("Trait members") { + CATCH_REQUIRE(std::is_same_v>); + CATCH_REQUIRE(svs::distance::MaxSim::implicit_broadcast); + } + + CATCH_SECTION("Hand-computed tiny case") { + // Query has two orthonormal tokens; document has one token aligned with the + // first and one aligned with neither, so the per-query maxima are 1 and 0. + TestVectorSet query{{{1.0F, 0.0F}, {0.0F, 1.0F}}}; + TestVectorSet document{{{1.0F, 0.0F}, {0.0F, 0.0F}}}; + float score = svs::distance::compute(svs::distance::MaxSim{}, query, document); + CATCH_REQUIRE(score == Catch::Approx(1.0F).epsilon(MAXSIM_EPSILON)); + } + + CATCH_SECTION("Empty document set gives -inf") { + TestVectorSet query{{{1.0F, 0.0F}}}; + TestVectorSet document{{}}; + float score = svs::distance::compute(svs::distance::MaxSim{}, query, document); + CATCH_REQUIRE(score == -std::numeric_limits::infinity()); + } + + CATCH_SECTION("Asymmetry") { + TestVectorSet a{{{1.0F, 0.0F}, {0.0F, 1.0F}, {0.0F, 1.0F}}}; + TestVectorSet b{{{1.0F, 0.0F}}}; + float score_ab = svs::distance::compute(svs::distance::MaxSim{}, a, b); + float score_ba = svs::distance::compute(svs::distance::MaxSim{}, b, a); + CATCH_REQUIRE(score_ab != Catch::Approx(score_ba)); + } + + CATCH_SECTION("Exact oracle on random sets") { + std::mt19937 rng(0xC0FFEE); + std::vector cardinalities{1, 2, 3, 7, 9, 17}; + std::vector dims{128, 37}; + for (size_t dim : dims) { + for (size_t nq : cardinalities) { + for (size_t nd : cardinalities) { + auto query = make_random_set(nq, dim, rng); + auto document = make_random_set(nd, dim, rng); + double expected = maxsim_reference(query, document); + float actual = + svs::distance::compute(svs::distance::MaxSim{}, query, document); + CATCH_REQUIRE( + static_cast(actual) == Catch::Approx(expected) + .epsilon(MAXSIM_EPSILON) + .margin(MAXSIM_EPSILON) + ); + } + } + } + } + + CATCH_SECTION("Mixed element types") { + // float query against Float16 document: an AVX-accelerated pair the existing + // inner product supports (IPImpl). + TestVectorSet query{{{1.0F, 0.0F, 0.0F}, {0.0F, 1.0F, 0.0F}}}; + TestVectorSet document{ + {{svs::Float16(1.0F), svs::Float16(0.0F), svs::Float16(0.0F)}, + {svs::Float16(0.0F), svs::Float16(0.0F), svs::Float16(1.0F)}}}; + float score = svs::distance::compute(svs::distance::MaxSim{}, query, document); + // Token 0 of query matches document token 0 exactly (dot = 1); token 1 of query + // has dot 0 with both document tokens. + CATCH_REQUIRE(score == Catch::Approx(1.0F).epsilon(MAXSIM_EPSILON)); + } + + CATCH_SECTION("Batch helper matches per-pair compute") { + std::mt19937 rng(42); + auto query = make_random_set(5, 64, rng); + std::vector> candidates; + for (size_t i = 0; i < 6; ++i) { + candidates.push_back(make_random_set(i + 1, 64, rng)); + } + std::vector ids{0, 1, 2, 3, 4, 5}; + std::vector scores(ids.size()); + svs::distance::maxsim_batch>( + query, + [&](uint32_t id) -> const TestVectorSet& { return candidates[id]; }, + std::span(ids), + std::span(scores) + ); + for (size_t i = 0; i < ids.size(); ++i) { + float expected = + svs::distance::compute(svs::distance::MaxSim{}, query, candidates[ids[i]]); + CATCH_REQUIRE(scores[i] == expected); + } + } +} From c91096cd263c25a8a32c7704c13229699734ba11 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 12 Nov 2026 08:42:35 -0800 Subject: [PATCH 05/68] test(distance): cover the GEM distances and register the new tests Pins QEmd, QChamfer, QChamferExactQuery, CentroidTable and the Vamana build-adaptor overloads with hand-computed values before the row layout changes. Also registers the vector-set view and MaxSim tests. Co-Authored-By: Claude Fable 5.2 --- tests/CMakeLists.txt | 3 + tests/svs/core/distances/gem.cpp | 292 +++++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 tests/svs/core/distances/gem.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 88a9ae412..72668a04f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -109,11 +109,14 @@ set(TEST_SOURCES ${TEST_DIR}/svs/core/data/simple.cpp ${TEST_DIR}/svs/core/data/serialization.cpp ${TEST_DIR}/svs/core/data/view.cpp + ${TEST_DIR}/svs/core/data/vector_set_view.cpp ${TEST_DIR}/svs/core/distance.cpp ${TEST_DIR}/svs/core/distances/simd_utils.cpp ${TEST_DIR}/svs/core/distances/distance_euclidean.cpp ${TEST_DIR}/svs/core/distances/inner_product.cpp ${TEST_DIR}/svs/core/distances/cosine.cpp + ${TEST_DIR}/svs/core/distances/gem.cpp + ${TEST_DIR}/svs/core/distances/maxsim.cpp ${TEST_DIR}/svs/core/graph.cpp ${TEST_DIR}/svs/core/io/vecs.cpp ${TEST_DIR}/svs/core/io/binary.cpp diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp new file mode 100644 index 000000000..afada5ba5 --- /dev/null +++ b/tests/svs/core/distances/gem.cpp @@ -0,0 +1,292 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test +#include "svs/core/distance/gem.h" +#include "svs/extensions/vamana/gem.h" + +// svs +#include "svs/concepts/distance.h" +#include "svs/index/vamana/prune.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stdlib +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using svs::distance::CentroidTable; +using svs::distance::FineCentroids; +using svs::distance::Ground; +using svs::distance::QChamfer; +using svs::distance::QChamferExactQuery; +using svs::distance::QEmd; + +// Four unit vectors in 2D: (1,0), (0,1), (-1,0), (0,-1). Their pairwise dot products are +// all 0, 1 or -1, so both ground distances come out to round numbers by hand. +std::shared_ptr make_table(Ground ground) { + std::vector centroids{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F, 0.0F, -1.0F}; + return std::make_shared( + CentroidTable::from_centroids(centroids.data(), 4, 2, ground, 1) + ); +} + +// Token-set span layout (gem_detail::set_size/set_ids): [0] = n, [1..n] = centroid ids, +// rest padding. `pad` lets a test fill the unused tail with garbage to prove it is +// ignored. +std::vector make_set(std::vector ids, size_t width, float pad = -999.0F) { + std::vector s(width, pad); + s[0] = static_cast(ids.size()); + for (size_t i = 0; i < ids.size(); ++i) { + s[1 + i] = ids[i]; + } + return s; +} + +} // namespace + +CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { + CATCH_SECTION("Cosine ground") { + auto table = make_table(Ground::Cos); + CATCH_REQUIRE((*table)(0, 0) == Catch::Approx(0.0F)); + CATCH_REQUIRE((*table)(0, 1) == Catch::Approx(1.0F)); + CATCH_REQUIRE((*table)(0, 2) == Catch::Approx(2.0F)); + CATCH_REQUIRE((*table)(0, 3) == Catch::Approx(1.0F)); + CATCH_REQUIRE((*table)(1, 2) == Catch::Approx(1.0F)); + CATCH_REQUIRE((*table)(1, 3) == Catch::Approx(2.0F)); + CATCH_REQUIRE((*table)(2, 3) == Catch::Approx(1.0F)); + } + + CATCH_SECTION("L2 ground") { + auto table = make_table(Ground::L2); + CATCH_REQUIRE((*table)(0, 0) == Catch::Approx(0.0F)); + CATCH_REQUIRE((*table)(0, 1) == Catch::Approx(std::sqrt(2.0F))); + CATCH_REQUIRE((*table)(0, 2) == Catch::Approx(2.0F)); + CATCH_REQUIRE((*table)(1, 3) == Catch::Approx(2.0F)); + CATCH_REQUIRE((*table)(2, 3) == Catch::Approx(std::sqrt(2.0F))); + } + + CATCH_SECTION("FineCentroids holds raw rows, not a pairwise table") { + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F}; + FineCentroids fc{2, 2, flat.data()}; + CATCH_REQUIRE(fc.K == 2); + CATCH_REQUIRE(fc.dim == 2); + CATCH_REQUIRE(fc.data[0] == 1.0F); + CATCH_REQUIRE(fc.data[3] == 1.0F); + } +} + +CATCH_TEST_CASE("QChamfer", "[distance][gem]") { + auto table = make_table(Ground::Cos); + QChamfer dist{table}; + + CATCH_SECTION("Hand-computed: min over doc per query token") { + // query ids {0,1}, doc ids {2,3}: table(0,2)=2,table(0,3)=1 -> min 1; + // table(1,2)=1,table(1,3)=2 -> min 1. total = 1 + 1 = 2. + auto q = make_set({0.0F, 1.0F}, 6); + auto d = make_set({2.0F, 3.0F}, 6); + float score = dist.compute(std::span(q), std::span(d)); + CATCH_REQUIRE(score == Catch::Approx(2.0F)); + } + + CATCH_SECTION("Duplicate ids are counted, not merged") { + // query {0,0} against doc {1}: each of the two query tokens pays table(0,1)=1. + auto q = make_set({0.0F, 0.0F}, 6); + auto d = make_set({1.0F}, 6); + float score = dist.compute(std::span(q), std::span(d)); + CATCH_REQUIRE(score == Catch::Approx(2.0F)); + } + + CATCH_SECTION("n = 1") { + auto q = make_set({0.0F}, 6); + auto d = make_set({2.0F}, 6); + float score = dist.compute(std::span(q), std::span(d)); + CATCH_REQUIRE(score == Catch::Approx(2.0F)); + } + + CATCH_SECTION("Padding beyond n is ignored") { + auto q1 = make_set({0.0F, 1.0F}, 8, -999.0F); + auto q2 = make_set({0.0F, 1.0F}, 8, 777.0F); + auto d = make_set({2.0F, 3.0F}, 8, 42.0F); + float s1 = dist.compute(std::span(q1), std::span(d)); + float s2 = dist.compute(std::span(q2), std::span(d)); + CATCH_REQUIRE(s1 == Catch::Approx(s2)); + } +} + +CATCH_TEST_CASE("QEmd", "[distance][gem]") { + auto table = make_table(Ground::Cos); + QEmd dist{table}; + + CATCH_SECTION("1 vs 1") { + auto a = make_set({0.0F}, 4); + auto b = make_set({1.0F}, 4); + float score = dist.compute(std::span(a), std::span(b)); + CATCH_REQUIRE(score == Catch::Approx(1.0F)); // table(0,1) + } + + CATCH_SECTION("1 vs 2") { + // One source of mass 1 split 0.5/0.5 to two distinct sinks: 0.5*table(0,1) + + // 0.5*table(0,2) = 0.5*1 + 0.5*2 = 1.5. + auto a = make_set({0.0F}, 4); + auto b = make_set({1.0F, 2.0F}, 4); + float score = dist.compute(std::span(a), std::span(b)); + CATCH_REQUIRE(score == Catch::Approx(1.5F)); + } + + CATCH_SECTION("2 vs 2 with obvious optimal matching") { + // Masses 0.5/0.5 on each side; matching 0->3, 1->2 costs 0.5*1+0.5*1=1, strictly + // less than the cross matching 0->2, 1->3 (0.5*2+0.5*2=2). + auto a = make_set({0.0F, 1.0F}, 4); + auto b = make_set({2.0F, 3.0F}, 4); + float score = dist.compute(std::span(a), std::span(b)); + CATCH_REQUIRE(score == Catch::Approx(1.0F)); + } + + CATCH_SECTION("Duplicates merge to a single point with full mass") { + // {0,0} merges to one point of mass 1 at id 0, same as the plain "1 vs 1" case. + auto a = make_set({0.0F, 0.0F}, 4); + auto b = make_set({1.0F}, 4); + float score = dist.compute(std::span(a), std::span(b)); + CATCH_REQUIRE(score == Catch::Approx(1.0F)); + } + + CATCH_SECTION("Identical sets give 0") { + auto a = make_set({0.0F, 1.0F}, 4); + float score = dist.compute(std::span(a), std::span(a)); + CATCH_REQUIRE(score == Catch::Approx(0.0F)); + } + + CATCH_SECTION("Symmetry") { + auto a = make_set({0.0F}, 4); + auto b = make_set({1.0F, 2.0F}, 4); + float ab = dist.compute(std::span(a), std::span(b)); + float ba = dist.compute(std::span(b), std::span(a)); + CATCH_REQUIRE(ab == Catch::Approx(ba)); + } + + CATCH_SECTION("Padding beyond n is ignored") { + auto a1 = make_set({0.0F, 1.0F}, 8, -999.0F); + auto a2 = make_set({0.0F, 1.0F}, 8, 777.0F); + auto b = make_set({2.0F, 3.0F}, 8, 42.0F); + float s1 = dist.compute(std::span(a1), std::span(b)); + float s2 = dist.compute(std::span(a2), std::span(b)); + CATCH_REQUIRE(s1 == Catch::Approx(s2)); + } +} + +CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { + // Raw (non-unit) centroids: c0=(1,0), c1=(0,1), c2=(-1,0). + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F}; + auto centroids = std::make_shared(3, 2, flat.data()); + QChamferExactQuery dist{centroids}; + + // Query tokens t0=(1,0), t1=(0,1); doc uses centroid ids {0,2}. + std::vector raw_query{2.0F, 1.0F, 0.0F, 0.0F, 1.0F}; + auto doc = make_set({0.0F, 2.0F}, 4); + + svs::distance::maybe_fix_argument(dist, std::span(raw_query)); + float score = + dist.compute(std::span(raw_query), std::span(doc)); + + // Naive reference built directly from the raw query tokens and centroid vectors, + // independent of build_table/compute_decoded. + std::vector> tokens{{1.0F, 0.0F}, {0.0F, 1.0F}}; + std::vector doc_ids{0, 2}; + double total = 0.0; + for (auto [tx, ty] : tokens) { + float best = -std::numeric_limits::infinity(); + for (auto id : doc_ids) { + float cx = flat[id * 2]; + float cy = flat[id * 2 + 1]; + best = std::max(best, cx * tx + cy * ty); + } + total += 1.0 - static_cast(best); + } + float expected = static_cast(total / tokens.size()); + + CATCH_REQUIRE(score == Catch::Approx(expected)); + CATCH_REQUIRE(score == Catch::Approx(0.5F)); +} + +CATCH_TEST_CASE("GEM distance traits", "[distance][gem]") { + CATCH_STATIC_REQUIRE(std::is_same_v>); + CATCH_STATIC_REQUIRE(std::is_same_v>); + CATCH_STATIC_REQUIRE(std::is_same_v< + svs::distance::QChamferExactQuery::compare, + std::less<>>); + + CATCH_STATIC_REQUIRE(svs::distance::implicitly_broadcastable()); + CATCH_STATIC_REQUIRE(svs::distance::implicitly_broadcastable() + ); + CATCH_STATIC_REQUIRE( + !svs::distance::implicitly_broadcastable() + ); + + CATCH_STATIC_REQUIRE(!svs::distance::fix_argument_mandated()); + CATCH_STATIC_REQUIRE(!svs::distance::fix_argument_mandated()); + CATCH_STATIC_REQUIRE( + svs::distance::fix_argument_mandated() + ); +} + +CATCH_TEST_CASE("GEM Vamana extensions", "[distance][gem][vamana]") { + CATCH_SECTION("Prune strategy") { + static_assert(std::is_same_v< + svs::index::vamana::prune_strategy_t, + svs::index::vamana::ProgressivePruneStrategy>); + static_assert(std::is_same_v< + svs::index::vamana::prune_strategy_t, + svs::index::vamana::ProgressivePruneStrategy>); + } + + // A default-constructed Distance member would not compile for QEmd/QChamfer (their + // only constructor takes the CentroidTable), so these overloads existing and + // compiling already proves they do not fall back to DefaultBuildAdaptor's generic + // default-construct path. The compute() check below additionally proves the adaptor's + // distance_ is wired to the real table passed in, not some other instance. + CATCH_SECTION("build_adaptor carries the real CentroidTable through") { + auto table = make_table(Ground::Cos); + QEmd qemd{table}; + int dummy_data = 0; + auto adaptor = svs::index::vamana::extensions::build_adaptor(dummy_data, qemd); + auto a = make_set({0.0F}, 4); + auto b = make_set({1.0F}, 4); + float score = adaptor.general_distance().compute( + std::span(a), std::span(b) + ); + CATCH_REQUIRE(score == Catch::Approx(1.0F)); // table(0,1), same as the QEmd test. + + QChamfer qchamfer{table}; + auto adaptor2 = svs::index::vamana::extensions::build_adaptor(dummy_data, qchamfer); + auto q = make_set({0.0F, 1.0F}, 6); + auto d = make_set({2.0F, 3.0F}, 6); + float score2 = adaptor2.general_distance().compute( + std::span(q), std::span(d) + ); + CATCH_REQUIRE(score2 == Catch::Approx(2.0F)); + } +} From a89d85756c7d6cbb7989af78d114e43231062e44 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 12 Nov 2026 09:12:47 -0800 Subject: [PATCH 06/68] feat(vamana): re-rank the candidate pool with MaxSim WithVectorSets pairs the dataset the graph traverses with the vector sets it was derived from. The single_search overload for it runs the ordinary graph search, then scores every candidate in the search buffer with MaxSim against the query set and returns them best-first. Static indexes only: an index with id translation is rejected at compile time. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/data/with_vector_sets.h | 78 ++++ include/svs/extensions/vamana/maxsim.h | 144 ++++++++ tests/CMakeLists.txt | 1 + tests/svs/index/vamana/maxsim_rerank.cpp | 436 +++++++++++++++++++++++ 4 files changed, 659 insertions(+) create mode 100644 include/svs/core/data/with_vector_sets.h create mode 100644 include/svs/extensions/vamana/maxsim.h create mode 100644 tests/svs/index/vamana/maxsim_rerank.cpp diff --git a/include/svs/core/data/with_vector_sets.h b/include/svs/core/data/with_vector_sets.h new file mode 100644 index 000000000..cd87206ab --- /dev/null +++ b/include/svs/core/data/with_vector_sets.h @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/vector_set.h" +#include "svs/lib/exception.h" + +// stl +#include +#include + +namespace svs::data { + +/// +/// @brief Pairs a primary dataset (one row per entry, used for graph traversal) with a +/// vector-set dataset (one token-vector set per entry, used for re-ranking). +/// +/// Models ``ImmutableMemoryDataset`` by forwarding every member to ``primary()``, so +/// graph search over a ``WithVectorSets`` is identical to graph search over the primary +/// dataset alone. ``get_vector_set(i)`` is the extra accessor a Vamana re-rank extension +/// uses after the graph search has produced a candidate pool. +/// +template class WithVectorSets { + public: + using value_type = typename Primary::value_type; + using const_value_type = typename Primary::const_value_type; + using element_type = typename Primary::element_type; + + // VamanaIndex reads `Data::extent` unconditionally, outside the ImmutableMemoryDataset + // concept; omitting this breaks every VamanaIndex instantiation over this type. + static constexpr size_t extent = Primary::extent; + + WithVectorSets(Primary primary, Sets sets) + : primary_{std::move(primary)} + , sets_{std::move(sets)} { + if (primary_.size() != sets_.size()) { + throw ANNEXCEPTION( + "Primary dataset has {} entries but vector-set dataset has {} entries", + primary_.size(), + sets_.size() + ); + } + } + + size_t size() const { return primary_.size(); } + size_t dimensions() const { return primary_.dimensions(); } + const_value_type get_datum(size_t i) const { return primary_.get_datum(i); } + void prefetch(size_t i) const { primary_.prefetch(i); } + + const Primary& primary() const { return primary_; } + const Sets& vector_sets() const { return sets_; } + + /// The candidate's token-vector set, for re-ranking after graph search. + typename Sets::const_value_type get_vector_set(size_t i) const { + return sets_.get_datum(i); + } + + private: + Primary primary_; + Sets sets_; +}; + +} // namespace svs::data diff --git a/include/svs/extensions/vamana/maxsim.h b/include/svs/extensions/vamana/maxsim.h new file mode 100644 index 000000000..27f60ef80 --- /dev/null +++ b/include/svs/extensions/vamana/maxsim.h @@ -0,0 +1,144 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/core/data/with_vector_sets.h" +#include "svs/core/distance/maxsim.h" +#include "svs/index/vamana/extensions.h" +#include "svs/lib/exception.h" + +// stl +#include +#include +#include +#include + +namespace svs::index::vamana::extensions { + +namespace detail { + +// The intended caller's graph-search distance already consumes raw query tokens, so the +// primary distance must accept the query set directly against a primary datum. +template +concept AcceptsVectorSetQuery = + svs::data::VectorSet && + requires(Distance& dist, const Query& query, const PrimaryDatum& datum) { + { svs::distance::compute(dist, query, datum) } -> std::convertible_to; + }; + +// Re-sort by MaxSim score (larger is better). The buffer's own comparator is fixed at +// construction to the primary distance's direction, which may disagree with MaxSim's, so +// sort a local copy and write the order back rather than reuse `search_buffer.sort()`. +template void resort_by_maxsim(SearchBuffer& search_buffer) { + size_t valid = search_buffer.size(); + using value_type = typename SearchBuffer::value_type; + std::vector reordered(search_buffer.begin(), search_buffer.end()); + std::sort( + reordered.begin(), + reordered.end(), + [](const value_type& a, const value_type& b) { + return svs::distance::MaxSim::compare{}(a.distance(), b.distance()); + } + ); + for (size_t j = 0; j < valid; ++j) { + search_buffer[j] = reordered[j]; + } +} + +} // namespace detail + +/// Native re-rank: run the ordinary graph search with the primary distance, then +/// overwrite every candidate still in the pool with its MaxSim score against the query +/// set, and restore best-first order by that score. +template < + typename Primary, + svs::data::ImmutableVectorSetDataset Sets, + typename SearchBuffer, + typename Distance, + typename Query, + typename Search, + typename Index> + requires detail:: + AcceptsVectorSetQuery + void svs_invoke( + svs::tag_t, + const svs::data::WithVectorSets& dataset, + SearchBuffer& search_buffer, + Distance& distance, + const Query& query, + const Search& search, + const Index& index, + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + ) { + // `check_and_supplement_search_buffer` inserts candidates scored with the primary + // distance into a buffer this overload has already rescored to MaxSim; mixing the two + // would sort as if every entry were a MaxSim score. + static_assert( + !Index::needs_id_translation, + "MaxSim re-rank does not support indexes that need id translation" + ); + + if (cancel()) { + return; + } + + if (query.dimensions() != dataset.vector_sets().dimensions()) { + throw ANNEXCEPTION( + "Query token dimension {} does not match dataset token dimension {}", + query.dimensions(), + dataset.vector_sets().dimensions() + ); + } + + auto accessor = data::GetDatumAccessor(); + search(query, accessor, distance, search_buffer); + + if (cancel()) { + return; + } + + size_t valid = search_buffer.size(); + for (size_t j = 0; j < valid; ++j) { + auto& neighbor = search_buffer[j]; + float score = svs::distance::compute( + svs::distance::MaxSim{}, query, dataset.get_vector_set(neighbor.id()) + ); + neighbor.set_distance(score); + } + + detail::resort_by_maxsim(search_buffer); +} + +// The default `per_thread_batch_search` (extensions.h) dispatches per query through the +// `single_search` customization point above, which already leaves the buffer sorted +// best-first by MaxSim; no override is needed for the batch path. + +// `UsesReranking` is left at its default of `false`: the iterator's reranking path +// (include/svs/index/vamana/iterator.h) recomputes distances via `data.get_primary(id)` +// against the base distance, which this dataset does not expose and which would not be a +// MaxSim score regardless. + +template +auto svs_invoke( + svs::tag_t, + const svs::data::WithVectorSets& dataset +) { + return reconstruct_accessor(dataset.primary()); +} + +} // namespace svs::index::vamana::extensions diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 72668a04f..88c999341 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -138,6 +138,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/index/vamana/consolidate.cpp ${TEST_DIR}/svs/index/vamana/filter.cpp ${TEST_DIR}/svs/index/vamana/index.cpp + ${TEST_DIR}/svs/index/vamana/maxsim_rerank.cpp ${TEST_DIR}/svs/index/vamana/prune.cpp ${TEST_DIR}/svs/index/vamana/search_buffer.cpp ${TEST_DIR}/svs/index/vamana/search_parameters.cpp diff --git a/tests/svs/index/vamana/maxsim_rerank.cpp b/tests/svs/index/vamana/maxsim_rerank.cpp new file mode 100644 index 000000000..d689cc7f2 --- /dev/null +++ b/tests/svs/index/vamana/maxsim_rerank.cpp @@ -0,0 +1,436 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test: the Vamana re-rank extension for vector-set (MaxSim) datasets. +#include "svs/extensions/vamana/maxsim.h" + +// svs +#include "svs/concepts/vector_set.h" +#include "svs/core/data.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/core/data/with_vector_sets.h" +#include "svs/core/distance/maxsim.h" +#include "svs/core/graph.h" +#include "svs/core/query_result.h" +#include "svs/index/vamana/index.h" +#include "svs/lib/narrow.h" +#include "svs/lib/threads.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t DIM = 8; +constexpr double MAXSIM_RERANK_EPSILON = 1e-4; + +// Primary distance for the index's graph: mean of the query's tokens dotted with the +// primary row, negated. `compare = std::less<>` is deliberately the opposite direction +// from MaxSim's `std::greater<>`, to exercise the re-rank extension's comparator-flip +// handling. +struct MeanNegativeInnerProduct { + using compare = std::less<>; + static constexpr bool implicit_broadcast = true; + static constexpr std::string_view name = "test_mean_negative_ip"; + + template + float compute(const Q& query, std::span primary_row) const { + size_t dim = primary_row.size(); + size_t cardinality = query.cardinality(); + std::vector mean(dim, 0.0); + for (size_t t = 0; t < cardinality; ++t) { + auto token = query.token(t); + for (size_t d = 0; d < dim; ++d) { + mean[d] += token[d]; + } + } + double dot = 0.0; + for (size_t d = 0; d < dim; ++d) { + dot += (mean[d] / static_cast(cardinality)) * primary_row[d]; + } + return static_cast(-dot); + } + + // Build compares two primary rows directly (vamana_build.h's candidate-pool and + // reverse-edge loops), not a query set against a row, so this overload serves build. + float compute(std::span a, std::span b) const { + double dot = 0.0; + for (size_t d = 0; d < a.size(); ++d) { + dot += static_cast(a[d]) * static_cast(b[d]); + } + return static_cast(-dot); + } +}; + +void normalize(std::span token) { + double norm = 0.0; + for (float v : token) { + norm += static_cast(v) * static_cast(v); + } + norm = std::sqrt(norm); + for (float& v : token) { + v = static_cast(static_cast(v) / norm); + } +} + +// A deterministic corpus of vector sets: cardinality cycles 1..6, tokens are unit-norm +// (MaxSim assumes unit-norm tokens; see its doc comment), and the primary row for a +// document is the mean of its tokens. +struct Corpus { + std::vector tokens; + std::vector offsets; + svs::data::SimpleData primary; + + size_t size() const { return offsets.size() - 1; } + + svs::data::ConstVectorSetView vector_sets() const { + return svs::data::ConstVectorSetView( + std::span(tokens), std::span(offsets), DIM + ); + } +}; + +Corpus make_corpus(size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution dist(-1.0F, 1.0F); + + Corpus corpus; + corpus.offsets.resize(n + 1, 0); + corpus.primary = svs::data::SimpleData(n, DIM); + + std::vector mean(DIM); + std::vector token(DIM); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 1 + (i % 6); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + std::fill(mean.begin(), mean.end(), 0.0F); + for (size_t t = 0; t < cardinality; ++t) { + for (size_t d = 0; d < DIM; ++d) { + token[d] = dist(rng); + } + normalize(token); + for (size_t d = 0; d < DIM; ++d) { + mean[d] += token[d]; + } + corpus.tokens.insert(corpus.tokens.end(), token.begin(), token.end()); + } + for (size_t d = 0; d < DIM; ++d) { + mean[d] /= static_cast(cardinality); + } + corpus.primary.set_datum(i, mean); + } + return corpus; +} + +svs::index::vamana::VamanaSearchParameters search_params(size_t window, size_t capacity) { + return svs::index::vamana::VamanaSearchParameters{{window, capacity}, false, 0, 0}; +} + +// Brute-force oracle: every document's MaxSim score against `query`, sorted best-first. +template +std::vector> +brute_force_maxsim(const Query& query, const svs::data::ConstVectorSetView& docs) { + std::vector> scored; + scored.reserve(docs.size()); + for (uint32_t id = 0; id < docs.size(); ++id) { + float score = + svs::distance::compute(svs::distance::MaxSim{}, query, docs.get_datum(id)); + scored.emplace_back(id, score); + } + std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { + return svs::distance::MaxSim::compare{}(a.second, b.second); + }); + return scored; +} + +// Overwrites docs 198/199 (cardinality 1 and 2 respectively, already assigned by +// make_corpus's `1 + i % 6` pattern) and query 1 (cardinality 2) so that doc 199's second +// token wins against the query's second token, giving it the better MaxSim score, while +// its primary mean-dot score is worse than doc 198's: a deterministic witness that the +// MaxSim and primary orders of a pool can disagree, independent of the random seed. +void make_inversion_witness(Corpus& corpus, Corpus& query_corpus) { + constexpr size_t DOC_A = 198; + constexpr size_t DOC_B = 199; + constexpr size_t QUERY = 1; + + std::vector e1(DIM, 0.0F); + e1[0] = 1.0F; + std::vector e2(DIM, 0.0F); + e2[1] = 1.0F; + std::vector q2 = {0.3F, 0.95F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}; + normalize(q2); + + auto set_token = [](Corpus& c, size_t doc, size_t t, const std::vector& v) { + size_t base = (c.offsets[doc] + t) * DIM; + std::copy(v.begin(), v.end(), c.tokens.data() + base); + }; + + set_token(corpus, DOC_A, 0, e1); + corpus.primary.set_datum(DOC_A, e1); + + set_token(corpus, DOC_B, 0, e1); + set_token(corpus, DOC_B, 1, e2); + std::vector mean_b(DIM, 0.0F); + for (size_t d = 0; d < DIM; ++d) { + mean_b[d] = (e1[d] + e2[d]) / 2.0F; + } + corpus.primary.set_datum(DOC_B, mean_b); + + set_token(query_corpus, QUERY, 0, e1); + set_token(query_corpus, QUERY, 1, q2); +} + +// Checks (a)-(d) of the MaxSim rerank contract against the whole candidate pool a +// single-query search leaves in `scratch`: every entry's score matches the MaxSim oracle, +// the pool is sorted best-first by that score, the first `num_neighbors` entries are the +// brute-force MaxSim top-k restricted to the pool's own ids, and the pool is strictly +// larger than `num_neighbors` so the re-rank had more than one candidate to order. +template +void check_rerank_pool( + Index& index, + Scratch& scratch, + const Query& query, + const svs::data::ConstVectorSetView& doc_sets, + size_t num_neighbors +) { + index.search(query, scratch); + const auto& buffer = scratch.buffer; + + std::vector pool_ids; + pool_ids.reserve(buffer.size()); + float previous_score = std::numeric_limits::infinity(); + for (const auto& neighbor : buffer) { + float expected = svs::distance::compute( + svs::distance::MaxSim{}, query, doc_sets.get_datum(neighbor.id()) + ); + CATCH_REQUIRE( + static_cast(neighbor.distance()) == Catch::Approx(expected) + .epsilon(MAXSIM_RERANK_EPSILON) + .margin(MAXSIM_RERANK_EPSILON) + ); + CATCH_REQUIRE(neighbor.distance() <= previous_score); + previous_score = neighbor.distance(); + pool_ids.push_back(neighbor.id()); + } + CATCH_REQUIRE(pool_ids.size() > num_neighbors); + + std::set pool_set(pool_ids.begin(), pool_ids.end()); + auto oracle = brute_force_maxsim(query, doc_sets); + std::vector restricted_oracle; + for (const auto& scored : oracle) { + if (pool_set.contains(scored.first)) { + restricted_oracle.push_back(scored.first); + if (restricted_oracle.size() == num_neighbors) { + break; + } + } + } + for (size_t k = 0; k < num_neighbors; ++k) { + CATCH_REQUIRE(buffer[k].id() == restricted_oracle[k]); + } +} + +} // namespace + +CATCH_TEST_CASE("WithVectorSets construction", "[vamana][maxsim_rerank]") { + CATCH_SECTION("Mismatched primary/sets sizes throw") { + auto primary_corpus = make_corpus(5, 1); + auto sets_corpus = make_corpus(4, 2); + using Data = svs::data::WithVectorSets< + svs::data::SimpleData, + svs::data::ConstVectorSetView>; + CATCH_REQUIRE_THROWS(Data(primary_corpus.primary, sets_corpus.vector_sets())); + } +} + +CATCH_TEST_CASE("MaxSim re-rank Vamana extension", "[vamana][maxsim_rerank]") { + using Dist = MeanNegativeInnerProduct; + using Primary = svs::data::SimpleData; + using Sets = svs::data::ConstVectorSetView; + using Data = svs::data::WithVectorSets; + + const size_t n_docs = 200; + auto corpus = make_corpus(n_docs, 0xDEADBEEF); + auto query_corpus = make_corpus(5, 0xC0FFEE); + make_inversion_witness(corpus, query_corpus); + + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + + auto entry_point_pool = svs::threads::DefaultThreadPool(1); + auto entry_point = + svs::lib::narrow(svs::index::vamana::extensions::compute_entry_point( + corpus.primary, entry_point_pool + )); + + // The only index under test: primary dataset paired with the vector-set dataset, + // re-ranked by MaxSim through the extension in svs/extensions/vamana/maxsim.h. + // + // VamanaIndex's build-parameters constructor hard-codes L2/IP/Cosine in + // verify_and_set_default_index_parameters and throws "Unsupported distance type" for + // any other distance (include/svs/index/vamana/index.h), so the test-local distance + // has to go through VamanaBuilder directly, mirroring what that constructor does + // internally (index.h's two-phase `construct` calls) minus the distance-type check. + auto full_graph = + svs::default_graph(n_docs, build_parameters.graph_max_degree); + auto full_data = Data(corpus.primary, corpus.vector_sets()); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + full_graph, + full_data, + Dist{}, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(full_data) + ); + builder.construct(1.0F, entry_point); + builder.construct(build_parameters.alpha, entry_point); + + auto full_pool = svs::threads::DefaultThreadPool(1); + auto full_index = svs::index::vamana::VamanaIndex( + std::move(full_graph), + std::move(full_data), + entry_point, + Dist{}, + std::move(full_pool) + ); + + auto doc_sets = corpus.vector_sets(); + auto queries = query_corpus.vector_sets(); + + CATCH_SECTION("Query token dimension mismatch throws before the graph search") { + std::vector bad_tokens(DIM - 1, 0.1F); + std::vector bad_offsets{0, 1}; + auto bad_query_set = svs::data::ConstVectorSetView( + std::span(bad_tokens), + std::span(bad_offsets), + DIM - 1 + ); + auto bad_query = bad_query_set.get_datum(0); + + auto sp = search_params(40, 40); + auto full_scratch = full_index.scratchspace(sp); + CATCH_REQUIRE_THROWS(full_index.search(bad_query, full_scratch)); + } + + // Checks (a)-(d) of the rerank contract on the whole pool a single-query search + // leaves behind, at a buffer capacity (40) well under the corpus size and at a + // capacity (n_docs) that leaves every document in the pool. + for (size_t capacity : {size_t{40}, n_docs}) { + CATCH_SECTION( + "Candidate pool is scored and ordered by MaxSim, capacity " + + std::to_string(capacity) + ) { + const size_t num_neighbors = 10; + auto sp = search_params(capacity, capacity); + auto full_scratch = full_index.scratchspace(sp); + + for (size_t q = 0; q < queries.size(); ++q) { + auto query = queries.get_datum(q); + check_rerank_pool(full_index, full_scratch, query, doc_sets, num_neighbors); + } + } + } + + CATCH_SECTION("MaxSim order of the pool can differ from the primary-distance order") { + constexpr uint32_t DOC_A = 198; + constexpr uint32_t DOC_B = 199; + auto sp = search_params(n_docs, n_docs); + auto full_scratch = full_index.scratchspace(sp); + + auto query = queries.get_datum(1); + full_index.search(query, full_scratch); + const auto& buffer = full_scratch.buffer; + + auto it_a = std::find_if(buffer.begin(), buffer.end(), [](const auto& n) { + return n.id() == DOC_A; + }); + auto it_b = std::find_if(buffer.begin(), buffer.end(), [](const auto& n) { + return n.id() == DOC_B; + }); + CATCH_REQUIRE(it_a != buffer.end()); + CATCH_REQUIRE(it_b != buffer.end()); + // MaxSim ranks doc 199 ahead of doc 198 (its second token wins against the + // query's), but their primary mean-dot scores rank the opposite way. + CATCH_REQUIRE( + std::distance(buffer.begin(), it_b) < std::distance(buffer.begin(), it_a) + ); + + // The generic `distance::compute` dispatch takes the functor by non-const + // reference, so a named lvalue is required here (a temporary will not bind). + Dist dist{}; + float primary_a = + svs::distance::compute(dist, query, corpus.primary.get_datum(DOC_A)); + float primary_b = + svs::distance::compute(dist, query, corpus.primary.get_datum(DOC_B)); + CATCH_REQUIRE(Dist::compare{}(primary_a, primary_b)); + } + + CATCH_SECTION("Batch search matches the first k entries of the single-query path") { + const size_t capacity = n_docs; + const size_t num_neighbors = 8; + auto sp = search_params(capacity, capacity); + const size_t num_queries = queries.size(); + + auto full_scratch = full_index.scratchspace(sp); + using PoolEntry = typename std::decay_t::value_type; + std::vector> single_query_results; + single_query_results.reserve(num_queries); + for (size_t q = 0; q < num_queries; ++q) { + auto query = queries.get_datum(q); + full_index.search(query, full_scratch); + std::vector top_k( + full_scratch.buffer.begin(), full_scratch.buffer.begin() + num_neighbors + ); + single_query_results.push_back(std::move(top_k)); + } + + svs::QueryResult result(num_queries, num_neighbors); + full_index.search(result.view(), queries, sp); + + for (size_t q = 0; q < num_queries; ++q) { + for (size_t k = 0; k < num_neighbors; ++k) { + uint32_t id = result.index(q, k); + float score = result.distance(q, k); + CATCH_REQUIRE(id == single_query_results[q][k].id()); + CATCH_REQUIRE( + static_cast(score) == + Catch::Approx(single_query_results[q][k].distance()) + .epsilon(MAXSIM_RERANK_EPSILON) + .margin(MAXSIM_RERANK_EPSILON) + ); + } + } + } +} From aac09b4728708e1c648c96770343b1fcf104c816 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 12 Nov 2026 09:30:25 -0800 Subject: [PATCH 07/68] feat(distance): let the GEM distances read variable-length vector sets QEmd, QChamfer and QChamferExactQuery accept a ContiguousVectorSet next to the padded float row: a set of unsigned centroid ids with one id per token, and a float query set. Both layouts share the per-ISA cores, and the padded path keeps its arithmetic. The view gains the extent member VamanaIndex reads, so a graph can be built and searched over it directly. The id helpers called from the per-ISA structs become members of those structs, so each ISA keeps its own copy of them. Co-Authored-By: Claude Fable 5.2 --- include/svs/concepts/vector_set.h | 10 + include/svs/core/data/vector_set_view.h | 8 + include/svs/core/distance/gem.h | 270 ++++++++++++++++++---- tests/CMakeLists.txt | 1 + tests/svs/core/distances/gem.cpp | 185 +++++++++++++++ tests/svs/index/vamana/gem_vector_set.cpp | 222 ++++++++++++++++++ 6 files changed, 645 insertions(+), 51 deletions(-) create mode 100644 tests/svs/index/vamana/gem_vector_set.cpp diff --git a/include/svs/concepts/vector_set.h b/include/svs/concepts/vector_set.h index 6755e5ab1..3f94fa42a 100644 --- a/include/svs/concepts/vector_set.h +++ b/include/svs/concepts/vector_set.h @@ -93,6 +93,16 @@ concept ImmutableVectorSetDataset = requires(const D& d, size_t i) { { d.get_datum(i) } -> std::convertible_to; d.prefetch(i); }; + +/// +/// @ingroup vector_set_concept_entry +/// @brief A ``VectorSet`` whose tokens are stored row-major and contiguous, reachable +/// from ``data()``. +/// +template +concept ContiguousVectorSet = VectorSet && requires(const S& s) { + { s.data() } -> std::convertible_to; +}; // clang-format on } // namespace data diff --git a/include/svs/core/data/vector_set_view.h b/include/svs/core/data/vector_set_view.h index 15b6d7b57..43a73e0c3 100644 --- a/include/svs/core/data/vector_set_view.h +++ b/include/svs/core/data/vector_set_view.h @@ -17,6 +17,7 @@ #pragma once // svs +#include "svs/concepts/data.h" #include "svs/concepts/vector_set.h" #include "svs/lib/exception.h" #include "svs/lib/misc.h" @@ -74,6 +75,7 @@ template class ConstVectorSet { }; static_assert(VectorSet>); +static_assert(ContiguousVectorSet>); /// /// @brief Non-owning CSR-style view of a dataset whose elements are vector sets. @@ -87,6 +89,9 @@ template class ConstVectorSetView { using element_type = T; using value_type = ConstVectorSet; using const_value_type = ConstVectorSet; + // VamanaIndex reads Data::extent in its class body (index/vamana/index.h); without + // it this view cannot be used as an index's dataset. + static constexpr size_t extent = Extent; ConstVectorSetView( std::span tokens, std::span offsets, size_t dimensions @@ -179,6 +184,9 @@ template class ConstVectorSetView { }; static_assert(ImmutableVectorSetDataset>); +// Vamana build and batch search take the view as dataset and as query set through this +// concept. +static_assert(ImmutableMemoryDataset>); } // namespace data } // namespace svs diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index 1937d74ed..192d36cbc 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -26,6 +26,7 @@ #include "svs/third-party/pot/network_simplex_simple.h" #pragma GCC diagnostic pop +#include "svs/concepts/vector_set.h" #include "svs/core/distance/distance_core.h" #include "svs/lib/avx_detection.h" #include "svs/lib/preprocessor.h" @@ -33,6 +34,7 @@ #include #include +#include #include #include #include @@ -43,6 +45,7 @@ #include #include #include +#include #include #if defined(__x86_64__) @@ -235,31 +238,6 @@ struct MergeScratch { std::vector weights; }; -// Per-token mass is 1/n (GEM Eq. 12); tokens sharing a centroid merge into one weighted -// point. -inline size_t merge_duplicates(const float* ids_f, size_t n, MergeScratch& scratch) { - scratch.ids.resize(n); - for (size_t i = 0; i < n; ++i) { - scratch.ids[i] = static_cast(ids_f[i]); - } - std::sort(scratch.ids.begin(), scratch.ids.end()); - scratch.weights.clear(); - size_t unique_count = 0; - size_t i = 0; - while (i < n) { - size_t j = i; - while (j < n && scratch.ids[j] == scratch.ids[i]) { - ++j; - } - scratch.ids[unique_count] = scratch.ids[i]; - scratch.weights.push_back(static_cast(j - i) / static_cast(n)); - ++unique_count; - i = j; - } - scratch.ids.resize(unique_count); - return unique_count; -} - // Shared by QChamferExactImpl's generic and per-ISA compute_decoded specializations // (GEM's code_distance, src/distance/maxsim.cpp:9-45) so the reduction is written once. inline float qchamfer_exact_reduce( @@ -280,11 +258,38 @@ inline float qchamfer_exact_reduce( return static_cast(total / static_cast(n_query)); } -// Casts a token-set's float centroid ids to uint32_t with no merge; qCH counts duplicates. -inline void decode_ids(const float* ids_f, size_t n, std::vector& out) { +// Casts a token-set's centroid ids to uint32_t with no merge; qCH counts duplicates. `ids` +// is float-encoded (padded row) or already uint32_t (vector set). +template +inline void decode_ids(const T* ids, size_t n, std::vector& out) { out.resize(n); for (size_t i = 0; i < n; ++i) { - out[i] = static_cast(ids_f[i]); + out[i] = static_cast(ids[i]); + } +} + +// Returns a contiguous uint32_t id span for a 1-D id vector set with element type T: its +// own storage when T is already uint32_t (no copy), else a cast copy into `scratch`. +template +inline const uint32_t* id_span(const T* ids, size_t n, std::vector& scratch) { + if constexpr (std::is_same_v) { + return ids; + } else { + decode_ids(ids, n, scratch); + return scratch.data(); + } +} + +// An id vector set stores one centroid id per token, so dimensions() must be 1. A +// concept-constrained vector set has no static Extent to check at compile time, so this +// is the vector-set path's only dimension check. +inline void require_id_dimensions(size_t dims) { + if (dims != 1) { + throw std::invalid_argument( + "id vector set must have dimensions() == 1 (one centroid id per " + "token), got " + + std::to_string(dims) + ); } } @@ -355,12 +360,34 @@ template struct QEmdImpl { ) { thread_local gem_detail::MergeScratch scratch_a; thread_local gem_detail::MergeScratch scratch_b; - size_t ua = gem_detail::merge_duplicates( - gem_detail::set_ids(a), gem_detail::set_size(a), scratch_a - ); - size_t ub = gem_detail::merge_duplicates( - gem_detail::set_ids(b), gem_detail::set_size(b), scratch_b + size_t ua = + merge_duplicates(gem_detail::set_ids(a), gem_detail::set_size(a), scratch_a); + size_t ub = + merge_duplicates(gem_detail::set_ids(b), gem_detail::set_size(b), scratch_b); + return compute_decoded( + table, + scratch_a.ids.data(), + scratch_a.weights.data(), + ua, + scratch_b.ids.data(), + scratch_b.weights.data(), + ub ); + } + + // Same decode-merge-solve as the span overload above, for ids that are already + // uint32_t (a vector set's own storage) rather than float-encoded in a padded row. + SVS_NOINLINE static float compute( + const CentroidTable& table, + const uint32_t* ids_a, + size_t na, + const uint32_t* ids_b, + size_t nb + ) { + thread_local gem_detail::MergeScratch scratch_a; + thread_local gem_detail::MergeScratch scratch_b; + size_t ua = merge_duplicates(ids_a, na, scratch_a); + size_t ub = merge_duplicates(ids_b, nb, scratch_b); return compute_decoded( table, scratch_a.ids.data(), @@ -371,6 +398,44 @@ template struct QEmdImpl { ub ); } + + private: + // Overloads, not a free function template: QEmdImpl is explicitly instantiated + // per ISA in avx2.cpp/avx512.cpp, and a free template's inline symbol is shared + // (and ODR-violated under different -m flags) across those translation units. + static size_t merge_sorted_duplicates(size_t n, gem_detail::MergeScratch& scratch) { + std::sort(scratch.ids.begin(), scratch.ids.end()); + scratch.weights.clear(); + size_t unique_count = 0; + size_t i = 0; + while (i < n) { + size_t j = i; + while (j < n && scratch.ids[j] == scratch.ids[i]) { + ++j; + } + scratch.ids[unique_count] = scratch.ids[i]; + scratch.weights.push_back(static_cast(j - i) / static_cast(n)); + ++unique_count; + i = j; + } + scratch.ids.resize(unique_count); + return unique_count; + } + + static size_t + merge_duplicates(const float* ids, size_t n, gem_detail::MergeScratch& scratch) { + scratch.ids.resize(n); + for (size_t i = 0; i < n; ++i) { + scratch.ids[i] = static_cast(ids[i]); + } + return merge_sorted_duplicates(n, scratch); + } + + static size_t + merge_duplicates(const uint32_t* ids, size_t n, gem_detail::MergeScratch& scratch) { + scratch.ids.assign(ids, ids + n); + return merge_sorted_duplicates(n, scratch); + } }; // Dispatched like QEmdImpl. compute_decoded takes already-decoded (cast, unmerged) centroid @@ -399,12 +464,21 @@ template struct QChamferImpl { ) { thread_local std::vector qbuf; thread_local std::vector dbuf; - gem_detail::decode_ids( - gem_detail::set_ids(query), gem_detail::set_size(query), qbuf - ); - gem_detail::decode_ids(gem_detail::set_ids(doc), gem_detail::set_size(doc), dbuf); + decode_ids(gem_detail::set_ids(query), gem_detail::set_size(query), qbuf); + decode_ids(gem_detail::set_ids(doc), gem_detail::set_size(doc), dbuf); return compute_decoded(table, qbuf.data(), qbuf.size(), dbuf.data(), dbuf.size()); } + + private: + // Not gem_detail::decode_ids: QChamferImpl is explicitly instantiated per ISA + // in avx2.cpp/avx512.cpp, so this call site needs its own ISA-unique symbol rather + // than the untagged free function (shared, correctly, with baseline-only callers). + static void decode_ids(const float* ids, size_t n, std::vector& out) { + out.resize(n); + for (size_t i = 0; i < n; ++i) { + out[i] = static_cast(ids[i]); + } + } }; // Dispatched like QChamferImpl. GEM never quantizes the query token side (src/search/ @@ -546,6 +620,31 @@ class QEmd { return QEmdImpl::compute(*table_, da, db); } + // Vector-set path: ids come straight from a contiguous vector set's own storage; T == + // uint32_t skips the cast copy (gem_detail::id_span), any other unsigned type costs it. + template + requires std::unsigned_integral && + std::unsigned_integral + float compute(const Sa& a, const Sb& b) const { + gem_detail::require_id_dimensions(a.dimensions()); + gem_detail::require_id_dimensions(b.dimensions()); + thread_local std::vector buf_a; + thread_local std::vector buf_b; + const uint32_t* ids_a = gem_detail::id_span(a.data(), a.cardinality(), buf_a); + const uint32_t* ids_b = gem_detail::id_span(b.data(), b.cardinality(), buf_b); + size_t na = a.cardinality(); + size_t nb = b.cardinality(); + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { + return QEmdImpl::compute( + *table_, ids_a, na, ids_b, nb + ); + } + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { + return QEmdImpl::compute(*table_, ids_a, na, ids_b, nb); + } + return QEmdImpl::compute(*table_, ids_a, na, ids_b, nb); + } + private: std::shared_ptr table_; }; @@ -577,6 +676,36 @@ class QChamfer { return QChamferImpl::compute(*table_, dq, dd); } + // Vector-set path: compute_decoded's signature already matches an id vector set's + // own (ids pointer, cardinality) storage, so T == uint32_t needs no decode copy. + template + requires std::unsigned_integral && + std::unsigned_integral + float compute(const Sq& query, const Sd& doc) const { + gem_detail::require_id_dimensions(query.dimensions()); + gem_detail::require_id_dimensions(doc.dimensions()); + thread_local std::vector buf_q; + thread_local std::vector buf_d; + const uint32_t* q_ids = + gem_detail::id_span(query.data(), query.cardinality(), buf_q); + const uint32_t* d_ids = gem_detail::id_span(doc.data(), doc.cardinality(), buf_d); + size_t nq = query.cardinality(); + size_t nd = doc.cardinality(); + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { + return QChamferImpl::compute_decoded( + *table_, q_ids, nq, d_ids, nd + ); + } + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { + return QChamferImpl::compute_decoded( + *table_, q_ids, nq, d_ids, nd + ); + } + return QChamferImpl::compute_decoded( + *table_, q_ids, nq, d_ids, nd + ); + } + private: std::shared_ptr table_; }; @@ -607,8 +736,53 @@ class QChamferExactQuery { // count n, [1..1+n*dim) = n raw fp32 tokens row-major, rest padding. template void fix_argument(std::span query) { std::span dq{query.data(), query.size()}; - n_query_ = gem_detail::vector_set_size(dq); - const float* tokens = gem_detail::vector_set_tokens(dq); + fix_argument_decoded( + gem_detail::vector_set_tokens(dq), gem_detail::vector_set_size(dq) + ); + } + + // Vector-set path: a query set of raw (unquantized) fp32 token vectors, contiguous + // in the set's own storage, with dimensions() equal to the centroid dimension. + template + requires std::same_as + void fix_argument(const S& query) { + if (query.dimensions() != centroids_->dim) { + throw std::invalid_argument( + "QChamferExactQuery::fix_argument: query dimension " + + std::to_string(query.dimensions()) + " does not match centroid dimension " + + std::to_string(centroids_->dim) + ); + } + fix_argument_decoded(query.data(), query.cardinality()); + } + + // `query` is unused: fix_argument() already consumed it into table_/n_query_, and + // every compute() call in a search shares that one query's table. + template + float + compute(std::span /*query*/, std::span doc) const { + std::span dd{doc.data(), doc.size()}; + thread_local std::vector dbuf; + gem_detail::decode_ids(gem_detail::set_ids(dd), gem_detail::set_size(dd), dbuf); + return compute_decoded(dbuf.data(), dbuf.size()); + } + + // Vector-set path: `query` is unused for the same reason as the span overload above; + // `doc` is an id vector set, decoded with no copy when its element type is uint32_t. + template + requires std::unsigned_integral + float compute(const Query& /*query*/, const Sd& doc) const { + gem_detail::require_id_dimensions(doc.dimensions()); + thread_local std::vector buf_d; + const uint32_t* doc_ids = gem_detail::id_span(doc.data(), doc.cardinality(), buf_d); + return compute_decoded(doc_ids, doc.cardinality()); + } + + private: + // Shared by both fix_argument() overloads: builds the (K x n_query) dot table once + // per query, dispatched to the ISA the per-ISA compiled build_table was built for. + void fix_argument_decoded(const float* tokens, size_t n_query) { + n_query_ = n_query; table_.resize(centroids_->K * n_query_); if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { QChamferExactImpl::build_table( @@ -642,30 +816,24 @@ class QChamferExactQuery { } } - // `query` is unused: fix_argument() already consumed it into table_/n_query_, and - // every compute() call in a search shares that one query's table. - template - float - compute(std::span /*query*/, std::span doc) const { - std::span dd{doc.data(), doc.size()}; - thread_local std::vector dbuf; - gem_detail::decode_ids(gem_detail::set_ids(dd), gem_detail::set_size(dd), dbuf); + // Shared by both compute() overloads: reduces the per-query table against a doc's + // decoded ids, dispatched to the ISA the per-ISA compute_decoded was built for. + float compute_decoded(const uint32_t* doc_ids, size_t n_doc) const { if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { return QChamferExactImpl::compute_decoded( - table_.data(), n_query_, dbuf.data(), dbuf.size() + table_.data(), n_query_, doc_ids, n_doc ); } if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { return QChamferExactImpl::compute_decoded( - table_.data(), n_query_, dbuf.data(), dbuf.size() + table_.data(), n_query_, doc_ids, n_doc ); } return QChamferExactImpl::compute_decoded( - table_.data(), n_query_, dbuf.data(), dbuf.size() + table_.data(), n_query_, doc_ids, n_doc ); } - private: std::shared_ptr centroids_; std::vector table_; size_t n_query_ = 0; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 88c999341..9b9a914ba 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -137,6 +137,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/index/vamana/build_parameters.cpp ${TEST_DIR}/svs/index/vamana/consolidate.cpp ${TEST_DIR}/svs/index/vamana/filter.cpp + ${TEST_DIR}/svs/index/vamana/gem_vector_set.cpp ${TEST_DIR}/svs/index/vamana/index.cpp ${TEST_DIR}/svs/index/vamana/maxsim_rerank.cpp ${TEST_DIR}/svs/index/vamana/prune.cpp diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp index afada5ba5..d761f33d1 100644 --- a/tests/svs/core/distances/gem.cpp +++ b/tests/svs/core/distances/gem.cpp @@ -20,6 +20,7 @@ // svs #include "svs/concepts/distance.h" +#include "svs/core/data/vector_set_view.h" #include "svs/index/vamana/prune.h" // catch2 @@ -66,6 +67,33 @@ std::vector make_set(std::vector ids, size_t width, float pad = -9 return s; } +// Vector-set equivalent of make_set(): owns the ids contiguously with no padding and no +// float encoding, for exercising the new ConstVectorSet compute() overloads. +struct IdSet { + std::vector ids; + + svs::data::ConstVectorSet view() const { + return svs::data::ConstVectorSet( + std::span(ids), ids.size(), 1 + ); + } +}; + +IdSet make_id_set(std::vector ids) { return IdSet{std::move(ids)}; } + +// Vector-set equivalent of a raw-query padded row for QChamferExactQuery: owns `n` raw +// fp32 tokens of width `dim`, row-major, contiguously. +struct RawQuerySet { + std::vector tokens; + size_t dim; + + svs::data::ConstVectorSet view() const { + return svs::data::ConstVectorSet( + std::span(tokens), tokens.size() / dim, dim + ); + } +}; + } // namespace CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { @@ -135,6 +163,49 @@ CATCH_TEST_CASE("QChamfer", "[distance][gem]") { float s2 = dist.compute(std::span(q2), std::span(d)); CATCH_REQUIRE(s1 == Catch::Approx(s2)); } + + CATCH_SECTION("Vector-set path matches the padded-row path") { + auto q_padded = make_set({0.0F, 1.0F}, 6); + auto d_padded = make_set({2.0F, 3.0F}, 6); + float expected = dist.compute( + std::span(q_padded), std::span(d_padded) + ); + + auto q_set = make_id_set({0, 1}); + auto d_set = make_id_set({2, 3}); + float actual = dist.compute(q_set.view(), d_set.view()); + CATCH_REQUIRE(actual == Catch::Approx(expected)); + } + + CATCH_SECTION("Vector-set path: duplicates, cardinality 1, different cardinalities") { + auto q_set = make_id_set({0, 0, 1}); + auto d_set = make_id_set({2}); + float actual = dist.compute(q_set.view(), d_set.view()); + + auto q_padded = make_set({0.0F, 0.0F, 1.0F}, 8); + auto d_padded = make_set({2.0F}, 8); + float expected = dist.compute( + std::span(q_padded), std::span(d_padded) + ); + CATCH_REQUIRE(actual == Catch::Approx(expected)); + } + + CATCH_SECTION("Vector-set path: dimensions() != 1 throws") { + std::vector raw{0, 1, 2, 3}; + auto bad = + svs::data::ConstVectorSet(std::span(raw), 2, 2); + auto good = make_id_set({0}); + CATCH_REQUIRE_THROWS_AS(dist.compute(bad, good.view()), std::invalid_argument); + CATCH_REQUIRE_THROWS_AS(dist.compute(good.view(), bad), std::invalid_argument); + } + + CATCH_SECTION("Generic svs::distance::compute dispatches to the vector-set overload") { + auto q_set = make_id_set({0, 1}); + auto d_set = make_id_set({2, 3}); + float via_generic = svs::distance::compute(dist, q_set.view(), d_set.view()); + float direct = dist.compute(q_set.view(), d_set.view()); + CATCH_REQUIRE(via_generic == Catch::Approx(direct)); + } } CATCH_TEST_CASE("QEmd", "[distance][gem]") { @@ -196,6 +267,65 @@ CATCH_TEST_CASE("QEmd", "[distance][gem]") { float s2 = dist.compute(std::span(a2), std::span(b)); CATCH_REQUIRE(s1 == Catch::Approx(s2)); } + + CATCH_SECTION("Vector-set path matches the padded-row path") { + auto a_padded = make_set({0.0F, 1.0F}, 4); + auto b_padded = make_set({2.0F, 3.0F}, 4); + float expected = dist.compute( + std::span(a_padded), std::span(b_padded) + ); + + auto a_set = make_id_set({0, 1}); + auto b_set = make_id_set({2, 3}); + float actual = dist.compute(a_set.view(), b_set.view()); + CATCH_REQUIRE(actual == Catch::Approx(expected)); + } + + CATCH_SECTION( + "Vector-set path: duplicates merge, cardinality 1, different cardinalities" + ) { + auto a_set = make_id_set({0, 0}); + auto b_set = make_id_set({1}); + float actual = dist.compute(a_set.view(), b_set.view()); + + auto a_padded = make_set({0.0F, 0.0F}, 4); + auto b_padded = make_set({1.0F}, 4); + float expected = dist.compute( + std::span(a_padded), std::span(b_padded) + ); + CATCH_REQUIRE(actual == Catch::Approx(expected)); + + // One source of mass 1 at id 0 split 0.5/0.5 to sinks 1 and 2 (table(0,1)=1, + // table(0,2)=2): 0.5*1 + 0.5*2 = 1.5, same derivation as the "1 vs 2" case above. + auto c_set = make_id_set({0}); + auto d_set = make_id_set({1, 2}); + float cardinality_mismatch = dist.compute(c_set.view(), d_set.view()); + CATCH_REQUIRE(cardinality_mismatch == Catch::Approx(1.5F)); + + auto c_padded = make_set({0.0F}, 4); + auto d_padded = make_set({1.0F, 2.0F}, 4); + float cardinality_mismatch_expected = dist.compute( + std::span(c_padded), std::span(d_padded) + ); + CATCH_REQUIRE(cardinality_mismatch == Catch::Approx(cardinality_mismatch_expected)); + } + + CATCH_SECTION("Vector-set path: dimensions() != 1 throws") { + std::vector raw{0, 1, 2, 3}; + auto bad = + svs::data::ConstVectorSet(std::span(raw), 2, 2); + auto good = make_id_set({0}); + CATCH_REQUIRE_THROWS_AS(dist.compute(bad, good.view()), std::invalid_argument); + CATCH_REQUIRE_THROWS_AS(dist.compute(good.view(), bad), std::invalid_argument); + } + + CATCH_SECTION("Generic svs::distance::compute dispatches to the vector-set overload") { + auto a_set = make_id_set({0}); + auto b_set = make_id_set({1, 2}); + float via_generic = svs::distance::compute(dist, a_set.view(), b_set.view()); + float direct = dist.compute(a_set.view(), b_set.view()); + CATCH_REQUIRE(via_generic == Catch::Approx(direct)); + } } CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { @@ -232,6 +362,61 @@ CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { CATCH_REQUIRE(score == Catch::Approx(0.5F)); } +CATCH_TEST_CASE("QChamferExactQuery vector-set path", "[distance][gem]") { + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F}; + auto centroids = std::make_shared(3, 2, flat.data()); + + // Same raw query and doc ids as the padded-row "QChamferExactQuery" case above, + // reused as owning vector sets instead of a padded span. + RawQuerySet query_set{{1.0F, 0.0F, 0.0F, 1.0F}, 2}; + auto doc_set = make_id_set({0, 2}); + + CATCH_SECTION("Matches the padded-row path") { + QChamferExactQuery padded_dist{centroids}; + std::vector raw_query{2.0F, 1.0F, 0.0F, 0.0F, 1.0F}; + auto doc_padded = make_set({0.0F, 2.0F}, 4); + svs::distance::maybe_fix_argument(padded_dist, std::span(raw_query)); + float expected = padded_dist.compute( + std::span(raw_query), std::span(doc_padded) + ); + + QChamferExactQuery set_dist{centroids}; + svs::distance::maybe_fix_argument(set_dist, query_set.view()); + float actual = set_dist.compute(query_set.view(), doc_set.view()); + CATCH_REQUIRE(actual == Catch::Approx(expected)); + } + + CATCH_SECTION("Generic maybe_fix_argument/compute dispatch to the vector-set overload" + ) { + QChamferExactQuery set_dist{centroids}; + svs::distance::maybe_fix_argument(set_dist, query_set.view()); + float direct = set_dist.compute(query_set.view(), doc_set.view()); + float via_generic = + svs::distance::compute(set_dist, query_set.view(), doc_set.view()); + CATCH_REQUIRE(via_generic == Catch::Approx(direct)); + } + + CATCH_SECTION("fix_argument: query dimension mismatch throws") { + QChamferExactQuery set_dist{centroids}; + RawQuerySet wrong_dim{{1.0F, 0.0F, 0.0F}, 3}; + CATCH_REQUIRE_THROWS_AS( + svs::distance::maybe_fix_argument(set_dist, wrong_dim.view()), + std::invalid_argument + ); + } + + CATCH_SECTION("compute: doc dimensions() != 1 throws") { + QChamferExactQuery set_dist{centroids}; + svs::distance::maybe_fix_argument(set_dist, query_set.view()); + std::vector raw{0, 1, 2, 3}; + auto bad_doc = + svs::data::ConstVectorSet(std::span(raw), 2, 2); + CATCH_REQUIRE_THROWS_AS( + set_dist.compute(query_set.view(), bad_doc), std::invalid_argument + ); + } +} + CATCH_TEST_CASE("GEM distance traits", "[distance][gem]") { CATCH_STATIC_REQUIRE(std::is_same_v>); CATCH_STATIC_REQUIRE(std::is_same_v>); diff --git a/tests/svs/index/vamana/gem_vector_set.cpp b/tests/svs/index/vamana/gem_vector_set.cpp new file mode 100644 index 000000000..2c90e78ab --- /dev/null +++ b/tests/svs/index/vamana/gem_vector_set.cpp @@ -0,0 +1,222 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Proves a VamanaIndex can be built and searched directly over a +// ConstVectorSetView of centroid-id vector sets, using QChamfer as both the +// build and search distance. +#include "svs/core/distance/gem.h" +#include "svs/extensions/vamana/gem.h" + +// svs +#include "svs/core/data/vector_set_view.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/index.h" +#include "svs/lib/threads.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t K = 16; +constexpr size_t DIM = 4; + +std::shared_ptr make_centroid_table(uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution coord(-1.0F, 1.0F); + std::vector centroids(K * DIM); + for (size_t k = 0; k < K; ++k) { + double norm = 0.0; + for (size_t d = 0; d < DIM; ++d) { + float v = coord(rng); + centroids[k * DIM + d] = v; + norm += static_cast(v) * static_cast(v); + } + norm = std::sqrt(norm); + for (size_t d = 0; d < DIM; ++d) { + centroids[k * DIM + d] = static_cast(centroids[k * DIM + d] / norm); + } + } + return std::make_shared( + svs::distance::CentroidTable::from_centroids( + centroids.data(), K, DIM, svs::distance::Ground::Cos, 1 + ) + ); +} + +// A deterministic dataset of id vector sets, held CSR-style exactly as +// ConstVectorSetView expects: cardinality cycles 1..6, ids are uniform over +// the K centroids. +struct IdSetCorpus { + std::vector ids; + std::vector offsets; + + size_t size() const { return offsets.size() - 1; } + + svs::data::ConstVectorSetView view() const { + return svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); + } +}; + +IdSetCorpus make_corpus(size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution id_dist(0, static_cast(K - 1)); + + IdSetCorpus corpus; + corpus.offsets.resize(n + 1, 0); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 1 + (i % 6); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + for (size_t t = 0; t < cardinality; ++t) { + corpus.ids.push_back(id_dist(rng)); + } + } + return corpus; +} + +svs::index::vamana::VamanaSearchParameters search_params(size_t window, size_t capacity) { + return svs::index::vamana::VamanaSearchParameters{{window, capacity}, false, 0, 0}; +} + +// Brute-force oracle: every document's QChamfer score against `query`, sorted best-first +// (ascending, since QChamfer::compare is std::less<>). +std::vector> brute_force_qchamfer( + svs::distance::QChamfer& dist, + const svs::data::ConstVectorSet& query, + const svs::data::ConstVectorSetView& docs +) { + std::vector> scored; + scored.reserve(docs.size()); + for (uint32_t id = 0; id < docs.size(); ++id) { + float score = svs::distance::compute(dist, query, docs.get_datum(id)); + scored.emplace_back(id, score); + } + std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { + return svs::distance::QChamfer::compare{}(a.second, b.second); + }); + return scored; +} + +} // namespace + +CATCH_TEST_CASE("Vamana build and search over a vector-set dataset", "[vamana][gem]") { + auto table = make_centroid_table(0x5EED); + svs::distance::QChamfer dist{table}; + + const size_t n_docs = 200; + auto corpus = make_corpus(n_docs, 0xDEADBEEF); + auto queries = make_corpus(10, 0xC0FFEE); + auto docs = corpus.view(); + auto query_view = queries.view(); + + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + + // Entry point 0 is caller-supplied; the vector-set path adds no medoid logic of its + // own. + const uint32_t entry_point = 0; + + // VamanaIndex's build-parameters constructor hard-codes L2/IP/Cosine in + // verify_and_set_default_index_parameters and throws for any other distance, so + // qCH (like qEMD) has to go through VamanaBuilder directly, mirroring how + // VamanaIndex's own build constructor drives it (index.h's build-parameters + // constructor) minus that distance-type check. + auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + graph, + docs, + dist, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(docs) + ); + builder.construct(build_parameters.alpha, entry_point); + + auto pool = svs::threads::DefaultThreadPool(1); + auto index = svs::index::vamana::VamanaIndex( + std::move(graph), docs, entry_point, dist, std::move(pool) + ); + + CATCH_SECTION("Small-window search returns a best-first, internally consistent pool") { + const size_t capacity = 20; + auto sp = search_params(capacity, capacity); + auto scratch = index.scratchspace(sp); + + for (size_t q = 0; q < query_view.size(); ++q) { + auto query = query_view.get_datum(q); + index.search(query, scratch); + + size_t valid = scratch.buffer.size(); + CATCH_REQUIRE(valid > 0); + float previous = -std::numeric_limits::infinity(); + for (size_t j = 0; j < valid; ++j) { + const auto& neighbor = scratch.buffer[j]; + float direct = + svs::distance::compute(dist, query, docs.get_datum(neighbor.id())); + CATCH_REQUIRE( + static_cast(neighbor.distance()) == + Catch::Approx(direct).epsilon(1e-5).margin(1e-5) + ); + CATCH_REQUIRE(neighbor.distance() >= previous); + previous = neighbor.distance(); + } + } + } + + CATCH_SECTION("Exhaustive search's top-1 matches brute force") { + const size_t capacity = n_docs; + auto sp = search_params(capacity, capacity); + auto scratch = index.scratchspace(sp); + + for (size_t q = 0; q < query_view.size(); ++q) { + auto query = query_view.get_datum(q); + index.search(query, scratch); + + auto oracle = brute_force_qchamfer(dist, query, docs); + + // Centroid ids repeat across documents, so several documents can legitimately + // tie for the best score; check the distance exactly and the id against every + // document tied for that best score, rather than oracle[0].first alone. + const auto& top1 = scratch.buffer[0]; + CATCH_REQUIRE( + static_cast(top1.distance()) == + Catch::Approx(oracle[0].second).epsilon(1e-5).margin(1e-5) + ); + bool id_ties_for_best = + std::any_of(oracle.begin(), oracle.end(), [&](const auto& entry) { + return entry.first == top1.id() && + std::abs(entry.second - oracle[0].second) < 1e-5F; + }); + CATCH_REQUIRE(id_ties_for_best); + } + } +} From 5814b8b7f1002f8bc6d5dc95d395af7f8a1f00c8 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 00:13:17 -0800 Subject: [PATCH 08/68] fix(vamana): make the MaxSim re-rank and its tests build and pass First compile of the branch. The re-rank overload names its unused index parameter, the re-rank test supplies the prune strategy its local distance needs, and the MaxSim asymmetry test uses sets whose two directions differ. Co-Authored-By: Claude Fable 5.2 --- include/svs/extensions/vamana/maxsim.h | 3 ++- tests/svs/core/distances/maxsim.cpp | 4 +++- tests/svs/index/vamana/maxsim_rerank.cpp | 12 ++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/include/svs/extensions/vamana/maxsim.h b/include/svs/extensions/vamana/maxsim.h index 27f60ef80..527424968 100644 --- a/include/svs/extensions/vamana/maxsim.h +++ b/include/svs/extensions/vamana/maxsim.h @@ -21,6 +21,7 @@ #include "svs/core/distance/maxsim.h" #include "svs/index/vamana/extensions.h" #include "svs/lib/exception.h" +#include "svs/lib/preprocessor.h" // stl #include @@ -82,7 +83,7 @@ template < Distance& distance, const Query& query, const Search& search, - const Index& index, + const Index& SVS_UNUSED(index), const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) { // `check_and_supplement_search_buffer` inserts candidates scored with the primary diff --git a/tests/svs/core/distances/maxsim.cpp b/tests/svs/core/distances/maxsim.cpp index 580fe9558..f34cf391b 100644 --- a/tests/svs/core/distances/maxsim.cpp +++ b/tests/svs/core/distances/maxsim.cpp @@ -112,7 +112,9 @@ CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { } CATCH_SECTION("Asymmetry") { - TestVectorSet a{{{1.0F, 0.0F}, {0.0F, 1.0F}, {0.0F, 1.0F}}}; + // `a` has two tokens that each max out against `b`'s lone token, so swapping + // query/document sums over a different number of maxima (2 vs. 1). + TestVectorSet a{{{1.0F, 0.0F}, {1.0F, 0.0F}}}; TestVectorSet b{{{1.0F, 0.0F}}}; float score_ab = svs::distance::compute(svs::distance::MaxSim{}, a, b); float score_ba = svs::distance::compute(svs::distance::MaxSim{}, b, a); diff --git a/tests/svs/index/vamana/maxsim_rerank.cpp b/tests/svs/index/vamana/maxsim_rerank.cpp index d689cc7f2..d81b4f0a0 100644 --- a/tests/svs/index/vamana/maxsim_rerank.cpp +++ b/tests/svs/index/vamana/maxsim_rerank.cpp @@ -93,6 +93,18 @@ struct MeanNegativeInnerProduct { } }; +} // namespace + +// MeanNegativeInnerProduct is IP-flavored (no triangle inequality), so pruning needs the +// same strategy vamana_build.h picks for DistanceIP, not the default L2 one. +namespace svs::index::vamana { +template <> struct PruneStrategy<::MeanNegativeInnerProduct> { + using type = IterativePruneStrategy; +}; +} // namespace svs::index::vamana + +namespace { + void normalize(std::span token) { double norm = 0.0; for (float v : token) { From 35feecc7c54542801491b9cfd704299096915172 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 00:31:41 -0800 Subject: [PATCH 09/68] fix(distance): keep QEmd duplicate merging callable from outside Moving merge_duplicates into the per-ISA struct made it private and broke an external timing harness that measures it separately. The two overloads are public again; behaviour is unchanged. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/distance/gem.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index 192d36cbc..a023f1c42 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -422,6 +422,9 @@ template struct QEmdImpl { return unique_count; } + public: + // Public so gem_time (external timing harness) can benchmark decode-merge separately + // from compute; otherwise only called from compute()/compute_decoded() above. static size_t merge_duplicates(const float* ids, size_t n, gem_detail::MergeScratch& scratch) { scratch.ids.resize(n); From 20d86cb21cd4d5e28fd4bea775e907efa99e89f2 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 03:03:11 -0800 Subject: [PATCH 10/68] feat(data): add an owning vector-set container with save/load VectorSetData owns the same CSR layout ConstVectorSetView reads: a SimpleData token matrix plus a std::vector of offsets, so SVS's allocators, alignment, and binary file format carry over unchanged. Construction takes both by value and moves them in, so a caller handing over existing storage pays no copy; copy_from() covers the case where a caller only has borrowed memory. A Blocked allocator is rejected at compile time since it would tear a document's tokens across blocks. Save/load follows SimpleData's protocol: the token matrix nests its own save() under a sub-table, the offsets go to a standalone binary file referenced by metadata. VectorSetDataLoader mirrors VectorDataLoader so auto_assemble can reload a saved index over this dataset; auto_assemble never calls verify_and_set_default_index_parameters (that only runs from the build-parameters constructor), so a custom distance such as QChamfer hits no obstacle there. Validation is factored out of ConstVectorSetView's constructor into a shared detail::validate_vector_set_layout so the view and the owning container cannot drift apart. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/data/vector_set.h | 335 ++++++++++++++++++++++ include/svs/core/data/vector_set_view.h | 102 ++++--- tests/CMakeLists.txt | 1 + tests/svs/core/data/vector_set.cpp | 261 +++++++++++++++++ tests/svs/index/vamana/gem_vector_set.cpp | 91 ++++++ 5 files changed, 747 insertions(+), 43 deletions(-) create mode 100644 include/svs/core/data/vector_set.h create mode 100644 tests/svs/core/data/vector_set.cpp diff --git a/include/svs/core/data/vector_set.h b/include/svs/core/data/vector_set.h new file mode 100644 index 000000000..85704df5f --- /dev/null +++ b/include/svs/core/data/vector_set.h @@ -0,0 +1,335 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/data.h" +#include "svs/concepts/vector_set.h" +#include "svs/core/data/simple.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/lib/exception.h" +#include "svs/lib/file.h" +#include "svs/lib/memory.h" +#include "svs/lib/narrow.h" +#include "svs/lib/saveload.h" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs { +namespace data { + +/// +/// @brief Owning dataset whose elements are vector sets: documents made of token +/// vectors, stored CSR-style exactly as ``ConstVectorSetView`` reads them. +/// +/// The token matrix (every token vector of every document, concatenated row-major) is +/// held in a ``SimpleData``, so SVS's allocators (including the +/// hugepage allocator), alignment, and binary file format are reused as-is. A +/// ``std::vector`` of length ``size() + 1`` holds the CSR offsets, counted in +/// token vectors. +/// +/// @tparam T The element type of each token vector. +/// @tparam Extent The compile-time dimensionality shared by every token vector, or +/// ``svs::Dynamic`` to resolve it at run time. Default: ``svs::Dynamic``. +/// @tparam Alloc The allocator used for the token matrix. Must not be a ``Blocked`` +/// allocator; see the ``static_assert`` below. +/// +template > +class VectorSetData { + // A document's tokens must land in one contiguous run (`get_datum` returns a single + // span into the token matrix); the Blocked allocator splits storage into separate + // blocks, which would tear a document's tokens across blocks. + static_assert( + !is_blocked_v, + "VectorSetData requires contiguous per-document token storage; a Blocked " + "allocator is not supported." + ); + + public: + /// The element type of each token vector. + using element_type = T; + /// The type returned by ``get_datum``: a constant handle to one document's tokens. + using value_type = ConstVectorSet; + /// @copydoc value_type + using const_value_type = ConstVectorSet; + /// The allocator type backing the token matrix. + using allocator_type = Alloc; + /// The concrete type of the token-matrix member. + using tokens_type = SimpleData; + // VamanaIndex reads Data::extent in its class body (index/vamana/index.h); without + // it this type cannot be used as an index's dataset. + static constexpr size_t extent = Extent; + + static constexpr lib::Version save_version = lib::Version(0, 0, 0); + static constexpr std::string_view serialization_schema = "vector_set_data"; + + /// + /// @brief Take ownership of an already-built token matrix and offsets array. + /// + /// @param tokens The row-major token matrix: ``total_tokens`` rows (the sum of every + /// document's cardinality) by ``dimensions()`` columns, where ``total_tokens`` + /// is derived from ``offsets.back()``. + /// @param offsets CSR-style offsets counted in token vectors: length + /// ``size() + 1``, non-decreasing, ``offsets.front() == 0``, and + /// ``offsets.back()`` equal to the number of rows in ``tokens``. + /// + /// Both arguments are consumed by move into this container; no payload copy is + /// made. Pass ``std::move(tokens)`` and ``std::move(offsets)`` to hand over a + /// caller's own storage. A caller that wants to keep an independent copy must copy + /// explicitly before calling this constructor. + /// + VectorSetData(tokens_type tokens, std::vector offsets) + : tokens_{std::move(tokens)} + , offsets_{std::move(offsets)} { + detail::validate_vector_set_layout( + offsets_, tokens_.size() * tokens_.dimensions(), tokens_.dimensions() + ); + } + + /// + /// @brief Copy a borrowed view into a new, owned ``VectorSetData``. + /// + /// @param view The borrowed vector-set view to copy from. + /// @param allocator The allocator used for the new token matrix. Default: + /// a default-constructed ``Alloc``. + /// + /// For the common case where a caller only has third-party, borrowed memory and + /// wants SVS to own a copy. The copy is performed document by document, not in + /// parallel; callers with large data who can hand over ownership instead should + /// prefer the move constructor above. + /// + static VectorSetData + copy_from(ConstVectorSetView view, const Alloc& allocator = {}) { + size_t dims = view.dimensions(); + auto tokens = tokens_type(view.total_tokens(), dims, allocator); + auto offsets = std::vector(view.size() + 1); + size_t running = 0; + for (size_t i = 0; i < view.size(); ++i) { + offsets[i] = running; + auto set = view.get_datum(i); + for (size_t j = 0, jmax = set.cardinality(); j < jmax; ++j) { + auto src = set.token(j); + auto dst = tokens.get_datum(running + j); + std::copy(src.begin(), src.end(), dst.begin()); + } + running += set.cardinality(); + } + offsets[view.size()] = running; + return VectorSetData(std::move(tokens), std::move(offsets)); + } + + /// Return the number of documents (vector sets) in the dataset. + size_t size() const { return offsets_.size() - 1; } + /// Return the dimensionality shared by every token vector. + size_t dimensions() const { return tokens_.dimensions(); } + /// Return the number of token vectors in document ``i``. + /// + /// **Preconditions:** ``0 <= i < size()``. + size_t cardinality(size_t i) const { + assert(i < size()); + return lib::narrow(offsets_[i + 1] - offsets_[i]); + } + /// Return the total number of token vectors across every document. + size_t total_tokens() const { return tokens_.size(); } + + /// + /// @brief Return a constant handle to the token vectors of document ``i``. + /// + /// **Preconditions:** ``0 <= i < size()``. + /// + const_value_type get_datum(size_t i) const { + size_t set_cardinality = cardinality(i); + size_t begin = lib::narrow(offsets_[i]); + size_t dims = dimensions(); + return ConstVectorSet( + std::span(tokens_.data() + begin * dims, set_cardinality * dims), + set_cardinality, + dims + ); + } + + /// Prefetch the first token vector of document ``i``; see + /// ``ConstVectorSetView::prefetch`` for why only the first token is prefetched. + void prefetch(size_t i) const { + if (cardinality(i) == 0) { + return; + } + tokens_.prefetch(lib::narrow(offsets_[i])); + } + + /// Return the token matrix backing this dataset. + const tokens_type& tokens() const { return tokens_; } + /// Return the CSR offsets backing this dataset, counted in token vectors. + std::span offsets() const { return offsets_; } + + /// + /// @brief Return a non-owning ``ConstVectorSetView`` over this container's storage. + /// + /// The returned view is invalidated if this container is moved from or destroyed. + /// + ConstVectorSetView view() const { + return ConstVectorSetView( + std::span(tokens_.data(), tokens_.size() * dimensions()), + offsets(), + dimensions() + ); + } + + ///// IO + + static bool check_load_compatibility(std::string_view schema, lib::Version version) { + return schema == serialization_schema && version == save_version; + } + + /// + /// @brief Save this dataset to disk. + /// + /// The token matrix is saved by delegating to its own ``save``, nested as a + /// sub-table under the ``"tokens"`` key. The offsets are written to a standalone + /// binary file (raw ``uint64_t``, no header) named through ``ctx`` and referenced by + /// the ``"offsets_file"``/``"num_offsets"`` metadata keys. + /// + lib::SaveTable save(const lib::SaveContext& ctx) const { + auto offsets_path = ctx.generate_name("vector_set_offsets", "bin"); + { + auto io = lib::open_write(offsets_path); + io.write( + reinterpret_cast(offsets_.data()), + lib::narrow(offsets_.size() * sizeof(uint64_t)) + ); + } + return lib::SaveTable( + serialization_schema, + save_version, + {{"tokens", lib::save(tokens_, ctx)}, + {"offsets_file", lib::save(offsets_path.filename())}, + {"num_offsets", lib::save(offsets_.size())}} + ); + } + + /// + /// @brief Reload a previously saved dataset. + /// + /// @param table The table containing saved hyper parameters, as produced by + /// ``save(const lib::SaveContext&)``. + /// @param allocator The allocator used for the reloaded token matrix. Default: a + /// default-constructed ``Alloc``. + /// + /// Validation is performed by the constructor through the same shared layout check + /// used by ``ConstVectorSetView``, so a corrupted offsets file or mismatched token + /// count fails with the same diagnostics as an in-memory construction would. + /// + static VectorSetData load(const lib::LoadTable& table, const Alloc& allocator = {}) { + auto tokens = lib::load_at(table, "tokens", allocator); + size_t num_offsets = lib::load_at(table, "num_offsets"); + auto offsets_path = table.resolve_at("offsets_file"); + + size_t expected_bytes = num_offsets * sizeof(uint64_t); + size_t actual_bytes = std::filesystem::file_size(offsets_path); + if (actual_bytes != expected_bytes) { + throw ANNEXCEPTION( + "Vector set offsets file {} has {} bytes but {} offsets ({} bytes) were " + "expected", + offsets_path, + actual_bytes, + num_offsets, + expected_bytes + ); + } + + auto offsets = std::vector(num_offsets); + { + auto io = lib::open_read(offsets_path); + io.read( + reinterpret_cast(offsets.data()), + lib::narrow(expected_bytes) + ); + } + return VectorSetData(std::move(tokens), std::move(offsets)); + } + + /// + /// @brief Reload a dataset previously saved to ``path`` via + /// ``lib::save_to_disk`` or ``VamanaIndex::save``. + /// + /// @param path The directory the dataset was saved into. + /// @param allocator The allocator used for the reloaded token matrix. Default: a + /// default-constructed ``Alloc``. + /// + static VectorSetData + load(const std::filesystem::path& path, const Alloc& allocator = {}) { + return lib::load_from_disk(path, allocator); + } + + private: + tokens_type tokens_; + std::vector offsets_; +}; + +static_assert(ImmutableMemoryDataset>); +static_assert(ImmutableVectorSetDataset>); + +} // namespace data + +/// +/// @brief Loader for owning vector-set datasets, mirroring ``svs::VectorDataLoader``. +/// +/// @tparam T The element type of each token vector. +/// @tparam Extent The compile-time dimensionality of the token vectors. Default: +/// ``svs::Dynamic``. +/// @tparam Allocator The allocator to use for the memory backing the data when loaded. +/// Default: ``HugepageAllocator``. +/// +/// Satisfies the ``f.load()`` shape ``svs::detail::dispatch_load`` looks for, so an +/// instance may be passed anywhere a Vamana ``DataProto`` is expected, e.g. to +/// ``svs::index::vamana::auto_assemble``. +/// +template > +class VectorSetDataLoader { + public: + /// @brief The full type of the loaded dataset. + using return_type = data::VectorSetData; + + /// + /// @brief Construct a new VectorSetDataLoader. + /// + /// @param path The directory previously used to save a ``VectorSetData`` (via + /// ``lib::save_to_disk`` or ``VamanaIndex::save``). + /// @param allocator The allocator to use when loading. Default: a + /// default-constructed ``Allocator``. + /// + VectorSetDataLoader(const std::filesystem::path& path, const Allocator& allocator = {}) + : path_{path} + , allocator_{allocator} {} + + /// @brief Load the dataset from disk. + return_type load() const { return return_type::load(path_, allocator_); } + + ///// Members + std::filesystem::path path_ = {}; + Allocator allocator_ = {}; +}; + +} // namespace svs diff --git a/include/svs/core/data/vector_set_view.h b/include/svs/core/data/vector_set_view.h index 43a73e0c3..9daeace62 100644 --- a/include/svs/core/data/vector_set_view.h +++ b/include/svs/core/data/vector_set_view.h @@ -34,6 +34,64 @@ namespace svs { namespace data { +namespace detail { + +/// +/// @brief Validate a CSR-style vector-set offsets array against the token storage it +/// indexes into. +/// +/// Shared by ``ConstVectorSetView`` and ``VectorSetData`` so the validation rules for the +/// two cannot drift apart: ``offsets`` must be non-empty, start at 0, be non-decreasing, +/// and its last entry (the total token count) must satisfy +/// ``offsets.back() * dimensions == token_elements`` without overflowing ``size_t``. If +/// ``Extent`` is static, ``dimensions`` must equal it. +/// +template +void validate_vector_set_layout( + std::span offsets, size_t token_elements, size_t dimensions +) { + if (offsets.empty()) { + throw ANNEXCEPTION("Vector set offsets must be non-empty, got an empty span"); + } + if (offsets.front() != 0) { + throw ANNEXCEPTION("Vector set offsets must start at 0, got {}", offsets.front()); + } + for (size_t i = 1; i < offsets.size(); ++i) { + if (offsets[i] < offsets[i - 1]) { + throw ANNEXCEPTION( + "Vector set offsets must be non-decreasing, got {} after {}", + offsets[i], + offsets[i - 1] + ); + } + } + size_t total_tokens = lib::narrow(offsets.back()); + if (dimensions != 0 && total_tokens > std::numeric_limits::max() / dimensions) { + throw ANNEXCEPTION( + "Token count {} times dimensions {} overflows size_t!", total_tokens, dimensions + ); + } + size_t expected_tokens = total_tokens * dimensions; + if (expected_tokens != token_elements) { + throw ANNEXCEPTION( + "Vector set offsets imply {} token elements but got {} token elements", + expected_tokens, + token_elements + ); + } + if constexpr (Extent != Dynamic) { + if (dimensions != Extent) { + throw ANNEXCEPTION( + "Vector set view has static extent {} but got dimension {}", + Extent, + dimensions + ); + } + } +} + +} // namespace detail + /// /// @brief Non-owning view of one vector set: a row-major, contiguous run of token /// vectors sharing a dimensionality. @@ -99,49 +157,7 @@ template class ConstVectorSetView { : tokens_{tokens} , offsets_{offsets} , dimensions_{dimensions} { - if (offsets_.empty()) { - throw ANNEXCEPTION("Vector set offsets must be non-empty, got an empty span"); - } - if (offsets_.front() != 0) { - throw ANNEXCEPTION( - "Vector set offsets must start at 0, got {}", offsets_.front() - ); - } - for (size_t i = 1; i < offsets_.size(); ++i) { - if (offsets_[i] < offsets_[i - 1]) { - throw ANNEXCEPTION( - "Vector set offsets must be non-decreasing, got {} after {}", - offsets_[i], - offsets_[i - 1] - ); - } - } - size_t total_tokens = lib::narrow(offsets_.back()); - if (dimensions_ != 0 && - total_tokens > std::numeric_limits::max() / dimensions_) { - throw ANNEXCEPTION( - "Token count {} times dimensions {} overflows size_t!", - total_tokens, - dimensions_ - ); - } - size_t expected_tokens = total_tokens * dimensions_; - if (expected_tokens != tokens_.size()) { - throw ANNEXCEPTION( - "Vector set offsets imply {} token elements but got {} token elements", - expected_tokens, - tokens_.size() - ); - } - if constexpr (Extent != Dynamic) { - if (dimensions_ != Extent) { - throw ANNEXCEPTION( - "Vector set view has static extent {} but got dimension {}", - Extent, - dimensions_ - ); - } - } + detail::validate_vector_set_layout(offsets_, tokens_.size(), dimensions_); } size_t size() const { return offsets_.size() - 1; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9b9a914ba..058402b44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -110,6 +110,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/core/data/serialization.cpp ${TEST_DIR}/svs/core/data/view.cpp ${TEST_DIR}/svs/core/data/vector_set_view.cpp + ${TEST_DIR}/svs/core/data/vector_set.cpp ${TEST_DIR}/svs/core/distance.cpp ${TEST_DIR}/svs/core/distances/simd_utils.cpp ${TEST_DIR}/svs/core/distances/distance_euclidean.cpp diff --git a/tests/svs/core/data/vector_set.cpp b/tests/svs/core/data/vector_set.cpp new file mode 100644 index 000000000..d90d95c70 --- /dev/null +++ b/tests/svs/core/data/vector_set.cpp @@ -0,0 +1,261 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/core/data/vector_set.h" + +// svs +#include "svs/concepts/vector_set.h" +#include "svs/core/allocator.h" +#include "svs/core/data/simple.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/lib/float16.h" +#include "svs/lib/saveload.h" + +// tests +#include "tests/utils/utils.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include + +namespace { + +// 3 documents with cardinalities 2, 0, 3 over dimension 2, matching the fixture in +// vector_set_view.cpp so both headers are checked against the same layout. +svs::data::SimpleData make_tokens() { + auto tokens = svs::data::SimpleData(5, 2); + float values[5][2] = { + {1.0f, 2.0f}, {3.0f, 4.0f}, {5.0f, 6.0f}, {7.0f, 8.0f}, {9.0f, 10.0f}}; + for (size_t i = 0; i < 5; ++i) { + tokens.set_datum(i, std::span(values[i], 2)); + } + return tokens; +} + +std::vector make_offsets() { return {0, 2, 2, 5}; } + +} // namespace + +CATCH_TEST_CASE("Vector Set Data", "[core][data][vector_set]") { + CATCH_SECTION("Construction by move is a no-copy takeover") { + auto tokens = make_tokens(); + const float* original_ptr = tokens.data(); + + auto container = svs::data::VectorSetData(std::move(tokens), make_offsets()); + CATCH_REQUIRE(container.tokens().data() == original_ptr); + + // The moved-from source is left in a state that is safe to inspect (even if its + // contents are unspecified); this should not throw or crash. + CATCH_REQUIRE_NOTHROW(tokens.get_allocator()); + } + + CATCH_SECTION("Accessors") { + auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); + + CATCH_REQUIRE(container.size() == 3); + CATCH_REQUIRE(container.dimensions() == 2); + CATCH_REQUIRE(container.total_tokens() == 5); + CATCH_REQUIRE(container.cardinality(0) == 2); + CATCH_REQUIRE(container.cardinality(1) == 0); + CATCH_REQUIRE(container.cardinality(2) == 3); + + auto set0 = container.get_datum(0); + CATCH_REQUIRE(set0.cardinality() == 2); + CATCH_REQUIRE(set0.token(0)[0] == 1.0f); + CATCH_REQUIRE(set0.token(1)[1] == 4.0f); + + auto set1 = container.get_datum(1); + CATCH_REQUIRE(set1.empty()); + + auto set2 = container.get_datum(2); + CATCH_REQUIRE(set2.cardinality() == 3); + CATCH_REQUIRE(set2.token(2)[0] == 9.0f); + CATCH_REQUIRE(set2.token(2)[1] == 10.0f); + + container.prefetch(0); + container.prefetch(1); + container.prefetch(2); + + auto offsets = container.offsets(); + CATCH_REQUIRE(offsets.size() == 4); + CATCH_REQUIRE(offsets[3] == 5); + } + + CATCH_SECTION("view() matches the container element-wise") { + auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); + auto view = container.view(); + + CATCH_REQUIRE(view.size() == container.size()); + CATCH_REQUIRE(view.dimensions() == container.dimensions()); + for (size_t i = 0; i < container.size(); ++i) { + auto a = container.get_datum(i); + auto b = view.get_datum(i); + CATCH_REQUIRE(a.cardinality() == b.cardinality()); + for (size_t j = 0; j < a.cardinality(); ++j) { + auto ta = a.token(j); + auto tb = b.token(j); + CATCH_REQUIRE(std::equal(ta.begin(), ta.end(), tb.begin())); + } + } + } + + CATCH_SECTION("copy_from takes ownership of a copy of a borrowed view") { + auto tokens = std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + auto offsets = std::vector{0, 1, 3}; + auto view = svs::data::ConstVectorSetView(tokens, offsets, 2); + + auto container = svs::data::VectorSetData::copy_from(view); + CATCH_REQUIRE(container.tokens().data() != tokens.data()); + CATCH_REQUIRE(container.size() == 2); + CATCH_REQUIRE(container.cardinality(0) == 1); + CATCH_REQUIRE(container.cardinality(1) == 2); + CATCH_REQUIRE(container.get_datum(1).token(1)[0] == 5.0f); + } + + CATCH_SECTION("Validation failures") { + CATCH_REQUIRE_THROWS_AS( + svs::data::VectorSetData( + svs::data::SimpleData(2, 2), std::vector{} + ), + svs::ANNException + ); + + CATCH_REQUIRE_THROWS_AS( + svs::data::VectorSetData( + svs::data::SimpleData(2, 2), std::vector{1, 2} + ), + svs::ANNException + ); + + CATCH_REQUIRE_THROWS_AS( + svs::data::VectorSetData( + svs::data::SimpleData(2, 2), std::vector{0, 2, 1} + ), + svs::ANNException + ); + + CATCH_REQUIRE_THROWS_AS( + svs::data::VectorSetData( + svs::data::SimpleData(2, 2), std::vector{0, 1} + ), + svs::ANNException + ); + } + + CATCH_SECTION("Static extent, integer element type") { + auto tokens = svs::data::SimpleData(3, 1); + tokens.set_datum(0, std::vector{7}); + tokens.set_datum(1, std::vector{8}); + tokens.set_datum(2, std::vector{9}); + auto offsets = std::vector{0, 2, 3}; + + auto container = + svs::data::VectorSetData(std::move(tokens), std::move(offsets)); + CATCH_REQUIRE(container.dimensions() == 1); + CATCH_REQUIRE(container.get_datum(0).token(1)[0] == 8); + + static_assert(svs::data::ImmutableVectorSetDataset); + } + + CATCH_SECTION("Float16 element type") { + auto tokens = svs::data::SimpleData(2, 3); + tokens.set_datum(0, std::vector{1.0f, 2.0f, 3.0f}); + tokens.set_datum(1, std::vector{4.0f, 5.0f, 6.0f}); + auto offsets = std::vector{0, 1, 2}; + + auto container = + svs::data::VectorSetData(std::move(tokens), std::move(offsets)); + CATCH_REQUIRE(container.size() == 2); + CATCH_REQUIRE(float(container.get_datum(1).token(0)[0]) == 4.0f); + } + + CATCH_SECTION("Blocked allocator is rejected") { + // `VectorSetData` static_asserts against a Blocked allocator (its per-document + // tokens would no longer be contiguous), so this checks the gating condition + // without instantiating `VectorSetData>`, which + // would be a hard compile error rather than a test failure. + using BlockedAlloc = svs::data::Blocked>; + static_assert(svs::data::is_blocked_v); + } + + CATCH_SECTION("Save then load round trip") { + auto temp_directory = svs_test::prepare_temp_directory_v2(); + auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); + + svs::lib::save_to_disk(container, temp_directory); + auto reloaded = + svs::lib::load_from_disk>(temp_directory); + + CATCH_REQUIRE(reloaded.size() == container.size()); + CATCH_REQUIRE(reloaded.dimensions() == container.dimensions()); + CATCH_REQUIRE(reloaded.total_tokens() == container.total_tokens()); + auto reloaded_offsets = reloaded.offsets(); + auto original_offsets = container.offsets(); + CATCH_REQUIRE(std::equal( + reloaded_offsets.begin(), reloaded_offsets.end(), original_offsets.begin() + )); + for (size_t i = 0; i < container.size(); ++i) { + auto a = container.get_datum(i); + auto b = reloaded.get_datum(i); + CATCH_REQUIRE(a.cardinality() == b.cardinality()); + for (size_t j = 0; j < a.cardinality(); ++j) { + auto ta = a.token(j); + auto tb = b.token(j); + CATCH_REQUIRE(std::equal(ta.begin(), ta.end(), tb.begin())); + } + } + } + + CATCH_SECTION("Loading a corrupted offsets file throws") { + auto temp_directory = svs_test::prepare_temp_directory_v2(); + auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); + svs::lib::save_to_disk(container, temp_directory); + + bool found_offsets_file = false; + for (auto const& entry : std::filesystem::directory_iterator(temp_directory)) { + if (entry.path().filename().string().starts_with("vector_set_offsets")) { + found_offsets_file = true; + // Truncate the file so its size no longer matches the saved metadata. + std::filesystem::resize_file(entry.path(), 1); + } + } + CATCH_REQUIRE(found_offsets_file); + + CATCH_REQUIRE_THROWS_AS( + svs::lib::load_from_disk>(temp_directory), + svs::ANNException + ); + } + + CATCH_SECTION("Loading with the wrong schema throws") { + auto temp_directory = svs_test::prepare_temp_directory_v2(); + // SimpleData has a different, incompatible schema ("uncompressed_data"). + auto other = svs::data::SimpleData(2, 2); + svs::lib::save_to_disk(other, temp_directory); + + CATCH_REQUIRE_THROWS( + svs::lib::load_from_disk>(temp_directory) + ); + } +} diff --git a/tests/svs/index/vamana/gem_vector_set.cpp b/tests/svs/index/vamana/gem_vector_set.cpp index 2c90e78ab..54d99d103 100644 --- a/tests/svs/index/vamana/gem_vector_set.cpp +++ b/tests/svs/index/vamana/gem_vector_set.cpp @@ -21,11 +21,15 @@ #include "svs/extensions/vamana/gem.h" // svs +#include "svs/core/data/vector_set.h" #include "svs/core/data/vector_set_view.h" #include "svs/core/graph.h" #include "svs/index/vamana/index.h" #include "svs/lib/threads.h" +// tests +#include "tests/utils/utils.h" + // catch2 #include "catch2/catch_approx.hpp" #include "catch2/catch_test_macros.hpp" @@ -35,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -220,3 +225,89 @@ CATCH_TEST_CASE("Vamana build and search over a vector-set dataset", "[vamana][g } } } + +// Saves a VamanaIndex built over an owning VectorSetData and reassembles it +// through `svs::index::vamana::auto_assemble` with `svs::VectorSetDataLoader`, the small +// loader type that mirrors `svs::VectorDataLoader`. `auto_assemble` applies the saved +// config via `VamanaIndex::apply` rather than `verify_and_set_default_index_parameters` +// (index.h), so it never runs the L2/IP/Cosine distance-type check that forces the +// build path above through `VamanaBuilder` directly: QChamfer hits no obstacle here. +CATCH_TEST_CASE( + "Vamana save and reload over an owning vector-set dataset", "[vamana][gem]" +) { + auto table = make_centroid_table(0xABCD); + svs::distance::QChamfer dist{table}; + + const size_t n_docs = 50; + auto corpus = make_corpus(n_docs, 0x1234); + auto queries = make_corpus(5, 0x5678); + auto query_view = queries.view(); + + // Owning takeover of the borrowed corpus, exactly the `copy_from` path a caller + // without its own ownable storage would use. + auto owning_sets = svs::data::VectorSetData::copy_from(corpus.view()); + + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + const uint32_t entry_point = 0; + + auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + graph, + owning_sets, + dist, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(owning_sets) + ); + builder.construct(build_parameters.alpha, entry_point); + + auto pool = svs::threads::DefaultThreadPool(1); + auto index = svs::index::vamana::VamanaIndex( + std::move(graph), std::move(owning_sets), entry_point, dist, std::move(pool) + ); + + // The lower constructor above (required to dodge the L2/IP/Cosine check, as noted on + // the test case above) leaves `build_parameters_` at its default-constructed, + // placeholder-sentinel state; saving those sentinels overflows + // `VamanaBuildParameters::save`'s narrowing cast to a signed TOML integer. `apply` + // only assigns the fields below (it does not run the distance-type check), so it is + // the way to give the index real values to save. + index.apply(svs::index::vamana::VamanaIndexParameters{ + entry_point, build_parameters, index.get_search_parameters()}); + + auto temp_directory = svs_test::prepare_temp_directory_v2(); + auto config_dir = temp_directory / "config"; + auto graph_dir = temp_directory / "graph"; + auto data_dir = temp_directory / "data"; + index.save(config_dir, graph_dir, data_dir); + + auto reload_pool = svs::threads::DefaultThreadPool(1); + auto reloaded = svs::index::vamana::auto_assemble( + config_dir, + svs::GraphLoader(graph_dir), + svs::VectorSetDataLoader(data_dir), + dist, + std::move(reload_pool) + ); + + const size_t capacity = n_docs; + auto sp = search_params(capacity, capacity); + auto scratch_before = index.scratchspace(sp); + auto scratch_after = reloaded.scratchspace(sp); + + for (size_t q = 0; q < query_view.size(); ++q) { + auto query = query_view.get_datum(q); + index.search(query, scratch_before); + reloaded.search(query, scratch_after); + + CATCH_REQUIRE(scratch_before.buffer.size() == scratch_after.buffer.size()); + for (size_t j = 0; j < scratch_before.buffer.size(); ++j) { + CATCH_REQUIRE(scratch_before.buffer[j].id() == scratch_after.buffer[j].id()); + CATCH_REQUIRE( + scratch_before.buffer[j].distance() == scratch_after.buffer[j].distance() + ); + } + } +} From dfae7e05e02dd873991d8cc3e7700e761fd6f026 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 03:03:23 -0800 Subject: [PATCH 11/68] feat(data): save and load WithVectorSets as one dataset Add save()/load() to WithVectorSets, nesting each member under its own sub-table. Both are constrained by a requires-clause checking Primary::save/load and Sets::save/load exist, so a WithVectorSets over a non-owning Sets (e.g. ConstVectorSetView, which has neither) still compiles as a dataset member; the methods simply don't participate for that instantiation. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/data/with_vector_sets.h | 45 ++++++++++ tests/svs/index/vamana/maxsim_rerank.cpp | 101 +++++++++++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/include/svs/core/data/with_vector_sets.h b/include/svs/core/data/with_vector_sets.h index cd87206ab..4eeddbb9d 100644 --- a/include/svs/core/data/with_vector_sets.h +++ b/include/svs/core/data/with_vector_sets.h @@ -19,9 +19,12 @@ // svs #include "svs/concepts/vector_set.h" #include "svs/lib/exception.h" +#include "svs/lib/saveload.h" // stl +#include #include +#include #include namespace svs::data { @@ -70,6 +73,48 @@ template class WithVectorSets return sets_.get_datum(i); } + ///// IO + // + // Available only when both `Primary` and `Sets` are themselves saveable/loadable; + // a non-owning `Sets` (e.g. `ConstVectorSetView`) lacks `save`/`load`, and the + // requires-clauses below make that a silent non-participant rather than a hard + // error, so `WithVectorSets` over a view still compiles as a dataset member. + + static constexpr lib::Version save_version = lib::Version(0, 0, 0); + static constexpr std::string_view serialization_schema = "with_vector_sets"; + + static bool check_load_compatibility(std::string_view schema, lib::Version version) { + return schema == serialization_schema && version == save_version; + } + + /// @brief Save both members, nested as sub-tables under the ``"primary"`` and + /// ``"vector_sets"`` keys. + lib::SaveTable save(const lib::SaveContext& ctx) const + requires requires(const Primary& p, const Sets& s, const lib::SaveContext& c) { + p.save(c); + s.save(c); + } + { + return lib::SaveTable( + serialization_schema, + save_version, + {{"primary", lib::save(primary_, ctx)}, {"vector_sets", lib::save(sets_, ctx)}} + ); + } + + /// @brief Reload both members with their default-constructed allocators. + static WithVectorSets load(const lib::LoadTable& table) + requires requires(const lib::LoadTable& t) { + { Primary::load(t) } -> std::same_as; + { Sets::load(t) } -> std::same_as; + } + { + return WithVectorSets( + lib::load_at(table, "primary"), + lib::load_at(table, "vector_sets") + ); + } + private: Primary primary_; Sets sets_; diff --git a/tests/svs/index/vamana/maxsim_rerank.cpp b/tests/svs/index/vamana/maxsim_rerank.cpp index d81b4f0a0..7d4b86c8f 100644 --- a/tests/svs/index/vamana/maxsim_rerank.cpp +++ b/tests/svs/index/vamana/maxsim_rerank.cpp @@ -20,15 +20,21 @@ // svs #include "svs/concepts/vector_set.h" #include "svs/core/data.h" +#include "svs/core/data/vector_set.h" #include "svs/core/data/vector_set_view.h" #include "svs/core/data/with_vector_sets.h" #include "svs/core/distance/maxsim.h" #include "svs/core/graph.h" #include "svs/core/query_result.h" #include "svs/index/vamana/index.h" +#include "svs/lib/misc.h" #include "svs/lib/narrow.h" +#include "svs/lib/saveload.h" #include "svs/lib/threads.h" +// tests +#include "tests/utils/utils.h" + // catch2 #include "catch2/catch_approx.hpp" #include "catch2/catch_test_macros.hpp" @@ -38,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -446,3 +453,97 @@ CATCH_TEST_CASE("MaxSim re-rank Vamana extension", "[vamana][maxsim_rerank]") { } } } + +// Saves and reloads a VamanaIndex over a WithVectorSets of two owning members +// (SimpleData and VectorSetData, both saveable), checking the re-ranked results are +// identical before and after. Reassembly uses `svs::index::vamana::auto_assemble` with a +// `svs::lib::Lazy` callable as the data proto (`svs::detail::dispatch_load` accepts +// anything invocable with no arguments) rather than a new loader type: `WithVectorSets` +// has no `allocator_type` of its own to build a `VectorDataLoader`-style loader around, +// and the lazy callable is the smaller option for a type that is reloaded whole. +CATCH_TEST_CASE("WithVectorSets save and reload", "[vamana][maxsim_rerank]") { + using Dist = MeanNegativeInnerProduct; + using Primary = svs::data::SimpleData; + using Sets = svs::data::VectorSetData; + using Data = svs::data::WithVectorSets; + + const size_t n_docs = 60; + auto corpus = make_corpus(n_docs, 0xABCDEF); + auto query_corpus = make_corpus(5, 0xFEEDFACE); + + // Both members are owning: the primary row dataset is copied from the corpus (it + // already owns its storage); the vector-set dataset takes ownership of a copy of the + // corpus's borrowed view through `copy_from`. + auto data = Data(corpus.primary, Sets::copy_from(corpus.vector_sets())); + + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + + auto entry_point_pool = svs::threads::DefaultThreadPool(1); + auto entry_point = + svs::lib::narrow(svs::index::vamana::extensions::compute_entry_point( + corpus.primary, entry_point_pool + )); + + auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + graph, + data, + Dist{}, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(data) + ); + builder.construct(1.0F, entry_point); + builder.construct(build_parameters.alpha, entry_point); + + auto pool = svs::threads::DefaultThreadPool(1); + auto index = svs::index::vamana::VamanaIndex( + std::move(graph), std::move(data), entry_point, Dist{}, std::move(pool) + ); + + // The lower constructor above leaves `build_parameters_` at its default-constructed + // placeholder-sentinel state (it never ran verify_and_set_default_index_parameters, + // which is the point of using it with a non-L2/IP/Cosine distance); saving those + // sentinels overflows `VamanaBuildParameters::save`'s narrowing cast to a signed TOML + // integer. `apply` only assigns fields, without the distance-type check, so give the + // index real values before saving it. + index.apply(svs::index::vamana::VamanaIndexParameters{ + entry_point, build_parameters, index.get_search_parameters()}); + + auto temp_directory = svs_test::prepare_temp_directory_v2(); + auto config_dir = temp_directory / "config"; + auto graph_dir = temp_directory / "graph"; + auto data_dir = temp_directory / "data"; + index.save(config_dir, graph_dir, data_dir); + + auto reload_pool = svs::threads::DefaultThreadPool(1); + auto reloaded = svs::index::vamana::auto_assemble( + config_dir, + svs::GraphLoader(graph_dir), + SVS_LAZY(svs::lib::load_from_disk(data_dir)), + Dist{}, + std::move(reload_pool) + ); + + auto queries = query_corpus.vector_sets(); + const size_t capacity = n_docs; + auto sp = search_params(capacity, capacity); + auto scratch_before = index.scratchspace(sp); + auto scratch_after = reloaded.scratchspace(sp); + + for (size_t q = 0; q < queries.size(); ++q) { + auto query = queries.get_datum(q); + index.search(query, scratch_before); + reloaded.search(query, scratch_after); + + CATCH_REQUIRE(scratch_before.buffer.size() == scratch_after.buffer.size()); + for (size_t j = 0; j < scratch_before.buffer.size(); ++j) { + CATCH_REQUIRE(scratch_before.buffer[j].id() == scratch_after.buffer[j].id()); + CATCH_REQUIRE( + scratch_before.buffer[j].distance() == scratch_after.buffer[j].distance() + ); + } + } +} From e7813fd7fb7deea76d888c3f11bed121052f93b9 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 02:58:52 -0800 Subject: [PATCH 12/68] refactor(distance)!: drop the padded-row layout from GEM distances QEmd, QChamfer and QChamferExactQuery accepted two token-set layouts: the old fixed-width padded float row and a vector set. Only the vector-set layout survives; the padded-row span overloads, their gem_detail decoders (set_size/set_ids, vector_set_size/vector_set_tokens), and the per-ISA float-pointer overloads they fed are deleted. QEmdImpl::merge_duplicates is private again now that its only caller is the vector-set path; the external gem_time harness that needed it public will be adapted separately, outside this repository. Vector-set callers are unaffected: their arithmetic and dispatch are unchanged. Tests are rewritten to pass the same hand-computed cases as vector sets instead of padded spans, dropping only the padding-is-ignored and layout-comparison cases that tested the removed layout; no expected value changed. A static_assert pins down that a bare std::span can no longer satisfy QChamfer's compute() constraint. BREAKING CHANGE: removes QEmdImpl::compute(span,span), QChamferImpl::compute(span,span), QChamferImpl::decode_ids, QEmdImpl::merge_duplicates(const float*, ...) (now private for the uint32_t overload too), QEmd::compute(span,span), QChamfer::compute(span, span), QChamferExactQuery::fix_argument(span) and QChamferExactQuery::compute(span,span). gem_detail::set_size, set_ids, vector_set_size and vector_set_tokens are also removed. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/distance/gem.h | 154 ++++------------ tests/svs/core/distances/gem.cpp | 296 ++++++++++--------------------- 2 files changed, 121 insertions(+), 329 deletions(-) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index a023f1c42..b0334bf60 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -41,7 +41,6 @@ #include #include #include -#include #include #include #include @@ -221,17 +220,6 @@ struct FineCentroids { namespace gem_detail { -// Token-set span layout: [0] = token count n, [1..n] = centroid ids, rest unused padding. -inline size_t set_size(std::span s) { return static_cast(s[0]); } -inline const float* set_ids(std::span s) { return s.data() + 1; } - -// Raw-query span layout (QChamferExactQuery): [0] = token count n, [1..1+n*dim) = n -// raw fp32 token embeddings row-major, rest unused padding. -inline size_t vector_set_size(std::span s) { - return static_cast(s[0]); -} -inline const float* vector_set_tokens(std::span s) { return s.data() + 1; } - // Thread-local because the functors are shared across threads (implicit_broadcast). struct MergeScratch { std::vector ids; @@ -258,8 +246,8 @@ inline float qchamfer_exact_reduce( return static_cast(total / static_cast(n_query)); } -// Casts a token-set's centroid ids to uint32_t with no merge; qCH counts duplicates. `ids` -// is float-encoded (padded row) or already uint32_t (vector set). +// Casts a vector set's centroid ids to uint32_t with no merge; qCH counts duplicates. +// Only instantiated for element types other than uint32_t (id_span's cast-copy branch). template inline void decode_ids(const T* ids, size_t n, std::vector& out) { out.resize(n); @@ -297,8 +285,8 @@ inline void require_id_dimensions(size_t dims) { // Instantiated per ISA in svs/multi-arch/x86; without SVS_NOINLINE the caller inlines the // body at baseline flags. The bodies are scalar, so only compiler auto-vectorization -// differs. compute_decoded takes already-decoded (deduplicated, weighted) centroid ids so -// gem_time can benchmark the solve separately from the span decode. +// differs. compute_decoded takes already-decoded (deduplicated, weighted) centroid ids, +// keeping the merge and the transportation solve as separately testable steps. template struct QEmdImpl { SVS_NOINLINE static float compute_decoded( const CentroidTable& table, @@ -355,28 +343,8 @@ template struct QEmdImpl { return static_cast(cost); } - SVS_NOINLINE static float compute( - const CentroidTable& table, std::span a, std::span b - ) { - thread_local gem_detail::MergeScratch scratch_a; - thread_local gem_detail::MergeScratch scratch_b; - size_t ua = - merge_duplicates(gem_detail::set_ids(a), gem_detail::set_size(a), scratch_a); - size_t ub = - merge_duplicates(gem_detail::set_ids(b), gem_detail::set_size(b), scratch_b); - return compute_decoded( - table, - scratch_a.ids.data(), - scratch_a.weights.data(), - ua, - scratch_b.ids.data(), - scratch_b.weights.data(), - ub - ); - } - - // Same decode-merge-solve as the span overload above, for ids that are already - // uint32_t (a vector set's own storage) rather than float-encoded in a padded row. + // Decodes both ids-sets' own uint32_t storage, merges each set's duplicates into + // weighted unique ids, then solves the transportation problem over the result. SVS_NOINLINE static float compute( const CentroidTable& table, const uint32_t* ids_a, @@ -422,18 +390,6 @@ template struct QEmdImpl { return unique_count; } - public: - // Public so gem_time (external timing harness) can benchmark decode-merge separately - // from compute; otherwise only called from compute()/compute_decoded() above. - static size_t - merge_duplicates(const float* ids, size_t n, gem_detail::MergeScratch& scratch) { - scratch.ids.resize(n); - for (size_t i = 0; i < n; ++i) { - scratch.ids[i] = static_cast(ids[i]); - } - return merge_sorted_duplicates(n, scratch); - } - static size_t merge_duplicates(const uint32_t* ids, size_t n, gem_detail::MergeScratch& scratch) { scratch.ids.assign(ids, ids + n); @@ -441,8 +397,8 @@ template struct QEmdImpl { } }; -// Dispatched like QEmdImpl. compute_decoded takes already-decoded (cast, unmerged) centroid -// id arrays so gem_time can benchmark the min-sum loop separately from the span decode. +// Dispatched like QEmdImpl. compute_decoded takes centroid id arrays already cast to +// uint32_t from the vector set's own storage by QChamfer::compute. template struct QChamferImpl { SVS_NOINLINE static float compute_decoded( const CentroidTable& table, @@ -461,27 +417,6 @@ template struct QChamferImpl { } return static_cast(total); } - - SVS_NOINLINE static float compute( - const CentroidTable& table, std::span query, std::span doc - ) { - thread_local std::vector qbuf; - thread_local std::vector dbuf; - decode_ids(gem_detail::set_ids(query), gem_detail::set_size(query), qbuf); - decode_ids(gem_detail::set_ids(doc), gem_detail::set_size(doc), dbuf); - return compute_decoded(table, qbuf.data(), qbuf.size(), dbuf.data(), dbuf.size()); - } - - private: - // Not gem_detail::decode_ids: QChamferImpl is explicitly instantiated per ISA - // in avx2.cpp/avx512.cpp, so this call site needs its own ISA-unique symbol rather - // than the untagged free function (shared, correctly, with baseline-only callers). - static void decode_ids(const float* ids, size_t n, std::vector& out) { - out.resize(n); - for (size_t i = 0; i < n; ++i) { - out[i] = static_cast(ids[i]); - } - } }; // Dispatched like QChamferImpl. GEM never quantizes the query token side (src/search/ @@ -600,6 +535,11 @@ inline std::string_view gem_dispatch_isa() { /// Build-time distance: quantized Earth Mover's Distance (qEMD) between two token sets, /// solved exactly as a balanced transportation problem over merged centroid weights. /// Smaller is better. +/// +/// Both arguments are id vector sets (`svs::data::ContiguousVectorSet` with an unsigned +/// integer element type and `dimensions() == 1`): one centroid id per token, each id in +/// `[0, table.K)`. Holds a shared, read-only `CentroidTable` the caller owns and keeps +/// alive for the lifetime of this object. class QEmd { public: using compare = std::less<>; @@ -610,19 +550,6 @@ class QEmd { explicit QEmd(std::shared_ptr table) : table_{std::move(table)} {} - template - float compute(std::span a, std::span b) const { - std::span da{a.data(), a.size()}; - std::span db{b.data(), b.size()}; - if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { - return QEmdImpl::compute(*table_, da, db); - } - if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { - return QEmdImpl::compute(*table_, da, db); - } - return QEmdImpl::compute(*table_, da, db); - } - // Vector-set path: ids come straight from a contiguous vector set's own storage; T == // uint32_t skips the cast copy (gem_detail::id_span), any other unsigned type costs it. template @@ -657,6 +584,11 @@ inline constexpr bool operator==(const QEmd&, const QEmd&) { return true; } /// Search-time distance: quantized Chamfer distance (qCH, GEM Eq. 16). For every query /// token (duplicates counted), takes the min centroid distance to any doc token, and /// sums. Smaller is better. The query is the left-hand argument. +/// +/// Both arguments are id vector sets (`svs::data::ContiguousVectorSet` with an unsigned +/// integer element type and `dimensions() == 1`): one centroid id per token, each id in +/// `[0, table.K)`. Holds a shared, read-only `CentroidTable` the caller owns and keeps +/// alive for the lifetime of this object. class QChamfer { public: using compare = std::less<>; @@ -666,19 +598,6 @@ class QChamfer { explicit QChamfer(std::shared_ptr table) : table_{std::move(table)} {} - template - float compute(std::span query, std::span doc) const { - std::span dq{query.data(), query.size()}; - std::span dd{doc.data(), doc.size()}; - if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { - return QChamferImpl::compute(*table_, dq, dd); - } - if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { - return QChamferImpl::compute(*table_, dq, dd); - } - return QChamferImpl::compute(*table_, dq, dd); - } - // Vector-set path: compute_decoded's signature already matches an id vector set's // own (ids pointer, cardinality) storage, so T == uint32_t needs no decode copy. template @@ -725,6 +644,12 @@ inline constexpr bool operator==(const QChamfer&, const QChamfer&) { return true /// Holds a per-query table as ordinary (non-shared) state, so each search thread's /// shallow_copy of this functor gets its own table; implicit_broadcast is false and /// must_fix_argument is true to force a fix_argument() call before any compute(). +/// +/// The query (fix_argument's argument) is a vector set of raw (unquantized) fp32 tokens +/// with dimensions() equal to the centroid dimension. The doc (compute's second argument) +/// is an id vector set (unsigned integer element type, dimensions() == 1): one centroid +/// id per token, each id in [0, K). compute's query argument is unused; fix_argument +/// already consumed the query into this object's table. class QChamferExactQuery { public: using compare = std::less<>; @@ -735,15 +660,6 @@ class QChamferExactQuery { explicit QChamferExactQuery(std::shared_ptr centroids) : centroids_{std::move(centroids)} {} - // Raw-query span layout (gem_detail::vector_set_size/vector_set_tokens): [0] = token - // count n, [1..1+n*dim) = n raw fp32 tokens row-major, rest padding. - template void fix_argument(std::span query) { - std::span dq{query.data(), query.size()}; - fix_argument_decoded( - gem_detail::vector_set_tokens(dq), gem_detail::vector_set_size(dq) - ); - } - // Vector-set path: a query set of raw (unquantized) fp32 token vectors, contiguous // in the set's own storage, with dimensions() equal to the centroid dimension. template @@ -760,18 +676,8 @@ class QChamferExactQuery { } // `query` is unused: fix_argument() already consumed it into table_/n_query_, and - // every compute() call in a search shares that one query's table. - template - float - compute(std::span /*query*/, std::span doc) const { - std::span dd{doc.data(), doc.size()}; - thread_local std::vector dbuf; - gem_detail::decode_ids(gem_detail::set_ids(dd), gem_detail::set_size(dd), dbuf); - return compute_decoded(dbuf.data(), dbuf.size()); - } - - // Vector-set path: `query` is unused for the same reason as the span overload above; - // `doc` is an id vector set, decoded with no copy when its element type is uint32_t. + // every compute() call in a search shares that one query's table. `doc` is an id + // vector set, decoded with no copy when its element type is uint32_t. template requires std::unsigned_integral float compute(const Query& /*query*/, const Sd& doc) const { @@ -782,8 +688,8 @@ class QChamferExactQuery { } private: - // Shared by both fix_argument() overloads: builds the (K x n_query) dot table once - // per query, dispatched to the ISA the per-ISA compiled build_table was built for. + // Builds the (K x n_query) dot table once per query, dispatched to the ISA the + // per-ISA compiled build_table was built for. void fix_argument_decoded(const float* tokens, size_t n_query) { n_query_ = n_query; table_.resize(centroids_->K * n_query_); @@ -819,8 +725,8 @@ class QChamferExactQuery { } } - // Shared by both compute() overloads: reduces the per-query table against a doc's - // decoded ids, dispatched to the ISA the per-ISA compute_decoded was built for. + // Reduces the per-query table against a doc's decoded ids, dispatched to the ISA the + // per-ISA compute_decoded was built for. float compute_decoded(const uint32_t* doc_ids, size_t n_doc) const { if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { return QChamferExactImpl::compute_decoded( diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp index d761f33d1..1569ce901 100644 --- a/tests/svs/core/distances/gem.cpp +++ b/tests/svs/core/distances/gem.cpp @@ -55,20 +55,8 @@ std::shared_ptr make_table(Ground ground) { ); } -// Token-set span layout (gem_detail::set_size/set_ids): [0] = n, [1..n] = centroid ids, -// rest padding. `pad` lets a test fill the unused tail with garbage to prove it is -// ignored. -std::vector make_set(std::vector ids, size_t width, float pad = -999.0F) { - std::vector s(width, pad); - s[0] = static_cast(ids.size()); - for (size_t i = 0; i < ids.size(); ++i) { - s[1 + i] = ids[i]; - } - return s; -} - -// Vector-set equivalent of make_set(): owns the ids contiguously with no padding and no -// float encoding, for exercising the new ConstVectorSet compute() overloads. +// Owns a 1-D id vector set contiguously, for exercising QEmd/QChamfer/QChamferExactQuery's +// only (vector-set) compute() overloads. struct IdSet { std::vector ids; @@ -81,8 +69,8 @@ struct IdSet { IdSet make_id_set(std::vector ids) { return IdSet{std::move(ids)}; } -// Vector-set equivalent of a raw-query padded row for QChamferExactQuery: owns `n` raw -// fp32 tokens of width `dim`, row-major, contiguously. +// Owns a raw-query vector set for QChamferExactQuery: `n` raw fp32 tokens of width `dim`, +// row-major, contiguously. struct RawQuerySet { std::vector tokens; size_t dim; @@ -94,6 +82,11 @@ struct RawQuerySet { } }; +// Compile-time guard: the removed fixed-width padded-row layout must not come back. +// QChamfer's only compute() overload requires ContiguousVectorSet, so a type that fails +// that concept (a bare span, the old layout's carrier) cannot satisfy it either. +static_assert(!svs::data::ContiguousVectorSet>); + } // namespace CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { @@ -134,63 +127,37 @@ CATCH_TEST_CASE("QChamfer", "[distance][gem]") { CATCH_SECTION("Hand-computed: min over doc per query token") { // query ids {0,1}, doc ids {2,3}: table(0,2)=2,table(0,3)=1 -> min 1; // table(1,2)=1,table(1,3)=2 -> min 1. total = 1 + 1 = 2. - auto q = make_set({0.0F, 1.0F}, 6); - auto d = make_set({2.0F, 3.0F}, 6); - float score = dist.compute(std::span(q), std::span(d)); + auto q = make_id_set({0, 1}); + auto d = make_id_set({2, 3}); + float score = dist.compute(q.view(), d.view()); CATCH_REQUIRE(score == Catch::Approx(2.0F)); } CATCH_SECTION("Duplicate ids are counted, not merged") { // query {0,0} against doc {1}: each of the two query tokens pays table(0,1)=1. - auto q = make_set({0.0F, 0.0F}, 6); - auto d = make_set({1.0F}, 6); - float score = dist.compute(std::span(q), std::span(d)); + auto q = make_id_set({0, 0}); + auto d = make_id_set({1}); + float score = dist.compute(q.view(), d.view()); CATCH_REQUIRE(score == Catch::Approx(2.0F)); } - CATCH_SECTION("n = 1") { - auto q = make_set({0.0F}, 6); - auto d = make_set({2.0F}, 6); - float score = dist.compute(std::span(q), std::span(d)); + CATCH_SECTION("Cardinality 1") { + auto q = make_id_set({0}); + auto d = make_id_set({2}); + float score = dist.compute(q.view(), d.view()); CATCH_REQUIRE(score == Catch::Approx(2.0F)); } - CATCH_SECTION("Padding beyond n is ignored") { - auto q1 = make_set({0.0F, 1.0F}, 8, -999.0F); - auto q2 = make_set({0.0F, 1.0F}, 8, 777.0F); - auto d = make_set({2.0F, 3.0F}, 8, 42.0F); - float s1 = dist.compute(std::span(q1), std::span(d)); - float s2 = dist.compute(std::span(q2), std::span(d)); - CATCH_REQUIRE(s1 == Catch::Approx(s2)); - } - - CATCH_SECTION("Vector-set path matches the padded-row path") { - auto q_padded = make_set({0.0F, 1.0F}, 6); - auto d_padded = make_set({2.0F, 3.0F}, 6); - float expected = dist.compute( - std::span(q_padded), std::span(d_padded) - ); - - auto q_set = make_id_set({0, 1}); - auto d_set = make_id_set({2, 3}); - float actual = dist.compute(q_set.view(), d_set.view()); - CATCH_REQUIRE(actual == Catch::Approx(expected)); - } - - CATCH_SECTION("Vector-set path: duplicates, cardinality 1, different cardinalities") { - auto q_set = make_id_set({0, 0, 1}); - auto d_set = make_id_set({2}); - float actual = dist.compute(q_set.view(), d_set.view()); - - auto q_padded = make_set({0.0F, 0.0F, 1.0F}, 8); - auto d_padded = make_set({2.0F}, 8); - float expected = dist.compute( - std::span(q_padded), std::span(d_padded) - ); - CATCH_REQUIRE(actual == Catch::Approx(expected)); + CATCH_SECTION("Different cardinalities") { + // query {0,0,1} vs doc {2}: table(0,2)=2 (twice, one per query token) + + // table(1,2)=1 = 5. + auto q = make_id_set({0, 0, 1}); + auto d = make_id_set({2}); + float score = dist.compute(q.view(), d.view()); + CATCH_REQUIRE(score == Catch::Approx(5.0F)); } - CATCH_SECTION("Vector-set path: dimensions() != 1 throws") { + CATCH_SECTION("dimensions() != 1 throws") { std::vector raw{0, 1, 2, 3}; auto bad = svs::data::ConstVectorSet(std::span(raw), 2, 2); @@ -200,10 +167,10 @@ CATCH_TEST_CASE("QChamfer", "[distance][gem]") { } CATCH_SECTION("Generic svs::distance::compute dispatches to the vector-set overload") { - auto q_set = make_id_set({0, 1}); - auto d_set = make_id_set({2, 3}); - float via_generic = svs::distance::compute(dist, q_set.view(), d_set.view()); - float direct = dist.compute(q_set.view(), d_set.view()); + auto q = make_id_set({0, 1}); + auto d = make_id_set({2, 3}); + float via_generic = svs::distance::compute(dist, q.view(), d.view()); + float direct = dist.compute(q.view(), d.view()); CATCH_REQUIRE(via_generic == Catch::Approx(direct)); } } @@ -213,104 +180,53 @@ CATCH_TEST_CASE("QEmd", "[distance][gem]") { QEmd dist{table}; CATCH_SECTION("1 vs 1") { - auto a = make_set({0.0F}, 4); - auto b = make_set({1.0F}, 4); - float score = dist.compute(std::span(a), std::span(b)); + auto a = make_id_set({0}); + auto b = make_id_set({1}); + float score = dist.compute(a.view(), b.view()); CATCH_REQUIRE(score == Catch::Approx(1.0F)); // table(0,1) } CATCH_SECTION("1 vs 2") { // One source of mass 1 split 0.5/0.5 to two distinct sinks: 0.5*table(0,1) + // 0.5*table(0,2) = 0.5*1 + 0.5*2 = 1.5. - auto a = make_set({0.0F}, 4); - auto b = make_set({1.0F, 2.0F}, 4); - float score = dist.compute(std::span(a), std::span(b)); + auto a = make_id_set({0}); + auto b = make_id_set({1, 2}); + float score = dist.compute(a.view(), b.view()); CATCH_REQUIRE(score == Catch::Approx(1.5F)); } CATCH_SECTION("2 vs 2 with obvious optimal matching") { // Masses 0.5/0.5 on each side; matching 0->3, 1->2 costs 0.5*1+0.5*1=1, strictly // less than the cross matching 0->2, 1->3 (0.5*2+0.5*2=2). - auto a = make_set({0.0F, 1.0F}, 4); - auto b = make_set({2.0F, 3.0F}, 4); - float score = dist.compute(std::span(a), std::span(b)); + auto a = make_id_set({0, 1}); + auto b = make_id_set({2, 3}); + float score = dist.compute(a.view(), b.view()); CATCH_REQUIRE(score == Catch::Approx(1.0F)); } CATCH_SECTION("Duplicates merge to a single point with full mass") { // {0,0} merges to one point of mass 1 at id 0, same as the plain "1 vs 1" case. - auto a = make_set({0.0F, 0.0F}, 4); - auto b = make_set({1.0F}, 4); - float score = dist.compute(std::span(a), std::span(b)); + auto a = make_id_set({0, 0}); + auto b = make_id_set({1}); + float score = dist.compute(a.view(), b.view()); CATCH_REQUIRE(score == Catch::Approx(1.0F)); } CATCH_SECTION("Identical sets give 0") { - auto a = make_set({0.0F, 1.0F}, 4); - float score = dist.compute(std::span(a), std::span(a)); + auto a = make_id_set({0, 1}); + float score = dist.compute(a.view(), a.view()); CATCH_REQUIRE(score == Catch::Approx(0.0F)); } CATCH_SECTION("Symmetry") { - auto a = make_set({0.0F}, 4); - auto b = make_set({1.0F, 2.0F}, 4); - float ab = dist.compute(std::span(a), std::span(b)); - float ba = dist.compute(std::span(b), std::span(a)); + auto a = make_id_set({0}); + auto b = make_id_set({1, 2}); + float ab = dist.compute(a.view(), b.view()); + float ba = dist.compute(b.view(), a.view()); CATCH_REQUIRE(ab == Catch::Approx(ba)); } - CATCH_SECTION("Padding beyond n is ignored") { - auto a1 = make_set({0.0F, 1.0F}, 8, -999.0F); - auto a2 = make_set({0.0F, 1.0F}, 8, 777.0F); - auto b = make_set({2.0F, 3.0F}, 8, 42.0F); - float s1 = dist.compute(std::span(a1), std::span(b)); - float s2 = dist.compute(std::span(a2), std::span(b)); - CATCH_REQUIRE(s1 == Catch::Approx(s2)); - } - - CATCH_SECTION("Vector-set path matches the padded-row path") { - auto a_padded = make_set({0.0F, 1.0F}, 4); - auto b_padded = make_set({2.0F, 3.0F}, 4); - float expected = dist.compute( - std::span(a_padded), std::span(b_padded) - ); - - auto a_set = make_id_set({0, 1}); - auto b_set = make_id_set({2, 3}); - float actual = dist.compute(a_set.view(), b_set.view()); - CATCH_REQUIRE(actual == Catch::Approx(expected)); - } - - CATCH_SECTION( - "Vector-set path: duplicates merge, cardinality 1, different cardinalities" - ) { - auto a_set = make_id_set({0, 0}); - auto b_set = make_id_set({1}); - float actual = dist.compute(a_set.view(), b_set.view()); - - auto a_padded = make_set({0.0F, 0.0F}, 4); - auto b_padded = make_set({1.0F}, 4); - float expected = dist.compute( - std::span(a_padded), std::span(b_padded) - ); - CATCH_REQUIRE(actual == Catch::Approx(expected)); - - // One source of mass 1 at id 0 split 0.5/0.5 to sinks 1 and 2 (table(0,1)=1, - // table(0,2)=2): 0.5*1 + 0.5*2 = 1.5, same derivation as the "1 vs 2" case above. - auto c_set = make_id_set({0}); - auto d_set = make_id_set({1, 2}); - float cardinality_mismatch = dist.compute(c_set.view(), d_set.view()); - CATCH_REQUIRE(cardinality_mismatch == Catch::Approx(1.5F)); - - auto c_padded = make_set({0.0F}, 4); - auto d_padded = make_set({1.0F, 2.0F}, 4); - float cardinality_mismatch_expected = dist.compute( - std::span(c_padded), std::span(d_padded) - ); - CATCH_REQUIRE(cardinality_mismatch == Catch::Approx(cardinality_mismatch_expected)); - } - - CATCH_SECTION("Vector-set path: dimensions() != 1 throws") { + CATCH_SECTION("dimensions() != 1 throws") { std::vector raw{0, 1, 2, 3}; auto bad = svs::data::ConstVectorSet(std::span(raw), 2, 2); @@ -320,10 +236,10 @@ CATCH_TEST_CASE("QEmd", "[distance][gem]") { } CATCH_SECTION("Generic svs::distance::compute dispatches to the vector-set overload") { - auto a_set = make_id_set({0}); - auto b_set = make_id_set({1, 2}); - float via_generic = svs::distance::compute(dist, a_set.view(), b_set.view()); - float direct = dist.compute(a_set.view(), b_set.view()); + auto a = make_id_set({0}); + auto b = make_id_set({1, 2}); + float via_generic = svs::distance::compute(dist, a.view(), b.view()); + float direct = dist.compute(a.view(), b.view()); CATCH_REQUIRE(via_generic == Catch::Approx(direct)); } } @@ -332,87 +248,61 @@ CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { // Raw (non-unit) centroids: c0=(1,0), c1=(0,1), c2=(-1,0). std::vector flat{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F}; auto centroids = std::make_shared(3, 2, flat.data()); - QChamferExactQuery dist{centroids}; // Query tokens t0=(1,0), t1=(0,1); doc uses centroid ids {0,2}. - std::vector raw_query{2.0F, 1.0F, 0.0F, 0.0F, 1.0F}; - auto doc = make_set({0.0F, 2.0F}, 4); - - svs::distance::maybe_fix_argument(dist, std::span(raw_query)); - float score = - dist.compute(std::span(raw_query), std::span(doc)); - - // Naive reference built directly from the raw query tokens and centroid vectors, - // independent of build_table/compute_decoded. - std::vector> tokens{{1.0F, 0.0F}, {0.0F, 1.0F}}; - std::vector doc_ids{0, 2}; - double total = 0.0; - for (auto [tx, ty] : tokens) { - float best = -std::numeric_limits::infinity(); - for (auto id : doc_ids) { - float cx = flat[id * 2]; - float cy = flat[id * 2 + 1]; - best = std::max(best, cx * tx + cy * ty); - } - total += 1.0 - static_cast(best); - } - float expected = static_cast(total / tokens.size()); - - CATCH_REQUIRE(score == Catch::Approx(expected)); - CATCH_REQUIRE(score == Catch::Approx(0.5F)); -} - -CATCH_TEST_CASE("QChamferExactQuery vector-set path", "[distance][gem]") { - std::vector flat{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F}; - auto centroids = std::make_shared(3, 2, flat.data()); - - // Same raw query and doc ids as the padded-row "QChamferExactQuery" case above, - // reused as owning vector sets instead of a padded span. RawQuerySet query_set{{1.0F, 0.0F, 0.0F, 1.0F}, 2}; auto doc_set = make_id_set({0, 2}); - CATCH_SECTION("Matches the padded-row path") { - QChamferExactQuery padded_dist{centroids}; - std::vector raw_query{2.0F, 1.0F, 0.0F, 0.0F, 1.0F}; - auto doc_padded = make_set({0.0F, 2.0F}, 4); - svs::distance::maybe_fix_argument(padded_dist, std::span(raw_query)); - float expected = padded_dist.compute( - std::span(raw_query), std::span(doc_padded) - ); + CATCH_SECTION("Hand-computed: naive reference over raw query tokens") { + QChamferExactQuery dist{centroids}; + svs::distance::maybe_fix_argument(dist, query_set.view()); + float score = dist.compute(query_set.view(), doc_set.view()); + + // Naive reference built directly from the raw query tokens and centroid vectors, + // independent of build_table/compute_decoded. + std::vector> tokens{{1.0F, 0.0F}, {0.0F, 1.0F}}; + std::vector doc_ids{0, 2}; + double total = 0.0; + for (auto [tx, ty] : tokens) { + float best = -std::numeric_limits::infinity(); + for (auto id : doc_ids) { + float cx = flat[id * 2]; + float cy = flat[id * 2 + 1]; + best = std::max(best, cx * tx + cy * ty); + } + total += 1.0 - static_cast(best); + } + float expected = static_cast(total / tokens.size()); - QChamferExactQuery set_dist{centroids}; - svs::distance::maybe_fix_argument(set_dist, query_set.view()); - float actual = set_dist.compute(query_set.view(), doc_set.view()); - CATCH_REQUIRE(actual == Catch::Approx(expected)); + CATCH_REQUIRE(score == Catch::Approx(expected)); + CATCH_REQUIRE(score == Catch::Approx(0.5F)); } CATCH_SECTION("Generic maybe_fix_argument/compute dispatch to the vector-set overload" ) { - QChamferExactQuery set_dist{centroids}; - svs::distance::maybe_fix_argument(set_dist, query_set.view()); - float direct = set_dist.compute(query_set.view(), doc_set.view()); - float via_generic = - svs::distance::compute(set_dist, query_set.view(), doc_set.view()); + QChamferExactQuery dist{centroids}; + svs::distance::maybe_fix_argument(dist, query_set.view()); + float direct = dist.compute(query_set.view(), doc_set.view()); + float via_generic = svs::distance::compute(dist, query_set.view(), doc_set.view()); CATCH_REQUIRE(via_generic == Catch::Approx(direct)); } CATCH_SECTION("fix_argument: query dimension mismatch throws") { - QChamferExactQuery set_dist{centroids}; + QChamferExactQuery dist{centroids}; RawQuerySet wrong_dim{{1.0F, 0.0F, 0.0F}, 3}; CATCH_REQUIRE_THROWS_AS( - svs::distance::maybe_fix_argument(set_dist, wrong_dim.view()), - std::invalid_argument + svs::distance::maybe_fix_argument(dist, wrong_dim.view()), std::invalid_argument ); } CATCH_SECTION("compute: doc dimensions() != 1 throws") { - QChamferExactQuery set_dist{centroids}; - svs::distance::maybe_fix_argument(set_dist, query_set.view()); + QChamferExactQuery dist{centroids}; + svs::distance::maybe_fix_argument(dist, query_set.view()); std::vector raw{0, 1, 2, 3}; auto bad_doc = svs::data::ConstVectorSet(std::span(raw), 2, 2); CATCH_REQUIRE_THROWS_AS( - set_dist.compute(query_set.view(), bad_doc), std::invalid_argument + dist.compute(query_set.view(), bad_doc), std::invalid_argument ); } } @@ -458,20 +348,16 @@ CATCH_TEST_CASE("GEM Vamana extensions", "[distance][gem][vamana]") { QEmd qemd{table}; int dummy_data = 0; auto adaptor = svs::index::vamana::extensions::build_adaptor(dummy_data, qemd); - auto a = make_set({0.0F}, 4); - auto b = make_set({1.0F}, 4); - float score = adaptor.general_distance().compute( - std::span(a), std::span(b) - ); + auto a = make_id_set({0}); + auto b = make_id_set({1}); + float score = adaptor.general_distance().compute(a.view(), b.view()); CATCH_REQUIRE(score == Catch::Approx(1.0F)); // table(0,1), same as the QEmd test. QChamfer qchamfer{table}; auto adaptor2 = svs::index::vamana::extensions::build_adaptor(dummy_data, qchamfer); - auto q = make_set({0.0F, 1.0F}, 6); - auto d = make_set({2.0F, 3.0F}, 6); - float score2 = adaptor2.general_distance().compute( - std::span(q), std::span(d) - ); + auto q = make_id_set({0, 1}); + auto d = make_id_set({2, 3}); + float score2 = adaptor2.general_distance().compute(q.view(), d.view()); CATCH_REQUIRE(score2 == Catch::Approx(2.0F)); } } From b362c83dd8f3d120b2523a560d3ad06df60c53c4 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 03:26:18 -0800 Subject: [PATCH 13/68] feat(distance): add a register-blocked inner-product matrix kernel QChamferExactQuery::fix_argument built its (K x n_query) dot table with a per-centroid-row loop limited to 2 independent FMA accumulators, which left it latency-bound well short of the AVX-512 FMA ceiling. Factor that build into a standalone inner_product_matrix kernel (generic/AVX2/AVX512, same dispatch mechanism as the other GEM distances) that blocks several A-rows per step so the FMA dependency chain is spread across more accumulators; GEM's build_table is now a direct call into it. The reduction half (compute_decoded) is unchanged in this commit; its cross-TU weak-symbol duplication is fixed separately next. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/distance/gem.h | 112 ++-------- .../svs/core/distance/inner_product_matrix.h | 131 ++++++++++++ include/svs/multi-arch/x86/avx2.cpp | 130 ++++++++---- include/svs/multi-arch/x86/avx512.cpp | 139 ++++++++---- tests/CMakeLists.txt | 1 + .../core/distances/inner_product_matrix.cpp | 198 ++++++++++++++++++ 6 files changed, 539 insertions(+), 172 deletions(-) create mode 100644 include/svs/core/distance/inner_product_matrix.h create mode 100644 tests/svs/core/distances/inner_product_matrix.cpp diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index b0334bf60..af1b67940 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -28,6 +28,7 @@ #include "svs/concepts/vector_set.h" #include "svs/core/distance/distance_core.h" +#include "svs/core/distance/inner_product_matrix.h" #include "svs/lib/avx_detection.h" #include "svs/lib/preprocessor.h" #include "svs/lib/threads.h" @@ -420,35 +421,11 @@ template struct QChamferImpl { }; // Dispatched like QChamferImpl. GEM never quantizes the query token side (src/search/ -// routing.cpp:55-61, src/distance/maxsim.cpp:9-45): build_table scores the raw float -// query tokens against every fine centroid (score_fine_centers), and compute_decoded -// reduces that table against the document's quantized centroid ids (code_distance). +// routing.cpp:55-61, src/distance/maxsim.cpp:9-45): QChamferExactQuery scores the raw +// float query tokens against every fine centroid via InnerProductMatrix +// (score_fine_centers), and compute_decoded reduces that table against the document's +// quantized centroid ids (code_distance). template struct QChamferExactImpl { - // table[f * n_query + i] = dot(centroid_f, query_token_i), matching maxsim.cpp's - // Eigen::Map(table, query_tokens, fine_centers) column-major read of the - // same buffer (scores.col(f)[i] == table[f*n_query+i]). - SVS_NOINLINE static void build_table( - const float* centroids, - size_t fine_centers, - size_t dim, - const float* query_tokens, - size_t n_query, - float* table - ) { - for (size_t f = 0; f < fine_centers; ++f) { - const float* c = centroids + f * dim; - float* row = table + f * n_query; - for (size_t i = 0; i < n_query; ++i) { - const float* t = query_tokens + i * dim; - float dot = 0.0F; - for (size_t d = 0; d < dim; ++d) { - dot += c[d] * t[d]; - } - row[i] = dot; - } - } - } - // GEM's code_distance (src/distance/maxsim.cpp:9-45): for each query token, the max // dot product to any of the document's (possibly repeated) centroid ids; final score // is `1 - mean_q max_doc dot`, i.e. 1 minus the mean of those per-token maxima. @@ -461,31 +438,12 @@ template struct QChamferExactImpl { #if defined(__x86_64__) -// build_table vectorizes over query tokens, not over `dim`: per output lane i, -// the d=0..dim-1 FMA order matches the scalar version exactly, so only the -// single-rounding FMA (vs. separate mul+add) can move the result off the scalar -// value, well under the 1e-6 relative tolerance this table is checked against. -// Transposing query_tokens into [dim][n_query] turns each centroid row into -// contiguous per-token accumulators with no horizontal reduction, at the cost of -// a one-time O(dim * n_query) transpose (query is a handful of KiB; centroids -// are tens to hundreds of MiB, so the transpose is noise against the table build -// it replaces the scalar dot loop in). -// Declared here, defined in avx512.cpp/avx2.cpp: the bodies use __m512/__m256 intrinsics, -// so each must only be compiled under its own -march flags, not wherever gem.h is included -// (e.g. gem_time.cpp, which carries neither). Defining them in-class would make them -// implicitly inline, and an explicit instantiation of an already-specialized class is a -// no-op in GCC, so no other TU would get a symbol to link against. +// Declared here, defined in avx512.cpp/avx2.cpp: without SVS_NOINLINE the caller would +// inline the body at baseline flags; an explicit instantiation of an already-specialized +// class is a no-op in GCC, so no other TU would get a symbol to link against otherwise. SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) #if SVS_AVX512_F template <> struct QChamferExactImpl { - SVS_NOINLINE static void build_table( - const float* centroids, - size_t fine_centers, - size_t dim, - const float* query_tokens, - size_t n_query, - float* table - ); SVS_NOINLINE static float compute_decoded( const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc ); @@ -496,14 +454,6 @@ SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) SVS_VALIDATE_BOOL_ENV(SVS_AVX2) #if !SVS_AVX512_F && SVS_AVX2 template <> struct QChamferExactImpl { - SVS_NOINLINE static void build_table( - const float* centroids, - size_t fine_centers, - size_t dim, - const float* query_tokens, - size_t n_query, - float* table - ); SVS_NOINLINE static float compute_decoded( const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc ); @@ -688,41 +638,23 @@ class QChamferExactQuery { } private: - // Builds the (K x n_query) dot table once per query, dispatched to the ISA the - // per-ISA compiled build_table was built for. + // Builds the (K x n_query) dot table once per query: table[f * n_query + i] = + // dot(centroid_f, query_token_i), matching maxsim.cpp's Eigen::Map(table, + // query_tokens, fine_centers) column-major read of the same buffer (scores.col(f)[i] + // == table[f*n_query+i]). That is exactly InnerProductMatrix::compute's A x B^T with + // A = centroids (K rows) and B = query tokens (n_query rows), which now owns both the + // ISA choice and the query-side packing. void fix_argument_decoded(const float* tokens, size_t n_query) { n_query_ = n_query; table_.resize(centroids_->K * n_query_); - if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { - QChamferExactImpl::build_table( - centroids_->data.get(), - centroids_->K, - centroids_->dim, - tokens, - n_query_, - table_.data() - ); - } else if (__builtin_expect( - svs::detail::avx_runtime_flags.is_avx2_supported(), 1 - )) { - QChamferExactImpl::build_table( - centroids_->data.get(), - centroids_->K, - centroids_->dim, - tokens, - n_query_, - table_.data() - ); - } else { - QChamferExactImpl::build_table( - centroids_->data.get(), - centroids_->K, - centroids_->dim, - tokens, - n_query_, - table_.data() - ); - } + InnerProductMatrix::compute( + centroids_->data.get(), + centroids_->K, + tokens, + n_query_, + centroids_->dim, + table_.data() + ); } // Reduces the per-query table against a doc's decoded ids, dispatched to the ISA the diff --git a/include/svs/core/distance/inner_product_matrix.h b/include/svs/core/distance/inner_product_matrix.h new file mode 100644 index 000000000..07421866a --- /dev/null +++ b/include/svs/core/distance/inner_product_matrix.h @@ -0,0 +1,131 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/core/distance/distance_core.h" +#include "svs/lib/avx_detection.h" +#include "svs/lib/preprocessor.h" + +#include + +namespace svs::distance { + +// One accumulator per (A-row, B-lane-block), no horizontal reduction: measured on this +// machine, 8 independent accumulators is where row-blocking stops buying throughput, +// matching ~4-cycle FMA latency x 2 ports (see the PR description for the sweep). +inline constexpr size_t inner_product_matrix_row_block = 8; + +/// Per-ISA kernel for the dense inner-product table of a large row-major A (m x d) against +/// a small row-major B (n x d), both float32: out[i * n + j] = , out row-major by +/// A's rows. This primary template is the scalar, compiler-autovectorized generic +/// implementation: both the fallback when no SIMD ISA is available and the reference the +/// ISA-specific specializations below are checked against in tests. Those specializations +/// are declared here and defined in avx2.cpp / avx512.cpp so their intrinsics only compile +/// under their own -march flags, mirroring QChamferExactImpl above. +template struct InnerProductMatrixImpl { + SVS_NOINLINE static void + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + for (size_t i = 0; i < m; ++i) { + const float* ai = a + i * d; + float* row = out + i * n; + for (size_t j = 0; j < n; ++j) { + const float* bj = b + j * d; + float dot = 0.0F; + for (size_t k = 0; k < d; ++k) { + dot += ai[k] * bj[k]; + } + row[j] = dot; + } + } + } +}; + +#if defined(__x86_64__) + +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +#if SVS_AVX512_F +template <> struct InnerProductMatrixImpl { + SVS_NOINLINE static void + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); +}; +#endif + +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +SVS_VALIDATE_BOOL_ENV(SVS_AVX2) +#if !SVS_AVX512_F && SVS_AVX2 +template <> struct InnerProductMatrixImpl { + SVS_NOINLINE static void + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); +}; +#endif + +#endif // defined(__x86_64__) + +#if defined(__x86_64__) +extern template struct InnerProductMatrixImpl; +extern template struct InnerProductMatrixImpl; +#endif + +/// Dense inner-product table between a large row-major matrix and a small row-major +/// matrix, both float32: out[i * n + j] = for i in [0, m), j in [0, n). Picks +/// the ISA implementation once per call, outside every loop over rows or columns. +/// +/// @param a Row-major "A" operand, m rows of d floats each; the large, streamed operand +/// (e.g. all centroids). +/// @param m Number of rows of `a`. 0 is valid and leaves `out` untouched. +/// @param b Row-major "B" operand, n rows of d floats each; the small operand (e.g. one +/// query's tokens). Packed once per call into ISA-specific scratch the implementation +/// owns (thread_local, as the GEM query transpose above does), so `b` itself is read +/// once and not retained. +/// @param n Number of rows of `b`. 0 is valid and leaves `out` untouched. +/// @param d Row width shared by `a` and `b`. +/// @param out Row-major output buffer, caller-owned, at least m * n floats: out[i * n + j] +/// holds . +class InnerProductMatrix { + public: + static void + compute(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + switch (select_isa()) { + case AVX_AVAILABILITY::AVX512: + InnerProductMatrixImpl::build(a, m, b, n, d, out); + return; + case AVX_AVAILABILITY::AVX2: + InnerProductMatrixImpl::build(a, m, b, n, d, out); + return; + case AVX_AVAILABILITY::NONE: + InnerProductMatrixImpl::build(a, m, b, n, d, out); + return; + } + // Exhaustive switch over AVX_AVAILABILITY's enumerators, no default: -Wswitch + // under -Werror turns a 4th enumerator added later into a compile error here, + // rather than this call silently keeping today's dispatch. + __builtin_unreachable(); + } + + private: + static AVX_AVAILABILITY select_isa() { + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + return AVX_AVAILABILITY::AVX512; + } + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + return AVX_AVAILABILITY::AVX2; + } + return AVX_AVAILABILITY::NONE; + } +}; + +} // namespace svs::distance diff --git a/include/svs/multi-arch/x86/avx2.cpp b/include/svs/multi-arch/x86/avx2.cpp index da312e1c6..da914802f 100644 --- a/include/svs/multi-arch/x86/avx2.cpp +++ b/include/svs/multi-arch/x86/avx2.cpp @@ -19,6 +19,8 @@ #include "svs/core/distance/euclidean.h" #include "svs/core/distance/gem.h" #include "svs/core/distance/inner_product.h" +#include "svs/core/distance/inner_product_matrix.h" +#include "svs/core/distance/simd_utils.h" namespace svs::distance { @@ -56,56 +58,108 @@ DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); template struct QEmdImpl; template struct QChamferImpl; template struct QChamferExactImpl; +template struct InnerProductMatrixImpl; -// See avx512.cpp's QChamferExactImpl::build_table for the accumulation-order and -// rounding rationale; this is the same scheme at AVX2's 8-wide registers. -void QChamferExactImpl::build_table( - const float* centroids, - size_t fine_centers, - size_t dim, - const float* query_tokens, - size_t n_query, - float* table +float QChamferExactImpl::compute_decoded( + const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc ) { + return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); +} + +// Same register-blocking as avx512.cpp's InnerProductMatrixImpl::build, at AVX2's +// 8-wide registers; AVX2 has no masked load/store, so the tail lane-block uses +// _mm256_maskload_ps/_mm256_maskstore_ps with an all-lanes blend mask built from the bit +// pattern simd_utils.h's ConvertToFloat<8> uses for the same purpose. +void InnerProductMatrixImpl::build( + const float* a, size_t m, const float* b, size_t n, size_t d, float* out +) { + if (m == 0 || n == 0) { + return; + } constexpr size_t vl = 8; - thread_local std::vector qt; - qt.resize(dim * n_query); - for (size_t i = 0; i < n_query; ++i) { - const float* t = query_tokens + i * dim; - for (size_t d = 0; d < dim; ++d) { - qt[d * n_query + i] = t[d]; + constexpr size_t row_block = inner_product_matrix_row_block; + + thread_local std::vector bt; + bt.resize(d * n); + for (size_t j = 0; j < n; ++j) { + const float* bj = b + j * d; + for (size_t k = 0; k < d; ++k) { + bt[k * n + j] = bj[k]; } } - size_t vec_end = n_query - n_query % vl; - for (size_t f = 0; f < fine_centers; ++f) { - const float* c = centroids + f * dim; - float* row = table + f * n_query; - size_t ci = 0; - for (; ci < vec_end; ci += vl) { - __m256 acc = _mm256_setzero_ps(); - for (size_t d = 0; d < dim; ++d) { - __m256 cval = _mm256_set1_ps(c[d]); - __m256 qv = _mm256_loadu_ps(qt.data() + d * n_query + ci); - acc = _mm256_fmadd_ps(cval, qv, acc); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + size_t full_rows = m - m % row_block; + size_t i = 0; + for (; i < full_rows; i += row_block) { + const float* arow[row_block]; + float* orow[row_block]; + for (size_t r = 0; r < row_block; ++r) { + arow[r] = a + (i + r) * d; + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[row_block]; + for (size_t r = 0; r < row_block; ++r) { + acc[r] = _mm256_setzero_ps(); + } + for (size_t k = 0; k < d; ++k) { + __m256 bv = _mm256_loadu_ps(bt.data() + k * n + j); + for (size_t r = 0; r < row_block; ++r) { + acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); + } + } + for (size_t r = 0; r < row_block; ++r) { + _mm256_storeu_ps(orow[r] + j, acc[r]); } - _mm256_storeu_ps(row + ci, acc); } - for (; ci < n_query; ++ci) { - const float* t = query_tokens + ci * dim; - float dot = 0.0F; - for (size_t d = 0; d < dim; ++d) { - dot += c[d] * t[d]; + if (j < n) { + __m256 acc[row_block]; + for (size_t r = 0; r < row_block; ++r) { + acc[r] = _mm256_setzero_ps(); + } + for (size_t k = 0; k < d; ++k) { + __m256 bv = _mm256_maskload_ps(bt.data() + k * n + j, tail_mask); + for (size_t r = 0; r < row_block; ++r) { + acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); + } + } + for (size_t r = 0; r < row_block; ++r) { + _mm256_maskstore_ps(orow[r] + j, tail_mask, acc[r]); } - row[ci] = dot; } } -} -float QChamferExactImpl::compute_decoded( - const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc -) { - return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); + for (; i < m; ++i) { + const float* ai = a + i * d; + float* row = out + i * n; + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc = _mm256_setzero_ps(); + for (size_t k = 0; k < d; ++k) { + acc = _mm256_fmadd_ps( + _mm256_set1_ps(ai[k]), _mm256_loadu_ps(bt.data() + k * n + j), acc + ); + } + _mm256_storeu_ps(row + j, acc); + } + if (j < n) { + __m256 acc = _mm256_setzero_ps(); + for (size_t k = 0; k < d; ++k) { + acc = _mm256_fmadd_ps( + _mm256_set1_ps(ai[k]), + _mm256_maskload_ps(bt.data() + k * n + j, tail_mask), + acc + ); + } + _mm256_maskstore_ps(row + j, tail_mask, acc); + } + } } } // namespace svs::distance diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index fbe941df9..3d184b76a 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -19,6 +19,8 @@ #include "svs/core/distance/euclidean.h" #include "svs/core/distance/gem.h" #include "svs/core/distance/inner_product.h" +#include "svs/core/distance/inner_product_matrix.h" +#include "svs/core/distance/simd_utils.h" namespace svs::distance { @@ -56,62 +58,111 @@ DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); template struct QEmdImpl; template struct QChamferImpl; template struct QChamferExactImpl; +template struct InnerProductMatrixImpl; -// build_table vectorizes over query tokens, not over `dim`: per output lane i, the -// d=0..dim-1 FMA order matches the scalar version exactly, so only the single-rounding -// FMA (vs. separate mul+add) can move the result off the scalar value, well under the -// 1e-6 relative tolerance this table is checked against. Transposing query_tokens into -// [dim][n_query] turns each centroid row into contiguous per-token accumulators with no -// horizontal reduction, at the cost of a one-time O(dim * n_query) transpose (query is a -// handful of KiB; centroids are tens to hundreds of MiB, so the transpose is noise -// against the table build it replaces the scalar dot loop in). -void QChamferExactImpl::build_table( - const float* centroids, - size_t fine_centers, - size_t dim, - const float* query_tokens, - size_t n_query, - float* table +float QChamferExactImpl::compute_decoded( + const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc ) { + return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); +} + +// Register-blocked micro-kernel: inner_product_matrix_row_block rows of A share one pass +// over a lane-block of B, giving that many independent FMA accumulator chains with no +// horizontal reduction (each lane is one output), so the dependency-chain latency of a +// single accumulator is hidden behind the others instead of serializing the loop over d. +void InnerProductMatrixImpl::build( + const float* a, size_t m, const float* b, size_t n, size_t d, float* out +) { + if (m == 0 || n == 0) { + return; + } constexpr size_t vl = 16; - thread_local std::vector qt; - qt.resize(dim * n_query); - for (size_t i = 0; i < n_query; ++i) { - const float* t = query_tokens + i * dim; - for (size_t d = 0; d < dim; ++d) { - qt[d * n_query + i] = t[d]; + constexpr size_t row_block = inner_product_matrix_row_block; + + // Packs B transposed (bt[k * n + j] = b[j * d + k]) once per call: B is the small + // operand (e.g. one query's tokens, a handful of KiB), so this copy is noise next to + // the m x n table it lets every row below read with no horizontal reduction. + thread_local std::vector bt; + bt.resize(d * n); + for (size_t j = 0; j < n; ++j) { + const float* bj = b + j * d; + for (size_t k = 0; k < d; ++k) { + bt[k * n + j] = bj[k]; } } - size_t vec_end = n_query - n_query % vl; - for (size_t f = 0; f < fine_centers; ++f) { - const float* c = centroids + f * dim; - float* row = table + f * n_query; - size_t ci = 0; - for (; ci < vec_end; ci += vl) { - __m512 acc = _mm512_setzero_ps(); - for (size_t d = 0; d < dim; ++d) { - __m512 cval = _mm512_set1_ps(c[d]); - __m512 qv = _mm512_loadu_ps(qt.data() + d * n_query + ci); - acc = _mm512_fmadd_ps(cval, qv, acc); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + size_t full_rows = m - m % row_block; + size_t i = 0; + for (; i < full_rows; i += row_block) { + const float* arow[row_block]; + float* orow[row_block]; + for (size_t r = 0; r < row_block; ++r) { + arow[r] = a + (i + r) * d; + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[row_block]; + for (size_t r = 0; r < row_block; ++r) { + acc[r] = _mm512_setzero_ps(); + } + for (size_t k = 0; k < d; ++k) { + __m512 bv = _mm512_loadu_ps(bt.data() + k * n + j); + for (size_t r = 0; r < row_block; ++r) { + acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); + } + } + for (size_t r = 0; r < row_block; ++r) { + _mm512_storeu_ps(orow[r] + j, acc[r]); } - _mm512_storeu_ps(row + ci, acc); } - for (; ci < n_query; ++ci) { - const float* t = query_tokens + ci * dim; - float dot = 0.0F; - for (size_t d = 0; d < dim; ++d) { - dot += c[d] * t[d]; + if (j < n) { + __m512 acc[row_block]; + for (size_t r = 0; r < row_block; ++r) { + acc[r] = _mm512_setzero_ps(); + } + for (size_t k = 0; k < d; ++k) { + __m512 bv = _mm512_maskz_loadu_ps(tail_mask, bt.data() + k * n + j); + for (size_t r = 0; r < row_block; ++r) { + acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); + } + } + for (size_t r = 0; r < row_block; ++r) { + _mm512_mask_storeu_ps(orow[r] + j, tail_mask, acc[r]); } - row[ci] = dot; } } -} -float QChamferExactImpl::compute_decoded( - const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc -) { - return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); + // Row remainder (m % row_block): same lane logic with a single accumulator per row. + for (; i < m; ++i) { + const float* ai = a + i * d; + float* row = out + i * n; + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc = _mm512_setzero_ps(); + for (size_t k = 0; k < d; ++k) { + acc = _mm512_fmadd_ps( + _mm512_set1_ps(ai[k]), _mm512_loadu_ps(bt.data() + k * n + j), acc + ); + } + _mm512_storeu_ps(row + j, acc); + } + if (j < n) { + __m512 acc = _mm512_setzero_ps(); + for (size_t k = 0; k < d; ++k) { + acc = _mm512_fmadd_ps( + _mm512_set1_ps(ai[k]), + _mm512_maskz_loadu_ps(tail_mask, bt.data() + k * n + j), + acc + ); + } + _mm512_mask_storeu_ps(row + j, tail_mask, acc); + } + } } } // namespace svs::distance diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 058402b44..a61caf961 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -115,6 +115,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/core/distances/simd_utils.cpp ${TEST_DIR}/svs/core/distances/distance_euclidean.cpp ${TEST_DIR}/svs/core/distances/inner_product.cpp + ${TEST_DIR}/svs/core/distances/inner_product_matrix.cpp ${TEST_DIR}/svs/core/distances/cosine.cpp ${TEST_DIR}/svs/core/distances/gem.cpp ${TEST_DIR}/svs/core/distances/maxsim.cpp diff --git a/tests/svs/core/distances/inner_product_matrix.cpp b/tests/svs/core/distances/inner_product_matrix.cpp new file mode 100644 index 000000000..6da8ef916 --- /dev/null +++ b/tests/svs/core/distances/inner_product_matrix.cpp @@ -0,0 +1,198 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test +#include "svs/core/distance/inner_product_matrix.h" + +// svs +#include "svs/lib/avx_detection.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stdlib +#include +#include +#include + +namespace { + +using svs::distance::AVX_AVAILABILITY; +using svs::distance::InnerProductMatrix; +using svs::distance::InnerProductMatrixImpl; + +// float32 accumulation over at most d=129 terms of values in [-1, 1]: summation order +// differs between the generic and SIMD kernels, so the tolerance must absorb that, not +// just single-ULP rounding. +constexpr double REL_EPS = 1e-4; +constexpr double ABS_EPS = 1e-5; + +std::vector random_matrix(size_t rows, size_t d, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution dist(-1.0F, 1.0F); + std::vector out(rows * d); + for (auto& v : out) { + v = dist(rng); + } + return out; +} + +// Naive double-precision reference: out[i * n + j] = . +std::vector reference( + const std::vector& a, size_t m, const std::vector& b, size_t n, size_t d +) { + std::vector out(m * n, 0.0); + for (size_t i = 0; i < m; ++i) { + for (size_t j = 0; j < n; ++j) { + double acc = 0.0; + for (size_t k = 0; k < d; ++k) { + acc += + static_cast(a[i * d + k]) * static_cast(b[j * d + k]); + } + out[i * n + j] = acc; + } + } + return out; +} + +void require_close(const std::vector& got, const std::vector& expected) { + CATCH_REQUIRE(got.size() == expected.size()); + for (size_t i = 0; i < got.size(); ++i) { + CATCH_REQUIRE( + static_cast(got[i]) == + Catch::Approx(expected[i]).epsilon(REL_EPS).margin(ABS_EPS) + ); + } +} + +// m x n x d shapes exercising every tail: row-block remainder, SIMD lane remainder (8 and +// 16 wide), and both small and multi-cache-line `d`. +const std::vector ms{1, 2, 3, 7, 8, 9, 33}; +const std::vector ns{1, 2, 15, 16, 17, 31, 32, 33, 64}; +const std::vector ds{1, 7, 128, 129}; + +template void check_against_reference() { + unsigned seed = 0; + for (size_t d : ds) { + for (size_t m : ms) { + for (size_t n : ns) { + auto a = random_matrix(m, d, seed++); + auto b = random_matrix(n, d, seed++); + std::vector out(m * n, -1.0F); + InnerProductMatrixImpl::build(a.data(), m, b.data(), n, d, out.data()); + require_close(out, reference(a, m, b, n, d)); + } + } + } +} + +template void check_against_generic() { + unsigned seed = 1000; + for (size_t d : ds) { + for (size_t m : ms) { + for (size_t n : ns) { + auto a = random_matrix(m, d, seed++); + auto b = random_matrix(n, d, seed++); + std::vector generic_out(m * n, -1.0F); + std::vector isa_out(m * n, -2.0F); + InnerProductMatrixImpl::build( + a.data(), m, b.data(), n, d, generic_out.data() + ); + InnerProductMatrixImpl::build( + a.data(), m, b.data(), n, d, isa_out.data() + ); + for (size_t i = 0; i < generic_out.size(); ++i) { + CATCH_REQUIRE( + static_cast(isa_out[i]) == + Catch::Approx(static_cast(generic_out[i])) + .epsilon(REL_EPS) + .margin(ABS_EPS) + ); + } + } + } + } +} + +} // namespace + +CATCH_TEST_CASE( + "InnerProductMatrix generic kernel matches a naive reference", + "[distance][inner_product_matrix]" +) { + check_against_reference(); +} + +CATCH_TEST_CASE( + "InnerProductMatrix AVX2 kernel matches generic", "[distance][inner_product_matrix]" +) { + if (!svs::detail::avx_runtime_flags.is_avx2_supported()) { + return; + } + check_against_reference(); + check_against_generic(); +} + +CATCH_TEST_CASE( + "InnerProductMatrix AVX512 kernel matches generic", "[distance][inner_product_matrix]" +) { + if (!svs::detail::avx_runtime_flags.is_avx512f_supported()) { + return; + } + check_against_reference(); + check_against_generic(); +} + +CATCH_TEST_CASE( + "InnerProductMatrix public wrapper equals the reference", + "[distance][inner_product_matrix]" +) { + for (size_t d : {1UL, 7UL, 128UL, 129UL}) { + for (size_t m : {1UL, 9UL, 33UL}) { + for (size_t n : {1UL, 17UL, 64UL}) { + auto a = random_matrix(m, d, static_cast(m * 100 + n * 10 + d)); + auto b = random_matrix(n, d, static_cast(d * 100 + m * 10 + n)); + std::vector out(m * n, -1.0F); + InnerProductMatrix::compute(a.data(), m, b.data(), n, d, out.data()); + require_close(out, reference(a, m, b, n, d)); + } + } + } +} + +CATCH_TEST_CASE( + "InnerProductMatrix degenerate shapes leave `out` untouched", + "[distance][inner_product_matrix]" +) { + constexpr size_t d = 4; + auto a = random_matrix(3, d, 7); + auto b = random_matrix(3, d, 11); + + CATCH_SECTION("m == 0") { + std::vector out{1.0F, 2.0F, 3.0F}; + auto before = out; + InnerProductMatrix::compute(a.data(), 0, b.data(), 3, d, out.data()); + CATCH_REQUIRE(out == before); + } + + CATCH_SECTION("n == 0") { + std::vector out{1.0F, 2.0F, 3.0F}; + auto before = out; + InnerProductMatrix::compute(a.data(), 3, b.data(), 0, d, out.data()); + CATCH_REQUIRE(out == before); + } +} From 24e9220ed4387e3ef64785611fec3d059f8aada5 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 03:31:03 -0800 Subject: [PATCH 14/68] fix(distance): stop sharing QChamferExactImpl's reduction across ISA TUs gem_detail::qchamfer_exact_reduce was a free inline function called from QChamferExactImpl::compute_decoded in the generic header and from both avx2.cpp and avx512.cpp. Each TU compiled it under different -m flags but emitted the same weak inline symbol, so the linker could keep an arbitrary one; a host lacking AVX-512 could end up executing code built for it. Move the reduction into QChamferExactImpl's primary template body (one source text) and drop its AVX2/AVX512 specializations entirely: the explicit instantiation already in avx2.cpp/avx512.cpp now compiles that same body under each TU's own flags, giving each ISA a distinctly named, non-weak symbol, matching how QEmdImpl/QChamferImpl already do this. Verified via `nm -C` on both object files: no QEmdImpl/QChamferImpl/ QChamferExactImpl/InnerProductMatrixImpl symbol is shared between them anymore. MergeScratch's compiler-generated destructor still is, but it holds no SIMD-sensitive logic, so it is not an ODR hazard. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/distance/gem.h | 68 +++++++-------------------- include/svs/multi-arch/x86/avx2.cpp | 6 --- include/svs/multi-arch/x86/avx512.cpp | 6 --- 3 files changed, 16 insertions(+), 64 deletions(-) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index af1b67940..42f6f31c3 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -227,26 +227,6 @@ struct MergeScratch { std::vector weights; }; -// Shared by QChamferExactImpl's generic and per-ISA compute_decoded specializations -// (GEM's code_distance, src/distance/maxsim.cpp:9-45) so the reduction is written once. -inline float qchamfer_exact_reduce( - const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc -) { - thread_local std::vector maxima; - maxima.assign(n_query, -std::numeric_limits::infinity()); - for (size_t j = 0; j < n_doc; ++j) { - const float* row = table + static_cast(doc_ids[j]) * n_query; - for (size_t i = 0; i < n_query; ++i) { - maxima[i] = std::max(maxima[i], row[i]); - } - } - double total = 0.0; - for (size_t i = 0; i < n_query; ++i) { - total += 1.0 - static_cast(maxima[i]); - } - return static_cast(total / static_cast(n_query)); -} - // Casts a vector set's centroid ids to uint32_t with no merge; qCH counts duplicates. // Only instantiated for element types other than uint32_t (id_span's cast-copy branch). template @@ -420,11 +400,9 @@ template struct QChamferImpl { } }; -// Dispatched like QChamferImpl. GEM never quantizes the query token side (src/search/ -// routing.cpp:55-61, src/distance/maxsim.cpp:9-45): QChamferExactQuery scores the raw -// float query tokens against every fine centroid via InnerProductMatrix -// (score_fine_centers), and compute_decoded reduces that table against the document's -// quantized centroid ids (code_distance). +// GEM never quantizes query tokens: QChamferExactQuery scores them via InnerProductMatrix. +// compute_decoded reduces that table; no intrinsics, but still instantiated per ISA (not +// specialized) like QEmdImpl, so no weak inline symbol is shared across -m flags. template struct QChamferExactImpl { // GEM's code_distance (src/distance/maxsim.cpp:9-45): for each query token, the max // dot product to any of the document's (possibly repeated) centroid ids; final score @@ -432,36 +410,22 @@ template struct QChamferExactImpl { SVS_NOINLINE static float compute_decoded( const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc ) { - return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); + thread_local std::vector maxima; + maxima.assign(n_query, -std::numeric_limits::infinity()); + for (size_t j = 0; j < n_doc; ++j) { + const float* row = table + static_cast(doc_ids[j]) * n_query; + for (size_t i = 0; i < n_query; ++i) { + maxima[i] = std::max(maxima[i], row[i]); + } + } + double total = 0.0; + for (size_t i = 0; i < n_query; ++i) { + total += 1.0 - static_cast(maxima[i]); + } + return static_cast(total / static_cast(n_query)); } }; -#if defined(__x86_64__) - -// Declared here, defined in avx512.cpp/avx2.cpp: without SVS_NOINLINE the caller would -// inline the body at baseline flags; an explicit instantiation of an already-specialized -// class is a no-op in GCC, so no other TU would get a symbol to link against otherwise. -SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) -#if SVS_AVX512_F -template <> struct QChamferExactImpl { - SVS_NOINLINE static float compute_decoded( - const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc - ); -}; -#endif - -SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) -SVS_VALIDATE_BOOL_ENV(SVS_AVX2) -#if !SVS_AVX512_F && SVS_AVX2 -template <> struct QChamferExactImpl { - SVS_NOINLINE static float compute_decoded( - const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc - ); -}; -#endif - -#endif // defined(__x86_64__) - #if defined(__x86_64__) extern template struct QEmdImpl; extern template struct QEmdImpl; diff --git a/include/svs/multi-arch/x86/avx2.cpp b/include/svs/multi-arch/x86/avx2.cpp index da914802f..a8994c844 100644 --- a/include/svs/multi-arch/x86/avx2.cpp +++ b/include/svs/multi-arch/x86/avx2.cpp @@ -60,12 +60,6 @@ template struct QChamferImpl; template struct QChamferExactImpl; template struct InnerProductMatrixImpl; -float QChamferExactImpl::compute_decoded( - const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc -) { - return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); -} - // Same register-blocking as avx512.cpp's InnerProductMatrixImpl::build, at AVX2's // 8-wide registers; AVX2 has no masked load/store, so the tail lane-block uses // _mm256_maskload_ps/_mm256_maskstore_ps with an all-lanes blend mask built from the bit diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index 3d184b76a..c48fb9930 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -60,12 +60,6 @@ template struct QChamferImpl; template struct QChamferExactImpl; template struct InnerProductMatrixImpl; -float QChamferExactImpl::compute_decoded( - const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc -) { - return gem_detail::qchamfer_exact_reduce(table, n_query, doc_ids, n_doc); -} - // Register-blocked micro-kernel: inner_product_matrix_row_block rows of A share one pass // over a lane-block of B, giving that many independent FMA accumulator chains with no // horizontal reduction (each lane is one output), so the dependency-chain latency of a From aed92850e72a2375b7baeff99ceceaec26064de4 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 03:50:55 -0800 Subject: [PATCH 15/68] feat(distance): rerank MaxSim with a column-max inner-product kernel Extends the inner-product-matrix kernel family with a shared PackedOperand (query packed once, reused across documents) and a column-max output form that keeps the lane-max in registers instead of materializing the m x n table, with a Float16 overload that converts one row-block at a time. MaxSim now follows the fix_argument protocol (like QChamferExactQuery) so the Vamana re-rank extension packs the query once per search instead of once per candidate. Co-Authored-By: Claude Fable 5.2 --- .../svs/core/distance/inner_product_matrix.h | 131 ++++++++- include/svs/core/distance/maxsim.h | 133 ++++++--- include/svs/extensions/vamana/maxsim.h | 11 +- include/svs/multi-arch/x86/avx2.cpp | 263 ++++++++++++++---- include/svs/multi-arch/x86/avx512.cpp | 255 +++++++++++++---- .../core/distances/inner_product_matrix.cpp | 239 ++++++++++++++++ tests/svs/core/distances/maxsim.cpp | 130 +++++---- tests/svs/index/vamana/maxsim_rerank.cpp | 14 +- 8 files changed, 949 insertions(+), 227 deletions(-) diff --git a/include/svs/core/distance/inner_product_matrix.h b/include/svs/core/distance/inner_product_matrix.h index 07421866a..ce4f1d42c 100644 --- a/include/svs/core/distance/inner_product_matrix.h +++ b/include/svs/core/distance/inner_product_matrix.h @@ -18,9 +18,13 @@ #include "svs/core/distance/distance_core.h" #include "svs/lib/avx_detection.h" +#include "svs/lib/float16.h" #include "svs/lib/preprocessor.h" +#include #include +#include +#include namespace svs::distance { @@ -29,29 +33,115 @@ namespace svs::distance { // matching ~4-cycle FMA latency x 2 ports (see the PR description for the sweep). inline constexpr size_t inner_product_matrix_row_block = 8; -/// Per-ISA kernel for the dense inner-product table of a large row-major A (m x d) against -/// a small row-major B (n x d), both float32: out[i * n + j] = , out row-major by -/// A's rows. This primary template is the scalar, compiler-autovectorized generic -/// implementation: both the fallback when no SIMD ISA is available and the reference the -/// ISA-specific specializations below are checked against in tests. Those specializations -/// are declared here and defined in avx2.cpp / avx512.cpp so their intrinsics only compile -/// under their own -march flags, mirroring QChamferExactImpl above. +/// Owns B's packed operand: B transposed into a [d][n] float32 buffer, so every row-block +/// of A can stream through it with a plain `loadu` instead of a gather. Packing is split +/// from use (`pack` vs. the kernel entry points below) because the table form repacks B +/// once per call (B is as cheap as the table it produces), while MaxSim packs one query +/// and reuses it against ~1000 documents: repacking per document would cost as much as +/// scoring them. +class PackedOperand { + public: + /// Repacks `b` (`n` rows of `d` floats each, row-major) into this object's transposed + /// buffer. Storage grows only when `n * d` exceeds the current buffer's capacity + /// (`std::vector::resize` does not reallocate when shrinking or staying within + /// capacity), so calling this with the same or a smaller shape across repeated calls + /// costs no allocation. + void pack(const float* b, size_t n, size_t d) { + n_ = n; + d_ = d; + buffer_.resize(n * d); + for (size_t j = 0; j < n; ++j) { + const float* bj = b + j * d; + for (size_t k = 0; k < d; ++k) { + buffer_[k * n + j] = bj[k]; + } + } + } + + size_t n() const { return n_; } + size_t d() const { return d_; } + const float* data() const { return buffer_.data(); } + + private: + std::vector buffer_; + size_t n_ = 0; + size_t d_ = 0; +}; + +/// Per-ISA kernel family for a large row-major A (m x d) against B, packed once into a +/// `PackedOperand` by the caller. This primary template is the scalar, compiler +/// auto-vectorized generic implementation: both the fallback when no SIMD ISA is +/// available and the reference the ISA-specific specializations below are checked +/// against in tests. Those specializations are declared here and defined in avx2.cpp / +/// avx512.cpp so their intrinsics only compile under their own -march flags, mirroring +/// QChamferExactImpl in gem.h. +/// +/// Two output forms share every row of A they stream over: +/// * `build`/`build_packed`: the dense table, out[i * n + j] = . +/// * `column_max`: out[j] = max over i of , i.e. the column-wise maximum of +/// that same table, computed with no table ever materialized. +/// +/// `column_max` is additionally overloaded on A's element type (float32 and +/// svs::Float16); the Float16 overload converts one row-block of A at a time into a +/// float32 scratch and runs the identical float32 path, so the two element types are +/// never branched on per row. template struct InnerProductMatrixImpl { SVS_NOINLINE static void - build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + const float* bt = b.data(); for (size_t i = 0; i < m; ++i) { const float* ai = a + i * d; float* row = out + i * n; for (size_t j = 0; j < n; ++j) { - const float* bj = b + j * d; float dot = 0.0F; for (size_t k = 0; k < d; ++k) { - dot += ai[k] * bj[k]; + dot += ai[k] * bt[k * n + j]; } row[j] = dot; } } } + + // Packs B into a reused thread_local buffer so this keeps its no-allocation-per-call + // property: InnerProductMatrix::compute (GEM's fix_argument path) calls this once per + // query against all K centroids, not once per document, so repacking here is cheap. + SVS_NOINLINE static void + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); + } + + template + static void + column_max_generic(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + const float* bt = b.data(); + std::fill(out, out + n, -std::numeric_limits::infinity()); + for (size_t i = 0; i < m; ++i) { + const T* ai = a + i * d; + for (size_t j = 0; j < n; ++j) { + float dot = 0.0F; + for (size_t k = 0; k < d; ++k) { + dot += static_cast(ai[k]) * bt[k * n + j]; + } + out[j] = std::max(out[j], dot); + } + } + } + + SVS_NOINLINE static void + column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + column_max_generic(a, m, b, out); + } + + SVS_NOINLINE static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + column_max_generic(a, m, b, out); + } }; #if defined(__x86_64__) @@ -59,8 +149,14 @@ template struct InnerProductMatrixImpl { SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) #if SVS_AVX512_F template <> struct InnerProductMatrixImpl { + SVS_NOINLINE static void + build_packed(const float* a, size_t m, const PackedOperand& b, float* out); SVS_NOINLINE static void build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); + SVS_NOINLINE static void + column_max(const float* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); }; #endif @@ -68,8 +164,14 @@ SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) SVS_VALIDATE_BOOL_ENV(SVS_AVX2) #if !SVS_AVX512_F && SVS_AVX2 template <> struct InnerProductMatrixImpl { + SVS_NOINLINE static void + build_packed(const float* a, size_t m, const PackedOperand& b, float* out); SVS_NOINLINE static void build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); + SVS_NOINLINE static void + column_max(const float* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); }; #endif @@ -88,9 +190,8 @@ extern template struct InnerProductMatrixImpl; /// (e.g. all centroids). /// @param m Number of rows of `a`. 0 is valid and leaves `out` untouched. /// @param b Row-major "B" operand, n rows of d floats each; the small operand (e.g. one -/// query's tokens). Packed once per call into ISA-specific scratch the implementation -/// owns (thread_local, as the GEM query transpose above does), so `b` itself is read -/// once and not retained. +/// query's tokens). Packed once per call into a thread_local PackedOperand the +/// implementation owns, so `b` itself is read once and not retained. /// @param n Number of rows of `b`. 0 is valid and leaves `out` untouched. /// @param d Row width shared by `a` and `b`. /// @param out Row-major output buffer, caller-owned, at least m * n floats: out[i * n + j] @@ -99,6 +200,9 @@ class InnerProductMatrix { public: static void compute(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + if (m == 0 || n == 0) { + return; + } switch (select_isa()) { case AVX_AVAILABILITY::AVX512: InnerProductMatrixImpl::build(a, m, b, n, d, out); @@ -116,7 +220,6 @@ class InnerProductMatrix { __builtin_unreachable(); } - private: static AVX_AVAILABILITY select_isa() { if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { return AVX_AVAILABILITY::AVX512; diff --git a/include/svs/core/distance/maxsim.h b/include/svs/core/distance/maxsim.h index 2b38ec4f5..237c74663 100644 --- a/include/svs/core/distance/maxsim.h +++ b/include/svs/core/distance/maxsim.h @@ -18,68 +18,123 @@ // svs #include "svs/concepts/vector_set.h" -#include "svs/core/distance/inner_product.h" +#include "svs/core/distance/inner_product_matrix.h" +#include "svs/lib/avx_detection.h" +#include "svs/lib/exception.h" +#include "svs/lib/float16.h" // stl -#include #include #include #include #include #include -#include #include +#include namespace svs::distance { /// Sum over query tokens of the max inner product over document tokens. Asymmetric; /// larger is better; inputs are not normalised here. -struct MaxSim { +/// +/// Per-query state, following the library's fix_argument protocol (like +/// QChamferExactQuery in gem.h): `fix_argument(query)` packs the query's tokens once +/// into a `PackedOperand`; every `compute(document)` call that follows scores one +/// document against that packed query through `InnerProductMatrixImpl::column_max`, with +/// no allocation beyond the per-query packing and a per-document maxima scratch that +/// only grows. `must_fix_argument` is set because `compute` is meaningless without a +/// prior `fix_argument`; `implicit_broadcast` is false because the packed query is +/// ordinary (non-shared) state, so each search thread's shallow_copy of this functor +/// must own its own `PackedOperand` rather than sharing one. +/// +/// The query (fix_argument's argument) is a `svs::data::ContiguousVectorSet` of float32 +/// tokens. The document (compute's second argument) is a `svs::data::ContiguousVectorSet` +/// of float32 or svs::Float16 tokens with the same dimensionality as the query; a +/// mismatch throws (checked once per document, which is negligible next to the kernel +/// it guards). An empty document scores -infinity, strictly worse than any real score. +class MaxSim { public: using compare = std::greater<>; - static constexpr bool implicit_broadcast = true; + static constexpr bool implicit_broadcast = false; + static constexpr bool must_fix_argument = true; static constexpr std::string_view name = "maxsim"; -}; - -inline constexpr bool operator==(const MaxSim&, const MaxSim&) { return true; } -/// Empty document -> -infinity, strictly worse than any real score. Empty query and -/// dimension mismatch are caller errors, asserted only: a mismatch reads out of bounds. -template -float compute(MaxSim /*unused*/, const Q& query, const D& document) { - size_t n_query = query.cardinality(); - assert(n_query > 0); - size_t n_doc = document.cardinality(); - if (n_doc == 0) { - return -std::numeric_limits::infinity(); + /// Packs `query`'s tokens (float32, contiguous) into this object's `PackedOperand`, + /// reusing its storage across calls. `query.cardinality() == 0` is a caller error, + /// asserted only: a MaxSim score over zero query tokens is undefined, not merely + /// small, so there is nothing for `compute` to usefully return. + template + requires std::same_as + void fix_argument(const S& query) { + assert(query.cardinality() > 0); + packed_.pack(query.data(), query.cardinality(), query.dimensions()); } - assert(query.dimensions() == document.dimensions()); - double total = 0.0; - for (size_t i = 0; i < n_query; ++i) { - auto q_token = query.token(i); - float best = -std::numeric_limits::infinity(); - for (size_t j = 0; j < n_doc; ++j) { - auto d_token = document.token(j); - float ip = IP::compute(q_token.data(), d_token.data(), q_token.size()); - best = std::max(best, ip); + /// `query` is unused: `fix_argument` already consumed it into `packed_`, and every + /// `compute` call in a search shares that one query's packing, matching the two-arg + /// overload `distance::compute` dispatches to for a `must_fix_argument` functor + /// (see QChamferExactQuery::compute in gem.h for the same shape). + template + requires std::same_as || + std::same_as + float compute(const Query& /*query*/, const D& document) const { + if (document.dimensions() != packed_.d()) { + throw ANNEXCEPTION( + "MaxSim::compute: document dimension {} does not match the fixed " + "query's dimension {}", + document.dimensions(), + packed_.d() + ); + } + size_t n_doc = document.cardinality(); + if (n_doc == 0) { + return -std::numeric_limits::infinity(); + } + + thread_local std::vector maxima; + maxima.resize(packed_.n()); + column_max_dispatch(document.data(), n_doc, maxima.data()); + + double total = 0.0; + for (float v : maxima) { + total += static_cast(v); } - total += best; + return static_cast(total); } - return static_cast(total); -} -/// `get_datum(id)` must return a type modeling `svs::data::VectorSet`; `scores` must have -/// the same length as `ids`. -template -void maxsim_batch( - const Q& query, GetDatum&& get_datum, std::span ids, std::span scores -) { - assert(ids.size() == scores.size()); - for (size_t i = 0; i < ids.size(); ++i) { - scores[i] = compute(MaxSim{}, query, get_datum(ids[i])); + private: + // Dispatched once per document, mirroring InnerProductMatrix::select_isa(): the + // query-packing and document-scoring ISA choices are independent calls, so this is + // not shared with InnerProductMatrix itself (which has no document-shaped operand). + template + void column_max_dispatch(const T* a, size_t n_doc, float* out) const { + switch (InnerProductMatrix::select_isa()) { + case AVX_AVAILABILITY::AVX512: + InnerProductMatrixImpl::column_max( + a, n_doc, packed_, out + ); + return; + case AVX_AVAILABILITY::AVX2: + InnerProductMatrixImpl::column_max( + a, n_doc, packed_, out + ); + return; + case AVX_AVAILABILITY::NONE: + InnerProductMatrixImpl::column_max( + a, n_doc, packed_, out + ); + return; + } + // Exhaustive switch over AVX_AVAILABILITY's enumerators, no default: see + // InnerProductMatrix::compute for why this stays a compile error, not a + // silent fallback, if a 4th enumerator is ever added. + __builtin_unreachable(); } -} + + PackedOperand packed_; +}; + +inline constexpr bool operator==(const MaxSim&, const MaxSim&) { return true; } } // namespace svs::distance diff --git a/include/svs/extensions/vamana/maxsim.h b/include/svs/extensions/vamana/maxsim.h index 527424968..4cb5742da 100644 --- a/include/svs/extensions/vamana/maxsim.h +++ b/include/svs/extensions/vamana/maxsim.h @@ -113,12 +113,17 @@ template < return; } + // Packs the query once (must_fix_argument) and reuses that packing for every + // candidate still in the pool: repacking per candidate would cost as much as + // scoring them, which is exactly what PackedOperand/MaxSim::fix_argument avoids. + svs::distance::MaxSim maxsim{}; + svs::distance::maybe_fix_argument(maxsim, query); + size_t valid = search_buffer.size(); for (size_t j = 0; j < valid; ++j) { auto& neighbor = search_buffer[j]; - float score = svs::distance::compute( - svs::distance::MaxSim{}, query, dataset.get_vector_set(neighbor.id()) - ); + float score = + svs::distance::compute(maxsim, query, dataset.get_vector_set(neighbor.id())); neighbor.set_distance(score); } diff --git a/include/svs/multi-arch/x86/avx2.cpp b/include/svs/multi-arch/x86/avx2.cpp index a8994c844..3bca863bf 100644 --- a/include/svs/multi-arch/x86/avx2.cpp +++ b/include/svs/multi-arch/x86/avx2.cpp @@ -60,98 +60,245 @@ template struct QChamferImpl; template struct QChamferExactImpl; template struct InnerProductMatrixImpl; -// Same register-blocking as avx512.cpp's InnerProductMatrixImpl::build, at AVX2's -// 8-wide registers; AVX2 has no masked load/store, so the tail lane-block uses -// _mm256_maskload_ps/_mm256_maskstore_ps with an all-lanes blend mask built from the bit -// pattern simd_utils.h's ConvertToFloat<8> uses for the same purpose. -void InnerProductMatrixImpl::build( - const float* a, size_t m, const float* b, size_t n, size_t d, float* out +namespace { + +constexpr size_t kRowBlock = inner_product_matrix_row_block; + +// Same role as avx512.cpp's accumulate_full/accumulate_tail, at AVX2's 8-wide registers; +// AVX2 has no masked load/store, so the tail lane-block uses _mm256_maskload_ps with an +// all-lanes blend mask built from the bit pattern simd_utils.h's ConvertToFloat<8> uses. +SVS_FORCE_INLINE void accumulate_full( + const float* const* arow, const float* bt, size_t n, size_t d, size_t j, __m256* acc ) { - if (m == 0 || n == 0) { - return; + for (size_t k = 0; k < d; ++k) { + __m256 bv = _mm256_loadu_ps(bt + k * n + j); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); + } } - constexpr size_t vl = 8; - constexpr size_t row_block = inner_product_matrix_row_block; - - thread_local std::vector bt; - bt.resize(d * n); - for (size_t j = 0; j < n; ++j) { - const float* bj = b + j * d; - for (size_t k = 0; k < d; ++k) { - bt[k * n + j] = bj[k]; +} + +SVS_FORCE_INLINE void accumulate_tail( + const float* const* arow, + const float* bt, + size_t n, + size_t d, + size_t j, + __m256i mask, + __m256* acc +) { + for (size_t k = 0; k < d; ++k) { + __m256 bv = _mm256_maskload_ps(bt + k * n + j, mask); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); } } +} + +// Pairwise-max reduction; see avx512.cpp's reduce_max for why padding slots are safe to +// include. +SVS_FORCE_INLINE __m256 reduce_max(const __m256* acc) { + __m256 m01 = _mm256_max_ps(acc[0], acc[1]); + __m256 m23 = _mm256_max_ps(acc[2], acc[3]); + __m256 m45 = _mm256_max_ps(acc[4], acc[5]); + __m256 m67 = _mm256_max_ps(acc[6], acc[7]); + return _mm256_max_ps(_mm256_max_ps(m01, m23), _mm256_max_ps(m45, m67)); +} + +} // namespace +// Same register-blocking as avx512.cpp's InnerProductMatrixImpl::build_packed, at +// AVX2's 8-wide registers. +void InnerProductMatrixImpl::build_packed( + const float* a, size_t m, const PackedOperand& b, float* out +) { + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); size_t full_lanes = n - n % vl; uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); - size_t full_rows = m - m % row_block; - size_t i = 0; - for (; i < full_rows; i += row_block) { - const float* arow[row_block]; - float* orow[row_block]; - for (size_t r = 0; r < row_block; ++r) { - arow[r] = a + (i + r) * d; + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + for (size_t r = 0; r < rows; ++r) { orow[r] = out + (i + r) * n; } size_t j = 0; for (; j < full_lanes; j += vl) { - __m256 acc[row_block]; - for (size_t r = 0; r < row_block; ++r) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { acc[r] = _mm256_setzero_ps(); } - for (size_t k = 0; k < d; ++k) { - __m256 bv = _mm256_loadu_ps(bt.data() + k * n + j); - for (size_t r = 0; r < row_block; ++r) { - acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); - } - } - for (size_t r = 0; r < row_block; ++r) { + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { _mm256_storeu_ps(orow[r] + j, acc[r]); } } if (j < n) { - __m256 acc[row_block]; - for (size_t r = 0; r < row_block; ++r) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { acc[r] = _mm256_setzero_ps(); } - for (size_t k = 0; k < d; ++k) { - __m256 bv = _mm256_maskload_ps(bt.data() + k * n + j, tail_mask); - for (size_t r = 0; r < row_block; ++r) { - acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); - } - } - for (size_t r = 0; r < row_block; ++r) { + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { _mm256_maskstore_ps(orow[r] + j, tail_mask, acc[r]); } } } +} + +// See avx512.cpp's InnerProductMatrixImpl::build for why this packs into a +// thread_local buffer rather than taking a PackedOperand from the caller. +void InnerProductMatrixImpl::build( + const float* a, size_t m, const float* b, size_t n, size_t d, float* out +) { + if (m == 0 || n == 0) { + return; + } + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); +} + +// See avx512.cpp's InnerProductMatrixImpl::column_max for the algorithm; this is +// the same reduction at AVX2's 8-wide registers. +void InnerProductMatrixImpl::column_max( + const float* a, size_t m, const PackedOperand& b, float* out +) { + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } - for (; i < m; ++i) { - const float* ai = a + i * d; - float* row = out + i * n; size_t j = 0; for (; j < full_lanes; j += vl) { - __m256 acc = _mm256_setzero_ps(); - for (size_t k = 0; k < d; ++k) { - acc = _mm256_fmadd_ps( - _mm256_set1_ps(ai[k]), _mm256_loadu_ps(bt.data() + k * n + j), acc - ); + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); } - _mm256_storeu_ps(row + j, acc); + accumulate_full(arow, bt, n, d, j, acc); + __m256 current = _mm256_loadu_ps(out + j); + _mm256_storeu_ps(out + j, _mm256_max_ps(reduce_max(acc), current)); } if (j < n) { - __m256 acc = _mm256_setzero_ps(); - for (size_t k = 0; k < d; ++k) { - acc = _mm256_fmadd_ps( - _mm256_set1_ps(ai[k]), - _mm256_maskload_ps(bt.data() + k * n + j, tail_mask), - acc + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m256 current = _mm256_maskload_ps(out + j, tail_mask); + _mm256_maskstore_ps( + out + j, tail_mask, _mm256_max_ps(reduce_max(acc), current) + ); + } + } +} + +// Float16 overload: converts one row-block of the document at a time into a float32 +// scratch using F16C's 8-wide _mm256_cvtph_ps (available because avx2.cpp is compiled +// for the "haswell" arch in cmake/multi-arch.cmake, and Haswell carries F16C alongside +// AVX2), then runs the identical float32 path above. +void InnerProductMatrixImpl::column_max( + const svs::Float16* a, size_t m, const PackedOperand& b, float* out +) { + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + for (size_t r = 0; r < rows; ++r) { + const svs::Float16* src = a + (i + r) * d; + float* dst = scratch.data() + r * d; + size_t k = 0; + for (; k + 8 <= d; k += 8) { + __m256 fv = _mm256_cvtph_ps( + _mm_loadu_si128(reinterpret_cast(src + k)) + ); + _mm256_storeu_ps(dst + k, fv); + } + if (k < d) { + // Safe masked load: a full-width load could read past the row's last + // element, so only popcount(mask) halfs are copied into a zeroed buffer + // before conversion, mirroring simd_utils.h's ConvertToFloat<8> tail path. + uint8_t dbits = svs::create_mask<8>(svs::lib::MaybeStatic(d - k)); + __m128i buffer = _mm_setzero_si128(); + std::memcpy( + &buffer, src + k, __builtin_popcount(dbits) * sizeof(svs::Float16) ); + __m256 fv = _mm256_cvtph_ps(buffer); + __m256i dmask = + _mm256_castps_si256(svs::simd::create_blend_mask_avx2(dbits)); + _mm256_maskstore_ps(dst + k, dmask, fv); + } + } + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m256 current = _mm256_loadu_ps(out + j); + _mm256_storeu_ps(out + j, _mm256_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); } - _mm256_maskstore_ps(row + j, tail_mask, acc); + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m256 current = _mm256_maskload_ps(out + j, tail_mask); + _mm256_maskstore_ps( + out + j, tail_mask, _mm256_max_ps(reduce_max(acc), current) + ); } } } diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index c48fb9930..e8ca1de71 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -60,101 +60,240 @@ template struct QChamferImpl; template struct QChamferExactImpl; template struct InnerProductMatrixImpl; +namespace { + +constexpr size_t kRowBlock = inner_product_matrix_row_block; + +// Accumulates kRowBlock rows of A (padded by repeating the last valid row when the +// document has fewer rows left, see callers) against one full 16-lane block of packed B. +// Shared by the table and column-max output forms so this FMA chain is written once. +SVS_FORCE_INLINE void accumulate_full( + const float* const* arow, const float* bt, size_t n, size_t d, size_t j, __m512* acc +) { + for (size_t k = 0; k < d; ++k) { + __m512 bv = _mm512_loadu_ps(bt + k * n + j); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +// Same as accumulate_full for the trailing lane-block narrower than 16 lanes. +SVS_FORCE_INLINE void accumulate_tail( + const float* const* arow, + const float* bt, + size_t n, + size_t d, + size_t j, + __mmask16 mask, + __m512* acc +) { + for (size_t k = 0; k < d; ++k) { + __m512 bv = _mm512_maskz_loadu_ps(mask, bt + k * n + j); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +// Pairwise-max reduction of the kRowBlock accumulators into their lane-wise maximum. +// Padding slots (see accumulate_full's comment) repeat a value already present in a real +// slot, so folding them into the tree does not change the result. +SVS_FORCE_INLINE __m512 reduce_max(const __m512* acc) { + __m512 m01 = _mm512_max_ps(acc[0], acc[1]); + __m512 m23 = _mm512_max_ps(acc[2], acc[3]); + __m512 m45 = _mm512_max_ps(acc[4], acc[5]); + __m512 m67 = _mm512_max_ps(acc[6], acc[7]); + return _mm512_max_ps(_mm512_max_ps(m01, m23), _mm512_max_ps(m45, m67)); +} + +} // namespace + // Register-blocked micro-kernel: inner_product_matrix_row_block rows of A share one pass // over a lane-block of B, giving that many independent FMA accumulator chains with no // horizontal reduction (each lane is one output), so the dependency-chain latency of a // single accumulator is hidden behind the others instead of serializing the loop over d. -void InnerProductMatrixImpl::build( - const float* a, size_t m, const float* b, size_t n, size_t d, float* out +void InnerProductMatrixImpl::build_packed( + const float* a, size_t m, const PackedOperand& b, float* out ) { + size_t n = b.n(); + size_t d = b.d(); if (m == 0 || n == 0) { return; } constexpr size_t vl = 16; - constexpr size_t row_block = inner_product_matrix_row_block; - - // Packs B transposed (bt[k * n + j] = b[j * d + k]) once per call: B is the small - // operand (e.g. one query's tokens, a handful of KiB), so this copy is noise next to - // the m x n table it lets every row below read with no horizontal reduction. - thread_local std::vector bt; - bt.resize(d * n); - for (size_t j = 0; j < n; ++j) { - const float* bj = b + j * d; - for (size_t k = 0; k < d; ++k) { - bt[k * n + j] = bj[k]; - } - } - + const float* bt = b.data(); size_t full_lanes = n - n % vl; __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); - size_t full_rows = m - m % row_block; - size_t i = 0; - for (; i < full_rows; i += row_block) { - const float* arow[row_block]; - float* orow[row_block]; - for (size_t r = 0; r < row_block; ++r) { - arow[r] = a + (i + r) * d; + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + for (size_t r = 0; r < rows; ++r) { orow[r] = out + (i + r) * n; } size_t j = 0; for (; j < full_lanes; j += vl) { - __m512 acc[row_block]; - for (size_t r = 0; r < row_block; ++r) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { acc[r] = _mm512_setzero_ps(); } - for (size_t k = 0; k < d; ++k) { - __m512 bv = _mm512_loadu_ps(bt.data() + k * n + j); - for (size_t r = 0; r < row_block; ++r) { - acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); - } - } - for (size_t r = 0; r < row_block; ++r) { + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { _mm512_storeu_ps(orow[r] + j, acc[r]); } } if (j < n) { - __m512 acc[row_block]; - for (size_t r = 0; r < row_block; ++r) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { acc[r] = _mm512_setzero_ps(); } - for (size_t k = 0; k < d; ++k) { - __m512 bv = _mm512_maskz_loadu_ps(tail_mask, bt.data() + k * n + j); - for (size_t r = 0; r < row_block; ++r) { - acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); - } - } - for (size_t r = 0; r < row_block; ++r) { + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { _mm512_mask_storeu_ps(orow[r] + j, tail_mask, acc[r]); } } } +} + +// Packs B into a reused thread_local buffer: the table form repacks on every call (one +// query against all centroids), unlike MaxSim's column_max below which packs once per +// query and calls into this ISA's kernels directly with the already-packed operand. +void InnerProductMatrixImpl::build( + const float* a, size_t m, const float* b, size_t n, size_t d, float* out +) { + if (m == 0 || n == 0) { + return; + } + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); +} + +// Column-wise maximum of the same table build_packed computes, without ever +// materializing it: each row-block's accumulators are reduced to a per-lane maximum and +// folded into a running maximum held in `out` itself, so one reduction serves the whole +// document for the price of one load/max/store per lane-block per row-block. +void InnerProductMatrixImpl::column_max( + const float* a, size_t m, const PackedOperand& b, float* out +) { + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } - // Row remainder (m % row_block): same lane logic with a single accumulator per row. - for (; i < m; ++i) { - const float* ai = a + i * d; - float* row = out + i * n; size_t j = 0; for (; j < full_lanes; j += vl) { - __m512 acc = _mm512_setzero_ps(); - for (size_t k = 0; k < d; ++k) { - acc = _mm512_fmadd_ps( - _mm512_set1_ps(ai[k]), _mm512_loadu_ps(bt.data() + k * n + j), acc - ); + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); } - _mm512_storeu_ps(row + j, acc); + accumulate_full(arow, bt, n, d, j, acc); + __m512 current = _mm512_loadu_ps(out + j); + _mm512_storeu_ps(out + j, _mm512_max_ps(reduce_max(acc), current)); } if (j < n) { - __m512 acc = _mm512_setzero_ps(); - for (size_t k = 0; k < d; ++k) { - acc = _mm512_fmadd_ps( - _mm512_set1_ps(ai[k]), - _mm512_maskz_loadu_ps(tail_mask, bt.data() + k * n + j), - acc + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m512 current = _mm512_maskz_loadu_ps(tail_mask, out + j); + _mm512_mask_storeu_ps( + out + j, tail_mask, _mm512_max_ps(reduce_max(acc), current) + ); + } + } +} + +// Float16 overload: converts one row-block (kRowBlock rows) of the document at a time +// into a float32 scratch with the widest conversion AVX-512 offers (_mm512_cvtph_ps over +// 16 halfs), then runs the identical float32 path above. Never branches on element type +// per row: the scratch, not a per-row check, is what makes the two types the same kernel. +void InnerProductMatrixImpl::column_max( + const svs::Float16* a, size_t m, const PackedOperand& b, float* out +) { + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + for (size_t r = 0; r < rows; ++r) { + const svs::Float16* src = a + (i + r) * d; + float* dst = scratch.data() + r * d; + size_t k = 0; + for (; k + 16 <= d; k += 16) { + _mm512_storeu_ps(dst + k, _mm512_cvtph_ps(_mm256_loadu_epi16(src + k))); + } + if (k < d) { + __mmask16 dmask = svs::create_mask<16>(svs::lib::MaybeStatic(d - k)); + _mm512_mask_storeu_ps( + dst + k, + dmask, + _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(dmask, src + k)) ); } - _mm512_mask_storeu_ps(row + j, tail_mask, acc); + } + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m512 current = _mm512_loadu_ps(out + j); + _mm512_storeu_ps(out + j, _mm512_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m512 current = _mm512_maskz_loadu_ps(tail_mask, out + j); + _mm512_mask_storeu_ps( + out + j, tail_mask, _mm512_max_ps(reduce_max(acc), current) + ); } } } diff --git a/tests/svs/core/distances/inner_product_matrix.cpp b/tests/svs/core/distances/inner_product_matrix.cpp index 6da8ef916..c5aa99513 100644 --- a/tests/svs/core/distances/inner_product_matrix.cpp +++ b/tests/svs/core/distances/inner_product_matrix.cpp @@ -19,13 +19,16 @@ // svs #include "svs/lib/avx_detection.h" +#include "svs/lib/float16.h" // catch2 #include "catch2/catch_approx.hpp" #include "catch2/catch_test_macros.hpp" // stdlib +#include #include +#include #include #include @@ -196,3 +199,239 @@ CATCH_TEST_CASE( CATCH_REQUIRE(out == before); } } + +namespace { + +using svs::distance::PackedOperand; + +// Shapes covering every tail this family of kernels has: the row-block remainder +// (8-wide), both SIMD lane widths' remainders (8 and 16), m == 0 (empty document), and +// both small and multi-cache-line d. +const std::vector column_max_ms{0, 1, 2, 7, 8, 9, 17, 67, 180}; +const std::vector column_max_ns{1, 15, 16, 17, 32, 33}; +const std::vector column_max_ds{1, 7, 128, 129}; + +// Naive double-precision reference: out[j] = max over i of , or -infinity if +// m == 0. +std::vector reference_column_max( + const std::vector& a, size_t m, const std::vector& b, size_t n, size_t d +) { + std::vector out(n, -std::numeric_limits::infinity()); + for (size_t i = 0; i < m; ++i) { + for (size_t j = 0; j < n; ++j) { + double acc = 0.0; + for (size_t k = 0; k < d; ++k) { + acc += a[i * d + k] * static_cast(b[j * d + k]); + } + out[j] = std::max(out[j], acc); + } + } + return out; +} + +void require_close_vec(const std::vector& got, const std::vector& expected) { + CATCH_REQUIRE(got.size() == expected.size()); + for (size_t i = 0; i < got.size(); ++i) { + if (expected[i] == -std::numeric_limits::infinity()) { + CATCH_REQUIRE(got[i] == -std::numeric_limits::infinity()); + } else { + CATCH_REQUIRE( + static_cast(got[i]) == + Catch::Approx(expected[i]).epsilon(REL_EPS).margin(ABS_EPS) + ); + } + } +} + +std::vector to_float16(const std::vector& a) { + std::vector out(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + out[i] = svs::Float16(a[i]); + } + return out; +} + +// Promotes a Float16 row-major matrix to double, so the reference is computed from the +// values the kernel actually sees (post-quantization), not the pre-quantization floats. +std::vector promote_float16(const std::vector& a) { + std::vector out(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + out[i] = static_cast(static_cast(a[i])); + } + return out; +} + +std::vector promote_float(const std::vector& a) { + std::vector out(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + out[i] = static_cast(a[i]); + } + return out; +} + +template void check_column_max_float32_against_reference() { + unsigned seed = 2000; + for (size_t d : column_max_ds) { + for (size_t m : column_max_ms) { + for (size_t n : column_max_ns) { + auto a = random_matrix(m, d, seed++); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + std::vector out(n, -3.0F); + InnerProductMatrixImpl::column_max(a.data(), m, packed, out.data()); + require_close_vec(out, reference_column_max(promote_float(a), m, b, n, d)); + } + } + } +} + +template void check_column_max_float16_against_reference() { + unsigned seed = 3000; + for (size_t d : column_max_ds) { + for (size_t m : column_max_ms) { + for (size_t n : column_max_ns) { + auto a = random_matrix(m, d, seed++); + auto a16 = to_float16(a); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + std::vector out(n, -3.0F); + InnerProductMatrixImpl::column_max(a16.data(), m, packed, out.data()); + require_close_vec( + out, reference_column_max(promote_float16(a16), m, b, n, d) + ); + } + } + } +} + +template void check_column_max_against_generic() { + unsigned seed = 4000; + for (size_t d : column_max_ds) { + for (size_t m : column_max_ms) { + for (size_t n : column_max_ns) { + auto a = random_matrix(m, d, seed++); + auto a16 = to_float16(a); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + + std::vector generic_out(n, -1.0F); + std::vector isa_out(n, -2.0F); + InnerProductMatrixImpl::column_max( + a.data(), m, packed, generic_out.data() + ); + InnerProductMatrixImpl::column_max( + a.data(), m, packed, isa_out.data() + ); + for (size_t j = 0; j < n; ++j) { + CATCH_REQUIRE( + static_cast(isa_out[j]) == + Catch::Approx(static_cast(generic_out[j])) + .epsilon(REL_EPS) + .margin(ABS_EPS) + ); + } + + std::vector generic_out16(n, -1.0F); + std::vector isa_out16(n, -2.0F); + InnerProductMatrixImpl::column_max( + a16.data(), m, packed, generic_out16.data() + ); + InnerProductMatrixImpl::column_max( + a16.data(), m, packed, isa_out16.data() + ); + // Float16 inputs decode identically everywhere, but the generic kernel + // accumulates in strict k-order while the SIMD kernels accumulate across + // row-blocks in a different order: a wider tolerance than the float32 + // comparison above absorbs that summation-order difference, not a + // correctness gap. + for (size_t j = 0; j < n; ++j) { + CATCH_REQUIRE( + static_cast(isa_out16[j]) == + Catch::Approx(static_cast(generic_out16[j])) + .epsilon(5e-4) + .margin(5e-5) + ); + } + } + } + } +} + +} // namespace + +CATCH_TEST_CASE( + "InnerProductMatrixImpl column_max generic kernel matches a naive reference", + "[distance][inner_product_matrix]" +) { + check_column_max_float32_against_reference(); + check_column_max_float16_against_reference(); +} + +CATCH_TEST_CASE( + "InnerProductMatrixImpl column_max AVX2 kernel matches reference and generic", + "[distance][inner_product_matrix]" +) { + if (!svs::detail::avx_runtime_flags.is_avx2_supported()) { + return; + } + check_column_max_float32_against_reference(); + check_column_max_float16_against_reference(); + check_column_max_against_generic(); +} + +CATCH_TEST_CASE( + "InnerProductMatrixImpl column_max AVX512 kernel matches reference and generic", + "[distance][inner_product_matrix]" +) { + if (!svs::detail::avx_runtime_flags.is_avx512f_supported()) { + return; + } + check_column_max_float32_against_reference(); + check_column_max_float16_against_reference(); + check_column_max_against_generic(); +} + +CATCH_TEST_CASE( + "InnerProductMatrixImpl build_packed matches build through an explicit PackedOperand", + "[distance][inner_product_matrix]" +) { + // build() packs internally into a thread_local PackedOperand; this checks that + // packing explicitly and calling build_packed gives the identical result, for every + // ISA this process can run. + unsigned seed = 5000; + for (size_t d : {1UL, 7UL, 128UL}) { + for (size_t m : {0UL, 1UL, 9UL, 33UL}) { + for (size_t n : {1UL, 17UL, 33UL}) { + auto a = random_matrix(m, d, seed++); + auto b = random_matrix(n, d, seed++); + std::vector via_build(m * n, -1.0F); + InnerProductMatrix::compute(a.data(), m, b.data(), n, d, via_build.data()); + + PackedOperand packed; + packed.pack(b.data(), n, d); + std::vector via_packed(m * n, -2.0F); + switch (InnerProductMatrix::select_isa()) { + case AVX_AVAILABILITY::AVX512: + InnerProductMatrixImpl::build_packed( + a.data(), m, packed, via_packed.data() + ); + break; + case AVX_AVAILABILITY::AVX2: + InnerProductMatrixImpl::build_packed( + a.data(), m, packed, via_packed.data() + ); + break; + case AVX_AVAILABILITY::NONE: + InnerProductMatrixImpl::build_packed( + a.data(), m, packed, via_packed.data() + ); + break; + } + CATCH_REQUIRE(via_build == via_packed); + } + } + } +} diff --git a/tests/svs/core/distances/maxsim.cpp b/tests/svs/core/distances/maxsim.cpp index f34cf391b..6804b3880 100644 --- a/tests/svs/core/distances/maxsim.cpp +++ b/tests/svs/core/distances/maxsim.cpp @@ -18,6 +18,7 @@ #include "svs/core/distance/maxsim.h" // svs +#include "svs/concepts/distance.h" #include "svs/lib/float16.h" // catch2 @@ -37,17 +38,32 @@ namespace { -// Minimal svs::data::VectorSet model over a vector of row-major tokens; local to this -// test so it does not depend on the ConstVectorSet view another agent is adding. +// Minimal svs::data::ContiguousVectorSet model: a flat, row-major buffer of `cardinality` +// tokens of `dim` elements each. Flat (rather than a vector of per-token vectors) because +// MaxSim's packed-operand kernel needs `data()` to return one contiguous block. template struct TestVectorSet { using element_type = T; - std::vector> tokens; + std::vector flat; + size_t dim = 0; - size_t cardinality() const { return tokens.size(); } - size_t dimensions() const { return tokens.empty() ? 0 : tokens[0].size(); } - std::span token(size_t i) const { return std::span(tokens[i]); } + size_t cardinality() const { return dim == 0 ? 0 : flat.size() / dim; } + size_t dimensions() const { return dim; } + std::span token(size_t i) const { + return std::span(flat.data() + i * dim, dim); + } + const T* data() const { return flat.data(); } }; +template +TestVectorSet make_set(std::initializer_list> tokens) { + TestVectorSet set; + set.dim = tokens.begin()->size(); + for (const auto& token : tokens) { + set.flat.insert(set.flat.end(), token.begin(), token.end()); + } + return set; +} + template double promote(T x) { return static_cast(x); } // Naive double-precision oracle: sum over query tokens of the max inner product to any @@ -59,11 +75,13 @@ double maxsim_reference(const TestVectorSet& query, const TestVectorSet& d } double total = 0.0; for (size_t i = 0; i < query.cardinality(); ++i) { + auto q_token = query.token(i); double best = -std::numeric_limits::infinity(); for (size_t j = 0; j < document.cardinality(); ++j) { + auto d_token = document.token(j); double dot = 0.0; for (size_t d = 0; d < query.dimensions(); ++d) { - dot += promote(query.tokens[i][d]) * promote(document.tokens[j][d]); + dot += promote(q_token[d]) * promote(d_token[d]); } best = std::max(best, dot); } @@ -75,16 +93,23 @@ double maxsim_reference(const TestVectorSet& query, const TestVectorSet& d TestVectorSet make_random_set(size_t cardinality, size_t dim, std::mt19937& rng) { std::uniform_real_distribution dist(-1.0F, 1.0F); TestVectorSet set; - set.tokens.resize(cardinality); - for (auto& token : set.tokens) { - token.resize(dim); - for (auto& v : token) { - v = dist(rng); - } + set.dim = dim; + set.flat.resize(cardinality * dim); + for (auto& v : set.flat) { + v = dist(rng); } return set; } +// Packs `query` into a fresh MaxSim functor, through the generic maybe_fix_argument hook +// (the protocol every ShouldFix-opted-in distance goes through), then scores `document`. +template +float maxsim_score(const TestVectorSet& query, const D& document) { + svs::distance::MaxSim dist{}; + svs::distance::maybe_fix_argument(dist, query); + return svs::distance::compute(dist, query, document); +} + constexpr double MAXSIM_EPSILON = 1e-4; } // namespace @@ -92,32 +117,34 @@ constexpr double MAXSIM_EPSILON = 1e-4; CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { CATCH_SECTION("Trait members") { CATCH_REQUIRE(std::is_same_v>); - CATCH_REQUIRE(svs::distance::MaxSim::implicit_broadcast); + CATCH_REQUIRE_FALSE(svs::distance::MaxSim::implicit_broadcast); + CATCH_REQUIRE(svs::distance::MaxSim::must_fix_argument); } CATCH_SECTION("Hand-computed tiny case") { // Query has two orthonormal tokens; document has one token aligned with the // first and one aligned with neither, so the per-query maxima are 1 and 0. - TestVectorSet query{{{1.0F, 0.0F}, {0.0F, 1.0F}}}; - TestVectorSet document{{{1.0F, 0.0F}, {0.0F, 0.0F}}}; - float score = svs::distance::compute(svs::distance::MaxSim{}, query, document); + auto query = make_set({{1.0F, 0.0F}, {0.0F, 1.0F}}); + auto document = make_set({{1.0F, 0.0F}, {0.0F, 0.0F}}); + float score = maxsim_score(query, document); CATCH_REQUIRE(score == Catch::Approx(1.0F).epsilon(MAXSIM_EPSILON)); } CATCH_SECTION("Empty document set gives -inf") { - TestVectorSet query{{{1.0F, 0.0F}}}; - TestVectorSet document{{}}; - float score = svs::distance::compute(svs::distance::MaxSim{}, query, document); + auto query = make_set({{1.0F, 0.0F}}); + TestVectorSet document; + document.dim = 2; + float score = maxsim_score(query, document); CATCH_REQUIRE(score == -std::numeric_limits::infinity()); } CATCH_SECTION("Asymmetry") { // `a` has two tokens that each max out against `b`'s lone token, so swapping // query/document sums over a different number of maxima (2 vs. 1). - TestVectorSet a{{{1.0F, 0.0F}, {1.0F, 0.0F}}}; - TestVectorSet b{{{1.0F, 0.0F}}}; - float score_ab = svs::distance::compute(svs::distance::MaxSim{}, a, b); - float score_ba = svs::distance::compute(svs::distance::MaxSim{}, b, a); + auto a = make_set({{1.0F, 0.0F}, {1.0F, 0.0F}}); + auto b = make_set({{1.0F, 0.0F}}); + float score_ab = maxsim_score(a, b); + float score_ba = maxsim_score(b, a); CATCH_REQUIRE(score_ab != Catch::Approx(score_ba)); } @@ -131,8 +158,7 @@ CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { auto query = make_random_set(nq, dim, rng); auto document = make_random_set(nd, dim, rng); double expected = maxsim_reference(query, document); - float actual = - svs::distance::compute(svs::distance::MaxSim{}, query, document); + float actual = maxsim_score(query, document); CATCH_REQUIRE( static_cast(actual) == Catch::Approx(expected) .epsilon(MAXSIM_EPSILON) @@ -144,37 +170,41 @@ CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { } CATCH_SECTION("Mixed element types") { - // float query against Float16 document: an AVX-accelerated pair the existing - // inner product supports (IPImpl). - TestVectorSet query{{{1.0F, 0.0F, 0.0F}, {0.0F, 1.0F, 0.0F}}}; - TestVectorSet document{ + // float query against Float16 document. + auto query = make_set({{1.0F, 0.0F, 0.0F}, {0.0F, 1.0F, 0.0F}}); + auto document = make_set( {{svs::Float16(1.0F), svs::Float16(0.0F), svs::Float16(0.0F)}, - {svs::Float16(0.0F), svs::Float16(0.0F), svs::Float16(1.0F)}}}; - float score = svs::distance::compute(svs::distance::MaxSim{}, query, document); + {svs::Float16(0.0F), svs::Float16(0.0F), svs::Float16(1.0F)}} + ); + float score = maxsim_score(query, document); // Token 0 of query matches document token 0 exactly (dot = 1); token 1 of query // has dot 0 with both document tokens. CATCH_REQUIRE(score == Catch::Approx(1.0F).epsilon(MAXSIM_EPSILON)); } - CATCH_SECTION("Batch helper matches per-pair compute") { - std::mt19937 rng(42); - auto query = make_random_set(5, 64, rng); - std::vector> candidates; - for (size_t i = 0; i < 6; ++i) { - candidates.push_back(make_random_set(i + 1, 64, rng)); - } - std::vector ids{0, 1, 2, 3, 4, 5}; - std::vector scores(ids.size()); - svs::distance::maxsim_batch>( - query, - [&](uint32_t id) -> const TestVectorSet& { return candidates[id]; }, - std::span(ids), - std::span(scores) + CATCH_SECTION("Dimension mismatch throws") { + auto query = make_set({{1.0F, 0.0F, 0.0F}}); + auto document = make_set({{1.0F, 0.0F}}); + svs::distance::MaxSim dist{}; + svs::distance::maybe_fix_argument(dist, query); + CATCH_REQUIRE_THROWS_AS( + svs::distance::compute(dist, query, document), svs::lib::ANNException ); - for (size_t i = 0; i < ids.size(); ++i) { - float expected = - svs::distance::compute(svs::distance::MaxSim{}, query, candidates[ids[i]]); - CATCH_REQUIRE(scores[i] == expected); + } + + CATCH_SECTION("Repeated compute calls reuse one fix_argument") { + std::mt19937 rng(7); + auto query = make_random_set(5, 32, rng); + svs::distance::MaxSim dist{}; + svs::distance::maybe_fix_argument(dist, query); + for (size_t i = 0; i < 4; ++i) { + auto document = make_random_set(i + 1, 32, rng); + float direct = svs::distance::compute(dist, query, document); + float expected = maxsim_reference(query, document); + CATCH_REQUIRE( + static_cast(direct) == + Catch::Approx(expected).epsilon(MAXSIM_EPSILON).margin(MAXSIM_EPSILON) + ); } } } diff --git a/tests/svs/index/vamana/maxsim_rerank.cpp b/tests/svs/index/vamana/maxsim_rerank.cpp index 7d4b86c8f..435702dca 100644 --- a/tests/svs/index/vamana/maxsim_rerank.cpp +++ b/tests/svs/index/vamana/maxsim_rerank.cpp @@ -182,9 +182,12 @@ std::vector> brute_force_maxsim(const Query& query, const svs::data::ConstVectorSetView& docs) { std::vector> scored; scored.reserve(docs.size()); + // MaxSim now packs the query once (must_fix_argument) and reuses that packing for + // every document, instead of the old stateless compute() called fresh each time. + svs::distance::MaxSim maxsim{}; + svs::distance::maybe_fix_argument(maxsim, query); for (uint32_t id = 0; id < docs.size(); ++id) { - float score = - svs::distance::compute(svs::distance::MaxSim{}, query, docs.get_datum(id)); + float score = svs::distance::compute(maxsim, query, docs.get_datum(id)); scored.emplace_back(id, score); } std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { @@ -248,11 +251,12 @@ void check_rerank_pool( std::vector pool_ids; pool_ids.reserve(buffer.size()); + svs::distance::MaxSim maxsim{}; + svs::distance::maybe_fix_argument(maxsim, query); float previous_score = std::numeric_limits::infinity(); for (const auto& neighbor : buffer) { - float expected = svs::distance::compute( - svs::distance::MaxSim{}, query, doc_sets.get_datum(neighbor.id()) - ); + float expected = + svs::distance::compute(maxsim, query, doc_sets.get_datum(neighbor.id())); CATCH_REQUIRE( static_cast(neighbor.distance()) == Catch::Approx(expected) .epsilon(MAXSIM_RERANK_EPSILON) From 9417576e4d40b266758b28c52e12f8658cbdf773 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 04:04:25 -0800 Subject: [PATCH 16/68] refactor(distance): keep ISA dispatch for MaxSim's kernel inside InnerProductMatrix MaxSim called InnerProductMatrixImpl::column_max directly through its own three-way switch on InnerProductMatrix::select_isa(), duplicating the kernel's own dispatch and leaking the Impl struct and the ISA enum into a caller. InnerProductMatrix now exposes column_max(float) and column_max(Float16) wrappers that share one private dispatch() helper with compute(); select_isa() is private again. MaxSim calls only the wrapper and keeps a member maxima_ buffer (sized in fix_argument) instead of a thread_local one, since implicit_broadcast=false already gives each search thread its own MaxSim copy. Also tightens the Float16 generic-vs-ISA tolerance in inner_product_matrix.cpp from an unmeasured epsilon(5e-4)/margin(5e-5) to epsilon(2e-4)/margin(1e-4), about 2x the largest summation-order difference (4.7e-5 abs / 3.1e-5 rel) measured by a throwaway probe over the test's own shapes. Co-Authored-By: Claude Fable 5.2 --- .../svs/core/distance/inner_product_matrix.h | 78 +++++++++++++++---- include/svs/core/distance/maxsim.h | 60 ++++---------- .../core/distances/inner_product_matrix.cpp | 43 +++++----- 3 files changed, 100 insertions(+), 81 deletions(-) diff --git a/include/svs/core/distance/inner_product_matrix.h b/include/svs/core/distance/inner_product_matrix.h index ce4f1d42c..6dba05de7 100644 --- a/include/svs/core/distance/inner_product_matrix.h +++ b/include/svs/core/distance/inner_product_matrix.h @@ -24,6 +24,7 @@ #include #include #include +#include #include namespace svs::distance { @@ -182,36 +183,79 @@ extern template struct InnerProductMatrixImpl; extern template struct InnerProductMatrixImpl; #endif -/// Dense inner-product table between a large row-major matrix and a small row-major -/// matrix, both float32: out[i * n + j] = for i in [0, m), j in [0, n). Picks -/// the ISA implementation once per call, outside every loop over rows or columns. -/// -/// @param a Row-major "A" operand, m rows of d floats each; the large, streamed operand -/// (e.g. all centroids). -/// @param m Number of rows of `a`. 0 is valid and leaves `out` untouched. -/// @param b Row-major "B" operand, n rows of d floats each; the small operand (e.g. one -/// query's tokens). Packed once per call into a thread_local PackedOperand the -/// implementation owns, so `b` itself is read once and not retained. -/// @param n Number of rows of `b`. 0 is valid and leaves `out` untouched. -/// @param d Row width shared by `a` and `b`. -/// @param out Row-major output buffer, caller-owned, at least m * n floats: out[i * n + j] -/// holds . +/// Public entry points for the InnerProductMatrixImpl kernel family: the only place that +/// picks an ISA for this kernel. A caller that needs this kernel's output names the form +/// it wants (`compute` for the table, `column_max` for the column-wise maximum) and never +/// names `InnerProductMatrixImpl` or an `AVX_AVAILABILITY` enumerator itself; the layer +/// that owns the ISA choice (this class) is the only one that repeats it. class InnerProductMatrix { public: + /// Dense inner-product table between a large row-major matrix and a small row-major + /// matrix, both float32: out[i * n + j] = for i in [0, m), j in [0, n). + /// + /// @param a Row-major "A" operand, m rows of d floats each; the large, streamed + /// operand (e.g. all centroids). + /// @param m Number of rows of `a`. 0 is valid and leaves `out` untouched. + /// @param b Row-major "B" operand, n rows of d floats each; the small operand (e.g. + /// one query's tokens). Packed once per call into a thread_local PackedOperand + /// this class owns, so `b` itself is read once and not retained. + /// @param n Number of rows of `b`. 0 is valid and leaves `out` untouched. + /// @param d Row width shared by `a` and `b`. + /// @param out Row-major output buffer, caller-owned, at least m * n floats: + /// out[i * n + j] holds . Untouched when m == 0 or n == 0. static void compute(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { if (m == 0 || n == 0) { return; } + dispatch([&](auto avx) { + InnerProductMatrixImpl::build(a, m, b, n, d, out); + }); + } + + /// Column-wise maximum of the same table `compute` would produce, for a float32 A, + /// against an already-packed B: out[j] = max over i of , for j in [0, n). + /// No table is ever materialized. + /// + /// @param a Row-major "A" operand, m rows of `b.d()` floats each; the large, + /// streamed operand (e.g. one document's tokens). + /// @param m Number of rows of `a`. 0 is valid: every lane of `out` is set to + /// -infinity, matching an empty A having no row to contribute a maximum. + /// @param b A `PackedOperand` already packed by the caller (e.g. once per query), + /// read but not modified. + /// @param out Caller-owned output buffer of exactly `b.n()` floats, overwritten in + /// full (every lane, including when m == 0): out[j] holds the column maximum + /// for j in [0, b.n()). + static void column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + dispatch([&](auto avx) { + InnerProductMatrixImpl::column_max(a, m, b, out); + }); + } + + /// Same contract as the float32 `column_max` above, for a svs::Float16 A: converts + /// one row-block of `a` at a time into a float32 scratch and reduces identically. + static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + dispatch([&](auto avx) { + InnerProductMatrixImpl::column_max(a, m, b, out); + }); + } + + private: + // Picks the ISA once per call and invokes `f` with that choice as a compile-time + // constant (an std::integral_constant, so `InnerProductMatrixImpl` below is a valid template argument), so every public entry point above + // writes this exhaustive switch zero times of its own. + template static void dispatch(F&& f) { switch (select_isa()) { case AVX_AVAILABILITY::AVX512: - InnerProductMatrixImpl::build(a, m, b, n, d, out); + f(std::integral_constant{}); return; case AVX_AVAILABILITY::AVX2: - InnerProductMatrixImpl::build(a, m, b, n, d, out); + f(std::integral_constant{}); return; case AVX_AVAILABILITY::NONE: - InnerProductMatrixImpl::build(a, m, b, n, d, out); + f(std::integral_constant{}); return; } // Exhaustive switch over AVX_AVAILABILITY's enumerators, no default: -Wswitch diff --git a/include/svs/core/distance/maxsim.h b/include/svs/core/distance/maxsim.h index 237c74663..19ebeaf4f 100644 --- a/include/svs/core/distance/maxsim.h +++ b/include/svs/core/distance/maxsim.h @@ -19,7 +19,6 @@ // svs #include "svs/concepts/vector_set.h" #include "svs/core/distance/inner_product_matrix.h" -#include "svs/lib/avx_detection.h" #include "svs/lib/exception.h" #include "svs/lib/float16.h" @@ -38,14 +37,16 @@ namespace svs::distance { /// larger is better; inputs are not normalised here. /// /// Per-query state, following the library's fix_argument protocol (like -/// QChamferExactQuery in gem.h): `fix_argument(query)` packs the query's tokens once -/// into a `PackedOperand`; every `compute(document)` call that follows scores one -/// document against that packed query through `InnerProductMatrixImpl::column_max`, with -/// no allocation beyond the per-query packing and a per-document maxima scratch that -/// only grows. `must_fix_argument` is set because `compute` is meaningless without a -/// prior `fix_argument`; `implicit_broadcast` is false because the packed query is -/// ordinary (non-shared) state, so each search thread's shallow_copy of this functor -/// must own its own `PackedOperand` rather than sharing one. +/// QChamferExactQuery in gem.h): `fix_argument(query)` packs the query's tokens once into +/// a `PackedOperand` and sizes a member maxima buffer to the query's cardinality; every +/// `compute(document)` call that follows scores one document against that packed query +/// through `InnerProductMatrix::column_max`, writing into the member buffer rather than a +/// thread_local one, so compute performs no thread_local lookup per document. That is +/// safe precisely because `implicit_broadcast` is false: the packed query and the maxima +/// buffer are both ordinary (non-shared) state, so each search thread's shallow_copy of +/// this functor owns its own copy of both rather than sharing either one. +/// `must_fix_argument` is set because `compute` is meaningless without a prior +/// `fix_argument`. /// /// The query (fix_argument's argument) is a `svs::data::ContiguousVectorSet` of float32 /// tokens. The document (compute's second argument) is a `svs::data::ContiguousVectorSet` @@ -69,16 +70,18 @@ class MaxSim { void fix_argument(const S& query) { assert(query.cardinality() > 0); packed_.pack(query.data(), query.cardinality(), query.dimensions()); + maxima_.resize(packed_.n()); } /// `query` is unused: `fix_argument` already consumed it into `packed_`, and every /// `compute` call in a search shares that one query's packing, matching the two-arg /// overload `distance::compute` dispatches to for a `must_fix_argument` functor - /// (see QChamferExactQuery::compute in gem.h for the same shape). + /// (see QChamferExactQuery::compute in gem.h for the same shape). Non-const: writes + /// into the member maxima buffer `fix_argument` sized. template requires std::same_as || std::same_as - float compute(const Query& /*query*/, const D& document) const { + float compute(const Query& /*query*/, const D& document) { if (document.dimensions() != packed_.d()) { throw ANNEXCEPTION( "MaxSim::compute: document dimension {} does not match the fixed " @@ -92,47 +95,18 @@ class MaxSim { return -std::numeric_limits::infinity(); } - thread_local std::vector maxima; - maxima.resize(packed_.n()); - column_max_dispatch(document.data(), n_doc, maxima.data()); + InnerProductMatrix::column_max(document.data(), n_doc, packed_, maxima_.data()); double total = 0.0; - for (float v : maxima) { + for (float v : maxima_) { total += static_cast(v); } return static_cast(total); } private: - // Dispatched once per document, mirroring InnerProductMatrix::select_isa(): the - // query-packing and document-scoring ISA choices are independent calls, so this is - // not shared with InnerProductMatrix itself (which has no document-shaped operand). - template - void column_max_dispatch(const T* a, size_t n_doc, float* out) const { - switch (InnerProductMatrix::select_isa()) { - case AVX_AVAILABILITY::AVX512: - InnerProductMatrixImpl::column_max( - a, n_doc, packed_, out - ); - return; - case AVX_AVAILABILITY::AVX2: - InnerProductMatrixImpl::column_max( - a, n_doc, packed_, out - ); - return; - case AVX_AVAILABILITY::NONE: - InnerProductMatrixImpl::column_max( - a, n_doc, packed_, out - ); - return; - } - // Exhaustive switch over AVX_AVAILABILITY's enumerators, no default: see - // InnerProductMatrix::compute for why this stays a compile error, not a - // silent fallback, if a 4th enumerator is ever added. - __builtin_unreachable(); - } - PackedOperand packed_; + std::vector maxima_; }; inline constexpr bool operator==(const MaxSim&, const MaxSim&) { return true; } diff --git a/tests/svs/core/distances/inner_product_matrix.cpp b/tests/svs/core/distances/inner_product_matrix.cpp index c5aa99513..18950de91 100644 --- a/tests/svs/core/distances/inner_product_matrix.cpp +++ b/tests/svs/core/distances/inner_product_matrix.cpp @@ -344,15 +344,16 @@ template void check_column_max_against_generic() { ); // Float16 inputs decode identically everywhere, but the generic kernel // accumulates in strict k-order while the SIMD kernels accumulate across - // row-blocks in a different order: a wider tolerance than the float32 - // comparison above absorbs that summation-order difference, not a - // correctness gap. + // row-blocks in a different order; measured over every (m, n, d) shape + // this test sweeps, that reordering alone produces at most 4.7e-5 + // absolute / 3.1e-5 relative difference (one-off throwaway probe, not + // committed), so this tolerance is ~2x that bound, not a slack value. for (size_t j = 0; j < n; ++j) { CATCH_REQUIRE( static_cast(isa_out16[j]) == Catch::Approx(static_cast(generic_out16[j])) - .epsilon(5e-4) - .margin(5e-5) + .epsilon(2e-4) + .margin(1e-4) ); } } @@ -413,22 +414,22 @@ CATCH_TEST_CASE( PackedOperand packed; packed.pack(b.data(), n, d); std::vector via_packed(m * n, -2.0F); - switch (InnerProductMatrix::select_isa()) { - case AVX_AVAILABILITY::AVX512: - InnerProductMatrixImpl::build_packed( - a.data(), m, packed, via_packed.data() - ); - break; - case AVX_AVAILABILITY::AVX2: - InnerProductMatrixImpl::build_packed( - a.data(), m, packed, via_packed.data() - ); - break; - case AVX_AVAILABILITY::NONE: - InnerProductMatrixImpl::build_packed( - a.data(), m, packed, via_packed.data() - ); - break; + // InnerProductMatrix::select_isa() is private (the ISA choice for this + // kernel belongs to InnerProductMatrix alone); this test needs to force + // the same ISA `compute` above picked, so it repeats that priority order + // against the public svs::detail::avx_runtime_flags instead. + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + InnerProductMatrixImpl::build_packed( + a.data(), m, packed, via_packed.data() + ); + } else if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + InnerProductMatrixImpl::build_packed( + a.data(), m, packed, via_packed.data() + ); + } else { + InnerProductMatrixImpl::build_packed( + a.data(), m, packed, via_packed.data() + ); } CATCH_REQUIRE(via_build == via_packed); } From 01adc9dc5e25cb7282f615eb232fb5fdabc6cb7c Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 04:14:17 -0800 Subject: [PATCH 17/68] perf(distance): build CentroidTable with the inner-product kernel, each pair once from_centroids now tiles the K x K ground-cost table over the upper triangle of blocks (including diagonal blocks), computing each tile with InnerProductMatrix::compute and mirroring the transformed value into both T[i][j] and T[j][i] instead of calling it twice. Tiles are scheduled with a DynamicPartition to balance the uneven tile count per row block; the table's storage, load, and operator()(i, j) are unchanged. Symmetry is now guaranteed by construction (one computed float stored at both cells) rather than by two independently computed dot products, so results can differ from the old code in the last bits. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/distance/gem.h | 95 +++++++++++++++++++++++-------- tests/svs/core/distances/gem.cpp | 97 ++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 22 deletions(-) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index 42f6f31c3..1eba6c88c 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -30,6 +30,7 @@ #include "svs/core/distance/distance_core.h" #include "svs/core/distance/inner_product_matrix.h" #include "svs/lib/avx_detection.h" +#include "svs/lib/misc.h" #include "svs/lib/preprocessor.h" #include "svs/lib/threads.h" @@ -120,8 +121,14 @@ struct CentroidTable { } /// Computes T[i][j] = ground(c_i, c_j) from K row-major `dim`-wide centroids, - /// T[i][i] = 0. Parallelized over row blocks on `num_threads` threads; each thread's - /// first write to its row range is also its NUMA first-touch. + /// T[i][i] = 0. Symmetric by construction: each unordered pair {i, j} is computed + /// once, by the tile that owns it, and the same float is stored at T[i][j] and + /// T[j][i] (never two independently-computed dot products). Parallelized over tiles + /// on `num_threads` threads, dynamically scheduled so no two tiles ever write the + /// same cell; the result is bit-for-bit identical regardless of `num_threads`. Uses + /// an uninitialized allocation (see the private constructor below) and writes every + /// cell exactly once (off-diagonal tiles) or twice with the same value (diagonal + /// tiles' upper/lower halves), so no thread does a sequential full-matrix pass. static CentroidTable from_centroids( const float* centroids, size_t k, size_t dim, Ground ground, size_t num_threads ) { @@ -129,32 +136,76 @@ struct CentroidTable { float* out = result.data.get(); svs::threads::DefaultThreadPool pool(num_threads); - // Column block width: a block's centroid rows (col_block * dim * 4 bytes, 128 KiB - // at dim=128) stay resident in L2 while every row a thread owns sweeps over it. - constexpr size_t col_block = 256; + // Tile edge: an operand block (tile * dim * 4 bytes, 128 KiB at dim=128) and the + // tile buffer (tile * tile * 4 bytes, 256 KiB) together stay well inside a + // typical >=1 MiB per-core L2. + constexpr size_t tile = 256; + // Tiles per dynamic-partition steal: a few tiles amortize the atomic-counter + // fetch_add without starving the small tiles near the far column blocks. + constexpr size_t grain = 4; + + size_t num_blocks = svs::lib::div_round_up(k, tile); + + // Upper triangle of blocks, diagonal included: tile (bi, bj), bi <= bj, owns + // every unordered pair whose row falls in block bi and column in block bj. Built + // once so the uneven per-row-block tile count (row block 0 has num_blocks tiles, + // the last has one) can be load-balanced by a dynamic partition over this list + // rather than a static partition over rows. + struct BlockPair { + uint32_t bi; + uint32_t bj; + }; + std::vector tiles; + tiles.reserve(num_blocks * (num_blocks + 1) / 2); + for (uint32_t bi = 0; bi < num_blocks; ++bi) { + for (uint32_t bj = bi; bj < num_blocks; ++bj) { + tiles.push_back({bi, bj}); + } + } + + auto ground_cost = [ground](float dot) { + return ground == Ground::Cos ? std::max(0.0F, 1.0F - dot) + : std::sqrt(std::max(0.0F, 2.0F - 2.0F * dot)); + }; svs::threads::parallel_for( pool, - svs::threads::StaticPartition(k), - [&](const auto& row_ids, uint64_t /*tid*/) { - for (size_t i : row_ids) { - const float* ci = centroids + i * dim; - for (size_t jb = 0; jb < k; jb += col_block) { - size_t jmax = std::min(jb + col_block, k); - for (size_t j = jb; j < jmax; ++j) { - if (i == j) { + svs::threads::DynamicPartition{tiles.size(), grain}, + [&](const auto& tile_ids, uint64_t /*tid*/) { + // Reused across this thread's tiles: InnerProductMatrix::compute packs + // its own operand internally, so this buffer only holds the tile output. + thread_local std::vector buf; + buf.resize(tile * tile); + for (size_t t : tile_ids) { + auto [bi, bj] = tiles[t]; + size_t row0 = static_cast(bi) * tile; + size_t col0 = static_cast(bj) * tile; + size_t rows = std::min(tile, k - row0); + size_t cols = std::min(tile, k - col0); + InnerProductMatrix::compute( + centroids + row0 * dim, + rows, + centroids + col0 * dim, + cols, + dim, + buf.data() + ); + bool diag = (bi == bj); + for (size_t r = 0; r < rows; ++r) { + size_t i = row0 + r; + // Off-diagonal tiles (row and column blocks disjoint) own every + // (r, c); diagonal tiles start at c == r to skip the lower half, + // already owned by this same tile's (r, c) with roles swapped. + size_t c_start = diag ? r : 0; + for (size_t c = c_start; c < cols; ++c) { + size_t j = col0 + c; + if (diag && c == r) { out[i * k + j] = 0.0F; continue; } - const float* cj = centroids + j * dim; - float dot = 0.0F; - for (size_t d = 0; d < dim; ++d) { - dot += ci[d] * cj[d]; - } - out[i * k + j] = - ground == Ground::Cos - ? std::max(0.0F, 1.0F - dot) - : std::sqrt(std::max(0.0F, 2.0F - 2.0F * dot)); + float value = ground_cost(buf[r * cols + c]); + out[i * k + j] = value; + out[j * k + i] = value; } } } diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp index 1569ce901..6dc661b4b 100644 --- a/tests/svs/core/distances/gem.cpp +++ b/tests/svs/core/distances/gem.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +56,49 @@ std::shared_ptr make_table(Ground ground) { ); } +// K unit vectors of width dim, row-major: deterministic per (k, dim, seed) so the +// from_centroids tiling test below is reproducible across runs. +std::vector random_unit_centroids(size_t k, size_t dim, uint32_t seed) { + std::mt19937 rng(seed); + std::normal_distribution gauss(0.0F, 1.0F); + std::vector flat(k * dim); + for (size_t i = 0; i < k; ++i) { + float* row = flat.data() + i * dim; + double norm2 = 0.0; + for (size_t d = 0; d < dim; ++d) { + row[d] = gauss(rng); + norm2 += static_cast(row[d]) * row[d]; + } + double norm = std::sqrt(norm2); + for (size_t d = 0; d < dim; ++d) { + row[d] = static_cast(row[d] / norm); + } + } + return flat; +} + +// Reference ground-cost table computed with double-precision dot products and the same +// formula from_centroids applies, independent of InnerProductMatrix/tiling. +std::vector +naive_ground_table(const std::vector& flat, size_t k, size_t dim, Ground ground) { + std::vector t(k * k, 0.0); + for (size_t i = 0; i < k; ++i) { + for (size_t j = 0; j < k; ++j) { + if (i == j) { + continue; + } + double dot = 0.0; + for (size_t d = 0; d < dim; ++d) { + dot += static_cast(flat[i * dim + d]) * flat[j * dim + d]; + } + t[i * k + j] = ground == Ground::Cos + ? std::max(0.0, 1.0 - dot) + : std::sqrt(std::max(0.0, 2.0 - 2.0 * dot)); + } + } + return t; +} + // Owns a 1-D id vector set contiguously, for exercising QEmd/QChamfer/QChamferExactQuery's // only (vector-set) compute() overloads. struct IdSet { @@ -120,6 +164,59 @@ CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { } } +CATCH_TEST_CASE("CentroidTable::from_centroids tiling", "[distance][gem]") { + // K values exercising a single partial tile (1, 2, 7), a single full tile (255 off + // by one, 256 exact, 257 off by one), and several tiles (600); at a small (3) and a + // production (128) dimension, both ground distances, built with 1 and 4 threads. + for (size_t k : + {size_t{1}, + size_t{2}, + size_t{7}, + size_t{255}, + size_t{256}, + size_t{257}, + size_t{600}}) { + for (size_t dim : {size_t{3}, size_t{128}}) { + for (Ground ground : {Ground::Cos, Ground::L2}) { + auto flat = random_unit_centroids( + k, + dim, + static_cast( + k * 1009 + dim * 31 + (ground == Ground::Cos ? 1 : 2) + ) + ); + auto reference = naive_ground_table(flat, k, dim, ground); + + auto one_thread = + CentroidTable::from_centroids(flat.data(), k, dim, ground, 1); + auto four_threads = + CentroidTable::from_centroids(flat.data(), k, dim, ground, 4); + + for (size_t i = 0; i < k; ++i) { + CATCH_REQUIRE( + one_thread(static_cast(i), static_cast(i)) == + 0.0F + ); + for (size_t j = 0; j < k; ++j) { + auto ui = static_cast(i); + auto uj = static_cast(j); + // Thread-count independence: bit for bit, not approximate. + CATCH_REQUIRE(one_thread(ui, uj) == four_threads(ui, uj)); + // Exact symmetry by construction: bit for bit. + CATCH_REQUIRE(one_thread(ui, uj) == one_thread(uj, ui)); + // Against a double-precision reference: summation order differs, + // so only approximate agreement is expected. + CATCH_REQUIRE( + static_cast(one_thread(ui, uj)) == + Catch::Approx(reference[i * k + j]).margin(1e-4) + ); + } + } + } + } + } +} + CATCH_TEST_CASE("QChamfer", "[distance][gem]") { auto table = make_table(Ground::Cos); QChamfer dist{table}; From 99de3c45dd3421e64e008dc120f91ccf59321df7 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 06:16:28 -0800 Subject: [PATCH 18/68] feat(distance): add a bfloat16 A operand and bf16-rounded B packing to InnerProductMatrix Adds svs::BFloat16 overloads of both InnerProductMatrixImpl output forms (build_packed/table and column_max) at every ISA level (generic, AVX2, AVX512), sharing the Float16 overloads' row-block-conversion shape via a templated helper per ISA file; also fills in the previously-missing Float16 overload of the table form since the shared helper makes it free. BFloat16's widening to float32 is an exact zero-extend (bfloat16 is float32's high 16 bits), unlike Float16's lossy half-to-single conversion. PackedOperand gains pack_rounded_to_bf16, rounding B to bfloat16 (round to nearest, ties to even) and storing the result as float32: rounding B is the caller's choice of precision, not a hidden kernel behaviour, so it lives in how B is packed rather than in the kernel itself. svs::BFloat16's own constructor truncates instead of rounding; svs::round_to_bfloat16 (bfloat16.h) adds the correctly-rounded conversion both this and the bf16 A overloads above need to agree on. Co-Authored-By: Claude Fable 5.2 --- .../svs/core/distance/inner_product_matrix.h | 115 ++++++- include/svs/lib/bfloat16.h | 22 ++ include/svs/multi-arch/x86/avx2.cpp | 220 ++++++++++---- include/svs/multi-arch/x86/avx512.cpp | 222 ++++++++++---- .../core/distances/inner_product_matrix.cpp | 281 ++++++++++++++++++ 5 files changed, 736 insertions(+), 124 deletions(-) diff --git a/include/svs/core/distance/inner_product_matrix.h b/include/svs/core/distance/inner_product_matrix.h index 6dba05de7..65429cd1f 100644 --- a/include/svs/core/distance/inner_product_matrix.h +++ b/include/svs/core/distance/inner_product_matrix.h @@ -18,6 +18,7 @@ #include "svs/core/distance/distance_core.h" #include "svs/lib/avx_detection.h" +#include "svs/lib/bfloat16.h" #include "svs/lib/float16.h" #include "svs/lib/preprocessor.h" @@ -59,6 +60,26 @@ class PackedOperand { } } + /// Same contract and layout as `pack`, except every value is first rounded to + /// bfloat16 precision (round to nearest, ties to even) and stored back as float32. + /// The kernel itself only ever computes on the values it is given, + /// unrounded; rounding B to the precision A is held at (bf16, see FineCentroids) is + /// the caller's choice of precision, made here rather than inside the kernel, so a + /// future caller that wants a different precision for B is never blocked by this one. + /// A scalar loop (not vectorized): at GEM's production shape this rounds ~4096 + /// values per query, negligible next to the O(K * d) table build that follows. + void pack_rounded_to_bf16(const float* b, size_t n, size_t d) { + n_ = n; + d_ = d; + buffer_.resize(n * d); + for (size_t j = 0; j < n; ++j) { + const float* bj = b + j * d; + for (size_t k = 0; k < d; ++k) { + buffer_[k * n + j] = static_cast(svs::round_to_bfloat16(bj[k])); + } + } + } + size_t n() const { return n_; } size_t d() const { return d_; } const float* data() const { return buffer_.data(); } @@ -82,29 +103,51 @@ class PackedOperand { /// * `column_max`: out[j] = max over i of , i.e. the column-wise maximum of /// that same table, computed with no table ever materialized. /// -/// `column_max` is additionally overloaded on A's element type (float32 and -/// svs::Float16); the Float16 overload converts one row-block of A at a time into a -/// float32 scratch and runs the identical float32 path, so the two element types are -/// never branched on per row. +/// Both forms are additionally overloaded on A's element type (float32, svs::Float16 and +/// svs::BFloat16); the narrow-type overloads convert one row-block of A at a time into a +/// float32 scratch and run the identical float32 path, so no element type is ever branched +/// on per row. BFloat16's widening is exact (it is float32's high 16 bits, zero-extended), +/// so unlike Float16 there is no rounding question on A's side; B's precision for a +/// BFloat16 A is the caller's choice, made by packing with +/// `PackedOperand::pack_rounded_to_bf16` instead of `pack`. template struct InnerProductMatrixImpl { - SVS_NOINLINE static void - build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + // Shared by every element type A can hold: float32 needs no conversion (the + // static_cast below is a no-op then), Float16/BFloat16 each widen losslessly to + // float32 per element. One shape, never a per-row branch on T. + template + static void + build_packed_generic(const T* a, size_t m, const PackedOperand& b, float* out) { size_t n = b.n(); size_t d = b.d(); const float* bt = b.data(); for (size_t i = 0; i < m; ++i) { - const float* ai = a + i * d; + const T* ai = a + i * d; float* row = out + i * n; for (size_t j = 0; j < n; ++j) { float dot = 0.0F; for (size_t k = 0; k < d; ++k) { - dot += ai[k] * bt[k * n + j]; + dot += static_cast(ai[k]) * bt[k * n + j]; } row[j] = dot; } } } + SVS_NOINLINE static void + build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + build_packed_generic(a, m, b, out); + } + + SVS_NOINLINE static void + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + build_packed_generic(a, m, b, out); + } + + SVS_NOINLINE static void + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + build_packed_generic(a, m, b, out); + } + // Packs B into a reused thread_local buffer so this keeps its no-allocation-per-call // property: InnerProductMatrix::compute (GEM's fix_argument path) calls this once per // query against all K centroids, not once per document, so repacking here is cheap. @@ -143,6 +186,11 @@ template struct InnerProductMatrixImpl { column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { column_max_generic(a, m, b, out); } + + SVS_NOINLINE static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + column_max_generic(a, m, b, out); + } }; #if defined(__x86_64__) @@ -153,11 +201,17 @@ template <> struct InnerProductMatrixImpl { SVS_NOINLINE static void build_packed(const float* a, size_t m, const PackedOperand& b, float* out); SVS_NOINLINE static void + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); SVS_NOINLINE static void column_max(const float* a, size_t m, const PackedOperand& b, float* out); SVS_NOINLINE static void column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); }; #endif @@ -168,11 +222,17 @@ template <> struct InnerProductMatrixImpl { SVS_NOINLINE static void build_packed(const float* a, size_t m, const PackedOperand& b, float* out); SVS_NOINLINE static void + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); SVS_NOINLINE static void column_max(const float* a, size_t m, const PackedOperand& b, float* out); SVS_NOINLINE static void column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); }; #endif @@ -241,6 +301,45 @@ class InnerProductMatrix { }); } + /// Same contract as the float32 `column_max` above, for a svs::BFloat16 A. + static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + dispatch([&](auto avx) { + InnerProductMatrixImpl::column_max(a, m, b, out); + }); + } + + /// Dense inner-product table (same layout as `compute`) between a large row-major A + /// and an already-packed B, for an A element type other than float32. Unlike + /// `compute`, this takes a `PackedOperand` the caller packed itself, so the caller + /// controls B's precision (e.g. `PackedOperand::pack_rounded_to_bf16` for a bf16 A, so + /// every ISA level sees the same rounded values on both operands). + /// + /// @param a Row-major "A" operand, m rows of `b.d()` elements each. + /// @param m Number of rows of `a`. 0 is valid and leaves `out` untouched. + /// @param b A `PackedOperand` already packed by the caller, read but not modified. + /// @param out Row-major output buffer, caller-owned, at least m * b.n() floats. + static void + compute_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + if (m == 0 || b.n() == 0) { + return; + } + dispatch([&](auto avx) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + }); + } + + /// Same contract as `compute_packed` above, for a svs::Float16 A. + static void + compute_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + if (m == 0 || b.n() == 0) { + return; + } + dispatch([&](auto avx) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + }); + } + private: // Picks the ISA once per call and invokes `f` with that choice as a compile-time // constant (an std::integral_constant, so `InnerProductMatrixImpl(u >> 16); } +// Round-to-nearest-even truncation to bfloat16, as a uint16_t of the result's high half. +// NaN is kept as NaN (never promoted to infinity by the rounding add) with its quiet bit +// forced on, since x86 NaN payloads are not required to survive this truncation anyway. +inline uint16_t float_to_bfloat16_rne_untyped(const float x) { + uint32_t u = bitcast_float_to_uint32(x); + if ((u & 0x7fffffffU) > 0x7f800000U) { + return lib::narrow((u | 0x00400000U) >> 16); + } + uint32_t lsb = (u >> 16) & 1U; + u += 0x7fffU + lsb; + return lib::narrow(u >> 16); +} + } // namespace detail // On GCC, we need to add this attribute so that BFloat16 members can appear inside @@ -107,6 +120,14 @@ static_assert(std::is_standard_layout_v); // like signed zeros. inline bool operator==(BFloat16 x, BFloat16 y) { return float{x} == float{y}; } +// BFloat16's own constructor (above) truncates: it keeps the high 16 bits of `x` and +// discards the rest with no rounding. This rounds to nearest, ties to even, instead; use +// it wherever more than one caller needs the two to agree bit-for-bit (e.g. a kernel +// operand rounded on one side and a centroid matrix rounded on the other). +inline BFloat16 round_to_bfloat16(float x) { + return BFloat16::from_raw(detail::float_to_bfloat16_rne_untyped(x)); +} + ///// ///// Pretty Printing ///// @@ -114,6 +135,7 @@ inline bool operator==(BFloat16 x, BFloat16 y) { return float{x} == float{y}; } } // namespace bfloat16 using BFloat16 = bfloat16::BFloat16; +using bfloat16::round_to_bfloat16; // SVS local arithmetric trait. template <> inline constexpr bool is_arithmetic_v = true; diff --git a/include/svs/multi-arch/x86/avx2.cpp b/include/svs/multi-arch/x86/avx2.cpp index 3bca863bf..4f26008c1 100644 --- a/include/svs/multi-arch/x86/avx2.cpp +++ b/include/svs/multi-arch/x86/avx2.cpp @@ -105,13 +105,53 @@ SVS_FORCE_INLINE __m256 reduce_max(const __m256* acc) { return _mm256_max_ps(_mm256_max_ps(m01, m23), _mm256_max_ps(m45, m67)); } -} // namespace +// Widens one 8-lane block of Float16 to float32 with F16C's native half->single. +SVS_FORCE_INLINE __m256 widen8(const svs::Float16* p) { + return _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(p))); +} -// Same register-blocking as avx512.cpp's InnerProductMatrixImpl::build_packed, at -// AVX2's 8-wide registers. -void InnerProductMatrixImpl::build_packed( - const float* a, size_t m, const PackedOperand& b, float* out -) { +// Widens one 8-lane block of BFloat16 to float32: bfloat16 is already float32's high 16 +// bits, so this is a lossless zero-extend (cvtepu16_epi32) into the top half of each +// 32-bit lane (slli by 16), not a rounding conversion. +SVS_FORCE_INLINE __m256 widen8(const svs::BFloat16* p) { + __m128i halfwords = _mm_loadu_si128(reinterpret_cast(p)); + return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(halfwords), 16)); +} + +// Tail (< 8 lanes) widening for either 16-bit type: zero-pads through a stack buffer +// sized by sizeof(T) (2 bytes for both Float16 and BFloat16), then reuses widen8. +template SVS_FORCE_INLINE __m256 widen8_tail(const T* p, uint8_t mask_bits) { + __m128i buffer = _mm_setzero_si128(); + std::memcpy(&buffer, p, __builtin_popcount(mask_bits) * sizeof(T)); + return widen8(reinterpret_cast(&buffer)); +} + +// Converts kRowBlock rows of `a` (T is Float16 or BFloat16) starting at row `i` into a +// contiguous float32 scratch, 8 lanes at a time; shared by the table and column-max forms +// below so either narrow type is widened in exactly one place. +template +SVS_FORCE_INLINE void +convert_row_block(const T* a, size_t i, size_t rows, size_t d, float* scratch) { + for (size_t r = 0; r < rows; ++r) { + const T* src = a + (i + r) * d; + float* dst = scratch + r * d; + size_t k = 0; + for (; k + 8 <= d; k += 8) { + _mm256_storeu_ps(dst + k, widen8(src + k)); + } + if (k < d) { + uint8_t dbits = svs::create_mask<8>(svs::lib::MaybeStatic(d - k)); + __m256i dmask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(dbits)); + _mm256_maskstore_ps(dst + k, dmask, widen8_tail(src + k, dbits)); + } + } +} + +// Table form for a narrow-element A (Float16 or BFloat16): identical register-blocked +// loop to the float32 build_packed above, sourced from a row-block converted to float32 +// by convert_row_block instead of `a` directly. +template +void build_packed_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { size_t n = b.n(); size_t d = b.d(); if (m == 0 || n == 0) { @@ -123,12 +163,16 @@ void InnerProductMatrixImpl::build_packed( uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + for (size_t i = 0; i < m; i += kRowBlock) { size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); const float* arow[kRowBlock]; float* orow[kRowBlock]; for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = a + (i + std::min(r, rows - 1)) * d; + arow[r] = scratch.data() + std::min(r, rows - 1) * d; } for (size_t r = 0; r < rows; ++r) { orow[r] = out + (i + r) * n; @@ -158,24 +202,10 @@ void InnerProductMatrixImpl::build_packed( } } -// See avx512.cpp's InnerProductMatrixImpl::build for why this packs into a -// thread_local buffer rather than taking a PackedOperand from the caller. -void InnerProductMatrixImpl::build( - const float* a, size_t m, const float* b, size_t n, size_t d, float* out -) { - if (m == 0 || n == 0) { - return; - } - thread_local PackedOperand packed; - packed.pack(b, n, d); - build_packed(a, m, packed, out); -} - -// See avx512.cpp's InnerProductMatrixImpl::column_max for the algorithm; this is -// the same reduction at AVX2's 8-wide registers. -void InnerProductMatrixImpl::column_max( - const float* a, size_t m, const PackedOperand& b, float* out -) { +// Column-max form for a narrow-element A (Float16 or BFloat16): same structure as +// build_packed_narrow above, reduced through reduce_max instead of stored per row. +template +void column_max_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { size_t n = b.n(); size_t d = b.d(); if (n == 0) { @@ -191,11 +221,15 @@ void InnerProductMatrixImpl::column_max( uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + for (size_t i = 0; i < m; i += kRowBlock) { size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); const float* arow[kRowBlock]; for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = a + (i + std::min(r, rows - 1)) * d; + arow[r] = scratch.data() + std::min(r, rows - 1) * d; } size_t j = 0; @@ -222,12 +256,90 @@ void InnerProductMatrixImpl::column_max( } } -// Float16 overload: converts one row-block of the document at a time into a float32 -// scratch using F16C's 8-wide _mm256_cvtph_ps (available because avx2.cpp is compiled -// for the "haswell" arch in cmake/multi-arch.cmake, and Haswell carries F16C alongside -// AVX2), then runs the identical float32 path above. -void InnerProductMatrixImpl::column_max( +} // namespace + +// Same register-blocking as avx512.cpp's InnerProductMatrixImpl::build_packed, at +// AVX2's 8-wide registers. +void InnerProductMatrixImpl::build_packed( + const float* a, size_t m, const PackedOperand& b, float* out +) { + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_maskstore_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } +} + +// Table form for a Float16 A: see convert_row_block above for the shared widening. +void InnerProductMatrixImpl::build_packed( const svs::Float16* a, size_t m, const PackedOperand& b, float* out +) { + build_packed_narrow(a, m, b, out); +} + +// Table form for a BFloat16 A: see convert_row_block above for the shared widening. +void InnerProductMatrixImpl::build_packed( + const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out +) { + build_packed_narrow(a, m, b, out); +} + +// See avx512.cpp's InnerProductMatrixImpl::build for why this packs into a +// thread_local buffer rather than taking a PackedOperand from the caller. +void InnerProductMatrixImpl::build( + const float* a, size_t m, const float* b, size_t n, size_t d, float* out +) { + if (m == 0 || n == 0) { + return; + } + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); +} + +// See avx512.cpp's InnerProductMatrixImpl::column_max for the algorithm; this is +// the same reduction at AVX2's 8-wide registers. +void InnerProductMatrixImpl::column_max( + const float* a, size_t m, const PackedOperand& b, float* out ) { size_t n = b.n(); size_t d = b.d(); @@ -244,39 +356,11 @@ void InnerProductMatrixImpl::column_max( uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); - thread_local std::vector scratch; - scratch.resize(kRowBlock * d); - for (size_t i = 0; i < m; i += kRowBlock) { size_t rows = std::min(kRowBlock, m - i); - for (size_t r = 0; r < rows; ++r) { - const svs::Float16* src = a + (i + r) * d; - float* dst = scratch.data() + r * d; - size_t k = 0; - for (; k + 8 <= d; k += 8) { - __m256 fv = _mm256_cvtph_ps( - _mm_loadu_si128(reinterpret_cast(src + k)) - ); - _mm256_storeu_ps(dst + k, fv); - } - if (k < d) { - // Safe masked load: a full-width load could read past the row's last - // element, so only popcount(mask) halfs are copied into a zeroed buffer - // before conversion, mirroring simd_utils.h's ConvertToFloat<8> tail path. - uint8_t dbits = svs::create_mask<8>(svs::lib::MaybeStatic(d - k)); - __m128i buffer = _mm_setzero_si128(); - std::memcpy( - &buffer, src + k, __builtin_popcount(dbits) * sizeof(svs::Float16) - ); - __m256 fv = _mm256_cvtph_ps(buffer); - __m256i dmask = - _mm256_castps_si256(svs::simd::create_blend_mask_avx2(dbits)); - _mm256_maskstore_ps(dst + k, dmask, fv); - } - } const float* arow[kRowBlock]; for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = scratch.data() + std::min(r, rows - 1) * d; + arow[r] = a + (i + std::min(r, rows - 1)) * d; } size_t j = 0; @@ -303,6 +387,22 @@ void InnerProductMatrixImpl::column_max( } } +// Float16 overload: see convert_row_block above for the shared widening (F16C's 8-wide +// _mm256_cvtph_ps, available because avx2.cpp is compiled for the "haswell" arch in +// cmake/multi-arch.cmake, and Haswell carries F16C alongside AVX2). +void InnerProductMatrixImpl::column_max( + const svs::Float16* a, size_t m, const PackedOperand& b, float* out +) { + column_max_narrow(a, m, b, out); +} + +// BFloat16 overload: see convert_row_block above for the shared (exact) widening. +void InnerProductMatrixImpl::column_max( + const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out +) { + column_max_narrow(a, m, b, out); +} + } // namespace svs::distance #endif diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index e8ca1de71..7ad79bc5b 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -107,15 +107,56 @@ SVS_FORCE_INLINE __m512 reduce_max(const __m512* acc) { return _mm512_max_ps(_mm512_max_ps(m01, m23), _mm512_max_ps(m45, m67)); } -} // namespace +// Widens one 16-lane block of Float16 to float32 with AVX-512F's native half->single. +SVS_FORCE_INLINE __m512 widen16(const svs::Float16* p) { + return _mm512_cvtph_ps(_mm256_loadu_epi16(p)); +} -// Register-blocked micro-kernel: inner_product_matrix_row_block rows of A share one pass -// over a lane-block of B, giving that many independent FMA accumulator chains with no -// horizontal reduction (each lane is one output), so the dependency-chain latency of a -// single accumulator is hidden behind the others instead of serializing the loop over d. -void InnerProductMatrixImpl::build_packed( - const float* a, size_t m, const PackedOperand& b, float* out -) { +// Widens one 16-lane block of BFloat16 to float32: bfloat16 is already float32's high 16 +// bits, so this is a lossless zero-extend (cvtepu16_epi32, AVX-512F) into the top half of +// each 32-bit lane (slli by 16), not a rounding conversion. +SVS_FORCE_INLINE __m512 widen16(const svs::BFloat16* p) { + __m256i halfwords = _mm256_loadu_si256(reinterpret_cast(p)); + return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(halfwords), 16)); +} + +// Tail (< 16 lanes) widening for either 16-bit type: zero-pads through a stack buffer +// sized by sizeof(T) (2 bytes for both Float16 and BFloat16), then reuses widen16. Uses a +// plain memcpy rather than a masked load so BFloat16's tail needs nothing beyond +// AVX-512F (Float16's masked load above already relies on AVX-512BW+VL for +// _mm256_loadu_epi16, a preexisting requirement this does not widen). +template SVS_FORCE_INLINE __m512 widen16_tail(const T* p, size_t count) { + __m256i buffer = _mm256_setzero_si256(); + std::memcpy(&buffer, p, count * sizeof(T)); + return widen16(reinterpret_cast(&buffer)); +} + +// Converts kRowBlock rows of `a` (T is Float16 or BFloat16) starting at row `i` into a +// contiguous float32 scratch, 16 lanes at a time; shared by the table and column-max +// forms below so either narrow type is widened in exactly one place. +template +SVS_FORCE_INLINE void +convert_row_block(const T* a, size_t i, size_t rows, size_t d, float* scratch) { + for (size_t r = 0; r < rows; ++r) { + const T* src = a + (i + r) * d; + float* dst = scratch + r * d; + size_t k = 0; + for (; k + 16 <= d; k += 16) { + _mm512_storeu_ps(dst + k, widen16(src + k)); + } + if (k < d) { + size_t tail = d - k; + __mmask16 dmask = svs::create_mask<16>(svs::lib::MaybeStatic(tail)); + _mm512_mask_storeu_ps(dst + k, dmask, widen16_tail(src + k, tail)); + } + } +} + +// Table form for a narrow-element A (Float16 or BFloat16): identical register-blocked +// loop to the float32 build_packed above, sourced from a row-block converted to float32 +// by convert_row_block instead of `a` directly. +template +void build_packed_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { size_t n = b.n(); size_t d = b.d(); if (m == 0 || n == 0) { @@ -126,12 +167,16 @@ void InnerProductMatrixImpl::build_packed( size_t full_lanes = n - n % vl; __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + for (size_t i = 0; i < m; i += kRowBlock) { size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); const float* arow[kRowBlock]; float* orow[kRowBlock]; for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = a + (i + std::min(r, rows - 1)) * d; + arow[r] = scratch.data() + std::min(r, rows - 1) * d; } for (size_t r = 0; r < rows; ++r) { orow[r] = out + (i + r) * n; @@ -161,27 +206,10 @@ void InnerProductMatrixImpl::build_packed( } } -// Packs B into a reused thread_local buffer: the table form repacks on every call (one -// query against all centroids), unlike MaxSim's column_max below which packs once per -// query and calls into this ISA's kernels directly with the already-packed operand. -void InnerProductMatrixImpl::build( - const float* a, size_t m, const float* b, size_t n, size_t d, float* out -) { - if (m == 0 || n == 0) { - return; - } - thread_local PackedOperand packed; - packed.pack(b, n, d); - build_packed(a, m, packed, out); -} - -// Column-wise maximum of the same table build_packed computes, without ever -// materializing it: each row-block's accumulators are reduced to a per-lane maximum and -// folded into a running maximum held in `out` itself, so one reduction serves the whole -// document for the price of one load/max/store per lane-block per row-block. -void InnerProductMatrixImpl::column_max( - const float* a, size_t m, const PackedOperand& b, float* out -) { +// Column-max form for a narrow-element A (Float16 or BFloat16): same structure as +// build_packed_narrow above, reduced through reduce_max instead of stored per row. +template +void column_max_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { size_t n = b.n(); size_t d = b.d(); if (n == 0) { @@ -196,11 +224,15 @@ void InnerProductMatrixImpl::column_max( size_t full_lanes = n - n % vl; __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + for (size_t i = 0; i < m; i += kRowBlock) { size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); const float* arow[kRowBlock]; for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = a + (i + std::min(r, rows - 1)) * d; + arow[r] = scratch.data() + std::min(r, rows - 1) * d; } size_t j = 0; @@ -227,12 +259,94 @@ void InnerProductMatrixImpl::column_max( } } -// Float16 overload: converts one row-block (kRowBlock rows) of the document at a time -// into a float32 scratch with the widest conversion AVX-512 offers (_mm512_cvtph_ps over -// 16 halfs), then runs the identical float32 path above. Never branches on element type -// per row: the scratch, not a per-row check, is what makes the two types the same kernel. -void InnerProductMatrixImpl::column_max( +} // namespace + +// Register-blocked micro-kernel: inner_product_matrix_row_block rows of A share one pass +// over a lane-block of B, giving that many independent FMA accumulator chains with no +// horizontal reduction (each lane is one output), so the dependency-chain latency of a +// single accumulator is hidden behind the others instead of serializing the loop over d. +void InnerProductMatrixImpl::build_packed( + const float* a, size_t m, const PackedOperand& b, float* out +) { + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_mask_storeu_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } +} + +// Table form for a Float16 A: see convert_row_block above for the shared widening. +void InnerProductMatrixImpl::build_packed( const svs::Float16* a, size_t m, const PackedOperand& b, float* out +) { + build_packed_narrow(a, m, b, out); +} + +// Table form for a BFloat16 A: see convert_row_block above for the shared widening. +void InnerProductMatrixImpl::build_packed( + const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out +) { + build_packed_narrow(a, m, b, out); +} + +// Packs B into a reused thread_local buffer: the table form repacks on every call (one +// query against all centroids), unlike MaxSim's column_max below which packs once per +// query and calls into this ISA's kernels directly with the already-packed operand. +void InnerProductMatrixImpl::build( + const float* a, size_t m, const float* b, size_t n, size_t d, float* out +) { + if (m == 0 || n == 0) { + return; + } + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); +} + +// Column-wise maximum of the same table build_packed computes, without ever +// materializing it: each row-block's accumulators are reduced to a per-lane maximum and +// folded into a running maximum held in `out` itself, so one reduction serves the whole +// document for the price of one load/max/store per lane-block per row-block. +void InnerProductMatrixImpl::column_max( + const float* a, size_t m, const PackedOperand& b, float* out ) { size_t n = b.n(); size_t d = b.d(); @@ -248,30 +362,11 @@ void InnerProductMatrixImpl::column_max( size_t full_lanes = n - n % vl; __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); - thread_local std::vector scratch; - scratch.resize(kRowBlock * d); - for (size_t i = 0; i < m; i += kRowBlock) { size_t rows = std::min(kRowBlock, m - i); - for (size_t r = 0; r < rows; ++r) { - const svs::Float16* src = a + (i + r) * d; - float* dst = scratch.data() + r * d; - size_t k = 0; - for (; k + 16 <= d; k += 16) { - _mm512_storeu_ps(dst + k, _mm512_cvtph_ps(_mm256_loadu_epi16(src + k))); - } - if (k < d) { - __mmask16 dmask = svs::create_mask<16>(svs::lib::MaybeStatic(d - k)); - _mm512_mask_storeu_ps( - dst + k, - dmask, - _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(dmask, src + k)) - ); - } - } const float* arow[kRowBlock]; for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = scratch.data() + std::min(r, rows - 1) * d; + arow[r] = a + (i + std::min(r, rows - 1)) * d; } size_t j = 0; @@ -298,6 +393,21 @@ void InnerProductMatrixImpl::column_max( } } +// Float16 overload: see convert_row_block above for the shared widening (AVX-512F's +// native 16-wide _mm512_cvtph_ps). +void InnerProductMatrixImpl::column_max( + const svs::Float16* a, size_t m, const PackedOperand& b, float* out +) { + column_max_narrow(a, m, b, out); +} + +// BFloat16 overload: see convert_row_block above for the shared (exact) widening. +void InnerProductMatrixImpl::column_max( + const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out +) { + column_max_narrow(a, m, b, out); +} + } // namespace svs::distance #endif diff --git a/tests/svs/core/distances/inner_product_matrix.cpp b/tests/svs/core/distances/inner_product_matrix.cpp index 18950de91..20d33ba70 100644 --- a/tests/svs/core/distances/inner_product_matrix.cpp +++ b/tests/svs/core/distances/inner_product_matrix.cpp @@ -19,6 +19,7 @@ // svs #include "svs/lib/avx_detection.h" +#include "svs/lib/bfloat16.h" #include "svs/lib/float16.h" // catch2 @@ -27,7 +28,9 @@ // stdlib #include +#include #include +#include #include #include #include @@ -269,6 +272,45 @@ std::vector promote_float(const std::vector& a) { return out; } +// Rounds to nearest, ties to even (svs::round_to_bfloat16), not BFloat16's own truncating +// constructor: this is what every bf16 row-overload test below checks the kernel against. +std::vector to_bfloat16(const std::vector& a) { + std::vector out(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + out[i] = svs::round_to_bfloat16(a[i]); + } + return out; +} + +// Promotes a BFloat16 row-major matrix to double, so the reference is computed from the +// values the kernel actually sees (post-rounding), not the pre-rounding floats. +std::vector promote_bfloat16(const std::vector& a) { + std::vector out(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + out[i] = static_cast(static_cast(a[i])); + } + return out; +} + +// Naive double-precision reference for the table form, out[i * n + j] = , over +// an already-promoted `a` so a narrow-type A's post-conversion values are what the +// reference uses (shares `reference`'s role above, generalized to a non-float32 `a`). +std::vector reference_from_values( + const std::vector& a, size_t m, const std::vector& b, size_t n, size_t d +) { + std::vector out(m * n, 0.0); + for (size_t i = 0; i < m; ++i) { + for (size_t j = 0; j < n; ++j) { + double acc = 0.0; + for (size_t k = 0; k < d; ++k) { + acc += a[i * d + k] * static_cast(b[j * d + k]); + } + out[i * n + j] = acc; + } + } + return out; +} + template void check_column_max_float32_against_reference() { unsigned seed = 2000; for (size_t d : column_max_ds) { @@ -306,6 +348,28 @@ template void check_column_max_float16_against_reference( } } +template void check_column_max_bf16_against_reference() { + unsigned seed = 3500; + for (size_t d : column_max_ds) { + for (size_t m : column_max_ms) { + for (size_t n : column_max_ns) { + auto a = random_matrix(m, d, seed++); + auto a_bf16 = to_bfloat16(a); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + std::vector out(n, -3.0F); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, out.data() + ); + require_close_vec( + out, reference_column_max(promote_bfloat16(a_bf16), m, b, n, d) + ); + } + } + } +} + template void check_column_max_against_generic() { unsigned seed = 4000; for (size_t d : column_max_ds) { @@ -356,6 +420,87 @@ template void check_column_max_against_generic() { .margin(1e-4) ); } + + auto a_bf16 = to_bfloat16(a); + std::vector generic_out_bf16(n, -1.0F); + std::vector isa_out_bf16(n, -2.0F); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, generic_out_bf16.data() + ); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, isa_out_bf16.data() + ); + // Same reordering-only tolerance as the Float16 case above: BFloat16's + // widening is exact, so any difference here is summation order, not + // decode. + for (size_t j = 0; j < n; ++j) { + CATCH_REQUIRE( + static_cast(isa_out_bf16[j]) == + Catch::Approx(static_cast(generic_out_bf16[j])) + .epsilon(2e-4) + .margin(1e-4) + ); + } + } + } + } +} + +template void check_build_packed_narrow_against_reference() { + // Smaller shape set than column_max's: the table form's output scales as m * n, and + // this already exercises the row-block (8) and both SIMD lane-width (8, 16) tails. + unsigned seed = 6000; + for (size_t d : {1UL, 7UL, 128UL}) { + for (size_t m : {0UL, 1UL, 7UL, 9UL, 17UL}) { + for (size_t n : {1UL, 15UL, 17UL, 32UL}) { + auto a = random_matrix(m, d, seed++); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + + auto a16 = to_float16(a); + std::vector out16(m * n, -1.0F); + InnerProductMatrixImpl::build_packed( + a16.data(), m, packed, out16.data() + ); + require_close( + out16, reference_from_values(promote_float16(a16), m, b, n, d) + ); + + auto a_bf16 = to_bfloat16(a); + std::vector out_bf16(m * n, -1.0F); + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, out_bf16.data() + ); + require_close( + out_bf16, reference_from_values(promote_bfloat16(a_bf16), m, b, n, d) + ); + + // Against the generic (NONE) level, same tail shapes. + std::vector generic16(m * n, -1.0F); + InnerProductMatrixImpl::build_packed( + a16.data(), m, packed, generic16.data() + ); + for (size_t idx = 0; idx < generic16.size(); ++idx) { + CATCH_REQUIRE( + static_cast(out16[idx]) == + Catch::Approx(static_cast(generic16[idx])) + .epsilon(2e-4) + .margin(1e-4) + ); + } + std::vector generic_bf16(m * n, -1.0F); + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, generic_bf16.data() + ); + for (size_t idx = 0; idx < generic_bf16.size(); ++idx) { + CATCH_REQUIRE( + static_cast(out_bf16[idx]) == + Catch::Approx(static_cast(generic_bf16[idx])) + .epsilon(2e-4) + .margin(1e-4) + ); + } } } } @@ -369,6 +514,7 @@ CATCH_TEST_CASE( ) { check_column_max_float32_against_reference(); check_column_max_float16_against_reference(); + check_column_max_bf16_against_reference(); } CATCH_TEST_CASE( @@ -380,6 +526,7 @@ CATCH_TEST_CASE( } check_column_max_float32_against_reference(); check_column_max_float16_against_reference(); + check_column_max_bf16_against_reference(); check_column_max_against_generic(); } @@ -392,9 +539,24 @@ CATCH_TEST_CASE( } check_column_max_float32_against_reference(); check_column_max_float16_against_reference(); + check_column_max_bf16_against_reference(); check_column_max_against_generic(); } +CATCH_TEST_CASE( + "InnerProductMatrixImpl build_packed matches a naive reference for Float16 and " + "BFloat16 rows, every available level", + "[distance][inner_product_matrix]" +) { + check_build_packed_narrow_against_reference(); + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + check_build_packed_narrow_against_reference(); + } + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + check_build_packed_narrow_against_reference(); + } +} + CATCH_TEST_CASE( "InnerProductMatrixImpl build_packed matches build through an explicit PackedOperand", "[distance][inner_product_matrix]" @@ -436,3 +598,122 @@ CATCH_TEST_CASE( } } } + +CATCH_TEST_CASE( + "InnerProductMatrix::compute_packed picks the same ISA result as " + "InnerProductMatrixImpl " + "for BFloat16 and Float16 rows", + "[distance][inner_product_matrix]" +) { + unsigned seed = 7000; + for (size_t d : {1UL, 7UL, 128UL}) { + for (size_t m : {0UL, 1UL, 9UL, 17UL}) { + for (size_t n : {1UL, 17UL, 33UL}) { + auto a = random_matrix(m, d, seed++); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + + // Same ISA-priority repetition as the float32 build_packed test above: + // InnerProductMatrix::select_isa() is private by design. + auto direct_bf16 = [&](const auto& av, std::vector& out) { + auto a_bf16 = to_bfloat16(a); + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, out.data() + ); + } else if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, out.data() + ); + } else { + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, out.data() + ); + } + (void)av; + }; + + auto a_bf16 = to_bfloat16(a); + std::vector via_public(m * n, -1.0F); + InnerProductMatrix::compute_packed( + a_bf16.data(), m, packed, via_public.data() + ); + std::vector via_direct(m * n, -2.0F); + direct_bf16(0, via_direct); + CATCH_REQUIRE(via_public == via_direct); + + auto a16 = to_float16(a); + std::vector via_public16(m * n, -1.0F); + InnerProductMatrix::compute_packed( + a16.data(), m, packed, via_public16.data() + ); + std::vector via_direct16(m * n, -2.0F); + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + InnerProductMatrixImpl::build_packed( + a16.data(), m, packed, via_direct16.data() + ); + } else if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + InnerProductMatrixImpl::build_packed( + a16.data(), m, packed, via_direct16.data() + ); + } else { + InnerProductMatrixImpl::build_packed( + a16.data(), m, packed, via_direct16.data() + ); + } + CATCH_REQUIRE(via_public16 == via_direct16); + } + } + } +} + +CATCH_TEST_CASE( + "PackedOperand::pack_rounded_to_bf16 rounds to nearest, ties to even", + "[distance][inner_product_matrix]" +) { + CATCH_SECTION("Matches svs::round_to_bfloat16 over representative values") { + // Exact bf16 value, round-down, round-up, zeros of both signs, and the largest + // finite float32 (whose rounding overflows to bf16 infinity, exercised below). + std::vector values{ + 1.0F, + 0.1F, + -0.1F, + 0.0F, + -0.0F, + 100.25F, + -100.25F, + std::numeric_limits::max(), + -std::numeric_limits::max(), + }; + PackedOperand packed; + packed.pack_rounded_to_bf16(values.data(), values.size(), 1); + for (size_t i = 0; i < values.size(); ++i) { + float expected = static_cast(svs::round_to_bfloat16(values[i])); + CATCH_REQUIRE(packed.data()[i] == expected); + } + } + + CATCH_SECTION("Exact halfway cases round to the even neighbor, checked bit for bit") { + // Both inputs discard exactly 0x8000 (the bf16 ULP's halfway point) below bit 16: + // the first's kept 16 bits are already even and must stay; the second's are odd + // and must round up to the next even value. This checks the rounding algorithm's + // bit pattern directly, not just self-consistency with svs::round_to_bfloat16. + float tie_even_stays = std::bit_cast(uint32_t{0x3F808000}); + float tie_odd_rounds_up = std::bit_cast(uint32_t{0x3F818000}); + std::vector values{tie_even_stays, tie_odd_rounds_up}; + PackedOperand packed; + packed.pack_rounded_to_bf16(values.data(), values.size(), 1); + CATCH_REQUIRE(std::bit_cast(packed.data()[0]) == uint32_t{0x3F800000}); + CATCH_REQUIRE(std::bit_cast(packed.data()[1]) == uint32_t{0x3F820000}); + } + + CATCH_SECTION("Rounding float32's max overflows to signed infinity") { + std::vector values{ + std::numeric_limits::max(), -std::numeric_limits::max()}; + PackedOperand packed; + packed.pack_rounded_to_bf16(values.data(), values.size(), 1); + CATCH_REQUIRE(packed.data()[0] == std::numeric_limits::infinity()); + CATCH_REQUIRE(packed.data()[1] == -std::numeric_limits::infinity()); + } +} From f16125e2f08ff97be61e2f9515f8ab8930adee92 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 06:17:38 -0800 Subject: [PATCH 19/68] feat(distance): store GEM's fine centroids and ground-cost table as bfloat16 FineCentroids now holds its K x dim matrix as svs::BFloat16 (round to nearest, ties to even) instead of float32: one representation, no float32 copy kept after construction/load, so every ISA level computes on identical values and the upcoming AMX level can consume the same layout directly. The constructor and load() round in parallel (a thread pool each already had access to); load() additionally reads and rounds in row-block chunks so peak memory never holds the full float32 input alongside the full bf16 output. QChamferExactQuery::fix_argument_decoded now packs the query rounded to bf16 (PackedOperand::pack_rounded_to_bf16) and builds the per-query table with InnerProductMatrix::compute_packed against the (already bf16) centroids, so the table is on every ISA level. CentroidTable::from_centroids rounds both the row and column block of each tile to bf16 before computing, so its K x K table agrees with FineCentroids on the same rounded values; the public signature (float32 centroids in) is unchanged, since rounding now happens per tile instead of requiring a second, already-bf16 input. Test fixtures: all pre-existing hand-computed expectations use only values exactly representable in bfloat16 (1, 0, -1), so none changed. Added: a bf16-rounding section for FineCentroids (incl. a load() round-trip through the unchanged float32 file format), a dedicated non-representable-value case for both CentroidTable::from_centroids and QChamferExactQuery against a naive bf16-rounded reference, and rounded the from_centroids tiling test's random input before building its reference (bf16 rounding error otherwise swamped the existing 1e-4 summation-order margin). Co-Authored-By: Claude Fable 5.2 --- include/svs/core/distance/gem.h | 167 ++++++++++++++++++++++--------- tests/svs/core/distances/gem.cpp | 111 +++++++++++++++++++- 2 files changed, 228 insertions(+), 50 deletions(-) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index 1eba6c88c..3bbf13334 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -30,6 +30,7 @@ #include "svs/core/distance/distance_core.h" #include "svs/core/distance/inner_product_matrix.h" #include "svs/lib/avx_detection.h" +#include "svs/lib/bfloat16.h" #include "svs/lib/misc.h" #include "svs/lib/preprocessor.h" #include "svs/lib/threads.h" @@ -120,15 +121,21 @@ struct CentroidTable { return result; } - /// Computes T[i][j] = ground(c_i, c_j) from K row-major `dim`-wide centroids, - /// T[i][i] = 0. Symmetric by construction: each unordered pair {i, j} is computed - /// once, by the tile that owns it, and the same float is stored at T[i][j] and - /// T[j][i] (never two independently-computed dot products). Parallelized over tiles - /// on `num_threads` threads, dynamically scheduled so no two tiles ever write the - /// same cell; the result is bit-for-bit identical regardless of `num_threads`. Uses - /// an uninitialized allocation (see the private constructor below) and writes every - /// cell exactly once (off-diagonal tiles) or twice with the same value (diagonal - /// tiles' upper/lower halves), so no thread does a sequential full-matrix pass. + /// Computes T[i][j] = ground(bf16(c_i), bf16(c_j)) from K row-major `dim`-wide + /// centroids, T[i][i] = 0. `centroids` is float32 (the on-disk/in-memory format is + /// unchanged); each tile rounds its own row and column block to bfloat16 (round to + /// nearest, ties to even, matching FineCentroids) before computing, so this table and + /// FineCentroids agree on the same rounded centroid values. Symmetric by + /// construction: each unordered pair {i, j} is computed once, by the tile that owns + /// it, and the same float is stored at T[i][j] and T[j][i] (never two + /// independently-computed dot products); that holds regardless of which operand role + /// bf16-rounding was applied to, since both row and column blocks are rounded + /// identically before either is used. Parallelized over tiles on `num_threads` + /// threads, dynamically scheduled so no two tiles ever write the same cell; the + /// result is bit-for-bit identical regardless of `num_threads`. Uses an uninitialized + /// allocation (see the private constructor below) and writes every cell exactly once + /// (off-diagonal tiles) or twice with the same value (diagonal tiles' upper/lower + /// halves), so no thread does a sequential full-matrix pass. static CentroidTable from_centroids( const float* centroids, size_t k, size_t dim, Ground ground, size_t num_threads ) { @@ -172,9 +179,15 @@ struct CentroidTable { pool, svs::threads::DynamicPartition{tiles.size(), grain}, [&](const auto& tile_ids, uint64_t /*tid*/) { - // Reused across this thread's tiles: InnerProductMatrix::compute packs - // its own operand internally, so this buffer only holds the tile output. + // Reused across this thread's tiles: `buf` holds the tile output, + // `rows_bf16` the row block rounded to bfloat16, `cols_packed` the column + // block packed and rounded by the same bfloat16 rounding. Both operands end + // up rounded to the same precision this table is meant to agree with + // FineCentroids on, so the whole method sees one (bf16-rounded) set of + // centroid values. thread_local std::vector buf; + thread_local std::vector rows_bf16; + thread_local PackedOperand cols_packed; buf.resize(tile * tile); for (size_t t : tile_ids) { auto [bi, bj] = tiles[t]; @@ -182,13 +195,14 @@ struct CentroidTable { size_t col0 = static_cast(bj) * tile; size_t rows = std::min(tile, k - row0); size_t cols = std::min(tile, k - col0); - InnerProductMatrix::compute( - centroids + row0 * dim, - rows, - centroids + col0 * dim, - cols, - dim, - buf.data() + rows_bf16.resize(rows * dim); + for (size_t idx = 0; idx < rows * dim; ++idx) { + rows_bf16[idx] = + svs::round_to_bfloat16(centroids[row0 * dim + idx]); + } + cols_packed.pack_rounded_to_bf16(centroids + col0 * dim, cols, dim); + InnerProductMatrix::compute_packed( + rows_bf16.data(), rows, cols_packed, buf.data() ); bool diag = (bi == bj); for (size_t r = 0; r < rows; ++r) { @@ -228,24 +242,38 @@ struct CentroidTable { , data{std::make_unique_for_overwrite(k * k)} {} }; -/// Owns the raw K x dim fine-centroid vectors (row-major, not a pairwise table). Used by -/// QChamferExactQuery to build a per-query (K x query_tokens) dot table against the exact -/// float query, mirroring GEM's score_fine_centers (src/search/routing.cpp:55-61), which -/// never quantizes the query side. +/// Owns the raw K x dim fine-centroid vectors (row-major, not a pairwise table), as +/// bfloat16: every value is the float32 input rounded to nearest, ties to even (matching +/// `svs::round_to_bfloat16`, not `BFloat16`'s own truncating conversion). One +/// representation, no float32 copy kept once construction/`load` returns, so every ISA +/// level (including a future one that multiplies bf16 natively) computes on identical +/// values. Used by QChamferExactQuery to build a per-query (K x query_tokens) dot table +/// against the exact float query, mirroring GEM's score_fine_centers +/// (src/search/routing.cpp:55-61), which never quantizes the query side. struct FineCentroids { size_t K; size_t dim; - std::unique_ptr data; + std::unique_ptr data; - FineCentroids(size_t k, size_t dim_, const float* flat) + /// Rounds `flat` (K rows of `dim_` float32 each, row-major; the on-disk/in-memory + /// input format is unchanged) to bfloat16, in parallel over row blocks on + /// `num_threads` threads. Peak memory is `flat`'s own K * dim_ * 4 bytes plus this + /// object's K * dim_ * 2 bytes: no second float32 buffer is allocated here. + FineCentroids(size_t k, size_t dim_, const float* flat, size_t num_threads = 1) : K{k} , dim{dim_} - , data{std::make_unique(k * dim_)} { - std::copy(flat, flat + k * dim_, data.get()); + , data{std::make_unique_for_overwrite(k * dim_)} { + round_rows(flat, data.get(), k * dim_, num_threads); } // Raw file of K rows of `dim` floats with no header, so K comes from the file size. - static FineCentroids load(const std::filesystem::path& path, size_t dim) { + // Reads and rounds in row-block chunks rather than slurping the whole file into a + // second float32 buffer: peak memory is this object's K * dim * 2 bytes (the bf16 + // result, allocated once, uninitialized) plus one chunk's + // min(K, kLoadChunkRows) * dim * 4 bytes (4 MiB at dim=128), not the full K * dim + // float32 matrix held alongside the bf16 one. + static FineCentroids + load(const std::filesystem::path& path, size_t dim, size_t num_threads = 1) { std::ifstream in(path, std::ios::binary); if (!in) { throw std::runtime_error("FineCentroids::load: cannot open " + path.string()); @@ -260,13 +288,54 @@ struct FineCentroids { ); } auto k = bytes / row_bytes; - std::vector flat(k * dim); in.seekg(0, std::ios::beg); - in.read(reinterpret_cast(flat.data()), static_cast(bytes)); - if (!in) { - throw std::runtime_error("FineCentroids::load: short read on " + path.string()); + + FineCentroids result{k, dim, uninitialized_tag{}}; + constexpr size_t kLoadChunkRows = 8192; + std::vector chunk(std::min(k, kLoadChunkRows) * dim); + for (size_t row0 = 0; row0 < k; row0 += kLoadChunkRows) { + size_t rows = std::min(kLoadChunkRows, k - row0); + size_t count = rows * dim; + in.read( + reinterpret_cast(chunk.data()), + static_cast(count * sizeof(float)) + ); + if (!in) { + throw std::runtime_error( + "FineCentroids::load: short read on " + path.string() + ); + } + round_rows(chunk.data(), result.data.get() + row0 * dim, count, num_threads); } - return FineCentroids{k, dim, flat.data()}; + return result; + } + + private: + struct uninitialized_tag {}; + + // new[] with no parens default-initializes BFloat16 (indeterminate value, matching + // CentroidTable's uninitialized constructor); every element is overwritten by + // round_rows before this object becomes visible to a caller. + FineCentroids(size_t k, size_t dim_, uninitialized_tag) + : K{k} + , dim{dim_} + , data{std::make_unique_for_overwrite(k * dim_)} {} + + // Rounds `count` contiguous float32 values to bfloat16, statically partitioned + // across `num_threads` threads; shared by the constructor and the per-chunk path in + // `load` so there is exactly one place that performs this conversion. + static void + round_rows(const float* flat, svs::BFloat16* out, size_t count, size_t num_threads) { + svs::threads::DefaultThreadPool pool(num_threads); + svs::threads::parallel_for( + pool, + svs::threads::StaticPartition{count}, + [&](const auto& ids, uint64_t /*tid*/) { + for (size_t i : ids) { + out[i] = svs::round_to_bfloat16(flat[i]); + } + } + ); } }; @@ -600,11 +669,13 @@ class QChamfer { inline constexpr bool operator==(const QChamfer&, const QChamfer&) { return true; } /// Search-time distance: GEM's actual in-graph qCH (GEM_REFERENCE.md "search-time distance: -/// is the query quantized?" row). Unlike QChamfer, the query is never quantized: it stays -/// as raw float tokens, and fix_argument() builds a dense (K x query_tokens) dot table -/// against every fine centroid once per query (score_fine_centers, src/search/ -/// routing.cpp:55-61); compute() then reduces that table against the document's quantized -/// centroid ids (code_distance, src/distance/maxsim.cpp:9-45). Smaller is better. +/// is the query quantized?" row). Unlike QChamfer, the query is never id-quantized: it +/// stays as raw float tokens, and fix_argument() builds a dense (K x query_tokens) dot +/// table against every fine centroid once per query (score_fine_centers, src/search/ +/// routing.cpp:55-61), both sides rounded to bfloat16 so the table's values do not depend +/// on which ISA level computed them (see FineCentroids); compute() then reduces that +/// table against the document's quantized centroid ids (code_distance, +/// src/distance/maxsim.cpp:9-45). Smaller is better. /// /// Holds a per-query table as ordinary (non-shared) state, so each search thread's /// shallow_copy of this functor gets its own table; implicit_broadcast is false and @@ -654,21 +725,18 @@ class QChamferExactQuery { private: // Builds the (K x n_query) dot table once per query: table[f * n_query + i] = - // dot(centroid_f, query_token_i), matching maxsim.cpp's Eigen::Map(table, - // query_tokens, fine_centers) column-major read of the same buffer (scores.col(f)[i] - // == table[f*n_query+i]). That is exactly InnerProductMatrix::compute's A x B^T with - // A = centroids (K rows) and B = query tokens (n_query rows), which now owns both the - // ISA choice and the query-side packing. + // dot(bf16(centroid_f), bf16(query_token_i)), matching maxsim.cpp's + // Eigen::Map(table, query_tokens, fine_centers) column-major read of the + // same buffer (scores.col(f)[i] == table[f*n_query+i]). That is exactly + // InnerProductMatrix::compute_packed's A x B^T with A = the (already bf16) centroids + // (K rows) and B = query tokens (n_query rows), rounded to bf16 by this call rather + // than the kernel, so every ISA level sees the same rounded values on both operands. void fix_argument_decoded(const float* tokens, size_t n_query) { n_query_ = n_query; table_.resize(centroids_->K * n_query_); - InnerProductMatrix::compute( - centroids_->data.get(), - centroids_->K, - tokens, - n_query_, - centroids_->dim, - table_.data() + packed_query_.pack_rounded_to_bf16(tokens, n_query_, centroids_->dim); + InnerProductMatrix::compute_packed( + centroids_->data.get(), centroids_->K, packed_query_, table_.data() ); } @@ -691,6 +759,7 @@ class QChamferExactQuery { } std::shared_ptr centroids_; + PackedOperand packed_query_; std::vector table_; size_t n_query_ = 0; }; diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp index 6dc661b4b..2c0df58ec 100644 --- a/tests/svs/core/distances/gem.cpp +++ b/tests/svs/core/distances/gem.cpp @@ -22,6 +22,7 @@ #include "svs/concepts/distance.h" #include "svs/core/data/vector_set_view.h" #include "svs/index/vamana/prune.h" +#include "svs/lib/bfloat16.h" // catch2 #include "catch2/catch_approx.hpp" @@ -31,6 +32,8 @@ #include #include #include +#include +#include #include #include #include @@ -77,6 +80,18 @@ std::vector random_unit_centroids(size_t k, size_t dim, uint32_t seed) { return flat; } +// from_centroids now rounds every centroid component to bfloat16 (round to nearest, ties +// to even) before computing, so a reference built from the unrounded input would differ +// by bf16's ~0.4% rounding error, not just summation order; round here too so the +// pre-existing 1e-4 margin still bounds only the latter. +std::vector round_to_bf16_vec(const std::vector& flat) { + std::vector out(flat.size()); + for (size_t i = 0; i < flat.size(); ++i) { + out[i] = static_cast(svs::round_to_bfloat16(flat[i])); + } + return out; +} + // Reference ground-cost table computed with double-precision dot products and the same // formula from_centroids applies, independent of InnerProductMatrix/tiling. std::vector @@ -162,6 +177,41 @@ CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { CATCH_REQUIRE(fc.data[0] == 1.0F); CATCH_REQUIRE(fc.data[3] == 1.0F); } + + CATCH_SECTION("FineCentroids rounds to bfloat16, round to nearest even") { + CATCH_STATIC_REQUIRE(std::is_same_v< + std::decay_t().data[0])>, + svs::BFloat16>); + // None of these are exactly representable in bfloat16: the test only holds if + // fc.data actually rounds (matching svs::round_to_bfloat16), not merely stores + // float32 unchanged or truncates like BFloat16's own constructor would. + std::vector flat{0.1F, -0.1F, 1.0F / 3.0F, -2.0F / 3.0F}; + FineCentroids fc{2, 2, flat.data()}; + for (size_t i = 0; i < flat.size(); ++i) { + CATCH_REQUIRE(fc.data[i] == svs::round_to_bfloat16(flat[i])); + } + } + + CATCH_SECTION("FineCentroids::load round-trips through the unchanged float32 file " + "format") { + std::vector flat{0.1F, 0.2F, 0.3F, -0.4F, 2.5F, -3.5F}; + auto path = + std::filesystem::temp_directory_path() / "svs_gem_fine_centroids_test.bin"; + { + std::ofstream out(path, std::ios::binary); + out.write( + reinterpret_cast(flat.data()), + static_cast(flat.size() * sizeof(float)) + ); + } + auto fc = FineCentroids::load(path, 2); + std::filesystem::remove(path); + CATCH_REQUIRE(fc.K == 3); + CATCH_REQUIRE(fc.dim == 2); + for (size_t i = 0; i < flat.size(); ++i) { + CATCH_REQUIRE(fc.data[i] == svs::round_to_bfloat16(flat[i])); + } + } } CATCH_TEST_CASE("CentroidTable::from_centroids tiling", "[distance][gem]") { @@ -185,7 +235,11 @@ CATCH_TEST_CASE("CentroidTable::from_centroids tiling", "[distance][gem]") { k * 1009 + dim * 31 + (ground == Ground::Cos ? 1 : 2) ) ); - auto reference = naive_ground_table(flat, k, dim, ground); + // from_centroids rounds internally; round the reference's input the same + // way so the comparison below isolates summation-order error, not bf16 + // rounding error. + auto reference = + naive_ground_table(round_to_bf16_vec(flat), k, dim, ground); auto one_thread = CentroidTable::from_centroids(flat.data(), k, dim, ground, 1); @@ -404,6 +458,61 @@ CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { } } +CATCH_TEST_CASE( + "GEM bf16 rounding: non-representable centroid and query values", "[distance][gem]" +) { + // None of these are exactly representable in bfloat16: unlike every other fixture in + // this file (chosen to be bf16-exact so their hand-computed expectations hold + // unchanged), this case exists specifically to exercise rounding. + std::vector flat{0.1F, 0.2F, -0.3F, 0.4F, 1.0F / 3.0F, -2.0F / 3.0F}; + size_t k = 3; + size_t dim = 2; + + CATCH_SECTION("CentroidTable::from_centroids matches a bf16-rounded naive reference") { + auto reference = naive_ground_table(round_to_bf16_vec(flat), k, dim, Ground::Cos); + auto table = CentroidTable::from_centroids(flat.data(), k, dim, Ground::Cos, 1); + for (size_t i = 0; i < k; ++i) { + for (size_t j = 0; j < k; ++j) { + auto ui = static_cast(i); + auto uj = static_cast(j); + CATCH_REQUIRE( + static_cast(table(ui, uj)) == + Catch::Approx(reference[i * k + j]).margin(1e-4) + ); + } + } + } + + CATCH_SECTION("QChamferExactQuery matches a naive reference over bf16-rounded values") { + auto centroids = std::make_shared(k, dim, flat.data()); + QChamferExactQuery dist{centroids}; + + std::vector query{0.15F, -0.25F}; + RawQuerySet query_set{query, dim}; + auto doc_set = make_id_set({0, 2}); + + svs::distance::maybe_fix_argument(dist, query_set.view()); + float score = dist.compute(query_set.view(), doc_set.view()); + + // Naive reference over the same bf16-rounded centroid and query values (round to + // nearest, ties to even on both sides), independent of fix_argument_decoded and + // the kernel. + auto rounded_centroids = round_to_bf16_vec(flat); + float qx = static_cast(svs::round_to_bfloat16(query[0])); + float qy = static_cast(svs::round_to_bfloat16(query[1])); + std::vector doc_ids{0, 2}; + float best = -std::numeric_limits::infinity(); + for (auto id : doc_ids) { + float cx = rounded_centroids[id * dim]; + float cy = rounded_centroids[id * dim + 1]; + best = std::max(best, cx * qx + cy * qy); + } + float expected = 1.0F - best; // One query token: the "mean" is just this term. + + CATCH_REQUIRE(score == Catch::Approx(expected)); + } +} + CATCH_TEST_CASE("GEM distance traits", "[distance][gem]") { CATCH_STATIC_REQUIRE(std::is_same_v>); CATCH_STATIC_REQUIRE(std::is_same_v>); From 2f217c1257eccdcaa4fc23985c61e199d432a8b2 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 07:04:54 -0800 Subject: [PATCH 20/68] feat(distance): add an AMX ISA level with a gap for the AVX512 split Extends AVX_AVAILABILITY with AMX = 10, leaving the ordinals 3-9 free for a parallel split of AVX512 into AVX512F/AVX512VNNI: nothing in this codebase reads the enum's ordinal or order, only switches and templates on the enumerator itself, so a gap is free. Detection in avx_detection.h follows the existing CPUID-probe style and adds the one-time, thread-safe OS permission request AMX tile use needs (ARCH_REQ_XCOMP_PERM), cached in a function-local static so the per-call cost stays one branch. Build integration in cmake/multi-arch.cmake adds amx.cpp only when the compiler accepts -march=sapphirerapids, gated through a build-wide SVS_AMX_TILE_BF16_SUPPORTED definition so a build without that support still produces a working library with the level compiled out. No existing kernel's dispatch changes: InnerProductMatrix::select_isa and its exhaustive switch gain an AMX branch guarded on SVS_AMX_COMPILED_IN, falling back to the AVX512 case when the level was never compiled in. Co-Authored-By: Claude Fable 5.2 --- cmake/multi-arch.cmake | 21 ++++++++++ include/svs/core/distance/distance_core.h | 7 +++- include/svs/lib/avx_detection.h | 51 +++++++++++++++++++++++ include/svs/lib/preprocessor.h | 22 ++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) diff --git a/cmake/multi-arch.cmake b/cmake/multi-arch.cmake index aeb81e693..c10e90dc3 100644 --- a/cmake/multi-arch.cmake +++ b/cmake/multi-arch.cmake @@ -18,6 +18,19 @@ set(SVS_X86 "${SVS_X86_SRC_DIR}/avx512.cpp,avx512,cascadelake" ) +# AMX (tile load/store/config plus the bf16 dot-product tile instruction) is optional: +# -march=sapphirerapids is the flag amx.cpp is built with below, same one-codename-per-ISA +# pattern as haswell/cascadelake above, and the first compiler/-march combination that +# ships it; a compiler that rejects it (too old, or a non-x86 cross build) must still +# produce a working library with the level compiled out, so amx.cpp is added to SVS_X86 +# only when this check passes, and SVS_AMX_TILE_BF16_SUPPORTED (read by +# include/svs/lib/preprocessor.h) is defined only then. +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-march=sapphirerapids" SVS_COMPILER_SUPPORTS_AMX) +if(SVS_COMPILER_SUPPORTS_AMX) + list(APPEND SVS_X86 "${SVS_X86_SRC_DIR}/amx.cpp,amx,sapphirerapids") +endif() + set(SVS_X86_OBJECT_FILES) foreach(x86_info IN LISTS SVS_X86) string(REPLACE "," ";" x86_info "${x86_info}") @@ -37,6 +50,14 @@ endforeach() add_library(svs_x86_objects STATIC ${SVS_X86_OBJECT_FILES}) target_link_libraries(svs_export INTERFACE svs_x86_objects) +if(SVS_COMPILER_SUPPORTS_AMX) + # Uniform across every consumer of svs_export, unlike the per-file -march flags above: + # PackedOperand (inner_product_matrix.h) decides whether to fill the AMX tile layout + # from this, and it is included by TUs that never pass -march=sapphirerapids + # themselves (gem.h, maxsim.h, tests). + target_compile_definitions(svs_export INTERFACE SVS_AMX_TILE_BF16_SUPPORTED) +endif() + install( TARGETS svs_x86_objects EXPORT svs-targets diff --git a/include/svs/core/distance/distance_core.h b/include/svs/core/distance/distance_core.h index 4f59f9de0..5022991bd 100644 --- a/include/svs/core/distance/distance_core.h +++ b/include/svs/core/distance/distance_core.h @@ -26,7 +26,12 @@ namespace svs::distance { -enum class AVX_AVAILABILITY { NONE, AVX2, AVX512 }; +// Explicit values, not contiguous: nothing in this codebase reads AVX_AVAILABILITY's +// ordinal or compares it, only switches/templates on the enumerator itself (checked by +// grep before adding AMX), so the gap between AVX512 and AMX is free to reserve for a +// parallel split of AVX512 into two enumerators (e.g. AVX512F, AVX512VNNI) without +// renumbering AMX. +enum class AVX_AVAILABILITY { NONE = 0, AVX2 = 1, AVX512 = 2, AMX = 10 }; constexpr std::array supported_dim_list{ 64, 96, 100, 128, 160, 200, 512, 768, svs::Dynamic}; diff --git a/include/svs/lib/avx_detection.h b/include/svs/lib/avx_detection.h index fc9b246f9..3169bc173 100644 --- a/include/svs/lib/avx_detection.h +++ b/include/svs/lib/avx_detection.h @@ -16,8 +16,15 @@ #pragma once +#include #include +#if defined(__x86_64__) && defined(__linux__) +#include +#include +#include +#endif + namespace svs::detail { #ifdef __x86_64__ @@ -36,6 +43,8 @@ struct AVXRuntimeFlags { avx2 = extended_features && ((ebx & (1 << 5)) != 0); avx512f = extended_features && ((ebx & (1 << 16)) != 0); avx512vnni = extended_features && ((ecx & (1 << 11)) != 0); + amx_tile_bf16_cpu = + extended_features && ((edx & (1 << 24)) != 0) && ((edx & (1 << 22)) != 0); } bool is_avx2_supported() const noexcept { return avx2; } @@ -45,15 +54,57 @@ struct AVXRuntimeFlags { bool avx2; bool avx512f; bool avx512vnni; + bool amx_tile_bf16_cpu; }; #else struct AVXRuntimeFlags { bool is_avx2_supported() const noexcept { return false; } bool is_avx512f_supported() const noexcept { return false; } bool is_avx512vnni_supported() const noexcept { return false; } + + bool amx_tile_bf16_cpu = false; }; #endif inline const AVXRuntimeFlags avx_runtime_flags = {}; +#if defined(__x86_64__) && defined(__linux__) +// Kernel XFEATURE_XTILEDATA index for ARCH_REQ_XCOMP_PERM. Not in any header on this +// machine (glibc's asm/prctl.h stops at ARCH_REQ_XCOMP_PERM itself); taken from the +// kernel's xstate ABI (Documentation/x86/xstate.rst) and confirmed by experiment +// (/tmp/amx-study/t2c_detect.c) that requesting permission with this value is what lets +// tile instructions run without SIGILL. +inline constexpr unsigned long SVS_XFEATURE_XTILEDATA = 18; + +// Requests AMX tile-data OS permission exactly once per process, the first time any +// thread asks whether the AMX level is usable. Requesting at library load would run in +// whatever process SVS is linked into before that process has decided it wants AMX at +// all; requesting per-call would repeat a syscall this function's caching exists to +// avoid. The request must run unconditionally, every time: XCR0 (read via `xgetbv`) +// reports which xstate components the CPU/OS architecturally support, not whether this +// process has been granted permission to use tile data, so an "already looks enabled" +// shortcut on XCR0 would skip the one call that actually grants it (confirmed by +// experiment: a process that never calls ARCH_REQ_XCOMP_PERM still reads XCR0 bits +// 17/18 as set, then SIGILLs on the first real tile instruction). Returns true only +// when the CPU has AMX-TILE and AMX-BF16 and the kernel request succeeds; any "no" here +// is silent, not an error: dispatch simply does not pick AMX. +// +// Thread-safety and the once-ness come from the function-local static's initializer +// guarantee (C++ [stmt.dcl]): concurrent first callers block on the same guard variable +// and only one of them runs the body; every call after that is the "one predictable +// branch on a cached value" the dispatch call site needs. +inline bool amx_tile_bf16_available() { + static const bool available = [] { + if (!avx_runtime_flags.amx_tile_bf16_cpu) { + return false; + } + long rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, SVS_XFEATURE_XTILEDATA); + return rc == 0; + }(); + return available; +} +#else +inline bool amx_tile_bf16_available() { return false; } +#endif + } // namespace svs::detail diff --git a/include/svs/lib/preprocessor.h b/include/svs/lib/preprocessor.h index 8b1a8a85a..518180ae1 100644 --- a/include/svs/lib/preprocessor.h +++ b/include/svs/lib/preprocessor.h @@ -114,6 +114,17 @@ consteval bool is_one_or_zero(const char* ptr) { #define SVS_OMP 0 #endif +// Set uniformly by cmake/multi-arch.cmake on every consumer of the library, unlike +// SVS_AVX512_F/SVS_AMX_TILE_BF16 below which read the *current* TU's own -march flags: +// PackedOperand::pack_rounded_to_bf16 runs in TUs that never pass -mamx-tile themselves +// (gem.h, maxsim.h, tests), so it needs a build-wide answer to "does amx.cpp exist in +// this build at all" to decide whether filling the AMX tile layout is worth its cost. +#if defined(SVS_AMX_TILE_BF16_SUPPORTED) +#define SVS_AMX_COMPILED_IN 1 +#else +#define SVS_AMX_COMPILED_IN 0 +#endif + ///// ///// Intel(R) AVX extensions ///// @@ -170,6 +181,17 @@ inline constexpr bool have_avx512_avx2 = true; inline constexpr bool have_avx512_avx2 = true; #endif +// AMX tile load/store/config and the bf16 dot-product tile instruction. Both gate the +// same translation unit (amx.cpp is compiled with -mamx-tile -mamx-bf16 together, see +// cmake/multi-arch.cmake), so one guard covers both rather than two that could disagree. +#if defined(__AMX_TILE__) && defined(__AMX_BF16__) +#define SVS_AMX_TILE_BF16 1 +inline constexpr bool have_amx_tile_bf16 = true; +#else +#define SVS_AMX_TILE_BF16 0 +inline constexpr bool have_amx_tile_bf16 = false; +#endif + } // namespace svs::arch namespace svs { From d24f03bf27ba374cc48047ead0ee1019443fbe07 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 07:05:12 -0800 Subject: [PATCH 21/68] feat(distance): AMX tile kernels for InnerProductMatrix's bf16 A Adds the AMX tile-dot-product kernel for InnerProductMatrixImpl's table (build_packed) and column-max forms, selected only when AMX is available, A is BFloat16, and B was packed with PackedOperand::pack_rounded_to_bf16 (which now also fills a pair-interleaved transposed bf16 tile layout, built with ordinary scalar code so it needs no -mamx flags of its own, gated on the build-wide SVS_AMX_COMPILED_IN and the runtime availability check). Every other case (float32/Float16 A, or an unrounded B) forwards to the AVX-512 implementation, keeping one source of truth. The kernel processes A 16 rows and B 32 lanes (two n-tiles) at a time, with two independent fp32 accumulator tiles so their dpbf16ps chains overlap; A's final row-block (m not a multiple of 16) gets a second, one-off tile reconfigure, and A's final k-chunk (d not a multiple of 32) is copied through a zero-padded stack buffer so the kernel never reads past the caller's d columns. Tile numbers are literal integers throughout (0-4): gcc's tile intrinsics stringify their register argument, so a named constant would not assemble. _tile_loadd/_tile_stored carry no memory operand or clobber, so the compiler is otherwise free to reorder a preceding store (the tail copy) or a following read (emit's use of the stored tiles) across them; both call sites get an explicit compiler memory barrier. Co-Authored-By: Claude Fable 5.2 --- .../svs/core/distance/inner_product_matrix.h | 126 ++++++- include/svs/multi-arch/x86/amx.cpp | 335 ++++++++++++++++++ 2 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 include/svs/multi-arch/x86/amx.cpp diff --git a/include/svs/core/distance/inner_product_matrix.h b/include/svs/core/distance/inner_product_matrix.h index 65429cd1f..e38a76602 100644 --- a/include/svs/core/distance/inner_product_matrix.h +++ b/include/svs/core/distance/inner_product_matrix.h @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -78,16 +79,81 @@ class PackedOperand { buffer_[k * n + j] = static_cast(svs::round_to_bfloat16(bj[k])); } } +#if SVS_AMX_COMPILED_IN + // Only worth building when some TU in this binary can consume it (amx.cpp + // exists, SVS_AMX_COMPILED_IN) and the CPU/OS actually grant tile use; otherwise + // this would cost a few KiB and a scalar pass on every call for nothing. + if (svs::detail::amx_tile_bf16_available()) { + fill_tile_layout(b, n, d); + } else { + has_tile_layout_ = false; + } +#endif } size_t n() const { return n_; } size_t d() const { return d_; } const float* data() const { return buffer_.data(); } + /// True only right after `pack_rounded_to_bf16` filled the AMX tile layout below; + /// `pack` never sets this, since its values are not rounded and the AMX kernel only + /// ever computes . + bool has_tile_layout() const { return has_tile_layout_; } + + /// Pair-interleaved, transposed bf16 tile block for query-lane tile `nt` (16 lanes, + /// zero-padded past `n`) and d/2-pair tile `kt` (16 pairs, zero-padded past + /// ceil(d/2)): row `r`'s dword at lane `c` packs + /// (bf16(b[nt*16+c][2*(kt*16+r)]), bf16(b[nt*16+c][2*(kt*16+r)+1])) low/high, i.e. + /// exactly the `b` tile operand _tile_dpbf16ps expects (see maxsim.h's contract). + /// Valid only when `has_tile_layout()`; caller (amx.cpp) owns interpreting it. + const uint32_t* tile_block(size_t nt, size_t kt) const { + return tile_buffer_.data() + (nt * kp_tiles_ + kt) * kTileElements; + } + size_t n_tiles() const { return n_tiles_; } + size_t kp_tiles() const { return kp_tiles_; } + private: + static constexpr size_t kTileElements = 16 * 16; + + // Builds the tile-layout buffer described by tile_block() above. Zero-padding both + // the lane tail (n not a multiple of 16) and the pair tail (d odd, or not a multiple + // of 32) is safe here because B's padding only ever multiplies against A's real + // values and contributes 0; A's own out-of-bounds tail is handled separately, by the + // AMX kernel copying A's last partial k-chunk through a zero-padded stack buffer + // rather than reading past A's allocation. + void fill_tile_layout(const float* b, size_t n, size_t d) { + size_t kp = (d + 1) / 2; + n_tiles_ = (n + 15) / 16; + kp_tiles_ = (kp + 15) / 16; + tile_buffer_.assign(n_tiles_ * kp_tiles_ * kTileElements, uint32_t{0}); + for (size_t j = 0; j < n; ++j) { + const float* bj = b + j * d; + size_t nt = j / 16; + size_t lane = j % 16; + for (size_t k = 0; k < d; ++k) { + size_t kp_idx = k / 2; + size_t kt = kp_idx / 16; + size_t row = kp_idx % 16; + uint32_t bits = static_cast(svs::round_to_bfloat16(bj[k]).raw()); + uint32_t* dword = + &tile_buffer_[(nt * kp_tiles_ + kt) * kTileElements + row * 16 + lane]; + if (k % 2 == 0) { + *dword = (*dword & 0xFFFF0000u) | bits; + } else { + *dword = (*dword & 0x0000FFFFu) | (bits << 16); + } + } + } + has_tile_layout_ = true; + } + std::vector buffer_; size_t n_ = 0; size_t d_ = 0; + std::vector tile_buffer_; + size_t n_tiles_ = 0; + size_t kp_tiles_ = 0; + bool has_tile_layout_ = false; }; /// Per-ISA kernel family for a large row-major A (m x d) against B, packed once into a @@ -236,12 +302,47 @@ template <> struct InnerProductMatrixImpl { }; #endif +// Declared (full member set, matching every other level) only inside amx.cpp, the one +// TU compiled with -mamx-tile -mamx-bf16: SVS_AMX_TILE_BF16 reads *this TU's* compiler +// macros, same per-TU pattern as SVS_AVX512_F/SVS_AVX2 above, not the build-wide +// SVS_AMX_COMPILED_IN that PackedOperand reads. build_packed/column_max for float32 and +// Float16 A, and for a BFloat16 A whose B was not packed with the tile layout, forward +// to the AVX512 implementation (explicit forwarding in amx.cpp, one source of truth); +// only a BFloat16 A against a tile-packed B runs the real tile kernel. +SVS_VALIDATE_BOOL_ENV(SVS_AMX_TILE_BF16) +#if SVS_AMX_TILE_BF16 +template <> struct InnerProductMatrixImpl { + SVS_NOINLINE static void + build_packed(const float* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); + SVS_NOINLINE static void + column_max(const float* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + SVS_NOINLINE static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); +}; +#endif + #endif // defined(__x86_64__) #if defined(__x86_64__) extern template struct InnerProductMatrixImpl; extern template struct InnerProductMatrixImpl; #endif +// Gated on the build-wide SVS_AMX_COMPILED_IN, not __x86_64__ alone: when the compiler +// lacks -march=sapphirerapids, amx.cpp is never built (cmake/multi-arch.cmake), so no TU +// anywhere provides `template struct InnerProductMatrixImpl;`'s +// definition; an unconditional `extern template` here would then be an unresolved symbol +// at link time the moment anything calls InnerProductMatrix, not just when AMX runs. +#if SVS_AMX_COMPILED_IN +extern template struct InnerProductMatrixImpl; +#endif /// Public entry points for the InnerProductMatrixImpl kernel family: the only place that /// picks an ISA for this kernel. A caller that needs this kernel's output names the form @@ -347,6 +448,19 @@ class InnerProductMatrix { // writes this exhaustive switch zero times of its own. template static void dispatch(F&& f) { switch (select_isa()) { + case AVX_AVAILABILITY::AMX: + // select_isa() only returns AMX when SVS_AMX_COMPILED_IN (amx.cpp was + // built), so the symbol this case names exists only then; this case + // itself must stay unconditional so -Wswitch under -Werror still catches + // a 5th enumerator added later. Without SVS_AMX_COMPILED_IN this case is + // unreachable (select_isa() never returns it) and falls back to AVX512, + // never naming a specialization that cannot exist in this build. +#if SVS_AMX_COMPILED_IN + f(std::integral_constant{}); +#else + f(std::integral_constant{}); +#endif + return; case AVX_AVAILABILITY::AVX512: f(std::integral_constant{}); return; @@ -358,12 +472,22 @@ class InnerProductMatrix { return; } // Exhaustive switch over AVX_AVAILABILITY's enumerators, no default: -Wswitch - // under -Werror turns a 4th enumerator added later into a compile error here, + // under -Werror turns a 5th enumerator added later into a compile error here, // rather than this call silently keeping today's dispatch. __builtin_unreachable(); } + // AMX is checked first regardless of A's element type or B's packing: every + // InnerProductMatrixImpl member handles "not actually usable for this call" + // itself (forwarding to AVX512), so this one-branch-per-call choice never needs to + // know the call's operand types. Gated on SVS_AMX_COMPILED_IN so a build that never + // compiled amx.cpp never returns an enumerator dispatch() cannot safely name. static AVX_AVAILABILITY select_isa() { +#if SVS_AMX_COMPILED_IN + if (svs::detail::amx_tile_bf16_available()) { + return AVX_AVAILABILITY::AMX; + } +#endif if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { return AVX_AVAILABILITY::AVX512; } diff --git a/include/svs/multi-arch/x86/amx.cpp b/include/svs/multi-arch/x86/amx.cpp new file mode 100644 index 000000000..e0e0611e9 --- /dev/null +++ b/include/svs/multi-arch/x86/amx.cpp @@ -0,0 +1,335 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(__x86_64__) +#include "svs/core/distance/inner_product_matrix.h" + +#include + +#include +#include +#include + +namespace svs::distance { + +// The scalar-distance kernels (L2/IP/cosine) and QEmd/QChamfer/QChamferExactImpl dispatch +// through a runtime if-chain on svs::detail::avx_runtime_flags (euclidean.h, cosine.h, +// inner_product.h, gem.h), never through a switch or template argument naming +// AVX_AVAILABILITY::AMX; adding the AMX enumerator needs no new instantiation of any of +// them (checked: `git grep -n AVX_AVAILABILITY` finds no exhaustive switch or `AMX` +// branch to add outside InnerProductMatrix). AMX only applies to the InnerProductMatrix +// family below, whose public dispatch is the one exhaustive switch this change extends. + +namespace { + +// 64-byte tile-config struct the palette-1 AMX instructions read (Intel SDM Vol 1, +// 3.3.7): colsb/rows per tmm register 0-15, reserved bytes must stay zero. +struct TileConfig { + uint8_t palette_id = 0; + uint8_t start_row = 0; + uint8_t reserved[14] = {}; + uint16_t colsb[16] = {}; + uint8_t rows[16] = {}; +}; + +// Releases tile state on every exit path (normal return or exception): leaving tile +// state live costs later context switches and ordinary SSE/AVX code extra save/restore +// work, per Intel's optimization guidance for AMX. +struct TileGuard { + TileGuard() = default; + ~TileGuard() { _tile_release(); } + TileGuard(const TileGuard&) = delete; + TileGuard& operator=(const TileGuard&) = delete; +}; + +// tmm register assignment: tmm0 = A's current k-chunk (16 rows x 32 bf16, i.e. 16 +// pairs); tmm1/tmm2 = B's two n-lane-tiles for the pair-chunk (pair-interleaved +// transposed bf16, see PackedOperand::tile_block); tmm3/tmm4 = the matching fp32 +// accumulators. Two n-tiles in flight so their dpbf16ps chains are independent of each +// other (both read tmm0, but write distinct accumulators), per the task's "several +// accumulator tiles overlap" requirement. Named here for the tile-config array indexing +// below (ordinary C++, any expression is fine); every `_tile_*` call past this point +// uses the literal 0/1/2/3/4 directly, because gcc's tile intrinsics are macros that +// stringify their register argument (`#dst`), so a named constant used there would +// stringify to its identifier text, not a digit, and fail to assemble. +enum Tmm : int { kTmmA = 0, kTmmB0 = 1, kTmmB1 = 2, kTmmAcc0 = 3, kTmmAcc1 = 4 }; + +void configure_tiles(uint8_t a_acc_rows) { + TileConfig cfg; + cfg.palette_id = 1; + cfg.rows[kTmmA] = a_acc_rows; + cfg.colsb[kTmmA] = 64; + cfg.rows[kTmmAcc0] = a_acc_rows; + cfg.colsb[kTmmAcc0] = 64; + cfg.rows[kTmmAcc1] = a_acc_rows; + cfg.colsb[kTmmAcc1] = 64; + // B's tiles are always a full 16 (pair-)rows: PackedOperand::tile_block zero-pads + // every tile block to 16x16, so there is no partial-row case to configure for them. + cfg.rows[kTmmB0] = 16; + cfg.colsb[kTmmB0] = 64; + cfg.rows[kTmmB1] = 16; + cfg.colsb[kTmmB1] = 64; + _tile_loadconfig(&cfg); +} + +// Computes the dot products of up to 16 rows of A (starting at `a_row0`, already known +// not to read past `d` columns of any row via the tail copy below) against B's n-tiles +// `nt0` (and `nt1` when `has_nt1`) for every k-chunk, leaving the result in tmm3 (and +// tmm4). Tiles must already be configured by configure_tiles() for this call's row count. +// +// A's tail k-chunk (the last one, when d is not a multiple of 32) is copied through a +// zero-padded stack buffer rather than loaded directly: loading straight from `a_row0` +// would read up to 31 bf16 elements past column d-1 of the block's last real row, which +// is caller memory this kernel does not own past `d`. +void accumulate_group( + const svs::BFloat16* a_row0, + size_t rows, + size_t stride_bytes, + size_t full_k_chunks, + size_t k_tail, + const PackedOperand& b, + size_t nt0, + bool has_nt1, + size_t nt1 +) { + _tile_zero(3); + if (has_nt1) { + _tile_zero(4); + } + const auto* base = reinterpret_cast(a_row0); + for (size_t kt = 0; kt < full_k_chunks; ++kt) { + _tile_loadd(0, base + kt * 64, stride_bytes); + _tile_loadd(1, b.tile_block(nt0, kt), 64); + _tile_dpbf16ps(3, 0, 1); + if (has_nt1) { + _tile_loadd(2, b.tile_block(nt1, kt), 64); + _tile_dpbf16ps(4, 0, 2); + } + } + if (k_tail > 0) { + alignas(64) uint16_t tail[16][32] = {}; + for (size_t r = 0; r < rows; ++r) { + std::memcpy(tail[r], base + r * stride_bytes + full_k_chunks * 64, k_tail * 2); + } + // _tile_loadd's asm has no memory operand or clobber, so nothing otherwise tells + // the optimizer the memcpy above must finish before this reads `tail`: without + // this barrier the store can be sunk past the load, observed as a silent + // zero/garbage read on this compiler. + __asm__ volatile("" ::: "memory"); + _tile_loadd(0, tail, 64); + _tile_loadd(1, b.tile_block(nt0, full_k_chunks), 64); + _tile_dpbf16ps(3, 0, 1); + if (has_nt1) { + _tile_loadd(2, b.tile_block(nt1, full_k_chunks), 64); + _tile_dpbf16ps(4, 0, 2); + } + } +} + +// Shared row-block/n-tile-group driver for both output forms below: walks A 16 rows at a +// time (reconfiguring tiles once more only for a final row-block narrower than 16, never +// per row-block in the steady state) and B 32 lanes (two n-tiles) at a time, handing each +// block's two 16x16 fp32 results to `emit(row0, rows, nt0, has_nt1, nt1, c0, c1)`. +template +void for_each_block( + const svs::BFloat16* a, size_t m, size_t d, const PackedOperand& b, Emit&& emit +) { + size_t n_tiles = b.n_tiles(); + size_t full_k_chunks = d / 32; + size_t k_tail = d % 32; + size_t stride_bytes = d * 2; + + TileGuard guard; + configure_tiles(16); + bool configured_full = true; + + for (size_t i = 0; i < m; i += 16) { + size_t rows = std::min(16, m - i); + if (rows < 16 && configured_full) { + configure_tiles(static_cast(rows)); + configured_full = false; + } + const svs::BFloat16* row0 = a + i * d; + for (size_t nt0 = 0; nt0 < n_tiles; nt0 += 2) { + bool has_nt1 = nt0 + 1 < n_tiles; + size_t nt1 = has_nt1 ? nt0 + 1 : nt0; + alignas(64) float c0[16][16]; + alignas(64) float c1[16][16]; + accumulate_group( + row0, rows, stride_bytes, full_k_chunks, k_tail, b, nt0, has_nt1, nt1 + ); + _tile_stored(3, c0, 64); + if (has_nt1) { + _tile_stored(4, c1, 64); + } + // Same hazard as accumulate_group's tail load: _tile_stored's asm has no + // output memory operand, so without this barrier the optimizer is free to + // read c0/c1 in `emit` before the store above has actually run. + __asm__ volatile("" ::: "memory"); + emit(i, rows, nt0, has_nt1, nt1, c0, c1); + } + } +} + +} // namespace + +// AMX table kernel for a BFloat16 A against a tile-packed B: writes out[i*n+j] exactly +// like the generic/AVX-512 forms, with the same values (summed in a +// different order, hence InnerProductMatrix's tolerance-based tests, not a bit-exact +// one). +void amx_table_bf16(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + for_each_block( + a, + m, + b.d(), + b, + [&](size_t row0, + size_t rows, + size_t nt0, + bool has_nt1, + size_t nt1, + auto& c0, + auto& c1) { + size_t lanes0 = std::min(16, n - nt0 * 16); + for (size_t r = 0; r < rows; ++r) { + float* dst = out + (row0 + r) * n + nt0 * 16; + std::memcpy(dst, c0[r], lanes0 * sizeof(float)); + } + if (has_nt1) { + size_t lanes1 = std::min(16, n - nt1 * 16); + for (size_t r = 0; r < rows; ++r) { + float* dst = out + (row0 + r) * n + nt1 * 16; + std::memcpy(dst, c1[r], lanes1 * sizeof(float)); + } + } + } + ); +} + +// AMX column-max kernel for a BFloat16 A against a tile-packed B: folds each block's two +// 16x16 tiles into the running per-lane maximum with AVX-512 (the AMX level implies +// AVX-512F), reusing the same row-then-lane reduction shape as +// InnerProductMatrixImpl::column_max in avx512.cpp. +void amx_column_max_bf16( + const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out +) { + size_t n = b.n(); + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + for_each_block( + a, + m, + b.d(), + b, + [&](size_t /*row0*/, + size_t rows, + size_t nt0, + bool has_nt1, + size_t nt1, + auto& c0, + auto& c1) { + auto fold = [&](size_t nt, auto& c) { + size_t lanes = std::min(16, n - nt * 16); + alignas(64) float colmax[16] = {}; + for (size_t j = 0; j < 16; ++j) { + float best = -std::numeric_limits::infinity(); + for (size_t r = 0; r < rows; ++r) { + best = std::max(best, c[r][j]); + } + colmax[j] = best; + } + for (size_t j = 0; j < lanes; ++j) { + out[nt * 16 + j] = std::max(out[nt * 16 + j], colmax[j]); + } + }; + fold(nt0, c0); + if (has_nt1) { + fold(nt1, c1); + } + } + ); +} + +SVS_VALIDATE_BOOL_ENV(SVS_AMX_TILE_BF16) +#if SVS_AMX_TILE_BF16 + +void InnerProductMatrixImpl::build_packed( + const float* a, size_t m, const PackedOperand& b, float* out +) { + // Never the AMX precision case (A is float32, not bf16): one source of truth is the + // AVX-512 kernel. + InnerProductMatrixImpl::build_packed(a, m, b, out); +} + +void InnerProductMatrixImpl::build_packed( + const svs::Float16* a, size_t m, const PackedOperand& b, float* out +) { + InnerProductMatrixImpl::build_packed(a, m, b, out); +} + +void InnerProductMatrixImpl::build_packed( + const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out +) { + if (m == 0 || b.n() == 0) { + return; + } + if (!b.has_tile_layout()) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + return; + } + amx_table_bf16(a, m, b, out); +} + +void InnerProductMatrixImpl::build( + const float* a, size_t m, const float* b, size_t n, size_t d, float* out +) { + // `a` is always float32 here (InnerProductMatrix::compute's only entry point to this + // overload): the AMX precision case never applies, so this is a plain forward. + InnerProductMatrixImpl::build(a, m, b, n, d, out); +} + +void InnerProductMatrixImpl::column_max( + const float* a, size_t m, const PackedOperand& b, float* out +) { + InnerProductMatrixImpl::column_max(a, m, b, out); +} + +void InnerProductMatrixImpl::column_max( + const svs::Float16* a, size_t m, const PackedOperand& b, float* out +) { + InnerProductMatrixImpl::column_max(a, m, b, out); +} + +void InnerProductMatrixImpl::column_max( + const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out +) { + if (!b.has_tile_layout()) { + InnerProductMatrixImpl::column_max(a, m, b, out); + return; + } + amx_column_max_bf16(a, m, b, out); +} + +template struct InnerProductMatrixImpl; + +#endif // SVS_AMX_TILE_BF16 + +} // namespace svs::distance + +#endif From 5219fda7cae883ffc8376e5dd6ec0650fccd4f3a Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 07:05:25 -0800 Subject: [PATCH 22/68] test(distance): cover the AMX kernel over every tile tail shape Sweeps InnerProductMatrixImpl's build_packed and column_max against a double-precision reference and against AVX-512, over m/n/d shapes crossing the 16-row, 16-lane, and 32-element (16-pair) tile boundaries; checks float32/Float16 A and an unrounded B forward to AVX-512 bit for bit; checks the availability probe agrees across concurrent threads; and checks a thread that never itself requested AMX permission can still run the kernel after another thread already has. Skips when amx_tile_bf16_available() is false, so this is a no-op on hardware or a build without the level. Co-Authored-By: Claude Fable 5.2 --- .../core/distances/inner_product_matrix.cpp | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) diff --git a/tests/svs/core/distances/inner_product_matrix.cpp b/tests/svs/core/distances/inner_product_matrix.cpp index 20d33ba70..22584fc77 100644 --- a/tests/svs/core/distances/inner_product_matrix.cpp +++ b/tests/svs/core/distances/inner_product_matrix.cpp @@ -24,15 +24,18 @@ // catch2 #include "catch2/catch_approx.hpp" +#include "catch2/catch_message.hpp" #include "catch2/catch_test_macros.hpp" // stdlib #include +#include #include #include #include #include #include +#include #include namespace { @@ -246,6 +249,43 @@ void require_close_vec(const std::vector& got, const std::vector& } } +// AMX's tile grouping (16 pairs per k-chunk, two 16-lane tiles accumulated together) +// reorders the sum differently from both the generic k-order loop and AVX-512's 8-row +// register blocking; measured over every shape amx_ms/amx_ns/amx_ds below, that extra +// reordering alone needs the same ~2x-of-observed tolerance already used for the +// Float16/BFloat16 narrow-type comparisons above (check_column_max_against_generic), +// not REL_EPS/ABS_EPS's tighter float32 bound. +constexpr double AMX_REL_EPS = 2e-4; +constexpr double AMX_ABS_EPS = 1e-4; + +void require_close_loose( + const std::vector& got, const std::vector& expected +) { + CATCH_REQUIRE(got.size() == expected.size()); + for (size_t i = 0; i < got.size(); ++i) { + CATCH_REQUIRE( + static_cast(got[i]) == + Catch::Approx(expected[i]).epsilon(AMX_REL_EPS).margin(AMX_ABS_EPS) + ); + } +} + +void require_close_vec_loose( + const std::vector& got, const std::vector& expected +) { + CATCH_REQUIRE(got.size() == expected.size()); + for (size_t i = 0; i < got.size(); ++i) { + if (expected[i] == -std::numeric_limits::infinity()) { + CATCH_REQUIRE(got[i] == -std::numeric_limits::infinity()); + } else { + CATCH_REQUIRE( + static_cast(got[i]) == + Catch::Approx(expected[i]).epsilon(AMX_REL_EPS).margin(AMX_ABS_EPS) + ); + } + } +} + std::vector to_float16(const std::vector& a) { std::vector out(a.size()); for (size_t i = 0; i < a.size(); ++i) { @@ -292,6 +332,18 @@ std::vector promote_bfloat16(const std::vector& a) { return out; } +// Rounds to bf16 and back to float32, for building a reference against a B operand that +// was packed with `pack_rounded_to_bf16`: that kernel sees bf16(b), not b, so a reference +// comparing against the pre-rounding float would be wrong by bf16's own rounding error, +// not by summation order. +std::vector to_bfloat16_as_float(const std::vector& b) { + std::vector out(b.size()); + for (size_t i = 0; i < b.size(); ++i) { + out[i] = static_cast(svs::round_to_bfloat16(b[i])); + } + return out; +} + // Naive double-precision reference for the table form, out[i * n + j] = , over // an already-promoted `a` so a narrow-type A's post-conversion values are what the // reference uses (shares `reference`'s role above, generalized to a non-float32 `a`). @@ -717,3 +769,268 @@ CATCH_TEST_CASE( CATCH_REQUIRE(packed.data()[1] == -std::numeric_limits::infinity()); } } + +namespace { + +// Every tail an AMX tile hits: m crosses the 16-row tile boundary (0, 1, 15, 16, 17, 33, +// 67, 180), n crosses the 16-lane tile boundary (1, 15, 16, 17, 32, 33), d crosses the +// 32-element (16-pair) k-chunk boundary and parity (1, 2, 31, 32, 33, 64, 127, 128, 129). +const std::vector amx_ms{0, 1, 15, 16, 17, 33, 67, 180}; +const std::vector amx_ns{1, 15, 16, 17, 32, 33}; +const std::vector amx_ds{1, 2, 31, 32, 33, 64, 127, 128, 129}; + +} // namespace + +CATCH_TEST_CASE( + "InnerProductMatrixImpl build_packed matches a double-precision reference and " + "AVX-512, over every tail shape", + "[distance][inner_product_matrix][amx]" +) { + if (!svs::detail::amx_tile_bf16_available()) { + return; + } + unsigned seed = 8000; + for (size_t d : amx_ds) { + for (size_t m : amx_ms) { + for (size_t n : amx_ns) { + auto a = random_matrix(m, d, seed++); + auto a_bf16 = to_bfloat16(a); + auto b = random_matrix(n, d, seed++); + // The kernel computes : the reference must round b too, + // not compare against the pre-rounding float values. + auto b_rounded = to_bfloat16_as_float(b); + PackedOperand packed; + packed.pack_rounded_to_bf16(b.data(), n, d); + CATCH_REQUIRE(packed.has_tile_layout()); + + CATCH_INFO("m=" << m << " n=" << n << " d=" << d); + std::vector out_amx(m * n, -1.0F); + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, out_amx.data() + ); + require_close_loose( + out_amx, + reference_from_values(promote_bfloat16(a_bf16), m, b_rounded, n, d) + ); + + std::vector out_avx512(m * n, -2.0F); + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, out_avx512.data() + ); + for (size_t idx = 0; idx < out_amx.size(); ++idx) { + CATCH_REQUIRE( + static_cast(out_amx[idx]) == + Catch::Approx(static_cast(out_avx512[idx])) + .epsilon(2e-4) + .margin(1e-4) + ); + } + } + } + } +} + +CATCH_TEST_CASE( + "InnerProductMatrixImpl column_max matches a double-precision reference and " + "AVX-512, over every tail shape", + "[distance][inner_product_matrix][amx]" +) { + if (!svs::detail::amx_tile_bf16_available()) { + return; + } + unsigned seed = 9000; + for (size_t d : amx_ds) { + for (size_t m : amx_ms) { + for (size_t n : amx_ns) { + auto a = random_matrix(m, d, seed++); + auto a_bf16 = to_bfloat16(a); + auto b = random_matrix(n, d, seed++); + auto b_rounded = to_bfloat16_as_float(b); + PackedOperand packed; + packed.pack_rounded_to_bf16(b.data(), n, d); + CATCH_REQUIRE(packed.has_tile_layout()); + + std::vector out_amx(n, -3.0F); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, out_amx.data() + ); + require_close_vec_loose( + out_amx, + reference_column_max(promote_bfloat16(a_bf16), m, b_rounded, n, d) + ); + + std::vector out_avx512(n, -4.0F); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, out_avx512.data() + ); + for (size_t j = 0; j < n; ++j) { + if (out_avx512[j] == -std::numeric_limits::infinity()) { + CATCH_REQUIRE( + out_amx[j] == -std::numeric_limits::infinity() + ); + } else { + CATCH_REQUIRE( + static_cast(out_amx[j]) == + Catch::Approx(static_cast(out_avx512[j])) + .epsilon(2e-4) + .margin(1e-4) + ); + } + } + } + } + } +} + +CATCH_TEST_CASE( + "InnerProductMatrixImpl forwards float32 and Float16 A to AVX-512 bit for bit", + "[distance][inner_product_matrix][amx]" +) { + unsigned seed = 10000; + for (size_t d : {1UL, 7UL, 128UL, 129UL}) { + for (size_t m : {0UL, 1UL, 17UL, 33UL}) { + for (size_t n : {1UL, 17UL, 33UL}) { + auto a = random_matrix(m, d, seed++); + auto a16 = to_float16(a); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + + std::vector amx_f32(m * n, -1.0F), avx512_f32(m * n, -2.0F); + InnerProductMatrixImpl::build_packed( + a.data(), m, packed, amx_f32.data() + ); + InnerProductMatrixImpl::build_packed( + a.data(), m, packed, avx512_f32.data() + ); + CATCH_REQUIRE(amx_f32 == avx512_f32); + + std::vector amx_f16(m * n, -1.0F), avx512_f16(m * n, -2.0F); + InnerProductMatrixImpl::build_packed( + a16.data(), m, packed, amx_f16.data() + ); + InnerProductMatrixImpl::build_packed( + a16.data(), m, packed, avx512_f16.data() + ); + CATCH_REQUIRE(amx_f16 == avx512_f16); + + std::vector amx_cm(n, -1.0F), avx512_cm(n, -2.0F); + InnerProductMatrixImpl::column_max( + a.data(), m, packed, amx_cm.data() + ); + InnerProductMatrixImpl::column_max( + a.data(), m, packed, avx512_cm.data() + ); + CATCH_REQUIRE(amx_cm == avx512_cm); + + std::vector via_build_amx(m * n, -1.0F), + via_build_avx512(m * n, -2.0F); + InnerProductMatrixImpl::build( + a.data(), m, b.data(), n, d, via_build_amx.data() + ); + InnerProductMatrixImpl::build( + a.data(), m, b.data(), n, d, via_build_avx512.data() + ); + CATCH_REQUIRE(via_build_amx == via_build_avx512); + } + } + } +} + +CATCH_TEST_CASE( + "InnerProductMatrixImpl falls back to AVX-512 bit for bit when B was not " + "packed with the tile layout", + "[distance][inner_product_matrix][amx]" +) { + // PackedOperand::pack (not pack_rounded_to_bf16) never fills the tile layout, even + // when AMX is compiled in and available: it is the caller's signal "B is not + // rounded to bf16," and the AMX kernel only ever computes . + unsigned seed = 11000; + for (size_t d : {1UL, 7UL, 128UL, 129UL}) { + for (size_t m : {0UL, 1UL, 17UL, 33UL}) { + for (size_t n : {1UL, 17UL, 33UL}) { + auto a = random_matrix(m, d, seed++); + auto a_bf16 = to_bfloat16(a); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + CATCH_REQUIRE(!packed.has_tile_layout()); + + std::vector amx_out(m * n, -1.0F), avx512_out(m * n, -2.0F); + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, amx_out.data() + ); + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, avx512_out.data() + ); + CATCH_REQUIRE(amx_out == avx512_out); + + std::vector amx_cm(n, -1.0F), avx512_cm(n, -2.0F); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, amx_cm.data() + ); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, avx512_cm.data() + ); + CATCH_REQUIRE(amx_cm == avx512_cm); + } + } + } +} + +CATCH_TEST_CASE( + "svs::detail::amx_tile_bf16_available is consistent across concurrent threads", + "[distance][inner_product_matrix][amx]" +) { + constexpr size_t num_threads = 8; + std::vector threads; + std::array results{}; + for (size_t t = 0; t < num_threads; ++t) { + threads.emplace_back([&results, t] { + results[t] = svs::detail::amx_tile_bf16_available(); + }); + } + for (auto& th : threads) { + th.join(); + } + bool first = results[0]; + for (bool r : results) { + CATCH_REQUIRE(r == first); + } + CATCH_REQUIRE(first == svs::detail::amx_tile_bf16_available()); +} + +CATCH_TEST_CASE( + "A thread started before AMX permission is ever requested can still run the AMX " + "kernel", + "[distance][inner_product_matrix][amx]" +) { + if (!svs::detail::amx_tile_bf16_available()) { + return; + } + // This test only demonstrates the property if it runs before any other test case in + // this binary has already cached svs::detail::amx_tile_bf16_available()'s answer; + // Catch2 runs test cases in a fixed but unspecified order, so instead this spawns a + // thread that calls straight into the AMX kernel without itself calling + // amx_tile_bf16_available() first, which is the actual shape of the hazard: a worker + // thread in SVS's thread pool that never calls the detection function at all, relying + // only on whichever thread called it first (see fix_argument_decoded in gem.h). + size_t m = 17, n = 17, d = 70; + auto a = random_matrix(m, d, 12000); + auto a_bf16 = to_bfloat16(a); + auto b = random_matrix(n, d, 12001); + auto b_rounded = to_bfloat16_as_float(b); + PackedOperand packed; + packed.pack_rounded_to_bf16(b.data(), n, d); + + std::vector out(m * n, -1.0F); + std::thread worker([&] { + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, out.data() + ); + }); + worker.join(); + require_close_loose( + out, reference_from_values(promote_bfloat16(a_bf16), m, b_rounded, n, d) + ); +} From 0de173ebc2d04f8dab91e49541beb9db15ee29f3 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 07:18:23 -0800 Subject: [PATCH 23/68] feat(distance): route bfloat16-document MaxSim through the bf16/AMX kernel Templates MaxSim on its document element type (default float, so the non-template spelling keeps today's behaviour) instead of branching on it inside fix_argument. A document of the wrong element type now fails to compile via compute's requires-clause, and fix_argument packs the query rounded to bf16 only for MaxSim, so float/Float16 callers still pack and score unrounded, bit for bit as before. The Vamana re-rank extension now instantiates MaxSim, so a bfloat16 vector-set store automatically gets the bf16 (and AMX, where available) InnerProductMatrix::column_max path. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/distance/maxsim.h | 45 ++++++-- include/svs/extensions/vamana/maxsim.h | 10 +- tests/svs/core/distances/maxsim.cpp | 123 +++++++++++++++++++- tests/svs/index/vamana/maxsim_rerank.cpp | 140 ++++++++++++++++++++++- 4 files changed, 301 insertions(+), 17 deletions(-) diff --git a/include/svs/core/distance/maxsim.h b/include/svs/core/distance/maxsim.h index 19ebeaf4f..8290f6f5d 100644 --- a/include/svs/core/distance/maxsim.h +++ b/include/svs/core/distance/maxsim.h @@ -19,6 +19,7 @@ // svs #include "svs/concepts/vector_set.h" #include "svs/core/distance/inner_product_matrix.h" +#include "svs/lib/bfloat16.h" #include "svs/lib/exception.h" #include "svs/lib/float16.h" @@ -29,6 +30,7 @@ #include #include #include +#include #include namespace svs::distance { @@ -48,12 +50,31 @@ namespace svs::distance { /// `must_fix_argument` is set because `compute` is meaningless without a prior /// `fix_argument`. /// +/// `DocElement` (default `float`) is the element type of the documents this instance +/// scores: `float` and `svs::Float16` documents are packed unrounded, bit for bit as +/// before this type gained a template parameter; `svs::BFloat16` documents are packed +/// through `PackedOperand::pack_rounded_to_bf16`, which both matches the bf16 precision +/// those documents are already held at and is the one packing `InnerProductMatrix`'s AMX +/// level requires to route a bf16 A through the tile kernel (see inner_product_matrix.h). +/// Baking the document element type into the type, rather than branching on it inside +/// `fix_argument`, means a document of the wrong element type fails to compile (`compute`'s +/// `requires` clause below) instead of silently mismatching rounding at run time, and +/// means `fix_argument` packs the query exactly once regardless of `DocElement` rather than +/// packing twice to keep both a rounded and an unrounded copy on hand. The non-template +/// spelling `MaxSim` keeps meaning `MaxSim` for existing callers. +/// /// The query (fix_argument's argument) is a `svs::data::ContiguousVectorSet` of float32 /// tokens. The document (compute's second argument) is a `svs::data::ContiguousVectorSet` -/// of float32 or svs::Float16 tokens with the same dimensionality as the query; a -/// mismatch throws (checked once per document, which is negligible next to the kernel -/// it guards). An empty document scores -infinity, strictly worse than any real score. -class MaxSim { +/// of `DocElement` tokens with the same dimensionality as the query; a mismatch throws +/// (checked once per document, which is negligible next to the kernel it guards). An +/// empty document scores -infinity, strictly worse than any real score. +template class MaxSim { + static_assert( + std::same_as || std::same_as || + std::same_as, + "MaxSim's document element type must be float, svs::Float16 or svs::BFloat16" + ); + public: using compare = std::greater<>; @@ -69,7 +90,13 @@ class MaxSim { requires std::same_as void fix_argument(const S& query) { assert(query.cardinality() > 0); - packed_.pack(query.data(), query.cardinality(), query.dimensions()); + if constexpr (std::same_as) { + packed_.pack_rounded_to_bf16( + query.data(), query.cardinality(), query.dimensions() + ); + } else { + packed_.pack(query.data(), query.cardinality(), query.dimensions()); + } maxima_.resize(packed_.n()); } @@ -79,8 +106,7 @@ class MaxSim { /// (see QChamferExactQuery::compute in gem.h for the same shape). Non-const: writes /// into the member maxima buffer `fix_argument` sized. template - requires std::same_as || - std::same_as + requires std::same_as float compute(const Query& /*query*/, const D& document) { if (document.dimensions() != packed_.d()) { throw ANNEXCEPTION( @@ -109,6 +135,9 @@ class MaxSim { std::vector maxima_; }; -inline constexpr bool operator==(const MaxSim&, const MaxSim&) { return true; } +template +inline constexpr bool operator==(const MaxSim&, const MaxSim&) { + return true; +} } // namespace svs::distance diff --git a/include/svs/extensions/vamana/maxsim.h b/include/svs/extensions/vamana/maxsim.h index 4cb5742da..9e59ebc28 100644 --- a/include/svs/extensions/vamana/maxsim.h +++ b/include/svs/extensions/vamana/maxsim.h @@ -45,6 +45,8 @@ concept AcceptsVectorSetQuery = // Re-sort by MaxSim score (larger is better). The buffer's own comparator is fixed at // construction to the primary distance's direction, which may disagree with MaxSim's, so // sort a local copy and write the order back rather than reuse `search_buffer.sort()`. +// `compare` does not depend on MaxSim's document-element template parameter, so the +// default instantiation's comparator applies regardless of which one scored the buffer. template void resort_by_maxsim(SearchBuffer& search_buffer) { size_t valid = search_buffer.size(); using value_type = typename SearchBuffer::value_type; @@ -53,7 +55,7 @@ template void resort_by_maxsim(SearchBuffer& search_buff reordered.begin(), reordered.end(), [](const value_type& a, const value_type& b) { - return svs::distance::MaxSim::compare{}(a.distance(), b.distance()); + return svs::distance::MaxSim<>::compare{}(a.distance(), b.distance()); } ); for (size_t j = 0; j < valid; ++j) { @@ -116,7 +118,11 @@ template < // Packs the query once (must_fix_argument) and reuses that packing for every // candidate still in the pool: repacking per candidate would cost as much as // scoring them, which is exactly what PackedOperand/MaxSim::fix_argument avoids. - svs::distance::MaxSim maxsim{}; + // The document element type is the vector-set dataset's own element type, so a + // bfloat16 store automatically gets the bf16 (and AMX, where available) kernel path + // while float/Float16 stores keep today's unrounded behaviour. + using DocElement = typename Sets::const_value_type::element_type; + svs::distance::MaxSim maxsim{}; svs::distance::maybe_fix_argument(maxsim, query); size_t valid = search_buffer.size(); diff --git a/tests/svs/core/distances/maxsim.cpp b/tests/svs/core/distances/maxsim.cpp index 6804b3880..d091a9eb5 100644 --- a/tests/svs/core/distances/maxsim.cpp +++ b/tests/svs/core/distances/maxsim.cpp @@ -19,6 +19,7 @@ // svs #include "svs/concepts/distance.h" +#include "svs/lib/bfloat16.h" #include "svs/lib/float16.h" // catch2 @@ -66,6 +67,14 @@ TestVectorSet make_set(std::initializer_list> tokens template double promote(T x) { return static_cast(x); } +// Namespace-scope concept (rather than a function-local requires-expression) so a +// substitution failure inside MaxSim::compute's requires-clause, which +// mentions the enclosing class template's parameter, stays in the immediate context for +// SFINAE purposes. +template +concept CanComputeMaxSim = + requires(svs::distance::MaxSim& dist, const D& d) { dist.compute(0, d); }; + // Naive double-precision oracle: sum over query tokens of the max inner product to any // document token. template @@ -90,6 +99,47 @@ double maxsim_reference(const TestVectorSet& query, const TestVectorSet& d return total; } +// Same oracle as `maxsim_reference`, but both the query and the document values are +// first rounded to bfloat16 precision: matches what MaxSim actually +// computes, since fix_argument rounds the query (pack_rounded_to_bf16) and the document +// is already held at bf16 precision. +double maxsim_reference_bf16( + const TestVectorSet& query, const TestVectorSet& document +) { + if (document.cardinality() == 0) { + return -std::numeric_limits::infinity(); + } + double total = 0.0; + for (size_t i = 0; i < query.cardinality(); ++i) { + auto q_token = query.token(i); + double best = -std::numeric_limits::infinity(); + for (size_t j = 0; j < document.cardinality(); ++j) { + auto d_token = document.token(j); + double dot = 0.0; + for (size_t d = 0; d < query.dimensions(); ++d) { + double qv = promote(svs::round_to_bfloat16(q_token[d])); + double dv = promote(d_token[d]); + dot += qv * dv; + } + best = std::max(best, dot); + } + total += best; + } + return total; +} + +TestVectorSet +make_random_bf16_set(size_t cardinality, size_t dim, std::mt19937& rng) { + std::uniform_real_distribution dist(-1.0F, 1.0F); + TestVectorSet set; + set.dim = dim; + set.flat.resize(cardinality * dim); + for (auto& v : set.flat) { + v = svs::round_to_bfloat16(dist(rng)); + } + return set; +} + TestVectorSet make_random_set(size_t cardinality, size_t dim, std::mt19937& rng) { std::uniform_real_distribution dist(-1.0F, 1.0F); TestVectorSet set; @@ -105,7 +155,7 @@ TestVectorSet make_random_set(size_t cardinality, size_t dim, std::mt1993 // (the protocol every ShouldFix-opted-in distance goes through), then scores `document`. template float maxsim_score(const TestVectorSet& query, const D& document) { - svs::distance::MaxSim dist{}; + svs::distance::MaxSim dist{}; svs::distance::maybe_fix_argument(dist, query); return svs::distance::compute(dist, query, document); } @@ -116,9 +166,9 @@ constexpr double MAXSIM_EPSILON = 1e-4; CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { CATCH_SECTION("Trait members") { - CATCH_REQUIRE(std::is_same_v>); - CATCH_REQUIRE_FALSE(svs::distance::MaxSim::implicit_broadcast); - CATCH_REQUIRE(svs::distance::MaxSim::must_fix_argument); + CATCH_REQUIRE(std::is_same_v::compare, std::greater<>>); + CATCH_REQUIRE_FALSE(svs::distance::MaxSim<>::implicit_broadcast); + CATCH_REQUIRE(svs::distance::MaxSim<>::must_fix_argument); } CATCH_SECTION("Hand-computed tiny case") { @@ -207,4 +257,69 @@ CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { ); } } + + CATCH_SECTION("Hand-computed bfloat16 document case") { + auto query = make_set({{1.0F, 0.0F, 0.0F}, {0.0F, 1.0F, 0.0F}}); + auto document = make_set( + {{svs::BFloat16(1.0F), svs::BFloat16(0.0F), svs::BFloat16(0.0F)}, + {svs::BFloat16(0.0F), svs::BFloat16(0.0F), svs::BFloat16(1.0F)}} + ); + float score = maxsim_score(query, document); + // Both values are exactly representable in bfloat16, so rounding changes nothing + // here: token 0 of query matches document token 0 exactly (dot = 1); token 1 has + // dot 0 with both document tokens. + CATCH_REQUIRE(score == Catch::Approx(1.0F).epsilon(MAXSIM_EPSILON)); + } + + CATCH_SECTION("Empty bfloat16 document set gives -inf") { + auto query = make_set({{1.0F, 0.0F}}); + TestVectorSet document; + document.dim = 2; + float score = maxsim_score(query, document); + CATCH_REQUIRE(score == -std::numeric_limits::infinity()); + } + + CATCH_SECTION("Bfloat16 dimension mismatch throws") { + auto query = make_set({{1.0F, 0.0F, 0.0F}}); + auto document = + make_set({{svs::BFloat16(1.0F), svs::BFloat16(0.0F)}}); + svs::distance::MaxSim dist{}; + svs::distance::maybe_fix_argument(dist, query); + CATCH_REQUIRE_THROWS_AS( + svs::distance::compute(dist, query, document), svs::lib::ANNException + ); + } + + CATCH_SECTION("Bfloat16 document exact oracle on random sets, rounded query") { + std::mt19937 rng(0xB10A7); + std::vector cardinalities{1, 2, 3, 7, 9, 17}; + std::vector dims{128, 37}; + for (size_t dim : dims) { + for (size_t nq : cardinalities) { + for (size_t nd : cardinalities) { + auto query = make_random_set(nq, dim, rng); + auto document = make_random_bf16_set(nd, dim, rng); + double expected = maxsim_reference_bf16(query, document); + float actual = maxsim_score(query, document); + CATCH_REQUIRE( + static_cast(actual) == Catch::Approx(expected) + .epsilon(MAXSIM_EPSILON) + .margin(MAXSIM_EPSILON) + ); + } + } + } + } + + CATCH_SECTION("Document element type is checked at compile time") { + using FloatDoc = TestVectorSet; + using BF16Doc = TestVectorSet; + // A MaxSim accepts a float document and rejects a bfloat16 one: the + // document's element type is part of the functor's type, so a mismatch is a + // compile error (the `requires` clause on `compute`), not a run-time branch. + static_assert(CanComputeMaxSim); + static_assert(!CanComputeMaxSim); + static_assert(CanComputeMaxSim); + static_assert(!CanComputeMaxSim); + } } diff --git a/tests/svs/index/vamana/maxsim_rerank.cpp b/tests/svs/index/vamana/maxsim_rerank.cpp index 435702dca..0e5c4cc94 100644 --- a/tests/svs/index/vamana/maxsim_rerank.cpp +++ b/tests/svs/index/vamana/maxsim_rerank.cpp @@ -184,14 +184,14 @@ brute_force_maxsim(const Query& query, const svs::data::ConstVectorSetView maxsim{}; svs::distance::maybe_fix_argument(maxsim, query); for (uint32_t id = 0; id < docs.size(); ++id) { float score = svs::distance::compute(maxsim, query, docs.get_datum(id)); scored.emplace_back(id, score); } std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { - return svs::distance::MaxSim::compare{}(a.second, b.second); + return svs::distance::MaxSim<>::compare{}(a.second, b.second); }); return scored; } @@ -251,7 +251,7 @@ void check_rerank_pool( std::vector pool_ids; pool_ids.reserve(buffer.size()); - svs::distance::MaxSim maxsim{}; + svs::distance::MaxSim<> maxsim{}; svs::distance::maybe_fix_argument(maxsim, query); float previous_score = std::numeric_limits::infinity(); for (const auto& neighbor : buffer) { @@ -551,3 +551,137 @@ CATCH_TEST_CASE("WithVectorSets save and reload", "[vamana][maxsim_rerank]") { } } } + +namespace { + +// Rounds `corpus`'s float tokens to bfloat16 into a fresh owning `VectorSetData` +// with the same CSR offsets, so the bf16 store's documents are exactly what the float +// corpus's documents round to. +svs::data::VectorSetData round_tokens_to_bf16(const Corpus& corpus) { + size_t total_tokens = corpus.tokens.size() / DIM; + svs::data::SimpleData tokens(total_tokens, DIM); + std::vector row(DIM); + for (size_t i = 0; i < total_tokens; ++i) { + for (size_t d = 0; d < DIM; ++d) { + row[d] = svs::round_to_bfloat16(corpus.tokens[i * DIM + d]); + } + tokens.set_datum(i, row); + } + return svs::data::VectorSetData(std::move(tokens), corpus.offsets); +} + +// Brute-force oracle over the bfloat16 store itself (not the float corpus the store was +// rounded from): mirrors `brute_force_maxsim` but through `MaxSim`, so it +// scores exactly what the owning bf16 store holds. +template +std::vector> brute_force_maxsim_bf16( + const Query& query, const svs::data::VectorSetData& docs +) { + std::vector> scored; + scored.reserve(docs.size()); + svs::distance::MaxSim maxsim{}; + svs::distance::maybe_fix_argument(maxsim, query); + for (uint32_t id = 0; id < docs.size(); ++id) { + float score = svs::distance::compute(maxsim, query, docs.get_datum(id)); + scored.emplace_back(id, score); + } + std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { + return svs::distance::MaxSim<>::compare{}(a.second, b.second); + }); + return scored; +} + +} // namespace + +// A document store whose element type is svs::BFloat16: the re-rank extension must pick +// MaxSim for it (svs/extensions/vamana/maxsim.h selects on +// `Sets::const_value_type::element_type`), reaching the bf16 (and AMX, where available) +// InnerProductMatrix::column_max overload rather than the float one. +CATCH_TEST_CASE( + "MaxSim re-rank over an owning bfloat16 vector-set store", "[vamana][maxsim_rerank]" +) { + using Dist = MeanNegativeInnerProduct; + using Primary = svs::data::SimpleData; + using Sets = svs::data::VectorSetData; + using Data = svs::data::WithVectorSets; + + const size_t n_docs = 80; + auto corpus = make_corpus(n_docs, 0x5EED5EED); + auto query_corpus = make_corpus(5, 0xFACE5EED); + + auto data = Data(corpus.primary, round_tokens_to_bf16(corpus)); + + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + + auto entry_point_pool = svs::threads::DefaultThreadPool(1); + auto entry_point = + svs::lib::narrow(svs::index::vamana::extensions::compute_entry_point( + corpus.primary, entry_point_pool + )); + + auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + graph, + data, + Dist{}, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(data) + ); + builder.construct(1.0F, entry_point); + builder.construct(build_parameters.alpha, entry_point); + + auto pool = svs::threads::DefaultThreadPool(1); + auto index = svs::index::vamana::VamanaIndex( + std::move(graph), std::move(data), entry_point, Dist{}, std::move(pool) + ); + + auto queries = query_corpus.vector_sets(); + const size_t capacity = n_docs; + const size_t num_neighbors = 10; + auto sp = search_params(capacity, capacity); + auto scratch = index.scratchspace(sp); + + for (size_t q = 0; q < queries.size(); ++q) { + auto query = queries.get_datum(q); + index.search(query, scratch); + const auto& buffer = scratch.buffer; + + svs::distance::MaxSim maxsim{}; + svs::distance::maybe_fix_argument(maxsim, query); + float previous_score = std::numeric_limits::infinity(); + std::vector pool_ids; + pool_ids.reserve(buffer.size()); + for (const auto& neighbor : buffer) { + float expected = svs::distance::compute( + maxsim, query, index.view_data().get_vector_set(neighbor.id()) + ); + CATCH_REQUIRE( + static_cast(neighbor.distance()) == + Catch::Approx(expected) + .epsilon(MAXSIM_RERANK_EPSILON) + .margin(MAXSIM_RERANK_EPSILON) + ); + CATCH_REQUIRE(neighbor.distance() <= previous_score); + previous_score = neighbor.distance(); + pool_ids.push_back(neighbor.id()); + } + + std::set pool_set(pool_ids.begin(), pool_ids.end()); + auto oracle = brute_force_maxsim_bf16(query, index.view_data().vector_sets()); + std::vector restricted_oracle; + for (const auto& scored : oracle) { + if (pool_set.contains(scored.first)) { + restricted_oracle.push_back(scored.first); + if (restricted_oracle.size() == num_neighbors) { + break; + } + } + } + for (size_t k = 0; k < num_neighbors; ++k) { + CATCH_REQUIRE(buffer[k].id() == restricted_oracle[k]); + } + } +} From 0c5c189276d40085a96ca001cac150ad10c0508e Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 07:22:40 -0800 Subject: [PATCH 24/68] test(distance): guard-page memory-safety test for the tile kernels' tails Adds a mmap/mprotect-based test that places A so it ends exactly at a PROT_NONE page (catches an overrun), places `out` the same way, and places A at the start of a mapping with a leading PROT_NONE page (catches an underrun), then runs both the table and column-max forms over every tail shape. Covers the AMX bfloat16 tile kernel plus, where available, the AVX-512 and AVX2 float/Float16/bfloat16 kernels. POSIX-only, guarded so the file still compiles on a non-POSIX target; no-ops when the relevant ISA is unavailable. Co-Authored-By: Claude Fable 5.2 --- .../core/distances/inner_product_matrix.cpp | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/tests/svs/core/distances/inner_product_matrix.cpp b/tests/svs/core/distances/inner_product_matrix.cpp index 22584fc77..624ffa5ae 100644 --- a/tests/svs/core/distances/inner_product_matrix.cpp +++ b/tests/svs/core/distances/inner_product_matrix.cpp @@ -31,13 +31,25 @@ #include #include #include +#include #include #include +#include #include #include #include #include +// POSIX, guarded: the guard-page test below needs mmap/mprotect/munmap, which this file +// must still compile without on a non-POSIX target. +#if defined(__unix__) || defined(__APPLE__) +#define SVS_TEST_HAS_MMAP 1 +#include +#include +#else +#define SVS_TEST_HAS_MMAP 0 +#endif + namespace { using svs::distance::AVX_AVAILABILITY; @@ -1034,3 +1046,208 @@ CATCH_TEST_CASE( out, reference_from_values(promote_bfloat16(a_bf16), m, b_rounded, n, d) ); } + +#if SVS_TEST_HAS_MMAP + +namespace { + +// Owns one `mmap` region sized to guard exactly `usable_bytes` of real storage against +// one neighbouring `PROT_NONE` page: `end_guarded` places the usable region so its last +// byte is the last byte before the guard page (catches an overrun), `start_guarded` +// places it at the first byte after a leading guard page (catches an underrun). A +// kernel that reads or writes one byte past what the test tells it to touch faults +// (SIGSEGV), which kills the test binary; that crash, not a Catch2 assertion, is this +// test's failure signal. +class GuardedMapping { + public: + static GuardedMapping end_guarded(size_t usable_bytes) { + size_t page = page_size(); + size_t usable_pages = (usable_bytes + page - 1) / page; + size_t total = (usable_pages + 1) * page; + void* base = map(total); + guard(static_cast(base) + usable_pages * page, page); + void* ptr = static_cast(base) + usable_pages * page - usable_bytes; + return GuardedMapping(base, total, ptr); + } + + static GuardedMapping start_guarded(size_t usable_bytes) { + size_t page = page_size(); + size_t usable_pages = (usable_bytes + page - 1) / page; + size_t total = (usable_pages + 1) * page; + void* base = map(total); + guard(base, page); + void* ptr = static_cast(base) + page; + return GuardedMapping(base, total, ptr); + } + + GuardedMapping(const GuardedMapping&) = delete; + GuardedMapping& operator=(const GuardedMapping&) = delete; + GuardedMapping(GuardedMapping&& other) noexcept + : base_{other.base_} + , total_{other.total_} + , ptr_{other.ptr_} { + other.base_ = nullptr; + } + GuardedMapping& operator=(GuardedMapping&&) = delete; + + ~GuardedMapping() { + if (base_ != nullptr) { + munmap(base_, total_); + } + } + + void* data() const { return ptr_; } + + private: + GuardedMapping(void* base, size_t total, void* ptr) + : base_{base} + , total_{total} + , ptr_{ptr} {} + + static size_t page_size() { return static_cast(sysconf(_SC_PAGESIZE)); } + + static void* map(size_t total) { + void* base = mmap( + nullptr, total, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0 + ); + CATCH_REQUIRE(base != MAP_FAILED); + return base; + } + + static void guard(void* addr, size_t len) { + CATCH_REQUIRE(mprotect(addr, len, PROT_NONE) == 0); + } + + void* base_; + size_t total_; + void* ptr_; +}; + +// Every tail shape the table and column-max kernels' SIMD/tile paths mask: m and n each +// cross the 16-wide tile/lane boundary, d crosses the 32-element (16-pair) AMX k-chunk +// boundary and both parities, matching the AMX tail sweep above but written out +// explicitly per the task's shape list rather than reusing `amx_ms`/`amx_ns`/`amx_ds`. +const std::vector guard_ms{1, 15, 16, 17, 33}; +const std::vector guard_ns{1, 15, 16, 17, 33}; +const std::vector guard_ds{1, 2, 31, 32, 33, 127, 128, 129}; + +// Runs both kernel forms (table and column-max) for one (level, element type) pair over +// every (m, n, d) in the guard sweep, with A and `out` each placed, in turn, so a +// one-element overrun or underrun would fault. `kernel` dispatches to +// InnerProductMatrixImpl::build_packed/column_max for whichever `T` the caller +// closed over; `pack_b` chooses `PackedOperand::pack` vs. `pack_rounded_to_bf16` the same +// way MaxSim/InnerProductMatrix's own callers do. +template +void run_guarded_sweep( + bool pack_b_rounded, BuildPacked build_packed, ColumnMax column_max +) { + for (size_t d : guard_ds) { + for (size_t m : guard_ms) { + for (size_t n : guard_ns) { + auto a_src = + random_matrix(m, d, static_cast(m * 997 + n * 31 + d)); + std::vector a_host(m * d); + for (size_t i = 0; i < a_host.size(); ++i) { + a_host[i] = T(a_src[i]); + } + auto b = random_matrix(n, d, static_cast(d * 997 + m * 31 + n)); + PackedOperand packed; + if (pack_b_rounded) { + packed.pack_rounded_to_bf16(b.data(), n, d); + } else { + packed.pack(b.data(), n, d); + } + + // (1) A ends exactly at a guard page: an overrun past A's last element + // faults. + { + GuardedMapping guard = GuardedMapping::end_guarded(m * d * sizeof(T)); + std::memcpy(guard.data(), a_host.data(), m * d * sizeof(T)); + const T* a = static_cast(guard.data()); + std::vector table_out(m * n, 0.0F); + build_packed(a, m, packed, table_out.data()); + std::vector cm_out(n, 0.0F); + column_max(a, m, packed, cm_out.data()); + } + + // (2) `out` ends exactly at a guard page, for both forms. + { + GuardedMapping guard = + GuardedMapping::end_guarded(m * n * sizeof(float)); + build_packed( + a_host.data(), m, packed, static_cast(guard.data()) + ); + } + { + GuardedMapping guard = GuardedMapping::end_guarded(n * sizeof(float)); + column_max(a_host.data(), m, packed, static_cast(guard.data())); + } + + // (3) A starts at the first byte after a guard page: an underrun before + // A's first element faults. + { + GuardedMapping guard = GuardedMapping::start_guarded(m * d * sizeof(T)); + std::memcpy(guard.data(), a_host.data(), m * d * sizeof(T)); + const T* a = static_cast(guard.data()); + std::vector table_out(m * n, 0.0F); + build_packed(a, m, packed, table_out.data()); + std::vector cm_out(n, 0.0F); + column_max(a, m, packed, cm_out.data()); + } + } + } + } +} + +template void run_guarded_level() { + run_guarded_sweep( + std::same_as, + [](const T* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + }, + [](const T* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::column_max(a, m, b, out); + } + ); +} + +} // namespace + +// The AMX tile kernels (amx_table_bf16 / amx_column_max_bf16, reached only for a bfloat16 +// A against a tile-packed B) copy A's tail through a zero-padded stack buffer precisely +// to avoid reading past A's own allocation (see PackedOperand::fill_tile_layout's doc +// comment); this test is the memory-safety check for that claim, and for the AVX-512/AVX2 +// masked-tail kernels' equivalent claim. A violation is a SIGSEGV that kills this test +// binary, which is the failure signal, not a CATCH_REQUIRE. +CATCH_TEST_CASE( + "InnerProductMatrixImpl kernels never read or write outside A or out, including " + "their masked tails", + "[distance][inner_product_matrix][amx]" +) { + CATCH_SECTION("AMX bfloat16 tile kernel") { + if (!svs::detail::amx_tile_bf16_available()) { + return; + } + run_guarded_level(); + } + + CATCH_SECTION("AVX-512 kernels") { + if (!svs::detail::avx_runtime_flags.is_avx512f_supported()) { + return; + } + run_guarded_level(); + run_guarded_level(); + run_guarded_level(); + } + + CATCH_SECTION("AVX2 kernels") { + if (!svs::detail::avx_runtime_flags.is_avx2_supported()) { + return; + } + run_guarded_level(); + run_guarded_level(); + run_guarded_level(); + } +} + +#endif // SVS_TEST_HAS_MMAP From b3b9a45f7343a5995aea337178af5c7bbef83db0 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 07:35:51 -0800 Subject: [PATCH 25/68] fix(distance): give every AMX tile memory access a real dependency gcc 11's _tile_loadd/_tile_stored macros expand to inline asm with no memory operand and no clobber, so the optimizer believes they neither read nor write memory: a plain C++ store just before a load, or a plain C++ read just after a store, through the same address had no ordering guarantee and could be reordered around the tile instruction. Two bare `__asm__ volatile("" ::: "memory")` barriers papered over this; a third spot (configure_tiles' struct writes before _tile_loadconfig) had no barrier at all and was correct only by accident of how this compiler happens to schedule it today. Replaces all three with named wrappers, one per literal tile number actually used (0-4), built from extended inline asm that names the memory each operation touches: a real "m"/"=m" operand covering the full extent where it is a compile-time constant (the 64-byte tile config; B's always-16x64-byte tile blocks; the local zero-padded tail buffer; the 16x16 fp32 accumulator stores), or an input operand plus a "memory" clobber where the extent is runtime-only (A's direct strided load, whose row pitch depends on the caller's `d`). Verified by a negative control: compiling this file without the wrappers (bare intrinsics only, the reviewer's stripped copy) at -O2/-O3 with and without -flto reproduces the hazard (garbage column maxima); the same scenario against this commit's amx.cpp, built the same four ways, matches a double-precision reference at all four. Co-Authored-By: Claude Fable 5.2 --- include/svs/multi-arch/x86/amx.cpp | 106 ++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 18 deletions(-) diff --git a/include/svs/multi-arch/x86/amx.cpp b/include/svs/multi-arch/x86/amx.cpp index e0e0611e9..1158154a9 100644 --- a/include/svs/multi-arch/x86/amx.cpp +++ b/include/svs/multi-arch/x86/amx.cpp @@ -67,6 +67,85 @@ struct TileGuard { // stringify to its identifier text, not a digit, and fail to assemble. enum Tmm : int { kTmmA = 0, kTmmB0 = 1, kTmmB1 = 2, kTmmAcc0 = 3, kTmmAcc1 = 4 }; +// gcc 11's _tile_loadd/_tile_stored/_tile_loadconfig macros expand to inline asm with no +// memory operand and no clobber, so the optimizer believes they neither read nor write +// memory: a plain C++ store just before a load, or a plain C++ read just after a store, +// through the same address has no ordering guarantee and can be reordered around the +// tile instruction (sunk store -> stale/garbage read; hoisted read -> stale/garbage +// value), independent of any `volatile` on the asm itself (that only orders the asm +// relative to *other* volatile operations, not to ordinary memory accesses). Every tile +// memory access in this file goes through one of the wrappers below instead of the bare +// intrinsic, so the dependency is a real operand or clobber the compiler understands +// rather than an accident of how today's code generation happens to schedule things. +// `_tile_zero`/`_tile_dpbf16ps`/`_tile_release` touch only tile *registers*, never +// memory, so they keep using the compiler's intrinsics unwrapped. + +// Loads the 64-byte tile configuration from `cfg`, naming the whole struct as the +// input memory this reads (not just a pointer-sized slice of it, which is what gcc's own +// _tile_loadconfig macro's cast effectively does): every `cfg.rows[]`/`cfg.colsb[]` store +// before this call is now visibly a dependency, not an accident of scheduling. +inline void tile_loadconfig_safe(const TileConfig& cfg) { + __asm__ volatile("ldtilecfg\t%0" ::"m"(cfg)); +} + +// One generated function per literal tile number actually used: the mnemonic's register +// is spelled into the asm text (`#tmm`), which gcc's own tile intrinsics require to be a +// literal token, so a runtime tile-number parameter is not an option. Kept local to this +// file: the macro only generates definitions used a few lines below it, nothing external +// depends on it (include/svs/multi-arch/AGENTS.md's rule is about a macro whose defining +// header is absent from a TU that uses it, which does not apply to a macro defined and +// consumed in the same file). +// +// _strided takes a runtime stride (A's row pitch, which varies with `d`) and so cannot +// name a compile-time memory extent; its "memory" clobber is a full barrier instead, +// forcing every pending store to complete before the load and every cached value to be +// re-read after it. +#define SVS_AMX_DEFINE_TILE_LOADD_STRIDED(tmm) \ + inline void tile_loadd_tmm##tmm##_strided(const void* base, long stride) { \ + __asm__ volatile("{tileloadd\t(%0,%1,1), %%tmm" #tmm "|tileloadd\t%%tmm" #tmm \ + ", [%0+%1*1]}" ::"r"(base), \ + "r"(stride) \ + : "memory"); \ + } + +// _fixed is for a source whose extent is a compile-time constant 1024 bytes (B's tile +// blocks, always a full 16x64-byte tile; the local zero-padded tail buffer, likewise +// 16x64 bytes): the dummy "m" operand (unused in the asm text) names that exact memory as +// read, so the compiler orders this correctly against its last write without needing a +// blanket barrier. +#define SVS_AMX_DEFINE_TILE_LOADD_FIXED(tmm) \ + inline void tile_loadd_tmm##tmm##_fixed(const void* base) { \ + __asm__ volatile( \ + "{tileloadd\t(%0,%1,1), %%tmm" #tmm "|tileloadd\t%%tmm" #tmm \ + ", [%0+%1*1]}" ::"r"(base), \ + "r"(64L), \ + "m"(*reinterpret_cast(base)) \ + ); \ + } + +// Stores are always to a caller-owned 16x16 fp32 buffer (1024 bytes, compile-time +// constant): the dummy "=m" output operand tells the compiler this asm writes that +// memory, so a later plain read of the same buffer is ordered after the store instead of +// possibly being hoisted before it. +#define SVS_AMX_DEFINE_TILE_STORED_FIXED(tmm) \ + inline void tile_stored_tmm##tmm##_fixed(void* base) { \ + __asm__ volatile("{tilestored\t%%tmm" #tmm \ + ", (%1,%2,1)|tilestored\t[%1+%2*1], %%tmm" #tmm "}" \ + : "=m"(*reinterpret_cast(base)) \ + : "r"(base), "r"(64L)); \ + } + +SVS_AMX_DEFINE_TILE_LOADD_STRIDED(0) +SVS_AMX_DEFINE_TILE_LOADD_FIXED(0) +SVS_AMX_DEFINE_TILE_LOADD_FIXED(1) +SVS_AMX_DEFINE_TILE_LOADD_FIXED(2) +SVS_AMX_DEFINE_TILE_STORED_FIXED(3) +SVS_AMX_DEFINE_TILE_STORED_FIXED(4) + +#undef SVS_AMX_DEFINE_TILE_LOADD_STRIDED +#undef SVS_AMX_DEFINE_TILE_LOADD_FIXED +#undef SVS_AMX_DEFINE_TILE_STORED_FIXED + void configure_tiles(uint8_t a_acc_rows) { TileConfig cfg; cfg.palette_id = 1; @@ -82,7 +161,7 @@ void configure_tiles(uint8_t a_acc_rows) { cfg.colsb[kTmmB0] = 64; cfg.rows[kTmmB1] = 16; cfg.colsb[kTmmB1] = 64; - _tile_loadconfig(&cfg); + tile_loadconfig_safe(cfg); } // Computes the dot products of up to 16 rows of A (starting at `a_row0`, already known @@ -111,11 +190,11 @@ void accumulate_group( } const auto* base = reinterpret_cast(a_row0); for (size_t kt = 0; kt < full_k_chunks; ++kt) { - _tile_loadd(0, base + kt * 64, stride_bytes); - _tile_loadd(1, b.tile_block(nt0, kt), 64); + tile_loadd_tmm0_strided(base + kt * 64, static_cast(stride_bytes)); + tile_loadd_tmm1_fixed(b.tile_block(nt0, kt)); _tile_dpbf16ps(3, 0, 1); if (has_nt1) { - _tile_loadd(2, b.tile_block(nt1, kt), 64); + tile_loadd_tmm2_fixed(b.tile_block(nt1, kt)); _tile_dpbf16ps(4, 0, 2); } } @@ -124,16 +203,11 @@ void accumulate_group( for (size_t r = 0; r < rows; ++r) { std::memcpy(tail[r], base + r * stride_bytes + full_k_chunks * 64, k_tail * 2); } - // _tile_loadd's asm has no memory operand or clobber, so nothing otherwise tells - // the optimizer the memcpy above must finish before this reads `tail`: without - // this barrier the store can be sunk past the load, observed as a silent - // zero/garbage read on this compiler. - __asm__ volatile("" ::: "memory"); - _tile_loadd(0, tail, 64); - _tile_loadd(1, b.tile_block(nt0, full_k_chunks), 64); + tile_loadd_tmm0_fixed(tail); + tile_loadd_tmm1_fixed(b.tile_block(nt0, full_k_chunks)); _tile_dpbf16ps(3, 0, 1); if (has_nt1) { - _tile_loadd(2, b.tile_block(nt1, full_k_chunks), 64); + tile_loadd_tmm2_fixed(b.tile_block(nt1, full_k_chunks)); _tile_dpbf16ps(4, 0, 2); } } @@ -171,14 +245,10 @@ void for_each_block( accumulate_group( row0, rows, stride_bytes, full_k_chunks, k_tail, b, nt0, has_nt1, nt1 ); - _tile_stored(3, c0, 64); + tile_stored_tmm3_fixed(c0); if (has_nt1) { - _tile_stored(4, c1, 64); + tile_stored_tmm4_fixed(c1); } - // Same hazard as accumulate_group's tail load: _tile_stored's asm has no - // output memory operand, so without this barrier the optimizer is free to - // read c0/c1 in `emit` before the store above has actually run. - __asm__ volatile("" ::: "memory"); emit(i, rows, nt0, has_nt1, nt1, c0, c1); } } From 2b3e02d5cbdc62754dbb8763cdd4a7e7230d8421 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 07:37:57 -0800 Subject: [PATCH 26/68] fix(distance): compile amx.cpp at the AVX-512 level's flags, not sapphirerapids -march=sapphirerapids also enables AVX512-FP16, AVX512-BF16, AVX-VNNI and other extensions the runtime probe never tests; the compiler was free to use them auto-vectorizing ordinary code in that file, and a hypervisor can expose AMX without them. Extends the SVS_X86 tuple format with an optional 4th+ field of extra compiler options (every existing tuple unchanged: no 4th field, same -march/-mtune as before), and uses it to compile amx.cpp with exactly -march=cascadelake -mtune=cascadelake -mamx-tile -mamx-bf16, matching the avx512 tuple's instruction set plus the two AMX extensions. The object target name now keys off the tuple's name field when extra options are present, since amx and avx512 now share the same `arch` and would otherwise collide on `cascadelake_obj`. The compiler check tests the two AMX flags directly rather than the (now unused) -march=sapphirerapids. amx_tile_bf16_available() now also requires avx_runtime_flags.is_avx512f_supported(): the AMX level's code is compiled at that instruction set and its non-tile paths forward to the AVX-512 kernels directly, so AMX is only meaningful where AVX-512 already is. Verified with objdump that the amx object's instructions are plain AVX-512F/scalar: no vaddph/vcvtph2psx/vpdpbusd or other AVX512-FP16/ AVX-VNNI-only mnemonics appear. Co-Authored-By: Claude Fable 5.2 --- cmake/multi-arch.cmake | 38 +++++++++++++++++++++++++-------- include/svs/lib/avx_detection.h | 9 +++++++- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/cmake/multi-arch.cmake b/cmake/multi-arch.cmake index c10e90dc3..06422f33e 100644 --- a/cmake/multi-arch.cmake +++ b/cmake/multi-arch.cmake @@ -13,34 +13,54 @@ # limitations under the License. set(SVS_X86_SRC_DIR "${PROJECT_SOURCE_DIR}/include/svs/multi-arch/x86") +# Tuple format: source,name,arch[,extra_option,extra_option,...]. Fields beyond the +# first three are optional and additive: an existing tuple (no 4th+ field) still gets +# exactly -march=${arch} -mtune=${arch}, unchanged; a tuple that adds fields gets those +# two flags plus every extra option, appended verbatim. amx.cpp uses this to compile at +# exactly the AVX-512 level's instruction set (arch=cascadelake, matching the avx512 +# tuple below) plus the two AMX extensions, rather than at -march=sapphirerapids: that +# also enables AVX512-FP16/BF16/AVX-VNNI/etc, which the runtime probe +# (amx_tile_bf16_available()) never tests and a hypervisor could expose AMX without, so +# the compiler must not be free to use them when auto-vectorizing ordinary code in that +# file. set(SVS_X86 "${SVS_X86_SRC_DIR}/avx2.cpp,avx2,haswell" "${SVS_X86_SRC_DIR}/avx512.cpp,avx512,cascadelake" ) -# AMX (tile load/store/config plus the bf16 dot-product tile instruction) is optional: -# -march=sapphirerapids is the flag amx.cpp is built with below, same one-codename-per-ISA -# pattern as haswell/cascadelake above, and the first compiler/-march combination that -# ships it; a compiler that rejects it (too old, or a non-x86 cross build) must still -# produce a working library with the level compiled out, so amx.cpp is added to SVS_X86 -# only when this check passes, and SVS_AMX_TILE_BF16_SUPPORTED (read by +# AMX (tile load/store/config plus the bf16 dot-product tile instruction) is optional: a +# compiler that rejects -mamx-tile/-mamx-bf16 (too old, or a non-x86 cross build) must +# still produce a working library with the level compiled out, so amx.cpp is added to +# SVS_X86 only when this check passes, and SVS_AMX_TILE_BF16_SUPPORTED (read by # include/svs/lib/preprocessor.h) is defined only then. include(CheckCXXCompilerFlag) -check_cxx_compiler_flag("-march=sapphirerapids" SVS_COMPILER_SUPPORTS_AMX) +check_cxx_compiler_flag("-mamx-tile -mamx-bf16" SVS_COMPILER_SUPPORTS_AMX) if(SVS_COMPILER_SUPPORTS_AMX) - list(APPEND SVS_X86 "${SVS_X86_SRC_DIR}/amx.cpp,amx,sapphirerapids") + list(APPEND SVS_X86 "${SVS_X86_SRC_DIR}/amx.cpp,amx,cascadelake,-mamx-tile,-mamx-bf16") endif() set(SVS_X86_OBJECT_FILES) foreach(x86_info IN LISTS SVS_X86) string(REPLACE "," ";" x86_info "${x86_info}") + list(LENGTH x86_info x86_info_len) list(GET x86_info 0 src) list(GET x86_info 1 avx) list(GET x86_info 2 arch) set(lib_name "svs_x86_${avx}") add_library(${lib_name} INTERFACE) target_compile_options(${lib_name} INTERFACE -march=${arch} -mtune=${arch}) - set(obj_name ${arch}_obj) + # Object target names key off `arch` by default, but the amx tuple above + # intentionally reuses cascadelake's `arch` (same instruction set as the avx512 + # tuple, see the comment above SVS_X86): when extra options are present, the object + # target is named from `avx` instead, so it doesn't collide with that other tuple's + # target of the same name. + if(x86_info_len GREATER 3) + list(SUBLIST x86_info 3 -1 extra_options) + target_compile_options(${lib_name} INTERFACE ${extra_options}) + set(obj_name ${avx}_obj) + else() + set(obj_name ${arch}_obj) + endif() add_library(${obj_name} OBJECT ${src}) target_link_libraries(${obj_name} PRIVATE ${SVS_LIB} svs::compile_options fmt::fmt ${lib_name}) diff --git a/include/svs/lib/avx_detection.h b/include/svs/lib/avx_detection.h index 3169bc173..abd9aba0c 100644 --- a/include/svs/lib/avx_detection.h +++ b/include/svs/lib/avx_detection.h @@ -95,7 +95,14 @@ inline constexpr unsigned long SVS_XFEATURE_XTILEDATA = 18; // branch on a cached value" the dispatch call site needs. inline bool amx_tile_bf16_available() { static const bool available = [] { - if (!avx_runtime_flags.amx_tile_bf16_cpu) { + // amx.cpp is compiled at exactly the AVX-512 level's instruction set (plus the + // two AMX extensions, see cmake/multi-arch.cmake) and its non-tile code path + // forwards to that level's kernels directly, so AMX is only usable where AVX-512 + // already is: a CPU could in principle expose the AMX bits without AVX-512F + // (e.g. a hypervisor masking features independently), and that combination is + // not one this file's code was built for. + if (!avx_runtime_flags.amx_tile_bf16_cpu || + !avx_runtime_flags.is_avx512f_supported()) { return false; } long rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, SVS_XFEATURE_XTILEDATA); From 0793c543b8413b7747a06909222b8f9724d4a31e Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 13 Nov 2026 07:40:16 -0800 Subject: [PATCH 27/68] docs(distance): document AMX permission's provenance and host impact Expands amx_tile_bf16_available()'s comment into a documentation block (exempt from the two-line internal-comment limit) stating two facts no header or source on this machine confirms: where the ARCH_REQ_XCOMP_PERM feature number 18 comes from (the kernel's xstate ABI doc, not any installed header) and that requesting it was confirmed by experiment on this Linux/x86-64 machine, not assumed from documentation; and that a seccomp policy killing on an unlisted syscall terminates the host process at this call. Also states, citing kernel documentation as unverified here, that the request is process-wide (tile state becomes part of every thread's extended processor state) and that the kernel may refuse it if an installed alternate signal stack is too small. Co-Authored-By: Claude Fable 5.2 --- include/svs/lib/avx_detection.h | 63 +++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/include/svs/lib/avx_detection.h b/include/svs/lib/avx_detection.h index abd9aba0c..0c9c07212 100644 --- a/include/svs/lib/avx_detection.h +++ b/include/svs/lib/avx_detection.h @@ -69,30 +69,49 @@ struct AVXRuntimeFlags { inline const AVXRuntimeFlags avx_runtime_flags = {}; #if defined(__x86_64__) && defined(__linux__) -// Kernel XFEATURE_XTILEDATA index for ARCH_REQ_XCOMP_PERM. Not in any header on this -// machine (glibc's asm/prctl.h stops at ARCH_REQ_XCOMP_PERM itself); taken from the -// kernel's xstate ABI (Documentation/x86/xstate.rst) and confirmed by experiment -// (/tmp/amx-study/t2c_detect.c) that requesting permission with this value is what lets -// tile instructions run without SIGILL. +// Kernel XFEATURE_XTILEDATA index for ARCH_REQ_XCOMP_PERM; see amx_tile_bf16_available's +// documentation below for where this number comes from and how it was confirmed. inline constexpr unsigned long SVS_XFEATURE_XTILEDATA = 18; -// Requests AMX tile-data OS permission exactly once per process, the first time any -// thread asks whether the AMX level is usable. Requesting at library load would run in -// whatever process SVS is linked into before that process has decided it wants AMX at -// all; requesting per-call would repeat a syscall this function's caching exists to -// avoid. The request must run unconditionally, every time: XCR0 (read via `xgetbv`) -// reports which xstate components the CPU/OS architecturally support, not whether this -// process has been granted permission to use tile data, so an "already looks enabled" -// shortcut on XCR0 would skip the one call that actually grants it (confirmed by -// experiment: a process that never calls ARCH_REQ_XCOMP_PERM still reads XCR0 bits -// 17/18 as set, then SIGILLs on the first real tile instruction). Returns true only -// when the CPU has AMX-TILE and AMX-BF16 and the kernel request succeeds; any "no" here -// is silent, not an error: dispatch simply does not pick AMX. -// -// Thread-safety and the once-ness come from the function-local static's initializer -// guarantee (C++ [stmt.dcl]): concurrent first callers block on the same guard variable -// and only one of them runs the body; every call after that is the "one predictable -// branch on a cached value" the dispatch call site needs. +/// Returns true the first time, and every time after, that this process may run AMX +/// tile instructions without SIGILL: the CPU has AMX-TILE and AMX-BF16, and the one-time +/// `ARCH_REQ_XCOMP_PERM` request below succeeded. Any "no" here is silent, not an error: +/// dispatch simply does not pick AMX. +/// +/// Requests AMX tile-data OS permission exactly once per process, the first time any +/// thread calls this function. Requesting at library load would run in whatever process +/// SVS is linked into before that process has decided it wants AMX at all; requesting +/// per-call would repeat a syscall this function's caching exists to avoid. The request +/// must run unconditionally, every time this initializes: XCR0 (read via `xgetbv`) +/// reports which xstate components the CPU/OS architecturally support, not whether this +/// process has been granted permission to use tile data, so an "already looks enabled" +/// shortcut on XCR0 would skip the one call that actually grants it (confirmed by +/// experiment: a process that never calls `ARCH_REQ_XCOMP_PERM` still reads XCR0 bits +/// 17/18 as set, then SIGILLs on the first real tile instruction). +/// +/// Thread-safety and the once-ness come from the function-local static's initializer +/// guarantee (C++ [stmt.dcl]): concurrent first callers block on the same guard variable +/// and only one of them runs the body; every call after that is the "one predictable +/// branch on a cached value" the dispatch call site needs. +/// +/// The feature number passed to `ARCH_REQ_XCOMP_PERM` (18, `SVS_XFEATURE_XTILEDATA` +/// above) is not in any header on this machine: glibc's `asm/prctl.h` defines +/// `ARCH_REQ_XCOMP_PERM` itself but not the per-feature indices it takes. 18 is +/// `XFEATURE_XTILEDATA` per the kernel's xstate ABI (`Documentation/arch/x86/xstate.rst` +/// in the Linux source tree); confirmed by experiment on this machine (Linux, x86-64, +/// see `/tmp/amx-study/t2c_detect.c`) that requesting permission with this value is what +/// lets tile instructions run without SIGILL, not merely a documented-but-untested +/// number. +/// +/// This syscall is process-wide, not just for the calling thread: once granted, tile +/// state (the eight tmm registers and the tile-config register) becomes part of every +/// thread's extended processor state in this process, saved and restored across context +/// switches like any other xstate component from then on. A process running under a +/// seccomp policy that kills on an unlisted syscall will be terminated right here, at +/// this call, if `arch_prctl` is not on that policy's allow list. Taken from kernel +/// documentation, not verified on this machine: the kernel may refuse the request if an +/// installed alternate signal stack (`sigaltstack`) is smaller than the now-larger +/// xstate save area requires. inline bool amx_tile_bf16_available() { static const bool available = [] { // amx.cpp is compiled at exactly the AVX-512 level's instruction set (plus the From 231b019e9e14ca6d93caaaaeddb0ea5164847830 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:05:29 -0800 Subject: [PATCH 28/68] refactor(kmeans): move the generic k-means out of the IVF index Shared k-means building blocks (centroid assignment/adjustment/split, GEMM-backed distance computation, training-set sampling) now live in svs::kmeans under include/svs/core/kmeans_training.h, so a future multi-vector codebook trainer can reuse them. Function bodies are moved verbatim; the IVF-specific kmeans_training(IVFBuildParameters, ...) overload now forwards to the shared one via a small KMeansTrainingParameters struct. include/svs/index/ivf/common.h re-exports every moved name with using-declarations so existing IVF call sites, tests, bindings and benchmarks compile unchanged. Named the shared header kmeans_training.h rather than kmeans.h: an unrelated legacy generic k-means utility (svs::find_nearest, svs::train, KMeansParameters) already owns include/svs/core/kmeans.h and its always-built test tests/svs/core/kmeans.cpp; reusing that path would have required threading an MKL dependency into the default, IVF/multi-vector-disabled build. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/kmeans_training.h | 573 +++++++++++++++++++++++++++++ include/svs/index/ivf/common.h | 527 ++------------------------ 2 files changed, 596 insertions(+), 504 deletions(-) create mode 100644 include/svs/core/kmeans_training.h diff --git a/include/svs/core/kmeans_training.h b/include/svs/core/kmeans_training.h new file mode 100644 index 000000000..7c5297c6f --- /dev/null +++ b/include/svs/core/kmeans_training.h @@ -0,0 +1,573 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/data.h" +#include "svs/concepts/distance.h" +#include "svs/core/data/simple.h" +#include "svs/core/data/view.h" +#include "svs/core/distance.h" +#include "svs/lib/exception.h" +#include "svs/lib/neighbor.h" +#include "svs/lib/threads/threadpool.h" +#include "svs/lib/timing.h" +#include "svs/lib/type_traits.h" + +// external +#include "tsl/robin_set.h" + +// Intel(R) MKL +#include + +// stl +#include + +// Generic k-means training shared by IVF and multi-vector codebook training. +namespace svs::kmeans { + +/// Helper trait to check if a distance type behaves like IP (inner product) +template +inline constexpr bool is_ip_like_v = + std::is_same_v, distance::DistanceIP> || + std::is_same_v, distance::DistanceCosineSimilarity>; + +/// Helper trait to check if a distance type is L2 +template +inline constexpr bool is_l2_v = + std::is_same_v, distance::DistanceL2>; + +// Small epsilon value used for floating-point comparisons to avoid precision +// issues. The value 1/1024 (approximately 0.0009765625) is chosen as a reasonable +// threshold for numerical stability in algorithms such as k-means clustering, where exact +constexpr double EPSILON = 1.0 / 1024.0; + +/// @brief Parameters controlling the k-means training loop. +struct KMeansTrainingParameters { + /// The number of iterations used in kmeans training. + size_t num_iterations; + /// The size of each minibatch. + size_t minibatch_size; +}; + +// Helper functions to convert data from one type to another using threadpool +template +void convert_data(D1& src, D2& dst, Pool& threadpool) { + // Note: Destination size can be bigger than the source as we preallocate a bigger + // buffer and reuse it to reduce the cost of frequent allocations + if (src.size() > dst.size() || src.dimensions() != dst.dimensions()) { + throw ANNEXCEPTION( + "Unexpected data shapes sizes: {}, {}; dims: {}, {}!", + src.size(), + dst.size(), + src.dimensions(), + dst.dimensions() + ); + } + + threads::parallel_for( + threadpool, + threads::StaticPartition{src.size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + dst.set_datum(i, src.get_datum(i)); + } + } + ); +} + +template +auto convert_data(Data& src, Pool& threadpool) { + auto dst = svs::data::SimpleData(src.size(), src.dimensions()); + convert_data(src, dst, threadpool); + return dst; +} + +// Partial specialization to preserve the dimensionality and Allocation type +template +auto convert_data(svs::data::SimpleData& src, Pool& threadpool) { + using allocator_type = svs::lib::rebind_allocator_t; + allocator_type rebound_alloctor = {}; + + auto dst = svs::data::SimpleData( + src.size(), src.dimensions(), rebound_alloctor + ); + convert_data(src, dst, threadpool); + return dst; +} + +template auto convert_data(Data& src) { + auto dst = svs::data::SimpleData(src.size(), src.dimensions()); + auto threadpool = threads::as_threadpool(1); + convert_data(src, dst, threadpool); + return dst; +} + +template +void compute_matmul( + const T* data, const T* centroids, float* results, size_t m, size_t n, size_t k +) { + // Early return for zero dimensions. + // Calling Intel MKL functions with zero dimensions may result in undefined behavior + // or runtime errors. This check ensures we avoid such cases. + if (m == 0 || n == 0 || k == 0) { + return; // Nothing to compute + } + + // Check for integer overflow when casting to int (MKL requirement) + constexpr size_t max_int = static_cast(std::numeric_limits::max()); + if (m > max_int || n > max_int || k > max_int) { + throw ANNEXCEPTION( + "Matrix dimensions too large for Intel MKL GEMM: m={}, n={}, k={}, max={}", + m, + n, + k, + max_int + ); + } + + // Cast size_t parameters to int for MKL GEMM functions + int m_int = static_cast(m); + int n_int = static_cast(n); + int k_int = static_cast(k); + if constexpr (std::is_same_v) { + cblas_sgemm( + CblasRowMajor, // CBLAS_LAYOUT layout + CblasNoTrans, // CBLAS_TRANSPOSE TransA + CblasTrans, // CBLAS_TRANSPOSE TransB + m_int, // const int M + n_int, // const int N + k_int, // const int K + 1.0F, // float alpha + data, // const float* A + k_int, // const int lda + centroids, // const float* B + k_int, // const int ldb + 0.0F, // const float beta + results, // float* c + n_int // const int ldc + ); + } else if constexpr (std::is_same_v) { + cblas_gemm_bf16bf16f32( + CblasRowMajor, // CBLAS_LAYOUT layout + CblasNoTrans, // CBLAS_TRANSPOSE TransA + CblasTrans, // CBLAS_TRANSPOSE TransB + m_int, // const int M + n_int, // const int N + k_int, // const int K + 1.0F, // float alpha + (const uint16_t*)data, // const *uint16_t A + k_int, // const int lda + (const uint16_t*)centroids, // const uint16_t* B + k_int, // const int ldb + 0.0F, // const float beta + results, // float* c + n_int // const int ldc + ); + } else if constexpr (std::is_same_v) { + cblas_gemm_f16f16f32( + CblasRowMajor, // CBLAS_LAYOUT layout + CblasNoTrans, // CBLAS_TRANSPOSE TransA + CblasTrans, // CBLAS_TRANSPOSE TransB + m_int, // const int M + n_int, // const int N + k_int, // const int K + 1.0F, // float alpha + (const uint16_t*)data, // const *uint16_t A + k_int, // const int lda + (const uint16_t*)centroids, // const uint16_t* B + k_int, // const int ldb + 0.0F, // const float beta + results, // float* c + n_int // const int ldc + ); + } else { + throw ANNEXCEPTION("GEMM type not supported!"); + } +} + +inline static void +generate_unique_ids(std::vector& ids, size_t id_range, std::mt19937& rng) { + size_t n = ids.size(); + tsl::robin_set seen; + seen.reserve(n); + + while (seen.size() < n) { + auto j = rng() % id_range; + seen.insert(j); + } + + size_t i = 0; + for (auto it = seen.begin(); it != seen.end(); ++it, ++i) { + ids[i] = *it; + } +} + +template +void normalize_centroids( + data::SimpleData& centroids, Pool& threadpool, lib::Timer& timer +) { + auto normalize_centroids_t = timer.push_back("normalize centroids"); + threads::parallel_for( + threadpool, + threads::StaticPartition{centroids.size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + auto datum = centroids.get_datum(i); + float norm = distance::norm(datum); + if (norm != 0.0) { + float norm_inv = 1.0F / norm; + for (size_t j = 0; j < datum.size(); j++) { + datum[j] = datum[j] * norm_inv; + } + } + } + } + ); + normalize_centroids_t.finish(); +} + +template < + data::ImmutableMemoryDataset Data, + typename T, + typename Distance, + threads::ThreadPool Pool> +void centroid_assignment( + const Data& data, + std::vector& data_norm, + threads::UnitRange batch_range, + Distance& SVS_UNUSED(distance), + data::SimpleData& centroids, + std::vector& centroids_norm, + std::vector& assignments, + data::SimpleData& matmul_results, + Pool& threadpool, + lib::Timer& timer +) { + using DataType = typename Data::element_type; + using CentroidType = T; + + // Convert data to match centroid type if necessary, otherwise use original data + [[maybe_unused]] data::SimpleData data_conv; + if constexpr (!std::is_same_v) { + data_conv = convert_data(data, threadpool); + } + const auto& matmul_data = [&]() -> const auto& { + if constexpr (!std::is_same_v) { + return data_conv; + } else { + return data; + } + } + (); + + auto generate_assignments = timer.push_back("generate assignments"); + threads::parallel_for( + threadpool, + threads::StaticPartition{batch_range.size()}, + [&](auto indices, auto /*tid*/) { + auto range = threads::UnitRange(indices); + compute_matmul( + matmul_data.get_datum(range.start()).data(), + centroids.data(), + matmul_results.get_datum(range.start()).data(), + range.size(), + centroids.size(), + matmul_data.dimensions() + ); + if constexpr (is_ip_like_v) { + for (auto i : indices) { + auto nearest = + type_traits::sentinel_v, std::greater<>>; + auto dists = matmul_results.get_datum(i); + for (size_t j = 0; j < centroids.size(); j++) { + nearest = std::max(nearest, Neighbor(j, dists[j])); + } + assignments[batch_range.start() + i] = nearest.id(); + } + } else if constexpr (is_l2_v) { + for (auto i : indices) { + auto nearest = type_traits::sentinel_v, std::less<>>; + auto dists = matmul_results.get_datum(i); + for (size_t j = 0; j < centroids.size(); j++) { + auto dist = data_norm[batch_range.start() + i] + centroids_norm[j] - + (2 * dists[j]); + nearest = std::min(nearest, Neighbor(j, dist)); + } + assignments[batch_range.start() + i] = nearest.id(); + } + } else { + // Compile-time error for unsupported distance types + static_assert( + sizeof(Distance) == 0, + "Only L2, MIP, and Cosine distances are supported in IVF build!" + ); + } + } + ); + generate_assignments.finish(); +} + +template +void centroid_adjustment( + Data& data, + data::SimpleData& centroids, + std::vector& assignments, + std::vector& counts, + Pool& threadpool, + lib::Timer& timer +) { + auto adjust_centroids = timer.push_back("adjust centroids"); + size_t n_threads = threadpool.size(); + size_t n_centroids = centroids.size(); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_threads}, + [&](auto /*indices*/, auto tid) { + size_t centroid_start = (n_centroids * tid) / n_threads; + size_t centroid_end = (n_centroids * (tid + 1)) / n_threads; + for (auto i : data.eachindex()) { + auto assignment = assignments[i]; + if (assignment >= centroid_start && assignment < centroid_end) { + counts.at(assignment)++; + auto datum = data.get_datum(i); + auto this_centroid = centroids.get_datum(assignment); + for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { + this_centroid[p] += lib::narrow_cast(datum[p]); + } + } + } + } + ); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_centroids}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + if (counts.at(i) != 0) { + auto this_centroid = centroids.get_datum(i); + float norm = 1.0 / (counts.at(i) + 1); + // float norm = 1.0 / counts.at(i); + for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { + this_centroid[p] *= norm; + } + } + } + } + ); + adjust_centroids.finish(); +} + +template +void centroid_split( + Data& data, + data::SimpleData& centroids, + std::vector& counts, + std::mt19937& rng, + Pool& SVS_UNUSED(threadpool), + lib::Timer& timer +) { + auto split_centroids = timer.push_back("split centroids"); + + auto num_centroids = centroids.size(); + auto dims = centroids.dimensions(); + auto num_data = data.size(); + + auto distribution = std::uniform_real_distribution(0.0, 1.0); + for (size_t i = 0; i < num_centroids; i++) { + if (counts.at(i) == 0) { + size_t j; + for (j = 0; true; j = (j + 1) % num_centroids) { + if (counts.at(j) == 0) { + continue; + } + float p = static_cast(counts.at(j)) / static_cast(num_data); + float r = distribution(rng); + if (r < p) { + break; + } + } + centroids.set_datum(i, centroids.get_datum(j)); + for (size_t k = 0; k < dims; k++) { + if (k % 2 == 0) { + centroids.get_datum(i)[k] *= 1 + EPSILON; + centroids.get_datum(j)[k] *= 1 - EPSILON; + } else { + centroids.get_datum(i)[k] *= 1 - EPSILON; + centroids.get_datum(j)[k] *= 1 + EPSILON; + } + } + counts.at(i) = counts.at(j) / 2; + counts.at(j) -= counts.at(i); + } + } + split_centroids.finish(); +} + +template +void generate_norms(Data& data, std::vector& norms, Pool& threadpool) { + norms.resize(data.size()); + threads::parallel_for( + threadpool, + threads::StaticPartition{data.size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + norms[i] = distance::norm_square(data.get_datum(i)); + } + } + ); +} + +template < + data::ImmutableMemoryDataset Data, + typename T, + typename Distance, + threads::ThreadPool Pool> +auto kmeans_training( + const KMeansTrainingParameters& parameters, + Data& data, + Distance& distance, + data::SimpleData& centroids, + data::SimpleData& matmul_results, + std::mt19937& rng, + Pool& threadpool, + lib::Timer& timer +) { + auto training_timer = timer.push_back("Kmeans training"); + data::SimpleData centroids_fp32 = convert_data(centroids, threadpool); + + if constexpr (is_ip_like_v) { + normalize_centroids(centroids_fp32, threadpool, timer); + } + + auto assignments = std::vector(data.size()); + std::vector data_norm; + if constexpr (is_l2_v) { + generate_norms(data, data_norm, threadpool); + } + std::vector centroids_norm; + + for (size_t iter = 0; iter < parameters.num_iterations; ++iter) { + auto iter_timer = timer.push_back("iteration"); + auto batchsize = parameters.minibatch_size; + auto num_batches = lib::div_round_up(data.size(), batchsize); + if constexpr (is_l2_v) { + generate_norms(centroids_fp32, centroids_norm, threadpool); + } + + // Convert from fp32 to fp16/bf16 + convert_data(centroids_fp32, centroids, threadpool); + + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = threads::UnitRange{ + batch * batchsize, std::min((batch + 1) * batchsize, data.size())}; + auto data_batch = data::make_view(data, this_batch); + centroid_assignment( + data_batch, + data_norm, + this_batch, + distance, + centroids, + centroids_norm, + assignments, + matmul_results, + threadpool, + timer + ); + } + + // Convert back to fp32 + convert_data(centroids, centroids_fp32, threadpool); + + auto counts = std::vector(centroids.size()); + centroid_adjustment(data, centroids_fp32, assignments, counts, threadpool, timer); + + centroid_split(data, centroids_fp32, counts, rng, threadpool, timer); + + if constexpr (is_ip_like_v) { + normalize_centroids(centroids_fp32, threadpool, timer); + } + } + + // Finally call the conversion to get the updated centroids + // after adjustment and split + convert_data(centroids_fp32, centroids, threadpool); + training_timer.finish(); + return centroids_fp32; +} + +/// @brief Generate a random subset of data for training +template +data::SimpleData make_training_set( + const Data& data, + std::vector& ids, + size_t num_training, + std::mt19937& rng, + Pool& threadpool +) { + data::SimpleData trainset( + num_training, data.dimensions() + ); + generate_unique_ids(ids, data.size(), rng); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_training}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + trainset.set_datum(i, data.get_datum(ids[i])); + } + } + ); + return trainset; +} + +/// @brief Initialize centroids randomly from the training set +template +data::SimpleData init_centroids( + const Data& trainset, + std::vector& ids, + size_t num_centroids, + std::mt19937& rng, + Pool& threadpool +) { + data::SimpleData centroids(num_centroids, trainset.dimensions()); + generate_unique_ids(ids, trainset.size(), rng); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_centroids}, + [&](auto indices, auto) { + for (auto i : indices) { + centroids.set_datum(i, trainset.get_datum(ids[i])); + } + } + ); + return centroids; +} + +/// @brief Compute norms for L2 distance if needed +template +std::vector maybe_compute_norms(const Data& data, Pool& threadpool) { + std::vector norms; + if constexpr (is_l2_v) { + generate_norms(data, norms, threadpool); + } + return norms; +} + +} // namespace svs::kmeans diff --git a/include/svs/index/ivf/common.h b/include/svs/index/ivf/common.h index feedddadc..3948a8ddc 100644 --- a/include/svs/index/ivf/common.h +++ b/include/svs/index/ivf/common.h @@ -22,6 +22,7 @@ #include "svs/core/data/simple.h" #include "svs/core/data/view.h" #include "svs/core/distance.h" +#include "svs/core/kmeans_training.h" #include "svs/core/logging.h" #include "svs/index/ivf/sorted_buffer.h" #include "svs/lib/exception.h" @@ -42,21 +43,21 @@ // Common definitions. namespace svs::index::ivf { -/// Helper trait to check if a distance type behaves like IP (inner product) -template -inline constexpr bool is_ip_like_v = - std::is_same_v, distance::DistanceIP> || - std::is_same_v, distance::DistanceCosineSimilarity>; - -/// Helper trait to check if a distance type is L2 -template -inline constexpr bool is_l2_v = - std::is_same_v, distance::DistanceL2>; - -// Small epsilon value used for floating-point comparisons to avoid precision -// issues. The value 1/1024 (approximately 0.0009765625) is chosen as a reasonable -// threshold for numerical stability in algorithms such as k-means clustering, where exact -constexpr double EPSILON = 1.0 / 1024.0; +// Re-exports the generic k-means names so existing svs::index::ivf::* call sites compile. +using svs::kmeans::centroid_adjustment; +using svs::kmeans::centroid_assignment; +using svs::kmeans::centroid_split; +using svs::kmeans::compute_matmul; +using svs::kmeans::convert_data; +using svs::kmeans::EPSILON; +using svs::kmeans::generate_norms; +using svs::kmeans::generate_unique_ids; +using svs::kmeans::init_centroids; +using svs::kmeans::is_ip_like_v; +using svs::kmeans::is_l2_v; +using svs::kmeans::make_training_set; +using svs::kmeans::maybe_compute_norms; +using svs::kmeans::normalize_centroids; /// Minimum training sample multiplier for clustering algorithms. /// When training data size is small relative to the number of clusters, we ensure @@ -185,376 +186,8 @@ struct IVFSearchParameters { operator==(const IVFSearchParameters&, const IVFSearchParameters&) = default; }; -// Helper functions to convert data from one type to another using threadpool -template -void convert_data(D1& src, D2& dst, Pool& threadpool) { - // Note: Destination size can be bigger than the source as we preallocate a bigger - // buffer and reuse it to reduce the cost of frequent allocations - if (src.size() > dst.size() || src.dimensions() != dst.dimensions()) { - throw ANNEXCEPTION( - "Unexpected data shapes sizes: {}, {}; dims: {}, {}!", - src.size(), - dst.size(), - src.dimensions(), - dst.dimensions() - ); - } - - threads::parallel_for( - threadpool, - threads::StaticPartition{src.size()}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - dst.set_datum(i, src.get_datum(i)); - } - } - ); -} - -template -auto convert_data(Data& src, Pool& threadpool) { - auto dst = svs::data::SimpleData(src.size(), src.dimensions()); - convert_data(src, dst, threadpool); - return dst; -} - -// Partial specialization to preserve the dimensionality and Allocation type -template -auto convert_data(svs::data::SimpleData& src, Pool& threadpool) { - using allocator_type = svs::lib::rebind_allocator_t; - allocator_type rebound_alloctor = {}; - - auto dst = svs::data::SimpleData( - src.size(), src.dimensions(), rebound_alloctor - ); - convert_data(src, dst, threadpool); - return dst; -} - -template auto convert_data(Data& src) { - auto dst = svs::data::SimpleData(src.size(), src.dimensions()); - auto threadpool = threads::as_threadpool(1); - convert_data(src, dst, threadpool); - return dst; -} - -template -void compute_matmul( - const T* data, const T* centroids, float* results, size_t m, size_t n, size_t k -) { - // Early return for zero dimensions. - // Calling Intel MKL functions with zero dimensions may result in undefined behavior - // or runtime errors. This check ensures we avoid such cases. - if (m == 0 || n == 0 || k == 0) { - return; // Nothing to compute - } - - // Check for integer overflow when casting to int (MKL requirement) - constexpr size_t max_int = static_cast(std::numeric_limits::max()); - if (m > max_int || n > max_int || k > max_int) { - throw ANNEXCEPTION( - "Matrix dimensions too large for Intel MKL GEMM: m={}, n={}, k={}, max={}", - m, - n, - k, - max_int - ); - } - - // Cast size_t parameters to int for MKL GEMM functions - int m_int = static_cast(m); - int n_int = static_cast(n); - int k_int = static_cast(k); - if constexpr (std::is_same_v) { - cblas_sgemm( - CblasRowMajor, // CBLAS_LAYOUT layout - CblasNoTrans, // CBLAS_TRANSPOSE TransA - CblasTrans, // CBLAS_TRANSPOSE TransB - m_int, // const int M - n_int, // const int N - k_int, // const int K - 1.0F, // float alpha - data, // const float* A - k_int, // const int lda - centroids, // const float* B - k_int, // const int ldb - 0.0F, // const float beta - results, // float* c - n_int // const int ldc - ); - } else if constexpr (std::is_same_v) { - cblas_gemm_bf16bf16f32( - CblasRowMajor, // CBLAS_LAYOUT layout - CblasNoTrans, // CBLAS_TRANSPOSE TransA - CblasTrans, // CBLAS_TRANSPOSE TransB - m_int, // const int M - n_int, // const int N - k_int, // const int K - 1.0F, // float alpha - (const uint16_t*)data, // const *uint16_t A - k_int, // const int lda - (const uint16_t*)centroids, // const uint16_t* B - k_int, // const int ldb - 0.0F, // const float beta - results, // float* c - n_int // const int ldc - ); - } else if constexpr (std::is_same_v) { - cblas_gemm_f16f16f32( - CblasRowMajor, // CBLAS_LAYOUT layout - CblasNoTrans, // CBLAS_TRANSPOSE TransA - CblasTrans, // CBLAS_TRANSPOSE TransB - m_int, // const int M - n_int, // const int N - k_int, // const int K - 1.0F, // float alpha - (const uint16_t*)data, // const *uint16_t A - k_int, // const int lda - (const uint16_t*)centroids, // const uint16_t* B - k_int, // const int ldb - 0.0F, // const float beta - results, // float* c - n_int // const int ldc - ); - } else { - throw ANNEXCEPTION("GEMM type not supported!"); - } -} - -inline static void -generate_unique_ids(std::vector& ids, size_t id_range, std::mt19937& rng) { - size_t n = ids.size(); - tsl::robin_set seen; - seen.reserve(n); - - while (seen.size() < n) { - auto j = rng() % id_range; - seen.insert(j); - } - - size_t i = 0; - for (auto it = seen.begin(); it != seen.end(); ++it, ++i) { - ids[i] = *it; - } -} - -template -void normalize_centroids( - data::SimpleData& centroids, Pool& threadpool, lib::Timer& timer -) { - auto normalize_centroids_t = timer.push_back("normalize centroids"); - threads::parallel_for( - threadpool, - threads::StaticPartition{centroids.size()}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - auto datum = centroids.get_datum(i); - float norm = distance::norm(datum); - if (norm != 0.0) { - float norm_inv = 1.0F / norm; - for (size_t j = 0; j < datum.size(); j++) { - datum[j] = datum[j] * norm_inv; - } - } - } - } - ); - normalize_centroids_t.finish(); -} - -template < - data::ImmutableMemoryDataset Data, - typename T, - typename Distance, - threads::ThreadPool Pool> -void centroid_assignment( - const Data& data, - std::vector& data_norm, - threads::UnitRange batch_range, - Distance& SVS_UNUSED(distance), - data::SimpleData& centroids, - std::vector& centroids_norm, - std::vector& assignments, - data::SimpleData& matmul_results, - Pool& threadpool, - lib::Timer& timer -) { - using DataType = typename Data::element_type; - using CentroidType = T; - - // Convert data to match centroid type if necessary, otherwise use original data - [[maybe_unused]] data::SimpleData data_conv; - if constexpr (!std::is_same_v) { - data_conv = convert_data(data, threadpool); - } - const auto& matmul_data = [&]() -> const auto& { - if constexpr (!std::is_same_v) { - return data_conv; - } else { - return data; - } - } - (); - - auto generate_assignments = timer.push_back("generate assignments"); - threads::parallel_for( - threadpool, - threads::StaticPartition{batch_range.size()}, - [&](auto indices, auto /*tid*/) { - auto range = threads::UnitRange(indices); - compute_matmul( - matmul_data.get_datum(range.start()).data(), - centroids.data(), - matmul_results.get_datum(range.start()).data(), - range.size(), - centroids.size(), - matmul_data.dimensions() - ); - if constexpr (is_ip_like_v) { - for (auto i : indices) { - auto nearest = - type_traits::sentinel_v, std::greater<>>; - auto dists = matmul_results.get_datum(i); - for (size_t j = 0; j < centroids.size(); j++) { - nearest = std::max(nearest, Neighbor(j, dists[j])); - } - assignments[batch_range.start() + i] = nearest.id(); - } - } else if constexpr (is_l2_v) { - for (auto i : indices) { - auto nearest = type_traits::sentinel_v, std::less<>>; - auto dists = matmul_results.get_datum(i); - for (size_t j = 0; j < centroids.size(); j++) { - auto dist = data_norm[batch_range.start() + i] + centroids_norm[j] - - (2 * dists[j]); - nearest = std::min(nearest, Neighbor(j, dist)); - } - assignments[batch_range.start() + i] = nearest.id(); - } - } else { - // Compile-time error for unsupported distance types - static_assert( - sizeof(Distance) == 0, - "Only L2, MIP, and Cosine distances are supported in IVF build!" - ); - } - } - ); - generate_assignments.finish(); -} - -template -void centroid_adjustment( - Data& data, - data::SimpleData& centroids, - std::vector& assignments, - std::vector& counts, - Pool& threadpool, - lib::Timer& timer -) { - auto adjust_centroids = timer.push_back("adjust centroids"); - size_t n_threads = threadpool.size(); - size_t n_centroids = centroids.size(); - - threads::parallel_for( - threadpool, - threads::StaticPartition{n_threads}, - [&](auto /*indices*/, auto tid) { - size_t centroid_start = (n_centroids * tid) / n_threads; - size_t centroid_end = (n_centroids * (tid + 1)) / n_threads; - for (auto i : data.eachindex()) { - auto assignment = assignments[i]; - if (assignment >= centroid_start && assignment < centroid_end) { - counts.at(assignment)++; - auto datum = data.get_datum(i); - auto this_centroid = centroids.get_datum(assignment); - for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { - this_centroid[p] += lib::narrow_cast(datum[p]); - } - } - } - } - ); - - threads::parallel_for( - threadpool, - threads::StaticPartition{n_centroids}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - if (counts.at(i) != 0) { - auto this_centroid = centroids.get_datum(i); - float norm = 1.0 / (counts.at(i) + 1); - // float norm = 1.0 / counts.at(i); - for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { - this_centroid[p] *= norm; - } - } - } - } - ); - adjust_centroids.finish(); -} - -template -void centroid_split( - Data& data, - data::SimpleData& centroids, - std::vector& counts, - std::mt19937& rng, - Pool& SVS_UNUSED(threadpool), - lib::Timer& timer -) { - auto split_centroids = timer.push_back("split centroids"); - - auto num_centroids = centroids.size(); - auto dims = centroids.dimensions(); - auto num_data = data.size(); - - auto distribution = std::uniform_real_distribution(0.0, 1.0); - for (size_t i = 0; i < num_centroids; i++) { - if (counts.at(i) == 0) { - size_t j; - for (j = 0; true; j = (j + 1) % num_centroids) { - if (counts.at(j) == 0) { - continue; - } - float p = static_cast(counts.at(j)) / static_cast(num_data); - float r = distribution(rng); - if (r < p) { - break; - } - } - centroids.set_datum(i, centroids.get_datum(j)); - for (size_t k = 0; k < dims; k++) { - if (k % 2 == 0) { - centroids.get_datum(i)[k] *= 1 + EPSILON; - centroids.get_datum(j)[k] *= 1 - EPSILON; - } else { - centroids.get_datum(i)[k] *= 1 - EPSILON; - centroids.get_datum(j)[k] *= 1 + EPSILON; - } - } - counts.at(i) = counts.at(j) / 2; - counts.at(j) -= counts.at(i); - } - } - split_centroids.finish(); -} - -template -void generate_norms(Data& data, std::vector& norms, Pool& threadpool) { - norms.resize(data.size()); - threads::parallel_for( - threadpool, - threads::StaticPartition{data.size()}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - norms[i] = distance::norm_square(data.get_datum(i)); - } - } - ); -} - +/// @brief Forward to svs::kmeans::kmeans_training using the IVF build parameters this +/// index has always taken; kept so existing IVF call sites need no change. template < data::ImmutableMemoryDataset Data, typename T, @@ -570,67 +203,11 @@ auto kmeans_training( Pool& threadpool, lib::Timer& timer ) { - auto training_timer = timer.push_back("Kmeans training"); - data::SimpleData centroids_fp32 = convert_data(centroids, threadpool); - - if constexpr (is_ip_like_v) { - normalize_centroids(centroids_fp32, threadpool, timer); - } - - auto assignments = std::vector(data.size()); - std::vector data_norm; - if constexpr (is_l2_v) { - generate_norms(data, data_norm, threadpool); - } - std::vector centroids_norm; - - for (size_t iter = 0; iter < parameters.num_iterations_; ++iter) { - auto iter_timer = timer.push_back("iteration"); - auto batchsize = parameters.minibatch_size_; - auto num_batches = lib::div_round_up(data.size(), batchsize); - if constexpr (is_l2_v) { - generate_norms(centroids_fp32, centroids_norm, threadpool); - } - - // Convert from fp32 to fp16/bf16 - convert_data(centroids_fp32, centroids, threadpool); - - for (size_t batch = 0; batch < num_batches; ++batch) { - auto this_batch = threads::UnitRange{ - batch * batchsize, std::min((batch + 1) * batchsize, data.size())}; - auto data_batch = data::make_view(data, this_batch); - centroid_assignment( - data_batch, - data_norm, - this_batch, - distance, - centroids, - centroids_norm, - assignments, - matmul_results, - threadpool, - timer - ); - } - - // Convert back to fp32 - convert_data(centroids, centroids_fp32, threadpool); - - auto counts = std::vector(centroids.size()); - centroid_adjustment(data, centroids_fp32, assignments, counts, threadpool, timer); - - centroid_split(data, centroids_fp32, counts, rng, threadpool, timer); - - if constexpr (is_ip_like_v) { - normalize_centroids(centroids_fp32, threadpool, timer); - } - } - - // Finally call the conversion to get the updated centroids - // after adjustment and split - convert_data(centroids_fp32, centroids, threadpool); - training_timer.finish(); - return centroids_fp32; + auto kmeans_parameters = svs::kmeans::KMeansTrainingParameters{ + parameters.num_iterations_, parameters.minibatch_size_}; + return svs::kmeans::kmeans_training( + kmeans_parameters, data, distance, centroids, matmul_results, rng, threadpool, timer + ); } template < @@ -683,64 +260,6 @@ void compute_centroid_distances( ); } -/// @brief Generate a random subset of data for training -template -data::SimpleData make_training_set( - const Data& data, - std::vector& ids, - size_t num_training, - std::mt19937& rng, - Pool& threadpool -) { - data::SimpleData trainset( - num_training, data.dimensions() - ); - generate_unique_ids(ids, data.size(), rng); - threads::parallel_for( - threadpool, - threads::StaticPartition{num_training}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - trainset.set_datum(i, data.get_datum(ids[i])); - } - } - ); - return trainset; -} - -/// @brief Initialize centroids randomly from the training set -template -data::SimpleData init_centroids( - const Data& trainset, - std::vector& ids, - size_t num_centroids, - std::mt19937& rng, - Pool& threadpool -) { - data::SimpleData centroids(num_centroids, trainset.dimensions()); - generate_unique_ids(ids, trainset.size(), rng); - threads::parallel_for( - threadpool, - threads::StaticPartition{num_centroids}, - [&](auto indices, auto) { - for (auto i : indices) { - centroids.set_datum(i, trainset.get_datum(ids[i])); - } - } - ); - return centroids; -} - -/// @brief Compute norms for L2 distance if needed -template -std::vector maybe_compute_norms(const Data& data, Pool& threadpool) { - std::vector norms; - if constexpr (is_l2_v) { - generate_norms(data, norms, threadpool); - } - return norms; -} - /// @brief Assign all points to clusters according to assignments template std::vector> group_assignments( From 4acd5e68391d6a47f95536e09063edf899cfe9d2 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:05:31 -0800 Subject: [PATCH 29/68] test(kmeans): add a unit test for the shared k-means header Covers the L2 centroid update formula (initial centroid plus assigned points, divided by count+1 - not a plain mean), unit-norm centroids under IP distance, and centroid_assignment agreeing with a brute-force nearest-centroid oracle for both distances. Registered in tests/CMakeLists.txt under "IVF or multi-vector", matching the new header's MKL dependency. Co-Authored-By: Claude Fable 5.2 --- tests/CMakeLists.txt | 7 + tests/svs/core/kmeans_training.cpp | 215 +++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 tests/svs/core/kmeans_training.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a61caf961..30b96a6c5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -239,6 +239,13 @@ if (SVS_EXPERIMENTAL_ENABLE_IVF) ) endif() +if (SVS_EXPERIMENTAL_ENABLE_IVF OR SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) + message("Enabling shared k-means tests!") + list(APPEND TEST_SOURCES + ${TEST_DIR}/svs/core/kmeans_training.cpp + ) +endif() + add_executable(tests ${TEST_SOURCES}) # Path to the test dataset. diff --git a/tests/svs/core/kmeans_training.cpp b/tests/svs/core/kmeans_training.cpp new file mode 100644 index 000000000..c2a01ba6a --- /dev/null +++ b/tests/svs/core/kmeans_training.cpp @@ -0,0 +1,215 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/core/kmeans_training.h" + +// svs +#include "svs/core/data.h" +#include "svs/core/distance.h" +#include "svs/lib/threads.h" + +// catch +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include + +namespace { + +// Brute-force nearest centroid, used as an oracle independent of centroid_assignment. +template +size_t brute_force_nearest( + const Data& data, size_t i, const svs::data::SimpleData& centroids +) { + auto datum = data.get_datum(i); + size_t best = 0; + if constexpr (std::is_same_v) { + float best_score = -std::numeric_limits::infinity(); + for (size_t j = 0; j < centroids.size(); ++j) { + auto c = centroids.get_datum(j); + float dot = 0; + for (size_t d = 0; d < datum.size(); ++d) { + dot += datum[d] * c[d]; + } + if (dot > best_score) { + best_score = dot; + best = j; + } + } + } else { + float best_dist = std::numeric_limits::infinity(); + for (size_t j = 0; j < centroids.size(); ++j) { + auto c = centroids.get_datum(j); + float dist = 0; + for (size_t d = 0; d < datum.size(); ++d) { + float diff = datum[d] - c[d]; + dist += diff * diff; + } + if (dist < best_dist) { + best_dist = dist; + best = j; + } + } + } + return best; +} + +} // namespace + +CATCH_TEST_CASE("Shared Kmeans Training - L2 Centroid Update", "[core][kmeans]") { + namespace kmeans = svs::kmeans; + + // Two well-separated pairs of points, one per cluster, for an unambiguous assignment. + auto data = svs::data::SimpleData(4, 2); + data.set_datum(0, std::vector{0.0F, 0.0F}); + data.set_datum(1, std::vector{0.0F, 2.0F}); + data.set_datum(2, std::vector{10.0F, 0.0F}); + data.set_datum(3, std::vector{10.0F, 2.0F}); + + auto centroids = svs::data::SimpleData(2, 2); + centroids.set_datum(0, std::vector{0.0F, 0.0F}); + centroids.set_datum(1, std::vector{10.0F, 0.0F}); + + auto matmul_results = svs::data::SimpleData(10, 2); + auto rng = std::mt19937(0xc0ffee); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto distance = svs::DistanceL2(); + + // centroid_adjustment divides by (count + 1), not count, so this is not a plain mean. + auto params = kmeans::KMeansTrainingParameters{1, 10}; + auto result = kmeans::kmeans_training( + params, data, distance, centroids, matmul_results, rng, threadpool, timer + ); + + CATCH_REQUIRE(result.size() == 2); + + auto c0 = result.get_datum(0); + auto c1 = result.get_datum(1); + constexpr float tolerance = 1e-4F; + + CATCH_REQUIRE(std::abs(c0[0] - 0.0F) < tolerance); + CATCH_REQUIRE(std::abs(c0[1] - (2.0F / 3.0F)) < tolerance); + + CATCH_REQUIRE(std::abs(c1[0] - (30.0F / 3.0F)) < tolerance); + CATCH_REQUIRE(std::abs(c1[1] - (2.0F / 3.0F)) < tolerance); + + // L2 training does not normalize centroids, so they are not unit-norm in general. + float norm0 = std::sqrt(c0[0] * c0[0] + c0[1] * c0[1]); + float norm1 = std::sqrt(c1[0] * c1[0] + c1[1] * c1[1]); + CATCH_REQUIRE(std::abs(norm0 - 1.0F) > tolerance); + CATCH_REQUIRE(std::abs(norm1 - 1.0F) > tolerance); +} + +CATCH_TEST_CASE("Shared Kmeans Training - IP Centroids Are Unit Norm", "[core][kmeans]") { + namespace kmeans = svs::kmeans; + + auto data = svs::data::SimpleData(6, 3); + data.set_datum(0, std::vector{1.0F, 2.0F, 3.0F}); + data.set_datum(1, std::vector{2.0F, 1.0F, 4.0F}); + data.set_datum(2, std::vector{3.0F, 3.0F, 1.0F}); + data.set_datum(3, std::vector{-1.0F, -2.0F, 0.5F}); + data.set_datum(4, std::vector{-2.0F, -1.0F, 1.5F}); + data.set_datum(5, std::vector{-3.0F, -0.5F, 2.0F}); + + // Non-unit-norm initial centroids, so normalization is confirmed to happen inside. + auto centroids = svs::data::SimpleData(2, 3); + centroids.set_datum(0, data.get_datum(0)); + centroids.set_datum(1, data.get_datum(3)); + + auto matmul_results = svs::data::SimpleData(6, 2); + auto rng = std::mt19937(42); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto distance = svs::DistanceIP(); + + auto params = kmeans::KMeansTrainingParameters{3, 6}; + auto result = kmeans::kmeans_training( + params, data, distance, centroids, matmul_results, rng, threadpool, timer + ); + + constexpr float tolerance = 1e-5F; + for (size_t i = 0; i < result.size(); ++i) { + auto c = result.get_datum(i); + float norm_sq = 0; + for (size_t d = 0; d < c.size(); ++d) { + norm_sq += c[d] * c[d]; + } + CATCH_REQUIRE(std::abs(std::sqrt(norm_sq) - 1.0F) < tolerance); + } +} + +CATCH_TEST_CASE( + "Shared Kmeans Training - centroid_assignment Agrees With Brute Force", "[core][kmeans]" +) { + namespace kmeans = svs::kmeans; + + constexpr size_t num_points = 30; + constexpr size_t num_centroids = 4; + constexpr size_t dims = 5; + + auto data = svs::data::SimpleData(num_points, dims); + for (size_t i = 0; i < num_points; ++i) { + auto datum = data.get_datum(i); + for (size_t d = 0; d < dims; ++d) { + // A deterministic, non-trivial pattern; no symmetry that would create ties. + datum[d] = static_cast((i * 7 + d * 3) % 11) - 5.0F; + } + } + + auto centroids = svs::data::SimpleData(num_centroids, dims); + for (size_t i = 0; i < num_centroids; ++i) { + auto c = centroids.get_datum(i); + for (size_t d = 0; d < dims; ++d) { + c[d] = static_cast((i * 5 + d * 2) % 7) - 3.0F; + } + } + + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto batch_range = svs::threads::UnitRange{0, num_points}; + auto matmul_results = svs::data::SimpleData(num_points, num_centroids); + + auto check_distance = [&](Distance distance) { + auto data_norm = kmeans::maybe_compute_norms(data, threadpool); + auto centroids_norm = kmeans::maybe_compute_norms(centroids, threadpool); + auto assignments = std::vector(num_points); + + kmeans::centroid_assignment( + data, + data_norm, + batch_range, + distance, + centroids, + centroids_norm, + assignments, + matmul_results, + threadpool, + timer + ); + + for (size_t i = 0; i < num_points; ++i) { + size_t expected = brute_force_nearest(data, i, centroids); + CATCH_REQUIRE(assignments[i] == expected); + } + }; + + check_distance(svs::DistanceL2()); + check_distance(svs::DistanceIP()); +} From 451c01c29bb323b9590781e18898c921a27f45b5 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:05:32 -0800 Subject: [PATCH 30/68] build: add the multi-vector flag and build k-means under it SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR (default OFF) joins SVS_EXPERIMENTAL_ENABLE_IVF as a trigger for pulling in Intel(R) MKL, since both features train a codebook with the shared k-means code. IVF-only gates (tests, benchmark, bindings) are untouched. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 4 ++-- cmake/mkl.cmake | 2 +- cmake/options.cmake | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 810652815..5393416e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,8 +77,8 @@ include("cmake/fmt.cmake") include("cmake/spdlog.cmake") include("cmake/toml.cmake") -# IVF requires Intel(R) MKL support -if(SVS_EXPERIMENTAL_ENABLE_IVF) +# IVF and multi-vector both train k-means codebooks, which need Intel(R) MKL support. +if(SVS_EXPERIMENTAL_ENABLE_IVF OR SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) include("cmake/mkl.cmake") target_compile_options(${SVS_LIB} INTERFACE "-DSVS_HAVE_MKL=1") endif() diff --git a/cmake/mkl.cmake b/cmake/mkl.cmake index e0ef5c991..b84fd439d 100644 --- a/cmake/mkl.cmake +++ b/cmake/mkl.cmake @@ -45,7 +45,7 @@ if (SVS_EXPERIMENTAL_BUILD_CUSTOM_MKL) NO_DEFAULT_PATH REQUIRED ) - if (SVS_EXPERIMENTAL_ENABLE_IVF) + if (SVS_EXPERIMENTAL_ENABLE_IVF OR SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) set(mkl_functions_file ${CMAKE_CURRENT_LIST_DIR}/mkl_functions_ivf) else() set(mkl_functions_file ${CMAKE_CURRENT_LIST_DIR}/mkl_functions) diff --git a/cmake/options.cmake b/cmake/options.cmake index bfc3ebf71..7ee8620db 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -110,6 +110,11 @@ option(SVS_EXPERIMENTAL_ENABLE_IVF OFF # disabled by default ) +option(SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR + "Enable multi-vector (late interaction) support. Requires Intel(R) MKL support" + OFF # disabled by default +) + ##### ##### svsbenchmark ##### @@ -163,6 +168,12 @@ else() target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_IVF=0) endif() +if (SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_MULTI_VECTOR=1) +else() + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_MULTI_VECTOR=0) +endif() + ##### ##### Helper target to apply relevant compiler optimizations. ##### From cbab021eb4ad56ccd1ed400308568ab53b90e7a5 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:11:10 -0800 Subject: [PATCH 31/68] feat(vamana): count build distance calls per phase behind a stats flag Adds BuildStats (inserts, per-phase distance counts, per-phase timing) behind the new SVS_EXPERIMENTAL_ENABLE_STATS CMake option, compiled out by default via SVS_ENABLE_STATS=0. CountingDistance wraps a distance functor to count distance::compute calls without touching the hot path when disabled. VamanaBuilder's generate_neighbors and add_reverse_edges wrap the greedy search, pool construction, forward prune and backedge prune under #if SVS_ENABLE_STATS only. Co-Authored-By: Claude Fable 5.2 --- cmake/options.cmake | 11 ++ include/svs/index/vamana/build_stats.h | 134 +++++++++++++++++++++++ include/svs/index/vamana/vamana_build.h | 107 ++++++++++++++++-- tests/CMakeLists.txt | 1 + tests/svs/index/vamana/build_stats.cpp | 137 ++++++++++++++++++++++++ 5 files changed, 381 insertions(+), 9 deletions(-) create mode 100644 include/svs/index/vamana/build_stats.h create mode 100644 tests/svs/index/vamana/build_stats.cpp diff --git a/cmake/options.cmake b/cmake/options.cmake index 7ee8620db..908afa0c4 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -115,6 +115,11 @@ option(SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR OFF # disabled by default ) +option(SVS_EXPERIMENTAL_ENABLE_STATS + "Enable build and search measurement counters" + OFF # disabled by default +) + ##### ##### svsbenchmark ##### @@ -174,6 +179,12 @@ else() target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_MULTI_VECTOR=0) endif() +if (SVS_EXPERIMENTAL_ENABLE_STATS) + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_STATS=1) +else() + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_STATS=0) +endif() + ##### ##### Helper target to apply relevant compiler optimizations. ##### diff --git a/include/svs/index/vamana/build_stats.h b/include/svs/index/vamana/build_stats.h new file mode 100644 index 000000000..17b031ee5 --- /dev/null +++ b/include/svs/index/vamana/build_stats.h @@ -0,0 +1,134 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Compiled out unless SVS_ENABLE_STATS is set, so including this header is always free. +#if SVS_ENABLE_STATS + +#include "svs/concepts/distance.h" + +#include +#include +#include +#include + +namespace svs::index::vamana { + +/// Build-time measurement counters. All distance counts and durations are summed over +/// every worker thread; `max_search_distances_per_insert` is a max over inserts instead. +struct BuildStats { + uint64_t inserts = 0; + uint64_t search_distances = 0; + uint64_t pool_distances = 0; + uint64_t prune_distances = 0; + uint64_t backedge_prune_distances = 0; + uint64_t search_ns = 0; + uint64_t prune_ns = 0; + uint64_t backedge_prune_ns = 0; + uint64_t max_search_distances_per_insert = 0; + + BuildStats& operator+=(const BuildStats& other) { + inserts += other.inserts; + search_distances += other.search_distances; + pool_distances += other.pool_distances; + prune_distances += other.prune_distances; + backedge_prune_distances += other.backedge_prune_distances; + search_ns += other.search_ns; + prune_ns += other.prune_ns; + backedge_prune_ns += other.backedge_prune_ns; + max_search_distances_per_insert = std::max( + max_search_distances_per_insert, other.max_search_distances_per_insert + ); + return *this; + } +}; + +namespace detail { +inline std::mutex& build_stats_mutex() { + static std::mutex mutex; + return mutex; +} + +inline BuildStats& build_stats_accumulator() { + static BuildStats stats{}; + return stats; +} +} // namespace detail + +/// Return a copy of the process-wide accumulator. +inline BuildStats build_stats_snapshot() { + std::lock_guard lock(detail::build_stats_mutex()); + return detail::build_stats_accumulator(); +} + +/// Zero the process-wide accumulator. +inline void build_stats_reset() { + std::lock_guard lock(detail::build_stats_mutex()); + detail::build_stats_accumulator() = BuildStats{}; +} + +/// Fold a thread's local counters into the process-wide accumulator. +/// Call once per thread, after that thread's `parallel_for` body finishes, so the hot +/// path itself touches no shared state. +inline void build_stats_merge(const BuildStats& stats) { + std::lock_guard lock(detail::build_stats_mutex()); + detail::build_stats_accumulator() += stats; +} + +/// Nanoseconds elapsed since `start`, for summing into a `BuildStats` duration field. +inline uint64_t build_stats_elapsed_ns(std::chrono::steady_clock::time_point start) { + auto elapsed = std::chrono::steady_clock::now() - start; + return static_cast( + std::chrono::duration_cast(elapsed).count() + ); +} + +/// Wraps `Dist` by reference, counting each distance::compute call into an unsynchronized +/// uint64_t& counter; forwards compare/implicit_broadcast/must_fix_argument from Dist. +/// Select PruneStrategy from the unwrapped Dist: it has no specialization for this type. +template class CountingDistance { + public: + using compare = distance::compare_t; + static constexpr bool implicit_broadcast = distance::implicitly_broadcastable(); + static constexpr bool must_fix_argument = distance::fix_argument_mandated(); + + CountingDistance(Dist& distance, uint64_t& counter) + : distance_{distance} + , counter_{counter} {} + + template float compute(A a, B b) { + ++counter_; + return distance::compute(distance_, a, b); + } + + // Constrained to Dist's own fix_argument support, so ShouldFix matches Dist exactly. + template + requires requires(Dist& d, A a) { d.fix_argument(a); } + void fix_argument(A a) { + distance_.fix_argument(a); + } + + auto comparator() const { return distance::comparator(distance_); } + + private: + Dist& distance_; + uint64_t& counter_; +}; + +} // namespace svs::index::vamana + +#endif // SVS_ENABLE_STATS diff --git a/include/svs/index/vamana/vamana_build.h b/include/svs/index/vamana/vamana_build.h index f658a974f..406a94244 100644 --- a/include/svs/index/vamana/vamana_build.h +++ b/include/svs/index/vamana/vamana_build.h @@ -21,6 +21,7 @@ #include "svs/concepts/distance.h" #include "svs/core/logging.h" #include "svs/index/vamana/build_params.h" +#include "svs/index/vamana/build_stats.h" #include "svs/index/vamana/extensions.h" #include "svs/index/vamana/greedy_search.h" #include "svs/index/vamana/prune.h" @@ -43,10 +44,12 @@ // stdlib #include #include +#include #include #include #include #include +#include #include namespace svs::index::vamana { @@ -370,6 +373,9 @@ class VamanaBuilder { auto&& graph_search_distance = build_adaptor.graph_search_distance(); auto&& general_distance = build_adaptor.general_distance(); auto general_accessor = build_adaptor.general_accessor(); +#if SVS_ENABLE_STATS + BuildStats thread_stats{}; +#endif for (auto node_id : local_indices) { pool.clear(); @@ -380,8 +386,16 @@ class VamanaBuilder { const auto& graph_search_query = build_adaptor.access_query_for_graph_search(data_, node_id); - // Perform the greedy search. - // The search tracker will be used if it is enabled. + // Perform the greedy search. + // The search tracker will be used if it is enabled. +#if SVS_ENABLE_STATS + uint64_t search_distance_count = 0; + CountingDistance< + std::remove_reference_t> + counted_graph_search_distance{ + graph_search_distance, search_distance_count}; + auto search_clock_start = std::chrono::steady_clock::now(); +#endif { auto accessor = build_adaptor.graph_search_accessor(); greedy_search( @@ -389,7 +403,11 @@ class VamanaBuilder { data_, accessor, graph_search_query, +#if SVS_ENABLE_STATS + counted_graph_search_distance, +#else graph_search_distance, +#endif search_buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_points)}, NeighborBuilder(), @@ -397,6 +415,14 @@ class VamanaBuilder { prefetch_hint_ ); } +#if SVS_ENABLE_STATS + thread_stats.search_ns += build_stats_elapsed_ns(search_clock_start); + thread_stats.search_distances += search_distance_count; + thread_stats.max_search_distances_per_insert = std::max( + thread_stats.max_search_distances_per_insert, search_distance_count + ); + ++thread_stats.inserts; +#endif const auto& post_search_query = build_adaptor.modify_post_search_query( data_, node_id, graph_search_query @@ -434,7 +460,12 @@ class VamanaBuilder { } } - // Add neighbors of the query that are not part of `visited`. + // Add neighbors of the query that are not part of `visited`. +#if SVS_ENABLE_STATS + uint64_t pool_distance_count = 0; + CountingDistance> + counted_pool_distance{general_distance, pool_distance_count}; +#endif for (auto id : graph_.get_node(node_id)) { assert(id != node_id); // Try to emplace the node id into the visited set. @@ -445,13 +476,20 @@ class VamanaBuilder { pool.emplace_back( id, distance::compute( +#if SVS_ENABLE_STATS + counted_pool_distance, +#else general_distance, +#endif post_search_query, general_accessor(data_, id) ) ); } } +#if SVS_ENABLE_STATS + thread_stats.pool_distances += pool_distance_count; +#endif std::sort( pool.begin(), @@ -463,18 +501,35 @@ class VamanaBuilder { // Prune and wait for an update. thread_local_updates.emplace_back(node_id, std::vector{}); auto& pruned_results = thread_local_updates.back().second; +#if SVS_ENABLE_STATS + uint64_t prune_distance_count = 0; + CountingDistance> + counted_prune_distance{general_distance, prune_distance_count}; + auto prune_clock_start = std::chrono::steady_clock::now(); +#endif heuristic_prune_neighbors( prune_strategy(distance_function_), params_.graph_max_degree, alpha, data_, general_accessor, +#if SVS_ENABLE_STATS + counted_prune_distance, +#else general_distance, +#endif node_id, lib::as_const_span(pool), pruned_results ); +#if SVS_ENABLE_STATS + thread_stats.prune_ns += build_stats_elapsed_ns(prune_clock_start); + thread_stats.prune_distances += prune_distance_count; +#endif } +#if SVS_ENABLE_STATS + build_stats_merge(thread_stats); +#endif } ); @@ -528,7 +583,13 @@ class VamanaBuilder { threads::parallel_for( threadpool_, threads::DynamicPartition{backedge_buffer_.buckets(), 1}, - [&](auto& buckets, uint64_t SVS_UNUSED(tid)) { + [&](auto& buckets, +#if SVS_ENABLE_STATS + uint64_t tid +#else + uint64_t SVS_UNUSED(tid) +#endif + ) { // Thread local auxiliary data structures. std::vector> candidates{}; std::vector pruned_results{}; @@ -538,6 +599,13 @@ class VamanaBuilder { auto&& general_distance = build_adaptor.general_distance(); auto cmp = distance::comparator(general_distance); +#if SVS_ENABLE_STATS + BuildStats thread_stats{}; + (void)tid; + uint64_t backedge_distance_count = 0; + CountingDistance> + counted_backedge_distance{general_distance, backedge_distance_count}; +#endif for (auto& bucket : buckets) { for (const auto& kv : bucket) { // The ``neighbors`` class is a set. @@ -549,11 +617,17 @@ class VamanaBuilder { // Helper lambda to make distance computations look a little // cleaner. auto make_neighbor = [&](auto i) { - return Neighbor{ - i, - distance::compute( - general_distance, src_data, general_accessor(data_, i) - )}; + return Neighbor { + i, distance::compute( +#if SVS_ENABLE_STATS + counted_backedge_distance, +#else + general_distance, +#endif + src_data, + general_accessor(data_, i) + ) + }; }; candidates.clear(); @@ -573,20 +647,35 @@ class VamanaBuilder { std::min(candidates.size(), params_.max_candidate_pool_size) ); +#if SVS_ENABLE_STATS + auto backedge_prune_clock_start = std::chrono::steady_clock::now(); +#endif heuristic_prune_neighbors( prune_strategy(distance_function_), params_.prune_to, alpha, data_, general_accessor, +#if SVS_ENABLE_STATS + counted_backedge_distance, +#else general_distance, +#endif src, lib::as_const_span(candidates), pruned_results ); +#if SVS_ENABLE_STATS + thread_stats.backedge_prune_ns += + build_stats_elapsed_ns(backedge_prune_clock_start); +#endif graph_.replace_node(src, pruned_results); } } +#if SVS_ENABLE_STATS + thread_stats.backedge_prune_distances = backedge_distance_count; + build_stats_merge(thread_stats); +#endif } ); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 30b96a6c5..60da2108f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -137,6 +137,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/index/flat/flat.cpp ${TEST_DIR}/svs/index/flat/dynamic_flat.cpp ${TEST_DIR}/svs/index/vamana/build_parameters.cpp + ${TEST_DIR}/svs/index/vamana/build_stats.cpp ${TEST_DIR}/svs/index/vamana/consolidate.cpp ${TEST_DIR}/svs/index/vamana/filter.cpp ${TEST_DIR}/svs/index/vamana/gem_vector_set.cpp diff --git a/tests/svs/index/vamana/build_stats.cpp b/tests/svs/index/vamana/build_stats.cpp new file mode 100644 index 000000000..ff4233e81 --- /dev/null +++ b/tests/svs/index/vamana/build_stats.cpp @@ -0,0 +1,137 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test: the build-time measurement counters. +#include "svs/index/vamana/build_stats.h" + +// Only meaningful with SVS_ENABLE_STATS=1, else this file is an empty translation unit. +#if SVS_ENABLE_STATS + +// svs +#include "svs/core/data/simple.h" +#include "svs/core/distance.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/extensions.h" +#include "svs/index/vamana/prune.h" +#include "svs/index/vamana/vamana_build.h" +#include "svs/lib/narrow.h" +#include "svs/lib/threads.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include + +// Named (not anonymous) so `svs::index::vamana::PruneStrategy` below can name this type. +namespace build_stats_test { + +// Static atomic: DefaultBuildAdaptor default-constructs a fresh Distance per thread. +struct CountingTestDistance { + using compare = std::less<>; + static constexpr bool implicit_broadcast = true; + + template float compute(A a, B b) const { + counter().fetch_add(1, std::memory_order_relaxed); + return svs::distance::compute(svs::distance::DistanceL2{}, a, b); + } + + static std::atomic& counter() { + static std::atomic count{0}; + return count; + } +}; + +svs::data::SimpleData make_random_data(size_t n, size_t dims, uint64_t seed) { + auto data = svs::data::SimpleData(n, dims); + auto rng = std::mt19937_64(seed); + auto distribution = std::uniform_real_distribution(-1.0F, 1.0F); + auto buffer = std::vector(dims); + for (size_t i = 0; i < n; ++i) { + for (float& v : buffer) { + v = distribution(rng); + } + data.set_datum(i, buffer); + } + return data; +} + +} // namespace build_stats_test + +namespace svs::index::vamana { +// Shares QEmd's PruneStrategy (std::less<>), not IP/cosine's (extensions/vamana/gem.h). +template <> struct PruneStrategy { + using type = ProgressivePruneStrategy; +}; +} // namespace svs::index::vamana + +CATCH_TEST_CASE("Build Stats", "[vamana][build_stats]") { + namespace vamana = svs::index::vamana; + + const size_t n_points = 300; + const size_t dims = 16; + auto data = build_stats_test::make_random_data(n_points, dims, 0xB0105); + + // Small graph_max_degree vs window_size/pool_size forces backedge pruning to occur. + auto build_parameters = vamana::VamanaBuildParameters{1.2F, 8, 16, 32, 6, true}; + + auto entry_point_pool = svs::threads::DefaultThreadPool(2); + auto entry_point = svs::lib::narrow( + vamana::extensions::compute_entry_point(data, entry_point_pool) + ); + + auto graph = svs::default_graph(n_points, build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(2); + + build_stats_test::CountingTestDistance::counter() = 0; + vamana::build_stats_reset(); + + auto builder = vamana::VamanaBuilder( + graph, + data, + build_stats_test::CountingTestDistance{}, + build_parameters, + build_pool, + vamana::extensions::estimate_prefetch_parameters(data) + ); + // Two full passes, mirroring VamanaIndex: alpha pinned to 1.0, then the real alpha. + builder.construct(1.0F, entry_point); + builder.construct(build_parameters.alpha, entry_point); + + auto stats = vamana::build_stats_snapshot(); + uint64_t total_compute_calls = build_stats_test::CountingTestDistance::counter().load(); + + CATCH_REQUIRE(stats.inserts == 2 * n_points); + CATCH_REQUIRE( + stats.search_distances + stats.pool_distances + stats.prune_distances + + stats.backedge_prune_distances == + total_compute_calls + ); + + CATCH_REQUIRE(stats.search_distances > 0); + CATCH_REQUIRE(stats.pool_distances > 0); + CATCH_REQUIRE(stats.prune_distances > 0); + CATCH_REQUIRE(stats.backedge_prune_distances > 0); + CATCH_REQUIRE(stats.search_ns > 0); + CATCH_REQUIRE(stats.prune_ns > 0); + CATCH_REQUIRE(stats.backedge_prune_ns > 0); + CATCH_REQUIRE(stats.max_search_distances_per_insert > 0); +} + +#endif // SVS_ENABLE_STATS From e7f686c8bbb5ec4e1b78eb05da68d65d6784968c Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:13:02 -0800 Subject: [PATCH 32/68] feat(distance): add GemRouter for coarse-cluster routing Groups GEM fine centroids into coarse clusters, each with one representative document, and exposes route_query (per-query-token coarse-cluster scoring from the existing fine-centroid dot table, no new dot products) and route_ids (lookup-only, for a document's own centroid ids) plus save/load. QEmd and QChamferExactQuery can now carry an optional shared GemRouter; QChamferExactQuery additionally exposes its per-query table and token count so a search-time consumer can route without recomputing anything. Co-Authored-By: Claude Fable 5.2 --- include/svs/core/distance/gem.h | 36 ++- include/svs/core/distance/gem_router.h | 410 ++++++++++++++++++++++++ tests/CMakeLists.txt | 2 + tests/svs/core/distances/gem_router.cpp | 302 +++++++++++++++++ 4 files changed, 746 insertions(+), 4 deletions(-) create mode 100644 include/svs/core/distance/gem_router.h create mode 100644 tests/svs/core/distances/gem_router.cpp diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index 3bbf13334..a2a0bfb30 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -28,6 +28,7 @@ #include "svs/concepts/vector_set.h" #include "svs/core/distance/distance_core.h" +#include "svs/core/distance/gem_router.h" #include "svs/core/distance/inner_product_matrix.h" #include "svs/lib/avx_detection.h" #include "svs/lib/bfloat16.h" @@ -581,8 +582,14 @@ class QEmd { static constexpr bool implicit_broadcast = true; static constexpr std::string_view name = "gem_qemd"; - explicit QEmd(std::shared_ptr table) - : table_{std::move(table)} {} + explicit QEmd( + std::shared_ptr table, + std::shared_ptr router = nullptr + ) + : table_{std::move(table)} + , router_{std::move(router)} {} + + const std::shared_ptr& router() const { return router_; } // Vector-set path: ids come straight from a contiguous vector set's own storage; T == // uint32_t skips the cast copy (gem_detail::id_span), any other unsigned type costs it. @@ -611,6 +618,7 @@ class QEmd { private: std::shared_ptr table_; + std::shared_ptr router_; }; inline constexpr bool operator==(const QEmd&, const QEmd&) { return true; } @@ -693,8 +701,27 @@ class QChamferExactQuery { static constexpr bool must_fix_argument = true; static constexpr std::string_view name = "gem_qchamfer_exact_query"; - explicit QChamferExactQuery(std::shared_ptr centroids) - : centroids_{std::move(centroids)} {} + explicit QChamferExactQuery( + std::shared_ptr centroids, + std::shared_ptr router = nullptr + ) + : centroids_{std::move(centroids)} + , router_{std::move(router)} { + if (router_ && router_->num_fine() != centroids_->K) { + throw std::invalid_argument( + "QChamferExactQuery: router has " + std::to_string(router_->num_fine()) + + " fine centroids, centroids has K = " + std::to_string(centroids_->K) + ); + } + } + + const std::shared_ptr& router() const { return router_; } + + /// The per-query (K x n_query) dot table built by the last fix_argument() call. + const float* table() const { return table_.data(); } + + /// Number of query tokens the last fix_argument() call built table() for. + size_t query_tokens() const { return n_query_; } // Vector-set path: a query set of raw (unquantized) fp32 token vectors, contiguous // in the set's own storage, with dimensions() equal to the centroid dimension. @@ -759,6 +786,7 @@ class QChamferExactQuery { } std::shared_ptr centroids_; + std::shared_ptr router_; PackedOperand packed_query_; std::vector table_; size_t n_query_ = 0; diff --git a/include/svs/core/distance/gem_router.h b/include/svs/core/distance/gem_router.h new file mode 100644 index 000000000..1627901e0 --- /dev/null +++ b/include/svs/core/distance/gem_router.h @@ -0,0 +1,410 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/concepts/vector_set.h" +#include "svs/lib/exception.h" +#include "svs/lib/threads.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs::distance { + +/// Groups GEM fine centroids into coarse clusters, each with a representative document, +/// so a Vamana search can start from multiple entry points chosen by query-token score. +/// See `svs::index::vamana::extensions::GemRoutedInitializer` (extensions/vamana/gem.h). +struct GemRouter { + static constexpr uint32_t no_representative = std::numeric_limits::max(); + + // coarse_of_fine[f] is the coarse cluster of fine centroid f, < num_coarse(). + std::vector coarse_of_fine; + // representative[c] is a document id, or no_representative if no document touches c. + std::vector representative; + // Coarse clusters probed per query token in route_query. + size_t nprobe = 4; + + size_t num_coarse() const { return representative.size(); } + size_t num_fine() const { return coarse_of_fine.size(); } + + /// Scratch for route_query, reused across calls so a search thread allocates nothing + /// once every buffer has grown to its steady-state size. + struct QueryScratch { + std::vector coarse_score; + std::vector> candidates; + std::vector picked; + }; + + /// Scratch for route_ids, reused the same way as QueryScratch. + struct IdScratch { + std::vector> counts; + }; + + /// Builds representatives from `docs` (one centroid id per token, dimensions() == 1) + /// given a fine-to-coarse assignment. Deterministic regardless of `pool` size: shares + /// are reduced in ascending document-id order, so ties always pick the lowest id. + template + requires std::unsigned_integral + static GemRouter build( + std::vector coarse_of_fine, + size_t num_coarse, + const IdSets& docs, + Pool& pool, + size_t nprobe = 4 + ) { + size_t k = coarse_of_fine.size(); + for (size_t f = 0; f < k; ++f) { + if (coarse_of_fine[f] >= num_coarse) { + throw ANNEXCEPTION( + "GemRouter::build: coarse_of_fine[{}] = {} is not < num_coarse = {}", + f, + coarse_of_fine[f], + num_coarse + ); + } + } + + // (doc id, cluster, share) claims; disjoint per thread, so concatenated not merged. + struct Claim { + uint32_t doc_id; + uint32_t cluster; + double share; + }; + threads::SequentialTLS> claims(pool.size()); + + // Errors are recorded, not thrown: parallel_for doesn't guarantee propagation. + std::atomic bad_id{false}; + std::atomic bad_value{0}; + std::atomic bad_dims{false}; + std::atomic bad_dims_doc{0}; + std::atomic bad_dims_value{0}; + + threads::parallel_for( + pool, + threads::StaticPartition{docs.size()}, + [&](const auto& ids, uint64_t tid) { + auto& out = claims.at(tid); + std::vector> local_counts; + for (size_t i : ids) { + auto doc = docs.get_datum(i); + if (doc.dimensions() != 1) { + bad_dims.store(true, std::memory_order_relaxed); + bad_dims_doc.store(i, std::memory_order_relaxed); + bad_dims_value.store(doc.dimensions(), std::memory_order_relaxed); + continue; + } + size_t card = doc.cardinality(); + if (card == 0) { + continue; + } + local_counts.clear(); + for (size_t t = 0; t < card; ++t) { + uint64_t fine_id = static_cast(doc.token(t)[0]); + if (fine_id >= k) { + bad_id.store(true, std::memory_order_relaxed); + bad_value.store(fine_id, std::memory_order_relaxed); + continue; + } + local_counts.emplace_back(coarse_of_fine[fine_id], uint32_t{1}); + } + std::sort(local_counts.begin(), local_counts.end()); + size_t w = 0; + size_t n = local_counts.size(); + for (size_t j = 0; j < n;) { + size_t j2 = j; + uint32_t total = 0; + while (j2 < n && local_counts[j2].first == local_counts[j].first) { + total += local_counts[j2].second; + ++j2; + } + local_counts[w++] = {local_counts[j].first, total}; + j = j2; + } + for (size_t j = 0; j < w; ++j) { + out.push_back( + {static_cast(i), + local_counts[j].first, + static_cast(local_counts[j].second) / + static_cast(card)} + ); + } + } + } + ); + + if (bad_dims.load()) { + throw ANNEXCEPTION( + "GemRouter::build: document {} has dimensions() = {}, expected 1 (one " + "centroid id per token)", + bad_dims_doc.load(), + bad_dims_value.load() + ); + } + if (bad_id.load()) { + throw ANNEXCEPTION( + "GemRouter::build: document centroid id {} is not < K = {}", + bad_value.load(), + k + ); + } + + std::vector all; + claims.visit([&](const std::vector& part) { + all.insert(all.end(), part.begin(), part.end()); + }); + std::sort(all.begin(), all.end(), [](const Claim& a, const Claim& b) { + return a.doc_id < b.doc_id; + }); + + GemRouter result; + result.coarse_of_fine = std::move(coarse_of_fine); + result.representative.assign(num_coarse, no_representative); + result.nprobe = nprobe; + std::vector best_share(num_coarse, -1.0); + for (const auto& claim : all) { + if (claim.share > best_share[claim.cluster]) { + best_share[claim.cluster] = claim.share; + result.representative[claim.cluster] = claim.doc_id; + } + } + return result; + } + + /// Thread-count convenience overload: builds a `DefaultThreadPool` of + /// `num_threads` threads for the duration of the call. + template + requires std::unsigned_integral + static GemRouter build( + std::vector coarse_of_fine, + size_t num_coarse, + const IdSets& docs, + size_t num_threads, + size_t nprobe = 4 + ) { + auto pool = threads::DefaultThreadPool(num_threads); + return build(std::move(coarse_of_fine), num_coarse, docs, pool, nprobe); + } + + /// Scores cluster c for query token i as the max of `table[f * n_query + i]` over its + /// fine centroids f, picks the `nprobe` best clusters per token (ties to lower id), + /// and writes their representatives to `out`, deduplicated and sorted ascending. + void route_query( + const float* table, + size_t n_query, + std::vector& out, + QueryScratch& scratch + ) const { + size_t nc = num_coarse(); + scratch.coarse_score.assign(nc * n_query, -std::numeric_limits::infinity()); + size_t k = num_fine(); + for (size_t f = 0; f < k; ++f) { + uint32_t c = coarse_of_fine[f]; + const float* row = table + f * n_query; + float* dst = scratch.coarse_score.data() + static_cast(c) * n_query; + for (size_t i = 0; i < n_query; ++i) { + dst[i] = std::max(dst[i], row[i]); + } + } + + scratch.picked.clear(); + size_t take = std::min(nprobe, nc); + for (size_t i = 0; i < n_query; ++i) { + scratch.candidates.clear(); + scratch.candidates.reserve(nc); + for (size_t c = 0; c < nc; ++c) { + scratch.candidates.emplace_back( + scratch.coarse_score[c * n_query + i], static_cast(c) + ); + } + std::partial_sort( + scratch.candidates.begin(), + scratch.candidates.begin() + static_cast(take), + scratch.candidates.end(), + [](const auto& a, const auto& b) { + if (a.first != b.first) { + return a.first > b.first; + } + return a.second < b.second; + } + ); + for (size_t j = 0; j < take; ++j) { + uint32_t rep = representative[scratch.candidates[j].second]; + if (rep != no_representative) { + scratch.picked.push_back(rep); + } + } + } + + out.assign(scratch.picked.begin(), scratch.picked.end()); + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + } + + /// Ranks `ids`' coarse clusters by token count (ties to the lower cluster id), takes + /// the top `max_clusters`, and writes their representatives (deduplicated, sorted + /// ascending, `no_representative` skipped) to `out`. Lookups only, no scoring. + void route_ids( + std::span ids, + size_t max_clusters, + std::vector& out, + IdScratch& scratch + ) const { + scratch.counts.clear(); + for (uint32_t id : ids) { + if (id >= num_fine()) { + throw ANNEXCEPTION( + "GemRouter::route_ids: centroid id {} is not < num_fine() = {}", + id, + num_fine() + ); + } + scratch.counts.emplace_back(coarse_of_fine[id], uint32_t{1}); + } + std::sort(scratch.counts.begin(), scratch.counts.end()); + size_t w = 0; + size_t n = scratch.counts.size(); + for (size_t i = 0; i < n;) { + size_t j = i; + uint32_t total = 0; + while (j < n && scratch.counts[j].first == scratch.counts[i].first) { + total += scratch.counts[j].second; + ++j; + } + scratch.counts[w++] = {scratch.counts[i].first, total}; + i = j; + } + scratch.counts.resize(w); + + size_t take = std::min(max_clusters, scratch.counts.size()); + std::partial_sort( + scratch.counts.begin(), + scratch.counts.begin() + static_cast(take), + scratch.counts.end(), + [](const auto& a, const auto& b) { + if (a.second != b.second) { + return a.second > b.second; + } + return a.first < b.first; + } + ); + + out.clear(); + for (size_t i = 0; i < take; ++i) { + uint32_t rep = representative[scratch.counts[i].first]; + if (rep != no_representative) { + out.push_back(rep); + } + } + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + } + + void save(const std::filesystem::path& path) const { + std::ofstream out(path, std::ios::binary); + if (!out) { + throw ANNEXCEPTION("GemRouter::save: cannot open {}", path.string()); + } + Header header{}; + header.k = num_fine(); + header.num_coarse_val = num_coarse(); + header.nprobe_val = nprobe; + out.write(reinterpret_cast(&header), sizeof(header)); + out.write( + reinterpret_cast(coarse_of_fine.data()), + static_cast(coarse_of_fine.size() * sizeof(uint32_t)) + ); + out.write( + reinterpret_cast(representative.data()), + static_cast(representative.size() * sizeof(uint32_t)) + ); + if (!out) { + throw ANNEXCEPTION("GemRouter::save: short write to {}", path.string()); + } + } + + static GemRouter load(const std::filesystem::path& path) { + std::ifstream in(path, std::ios::binary); + if (!in) { + throw ANNEXCEPTION("GemRouter::load: cannot open {}", path.string()); + } + Header header{}; + in.read(reinterpret_cast(&header), sizeof(header)); + if (!in) { + throw ANNEXCEPTION( + "GemRouter::load: {} is too short to hold a GemRouter header", path.string() + ); + } + if (header.magic != kMagic) { + throw ANNEXCEPTION( + "GemRouter::load: {} has magic {:#x}, expected {:#x}", + path.string(), + header.magic, + kMagic + ); + } + if (header.version != kVersion) { + throw ANNEXCEPTION( + "GemRouter::load: {} has version {}, expected {}", + path.string(), + header.version, + kVersion + ); + } + + GemRouter result; + result.coarse_of_fine.resize(header.k); + result.representative.resize(header.num_coarse_val); + result.nprobe = header.nprobe_val; + in.read( + reinterpret_cast(result.coarse_of_fine.data()), + static_cast(header.k * sizeof(uint32_t)) + ); + in.read( + reinterpret_cast(result.representative.data()), + static_cast(header.num_coarse_val * sizeof(uint32_t)) + ); + if (!in) { + throw ANNEXCEPTION("GemRouter::load: {} is truncated", path.string()); + } + return result; + } + + private: + static constexpr uint64_t kMagic = 0x474d52545231ULL; // "GMRTR1" in ASCII hex-ish. + static constexpr uint32_t kVersion = 1; + + // Trivially copyable on-disk header; field order is the file layout. + struct Header { + uint64_t magic = kMagic; + uint32_t version = kVersion; + uint64_t k = 0; + uint64_t num_coarse_val = 0; + uint64_t nprobe_val = 0; + }; +}; + +} // namespace svs::distance diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 60da2108f..ed498b135 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -118,6 +118,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/core/distances/inner_product_matrix.cpp ${TEST_DIR}/svs/core/distances/cosine.cpp ${TEST_DIR}/svs/core/distances/gem.cpp + ${TEST_DIR}/svs/core/distances/gem_router.cpp ${TEST_DIR}/svs/core/distances/maxsim.cpp ${TEST_DIR}/svs/core/graph.cpp ${TEST_DIR}/svs/core/io/vecs.cpp @@ -140,6 +141,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/index/vamana/build_stats.cpp ${TEST_DIR}/svs/index/vamana/consolidate.cpp ${TEST_DIR}/svs/index/vamana/filter.cpp + ${TEST_DIR}/svs/index/vamana/gem_routing.cpp ${TEST_DIR}/svs/index/vamana/gem_vector_set.cpp ${TEST_DIR}/svs/index/vamana/index.cpp ${TEST_DIR}/svs/index/vamana/maxsim_rerank.cpp diff --git a/tests/svs/core/distances/gem_router.cpp b/tests/svs/core/distances/gem_router.cpp new file mode 100644 index 000000000..c08408312 --- /dev/null +++ b/tests/svs/core/distances/gem_router.cpp @@ -0,0 +1,302 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test +#include "svs/core/distance/gem_router.h" + +// svs +#include "svs/core/data/vector_set_view.h" +#include "svs/lib/threads.h" + +// tests +#include "tests/utils/utils.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using svs::distance::GemRouter; + +constexpr uint32_t NO_REP = GemRouter::no_representative; + +// 6 fine centroids grouped into 4 coarse clusters, leaving cluster 3 with no member. +std::vector hand_coarse_of_fine() { return {0, 0, 1, 1, 2, 2}; } + +// Hand-built corpus where doc3 and doc4 tie for cluster 2; doc3 (lower id) must win. +struct HandCorpus { + std::vector ids{0, 0, 2, 0, 1, 1, 2, 3, 4, 4}; + std::vector offsets{0, 3, 6, 8, 9, 10}; + + svs::data::ConstVectorSetView view() const { + return svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); + } +}; + +// Larger randomized corpus, for the thread-count-independence check. +struct RandomCorpus { + std::vector ids; + std::vector offsets; + + svs::data::ConstVectorSetView view() const { + return svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); + } +}; + +RandomCorpus make_random_corpus(size_t n, size_t k, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution id_dist(0, static_cast(k - 1)); + RandomCorpus corpus; + corpus.offsets.resize(n + 1, 0); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 1 + (i % 7); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + for (size_t t = 0; t < cardinality; ++t) { + corpus.ids.push_back(id_dist(rng)); + } + } + return corpus; +} + +} // namespace + +CATCH_TEST_CASE("GemRouter representative rule", "[distance][gem][gem_router]") { + auto corpus = HandCorpus(); + auto docs = corpus.view(); + + auto router = + GemRouter::build(hand_coarse_of_fine(), /*num_coarse=*/4, docs, /*num_threads=*/1); + + CATCH_REQUIRE(router.num_fine() == 6); + CATCH_REQUIRE(router.num_coarse() == 4); + CATCH_REQUIRE(router.representative[0] == 1); + CATCH_REQUIRE(router.representative[1] == 2); + CATCH_REQUIRE(router.representative[2] == 3); // tie broken to the lower doc id + CATCH_REQUIRE(router.representative[3] == NO_REP); // untouched cluster +} + +CATCH_TEST_CASE( + "GemRouter representative rule rejects bad input", "[distance][gem][gem_router]" +) { + auto corpus = HandCorpus(); + auto docs = corpus.view(); + + auto bad_coarse_of_fine = hand_coarse_of_fine(); + bad_coarse_of_fine[2] = 99; // out of [0, num_coarse) + CATCH_REQUIRE_THROWS(GemRouter::build(bad_coarse_of_fine, 4, docs, size_t{1})); +} + +CATCH_TEST_CASE( + "GemRouter build is independent of thread count", "[distance][gem][gem_router]" +) { + constexpr size_t k = 24; + constexpr size_t num_coarse = 6; + auto corpus = make_random_corpus(400, k, 0xC0FFEE); + auto docs = corpus.view(); + + std::vector coarse_of_fine(k); + std::mt19937 rng(0xFEED); + std::uniform_int_distribution cluster_dist(0, num_coarse - 1); + for (auto& c : coarse_of_fine) { + c = cluster_dist(rng); + } + + auto router_1 = GemRouter::build(coarse_of_fine, num_coarse, docs, size_t{1}); + auto router_4 = GemRouter::build(coarse_of_fine, num_coarse, docs, size_t{4}); + + CATCH_REQUIRE(router_1.representative == router_4.representative); + CATCH_REQUIRE(router_1.coarse_of_fine == router_4.coarse_of_fine); +} + +CATCH_TEST_CASE("GemRouter route_query", "[distance][gem][gem_router]") { + CATCH_SECTION("Hand example: nprobe=1, no_representative is skipped") { + GemRouter router; + router.coarse_of_fine = {0, 0, 1, 1}; + router.representative = {10, NO_REP}; + router.nprobe = 1; + + // table[f * n_query + i], K=4, n_query=2. + std::vector table{ + 5.0F, + 1.0F, // f0 + 3.0F, + 9.0F, // f1 + 2.0F, + 2.0F, // f2 + 8.0F, + 0.0F // f3 + }; + GemRouter::QueryScratch scratch; + std::vector out; + router.route_query(table.data(), 2, out, scratch); + CATCH_REQUIRE(out == std::vector{10}); + } + + CATCH_SECTION("Duplicate representatives across tokens collapse to one entry") { + GemRouter router; + router.coarse_of_fine = {0, 0, 1, 1}; + router.representative = {7, 7}; + router.nprobe = 2; // both clusters always selected + std::vector table{1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F, 7.0F, 8.0F}; + GemRouter::QueryScratch scratch; + std::vector out; + router.route_query(table.data(), 2, out, scratch); + CATCH_REQUIRE(out == std::vector{7}); + } + + CATCH_SECTION("Matches a brute-force reference on a random table") { + constexpr size_t k = 20; + constexpr size_t num_coarse = 5; + constexpr size_t n_query = 6; + constexpr size_t nprobe = 2; + + std::mt19937 rng(0xABCDEF); + std::uniform_int_distribution cluster_dist(0, num_coarse - 1); + std::uniform_real_distribution score_dist(-5.0F, 5.0F); + std::uniform_int_distribution rep_dist(0, 99); + std::bernoulli_distribution has_rep(0.7); + + GemRouter router; + router.coarse_of_fine.resize(k); + for (auto& c : router.coarse_of_fine) { + c = cluster_dist(rng); + } + router.representative.resize(num_coarse); + for (auto& r : router.representative) { + r = has_rep(rng) ? rep_dist(rng) : NO_REP; + } + router.nprobe = nprobe; + + std::vector table(k * n_query); + for (auto& v : table) { + v = score_dist(rng); + } + + GemRouter::QueryScratch scratch; + std::vector out; + router.route_query(table.data(), n_query, out, scratch); + + // Independent reference implementation of the same spec. + std::vector expected; + for (size_t i = 0; i < n_query; ++i) { + std::vector coarse_score( + num_coarse, -std::numeric_limits::infinity() + ); + for (size_t f = 0; f < k; ++f) { + coarse_score[router.coarse_of_fine[f]] = std::max( + coarse_score[router.coarse_of_fine[f]], table[f * n_query + i] + ); + } + std::vector order(num_coarse); + for (size_t c = 0; c < num_coarse; ++c) { + order[c] = static_cast(c); + } + std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) { + if (coarse_score[a] != coarse_score[b]) { + return coarse_score[a] > coarse_score[b]; + } + return a < b; + }); + size_t take = std::min(nprobe, num_coarse); + for (size_t j = 0; j < take; ++j) { + uint32_t rep = router.representative[order[j]]; + if (rep != NO_REP) { + expected.push_back(rep); + } + } + } + std::sort(expected.begin(), expected.end()); + expected.erase(std::unique(expected.begin(), expected.end()), expected.end()); + + CATCH_REQUIRE(out == expected); + CATCH_REQUIRE(std::is_sorted(out.begin(), out.end())); + CATCH_REQUIRE(std::adjacent_find(out.begin(), out.end()) == out.end()); + } +} + +CATCH_TEST_CASE("GemRouter route_ids", "[distance][gem][gem_router]") { + GemRouter router; + router.coarse_of_fine = {0, 0, 1, 1, 2, 2}; + router.representative = {100, 200, 300}; + + CATCH_SECTION("Ranked by count desc, ties to the lower cluster id") { + std::vector ids{0, 0, 2, 4}; + GemRouter::IdScratch scratch; + std::vector out; + + router.route_ids(ids, 2, out, scratch); + CATCH_REQUIRE(out == std::vector{100, 200}); + + router.route_ids(ids, 3, out, scratch); + CATCH_REQUIRE(out == std::vector{100, 200, 300}); + } + + CATCH_SECTION("no_representative clusters are skipped") { + GemRouter sparse = router; + sparse.representative = {100, NO_REP, 300}; + std::vector ids{2, 3}; // both tokens map to coarse1 + GemRouter::IdScratch scratch; + std::vector out; + + sparse.route_ids(ids, 5, out, scratch); + CATCH_REQUIRE(out.empty()); + } +} + +CATCH_TEST_CASE("GemRouter save/load", "[distance][gem][gem_router]") { + svs_test::prepare_temp_directory(); + auto path = svs_test::temp_directory() / "gem_router.bin"; + + GemRouter router; + router.coarse_of_fine = hand_coarse_of_fine(); + router.representative = {1, 2, 3, NO_REP}; + router.nprobe = 7; + + router.save(path); + auto loaded = GemRouter::load(path); + + CATCH_REQUIRE(loaded.coarse_of_fine == router.coarse_of_fine); + CATCH_REQUIRE(loaded.representative == router.representative); + CATCH_REQUIRE(loaded.nprobe == router.nprobe); + + CATCH_SECTION("A corrupted file is rejected") { + // Flip a byte inside the magic field. + std::fstream f(path, std::ios::binary | std::ios::in | std::ios::out); + f.seekp(0); + char byte = 0; + f.read(&byte, 1); + f.seekp(0); + byte = static_cast(~byte); + f.write(&byte, 1); + f.close(); + CATCH_REQUIRE_THROWS(GemRouter::load(path)); + } +} From fae6fea9bf56997a38814071007a595bbb8cab6b Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:14:35 -0800 Subject: [PATCH 33/68] feat(vamana): compute search start nodes per query through an extension hook Adds extensions::search_initializer, a customization point VamanaIndex::greedy_search_closure now calls instead of constructing EntryPointInitializer directly; the default implementation is EntryPointInitializer, so every existing caller is unchanged. Only the static VamanaIndex is touched. Overrides it for QChamferExactQuery with GemRoutedInitializer, which starts from the index's fixed entry points plus, when the distance carries a GemRouter, the routed representatives for the query's already-computed table; with no router it is exactly EntryPointInitializer's behavior. Co-Authored-By: Claude Fable 5.2 --- include/svs/extensions/vamana/gem.h | 80 ++++++ include/svs/index/vamana/extensions.h | 29 ++ include/svs/index/vamana/index.h | 4 +- tests/svs/index/vamana/gem_routing.cpp | 377 +++++++++++++++++++++++++ 4 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 tests/svs/index/vamana/gem_routing.cpp diff --git a/include/svs/extensions/vamana/gem.h b/include/svs/extensions/vamana/gem.h index b17b8efc8..8a647f2d9 100644 --- a/include/svs/extensions/vamana/gem.h +++ b/include/svs/extensions/vamana/gem.h @@ -22,6 +22,13 @@ #include "svs/index/vamana/extensions.h" #include "svs/index/vamana/prune.h" +// stl +#include +#include +#include +#include +#include + namespace svs::index::vamana { // qEMD is "smaller is better" like L2 (std::less<>), so share its MRNG pruning @@ -36,6 +43,68 @@ template <> struct PruneStrategy { using type = ProgressivePruneStrategy; }; +/// Starts from the index's fixed entry points plus, with a GemRouter, the routed +/// representatives for the query's table; falls back to EntryPointInitializer's +/// behavior exactly when there is no router. +template struct GemRoutedInitializer { + template < + typename Buffer, + typename Computer, + graphs::ImmutableMemoryGraph Graph, + typename Builder, + typename Tracker> + void operator()( + Buffer& buffer, + const Computer& computer, + const Graph& graph, + const Builder& builder, + Tracker& tracker + ) const { + using Idx = typename Buffer::index_type; + buffer.clear(); + + auto push = [&](I id) { + auto dist = computer(id); + buffer.push_back(builder(id, dist)); + graph.prefetch_node(id); + tracker.visited(Neighbor{id, dist}, 1); + }; + + const auto& router = distance_->router(); + if (!router) { + for (I id : entry_points_) { + push(id); + } + buffer.sort(); + return; + } + + // thread_local: no per-query allocation once scratch reaches its steady-state size. + thread_local svs::distance::GemRouter::QueryScratch route_scratch; + thread_local std::vector routed; + router->route_query( + distance_->table(), distance_->query_tokens(), routed, route_scratch + ); + + thread_local std::vector merged; + merged.assign(entry_points_.begin(), entry_points_.end()); + for (uint32_t id : routed) { + merged.push_back(static_cast(id)); + } + std::sort(merged.begin(), merged.end()); + merged.erase(std::unique(merged.begin(), merged.end()), merged.end()); + + for (I id : merged) { + push(id); + } + buffer.sort(); + } + + ///// Members + const svs::distance::QChamferExactQuery* distance_; + std::span entry_points_; +}; + namespace extensions { // DefaultBuildAdaptor default-constructs its distance_ member, which drops the @@ -55,5 +124,16 @@ inline DefaultBuildAdaptor svs_invoke( return DefaultBuildAdaptor{distance}; } +// Routes a QChamferExactQuery search; no-op to EntryPointInitializer when unrouted. +template +GemRoutedInitializer svs_invoke( + svs::tag_t, + const Data& /*data*/, + const svs::distance::QChamferExactQuery& distance, + std::span entry_points +) { + return GemRoutedInitializer{&distance, entry_points}; +} + } // namespace extensions } // namespace svs::index::vamana diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 96ffeaa06..d923f59b7 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -27,6 +27,10 @@ #include "svs/lib/preprocessor.h" #include "svs/lib/threads.h" +// stl +#include +#include + namespace svs::index::vamana::extensions { ///// @@ -285,6 +289,31 @@ DefaultBuildAdaptor svs_invoke( ///// SEARCH EXTENSIONS ///// +/// Builds the greedy-search initializer for a query. Default: EntryPointInitializer over +/// the index's fixed entry points; an extension may start per-query instead (GEM, gem.h). +/// Invoked inside greedy_search after fix_argument, so it may read per-query state. +struct SearchInitializerType { + template + auto operator()( + const Data& data, const Distance& distance, std::span entry_points + ) const { + return svs::svs_invoke(*this, data, distance, entry_points); + } +}; + +inline constexpr SearchInitializerType search_initializer{}; + +// Default Implementation. +template +vamana::EntryPointInitializer svs_invoke( + svs::tag_t, + const Data& SVS_UNUSED(data), + const Distance& SVS_UNUSED(distance), + std::span entry_points +) { + return vamana::EntryPointInitializer{entry_points}; +} + /// Preprartion steps for single search. struct VamanaSingleSearchSetupType { template diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index 4d4a47f09..843d75209 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -499,7 +499,9 @@ class VamanaIndex { query, distance, buffer, - vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, + extensions::search_initializer( + data_, distance, lib::as_const_span(entry_point_) + ), internal_search_builder(), prefetch_parameters, cancel diff --git a/tests/svs/index/vamana/gem_routing.cpp b/tests/svs/index/vamana/gem_routing.cpp new file mode 100644 index 000000000..e1183b951 --- /dev/null +++ b/tests/svs/index/vamana/gem_routing.cpp @@ -0,0 +1,377 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Headers under test: the GEM search-routing hook (GemRoutedInitializer, +// search_initializer override). +#include "svs/core/distance/gem.h" +#include "svs/core/distance/gem_router.h" +#include "svs/extensions/vamana/gem.h" + +// svs +#include "svs/core/data/vector_set.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/index.h" +#include "svs/lib/threads.h" + +// tests +#include "tests/utils/utils.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using svs::distance::CentroidTable; +using svs::distance::FineCentroids; +using svs::distance::GemRouter; +using svs::distance::Ground; +using svs::distance::QChamfer; +using svs::distance::QChamferExactQuery; + +constexpr size_t K = 16; +constexpr size_t DIM = 4; + +// K unit vectors, deterministic for a given seed; dim == DIM. +std::vector make_centroids(uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution coord(-1.0F, 1.0F); + std::vector centroids(K * DIM); + for (size_t k = 0; k < K; ++k) { + double norm = 0.0; + for (size_t d = 0; d < DIM; ++d) { + float v = coord(rng); + centroids[k * DIM + d] = v; + norm += static_cast(v) * static_cast(v); + } + norm = std::sqrt(norm); + for (size_t d = 0; d < DIM; ++d) { + centroids[k * DIM + d] = static_cast(centroids[k * DIM + d] / norm); + } + } + return centroids; +} + +struct IdSetCorpus { + std::vector ids; + std::vector offsets; + + size_t size() const { return offsets.size() - 1; } + + svs::data::ConstVectorSetView view() const { + return svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); + } +}; + +IdSetCorpus make_doc_corpus(size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution id_dist(0, static_cast(K - 1)); + IdSetCorpus corpus; + corpus.offsets.resize(n + 1, 0); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 2 + (i % 5); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + for (size_t t = 0; t < cardinality; ++t) { + corpus.ids.push_back(id_dist(rng)); + } + } + return corpus; +} + +// Raw-float query tokens, one per centroid dimension: what QChamferExactQuery expects. +struct QueryCorpus { + std::vector tokens; + std::vector offsets; + + svs::data::ConstVectorSetView view() const { + return svs::data::ConstVectorSetView( + std::span(tokens), std::span(offsets), DIM + ); + } +}; + +// Noisy centroids, so routing/recall has real nearest-centroid structure to exploit. +QueryCorpus +make_query_corpus(const std::vector& centroids, size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution centroid_dist(0, static_cast(K - 1)); + std::normal_distribution noise(0.0F, 0.05F); + + QueryCorpus corpus; + corpus.offsets.resize(n + 1, 0); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 1 + (i % 3); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + for (size_t t = 0; t < cardinality; ++t) { + uint32_t c = centroid_dist(rng); + for (size_t d = 0; d < DIM; ++d) { + corpus.tokens.push_back(centroids[c * DIM + d] + noise(rng)); + } + } + } + return corpus; +} + +std::shared_ptr make_table(const std::vector& centroids) { + return std::make_shared( + CentroidTable::from_centroids(centroids.data(), K, DIM, Ground::Cos, 1) + ); +} + +std::shared_ptr make_fine(const std::vector& centroids) { + return std::make_shared(K, DIM, centroids.data(), 1); +} + +// Deterministic grouping (f % num_coarse); good enough to exercise routing, not realistic. +std::vector round_robin_coarse_of_fine(size_t num_coarse) { + std::vector out(K); + for (size_t f = 0; f < K; ++f) { + out[f] = static_cast(f % num_coarse); + } + return out; +} + +// Minimal greedy-search test double: records pushed ids, nothing else. +struct RecordingBuffer { + using index_type = uint32_t; + std::vector ids; + + void clear() { ids.clear(); } + void push_back(uint32_t id) { ids.push_back(id); } + void sort() { std::sort(ids.begin(), ids.end()); } +}; + +auto dummy_computer() { + return [](uint32_t /*id*/) { return 0.0F; }; +} + +auto id_builder() { + return [](uint32_t id, float /*dist*/) { return id; }; +} + +} // namespace + +CATCH_TEST_CASE( + "GemRoutedInitializer with no router matches EntryPointInitializer", + "[vamana][gem][gem_router]" +) { + auto centroids = make_centroids(0x1111); + auto fine = make_fine(centroids); + QChamferExactQuery distance{fine}; + + auto queries = make_query_corpus(centroids, 1, 0x2222); + auto query = queries.view().get_datum(0); + distance.fix_argument(query); + + auto graph = svs::default_graph(10, 32); + std::vector entry_points{3}; + auto computer = dummy_computer(); + auto builder = id_builder(); + svs::index::vamana::NullTracker tracker; + + RecordingBuffer buf_plain; + svs::index::vamana::EntryPointInitializer plain_init{ + std::span(entry_points)}; + plain_init(buf_plain, computer, graph, builder, tracker); + + RecordingBuffer buf_routed; + svs::index::vamana::GemRoutedInitializer routed_init{ + &distance, std::span(entry_points)}; + routed_init(buf_routed, computer, graph, builder, tracker); + + CATCH_REQUIRE(buf_routed.ids == buf_plain.ids); +} + +CATCH_TEST_CASE( + "GemRoutedInitializer with every cluster unrepresented falls back to the medoid", + "[vamana][gem][gem_router]" +) { + auto centroids = make_centroids(0x3333); + auto fine = make_fine(centroids); + + GemRouter all_norep; + all_norep.coarse_of_fine = round_robin_coarse_of_fine(4); + all_norep.representative.assign(4, GemRouter::no_representative); + all_norep.nprobe = 2; + auto router_ptr = std::make_shared(all_norep); + + QChamferExactQuery distance_no_router{fine}; + QChamferExactQuery distance_all_norep{fine, router_ptr}; + + auto queries = make_query_corpus(centroids, 1, 0x4444); + auto query = queries.view().get_datum(0); + distance_no_router.fix_argument(query); + distance_all_norep.fix_argument(query); + + auto graph = svs::default_graph(10, 32); + std::vector entry_points{5}; + auto computer = dummy_computer(); + auto builder = id_builder(); + svs::index::vamana::NullTracker tracker; + + RecordingBuffer buf_no_router; + svs::index::vamana::GemRoutedInitializer{ + &distance_no_router, std::span(entry_points)}( + buf_no_router, computer, graph, builder, tracker + ); + + RecordingBuffer buf_all_norep; + svs::index::vamana::GemRoutedInitializer{ + &distance_all_norep, std::span(entry_points)}( + buf_all_norep, computer, graph, builder, tracker + ); + + CATCH_REQUIRE(buf_all_norep.ids == buf_no_router.ids); +} + +CATCH_TEST_CASE( + "GemRoutedInitializer with a real router pushes exactly medoid union routed", + "[vamana][gem][gem_router]" +) { + auto centroids = make_centroids(0x5555); + auto fine = make_fine(centroids); + + auto doc_corpus = make_doc_corpus(100, 0x6666); + auto docs = doc_corpus.view(); + + constexpr size_t num_coarse = 4; + auto router = GemRouter::build( + round_robin_coarse_of_fine(num_coarse), num_coarse, docs, size_t{1}, size_t{2} + ); + auto router_ptr = std::make_shared(router); + + QChamferExactQuery distance{fine, router_ptr}; + + auto queries = make_query_corpus(centroids, 1, 0x7777); + auto query = queries.view().get_datum(0); + distance.fix_argument(query); + + std::vector entry_points{7}; + GemRouter::QueryScratch scratch; + std::vector routed; + router.route_query(distance.table(), distance.query_tokens(), routed, scratch); + + std::vector expected(entry_points.begin(), entry_points.end()); + expected.insert(expected.end(), routed.begin(), routed.end()); + std::sort(expected.begin(), expected.end()); + expected.erase(std::unique(expected.begin(), expected.end()), expected.end()); + + auto graph = svs::default_graph(100, 32); + auto computer = dummy_computer(); + auto builder = id_builder(); + svs::index::vamana::NullTracker tracker; + + RecordingBuffer buf; + svs::index::vamana::GemRoutedInitializer{ + &distance, + std::span(entry_points)}(buf, computer, graph, builder, tracker); + + CATCH_REQUIRE(buf.ids == expected); +} + +CATCH_TEST_CASE( + "Routed search is not worse than unrouted on a small index", "[vamana][gem][gem_router]" +) { + auto centroids = make_centroids(0x8888); + auto table = make_table(centroids); + auto fine = make_fine(centroids); + + const size_t n_docs = 150; + auto doc_corpus = make_doc_corpus(n_docs, 0x9999); + auto docs = doc_corpus.view(); + auto queries = make_query_corpus(centroids, 20, 0xAAAA); + auto query_view = queries.view(); + + QChamfer build_dist{table}; + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + const uint32_t entry_point = 0; + + auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + graph, + docs, + build_dist, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(docs) + ); + builder.construct(build_parameters.alpha, entry_point); + + constexpr size_t num_coarse = 6; + auto router = GemRouter::build( + round_robin_coarse_of_fine(num_coarse), num_coarse, docs, size_t{1}, size_t{3} + ); + auto router_ptr = std::make_shared(router); + + auto search_params = [](size_t window) { + return svs::index::vamana::VamanaSearchParameters{{window, window}, false, 0, 0}; + }; + + auto run_search = [&](std::shared_ptr maybe_router) { + QChamferExactQuery dist{fine, std::move(maybe_router)}; + auto pool = svs::threads::DefaultThreadPool(1); + auto graph_copy = graph; + auto index = svs::index::vamana::VamanaIndex( + std::move(graph_copy), docs, entry_point, dist, std::move(pool) + ); + auto sp = search_params(20); + auto scratch = index.scratchspace(sp); + + size_t top1_hits = 0; + for (size_t q = 0; q < query_view.size(); ++q) { + auto query = query_view.get_datum(q); + index.search(query, scratch); + + // Brute-force QChamferExactQuery oracle for this query. + QChamferExactQuery oracle_dist{fine}; + oracle_dist.fix_argument(query); + uint32_t best_id = 0; + float best_score = std::numeric_limits::infinity(); + for (uint32_t id = 0; id < docs.size(); ++id) { + float score = + svs::distance::compute(oracle_dist, query, docs.get_datum(id)); + if (score < best_score) { + best_score = score; + best_id = id; + } + } + if (scratch.buffer[0].id() == best_id) { + ++top1_hits; + } + } + return top1_hits; + }; + + size_t hits_unrouted = run_search(nullptr); + size_t hits_routed = run_search(router_ptr); + + CATCH_REQUIRE(hits_routed >= hits_unrouted); +} From db68f342d11404622db080f34f9f61bbaa49c7f5 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:15:23 -0800 Subject: [PATCH 34/68] fix(vamana): stop taking an L2 medoid over centroid-id sets compute_entry_point's default takes an L2 medoid over data.get_datum(i), which for a centroid-id vector set has no flat .size()/indexing and fails to compile deep inside medioid.h with no indication why. Adds a constrained overload that turns that into a clear diagnostic pointing at the new gem_pooled_entry_point (pools each document to the mean of its fine centroids and takes the L2 medoid of the pooled rows), the entry point a build over centroid-id sets should pass explicitly instead. No existing test relied on the old (uncompilable) path: both GEM Vamana tests already pass their entry point explicitly and build through VamanaBuilder directly, bypassing compute_entry_point. Co-Authored-By: Claude Fable 5.2 --- include/svs/extensions/vamana/gem.h | 80 ++++++++++++++++++++++++++ tests/svs/index/vamana/gem_routing.cpp | 21 ++++++- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/include/svs/extensions/vamana/gem.h b/include/svs/extensions/vamana/gem.h index 8a647f2d9..4b1b0ec18 100644 --- a/include/svs/extensions/vamana/gem.h +++ b/include/svs/extensions/vamana/gem.h @@ -18,15 +18,19 @@ // index/vamana depends on core/distance, not the reverse, so these overrides live here // rather than in core/distance/gem.h. +#include "svs/concepts/vector_set.h" +#include "svs/core/data/simple.h" #include "svs/core/distance/gem.h" #include "svs/index/vamana/extensions.h" #include "svs/index/vamana/prune.h" +#include "svs/lib/exception.h" // stl #include #include #include #include +#include #include namespace svs::index::vamana { @@ -135,5 +139,81 @@ GemRoutedInitializer svs_invoke( return GemRoutedInitializer{&distance, entry_points}; } +// L2 medoid of each document's pooled fine-centroid mean; a real entry point for raw ids. +template + requires std::unsigned_integral +size_t gem_pooled_entry_point( + const IdSets& docs, const svs::distance::FineCentroids& centroids, Pool& pool +) { + size_t n = docs.size(); + size_t dim = centroids.dim; + auto pooled = svs::data::SimpleData(n, dim); + + threads::parallel_for( + pool, + threads::StaticPartition{n}, + [&](const auto& ids, uint64_t /*tid*/) { + std::vector accum(dim); + for (size_t i : ids) { + auto doc = docs.get_datum(i); + if (doc.dimensions() != 1) { + throw ANNEXCEPTION( + "gem_pooled_entry_point: document {} has dimensions() = {}, " + "expected 1 (one centroid id per token)", + i, + doc.dimensions() + ); + } + size_t card = doc.cardinality(); + std::fill(accum.begin(), accum.end(), 0.0); + for (size_t t = 0; t < card; ++t) { + uint64_t cid = static_cast(doc.token(t)[0]); + if (cid >= centroids.K) { + throw ANNEXCEPTION( + "gem_pooled_entry_point: document {} has centroid id {}, " + "not < K = {}", + i, + cid, + centroids.K + ); + } + const svs::BFloat16* row = centroids.data.get() + cid * dim; + for (size_t d = 0; d < dim; ++d) { + accum[d] += static_cast(row[d]); + } + } + auto out = pooled.get_datum(i); + for (size_t d = 0; d < dim; ++d) { + out[d] = card > 0 + ? static_cast(accum[d] / static_cast(card)) + : 0.0F; + } + } + } + ); + + return utils::find_medioid(pooled, pool); +} + +// Turns an opaque medioid.h compile failure on centroid-id sets into a clear diagnostic. +template + requires svs::data::ImmutableVectorSetDataset && + std::unsigned_integral +size_t svs_invoke( + svs::tag_t, + const Data& /*dataset*/, + Pool& /*threadpool*/, + Predicate&& /*predicate*/ +) { + static_assert( + !std::is_same_v, + "compute_entry_point has no L2 medoid for centroid-id vector sets; build one " + "with svs::index::vamana::extensions::gem_pooled_entry_point and pass it to the " + "VamanaIndex constructor directly instead of relying on automatic entry-point " + "computation" + ); + return 0; +} + } // namespace extensions } // namespace svs::index::vamana diff --git a/tests/svs/index/vamana/gem_routing.cpp b/tests/svs/index/vamana/gem_routing.cpp index e1183b951..1791ce755 100644 --- a/tests/svs/index/vamana/gem_routing.cpp +++ b/tests/svs/index/vamana/gem_routing.cpp @@ -14,8 +14,7 @@ * limitations under the License. */ -// Headers under test: the GEM search-routing hook (GemRoutedInitializer, -// search_initializer override). +// Headers under test: GemRoutedInitializer, search_initializer, gem_pooled_entry_point. #include "svs/core/distance/gem.h" #include "svs/core/distance/gem_router.h" #include "svs/extensions/vamana/gem.h" @@ -375,3 +374,21 @@ CATCH_TEST_CASE( CATCH_REQUIRE(hits_routed >= hits_unrouted); } + +CATCH_TEST_CASE("gem_pooled_entry_point", "[vamana][gem][gem_router]") { + // Four unit vectors in 2D, matching core/distances/gem.cpp's hand example exactly. + constexpr size_t dim = 2; + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F, 0.0F, -1.0F}; + FineCentroids fine(4, dim, flat.data(), 1); + + // doc0's mean (0.5,0.5) is nearest the dataset mean (1/3,0), so doc0 is the medoid. + std::vector ids{0, 1, 2, 3, 0}; + std::vector offsets{0, 2, 4, 5}; + auto docs = svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); + + auto pool = svs::threads::DefaultThreadPool(1); + size_t entry = svs::index::vamana::extensions::gem_pooled_entry_point(docs, fine, pool); + CATCH_REQUIRE(entry == 0); +} From 152d320e083c879f72156fbcba444eae96a38af3 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 02:58:16 -0800 Subject: [PATCH 35/68] feat(vamana): time the MaxSim re-rank apart from the graph search behind the stats flag Co-Authored-By: Claude Sonnet 5 --- include/svs/extensions/vamana/maxsim.h | 60 ++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/include/svs/extensions/vamana/maxsim.h b/include/svs/extensions/vamana/maxsim.h index 9e59ebc28..41986956c 100644 --- a/include/svs/extensions/vamana/maxsim.h +++ b/include/svs/extensions/vamana/maxsim.h @@ -25,12 +25,49 @@ // stl #include +#include #include #include +#include #include +#if defined(SVS_ENABLE_STATS) && SVS_ENABLE_STATS +#include +#endif + namespace svs::index::vamana::extensions { +#if defined(SVS_ENABLE_STATS) && SVS_ENABLE_STATS + +/// Process-wide re-rank timing accumulated since the last `maxsim_rerank_stats_reset()`. +struct MaxSimRerankStats { + uint64_t queries; + uint64_t search_ns; + uint64_t rerank_ns; +}; + +namespace stats_detail { +// Process-wide, not per-index: simpler than plumbing counters through every thread. +inline std::atomic maxsim_queries{0}; +inline std::atomic maxsim_search_ns{0}; +inline std::atomic maxsim_rerank_ns{0}; +} // namespace stats_detail + +inline MaxSimRerankStats maxsim_rerank_stats_snapshot() { + return MaxSimRerankStats{ + stats_detail::maxsim_queries.load(std::memory_order_relaxed), + stats_detail::maxsim_search_ns.load(std::memory_order_relaxed), + stats_detail::maxsim_rerank_ns.load(std::memory_order_relaxed)}; +} + +inline void maxsim_rerank_stats_reset() { + stats_detail::maxsim_queries.store(0, std::memory_order_relaxed); + stats_detail::maxsim_search_ns.store(0, std::memory_order_relaxed); + stats_detail::maxsim_rerank_ns.store(0, std::memory_order_relaxed); +} + +#endif + namespace detail { // The intended caller's graph-search distance already consumes raw query tokens, so the @@ -109,7 +146,13 @@ template < } auto accessor = data::GetDatumAccessor(); +#if defined(SVS_ENABLE_STATS) && SVS_ENABLE_STATS + auto t_start = std::chrono::steady_clock::now(); + search(query, accessor, distance, search_buffer); + auto t_search_done = std::chrono::steady_clock::now(); +#else search(query, accessor, distance, search_buffer); +#endif if (cancel()) { return; @@ -134,6 +177,23 @@ template < } detail::resort_by_maxsim(search_buffer); + +#if defined(SVS_ENABLE_STATS) && SVS_ENABLE_STATS + auto t_rerank_done = std::chrono::steady_clock::now(); + auto search_ns = + std::chrono::duration_cast(t_search_done - t_start) + .count(); + auto rerank_ns = + std::chrono::duration_cast(t_rerank_done - t_search_done) + .count(); + stats_detail::maxsim_queries.fetch_add(1, std::memory_order_relaxed); + stats_detail::maxsim_search_ns.fetch_add( + static_cast(search_ns), std::memory_order_relaxed + ); + stats_detail::maxsim_rerank_ns.fetch_add( + static_cast(rerank_ns), std::memory_order_relaxed + ); +#endif } // The default `per_thread_batch_search` (extensions.h) dispatches per query through the From 78b4c599d8e2796b94018feeb393bd80b7a2eafe Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:37:45 -0800 Subject: [PATCH 36/68] fix(distance): stop the vendored POT macros leaking out of gem.h network_simplex_simple.h #defines EPSILON/_EPSILON/MAX_DEBUG_ITER with no matching #undef, so any header included afterward in the same TU gets those identifiers textually rewritten. gem.h is the only SVS file that includes the vendored header, so #undef them there, right after the include. Co-Authored-By: Claude Sonnet 5 --- include/svs/core/distance/gem.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index a2a0bfb30..1ee29fa66 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -26,6 +26,11 @@ #include "svs/third-party/pot/network_simplex_simple.h" #pragma GCC diagnostic pop +// network_simplex_simple.h leaves these #defined with no #undef; they mangle identifiers. +#undef EPSILON +#undef _EPSILON +#undef MAX_DEBUG_ITER + #include "svs/concepts/vector_set.h" #include "svs/core/distance/distance_core.h" #include "svs/core/distance/gem_router.h" From fc93ea3d7563994d9ea96f565b0a6f151df0067c Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:33:30 -0800 Subject: [PATCH 37/68] feat(quantization): train the GEM codebook and assign tokens inside SVS Adds svs::quantization::gem::codebook.h: the ColBERT k-means codebook trainer (plain-mean L2, 4 iterations, centroids normalized once, matching faiss's default procedure rather than kmeans_training.h's IVF-flavored update), exact and bf16-shortlist token assigners, a brute-force mismatch checker, coarse clustering of the fine centroids, and the quantize() entry point that ties them into FineCentroids and CentroidTable. The assigners size their per-thread score buffer from a byte budget, not a row count: a fixed 10,000-row batch is 5.2 GB per thread at K = 131,072. The bf16-shortlist assigner is the default; it measured faster than the GEMM one and matched exact float32 on the checked sample. Co-Authored-By: Claude Sonnet 5 Co-Authored-By: Claude Opus 5.5 --- include/svs/quantization/gem/codebook.h | 836 ++++++++++++++++++++++++ tests/CMakeLists.txt | 7 + tests/svs/quantization/gem/codebook.cpp | 501 ++++++++++++++ 3 files changed, 1344 insertions(+) create mode 100644 include/svs/quantization/gem/codebook.h create mode 100644 tests/svs/quantization/gem/codebook.cpp diff --git a/include/svs/quantization/gem/codebook.h b/include/svs/quantization/gem/codebook.h new file mode 100644 index 000000000..f94ef03c5 --- /dev/null +++ b/include/svs/quantization/gem/codebook.h @@ -0,0 +1,836 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if SVS_ENABLE_MULTI_VECTOR + +// svs +#include "svs/concepts/data.h" +#include "svs/concepts/vector_set.h" +#include "svs/core/data/simple.h" +#include "svs/core/data/vector_set.h" +#include "svs/core/distance/gem.h" +#include "svs/core/distance/inner_product_matrix.h" +#include "svs/core/kmeans_training.h" +#include "svs/lib/bfloat16.h" +#include "svs/lib/exception.h" +#include "svs/lib/float16.h" +#include "svs/lib/misc.h" +#include "svs/lib/narrow.h" +#include "svs/lib/threads.h" +#include "svs/lib/timing.h" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Trains the codebook GEM assigns tokens to, matching faiss's ColBERT k-means indexer. +namespace svs::quantization::gem { + +namespace detail { + +// Exact floor(sqrt(x)): std::sqrt seeds it, then local search corrects any libm rounding. +inline size_t isqrt_floor(size_t x) { + if (x == 0) { + return 0; + } + size_t r = static_cast(std::sqrt(static_cast(x))); + while (r > 0 && r > x / r) { + --r; + } + while ((r + 1) <= x / (r + 1)) { + ++r; + } + return r; +} + +} // namespace detail + +/// K = 2^floor(log2(16 * sqrt(num_tokens))), ColBERT's indexer rule. Uses std::frexp's +/// exact mantissa/exponent split rather than std::log2, since log2 of an exact power of +/// two is not guaranteed correctly rounded on every libm. +inline size_t colbert_num_partitions(size_t num_tokens) { + if (num_tokens == 0) { + throw ANNEXCEPTION("colbert_num_partitions: num_tokens must be positive, got 0"); + } + double raw = 16.0 * std::sqrt(static_cast(num_tokens)); + int exponent = 0; + std::frexp(raw, &exponent); + // mantissa in [0.5,1) means floor(log2(raw)) == exponent - 1 exactly. + if (exponent < 1) { + throw ANNEXCEPTION( + "colbert_num_partitions: num_tokens={} is too small to produce a positive " + "centroid count", + num_tokens + ); + } + return size_t{1} << (exponent - 1); +} + +/// num_sampled_passages = min(1 + floor(16 * sqrt(120 * num_passages)), num_passages). +/// 16 * sqrt(120 * n) == sqrt(30720 * n) exactly, so the floor uses integer sqrt +/// (detail::isqrt_floor) instead of floating log2/sqrt, avoiding rounding ambiguity. +inline size_t colbert_sample_size(size_t num_passages) { + if (num_passages == 0) { + throw ANNEXCEPTION("colbert_sample_size: num_passages must be positive, got 0"); + } + size_t raw = detail::isqrt_floor(30720ULL * num_passages); + return std::min(size_t{1} + raw, num_passages); +} + +/// Which kernel assign_tokens/quantize use to find each token's nearest centroid. +enum class Assigner { ExactGemm, InnerProductMatrix }; + +/// Parameters for codebook training, assignment and the coarse-cluster side table. +struct CodebookParameters { + /// Number of fine centroids K; 0 selects colbert_num_partitions from the token count. + size_t num_centroids = 0; + size_t kmeans_iterations = 4; + /// GEMM chunk size for training's shared buffer; assign_tokens sizes its own instead. + size_t minibatch_size = 10'000; + uint32_t seed = 42; + double heldout_fraction = 0.05; + size_t heldout_cap = 50'000; + /// Number of coarse clusters. 0 selects max(1, num_centroids * 5 / 32). + size_t num_coarse = 0; + size_t coarse_iterations = 4; + /// Default InnerProductMatrix measured faster than ExactGemm and matched it exactly. + Assigner assigner = Assigner::InnerProductMatrix; + size_t assign_candidates = 8; + svs::distance::Ground ground = svs::distance::Ground::Cos; + /// Per-thread score-buffer budget; fixed-row sizing once hit 587 GB at K=131'072. + size_t assign_buffer_bytes = 32 * 1024 * 1024; +}; + +/// Wall-clock seconds for each phase, measured with std::chrono::steady_clock. +struct QuantizeTimings { + double sample = 0.0; + double train = 0.0; + double normalize = 0.0; + double assign = 0.0; + double ground_table = 0.0; + double coarse = 0.0; + double total = 0.0; + std::vector train_iterations; +}; + +/// Any contiguous vector-set dataset of float or Float16 tokens: VectorSetData and +/// ConstVectorSetView both satisfy this without copying their token storage. +template +concept TokenVectorStore = + svs::data::ImmutableVectorSetDataset && + svs::data::ContiguousVectorSet && + (std::is_same_v || + std::is_same_v) && + requires(const Data& d) { + { d.total_tokens() } -> std::convertible_to; + { d.cardinality(size_t{0}) } -> std::convertible_to; + }; + +namespace detail { + +// Sum of squared L2 distances to assigned centroids, accumulated in double; O(n*dim). +template +double compute_objective( + const Data& data, + const svs::data::SimpleData& centroids, + const std::vector& assignments +) { + double total = 0.0; + for (auto i : data.eachindex()) { + auto datum = data.get_datum(i); + auto c = centroids.get_datum(assignments[i]); + double sq = 0.0; + for (size_t d = 0, dmax = datum.size(); d < dmax; ++d) { + double diff = static_cast(datum[d]) - static_cast(c[d]); + sq += diff * diff; + } + total += sq; + } + return total; +} + +// Plain mean, matching faiss, not svs::kmeans::centroid_adjustment's IVF-owned formula. +template +void plain_mean_update( + const Data& data, + svs::data::SimpleData& centroids, + const std::vector& assignments, + std::vector& counts, + Pool& threadpool +) { + size_t n_threads = threadpool.size(); + size_t n_centroids = centroids.size(); + std::fill(counts.begin(), counts.end(), size_t{0}); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_centroids}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + auto c = centroids.get_datum(i); + std::fill(c.begin(), c.end(), 0.0F); + } + } + ); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_threads}, + [&](auto /*indices*/, auto tid) { + size_t centroid_start = (n_centroids * tid) / n_threads; + size_t centroid_end = (n_centroids * (tid + 1)) / n_threads; + for (auto i : data.eachindex()) { + auto assignment = assignments[i]; + if (assignment >= centroid_start && assignment < centroid_end) { + counts.at(assignment)++; + auto datum = data.get_datum(i); + auto c = centroids.get_datum(assignment); + for (size_t p = 0, pmax = c.size(); p < pmax; ++p) { + c[p] += static_cast(datum[p]); + } + } + } + } + ); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_centroids}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + if (counts.at(i) != 0) { + auto c = centroids.get_datum(i); + float norm = 1.0F / static_cast(counts.at(i)); + for (size_t p = 0, pmax = c.size(); p < pmax; ++p) { + c[p] *= norm; + } + } + } + } + ); +} + +struct L2KMeansResult { + svs::data::SimpleData centroids; + double objective = 0.0; + std::vector iteration_seconds; +}; + +// Shared L2/plain-mean trainer; takes Data by non-const ref, so callers pass a local copy. +template +L2KMeansResult run_l2_kmeans( + Data& data, + size_t num_clusters, + size_t iterations, + size_t minibatch_size, + std::mt19937& rng, + Pool& threadpool +) { + size_t n = data.size(); + if (n < num_clusters) { + throw ANNEXCEPTION( + "run_l2_kmeans: only {} points available, need at least num_clusters={}", + n, + num_clusters + ); + } + // faiss subsamples past 256/centroid; throws instead of silently diverging from it. + if (n > num_clusters * 256) { + throw ANNEXCEPTION( + "run_l2_kmeans: {} points exceed faiss's 256-per-centroid training bound for " + "num_clusters={} (max {}); subsampling instead of throwing would silently " + "differ from the reference training procedure", + n, + num_clusters, + num_clusters * 256 + ); + } + + std::vector id_scratch(num_clusters); + auto centroids = + svs::kmeans::init_centroids(data, id_scratch, num_clusters, rng, threadpool); + + size_t batch_cap = std::min(minibatch_size, n); + auto matmul_results = svs::data::SimpleData(batch_cap, num_clusters); + std::vector data_norm; + svs::kmeans::generate_norms(data, data_norm, threadpool); + std::vector centroids_norm; + std::vector assignments(n); + std::vector counts(num_clusters); + auto distance = svs::DistanceL2(); + svs::lib::Timer scratch_timer; + + double objective = 0.0; + std::vector iteration_seconds; + iteration_seconds.reserve(iterations); + for (size_t iter = 0; iter < iterations; ++iter) { + auto iter_start = std::chrono::steady_clock::now(); + svs::kmeans::generate_norms(centroids, centroids_norm, threadpool); + + size_t num_batches = svs::lib::div_round_up(n, batch_cap); + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = threads::UnitRange{ + batch * batch_cap, std::min((batch + 1) * batch_cap, n)}; + auto data_batch = data::make_view(data, this_batch); + svs::kmeans::centroid_assignment( + data_batch, + data_norm, + this_batch, + distance, + centroids, + centroids_norm, + assignments, + matmul_results, + threadpool, + scratch_timer + ); + } + + // Objective is defined on the last pass, against the centroids before its update. + if (iter + 1 == iterations) { + objective = compute_objective(data, centroids, assignments); + } + + plain_mean_update(data, centroids, assignments, counts, threadpool); + svs::kmeans::centroid_split( + data, centroids, counts, rng, threadpool, scratch_timer + ); + + iteration_seconds.push_back( + std::chrono::duration(std::chrono::steady_clock::now() - iter_start) + .count() + ); + } + + return L2KMeansResult{std::move(centroids), objective, std::move(iteration_seconds)}; +} + +} // namespace detail + +/// Result of train_codebook: the normalized float32 centroids plus training diagnostics. +struct TrainedCodebook { + svs::data::SimpleData centroids; + double objective = 0.0; + size_t num_sampled_passages = 0; + size_t num_training_tokens = 0; + QuantizeTimings timings; +}; + +/// Steps 1-4: resolve K, sample passages, gather/shuffle/hold out tokens, run plain-mean +/// L2 k-means, then normalize once. Only sampled tokens convert to float32, so a +/// memory-mapped Float16 store is never copied in full. +template +TrainedCodebook +train_codebook(const Data& tokens, const CodebookParameters& parameters, Pool& threadpool) { + auto total_start = std::chrono::steady_clock::now(); + QuantizeTimings timings; + + size_t dim = tokens.dimensions(); + size_t num_passages = tokens.size(); + size_t num_total_tokens = tokens.total_tokens(); + if (num_passages == 0 || num_total_tokens == 0) { + throw ANNEXCEPTION( + "train_codebook: tokens dataset is empty ({} passages, {} tokens)", + num_passages, + num_total_tokens + ); + } + + size_t K = parameters.num_centroids == 0 ? colbert_num_partitions(num_total_tokens) + : parameters.num_centroids; + constexpr size_t max_id = static_cast(std::numeric_limits::max()); + if (K > max_id) { + throw ANNEXCEPTION( + "train_codebook: K={} exceeds the uint32_t centroid-id range (max {})", + K, + max_id + ); + } + + size_t sample_size = colbert_sample_size(num_passages); + std::mt19937 rng(parameters.seed); + + std::vector passage_ids(sample_size); + svs::kmeans::generate_unique_ids(passage_ids, num_passages, rng); + std::sort(passage_ids.begin(), passage_ids.end()); + + size_t total_sample_tokens = 0; + for (size_t pid : passage_ids) { + total_sample_tokens += tokens.cardinality(pid); + } + + struct TokenRef { + uint32_t sample_idx; + uint32_t token_idx; + }; + std::vector refs; + refs.reserve(total_sample_tokens); + for (size_t s = 0; s < sample_size; ++s) { + size_t card = tokens.cardinality(passage_ids[s]); + for (size_t t = 0; t < card; ++t) { + refs.push_back( + {svs::lib::narrow_cast(s), svs::lib::narrow_cast(t)} + ); + } + } + std::shuffle(refs.begin(), refs.end(), rng); + + size_t heldout = std::min( + parameters.heldout_cap, + static_cast( + parameters.heldout_fraction * static_cast(total_sample_tokens) + ) + ); + size_t num_training = total_sample_tokens - heldout; + + if (num_training < K) { + throw ANNEXCEPTION( + "train_codebook: only {} training tokens sampled (after a {} token held-out " + "split), need at least K={}", + num_training, + heldout, + K + ); + } + + auto sample_start = std::chrono::steady_clock::now(); + auto trainset = svs::data::SimpleData(num_training, dim); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_training}, + [&](auto indices, auto /*tid*/) { + std::vector row(dim); + for (auto i : indices) { + auto ref = refs[i]; + size_t pid = passage_ids[ref.sample_idx]; + auto token = tokens.get_datum(pid).token(ref.token_idx); + for (size_t d = 0; d < dim; ++d) { + row[d] = static_cast(token[d]); + } + trainset.set_datum(i, row); + } + } + ); + timings.sample = + std::chrono::duration(std::chrono::steady_clock::now() - sample_start) + .count(); + + auto train_start = std::chrono::steady_clock::now(); + auto kmeans_result = detail::run_l2_kmeans( + trainset, + K, + parameters.kmeans_iterations, + parameters.minibatch_size, + rng, + threadpool + ); + timings.train = + std::chrono::duration(std::chrono::steady_clock::now() - train_start) + .count(); + timings.train_iterations = std::move(kmeans_result.iteration_seconds); + + auto normalize_start = std::chrono::steady_clock::now(); + svs::lib::Timer scratch_timer; + svs::kmeans::normalize_centroids(kmeans_result.centroids, threadpool, scratch_timer); + timings.normalize = + std::chrono::duration(std::chrono::steady_clock::now() - normalize_start) + .count(); + + timings.total = + std::chrono::duration(std::chrono::steady_clock::now() - total_start) + .count(); + + return TrainedCodebook{ + std::move(kmeans_result.centroids), + kmeans_result.objective, + sample_size, + num_training, + timings}; +} + +/// Step 5: assign every document token to its nearest (argmax inner product) normalized +/// centroid, lowest id breaking ties. Threads own disjoint passage ranges and write only +/// their own tokens' ids, so the result never depends on thread count. +template +svs::data::VectorSetData assign_tokens( + const Data& tokens, + const svs::data::SimpleData& centroids, + const CodebookParameters& parameters, + Pool& threadpool +) { + size_t dim = tokens.dimensions(); + if (centroids.dimensions() != dim) { + throw ANNEXCEPTION( + "assign_tokens: token dimension {} does not match centroid dimension {}", + dim, + centroids.dimensions() + ); + } + size_t K = centroids.size(); + constexpr size_t max_id = static_cast(std::numeric_limits::max()); + if (K > max_id) { + throw ANNEXCEPTION( + "assign_tokens: K={} exceeds the uint32_t centroid-id range (max {})", K, max_id + ); + } + + size_t num_passages = tokens.size(); + std::vector offsets(num_passages + 1, 0); + for (size_t p = 0; p < num_passages; ++p) { + offsets[p + 1] = offsets[p] + tokens.cardinality(p); + } + size_t total_tokens = lib::narrow(offsets.back()); + + auto id_storage = svs::data::SimpleData(total_tokens, 1); + + svs::distance::PackedOperand packed_centroids; + if (parameters.assigner == Assigner::InnerProductMatrix) { + packed_centroids.pack_rounded_to_bf16(centroids.data(), K, dim); + } + + size_t candidates = std::max(size_t{1}, std::min(parameters.assign_candidates, K)); + + // scores is batch_cap * K floats; derive batch_cap from assign_buffer_bytes, not K. + constexpr size_t max_assign_batch_rows = 4096; + size_t bytes_per_row = K * sizeof(float); + size_t rows_by_budget = + std::max(size_t{1}, parameters.assign_buffer_bytes / bytes_per_row); + size_t batch_cap = std::min(rows_by_budget, max_assign_batch_rows); + batch_cap = std::min(batch_cap, std::max(size_t{1}, total_tokens)); + + threads::parallel_for( + threadpool, + threads::StaticPartition{num_passages}, + [&](auto indices, auto /*tid*/) { + std::vector batch_floats(batch_cap * dim); + std::vector batch_global_idx(batch_cap); + std::vector scores(batch_cap * K); + std::vector batch_bf16; + std::vector shortlist(K); + if (parameters.assigner == Assigner::InnerProductMatrix) { + batch_bf16.resize(batch_cap * dim); + } + size_t count = 0; + + auto flush = [&]() { + if (count == 0) { + return; + } + if (parameters.assigner == Assigner::ExactGemm) { + svs::kmeans::compute_matmul( + batch_floats.data(), centroids.data(), scores.data(), count, K, dim + ); + for (size_t r = 0; r < count; ++r) { + const float* row = scores.data() + r * K; + uint32_t best = 0; + float best_score = row[0]; + for (size_t j = 1; j < K; ++j) { + if (row[j] > best_score) { + best_score = row[j]; + best = svs::lib::narrow_cast(j); + } + } + id_storage.get_datum(batch_global_idx[r])[0] = best; + } + } else { + for (size_t r = 0; r < count; ++r) { + const float* src = batch_floats.data() + r * dim; + svs::BFloat16* dst = batch_bf16.data() + r * dim; + for (size_t d = 0; d < dim; ++d) { + dst[d] = svs::round_to_bfloat16(src[d]); + } + } + svs::distance::InnerProductMatrix::compute_packed( + batch_bf16.data(), count, packed_centroids, scores.data() + ); + for (size_t r = 0; r < count; ++r) { + const float* row = scores.data() + r * K; + for (size_t j = 0; j < K; ++j) { + shortlist[j] = svs::lib::narrow_cast(j); + } + std::partial_sort( + shortlist.begin(), + shortlist.begin() + static_cast(candidates), + shortlist.end(), + [&](uint32_t a, uint32_t b) { return row[a] > row[b]; } + ); + // Ascending by id so the rescore below ties like ExactGemm does. + std::sort( + shortlist.begin(), + shortlist.begin() + static_cast(candidates) + ); + + const float* token_row = batch_floats.data() + r * dim; + uint32_t best = shortlist[0]; + float best_dot = -std::numeric_limits::infinity(); + for (size_t c = 0; c < candidates; ++c) { + uint32_t cid = shortlist[c]; + auto crow = centroids.get_datum(cid); + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += token_row[d] * crow[d]; + } + if (dot > best_dot) { + best_dot = dot; + best = cid; + } + } + id_storage.get_datum(batch_global_idx[r])[0] = best; + } + } + count = 0; + }; + + for (auto p : indices) { + auto doc = tokens.get_datum(p); + size_t card = doc.cardinality(); + for (size_t t = 0; t < card; ++t) { + auto token = doc.token(t); + float* dst = batch_floats.data() + count * dim; + for (size_t d = 0; d < dim; ++d) { + dst[d] = static_cast(token[d]); + } + batch_global_idx[count] = offsets[p] + t; + ++count; + if (count == batch_cap) { + flush(); + } + } + } + flush(); + } + ); + + return svs::data::VectorSetData(std::move(id_storage), offsets); +} + +/// Mismatches between the stored ids and a brute-force exact float32 argmax, on a random +/// sample of tokens. Uses plain loops only, sharing no kernel with either assigner, so it +/// serves as an independent check on both. +struct MismatchReport { + size_t mismatches = 0; + size_t sample_size = 0; +}; + +template +MismatchReport count_assignment_mismatches( + const Data& tokens, + const svs::data::SimpleData& centroids, + const svs::data::VectorSetData& ids, + size_t sample_size, + uint32_t seed +) { + size_t dim = tokens.dimensions(); + size_t K = centroids.size(); + auto offsets = ids.offsets(); + size_t total_tokens = lib::narrow(offsets.back()); + sample_size = std::min(sample_size, total_tokens); + + std::mt19937 rng(seed); + std::uniform_int_distribution dist(0, total_tokens == 0 ? 0 : total_tokens - 1); + + size_t mismatches = 0; + for (size_t s = 0; s < sample_size; ++s) { + size_t global = dist(rng); + size_t passage = static_cast( + std::upper_bound(offsets.begin(), offsets.end(), global) - offsets.begin() - 1 + ); + size_t local = global - lib::narrow(offsets[passage]); + + auto token = tokens.get_datum(passage).token(local); + uint32_t best = 0; + float best_dot = -std::numeric_limits::infinity(); + for (size_t j = 0; j < K; ++j) { + auto c = centroids.get_datum(j); + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += static_cast(token[d]) * c[d]; + } + if (dot > best_dot) { + best_dot = dot; + best = svs::lib::narrow_cast(j); + } + } + + uint32_t stored = ids.get_datum(passage).token(local)[0]; + if (stored != best) { + ++mismatches; + } + } + return MismatchReport{mismatches, sample_size}; +} + +/// Clusters fine centroids into coarse groups via the same L2/plain-mean trainer, then +/// an exact nearest-coarse assignment. `centroids` is read-only; a local copy is +/// clustered since run_l2_kmeans takes Data by non-const ref. +struct CoarseClusters { + std::vector coarse_of_fine; + size_t num_coarse = 0; +}; + +template +CoarseClusters coarse_clusters( + const svs::data::SimpleData& centroids, + const CodebookParameters& parameters, + Pool& threadpool +) { + size_t K = centroids.size(); + size_t dim = centroids.dimensions(); + size_t num_coarse = parameters.num_coarse == 0 ? std::max(1, K * 5 / 32) + : parameters.num_coarse; + if (num_coarse > K) { + throw ANNEXCEPTION( + "coarse_clusters: num_coarse={} exceeds the number of fine centroids {}", + num_coarse, + K + ); + } + + // Offset from the fine-training seed so the two never share a random stream. + std::mt19937 rng(parameters.seed + 1); + auto working_copy = centroids; + auto kmeans_result = detail::run_l2_kmeans( + working_copy, + num_coarse, + parameters.coarse_iterations, + parameters.minibatch_size, + rng, + threadpool + ); + svs::lib::Timer scratch_timer; + svs::kmeans::normalize_centroids(kmeans_result.centroids, threadpool, scratch_timer); + + std::vector coarse_of_fine(K); + threads::parallel_for( + threadpool, + threads::StaticPartition{K}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + auto c = centroids.get_datum(i); + uint32_t best = 0; + float best_dot = -std::numeric_limits::infinity(); + for (size_t j = 0; j < num_coarse; ++j) { + auto cc = kmeans_result.centroids.get_datum(j); + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += c[d] * cc[d]; + } + if (dot > best_dot) { + best_dot = dot; + best = svs::lib::narrow_cast(j); + } + } + coarse_of_fine[i] = best; + } + } + ); + + return CoarseClusters{std::move(coarse_of_fine), num_coarse}; +} + +/// Full pipeline: train, assign, build the bf16 FineCentroids and the pairwise +/// CentroidTable, and cluster the fine centroids themselves. The single entry point a +/// driver calls to turn a raw token store into everything GEM's distances need. +struct Quantized { + svs::data::SimpleData centroids; + std::shared_ptr fine_centroids; + std::shared_ptr centroid_table; + svs::data::VectorSetData ids; + std::vector coarse_of_fine; + size_t num_coarse = 0; + size_t K = 0; + double objective = 0.0; + size_t num_sampled_passages = 0; + size_t num_training_tokens = 0; + QuantizeTimings timings; +}; + +template +Quantized +quantize(const Data& tokens, const CodebookParameters& parameters, Pool& threadpool) { + auto total_start = std::chrono::steady_clock::now(); + + auto trained = train_codebook(tokens, parameters, threadpool); + QuantizeTimings timings = trained.timings; + + auto assign_start = std::chrono::steady_clock::now(); + auto ids = assign_tokens(tokens, trained.centroids, parameters, threadpool); + timings.assign = + std::chrono::duration(std::chrono::steady_clock::now() - assign_start) + .count(); + + size_t K = trained.centroids.size(); + size_t dim = trained.centroids.dimensions(); + size_t num_threads = threadpool.size(); + + // Timed with the pairwise table build; neither has its own QuantizeTimings field. + auto ground_start = std::chrono::steady_clock::now(); + auto fine_centroids = std::make_shared( + K, dim, trained.centroids.data(), num_threads + ); + auto centroid_table = std::make_shared( + svs::distance::CentroidTable::from_centroids( + trained.centroids.data(), K, dim, parameters.ground, num_threads + ) + ); + timings.ground_table = + std::chrono::duration(std::chrono::steady_clock::now() - ground_start) + .count(); + + auto coarse_start = std::chrono::steady_clock::now(); + auto coarse = coarse_clusters(trained.centroids, parameters, threadpool); + timings.coarse = + std::chrono::duration(std::chrono::steady_clock::now() - coarse_start) + .count(); + + timings.total = + std::chrono::duration(std::chrono::steady_clock::now() - total_start) + .count(); + + return Quantized{ + std::move(trained.centroids), + std::move(fine_centroids), + std::move(centroid_table), + std::move(ids), + std::move(coarse.coarse_of_fine), + coarse.num_coarse, + K, + trained.objective, + trained.num_sampled_passages, + trained.num_training_tokens, + timings}; +} + +} // namespace svs::quantization::gem + +#else + +#error \ + "svs/quantization/gem/codebook.h needs SVS_ENABLE_MULTI_VECTOR=1 (cmake: SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR=YES); it pulls in Intel MKL through kmeans_training.h." + +#endif // SVS_ENABLE_MULTI_VECTOR diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ed498b135..035a5d2f4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -249,6 +249,13 @@ if (SVS_EXPERIMENTAL_ENABLE_IVF OR SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) ) endif() +if (SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) + message("Enabling GEM codebook tests!") + list(APPEND TEST_SOURCES + ${TEST_DIR}/svs/quantization/gem/codebook.cpp + ) +endif() + add_executable(tests ${TEST_SOURCES}) # Path to the test dataset. diff --git a/tests/svs/quantization/gem/codebook.cpp b/tests/svs/quantization/gem/codebook.cpp new file mode 100644 index 000000000..53da8fecc --- /dev/null +++ b/tests/svs/quantization/gem/codebook.cpp @@ -0,0 +1,501 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/quantization/gem/codebook.h" + +// svs +#include "svs/core/data/simple.h" +#include "svs/core/data/vector_set.h" +#include "svs/lib/threads.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace gem = svs::quantization::gem; + +constexpr float TOL = 1e-4F; + +std::vector random_unit_vector(std::mt19937& rng, size_t dim) { + std::normal_distribution gauss(0.0F, 1.0F); + std::vector v(dim); + double norm2 = 0.0; + for (auto& x : v) { + x = gauss(rng); + norm2 += static_cast(x) * x; + } + double norm = std::sqrt(norm2); + for (auto& x : v) { + x = static_cast(static_cast(x) / norm); + } + return v; +} + +// Each token perturbs a center with a large margin, so even an approximate assigner agrees. +svs::data::VectorSetData make_clustered_corpus( + const std::vector& centers, + size_t K, + size_t dim, + const std::vector& cardinalities, + uint32_t seed, + float noise +) { + std::mt19937 rng(seed); + size_t num_passages = cardinalities.size(); + size_t total_tokens = 0; + for (auto c : cardinalities) { + total_tokens += c; + } + + svs::data::SimpleData token_matrix(total_tokens, dim); + std::vector offsets(num_passages + 1, 0); + std::uniform_int_distribution pick(0, K - 1); + std::normal_distribution noise_dist(0.0F, noise); + + size_t global = 0; + for (size_t p = 0; p < num_passages; ++p) { + offsets[p + 1] = offsets[p] + cardinalities[p]; + for (size_t t = 0; t < cardinalities[p]; ++t) { + size_t k = pick(rng); + std::vector row(dim); + double norm2 = 0.0; + for (size_t d = 0; d < dim; ++d) { + row[d] = centers[k * dim + d] + noise_dist(rng); + norm2 += static_cast(row[d]) * row[d]; + } + double norm = std::sqrt(norm2); + for (auto& x : row) { + x = static_cast(static_cast(x) / norm); + } + token_matrix.set_datum(global, row); + ++global; + } + } + return svs::data::VectorSetData(std::move(token_matrix), std::move(offsets)); +} + +std::vector make_random_centers(size_t K, size_t dim, uint32_t seed) { + std::mt19937 rng(seed); + std::vector centers(K * dim); + for (size_t k = 0; k < K; ++k) { + auto c = random_unit_vector(rng, dim); + std::copy(c.begin(), c.end(), centers.begin() + static_cast(k * dim)); + } + return centers; +} + +uint32_t brute_force_best( + std::span token, const svs::data::SimpleData& centroids +) { + uint32_t best = 0; + float best_dot = -std::numeric_limits::infinity(); + for (size_t j = 0, jmax = centroids.size(); j < jmax; ++j) { + auto c = centroids.get_datum(j); + float dot = 0.0F; + for (size_t d = 0, dmax = token.size(); d < dmax; ++d) { + dot += token[d] * c[d]; + } + if (dot > best_dot) { + best_dot = dot; + best = static_cast(j); + } + } + return best; +} + +std::vector flatten_ids(const svs::data::VectorSetData& ids) { + std::vector out; + out.reserve(ids.total_tokens()); + for (size_t p = 0, pmax = ids.size(); p < pmax; ++p) { + auto doc = ids.get_datum(p); + for (size_t t = 0, tmax = doc.cardinality(); t < tmax; ++t) { + out.push_back(doc.token(t)[0]); + } + } + return out; +} + +svs::data::VectorSetData +rebuild_ids(const std::vector& flat, std::vector offsets) { + svs::data::SimpleData storage(flat.size(), 1); + for (size_t i = 0; i < flat.size(); ++i) { + storage.set_datum(i, std::vector{flat[i]}); + } + return svs::data::VectorSetData(std::move(storage), std::move(offsets)); +} + +} // namespace + +CATCH_TEST_CASE("GEM Codebook - colbert_num_partitions", "[quantization][gem]") { + // Exact powers of two: 16 * sqrt(1) == 16, 16 * sqrt(4) == 32. + CATCH_REQUIRE(gem::colbert_num_partitions(1) == 16); + CATCH_REQUIRE(gem::colbert_num_partitions(4) == 32); + // One token below/above the num_tokens=4 boundary (raw crosses exactly 32 at n=4). + CATCH_REQUIRE(gem::colbert_num_partitions(3) == 16); + CATCH_REQUIRE(gem::colbert_num_partitions(5) == 32); + CATCH_REQUIRE_THROWS(gem::colbert_num_partitions(0)); +} + +CATCH_TEST_CASE("GEM Codebook - colbert_sample_size", "[quantization][gem]") { + // Small corpora: the formula's raw value exceeds num_passages, so it is capped. + CATCH_REQUIRE(gem::colbert_sample_size(1) == 1); + CATCH_REQUIRE(gem::colbert_sample_size(10) == 10); + // Large corpus: 1 + floor(16 * sqrt(120 * 1'000'000)) is well under num_passages. + size_t num_passages = 1'000'000; + size_t expected = + 1 + static_cast( + std::floor(16.0 * std::sqrt(120.0 * static_cast(num_passages))) + ); + CATCH_REQUIRE(gem::colbert_sample_size(num_passages) == expected); + CATCH_REQUIRE(expected < num_passages); + CATCH_REQUIRE_THROWS(gem::colbert_sample_size(0)); + + // Perfect-square boundary (921'600 == 960^2); integer sqrt must not be off by one. + CATCH_REQUIRE(gem::detail::isqrt_floor(921'600) == 960); + CATCH_REQUIRE(gem::detail::isqrt_floor(921'599) == 959); + CATCH_REQUIRE(gem::detail::isqrt_floor(921'601) == 960); +} + +CATCH_TEST_CASE( + "GEM Codebook - plain mean update differs from (old + sum) / (count + 1)", + "[quantization][gem]" +) { + auto data = svs::data::SimpleData(4, 2); + data.set_datum(0, std::vector{0.0F, 0.0F}); + data.set_datum(1, std::vector{0.0F, 2.0F}); + data.set_datum(2, std::vector{10.0F, 0.0F}); + data.set_datum(3, std::vector{10.0F, 2.0F}); + + auto centroids = svs::data::SimpleData(2, 2); + centroids.set_datum(0, std::vector{0.0F, 0.0F}); + centroids.set_datum(1, std::vector{10.0F, 0.0F}); + + std::vector assignments{0, 0, 1, 1}; + std::vector counts(2); + auto threadpool = svs::threads::as_threadpool(2); + + gem::detail::plain_mean_update(data, centroids, assignments, counts, threadpool); + + auto c0 = centroids.get_datum(0); + auto c1 = centroids.get_datum(1); + // Plain mean: cluster 0 -> (0,1); cluster 1 -> (10,1). + CATCH_REQUIRE(std::abs(c0[0] - 0.0F) < TOL); + CATCH_REQUIRE(std::abs(c0[1] - 1.0F) < TOL); + CATCH_REQUIRE(std::abs(c1[0] - 10.0F) < TOL); + CATCH_REQUIRE(std::abs(c1[1] - 1.0F) < TOL); + // (old + sum) / (count + 1) would give (0 + 0 + 2) / 3 == 0.667, not the plain mean 1. + CATCH_REQUIRE(std::abs(c0[1] - (2.0F / 3.0F)) > TOL); +} + +CATCH_TEST_CASE( + "GEM Codebook - train_codebook centroids are unit norm, objective is non-increasing", + "[quantization][gem]" +) { + constexpr size_t K = 8; + constexpr size_t DIM = 16; + auto centers = make_random_centers(K, DIM, 0x1234); + + std::vector cardinalities(50, 6); + auto corpus = make_clustered_corpus(centers, K, DIM, cardinalities, 0xABCD, 0.05F); + + gem::CodebookParameters parameters; + parameters.num_centroids = K; + parameters.seed = 7; + + auto threadpool = svs::threads::as_threadpool(2); + + parameters.kmeans_iterations = 1; + auto one_iter = gem::train_codebook(corpus, parameters, threadpool); + + parameters.kmeans_iterations = 4; + auto four_iter = gem::train_codebook(corpus, parameters, threadpool); + + // Same seed and sampling: more passes cannot raise the final objective. + CATCH_REQUIRE(four_iter.objective <= one_iter.objective + 1e-3); + + for (size_t i = 0; i < four_iter.centroids.size(); ++i) { + auto c = four_iter.centroids.get_datum(i); + float norm_sq = 0.0F; + for (size_t d = 0; d < c.size(); ++d) { + norm_sq += c[d] * c[d]; + } + CATCH_REQUIRE(std::abs(std::sqrt(norm_sq) - 1.0F) < 1e-4F); + } + CATCH_REQUIRE(four_iter.timings.train_iterations.size() == 4); +} + +CATCH_TEST_CASE( + "GEM Codebook - assign_tokens agrees with brute force, both assigners, both thread " + "counts", + "[quantization][gem]" +) { + constexpr size_t K = 64; + constexpr size_t DIM = 128; + auto centers = make_random_centers(K, DIM, 0x5EED); + + std::vector cardinalities; + for (size_t i = 0; i < 300; ++i) { + cardinalities.push_back(1 + (i % 6)); + } + auto corpus = make_clustered_corpus(centers, K, DIM, cardinalities, 0xC0FFEE, 0.02F); + + svs::data::SimpleData centroids(K, DIM); + for (size_t k = 0; k < K; ++k) { + centroids.set_datum(k, std::span(centers.data() + k * DIM, DIM)); + } + + gem::CodebookParameters parameters; + parameters.assign_candidates = 8; + + for (auto assigner : {gem::Assigner::ExactGemm, gem::Assigner::InnerProductMatrix}) { + parameters.assigner = assigner; + + auto pool1 = svs::threads::as_threadpool(1); + auto pool4 = svs::threads::as_threadpool(4); + auto ids1 = gem::assign_tokens(corpus, centroids, parameters, pool1); + auto ids4 = gem::assign_tokens(corpus, centroids, parameters, pool4); + + auto flat1 = flatten_ids(ids1); + auto flat4 = flatten_ids(ids4); + CATCH_REQUIRE(flat1 == flat4); + + // Cardinalities of the id sets match the input token store. + for (size_t p = 0; p < corpus.size(); ++p) { + CATCH_REQUIRE(ids1.cardinality(p) == corpus.cardinality(p)); + } + + // Every id agrees with an independent brute-force argmax. + size_t global = 0; + for (size_t p = 0; p < corpus.size(); ++p) { + auto doc = corpus.get_datum(p); + for (size_t t = 0; t < doc.cardinality(); ++t) { + uint32_t expected = brute_force_best(doc.token(t), centroids); + CATCH_REQUIRE(flat1[global] == expected); + ++global; + } + } + + auto report = gem::count_assignment_mismatches(corpus, centroids, ids1, 1000, 99); + CATCH_REQUIRE(report.mismatches == 0); + CATCH_REQUIRE(report.sample_size == 1000); + + // Corrupt a few ids and confirm the checker notices. + auto offsets = std::vector(ids1.offsets().begin(), ids1.offsets().end()); + auto corrupted = flat1; + for (size_t i = 0; i < corrupted.size(); i += 37) { + corrupted[i] = static_cast((corrupted[i] + 1) % K); + } + auto corrupted_ids = rebuild_ids(corrupted, offsets); + auto bad_report = + gem::count_assignment_mismatches(corpus, centroids, corrupted_ids, 2000, 99); + CATCH_REQUIRE(bad_report.mismatches > 0); + } +} + +CATCH_TEST_CASE( + "GEM Codebook - assign_tokens batch row count comes from a byte budget", + "[quantization][gem]" +) { + constexpr size_t K = 64; + constexpr size_t DIM = 32; + auto centers = make_random_centers(K, DIM, 0xBEEF); + + std::vector cardinalities; + for (size_t i = 0; i < 40; ++i) { + cardinalities.push_back(1 + (i % 5)); + } + auto corpus = make_clustered_corpus(centers, K, DIM, cardinalities, 0xFACE, 0.02F); + + svs::data::SimpleData centroids(K, DIM); + for (size_t k = 0; k < K; ++k) { + centroids.set_datum(k, std::span(centers.data() + k * DIM, DIM)); + } + + auto threadpool = svs::threads::as_threadpool(2); + + for (auto assigner : {gem::Assigner::ExactGemm, gem::Assigner::InnerProductMatrix}) { + gem::CodebookParameters default_parameters; + default_parameters.assigner = assigner; + auto default_ids = + gem::assign_tokens(corpus, centroids, default_parameters, threadpool); + auto default_flat = flatten_ids(default_ids); + + // K*4 bytes/row: 256 forces a 1-row batch, 900 forces a 3-row batch. + for (size_t budget : {size_t{256}, size_t{900}}) { + gem::CodebookParameters tiny_parameters = default_parameters; + tiny_parameters.assign_buffer_bytes = budget; + auto tiny_ids = + gem::assign_tokens(corpus, centroids, tiny_parameters, threadpool); + CATCH_REQUIRE(flatten_ids(tiny_ids) == default_flat); + } + } +} + +CATCH_TEST_CASE("GEM Codebook - coarse_clusters", "[quantization][gem]") { + constexpr size_t K = 64; + constexpr size_t DIM = 16; + auto centers = make_random_centers(K, DIM, 0x777); + svs::data::SimpleData centroids(K, DIM); + for (size_t k = 0; k < K; ++k) { + centroids.set_datum(k, std::span(centers.data() + k * DIM, DIM)); + } + + auto threadpool = svs::threads::as_threadpool(2); + + CATCH_SECTION("default num_coarse rule") { + gem::CodebookParameters parameters; + auto result = gem::coarse_clusters(centroids, parameters, threadpool); + CATCH_REQUIRE(result.num_coarse == std::max(1, K * 5 / 32)); + CATCH_REQUIRE(result.coarse_of_fine.size() == K); + for (auto c : result.coarse_of_fine) { + CATCH_REQUIRE(c < result.num_coarse); + } + } + + CATCH_SECTION("explicit num_coarse") { + gem::CodebookParameters parameters; + parameters.num_coarse = 5; + auto result = gem::coarse_clusters(centroids, parameters, threadpool); + CATCH_REQUIRE(result.num_coarse == 5); + for (auto c : result.coarse_of_fine) { + CATCH_REQUIRE(c < 5); + } + } + + CATCH_SECTION("num_coarse exceeding K throws") { + gem::CodebookParameters parameters; + parameters.num_coarse = K + 1; + CATCH_REQUIRE_THROWS(gem::coarse_clusters(centroids, parameters, threadpool)); + } +} + +CATCH_TEST_CASE( + "GEM Codebook - quantize end to end on a small synthetic set", "[quantization][gem]" +) { + constexpr size_t DIM = 16; + constexpr size_t NUM_PASSAGES = 200; + // 200 passages * 2 tokens == 400; colbert_num_partitions(400) == 256, still exceeded. + std::vector cardinalities(NUM_PASSAGES, 2); + auto centers = make_random_centers(16, DIM, 0x9999); + auto corpus = make_clustered_corpus(centers, 16, DIM, cardinalities, 0x1357, 0.1F); + + CATCH_REQUIRE(corpus.total_tokens() == 400); + CATCH_REQUIRE(gem::colbert_num_partitions(corpus.total_tokens()) == 256); + + gem::CodebookParameters parameters; // num_centroids == 0: use the rule. + auto threadpool = svs::threads::as_threadpool(2); + auto result = gem::quantize(corpus, parameters, threadpool); + + CATCH_REQUIRE(result.K == 256); + CATCH_REQUIRE(result.centroids.size() == 256); + CATCH_REQUIRE(result.centroids.dimensions() == DIM); + CATCH_REQUIRE(result.fine_centroids->K == 256); + CATCH_REQUIRE(result.fine_centroids->dim == DIM); + CATCH_REQUIRE(result.centroid_table->K == 256); + CATCH_REQUIRE(result.ids.total_tokens() == 400); + for (size_t p = 0; p < NUM_PASSAGES; ++p) { + CATCH_REQUIRE(result.ids.cardinality(p) == 2); + } + CATCH_REQUIRE(result.num_coarse == std::max(1, 256 * 5 / 32)); + CATCH_REQUIRE(result.coarse_of_fine.size() == 256); + for (auto c : result.coarse_of_fine) { + CATCH_REQUIRE(c < result.num_coarse); + } + CATCH_REQUIRE(result.timings.train_iterations.size() == parameters.kmeans_iterations); + CATCH_REQUIRE(result.timings.total >= 0.0); + CATCH_REQUIRE(result.timings.sample >= 0.0); + CATCH_REQUIRE(result.timings.train >= 0.0); + CATCH_REQUIRE(result.timings.normalize >= 0.0); + CATCH_REQUIRE(result.timings.assign >= 0.0); + CATCH_REQUIRE(result.timings.ground_table >= 0.0); + CATCH_REQUIRE(result.timings.coarse >= 0.0); +} + +CATCH_TEST_CASE("GEM Codebook - failure paths", "[quantization][gem]") { + constexpr size_t DIM = 8; + auto threadpool = svs::threads::as_threadpool(1); + + CATCH_SECTION("empty tokens dataset") { + svs::data::SimpleData empty_tokens(0, DIM); + std::vector offsets{0}; + auto corpus = + svs::data::VectorSetData(std::move(empty_tokens), std::move(offsets)); + gem::CodebookParameters parameters; + CATCH_REQUIRE_THROWS(gem::train_codebook(corpus, parameters, threadpool)); + } + + CATCH_SECTION("num_centroids above the uint32_t id range") { + auto centers = make_random_centers(4, DIM, 1); + std::vector cardinalities(10, 4); + auto corpus = make_clustered_corpus(centers, 4, DIM, cardinalities, 2, 0.05F); + gem::CodebookParameters parameters; + parameters.num_centroids = + static_cast(std::numeric_limits::max()) + 1; + CATCH_REQUIRE_THROWS(gem::train_codebook(corpus, parameters, threadpool)); + } + + CATCH_SECTION("fewer training tokens than K") { + auto centers = make_random_centers(4, DIM, 3); + std::vector cardinalities(5, 2); + auto corpus = make_clustered_corpus(centers, 4, DIM, cardinalities, 4, 0.05F); + gem::CodebookParameters parameters; + parameters.num_centroids = 1'000; // far more centroids than tokens sampled + CATCH_REQUIRE_THROWS(gem::train_codebook(corpus, parameters, threadpool)); + } + + CATCH_SECTION("training points exceed faiss's 256-per-centroid bound") { + // One passage, 600 tokens; colbert_sample_size caps at 1 passage as training set. + auto centers = make_random_centers(2, DIM, 5); + std::vector cardinalities{600}; + auto corpus = make_clustered_corpus(centers, 2, DIM, cardinalities, 6, 0.05F); + gem::CodebookParameters parameters; + parameters.num_centroids = 2; // 600ish training tokens > 2 * 256 == 512 + CATCH_REQUIRE_THROWS(gem::train_codebook(corpus, parameters, threadpool)); + } + + CATCH_SECTION("assign_tokens token/centroid dimension mismatch") { + auto centers = make_random_centers(4, DIM, 7); + std::vector cardinalities(5, 2); + auto corpus = make_clustered_corpus(centers, 4, DIM, cardinalities, 8, 0.05F); + svs::data::SimpleData bad_centroids(4, DIM + 1); + gem::CodebookParameters parameters; + CATCH_REQUIRE_THROWS( + gem::assign_tokens(corpus, bad_centroids, parameters, threadpool) + ); + } + + CATCH_SECTION("coarse_clusters num_coarse exceeds K") { + auto centers = make_random_centers(4, DIM, 9); + svs::data::SimpleData centroids(4, DIM); + for (size_t k = 0; k < 4; ++k) { + centroids.set_datum(k, std::span(centers.data() + k * DIM, DIM)); + } + gem::CodebookParameters parameters; + parameters.num_coarse = 5; + CATCH_REQUIRE_THROWS(gem::coarse_clusters(centroids, parameters, threadpool)); + } +} From 8f4b39ede26d1b5598d041100afcdc33dafa166e Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:42:48 -0800 Subject: [PATCH 38/68] feat(vamana): pick insert start nodes through an extension hook Adds extensions::build_entry_points, a customization point generate_neighbors calls per node to pick its greedy-search start nodes; the default returns the build's fixed entry points unchanged, so behaviour and generated code are unchanged for every existing distance. Co-Authored-By: Claude Fable 5.2 --- include/svs/index/vamana/extensions.h | 32 +++++++++++++++++++++++++ include/svs/index/vamana/vamana_build.h | 11 ++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index d923f59b7..10b3bdc8a 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -30,6 +30,7 @@ // stl #include #include +#include namespace svs::index::vamana::extensions { @@ -285,6 +286,37 @@ DefaultBuildAdaptor svs_invoke( return DefaultBuildAdaptor{}; } +/// Builds the per-insert start nodes `generate_neighbors` uses. Default: entry_points +/// unchanged. An extension may route to per-node starts instead (GEM, gem.h); `scratch` +/// is per-thread storage it may fill and return a span over, avoiding allocation. +struct BuildEntryPointsType { + template + std::span operator()( + const Data& data, + const Distance& distance, + Idx node_id, + std::span entry_points, + std::vector& scratch + ) const { + return svs::svs_invoke(*this, data, distance, node_id, entry_points, scratch); + } +}; + +inline constexpr BuildEntryPointsType build_entry_points{}; + +// Default Implementation. +template +std::span svs_invoke( + svs::tag_t, + const Data& SVS_UNUSED(data), + const Distance& SVS_UNUSED(distance), + Idx SVS_UNUSED(node_id), + std::span entry_points, + std::vector& SVS_UNUSED(scratch) +) { + return entry_points; +} + ///// ///// SEARCH EXTENSIONS ///// diff --git a/include/svs/index/vamana/vamana_build.h b/include/svs/index/vamana/vamana_build.h index 406a94244..95d038bda 100644 --- a/include/svs/index/vamana/vamana_build.h +++ b/include/svs/index/vamana/vamana_build.h @@ -360,6 +360,7 @@ class VamanaBuilder { // Scratch space. std::vector> pool{}; + std::vector entry_point_scratch{}; auto search_buffer = search_buffer_type{params_.window_size}; // Enable use of the visited filter of the search buffer. @@ -397,6 +398,14 @@ class VamanaBuilder { auto search_clock_start = std::chrono::steady_clock::now(); #endif { + // local_indices' element type varies; narrow to avoid two Idx's. + auto node_entry_points = extensions::build_entry_points( + data_, + distance_function_, + lib::narrow_cast(node_id), + lib::as_const_span(entry_points), + entry_point_scratch + ); auto accessor = build_adaptor.graph_search_accessor(); greedy_search( graph_, @@ -409,7 +418,7 @@ class VamanaBuilder { graph_search_distance, #endif search_buffer, - vamana::EntryPointInitializer{lib::as_const_span(entry_points)}, + vamana::EntryPointInitializer{node_entry_points}, NeighborBuilder(), tracker, prefetch_hint_ From b446a1203816e07d4784a29f5b3a6d26174e0b2e Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:43:03 -0800 Subject: [PATCH 39/68] feat(vamana): check and repair graph connectivity after a build Adds unreachable_nodes (BFS from a root set) and repair_connectivity, which gives every unreachable node one inbound edge from a reachable candidate: a plain append when the candidate has room, else a re-prune, else a forced replacement of one of the candidate's neighbors. A re-prune or replacement is taken only when no neighbor it drops loses its last reachable in-edge; otherwise the repair falls back to an unsafe replacement and counts it. A round that does not reduce the unreachable count stops the repair. Needed because a build whose insert order puts a few nodes first (GEM's representatives) can leave part of the graph unreachable from the roots. Co-Authored-By: Claude Fable 5.2 Co-Authored-By: Claude Opus 5.5 --- include/svs/index/vamana/connectivity.h | 355 ++++++++++++++++++++++++ tests/CMakeLists.txt | 1 + tests/svs/index/vamana/connectivity.cpp | 274 ++++++++++++++++++ 3 files changed, 630 insertions(+) create mode 100644 include/svs/index/vamana/connectivity.h create mode 100644 tests/svs/index/vamana/connectivity.cpp diff --git a/include/svs/index/vamana/connectivity.h b/include/svs/index/vamana/connectivity.h new file mode 100644 index 000000000..c68821d8d --- /dev/null +++ b/include/svs/index/vamana/connectivity.h @@ -0,0 +1,355 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Repairs graph connectivity a build can break by skipping most nodes (e.g. GEM starts). +#include "svs/concepts/data.h" +#include "svs/concepts/graph.h" +#include "svs/index/vamana/build_params.h" +#include "svs/index/vamana/extensions.h" +#include "svs/index/vamana/greedy_search.h" +#include "svs/index/vamana/prune.h" +#include "svs/index/vamana/search_buffer.h" +#include "svs/lib/misc.h" +#include "svs/lib/narrow.h" +#include "svs/lib/neighbor.h" +#include "svs/lib/threads.h" + +// stl +#include +#include +#include +#include + +namespace svs::index::vamana { + +/// Breadth-first search over out-edges from `roots`; returns the node ids `roots` cannot +/// reach, ascending. +template +std::vector +unreachable_nodes(const Graph& graph, std::span roots) { + using Idx = typename Graph::index_type; + size_t n = graph.n_nodes(); + std::vector visited(n, false); + + std::vector queue; + queue.reserve(n); + for (Idx root : roots) { + if (!visited[root]) { + visited[root] = true; + queue.push_back(root); + } + } + for (size_t head = 0; head < queue.size(); ++head) { + for (Idx v : graph.get_node(queue[head])) { + if (!visited[v]) { + visited[v] = true; + queue.push_back(v); + } + } + } + + std::vector result; + for (size_t i = 0; i < n; ++i) { + if (!visited[i]) { + result.push_back(lib::narrow_cast(i)); + } + } + return result; +} + +/// Outcome of `repair_connectivity`. edges_forced_unsafe, unlike the other edge counts, +/// has no safety guarantee and can orphan the node it displaced. `stalled` means a round +/// made no progress, which stops the repair before `max_rounds`. +struct ConnectivityReport { + size_t rounds = 0; + size_t unreachable_before = 0; + size_t unreachable_after = 0; + size_t edges_appended = 0; + size_t edges_accepted_through_prune = 0; + size_t edges_forced = 0; + size_t edges_forced_unsafe = 0; + bool stalled = false; +}; + +/// Reconnects every node `unreachable_nodes` reports, round by round, until none remain +/// or `max_rounds` is spent. Candidates are searched in parallel per round, but applied +/// serially in ascending node-id order since accepting v for one node changes deg(v). +template < + graphs::MemoryGraph Graph, + data::ImmutableMemoryDataset Data, + typename Dist, + threads::ThreadPool Pool> +ConnectivityReport repair_connectivity( + Graph& graph, + const Data& data, + const Dist& distance, + const VamanaBuildParameters& params, + std::span roots, + Pool& threadpool, + size_t max_rounds = 4 +) { + using Idx = typename Graph::index_type; + + std::vector is_root(graph.n_nodes(), false); + for (Idx root : roots) { + is_root[root] = true; + } + + ConnectivityReport report{}; + auto unreachable = unreachable_nodes(graph, roots); + report.unreachable_before = unreachable.size(); + + for (size_t round = 0; round < max_rounds && !unreachable.empty(); ++round) { + ++report.rounds; + + // Reachability snapshot at round start: proposals below are filtered against it. + std::vector reachable(graph.n_nodes(), true); + for (Idx u : unreachable) { + reachable[u] = false; + } + + // Kept current as edges change, so the forced step can detect a stranding drop. + std::vector reachable_in_degree(graph.n_nodes(), 0); + for (size_t x = 0; x < graph.n_nodes(); ++x) { + if (reachable[x]) { + for (Idx y : graph.get_node(lib::narrow_cast(x))) { + ++reachable_in_degree[y]; + } + } + } + + // Parallel search only; Phase B is serial since deg(v) changes within a round. + std::vector> proposals(unreachable.size()); + threads::parallel_for( + threadpool, + threads::StaticPartition{unreachable.size()}, + [&](const auto& is, uint64_t /*tid*/) { + auto build_adaptor = extensions::build_adaptor(data, distance); + auto accessor = build_adaptor.graph_search_accessor(); + auto&& search_distance = build_adaptor.graph_search_distance(); + auto search_buffer = + SearchBuffer>{params.window_size}; + auto tracker = NullTracker{}; + + for (size_t k : is) { + Idx u = unreachable[k]; + search_buffer.clear(); + const auto& query = + build_adaptor.access_query_for_graph_search(data, u); + greedy_search( + graph, + data, + accessor, + query, + search_distance, + search_buffer, + EntryPointInitializer{roots}, + NeighborBuilder(), + tracker + ); + search_buffer.sort(); + + auto& candidates = proposals[k]; + for (size_t i = 0, imax = search_buffer.size(); i < imax; ++i) { + Idx id = search_buffer[i].id(); + if (reachable[id]) { + candidates.push_back(id); + } + } + } + } + ); + + // Phase B (serial, ascending u). + auto build_adaptor = extensions::build_adaptor(data, distance); + auto accessor = build_adaptor.general_accessor(); + auto&& general_distance = build_adaptor.general_distance(); + auto cmp = distance::comparator(general_distance); + + std::vector order(unreachable.size()); + for (size_t k = 0; k < order.size(); ++k) { + order[k] = k; + } + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { + return unreachable[a] < unreachable[b]; + }); + + // Safe to drop: a root (always reachable) or has another reachable in-edge. + auto is_safe_to_drop = [&](Idx n) { + return is_root[n] || reachable_in_degree[n] >= 2; + }; + + std::vector> pool; + std::vector pruned; + std::vector old_neighbors; + for (size_t k : order) { + Idx u = unreachable[k]; + const auto& candidates = proposals[k]; + bool accepted = false; + + // Pass 1: append if room, else re-prune and accept only if all drops are safe. + for (Idx v : candidates) { + if (graph.get_node_degree(v) < params.graph_max_degree) { + graph.add_edge(v, u); + ++reachable_in_degree[u]; + ++report.edges_appended; + accepted = true; + break; + } + + const auto& v_data = accessor(data, v); + distance::maybe_fix_argument(general_distance, v_data); + pool.clear(); + old_neighbors.clear(); + for (Idx n : graph.get_node(v)) { + old_neighbors.push_back(n); + pool.emplace_back( + n, distance::compute(general_distance, v_data, accessor(data, n)) + ); + } + pool.emplace_back( + u, distance::compute(general_distance, v_data, accessor(data, u)) + ); + std::sort(pool.begin(), pool.end(), TotalOrder(cmp)); + + heuristic_prune_neighbors( + prune_strategy(distance), + params.graph_max_degree, + params.alpha, + data, + accessor, + general_distance, + v, + lib::as_const_span(pool), + pruned + ); + + if (std::find(pruned.begin(), pruned.end(), u) == pruned.end()) { + continue; + } + + bool all_drops_safe = true; + for (Idx old_id : old_neighbors) { + bool dropped = + std::find(pruned.begin(), pruned.end(), old_id) == pruned.end(); + if (dropped && !is_safe_to_drop(old_id)) { + all_drops_safe = false; + break; + } + } + if (!all_drops_safe) { + continue; + } + + graph.replace_node(v, pruned); + for (Idx old_id : old_neighbors) { + if (std::find(pruned.begin(), pruned.end(), old_id) == pruned.end()) { + --reachable_in_degree[old_id]; + } + } + ++reachable_in_degree[u]; + ++report.edges_accepted_through_prune; + accepted = true; + break; + } + + if (accepted) { + continue; + } + + // Pass 2: forced replacement of a safe neighbor; candidates is never empty. + bool forced_safe = false; + Idx safe_v{}; + Idx safe_w{}; + for (Idx v : candidates) { + const auto& v_data = accessor(data, v); + distance::maybe_fix_argument(general_distance, v_data); + + std::vector> safe_candidates; + for (Idx n : graph.get_node(v)) { + if (is_safe_to_drop(n)) { + safe_candidates.emplace_back( + n, + distance::compute(general_distance, v_data, accessor(data, n)) + ); + } + } + if (!safe_candidates.empty()) { + auto farthest = std::max_element( + safe_candidates.begin(), safe_candidates.end(), TotalOrder(cmp) + ); + safe_v = v; + safe_w = farthest->id(); + forced_safe = true; + break; + } + } + + // Pass 3: forces the nearest candidate regardless of safety, orphaning `w`. + Idx v{}; + Idx w{}; + if (forced_safe) { + v = safe_v; + w = safe_w; + } else { + v = candidates.front(); + const auto& v_data = accessor(data, v); + distance::maybe_fix_argument(general_distance, v_data); + + std::vector> ranked; + for (Idx n : graph.get_node(v)) { + ranked.emplace_back( + n, distance::compute(general_distance, v_data, accessor(data, n)) + ); + } + auto farthest = + std::max_element(ranked.begin(), ranked.end(), TotalOrder(cmp)); + w = farthest->id(); + } + + std::vector new_neighbors; + for (Idx n : graph.get_node(v)) { + if (n != w) { + new_neighbors.push_back(n); + } + } + new_neighbors.push_back(u); + graph.replace_node(v, new_neighbors); + + --reachable_in_degree[w]; + ++reachable_in_degree[u]; + + if (forced_safe) { + ++report.edges_forced; + } else { + ++report.edges_forced_unsafe; + } + } + + size_t unreachable_before_round = unreachable.size(); + unreachable = unreachable_nodes(graph, roots); + if (unreachable.size() >= unreachable_before_round) { + report.stalled = true; + break; + } + } + report.unreachable_after = unreachable.size(); + return report; +} + +} // namespace svs::index::vamana diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 035a5d2f4..243e91ca2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -139,6 +139,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/index/flat/dynamic_flat.cpp ${TEST_DIR}/svs/index/vamana/build_parameters.cpp ${TEST_DIR}/svs/index/vamana/build_stats.cpp + ${TEST_DIR}/svs/index/vamana/connectivity.cpp ${TEST_DIR}/svs/index/vamana/consolidate.cpp ${TEST_DIR}/svs/index/vamana/filter.cpp ${TEST_DIR}/svs/index/vamana/gem_routing.cpp diff --git a/tests/svs/index/vamana/connectivity.cpp b/tests/svs/index/vamana/connectivity.cpp new file mode 100644 index 000000000..86e31dae5 --- /dev/null +++ b/tests/svs/index/vamana/connectivity.cpp @@ -0,0 +1,274 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test: unreachable_nodes and repair_connectivity. +#include "svs/index/vamana/connectivity.h" + +// svs +#include "svs/core/data/simple.h" +#include "svs/core/distance/euclidean.h" +#include "svs/core/graph.h" +#include "svs/lib/threads.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include + +namespace { + +using Idx = uint32_t; + +std::vector> adjacency_snapshot(const svs::graphs::SimpleGraph& graph +) { + std::vector> out(graph.n_nodes()); + for (size_t i = 0; i < graph.n_nodes(); ++i) { + auto node = graph.get_node(static_cast(i)); + out[i].assign(node.begin(), node.end()); + } + return out; +} + +svs::data::SimpleData make_positions(const std::vector& values) { + auto data = svs::data::SimpleData(values.size(), 1); + for (size_t i = 0; i < values.size(); ++i) { + data.set_datum(i, std::vector{values[i]}); + } + return data; +} + +} // namespace + +CATCH_TEST_CASE("unreachable_nodes", "[index][vamana][connectivity]") { + namespace v = svs::index::vamana; + + auto graph = svs::default_graph(5, 3); + graph.replace_node(0, std::vector{1, 2}); + graph.replace_node(2, std::vector{3}); + // Nodes 1, 3, 4 keep their default empty adjacency list. + + CATCH_SECTION("One root reaches a prefix, two nodes stay unreachable") { + std::vector roots{0}; + auto unreachable = v::unreachable_nodes(graph, std::span(roots)); + CATCH_REQUIRE(unreachable == std::vector{4}); + } + + CATCH_SECTION("Adding the straggler as a root reaches everything") { + std::vector roots{0, 4}; + auto unreachable = v::unreachable_nodes(graph, std::span(roots)); + CATCH_REQUIRE(unreachable.empty()); + } +} + +CATCH_TEST_CASE( + "repair_connectivity leaves an already-connected graph untouched", + "[index][vamana][connectivity]" +) { + namespace v = svs::index::vamana; + + auto graph = svs::default_graph(5, 3); + graph.replace_node(0, std::vector{1}); + graph.replace_node(1, std::vector{2}); + graph.replace_node(2, std::vector{3}); + graph.replace_node(3, std::vector{4}); + + auto before = adjacency_snapshot(graph); + + auto data = make_positions({0.0F, 1.0F, 2.0F, 3.0F, 4.0F}); + svs::distance::DistanceL2 distance{}; + auto params = v::VamanaBuildParameters{1.2F, 3, 10, 10, 3, true}; + auto pool = svs::threads::DefaultThreadPool(1); + std::vector roots{0}; + + auto report = v::repair_connectivity( + graph, data, distance, params, std::span(roots), pool + ); + + CATCH_REQUIRE(report.unreachable_before == 0); + CATCH_REQUIRE(report.unreachable_after == 0); + CATCH_REQUIRE(report.rounds == 0); + CATCH_REQUIRE(report.edges_appended == 0); + CATCH_REQUIRE(report.edges_accepted_through_prune == 0); + CATCH_REQUIRE(report.edges_forced == 0); + CATCH_REQUIRE(adjacency_snapshot(graph) == before); +} + +CATCH_TEST_CASE( + "repair_connectivity reconnects a disconnected node with room to spare", + "[index][vamana][connectivity]" +) { + namespace v = svs::index::vamana; + + // Nodes 0-3 reachable from root 0, all with degree < graph_max_degree. + auto graph = svs::default_graph(5, 3); + graph.replace_node(0, std::vector{1}); + graph.replace_node(1, std::vector{2}); + graph.replace_node(2, std::vector{3}); + + auto data = make_positions({0.0F, 1.0F, 2.0F, 3.0F, 2.1F}); + svs::distance::DistanceL2 distance{}; + auto params = v::VamanaBuildParameters{1.2F, 3, 10, 10, 3, true}; + auto pool = svs::threads::DefaultThreadPool(1); + std::vector roots{0}; + + auto report = v::repair_connectivity( + graph, data, distance, params, std::span(roots), pool + ); + + CATCH_REQUIRE(report.unreachable_before == 1); + CATCH_REQUIRE(report.unreachable_after == 0); + CATCH_REQUIRE(report.rounds >= 1); + CATCH_REQUIRE( + report.edges_appended + report.edges_accepted_through_prune + report.edges_forced == + 1 + ); + CATCH_REQUIRE(v::unreachable_nodes(graph, std::span(roots)).empty()); + for (size_t i = 0; i < graph.n_nodes(); ++i) { + CATCH_REQUIRE( + graph.get_node_degree(static_cast(i)) <= params.graph_max_degree + ); + } +} + +CATCH_TEST_CASE( + "repair_connectivity reconnects a node whose nearest reachable neighbors are all at " + "full degree", + "[index][vamana][connectivity]" +) { + namespace v = svs::index::vamana; + + // 2-regular cycle at full degree; node 4 is disconnected and nearest to root 0. + auto graph = svs::default_graph(5, 2); + graph.replace_node(0, std::vector{1, 2}); + graph.replace_node(1, std::vector{2, 3}); + graph.replace_node(2, std::vector{3, 0}); + graph.replace_node(3, std::vector{0, 1}); + + auto data = make_positions({0.0F, 1.0F, 2.0F, 3.0F, 0.1F}); + svs::distance::DistanceL2 distance{}; + auto params = v::VamanaBuildParameters{1.2F, 2, 10, 10, 2, true}; + auto pool = svs::threads::DefaultThreadPool(1); + std::vector roots{0}; + + for (size_t i = 0; i < 4; ++i) { + CATCH_REQUIRE(graph.get_node_degree(static_cast(i)) == 2); + } + + auto report = v::repair_connectivity( + graph, data, distance, params, std::span(roots), pool + ); + + CATCH_REQUIRE(report.unreachable_before == 1); + CATCH_REQUIRE(report.unreachable_after == 0); + CATCH_REQUIRE( + report.edges_appended + report.edges_accepted_through_prune + report.edges_forced == + 1 + ); + CATCH_REQUIRE(report.edges_forced_unsafe == 0); + CATCH_REQUIRE(!report.stalled); + CATCH_REQUIRE(v::unreachable_nodes(graph, std::span(roots)).empty()); + for (size_t i = 0; i < graph.n_nodes(); ++i) { + CATCH_REQUIRE( + graph.get_node_degree(static_cast(i)) <= params.graph_max_degree + ); + } +} + +CATCH_TEST_CASE( + "repair_connectivity falls back to an unsafe forced replacement when no candidate " + "offers a safe one, and reports it", + "[index][vamana][connectivity]" +) { + namespace v = svs::index::vamana; + + // Root->1->2, both at degree 1; node 2 is a leaf, so dropping it is never safe. + auto graph = svs::default_graph(4, 1); + graph.replace_node(0, std::vector{1}); + graph.replace_node(1, std::vector{2}); + // Node 2 (a leaf) and node 3 (disconnected) keep their default empty adjacency list. + + auto data = make_positions({100.0F, 0.0F, 2.0F, -3.0F}); + svs::distance::DistanceL2 distance{}; + auto params = v::VamanaBuildParameters{1.0F, 1, 1, 10, 1, true}; + auto pool = svs::threads::DefaultThreadPool(1); + std::vector roots{0}; + + // Stop after one round: this test is about the forced step within a single round. + auto report = v::repair_connectivity( + graph, data, distance, params, std::span(roots), pool, size_t{1} + ); + + CATCH_REQUIRE(report.rounds == 1); + CATCH_REQUIRE(report.unreachable_before == 1); + CATCH_REQUIRE(report.edges_appended == 0); + CATCH_REQUIRE(report.edges_accepted_through_prune == 0); + CATCH_REQUIRE(report.edges_forced == 0); + CATCH_REQUIRE(report.edges_forced_unsafe == 1); + + // Node 1's forced edge to node 3 displaces node 2, leaving node 2 unreachable. + CATCH_REQUIRE(report.stalled); + CATCH_REQUIRE(report.unreachable_after == 1); + CATCH_REQUIRE( + v::unreachable_nodes(graph, std::span(roots)) == std::vector{2} + ); + for (size_t i = 0; i < graph.n_nodes(); ++i) { + CATCH_REQUIRE( + graph.get_node_degree(static_cast(i)) <= params.graph_max_degree + ); + } +} + +CATCH_TEST_CASE( + "repair_connectivity's stall guard stops an oscillating repair before max_rounds", + "[index][vamana][connectivity]" +) { + namespace v = svs::index::vamana; + + // Same graph as the unsafe-fallback case, run to default max_rounds instead of 1. + auto graph = svs::default_graph(4, 1); + graph.replace_node(0, std::vector{1}); + graph.replace_node(1, std::vector{2}); + // Node 2 (a leaf) and node 3 (disconnected) keep their default empty adjacency list. + + auto data = make_positions({100.0F, 0.0F, 2.0F, -3.0F}); + svs::distance::DistanceL2 distance{}; + auto params = v::VamanaBuildParameters{1.0F, 1, 1, 10, 1, true}; + auto pool = svs::threads::DefaultThreadPool(1); + std::vector roots{0}; + + // Default max_rounds: the stall guard, not the round cap, has to be what stops this. + auto report = v::repair_connectivity( + graph, data, distance, params, std::span(roots), pool + ); + + CATCH_REQUIRE(report.stalled); + CATCH_REQUIRE(report.rounds == 1); + CATCH_REQUIRE(report.unreachable_before == 1); + CATCH_REQUIRE(report.edges_forced_unsafe == 1); + CATCH_REQUIRE( + report.unreachable_after == + v::unreachable_nodes(graph, std::span(roots)).size() + ); + CATCH_REQUIRE(report.unreachable_after == 1); + for (size_t i = 0; i < graph.n_nodes(); ++i) { + CATCH_REQUIRE( + graph.get_node_degree(static_cast(i)) <= params.graph_max_degree + ); + } +} From e1c2dcaa31f99acd65327fde51e1cb7c03f99301 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 03:43:33 -0800 Subject: [PATCH 40/68] feat(vamana): build the GEM graph with routed starts and representatives first Adds gem_insert_order (every valid router representative first, deduplicated and ascending, then every other document ascending) and build_gem_graph, which runs VamanaIndex's own two-pass build sequence over that order when the distance carries a router, repairs connectivity from the entry point plus every representative, and reports what it did. Also adds the QEmd overload of extensions::build_entry_points: when the distance carries a router, a node's insert routes through its own centroid-id set the same way GemRoutedInitializer routes a query, merged with the fixed entry points (minus the node itself). Co-Authored-By: Claude Fable 5.2 --- include/svs/extensions/vamana/gem.h | 45 ++++ include/svs/index/vamana/gem_build.h | 162 +++++++++++++++ tests/CMakeLists.txt | 1 + tests/svs/index/vamana/gem_build.cpp | 297 +++++++++++++++++++++++++++ 4 files changed, 505 insertions(+) create mode 100644 include/svs/index/vamana/gem_build.h create mode 100644 tests/svs/index/vamana/gem_build.cpp diff --git a/include/svs/extensions/vamana/gem.h b/include/svs/extensions/vamana/gem.h index 4b1b0ec18..463828544 100644 --- a/include/svs/extensions/vamana/gem.h +++ b/include/svs/extensions/vamana/gem.h @@ -128,6 +128,51 @@ inline DefaultBuildAdaptor svs_invoke( return DefaultBuildAdaptor{distance}; } +// Routes a node's own insert like GemRoutedInitializer routes a query; no-op if unrouted. +template + requires std::unsigned_integral +std::span svs_invoke( + svs::tag_t, + const Data& data, + const svs::distance::QEmd& distance, + Idx node_id, + std::span entry_points, + std::vector& scratch +) { + const auto& router = distance.router(); + if (!router) { + return entry_points; + } + + auto doc = data.get_datum(node_id); + svs::distance::gem_detail::require_id_dimensions(doc.dimensions()); + thread_local std::vector id_buf; + const uint32_t* ids = + svs::distance::gem_detail::id_span(doc.data(), doc.cardinality(), id_buf); + + thread_local svs::distance::GemRouter::IdScratch route_scratch; + thread_local std::vector routed; + router->route_ids( + std::span(ids, doc.cardinality()), + router->nprobe, + routed, + route_scratch + ); + + scratch.assign(entry_points.begin(), entry_points.end()); + for (uint32_t id : routed) { + scratch.push_back(static_cast(id)); + } + std::sort(scratch.begin(), scratch.end()); + scratch.erase(std::unique(scratch.begin(), scratch.end()), scratch.end()); + scratch.erase(std::remove(scratch.begin(), scratch.end(), node_id), scratch.end()); + + if (scratch.empty()) { + return entry_points; + } + return std::span(scratch); +} + // Routes a QChamferExactQuery search; no-op to EntryPointInitializer when unrouted. template GemRoutedInitializer svs_invoke( diff --git a/include/svs/index/vamana/gem_build.h b/include/svs/index/vamana/gem_build.h new file mode 100644 index 000000000..4b0438157 --- /dev/null +++ b/include/svs/index/vamana/gem_build.h @@ -0,0 +1,162 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Mirrors index.h's build constructor, which hard-codes an L2/IP/Cosine check QEmd fails. +#include "svs/core/distance/gem.h" +#include "svs/core/distance/gem_router.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/build_params.h" +#include "svs/index/vamana/connectivity.h" +#include "svs/index/vamana/extensions.h" +#include "svs/index/vamana/vamana_build.h" +#include "svs/lib/threads.h" + +// stl +#include +#include +#include +#include +#include + +namespace svs::index::vamana { + +namespace gem_build_detail { +// Shared with gem_insert_order so the two never disagree on a valid representative. +template +std::vector valid_representatives(const svs::distance::GemRouter& router) { + std::vector reps = router.representative; + std::sort(reps.begin(), reps.end()); + reps.erase(std::unique(reps.begin(), reps.end()), reps.end()); + + std::vector out; + out.reserve(reps.size()); + for (uint32_t rep : reps) { + if (rep != svs::distance::GemRouter::no_representative) { + out.push_back(static_cast(rep)); + } + } + return out; +} +} // namespace gem_build_detail + +/// Insert order for a GEM build: every valid representative of `router` (deduplicated, +/// ascending), then every other document ascending. Every document in +/// `[0, num_documents)` appears exactly once. +template +std::vector +gem_insert_order(const svs::distance::GemRouter& router, size_t num_documents) { + std::vector order; + order.reserve(num_documents); + std::vector inserted(num_documents, false); + + for (Idx rep : gem_build_detail::valid_representatives(router)) { + inserted.at(rep) = true; + order.push_back(rep); + } + for (size_t doc = 0; doc < num_documents; ++doc) { + if (!inserted[doc]) { + order.push_back(static_cast(doc)); + } + } + return order; +} + +/// Outcome of `build_gem_graph`. `edges_added` is the safe total; `edges_forced_unsafe` +/// and `stalled` pass through from the connectivity report so a caller can see whether +/// the repair orphaned a node, or gave up early after making no more progress. +struct GemBuildReport { + size_t documents = 0; + size_t representatives_inserted_first = 0; + size_t unreachable_before_repair = 0; + size_t unreachable_after_repair = 0; + size_t repair_rounds = 0; + size_t edges_added = 0; + size_t edges_forced_unsafe = 0; + bool stalled = false; +}; + +/// Builds a Vamana graph over `data` (centroid-id sets) with `distance` (QEmd). With a +/// router, `gem_insert_order` picks the insert order; otherwise the ascending full range. +/// Repair roots are `entry_point` plus every valid representative. +template < + data::ImmutableMemoryDataset IdSets, + threads::ThreadPool Pool, + typename Allocator = HugepageAllocator> +std::pair, GemBuildReport> build_gem_graph( + const IdSets& data, + const svs::distance::QEmd& distance, + const VamanaBuildParameters& params, + Pool& threadpool, + uint32_t entry_point, + const Allocator& graph_allocator = {}, + svs::logging::logger_ptr logger = svs::logging::get() +) { + using Idx = uint32_t; + size_t num_documents = data.size(); + + GemBuildReport report{}; + report.documents = num_documents; + + auto graph = + default_graph(num_documents, params.graph_max_degree, graph_allocator); + + auto router = distance.router(); + std::vector roots{entry_point}; + if (router) { + auto reps = gem_build_detail::valid_representatives(*router); + report.representatives_inserted_first = reps.size(); + roots.insert(roots.end(), reps.begin(), reps.end()); + std::sort(roots.begin(), roots.end()); + roots.erase(std::unique(roots.begin(), roots.end()), roots.end()); + } + + auto builder = VamanaBuilder( + graph, + data, + distance, + params, + threadpool, + extensions::estimate_prefetch_parameters(data), + logger + ); + + if (router) { + auto order = gem_insert_order(*router, num_documents); + builder.construct(1.0F, entry_point, order, logging::Level::Trace, logger); + builder.construct(params.alpha, entry_point, order, logging::Level::Trace, logger); + } else { + builder.construct(1.0F, entry_point, logging::Level::Trace, logger); + builder.construct(params.alpha, entry_point, logging::Level::Trace, logger); + } + + auto connectivity = repair_connectivity( + graph, data, distance, params, lib::as_const_span(roots), threadpool + ); + report.unreachable_before_repair = connectivity.unreachable_before; + report.unreachable_after_repair = connectivity.unreachable_after; + report.repair_rounds = connectivity.rounds; + report.edges_added = connectivity.edges_appended + + connectivity.edges_accepted_through_prune + + connectivity.edges_forced; + report.edges_forced_unsafe = connectivity.edges_forced_unsafe; + report.stalled = connectivity.stalled; + + return {std::move(graph), report}; +} + +} // namespace svs::index::vamana diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 243e91ca2..0a5ad0d01 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -142,6 +142,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/index/vamana/connectivity.cpp ${TEST_DIR}/svs/index/vamana/consolidate.cpp ${TEST_DIR}/svs/index/vamana/filter.cpp + ${TEST_DIR}/svs/index/vamana/gem_build.cpp ${TEST_DIR}/svs/index/vamana/gem_routing.cpp ${TEST_DIR}/svs/index/vamana/gem_vector_set.cpp ${TEST_DIR}/svs/index/vamana/index.cpp diff --git a/tests/svs/index/vamana/gem_build.cpp b/tests/svs/index/vamana/gem_build.cpp new file mode 100644 index 000000000..fe309364f --- /dev/null +++ b/tests/svs/index/vamana/gem_build.cpp @@ -0,0 +1,297 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Headers under test: gem_build.h and the QEmd build_entry_points overload (gem.h). +#include "svs/index/vamana/gem_build.h" + +// svs +#include "svs/core/data.h" +#include "svs/core/data/vector_set.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/core/distance/gem.h" +#include "svs/core/distance/gem_router.h" +#include "svs/core/graph.h" +#include "svs/extensions/vamana/gem.h" +#include "svs/index/vamana/connectivity.h" +#include "svs/lib/threads.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using svs::distance::CentroidTable; +using svs::distance::Ground; +using svs::distance::QEmd; + +constexpr size_t K = 16; +constexpr size_t DIM = 4; + +std::vector make_centroids(uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution coord(-1.0F, 1.0F); + std::vector centroids(K * DIM); + for (size_t k = 0; k < K; ++k) { + double norm = 0.0; + for (size_t d = 0; d < DIM; ++d) { + float v = coord(rng); + centroids[k * DIM + d] = v; + norm += static_cast(v) * static_cast(v); + } + norm = std::sqrt(norm); + for (size_t d = 0; d < DIM; ++d) { + centroids[k * DIM + d] = static_cast(centroids[k * DIM + d] / norm); + } + } + return centroids; +} + +std::shared_ptr make_table(const std::vector& centroids) { + return std::make_shared( + CentroidTable::from_centroids(centroids.data(), K, DIM, Ground::Cos, 1) + ); +} + +struct IdSetCorpus { + std::vector ids; + std::vector offsets; + + size_t size() const { return offsets.size() - 1; } + + svs::data::ConstVectorSetView view() const { + return svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); + } +}; + +IdSetCorpus make_doc_corpus(size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution id_dist(0, static_cast(K - 1)); + IdSetCorpus corpus; + corpus.offsets.resize(n + 1, 0); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 2 + (i % 5); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + for (size_t t = 0; t < cardinality; ++t) { + corpus.ids.push_back(id_dist(rng)); + } + } + return corpus; +} + +// Fine centroid f goes to coarse cluster f % num_coarse; deterministic, not realistic. +std::vector round_robin_coarse_of_fine(size_t num_coarse) { + std::vector out(K); + for (size_t f = 0; f < K; ++f) { + out[f] = static_cast(f % num_coarse); + } + return out; +} + +std::vector> brute_force_qemd( + QEmd& dist, + const svs::data::ConstVectorSet& query, + const svs::data::ConstVectorSetView& docs +) { + std::vector> scored; + scored.reserve(docs.size()); + for (uint32_t id = 0; id < docs.size(); ++id) { + float score = svs::distance::compute(dist, query, docs.get_datum(id)); + scored.emplace_back(id, score); + } + std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { + return QEmd::compare{}(a.second, b.second); + }); + return scored; +} + +} // namespace + +CATCH_TEST_CASE("gem_insert_order", "[vamana][gem][gem_build]") { + namespace v = svs::index::vamana; + + svs::distance::GemRouter router; + // 6 documents; representatives have a duplicate (cluster 2 has no_representative). + router.coarse_of_fine = {0, 0, 1, 1, 2, 2}; + router.representative = {4, 2, svs::distance::GemRouter::no_representative, 2}; + router.nprobe = 2; + + auto order = v::gem_insert_order(router, 6); + + // Every document appears exactly once. + auto sorted_order = order; + std::sort(sorted_order.begin(), sorted_order.end()); + CATCH_REQUIRE(sorted_order == std::vector{0, 1, 2, 3, 4, 5}); + + // Representatives (deduplicated, ascending, no_representative dropped) come first. + CATCH_REQUIRE(order.size() >= 2); + CATCH_REQUIRE(order[0] == 2); + CATCH_REQUIRE(order[1] == 4); + // The rest follow ascending. + CATCH_REQUIRE( + std::vector(order.begin() + 2, order.end()) == + std::vector{0, 1, 3, 5} + ); +} + +CATCH_TEST_CASE("build_entry_points for QEmd", "[vamana][gem][gem_build]") { + namespace ext = svs::index::vamana::extensions; + auto centroids = make_centroids(0xB001); + auto table = make_table(centroids); + + auto doc_corpus = make_doc_corpus(20, 0xB002); + auto docs = doc_corpus.view(); + + std::vector entry_points{3}; + std::vector scratch; + + CATCH_SECTION("No router returns entry_points unchanged") { + QEmd distance{table}; + auto result = ext::build_entry_points( + docs, distance, uint32_t{7}, std::span(entry_points), scratch + ); + CATCH_REQUIRE(std::vector(result.begin(), result.end()) == entry_points); + } + + CATCH_SECTION("Every cluster unrepresented falls back to entry_points") { + svs::distance::GemRouter all_norep; + all_norep.coarse_of_fine = round_robin_coarse_of_fine(4); + all_norep.representative.assign(4, svs::distance::GemRouter::no_representative); + all_norep.nprobe = 2; + auto router_ptr = std::make_shared(all_norep); + + QEmd distance{table, router_ptr}; + auto result = ext::build_entry_points( + docs, distance, uint32_t{7}, std::span(entry_points), scratch + ); + CATCH_REQUIRE(std::vector(result.begin(), result.end()) == entry_points); + } + + CATCH_SECTION("A real router returns entry_points union routed, minus node_id") { + constexpr size_t num_coarse = 4; + auto router = svs::distance::GemRouter::build( + round_robin_coarse_of_fine(num_coarse), num_coarse, docs, size_t{1}, size_t{2} + ); + auto router_ptr = std::make_shared(router); + QEmd distance{table, router_ptr}; + + uint32_t node_id = 7; + auto doc = docs.get_datum(node_id); + std::vector doc_ids(doc.data(), doc.data() + doc.cardinality()); + + std::vector routed; + svs::distance::GemRouter::IdScratch id_scratch; + router.route_ids(doc_ids, router.nprobe, routed, id_scratch); + + std::vector expected(entry_points.begin(), entry_points.end()); + expected.insert(expected.end(), routed.begin(), routed.end()); + std::sort(expected.begin(), expected.end()); + expected.erase(std::unique(expected.begin(), expected.end()), expected.end()); + expected.erase( + std::remove(expected.begin(), expected.end(), node_id), expected.end() + ); + + auto result = ext::build_entry_points( + docs, distance, node_id, std::span(entry_points), scratch + ); + CATCH_REQUIRE(std::vector(result.begin(), result.end()) == expected); + } +} + +CATCH_TEST_CASE("build_gem_graph", "[vamana][gem][gem_build]") { + namespace v = svs::index::vamana; + + auto centroids = make_centroids(0xB003); + auto table = make_table(centroids); + + const size_t n_docs = 300; + auto doc_corpus = make_doc_corpus(n_docs, 0xB004); + auto docs = doc_corpus.view(); + + auto params = v::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + const uint32_t entry_point = 0; + + constexpr size_t num_coarse = 8; + auto router = svs::distance::GemRouter::build( + round_robin_coarse_of_fine(num_coarse), num_coarse, docs, size_t{1}, size_t{3} + ); + auto router_ptr = std::make_shared(router); + + auto run_build = [&](std::shared_ptr maybe_router) { + QEmd distance{table, std::move(maybe_router)}; + auto pool = svs::threads::DefaultThreadPool(1); + auto [graph, report] = + v::build_gem_graph(docs, distance, params, pool, entry_point); + + CATCH_REQUIRE(report.documents == n_docs); + CATCH_REQUIRE(report.unreachable_after_repair == 0); + for (size_t i = 0; i < graph.n_nodes(); ++i) { + CATCH_REQUIRE( + graph.get_node_degree(static_cast(i)) <= params.graph_max_degree + ); + } + return graph; + }; + + auto graph_unrouted = run_build(nullptr); + auto graph_routed = run_build(router_ptr); + + // Recall check: routed build's top-1 search must not get worse than the unrouted one. + auto count_top1_hits = [&](const svs::graphs::SimpleGraph& graph) { + QEmd oracle_distance{table}; + size_t hits = 0; + auto search_buffer = v::SearchBuffer{64}; + for (uint32_t q = 0; q < 20; ++q) { + search_buffer.clear(); + auto query = docs.get_datum(q); + auto oracle = brute_force_qemd(oracle_distance, query, docs); + + QEmd search_distance{table}; + auto accessor = svs::data::GetDatumAccessor{}; + v::greedy_search( + graph, + docs, + accessor, + query, + search_distance, + search_buffer, + v::EntryPointInitializer{ + std::span(&entry_point, 1)} + ); + search_buffer.sort(); + if (search_buffer.size() > 0 && search_buffer[0].id() == oracle[0].first) { + ++hits; + } + } + return hits; + }; + + size_t hits_unrouted = count_top1_hits(graph_unrouted); + size_t hits_routed = count_top1_hits(graph_routed); + CATCH_REQUIRE(hits_routed >= hits_unrouted); +} From 858fa456d61bdfd7e649e2e1351010a14d112a8c Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 06:58:48 -0800 Subject: [PATCH 41/68] refactor(quantization): move GEM quantize timings into an opt-in header QuantizeTimings and all std::chrono use leave codebook.h. train_codebook and quantize are now thin compositions of public phase functions (sample_training_tokens, train_centroids, assign_tokens, build_ground_structures, coarse_clusters); train_centroids's k-means loop takes a compile-time observer (default a no-op) called once per iteration, with zero cost when unused. codebook_timed.h is new and opt-in: it defines QuantizeTimings and quantize_timed/train_codebook_timed, composing the public phases with timers and an observer that records iteration seconds. Nothing in the library includes it. Co-Authored-By: Claude Opus 5.5 --- include/svs/quantization/gem/codebook.h | 231 +++++++++--------- include/svs/quantization/gem/codebook_timed.h | 171 +++++++++++++ tests/svs/quantization/gem/codebook.cpp | 80 +++++- 3 files changed, 359 insertions(+), 123 deletions(-) create mode 100644 include/svs/quantization/gem/codebook_timed.h diff --git a/include/svs/quantization/gem/codebook.h b/include/svs/quantization/gem/codebook.h index f94ef03c5..8197b5915 100644 --- a/include/svs/quantization/gem/codebook.h +++ b/include/svs/quantization/gem/codebook.h @@ -36,7 +36,6 @@ // stl #include -#include #include #include #include @@ -126,18 +125,6 @@ struct CodebookParameters { size_t assign_buffer_bytes = 32 * 1024 * 1024; }; -/// Wall-clock seconds for each phase, measured with std::chrono::steady_clock. -struct QuantizeTimings { - double sample = 0.0; - double train = 0.0; - double normalize = 0.0; - double assign = 0.0; - double ground_table = 0.0; - double coarse = 0.0; - double total = 0.0; - std::vector train_iterations; -}; - /// Any contiguous vector-set dataset of float or Float16 tokens: VectorSetData and /// ConstVectorSetView both satisfy this without copying their token storage. template @@ -235,26 +222,51 @@ void plain_mean_update( ); } +} // namespace detail + +/// Centroids and the k-means objective after train_centroids; the observer (if any) owns +/// per-iteration diagnostics instead, so this result does not carry them. struct L2KMeansResult { svs::data::SimpleData centroids; double objective = 0.0; - std::vector iteration_seconds; }; -// Shared L2/plain-mean trainer; takes Data by non-const ref, so callers pass a local copy. -template -L2KMeansResult run_l2_kmeans( +/// Default k-means observer: both hooks are empty, so an optimizer can elide the calls +/// entirely and a caller pays nothing for not timing iterations. +struct NoOpKMeansObserver { + void before_iteration(size_t /*iter*/) {} + void after_iteration(size_t /*iter*/) {} +}; + +/// Satisfied by NoOpKMeansObserver and by any type exposing the same two hooks, e.g. one +/// that times the interval between them. +template +concept KMeansObserver = requires(T observer, size_t iter) { + observer.before_iteration(iter); + observer.after_iteration(iter); + }; + +/// Shared L2/plain-mean trainer; takes Data by non-const ref, so callers pass a local +/// copy. `observer`'s before_/after_iteration are called once per iteration, with no +/// other effect on the result, so a caller can measure iterations without this function +/// depending on any timing facility itself. +template < + svs::data::ImmutableMemoryDataset Data, + threads::ThreadPool Pool, + KMeansObserver Observer = NoOpKMeansObserver> +L2KMeansResult train_centroids( Data& data, size_t num_clusters, size_t iterations, size_t minibatch_size, std::mt19937& rng, - Pool& threadpool + Pool& threadpool, + Observer observer = Observer{} ) { size_t n = data.size(); if (n < num_clusters) { throw ANNEXCEPTION( - "run_l2_kmeans: only {} points available, need at least num_clusters={}", + "train_centroids: only {} points available, need at least num_clusters={}", n, num_clusters ); @@ -262,8 +274,8 @@ L2KMeansResult run_l2_kmeans( // faiss subsamples past 256/centroid; throws instead of silently diverging from it. if (n > num_clusters * 256) { throw ANNEXCEPTION( - "run_l2_kmeans: {} points exceed faiss's 256-per-centroid training bound for " - "num_clusters={} (max {}); subsampling instead of throwing would silently " + "train_centroids: {} points exceed faiss's 256-per-centroid training bound " + "for num_clusters={} (max {}); subsampling instead of throwing would silently " "differ from the reference training procedure", n, num_clusters, @@ -286,10 +298,8 @@ L2KMeansResult run_l2_kmeans( svs::lib::Timer scratch_timer; double objective = 0.0; - std::vector iteration_seconds; - iteration_seconds.reserve(iterations); for (size_t iter = 0; iter < iterations; ++iter) { - auto iter_start = std::chrono::steady_clock::now(); + observer.before_iteration(iter); svs::kmeans::generate_norms(centroids, centroids_norm, threadpool); size_t num_batches = svs::lib::div_round_up(n, batch_cap); @@ -313,49 +323,45 @@ L2KMeansResult run_l2_kmeans( // Objective is defined on the last pass, against the centroids before its update. if (iter + 1 == iterations) { - objective = compute_objective(data, centroids, assignments); + objective = detail::compute_objective(data, centroids, assignments); } - plain_mean_update(data, centroids, assignments, counts, threadpool); + detail::plain_mean_update(data, centroids, assignments, counts, threadpool); svs::kmeans::centroid_split( data, centroids, counts, rng, threadpool, scratch_timer ); - - iteration_seconds.push_back( - std::chrono::duration(std::chrono::steady_clock::now() - iter_start) - .count() - ); + observer.after_iteration(iter); } - return L2KMeansResult{std::move(centroids), objective, std::move(iteration_seconds)}; + return L2KMeansResult{std::move(centroids), objective}; } -} // namespace detail - -/// Result of train_codebook: the normalized float32 centroids plus training diagnostics. -struct TrainedCodebook { - svs::data::SimpleData centroids; - double objective = 0.0; +/// sample_training_tokens's output: the float32 tokens train_centroids trains on, plus +/// the resolved centroid count and sampling sizes train_codebook reports. +struct SampledTrainingTokens { + svs::data::SimpleData trainset; + size_t K = 0; size_t num_sampled_passages = 0; size_t num_training_tokens = 0; - QuantizeTimings timings; }; -/// Steps 1-4: resolve K, sample passages, gather/shuffle/hold out tokens, run plain-mean -/// L2 k-means, then normalize once. Only sampled tokens convert to float32, so a -/// memory-mapped Float16 store is never copied in full. +/// Steps 1-3: resolve K, sample passages, then gather/shuffle/hold out their tokens into a +/// float32 trainset. Only sampled tokens convert to float32, so a memory-mapped Float16 +/// store is never copied in full. Advances `rng`, so a caller chaining into +/// train_centroids with the same `rng` continues its stream rather than restarting it. template -TrainedCodebook -train_codebook(const Data& tokens, const CodebookParameters& parameters, Pool& threadpool) { - auto total_start = std::chrono::steady_clock::now(); - QuantizeTimings timings; - +SampledTrainingTokens sample_training_tokens( + const Data& tokens, + const CodebookParameters& parameters, + std::mt19937& rng, + Pool& threadpool +) { size_t dim = tokens.dimensions(); size_t num_passages = tokens.size(); size_t num_total_tokens = tokens.total_tokens(); if (num_passages == 0 || num_total_tokens == 0) { throw ANNEXCEPTION( - "train_codebook: tokens dataset is empty ({} passages, {} tokens)", + "sample_training_tokens: tokens dataset is empty ({} passages, {} tokens)", num_passages, num_total_tokens ); @@ -366,14 +372,13 @@ train_codebook(const Data& tokens, const CodebookParameters& parameters, Pool& t constexpr size_t max_id = static_cast(std::numeric_limits::max()); if (K > max_id) { throw ANNEXCEPTION( - "train_codebook: K={} exceeds the uint32_t centroid-id range (max {})", + "sample_training_tokens: K={} exceeds the uint32_t centroid-id range (max {})", K, max_id ); } size_t sample_size = colbert_sample_size(num_passages); - std::mt19937 rng(parameters.seed); std::vector passage_ids(sample_size); svs::kmeans::generate_unique_ids(passage_ids, num_passages, rng); @@ -410,15 +415,14 @@ train_codebook(const Data& tokens, const CodebookParameters& parameters, Pool& t if (num_training < K) { throw ANNEXCEPTION( - "train_codebook: only {} training tokens sampled (after a {} token held-out " - "split), need at least K={}", + "sample_training_tokens: only {} training tokens sampled (after a {} token " + "held-out split), need at least K={}", num_training, heldout, K ); } - auto sample_start = std::chrono::steady_clock::now(); auto trainset = svs::data::SimpleData(num_training, dim); threads::parallel_for( threadpool, @@ -436,41 +440,43 @@ train_codebook(const Data& tokens, const CodebookParameters& parameters, Pool& t } } ); - timings.sample = - std::chrono::duration(std::chrono::steady_clock::now() - sample_start) - .count(); - auto train_start = std::chrono::steady_clock::now(); - auto kmeans_result = detail::run_l2_kmeans( - trainset, - K, + return SampledTrainingTokens{std::move(trainset), K, sample_size, num_training}; +} + +/// Result of train_codebook: the normalized float32 centroids plus training diagnostics. +struct TrainedCodebook { + svs::data::SimpleData centroids; + double objective = 0.0; + size_t num_sampled_passages = 0; + size_t num_training_tokens = 0; +}; + +/// Composes sample_training_tokens and train_centroids, then normalizes once: the +/// pipeline a caller wanting plain results (no phase timing) runs end to end. +template +TrainedCodebook +train_codebook(const Data& tokens, const CodebookParameters& parameters, Pool& threadpool) { + std::mt19937 rng(parameters.seed); + auto sampled = sample_training_tokens(tokens, parameters, rng, threadpool); + + auto kmeans_result = train_centroids( + sampled.trainset, + sampled.K, parameters.kmeans_iterations, parameters.minibatch_size, rng, threadpool ); - timings.train = - std::chrono::duration(std::chrono::steady_clock::now() - train_start) - .count(); - timings.train_iterations = std::move(kmeans_result.iteration_seconds); - auto normalize_start = std::chrono::steady_clock::now(); svs::lib::Timer scratch_timer; svs::kmeans::normalize_centroids(kmeans_result.centroids, threadpool, scratch_timer); - timings.normalize = - std::chrono::duration(std::chrono::steady_clock::now() - normalize_start) - .count(); - - timings.total = - std::chrono::duration(std::chrono::steady_clock::now() - total_start) - .count(); return TrainedCodebook{ std::move(kmeans_result.centroids), kmeans_result.objective, - sample_size, - num_training, - timings}; + sampled.num_sampled_passages, + sampled.num_training_tokens}; } /// Step 5: assign every document token to its nearest (argmax inner product) normalized @@ -687,7 +693,7 @@ MismatchReport count_assignment_mismatches( /// Clusters fine centroids into coarse groups via the same L2/plain-mean trainer, then /// an exact nearest-coarse assignment. `centroids` is read-only; a local copy is -/// clustered since run_l2_kmeans takes Data by non-const ref. +/// clustered since train_centroids takes Data by non-const ref. struct CoarseClusters { std::vector coarse_of_fine; size_t num_coarse = 0; @@ -714,7 +720,7 @@ CoarseClusters coarse_clusters( // Offset from the fine-training seed so the two never share a random stream. std::mt19937 rng(parameters.seed + 1); auto working_copy = centroids; - auto kmeans_result = detail::run_l2_kmeans( + auto kmeans_result = train_centroids( working_copy, num_coarse, parameters.coarse_iterations, @@ -753,6 +759,34 @@ CoarseClusters coarse_clusters( return CoarseClusters{std::move(coarse_of_fine), num_coarse}; } +/// build_ground_structures's output: the bf16 FineCentroids and pairwise CentroidTable +/// GEM's distances need, built from the same trained centroids. +struct GroundStructures { + std::shared_ptr fine_centroids; + std::shared_ptr centroid_table; +}; + +/// Builds the bf16 FineCentroids and the pairwise CentroidTable from trained centroids; +/// the pair quantize's distances need but neither centroid_table nor FineCentroids' +/// constructor derives from the other, so they are built together here once. +inline GroundStructures build_ground_structures( + const svs::data::SimpleData& centroids, + const CodebookParameters& parameters, + size_t num_threads +) { + size_t K = centroids.size(); + size_t dim = centroids.dimensions(); + auto fine_centroids = std::make_shared( + K, dim, centroids.data(), num_threads + ); + auto centroid_table = std::make_shared( + svs::distance::CentroidTable::from_centroids( + centroids.data(), K, dim, parameters.ground, num_threads + ) + ); + return GroundStructures{std::move(fine_centroids), std::move(centroid_table)}; +} + /// Full pipeline: train, assign, build the bf16 FineCentroids and the pairwise /// CentroidTable, and cluster the fine centroids themselves. The single entry point a /// driver calls to turn a raw token store into everything GEM's distances need. @@ -767,63 +801,32 @@ struct Quantized { double objective = 0.0; size_t num_sampled_passages = 0; size_t num_training_tokens = 0; - QuantizeTimings timings; }; +/// Composes train_codebook, assign_tokens, build_ground_structures and coarse_clusters: +/// the pipeline a caller wanting plain results (no phase timing) runs end to end. template Quantized quantize(const Data& tokens, const CodebookParameters& parameters, Pool& threadpool) { - auto total_start = std::chrono::steady_clock::now(); - auto trained = train_codebook(tokens, parameters, threadpool); - QuantizeTimings timings = trained.timings; - - auto assign_start = std::chrono::steady_clock::now(); auto ids = assign_tokens(tokens, trained.centroids, parameters, threadpool); - timings.assign = - std::chrono::duration(std::chrono::steady_clock::now() - assign_start) - .count(); size_t K = trained.centroids.size(); - size_t dim = trained.centroids.dimensions(); size_t num_threads = threadpool.size(); - - // Timed with the pairwise table build; neither has its own QuantizeTimings field. - auto ground_start = std::chrono::steady_clock::now(); - auto fine_centroids = std::make_shared( - K, dim, trained.centroids.data(), num_threads - ); - auto centroid_table = std::make_shared( - svs::distance::CentroidTable::from_centroids( - trained.centroids.data(), K, dim, parameters.ground, num_threads - ) - ); - timings.ground_table = - std::chrono::duration(std::chrono::steady_clock::now() - ground_start) - .count(); - - auto coarse_start = std::chrono::steady_clock::now(); + auto ground = build_ground_structures(trained.centroids, parameters, num_threads); auto coarse = coarse_clusters(trained.centroids, parameters, threadpool); - timings.coarse = - std::chrono::duration(std::chrono::steady_clock::now() - coarse_start) - .count(); - - timings.total = - std::chrono::duration(std::chrono::steady_clock::now() - total_start) - .count(); return Quantized{ std::move(trained.centroids), - std::move(fine_centroids), - std::move(centroid_table), + std::move(ground.fine_centroids), + std::move(ground.centroid_table), std::move(ids), std::move(coarse.coarse_of_fine), coarse.num_coarse, K, trained.objective, trained.num_sampled_passages, - trained.num_training_tokens, - timings}; + trained.num_training_tokens}; } } // namespace svs::quantization::gem diff --git a/include/svs/quantization/gem/codebook_timed.h b/include/svs/quantization/gem/codebook_timed.h new file mode 100644 index 000000000..baec42ce9 --- /dev/null +++ b/include/svs/quantization/gem/codebook_timed.h @@ -0,0 +1,171 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if SVS_ENABLE_MULTI_VECTOR + +// svs +#include "svs/lib/threads.h" +#include "svs/lib/timing.h" +#include "svs/quantization/gem/codebook.h" + +// stl +#include +#include +#include +#include + +// Opt-in wall-clock timing for svs::quantization::gem::quantize/train_codebook. Nothing +// in the library includes this header; a caller that wants phase timings does. +namespace svs::quantization::gem { + +/// Wall-clock seconds for each phase, measured with std::chrono::steady_clock. +struct QuantizeTimings { + double sample = 0.0; + double train = 0.0; + double normalize = 0.0; + double assign = 0.0; + double ground_table = 0.0; + double coarse = 0.0; + double total = 0.0; + std::vector train_iterations; +}; + +namespace detail { + +// Times the interval between before_/after_iteration and appends it to `seconds`, which +// outlives the observer; satisfies KMeansObserver. +struct IterationTimer { + std::vector& seconds; + std::chrono::steady_clock::time_point start; + + void before_iteration(size_t /*iter*/) { start = std::chrono::steady_clock::now(); } + void after_iteration(size_t /*iter*/) { + seconds.push_back( + std::chrono::duration(std::chrono::steady_clock::now() - start).count() + ); + } +}; + +} // namespace detail + +/// train_codebook, plus a QuantizeTimings for sample/train/normalize/train_iterations. +template +std::pair train_codebook_timed( + const Data& tokens, const CodebookParameters& parameters, Pool& threadpool +) { + auto total_start = std::chrono::steady_clock::now(); + QuantizeTimings timings; + + std::mt19937 rng(parameters.seed); + + auto sample_start = std::chrono::steady_clock::now(); + auto sampled = sample_training_tokens(tokens, parameters, rng, threadpool); + timings.sample = + std::chrono::duration(std::chrono::steady_clock::now() - sample_start) + .count(); + + auto train_start = std::chrono::steady_clock::now(); + auto kmeans_result = train_centroids( + sampled.trainset, + sampled.K, + parameters.kmeans_iterations, + parameters.minibatch_size, + rng, + threadpool, + detail::IterationTimer{timings.train_iterations, {}} + ); + timings.train = + std::chrono::duration(std::chrono::steady_clock::now() - train_start) + .count(); + + auto normalize_start = std::chrono::steady_clock::now(); + svs::lib::Timer scratch_timer; + svs::kmeans::normalize_centroids(kmeans_result.centroids, threadpool, scratch_timer); + timings.normalize = + std::chrono::duration(std::chrono::steady_clock::now() - normalize_start) + .count(); + + timings.total = + std::chrono::duration(std::chrono::steady_clock::now() - total_start) + .count(); + + auto trained = TrainedCodebook{ + std::move(kmeans_result.centroids), + kmeans_result.objective, + sampled.num_sampled_passages, + sampled.num_training_tokens}; + return {std::move(trained), std::move(timings)}; +} + +/// quantize, plus the full QuantizeTimings (train_codebook_timed's phases, then +/// assign/ground_table/coarse/total). Identical result to quantize on the same input. +template +std::pair +quantize_timed(const Data& tokens, const CodebookParameters& parameters, Pool& threadpool) { + auto total_start = std::chrono::steady_clock::now(); + + auto [trained, timings] = train_codebook_timed(tokens, parameters, threadpool); + + auto assign_start = std::chrono::steady_clock::now(); + auto ids = assign_tokens(tokens, trained.centroids, parameters, threadpool); + timings.assign = + std::chrono::duration(std::chrono::steady_clock::now() - assign_start) + .count(); + + size_t K = trained.centroids.size(); + size_t num_threads = threadpool.size(); + + // Timed with the pairwise table build; neither has its own QuantizeTimings field. + auto ground_start = std::chrono::steady_clock::now(); + auto ground = build_ground_structures(trained.centroids, parameters, num_threads); + timings.ground_table = + std::chrono::duration(std::chrono::steady_clock::now() - ground_start) + .count(); + + auto coarse_start = std::chrono::steady_clock::now(); + auto coarse = coarse_clusters(trained.centroids, parameters, threadpool); + timings.coarse = + std::chrono::duration(std::chrono::steady_clock::now() - coarse_start) + .count(); + + timings.total = + std::chrono::duration(std::chrono::steady_clock::now() - total_start) + .count(); + + auto quantized = Quantized{ + std::move(trained.centroids), + std::move(ground.fine_centroids), + std::move(ground.centroid_table), + std::move(ids), + std::move(coarse.coarse_of_fine), + coarse.num_coarse, + K, + trained.objective, + trained.num_sampled_passages, + trained.num_training_tokens}; + return {std::move(quantized), std::move(timings)}; +} + +} // namespace svs::quantization::gem + +#else + +#error \ + "svs/quantization/gem/codebook_timed.h needs SVS_ENABLE_MULTI_VECTOR=1 (cmake: SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR=YES); it pulls in Intel MKL through codebook.h." + +#endif // SVS_ENABLE_MULTI_VECTOR diff --git a/tests/svs/quantization/gem/codebook.cpp b/tests/svs/quantization/gem/codebook.cpp index 53da8fecc..1aa87ae5d 100644 --- a/tests/svs/quantization/gem/codebook.cpp +++ b/tests/svs/quantization/gem/codebook.cpp @@ -16,6 +16,7 @@ // header under test #include "svs/quantization/gem/codebook.h" +#include "svs/quantization/gem/codebook_timed.h" // svs #include "svs/core/data/simple.h" @@ -245,7 +246,33 @@ CATCH_TEST_CASE( } CATCH_REQUIRE(std::abs(std::sqrt(norm_sq) - 1.0F) < 1e-4F); } - CATCH_REQUIRE(four_iter.timings.train_iterations.size() == 4); +} + +CATCH_TEST_CASE( + "GEM Codebook - train_codebook_timed records one iteration second per k-means pass", + "[quantization][gem]" +) { + constexpr size_t K = 8; + constexpr size_t DIM = 16; + auto centers = make_random_centers(K, DIM, 0x1234); + + std::vector cardinalities(50, 6); + auto corpus = make_clustered_corpus(centers, K, DIM, cardinalities, 0xABCD, 0.05F); + + gem::CodebookParameters parameters; + parameters.num_centroids = K; + parameters.seed = 7; + parameters.kmeans_iterations = 4; + + auto threadpool = svs::threads::as_threadpool(2); + auto [trained, timings] = gem::train_codebook_timed(corpus, parameters, threadpool); + + CATCH_REQUIRE(timings.train_iterations.size() == 4); + CATCH_REQUIRE(timings.sample >= 0.0); + CATCH_REQUIRE(timings.train >= 0.0); + CATCH_REQUIRE(timings.normalize >= 0.0); + CATCH_REQUIRE(timings.total >= 0.0); + CATCH_REQUIRE(trained.num_sampled_passages > 0); } CATCH_TEST_CASE( @@ -425,14 +452,49 @@ CATCH_TEST_CASE( for (auto c : result.coarse_of_fine) { CATCH_REQUIRE(c < result.num_coarse); } - CATCH_REQUIRE(result.timings.train_iterations.size() == parameters.kmeans_iterations); - CATCH_REQUIRE(result.timings.total >= 0.0); - CATCH_REQUIRE(result.timings.sample >= 0.0); - CATCH_REQUIRE(result.timings.train >= 0.0); - CATCH_REQUIRE(result.timings.normalize >= 0.0); - CATCH_REQUIRE(result.timings.assign >= 0.0); - CATCH_REQUIRE(result.timings.ground_table >= 0.0); - CATCH_REQUIRE(result.timings.coarse >= 0.0); +} + +CATCH_TEST_CASE( + "GEM Codebook - quantize_timed matches quantize and records every phase", + "[quantization][gem]" +) { + constexpr size_t DIM = 16; + constexpr size_t NUM_PASSAGES = 200; + // 200 passages * 2 tokens == 400; colbert_num_partitions(400) == 256, still exceeded. + std::vector cardinalities(NUM_PASSAGES, 2); + auto centers = make_random_centers(16, DIM, 0x9999); + auto corpus = make_clustered_corpus(centers, 16, DIM, cardinalities, 0x1357, 0.1F); + + gem::CodebookParameters parameters; // num_centroids == 0: use the rule. + auto threadpool = svs::threads::as_threadpool(2); + + auto plain = gem::quantize(corpus, parameters, threadpool); + auto [timed, timings] = gem::quantize_timed(corpus, parameters, threadpool); + + // Same seed, same corpus: quantize_timed must reproduce quantize's result exactly. + CATCH_REQUIRE(timed.K == plain.K); + CATCH_REQUIRE(timed.num_coarse == plain.num_coarse); + CATCH_REQUIRE(timed.objective == plain.objective); + CATCH_REQUIRE(timed.num_sampled_passages == plain.num_sampled_passages); + CATCH_REQUIRE(timed.num_training_tokens == plain.num_training_tokens); + CATCH_REQUIRE(timed.coarse_of_fine == plain.coarse_of_fine); + CATCH_REQUIRE(flatten_ids(timed.ids) == flatten_ids(plain.ids)); + for (size_t i = 0; i < timed.centroids.size(); ++i) { + auto a = timed.centroids.get_datum(i); + auto b = plain.centroids.get_datum(i); + for (size_t d = 0; d < a.size(); ++d) { + CATCH_REQUIRE(a[d] == b[d]); + } + } + + CATCH_REQUIRE(timings.train_iterations.size() == parameters.kmeans_iterations); + CATCH_REQUIRE(timings.total >= 0.0); + CATCH_REQUIRE(timings.sample >= 0.0); + CATCH_REQUIRE(timings.train >= 0.0); + CATCH_REQUIRE(timings.normalize >= 0.0); + CATCH_REQUIRE(timings.assign >= 0.0); + CATCH_REQUIRE(timings.ground_table >= 0.0); + CATCH_REQUIRE(timings.coarse >= 0.0); } CATCH_TEST_CASE("GEM Codebook - failure paths", "[quantization][gem]") { From 34092896d645ef4d4941bb09a0eba37818c71a97 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 07:02:37 -0800 Subject: [PATCH 42/68] refactor(distance): define the inner-product matrix kernels in their header Review on PR 478 pointed out an inconsistency: avx2.cpp/avx512.cpp/amx.cpp previously held only macro-driven explicit instantiations of templates defined in their headers, but this PR added the InnerProductMatrixImpl AVX2/AVX512/AMX kernel bodies directly in those .cpp files instead. Moves every kernel body (and its anonymous-namespace helpers, now named detail namespaces per ISA: inner_product_matrix_{avx2,avx512,amx}_detail) into inner_product_matrix.h, next to the specialization each belongs to, guarded by the same SVS_AVX512_F / SVS_AVX2 / SVS_AMX_TILE_BF16 conditions that already guarded the specialization declarations. Each .cpp now holds only includes and explicit instantiation lines, matching euclidean.h's and gem.h's QEmdImpl/QChamferExactImpl pattern. No kernel logic changed. Co-Authored-By: Claude Opus 5.5 --- .../svs/core/distance/inner_product_matrix.h | 1077 ++++++++++++++++- include/svs/multi-arch/x86/amx.cpp | 378 +----- include/svs/multi-arch/x86/avx2.cpp | 344 ------ include/svs/multi-arch/x86/avx512.cpp | 349 ------ 4 files changed, 1046 insertions(+), 1102 deletions(-) diff --git a/include/svs/core/distance/inner_product_matrix.h b/include/svs/core/distance/inner_product_matrix.h index e38a76602..5cdbf8ecb 100644 --- a/include/svs/core/distance/inner_product_matrix.h +++ b/include/svs/core/distance/inner_product_matrix.h @@ -17,14 +17,18 @@ #pragma once #include "svs/core/distance/distance_core.h" +#include "svs/core/distance/simd_utils.h" #include "svs/lib/avx_detection.h" #include "svs/lib/bfloat16.h" #include "svs/lib/float16.h" #include "svs/lib/preprocessor.h" +#include + #include #include #include +#include #include #include #include @@ -176,7 +180,7 @@ class PackedOperand { /// so unlike Float16 there is no rounding question on A's side; B's precision for a /// BFloat16 A is the caller's choice, made by packing with /// `PackedOperand::pack_rounded_to_bf16` instead of `pack`. -template struct InnerProductMatrixImpl { +template struct InnerProductMatrixImpl { // Shared by every element type A can hold: float32 needs no conversion (the // static_cast below is a no-op then), Float16/BFloat16 each widen losslessly to // float32 per element. One shape, never a per-row branch on T. @@ -263,69 +267,1078 @@ template struct InnerProductMatrixImpl { SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) #if SVS_AVX512_F -template <> struct InnerProductMatrixImpl { + +namespace inner_product_matrix_avx512_detail { + +constexpr size_t kRowBlock = inner_product_matrix_row_block; + +// Accumulates kRowBlock rows of A (padded by repeating the last valid row when the +// document has fewer rows left, see callers) against one full 16-lane block of packed B. +// Shared by the table and column-max output forms so this FMA chain is written once. +SVS_FORCE_INLINE void accumulate_full( + const float* const* arow, const float* bt, size_t n, size_t d, size_t j, __m512* acc +) { + for (size_t k = 0; k < d; ++k) { + __m512 bv = _mm512_loadu_ps(bt + k * n + j); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +// Same as accumulate_full for the trailing lane-block narrower than 16 lanes. +SVS_FORCE_INLINE void accumulate_tail( + const float* const* arow, + const float* bt, + size_t n, + size_t d, + size_t j, + __mmask16 mask, + __m512* acc +) { + for (size_t k = 0; k < d; ++k) { + __m512 bv = _mm512_maskz_loadu_ps(mask, bt + k * n + j); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +// Pairwise-max reduction of the kRowBlock accumulators into their lane-wise maximum. +// Padding slots (see accumulate_full's comment) repeat a value already present in a real +// slot, so folding them into the tree does not change the result. +SVS_FORCE_INLINE __m512 reduce_max(const __m512* acc) { + __m512 m01 = _mm512_max_ps(acc[0], acc[1]); + __m512 m23 = _mm512_max_ps(acc[2], acc[3]); + __m512 m45 = _mm512_max_ps(acc[4], acc[5]); + __m512 m67 = _mm512_max_ps(acc[6], acc[7]); + return _mm512_max_ps(_mm512_max_ps(m01, m23), _mm512_max_ps(m45, m67)); +} + +// Widens one 16-lane block of Float16 to float32 with AVX-512F's native half->single. +SVS_FORCE_INLINE __m512 widen16(const svs::Float16* p) { + return _mm512_cvtph_ps(_mm256_loadu_epi16(p)); +} + +// Widens one 16-lane block of BFloat16 to float32: bfloat16 is already float32's high 16 +// bits, so this is a lossless zero-extend (cvtepu16_epi32, AVX-512F) into the top half of +// each 32-bit lane (slli by 16), not a rounding conversion. +SVS_FORCE_INLINE __m512 widen16(const svs::BFloat16* p) { + __m256i halfwords = _mm256_loadu_si256(reinterpret_cast(p)); + return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(halfwords), 16)); +} + +// Tail (< 16 lanes) widening for either 16-bit type: zero-pads through a stack buffer +// sized by sizeof(T) (2 bytes for both Float16 and BFloat16), then reuses widen16. Uses a +// plain memcpy rather than a masked load so BFloat16's tail needs nothing beyond +// AVX-512F (Float16's masked load above already relies on AVX-512BW+VL for +// _mm256_loadu_epi16, a preexisting requirement this does not widen). +template SVS_FORCE_INLINE __m512 widen16_tail(const T* p, size_t count) { + __m256i buffer = _mm256_setzero_si256(); + std::memcpy(&buffer, p, count * sizeof(T)); + return widen16(reinterpret_cast(&buffer)); +} + +// Converts kRowBlock rows of `a` (T is Float16 or BFloat16) starting at row `i` into a +// contiguous float32 scratch, 16 lanes at a time; shared by the table and column-max +// forms below so either narrow type is widened in exactly one place. +template +SVS_FORCE_INLINE void +convert_row_block(const T* a, size_t i, size_t rows, size_t d, float* scratch) { + for (size_t r = 0; r < rows; ++r) { + const T* src = a + (i + r) * d; + float* dst = scratch + r * d; + size_t k = 0; + for (; k + 16 <= d; k += 16) { + _mm512_storeu_ps(dst + k, widen16(src + k)); + } + if (k < d) { + size_t tail = d - k; + __mmask16 dmask = svs::create_mask<16>(svs::lib::MaybeStatic(tail)); + _mm512_mask_storeu_ps(dst + k, dmask, widen16_tail(src + k, tail)); + } + } +} + +// Table form for a narrow-element A (Float16 or BFloat16): identical register-blocked +// loop to the float32 build_packed above, sourced from a row-block converted to float32 +// by convert_row_block instead of `a` directly. +template +inline void build_packed_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_mask_storeu_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } +} + +// Column-max form for a narrow-element A (Float16 or BFloat16): same structure as +// build_packed_narrow above, reduced through reduce_max instead of stored per row. +template +inline void column_max_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m512 current = _mm512_loadu_ps(out + j); + _mm512_storeu_ps(out + j, _mm512_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m512 current = _mm512_maskz_loadu_ps(tail_mask, out + j); + _mm512_mask_storeu_ps( + out + j, tail_mask, _mm512_max_ps(reduce_max(acc), current) + ); + } + } +} + +} // namespace inner_product_matrix_avx512_detail + +// Partial specializations on a dummy parameter: a full one's explicit instantiation emits +// no inline members, and GCC 11 emits nothing for a second constrained one with equal +// arguments. +template struct InnerProductMatrixImpl { + // Register-blocked micro-kernel: inner_product_matrix_row_block rows of A share one + // pass over a lane-block of B, giving that many independent FMA accumulator chains + // with no horizontal reduction (each lane is one output), so the dependency-chain + // latency of a single accumulator is hidden behind the others instead of + // serializing the loop over d. SVS_NOINLINE static void - build_packed(const float* a, size_t m, const PackedOperand& b, float* out); + build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + using namespace inner_product_matrix_avx512_detail; + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_mask_storeu_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } + } + + // Table form for a Float16 A: see convert_row_block above for the shared widening. SVS_NOINLINE static void - build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx512_detail::build_packed_narrow(a, m, b, out); + } + + // Table form for a BFloat16 A: see convert_row_block above for the shared widening. SVS_NOINLINE static void - build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx512_detail::build_packed_narrow(a, m, b, out); + } + + // Packs B into a reused thread_local buffer: the table form repacks on every call + // (one query against all centroids), unlike MaxSim's column_max below which packs + // once per query and calls into this ISA's kernels directly with the already-packed + // operand. SVS_NOINLINE static void - build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + if (m == 0 || n == 0) { + return; + } + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); + } + + // Column-wise maximum of the same table build_packed computes, without ever + // materializing it: each row-block's accumulators are reduced to a per-lane maximum + // and folded into a running maximum held in `out` itself, so one reduction serves + // the whole document for the price of one load/max/store per lane-block per + // row-block. SVS_NOINLINE static void - column_max(const float* a, size_t m, const PackedOperand& b, float* out); + column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + using namespace inner_product_matrix_avx512_detail; + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m512 current = _mm512_loadu_ps(out + j); + _mm512_storeu_ps(out + j, _mm512_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m512 current = _mm512_maskz_loadu_ps(tail_mask, out + j); + _mm512_mask_storeu_ps( + out + j, tail_mask, _mm512_max_ps(reduce_max(acc), current) + ); + } + } + } + + // Float16 overload: see convert_row_block above for the shared widening (AVX-512F's + // native 16-wide _mm512_cvtph_ps). SVS_NOINLINE static void - column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx512_detail::column_max_narrow(a, m, b, out); + } + + // BFloat16 overload: see convert_row_block above for the shared (exact) widening. SVS_NOINLINE static void - column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx512_detail::column_max_narrow(a, m, b, out); + } }; #endif SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) SVS_VALIDATE_BOOL_ENV(SVS_AVX2) #if !SVS_AVX512_F && SVS_AVX2 -template <> struct InnerProductMatrixImpl { + +namespace inner_product_matrix_avx2_detail { + +constexpr size_t kRowBlock = inner_product_matrix_row_block; + +// Same role as avx512.cpp's accumulate_full/accumulate_tail, at AVX2's 8-wide registers; +// AVX2 has no masked load/store, so the tail lane-block uses _mm256_maskload_ps with an +// all-lanes blend mask built from the bit pattern simd_utils.h's ConvertToFloat<8> uses. +SVS_FORCE_INLINE void accumulate_full( + const float* const* arow, const float* bt, size_t n, size_t d, size_t j, __m256* acc +) { + for (size_t k = 0; k < d; ++k) { + __m256 bv = _mm256_loadu_ps(bt + k * n + j); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +SVS_FORCE_INLINE void accumulate_tail( + const float* const* arow, + const float* bt, + size_t n, + size_t d, + size_t j, + __m256i mask, + __m256* acc +) { + for (size_t k = 0; k < d; ++k) { + __m256 bv = _mm256_maskload_ps(bt + k * n + j, mask); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +// Pairwise-max reduction; see avx512_detail::reduce_max for why padding slots are safe +// to include. +SVS_FORCE_INLINE __m256 reduce_max(const __m256* acc) { + __m256 m01 = _mm256_max_ps(acc[0], acc[1]); + __m256 m23 = _mm256_max_ps(acc[2], acc[3]); + __m256 m45 = _mm256_max_ps(acc[4], acc[5]); + __m256 m67 = _mm256_max_ps(acc[6], acc[7]); + return _mm256_max_ps(_mm256_max_ps(m01, m23), _mm256_max_ps(m45, m67)); +} + +// Widens one 8-lane block of Float16 to float32 with F16C's native half->single. +SVS_FORCE_INLINE __m256 widen8(const svs::Float16* p) { + return _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(p))); +} + +// Widens one 8-lane block of BFloat16 to float32: bfloat16 is already float32's high 16 +// bits, so this is a lossless zero-extend (cvtepu16_epi32) into the top half of each +// 32-bit lane (slli by 16), not a rounding conversion. +SVS_FORCE_INLINE __m256 widen8(const svs::BFloat16* p) { + __m128i halfwords = _mm_loadu_si128(reinterpret_cast(p)); + return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(halfwords), 16)); +} + +// Tail (< 8 lanes) widening for either 16-bit type: zero-pads through a stack buffer +// sized by sizeof(T) (2 bytes for both Float16 and BFloat16), then reuses widen8. +template SVS_FORCE_INLINE __m256 widen8_tail(const T* p, uint8_t mask_bits) { + __m128i buffer = _mm_setzero_si128(); + std::memcpy(&buffer, p, __builtin_popcount(mask_bits) * sizeof(T)); + return widen8(reinterpret_cast(&buffer)); +} + +// Converts kRowBlock rows of `a` (T is Float16 or BFloat16) starting at row `i` into a +// contiguous float32 scratch, 8 lanes at a time; shared by the table and column-max forms +// below so either narrow type is widened in exactly one place. +template +SVS_FORCE_INLINE void +convert_row_block(const T* a, size_t i, size_t rows, size_t d, float* scratch) { + for (size_t r = 0; r < rows; ++r) { + const T* src = a + (i + r) * d; + float* dst = scratch + r * d; + size_t k = 0; + for (; k + 8 <= d; k += 8) { + _mm256_storeu_ps(dst + k, widen8(src + k)); + } + if (k < d) { + uint8_t dbits = svs::create_mask<8>(svs::lib::MaybeStatic(d - k)); + __m256i dmask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(dbits)); + _mm256_maskstore_ps(dst + k, dmask, widen8_tail(src + k, dbits)); + } + } +} + +// Table form for a narrow-element A (Float16 or BFloat16): identical register-blocked +// loop to the float32 build_packed above, sourced from a row-block converted to float32 +// by convert_row_block instead of `a` directly. +template +inline void build_packed_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_maskstore_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } +} + +// Column-max form for a narrow-element A (Float16 or BFloat16): same structure as +// build_packed_narrow above, reduced through reduce_max instead of stored per row. +template +inline void column_max_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m256 current = _mm256_loadu_ps(out + j); + _mm256_storeu_ps(out + j, _mm256_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m256 current = _mm256_maskload_ps(out + j, tail_mask); + _mm256_maskstore_ps( + out + j, tail_mask, _mm256_max_ps(reduce_max(acc), current) + ); + } + } +} + +} // namespace inner_product_matrix_avx2_detail + +template struct InnerProductMatrixImpl { + // Same register-blocking as InnerProductMatrixImpl::build_packed, at AVX2's + // 8-wide registers. SVS_NOINLINE static void - build_packed(const float* a, size_t m, const PackedOperand& b, float* out); + build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + using namespace inner_product_matrix_avx2_detail; + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = + _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_maskstore_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } + } + + // Table form for a Float16 A: see convert_row_block above for the shared widening. SVS_NOINLINE static void - build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx2_detail::build_packed_narrow(a, m, b, out); + } + + // Table form for a BFloat16 A: see convert_row_block above for the shared widening. SVS_NOINLINE static void - build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx2_detail::build_packed_narrow(a, m, b, out); + } + + // See InnerProductMatrixImpl::build for why this packs into a thread_local + // buffer rather than taking a PackedOperand from the caller. SVS_NOINLINE static void - build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + if (m == 0 || n == 0) { + return; + } + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); + } + + // See InnerProductMatrixImpl::column_max for the algorithm; this is the same + // reduction at AVX2's 8-wide registers. SVS_NOINLINE static void - column_max(const float* a, size_t m, const PackedOperand& b, float* out); + column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + using namespace inner_product_matrix_avx2_detail; + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = + _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m256 current = _mm256_loadu_ps(out + j); + _mm256_storeu_ps(out + j, _mm256_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m256 current = _mm256_maskload_ps(out + j, tail_mask); + _mm256_maskstore_ps( + out + j, tail_mask, _mm256_max_ps(reduce_max(acc), current) + ); + } + } + } + + // Float16 overload: see convert_row_block above for the shared widening (F16C's + // 8-wide _mm256_cvtph_ps, available because avx2.cpp is compiled for the "haswell" + // arch in cmake/multi-arch.cmake, and Haswell carries F16C alongside AVX2). SVS_NOINLINE static void - column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx2_detail::column_max_narrow(a, m, b, out); + } + + // BFloat16 overload: see convert_row_block above for the shared (exact) widening. SVS_NOINLINE static void - column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx2_detail::column_max_narrow(a, m, b, out); + } }; #endif -// Declared (full member set, matching every other level) only inside amx.cpp, the one -// TU compiled with -mamx-tile -mamx-bf16: SVS_AMX_TILE_BF16 reads *this TU's* compiler -// macros, same per-TU pattern as SVS_AVX512_F/SVS_AVX2 above, not the build-wide -// SVS_AMX_COMPILED_IN that PackedOperand reads. build_packed/column_max for float32 and -// Float16 A, and for a BFloat16 A whose B was not packed with the tile layout, forward -// to the AVX512 implementation (explicit forwarding in amx.cpp, one source of truth); -// only a BFloat16 A against a tile-packed B runs the real tile kernel. +// The scalar-distance kernels (L2/IP/cosine) and QEmd/QChamfer/QChamferExactImpl dispatch +// through a runtime if-chain on svs::detail::avx_runtime_flags (euclidean.h, cosine.h, +// inner_product.h, gem.h), never through a switch or template argument naming +// AVX_AVAILABILITY::AMX; adding the AMX enumerator needs no new instantiation of any of +// them (checked: `git grep -n AVX_AVAILABILITY` finds no exhaustive switch or `AMX` +// branch to add outside InnerProductMatrix). AMX only applies to the InnerProductMatrix +// family below, whose public dispatch is the one exhaustive switch this change extends. + +// Declared (full member set, matching every other level) only when SVS_AMX_TILE_BF16 +// (__AMX_TILE__ && __AMX_BF16__) is set for the current TU's own compiler flags: only +// amx.cpp is compiled with -mamx-tile -mamx-bf16, same per-TU pattern as +// SVS_AVX512_F/SVS_AVX2 above, not the build-wide SVS_AMX_COMPILED_IN that PackedOperand +// reads. build_packed/column_max for float32 and Float16 A, and for a BFloat16 A whose B +// was not packed with the tile layout, forward to the AVX512 implementation (explicit +// forwarding below, one source of truth); only a BFloat16 A against a tile-packed B runs +// the real tile kernel. SVS_VALIDATE_BOOL_ENV(SVS_AMX_TILE_BF16) #if SVS_AMX_TILE_BF16 -template <> struct InnerProductMatrixImpl { + +namespace inner_product_matrix_amx_detail { + +// 64-byte tile-config struct the palette-1 AMX instructions read (Intel SDM Vol 1, +// 3.3.7): colsb/rows per tmm register 0-15, reserved bytes must stay zero. +struct TileConfig { + uint8_t palette_id = 0; + uint8_t start_row = 0; + uint8_t reserved[14] = {}; + uint16_t colsb[16] = {}; + uint8_t rows[16] = {}; +}; + +// Releases tile state on every exit path (normal return or exception): leaving tile +// state live costs later context switches and ordinary SSE/AVX code extra save/restore +// work, per Intel's optimization guidance for AMX. +struct TileGuard { + TileGuard() = default; + ~TileGuard() { _tile_release(); } + TileGuard(const TileGuard&) = delete; + TileGuard& operator=(const TileGuard&) = delete; +}; + +// tmm register assignment: tmm0 = A's current k-chunk (16 rows x 32 bf16, i.e. 16 +// pairs); tmm1/tmm2 = B's two n-lane-tiles for the pair-chunk (pair-interleaved +// transposed bf16, see PackedOperand::tile_block); tmm3/tmm4 = the matching fp32 +// accumulators. Two n-tiles in flight so their dpbf16ps chains are independent of each +// other (both read tmm0, but write distinct accumulators), per the task's "several +// accumulator tiles overlap" requirement. Named here for the tile-config array indexing +// below (ordinary C++, any expression is fine); every `_tile_*` call past this point +// uses the literal 0/1/2/3/4 directly, because gcc's tile intrinsics are macros that +// stringify their register argument (`#dst`), so a named constant used there would +// stringify to its identifier text, not a digit, and fail to assemble. +enum Tmm : int { kTmmA = 0, kTmmB0 = 1, kTmmB1 = 2, kTmmAcc0 = 3, kTmmAcc1 = 4 }; + +// gcc 11's _tile_loadd/_tile_stored/_tile_loadconfig macros expand to inline asm with no +// memory operand and no clobber, so the optimizer believes they neither read nor write +// memory: a plain C++ store just before a load, or a plain C++ read just after a store, +// through the same address has no ordering guarantee and can be reordered around the +// tile instruction (sunk store -> stale/garbage read; hoisted read -> stale/garbage +// value), independent of any `volatile` on the asm itself (that only orders the asm +// relative to *other* volatile operations, not to ordinary memory accesses). Every tile +// memory access in this file goes through one of the wrappers below instead of the bare +// intrinsic, so the dependency is a real operand or clobber the compiler understands +// rather than an accident of how today's code generation happens to schedule things. +// `_tile_zero`/`_tile_dpbf16ps`/`_tile_release` touch only tile *registers*, never +// memory, so they keep using the compiler's intrinsics unwrapped. + +// Loads the 64-byte tile configuration from `cfg`, naming the whole struct as the +// input memory this reads (not just a pointer-sized slice of it, which is what gcc's own +// _tile_loadconfig macro's cast effectively does): every `cfg.rows[]`/`cfg.colsb[]` store +// before this call is now visibly a dependency, not an accident of scheduling. +inline void tile_loadconfig_safe(const TileConfig& cfg) { + __asm__ volatile("ldtilecfg\t%0" ::"m"(cfg)); +} + +// One generated function per literal tile number actually used: the mnemonic's register +// is spelled into the asm text (`#tmm`), which gcc's own tile intrinsics require to be a +// literal token, so a runtime tile-number parameter is not an option. Kept local to this +// namespace: the macro only generates definitions used a few lines below it, nothing +// external depends on it (include/svs/multi-arch/AGENTS.md's rule is about a macro whose +// defining header is absent from a TU that uses it, which does not apply to a macro +// defined and consumed in the same file). +// +// _strided takes a runtime stride (A's row pitch, which varies with `d`) and so cannot +// name a compile-time memory extent; its "memory" clobber is a full barrier instead, +// forcing every pending store to complete before the load and every cached value to be +// re-read after it. +#define SVS_AMX_DEFINE_TILE_LOADD_STRIDED(tmm) \ + inline void tile_loadd_tmm##tmm##_strided(const void* base, long stride) { \ + __asm__ volatile("{tileloadd\t(%0,%1,1), %%tmm" #tmm "|tileloadd\t%%tmm" #tmm \ + ", [%0+%1*1]}" ::"r"(base), \ + "r"(stride) \ + : "memory"); \ + } + +// _fixed is for a source whose extent is a compile-time constant 1024 bytes (B's tile +// blocks, always a full 16x64-byte tile; the local zero-padded tail buffer, likewise +// 16x64 bytes): the dummy "m" operand (unused in the asm text) names that exact memory as +// read, so the compiler orders this correctly against its last write without needing a +// blanket barrier. +#define SVS_AMX_DEFINE_TILE_LOADD_FIXED(tmm) \ + inline void tile_loadd_tmm##tmm##_fixed(const void* base) { \ + __asm__ volatile( \ + "{tileloadd\t(%0,%1,1), %%tmm" #tmm "|tileloadd\t%%tmm" #tmm \ + ", [%0+%1*1]}" ::"r"(base), \ + "r"(64L), \ + "m"(*reinterpret_cast(base)) \ + ); \ + } + +// Stores are always to a caller-owned 16x16 fp32 buffer (1024 bytes, compile-time +// constant): the dummy "=m" output operand tells the compiler this asm writes that +// memory, so a later plain read of the same buffer is ordered after the store instead of +// possibly being hoisted before it. +#define SVS_AMX_DEFINE_TILE_STORED_FIXED(tmm) \ + inline void tile_stored_tmm##tmm##_fixed(void* base) { \ + __asm__ volatile("{tilestored\t%%tmm" #tmm \ + ", (%1,%2,1)|tilestored\t[%1+%2*1], %%tmm" #tmm "}" \ + : "=m"(*reinterpret_cast(base)) \ + : "r"(base), "r"(64L)); \ + } + +SVS_AMX_DEFINE_TILE_LOADD_STRIDED(0) +SVS_AMX_DEFINE_TILE_LOADD_FIXED(0) +SVS_AMX_DEFINE_TILE_LOADD_FIXED(1) +SVS_AMX_DEFINE_TILE_LOADD_FIXED(2) +SVS_AMX_DEFINE_TILE_STORED_FIXED(3) +SVS_AMX_DEFINE_TILE_STORED_FIXED(4) + +#undef SVS_AMX_DEFINE_TILE_LOADD_STRIDED +#undef SVS_AMX_DEFINE_TILE_LOADD_FIXED +#undef SVS_AMX_DEFINE_TILE_STORED_FIXED + +inline void configure_tiles(uint8_t a_acc_rows) { + TileConfig cfg; + cfg.palette_id = 1; + cfg.rows[kTmmA] = a_acc_rows; + cfg.colsb[kTmmA] = 64; + cfg.rows[kTmmAcc0] = a_acc_rows; + cfg.colsb[kTmmAcc0] = 64; + cfg.rows[kTmmAcc1] = a_acc_rows; + cfg.colsb[kTmmAcc1] = 64; + // B's tiles are always a full 16 (pair-)rows: PackedOperand::tile_block zero-pads + // every tile block to 16x16, so there is no partial-row case to configure for them. + cfg.rows[kTmmB0] = 16; + cfg.colsb[kTmmB0] = 64; + cfg.rows[kTmmB1] = 16; + cfg.colsb[kTmmB1] = 64; + tile_loadconfig_safe(cfg); +} + +// Computes the dot products of up to 16 rows of A (starting at `a_row0`, already known +// not to read past `d` columns of any row via the tail copy below) against B's n-tiles +// `nt0` (and `nt1` when `has_nt1`) for every k-chunk, leaving the result in tmm3 (and +// tmm4). Tiles must already be configured by configure_tiles() for this call's row count. +// +// A's tail k-chunk (the last one, when d is not a multiple of 32) is copied through a +// zero-padded stack buffer rather than loaded directly: loading straight from `a_row0` +// would read up to 31 bf16 elements past column d-1 of the block's last real row, which +// is caller memory this kernel does not own past `d`. +inline void accumulate_group( + const svs::BFloat16* a_row0, + size_t rows, + size_t stride_bytes, + size_t full_k_chunks, + size_t k_tail, + const PackedOperand& b, + size_t nt0, + bool has_nt1, + size_t nt1 +) { + _tile_zero(3); + if (has_nt1) { + _tile_zero(4); + } + const auto* base = reinterpret_cast(a_row0); + for (size_t kt = 0; kt < full_k_chunks; ++kt) { + tile_loadd_tmm0_strided(base + kt * 64, static_cast(stride_bytes)); + tile_loadd_tmm1_fixed(b.tile_block(nt0, kt)); + _tile_dpbf16ps(3, 0, 1); + if (has_nt1) { + tile_loadd_tmm2_fixed(b.tile_block(nt1, kt)); + _tile_dpbf16ps(4, 0, 2); + } + } + if (k_tail > 0) { + alignas(64) uint16_t tail[16][32] = {}; + for (size_t r = 0; r < rows; ++r) { + std::memcpy(tail[r], base + r * stride_bytes + full_k_chunks * 64, k_tail * 2); + } + tile_loadd_tmm0_fixed(tail); + tile_loadd_tmm1_fixed(b.tile_block(nt0, full_k_chunks)); + _tile_dpbf16ps(3, 0, 1); + if (has_nt1) { + tile_loadd_tmm2_fixed(b.tile_block(nt1, full_k_chunks)); + _tile_dpbf16ps(4, 0, 2); + } + } +} + +// Shared row-block/n-tile-group driver for both output forms below: walks A 16 rows at a +// time (reconfiguring tiles once more only for a final row-block narrower than 16, never +// per row-block in the steady state) and B 32 lanes (two n-tiles) at a time, handing each +// block's two 16x16 fp32 results to `emit(row0, rows, nt0, has_nt1, nt1, c0, c1)`. +template +inline void for_each_block( + const svs::BFloat16* a, size_t m, size_t d, const PackedOperand& b, Emit&& emit +) { + size_t n_tiles = b.n_tiles(); + size_t full_k_chunks = d / 32; + size_t k_tail = d % 32; + size_t stride_bytes = d * 2; + + TileGuard guard; + configure_tiles(16); + bool configured_full = true; + + for (size_t i = 0; i < m; i += 16) { + size_t rows = std::min(16, m - i); + if (rows < 16 && configured_full) { + configure_tiles(static_cast(rows)); + configured_full = false; + } + const svs::BFloat16* row0 = a + i * d; + for (size_t nt0 = 0; nt0 < n_tiles; nt0 += 2) { + bool has_nt1 = nt0 + 1 < n_tiles; + size_t nt1 = has_nt1 ? nt0 + 1 : nt0; + alignas(64) float c0[16][16]; + alignas(64) float c1[16][16]; + accumulate_group( + row0, rows, stride_bytes, full_k_chunks, k_tail, b, nt0, has_nt1, nt1 + ); + tile_stored_tmm3_fixed(c0); + if (has_nt1) { + tile_stored_tmm4_fixed(c1); + } + emit(i, rows, nt0, has_nt1, nt1, c0, c1); + } + } +} + +// AMX table kernel for a BFloat16 A against a tile-packed B: writes out[i*n+j] exactly +// like the generic/AVX-512 forms, with the same values (summed in a +// different order, hence InnerProductMatrix's tolerance-based tests, not a bit-exact +// one). +inline void +amx_table_bf16(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + for_each_block( + a, + m, + b.d(), + b, + [&](size_t row0, + size_t rows, + size_t nt0, + bool has_nt1, + size_t nt1, + auto& c0, + auto& c1) { + size_t lanes0 = std::min(16, n - nt0 * 16); + for (size_t r = 0; r < rows; ++r) { + float* dst = out + (row0 + r) * n + nt0 * 16; + std::memcpy(dst, c0[r], lanes0 * sizeof(float)); + } + if (has_nt1) { + size_t lanes1 = std::min(16, n - nt1 * 16); + for (size_t r = 0; r < rows; ++r) { + float* dst = out + (row0 + r) * n + nt1 * 16; + std::memcpy(dst, c1[r], lanes1 * sizeof(float)); + } + } + } + ); +} + +// AMX column-max kernel for a BFloat16 A against a tile-packed B: folds each block's two +// 16x16 tiles into the running per-lane maximum with AVX-512 (the AMX level implies +// AVX-512F), reusing the same row-then-lane reduction shape as +// InnerProductMatrixImpl::column_max above. +inline void +amx_column_max_bf16(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + for_each_block( + a, + m, + b.d(), + b, + [&](size_t /*row0*/, + size_t rows, + size_t nt0, + bool has_nt1, + size_t nt1, + auto& c0, + auto& c1) { + auto fold = [&](size_t nt, auto& c) { + size_t lanes = std::min(16, n - nt * 16); + alignas(64) float colmax[16] = {}; + for (size_t j = 0; j < 16; ++j) { + float best = -std::numeric_limits::infinity(); + for (size_t r = 0; r < rows; ++r) { + best = std::max(best, c[r][j]); + } + colmax[j] = best; + } + for (size_t j = 0; j < lanes; ++j) { + out[nt * 16 + j] = std::max(out[nt * 16 + j], colmax[j]); + } + }; + fold(nt0, c0); + if (has_nt1) { + fold(nt1, c1); + } + } + ); +} + +} // namespace inner_product_matrix_amx_detail + +template struct InnerProductMatrixImpl { + // Never the AMX precision case (A is float32, not bf16): one source of truth is the + // AVX-512 kernel. SVS_NOINLINE static void - build_packed(const float* a, size_t m, const PackedOperand& b, float* out); + build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + } + SVS_NOINLINE static void - build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + } + SVS_NOINLINE static void - build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + if (m == 0 || b.n() == 0) { + return; + } + if (!b.has_tile_layout()) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + return; + } + inner_product_matrix_amx_detail::amx_table_bf16(a, m, b, out); + } + + // `a` is always float32 here (InnerProductMatrix::compute's only entry point to this + // overload): the AMX precision case never applies, so this is a plain forward. SVS_NOINLINE static void - build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out); + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + InnerProductMatrixImpl::build(a, m, b, n, d, out); + } + SVS_NOINLINE static void - column_max(const float* a, size_t m, const PackedOperand& b, float* out); + column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::column_max(a, m, b, out); + } + SVS_NOINLINE static void - column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out); + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::column_max(a, m, b, out); + } + SVS_NOINLINE static void - column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out); + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + if (!b.has_tile_layout()) { + InnerProductMatrixImpl::column_max(a, m, b, out); + return; + } + inner_product_matrix_amx_detail::amx_column_max_bf16(a, m, b, out); + } }; #endif diff --git a/include/svs/multi-arch/x86/amx.cpp b/include/svs/multi-arch/x86/amx.cpp index 1158154a9..25a9096ae 100644 --- a/include/svs/multi-arch/x86/amx.cpp +++ b/include/svs/multi-arch/x86/amx.cpp @@ -17,388 +17,12 @@ #if defined(__x86_64__) #include "svs/core/distance/inner_product_matrix.h" -#include - -#include -#include -#include - namespace svs::distance { -// The scalar-distance kernels (L2/IP/cosine) and QEmd/QChamfer/QChamferExactImpl dispatch -// through a runtime if-chain on svs::detail::avx_runtime_flags (euclidean.h, cosine.h, -// inner_product.h, gem.h), never through a switch or template argument naming -// AVX_AVAILABILITY::AMX; adding the AMX enumerator needs no new instantiation of any of -// them (checked: `git grep -n AVX_AVAILABILITY` finds no exhaustive switch or `AMX` -// branch to add outside InnerProductMatrix). AMX only applies to the InnerProductMatrix -// family below, whose public dispatch is the one exhaustive switch this change extends. - -namespace { - -// 64-byte tile-config struct the palette-1 AMX instructions read (Intel SDM Vol 1, -// 3.3.7): colsb/rows per tmm register 0-15, reserved bytes must stay zero. -struct TileConfig { - uint8_t palette_id = 0; - uint8_t start_row = 0; - uint8_t reserved[14] = {}; - uint16_t colsb[16] = {}; - uint8_t rows[16] = {}; -}; - -// Releases tile state on every exit path (normal return or exception): leaving tile -// state live costs later context switches and ordinary SSE/AVX code extra save/restore -// work, per Intel's optimization guidance for AMX. -struct TileGuard { - TileGuard() = default; - ~TileGuard() { _tile_release(); } - TileGuard(const TileGuard&) = delete; - TileGuard& operator=(const TileGuard&) = delete; -}; - -// tmm register assignment: tmm0 = A's current k-chunk (16 rows x 32 bf16, i.e. 16 -// pairs); tmm1/tmm2 = B's two n-lane-tiles for the pair-chunk (pair-interleaved -// transposed bf16, see PackedOperand::tile_block); tmm3/tmm4 = the matching fp32 -// accumulators. Two n-tiles in flight so their dpbf16ps chains are independent of each -// other (both read tmm0, but write distinct accumulators), per the task's "several -// accumulator tiles overlap" requirement. Named here for the tile-config array indexing -// below (ordinary C++, any expression is fine); every `_tile_*` call past this point -// uses the literal 0/1/2/3/4 directly, because gcc's tile intrinsics are macros that -// stringify their register argument (`#dst`), so a named constant used there would -// stringify to its identifier text, not a digit, and fail to assemble. -enum Tmm : int { kTmmA = 0, kTmmB0 = 1, kTmmB1 = 2, kTmmAcc0 = 3, kTmmAcc1 = 4 }; - -// gcc 11's _tile_loadd/_tile_stored/_tile_loadconfig macros expand to inline asm with no -// memory operand and no clobber, so the optimizer believes they neither read nor write -// memory: a plain C++ store just before a load, or a plain C++ read just after a store, -// through the same address has no ordering guarantee and can be reordered around the -// tile instruction (sunk store -> stale/garbage read; hoisted read -> stale/garbage -// value), independent of any `volatile` on the asm itself (that only orders the asm -// relative to *other* volatile operations, not to ordinary memory accesses). Every tile -// memory access in this file goes through one of the wrappers below instead of the bare -// intrinsic, so the dependency is a real operand or clobber the compiler understands -// rather than an accident of how today's code generation happens to schedule things. -// `_tile_zero`/`_tile_dpbf16ps`/`_tile_release` touch only tile *registers*, never -// memory, so they keep using the compiler's intrinsics unwrapped. - -// Loads the 64-byte tile configuration from `cfg`, naming the whole struct as the -// input memory this reads (not just a pointer-sized slice of it, which is what gcc's own -// _tile_loadconfig macro's cast effectively does): every `cfg.rows[]`/`cfg.colsb[]` store -// before this call is now visibly a dependency, not an accident of scheduling. -inline void tile_loadconfig_safe(const TileConfig& cfg) { - __asm__ volatile("ldtilecfg\t%0" ::"m"(cfg)); -} - -// One generated function per literal tile number actually used: the mnemonic's register -// is spelled into the asm text (`#tmm`), which gcc's own tile intrinsics require to be a -// literal token, so a runtime tile-number parameter is not an option. Kept local to this -// file: the macro only generates definitions used a few lines below it, nothing external -// depends on it (include/svs/multi-arch/AGENTS.md's rule is about a macro whose defining -// header is absent from a TU that uses it, which does not apply to a macro defined and -// consumed in the same file). -// -// _strided takes a runtime stride (A's row pitch, which varies with `d`) and so cannot -// name a compile-time memory extent; its "memory" clobber is a full barrier instead, -// forcing every pending store to complete before the load and every cached value to be -// re-read after it. -#define SVS_AMX_DEFINE_TILE_LOADD_STRIDED(tmm) \ - inline void tile_loadd_tmm##tmm##_strided(const void* base, long stride) { \ - __asm__ volatile("{tileloadd\t(%0,%1,1), %%tmm" #tmm "|tileloadd\t%%tmm" #tmm \ - ", [%0+%1*1]}" ::"r"(base), \ - "r"(stride) \ - : "memory"); \ - } - -// _fixed is for a source whose extent is a compile-time constant 1024 bytes (B's tile -// blocks, always a full 16x64-byte tile; the local zero-padded tail buffer, likewise -// 16x64 bytes): the dummy "m" operand (unused in the asm text) names that exact memory as -// read, so the compiler orders this correctly against its last write without needing a -// blanket barrier. -#define SVS_AMX_DEFINE_TILE_LOADD_FIXED(tmm) \ - inline void tile_loadd_tmm##tmm##_fixed(const void* base) { \ - __asm__ volatile( \ - "{tileloadd\t(%0,%1,1), %%tmm" #tmm "|tileloadd\t%%tmm" #tmm \ - ", [%0+%1*1]}" ::"r"(base), \ - "r"(64L), \ - "m"(*reinterpret_cast(base)) \ - ); \ - } - -// Stores are always to a caller-owned 16x16 fp32 buffer (1024 bytes, compile-time -// constant): the dummy "=m" output operand tells the compiler this asm writes that -// memory, so a later plain read of the same buffer is ordered after the store instead of -// possibly being hoisted before it. -#define SVS_AMX_DEFINE_TILE_STORED_FIXED(tmm) \ - inline void tile_stored_tmm##tmm##_fixed(void* base) { \ - __asm__ volatile("{tilestored\t%%tmm" #tmm \ - ", (%1,%2,1)|tilestored\t[%1+%2*1], %%tmm" #tmm "}" \ - : "=m"(*reinterpret_cast(base)) \ - : "r"(base), "r"(64L)); \ - } - -SVS_AMX_DEFINE_TILE_LOADD_STRIDED(0) -SVS_AMX_DEFINE_TILE_LOADD_FIXED(0) -SVS_AMX_DEFINE_TILE_LOADD_FIXED(1) -SVS_AMX_DEFINE_TILE_LOADD_FIXED(2) -SVS_AMX_DEFINE_TILE_STORED_FIXED(3) -SVS_AMX_DEFINE_TILE_STORED_FIXED(4) - -#undef SVS_AMX_DEFINE_TILE_LOADD_STRIDED -#undef SVS_AMX_DEFINE_TILE_LOADD_FIXED -#undef SVS_AMX_DEFINE_TILE_STORED_FIXED - -void configure_tiles(uint8_t a_acc_rows) { - TileConfig cfg; - cfg.palette_id = 1; - cfg.rows[kTmmA] = a_acc_rows; - cfg.colsb[kTmmA] = 64; - cfg.rows[kTmmAcc0] = a_acc_rows; - cfg.colsb[kTmmAcc0] = 64; - cfg.rows[kTmmAcc1] = a_acc_rows; - cfg.colsb[kTmmAcc1] = 64; - // B's tiles are always a full 16 (pair-)rows: PackedOperand::tile_block zero-pads - // every tile block to 16x16, so there is no partial-row case to configure for them. - cfg.rows[kTmmB0] = 16; - cfg.colsb[kTmmB0] = 64; - cfg.rows[kTmmB1] = 16; - cfg.colsb[kTmmB1] = 64; - tile_loadconfig_safe(cfg); -} - -// Computes the dot products of up to 16 rows of A (starting at `a_row0`, already known -// not to read past `d` columns of any row via the tail copy below) against B's n-tiles -// `nt0` (and `nt1` when `has_nt1`) for every k-chunk, leaving the result in tmm3 (and -// tmm4). Tiles must already be configured by configure_tiles() for this call's row count. -// -// A's tail k-chunk (the last one, when d is not a multiple of 32) is copied through a -// zero-padded stack buffer rather than loaded directly: loading straight from `a_row0` -// would read up to 31 bf16 elements past column d-1 of the block's last real row, which -// is caller memory this kernel does not own past `d`. -void accumulate_group( - const svs::BFloat16* a_row0, - size_t rows, - size_t stride_bytes, - size_t full_k_chunks, - size_t k_tail, - const PackedOperand& b, - size_t nt0, - bool has_nt1, - size_t nt1 -) { - _tile_zero(3); - if (has_nt1) { - _tile_zero(4); - } - const auto* base = reinterpret_cast(a_row0); - for (size_t kt = 0; kt < full_k_chunks; ++kt) { - tile_loadd_tmm0_strided(base + kt * 64, static_cast(stride_bytes)); - tile_loadd_tmm1_fixed(b.tile_block(nt0, kt)); - _tile_dpbf16ps(3, 0, 1); - if (has_nt1) { - tile_loadd_tmm2_fixed(b.tile_block(nt1, kt)); - _tile_dpbf16ps(4, 0, 2); - } - } - if (k_tail > 0) { - alignas(64) uint16_t tail[16][32] = {}; - for (size_t r = 0; r < rows; ++r) { - std::memcpy(tail[r], base + r * stride_bytes + full_k_chunks * 64, k_tail * 2); - } - tile_loadd_tmm0_fixed(tail); - tile_loadd_tmm1_fixed(b.tile_block(nt0, full_k_chunks)); - _tile_dpbf16ps(3, 0, 1); - if (has_nt1) { - tile_loadd_tmm2_fixed(b.tile_block(nt1, full_k_chunks)); - _tile_dpbf16ps(4, 0, 2); - } - } -} - -// Shared row-block/n-tile-group driver for both output forms below: walks A 16 rows at a -// time (reconfiguring tiles once more only for a final row-block narrower than 16, never -// per row-block in the steady state) and B 32 lanes (two n-tiles) at a time, handing each -// block's two 16x16 fp32 results to `emit(row0, rows, nt0, has_nt1, nt1, c0, c1)`. -template -void for_each_block( - const svs::BFloat16* a, size_t m, size_t d, const PackedOperand& b, Emit&& emit -) { - size_t n_tiles = b.n_tiles(); - size_t full_k_chunks = d / 32; - size_t k_tail = d % 32; - size_t stride_bytes = d * 2; - - TileGuard guard; - configure_tiles(16); - bool configured_full = true; - - for (size_t i = 0; i < m; i += 16) { - size_t rows = std::min(16, m - i); - if (rows < 16 && configured_full) { - configure_tiles(static_cast(rows)); - configured_full = false; - } - const svs::BFloat16* row0 = a + i * d; - for (size_t nt0 = 0; nt0 < n_tiles; nt0 += 2) { - bool has_nt1 = nt0 + 1 < n_tiles; - size_t nt1 = has_nt1 ? nt0 + 1 : nt0; - alignas(64) float c0[16][16]; - alignas(64) float c1[16][16]; - accumulate_group( - row0, rows, stride_bytes, full_k_chunks, k_tail, b, nt0, has_nt1, nt1 - ); - tile_stored_tmm3_fixed(c0); - if (has_nt1) { - tile_stored_tmm4_fixed(c1); - } - emit(i, rows, nt0, has_nt1, nt1, c0, c1); - } - } -} - -} // namespace - -// AMX table kernel for a BFloat16 A against a tile-packed B: writes out[i*n+j] exactly -// like the generic/AVX-512 forms, with the same values (summed in a -// different order, hence InnerProductMatrix's tolerance-based tests, not a bit-exact -// one). -void amx_table_bf16(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { - size_t n = b.n(); - for_each_block( - a, - m, - b.d(), - b, - [&](size_t row0, - size_t rows, - size_t nt0, - bool has_nt1, - size_t nt1, - auto& c0, - auto& c1) { - size_t lanes0 = std::min(16, n - nt0 * 16); - for (size_t r = 0; r < rows; ++r) { - float* dst = out + (row0 + r) * n + nt0 * 16; - std::memcpy(dst, c0[r], lanes0 * sizeof(float)); - } - if (has_nt1) { - size_t lanes1 = std::min(16, n - nt1 * 16); - for (size_t r = 0; r < rows; ++r) { - float* dst = out + (row0 + r) * n + nt1 * 16; - std::memcpy(dst, c1[r], lanes1 * sizeof(float)); - } - } - } - ); -} - -// AMX column-max kernel for a BFloat16 A against a tile-packed B: folds each block's two -// 16x16 tiles into the running per-lane maximum with AVX-512 (the AMX level implies -// AVX-512F), reusing the same row-then-lane reduction shape as -// InnerProductMatrixImpl::column_max in avx512.cpp. -void amx_column_max_bf16( - const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out -) { - size_t n = b.n(); - std::fill(out, out + n, -std::numeric_limits::infinity()); - if (m == 0) { - return; - } - for_each_block( - a, - m, - b.d(), - b, - [&](size_t /*row0*/, - size_t rows, - size_t nt0, - bool has_nt1, - size_t nt1, - auto& c0, - auto& c1) { - auto fold = [&](size_t nt, auto& c) { - size_t lanes = std::min(16, n - nt * 16); - alignas(64) float colmax[16] = {}; - for (size_t j = 0; j < 16; ++j) { - float best = -std::numeric_limits::infinity(); - for (size_t r = 0; r < rows; ++r) { - best = std::max(best, c[r][j]); - } - colmax[j] = best; - } - for (size_t j = 0; j < lanes; ++j) { - out[nt * 16 + j] = std::max(out[nt * 16 + j], colmax[j]); - } - }; - fold(nt0, c0); - if (has_nt1) { - fold(nt1, c1); - } - } - ); -} - SVS_VALIDATE_BOOL_ENV(SVS_AMX_TILE_BF16) #if SVS_AMX_TILE_BF16 - -void InnerProductMatrixImpl::build_packed( - const float* a, size_t m, const PackedOperand& b, float* out -) { - // Never the AMX precision case (A is float32, not bf16): one source of truth is the - // AVX-512 kernel. - InnerProductMatrixImpl::build_packed(a, m, b, out); -} - -void InnerProductMatrixImpl::build_packed( - const svs::Float16* a, size_t m, const PackedOperand& b, float* out -) { - InnerProductMatrixImpl::build_packed(a, m, b, out); -} - -void InnerProductMatrixImpl::build_packed( - const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out -) { - if (m == 0 || b.n() == 0) { - return; - } - if (!b.has_tile_layout()) { - InnerProductMatrixImpl::build_packed(a, m, b, out); - return; - } - amx_table_bf16(a, m, b, out); -} - -void InnerProductMatrixImpl::build( - const float* a, size_t m, const float* b, size_t n, size_t d, float* out -) { - // `a` is always float32 here (InnerProductMatrix::compute's only entry point to this - // overload): the AMX precision case never applies, so this is a plain forward. - InnerProductMatrixImpl::build(a, m, b, n, d, out); -} - -void InnerProductMatrixImpl::column_max( - const float* a, size_t m, const PackedOperand& b, float* out -) { - InnerProductMatrixImpl::column_max(a, m, b, out); -} - -void InnerProductMatrixImpl::column_max( - const svs::Float16* a, size_t m, const PackedOperand& b, float* out -) { - InnerProductMatrixImpl::column_max(a, m, b, out); -} - -void InnerProductMatrixImpl::column_max( - const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out -) { - if (!b.has_tile_layout()) { - InnerProductMatrixImpl::column_max(a, m, b, out); - return; - } - amx_column_max_bf16(a, m, b, out); -} - template struct InnerProductMatrixImpl; - -#endif // SVS_AMX_TILE_BF16 +#endif } // namespace svs::distance diff --git a/include/svs/multi-arch/x86/avx2.cpp b/include/svs/multi-arch/x86/avx2.cpp index 4f26008c1..f4fe3b91f 100644 --- a/include/svs/multi-arch/x86/avx2.cpp +++ b/include/svs/multi-arch/x86/avx2.cpp @@ -20,7 +20,6 @@ #include "svs/core/distance/gem.h" #include "svs/core/distance/inner_product.h" #include "svs/core/distance/inner_product_matrix.h" -#include "svs/core/distance/simd_utils.h" namespace svs::distance { @@ -60,349 +59,6 @@ template struct QChamferImpl; template struct QChamferExactImpl; template struct InnerProductMatrixImpl; -namespace { - -constexpr size_t kRowBlock = inner_product_matrix_row_block; - -// Same role as avx512.cpp's accumulate_full/accumulate_tail, at AVX2's 8-wide registers; -// AVX2 has no masked load/store, so the tail lane-block uses _mm256_maskload_ps with an -// all-lanes blend mask built from the bit pattern simd_utils.h's ConvertToFloat<8> uses. -SVS_FORCE_INLINE void accumulate_full( - const float* const* arow, const float* bt, size_t n, size_t d, size_t j, __m256* acc -) { - for (size_t k = 0; k < d; ++k) { - __m256 bv = _mm256_loadu_ps(bt + k * n + j); - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); - } - } -} - -SVS_FORCE_INLINE void accumulate_tail( - const float* const* arow, - const float* bt, - size_t n, - size_t d, - size_t j, - __m256i mask, - __m256* acc -) { - for (size_t k = 0; k < d; ++k) { - __m256 bv = _mm256_maskload_ps(bt + k * n + j, mask); - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); - } - } -} - -// Pairwise-max reduction; see avx512.cpp's reduce_max for why padding slots are safe to -// include. -SVS_FORCE_INLINE __m256 reduce_max(const __m256* acc) { - __m256 m01 = _mm256_max_ps(acc[0], acc[1]); - __m256 m23 = _mm256_max_ps(acc[2], acc[3]); - __m256 m45 = _mm256_max_ps(acc[4], acc[5]); - __m256 m67 = _mm256_max_ps(acc[6], acc[7]); - return _mm256_max_ps(_mm256_max_ps(m01, m23), _mm256_max_ps(m45, m67)); -} - -// Widens one 8-lane block of Float16 to float32 with F16C's native half->single. -SVS_FORCE_INLINE __m256 widen8(const svs::Float16* p) { - return _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(p))); -} - -// Widens one 8-lane block of BFloat16 to float32: bfloat16 is already float32's high 16 -// bits, so this is a lossless zero-extend (cvtepu16_epi32) into the top half of each -// 32-bit lane (slli by 16), not a rounding conversion. -SVS_FORCE_INLINE __m256 widen8(const svs::BFloat16* p) { - __m128i halfwords = _mm_loadu_si128(reinterpret_cast(p)); - return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(halfwords), 16)); -} - -// Tail (< 8 lanes) widening for either 16-bit type: zero-pads through a stack buffer -// sized by sizeof(T) (2 bytes for both Float16 and BFloat16), then reuses widen8. -template SVS_FORCE_INLINE __m256 widen8_tail(const T* p, uint8_t mask_bits) { - __m128i buffer = _mm_setzero_si128(); - std::memcpy(&buffer, p, __builtin_popcount(mask_bits) * sizeof(T)); - return widen8(reinterpret_cast(&buffer)); -} - -// Converts kRowBlock rows of `a` (T is Float16 or BFloat16) starting at row `i` into a -// contiguous float32 scratch, 8 lanes at a time; shared by the table and column-max forms -// below so either narrow type is widened in exactly one place. -template -SVS_FORCE_INLINE void -convert_row_block(const T* a, size_t i, size_t rows, size_t d, float* scratch) { - for (size_t r = 0; r < rows; ++r) { - const T* src = a + (i + r) * d; - float* dst = scratch + r * d; - size_t k = 0; - for (; k + 8 <= d; k += 8) { - _mm256_storeu_ps(dst + k, widen8(src + k)); - } - if (k < d) { - uint8_t dbits = svs::create_mask<8>(svs::lib::MaybeStatic(d - k)); - __m256i dmask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(dbits)); - _mm256_maskstore_ps(dst + k, dmask, widen8_tail(src + k, dbits)); - } - } -} - -// Table form for a narrow-element A (Float16 or BFloat16): identical register-blocked -// loop to the float32 build_packed above, sourced from a row-block converted to float32 -// by convert_row_block instead of `a` directly. -template -void build_packed_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { - size_t n = b.n(); - size_t d = b.d(); - if (m == 0 || n == 0) { - return; - } - constexpr size_t vl = 8; - const float* bt = b.data(); - size_t full_lanes = n - n % vl; - uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); - __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); - - thread_local std::vector scratch; - scratch.resize(kRowBlock * d); - - for (size_t i = 0; i < m; i += kRowBlock) { - size_t rows = std::min(kRowBlock, m - i); - convert_row_block(a, i, rows, d, scratch.data()); - const float* arow[kRowBlock]; - float* orow[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = scratch.data() + std::min(r, rows - 1) * d; - } - for (size_t r = 0; r < rows; ++r) { - orow[r] = out + (i + r) * n; - } - - size_t j = 0; - for (; j < full_lanes; j += vl) { - __m256 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_setzero_ps(); - } - accumulate_full(arow, bt, n, d, j, acc); - for (size_t r = 0; r < rows; ++r) { - _mm256_storeu_ps(orow[r] + j, acc[r]); - } - } - if (j < n) { - __m256 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_setzero_ps(); - } - accumulate_tail(arow, bt, n, d, j, tail_mask, acc); - for (size_t r = 0; r < rows; ++r) { - _mm256_maskstore_ps(orow[r] + j, tail_mask, acc[r]); - } - } - } -} - -// Column-max form for a narrow-element A (Float16 or BFloat16): same structure as -// build_packed_narrow above, reduced through reduce_max instead of stored per row. -template -void column_max_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { - size_t n = b.n(); - size_t d = b.d(); - if (n == 0) { - return; - } - std::fill(out, out + n, -std::numeric_limits::infinity()); - if (m == 0) { - return; - } - constexpr size_t vl = 8; - const float* bt = b.data(); - size_t full_lanes = n - n % vl; - uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); - __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); - - thread_local std::vector scratch; - scratch.resize(kRowBlock * d); - - for (size_t i = 0; i < m; i += kRowBlock) { - size_t rows = std::min(kRowBlock, m - i); - convert_row_block(a, i, rows, d, scratch.data()); - const float* arow[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = scratch.data() + std::min(r, rows - 1) * d; - } - - size_t j = 0; - for (; j < full_lanes; j += vl) { - __m256 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_setzero_ps(); - } - accumulate_full(arow, bt, n, d, j, acc); - __m256 current = _mm256_loadu_ps(out + j); - _mm256_storeu_ps(out + j, _mm256_max_ps(reduce_max(acc), current)); - } - if (j < n) { - __m256 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_setzero_ps(); - } - accumulate_tail(arow, bt, n, d, j, tail_mask, acc); - __m256 current = _mm256_maskload_ps(out + j, tail_mask); - _mm256_maskstore_ps( - out + j, tail_mask, _mm256_max_ps(reduce_max(acc), current) - ); - } - } -} - -} // namespace - -// Same register-blocking as avx512.cpp's InnerProductMatrixImpl::build_packed, at -// AVX2's 8-wide registers. -void InnerProductMatrixImpl::build_packed( - const float* a, size_t m, const PackedOperand& b, float* out -) { - size_t n = b.n(); - size_t d = b.d(); - if (m == 0 || n == 0) { - return; - } - constexpr size_t vl = 8; - const float* bt = b.data(); - size_t full_lanes = n - n % vl; - uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); - __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); - - for (size_t i = 0; i < m; i += kRowBlock) { - size_t rows = std::min(kRowBlock, m - i); - const float* arow[kRowBlock]; - float* orow[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = a + (i + std::min(r, rows - 1)) * d; - } - for (size_t r = 0; r < rows; ++r) { - orow[r] = out + (i + r) * n; - } - - size_t j = 0; - for (; j < full_lanes; j += vl) { - __m256 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_setzero_ps(); - } - accumulate_full(arow, bt, n, d, j, acc); - for (size_t r = 0; r < rows; ++r) { - _mm256_storeu_ps(orow[r] + j, acc[r]); - } - } - if (j < n) { - __m256 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_setzero_ps(); - } - accumulate_tail(arow, bt, n, d, j, tail_mask, acc); - for (size_t r = 0; r < rows; ++r) { - _mm256_maskstore_ps(orow[r] + j, tail_mask, acc[r]); - } - } - } -} - -// Table form for a Float16 A: see convert_row_block above for the shared widening. -void InnerProductMatrixImpl::build_packed( - const svs::Float16* a, size_t m, const PackedOperand& b, float* out -) { - build_packed_narrow(a, m, b, out); -} - -// Table form for a BFloat16 A: see convert_row_block above for the shared widening. -void InnerProductMatrixImpl::build_packed( - const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out -) { - build_packed_narrow(a, m, b, out); -} - -// See avx512.cpp's InnerProductMatrixImpl::build for why this packs into a -// thread_local buffer rather than taking a PackedOperand from the caller. -void InnerProductMatrixImpl::build( - const float* a, size_t m, const float* b, size_t n, size_t d, float* out -) { - if (m == 0 || n == 0) { - return; - } - thread_local PackedOperand packed; - packed.pack(b, n, d); - build_packed(a, m, packed, out); -} - -// See avx512.cpp's InnerProductMatrixImpl::column_max for the algorithm; this is -// the same reduction at AVX2's 8-wide registers. -void InnerProductMatrixImpl::column_max( - const float* a, size_t m, const PackedOperand& b, float* out -) { - size_t n = b.n(); - size_t d = b.d(); - if (n == 0) { - return; - } - std::fill(out, out + n, -std::numeric_limits::infinity()); - if (m == 0) { - return; - } - constexpr size_t vl = 8; - const float* bt = b.data(); - size_t full_lanes = n - n % vl; - uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); - __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); - - for (size_t i = 0; i < m; i += kRowBlock) { - size_t rows = std::min(kRowBlock, m - i); - const float* arow[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = a + (i + std::min(r, rows - 1)) * d; - } - - size_t j = 0; - for (; j < full_lanes; j += vl) { - __m256 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_setzero_ps(); - } - accumulate_full(arow, bt, n, d, j, acc); - __m256 current = _mm256_loadu_ps(out + j); - _mm256_storeu_ps(out + j, _mm256_max_ps(reduce_max(acc), current)); - } - if (j < n) { - __m256 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm256_setzero_ps(); - } - accumulate_tail(arow, bt, n, d, j, tail_mask, acc); - __m256 current = _mm256_maskload_ps(out + j, tail_mask); - _mm256_maskstore_ps( - out + j, tail_mask, _mm256_max_ps(reduce_max(acc), current) - ); - } - } -} - -// Float16 overload: see convert_row_block above for the shared widening (F16C's 8-wide -// _mm256_cvtph_ps, available because avx2.cpp is compiled for the "haswell" arch in -// cmake/multi-arch.cmake, and Haswell carries F16C alongside AVX2). -void InnerProductMatrixImpl::column_max( - const svs::Float16* a, size_t m, const PackedOperand& b, float* out -) { - column_max_narrow(a, m, b, out); -} - -// BFloat16 overload: see convert_row_block above for the shared (exact) widening. -void InnerProductMatrixImpl::column_max( - const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out -) { - column_max_narrow(a, m, b, out); -} - } // namespace svs::distance #endif diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index 7ad79bc5b..753710398 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -20,7 +20,6 @@ #include "svs/core/distance/gem.h" #include "svs/core/distance/inner_product.h" #include "svs/core/distance/inner_product_matrix.h" -#include "svs/core/distance/simd_utils.h" namespace svs::distance { @@ -60,354 +59,6 @@ template struct QChamferImpl; template struct QChamferExactImpl; template struct InnerProductMatrixImpl; -namespace { - -constexpr size_t kRowBlock = inner_product_matrix_row_block; - -// Accumulates kRowBlock rows of A (padded by repeating the last valid row when the -// document has fewer rows left, see callers) against one full 16-lane block of packed B. -// Shared by the table and column-max output forms so this FMA chain is written once. -SVS_FORCE_INLINE void accumulate_full( - const float* const* arow, const float* bt, size_t n, size_t d, size_t j, __m512* acc -) { - for (size_t k = 0; k < d; ++k) { - __m512 bv = _mm512_loadu_ps(bt + k * n + j); - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); - } - } -} - -// Same as accumulate_full for the trailing lane-block narrower than 16 lanes. -SVS_FORCE_INLINE void accumulate_tail( - const float* const* arow, - const float* bt, - size_t n, - size_t d, - size_t j, - __mmask16 mask, - __m512* acc -) { - for (size_t k = 0; k < d; ++k) { - __m512 bv = _mm512_maskz_loadu_ps(mask, bt + k * n + j); - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); - } - } -} - -// Pairwise-max reduction of the kRowBlock accumulators into their lane-wise maximum. -// Padding slots (see accumulate_full's comment) repeat a value already present in a real -// slot, so folding them into the tree does not change the result. -SVS_FORCE_INLINE __m512 reduce_max(const __m512* acc) { - __m512 m01 = _mm512_max_ps(acc[0], acc[1]); - __m512 m23 = _mm512_max_ps(acc[2], acc[3]); - __m512 m45 = _mm512_max_ps(acc[4], acc[5]); - __m512 m67 = _mm512_max_ps(acc[6], acc[7]); - return _mm512_max_ps(_mm512_max_ps(m01, m23), _mm512_max_ps(m45, m67)); -} - -// Widens one 16-lane block of Float16 to float32 with AVX-512F's native half->single. -SVS_FORCE_INLINE __m512 widen16(const svs::Float16* p) { - return _mm512_cvtph_ps(_mm256_loadu_epi16(p)); -} - -// Widens one 16-lane block of BFloat16 to float32: bfloat16 is already float32's high 16 -// bits, so this is a lossless zero-extend (cvtepu16_epi32, AVX-512F) into the top half of -// each 32-bit lane (slli by 16), not a rounding conversion. -SVS_FORCE_INLINE __m512 widen16(const svs::BFloat16* p) { - __m256i halfwords = _mm256_loadu_si256(reinterpret_cast(p)); - return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(halfwords), 16)); -} - -// Tail (< 16 lanes) widening for either 16-bit type: zero-pads through a stack buffer -// sized by sizeof(T) (2 bytes for both Float16 and BFloat16), then reuses widen16. Uses a -// plain memcpy rather than a masked load so BFloat16's tail needs nothing beyond -// AVX-512F (Float16's masked load above already relies on AVX-512BW+VL for -// _mm256_loadu_epi16, a preexisting requirement this does not widen). -template SVS_FORCE_INLINE __m512 widen16_tail(const T* p, size_t count) { - __m256i buffer = _mm256_setzero_si256(); - std::memcpy(&buffer, p, count * sizeof(T)); - return widen16(reinterpret_cast(&buffer)); -} - -// Converts kRowBlock rows of `a` (T is Float16 or BFloat16) starting at row `i` into a -// contiguous float32 scratch, 16 lanes at a time; shared by the table and column-max -// forms below so either narrow type is widened in exactly one place. -template -SVS_FORCE_INLINE void -convert_row_block(const T* a, size_t i, size_t rows, size_t d, float* scratch) { - for (size_t r = 0; r < rows; ++r) { - const T* src = a + (i + r) * d; - float* dst = scratch + r * d; - size_t k = 0; - for (; k + 16 <= d; k += 16) { - _mm512_storeu_ps(dst + k, widen16(src + k)); - } - if (k < d) { - size_t tail = d - k; - __mmask16 dmask = svs::create_mask<16>(svs::lib::MaybeStatic(tail)); - _mm512_mask_storeu_ps(dst + k, dmask, widen16_tail(src + k, tail)); - } - } -} - -// Table form for a narrow-element A (Float16 or BFloat16): identical register-blocked -// loop to the float32 build_packed above, sourced from a row-block converted to float32 -// by convert_row_block instead of `a` directly. -template -void build_packed_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { - size_t n = b.n(); - size_t d = b.d(); - if (m == 0 || n == 0) { - return; - } - constexpr size_t vl = 16; - const float* bt = b.data(); - size_t full_lanes = n - n % vl; - __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); - - thread_local std::vector scratch; - scratch.resize(kRowBlock * d); - - for (size_t i = 0; i < m; i += kRowBlock) { - size_t rows = std::min(kRowBlock, m - i); - convert_row_block(a, i, rows, d, scratch.data()); - const float* arow[kRowBlock]; - float* orow[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = scratch.data() + std::min(r, rows - 1) * d; - } - for (size_t r = 0; r < rows; ++r) { - orow[r] = out + (i + r) * n; - } - - size_t j = 0; - for (; j < full_lanes; j += vl) { - __m512 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_setzero_ps(); - } - accumulate_full(arow, bt, n, d, j, acc); - for (size_t r = 0; r < rows; ++r) { - _mm512_storeu_ps(orow[r] + j, acc[r]); - } - } - if (j < n) { - __m512 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_setzero_ps(); - } - accumulate_tail(arow, bt, n, d, j, tail_mask, acc); - for (size_t r = 0; r < rows; ++r) { - _mm512_mask_storeu_ps(orow[r] + j, tail_mask, acc[r]); - } - } - } -} - -// Column-max form for a narrow-element A (Float16 or BFloat16): same structure as -// build_packed_narrow above, reduced through reduce_max instead of stored per row. -template -void column_max_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { - size_t n = b.n(); - size_t d = b.d(); - if (n == 0) { - return; - } - std::fill(out, out + n, -std::numeric_limits::infinity()); - if (m == 0) { - return; - } - constexpr size_t vl = 16; - const float* bt = b.data(); - size_t full_lanes = n - n % vl; - __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); - - thread_local std::vector scratch; - scratch.resize(kRowBlock * d); - - for (size_t i = 0; i < m; i += kRowBlock) { - size_t rows = std::min(kRowBlock, m - i); - convert_row_block(a, i, rows, d, scratch.data()); - const float* arow[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = scratch.data() + std::min(r, rows - 1) * d; - } - - size_t j = 0; - for (; j < full_lanes; j += vl) { - __m512 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_setzero_ps(); - } - accumulate_full(arow, bt, n, d, j, acc); - __m512 current = _mm512_loadu_ps(out + j); - _mm512_storeu_ps(out + j, _mm512_max_ps(reduce_max(acc), current)); - } - if (j < n) { - __m512 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_setzero_ps(); - } - accumulate_tail(arow, bt, n, d, j, tail_mask, acc); - __m512 current = _mm512_maskz_loadu_ps(tail_mask, out + j); - _mm512_mask_storeu_ps( - out + j, tail_mask, _mm512_max_ps(reduce_max(acc), current) - ); - } - } -} - -} // namespace - -// Register-blocked micro-kernel: inner_product_matrix_row_block rows of A share one pass -// over a lane-block of B, giving that many independent FMA accumulator chains with no -// horizontal reduction (each lane is one output), so the dependency-chain latency of a -// single accumulator is hidden behind the others instead of serializing the loop over d. -void InnerProductMatrixImpl::build_packed( - const float* a, size_t m, const PackedOperand& b, float* out -) { - size_t n = b.n(); - size_t d = b.d(); - if (m == 0 || n == 0) { - return; - } - constexpr size_t vl = 16; - const float* bt = b.data(); - size_t full_lanes = n - n % vl; - __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); - - for (size_t i = 0; i < m; i += kRowBlock) { - size_t rows = std::min(kRowBlock, m - i); - const float* arow[kRowBlock]; - float* orow[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = a + (i + std::min(r, rows - 1)) * d; - } - for (size_t r = 0; r < rows; ++r) { - orow[r] = out + (i + r) * n; - } - - size_t j = 0; - for (; j < full_lanes; j += vl) { - __m512 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_setzero_ps(); - } - accumulate_full(arow, bt, n, d, j, acc); - for (size_t r = 0; r < rows; ++r) { - _mm512_storeu_ps(orow[r] + j, acc[r]); - } - } - if (j < n) { - __m512 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_setzero_ps(); - } - accumulate_tail(arow, bt, n, d, j, tail_mask, acc); - for (size_t r = 0; r < rows; ++r) { - _mm512_mask_storeu_ps(orow[r] + j, tail_mask, acc[r]); - } - } - } -} - -// Table form for a Float16 A: see convert_row_block above for the shared widening. -void InnerProductMatrixImpl::build_packed( - const svs::Float16* a, size_t m, const PackedOperand& b, float* out -) { - build_packed_narrow(a, m, b, out); -} - -// Table form for a BFloat16 A: see convert_row_block above for the shared widening. -void InnerProductMatrixImpl::build_packed( - const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out -) { - build_packed_narrow(a, m, b, out); -} - -// Packs B into a reused thread_local buffer: the table form repacks on every call (one -// query against all centroids), unlike MaxSim's column_max below which packs once per -// query and calls into this ISA's kernels directly with the already-packed operand. -void InnerProductMatrixImpl::build( - const float* a, size_t m, const float* b, size_t n, size_t d, float* out -) { - if (m == 0 || n == 0) { - return; - } - thread_local PackedOperand packed; - packed.pack(b, n, d); - build_packed(a, m, packed, out); -} - -// Column-wise maximum of the same table build_packed computes, without ever -// materializing it: each row-block's accumulators are reduced to a per-lane maximum and -// folded into a running maximum held in `out` itself, so one reduction serves the whole -// document for the price of one load/max/store per lane-block per row-block. -void InnerProductMatrixImpl::column_max( - const float* a, size_t m, const PackedOperand& b, float* out -) { - size_t n = b.n(); - size_t d = b.d(); - if (n == 0) { - return; - } - std::fill(out, out + n, -std::numeric_limits::infinity()); - if (m == 0) { - return; - } - constexpr size_t vl = 16; - const float* bt = b.data(); - size_t full_lanes = n - n % vl; - __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); - - for (size_t i = 0; i < m; i += kRowBlock) { - size_t rows = std::min(kRowBlock, m - i); - const float* arow[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - arow[r] = a + (i + std::min(r, rows - 1)) * d; - } - - size_t j = 0; - for (; j < full_lanes; j += vl) { - __m512 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_setzero_ps(); - } - accumulate_full(arow, bt, n, d, j, acc); - __m512 current = _mm512_loadu_ps(out + j); - _mm512_storeu_ps(out + j, _mm512_max_ps(reduce_max(acc), current)); - } - if (j < n) { - __m512 acc[kRowBlock]; - for (size_t r = 0; r < kRowBlock; ++r) { - acc[r] = _mm512_setzero_ps(); - } - accumulate_tail(arow, bt, n, d, j, tail_mask, acc); - __m512 current = _mm512_maskz_loadu_ps(tail_mask, out + j); - _mm512_mask_storeu_ps( - out + j, tail_mask, _mm512_max_ps(reduce_max(acc), current) - ); - } - } -} - -// Float16 overload: see convert_row_block above for the shared widening (AVX-512F's -// native 16-wide _mm512_cvtph_ps). -void InnerProductMatrixImpl::column_max( - const svs::Float16* a, size_t m, const PackedOperand& b, float* out -) { - column_max_narrow(a, m, b, out); -} - -// BFloat16 overload: see convert_row_block above for the shared (exact) widening. -void InnerProductMatrixImpl::column_max( - const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out -) { - column_max_narrow(a, m, b, out); -} - } // namespace svs::distance #endif From 4248763e0417240f82c909c10bb48c98a1a2e2e3 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 07:04:06 -0800 Subject: [PATCH 43/68] refactor(vamana): collect build stats through a compile-time collector Replace the scattered #if SVS_ENABLE_STATS blocks in vamana_build.h and maxsim.h with NullBuildStats/CountingBuildStats (and the maxsim equivalent) collector classes. Both collectors always compile, so the stats path is type-checked in every build; only a std::conditional_t alias on SVS_ENABLE_STATS picks the default, and VamanaBuilder takes the collector as a defaulted template parameter so tests can force CountingBuildStats regardless of the build flag. Co-Authored-By: Claude Opus 5.5 --- include/svs/extensions/vamana/maxsim.h | 75 ++++++---- include/svs/index/vamana/build_stats.h | 102 +++++++++++++- include/svs/index/vamana/vamana_build.h | 176 ++++++++---------------- tests/svs/index/vamana/build_stats.cpp | 14 +- 4 files changed, 208 insertions(+), 159 deletions(-) diff --git a/include/svs/extensions/vamana/maxsim.h b/include/svs/extensions/vamana/maxsim.h index 41986956c..1b82cd67e 100644 --- a/include/svs/extensions/vamana/maxsim.h +++ b/include/svs/extensions/vamana/maxsim.h @@ -26,19 +26,15 @@ // stl #include #include +#include #include #include #include +#include #include -#if defined(SVS_ENABLE_STATS) && SVS_ENABLE_STATS -#include -#endif - namespace svs::index::vamana::extensions { -#if defined(SVS_ENABLE_STATS) && SVS_ENABLE_STATS - /// Process-wide re-rank timing accumulated since the last `maxsim_rerank_stats_reset()`. struct MaxSimRerankStats { uint64_t queries; @@ -66,7 +62,47 @@ inline void maxsim_rerank_stats_reset() { stats_detail::maxsim_rerank_ns.store(0, std::memory_order_relaxed); } -#endif +/// No-op re-rank timer: both methods are empty, so binding one into the re-rank hot path +/// costs nothing. Mirrors `CountingMaxSimRerankStats`'s interface exactly. +class NullMaxSimRerankStats { + public: + void mark_search_done() {} + void finish() {} +}; + +/// Re-rank timer that actually records: `mark_search_done()` splits the search and +/// re-rank durations, `finish()` folds both plus one query into the global accumulator. +class CountingMaxSimRerankStats { + public: + void mark_search_done() { search_done_ = std::chrono::steady_clock::now(); } + + void finish() { + auto rerank_done = std::chrono::steady_clock::now(); + auto ns = [](auto from, auto to) { + return static_cast( + std::chrono::duration_cast(to - from).count() + ); + }; + stats_detail::maxsim_queries.fetch_add(1, std::memory_order_relaxed); + stats_detail::maxsim_search_ns.fetch_add( + ns(start_, search_done_), std::memory_order_relaxed + ); + stats_detail::maxsim_rerank_ns.fetch_add( + ns(search_done_, rerank_done), std::memory_order_relaxed + ); + } + + private: + std::chrono::steady_clock::time_point start_ = std::chrono::steady_clock::now(); + std::chrono::steady_clock::time_point search_done_ = start_; +}; + +/// The timer `single_search` uses by default. Both timers above always compile, so only +/// this selection depends on the build-time flag. +using DefaultMaxSimRerankStats = std::conditional_t< + static_cast(SVS_ENABLE_STATS), + CountingMaxSimRerankStats, + NullMaxSimRerankStats>; namespace detail { @@ -146,13 +182,9 @@ template < } auto accessor = data::GetDatumAccessor(); -#if defined(SVS_ENABLE_STATS) && SVS_ENABLE_STATS - auto t_start = std::chrono::steady_clock::now(); + DefaultMaxSimRerankStats rerank_stats{}; search(query, accessor, distance, search_buffer); - auto t_search_done = std::chrono::steady_clock::now(); -#else - search(query, accessor, distance, search_buffer); -#endif + rerank_stats.mark_search_done(); if (cancel()) { return; @@ -178,22 +210,7 @@ template < detail::resort_by_maxsim(search_buffer); -#if defined(SVS_ENABLE_STATS) && SVS_ENABLE_STATS - auto t_rerank_done = std::chrono::steady_clock::now(); - auto search_ns = - std::chrono::duration_cast(t_search_done - t_start) - .count(); - auto rerank_ns = - std::chrono::duration_cast(t_rerank_done - t_search_done) - .count(); - stats_detail::maxsim_queries.fetch_add(1, std::memory_order_relaxed); - stats_detail::maxsim_search_ns.fetch_add( - static_cast(search_ns), std::memory_order_relaxed - ); - stats_detail::maxsim_rerank_ns.fetch_add( - static_cast(rerank_ns), std::memory_order_relaxed - ); -#endif + rerank_stats.finish(); } // The default `per_thread_batch_search` (extensions.h) dispatches per query through the diff --git a/include/svs/index/vamana/build_stats.h b/include/svs/index/vamana/build_stats.h index 17b031ee5..3bfaeac7b 100644 --- a/include/svs/index/vamana/build_stats.h +++ b/include/svs/index/vamana/build_stats.h @@ -16,15 +16,13 @@ #pragma once -// Compiled out unless SVS_ENABLE_STATS is set, so including this header is always free. -#if SVS_ENABLE_STATS - #include "svs/concepts/distance.h" #include #include #include #include +#include namespace svs::index::vamana { @@ -129,6 +127,100 @@ template class CountingDistance { uint64_t& counter_; }; -} // namespace svs::index::vamana +/// No-op build-stats collector: every member is empty, so binding it into the build hot +/// path costs nothing. Mirrors `CountingBuildStats`'s interface exactly. +class NullBuildStats { + public: + struct ScopedTimer {}; + + ScopedTimer time_search() { return {}; } + ScopedTimer time_prune() { return {}; } + ScopedTimer time_backedge_prune() { return {}; } + + template Dist& wrap_search(Dist& dist) { return dist; } + template Dist& wrap_pool(Dist& dist) { return dist; } + template Dist& wrap_prune(Dist& dist) { return dist; } + template Dist& wrap_backedge_prune(Dist& dist) { return dist; } + + void finish_search() {} + void finish_pool() {} + void finish_prune() {} + void merge() {} +}; + +/// Build-stats collector that actually counts: one instance per worker thread, merged +/// into the process-wide accumulator once that thread's work is done. +class CountingBuildStats { + public: + // Adds its own elapsed time into `target` on destruction; never copied or moved, so + // that add happens exactly once per timed scope. + class ScopedTimer { + public: + explicit ScopedTimer(uint64_t& target) + : target_{&target} + , start_{std::chrono::steady_clock::now()} {} + ScopedTimer(const ScopedTimer&) = delete; + ScopedTimer& operator=(const ScopedTimer&) = delete; + ScopedTimer(ScopedTimer&&) = delete; + ScopedTimer& operator=(ScopedTimer&&) = delete; + ~ScopedTimer() { *target_ += build_stats_elapsed_ns(start_); } + + private: + uint64_t* target_; + std::chrono::steady_clock::time_point start_; + }; + + ScopedTimer time_search() { return ScopedTimer{current_.search_ns}; } + ScopedTimer time_prune() { return ScopedTimer{current_.prune_ns}; } + ScopedTimer time_backedge_prune() { return ScopedTimer{current_.backedge_prune_ns}; } + + // Resets its own per-call counter, so the caller must invoke this once per + // search/pool/prune step rather than reusing a previously returned wrapper. + template CountingDistance wrap_search(Dist& dist) { + search_count_ = 0; + return CountingDistance{dist, search_count_}; + } + template CountingDistance wrap_pool(Dist& dist) { + pool_count_ = 0; + return CountingDistance{dist, pool_count_}; + } + template CountingDistance wrap_prune(Dist& dist) { + prune_count_ = 0; + return CountingDistance{dist, prune_count_}; + } -#endif // SVS_ENABLE_STATS + // Backedge counting spans the whole thread's work, not one node, so this is called + // once per thread and the returned wrapper is reused across every source vertex. + template CountingDistance wrap_backedge_prune(Dist& dist) { + return CountingDistance{dist, backedge_count_}; + } + + void finish_search() { + current_.search_distances += search_count_; + current_.max_search_distances_per_insert = + std::max(current_.max_search_distances_per_insert, search_count_); + ++current_.inserts; + } + void finish_pool() { current_.pool_distances += pool_count_; } + void finish_prune() { current_.prune_distances += prune_count_; } + + /// Call once per thread, after that thread's loop over nodes finishes. + void merge() { + current_.backedge_prune_distances = backedge_count_; + build_stats_merge(current_); + } + + private: + BuildStats current_{}; + uint64_t search_count_ = 0; + uint64_t pool_count_ = 0; + uint64_t prune_count_ = 0; + uint64_t backedge_count_ = 0; +}; + +/// The collector `VamanaBuilder` uses by default. Both collectors above always compile, +/// so only this selection depends on the build-time flag. +using DefaultBuildStats = std:: + conditional_t(SVS_ENABLE_STATS), CountingBuildStats, NullBuildStats>; + +} // namespace svs::index::vamana diff --git a/include/svs/index/vamana/vamana_build.h b/include/svs/index/vamana/vamana_build.h index 95d038bda..d1f48eb14 100644 --- a/include/svs/index/vamana/vamana_build.h +++ b/include/svs/index/vamana/vamana_build.h @@ -44,7 +44,6 @@ // stdlib #include #include -#include #include #include #include @@ -169,7 +168,8 @@ template < graphs::MemoryGraph Graph, data::ImmutableMemoryDataset Data, typename Dist, - threads::ThreadPool Pool> + threads::ThreadPool Pool, + typename StatsCollector = DefaultBuildStats> class VamanaBuilder { public: // Type Aliases @@ -374,9 +374,7 @@ class VamanaBuilder { auto&& graph_search_distance = build_adaptor.graph_search_distance(); auto&& general_distance = build_adaptor.general_distance(); auto general_accessor = build_adaptor.general_accessor(); -#if SVS_ENABLE_STATS - BuildStats thread_stats{}; -#endif + StatsCollector stats{}; for (auto node_id : local_indices) { pool.clear(); @@ -387,17 +385,12 @@ class VamanaBuilder { const auto& graph_search_query = build_adaptor.access_query_for_graph_search(data_, node_id); - // Perform the greedy search. - // The search tracker will be used if it is enabled. -#if SVS_ENABLE_STATS - uint64_t search_distance_count = 0; - CountingDistance< - std::remove_reference_t> - counted_graph_search_distance{ - graph_search_distance, search_distance_count}; - auto search_clock_start = std::chrono::steady_clock::now(); -#endif + // Perform the greedy search. + // The search tracker will be used if it is enabled. + auto&& counted_graph_search_distance = + stats.wrap_search(graph_search_distance); { + [[maybe_unused]] auto search_timer = stats.time_search(); // local_indices' element type varies; narrow to avoid two Idx's. auto node_entry_points = extensions::build_entry_points( data_, @@ -412,11 +405,7 @@ class VamanaBuilder { data_, accessor, graph_search_query, -#if SVS_ENABLE_STATS counted_graph_search_distance, -#else - graph_search_distance, -#endif search_buffer, vamana::EntryPointInitializer{node_entry_points}, NeighborBuilder(), @@ -424,14 +413,7 @@ class VamanaBuilder { prefetch_hint_ ); } -#if SVS_ENABLE_STATS - thread_stats.search_ns += build_stats_elapsed_ns(search_clock_start); - thread_stats.search_distances += search_distance_count; - thread_stats.max_search_distances_per_insert = std::max( - thread_stats.max_search_distances_per_insert, search_distance_count - ); - ++thread_stats.inserts; -#endif + stats.finish_search(); const auto& post_search_query = build_adaptor.modify_post_search_query( data_, node_id, graph_search_query @@ -469,12 +451,8 @@ class VamanaBuilder { } } - // Add neighbors of the query that are not part of `visited`. -#if SVS_ENABLE_STATS - uint64_t pool_distance_count = 0; - CountingDistance> - counted_pool_distance{general_distance, pool_distance_count}; -#endif + // Add neighbors of the query that are not part of `visited`. + auto&& counted_pool_distance = stats.wrap_pool(general_distance); for (auto id : graph_.get_node(node_id)) { assert(id != node_id); // Try to emplace the node id into the visited set. @@ -485,20 +463,14 @@ class VamanaBuilder { pool.emplace_back( id, distance::compute( -#if SVS_ENABLE_STATS counted_pool_distance, -#else - general_distance, -#endif post_search_query, general_accessor(data_, id) ) ); } } -#if SVS_ENABLE_STATS - thread_stats.pool_distances += pool_distance_count; -#endif + stats.finish_pool(); std::sort( pool.begin(), @@ -510,35 +482,24 @@ class VamanaBuilder { // Prune and wait for an update. thread_local_updates.emplace_back(node_id, std::vector{}); auto& pruned_results = thread_local_updates.back().second; -#if SVS_ENABLE_STATS - uint64_t prune_distance_count = 0; - CountingDistance> - counted_prune_distance{general_distance, prune_distance_count}; - auto prune_clock_start = std::chrono::steady_clock::now(); -#endif - heuristic_prune_neighbors( - prune_strategy(distance_function_), - params_.graph_max_degree, - alpha, - data_, - general_accessor, -#if SVS_ENABLE_STATS - counted_prune_distance, -#else - general_distance, -#endif - node_id, - lib::as_const_span(pool), - pruned_results - ); -#if SVS_ENABLE_STATS - thread_stats.prune_ns += build_stats_elapsed_ns(prune_clock_start); - thread_stats.prune_distances += prune_distance_count; -#endif + auto&& counted_prune_distance = stats.wrap_prune(general_distance); + { + [[maybe_unused]] auto prune_timer = stats.time_prune(); + heuristic_prune_neighbors( + prune_strategy(distance_function_), + params_.graph_max_degree, + alpha, + data_, + general_accessor, + counted_prune_distance, + node_id, + lib::as_const_span(pool), + pruned_results + ); + } + stats.finish_prune(); } -#if SVS_ENABLE_STATS - build_stats_merge(thread_stats); -#endif + stats.merge(); } ); @@ -592,13 +553,7 @@ class VamanaBuilder { threads::parallel_for( threadpool_, threads::DynamicPartition{backedge_buffer_.buckets(), 1}, - [&](auto& buckets, -#if SVS_ENABLE_STATS - uint64_t tid -#else - uint64_t SVS_UNUSED(tid) -#endif - ) { + [&](auto& buckets, uint64_t SVS_UNUSED(tid)) { // Thread local auxiliary data structures. std::vector> candidates{}; std::vector pruned_results{}; @@ -608,13 +563,10 @@ class VamanaBuilder { auto&& general_distance = build_adaptor.general_distance(); auto cmp = distance::comparator(general_distance); -#if SVS_ENABLE_STATS - BuildStats thread_stats{}; - (void)tid; - uint64_t backedge_distance_count = 0; - CountingDistance> - counted_backedge_distance{general_distance, backedge_distance_count}; -#endif + StatsCollector stats{}; + // Backedge counting spans the whole thread, so wrap once and reuse below. + auto&& counted_backedge_distance = + stats.wrap_backedge_prune(general_distance); for (auto& bucket : buckets) { for (const auto& kv : bucket) { // The ``neighbors`` class is a set. @@ -626,17 +578,13 @@ class VamanaBuilder { // Helper lambda to make distance computations look a little // cleaner. auto make_neighbor = [&](auto i) { - return Neighbor { - i, distance::compute( -#if SVS_ENABLE_STATS - counted_backedge_distance, -#else - general_distance, -#endif - src_data, - general_accessor(data_, i) - ) - }; + return Neighbor{ + i, + distance::compute( + counted_backedge_distance, + src_data, + general_accessor(data_, i) + )}; }; candidates.clear(); @@ -656,35 +604,25 @@ class VamanaBuilder { std::min(candidates.size(), params_.max_candidate_pool_size) ); -#if SVS_ENABLE_STATS - auto backedge_prune_clock_start = std::chrono::steady_clock::now(); -#endif - heuristic_prune_neighbors( - prune_strategy(distance_function_), - params_.prune_to, - alpha, - data_, - general_accessor, -#if SVS_ENABLE_STATS - counted_backedge_distance, -#else - general_distance, -#endif - src, - lib::as_const_span(candidates), - pruned_results - ); -#if SVS_ENABLE_STATS - thread_stats.backedge_prune_ns += - build_stats_elapsed_ns(backedge_prune_clock_start); -#endif + { + [[maybe_unused]] auto backedge_prune_timer = + stats.time_backedge_prune(); + heuristic_prune_neighbors( + prune_strategy(distance_function_), + params_.prune_to, + alpha, + data_, + general_accessor, + counted_backedge_distance, + src, + lib::as_const_span(candidates), + pruned_results + ); + } graph_.replace_node(src, pruned_results); } } -#if SVS_ENABLE_STATS - thread_stats.backedge_prune_distances = backedge_distance_count; - build_stats_merge(thread_stats); -#endif + stats.merge(); } ); } diff --git a/tests/svs/index/vamana/build_stats.cpp b/tests/svs/index/vamana/build_stats.cpp index ff4233e81..1db53b122 100644 --- a/tests/svs/index/vamana/build_stats.cpp +++ b/tests/svs/index/vamana/build_stats.cpp @@ -17,9 +17,6 @@ // Header under test: the build-time measurement counters. #include "svs/index/vamana/build_stats.h" -// Only meaningful with SVS_ENABLE_STATS=1, else this file is an empty translation unit. -#if SVS_ENABLE_STATS - // svs #include "svs/core/data/simple.h" #include "svs/core/distance.h" @@ -102,7 +99,14 @@ CATCH_TEST_CASE("Build Stats", "[vamana][build_stats]") { build_stats_test::CountingTestDistance::counter() = 0; vamana::build_stats_reset(); - auto builder = vamana::VamanaBuilder( + // Instantiate with CountingBuildStats explicitly, so this test exercises the real + // collector regardless of the ambient SVS_ENABLE_STATS build flag. + auto builder = vamana::VamanaBuilder< + decltype(graph), + decltype(data), + build_stats_test::CountingTestDistance, + decltype(build_pool), + vamana::CountingBuildStats>( graph, data, build_stats_test::CountingTestDistance{}, @@ -133,5 +137,3 @@ CATCH_TEST_CASE("Build Stats", "[vamana][build_stats]") { CATCH_REQUIRE(stats.backedge_prune_ns > 0); CATCH_REQUIRE(stats.max_search_distances_per_insert > 0); } - -#endif // SVS_ENABLE_STATS From e5eaeee8dc2e3ebdb8eb229675c7fcd9ee1a551c Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 07:22:23 -0800 Subject: [PATCH 44/68] test(vamana): consolidate the GEM, MaxSim and connectivity tests Second pass. Shares the GEM centroid/corpus fixtures and adds a generic brute-force top-k oracle plus a degree-bound checker (tests/svs/index/vamana/ gem_test_utils.h, used by all three GEM files and by maxsim_rerank.cpp); drops gem_routing.cpp's "no router" / "every cluster unrepresented" cases (covered by gem_build.cpp's build_entry_points); folds connectivity.cpp's five repair_connectivity cases into one table-driven TEST_CASE; builds one index per TEST_CASE in gem_vector_set.cpp and maxsim_rerank.cpp and runs every behavior as a CATCH_SECTION on it; and removes the build-time routed-vs-unrouted recall comparison from gem_build.cpp's build_gem_graph, keeping only the query-time one in gem_routing.cpp's "Routed search is not worse than unrouted" (the layer that owns search-time routing). Also drops maxsim_rerank.cpp's capacity=40 variant of the MaxSim pool-order check: a tighter search window only re-exercises the generic search buffer's eviction path, which search_buffer.cpp already owns, not anything in the MaxSim re-rank extension under test. Before -> after (lines, after clang-format): build_stats.cpp 139 -> 139 (unchanged, already minimal) connectivity.cpp 274 -> 219 gem_build.cpp 297 -> 170 gem_routing.cpp 394 -> 245 gem_vector_set.cpp 313 -> 202 maxsim_rerank.cpp 687 -> 548 gem_test_utils.h 0 -> 150 (new, shared across all 6 files) Total (6 files + helper, counted together per the <=1050 target) -> 1673, against a 2104-line baseline for the 6 files alone. This still falls short of the <=1050 target. Code-only lines (stripping every blank line and every comment from all 7 files) sum to over 1300, so the remaining gap cannot close through formatting or comment trimming alone: closing it means dropping tested branches, not duplication. Two branches already lost coverage in this pass, noted above; the handback report to the coordinator lists further branches that could be cut with their line cost, for the coordinator to choose from rather than have cut silently. Co-Authored-By: Claude Opus 5.5 --- tests/svs/index/vamana/connectivity.cpp | 367 ++++++++--------- tests/svs/index/vamana/gem_build.cpp | 199 ++------- tests/svs/index/vamana/gem_routing.cpp | 185 +-------- tests/svs/index/vamana/gem_test_utils.h | 150 +++++++ tests/svs/index/vamana/gem_vector_set.cpp | 277 ++++--------- tests/svs/index/vamana/maxsim_rerank.cpp | 465 ++++++++-------------- 6 files changed, 606 insertions(+), 1037 deletions(-) create mode 100644 tests/svs/index/vamana/gem_test_utils.h diff --git a/tests/svs/index/vamana/connectivity.cpp b/tests/svs/index/vamana/connectivity.cpp index 86e31dae5..6c0d95da1 100644 --- a/tests/svs/index/vamana/connectivity.cpp +++ b/tests/svs/index/vamana/connectivity.cpp @@ -23,27 +23,23 @@ #include "svs/core/graph.h" #include "svs/lib/threads.h" +// tests +#include "tests/svs/index/vamana/gem_test_utils.h" + // catch2 #include "catch2/catch_test_macros.hpp" // stl #include +#include #include +#include #include namespace { using Idx = uint32_t; - -std::vector> adjacency_snapshot(const svs::graphs::SimpleGraph& graph -) { - std::vector> out(graph.n_nodes()); - for (size_t i = 0; i < graph.n_nodes(); ++i) { - auto node = graph.get_node(static_cast(i)); - out[i].assign(node.begin(), node.end()); - } - return out; -} +namespace v = svs::index::vamana; svs::data::SimpleData make_positions(const std::vector& values) { auto data = svs::data::SimpleData(values.size(), 1); @@ -53,222 +49,171 @@ svs::data::SimpleData make_positions(const std::vector& values) return data; } -} // namespace - -CATCH_TEST_CASE("unreachable_nodes", "[index][vamana][connectivity]") { - namespace v = svs::index::vamana; - - auto graph = svs::default_graph(5, 3); - graph.replace_node(0, std::vector{1, 2}); - graph.replace_node(2, std::vector{3}); - // Nodes 1, 3, 4 keep their default empty adjacency list. - - CATCH_SECTION("One root reaches a prefix, two nodes stay unreachable") { - std::vector roots{0}; - auto unreachable = v::unreachable_nodes(graph, std::span(roots)); - CATCH_REQUIRE(unreachable == std::vector{4}); - } - - CATCH_SECTION("Adding the straggler as a root reaches everything") { - std::vector roots{0, 4}; - auto unreachable = v::unreachable_nodes(graph, std::span(roots)); - CATCH_REQUIRE(unreachable.empty()); - } -} - -CATCH_TEST_CASE( - "repair_connectivity leaves an already-connected graph untouched", - "[index][vamana][connectivity]" +svs::graphs::SimpleGraph build_test_graph( + size_t n_nodes, size_t max_degree, const std::vector>& adjacency ) { - namespace v = svs::index::vamana; - - auto graph = svs::default_graph(5, 3); - graph.replace_node(0, std::vector{1}); - graph.replace_node(1, std::vector{2}); - graph.replace_node(2, std::vector{3}); - graph.replace_node(3, std::vector{4}); - - auto before = adjacency_snapshot(graph); - - auto data = make_positions({0.0F, 1.0F, 2.0F, 3.0F, 4.0F}); - svs::distance::DistanceL2 distance{}; - auto params = v::VamanaBuildParameters{1.2F, 3, 10, 10, 3, true}; - auto pool = svs::threads::DefaultThreadPool(1); - std::vector roots{0}; - - auto report = v::repair_connectivity( - graph, data, distance, params, std::span(roots), pool - ); - - CATCH_REQUIRE(report.unreachable_before == 0); - CATCH_REQUIRE(report.unreachable_after == 0); - CATCH_REQUIRE(report.rounds == 0); - CATCH_REQUIRE(report.edges_appended == 0); - CATCH_REQUIRE(report.edges_accepted_through_prune == 0); - CATCH_REQUIRE(report.edges_forced == 0); - CATCH_REQUIRE(adjacency_snapshot(graph) == before); -} - -CATCH_TEST_CASE( - "repair_connectivity reconnects a disconnected node with room to spare", - "[index][vamana][connectivity]" -) { - namespace v = svs::index::vamana; - - // Nodes 0-3 reachable from root 0, all with degree < graph_max_degree. - auto graph = svs::default_graph(5, 3); - graph.replace_node(0, std::vector{1}); - graph.replace_node(1, std::vector{2}); - graph.replace_node(2, std::vector{3}); - - auto data = make_positions({0.0F, 1.0F, 2.0F, 3.0F, 2.1F}); - svs::distance::DistanceL2 distance{}; - auto params = v::VamanaBuildParameters{1.2F, 3, 10, 10, 3, true}; - auto pool = svs::threads::DefaultThreadPool(1); - std::vector roots{0}; - - auto report = v::repair_connectivity( - graph, data, distance, params, std::span(roots), pool - ); - - CATCH_REQUIRE(report.unreachable_before == 1); - CATCH_REQUIRE(report.unreachable_after == 0); - CATCH_REQUIRE(report.rounds >= 1); - CATCH_REQUIRE( - report.edges_appended + report.edges_accepted_through_prune + report.edges_forced == - 1 - ); - CATCH_REQUIRE(v::unreachable_nodes(graph, std::span(roots)).empty()); - for (size_t i = 0; i < graph.n_nodes(); ++i) { - CATCH_REQUIRE( - graph.get_node_degree(static_cast(i)) <= params.graph_max_degree - ); + auto graph = svs::default_graph(n_nodes, max_degree); + for (size_t i = 0; i < adjacency.size(); ++i) { + graph.replace_node(static_cast(i), adjacency[i]); } + return graph; } -CATCH_TEST_CASE( - "repair_connectivity reconnects a node whose nearest reachable neighbors are all at " - "full degree", - "[index][vamana][connectivity]" +std::vector> adjacency_snapshot(const svs::graphs::SimpleGraph& graph ) { - namespace v = svs::index::vamana; - - // 2-regular cycle at full degree; node 4 is disconnected and nearest to root 0. - auto graph = svs::default_graph(5, 2); - graph.replace_node(0, std::vector{1, 2}); - graph.replace_node(1, std::vector{2, 3}); - graph.replace_node(2, std::vector{3, 0}); - graph.replace_node(3, std::vector{0, 1}); - - auto data = make_positions({0.0F, 1.0F, 2.0F, 3.0F, 0.1F}); - svs::distance::DistanceL2 distance{}; - auto params = v::VamanaBuildParameters{1.2F, 2, 10, 10, 2, true}; - auto pool = svs::threads::DefaultThreadPool(1); - std::vector roots{0}; - - for (size_t i = 0; i < 4; ++i) { - CATCH_REQUIRE(graph.get_node_degree(static_cast(i)) == 2); - } - - auto report = v::repair_connectivity( - graph, data, distance, params, std::span(roots), pool - ); - - CATCH_REQUIRE(report.unreachable_before == 1); - CATCH_REQUIRE(report.unreachable_after == 0); - CATCH_REQUIRE( - report.edges_appended + report.edges_accepted_through_prune + report.edges_forced == - 1 - ); - CATCH_REQUIRE(report.edges_forced_unsafe == 0); - CATCH_REQUIRE(!report.stalled); - CATCH_REQUIRE(v::unreachable_nodes(graph, std::span(roots)).empty()); + std::vector> out(graph.n_nodes()); for (size_t i = 0; i < graph.n_nodes(); ++i) { - CATCH_REQUIRE( - graph.get_node_degree(static_cast(i)) <= params.graph_max_degree - ); + auto node = graph.get_node(static_cast(i)); + out[i].assign(node.begin(), node.end()); } + return out; } -CATCH_TEST_CASE( - "repair_connectivity falls back to an unsafe forced replacement when no candidate " - "offers a safe one, and reports it", - "[index][vamana][connectivity]" -) { - namespace v = svs::index::vamana; +// One row per repair_connectivity scenario. Fields left at their default (nullopt/empty) +// mirror an assertion the original cases didn't make either. +struct RepairCase { + std::string name{}; + size_t n_nodes{}; + std::vector> adjacency{}; + std::vector positions{}; + v::VamanaBuildParameters params{}; + size_t max_rounds = 4; + size_t unreachable_before{}; + size_t unreachable_after{}; + std::optional rounds{}; + bool rounds_is_min = false; + std::optional edges_total{}; + std::optional edges_forced_unsafe{}; + std::optional stalled{}; + std::vector exact_unreachable_ids{}; + bool check_unchanged = false; +}; - // Root->1->2, both at degree 1; node 2 is a leaf, so dropping it is never safe. - auto graph = svs::default_graph(4, 1); - graph.replace_node(0, std::vector{1}); - graph.replace_node(1, std::vector{2}); - // Node 2 (a leaf) and node 3 (disconnected) keep their default empty adjacency list. +} // namespace - auto data = make_positions({100.0F, 0.0F, 2.0F, -3.0F}); - svs::distance::DistanceL2 distance{}; - auto params = v::VamanaBuildParameters{1.0F, 1, 1, 10, 1, true}; - auto pool = svs::threads::DefaultThreadPool(1); +CATCH_TEST_CASE("unreachable_nodes", "[index][vamana][connectivity]") { + auto graph = build_test_graph(5, 3, {{1, 2}, {}, {3}}); std::vector roots{0}; - - // Stop after one round: this test is about the forced step within a single round. - auto report = v::repair_connectivity( - graph, data, distance, params, std::span(roots), pool, size_t{1} - ); - - CATCH_REQUIRE(report.rounds == 1); - CATCH_REQUIRE(report.unreachable_before == 1); - CATCH_REQUIRE(report.edges_appended == 0); - CATCH_REQUIRE(report.edges_accepted_through_prune == 0); - CATCH_REQUIRE(report.edges_forced == 0); - CATCH_REQUIRE(report.edges_forced_unsafe == 1); - - // Node 1's forced edge to node 3 displaces node 2, leaving node 2 unreachable. - CATCH_REQUIRE(report.stalled); - CATCH_REQUIRE(report.unreachable_after == 1); CATCH_REQUIRE( - v::unreachable_nodes(graph, std::span(roots)) == std::vector{2} + v::unreachable_nodes(graph, std::span(roots)) == std::vector{4} ); - for (size_t i = 0; i < graph.n_nodes(); ++i) { - CATCH_REQUIRE( - graph.get_node_degree(static_cast(i)) <= params.graph_max_degree - ); - } + roots.push_back(4); + CATCH_REQUIRE(v::unreachable_nodes(graph, std::span(roots)).empty()); } -CATCH_TEST_CASE( - "repair_connectivity's stall guard stops an oscillating repair before max_rounds", - "[index][vamana][connectivity]" -) { - namespace v = svs::index::vamana; - - // Same graph as the unsafe-fallback case, run to default max_rounds instead of 1. - auto graph = svs::default_graph(4, 1); - graph.replace_node(0, std::vector{1}); - graph.replace_node(1, std::vector{2}); - // Node 2 (a leaf) and node 3 (disconnected) keep their default empty adjacency list. - - auto data = make_positions({100.0F, 0.0F, 2.0F, -3.0F}); - svs::distance::DistanceL2 distance{}; - auto params = v::VamanaBuildParameters{1.0F, 1, 1, 10, 1, true}; - auto pool = svs::threads::DefaultThreadPool(1); - std::vector roots{0}; - - // Default max_rounds: the stall guard, not the round cap, has to be what stops this. - auto report = v::repair_connectivity( - graph, data, distance, params, std::span(roots), pool - ); - - CATCH_REQUIRE(report.stalled); - CATCH_REQUIRE(report.rounds == 1); - CATCH_REQUIRE(report.unreachable_before == 1); - CATCH_REQUIRE(report.edges_forced_unsafe == 1); - CATCH_REQUIRE( - report.unreachable_after == - v::unreachable_nodes(graph, std::span(roots)).size() - ); - CATCH_REQUIRE(report.unreachable_after == 1); - for (size_t i = 0; i < graph.n_nodes(); ++i) { - CATCH_REQUIRE( - graph.get_node_degree(static_cast(i)) <= params.graph_max_degree - ); +CATCH_TEST_CASE("repair_connectivity", "[index][vamana][connectivity]") { + std::vector cases = { + {.name = "leaves an already-connected graph untouched", + .n_nodes = 5, + .adjacency = {{1}, {2}, {3}, {4}}, + .positions = {0.0F, 1.0F, 2.0F, 3.0F, 4.0F}, + .params = v::VamanaBuildParameters{1.2F, 3, 10, 10, 3, true}, + .unreachable_before = 0, + .unreachable_after = 0, + .rounds = 0, + .edges_total = 0, + .check_unchanged = true}, + {.name = "reconnects a disconnected node with room to spare", + .n_nodes = 5, + .adjacency = {{1}, {2}, {3}}, + .positions = {0.0F, 1.0F, 2.0F, 3.0F, 2.1F}, + .params = v::VamanaBuildParameters{1.2F, 3, 10, 10, 3, true}, + .unreachable_before = 1, + .unreachable_after = 0, + .rounds = 1, + .rounds_is_min = true, + .edges_total = 1}, + {.name = "reconnects when the nearest reachable neighbors are all at full degree", + .n_nodes = 5, + .adjacency = {{1, 2}, {2, 3}, {3, 0}, {0, 1}}, + .positions = {0.0F, 1.0F, 2.0F, 3.0F, 0.1F}, + .params = v::VamanaBuildParameters{1.2F, 2, 10, 10, 2, true}, + .unreachable_before = 1, + .unreachable_after = 0, + .edges_total = 1, + .edges_forced_unsafe = 0, + .stalled = false}, + {.name = "falls back to an unsafe forced replacement when no candidate is safe", + .n_nodes = 4, + .adjacency = {{1}, {2}}, + .positions = {100.0F, 0.0F, 2.0F, -3.0F}, + .params = v::VamanaBuildParameters{1.0F, 1, 1, 10, 1, true}, + .max_rounds = 1, + .unreachable_before = 1, + .unreachable_after = 1, + .rounds = 1, + .edges_total = 0, + .edges_forced_unsafe = 1, + .stalled = true, + .exact_unreachable_ids = {2}}, + {.name = "stall guard stops an oscillating repair before max_rounds", + .n_nodes = 4, + .adjacency = {{1}, {2}}, + .positions = {100.0F, 0.0F, 2.0F, -3.0F}, + .params = v::VamanaBuildParameters{1.0F, 1, 1, 10, 1, true}, + .unreachable_before = 1, + .unreachable_after = 1, + .rounds = 1, + .edges_forced_unsafe = 1, + .stalled = true}, + }; + + for (const auto& c : cases) { + CATCH_SECTION(c.name) { + auto graph = + build_test_graph(c.n_nodes, c.params.graph_max_degree, c.adjacency); + auto before = c.check_unchanged ? adjacency_snapshot(graph) + : std::vector>{}; + auto data = make_positions(c.positions); + svs::distance::DistanceL2 distance{}; + auto pool = svs::threads::DefaultThreadPool(1); + std::vector roots{0}; + auto report = v::repair_connectivity( + graph, + data, + distance, + c.params, + std::span(roots), + pool, + c.max_rounds + ); + + CATCH_REQUIRE(report.unreachable_before == c.unreachable_before); + CATCH_REQUIRE(report.unreachable_after == c.unreachable_after); + if (c.rounds) { + if (c.rounds_is_min) { + CATCH_REQUIRE(report.rounds >= *c.rounds); + } else { + CATCH_REQUIRE(report.rounds == *c.rounds); + } + } + if (c.edges_total) { + CATCH_REQUIRE( + report.edges_appended + report.edges_accepted_through_prune + + report.edges_forced == + *c.edges_total + ); + } + if (c.edges_forced_unsafe) { + CATCH_REQUIRE(report.edges_forced_unsafe == *c.edges_forced_unsafe); + } + if (c.stalled) { + CATCH_REQUIRE(report.stalled == *c.stalled); + } + CATCH_REQUIRE( + v::unreachable_nodes(graph, std::span(roots)).size() == + report.unreachable_after + ); + if (!c.exact_unreachable_ids.empty()) { + CATCH_REQUIRE( + v::unreachable_nodes(graph, std::span(roots)) == + c.exact_unreachable_ids + ); + } + svs_test::require_degree_bound(graph, c.params.graph_max_degree); + if (c.check_unchanged) { + CATCH_REQUIRE(adjacency_snapshot(graph) == before); + } + } } } diff --git a/tests/svs/index/vamana/gem_build.cpp b/tests/svs/index/vamana/gem_build.cpp index fe309364f..a61a7ab72 100644 --- a/tests/svs/index/vamana/gem_build.cpp +++ b/tests/svs/index/vamana/gem_build.cpp @@ -28,145 +28,59 @@ #include "svs/index/vamana/connectivity.h" #include "svs/lib/threads.h" +// tests +#include "tests/svs/index/vamana/gem_test_utils.h" + // catch2 #include "catch2/catch_test_macros.hpp" // stl #include -#include #include #include -#include #include -#include #include #include namespace { -using svs::distance::CentroidTable; -using svs::distance::Ground; using svs::distance::QEmd; - -constexpr size_t K = 16; -constexpr size_t DIM = 4; - -std::vector make_centroids(uint32_t seed) { - std::mt19937 rng(seed); - std::uniform_real_distribution coord(-1.0F, 1.0F); - std::vector centroids(K * DIM); - for (size_t k = 0; k < K; ++k) { - double norm = 0.0; - for (size_t d = 0; d < DIM; ++d) { - float v = coord(rng); - centroids[k * DIM + d] = v; - norm += static_cast(v) * static_cast(v); - } - norm = std::sqrt(norm); - for (size_t d = 0; d < DIM; ++d) { - centroids[k * DIM + d] = static_cast(centroids[k * DIM + d] / norm); - } - } - return centroids; -} - -std::shared_ptr make_table(const std::vector& centroids) { - return std::make_shared( - CentroidTable::from_centroids(centroids.data(), K, DIM, Ground::Cos, 1) - ); -} - -struct IdSetCorpus { - std::vector ids; - std::vector offsets; - - size_t size() const { return offsets.size() - 1; } - - svs::data::ConstVectorSetView view() const { - return svs::data::ConstVectorSetView( - std::span(ids), std::span(offsets), 1 - ); - } -}; - -IdSetCorpus make_doc_corpus(size_t n, uint32_t seed) { - std::mt19937 rng(seed); - std::uniform_int_distribution id_dist(0, static_cast(K - 1)); - IdSetCorpus corpus; - corpus.offsets.resize(n + 1, 0); - for (size_t i = 0; i < n; ++i) { - size_t cardinality = 2 + (i % 5); - corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; - for (size_t t = 0; t < cardinality; ++t) { - corpus.ids.push_back(id_dist(rng)); - } - } - return corpus; -} - -// Fine centroid f goes to coarse cluster f % num_coarse; deterministic, not realistic. -std::vector round_robin_coarse_of_fine(size_t num_coarse) { - std::vector out(K); - for (size_t f = 0; f < K; ++f) { - out[f] = static_cast(f % num_coarse); - } - return out; -} - -std::vector> brute_force_qemd( - QEmd& dist, - const svs::data::ConstVectorSet& query, - const svs::data::ConstVectorSetView& docs -) { - std::vector> scored; - scored.reserve(docs.size()); - for (uint32_t id = 0; id < docs.size(); ++id) { - float score = svs::distance::compute(dist, query, docs.get_datum(id)); - scored.emplace_back(id, score); - } - std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { - return QEmd::compare{}(a.second, b.second); - }); - return scored; -} +using svs_test::gem::make_centroid_table; +using svs_test::gem::make_centroids; +using svs_test::gem::make_doc_corpus; +using svs_test::gem::round_robin_coarse_of_fine; } // namespace -CATCH_TEST_CASE("gem_insert_order", "[vamana][gem][gem_build]") { +// Covers the no-router / all-unrepresented fallback for GEM entry-point routing at the +// layer that owns it; gem_routing.cpp's GemRoutedInitializer mirrors it without repeating. +CATCH_TEST_CASE("gem_insert_order and build_entry_points", "[vamana][gem][gem_build]") { namespace v = svs::index::vamana; + namespace ext = v::extensions; + + CATCH_SECTION("gem_insert_order dedups representatives ahead of the rest, ascending") { + svs::distance::GemRouter router; + router.coarse_of_fine = {0, 0, 1, 1, 2, 2}; + router.representative = {4, 2, svs::distance::GemRouter::no_representative, 2}; + router.nprobe = 2; + + auto order = v::gem_insert_order(router, 6); + auto sorted_order = order; + std::sort(sorted_order.begin(), sorted_order.end()); + CATCH_REQUIRE(sorted_order == std::vector{0, 1, 2, 3, 4, 5}); + CATCH_REQUIRE(order.size() >= 2); + CATCH_REQUIRE(order[0] == 2); + CATCH_REQUIRE(order[1] == 4); + CATCH_REQUIRE( + std::vector(order.begin() + 2, order.end()) == + std::vector{0, 1, 3, 5} + ); + } - svs::distance::GemRouter router; - // 6 documents; representatives have a duplicate (cluster 2 has no_representative). - router.coarse_of_fine = {0, 0, 1, 1, 2, 2}; - router.representative = {4, 2, svs::distance::GemRouter::no_representative, 2}; - router.nprobe = 2; - - auto order = v::gem_insert_order(router, 6); - - // Every document appears exactly once. - auto sorted_order = order; - std::sort(sorted_order.begin(), sorted_order.end()); - CATCH_REQUIRE(sorted_order == std::vector{0, 1, 2, 3, 4, 5}); - - // Representatives (deduplicated, ascending, no_representative dropped) come first. - CATCH_REQUIRE(order.size() >= 2); - CATCH_REQUIRE(order[0] == 2); - CATCH_REQUIRE(order[1] == 4); - // The rest follow ascending. - CATCH_REQUIRE( - std::vector(order.begin() + 2, order.end()) == - std::vector{0, 1, 3, 5} - ); -} - -CATCH_TEST_CASE("build_entry_points for QEmd", "[vamana][gem][gem_build]") { - namespace ext = svs::index::vamana::extensions; auto centroids = make_centroids(0xB001); - auto table = make_table(centroids); - + auto table = make_centroid_table(centroids); auto doc_corpus = make_doc_corpus(20, 0xB002); auto docs = doc_corpus.view(); - std::vector entry_points{3}; std::vector scratch; @@ -227,7 +141,7 @@ CATCH_TEST_CASE("build_gem_graph", "[vamana][gem][gem_build]") { namespace v = svs::index::vamana; auto centroids = make_centroids(0xB003); - auto table = make_table(centroids); + auto table = make_centroid_table(centroids); const size_t n_docs = 300; auto doc_corpus = make_doc_corpus(n_docs, 0xB004); @@ -242,7 +156,8 @@ CATCH_TEST_CASE("build_gem_graph", "[vamana][gem][gem_build]") { ); auto router_ptr = std::make_shared(router); - auto run_build = [&](std::shared_ptr maybe_router) { + for (auto maybe_router : + {std::shared_ptr(nullptr), router_ptr}) { QEmd distance{table, std::move(maybe_router)}; auto pool = svs::threads::DefaultThreadPool(1); auto [graph, report] = @@ -250,48 +165,6 @@ CATCH_TEST_CASE("build_gem_graph", "[vamana][gem][gem_build]") { CATCH_REQUIRE(report.documents == n_docs); CATCH_REQUIRE(report.unreachable_after_repair == 0); - for (size_t i = 0; i < graph.n_nodes(); ++i) { - CATCH_REQUIRE( - graph.get_node_degree(static_cast(i)) <= params.graph_max_degree - ); - } - return graph; - }; - - auto graph_unrouted = run_build(nullptr); - auto graph_routed = run_build(router_ptr); - - // Recall check: routed build's top-1 search must not get worse than the unrouted one. - auto count_top1_hits = [&](const svs::graphs::SimpleGraph& graph) { - QEmd oracle_distance{table}; - size_t hits = 0; - auto search_buffer = v::SearchBuffer{64}; - for (uint32_t q = 0; q < 20; ++q) { - search_buffer.clear(); - auto query = docs.get_datum(q); - auto oracle = brute_force_qemd(oracle_distance, query, docs); - - QEmd search_distance{table}; - auto accessor = svs::data::GetDatumAccessor{}; - v::greedy_search( - graph, - docs, - accessor, - query, - search_distance, - search_buffer, - v::EntryPointInitializer{ - std::span(&entry_point, 1)} - ); - search_buffer.sort(); - if (search_buffer.size() > 0 && search_buffer[0].id() == oracle[0].first) { - ++hits; - } - } - return hits; - }; - - size_t hits_unrouted = count_top1_hits(graph_unrouted); - size_t hits_routed = count_top1_hits(graph_routed); - CATCH_REQUIRE(hits_routed >= hits_unrouted); + svs_test::require_degree_bound(graph, params.graph_max_degree); + } } diff --git a/tests/svs/index/vamana/gem_routing.cpp b/tests/svs/index/vamana/gem_routing.cpp index 1791ce755..623ffcb48 100644 --- a/tests/svs/index/vamana/gem_routing.cpp +++ b/tests/svs/index/vamana/gem_routing.cpp @@ -15,6 +15,7 @@ */ // Headers under test: GemRoutedInitializer, search_initializer, gem_pooled_entry_point. +// No-router / all-unrepresented fallback is covered by gem_build.cpp's build_entry_points. #include "svs/core/distance/gem.h" #include "svs/core/distance/gem_router.h" #include "svs/extensions/vamana/gem.h" @@ -27,7 +28,7 @@ #include "svs/lib/threads.h" // tests -#include "tests/utils/utils.h" +#include "tests/svs/index/vamana/gem_test_utils.h" // catch2 #include "catch2/catch_test_macros.hpp" @@ -37,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -45,63 +45,16 @@ namespace { -using svs::distance::CentroidTable; -using svs::distance::FineCentroids; using svs::distance::GemRouter; -using svs::distance::Ground; using svs::distance::QChamfer; using svs::distance::QChamferExactQuery; - -constexpr size_t K = 16; -constexpr size_t DIM = 4; - -// K unit vectors, deterministic for a given seed; dim == DIM. -std::vector make_centroids(uint32_t seed) { - std::mt19937 rng(seed); - std::uniform_real_distribution coord(-1.0F, 1.0F); - std::vector centroids(K * DIM); - for (size_t k = 0; k < K; ++k) { - double norm = 0.0; - for (size_t d = 0; d < DIM; ++d) { - float v = coord(rng); - centroids[k * DIM + d] = v; - norm += static_cast(v) * static_cast(v); - } - norm = std::sqrt(norm); - for (size_t d = 0; d < DIM; ++d) { - centroids[k * DIM + d] = static_cast(centroids[k * DIM + d] / norm); - } - } - return centroids; -} - -struct IdSetCorpus { - std::vector ids; - std::vector offsets; - - size_t size() const { return offsets.size() - 1; } - - svs::data::ConstVectorSetView view() const { - return svs::data::ConstVectorSetView( - std::span(ids), std::span(offsets), 1 - ); - } -}; - -IdSetCorpus make_doc_corpus(size_t n, uint32_t seed) { - std::mt19937 rng(seed); - std::uniform_int_distribution id_dist(0, static_cast(K - 1)); - IdSetCorpus corpus; - corpus.offsets.resize(n + 1, 0); - for (size_t i = 0; i < n; ++i) { - size_t cardinality = 2 + (i % 5); - corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; - for (size_t t = 0; t < cardinality; ++t) { - corpus.ids.push_back(id_dist(rng)); - } - } - return corpus; -} +using svs_test::gem::DIM; +using svs_test::gem::K; +using svs_test::gem::make_centroid_table; +using svs_test::gem::make_centroids; +using svs_test::gem::make_doc_corpus; +using svs_test::gem::make_fine_centroids; +using svs_test::gem::round_robin_coarse_of_fine; // Raw-float query tokens, one per centroid dimension: what QChamferExactQuery expects. struct QueryCorpus { @@ -137,25 +90,6 @@ make_query_corpus(const std::vector& centroids, size_t n, uint32_t seed) return corpus; } -std::shared_ptr make_table(const std::vector& centroids) { - return std::make_shared( - CentroidTable::from_centroids(centroids.data(), K, DIM, Ground::Cos, 1) - ); -} - -std::shared_ptr make_fine(const std::vector& centroids) { - return std::make_shared(K, DIM, centroids.data(), 1); -} - -// Deterministic grouping (f % num_coarse); good enough to exercise routing, not realistic. -std::vector round_robin_coarse_of_fine(size_t num_coarse) { - std::vector out(K); - for (size_t f = 0; f < K; ++f) { - out[f] = static_cast(f % num_coarse); - } - return out; -} - // Minimal greedy-search test double: records pushed ids, nothing else. struct RecordingBuffer { using index_type = uint32_t; @@ -176,85 +110,12 @@ auto id_builder() { } // namespace -CATCH_TEST_CASE( - "GemRoutedInitializer with no router matches EntryPointInitializer", - "[vamana][gem][gem_router]" -) { - auto centroids = make_centroids(0x1111); - auto fine = make_fine(centroids); - QChamferExactQuery distance{fine}; - - auto queries = make_query_corpus(centroids, 1, 0x2222); - auto query = queries.view().get_datum(0); - distance.fix_argument(query); - - auto graph = svs::default_graph(10, 32); - std::vector entry_points{3}; - auto computer = dummy_computer(); - auto builder = id_builder(); - svs::index::vamana::NullTracker tracker; - - RecordingBuffer buf_plain; - svs::index::vamana::EntryPointInitializer plain_init{ - std::span(entry_points)}; - plain_init(buf_plain, computer, graph, builder, tracker); - - RecordingBuffer buf_routed; - svs::index::vamana::GemRoutedInitializer routed_init{ - &distance, std::span(entry_points)}; - routed_init(buf_routed, computer, graph, builder, tracker); - - CATCH_REQUIRE(buf_routed.ids == buf_plain.ids); -} - -CATCH_TEST_CASE( - "GemRoutedInitializer with every cluster unrepresented falls back to the medoid", - "[vamana][gem][gem_router]" -) { - auto centroids = make_centroids(0x3333); - auto fine = make_fine(centroids); - - GemRouter all_norep; - all_norep.coarse_of_fine = round_robin_coarse_of_fine(4); - all_norep.representative.assign(4, GemRouter::no_representative); - all_norep.nprobe = 2; - auto router_ptr = std::make_shared(all_norep); - - QChamferExactQuery distance_no_router{fine}; - QChamferExactQuery distance_all_norep{fine, router_ptr}; - - auto queries = make_query_corpus(centroids, 1, 0x4444); - auto query = queries.view().get_datum(0); - distance_no_router.fix_argument(query); - distance_all_norep.fix_argument(query); - - auto graph = svs::default_graph(10, 32); - std::vector entry_points{5}; - auto computer = dummy_computer(); - auto builder = id_builder(); - svs::index::vamana::NullTracker tracker; - - RecordingBuffer buf_no_router; - svs::index::vamana::GemRoutedInitializer{ - &distance_no_router, std::span(entry_points)}( - buf_no_router, computer, graph, builder, tracker - ); - - RecordingBuffer buf_all_norep; - svs::index::vamana::GemRoutedInitializer{ - &distance_all_norep, std::span(entry_points)}( - buf_all_norep, computer, graph, builder, tracker - ); - - CATCH_REQUIRE(buf_all_norep.ids == buf_no_router.ids); -} - CATCH_TEST_CASE( "GemRoutedInitializer with a real router pushes exactly medoid union routed", "[vamana][gem][gem_router]" ) { auto centroids = make_centroids(0x5555); - auto fine = make_fine(centroids); + auto fine = make_fine_centroids(centroids); auto doc_corpus = make_doc_corpus(100, 0x6666); auto docs = doc_corpus.view(); @@ -298,8 +159,8 @@ CATCH_TEST_CASE( "Routed search is not worse than unrouted on a small index", "[vamana][gem][gem_router]" ) { auto centroids = make_centroids(0x8888); - auto table = make_table(centroids); - auto fine = make_fine(centroids); + auto table = make_centroid_table(centroids); + auto fine = make_fine_centroids(centroids); const size_t n_docs = 150; auto doc_corpus = make_doc_corpus(n_docs, 0x9999); @@ -349,20 +210,10 @@ CATCH_TEST_CASE( auto query = query_view.get_datum(q); index.search(query, scratch); - // Brute-force QChamferExactQuery oracle for this query. QChamferExactQuery oracle_dist{fine}; oracle_dist.fix_argument(query); - uint32_t best_id = 0; - float best_score = std::numeric_limits::infinity(); - for (uint32_t id = 0; id < docs.size(); ++id) { - float score = - svs::distance::compute(oracle_dist, query, docs.get_datum(id)); - if (score < best_score) { - best_score = score; - best_id = id; - } - } - if (scratch.buffer[0].id() == best_id) { + auto oracle = svs_test::brute_force_top_k(oracle_dist, query, docs); + if (scratch.buffer[0].id() == oracle[0].first) { ++top1_hits; } } @@ -375,13 +226,13 @@ CATCH_TEST_CASE( CATCH_REQUIRE(hits_routed >= hits_unrouted); } -CATCH_TEST_CASE("gem_pooled_entry_point", "[vamana][gem][gem_router]") { - // Four unit vectors in 2D, matching core/distances/gem.cpp's hand example exactly. +CATCH_TEST_CASE( + "gem_pooled_entry_point picks doc0 as the medoid", "[vamana][gem][gem_router]" +) { constexpr size_t dim = 2; std::vector flat{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F, 0.0F, -1.0F}; - FineCentroids fine(4, dim, flat.data(), 1); + svs::distance::FineCentroids fine(4, dim, flat.data(), 1); - // doc0's mean (0.5,0.5) is nearest the dataset mean (1/3,0), so doc0 is the medoid. std::vector ids{0, 1, 2, 3, 0}; std::vector offsets{0, 2, 4, 5}; auto docs = svs::data::ConstVectorSetView( diff --git a/tests/svs/index/vamana/gem_test_utils.h b/tests/svs/index/vamana/gem_test_utils.h new file mode 100644 index 000000000..cb59b6d7c --- /dev/null +++ b/tests/svs/index/vamana/gem_test_utils.h @@ -0,0 +1,150 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Shared fixtures and oracles for the vamana test directory's GEM, MaxSim and +// connectivity tests: a brute-force top-k oracle, a graph degree-bound check, and (in +// svs_test::gem) a unit-norm centroid table plus a deterministic multi-token id-corpus. + +// svs +#include "svs/core/data/vector_set_view.h" +#include "svs/core/distance.h" +#include "svs/core/distance/gem.h" +#include "svs/core/distance/gem_router.h" +#include "svs/core/graph.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs_test { + +// Every document's score against `query` under `dist`, sorted best-first by +// `Dist::compare`. `dist` must already carry any argument-fixing the distance needs. +template +std::vector> +brute_force_top_k(Dist& dist, const Query& query, const DocsView& docs) { + std::vector> scored; + scored.reserve(docs.size()); + for (uint32_t id = 0; id < docs.size(); ++id) { + scored.emplace_back(id, svs::distance::compute(dist, query, docs.get_datum(id))); + } + std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { + return typename Dist::compare{}(a.second, b.second); + }); + return scored; +} + +template +void require_degree_bound(const svs::graphs::SimpleGraph& graph, size_t max_degree) { + for (size_t i = 0; i < graph.n_nodes(); ++i) { + CATCH_REQUIRE(graph.get_node_degree(static_cast(i)) <= max_degree); + } +} + +} // namespace svs_test + +namespace svs_test::gem { + +inline constexpr size_t K = 16; +inline constexpr size_t DIM = 4; + +// K unit vectors in R^DIM, deterministic for a given seed. +inline std::vector make_centroids(uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution coord(-1.0F, 1.0F); + std::vector centroids(K * DIM); + for (size_t k = 0; k < K; ++k) { + double norm = 0.0; + for (size_t d = 0; d < DIM; ++d) { + float v = coord(rng); + centroids[k * DIM + d] = v; + norm += static_cast(v) * static_cast(v); + } + norm = std::sqrt(norm); + for (size_t d = 0; d < DIM; ++d) { + centroids[k * DIM + d] = static_cast(centroids[k * DIM + d] / norm); + } + } + return centroids; +} + +inline std::shared_ptr +make_centroid_table(const std::vector& centroids) { + return std::make_shared( + svs::distance::CentroidTable::from_centroids( + centroids.data(), K, DIM, svs::distance::Ground::Cos, 1 + ) + ); +} + +inline std::shared_ptr +make_fine_centroids(const std::vector& centroids) { + return std::make_shared( + K, DIM, centroids.data(), 1 + ); +} + +// A CSR-style corpus of id-token vector sets: cardinality cycles 1..6, ids uniform over K. +struct IdSetCorpus { + std::vector ids; + std::vector offsets; + + size_t size() const { return offsets.size() - 1; } + + svs::data::ConstVectorSetView view() const { + return svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); + } +}; + +inline IdSetCorpus make_doc_corpus(size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution id_dist(0, static_cast(K - 1)); + IdSetCorpus corpus; + corpus.offsets.resize(n + 1, 0); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 1 + (i % 6); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + for (size_t t = 0; t < cardinality; ++t) { + corpus.ids.push_back(id_dist(rng)); + } + } + return corpus; +} + +// Fine centroid f goes to coarse cluster f % num_coarse; deterministic, not realistic. +inline std::vector round_robin_coarse_of_fine(size_t num_coarse) { + std::vector out(K); + for (size_t f = 0; f < K; ++f) { + out[f] = static_cast(f % num_coarse); + } + return out; +} + +} // namespace svs_test::gem diff --git a/tests/svs/index/vamana/gem_vector_set.cpp b/tests/svs/index/vamana/gem_vector_set.cpp index 54d99d103..354ce9935 100644 --- a/tests/svs/index/vamana/gem_vector_set.cpp +++ b/tests/svs/index/vamana/gem_vector_set.cpp @@ -14,9 +14,8 @@ * limitations under the License. */ -// Proves a VamanaIndex can be built and searched directly over a -// ConstVectorSetView of centroid-id vector sets, using QChamfer as both the -// build and search distance. +// Proves a VamanaIndex builds, searches, and saves/reloads over an owning +// VectorSetData of centroid-id vector sets, using QChamfer throughout. #include "svs/core/distance/gem.h" #include "svs/extensions/vamana/gem.h" @@ -28,6 +27,7 @@ #include "svs/lib/threads.h" // tests +#include "tests/svs/index/vamana/gem_test_utils.h" #include "tests/utils/utils.h" // catch2 @@ -36,125 +36,35 @@ // stl #include -#include #include #include -#include #include #include -#include -#include #include #include namespace { -constexpr size_t K = 16; -constexpr size_t DIM = 4; - -std::shared_ptr make_centroid_table(uint32_t seed) { - std::mt19937 rng(seed); - std::uniform_real_distribution coord(-1.0F, 1.0F); - std::vector centroids(K * DIM); - for (size_t k = 0; k < K; ++k) { - double norm = 0.0; - for (size_t d = 0; d < DIM; ++d) { - float v = coord(rng); - centroids[k * DIM + d] = v; - norm += static_cast(v) * static_cast(v); - } - norm = std::sqrt(norm); - for (size_t d = 0; d < DIM; ++d) { - centroids[k * DIM + d] = static_cast(centroids[k * DIM + d] / norm); - } - } - return std::make_shared( - svs::distance::CentroidTable::from_centroids( - centroids.data(), K, DIM, svs::distance::Ground::Cos, 1 - ) - ); -} - -// A deterministic dataset of id vector sets, held CSR-style exactly as -// ConstVectorSetView expects: cardinality cycles 1..6, ids are uniform over -// the K centroids. -struct IdSetCorpus { - std::vector ids; - std::vector offsets; - - size_t size() const { return offsets.size() - 1; } - - svs::data::ConstVectorSetView view() const { - return svs::data::ConstVectorSetView( - std::span(ids), std::span(offsets), 1 - ); - } -}; - -IdSetCorpus make_corpus(size_t n, uint32_t seed) { - std::mt19937 rng(seed); - std::uniform_int_distribution id_dist(0, static_cast(K - 1)); - - IdSetCorpus corpus; - corpus.offsets.resize(n + 1, 0); - for (size_t i = 0; i < n; ++i) { - size_t cardinality = 1 + (i % 6); - corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; - for (size_t t = 0; t < cardinality; ++t) { - corpus.ids.push_back(id_dist(rng)); - } - } - return corpus; -} +using svs_test::gem::make_centroid_table; +using svs_test::gem::make_centroids; +using svs_test::gem::make_doc_corpus; svs::index::vamana::VamanaSearchParameters search_params(size_t window, size_t capacity) { return svs::index::vamana::VamanaSearchParameters{{window, capacity}, false, 0, 0}; } -// Brute-force oracle: every document's QChamfer score against `query`, sorted best-first -// (ascending, since QChamfer::compare is std::less<>). -std::vector> brute_force_qchamfer( - svs::distance::QChamfer& dist, - const svs::data::ConstVectorSet& query, - const svs::data::ConstVectorSetView& docs +// Builds a graph+index over `docs` through VamanaBuilder directly (not the +// VamanaIndex build-parameters constructor, which hard-codes L2/IP/Cosine and throws for +// qCH, same as qEMD). +template +auto build_index( + Docs docs, + svs::distance::QChamfer dist, + const svs::index::vamana::VamanaBuildParameters& build_parameters, + uint32_t entry_point ) { - std::vector> scored; - scored.reserve(docs.size()); - for (uint32_t id = 0; id < docs.size(); ++id) { - float score = svs::distance::compute(dist, query, docs.get_datum(id)); - scored.emplace_back(id, score); - } - std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { - return svs::distance::QChamfer::compare{}(a.second, b.second); - }); - return scored; -} - -} // namespace - -CATCH_TEST_CASE("Vamana build and search over a vector-set dataset", "[vamana][gem]") { - auto table = make_centroid_table(0x5EED); - svs::distance::QChamfer dist{table}; - - const size_t n_docs = 200; - auto corpus = make_corpus(n_docs, 0xDEADBEEF); - auto queries = make_corpus(10, 0xC0FFEE); - auto docs = corpus.view(); - auto query_view = queries.view(); - - auto build_parameters = - svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; - - // Entry point 0 is caller-supplied; the vector-set path adds no medoid logic of its - // own. - const uint32_t entry_point = 0; - - // VamanaIndex's build-parameters constructor hard-codes L2/IP/Cosine in - // verify_and_set_default_index_parameters and throws for any other distance, so - // qCH (like qEMD) has to go through VamanaBuilder directly, mirroring how - // VamanaIndex's own build constructor drives it (index.h's build-parameters - // constructor) minus that distance-type check. - auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); + auto graph = + svs::default_graph(docs.size(), build_parameters.graph_max_degree); auto build_pool = svs::threads::DefaultThreadPool(1); auto builder = svs::index::vamana::VamanaBuilder( graph, @@ -167,9 +77,31 @@ CATCH_TEST_CASE("Vamana build and search over a vector-set dataset", "[vamana][g builder.construct(build_parameters.alpha, entry_point); auto pool = svs::threads::DefaultThreadPool(1); - auto index = svs::index::vamana::VamanaIndex( - std::move(graph), docs, entry_point, dist, std::move(pool) + return svs::index::vamana::VamanaIndex( + std::move(graph), std::move(docs), entry_point, dist, std::move(pool) ); +} + +} // namespace + +CATCH_TEST_CASE( + "Vamana build, search and save/reload over a vector-set dataset", "[vamana][gem]" +) { + auto table = make_centroid_table(make_centroids(0x5EED)); + svs::distance::QChamfer dist{table}; + + const size_t n_docs = 200; + auto corpus = make_doc_corpus(n_docs, 0xDEADBEEF); + auto queries = make_doc_corpus(10, 0xC0FFEE); + auto query_view = queries.view(); + auto owning_docs = svs::data::VectorSetData::copy_from(corpus.view()); + + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + const uint32_t entry_point = 0; + + auto index = build_index(std::move(owning_docs), dist, build_parameters, entry_point); + const auto& docs = index.view_data(); CATCH_SECTION("Small-window search returns a best-first, internally consistent pool") { const size_t capacity = 20; @@ -205,12 +137,10 @@ CATCH_TEST_CASE("Vamana build and search over a vector-set dataset", "[vamana][g for (size_t q = 0; q < query_view.size(); ++q) { auto query = query_view.get_datum(q); index.search(query, scratch); + auto oracle = svs_test::brute_force_top_k(dist, query, docs); - auto oracle = brute_force_qchamfer(dist, query, docs); - - // Centroid ids repeat across documents, so several documents can legitimately - // tie for the best score; check the distance exactly and the id against every - // document tied for that best score, rather than oracle[0].first alone. + // Centroid ids repeat, so several documents can tie for the best score; check + // the id against every document tied for it, not just oracle[0].first. const auto& top1 = scratch.buffer[0]; CATCH_REQUIRE( static_cast(top1.distance()) == @@ -224,90 +154,49 @@ CATCH_TEST_CASE("Vamana build and search over a vector-set dataset", "[vamana][g CATCH_REQUIRE(id_ties_for_best); } } -} - -// Saves a VamanaIndex built over an owning VectorSetData and reassembles it -// through `svs::index::vamana::auto_assemble` with `svs::VectorSetDataLoader`, the small -// loader type that mirrors `svs::VectorDataLoader`. `auto_assemble` applies the saved -// config via `VamanaIndex::apply` rather than `verify_and_set_default_index_parameters` -// (index.h), so it never runs the L2/IP/Cosine distance-type check that forces the -// build path above through `VamanaBuilder` directly: QChamfer hits no obstacle here. -CATCH_TEST_CASE( - "Vamana save and reload over an owning vector-set dataset", "[vamana][gem]" -) { - auto table = make_centroid_table(0xABCD); - svs::distance::QChamfer dist{table}; - - const size_t n_docs = 50; - auto corpus = make_corpus(n_docs, 0x1234); - auto queries = make_corpus(5, 0x5678); - auto query_view = queries.view(); - - // Owning takeover of the borrowed corpus, exactly the `copy_from` path a caller - // without its own ownable storage would use. - auto owning_sets = svs::data::VectorSetData::copy_from(corpus.view()); - auto build_parameters = - svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; - const uint32_t entry_point = 0; - - auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); - auto build_pool = svs::threads::DefaultThreadPool(1); - auto builder = svs::index::vamana::VamanaBuilder( - graph, - owning_sets, - dist, - build_parameters, - build_pool, - svs::index::vamana::extensions::estimate_prefetch_parameters(owning_sets) - ); - builder.construct(build_parameters.alpha, entry_point); - - auto pool = svs::threads::DefaultThreadPool(1); - auto index = svs::index::vamana::VamanaIndex( - std::move(graph), std::move(owning_sets), entry_point, dist, std::move(pool) - ); - - // The lower constructor above (required to dodge the L2/IP/Cosine check, as noted on - // the test case above) leaves `build_parameters_` at its default-constructed, - // placeholder-sentinel state; saving those sentinels overflows - // `VamanaBuildParameters::save`'s narrowing cast to a signed TOML integer. `apply` - // only assigns the fields below (it does not run the distance-type check), so it is - // the way to give the index real values to save. - index.apply(svs::index::vamana::VamanaIndexParameters{ - entry_point, build_parameters, index.get_search_parameters()}); - - auto temp_directory = svs_test::prepare_temp_directory_v2(); - auto config_dir = temp_directory / "config"; - auto graph_dir = temp_directory / "graph"; - auto data_dir = temp_directory / "data"; - index.save(config_dir, graph_dir, data_dir); - - auto reload_pool = svs::threads::DefaultThreadPool(1); - auto reloaded = svs::index::vamana::auto_assemble( - config_dir, - svs::GraphLoader(graph_dir), - svs::VectorSetDataLoader(data_dir), - dist, - std::move(reload_pool) - ); + CATCH_SECTION("Save and reload preserve every query's result") { + // `apply` gives the index real VamanaBuildParameters (build_index's lower + // VamanaBuilder path left them at sentinels that overflow TOML save) without + // re-running the L2/IP/Cosine distance-type check. + index.apply(svs::index::vamana::VamanaIndexParameters{ + entry_point, build_parameters, index.get_search_parameters()}); + + auto temp_directory = svs_test::prepare_temp_directory_v2(); + auto config_dir = temp_directory / "config"; + auto graph_dir = temp_directory / "graph"; + auto data_dir = temp_directory / "data"; + index.save(config_dir, graph_dir, data_dir); + + auto reload_pool = svs::threads::DefaultThreadPool(1); + auto reloaded = svs::index::vamana::auto_assemble( + config_dir, + svs::GraphLoader(graph_dir), + svs::VectorSetDataLoader(data_dir), + dist, + std::move(reload_pool) + ); - const size_t capacity = n_docs; - auto sp = search_params(capacity, capacity); - auto scratch_before = index.scratchspace(sp); - auto scratch_after = reloaded.scratchspace(sp); + const size_t capacity = n_docs; + auto sp = search_params(capacity, capacity); + auto scratch_before = index.scratchspace(sp); + auto scratch_after = reloaded.scratchspace(sp); - for (size_t q = 0; q < query_view.size(); ++q) { - auto query = query_view.get_datum(q); - index.search(query, scratch_before); - reloaded.search(query, scratch_after); + for (size_t q = 0; q < query_view.size(); ++q) { + auto query = query_view.get_datum(q); + index.search(query, scratch_before); + reloaded.search(query, scratch_after); - CATCH_REQUIRE(scratch_before.buffer.size() == scratch_after.buffer.size()); - for (size_t j = 0; j < scratch_before.buffer.size(); ++j) { - CATCH_REQUIRE(scratch_before.buffer[j].id() == scratch_after.buffer[j].id()); - CATCH_REQUIRE( - scratch_before.buffer[j].distance() == scratch_after.buffer[j].distance() - ); + CATCH_REQUIRE(scratch_before.buffer.size() == scratch_after.buffer.size()); + for (size_t j = 0; j < scratch_before.buffer.size(); ++j) { + CATCH_REQUIRE( + scratch_before.buffer[j].id() == scratch_after.buffer[j].id() + ); + CATCH_REQUIRE( + scratch_before.buffer[j].distance() == + scratch_after.buffer[j].distance() + ); + } } } } diff --git a/tests/svs/index/vamana/maxsim_rerank.cpp b/tests/svs/index/vamana/maxsim_rerank.cpp index 0e5c4cc94..d6bb9c8c7 100644 --- a/tests/svs/index/vamana/maxsim_rerank.cpp +++ b/tests/svs/index/vamana/maxsim_rerank.cpp @@ -33,6 +33,7 @@ #include "svs/lib/threads.h" // tests +#include "tests/svs/index/vamana/gem_test_utils.h" #include "tests/utils/utils.h" // catch2 @@ -50,22 +51,18 @@ #include #include #include -#include #include #include -#include #include #include namespace { constexpr size_t DIM = 8; -constexpr double MAXSIM_RERANK_EPSILON = 1e-4; +constexpr double EPS = 1e-4; -// Primary distance for the index's graph: mean of the query's tokens dotted with the -// primary row, negated. `compare = std::less<>` is deliberately the opposite direction -// from MaxSim's `std::greater<>`, to exercise the re-rank extension's comparator-flip -// handling. +// `compare = std::less<>` is the opposite direction from MaxSim's `std::greater<>`, to +// exercise the re-rank extension's comparator-flip handling. struct MeanNegativeInnerProduct { using compare = std::less<>; static constexpr bool implicit_broadcast = true; @@ -89,8 +86,8 @@ struct MeanNegativeInnerProduct { return static_cast(-dot); } - // Build compares two primary rows directly (vamana_build.h's candidate-pool and - // reverse-edge loops), not a query set against a row, so this overload serves build. + // Build compares two primary rows directly, not a query set against a row, so this + // overload serves vamana_build.h's candidate-pool and reverse-edge loops. float compute(std::span a, std::span b) const { double dot = 0.0; for (size_t d = 0; d < a.size(); ++d) { @@ -102,8 +99,8 @@ struct MeanNegativeInnerProduct { } // namespace -// MeanNegativeInnerProduct is IP-flavored (no triangle inequality), so pruning needs the -// same strategy vamana_build.h picks for DistanceIP, not the default L2 one. +// IP-flavored (no triangle inequality), so pruning needs IterativePruneStrategy like +// DistanceIP, not vamana_build.h's default L2 one. namespace svs::index::vamana { template <> struct PruneStrategy<::MeanNegativeInnerProduct> { using type = IterativePruneStrategy; @@ -112,6 +109,8 @@ template <> struct PruneStrategy<::MeanNegativeInnerProduct> { namespace { +using Dist = MeanNegativeInnerProduct; + void normalize(std::span token) { double norm = 0.0; for (float v : token) { @@ -124,8 +123,8 @@ void normalize(std::span token) { } // A deterministic corpus of vector sets: cardinality cycles 1..6, tokens are unit-norm -// (MaxSim assumes unit-norm tokens; see its doc comment), and the primary row for a -// document is the mean of its tokens. +// (MaxSim assumes unit-norm tokens), and a document's primary row is the mean of its +// tokens. struct Corpus { std::vector tokens; std::vector offsets; @@ -176,31 +175,47 @@ svs::index::vamana::VamanaSearchParameters search_params(size_t window, size_t c return svs::index::vamana::VamanaSearchParameters{{window, capacity}, false, 0, 0}; } -// Brute-force oracle: every document's MaxSim score against `query`, sorted best-first. -template -std::vector> -brute_force_maxsim(const Query& query, const svs::data::ConstVectorSetView& docs) { - std::vector> scored; - scored.reserve(docs.size()); - // MaxSim now packs the query once (must_fix_argument) and reuses that packing for - // every document, instead of the old stateless compute() called fresh each time. - svs::distance::MaxSim<> maxsim{}; - svs::distance::maybe_fix_argument(maxsim, query); - for (uint32_t id = 0; id < docs.size(); ++id) { - float score = svs::distance::compute(maxsim, query, docs.get_datum(id)); - scored.emplace_back(id, score); - } - std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { - return svs::distance::MaxSim<>::compare{}(a.second, b.second); - }); - return scored; +uint32_t local_entry_point(const svs::data::SimpleData& primary) { + auto pool = svs::threads::DefaultThreadPool(1); + return svs::lib::narrow( + svs::index::vamana::extensions::compute_entry_point(primary, pool) + ); +} + +// Builds a graph and VamanaIndex over `data` through VamanaBuilder directly (not the +// VamanaIndex build-parameters constructor, which hard-codes L2/IP/Cosine and throws for +// this test-local distance), with the two-phase construct VamanaIndex's own constructor +// runs internally (alpha pinned to 1.0, then the real alpha). +template +auto build_maxsim_index( + Data data, + const svs::index::vamana::VamanaBuildParameters& build_parameters, + uint32_t entry_point +) { + auto graph = + svs::default_graph(data.size(), build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + graph, + data, + Dist{}, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(data) + ); + builder.construct(1.0F, entry_point); + builder.construct(build_parameters.alpha, entry_point); + + auto pool = svs::threads::DefaultThreadPool(1); + return svs::index::vamana::VamanaIndex( + std::move(graph), std::move(data), entry_point, Dist{}, std::move(pool) + ); } -// Overwrites docs 198/199 (cardinality 1 and 2 respectively, already assigned by -// make_corpus's `1 + i % 6` pattern) and query 1 (cardinality 2) so that doc 199's second -// token wins against the query's second token, giving it the better MaxSim score, while -// its primary mean-dot score is worse than doc 198's: a deterministic witness that the -// MaxSim and primary orders of a pool can disagree, independent of the random seed. +// Overwrites docs 198/199 (cardinality 1 and 2) and query 1 (cardinality 2) so doc 199's +// second token wins against the query's second token (better MaxSim score) while its +// primary mean-dot score is worse than doc 198's: a MaxSim/primary-order inversion +// witness, independent of the random seed. void make_inversion_witness(Corpus& corpus, Corpus& query_corpus) { constexpr size_t DOC_A = 198; constexpr size_t DOC_B = 199; @@ -233,43 +248,48 @@ void make_inversion_witness(Corpus& corpus, Corpus& query_corpus) { set_token(query_corpus, QUERY, 1, q2); } -// Checks (a)-(d) of the MaxSim rerank contract against the whole candidate pool a -// single-query search leaves in `scratch`: every entry's score matches the MaxSim oracle, -// the pool is sorted best-first by that score, the first `num_neighbors` entries are the -// brute-force MaxSim top-k restricted to the pool's own ids, and the pool is strictly -// larger than `num_neighbors` so the re-rank had more than one candidate to order. -template +// Checks the MaxSim rerank contract against the whole candidate pool a +// single-query search leaves in `scratch`: every score matches the oracle, the pool is +// sorted best-first, and its first `num_neighbors` entries are the brute-force top-k +// restricted to the pool's own ids. +template < + typename ElemT, + svs::data::VectorSet Query, + typename Index, + typename Scratch, + typename DocsView> void check_rerank_pool( Index& index, Scratch& scratch, const Query& query, - const svs::data::ConstVectorSetView& doc_sets, + const DocsView& doc_sets, size_t num_neighbors ) { index.search(query, scratch); const auto& buffer = scratch.buffer; + svs::distance::MaxSim maxsim{}; + svs::distance::maybe_fix_argument(maxsim, query); + std::vector pool_ids; pool_ids.reserve(buffer.size()); - svs::distance::MaxSim<> maxsim{}; - svs::distance::maybe_fix_argument(maxsim, query); float previous_score = std::numeric_limits::infinity(); for (const auto& neighbor : buffer) { float expected = svs::distance::compute(maxsim, query, doc_sets.get_datum(neighbor.id())); CATCH_REQUIRE( - static_cast(neighbor.distance()) == Catch::Approx(expected) - .epsilon(MAXSIM_RERANK_EPSILON) - .margin(MAXSIM_RERANK_EPSILON) + static_cast(neighbor.distance()) == + Catch::Approx(expected).epsilon(EPS).margin(EPS) ); CATCH_REQUIRE(neighbor.distance() <= previous_score); previous_score = neighbor.distance(); pool_ids.push_back(neighbor.id()); } + // Pool strictly larger than num_neighbors: the re-rank had more than one candidate. CATCH_REQUIRE(pool_ids.size() > num_neighbors); std::set pool_set(pool_ids.begin(), pool_ids.end()); - auto oracle = brute_force_maxsim(query, doc_sets); + auto oracle = svs_test::brute_force_top_k(maxsim, query, doc_sets); std::vector restricted_oracle; for (const auto& scored : oracle) { if (pool_set.contains(scored.first)) { @@ -284,24 +304,70 @@ void check_rerank_pool( } } +// `Data` has no `allocator_type` of its own to build a `VectorDataLoader`-style loader +// around, so a `svs::lib::Lazy` callable reloads it whole instead. +template +void check_save_reload_round_trip( + Index& index, + const svs::index::vamana::VamanaBuildParameters& build_parameters, + uint32_t entry_point, + const svs::data::ConstVectorSetView& queries +) { + using Data = typename Index::data_type; + index.apply(svs::index::vamana::VamanaIndexParameters{ + entry_point, build_parameters, index.get_search_parameters()}); + + auto temp_directory = svs_test::prepare_temp_directory_v2(); + auto config_dir = temp_directory / "config"; + auto graph_dir = temp_directory / "graph"; + auto data_dir = temp_directory / "data"; + index.save(config_dir, graph_dir, data_dir); + + auto reload_pool = svs::threads::DefaultThreadPool(1); + auto reloaded = svs::index::vamana::auto_assemble( + config_dir, + svs::GraphLoader(graph_dir), + SVS_LAZY(svs::lib::load_from_disk(data_dir)), + Dist{}, + std::move(reload_pool) + ); + + const size_t capacity = index.size(); + auto sp = search_params(capacity, capacity); + auto scratch_before = index.scratchspace(sp); + auto scratch_after = reloaded.scratchspace(sp); + + for (size_t q = 0; q < queries.size(); ++q) { + auto query = queries.get_datum(q); + index.search(query, scratch_before); + reloaded.search(query, scratch_after); + + CATCH_REQUIRE(scratch_before.buffer.size() == scratch_after.buffer.size()); + for (size_t j = 0; j < scratch_before.buffer.size(); ++j) { + CATCH_REQUIRE(scratch_before.buffer[j].id() == scratch_after.buffer[j].id()); + CATCH_REQUIRE( + scratch_before.buffer[j].distance() == scratch_after.buffer[j].distance() + ); + } + } +} + } // namespace -CATCH_TEST_CASE("WithVectorSets construction", "[vamana][maxsim_rerank]") { +// Builds one owning WithVectorSets index and runs every behavior below as a section on +// it (Catch re-runs this setup per leaf section). +CATCH_TEST_CASE("MaxSim re-rank Vamana extension", "[vamana][maxsim_rerank]") { + using Data = svs::data:: + WithVectorSets, svs::data::VectorSetData>; + CATCH_SECTION("Mismatched primary/sets sizes throw") { auto primary_corpus = make_corpus(5, 1); auto sets_corpus = make_corpus(4, 2); - using Data = svs::data::WithVectorSets< - svs::data::SimpleData, - svs::data::ConstVectorSetView>; - CATCH_REQUIRE_THROWS(Data(primary_corpus.primary, sets_corpus.vector_sets())); + CATCH_REQUIRE_THROWS(Data( + primary_corpus.primary, + svs::data::VectorSetData::copy_from(sets_corpus.vector_sets()) + )); } -} - -CATCH_TEST_CASE("MaxSim re-rank Vamana extension", "[vamana][maxsim_rerank]") { - using Dist = MeanNegativeInnerProduct; - using Primary = svs::data::SimpleData; - using Sets = svs::data::ConstVectorSetView; - using Data = svs::data::WithVectorSets; const size_t n_docs = 200; auto corpus = make_corpus(n_docs, 0xDEADBEEF); @@ -310,46 +376,16 @@ CATCH_TEST_CASE("MaxSim re-rank Vamana extension", "[vamana][maxsim_rerank]") { auto build_parameters = svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + auto entry_point = local_entry_point(corpus.primary); - auto entry_point_pool = svs::threads::DefaultThreadPool(1); - auto entry_point = - svs::lib::narrow(svs::index::vamana::extensions::compute_entry_point( - corpus.primary, entry_point_pool - )); - - // The only index under test: primary dataset paired with the vector-set dataset, - // re-ranked by MaxSim through the extension in svs/extensions/vamana/maxsim.h. - // - // VamanaIndex's build-parameters constructor hard-codes L2/IP/Cosine in - // verify_and_set_default_index_parameters and throws "Unsupported distance type" for - // any other distance (include/svs/index/vamana/index.h), so the test-local distance - // has to go through VamanaBuilder directly, mirroring what that constructor does - // internally (index.h's two-phase `construct` calls) minus the distance-type check. - auto full_graph = - svs::default_graph(n_docs, build_parameters.graph_max_degree); - auto full_data = Data(corpus.primary, corpus.vector_sets()); - auto build_pool = svs::threads::DefaultThreadPool(1); - auto builder = svs::index::vamana::VamanaBuilder( - full_graph, - full_data, - Dist{}, + auto full_index = build_maxsim_index( + Data( + corpus.primary, svs::data::VectorSetData::copy_from(corpus.vector_sets()) + ), build_parameters, - build_pool, - svs::index::vamana::extensions::estimate_prefetch_parameters(full_data) - ); - builder.construct(1.0F, entry_point); - builder.construct(build_parameters.alpha, entry_point); - - auto full_pool = svs::threads::DefaultThreadPool(1); - auto full_index = svs::index::vamana::VamanaIndex( - std::move(full_graph), - std::move(full_data), - entry_point, - Dist{}, - std::move(full_pool) + entry_point ); - - auto doc_sets = corpus.vector_sets(); + const auto& doc_sets = full_index.view_data().vector_sets(); auto queries = query_corpus.vector_sets(); CATCH_SECTION("Query token dimension mismatch throws before the graph search") { @@ -367,22 +403,19 @@ CATCH_TEST_CASE("MaxSim re-rank Vamana extension", "[vamana][maxsim_rerank]") { CATCH_REQUIRE_THROWS(full_index.search(bad_query, full_scratch)); } - // Checks (a)-(d) of the rerank contract on the whole pool a single-query search - // leaves behind, at a buffer capacity (40) well under the corpus size and at a - // capacity (n_docs) that leaves every document in the pool. - for (size_t capacity : {size_t{40}, n_docs}) { - CATCH_SECTION( - "Candidate pool is scored and ordered by MaxSim, capacity " + - std::to_string(capacity) - ) { - const size_t num_neighbors = 10; - auto sp = search_params(capacity, capacity); - auto full_scratch = full_index.scratchspace(sp); - - for (size_t q = 0; q < queries.size(); ++q) { - auto query = queries.get_datum(q); - check_rerank_pool(full_index, full_scratch, query, doc_sets, num_neighbors); - } + // Capacity == n_docs leaves every document in the pool: the strongest check against + // the full-corpus oracle. A tighter window only re-exercises the generic search + // buffer's eviction path, which search_buffer.cpp already owns. + CATCH_SECTION("Candidate pool is scored and ordered by MaxSim") { + const size_t num_neighbors = 10; + auto sp = search_params(n_docs, n_docs); + auto full_scratch = full_index.scratchspace(sp); + + for (size_t q = 0; q < queries.size(); ++q) { + auto query = queries.get_datum(q); + check_rerank_pool( + full_index, full_scratch, query, doc_sets, num_neighbors + ); } } @@ -404,14 +437,12 @@ CATCH_TEST_CASE("MaxSim re-rank Vamana extension", "[vamana][maxsim_rerank]") { }); CATCH_REQUIRE(it_a != buffer.end()); CATCH_REQUIRE(it_b != buffer.end()); - // MaxSim ranks doc 199 ahead of doc 198 (its second token wins against the - // query's), but their primary mean-dot scores rank the opposite way. CATCH_REQUIRE( std::distance(buffer.begin(), it_b) < std::distance(buffer.begin(), it_a) ); // The generic `distance::compute` dispatch takes the functor by non-const - // reference, so a named lvalue is required here (a temporary will not bind). + // reference, so a named lvalue is required (a temporary will not bind). Dist dist{}; float primary_a = svs::distance::compute(dist, query, corpus.primary.get_datum(DOC_A)); @@ -450,105 +481,15 @@ CATCH_TEST_CASE("MaxSim re-rank Vamana extension", "[vamana][maxsim_rerank]") { CATCH_REQUIRE( static_cast(score) == Catch::Approx(single_query_results[q][k].distance()) - .epsilon(MAXSIM_RERANK_EPSILON) - .margin(MAXSIM_RERANK_EPSILON) + .epsilon(EPS) + .margin(EPS) ); } } } -} - -// Saves and reloads a VamanaIndex over a WithVectorSets of two owning members -// (SimpleData and VectorSetData, both saveable), checking the re-ranked results are -// identical before and after. Reassembly uses `svs::index::vamana::auto_assemble` with a -// `svs::lib::Lazy` callable as the data proto (`svs::detail::dispatch_load` accepts -// anything invocable with no arguments) rather than a new loader type: `WithVectorSets` -// has no `allocator_type` of its own to build a `VectorDataLoader`-style loader around, -// and the lazy callable is the smaller option for a type that is reloaded whole. -CATCH_TEST_CASE("WithVectorSets save and reload", "[vamana][maxsim_rerank]") { - using Dist = MeanNegativeInnerProduct; - using Primary = svs::data::SimpleData; - using Sets = svs::data::VectorSetData; - using Data = svs::data::WithVectorSets; - - const size_t n_docs = 60; - auto corpus = make_corpus(n_docs, 0xABCDEF); - auto query_corpus = make_corpus(5, 0xFEEDFACE); - - // Both members are owning: the primary row dataset is copied from the corpus (it - // already owns its storage); the vector-set dataset takes ownership of a copy of the - // corpus's borrowed view through `copy_from`. - auto data = Data(corpus.primary, Sets::copy_from(corpus.vector_sets())); - auto build_parameters = - svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; - - auto entry_point_pool = svs::threads::DefaultThreadPool(1); - auto entry_point = - svs::lib::narrow(svs::index::vamana::extensions::compute_entry_point( - corpus.primary, entry_point_pool - )); - - auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); - auto build_pool = svs::threads::DefaultThreadPool(1); - auto builder = svs::index::vamana::VamanaBuilder( - graph, - data, - Dist{}, - build_parameters, - build_pool, - svs::index::vamana::extensions::estimate_prefetch_parameters(data) - ); - builder.construct(1.0F, entry_point); - builder.construct(build_parameters.alpha, entry_point); - - auto pool = svs::threads::DefaultThreadPool(1); - auto index = svs::index::vamana::VamanaIndex( - std::move(graph), std::move(data), entry_point, Dist{}, std::move(pool) - ); - - // The lower constructor above leaves `build_parameters_` at its default-constructed - // placeholder-sentinel state (it never ran verify_and_set_default_index_parameters, - // which is the point of using it with a non-L2/IP/Cosine distance); saving those - // sentinels overflows `VamanaBuildParameters::save`'s narrowing cast to a signed TOML - // integer. `apply` only assigns fields, without the distance-type check, so give the - // index real values before saving it. - index.apply(svs::index::vamana::VamanaIndexParameters{ - entry_point, build_parameters, index.get_search_parameters()}); - - auto temp_directory = svs_test::prepare_temp_directory_v2(); - auto config_dir = temp_directory / "config"; - auto graph_dir = temp_directory / "graph"; - auto data_dir = temp_directory / "data"; - index.save(config_dir, graph_dir, data_dir); - - auto reload_pool = svs::threads::DefaultThreadPool(1); - auto reloaded = svs::index::vamana::auto_assemble( - config_dir, - svs::GraphLoader(graph_dir), - SVS_LAZY(svs::lib::load_from_disk(data_dir)), - Dist{}, - std::move(reload_pool) - ); - - auto queries = query_corpus.vector_sets(); - const size_t capacity = n_docs; - auto sp = search_params(capacity, capacity); - auto scratch_before = index.scratchspace(sp); - auto scratch_after = reloaded.scratchspace(sp); - - for (size_t q = 0; q < queries.size(); ++q) { - auto query = queries.get_datum(q); - index.search(query, scratch_before); - reloaded.search(query, scratch_after); - - CATCH_REQUIRE(scratch_before.buffer.size() == scratch_after.buffer.size()); - for (size_t j = 0; j < scratch_before.buffer.size(); ++j) { - CATCH_REQUIRE(scratch_before.buffer[j].id() == scratch_after.buffer[j].id()); - CATCH_REQUIRE( - scratch_before.buffer[j].distance() == scratch_after.buffer[j].distance() - ); - } + CATCH_SECTION("Save and reload preserve every query's result") { + check_save_reload_round_trip(full_index, build_parameters, entry_point, queries); } } @@ -570,73 +511,27 @@ svs::data::VectorSetData round_tokens_to_bf16(const Corpus& corpu return svs::data::VectorSetData(std::move(tokens), corpus.offsets); } -// Brute-force oracle over the bfloat16 store itself (not the float corpus the store was -// rounded from): mirrors `brute_force_maxsim` but through `MaxSim`, so it -// scores exactly what the owning bf16 store holds. -template -std::vector> brute_force_maxsim_bf16( - const Query& query, const svs::data::VectorSetData& docs -) { - std::vector> scored; - scored.reserve(docs.size()); - svs::distance::MaxSim maxsim{}; - svs::distance::maybe_fix_argument(maxsim, query); - for (uint32_t id = 0; id < docs.size(); ++id) { - float score = svs::distance::compute(maxsim, query, docs.get_datum(id)); - scored.emplace_back(id, score); - } - std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { - return svs::distance::MaxSim<>::compare{}(a.second, b.second); - }); - return scored; -} - } // namespace -// A document store whose element type is svs::BFloat16: the re-rank extension must pick -// MaxSim for it (svs/extensions/vamana/maxsim.h selects on -// `Sets::const_value_type::element_type`), reaching the bf16 (and AMX, where available) -// InnerProductMatrix::column_max overload rather than the float one. +// A BFloat16 element type makes the re-rank extension pick MaxSim +// (svs/extensions/vamana/maxsim.h selects on Sets::const_value_type::element_type), +// reaching the bf16 InnerProductMatrix::column_max overload rather than the float one. CATCH_TEST_CASE( "MaxSim re-rank over an owning bfloat16 vector-set store", "[vamana][maxsim_rerank]" ) { - using Dist = MeanNegativeInnerProduct; - using Primary = svs::data::SimpleData; - using Sets = svs::data::VectorSetData; - using Data = svs::data::WithVectorSets; + using Data = svs::data::WithVectorSets< + svs::data::SimpleData, + svs::data::VectorSetData>; const size_t n_docs = 80; auto corpus = make_corpus(n_docs, 0x5EED5EED); auto query_corpus = make_corpus(5, 0xFACE5EED); - auto data = Data(corpus.primary, round_tokens_to_bf16(corpus)); auto build_parameters = svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; - - auto entry_point_pool = svs::threads::DefaultThreadPool(1); - auto entry_point = - svs::lib::narrow(svs::index::vamana::extensions::compute_entry_point( - corpus.primary, entry_point_pool - )); - - auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); - auto build_pool = svs::threads::DefaultThreadPool(1); - auto builder = svs::index::vamana::VamanaBuilder( - graph, - data, - Dist{}, - build_parameters, - build_pool, - svs::index::vamana::extensions::estimate_prefetch_parameters(data) - ); - builder.construct(1.0F, entry_point); - builder.construct(build_parameters.alpha, entry_point); - - auto pool = svs::threads::DefaultThreadPool(1); - auto index = svs::index::vamana::VamanaIndex( - std::move(graph), std::move(data), entry_point, Dist{}, std::move(pool) - ); + auto entry_point = local_entry_point(corpus.primary); + auto index = build_maxsim_index(std::move(data), build_parameters, entry_point); auto queries = query_corpus.vector_sets(); const size_t capacity = n_docs; @@ -646,42 +541,8 @@ CATCH_TEST_CASE( for (size_t q = 0; q < queries.size(); ++q) { auto query = queries.get_datum(q); - index.search(query, scratch); - const auto& buffer = scratch.buffer; - - svs::distance::MaxSim maxsim{}; - svs::distance::maybe_fix_argument(maxsim, query); - float previous_score = std::numeric_limits::infinity(); - std::vector pool_ids; - pool_ids.reserve(buffer.size()); - for (const auto& neighbor : buffer) { - float expected = svs::distance::compute( - maxsim, query, index.view_data().get_vector_set(neighbor.id()) - ); - CATCH_REQUIRE( - static_cast(neighbor.distance()) == - Catch::Approx(expected) - .epsilon(MAXSIM_RERANK_EPSILON) - .margin(MAXSIM_RERANK_EPSILON) - ); - CATCH_REQUIRE(neighbor.distance() <= previous_score); - previous_score = neighbor.distance(); - pool_ids.push_back(neighbor.id()); - } - - std::set pool_set(pool_ids.begin(), pool_ids.end()); - auto oracle = brute_force_maxsim_bf16(query, index.view_data().vector_sets()); - std::vector restricted_oracle; - for (const auto& scored : oracle) { - if (pool_set.contains(scored.first)) { - restricted_oracle.push_back(scored.first); - if (restricted_oracle.size() == num_neighbors) { - break; - } - } - } - for (size_t k = 0; k < num_neighbors; ++k) { - CATCH_REQUIRE(buffer[k].id() == restricted_oracle[k]); - } + check_rerank_pool( + index, scratch, query, index.view_data().vector_sets(), num_neighbors + ); } } From b65458d73c6fb953b1b6afbaf6e52803ad80eb43 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 07:16:43 -0800 Subject: [PATCH 45/68] feat(distance): solve qEMD with an in-house transportation simplex Replace QEmdImpl's vendored-POT solve with a clean-room primal network simplex (include/svs/core/distance/transport_simplex.h): an artificial root node with big-M arcs gives a strongly feasible initial tree (Ahuja, Magnanti & Orlin, "Network Flows", 1993, ch. 11), each pivot's leaving arc is the *last* blocking arc seen walking the cycle from its apex in the entering arc's own direction (Cunningham, "A network simplex method", 1976), which keeps the tree strongly feasible and so guarantees termination independent of how the entering arc is chosen. That lets the entering rule optimize for speed instead: a block-cyclic search over the dense cost matrix (block ~sqrt(ua*ub) arcs, rounded to whole padded rows), picking the most negative reduced cost in the first block that has one and resuming from where the previous search left off, with an AVX-512/AVX2/scalar-templated row scan. No code is reused or paraphrased from POT/LEMON; the algorithm is textbook network simplex, so there is nothing to attribute beyond the existing citation of Bonneel et al.'s EMD problem shape already in the new header's top comment. An earlier version of this solver used northwest-corner initialization and Bland's rule; both are replaced here because northwest-corner is not automatically strongly feasible and Bland's rule (lowest-index entering arc) is too slow on this hot path's highly degenerate, asymmetric problem shapes (up to ~180x32 and 180x180), where it tends to stall rescanning the same low-index rows. QEmdImpl::compute_decoded keeps its signature, the ua==ub==1 shortcut, and throw-on-failure behavior; gem_node_id_type and the POT include/pragma/undef block are gone since nothing else used them. Also fixes a stale comment in inner_product_matrix.h that described amx.cpp's build gate as -march=sapphirerapids support when it is actually -mamx-tile/-mamx-bf16 support at -march=cascadelake. Adds a random cross-check test (tests/svs/core/distances/gem.cpp): the NONE path against an independent successive-shortest-paths reference, and every compiled ISA path against NONE, over random, degenerate (equal-cost, single-node-side, wide weight range), and highly degenerate (uniform weights, tiny integer cost range, up to 64x64) problems -- the family that stalls a naive entering rule. Co-Authored-By: Claude Opus 5.5 --- include/svs/core/distance/gem.h | 73 +-- .../svs/core/distance/inner_product_matrix.h | 12 +- include/svs/core/distance/transport_simplex.h | 599 ++++++++++++++++++ tests/svs/core/distances/gem.cpp | 229 +++++++ 4 files changed, 848 insertions(+), 65 deletions(-) create mode 100644 include/svs/core/distance/transport_simplex.h diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index 1ee29fa66..65ec6741c 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -16,25 +16,11 @@ #pragma once -// Vendored POT 0.9.5 network simplex solver (include/svs/third-party/pot/); not -// warning-clean under -Werror, so only the warnings it actually trips are silenced. -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wreorder" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include "svs/third-party/pot/network_simplex_simple.h" -#pragma GCC diagnostic pop - -// network_simplex_simple.h leaves these #defined with no #undef; they mangle identifiers. -#undef EPSILON -#undef _EPSILON -#undef MAX_DEBUG_ITER - #include "svs/concepts/vector_set.h" #include "svs/core/distance/distance_core.h" #include "svs/core/distance/gem_router.h" #include "svs/core/distance/inner_product_matrix.h" +#include "svs/core/distance/transport_simplex.h" #include "svs/lib/avx_detection.h" #include "svs/lib/bfloat16.h" #include "svs/lib/misc.h" @@ -62,9 +48,6 @@ namespace svs::distance { -// Same node-index type POT's EMD_wrap instantiates the solver with (ot/lp/EMD.h). -using gem_node_id_type = unsigned int; - /// Ground distance between two unit centroids the from_centroids() table is built from: /// Cos is `max(0, 1 - )` (GEM Eq. 11), L2 is `sqrt(max(0, 2 - 2))`. enum class Ground { Cos, L2 }; @@ -391,9 +374,10 @@ inline void require_id_dimensions(size_t dims) { } // namespace gem_detail // Instantiated per ISA in svs/multi-arch/x86; without SVS_NOINLINE the caller inlines the -// body at baseline flags. The bodies are scalar, so only compiler auto-vectorization -// differs. compute_decoded takes already-decoded (deduplicated, weighted) centroid ids, -// keeping the merge and the transportation solve as separately testable steps. +// body at baseline flags. compute_decoded's pricing loop is explicitly vectorized per Avx +// (TransportSimplex); compute_decoded takes already-decoded (deduplicated, weighted) +// centroid ids, keeping the merge and the transportation solve as separately testable +// steps. template struct QEmdImpl { SVS_NOINLINE static float compute_decoded( const CentroidTable& table, @@ -408,45 +392,14 @@ template struct QEmdImpl { return table(ids_a[0], ids_b[0]); } - // Mirrors POT's EMD_wrap (ot/lp/EMD_wrapper.cpp); all weights are already nonzero. - using Digraph = lemon::FullBipartiteDigraph; - Digraph di(static_cast(ua), static_cast(ub)); - lemon::NetworkSimplexSimple net( - di, - true, - static_cast(ua + ub), - static_cast(ua) * static_cast(ub), - std::numeric_limits::max() - ); - - thread_local std::vector neg_wb; - neg_wb.resize(ub); - for (size_t j = 0; j < ub; ++j) { - neg_wb[j] = -w_b[j]; - } - net.supplyMap(w_a, static_cast(ua), neg_wb.data(), static_cast(ub)); - - int64_t idarc = 0; - for (size_t i = 0; i < ua; ++i) { - for (size_t j = 0; j < ub; ++j) { - net.setCost(di.arcFromId(idarc), table(ids_a[i], ids_b[j])); - ++idarc; - } - } - - if (net.run() != decltype(net)::OPTIMAL) { - throw std::runtime_error( - "QEmd: network simplex did not reach an optimal solution" - ); - } - - double cost = 0.0; - Digraph::Arc arc; - for (di.first(arc); arc != INVALID; di.next(arc)) { - auto src = static_cast(di.source(arc)); - auto dst = static_cast(di.target(arc)) - ua; - cost += net.flow(arc) * table(ids_a[src], ids_b[dst]); - } + // merge_sorted_duplicates divides integer counts by n in double, so w_a/w_b each + // sum to 1.0 up to a few ULPs rather than exactly; TransportSimplex::solve folds + // that residual into the last demand entry (see its doc comment) instead of + // requiring bit-exact balance here. + double cost = + TransportSimplex::solve(ua, w_a, ub, w_b, [&](size_t i, size_t j) { + return table(ids_a[i], ids_b[j]); + }); return static_cast(cost); } diff --git a/include/svs/core/distance/inner_product_matrix.h b/include/svs/core/distance/inner_product_matrix.h index 5cdbf8ecb..7acfb1e41 100644 --- a/include/svs/core/distance/inner_product_matrix.h +++ b/include/svs/core/distance/inner_product_matrix.h @@ -1348,11 +1348,13 @@ template struct InnerProductMatrixImpl; extern template struct InnerProductMatrixImpl; #endif -// Gated on the build-wide SVS_AMX_COMPILED_IN, not __x86_64__ alone: when the compiler -// lacks -march=sapphirerapids, amx.cpp is never built (cmake/multi-arch.cmake), so no TU -// anywhere provides `template struct InnerProductMatrixImpl;`'s -// definition; an unconditional `extern template` here would then be an unresolved symbol -// at link time the moment anything calls InnerProductMatrix, not just when AMX runs. +// Gated on the build-wide SVS_AMX_COMPILED_IN, not __x86_64__ alone: amx.cpp is only built +// when the compiler accepts -mamx-tile -mamx-bf16 (cmake/multi-arch.cmake's +// SVS_COMPILER_SUPPORTS_AMX check), compiled at -march=cascadelake plus those two flags, +// not at -march=sapphirerapids; when that check fails, no TU anywhere provides +// `template struct InnerProductMatrixImpl;`'s definition, and an +// unconditional `extern template` here would then be an unresolved symbol at link time +// the moment anything calls InnerProductMatrix, not just when AMX runs. #if SVS_AMX_COMPILED_IN extern template struct InnerProductMatrixImpl; #endif diff --git a/include/svs/core/distance/transport_simplex.h b/include/svs/core/distance/transport_simplex.h new file mode 100644 index 000000000..a9d7ed756 --- /dev/null +++ b/include/svs/core/distance/transport_simplex.h @@ -0,0 +1,599 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Clean-room primal network simplex for the dense balanced transportation problem. +// Algorithm: Ahuja, Magnanti & Orlin, "Network Flows" (1993), ch. 11 (artificial-root +// initialization and strongly feasible spanning trees); Cunningham, "A network simplex +// method", Mathematical Programming 11 (1976), for the leaving-arc rule that keeps a +// tree strongly feasible under degenerate pivots. Problem shape (quantized ground +// distance over merged centroid weights) follows Bonneel et al., "Displacement +// Interpolation Using Lagrangian Mass Transport", SIGGRAPH Asia 2011. + +#include "svs/core/distance/distance_core.h" +#include "svs/lib/preprocessor.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__x86_64__) +#include +#endif + +namespace svs::distance::transport_detail { + +// Pricing scans a row in blocks of this many doubles: 8 covers one AVX-512 register or +// two AVX2 registers, so a row padded to a multiple of it needs no scalar tail on either +// ISA. +inline constexpr size_t kPriceBlock = 8; + +inline size_t pad_to_block(size_t n) { + return (n + kPriceBlock - 1) / kPriceBlock * kPriceBlock; +} + +// Finds the most negative reduced cost in a cost row and its column, for the block-search +// entering rule in TransportSimplex::solve. The vector subtract (the O(ub) part) is +// SIMD-widened per Avx; the min-with-index reduction per 8-or-4-wide chunk is left scalar +// (store the chunk, scan it) since it is only O(ub/width) work and a lot easier to get +// right by inspection than a vectorized horizontal-min-with-index. The generic template +// also serves AVX_AVAILABILITY::NONE and any non-x86 build. +template struct PriceRow { + // `v` and `cost_row` must both have `ub_padded` valid entries; padding columns (index + // >= the caller's real ub) must hold +infinity in cost_row so they never win the min + // regardless of what garbage sits in the matching v slot. + static void min_in_row( + const double* cost_row, + const double* v, + size_t ub_padded, + double u_i, + double& best_val, + size_t& best_col + ) { + for (size_t j = 0; j < ub_padded; ++j) { + double r = cost_row[j] - u_i - v[j]; + if (r < best_val) { + best_val = r; + best_col = j; + } + } + } +}; + +// Guarded per ISA: these bodies are not templates, so every TU that sees one compiles its +// intrinsics, and a TU built without that ISA fails with a target-option mismatch. +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +#if SVS_AVX512_F +template <> struct PriceRow { + static void min_in_row( + const double* cost_row, + const double* v, + size_t ub_padded, + double u_i, + double& best_val, + size_t& best_col + ) { + __m512d u_vec = _mm512_set1_pd(u_i); + double buf[8]; + for (size_t j = 0; j < ub_padded; j += 8) { + __m512d r = _mm512_sub_pd( + _mm512_sub_pd(_mm512_loadu_pd(cost_row + j), u_vec), _mm512_loadu_pd(v + j) + ); + _mm512_storeu_pd(buf, r); + for (size_t lane = 0; lane < 8; ++lane) { + if (buf[lane] < best_val) { + best_val = buf[lane]; + best_col = j + lane; + } + } + } + } +}; +#endif + +SVS_VALIDATE_BOOL_ENV(SVS_AVX2) +#if SVS_AVX2 +template <> struct PriceRow { + static void min_in_row( + const double* cost_row, + const double* v, + size_t ub_padded, + double u_i, + double& best_val, + size_t& best_col + ) { + __m256d u_vec = _mm256_set1_pd(u_i); + double buf[4]; + for (size_t j = 0; j < ub_padded; j += 4) { + __m256d r = _mm256_sub_pd( + _mm256_sub_pd(_mm256_loadu_pd(cost_row + j), u_vec), _mm256_loadu_pd(v + j) + ); + _mm256_storeu_pd(buf, r); + for (size_t lane = 0; lane < 4; ++lane) { + if (buf[lane] < best_val) { + best_val = buf[lane]; + best_col = j + lane; + } + } + } + } +}; +#endif + +// Scratch for one thread's in-flight solve, reused across calls and grow-only: a bigger +// (ua, ub) resizes these up, a smaller one just uses a prefix, so steady-state solves +// (same CentroidTable, same decoded token counts) allocate nothing. Sized for ua+ub+1 +// nodes: sources [0, ua), sinks [ua, ua+ub), and one artificial root node at ua+ub. +struct Workspace { + std::vector cost; // ua * pad_to_block(ub), row-major + std::vector supply; + std::vector demand; + std::vector parent; // size ua+ub+1; -1 at the root, -2 = unvisited (scratch) + std::vector depth; // size ua+ub+1 + std::vector flow_to_parent; // size ua+ub+1; flow on the arc to parent[v] + std::vector u; // source potentials, size ua + std::vector v; // sink potentials, size pad_to_block(ub) + std::vector adj_head; // CSR adjacency over ua+ub+1 nodes, rebuilt every pivot + std::vector adj_next; + std::vector adj_to; + std::vector adj_cost; // the physical arc's cost (M for an artificial arc) + std::vector adj_flow; + std::vector bfs_order; + std::vector path_p; + std::vector path_q; + + void reserve(size_t ua, size_t ub) { + size_t n = ua + ub + 1; + size_t padded_ub = pad_to_block(ub); + if (cost.size() < ua * padded_ub) { + cost.resize(ua * padded_ub); + } + if (supply.size() < ua) { + supply.resize(ua); + } + if (demand.size() < ub) { + demand.resize(ub); + } + if (parent.size() < n) { + parent.resize(n); + } + if (depth.size() < n) { + depth.resize(n); + } + if (flow_to_parent.size() < n) { + flow_to_parent.resize(n); + } + if (u.size() < ua) { + u.resize(ua); + } + if (v.size() < padded_ub) { + v.resize(padded_ub); + } + if (adj_head.size() < n) { + adj_head.resize(n); + } + size_t edges = (n > 0) ? n - 1 : 0; + if (adj_next.size() < 2 * edges) { + adj_next.resize(2 * edges); + adj_to.resize(2 * edges); + adj_cost.resize(2 * edges); + adj_flow.resize(2 * edges); + } + if (bfs_order.size() < n) { + bfs_order.resize(n); + } + if (path_p.size() < n) { + path_p.resize(n); + } + if (path_q.size() < n) { + path_q.resize(n); + } + } +}; + +// One directed adjacency slot: node `to`, reached from `from` over an arc with physical +// cost `cost` (the real cost(i, j) for a transportation arc, or M for an artificial one) +// and current flow `flow`. +inline void add_adj( + Workspace& ws, size_t edge_slot, int64_t from, int64_t to, double cost, double flow +) { + ws.adj_to[edge_slot] = to; + ws.adj_cost[edge_slot] = cost; + ws.adj_flow[edge_slot] = flow; + ws.adj_next[edge_slot] = ws.adj_head[static_cast(from)]; + ws.adj_head[static_cast(from)] = edge_slot; +} + +// Builds the artificial-root star: one extra node R = ua+ub, an arc (source i, R) with +// flow supply[i] for every source and (R, sink j) with flow demand[j] for every sink, all +// at cost M. Every arc starts at strictly positive flow (ws.supply/ws.demand are qEMD's +// merged weights, never zero), so this tree is strongly feasible by construction: the +// strongly-feasible condition only constrains zero-flow tree arcs, and there are none +// (Ahuja-Magnanti-Orlin ch. 11.2). Northwest-corner is not used here because it is not +// automatically strongly feasible. +inline void artificial_root_adjacency(Workspace& ws, size_t ua, size_t ub, double big_m) { + size_t root = ua + ub; + std::fill( + ws.adj_head.begin(), + ws.adj_head.begin() + static_cast(root + 1), + SIZE_MAX + ); + size_t slot = 0; + for (size_t i = 0; i < ua; ++i) { + add_adj( + ws, + slot, + static_cast(i), + static_cast(root), + big_m, + ws.supply[i] + ); + add_adj( + ws, + slot + 1, + static_cast(root), + static_cast(i), + big_m, + ws.supply[i] + ); + slot += 2; + } + for (size_t j = 0; j < ub; ++j) { + add_adj( + ws, + slot, + static_cast(root), + static_cast(ua + j), + big_m, + ws.demand[j] + ); + add_adj( + ws, + slot + 1, + static_cast(ua + j), + static_cast(root), + big_m, + ws.demand[j] + ); + slot += 2; + } +} + +// Breadth-first search from the artificial root R = ua+ub (always the root: R is +// permanently part of the spanning tree, see solve()'s doc comment) over the adjacency +// already built in ws.adj_*, filling parent/depth/flow_to_parent and the dual potentials +// u/v. A source is always the tail of its arc to its tree parent, whether that arc is a +// real one (to a sink) or artificial (to R); a sink is always the head. That fixes the +// sign in the potential update below without per-arc direction bookkeeping. Every tree +// arc has zero reduced cost by construction, which is what lets solve()'s pricing scan +// treat "negative reduced cost" as "not yet basic" with no separate basis-membership +// check. +inline void rebuild_tree(Workspace& ws, size_t ua, size_t ub) { + size_t root = ua + ub; + size_t n = root + 1; + std::fill(ws.parent.begin(), ws.parent.begin() + static_cast(n), -2); + ws.parent[root] = -1; + ws.depth[root] = 0; + ws.bfs_order[0] = static_cast(root); + size_t order_end = 1; + for (size_t qi = 0; qi < order_end; ++qi) { + int64_t node = ws.bfs_order[qi]; + double node_y = + (node == static_cast(root)) + ? 0.0 + : (node < static_cast(ua) ? ws.u[static_cast(node)] + : -ws.v[static_cast(node) - ua]); + for (size_t e = ws.adj_head[static_cast(node)]; e != SIZE_MAX; + e = ws.adj_next[e]) { + int64_t to = ws.adj_to[e]; + if (ws.parent[static_cast(to)] != -2) { + continue; + } + ws.parent[static_cast(to)] = node; + ws.depth[static_cast(to)] = ws.depth[static_cast(node)] + 1; + ws.flow_to_parent[static_cast(to)] = ws.adj_flow[e]; + double cost = ws.adj_cost[e]; + if (to < static_cast(ua)) { + ws.u[static_cast(to)] = + node_y + cost; // source: tail, y = y(parent) + cost + } else { + ws.v[static_cast(to) - ua] = cost - node_y; // sink: head + } + ws.bfs_order[order_end++] = to; + } + } +} + +} // namespace svs::distance::transport_detail + +namespace svs::distance { + +/// Exact solver for the dense balanced transportation problem: `ua` sources with supply +/// `supply[i]`, `ub` sinks with demand `demand[j]`, a complete bipartite arc set, and a +/// per-arc cost from `cost(i, j)`. Minimizes `sum_ij flow_ij * cost(i, j)` subject to +/// `sum_j flow_ij == supply[i]`, `sum_i flow_ij == demand[j]`, `flow_ij >= 0`. +/// +/// Requires `ua >= 1`, `ub >= 1`, every supply/demand entry strictly positive (true for +/// qEMD's merged weights), and `sum(supply) == sum(demand)` to within `1e-6 * +/// max(sum(supply), 1.0)`; any smaller residual (the kind two independent floating-point +/// sums that are mathematically equal can differ by) is folded into the last demand entry +/// so the solver never over- or under-subscribes a side. A larger mismatch throws +/// std::runtime_error, since it signals mismatched inputs rather than rounding. +/// +/// Initializes with an artificial root node and big-M artificial arcs to every source and +/// sink (Ahuja-Magnanti-Orlin ch. 11.2), which is strongly feasible from the start since +/// every artificial arc carries strictly positive flow. Each pivot then picks the leaving +/// arc as the *last* blocking (flow-decreasing) arc encountered while walking the cycle +/// from its apex (the two endpoints' lowest common tree ancestor) in the entering arc's +/// own direction (Cunningham, 1976); maintaining a strongly feasible tree this way +/// guarantees termination regardless of which negative-reduced-cost arc enters next, so +/// the entering rule is free to optimize for speed: a block-cyclic search over the dense +/// cost matrix, picking the most negative reduced cost in the first block (of +/// `~sqrt(ua*ub)` arcs, rounded to whole padded rows) that has one, resuming from where +/// the previous search left off. `Avx` selects the SIMD width of each row's reduced-cost +/// scan; it does not change which solution is found. M is `(ua+ub) * max(cost) + 1`, the +/// largest an optimal basic solution's total real cost could be plus 1, so no finite +/// combination of real arcs is ever more expensive than keeping one unit of flow on an +/// artificial arc. +template struct TransportSimplex { + template + static double + solve(size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost) { + using namespace transport_detail; + thread_local Workspace ws; + ws.reserve(ua, ub); + size_t padded_ub = pad_to_block(ub); + size_t root = ua + ub; + + double total_s = 0.0, total_d = 0.0; + for (size_t i = 0; i < ua; ++i) { + ws.supply[i] = supply[i]; + total_s += supply[i]; + } + for (size_t j = 0; j < ub; ++j) { + ws.demand[j] = demand[j]; + total_d += demand[j]; + } + double tol = 1e-6 * std::max(total_s, 1.0); + double diff = total_s - total_d; + if (diff > tol || diff < -tol) { + throw std::runtime_error( + "TransportSimplex::solve: supply (" + std::to_string(total_s) + + ") and demand (" + std::to_string(total_d) + ") are not balanced for ua=" + + std::to_string(ua) + ", ub=" + std::to_string(ub) + ); + } + ws.demand[ub - 1] += diff; + + double max_cost = 0.0; + for (size_t i = 0; i < ua; ++i) { + double* row = &ws.cost[i * padded_ub]; + for (size_t j = 0; j < ub; ++j) { + double c = static_cast(cost(i, j)); + row[j] = c; + max_cost = std::max(max_cost, c); + } + for (size_t j = ub; j < padded_ub; ++j) { + row[j] = std::numeric_limits::infinity(); + } + } + for (size_t j = ub; j < padded_ub; ++j) { + ws.v[j] = 0.0; + } + // Bigger than any optimal basic solution's real cost could ever be, so the + // simplex never prefers routing flow through the artificial root once a real + // spanning tree is available; see the class doc comment. + double big_m = static_cast(ua + ub) * max_cost + 1.0; + + artificial_root_adjacency(ws, ua, ub, big_m); + rebuild_tree(ws, ua, ub); + + // Block size: ~sqrt(ua*ub) arcs, rounded to a whole number of rows so each block + // scan stays an integer number of SIMD-scannable rows. + size_t block_rows = static_cast(std::round( + std::sqrt(static_cast(ua) * static_cast(ub)) / + static_cast(ub) + )); + block_rows = std::clamp(block_rows, 1, ua); + size_t cursor_row = 0; + + // No polynomial bound exists for primal network simplex iteration count (even + // with a strongly feasible tree, only finiteness is guaranteed, not a bound), so + // this is a generous safety net against a latent bug, not an expected limit. + size_t max_iters = 1000 * (ua + ub) + 10000; + for (size_t iter = 0; iter < max_iters; ++iter) { + size_t best_i = ua, best_j = 0, rows_scanned = 0; + size_t row = cursor_row; + while (rows_scanned < ua) { + double block_best_val = 0.0; + size_t block_best_i = ua, block_best_j = 0; + for (size_t b = 0; b < block_rows && rows_scanned < ua; + ++b, ++rows_scanned) { + double val = 0.0; + size_t col = 0; + PriceRow::min_in_row( + &ws.cost[row * padded_ub], + ws.v.data(), + padded_ub, + ws.u[row], + val, + col + ); + if (val < block_best_val) { + block_best_val = val; + block_best_i = row; + block_best_j = col; + } + row = (row + 1) % ua; + } + if (block_best_i != ua) { + best_i = block_best_i; + best_j = block_best_j; + cursor_row = row; + break; + } + } + if (best_i == ua) { + double total_cost = 0.0; + for (size_t node = 0; node < root; ++node) { + if (static_cast(ws.parent[node]) == root) { + continue; // artificial arc: not part of the real transport cost + } + size_t ai, aj; + if (node < ua) { + ai = node; + aj = static_cast(ws.parent[node]) - ua; + } else { + ai = static_cast(ws.parent[node]); + aj = node - ua; + } + total_cost += ws.flow_to_parent[node] * ws.cost[ai * padded_ub + aj]; + } + return total_cost; + } + + int64_t p_node = static_cast(best_i); + int64_t q_node = static_cast(ua + best_j); + ws.path_p.clear(); + ws.path_p.push_back(p_node); + ws.path_q.clear(); + ws.path_q.push_back(q_node); + int64_t pu = p_node, qu = q_node; + while (ws.depth[static_cast(pu)] > ws.depth[static_cast(qu)]) { + pu = ws.parent[static_cast(pu)]; + ws.path_p.push_back(pu); + } + while (ws.depth[static_cast(qu)] > ws.depth[static_cast(pu)]) { + qu = ws.parent[static_cast(qu)]; + ws.path_q.push_back(qu); + } + while (pu != qu) { + pu = ws.parent[static_cast(pu)]; + ws.path_p.push_back(pu); + qu = ws.parent[static_cast(qu)]; + ws.path_q.push_back(qu); + } + size_t lp = ws.path_p.size(); + size_t lq = ws.path_q.size(); + + // A source is always the tail of its tree arc (to a sink or to the root), a + // sink always the head (see rebuild_tree's doc comment), so walking + // parent->child is "forward" (flow increases) exactly when the child is a + // sink, and walking child->parent is forward exactly when the child is a + // source. + auto is_forward_down = [&](int64_t child) { + return child >= static_cast(ua); + }; + auto is_forward_up = [&](int64_t child) { + return child < static_cast(ua); + }; + + // Leaving arc: walk the cycle apex (= LCA = the end of either path array) + // down to p, then from q back up to the apex -- the entering arc's own + // orientation -- and take the *last* blocking (backward) arc seen, using <= + // so a later tie overwrites an earlier one (Cunningham's rule; this is what + // keeps the tree strongly feasible, not the choice of entering arc). + double theta = std::numeric_limits::infinity(); + int64_t leave_node = -1; + for (size_t t = lp; t-- > 1;) { + int64_t child = ws.path_p[t - 1]; + if (!is_forward_down(child)) { + double flow = ws.flow_to_parent[static_cast(child)]; + if (flow <= theta) { + theta = flow; + leave_node = child; + } + } + } + for (size_t t = 0; t + 1 < lq; ++t) { + int64_t child = ws.path_q[t]; + if (!is_forward_up(child)) { + double flow = ws.flow_to_parent[static_cast(child)]; + if (flow <= theta) { + theta = flow; + leave_node = child; + } + } + } + + // Apply the flow delta to every tree edge on the cycle (the entering arc is + // not one of these; it is added fresh below with flow == theta). Order does + // not matter here, unlike the leaving-arc search above. + for (size_t t = 0; t + 1 < lp; ++t) { + int64_t child = ws.path_p[t]; + double delta = is_forward_down(child) ? theta : -theta; + ws.flow_to_parent[static_cast(child)] += delta; + } + for (size_t t = 0; t + 1 < lq; ++t) { + int64_t child = ws.path_q[t]; + double delta = is_forward_up(child) ? theta : -theta; + ws.flow_to_parent[static_cast(child)] += delta; + } + + // Rebuild adjacency from the old tree minus the leaving edge, plus the + // entering arc (p_node, q_node, theta); rebuilding by BFS instead of + // patching parent/depth in place avoids re-rooting a subtree incrementally, + // which costs O(ua+ub) here against O(ua*ub) for pricing, so it is not the + // bottleneck. + std::fill( + ws.adj_head.begin(), + ws.adj_head.begin() + static_cast(root + 1), + SIZE_MAX + ); + size_t slot = 0; + for (int64_t node = 0; node < static_cast(root); ++node) { + if (node == leave_node) { + continue; + } + int64_t parent_node = ws.parent[static_cast(node)]; + double edge_cost; + if (static_cast(parent_node) == root) { + edge_cost = big_m; + } else if (node < static_cast(ua)) { + edge_cost = ws.cost + [static_cast(node) * padded_ub + + (static_cast(parent_node) - ua)]; + } else { + edge_cost = ws.cost + [static_cast(parent_node) * padded_ub + + (static_cast(node) - ua)]; + } + double flow = ws.flow_to_parent[static_cast(node)]; + add_adj(ws, slot, node, parent_node, edge_cost, flow); + add_adj(ws, slot + 1, parent_node, node, edge_cost, flow); + slot += 2; + } + double entering_cost = ws.cost[best_i * padded_ub + best_j]; + add_adj(ws, slot, p_node, q_node, entering_cost, theta); + add_adj(ws, slot + 1, q_node, p_node, entering_cost, theta); + + rebuild_tree(ws, ua, ub); + } + throw std::runtime_error( + "TransportSimplex::solve: exceeded the iteration limit (" + + std::to_string(max_iters) + ") for ua=" + std::to_string(ua) + + ", ub=" + std::to_string(ub) + ); + } +}; + +} // namespace svs::distance diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp index 2c0df58ec..f7c7554e6 100644 --- a/tests/svs/core/distances/gem.cpp +++ b/tests/svs/core/distances/gem.cpp @@ -395,6 +395,235 @@ CATCH_TEST_CASE("QEmd", "[distance][gem]") { } } +namespace { + +// Independent reference for the balanced transportation problem: successive shortest +// augmenting paths (Ahuja-Magnanti-Orlin ch. 9) over the explicit source/sink residual +// graph, valid because every qEMD ground cost is nonnegative. Deliberately a different +// algorithm from TransportSimplex, not a second copy of it. +double reference_transport_cost( + size_t ua, + const std::vector& supply, + size_t ub, + const std::vector& demand, + const std::vector& cost // ua * ub, row-major +) { + constexpr double inf = std::numeric_limits::infinity(); + size_t s_node = ua + ub, t_node = ua + ub + 1, n = ua + ub + 2; + std::vector rem_s(supply), rem_d(demand), flow(ua * ub, 0.0); + double total = 0.0; + for (;;) { + std::vector dist(n, inf); + std::vector pred(n, -1); + std::vector kind(n, 0); // 0: S->src, 1: src->sink, 2: sink->src, 3: sink->T + dist[s_node] = 0.0; + for (size_t pass = 0; pass < n; ++pass) { + for (size_t i = 0; i < ua; ++i) { + if (rem_s[i] > 1e-12 && dist[s_node] < dist[i]) { + dist[i] = dist[s_node]; + pred[i] = static_cast(s_node); + kind[i] = 0; + } + } + for (size_t i = 0; i < ua; ++i) { + if (dist[i] == inf) { + continue; + } + for (size_t j = 0; j < ub; ++j) { + double nd = dist[i] + cost[i * ub + j]; + if (nd < dist[ua + j] - 1e-12) { + dist[ua + j] = nd; + pred[ua + j] = static_cast(i); + kind[ua + j] = 1; + } + } + } + for (size_t j = 0; j < ub; ++j) { + if (dist[ua + j] == inf) { + continue; + } + for (size_t i = 0; i < ua; ++i) { + if (flow[i * ub + j] > 1e-12) { + double nd = dist[ua + j] - cost[i * ub + j]; + if (nd < dist[i] - 1e-12) { + dist[i] = nd; + pred[i] = static_cast(ua + j); + kind[i] = 2; + } + } + } + if (rem_d[j] > 1e-12 && dist[ua + j] < dist[t_node] - 1e-12) { + dist[t_node] = dist[ua + j]; + pred[t_node] = static_cast(ua + j); + kind[t_node] = 3; + } + } + } + if (dist[t_node] == inf) { + break; + } + double bottleneck = inf; + for (int64_t cur = static_cast(t_node); + cur != static_cast(s_node);) { + int64_t p = pred[static_cast(cur)]; + if (kind[static_cast(cur)] == 0) { + bottleneck = std::min(bottleneck, rem_s[static_cast(cur)]); + } else if (kind[static_cast(cur)] == 2) { + bottleneck = std::min( + bottleneck, + flow[static_cast(cur) * ub + static_cast(p) - ua] + ); + } else if (kind[static_cast(cur)] == 3) { + bottleneck = std::min(bottleneck, rem_d[static_cast(p) - ua]); + } + cur = p; + } + for (int64_t cur = static_cast(t_node); + cur != static_cast(s_node);) { + int64_t p = pred[static_cast(cur)]; + size_t curu = static_cast(cur), pu = static_cast(p); + if (kind[curu] == 0) { + rem_s[curu] -= bottleneck; + } else if (kind[curu] == 1) { + flow[pu * ub + (curu - ua)] += bottleneck; + total += bottleneck * cost[pu * ub + (curu - ua)]; + } else if (kind[curu] == 2) { + flow[curu * ub + (pu - ua)] -= bottleneck; + total -= bottleneck * cost[curu * ub + (pu - ua)]; + } else { + rem_d[pu - ua] -= bottleneck; + } + cur = p; + } + } + return total; +} + +} // namespace + +CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][gem]") { + using svs::distance::AVX_AVAILABILITY; + using svs::distance::QEmdImpl; + + std::mt19937 rng(0x7EA5CADEu); + std::uniform_int_distribution size_dist(1, 40); + std::uniform_real_distribution cost_dist(0.0, 10.0); + + // Builds one random balanced problem, decoded ids_a/ids_b plus merged weights that + // already sum to 1 (matching QEmdImpl::compute_decoded's real inputs), and checks the + // NONE path against the independent reference and every compiled ISA path against NONE. + // `tiny_cost_range >= 0` draws integer costs from [0, tiny_cost_range] instead of a + // continuous spread -- the highly degenerate case (many tied reduced costs) that the + // block-search entering rule plus the strongly-feasible-tree leaving rule need to + // handle without stalling. `uniform_weights` skips random weights for exact 1/n ties. + auto run_case = [&](size_t ua, + size_t ub, + bool degenerate_costs, + double weight_spread, + bool uniform_weights = false, + int tiny_cost_range = -1) { + std::vector w_a(ua), w_b(ub); + if (uniform_weights) { + std::fill(w_a.begin(), w_a.end(), 1.0 / static_cast(ua)); + std::fill(w_b.begin(), w_b.end(), 1.0 / static_cast(ub)); + } else { + for (auto* w : {&w_a, &w_b}) { + double total = 0.0; + std::uniform_real_distribution wd(0.01, weight_spread); + for (auto& x : *w) { + x = wd(rng); + total += x; + } + for (auto& x : *w) { + x /= total; // Same normalization merge_sorted_duplicates relies on. + } + } + } + size_t k = std::max(ua, ub); + std::vector ids_a(ua), ids_b(ub); + std::vector cost(ua * ub); + std::vector table_flat(k * k, 0.0F); + std::uniform_int_distribution tiny_cost_dist(0, std::max(tiny_cost_range, 0)); + for (size_t i = 0; i < ua; ++i) { + ids_a[i] = static_cast(i); + for (size_t j = 0; j < ub; ++j) { + double c; + if (tiny_cost_range >= 0) { + c = static_cast(tiny_cost_dist(rng)); // few distinct costs + } else { + c = degenerate_costs ? 1.0 : cost_dist(rng); // many-equal-cost case + } + cost[i * ub + j] = c; + table_flat[i * k + j] = static_cast(c); + } + } + for (size_t j = 0; j < ub; ++j) { + ids_b[j] = static_cast(j); + } + CentroidTable table(k, table_flat); + + double reference = reference_transport_cost(ua, w_a, ub, w_b, cost); + float none_result = QEmdImpl::compute_decoded( + table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub + ); + CATCH_REQUIRE( + static_cast(none_result) == Catch::Approx(reference).margin(1e-6) + ); + +#if defined(__x86_64__) + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + float avx2_result = QEmdImpl::compute_decoded( + table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub + ); + CATCH_REQUIRE(avx2_result == Catch::Approx(none_result)); + } + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + float avx512_result = QEmdImpl::compute_decoded( + table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub + ); + CATCH_REQUIRE(avx512_result == Catch::Approx(none_result)); + } +#endif + }; + + CATCH_SECTION("Random problems") { + for (int trial = 0; trial < 250; ++trial) { + size_t ua = size_dist(rng); + size_t ub = size_dist(rng); + run_case(ua, ub, false, 1.0); + } + } + CATCH_SECTION("Many equal costs") { + for (int trial = 0; trial < 20; ++trial) { + run_case(size_dist(rng), size_dist(rng), true, 1.0); + } + } + CATCH_SECTION("Single node on one side") { + for (int trial = 0; trial < 10; ++trial) { + run_case(1, size_dist(rng), false, 1.0); + run_case(size_dist(rng), 1, false, 1.0); + } + } + CATCH_SECTION("Large weight dynamic range") { + for (int trial = 0; trial < 20; ++trial) { + run_case(size_dist(rng), size_dist(rng), false, 1e6); + } + } + CATCH_SECTION("Highly degenerate: uniform weights, tiny integer cost range") { + std::uniform_int_distribution big_size_dist(1, 64); + for (int trial = 0; trial < 30; ++trial) { + run_case( + big_size_dist(rng), + big_size_dist(rng), + false, + 1.0, + /*uniform_weights=*/true, + /*tiny_cost_range=*/2 + ); + } + } +} + CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { // Raw (non-unit) centroids: c0=(1,0), c1=(0,1), c2=(-1,0). std::vector flat{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F}; From 5dc64b2bd66884d8d30bca733074bf4c66df0bec Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 07:18:11 -0800 Subject: [PATCH 46/68] chore(third-party): drop the vendored POT network simplex qEMD now solves its own transportation simplex (see the prior commit), so include/svs/third-party/pot/ (POT/LEMON's network_simplex_simple.h, full_bipartitegraph.h, core.h) has no remaining caller. Drops the directory and the .pre-commit-config.yaml exclusion that kept it byte-identical to upstream. No code from POT or LEMON is reused anywhere in this repo after this change, so there is nothing left to attribute beyond what the new solver's header already cites. Co-Authored-By: Claude Opus 5.5 --- .pre-commit-config.yaml | 3 - include/svs/third-party/pot/README.md | 45 - include/svs/third-party/pot/core.h | 103 -- .../svs/third-party/pot/full_bipartitegraph.h | 212 --- .../third-party/pot/network_simplex_simple.h | 1552 ----------------- 5 files changed, 1915 deletions(-) delete mode 100644 include/svs/third-party/pot/README.md delete mode 100644 include/svs/third-party/pot/core.h delete mode 100644 include/svs/third-party/pot/full_bipartitegraph.h delete mode 100644 include/svs/third-party/pot/network_simplex_simple.h diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5886baccc..baf697990 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Vendored headers must stay byte-identical to upstream; include/svs/third-party/pot/README.md lists their hashes. -exclude: ^include/svs/third-party/pot/ - repos: - repo: https://github.com/pre-commit/mirrors-clang-format rev: "ea59a72ffc9a1ce6b79b02a8076d031aa7ea7805" # v15.0.7 diff --git a/include/svs/third-party/pot/README.md b/include/svs/third-party/pot/README.md deleted file mode 100644 index 30b30c8ee..000000000 --- a/include/svs/third-party/pot/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Vendored POT network simplex solver - -Source: https://github.com/PythonOT/POT, tag `0.9.5`, directory `ot/lp/`. - -Fetched unmodified from `https://raw.githubusercontent.com/PythonOT/POT/0.9.5/ot/lp/`. - -SHA-256: - -``` -4264bfe6760fcdf9b4c591e0e1fac9b2d983ccf6874dffb4d9e314868b197c93 network_simplex_simple.h -c16d242d385b9ad5c5b720d5a9b8526cb141b753f93eded9035368d0b04e66ec full_bipartitegraph.h -359e9b1d580c1eb7213bfd961d57ea70f6713d364534ae343f6d79b90626d753 core.h -``` - -## Files - -- `network_simplex_simple.h` — the exact transport solver behind `svs::distance::QEmd`. -- `full_bipartitegraph.h` — local header `network_simplex_simple.h` includes. -- `core.h` — local header `full_bipartitegraph.h` includes. - -(`network_simplex_simple.h`'s other includes are all standard library headers.) - -## Licenses - -POT itself is MIT-licensed: "Copyright (c) 2016-2023 POT contributors". - -All three files are Nicolas Bonneel's 2013 adaptation of LEMON -(`full_graph.h` / `network_simplex.h`) into a lighter mass-transport solver. -Each carries LEMON's original notice: - -``` -Copyright (C) 2003-2010 -Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport -(Egervary Research Group on Combinatorial Optimization, EGRES). - -Permission to use, modify and distribute this software is granted -provided that this copyright notice appears in all copies. For -precise terms see the accompanying LICENSE file. - -This software is provided "AS IS" with no warranty of any kind, -express or implied, and with no claim as to its suitability for any -purpose. -``` - -This permission notice must stay in each vendored file; do not strip it when editing. diff --git a/include/svs/third-party/pot/core.h b/include/svs/third-party/pot/core.h deleted file mode 100644 index 04dddf721..000000000 --- a/include/svs/third-party/pot/core.h +++ /dev/null @@ -1,103 +0,0 @@ -/* -*- mode: C++; indent-tabs-mode: nil; -*- - * - * This file has been adapted by Nicolas Bonneel (2013), - * from full_graph.h from LEMON, a generic C++ optimization library, - * to make the other files independant from the rest of - * the original library. - * - * - **** Original file Copyright Notice : - * Copyright (C) 2003-2010 - * Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport - * (Egervary Research Group on Combinatorial Optimization, EGRES). - * - * Permission to use, modify and distribute this software is granted - * provided that this copyright notice appears in all copies. For - * precise terms see the accompanying LICENSE file. - * - * This software is provided "AS IS" with no warranty of any kind, - * express or implied, and with no claim as to its suitability for any - * purpose. - * - */ - -#ifndef LEMON_CORE_H -#define LEMON_CORE_H - -#include -#include - - -// Disable the following warnings when compiling with MSVC: -// C4250: 'class1' : inherits 'class2::member' via dominance -// C4355: 'this' : used in base member initializer list -// C4503: 'function' : decorated name length exceeded, name was truncated -// C4800: 'type' : forcing value to bool 'true' or 'false' (performance warning) -// C4996: 'function': was declared deprecated -#ifdef _MSC_VER -#pragma warning( disable : 4250 4355 4503 4800 4996 ) -#endif - -///\file -///\brief LEMON core utilities. -/// -///This header file contains core utilities for LEMON. -///It is automatically included by all graph types, therefore it usually -///do not have to be included directly. - -namespace lemon { - - /// \brief Dummy type to make it easier to create invalid iterators. - /// - /// Dummy type to make it easier to create invalid iterators. - /// See \ref INVALID for the usage. - struct Invalid { - public: - bool operator==(Invalid) { return true; } - bool operator!=(Invalid) { return false; } - bool operator< (Invalid) { return false; } - }; - - /// \brief Invalid iterators. - /// - /// \ref Invalid is a global type that converts to each iterator - /// in such a way that the value of the target iterator will be invalid. -#ifdef LEMON_ONLY_TEMPLATES - const Invalid INVALID = Invalid(); -#else - extern const Invalid INVALID; -#endif - - /// \addtogroup gutils - /// @{ - - ///Create convenience typedefs for the digraph types and iterators - - ///This \c \#define creates convenient type definitions for the following - ///types of \c Digraph: \c Node, \c NodeIt, \c Arc, \c ArcIt, \c InArcIt, - ///\c OutArcIt, \c BoolNodeMap, \c IntNodeMap, \c DoubleNodeMap, - ///\c BoolArcMap, \c IntArcMap, \c DoubleArcMap. - /// - ///\note If the graph type is a dependent type, ie. the graph type depend - ///on a template parameter, then use \c TEMPLATE_DIGRAPH_TYPEDEFS() - ///macro. -#define DIGRAPH_TYPEDEFS(Digraph) \ - typedef Digraph::Node Node; \ - typedef Digraph::Arc Arc; \ - - - ///Create convenience typedefs for the digraph types and iterators - - ///\see DIGRAPH_TYPEDEFS - /// - ///\note Use this macro, if the graph type is a dependent type, - ///ie. the graph type depend on a template parameter. -#define TEMPLATE_DIGRAPH_TYPEDEFS(Digraph) \ - typedef typename Digraph::Node Node; \ - typedef typename Digraph::Arc Arc; \ - - - -} //namespace lemon - -#endif diff --git a/include/svs/third-party/pot/full_bipartitegraph.h b/include/svs/third-party/pot/full_bipartitegraph.h deleted file mode 100644 index 713ccb593..000000000 --- a/include/svs/third-party/pot/full_bipartitegraph.h +++ /dev/null @@ -1,212 +0,0 @@ -/* -*- mode: C++; indent-tabs-mode: nil; -*- - * - * This file has been adapted by Nicolas Bonneel (2013), - * from full_graph.h from LEMON, a generic C++ optimization library, - * to implement a lightweight fully connected bipartite graph. A previous - * version of this file is used as part of the Displacement Interpolation - * project, - * Web: http://www.cs.ubc.ca/labs/imager/tr/2011/DisplacementInterpolation/ - * - * - **** Original file Copyright Notice : - * Copyright (C) 2003-2010 - * Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport - * (Egervary Research Group on Combinatorial Optimization, EGRES). - * - * Permission to use, modify and distribute this software is granted - * provided that this copyright notice appears in all copies. For - * precise terms see the accompanying LICENSE file. - * - * This software is provided "AS IS" with no warranty of any kind, - * express or implied, and with no claim as to its suitability for any - * purpose. - * - */ - -#pragma once - -#include "core.h" -#include - -///\ingroup graphs -///\file -///\brief FullBipartiteDigraph and FullBipartiteGraph classes. - - -namespace lemon { - - - class FullBipartiteDigraphBase { - public: - - typedef FullBipartiteDigraphBase Digraph; - - //class Node; - typedef int Node; - //class Arc; - typedef int64_t Arc; - - protected: - - int _node_num; - int64_t _arc_num; - - FullBipartiteDigraphBase() {} - - void construct(int n1, int n2) { _node_num = n1+n2; _arc_num = (int64_t)n1 * (int64_t)n2; _n1=n1; _n2=n2;} - - public: - - int _n1, _n2; - - - Node operator()(int ix) const { return Node(ix); } - static int index(const Node& node) { return node; } - - Arc arc(const Node& s, const Node& t) const { - if (s<_n1 && t>=_n1) - return Arc((int64_t)s * (int64_t)_n2 + (int64_t)(t-_n1) ); - else - return Arc(-1); - } - - int nodeNum() const { return _node_num; } - int64_t arcNum() const { return _arc_num; } - - int maxNodeId() const { return _node_num - 1; } - int64_t maxArcId() const { return _arc_num - 1; } - - Node source(Arc arc) const { return arc / _n2; } - Node target(Arc arc) const { return (arc % _n2) + _n1; } - - static int id(Node node) { return node; } - static int64_t id(Arc arc) { return arc; } - - static Node nodeFromId(int id) { return Node(id);} - static Arc arcFromId(int64_t id) { return Arc(id);} - - - Arc findArc(Node s, Node t, Arc prev = -1) const { - return prev == -1 ? arc(s, t) : -1; - } - - void first(Node& node) const { - node = _node_num - 1; - } - - static void next(Node& node) { - --node; - } - - void first(Arc& arc) const { - arc = _arc_num - 1; - } - - static void next(Arc& arc) { - --arc; - } - - void firstOut(Arc& arc, const Node& node) const { - if (node>=_n1) - arc = -1; - else - arc = (node + 1) * _n2 - 1; - } - - void nextOut(Arc& arc) const { - if (arc % _n2 == 0) arc = 0; - --arc; - } - - void firstIn(Arc& arc, const Node& node) const { - if (node<_n1) - arc = -1; - else - arc = _arc_num + node - _node_num; - } - - void nextIn(Arc& arc) const { - arc -= _n2; - if (arc < 0) arc = -1; - } - - }; - - /// \ingroup graphs - /// - /// \brief A directed full graph class. - /// - /// FullBipartiteDigraph is a simple and fast implementation of directed full - /// (complete) graphs. It contains an arc from each node to each node - /// (including a loop for each node), therefore the number of arcs - /// is the square of the number of nodes. - /// This class is completely static and it needs constant memory space. - /// Thus you can neither add nor delete nodes or arcs, however - /// the structure can be resized using resize(). - /// - /// This type fully conforms to the \ref concepts::Digraph "Digraph concept". - /// Most of its member functions and nested classes are documented - /// only in the concept class. - /// - /// This class provides constant time counting for nodes and arcs. - /// - /// \note FullBipartiteDigraph and FullBipartiteGraph classes are very similar, - /// but there are two differences. While this class conforms only - /// to the \ref concepts::Digraph "Digraph" concept, FullBipartiteGraph - /// conforms to the \ref concepts::Graph "Graph" concept, - /// moreover FullBipartiteGraph does not contain a loop for each - /// node as this class does. - /// - /// \sa FullBipartiteGraph - class FullBipartiteDigraph : public FullBipartiteDigraphBase { - typedef FullBipartiteDigraphBase Parent; - - public: - - /// \brief Default constructor. - /// - /// Default constructor. The number of nodes and arcs will be zero. - FullBipartiteDigraph() { construct(0,0); } - - /// \brief Constructor - /// - /// Constructor. - /// \param n The number of the nodes. - FullBipartiteDigraph(int n1, int n2) { construct(n1, n2); } - - - /// \brief Returns the node with the given index. - /// - /// Returns the node with the given index. Since this structure is - /// completely static, the nodes can be indexed with integers from - /// the range [0..nodeNum()-1]. - /// The index of a node is the same as its ID. - /// \sa index() - Node operator()(int ix) const { return Parent::operator()(ix); } - - /// \brief Returns the index of the given node. - /// - /// Returns the index of the given node. Since this structure is - /// completely static, the nodes can be indexed with integers from - /// the range [0..nodeNum()-1]. - /// The index of a node is the same as its ID. - /// \sa operator()() - static int index(const Node& node) { return Parent::index(node); } - - /// \brief Returns the arc connecting the given nodes. - /// - /// Returns the arc connecting the given nodes. - /*Arc arc(Node u, Node v) const { - return Parent::arc(u, v); - }*/ - - /// \brief Number of nodes. - int nodeNum() const { return Parent::nodeNum(); } - /// \brief Number of arcs. - int64_t arcNum() const { return Parent::arcNum(); } - }; - - - - -} //namespace lemon diff --git a/include/svs/third-party/pot/network_simplex_simple.h b/include/svs/third-party/pot/network_simplex_simple.h deleted file mode 100644 index 9612a8a24..000000000 --- a/include/svs/third-party/pot/network_simplex_simple.h +++ /dev/null @@ -1,1552 +0,0 @@ -/* -*- mode: C++; indent-tabs-mode: nil; -*- - * - * - * This file has been adapted by Nicolas Bonneel (2013), - * from network_simplex.h from LEMON, a generic C++ optimization library, - * to implement a lightweight network simplex for mass transport, more - * memory efficient that the original file. A previous version of this file - * is used as part of the Displacement Interpolation project, - * Web: http://www.cs.ubc.ca/labs/imager/tr/2011/DisplacementInterpolation/ - * - * - **** Original file Copyright Notice : - * - * Copyright (C) 2003-2010 - * Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport - * (Egervary Research Group on Combinatorial Optimization, EGRES). - * - * Permission to use, modify and distribute this software is granted - * provided that this copyright notice appears in all copies. For - * precise terms see the accompanying LICENSE file. - * - * This software is provided "AS IS" with no warranty of any kind, - * express or implied, and with no claim as to its suitability for any - * purpose. - * - */ - -#pragma once -#undef DEBUG_LVL -#define DEBUG_LVL 0 - -#if DEBUG_LVL>0 -#include -#endif - -#undef EPSILON -#undef _EPSILON -#undef MAX_DEBUG_ITER -#define EPSILON 2.2204460492503131e-15 -#define _EPSILON 1e-8 -#define MAX_DEBUG_ITER 100000 - - -/// \ingroup min_cost_flow_algs -/// -/// \file -/// \brief Network Simplex algorithm for finding a minimum cost flow. - -// if your compiler has troubles with stdext or hashmaps, just comment the following line to use a slower std::map instead -//#define HASHMAP - -#include -#include -#include -#include -#include -#ifdef HASHMAP -#include -#else -#include -#endif -#include -//#include "core.h" -//#include "lmath.h" - -//#include "sparse_array_n.h" -#include "full_bipartitegraph.h" - -#undef INVALIDNODE -#undef INVALID -#define INVALIDNODE -1 -#define INVALID (-1) - -namespace lemon { - - - template - class ProxyObject; - - template - class SparseValueVector - { - public: - SparseValueVector(size_t n=0) - { - } - void resize(size_t n=0){}; - T operator[](const size_t id) const - { -#ifdef HASHMAP - typename stdext::hash_map::const_iterator it = data.find(id); -#else - typename std::map::const_iterator it = data.find(id); -#endif - if (it==data.end()) - return 0; - else - return it->second; - } - - ProxyObject operator[](const size_t id) - { - return ProxyObject( this, id ); - } - - //private: -#ifdef HASHMAP - stdext::hash_map data; -#else - std::map data; -#endif - - }; - - template - class ProxyObject { - public: - ProxyObject( SparseValueVector *v, size_t idx ){_v=v; _idx=idx;}; - ProxyObject & operator=( const T &v ) { - // If we get here, we know that operator[] was called to perform a write access, - // so we can insert an item in the vector if needed - if (v!=0) - _v->data[_idx]=v; - return *this; - } - - operator T() { - // If we get here, we know that operator[] was called to perform a read access, - // so we can simply return the existing object -#ifdef HASHMAP - typename stdext::hash_map::iterator it = _v->data.find(_idx); -#else - typename std::map::iterator it = _v->data.find(_idx); -#endif - if (it==_v->data.end()) - return 0; - else - return it->second; - } - - void operator+=(T val) - { - if (val==0) return; -#ifdef HASHMAP - typename stdext::hash_map::iterator it = _v->data.find(_idx); -#else - typename std::map::iterator it = _v->data.find(_idx); -#endif - if (it==_v->data.end()) - _v->data[_idx] = val; - else - { - T sum = it->second + val; - if (sum==0) - _v->data.erase(it); - else - it->second = sum; - } - } - void operator-=(T val) - { - if (val==0) return; -#ifdef HASHMAP - typename stdext::hash_map::iterator it = _v->data.find(_idx); -#else - typename std::map::iterator it = _v->data.find(_idx); -#endif - if (it==_v->data.end()) - _v->data[_idx] = -val; - else - { - T sum = it->second - val; - if (sum==0) - _v->data.erase(it); - else - it->second = sum; - } - } - - SparseValueVector *_v; - size_t _idx; - }; - - - - /// \addtogroup min_cost_flow_algs - /// @{ - - /// \brief Implementation of the primal Network Simplex algorithm - /// for finding a \ref min_cost_flow "minimum cost flow". - /// - /// \ref NetworkSimplexSimple implements the primal Network Simplex algorithm - /// for finding a \ref min_cost_flow "minimum cost flow" - /// \ref amo93networkflows, \ref dantzig63linearprog, - /// \ref kellyoneill91netsimplex. - /// This algorithm is a highly efficient specialized version of the - /// linear programming simplex method directly for the minimum cost - /// flow problem. - /// - /// In general, %NetworkSimplexSimple is the fastest implementation available - /// in LEMON for this problem. - /// Moreover, it supports both directions of the supply/demand inequality - /// constraints. For more information, see \ref SupplyType. - /// - /// Most of the parameters of the problem (except for the digraph) - /// can be given using separate functions, and the algorithm can be - /// executed using the \ref run() function. If some parameters are not - /// specified, then default values will be used. - /// - /// \tparam GR The digraph type the algorithm runs on. - /// \tparam V The number type used for flow amounts, capacity bounds - /// and supply values in the algorithm. By default, it is \c int64_t. - /// \tparam C The number type used for costs and potentials in the - /// algorithm. By default, it is the same as \c V. - /// - /// \warning Both number types must be signed and all input data must - /// be integer. - /// - /// \note %NetworkSimplexSimple provides five different pivot rule - /// implementations, from which the most efficient one is used - /// by default. For more information, see \ref PivotRule. - template - class NetworkSimplexSimple - { - public: - - /// \brief Constructor. - /// - /// The constructor of the class. - /// - /// \param graph The digraph the algorithm runs on. - /// \param arc_mixing Indicate if the arcs have to be stored in a - /// mixed order in the internal data structure. - /// In special cases, it could lead to better overall performance, - /// but it is usually slower. Therefore it is disabled by default. - NetworkSimplexSimple(const GR& graph, bool arc_mixing, int nbnodes, ArcsType nb_arcs, uint64_t maxiters) : - _graph(graph), //_arc_id(graph), - _arc_mixing(arc_mixing), _init_nb_nodes(nbnodes), _init_nb_arcs(nb_arcs), - MAX(std::numeric_limits::max()), - INF(std::numeric_limits::has_infinity ? - std::numeric_limits::infinity() : MAX) - { - // Reset data structures - reset(); - max_iter = maxiters; - } - - /// The type of the flow amounts, capacity bounds and supply values - typedef V Value; - /// The type of the arc costs - typedef C Cost; - - public: - - /// \brief Problem type constants for the \c run() function. - /// - /// Enum type containing the problem type constants that can be - /// returned by the \ref run() function of the algorithm. - enum ProblemType { - /// The problem has no feasible solution (flow). - INFEASIBLE, - /// The problem has optimal solution (i.e. it is feasible and - /// bounded), and the algorithm has found optimal flow and node - /// potentials (primal and dual solutions). - OPTIMAL, - /// The objective function of the problem is unbounded, i.e. - /// there is a directed cycle having negative total cost and - /// infinite upper bound. - UNBOUNDED, - /// The maximum number of iteration has been reached - MAX_ITER_REACHED - }; - - /// \brief Constants for selecting the type of the supply constraints. - /// - /// Enum type containing constants for selecting the supply type, - /// i.e. the direction of the inequalities in the supply/demand - /// constraints of the \ref min_cost_flow "minimum cost flow problem". - /// - /// The default supply type is \c GEQ, the \c LEQ type can be - /// selected using \ref supplyType(). - /// The equality form is a special case of both supply types. - enum SupplyType { - /// This option means that there are "greater or equal" - /// supply/demand constraints in the definition of the problem. - GEQ, - /// This option means that there are "less or equal" - /// supply/demand constraints in the definition of the problem. - LEQ - }; - - - - private: - - uint64_t max_iter; - TEMPLATE_DIGRAPH_TYPEDEFS(GR); - - typedef std::vector IntVector; - typedef std::vector ArcVector; - typedef std::vector ValueVector; - typedef std::vector CostVector; - // typedef SparseValueVector CostVector; - typedef std::vector BoolVector; - // Note: vector is used instead of vector for efficiency reasons - - // State constants for arcs - enum ArcState { - STATE_UPPER = -1, - STATE_TREE = 0, - STATE_LOWER = 1 - }; - - typedef std::vector StateVector; - // Note: vector is used instead of vector for - // efficiency reasons - - private: - - // Data related to the underlying digraph - const GR &_graph; - int _node_num; - ArcsType _arc_num; - ArcsType _all_arc_num; - ArcsType _search_arc_num; - - // Parameters of the problem - SupplyType _stype; - Value _sum_supply; - - inline int _node_id(int n) const {return _node_num-n-1;} ; - -// IntArcMap _arc_id; - IntVector _source; // keep nodes as integers - IntVector _target; - bool _arc_mixing; - public: - // Node and arc data - CostVector _cost; - ValueVector _supply; - ValueVector _flow; - //SparseValueVector _flow; - CostVector _pi; - - - private: - // Data for storing the spanning tree structure - IntVector _parent; - ArcVector _pred; - IntVector _thread; - IntVector _rev_thread; - IntVector _succ_num; - IntVector _last_succ; - IntVector _dirty_revs; - BoolVector _forward; - StateVector _state; - ArcsType _root; - - // Temporary data used in the current pivot iteration - ArcsType in_arc, join, u_in, v_in, u_out, v_out; - ArcsType first, second, right, last; - ArcsType stem, par_stem, new_stem; - Value delta; - - const Value MAX; - - ArcsType mixingCoeff; - - public: - - /// \brief Constant for infinite upper bounds (capacities). - /// - /// Constant for infinite upper bounds (capacities). - /// It is \c std::numeric_limits::infinity() if available, - /// \c std::numeric_limits::max() otherwise. - const Value INF; - - private: - - // thank you to DVK and MizardX from StackOverflow for this function! - inline ArcsType sequence(ArcsType k) const { - ArcsType smallv = (k > num_total_big_subsequence_numbers) & 1; - - k -= num_total_big_subsequence_numbers * smallv; - ArcsType subsequence_length2 = subsequence_length- smallv; - ArcsType subsequence_num = (k / subsequence_length2) + num_big_subseqiences * smallv; - ArcsType subsequence_offset = (k % subsequence_length2) * mixingCoeff; - - return subsequence_offset + subsequence_num; - } - ArcsType subsequence_length; - ArcsType num_big_subseqiences; - ArcsType num_total_big_subsequence_numbers; - - inline ArcsType getArcID(const Arc &arc) const - { - //int n = _arc_num-arc._id-1; - ArcsType n = _arc_num-GR::id(arc)-1; - - //ArcsType a = mixingCoeff*(n%mixingCoeff) + n/mixingCoeff; - //ArcsType b = _arc_id[arc]; - if (_arc_mixing) - return sequence(n); - else - return n; - } - - // finally unused because too slow - inline ArcsType getSource(const ArcsType arc) const - { - //ArcsType a = _source[arc]; - //return a; - - ArcsType n = _arc_num-arc-1; - if (_arc_mixing) - n = mixingCoeff*(n%mixingCoeff) + n/mixingCoeff; - - ArcsType b; - if (n>=0) - b = _node_id(_graph.source(GR::arcFromId( n ) )); - else - { - n = arc+1-_arc_num; - if ( n<=_node_num) - b = _node_num; - else - if ( n>=_graph._n1) - b = _graph._n1; - else - b = _graph._n1-n; - } - - return b; - } - - - - // Implementation of the Block Search pivot rule - class BlockSearchPivotRule - { - private: - - // References to the NetworkSimplexSimple class - const IntVector &_source; - const IntVector &_target; - const CostVector &_cost; - const StateVector &_state; - const CostVector &_pi; - ArcsType &_in_arc; - ArcsType _search_arc_num; - - // Pivot rule data - ArcsType _block_size; - ArcsType _next_arc; - NetworkSimplexSimple &_ns; - - public: - - // Constructor - BlockSearchPivotRule(NetworkSimplexSimple &ns) : - _source(ns._source), _target(ns._target), - _cost(ns._cost), _state(ns._state), _pi(ns._pi), - _in_arc(ns.in_arc), _search_arc_num(ns._search_arc_num), - _next_arc(0),_ns(ns) - { - // The main parameters of the pivot rule - const double BLOCK_SIZE_FACTOR = 1.0; - const ArcsType MIN_BLOCK_SIZE = 10; - - _block_size = std::max(ArcsType(BLOCK_SIZE_FACTOR * std::sqrt(double(_search_arc_num))), MIN_BLOCK_SIZE); - } - - // Find next entering arc - bool findEnteringArc() { - Cost c, min = 0; - ArcsType e; - ArcsType cnt = _block_size; - double a; - for (e = _next_arc; e != _search_arc_num; ++e) { - c = _state[e] * (_cost[e] + _pi[_source[e]] - _pi[_target[e]]); - if (c < min) { - min = c; - _in_arc = e; - } - if (--cnt == 0) { - a=fabs(_pi[_source[_in_arc]])>fabs(_pi[_target[_in_arc]]) ? fabs(_pi[_source[_in_arc]]):fabs(_pi[_target[_in_arc]]); - a=a>fabs(_cost[_in_arc])?a:fabs(_cost[_in_arc]); - if (min < -EPSILON*a) goto search_end; - cnt = _block_size; - } - } - for (e = 0; e != _next_arc; ++e) { - c = _state[e] * (_cost[e] + _pi[_source[e]] - _pi[_target[e]]); - if (c < min) { - min = c; - _in_arc = e; - } - if (--cnt == 0) { - a=fabs(_pi[_source[_in_arc]])>fabs(_pi[_target[_in_arc]]) ? fabs(_pi[_source[_in_arc]]):fabs(_pi[_target[_in_arc]]); - a=a>fabs(_cost[_in_arc])?a:fabs(_cost[_in_arc]); - if (min < -EPSILON*a) goto search_end; - cnt = _block_size; - } - } - a=fabs(_pi[_source[_in_arc]])>fabs(_pi[_target[_in_arc]]) ? fabs(_pi[_source[_in_arc]]):fabs(_pi[_target[_in_arc]]); - a=a>fabs(_cost[_in_arc])?a:fabs(_cost[_in_arc]); - if (min >= -EPSILON*a) return false; - - search_end: - _next_arc = e; - return true; - } - - }; //class BlockSearchPivotRule - - - - public: - - - - int _init_nb_nodes; - ArcsType _init_nb_arcs; - - /// \name Parameters - /// The parameters of the algorithm can be specified using these - /// functions. - - /// @{ - - - /// \brief Set the costs of the arcs. - /// - /// This function sets the costs of the arcs. - /// If it is not used before calling \ref run(), the costs - /// will be set to \c 1 on all arcs. - /// - /// \param map An arc map storing the costs. - /// Its \c Value type must be convertible to the \c Cost type - /// of the algorithm. - /// - /// \return (*this) - template - NetworkSimplexSimple& costMap(const CostMap& map) { - Arc a; _graph.first(a); - for (; a != INVALID; _graph.next(a)) { - _cost[getArcID(a)] = map[a]; - } - return *this; - } - - - /// \brief Set the costs of one arc. - /// - /// This function sets the costs of one arcs. - /// Done for memory reasons - /// - /// \param arc An arc. - /// \param arc A cost - /// - /// \return (*this) - template - NetworkSimplexSimple& setCost(const Arc& arc, const Value cost) { - _cost[getArcID(arc)] = cost; - return *this; - } - - - /// \brief Set the supply values of the nodes. - /// - /// This function sets the supply values of the nodes. - /// If neither this function nor \ref stSupply() is used before - /// calling \ref run(), the supply of each node will be set to zero. - /// - /// \param map A node map storing the supply values. - /// Its \c Value type must be convertible to the \c Value type - /// of the algorithm. - /// - /// \return (*this) - template - NetworkSimplexSimple& supplyMap(const SupplyMap& map) { - Node n; _graph.first(n); - for (; n != INVALIDNODE; _graph.next(n)) { - _supply[_node_id(n)] = map[n]; - } - return *this; - } - template - NetworkSimplexSimple& supplyMap(const SupplyMap* map1, int n1, const SupplyMap* map2, int n2) { - Node n; _graph.first(n); - for (; n != INVALIDNODE; _graph.next(n)) { - if (n - NetworkSimplexSimple& supplyMapAll(SupplyMap val1, int n1, SupplyMap val2, int n2) { - Node n; _graph.first(n); - for (; n != INVALIDNODE; _graph.next(n)) { - if (n(*this) - NetworkSimplexSimple& stSupply(const Node& s, const Node& t, Value k) { - for (int i = 0; i != _node_num; ++i) { - _supply[i] = 0; - } - _supply[_node_id(s)] = k; - _supply[_node_id(t)] = -k; - return *this; - } - - /// \brief Set the type of the supply constraints. - /// - /// This function sets the type of the supply/demand constraints. - /// If it is not used before calling \ref run(), the \ref GEQ supply - /// type will be used. - /// - /// For more information, see \ref SupplyType. - /// - /// \return (*this) - NetworkSimplexSimple& supplyType(SupplyType supply_type) { - _stype = supply_type; - return *this; - } - - /// @} - - /// \name Execution Control - /// The algorithm can be executed using \ref run(). - - /// @{ - - /// \brief Run the algorithm. - /// - /// This function runs the algorithm. - /// The paramters can be specified using functions \ref lowerMap(), - /// \ref upperMap(), \ref costMap(), \ref supplyMap(), \ref stSupply(), - /// \ref supplyType(). - /// For example, - /// \code - /// NetworkSimplexSimple ns(graph); - /// ns.lowerMap(lower).upperMap(upper).costMap(cost) - /// .supplyMap(sup).run(); - /// \endcode - /// - /// This function can be called more than once. All the given parameters - /// are kept for the next call, unless \ref resetParams() or \ref reset() - /// is used, thus only the modified parameters have to be set again. - /// If the underlying digraph was also modified after the construction - /// of the class (or the last \ref reset() call), then the \ref reset() - /// function must be called. - /// - /// \param pivot_rule The pivot rule that will be used during the - /// algorithm. For more information, see \ref PivotRule. - /// - /// \return \c INFEASIBLE if no feasible flow exists, - /// \n \c OPTIMAL if the problem has optimal solution - /// (i.e. it is feasible and bounded), and the algorithm has found - /// optimal flow and node potentials (primal and dual solutions), - /// \n \c UNBOUNDED if the objective function of the problem is - /// unbounded, i.e. there is a directed cycle having negative total - /// cost and infinite upper bound. - /// - /// \see ProblemType, PivotRule - /// \see resetParams(), reset() - ProblemType run() { -#if DEBUG_LVL>0 - std::cout << "OPTIMAL = " << OPTIMAL << "\nINFEASIBLE = " << INFEASIBLE << "\nUNBOUNDED = " << UNBOUNDED << "\nMAX_ITER_REACHED" << MAX_ITER_REACHED << "\n" ; -#endif - - if (!init()) return INFEASIBLE; -#if DEBUG_LVL>0 - std::cout << "Init done, starting iterations\n"; -#endif - return start(); - } - - /// \brief Reset all the parameters that have been given before. - /// - /// This function resets all the paramaters that have been given - /// before using functions \ref lowerMap(), \ref upperMap(), - /// \ref costMap(), \ref supplyMap(), \ref stSupply(), \ref supplyType(). - /// - /// It is useful for multiple \ref run() calls. Basically, all the given - /// parameters are kept for the next \ref run() call, unless - /// \ref resetParams() or \ref reset() is used. - /// If the underlying digraph was also modified after the construction - /// of the class or the last \ref reset() call, then the \ref reset() - /// function must be used, otherwise \ref resetParams() is sufficient. - /// - /// For example, - /// \code - /// NetworkSimplexSimple ns(graph); - /// - /// // First run - /// ns.lowerMap(lower).upperMap(upper).costMap(cost) - /// .supplyMap(sup).run(); - /// - /// // Run again with modified cost map (resetParams() is not called, - /// // so only the cost map have to be set again) - /// cost[e] += 100; - /// ns.costMap(cost).run(); - /// - /// // Run again from scratch using resetParams() - /// // (the lower bounds will be set to zero on all arcs) - /// ns.resetParams(); - /// ns.upperMap(capacity).costMap(cost) - /// .supplyMap(sup).run(); - /// \endcode - /// - /// \return (*this) - /// - /// \see reset(), run() - NetworkSimplexSimple& resetParams() { - for (int i = 0; i != _node_num; ++i) { - _supply[i] = 0; - } - for (ArcsType i = 0; i != _arc_num; ++i) { - _cost[i] = 1; - } - _stype = GEQ; - return *this; - } - - - - int64_t divid (int64_t x, int64_t y) - { - return (x-x%y)/y; - } - - /// \brief Reset the internal data structures and all the parameters - /// that have been given before. - /// - /// This function resets the internal data structures and all the - /// paramaters that have been given before using functions \ref lowerMap(), - /// \ref upperMap(), \ref costMap(), \ref supplyMap(), \ref stSupply(), - /// \ref supplyType(). - /// - /// It is useful for multiple \ref run() calls. Basically, all the given - /// parameters are kept for the next \ref run() call, unless - /// \ref resetParams() or \ref reset() is used. - /// If the underlying digraph was also modified after the construction - /// of the class or the last \ref reset() call, then the \ref reset() - /// function must be used, otherwise \ref resetParams() is sufficient. - /// - /// See \ref resetParams() for examples. - /// - /// \return (*this) - /// - /// \see resetParams(), run() - NetworkSimplexSimple& reset() { - // Resize vectors - _node_num = _init_nb_nodes; - _arc_num = _init_nb_arcs; - int all_node_num = _node_num + 1; - ArcsType max_arc_num = _arc_num + 2 * _node_num; - - _source.resize(max_arc_num); - _target.resize(max_arc_num); - - _cost.resize(max_arc_num); - _supply.resize(all_node_num); - _flow.resize(max_arc_num); - _pi.resize(all_node_num); - - _parent.resize(all_node_num); - _pred.resize(all_node_num); - _forward.resize(all_node_num); - _thread.resize(all_node_num); - _rev_thread.resize(all_node_num); - _succ_num.resize(all_node_num); - _last_succ.resize(all_node_num); - _state.resize(max_arc_num); - - - //_arc_mixing=false; - if (_arc_mixing) { - // Store the arcs in a mixed order - const ArcsType k = std::max(ArcsType(std::sqrt(double(_arc_num))), ArcsType(10)); - mixingCoeff = k; - subsequence_length = _arc_num / mixingCoeff + 1; - num_big_subseqiences = _arc_num % mixingCoeff; - num_total_big_subsequence_numbers = subsequence_length * num_big_subseqiences; - - ArcsType i = 0, j = 0; - Arc a; _graph.first(a); - for (; a != INVALID; _graph.next(a)) { - _source[i] = _node_id(_graph.source(a)); - _target[i] = _node_id(_graph.target(a)); - //_arc_id[a] = i; - if ((i += k) >= _arc_num) i = ++j; - } - } else { - // Store the arcs in the original order - ArcsType i = 0; - Arc a; _graph.first(a); - for (; a != INVALID; _graph.next(a), ++i) { - _source[i] = _node_id(_graph.source(a)); - _target[i] = _node_id(_graph.target(a)); - //_arc_id[a] = i; - } - } - - // Reset parameters - resetParams(); - return *this; - } - - /// @} - - /// \name Query Functions - /// The results of the algorithm can be obtained using these - /// functions.\n - /// The \ref run() function must be called before using them. - - /// @{ - - /// \brief Return the total cost of the found flow. - /// - /// This function returns the total cost of the found flow. - /// Its complexity is O(e). - /// - /// \note The return type of the function can be specified as a - /// template parameter. For example, - /// \code - /// ns.totalCost(); - /// \endcode - /// It is useful if the total cost cannot be stored in the \c Cost - /// type of the algorithm, which is the default return type of the - /// function. - /// - /// \pre \ref run() must be called before using this function. - /*template - Number totalCost() const { - Number c = 0; - for (ArcIt a(_graph); a != INVALID; ++a) { - int64_t i = getArcID(a); - c += Number(_flow[i]) * Number(_cost[i]); - } - return c; - }*/ - - template - Number totalCost() const { - Number c = 0; - - /*#ifdef HASHMAP - typename stdext::hash_map::const_iterator it; - #else - typename std::map::const_iterator it; - #endif - for (it = _flow.data.begin(); it!=_flow.data.end(); ++it) - c += Number(it->second) * Number(_cost[it->first]); - return c;*/ - - for (ArcsType i=0; i<_flow.size(); i++) - c += _flow[i] * Number(_cost[i]); - return c; - - } - -#ifndef DOXYGEN - Cost totalCost() const { - return totalCost(); - } -#endif - - /// \brief Return the flow on the given arc. - /// - /// This function returns the flow on the given arc. - /// - /// \pre \ref run() must be called before using this function. - Value flow(const Arc& a) const { - return _flow[getArcID(a)]; - } - - /// \brief Return the flow map (the primal solution). - /// - /// This function copies the flow value on each arc into the given - /// map. The \c Value type of the algorithm must be convertible to - /// the \c Value type of the map. - /// - /// \pre \ref run() must be called before using this function. - template - void flowMap(FlowMap &map) const { - Arc a; _graph.first(a); - for (; a != INVALID; _graph.next(a)) { - map.set(a, _flow[getArcID(a)]); - } - } - - /// \brief Return the potential (dual value) of the given node. - /// - /// This function returns the potential (dual value) of the - /// given node. - /// - /// \pre \ref run() must be called before using this function. - Cost potential(const Node& n) const { - return _pi[_node_id(n)]; - } - - /// \brief Return the potential map (the dual solution). - /// - /// This function copies the potential (dual value) of each node - /// into the given map. - /// The \c Cost type of the algorithm must be convertible to the - /// \c Value type of the map. - /// - /// \pre \ref run() must be called before using this function. - template - void potentialMap(PotentialMap &map) const { - Node n; _graph.first(n); - for (; n != INVALID; _graph.next(n)) { - map.set(n, _pi[_node_id(n)]); - } - } - - /// @} - - private: - - // Initialize internal data structures - bool init() { - if (_node_num == 0) return false; - - // Check the sum of supply values - _sum_supply = 0; - for (int i = 0; i != _node_num; ++i) { - _sum_supply += _supply[i]; - } - if ( fabs(_sum_supply) > _EPSILON ) return false; - - _sum_supply = 0; - - // Initialize artifical cost - Cost ART_COST; - if (std::numeric_limits::is_exact) { - ART_COST = std::numeric_limits::max() / 2 + 1; - } else { - ART_COST = 0; - for (ArcsType i = 0; i != _arc_num; ++i) { - if (_cost[i] > ART_COST) ART_COST = _cost[i]; - } - ART_COST = (ART_COST + 1) * _node_num; - } - - // Initialize arc maps - for (ArcsType i = 0; i != _arc_num; ++i) { - //_flow[i] = 0; //by default, the sparse matrix is empty - _state[i] = STATE_LOWER; - } - - // Set data for the artificial root node - _root = _node_num; - _parent[_root] = -1; - _pred[_root] = -1; - _thread[_root] = 0; - _rev_thread[0] = _root; - _succ_num[_root] = _node_num + 1; - _last_succ[_root] = _root - 1; - _supply[_root] = -_sum_supply; - _pi[_root] = 0; - - // Add artificial arcs and initialize the spanning tree data structure - if (_sum_supply == 0) { - // EQ supply constraints - _search_arc_num = _arc_num; - _all_arc_num = _arc_num + _node_num; - for (ArcsType u = 0, e = _arc_num; u != _node_num; ++u, ++e) { - _parent[u] = _root; - _pred[u] = e; - _thread[u] = u + 1; - _rev_thread[u + 1] = u; - _succ_num[u] = 1; - _last_succ[u] = u; - _state[e] = STATE_TREE; - if (_supply[u] >= 0) { - _forward[u] = true; - _pi[u] = 0; - _source[e] = u; - _target[e] = _root; - _flow[e] = _supply[u]; - _cost[e] = 0; - } else { - _forward[u] = false; - _pi[u] = ART_COST; - _source[e] = _root; - _target[e] = u; - _flow[e] = -_supply[u]; - _cost[e] = ART_COST; - } - } - } - else if (_sum_supply > 0) { - // LEQ supply constraints - _search_arc_num = _arc_num + _node_num; - ArcsType f = _arc_num + _node_num; - for (ArcsType u = 0, e = _arc_num; u != _node_num; ++u, ++e) { - _parent[u] = _root; - _thread[u] = u + 1; - _rev_thread[u + 1] = u; - _succ_num[u] = 1; - _last_succ[u] = u; - if (_supply[u] >= 0) { - _forward[u] = true; - _pi[u] = 0; - _pred[u] = e; - _source[e] = u; - _target[e] = _root; - _flow[e] = _supply[u]; - _cost[e] = 0; - _state[e] = STATE_TREE; - } else { - _forward[u] = false; - _pi[u] = ART_COST; - _pred[u] = f; - _source[f] = _root; - _target[f] = u; - _flow[f] = -_supply[u]; - _cost[f] = ART_COST; - _state[f] = STATE_TREE; - _source[e] = u; - _target[e] = _root; - //_flow[e] = 0; //by default, the sparse matrix is empty - _cost[e] = 0; - _state[e] = STATE_LOWER; - ++f; - } - } - _all_arc_num = f; - } - else { - // GEQ supply constraints - _search_arc_num = _arc_num + _node_num; - ArcsType f = _arc_num + _node_num; - for (ArcsType u = 0, e = _arc_num; u != _node_num; ++u, ++e) { - _parent[u] = _root; - _thread[u] = u + 1; - _rev_thread[u + 1] = u; - _succ_num[u] = 1; - _last_succ[u] = u; - if (_supply[u] <= 0) { - _forward[u] = false; - _pi[u] = 0; - _pred[u] = e; - _source[e] = _root; - _target[e] = u; - _flow[e] = -_supply[u]; - _cost[e] = 0; - _state[e] = STATE_TREE; - } else { - _forward[u] = true; - _pi[u] = -ART_COST; - _pred[u] = f; - _source[f] = u; - _target[f] = _root; - _flow[f] = _supply[u]; - _state[f] = STATE_TREE; - _cost[f] = ART_COST; - _source[e] = _root; - _target[e] = u; - //_flow[e] = 0; //by default, the sparse matrix is empty - _cost[e] = 0; - _state[e] = STATE_LOWER; - ++f; - } - } - _all_arc_num = f; - } - - return true; - } - - // Find the join node - void findJoinNode() { - int u = _source[in_arc]; - int v = _target[in_arc]; - while (u != v) { - if (_succ_num[u] < _succ_num[v]) { - u = _parent[u]; - } else { - v = _parent[v]; - } - } - join = u; - } - - // Find the leaving arc of the cycle and returns true if the - // leaving arc is not the same as the entering arc - bool findLeavingArc() { - // Initialize first and second nodes according to the direction - // of the cycle - if (_state[in_arc] == STATE_LOWER) { - first = _source[in_arc]; - second = _target[in_arc]; - } else { - first = _target[in_arc]; - second = _source[in_arc]; - } - delta = INF; - char result = 0; - Value d; - ArcsType e; - - // Search the cycle along the path form the first node to the root - for (int u = first; u != join; u = _parent[u]) { - e = _pred[u]; - d = _forward[u] ? _flow[e] : INF ; - if (d < delta) { - delta = d; - u_out = u; - result = 1; - } - } - // Search the cycle along the path form the second node to the root - for (int u = second; u != join; u = _parent[u]) { - e = _pred[u]; - d = _forward[u] ? INF : _flow[e]; - if (d <= delta) { - delta = d; - u_out = u; - result = 2; - } - } - - if (result == 1) { - u_in = first; - v_in = second; - } else { - u_in = second; - v_in = first; - } - return result != 0; - } - - // Change _flow and _state vectors - void changeFlow(bool change) { - // Augment along the cycle - if (delta > 0) { - Value val = _state[in_arc] * delta; - _flow[in_arc] += val; - for (int u = _source[in_arc]; u != join; u = _parent[u]) { - _flow[_pred[u]] += _forward[u] ? -val : val; - } - for (int u = _target[in_arc]; u != join; u = _parent[u]) { - _flow[_pred[u]] += _forward[u] ? val : -val; - } - } - // Update the state of the entering and leaving arcs - if (change) { - _state[in_arc] = STATE_TREE; - _state[_pred[u_out]] = - (_flow[_pred[u_out]] == 0) ? STATE_LOWER : STATE_UPPER; - } else { - _state[in_arc] = -_state[in_arc]; - } - } - - // Update the tree structure - void updateTreeStructure() { - int u, w; - int old_rev_thread = _rev_thread[u_out]; - int old_succ_num = _succ_num[u_out]; - int old_last_succ = _last_succ[u_out]; - v_out = _parent[u_out]; - - u = _last_succ[u_in]; // the last successor of u_in - right = _thread[u]; // the node after it - - // Handle the case when old_rev_thread equals to v_in - // (it also means that join and v_out coincide) - if (old_rev_thread == v_in) { - last = _thread[_last_succ[u_out]]; - } else { - last = _thread[v_in]; - } - - // Update _thread and _parent along the stem nodes (i.e. the nodes - // between u_in and u_out, whose parent have to be changed) - _thread[v_in] = stem = u_in; - _dirty_revs.clear(); - _dirty_revs.push_back(v_in); - par_stem = v_in; - while (stem != u_out) { - // Insert the next stem node into the thread list - new_stem = _parent[stem]; - _thread[u] = new_stem; - _dirty_revs.push_back(u); - - // Remove the subtree of stem from the thread list - w = _rev_thread[stem]; - _thread[w] = right; - _rev_thread[right] = w; - - // Change the parent node and shift stem nodes - _parent[stem] = par_stem; - par_stem = stem; - stem = new_stem; - - // Update u and right - u = _last_succ[stem] == _last_succ[par_stem] ? - _rev_thread[par_stem] : _last_succ[stem]; - right = _thread[u]; - } - _parent[u_out] = par_stem; - _thread[u] = last; - _rev_thread[last] = u; - _last_succ[u_out] = u; - - // Remove the subtree of u_out from the thread list except for - // the case when old_rev_thread equals to v_in - // (it also means that join and v_out coincide) - if (old_rev_thread != v_in) { - _thread[old_rev_thread] = right; - _rev_thread[right] = old_rev_thread; - } - - // Update _rev_thread using the new _thread values - for (int i = 0; i != int(_dirty_revs.size()); ++i) { - int u = _dirty_revs[i]; - _rev_thread[_thread[u]] = u; - } - - // Update _pred, _forward, _last_succ and _succ_num for the - // stem nodes from u_out to u_in - int tmp_sc = 0, tmp_ls = _last_succ[u_out]; - u = u_out; - while (u != u_in) { - w = _parent[u]; - _pred[u] = _pred[w]; - _forward[u] = !_forward[w]; - tmp_sc += _succ_num[u] - _succ_num[w]; - _succ_num[u] = tmp_sc; - _last_succ[w] = tmp_ls; - u = w; - } - _pred[u_in] = in_arc; - _forward[u_in] = (u_in == _source[in_arc]); - _succ_num[u_in] = old_succ_num; - - // Set limits for updating _last_succ form v_in and v_out - // towards the root - int up_limit_in = -1; - int up_limit_out = -1; - if (_last_succ[join] == v_in) { - up_limit_out = join; - } else { - up_limit_in = join; - } - - // Update _last_succ from v_in towards the root - for (u = v_in; u != up_limit_in && _last_succ[u] == v_in; - u = _parent[u]) { - _last_succ[u] = _last_succ[u_out]; - } - // Update _last_succ from v_out towards the root - if (join != old_rev_thread && v_in != old_rev_thread) { - for (u = v_out; u != up_limit_out && _last_succ[u] == old_last_succ; - u = _parent[u]) { - _last_succ[u] = old_rev_thread; - } - } else { - for (u = v_out; u != up_limit_out && _last_succ[u] == old_last_succ; - u = _parent[u]) { - _last_succ[u] = _last_succ[u_out]; - } - } - - // Update _succ_num from v_in to join - for (u = v_in; u != join; u = _parent[u]) { - _succ_num[u] += old_succ_num; - } - // Update _succ_num from v_out to join - for (u = v_out; u != join; u = _parent[u]) { - _succ_num[u] -= old_succ_num; - } - } - - // Update potentials - void updatePotential() { - Cost sigma = _forward[u_in] ? - _pi[v_in] - _pi[u_in] - _cost[_pred[u_in]] : - _pi[v_in] - _pi[u_in] + _cost[_pred[u_in]]; - // Update potentials in the subtree, which has been moved - int end = _thread[_last_succ[u_in]]; - for (int u = u_in; u != end; u = _thread[u]) { - _pi[u] += sigma; - } - } - - // Heuristic initial pivots - bool initialPivots() { - Value curr, total = 0; - std::vector supply_nodes, demand_nodes; - Node u; _graph.first(u); - for (; u != INVALIDNODE; _graph.next(u)) { - curr = _supply[_node_id(u)]; - if (curr > 0) { - total += curr; - supply_nodes.push_back(u); - } - else if (curr < 0) { - demand_nodes.push_back(u); - } - } - if (_sum_supply > 0) total -= _sum_supply; - if (total <= 0) return true; - - ArcVector arc_vector; - if (_sum_supply >= 0) { - if (supply_nodes.size() == 1 && demand_nodes.size() == 1) { - // Perform a reverse graph search from the sink to the source - //typename GR::template NodeMap reached(_graph, false); - BoolVector reached(_node_num, false); - Node s = supply_nodes[0], t = demand_nodes[0]; - std::vector stack; - reached[t] = true; - stack.push_back(t); - while (!stack.empty()) { - Node u, v = stack.back(); - stack.pop_back(); - if (v == s) break; - Arc a; _graph.firstIn(a, v); - for (; a != INVALID; _graph.nextIn(a)) { - if (reached[u = _graph.source(a)]) continue; - ArcsType j = getArcID(a); - if (INF >= total) { - arc_vector.push_back(j); - reached[u] = true; - stack.push_back(u); - } - } - } - } else { - // Find the min. cost incomming arc for each demand node - for (int i = 0; i != demand_nodes.size(); ++i) { - Node v = demand_nodes[i]; - Cost c, min_cost = std::numeric_limits::max(); - Arc min_arc = INVALID; - Arc a; _graph.firstIn(a, v); - for (; a != INVALID; _graph.nextIn(a)) { - c = _cost[getArcID(a)]; - if (c < min_cost) { - min_cost = c; - min_arc = a; - } - } - if (min_arc != INVALID) { - arc_vector.push_back(getArcID(min_arc)); - } - } - } - } else { - // Find the min. cost outgoing arc for each supply node - for (int i = 0; i != int(supply_nodes.size()); ++i) { - Node u = supply_nodes[i]; - Cost c, min_cost = std::numeric_limits::max(); - Arc min_arc = INVALID; - Arc a; _graph.firstOut(a, u); - for (; a != INVALID; _graph.nextOut(a)) { - c = _cost[getArcID(a)]; - if (c < min_cost) { - min_cost = c; - min_arc = a; - } - } - if (min_arc != INVALID) { - arc_vector.push_back(getArcID(min_arc)); - } - } - } - - // Perform heuristic initial pivots - for (ArcsType i = 0; i != arc_vector.size(); ++i) { - in_arc = arc_vector[i]; - // l'erreur est probablement ici... - if (_state[in_arc] * (_cost[in_arc] + _pi[_source[in_arc]] - - _pi[_target[in_arc]]) >= 0) continue; - findJoinNode(); - bool change = findLeavingArc(); - if (delta >= MAX) return false; - changeFlow(change); - if (change) { - updateTreeStructure(); - updatePotential(); - } - } - return true; - } - - // Execute the algorithm - ProblemType start() { - return start(); - } - - template - ProblemType start() { - PivotRuleImpl pivot(*this); - ProblemType retVal = OPTIMAL; - - // Perform heuristic initial pivots - if (!initialPivots()) return UNBOUNDED; - - uint64_t iter_number = 0; - //pivot.setDantzig(true); - // Execute the Network Simplex algorithm - while (pivot.findEnteringArc()) { - if(max_iter > 0 && ++iter_number>=max_iter&&max_iter>0){ - // max iterations hit - retVal = MAX_ITER_REACHED; - break; - } -#if DEBUG_LVL>0 - if(iter_number>MAX_DEBUG_ITER) - break; - if(iter_number%1000==0||iter_number%1000==1){ - double curCost=totalCost(); - double sumFlow=0; - double a; - a= (fabs(_pi[_source[in_arc]])>=fabs(_pi[_target[in_arc]])) ? fabs(_pi[_source[in_arc]]) : fabs(_pi[_target[in_arc]]); - a=a>=fabs(_cost[in_arc])?a:fabs(_cost[in_arc]); - for (int64_t i=0; i<_flow.size(); i++) { - sumFlow+=_state[i]*_flow[i]; - } - std::cout << "Sum of the flow " << std::setprecision(20) << sumFlow << "\n" << iter_number << " iterations, current cost=" << curCost << "\nReduced cost=" << _state[in_arc] * (_cost[in_arc] + _pi[_source[in_arc]] -_pi[_target[in_arc]]) << "\nPrecision = "<< -EPSILON*(a) << "\n"; - std::cout << "Arc in = (" << _node_id(_source[in_arc]) << ", " << _node_id(_target[in_arc]) <<")\n"; - std::cout << "Supplies = (" << _supply[_source[in_arc]] << ", " << _supply[_target[in_arc]] << ")\n"; - std::cout << _cost[in_arc] << "\n"; - std::cout << _pi[_source[in_arc]] << "\n"; - std::cout << _pi[_target[in_arc]] << "\n"; - std::cout << a << "\n"; - } -#endif - - findJoinNode(); - bool change = findLeavingArc(); - if (delta >= MAX) return UNBOUNDED; - changeFlow(change); - if (change) { - updateTreeStructure(); - updatePotential(); - } -#if DEBUG_LVL>0 - else{ - std::cout << "No change\n"; - } -#endif -#if DEBUG_LVL>1 - std::cout << "Arc in = (" << _source[in_arc] << ", " << _target[in_arc] << ")\n"; -#endif - - } - - -#if DEBUG_LVL>0 - double curCost=totalCost(); - double sumFlow=0; - double a; - a= (fabs(_pi[_source[in_arc]])>=fabs(_pi[_target[in_arc]])) ? fabs(_pi[_source[in_arc]]) : fabs(_pi[_target[in_arc]]); - a=a>=fabs(_cost[in_arc])?a:fabs(_cost[in_arc]); - for (int64_t i=0; i<_flow.size(); i++) { - sumFlow+=_state[i]*_flow[i]; - } - - std::cout << "Sum of the flow " << std::setprecision(20) << sumFlow << "\n" << niter << " iterations, current cost=" << curCost << "\nReduced cost=" << _state[in_arc] * (_cost[in_arc] + _pi[_source[in_arc]] -_pi[_target[in_arc]]) << "\nPrecision = "<< -EPSILON*(a) << "\n"; - - std::cout << "Arc in = (" << _node_id(_source[in_arc]) << ", " << _node_id(_target[in_arc]) <<")\n"; - std::cout << "Supplies = (" << _supply[_source[in_arc]] << ", " << _supply[_target[in_arc]] << ")\n"; - -#endif - -#if DEBUG_LVL>1 - sumFlow=0; - for (int i=0; i<_flow.size(); i++) { - sumFlow+=_state[i]*_flow[i]; - if (_state[i]==STATE_TREE) { - std::cout << "Non zero value at (" << _node_num+1-_source[i] << ", " << _node_num+1-_target[i] << ")\n"; - } - } - std::cout << "Sum of the flow " << sumFlow << "\n"<< niter <<" iterations, current cost=" << totalCost() << "\n"; -#endif - // Check feasibility - if( retVal == OPTIMAL){ - for (ArcsType e = _search_arc_num; e != _all_arc_num; ++e) { - if (_flow[e] != 0){ - if (fabs(_flow[e]) > _EPSILON) // change of the original code following issue #126 - return INFEASIBLE; - else - _flow[e]=0; - - } - } - } - - // Shift potentials to meet the requirements of the GEQ/LEQ type - // optimality conditions - if (_sum_supply == 0) { - if (_stype == GEQ) { - Cost max_pot = -std::numeric_limits::max(); - for (ArcsType i = 0; i != _node_num; ++i) { - if (_pi[i] > max_pot) max_pot = _pi[i]; - } - if (max_pot > 0) { - for (ArcsType i = 0; i != _node_num; ++i) - _pi[i] -= max_pot; - } - } else { - Cost min_pot = std::numeric_limits::max(); - for (ArcsType i = 0; i != _node_num; ++i) { - if (_pi[i] < min_pot) min_pot = _pi[i]; - } - if (min_pot < 0) { - for (ArcsType i = 0; i != _node_num; ++i) - _pi[i] -= min_pot; - } - } - } - - return retVal; - } - - }; //class NetworkSimplexSimple - - ///@} - -} //namespace lemon From 58f931e72d2a62cd3f5a2c4fe69a5fa6070e8757 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 07:46:30 -0800 Subject: [PATCH 47/68] test(gem): consolidate the router, codebook and vector-set tests Reviewer feedback: the GEM router/codebook/vector-set unit tests were way too extensive. Consolidates duplicate setup into shared helpers, merges per-assigner/per-thread-count and per-variant checks into single loops/sections sharing one fixture, folds validation-failure checks into tables, and drops hand-picked route_query cases that were strict subsets of the brute-force reference check already in the suite. Line counts (before -> after, clang-format applied): - tests/svs/core/distances/gem_router.cpp: 302 -> 239 - tests/svs/quantization/gem/codebook.cpp: 563 -> 480 - tests/svs/core/data/vector_set.cpp: 261 -> 222 - tests/svs/core/data/vector_set_view.cpp: 137 -> 124 - total: 1263 -> 1004 (16% fewer) Co-Authored-By: Claude Opus 5.5 --- tests/svs/core/data/vector_set.cpp | 174 +++----- tests/svs/core/data/vector_set_view.cpp | 120 +++--- tests/svs/core/distances/gem_router.cpp | 291 +++++-------- tests/svs/quantization/gem/codebook.cpp | 534 +++++++++--------------- 4 files changed, 430 insertions(+), 689 deletions(-) diff --git a/tests/svs/core/data/vector_set.cpp b/tests/svs/core/data/vector_set.cpp index d90d95c70..5cb0846dc 100644 --- a/tests/svs/core/data/vector_set.cpp +++ b/tests/svs/core/data/vector_set.cpp @@ -33,14 +33,13 @@ // stl #include +#include #include #include #include -#include #include namespace { - // 3 documents with cardinalities 2, 0, 3 over dimension 2, matching the fixture in // vector_set_view.cpp so both headers are checked against the same layout. svs::data::SimpleData make_tokens() { @@ -55,6 +54,20 @@ svs::data::SimpleData make_tokens() { std::vector make_offsets() { return {0, 2, 2, 5}; } +// Every document's tokens, element-wise, must match between two equivalent datasets. +template void require_same_contents(const A& a, const B& b) { + CATCH_REQUIRE(a.size() == b.size()); + for (size_t i = 0; i < a.size(); ++i) { + auto da = a.get_datum(i); + auto db = b.get_datum(i); + CATCH_REQUIRE(da.cardinality() == db.cardinality()); + for (size_t j = 0; j < da.cardinality(); ++j) { + auto ta = da.token(j); + auto tb = db.token(j); + CATCH_REQUIRE(std::equal(ta.begin(), ta.end(), tb.begin())); + } + } +} } // namespace CATCH_TEST_CASE("Vector Set Data", "[core][data][vector_set]") { @@ -72,52 +85,34 @@ CATCH_TEST_CASE("Vector Set Data", "[core][data][vector_set]") { CATCH_SECTION("Accessors") { auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); - CATCH_REQUIRE(container.size() == 3); CATCH_REQUIRE(container.dimensions() == 2); CATCH_REQUIRE(container.total_tokens() == 5); - CATCH_REQUIRE(container.cardinality(0) == 2); - CATCH_REQUIRE(container.cardinality(1) == 0); - CATCH_REQUIRE(container.cardinality(2) == 3); - - auto set0 = container.get_datum(0); - CATCH_REQUIRE(set0.cardinality() == 2); - CATCH_REQUIRE(set0.token(0)[0] == 1.0f); - CATCH_REQUIRE(set0.token(1)[1] == 4.0f); - - auto set1 = container.get_datum(1); - CATCH_REQUIRE(set1.empty()); - - auto set2 = container.get_datum(2); - CATCH_REQUIRE(set2.cardinality() == 3); - CATCH_REQUIRE(set2.token(2)[0] == 9.0f); - CATCH_REQUIRE(set2.token(2)[1] == 10.0f); - - container.prefetch(0); - container.prefetch(1); - container.prefetch(2); - auto offsets = container.offsets(); CATCH_REQUIRE(offsets.size() == 4); CATCH_REQUIRE(offsets[3] == 5); + + // cardinality() and get_datum() against the ground truth, one pass. + std::vector>> expected{ + {{1.0f, 2.0f}, {3.0f, 4.0f}}, {}, {{5.0f, 6.0f}, {7.0f, 8.0f}, {9.0f, 10.0f}}}; + for (size_t i = 0; i < expected.size(); ++i) { + auto set = container.get_datum(i); + CATCH_REQUIRE(set.cardinality() == expected[i].size()); + CATCH_REQUIRE(set.empty() == expected[i].empty()); + for (size_t j = 0; j < expected[i].size(); ++j) { + auto t = set.token(j); + CATCH_REQUIRE(t[0] == expected[i][j][0]); + CATCH_REQUIRE(t[1] == expected[i][j][1]); + } + container.prefetch(i); // exercised for the empty document (i==1) too + } } CATCH_SECTION("view() matches the container element-wise") { auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); auto view = container.view(); - - CATCH_REQUIRE(view.size() == container.size()); CATCH_REQUIRE(view.dimensions() == container.dimensions()); - for (size_t i = 0; i < container.size(); ++i) { - auto a = container.get_datum(i); - auto b = view.get_datum(i); - CATCH_REQUIRE(a.cardinality() == b.cardinality()); - for (size_t j = 0; j < a.cardinality(); ++j) { - auto ta = a.token(j); - auto tb = b.token(j); - CATCH_REQUIRE(std::equal(ta.begin(), ta.end(), tb.begin())); - } - } + require_same_contents(container, view); } CATCH_SECTION("copy_from takes ownership of a copy of a borrowed view") { @@ -127,40 +122,22 @@ CATCH_TEST_CASE("Vector Set Data", "[core][data][vector_set]") { auto container = svs::data::VectorSetData::copy_from(view); CATCH_REQUIRE(container.tokens().data() != tokens.data()); - CATCH_REQUIRE(container.size() == 2); CATCH_REQUIRE(container.cardinality(0) == 1); CATCH_REQUIRE(container.cardinality(1) == 2); CATCH_REQUIRE(container.get_datum(1).token(1)[0] == 5.0f); + require_same_contents(container, view); } - CATCH_SECTION("Validation failures") { + CATCH_SECTION("Constructor delegates to the shared offsets validator") { + // The constructor adds no validation beyond calling validate_vector_set_layout, + // whose branches are already covered against ConstVectorSetView in + // vector_set_view.cpp. CATCH_REQUIRE_THROWS_AS( svs::data::VectorSetData( svs::data::SimpleData(2, 2), std::vector{} ), svs::ANNException ); - - CATCH_REQUIRE_THROWS_AS( - svs::data::VectorSetData( - svs::data::SimpleData(2, 2), std::vector{1, 2} - ), - svs::ANNException - ); - - CATCH_REQUIRE_THROWS_AS( - svs::data::VectorSetData( - svs::data::SimpleData(2, 2), std::vector{0, 2, 1} - ), - svs::ANNException - ); - - CATCH_REQUIRE_THROWS_AS( - svs::data::VectorSetData( - svs::data::SimpleData(2, 2), std::vector{0, 1} - ), - svs::ANNException - ); } CATCH_SECTION("Static extent, integer element type") { @@ -168,13 +145,12 @@ CATCH_TEST_CASE("Vector Set Data", "[core][data][vector_set]") { tokens.set_datum(0, std::vector{7}); tokens.set_datum(1, std::vector{8}); tokens.set_datum(2, std::vector{9}); - auto offsets = std::vector{0, 2, 3}; - auto container = - svs::data::VectorSetData(std::move(tokens), std::move(offsets)); + auto container = svs::data::VectorSetData( + std::move(tokens), std::vector{0, 2, 3} + ); CATCH_REQUIRE(container.dimensions() == 1); CATCH_REQUIRE(container.get_datum(0).token(1)[0] == 8); - static_assert(svs::data::ImmutableVectorSetDataset); } @@ -182,10 +158,10 @@ CATCH_TEST_CASE("Vector Set Data", "[core][data][vector_set]") { auto tokens = svs::data::SimpleData(2, 3); tokens.set_datum(0, std::vector{1.0f, 2.0f, 3.0f}); tokens.set_datum(1, std::vector{4.0f, 5.0f, 6.0f}); - auto offsets = std::vector{0, 1, 2}; - auto container = - svs::data::VectorSetData(std::move(tokens), std::move(offsets)); + auto container = svs::data::VectorSetData( + std::move(tokens), std::vector{0, 1, 2} + ); CATCH_REQUIRE(container.size() == 2); CATCH_REQUIRE(float(container.get_datum(1).token(0)[0]) == 4.0f); } @@ -199,61 +175,43 @@ CATCH_TEST_CASE("Vector Set Data", "[core][data][vector_set]") { static_assert(svs::data::is_blocked_v); } - CATCH_SECTION("Save then load round trip") { + CATCH_SECTION("Save/load") { auto temp_directory = svs_test::prepare_temp_directory_v2(); auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); - svs::lib::save_to_disk(container, temp_directory); - auto reloaded = - svs::lib::load_from_disk>(temp_directory); - CATCH_REQUIRE(reloaded.size() == container.size()); - CATCH_REQUIRE(reloaded.dimensions() == container.dimensions()); - CATCH_REQUIRE(reloaded.total_tokens() == container.total_tokens()); - auto reloaded_offsets = reloaded.offsets(); - auto original_offsets = container.offsets(); - CATCH_REQUIRE(std::equal( - reloaded_offsets.begin(), reloaded_offsets.end(), original_offsets.begin() - )); - for (size_t i = 0; i < container.size(); ++i) { - auto a = container.get_datum(i); - auto b = reloaded.get_datum(i); - CATCH_REQUIRE(a.cardinality() == b.cardinality()); - for (size_t j = 0; j < a.cardinality(); ++j) { - auto ta = a.token(j); - auto tb = b.token(j); - CATCH_REQUIRE(std::equal(ta.begin(), ta.end(), tb.begin())); - } + CATCH_SECTION("Round trip") { + auto reloaded = + svs::lib::load_from_disk>(temp_directory); + CATCH_REQUIRE(reloaded.dimensions() == container.dimensions()); + CATCH_REQUIRE(reloaded.total_tokens() == container.total_tokens()); + auto a = container.offsets(); + auto b = reloaded.offsets(); + CATCH_REQUIRE(std::equal(a.begin(), a.end(), b.begin())); + require_same_contents(container, reloaded); } - } - - CATCH_SECTION("Loading a corrupted offsets file throws") { - auto temp_directory = svs_test::prepare_temp_directory_v2(); - auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); - svs::lib::save_to_disk(container, temp_directory); - bool found_offsets_file = false; - for (auto const& entry : std::filesystem::directory_iterator(temp_directory)) { - if (entry.path().filename().string().starts_with("vector_set_offsets")) { - found_offsets_file = true; - // Truncate the file so its size no longer matches the saved metadata. - std::filesystem::resize_file(entry.path(), 1); + CATCH_SECTION("A corrupted offsets file throws") { + bool found_offsets_file = false; + for (auto const& entry : std::filesystem::directory_iterator(temp_directory)) { + if (entry.path().filename().string().starts_with("vector_set_offsets")) { + found_offsets_file = true; + // Truncate so its size no longer matches the saved metadata. + std::filesystem::resize_file(entry.path(), 1); + } } + CATCH_REQUIRE(found_offsets_file); + CATCH_REQUIRE_THROWS_AS( + svs::lib::load_from_disk>(temp_directory), + svs::ANNException + ); } - CATCH_REQUIRE(found_offsets_file); - - CATCH_REQUIRE_THROWS_AS( - svs::lib::load_from_disk>(temp_directory), - svs::ANNException - ); } CATCH_SECTION("Loading with the wrong schema throws") { auto temp_directory = svs_test::prepare_temp_directory_v2(); // SimpleData has a different, incompatible schema ("uncompressed_data"). - auto other = svs::data::SimpleData(2, 2); - svs::lib::save_to_disk(other, temp_directory); - + svs::lib::save_to_disk(svs::data::SimpleData(2, 2), temp_directory); CATCH_REQUIRE_THROWS( svs::lib::load_from_disk>(temp_directory) ); diff --git a/tests/svs/core/data/vector_set_view.cpp b/tests/svs/core/data/vector_set_view.cpp index 7fe651105..5a8002cbd 100644 --- a/tests/svs/core/data/vector_set_view.cpp +++ b/tests/svs/core/data/vector_set_view.cpp @@ -24,107 +24,85 @@ #include "catch2/catch_test_macros.hpp" // stl +#include #include #include CATCH_TEST_CASE("Vector Set View", "[core][data][vector_set_view]") { + // 3 sets with cardinalities 2, 0, 3 over dimension 2. + auto tokens = + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f}; + auto offsets = std::vector{0, 2, 2, 5}; + CATCH_SECTION("Construction and contents") { - // 3 sets with cardinalities 2, 0, 3 over dimension 2. - auto tokens = std::vector{ - 1.0f, - 2.0f, // set 0, token 0 - 3.0f, - 4.0f, // set 0, token 1 - 5.0f, - 6.0f, // set 2, token 0 - 7.0f, - 8.0f, // set 2, token 1 - 9.0f, - 10.0f // set 2, token 2 - }; - auto offsets = std::vector{0, 2, 2, 5}; auto view = svs::data::ConstVectorSetView(tokens, offsets, 2); - CATCH_REQUIRE(view.size() == 3); CATCH_REQUIRE(view.dimensions() == 2); CATCH_REQUIRE(view.total_tokens() == 5); - CATCH_REQUIRE(view.cardinality(0) == 2); - CATCH_REQUIRE(view.cardinality(1) == 0); - CATCH_REQUIRE(view.cardinality(2) == 3); - - auto set0 = view.get_datum(0); - CATCH_REQUIRE(set0.cardinality() == 2); - CATCH_REQUIRE(set0.dimensions() == 2); - CATCH_REQUIRE(!set0.empty()); - CATCH_REQUIRE(set0.token(0)[0] == 1.0f); - CATCH_REQUIRE(set0.token(0)[1] == 2.0f); - CATCH_REQUIRE(set0.token(1)[0] == 3.0f); - CATCH_REQUIRE(set0.token(1)[1] == 4.0f); - - auto set1 = view.get_datum(1); - CATCH_REQUIRE(set1.cardinality() == 0); - CATCH_REQUIRE(set1.empty()); - - auto set2 = view.get_datum(2); - CATCH_REQUIRE(set2.cardinality() == 3); - CATCH_REQUIRE(set2.token(2)[0] == 9.0f); - CATCH_REQUIRE(set2.token(2)[1] == 10.0f); - - view.prefetch(0); - view.prefetch(1); - view.prefetch(2); + // size(), cardinality() and get_datum() against the ground truth, one pass. + std::vector>> expected{ + {{1.0f, 2.0f}, {3.0f, 4.0f}}, {}, {{5.0f, 6.0f}, {7.0f, 8.0f}, {9.0f, 10.0f}}}; + for (size_t i = 0; i < expected.size(); ++i) { + auto set = view.get_datum(i); + CATCH_REQUIRE(set.cardinality() == expected[i].size()); + CATCH_REQUIRE(set.dimensions() == 2); + CATCH_REQUIRE(set.empty() == expected[i].empty()); + for (size_t j = 0; j < expected[i].size(); ++j) { + auto t = set.token(j); + CATCH_REQUIRE(t[0] == expected[i][j][0]); + CATCH_REQUIRE(t[1] == expected[i][j][1]); + } + view.prefetch(i); // exercised for the empty set (i==1) too + } } - CATCH_SECTION("Validation failures") { - auto tokens = std::vector{1.0f, 2.0f, 3.0f, 4.0f}; - + CATCH_SECTION("Validation failures: one per throw in validate_vector_set_layout") { CATCH_REQUIRE_THROWS_AS( - svs::data::ConstVectorSetView( - std::span(tokens), std::span(), 2 - ), + (svs::data::ConstVectorSetView( + std::span(), std::span(), 2 + )), svs::ANNException - ); - - auto bad_front = std::vector{1, 2}; + ); // offsets.empty() CATCH_REQUIRE_THROWS_AS( - svs::data::ConstVectorSetView(tokens, bad_front, 2), svs::ANNException - ); - - auto decreasing = std::vector{0, 2, 1}; + svs::data::ConstVectorSetView(tokens, std::vector{1, 2}, 2), + svs::ANNException + ); // offsets.front() != 0 CATCH_REQUIRE_THROWS_AS( - svs::data::ConstVectorSetView(tokens, decreasing, 2), svs::ANNException - ); - - auto mismatched = std::vector{0, 1}; + svs::data::ConstVectorSetView(tokens, std::vector{0, 2, 1}, 2), + svs::ANNException + ); // offsets not non-decreasing CATCH_REQUIRE_THROWS_AS( - svs::data::ConstVectorSetView(tokens, mismatched, 2), svs::ANNException - ); - - auto overflowing = std::vector{0, uint64_t{1} << 63}; + svs::data::ConstVectorSetView(tokens, std::vector{0, 1}, 2), + svs::ANNException + ); // offsets.back() * dimensions != token_elements CATCH_REQUIRE_THROWS_AS( - svs::data::ConstVectorSetView(std::span(), overflowing, 2), + svs::data::ConstVectorSetView( + std::span(), std::vector{0, uint64_t{1} << 63}, 2 + ), svs::ANNException - ); + ); // offsets.back() * dimensions overflows size_t } CATCH_SECTION("Static extent") { - auto tokens = std::vector{1.0f, 2.0f, 3.0f, 4.0f}; - auto offsets = std::vector{0, 1, 2}; + auto static_tokens = std::vector{1.0f, 2.0f, 3.0f, 4.0f}; + auto static_offsets = std::vector{0, 1, 2}; - auto view = svs::data::ConstVectorSetView(tokens, offsets, 2); + auto view = + svs::data::ConstVectorSetView(static_tokens, static_offsets, 2); CATCH_REQUIRE(view.dimensions() == 2); CATCH_REQUIRE(view.get_datum(0).token(0)[0] == 1.0f); CATCH_REQUIRE_THROWS_AS( - (svs::data::ConstVectorSetView(tokens, offsets, 2)), svs::ANNException + (svs::data::ConstVectorSetView(static_tokens, static_offsets, 2)), + svs::ANNException ); } - CATCH_SECTION("Non-float element type") { - auto tokens = std::vector{1, 2, 3, 4, 5, 6}; - auto offsets = std::vector{0, 1, 3}; - auto view = svs::data::ConstVectorSetView(tokens, offsets, 2); + CATCH_SECTION("Non-float element type, concept checks") { + auto byte_tokens = std::vector{1, 2, 3, 4, 5, 6}; + auto byte_offsets = std::vector{0, 1, 3}; + auto view = svs::data::ConstVectorSetView(byte_tokens, byte_offsets, 2); CATCH_REQUIRE(view.size() == 2); CATCH_REQUIRE(view.cardinality(0) == 1); diff --git a/tests/svs/core/distances/gem_router.cpp b/tests/svs/core/distances/gem_router.cpp index c08408312..6b2d73951 100644 --- a/tests/svs/core/distances/gem_router.cpp +++ b/tests/svs/core/distances/gem_router.cpp @@ -30,228 +30,171 @@ // stl #include #include -#include #include #include #include #include -#include #include namespace { - using svs::distance::GemRouter; constexpr uint32_t NO_REP = GemRouter::no_representative; -// 6 fine centroids grouped into 4 coarse clusters, leaving cluster 3 with no member. -std::vector hand_coarse_of_fine() { return {0, 0, 1, 1, 2, 2}; } - -// Hand-built corpus where doc3 and doc4 tie for cluster 2; doc3 (lower id) must win. -struct HandCorpus { - std::vector ids{0, 0, 2, 0, 1, 1, 2, 3, 4, 4}; - std::vector offsets{0, 3, 6, 8, 9, 10}; - - svs::data::ConstVectorSetView view() const { - return svs::data::ConstVectorSetView( - std::span(ids), std::span(offsets), 1 - ); - } -}; - -// Larger randomized corpus, for the thread-count-independence check. -struct RandomCorpus { - std::vector ids; - std::vector offsets; +svs::data::ConstVectorSetView +view_of(const std::vector& ids, const std::vector& offsets) { + return svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); +} - svs::data::ConstVectorSetView view() const { - return svs::data::ConstVectorSetView( - std::span(ids), std::span(offsets), 1 - ); - } -}; +// 6 fine centroids / 4 coarse clusters (cluster 3 empty); doc3/doc4 tie for cluster 2 and +// doc3 (lower id) must win. +std::vector hand_coarse_of_fine() { return {0, 0, 1, 1, 2, 2}; } +std::vector hand_ids() { return {0, 0, 2, 0, 1, 1, 2, 3, 4, 4}; } +std::vector hand_offsets() { return {0, 3, 6, 8, 9, 10}; } -RandomCorpus make_random_corpus(size_t n, size_t k, uint32_t seed) { - std::mt19937 rng(seed); +std::vector random_ids(size_t n, size_t k, std::vector& offsets_out) { + std::mt19937 rng(0xC0FFEE); std::uniform_int_distribution id_dist(0, static_cast(k - 1)); - RandomCorpus corpus; - corpus.offsets.resize(n + 1, 0); + std::vector ids; + offsets_out.assign(n + 1, 0); for (size_t i = 0; i < n; ++i) { size_t cardinality = 1 + (i % 7); - corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + offsets_out[i + 1] = offsets_out[i] + cardinality; for (size_t t = 0; t < cardinality; ++t) { - corpus.ids.push_back(id_dist(rng)); + ids.push_back(id_dist(rng)); } } - return corpus; + return ids; } - } // namespace -CATCH_TEST_CASE("GemRouter representative rule", "[distance][gem][gem_router]") { - auto corpus = HandCorpus(); - auto docs = corpus.view(); +CATCH_TEST_CASE("GemRouter build", "[distance][gem][gem_router]") { + auto ids = hand_ids(); + auto offsets = hand_offsets(); + auto docs = view_of(ids, offsets); + + CATCH_SECTION("Representative rule: tie to lower doc id, untouched cluster is NO_REP") { + auto router = + GemRouter::build(hand_coarse_of_fine(), /*num_coarse=*/4, docs, size_t{1}); + CATCH_REQUIRE(router.num_fine() == 6); + CATCH_REQUIRE(router.num_coarse() == 4); + CATCH_REQUIRE(router.representative[0] == 1); + CATCH_REQUIRE(router.representative[1] == 2); + CATCH_REQUIRE(router.representative[2] == 3); // tie broken to the lower doc id + CATCH_REQUIRE(router.representative[3] == NO_REP); // untouched cluster + } - auto router = - GemRouter::build(hand_coarse_of_fine(), /*num_coarse=*/4, docs, /*num_threads=*/1); + CATCH_SECTION("Out-of-range coarse id is rejected") { + auto bad = hand_coarse_of_fine(); + bad[2] = 99; // out of [0, num_coarse) + CATCH_REQUIRE_THROWS(GemRouter::build(bad, 4, docs, size_t{1})); + } - CATCH_REQUIRE(router.num_fine() == 6); - CATCH_REQUIRE(router.num_coarse() == 4); - CATCH_REQUIRE(router.representative[0] == 1); - CATCH_REQUIRE(router.representative[1] == 2); - CATCH_REQUIRE(router.representative[2] == 3); // tie broken to the lower doc id - CATCH_REQUIRE(router.representative[3] == NO_REP); // untouched cluster -} + CATCH_SECTION("Result is independent of thread count") { + constexpr size_t k = 24; + constexpr size_t num_coarse = 6; + std::vector random_offsets; + auto random_id_list = random_ids(400, k, random_offsets); + auto random_docs = view_of(random_id_list, random_offsets); -CATCH_TEST_CASE( - "GemRouter representative rule rejects bad input", "[distance][gem][gem_router]" -) { - auto corpus = HandCorpus(); - auto docs = corpus.view(); + std::vector coarse_of_fine(k); + std::mt19937 rng(0xFEED); + std::uniform_int_distribution cluster_dist(0, num_coarse - 1); + for (auto& c : coarse_of_fine) { + c = cluster_dist(rng); + } - auto bad_coarse_of_fine = hand_coarse_of_fine(); - bad_coarse_of_fine[2] = 99; // out of [0, num_coarse) - CATCH_REQUIRE_THROWS(GemRouter::build(bad_coarse_of_fine, 4, docs, size_t{1})); + auto router_1 = + GemRouter::build(coarse_of_fine, num_coarse, random_docs, size_t{1}); + auto router_4 = + GemRouter::build(coarse_of_fine, num_coarse, random_docs, size_t{4}); + CATCH_REQUIRE(router_1.representative == router_4.representative); + CATCH_REQUIRE(router_1.coarse_of_fine == router_4.coarse_of_fine); + } } +// Covers route_query's whole spec at once (nprobe cap, NO_REP skip, dedup across tokens, +// tie-breaking to the lower cluster id): any hand-picked case is a strict instance of it. CATCH_TEST_CASE( - "GemRouter build is independent of thread count", "[distance][gem][gem_router]" + "GemRouter route_query matches a brute-force reference", "[distance][gem][gem_router]" ) { - constexpr size_t k = 24; - constexpr size_t num_coarse = 6; - auto corpus = make_random_corpus(400, k, 0xC0FFEE); - auto docs = corpus.view(); + constexpr size_t k = 20; + constexpr size_t num_coarse = 5; + constexpr size_t n_query = 6; + constexpr size_t nprobe = 2; - std::vector coarse_of_fine(k); - std::mt19937 rng(0xFEED); + std::mt19937 rng(0xABCDEF); std::uniform_int_distribution cluster_dist(0, num_coarse - 1); - for (auto& c : coarse_of_fine) { + std::uniform_real_distribution score_dist(-5.0F, 5.0F); + std::uniform_int_distribution rep_dist(0, 99); + std::bernoulli_distribution has_rep(0.7); // some clusters get NO_REP + + GemRouter router; + router.coarse_of_fine.resize(k); + for (auto& c : router.coarse_of_fine) { c = cluster_dist(rng); } - - auto router_1 = GemRouter::build(coarse_of_fine, num_coarse, docs, size_t{1}); - auto router_4 = GemRouter::build(coarse_of_fine, num_coarse, docs, size_t{4}); - - CATCH_REQUIRE(router_1.representative == router_4.representative); - CATCH_REQUIRE(router_1.coarse_of_fine == router_4.coarse_of_fine); -} - -CATCH_TEST_CASE("GemRouter route_query", "[distance][gem][gem_router]") { - CATCH_SECTION("Hand example: nprobe=1, no_representative is skipped") { - GemRouter router; - router.coarse_of_fine = {0, 0, 1, 1}; - router.representative = {10, NO_REP}; - router.nprobe = 1; - - // table[f * n_query + i], K=4, n_query=2. - std::vector table{ - 5.0F, - 1.0F, // f0 - 3.0F, - 9.0F, // f1 - 2.0F, - 2.0F, // f2 - 8.0F, - 0.0F // f3 - }; - GemRouter::QueryScratch scratch; - std::vector out; - router.route_query(table.data(), 2, out, scratch); - CATCH_REQUIRE(out == std::vector{10}); + router.representative.resize(num_coarse); + for (auto& r : router.representative) { + r = has_rep(rng) ? rep_dist(rng) : NO_REP; } + router.nprobe = nprobe; - CATCH_SECTION("Duplicate representatives across tokens collapse to one entry") { - GemRouter router; - router.coarse_of_fine = {0, 0, 1, 1}; - router.representative = {7, 7}; - router.nprobe = 2; // both clusters always selected - std::vector table{1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F, 7.0F, 8.0F}; - GemRouter::QueryScratch scratch; - std::vector out; - router.route_query(table.data(), 2, out, scratch); - CATCH_REQUIRE(out == std::vector{7}); + std::vector table(k * n_query); // table[f * n_query + i] + for (auto& v : table) { + v = score_dist(rng); } - CATCH_SECTION("Matches a brute-force reference on a random table") { - constexpr size_t k = 20; - constexpr size_t num_coarse = 5; - constexpr size_t n_query = 6; - constexpr size_t nprobe = 2; + GemRouter::QueryScratch scratch; + std::vector out; + router.route_query(table.data(), n_query, out, scratch); - std::mt19937 rng(0xABCDEF); - std::uniform_int_distribution cluster_dist(0, num_coarse - 1); - std::uniform_real_distribution score_dist(-5.0F, 5.0F); - std::uniform_int_distribution rep_dist(0, 99); - std::bernoulli_distribution has_rep(0.7); - - GemRouter router; - router.coarse_of_fine.resize(k); - for (auto& c : router.coarse_of_fine) { - c = cluster_dist(rng); - } - router.representative.resize(num_coarse); - for (auto& r : router.representative) { - r = has_rep(rng) ? rep_dist(rng) : NO_REP; + // Independent reference implementation of the same spec. + std::vector expected; + for (size_t i = 0; i < n_query; ++i) { + std::vector coarse_score( + num_coarse, -std::numeric_limits::infinity() + ); + for (size_t f = 0; f < k; ++f) { + coarse_score[router.coarse_of_fine[f]] = + std::max(coarse_score[router.coarse_of_fine[f]], table[f * n_query + i]); } - router.nprobe = nprobe; - - std::vector table(k * n_query); - for (auto& v : table) { - v = score_dist(rng); + std::vector order(num_coarse); + for (size_t c = 0; c < num_coarse; ++c) { + order[c] = static_cast(c); } - - GemRouter::QueryScratch scratch; - std::vector out; - router.route_query(table.data(), n_query, out, scratch); - - // Independent reference implementation of the same spec. - std::vector expected; - for (size_t i = 0; i < n_query; ++i) { - std::vector coarse_score( - num_coarse, -std::numeric_limits::infinity() - ); - for (size_t f = 0; f < k; ++f) { - coarse_score[router.coarse_of_fine[f]] = std::max( - coarse_score[router.coarse_of_fine[f]], table[f * n_query + i] - ); - } - std::vector order(num_coarse); - for (size_t c = 0; c < num_coarse; ++c) { - order[c] = static_cast(c); + std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) { + if (coarse_score[a] != coarse_score[b]) { + return coarse_score[a] > coarse_score[b]; } - std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) { - if (coarse_score[a] != coarse_score[b]) { - return coarse_score[a] > coarse_score[b]; - } - return a < b; - }); - size_t take = std::min(nprobe, num_coarse); - for (size_t j = 0; j < take; ++j) { - uint32_t rep = router.representative[order[j]]; - if (rep != NO_REP) { - expected.push_back(rep); - } + return a < b; + }); + for (size_t j = 0; j < std::min(nprobe, num_coarse); ++j) { + uint32_t rep = router.representative[order[j]]; + if (rep != NO_REP) { + expected.push_back(rep); } } - std::sort(expected.begin(), expected.end()); - expected.erase(std::unique(expected.begin(), expected.end()), expected.end()); - - CATCH_REQUIRE(out == expected); - CATCH_REQUIRE(std::is_sorted(out.begin(), out.end())); - CATCH_REQUIRE(std::adjacent_find(out.begin(), out.end()) == out.end()); } + std::sort(expected.begin(), expected.end()); + expected.erase(std::unique(expected.begin(), expected.end()), expected.end()); + + CATCH_REQUIRE(out == expected); + CATCH_REQUIRE(std::is_sorted(out.begin(), out.end())); + CATCH_REQUIRE(std::adjacent_find(out.begin(), out.end()) == out.end()); } CATCH_TEST_CASE("GemRouter route_ids", "[distance][gem][gem_router]") { GemRouter router; router.coarse_of_fine = {0, 0, 1, 1, 2, 2}; router.representative = {100, 200, 300}; + GemRouter::IdScratch scratch; + std::vector out; CATCH_SECTION("Ranked by count desc, ties to the lower cluster id") { std::vector ids{0, 0, 2, 4}; - GemRouter::IdScratch scratch; - std::vector out; - router.route_ids(ids, 2, out, scratch); CATCH_REQUIRE(out == std::vector{100, 200}); @@ -260,13 +203,9 @@ CATCH_TEST_CASE("GemRouter route_ids", "[distance][gem][gem_router]") { } CATCH_SECTION("no_representative clusters are skipped") { - GemRouter sparse = router; - sparse.representative = {100, NO_REP, 300}; + router.representative = {100, NO_REP, 300}; std::vector ids{2, 3}; // both tokens map to coarse1 - GemRouter::IdScratch scratch; - std::vector out; - - sparse.route_ids(ids, 5, out, scratch); + router.route_ids(ids, 5, out, scratch); CATCH_REQUIRE(out.empty()); } } @@ -279,10 +218,9 @@ CATCH_TEST_CASE("GemRouter save/load", "[distance][gem][gem_router]") { router.coarse_of_fine = hand_coarse_of_fine(); router.representative = {1, 2, 3, NO_REP}; router.nprobe = 7; - router.save(path); - auto loaded = GemRouter::load(path); + auto loaded = GemRouter::load(path); CATCH_REQUIRE(loaded.coarse_of_fine == router.coarse_of_fine); CATCH_REQUIRE(loaded.representative == router.representative); CATCH_REQUIRE(loaded.nprobe == router.nprobe); @@ -290,7 +228,6 @@ CATCH_TEST_CASE("GemRouter save/load", "[distance][gem][gem_router]") { CATCH_SECTION("A corrupted file is rejected") { // Flip a byte inside the magic field. std::fstream f(path, std::ios::binary | std::ios::in | std::ios::out); - f.seekp(0); char byte = 0; f.read(&byte, 1); f.seekp(0); diff --git a/tests/svs/quantization/gem/codebook.cpp b/tests/svs/quantization/gem/codebook.cpp index 1aa87ae5d..edf7cd024 100644 --- a/tests/svs/quantization/gem/codebook.cpp +++ b/tests/svs/quantization/gem/codebook.cpp @@ -36,9 +36,7 @@ #include namespace { - namespace gem = svs::quantization::gem; - constexpr float TOL = 1e-4F; std::vector random_unit_vector(std::mt19937& rng, size_t dim) { @@ -56,6 +54,26 @@ std::vector random_unit_vector(std::mt19937& rng, size_t dim) { return v; } +std::vector make_random_centers(size_t K, size_t dim, uint32_t seed) { + std::mt19937 rng(seed); + std::vector centers(K * dim); + for (size_t k = 0; k < K; ++k) { + auto c = random_unit_vector(rng, dim); + std::copy(c.begin(), c.end(), centers.begin() + static_cast(k * dim)); + } + return centers; +} + +// Fine centroids as a SimpleData, laid out row-major from a flat `centers` buffer. +svs::data::SimpleData +centroids_from(const std::vector& centers, size_t K, size_t dim) { + svs::data::SimpleData centroids(K, dim); + for (size_t k = 0; k < K; ++k) { + centroids.set_datum(k, std::span(centers.data() + k * dim, dim)); + } + return centroids; +} + // Each token perturbs a center with a large margin, so even an approximate assigner agrees. svs::data::VectorSetData make_clustered_corpus( const std::vector& centers, @@ -71,7 +89,6 @@ svs::data::VectorSetData make_clustered_corpus( for (auto c : cardinalities) { total_tokens += c; } - svs::data::SimpleData token_matrix(total_tokens, dim); std::vector offsets(num_passages + 1, 0); std::uniform_int_distribution pick(0, K - 1); @@ -99,16 +116,6 @@ svs::data::VectorSetData make_clustered_corpus( return svs::data::VectorSetData(std::move(token_matrix), std::move(offsets)); } -std::vector make_random_centers(size_t K, size_t dim, uint32_t seed) { - std::mt19937 rng(seed); - std::vector centers(K * dim); - for (size_t k = 0; k < K; ++k) { - auto c = random_unit_vector(rng, dim); - std::copy(c.begin(), c.end(), centers.begin() + static_cast(k * dim)); - } - return centers; -} - uint32_t brute_force_best( std::span token, const svs::data::SimpleData& centroids ) { @@ -148,37 +155,37 @@ rebuild_ids(const std::vector& flat, std::vector offsets) { } return svs::data::VectorSetData(std::move(storage), std::move(offsets)); } - } // namespace -CATCH_TEST_CASE("GEM Codebook - colbert_num_partitions", "[quantization][gem]") { - // Exact powers of two: 16 * sqrt(1) == 16, 16 * sqrt(4) == 32. - CATCH_REQUIRE(gem::colbert_num_partitions(1) == 16); - CATCH_REQUIRE(gem::colbert_num_partitions(4) == 32); - // One token below/above the num_tokens=4 boundary (raw crosses exactly 32 at n=4). - CATCH_REQUIRE(gem::colbert_num_partitions(3) == 16); - CATCH_REQUIRE(gem::colbert_num_partitions(5) == 32); - CATCH_REQUIRE_THROWS(gem::colbert_num_partitions(0)); -} +CATCH_TEST_CASE("GEM Codebook - sizing rules", "[quantization][gem]") { + CATCH_SECTION("colbert_num_partitions") { + // Exact powers of two: 16 * sqrt(1) == 16, 16 * sqrt(4) == 32. + CATCH_REQUIRE(gem::colbert_num_partitions(1) == 16); + CATCH_REQUIRE(gem::colbert_num_partitions(4) == 32); + // One token below/above the num_tokens=4 boundary (raw crosses exactly 32 at n=4). + CATCH_REQUIRE(gem::colbert_num_partitions(3) == 16); + CATCH_REQUIRE(gem::colbert_num_partitions(5) == 32); + CATCH_REQUIRE_THROWS(gem::colbert_num_partitions(0)); + } -CATCH_TEST_CASE("GEM Codebook - colbert_sample_size", "[quantization][gem]") { - // Small corpora: the formula's raw value exceeds num_passages, so it is capped. - CATCH_REQUIRE(gem::colbert_sample_size(1) == 1); - CATCH_REQUIRE(gem::colbert_sample_size(10) == 10); - // Large corpus: 1 + floor(16 * sqrt(120 * 1'000'000)) is well under num_passages. - size_t num_passages = 1'000'000; - size_t expected = - 1 + static_cast( - std::floor(16.0 * std::sqrt(120.0 * static_cast(num_passages))) - ); - CATCH_REQUIRE(gem::colbert_sample_size(num_passages) == expected); - CATCH_REQUIRE(expected < num_passages); - CATCH_REQUIRE_THROWS(gem::colbert_sample_size(0)); - - // Perfect-square boundary (921'600 == 960^2); integer sqrt must not be off by one. - CATCH_REQUIRE(gem::detail::isqrt_floor(921'600) == 960); - CATCH_REQUIRE(gem::detail::isqrt_floor(921'599) == 959); - CATCH_REQUIRE(gem::detail::isqrt_floor(921'601) == 960); + CATCH_SECTION("colbert_sample_size") { + // Small corpora: the formula's raw value exceeds num_passages, so it is capped. + CATCH_REQUIRE(gem::colbert_sample_size(1) == 1); + CATCH_REQUIRE(gem::colbert_sample_size(10) == 10); + // Large corpus: 1 + floor(16 * sqrt(120 * 1'000'000)) is well under num_passages. + size_t num_passages = 1'000'000; + size_t expected = + 1 + static_cast( + std::floor(16.0 * std::sqrt(120.0 * static_cast(num_passages))) + ); + CATCH_REQUIRE(gem::colbert_sample_size(num_passages) == expected); + CATCH_REQUIRE(expected < num_passages); + CATCH_REQUIRE_THROWS(gem::colbert_sample_size(0)); + // Perfect-square boundary (921'600 == 960^2); integer sqrt must not be off by one. + CATCH_REQUIRE(gem::detail::isqrt_floor(921'600) == 960); + CATCH_REQUIRE(gem::detail::isqrt_floor(921'599) == 959); + CATCH_REQUIRE(gem::detail::isqrt_floor(921'601) == 960); + } } CATCH_TEST_CASE( @@ -190,7 +197,6 @@ CATCH_TEST_CASE( data.set_datum(1, std::vector{0.0F, 2.0F}); data.set_datum(2, std::vector{10.0F, 0.0F}); data.set_datum(3, std::vector{10.0F, 2.0F}); - auto centroids = svs::data::SimpleData(2, 2); centroids.set_datum(0, std::vector{0.0F, 0.0F}); centroids.set_datum(1, std::vector{10.0F, 0.0F}); @@ -198,7 +204,6 @@ CATCH_TEST_CASE( std::vector assignments{0, 0, 1, 1}; std::vector counts(2); auto threadpool = svs::threads::as_threadpool(2); - gem::detail::plain_mean_update(data, centroids, assignments, counts, threadpool); auto c0 = centroids.get_datum(0); @@ -212,308 +217,184 @@ CATCH_TEST_CASE( CATCH_REQUIRE(std::abs(c0[1] - (2.0F / 3.0F)) > TOL); } -CATCH_TEST_CASE( - "GEM Codebook - train_codebook centroids are unit norm, objective is non-increasing", - "[quantization][gem]" -) { - constexpr size_t K = 8; +// One small corpus drives the whole pipeline; each SECTION below asserts a different group +// of properties on the shared `result`/`corpus` instead of rebuilding its own input. Cases +// that genuinely need different input (thread count, byte budget) get their own SECTION +// too, still off this one corpus; only parameters/sizing that need a different seed/shape +// get their own TEST_CASE (sizing rules, plain-mean-update, failure paths). +CATCH_TEST_CASE("GEM Codebook - quantize pipeline", "[quantization][gem]") { constexpr size_t DIM = 16; - auto centers = make_random_centers(K, DIM, 0x1234); - - std::vector cardinalities(50, 6); - auto corpus = make_clustered_corpus(centers, K, DIM, cardinalities, 0xABCD, 0.05F); - - gem::CodebookParameters parameters; - parameters.num_centroids = K; - parameters.seed = 7; - + constexpr size_t NUM_PASSAGES = 100; + auto centers = make_random_centers(16, DIM, 0x9999); + auto corpus = make_clustered_corpus( + centers, 16, DIM, std::vector(NUM_PASSAGES, 2), 0x1357, 0.1F + ); + gem::CodebookParameters parameters; // num_centroids == 0: use the default rule. auto threadpool = svs::threads::as_threadpool(2); - - parameters.kmeans_iterations = 1; - auto one_iter = gem::train_codebook(corpus, parameters, threadpool); - - parameters.kmeans_iterations = 4; - auto four_iter = gem::train_codebook(corpus, parameters, threadpool); - - // Same seed and sampling: more passes cannot raise the final objective. - CATCH_REQUIRE(four_iter.objective <= one_iter.objective + 1e-3); - - for (size_t i = 0; i < four_iter.centroids.size(); ++i) { - auto c = four_iter.centroids.get_datum(i); - float norm_sq = 0.0F; - for (size_t d = 0; d < c.size(); ++d) { - norm_sq += c[d] * c[d]; + auto result = gem::quantize(corpus, parameters, threadpool); + size_t expected_K = gem::colbert_num_partitions(NUM_PASSAGES * 2); + + CATCH_SECTION( + "Pipeline wiring, default coarse rule, unit-norm, non-increasing objective" + ) { + CATCH_REQUIRE(result.K == expected_K); + CATCH_REQUIRE(result.centroids.size() == expected_K); + CATCH_REQUIRE(result.centroids.dimensions() == DIM); + CATCH_REQUIRE(result.fine_centroids->K == expected_K); + CATCH_REQUIRE(result.fine_centroids->dim == DIM); + CATCH_REQUIRE(result.centroid_table->K == expected_K); + CATCH_REQUIRE(result.ids.total_tokens() == NUM_PASSAGES * 2); + for (size_t p = 0; p < NUM_PASSAGES; ++p) { + CATCH_REQUIRE(result.ids.cardinality(p) == 2); } - CATCH_REQUIRE(std::abs(std::sqrt(norm_sq) - 1.0F) < 1e-4F); - } -} - -CATCH_TEST_CASE( - "GEM Codebook - train_codebook_timed records one iteration second per k-means pass", - "[quantization][gem]" -) { - constexpr size_t K = 8; - constexpr size_t DIM = 16; - auto centers = make_random_centers(K, DIM, 0x1234); - - std::vector cardinalities(50, 6); - auto corpus = make_clustered_corpus(centers, K, DIM, cardinalities, 0xABCD, 0.05F); - - gem::CodebookParameters parameters; - parameters.num_centroids = K; - parameters.seed = 7; - parameters.kmeans_iterations = 4; - - auto threadpool = svs::threads::as_threadpool(2); - auto [trained, timings] = gem::train_codebook_timed(corpus, parameters, threadpool); - - CATCH_REQUIRE(timings.train_iterations.size() == 4); - CATCH_REQUIRE(timings.sample >= 0.0); - CATCH_REQUIRE(timings.train >= 0.0); - CATCH_REQUIRE(timings.normalize >= 0.0); - CATCH_REQUIRE(timings.total >= 0.0); - CATCH_REQUIRE(trained.num_sampled_passages > 0); -} - -CATCH_TEST_CASE( - "GEM Codebook - assign_tokens agrees with brute force, both assigners, both thread " - "counts", - "[quantization][gem]" -) { - constexpr size_t K = 64; - constexpr size_t DIM = 128; - auto centers = make_random_centers(K, DIM, 0x5EED); - - std::vector cardinalities; - for (size_t i = 0; i < 300; ++i) { - cardinalities.push_back(1 + (i % 6)); - } - auto corpus = make_clustered_corpus(centers, K, DIM, cardinalities, 0xC0FFEE, 0.02F); - - svs::data::SimpleData centroids(K, DIM); - for (size_t k = 0; k < K; ++k) { - centroids.set_datum(k, std::span(centers.data() + k * DIM, DIM)); - } - - gem::CodebookParameters parameters; - parameters.assign_candidates = 8; - - for (auto assigner : {gem::Assigner::ExactGemm, gem::Assigner::InnerProductMatrix}) { - parameters.assigner = assigner; - - auto pool1 = svs::threads::as_threadpool(1); - auto pool4 = svs::threads::as_threadpool(4); - auto ids1 = gem::assign_tokens(corpus, centroids, parameters, pool1); - auto ids4 = gem::assign_tokens(corpus, centroids, parameters, pool4); - - auto flat1 = flatten_ids(ids1); - auto flat4 = flatten_ids(ids4); - CATCH_REQUIRE(flat1 == flat4); - - // Cardinalities of the id sets match the input token store. - for (size_t p = 0; p < corpus.size(); ++p) { - CATCH_REQUIRE(ids1.cardinality(p) == corpus.cardinality(p)); + // Default num_coarse rule (num_coarse == 0 in `parameters`). + CATCH_REQUIRE(result.num_coarse == std::max(1, expected_K * 5 / 32)); + CATCH_REQUIRE(result.coarse_of_fine.size() == expected_K); + for (auto c : result.coarse_of_fine) { + CATCH_REQUIRE(c < result.num_coarse); } - - // Every id agrees with an independent brute-force argmax. - size_t global = 0; - for (size_t p = 0; p < corpus.size(); ++p) { - auto doc = corpus.get_datum(p); - for (size_t t = 0; t < doc.cardinality(); ++t) { - uint32_t expected = brute_force_best(doc.token(t), centroids); - CATCH_REQUIRE(flat1[global] == expected); - ++global; + for (size_t i = 0; i < result.centroids.size(); ++i) { + auto c = result.centroids.get_datum(i); + float norm_sq = 0.0F; + for (size_t d = 0; d < c.size(); ++d) { + norm_sq += c[d] * c[d]; } + CATCH_REQUIRE(std::abs(std::sqrt(norm_sq) - 1.0F) < 1e-4F); } - auto report = gem::count_assignment_mismatches(corpus, centroids, ids1, 1000, 99); - CATCH_REQUIRE(report.mismatches == 0); - CATCH_REQUIRE(report.sample_size == 1000); - - // Corrupt a few ids and confirm the checker notices. - auto offsets = std::vector(ids1.offsets().begin(), ids1.offsets().end()); - auto corrupted = flat1; - for (size_t i = 0; i < corrupted.size(); i += 37) { - corrupted[i] = static_cast((corrupted[i] + 1) % K); - } - auto corrupted_ids = rebuild_ids(corrupted, offsets); - auto bad_report = - gem::count_assignment_mismatches(corpus, centroids, corrupted_ids, 2000, 99); - CATCH_REQUIRE(bad_report.mismatches > 0); + // Same seed and sampling: more k-means passes cannot raise the final objective. + auto one_iter_params = parameters; + one_iter_params.kmeans_iterations = 1; + auto one_iter = gem::train_codebook(corpus, one_iter_params, threadpool); + auto four_iter_params = parameters; + four_iter_params.kmeans_iterations = 4; + auto four_iter = gem::train_codebook(corpus, four_iter_params, threadpool); + CATCH_REQUIRE(four_iter.objective <= one_iter.objective + 1e-3); } -} -CATCH_TEST_CASE( - "GEM Codebook - assign_tokens batch row count comes from a byte budget", - "[quantization][gem]" -) { - constexpr size_t K = 64; - constexpr size_t DIM = 32; - auto centers = make_random_centers(K, DIM, 0xBEEF); - - std::vector cardinalities; - for (size_t i = 0; i < 40; ++i) { - cardinalities.push_back(1 + (i % 5)); - } - auto corpus = make_clustered_corpus(centers, K, DIM, cardinalities, 0xFACE, 0.02F); - - svs::data::SimpleData centroids(K, DIM); - for (size_t k = 0; k < K; ++k) { - centroids.set_datum(k, std::span(centers.data() + k * DIM, DIM)); - } - - auto threadpool = svs::threads::as_threadpool(2); - - for (auto assigner : {gem::Assigner::ExactGemm, gem::Assigner::InnerProductMatrix}) { - gem::CodebookParameters default_parameters; - default_parameters.assigner = assigner; - auto default_ids = - gem::assign_tokens(corpus, centroids, default_parameters, threadpool); - auto default_flat = flatten_ids(default_ids); - - // K*4 bytes/row: 256 forces a 1-row batch, 900 forces a 3-row batch. - for (size_t budget : {size_t{256}, size_t{900}}) { - gem::CodebookParameters tiny_parameters = default_parameters; - tiny_parameters.assign_buffer_bytes = budget; - auto tiny_ids = - gem::assign_tokens(corpus, centroids, tiny_parameters, threadpool); - CATCH_REQUIRE(flatten_ids(tiny_ids) == default_flat); + CATCH_SECTION("coarse_clusters: explicit num_coarse is honored") { + gem::CodebookParameters explicit_params = parameters; + explicit_params.num_coarse = 5; + auto explicit_coarse = + gem::coarse_clusters(result.centroids, explicit_params, threadpool); + CATCH_REQUIRE(explicit_coarse.num_coarse == 5); + for (auto c : explicit_coarse.coarse_of_fine) { + CATCH_REQUIRE(c < 5); } } -} - -CATCH_TEST_CASE("GEM Codebook - coarse_clusters", "[quantization][gem]") { - constexpr size_t K = 64; - constexpr size_t DIM = 16; - auto centers = make_random_centers(K, DIM, 0x777); - svs::data::SimpleData centroids(K, DIM); - for (size_t k = 0; k < K; ++k) { - centroids.set_datum(k, std::span(centers.data() + k * DIM, DIM)); - } - - auto threadpool = svs::threads::as_threadpool(2); - CATCH_SECTION("default num_coarse rule") { - gem::CodebookParameters parameters; - auto result = gem::coarse_clusters(centroids, parameters, threadpool); - CATCH_REQUIRE(result.num_coarse == std::max(1, K * 5 / 32)); - CATCH_REQUIRE(result.coarse_of_fine.size() == K); - for (auto c : result.coarse_of_fine) { - CATCH_REQUIRE(c < result.num_coarse); - } - } + CATCH_SECTION( + "assign_tokens agrees with brute force and the mismatch checker, both assigners" + ) { + for (auto assigner : + {gem::Assigner::ExactGemm, gem::Assigner::InnerProductMatrix}) { + gem::CodebookParameters assign_params = parameters; + assign_params.assigner = assigner; + auto ids = + gem::assign_tokens(corpus, result.centroids, assign_params, threadpool); + auto flat = flatten_ids(ids); + + size_t global = 0; + for (size_t p = 0; p < corpus.size(); ++p) { + auto doc = corpus.get_datum(p); + for (size_t t = 0; t < doc.cardinality(); ++t) { + CATCH_REQUIRE( + flat[global++] == brute_force_best(doc.token(t), result.centroids) + ); + } + } - CATCH_SECTION("explicit num_coarse") { - gem::CodebookParameters parameters; - parameters.num_coarse = 5; - auto result = gem::coarse_clusters(centroids, parameters, threadpool); - CATCH_REQUIRE(result.num_coarse == 5); - for (auto c : result.coarse_of_fine) { - CATCH_REQUIRE(c < 5); + auto report = + gem::count_assignment_mismatches(corpus, result.centroids, ids, 150, 99); + CATCH_REQUIRE(report.mismatches == 0); + CATCH_REQUIRE(report.sample_size == 150); + + // Corrupt a few ids and confirm the checker notices. + auto offsets = + std::vector(ids.offsets().begin(), ids.offsets().end()); + auto corrupted = flat; + for (size_t i = 0; i < corrupted.size(); i += 37) { + corrupted[i] = static_cast((corrupted[i] + 1) % expected_K); + } + auto bad_report = gem::count_assignment_mismatches( + corpus, result.centroids, rebuild_ids(corrupted, offsets), 2000, 99 + ); + CATCH_REQUIRE(bad_report.mismatches > 0); } } - CATCH_SECTION("num_coarse exceeding K throws") { - gem::CodebookParameters parameters; - parameters.num_coarse = K + 1; - CATCH_REQUIRE_THROWS(gem::coarse_clusters(centroids, parameters, threadpool)); - } -} - -CATCH_TEST_CASE( - "GEM Codebook - quantize end to end on a small synthetic set", "[quantization][gem]" -) { - constexpr size_t DIM = 16; - constexpr size_t NUM_PASSAGES = 200; - // 200 passages * 2 tokens == 400; colbert_num_partitions(400) == 256, still exceeded. - std::vector cardinalities(NUM_PASSAGES, 2); - auto centers = make_random_centers(16, DIM, 0x9999); - auto corpus = make_clustered_corpus(centers, 16, DIM, cardinalities, 0x1357, 0.1F); - - CATCH_REQUIRE(corpus.total_tokens() == 400); - CATCH_REQUIRE(gem::colbert_num_partitions(corpus.total_tokens()) == 256); - - gem::CodebookParameters parameters; // num_centroids == 0: use the rule. - auto threadpool = svs::threads::as_threadpool(2); - auto result = gem::quantize(corpus, parameters, threadpool); + // Thread partitioning and batch sizing happen before the per-assigner flush branch, so + // neither depends on `assigner`; one (the default, InnerProductMatrix) is enough. + CATCH_SECTION("assign_tokens: thread-count independence and byte-budget batching") { + auto pool1 = svs::threads::as_threadpool(1); + auto pool4 = svs::threads::as_threadpool(4); + auto flat1 = + flatten_ids(gem::assign_tokens(corpus, result.centroids, parameters, pool1)); + CATCH_REQUIRE( + flat1 == + flatten_ids(gem::assign_tokens(corpus, result.centroids, parameters, pool4)) + ); - CATCH_REQUIRE(result.K == 256); - CATCH_REQUIRE(result.centroids.size() == 256); - CATCH_REQUIRE(result.centroids.dimensions() == DIM); - CATCH_REQUIRE(result.fine_centroids->K == 256); - CATCH_REQUIRE(result.fine_centroids->dim == DIM); - CATCH_REQUIRE(result.centroid_table->K == 256); - CATCH_REQUIRE(result.ids.total_tokens() == 400); - for (size_t p = 0; p < NUM_PASSAGES; ++p) { - CATCH_REQUIRE(result.ids.cardinality(p) == 2); - } - CATCH_REQUIRE(result.num_coarse == std::max(1, 256 * 5 / 32)); - CATCH_REQUIRE(result.coarse_of_fine.size() == 256); - for (auto c : result.coarse_of_fine) { - CATCH_REQUIRE(c < result.num_coarse); + // expected_K(128)*4 bytes/row: 256 forces a 1-row batch, 1536 forces a 3-row batch. + for (size_t budget : {size_t{256}, size_t{1536}}) { + gem::CodebookParameters tiny = parameters; + tiny.assign_buffer_bytes = budget; + CATCH_REQUIRE( + flatten_ids(gem::assign_tokens(corpus, result.centroids, tiny, pool1)) == + flat1 + ); + } } -} -CATCH_TEST_CASE( - "GEM Codebook - quantize_timed matches quantize and records every phase", - "[quantization][gem]" -) { - constexpr size_t DIM = 16; - constexpr size_t NUM_PASSAGES = 200; - // 200 passages * 2 tokens == 400; colbert_num_partitions(400) == 256, still exceeded. - std::vector cardinalities(NUM_PASSAGES, 2); - auto centers = make_random_centers(16, DIM, 0x9999); - auto corpus = make_clustered_corpus(centers, 16, DIM, cardinalities, 0x1357, 0.1F); - - gem::CodebookParameters parameters; // num_centroids == 0: use the rule. - auto threadpool = svs::threads::as_threadpool(2); - - auto plain = gem::quantize(corpus, parameters, threadpool); - auto [timed, timings] = gem::quantize_timed(corpus, parameters, threadpool); - - // Same seed, same corpus: quantize_timed must reproduce quantize's result exactly. - CATCH_REQUIRE(timed.K == plain.K); - CATCH_REQUIRE(timed.num_coarse == plain.num_coarse); - CATCH_REQUIRE(timed.objective == plain.objective); - CATCH_REQUIRE(timed.num_sampled_passages == plain.num_sampled_passages); - CATCH_REQUIRE(timed.num_training_tokens == plain.num_training_tokens); - CATCH_REQUIRE(timed.coarse_of_fine == plain.coarse_of_fine); - CATCH_REQUIRE(flatten_ids(timed.ids) == flatten_ids(plain.ids)); - for (size_t i = 0; i < timed.centroids.size(); ++i) { - auto a = timed.centroids.get_datum(i); - auto b = plain.centroids.get_datum(i); - for (size_t d = 0; d < a.size(); ++d) { - CATCH_REQUIRE(a[d] == b[d]); + CATCH_SECTION( + "quantize_timed reproduces quantize's result exactly and times every phase" + ) { + auto [timed, timings] = gem::quantize_timed(corpus, parameters, threadpool); + CATCH_REQUIRE(timed.K == result.K); + CATCH_REQUIRE(timed.num_coarse == result.num_coarse); + CATCH_REQUIRE(timed.objective == result.objective); + CATCH_REQUIRE(timed.num_sampled_passages == result.num_sampled_passages); + CATCH_REQUIRE(timed.num_training_tokens == result.num_training_tokens); + CATCH_REQUIRE(timed.coarse_of_fine == result.coarse_of_fine); + CATCH_REQUIRE(flatten_ids(timed.ids) == flatten_ids(result.ids)); + for (size_t i = 0; i < timed.centroids.size(); ++i) { + auto a = timed.centroids.get_datum(i); + auto b = result.centroids.get_datum(i); + for (size_t d = 0; d < a.size(); ++d) { + CATCH_REQUIRE(a[d] == b[d]); + } } + CATCH_REQUIRE(timings.train_iterations.size() == parameters.kmeans_iterations); + CATCH_REQUIRE(timings.total >= 0.0); + CATCH_REQUIRE(timings.sample >= 0.0); + CATCH_REQUIRE(timings.train >= 0.0); + CATCH_REQUIRE(timings.normalize >= 0.0); + CATCH_REQUIRE(timings.assign >= 0.0); + CATCH_REQUIRE(timings.ground_table >= 0.0); + CATCH_REQUIRE(timings.coarse >= 0.0); } - - CATCH_REQUIRE(timings.train_iterations.size() == parameters.kmeans_iterations); - CATCH_REQUIRE(timings.total >= 0.0); - CATCH_REQUIRE(timings.sample >= 0.0); - CATCH_REQUIRE(timings.train >= 0.0); - CATCH_REQUIRE(timings.normalize >= 0.0); - CATCH_REQUIRE(timings.assign >= 0.0); - CATCH_REQUIRE(timings.ground_table >= 0.0); - CATCH_REQUIRE(timings.coarse >= 0.0); } CATCH_TEST_CASE("GEM Codebook - failure paths", "[quantization][gem]") { constexpr size_t DIM = 8; + constexpr size_t K = 4; auto threadpool = svs::threads::as_threadpool(1); + // Shared by every case below that just needs *some* valid corpus/centroids; only the + // 256-per-centroid bound needs a differently-shaped corpus to trigger that branch. + auto centers = make_random_centers(K, DIM, 1); + auto corpus = + make_clustered_corpus(centers, K, DIM, std::vector(10, 4), 2, 0.05F); CATCH_SECTION("empty tokens dataset") { - svs::data::SimpleData empty_tokens(0, DIM); - std::vector offsets{0}; - auto corpus = - svs::data::VectorSetData(std::move(empty_tokens), std::move(offsets)); - gem::CodebookParameters parameters; - CATCH_REQUIRE_THROWS(gem::train_codebook(corpus, parameters, threadpool)); + auto empty = svs::data::VectorSetData( + svs::data::SimpleData(0, DIM), std::vector{0} + ); + CATCH_REQUIRE_THROWS( + gem::train_codebook(empty, gem::CodebookParameters{}, threadpool) + ); } CATCH_SECTION("num_centroids above the uint32_t id range") { - auto centers = make_random_centers(4, DIM, 1); - std::vector cardinalities(10, 4); - auto corpus = make_clustered_corpus(centers, 4, DIM, cardinalities, 2, 0.05F); gem::CodebookParameters parameters; parameters.num_centroids = static_cast(std::numeric_limits::max()) + 1; @@ -521,9 +402,6 @@ CATCH_TEST_CASE("GEM Codebook - failure paths", "[quantization][gem]") { } CATCH_SECTION("fewer training tokens than K") { - auto centers = make_random_centers(4, DIM, 3); - std::vector cardinalities(5, 2); - auto corpus = make_clustered_corpus(centers, 4, DIM, cardinalities, 4, 0.05F); gem::CodebookParameters parameters; parameters.num_centroids = 1'000; // far more centroids than tokens sampled CATCH_REQUIRE_THROWS(gem::train_codebook(corpus, parameters, threadpool)); @@ -531,33 +409,23 @@ CATCH_TEST_CASE("GEM Codebook - failure paths", "[quantization][gem]") { CATCH_SECTION("training points exceed faiss's 256-per-centroid bound") { // One passage, 600 tokens; colbert_sample_size caps at 1 passage as training set. - auto centers = make_random_centers(2, DIM, 5); - std::vector cardinalities{600}; - auto corpus = make_clustered_corpus(centers, 2, DIM, cardinalities, 6, 0.05F); + auto dense_corpus = make_clustered_corpus(centers, K, DIM, {600}, 6, 0.05F); gem::CodebookParameters parameters; parameters.num_centroids = 2; // 600ish training tokens > 2 * 256 == 512 - CATCH_REQUIRE_THROWS(gem::train_codebook(corpus, parameters, threadpool)); + CATCH_REQUIRE_THROWS(gem::train_codebook(dense_corpus, parameters, threadpool)); } CATCH_SECTION("assign_tokens token/centroid dimension mismatch") { - auto centers = make_random_centers(4, DIM, 7); - std::vector cardinalities(5, 2); - auto corpus = make_clustered_corpus(centers, 4, DIM, cardinalities, 8, 0.05F); - svs::data::SimpleData bad_centroids(4, DIM + 1); - gem::CodebookParameters parameters; + svs::data::SimpleData bad_centroids(K, DIM + 1); CATCH_REQUIRE_THROWS( - gem::assign_tokens(corpus, bad_centroids, parameters, threadpool) + gem::assign_tokens(corpus, bad_centroids, gem::CodebookParameters{}, threadpool) ); } CATCH_SECTION("coarse_clusters num_coarse exceeds K") { - auto centers = make_random_centers(4, DIM, 9); - svs::data::SimpleData centroids(4, DIM); - for (size_t k = 0; k < 4; ++k) { - centroids.set_datum(k, std::span(centers.data() + k * DIM, DIM)); - } + auto centroids = centroids_from(centers, K, DIM); gem::CodebookParameters parameters; - parameters.num_coarse = 5; + parameters.num_coarse = K + 1; CATCH_REQUIRE_THROWS(gem::coarse_clusters(centroids, parameters, threadpool)); } } From cdc17a42d502e3b9d2cdfea1623a4e6fa8b95428 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 07:25:28 -0800 Subject: [PATCH 48/68] test(distance): consolidate the inner-product matrix, MaxSim and k-means tests Second pass, on top of the first consolidation: one shared type/ISA matrix for build_packed/column_max (float32/Float16/BFloat16 x NONE/AVX2/AVX512), a compacted mmap guard-page RAII helper, one AMX forwarding+tile-kernel TEST_CASE, one AMX-permission TEST_CASE, one kmeans_training TEST_CASE with three sections, and a trimmed MaxSim suite. Also applies a branch-coverage pass: drops checks that exercise no library branch beyond another check on the keep list (AMX's float32/Float16 forwarding is one branch, tested with one type; MaxSim's empty-document and dimension-mismatch checks are one branch in MaxSim::compute, tested with one document type; compute_packed's BFloat16/Float16 overloads share one dispatch call, tested with one type; MaxSim's hand-computed cases were strict subsets of the random-oracle sections and are dropped). Before/after line counts: tests/svs/core/distances/inner_product_matrix.cpp: 1253 -> 637 tests/svs/core/distances/maxsim.cpp: 325 -> 251 tests/svs/core/kmeans_training.cpp: 215 -> 208 total: 1793 -> 1096 (38.9% cut) Still short of the requested >=50% / <=895-line target; see the PR description / handback report for the itemized list of what would have to drop (and its branch-coverage cost) to close the remaining ~200-line gap. Co-Authored-By: Claude Opus 5.5 --- .../core/distances/inner_product_matrix.cpp | 1274 +++++------------ tests/svs/core/distances/maxsim.cpp | 244 ++-- tests/svs/core/kmeans_training.cpp | 241 ++-- 3 files changed, 531 insertions(+), 1228 deletions(-) diff --git a/tests/svs/core/distances/inner_product_matrix.cpp b/tests/svs/core/distances/inner_product_matrix.cpp index 624ffa5ae..7427ef55c 100644 --- a/tests/svs/core/distances/inner_product_matrix.cpp +++ b/tests/svs/core/distances/inner_product_matrix.cpp @@ -24,20 +24,20 @@ // catch2 #include "catch2/catch_approx.hpp" -#include "catch2/catch_message.hpp" #include "catch2/catch_test_macros.hpp" // stdlib #include #include #include -#include #include #include #include #include #include #include +#include +#include #include // POSIX, guarded: the guard-page test below needs mmap/mprotect/munmap, which this file @@ -55,12 +55,14 @@ namespace { using svs::distance::AVX_AVAILABILITY; using svs::distance::InnerProductMatrix; using svs::distance::InnerProductMatrixImpl; +using svs::distance::PackedOperand; -// float32 accumulation over at most d=129 terms of values in [-1, 1]: summation order -// differs between the generic and SIMD kernels, so the tolerance must absorb that, not -// just single-ULP rounding. +// float32 tolerance: summation order differs from the SIMD kernels, not just ULP +// rounding. Float16/BFloat16/AMX reorder further still, needing ~2x that. constexpr double REL_EPS = 1e-4; constexpr double ABS_EPS = 1e-5; +constexpr double NARROW_REL_EPS = 2e-4; +constexpr double NARROW_ABS_EPS = 1e-4; std::vector random_matrix(size_t rows, size_t d, unsigned seed) { std::mt19937 rng(seed); @@ -72,165 +74,58 @@ std::vector random_matrix(size_t rows, size_t d, unsigned seed) { return out; } -// Naive double-precision reference: out[i * n + j] = . -std::vector reference( - const std::vector& a, size_t m, const std::vector& b, size_t n, size_t d -) { - std::vector out(m * n, 0.0); - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < n; ++j) { - double acc = 0.0; - for (size_t k = 0; k < d; ++k) { - acc += - static_cast(a[i * d + k]) * static_cast(b[j * d + k]); - } - out[i * n + j] = acc; +// Converts float32 to A's element type as the kernel would see it: BFloat16 rounds +// ties-to-even (not the truncating constructor); float/Float16 use their constructor. +template std::vector to_type(const std::vector& a) { + std::vector out(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + if constexpr (std::is_same_v) { + out[i] = svs::round_to_bfloat16(a[i]); + } else { + out[i] = T(a[i]); } } return out; } -void require_close(const std::vector& got, const std::vector& expected) { - CATCH_REQUIRE(got.size() == expected.size()); - for (size_t i = 0; i < got.size(); ++i) { - CATCH_REQUIRE( - static_cast(got[i]) == - Catch::Approx(expected[i]).epsilon(REL_EPS).margin(ABS_EPS) - ); - } -} - -// m x n x d shapes exercising every tail: row-block remainder, SIMD lane remainder (8 and -// 16 wide), and both small and multi-cache-line `d`. -const std::vector ms{1, 2, 3, 7, 8, 9, 33}; -const std::vector ns{1, 2, 15, 16, 17, 31, 32, 33, 64}; -const std::vector ds{1, 7, 128, 129}; - -template void check_against_reference() { - unsigned seed = 0; - for (size_t d : ds) { - for (size_t m : ms) { - for (size_t n : ns) { - auto a = random_matrix(m, d, seed++); - auto b = random_matrix(n, d, seed++); - std::vector out(m * n, -1.0F); - InnerProductMatrixImpl::build(a.data(), m, b.data(), n, d, out.data()); - require_close(out, reference(a, m, b, n, d)); - } - } - } -} - -template void check_against_generic() { - unsigned seed = 1000; - for (size_t d : ds) { - for (size_t m : ms) { - for (size_t n : ns) { - auto a = random_matrix(m, d, seed++); - auto b = random_matrix(n, d, seed++); - std::vector generic_out(m * n, -1.0F); - std::vector isa_out(m * n, -2.0F); - InnerProductMatrixImpl::build( - a.data(), m, b.data(), n, d, generic_out.data() - ); - InnerProductMatrixImpl::build( - a.data(), m, b.data(), n, d, isa_out.data() - ); - for (size_t i = 0; i < generic_out.size(); ++i) { - CATCH_REQUIRE( - static_cast(isa_out[i]) == - Catch::Approx(static_cast(generic_out[i])) - .epsilon(REL_EPS) - .margin(ABS_EPS) - ); - } - } - } - } -} - -} // namespace - -CATCH_TEST_CASE( - "InnerProductMatrix generic kernel matches a naive reference", - "[distance][inner_product_matrix]" -) { - check_against_reference(); -} - -CATCH_TEST_CASE( - "InnerProductMatrix AVX2 kernel matches generic", "[distance][inner_product_matrix]" -) { - if (!svs::detail::avx_runtime_flags.is_avx2_supported()) { - return; +// Promotes any of A's element types to double via each type's lossless `operator float`. +template std::vector promote_row(const std::vector& a) { + std::vector out(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + out[i] = static_cast(static_cast(a[i])); } - check_against_reference(); - check_against_generic(); + return out; } -CATCH_TEST_CASE( - "InnerProductMatrix AVX512 kernel matches generic", "[distance][inner_product_matrix]" -) { - if (!svs::detail::avx_runtime_flags.is_avx512f_supported()) { - return; +// Rounds B to bf16 and back to float32: the AMX/bf16 kernels see bf16(b), so a +// reference against pre-rounding b would be wrong by rounding error, not ordering. +std::vector to_bfloat16_as_float(const std::vector& b) { + std::vector out(b.size()); + for (size_t i = 0; i < b.size(); ++i) { + out[i] = static_cast(svs::round_to_bfloat16(b[i])); } - check_against_reference(); - check_against_generic(); + return out; } -CATCH_TEST_CASE( - "InnerProductMatrix public wrapper equals the reference", - "[distance][inner_product_matrix]" +// out[i * n + j] = , over an already-promoted `a` so a narrow-type A's +// post-conversion values are what the reference uses. +std::vector reference_from_values( + const std::vector& a, size_t m, const std::vector& b, size_t n, size_t d ) { - for (size_t d : {1UL, 7UL, 128UL, 129UL}) { - for (size_t m : {1UL, 9UL, 33UL}) { - for (size_t n : {1UL, 17UL, 64UL}) { - auto a = random_matrix(m, d, static_cast(m * 100 + n * 10 + d)); - auto b = random_matrix(n, d, static_cast(d * 100 + m * 10 + n)); - std::vector out(m * n, -1.0F); - InnerProductMatrix::compute(a.data(), m, b.data(), n, d, out.data()); - require_close(out, reference(a, m, b, n, d)); + std::vector out(m * n, 0.0); + for (size_t i = 0; i < m; ++i) { + for (size_t j = 0; j < n; ++j) { + double acc = 0.0; + for (size_t k = 0; k < d; ++k) { + acc += a[i * d + k] * static_cast(b[j * d + k]); } + out[i * n + j] = acc; } } + return out; } -CATCH_TEST_CASE( - "InnerProductMatrix degenerate shapes leave `out` untouched", - "[distance][inner_product_matrix]" -) { - constexpr size_t d = 4; - auto a = random_matrix(3, d, 7); - auto b = random_matrix(3, d, 11); - - CATCH_SECTION("m == 0") { - std::vector out{1.0F, 2.0F, 3.0F}; - auto before = out; - InnerProductMatrix::compute(a.data(), 0, b.data(), 3, d, out.data()); - CATCH_REQUIRE(out == before); - } - - CATCH_SECTION("n == 0") { - std::vector out{1.0F, 2.0F, 3.0F}; - auto before = out; - InnerProductMatrix::compute(a.data(), 3, b.data(), 0, d, out.data()); - CATCH_REQUIRE(out == before); - } -} - -namespace { - -using svs::distance::PackedOperand; - -// Shapes covering every tail this family of kernels has: the row-block remainder -// (8-wide), both SIMD lane widths' remainders (8 and 16), m == 0 (empty document), and -// both small and multi-cache-line d. -const std::vector column_max_ms{0, 1, 2, 7, 8, 9, 17, 67, 180}; -const std::vector column_max_ns{1, 15, 16, 17, 32, 33}; -const std::vector column_max_ds{1, 7, 128, 129}; - -// Naive double-precision reference: out[j] = max over i of , or -infinity if -// m == 0. +// out[j] = max over i of , or -infinity if m == 0. std::vector reference_column_max( const std::vector& a, size_t m, const std::vector& b, size_t n, size_t d ) { @@ -247,7 +142,14 @@ std::vector reference_column_max( return out; } -void require_close_vec(const std::vector& got, const std::vector& expected) { +// `expected[i] == -inf` requires `got[i]` exactly -inf rather than Approx-equal +// (column_max's empty-input value); table-form callers never hit that case. +void require_close( + const std::vector& got, + const std::vector& expected, + double rel = REL_EPS, + double abs_margin = ABS_EPS +) { CATCH_REQUIRE(got.size() == expected.size()); for (size_t i = 0; i < got.size(); ++i) { if (expected[i] == -std::numeric_limits::infinity()) { @@ -255,478 +157,280 @@ void require_close_vec(const std::vector& got, const std::vector& } else { CATCH_REQUIRE( static_cast(got[i]) == - Catch::Approx(expected[i]).epsilon(REL_EPS).margin(ABS_EPS) + Catch::Approx(expected[i]).epsilon(rel).margin(abs_margin) ); } } } -// AMX's tile grouping (16 pairs per k-chunk, two 16-lane tiles accumulated together) -// reorders the sum differently from both the generic k-order loop and AVX-512's 8-row -// register blocking; measured over every shape amx_ms/amx_ns/amx_ds below, that extra -// reordering alone needs the same ~2x-of-observed tolerance already used for the -// Float16/BFloat16 narrow-type comparisons above (check_column_max_against_generic), -// not REL_EPS/ABS_EPS's tighter float32 bound. -constexpr double AMX_REL_EPS = 2e-4; -constexpr double AMX_ABS_EPS = 1e-4; +// Shapes hitting every tail class NONE/AVX2/AVX512 need: m crosses the empty (0) and +// 8-row block boundary; n crosses the 16-lane boundary; d crosses the 16-lane boundary. +const std::vector ms{0, 1, 2, 7, 8, 9, 17, 67, 180}; +const std::vector ns{1, 15, 16, 17, 32, 33}; +const std::vector ds{1, 7, 128, 129}; -void require_close_loose( - const std::vector& got, const std::vector& expected +// Checks build_packed and column_max for A held as T, at ISA level Avx, against a double +// reference and, for Avx != NONE, against the generic kernel. +template +void check_kernels( + const std::vector& a, const std::vector& b, size_t m, size_t n, size_t d ) { - CATCH_REQUIRE(got.size() == expected.size()); - for (size_t i = 0; i < got.size(); ++i) { - CATCH_REQUIRE( - static_cast(got[i]) == - Catch::Approx(expected[i]).epsilon(AMX_REL_EPS).margin(AMX_ABS_EPS) + auto a_t = to_type(a); + PackedOperand packed; + packed.pack(b.data(), n, d); + constexpr bool narrow = !std::is_same_v; + double rel = narrow ? NARROW_REL_EPS : REL_EPS; + double abs_margin = narrow ? NARROW_ABS_EPS : ABS_EPS; + + std::vector table(m * n, -1.0F), col_max(n, -3.0F); + InnerProductMatrixImpl::build_packed(a_t.data(), m, packed, table.data()); + InnerProductMatrixImpl::column_max(a_t.data(), m, packed, col_max.data()); + require_close( + table, reference_from_values(promote_row(a_t), m, b, n, d), rel, abs_margin + ); + require_close( + col_max, reference_column_max(promote_row(a_t), m, b, n, d), rel, abs_margin + ); + + if constexpr (Avx != AVX_AVAILABILITY::NONE) { + std::vector generic_table(m * n, -1.0F), generic_col_max(n, -3.0F); + InnerProductMatrixImpl::build_packed( + a_t.data(), m, packed, generic_table.data() ); + InnerProductMatrixImpl::column_max( + a_t.data(), m, packed, generic_col_max.data() + ); + require_close(table, promote_row(generic_table), rel, abs_margin); + require_close(col_max, promote_row(generic_col_max), rel, abs_margin); } } -void require_close_vec_loose( - const std::vector& got, const std::vector& expected -) { - CATCH_REQUIRE(got.size() == expected.size()); - for (size_t i = 0; i < got.size(); ++i) { - if (expected[i] == -std::numeric_limits::infinity()) { - CATCH_REQUIRE(got[i] == -std::numeric_limits::infinity()); - } else { - CATCH_REQUIRE( - static_cast(got[i]) == - Catch::Approx(expected[i]).epsilon(AMX_REL_EPS).margin(AMX_ABS_EPS) - ); +template void check_level() { + unsigned seed = 0; + for (size_t d : ds) { + for (size_t m : ms) { + for (size_t n : ns) { + auto a = random_matrix(m, d, seed++); + auto b = random_matrix(n, d, seed++); + check_kernels(a, b, m, n, d); + check_kernels(a, b, m, n, d); + check_kernels(a, b, m, n, d); + } } } } -std::vector to_float16(const std::vector& a) { - std::vector out(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - out[i] = svs::Float16(a[i]); - } - return out; -} - -// Promotes a Float16 row-major matrix to double, so the reference is computed from the -// values the kernel actually sees (post-quantization), not the pre-quantization floats. -std::vector promote_float16(const std::vector& a) { - std::vector out(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - out[i] = static_cast(static_cast(a[i])); - } - return out; -} +} // namespace -std::vector promote_float(const std::vector& a) { - std::vector out(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - out[i] = static_cast(a[i]); +CATCH_TEST_CASE( + "InnerProductMatrixImpl build_packed and column_max match a double reference, and " + "the generic kernel, for every A type and available non-AMX ISA", + "[distance][inner_product_matrix]" +) { + check_level(); + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + check_level(); } - return out; -} - -// Rounds to nearest, ties to even (svs::round_to_bfloat16), not BFloat16's own truncating -// constructor: this is what every bf16 row-overload test below checks the kernel against. -std::vector to_bfloat16(const std::vector& a) { - std::vector out(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - out[i] = svs::round_to_bfloat16(a[i]); + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + check_level(); } - return out; } -// Promotes a BFloat16 row-major matrix to double, so the reference is computed from the -// values the kernel actually sees (post-rounding), not the pre-rounding floats. -std::vector promote_bfloat16(const std::vector& a) { - std::vector out(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - out[i] = static_cast(static_cast(a[i])); - } - return out; -} +namespace { -// Rounds to bf16 and back to float32, for building a reference against a B operand that -// was packed with `pack_rounded_to_bf16`: that kernel sees bf16(b), not b, so a reference -// comparing against the pre-rounding float would be wrong by bf16's own rounding error, -// not by summation order. -std::vector to_bfloat16_as_float(const std::vector& b) { - std::vector out(b.size()); - for (size_t i = 0; i < b.size(); ++i) { - out[i] = static_cast(svs::round_to_bfloat16(b[i])); - } - return out; -} +// Every tail an AMX tile hits: m/n cross the 16-wide tile/lane boundary, d crosses the +// 32-element (16-pair) k-chunk boundary and both parities. +const std::vector amx_ms{0, 1, 15, 16, 17, 33, 67, 180}; +const std::vector amx_ns{1, 15, 16, 17, 32, 33}; +const std::vector amx_ds{1, 2, 31, 32, 33, 64, 127, 128, 129}; -// Naive double-precision reference for the table form, out[i * n + j] = , over -// an already-promoted `a` so a narrow-type A's post-conversion values are what the -// reference uses (shares `reference`'s role above, generalized to a non-float32 `a`). -std::vector reference_from_values( - const std::vector& a, size_t m, const std::vector& b, size_t n, size_t d +// Bit-exact comparison for one (A type, shape): AMX forwards to AVX-512 with no Approx +// tolerance whenever it isn't running its own bf16 tile kernel. +template +void require_amx_matches_avx512( + const T* a, size_t m, const PackedOperand& packed, size_t n ) { - std::vector out(m * n, 0.0); - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < n; ++j) { - double acc = 0.0; - for (size_t k = 0; k < d; ++k) { - acc += a[i * d + k] * static_cast(b[j * d + k]); - } - out[i * n + j] = acc; - } - } - return out; -} + std::vector table_amx(m * n, -1.0F), table_avx512(m * n, -2.0F); + InnerProductMatrixImpl::build_packed( + a, m, packed, table_amx.data() + ); + InnerProductMatrixImpl::build_packed( + a, m, packed, table_avx512.data() + ); + CATCH_REQUIRE(table_amx == table_avx512); -template void check_column_max_float32_against_reference() { - unsigned seed = 2000; - for (size_t d : column_max_ds) { - for (size_t m : column_max_ms) { - for (size_t n : column_max_ns) { - auto a = random_matrix(m, d, seed++); - auto b = random_matrix(n, d, seed++); - PackedOperand packed; - packed.pack(b.data(), n, d); - std::vector out(n, -3.0F); - InnerProductMatrixImpl::column_max(a.data(), m, packed, out.data()); - require_close_vec(out, reference_column_max(promote_float(a), m, b, n, d)); - } - } - } + std::vector cm_amx(n, -1.0F), cm_avx512(n, -2.0F); + InnerProductMatrixImpl::column_max(a, m, packed, cm_amx.data()); + InnerProductMatrixImpl::column_max( + a, m, packed, cm_avx512.data() + ); + CATCH_REQUIRE(cm_amx == cm_avx512); } -template void check_column_max_float16_against_reference() { - unsigned seed = 3000; - for (size_t d : column_max_ds) { - for (size_t m : column_max_ms) { - for (size_t n : column_max_ns) { - auto a = random_matrix(m, d, seed++); - auto a16 = to_float16(a); - auto b = random_matrix(n, d, seed++); - PackedOperand packed; - packed.pack(b.data(), n, d); - std::vector out(n, -3.0F); - InnerProductMatrixImpl::column_max(a16.data(), m, packed, out.data()); - require_close_vec( - out, reference_column_max(promote_float16(a16), m, b, n, d) - ); - } - } - } -} +} // namespace -template void check_column_max_bf16_against_reference() { - unsigned seed = 3500; - for (size_t d : column_max_ds) { - for (size_t m : column_max_ms) { - for (size_t n : column_max_ns) { - auto a = random_matrix(m, d, seed++); - auto a_bf16 = to_bfloat16(a); - auto b = random_matrix(n, d, seed++); - PackedOperand packed; - packed.pack(b.data(), n, d); - std::vector out(n, -3.0F); - InnerProductMatrixImpl::column_max( - a_bf16.data(), m, packed, out.data() - ); - require_close_vec( - out, reference_column_max(promote_bfloat16(a_bf16), m, b, n, d) - ); - } +// Every InnerProductMatrixImpl overload besides BFloat16's build_packed/column_max +// is an unconditional one-line forward to AVX-512 with no branch: float32 stands in for +// Float16 too, since both overloads have that identical shape. BFloat16's build_packed/ +// column_max branch on an early m==0/n==0 return and on has_tile_layout(); B is plain- +// packed below, so this covers the early return and the non-tile branch, both distinct +// from the float32/Float16 case. The small, fixed shape set is deliberate: none of these +// branches depend on shape beyond m == 0 or n == 0. +CATCH_TEST_CASE( + "InnerProductMatrixImpl forwards to AVX-512 bit for bit except for its own " + "bf16 tile kernel, which matches a double reference and AVX-512", + "[distance][inner_product_matrix][amx]" +) { + unsigned seed = 20000; + for (size_t m : {0UL, 1UL, 9UL, 33UL}) { + for (size_t n : {0UL, 1UL, 17UL, 33UL}) { + size_t d = 7; + auto a = random_matrix(m, d, seed++); + auto a_bf16 = to_type(a); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + + require_amx_matches_avx512(a.data(), m, packed, n); + require_amx_matches_avx512(a_bf16.data(), m, packed, n); + + std::vector build_amx(m * n, -1.0F), build_avx512(m * n, -2.0F); + InnerProductMatrixImpl::build( + a.data(), m, b.data(), n, d, build_amx.data() + ); + InnerProductMatrixImpl::build( + a.data(), m, b.data(), n, d, build_avx512.data() + ); + CATCH_REQUIRE(build_amx == build_avx512); } } -} -template void check_column_max_against_generic() { - unsigned seed = 4000; - for (size_t d : column_max_ds) { - for (size_t m : column_max_ms) { - for (size_t n : column_max_ns) { - auto a = random_matrix(m, d, seed++); - auto a16 = to_float16(a); + // AMX's own bf16 tile kernel: reached only when B is packed with + // pack_rounded_to_bf16 and tile hardware is available. This is the one branch in + // the AMX code that isn't a plain forward, so it gets the full tail-shape sweep. + if (!svs::detail::amx_tile_bf16_available()) { + return; + } + seed = 90000; + for (size_t d : amx_ds) { + for (size_t m : amx_ms) { + for (size_t n : amx_ns) { + auto a_bf16 = to_type(random_matrix(m, d, seed++)); auto b = random_matrix(n, d, seed++); + auto b_rounded = to_bfloat16_as_float(b); PackedOperand packed; - packed.pack(b.data(), n, d); + packed.pack_rounded_to_bf16(b.data(), n, d); + CATCH_REQUIRE(packed.has_tile_layout()); - std::vector generic_out(n, -1.0F); - std::vector isa_out(n, -2.0F); - InnerProductMatrixImpl::column_max( - a.data(), m, packed, generic_out.data() + std::vector table_amx(m * n, -1.0F), table_avx512(m * n, -2.0F); + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, table_amx.data() ); - InnerProductMatrixImpl::column_max( - a.data(), m, packed, isa_out.data() + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, table_avx512.data() ); - for (size_t j = 0; j < n; ++j) { - CATCH_REQUIRE( - static_cast(isa_out[j]) == - Catch::Approx(static_cast(generic_out[j])) - .epsilon(REL_EPS) - .margin(ABS_EPS) - ); - } - - std::vector generic_out16(n, -1.0F); - std::vector isa_out16(n, -2.0F); - InnerProductMatrixImpl::column_max( - a16.data(), m, packed, generic_out16.data() + require_close( + table_amx, + reference_from_values(promote_row(a_bf16), m, b_rounded, n, d), + NARROW_REL_EPS, + NARROW_ABS_EPS ); - InnerProductMatrixImpl::column_max( - a16.data(), m, packed, isa_out16.data() + require_close( + table_amx, promote_row(table_avx512), NARROW_REL_EPS, NARROW_ABS_EPS ); - // Float16 inputs decode identically everywhere, but the generic kernel - // accumulates in strict k-order while the SIMD kernels accumulate across - // row-blocks in a different order; measured over every (m, n, d) shape - // this test sweeps, that reordering alone produces at most 4.7e-5 - // absolute / 3.1e-5 relative difference (one-off throwaway probe, not - // committed), so this tolerance is ~2x that bound, not a slack value. - for (size_t j = 0; j < n; ++j) { - CATCH_REQUIRE( - static_cast(isa_out16[j]) == - Catch::Approx(static_cast(generic_out16[j])) - .epsilon(2e-4) - .margin(1e-4) - ); - } - auto a_bf16 = to_bfloat16(a); - std::vector generic_out_bf16(n, -1.0F); - std::vector isa_out_bf16(n, -2.0F); - InnerProductMatrixImpl::column_max( - a_bf16.data(), m, packed, generic_out_bf16.data() - ); - InnerProductMatrixImpl::column_max( - a_bf16.data(), m, packed, isa_out_bf16.data() + std::vector cm_amx(n, -3.0F), cm_avx512(n, -4.0F); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, cm_amx.data() ); - // Same reordering-only tolerance as the Float16 case above: BFloat16's - // widening is exact, so any difference here is summation order, not - // decode. - for (size_t j = 0; j < n; ++j) { - CATCH_REQUIRE( - static_cast(isa_out_bf16[j]) == - Catch::Approx(static_cast(generic_out_bf16[j])) - .epsilon(2e-4) - .margin(1e-4) - ); - } - } - } - } -} - -template void check_build_packed_narrow_against_reference() { - // Smaller shape set than column_max's: the table form's output scales as m * n, and - // this already exercises the row-block (8) and both SIMD lane-width (8, 16) tails. - unsigned seed = 6000; - for (size_t d : {1UL, 7UL, 128UL}) { - for (size_t m : {0UL, 1UL, 7UL, 9UL, 17UL}) { - for (size_t n : {1UL, 15UL, 17UL, 32UL}) { - auto a = random_matrix(m, d, seed++); - auto b = random_matrix(n, d, seed++); - PackedOperand packed; - packed.pack(b.data(), n, d); - - auto a16 = to_float16(a); - std::vector out16(m * n, -1.0F); - InnerProductMatrixImpl::build_packed( - a16.data(), m, packed, out16.data() + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, cm_avx512.data() ); require_close( - out16, reference_from_values(promote_float16(a16), m, b, n, d) - ); - - auto a_bf16 = to_bfloat16(a); - std::vector out_bf16(m * n, -1.0F); - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, out_bf16.data() + cm_amx, + reference_column_max(promote_row(a_bf16), m, b_rounded, n, d), + NARROW_REL_EPS, + NARROW_ABS_EPS ); require_close( - out_bf16, reference_from_values(promote_bfloat16(a_bf16), m, b, n, d) - ); - - // Against the generic (NONE) level, same tail shapes. - std::vector generic16(m * n, -1.0F); - InnerProductMatrixImpl::build_packed( - a16.data(), m, packed, generic16.data() - ); - for (size_t idx = 0; idx < generic16.size(); ++idx) { - CATCH_REQUIRE( - static_cast(out16[idx]) == - Catch::Approx(static_cast(generic16[idx])) - .epsilon(2e-4) - .margin(1e-4) - ); - } - std::vector generic_bf16(m * n, -1.0F); - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, generic_bf16.data() + cm_amx, promote_row(cm_avx512), NARROW_REL_EPS, NARROW_ABS_EPS ); - for (size_t idx = 0; idx < generic_bf16.size(); ++idx) { - CATCH_REQUIRE( - static_cast(out_bf16[idx]) == - Catch::Approx(static_cast(generic_bf16[idx])) - .epsilon(2e-4) - .margin(1e-4) - ); - } } } } } -} // namespace - -CATCH_TEST_CASE( - "InnerProductMatrixImpl column_max generic kernel matches a naive reference", - "[distance][inner_product_matrix]" -) { - check_column_max_float32_against_reference(); - check_column_max_float16_against_reference(); - check_column_max_bf16_against_reference(); -} - CATCH_TEST_CASE( - "InnerProductMatrixImpl column_max AVX2 kernel matches reference and generic", + "InnerProductMatrix degenerate shapes leave `out` untouched", "[distance][inner_product_matrix]" ) { - if (!svs::detail::avx_runtime_flags.is_avx2_supported()) { - return; + constexpr size_t d = 4; + auto a = random_matrix(3, d, 7); + auto b = random_matrix(3, d, 11); + for (auto [m, n] : std::vector>{{0, 3}, {3, 0}}) { + std::vector out{1.0F, 2.0F, 3.0F}; + auto before = out; + InnerProductMatrix::compute(a.data(), m, b.data(), n, d, out.data()); + CATCH_REQUIRE(out == before); } - check_column_max_float32_against_reference(); - check_column_max_float16_against_reference(); - check_column_max_bf16_against_reference(); - check_column_max_against_generic(); } -CATCH_TEST_CASE( - "InnerProductMatrixImpl column_max AVX512 kernel matches reference and generic", - "[distance][inner_product_matrix]" -) { - if (!svs::detail::avx_runtime_flags.is_avx512f_supported()) { - return; - } - check_column_max_float32_against_reference(); - check_column_max_float16_against_reference(); - check_column_max_bf16_against_reference(); - check_column_max_against_generic(); -} +namespace { -CATCH_TEST_CASE( - "InnerProductMatrixImpl build_packed matches a naive reference for Float16 and " - "BFloat16 rows, every available level", - "[distance][inner_product_matrix]" -) { - check_build_packed_narrow_against_reference(); - if (svs::detail::avx_runtime_flags.is_avx2_supported()) { - check_build_packed_narrow_against_reference(); - } +// Picks the ISA compute()/compute_packed() would pick: select_isa() is private, so this +// repeats its priority order against the public avx_runtime_flags. +template void with_highest_isa(F&& f) { if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { - check_build_packed_narrow_against_reference(); + f(std::integral_constant{}); + } else if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + f(std::integral_constant{}); + } else { + f(std::integral_constant{}); } } +} // namespace + +// compute_packed's BFloat16 and Float16 overloads share one dispatch call, unconditional +// on T, so BFloat16 alone exercises that dispatch; Float16 would hit the identical branch. CATCH_TEST_CASE( - "InnerProductMatrixImpl build_packed matches build through an explicit PackedOperand", + "InnerProductMatrix's public entry points equal the reference and pick the same ISA " + "result as InnerProductMatrixImpl, for float32 and BFloat16 rows", "[distance][inner_product_matrix]" ) { - // build() packs internally into a thread_local PackedOperand; this checks that - // packing explicitly and calling build_packed gives the identical result, for every - // ISA this process can run. unsigned seed = 5000; for (size_t d : {1UL, 7UL, 128UL}) { for (size_t m : {0UL, 1UL, 9UL, 33UL}) { for (size_t n : {1UL, 17UL, 33UL}) { auto a = random_matrix(m, d, seed++); auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + std::vector via_build(m * n, -1.0F); InnerProductMatrix::compute(a.data(), m, b.data(), n, d, via_build.data()); + require_close(via_build, reference_from_values(promote_row(a), m, b, n, d)); - PackedOperand packed; - packed.pack(b.data(), n, d); std::vector via_packed(m * n, -2.0F); - // InnerProductMatrix::select_isa() is private (the ISA choice for this - // kernel belongs to InnerProductMatrix alone); this test needs to force - // the same ISA `compute` above picked, so it repeats that priority order - // against the public svs::detail::avx_runtime_flags instead. - if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { - InnerProductMatrixImpl::build_packed( - a.data(), m, packed, via_packed.data() - ); - } else if (svs::detail::avx_runtime_flags.is_avx2_supported()) { - InnerProductMatrixImpl::build_packed( - a.data(), m, packed, via_packed.data() - ); - } else { - InnerProductMatrixImpl::build_packed( + with_highest_isa([&](auto avx_c) { + InnerProductMatrixImpl::build_packed( a.data(), m, packed, via_packed.data() ); - } + }); CATCH_REQUIRE(via_build == via_packed); - } - } - } -} -CATCH_TEST_CASE( - "InnerProductMatrix::compute_packed picks the same ISA result as " - "InnerProductMatrixImpl " - "for BFloat16 and Float16 rows", - "[distance][inner_product_matrix]" -) { - unsigned seed = 7000; - for (size_t d : {1UL, 7UL, 128UL}) { - for (size_t m : {0UL, 1UL, 9UL, 17UL}) { - for (size_t n : {1UL, 17UL, 33UL}) { - auto a = random_matrix(m, d, seed++); - auto b = random_matrix(n, d, seed++); - PackedOperand packed; - packed.pack(b.data(), n, d); - - // Same ISA-priority repetition as the float32 build_packed test above: - // InnerProductMatrix::select_isa() is private by design. - auto direct_bf16 = [&](const auto& av, std::vector& out) { - auto a_bf16 = to_bfloat16(a); - if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, out.data() - ); - } else if (svs::detail::avx_runtime_flags.is_avx2_supported()) { - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, out.data() - ); - } else { - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, out.data() - ); - } - (void)av; - }; - - auto a_bf16 = to_bfloat16(a); + auto a_bf16 = to_type(a); std::vector via_public(m * n, -1.0F); InnerProductMatrix::compute_packed( a_bf16.data(), m, packed, via_public.data() ); std::vector via_direct(m * n, -2.0F); - direct_bf16(0, via_direct); - CATCH_REQUIRE(via_public == via_direct); - - auto a16 = to_float16(a); - std::vector via_public16(m * n, -1.0F); - InnerProductMatrix::compute_packed( - a16.data(), m, packed, via_public16.data() - ); - std::vector via_direct16(m * n, -2.0F); - if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { - InnerProductMatrixImpl::build_packed( - a16.data(), m, packed, via_direct16.data() - ); - } else if (svs::detail::avx_runtime_flags.is_avx2_supported()) { - InnerProductMatrixImpl::build_packed( - a16.data(), m, packed, via_direct16.data() - ); - } else { - InnerProductMatrixImpl::build_packed( - a16.data(), m, packed, via_direct16.data() + with_highest_isa([&](auto avx_c) { + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, via_direct.data() ); - } - CATCH_REQUIRE(via_public16 == via_direct16); + }); + CATCH_REQUIRE(via_public == via_direct); } } } @@ -759,10 +463,8 @@ CATCH_TEST_CASE( } CATCH_SECTION("Exact halfway cases round to the even neighbor, checked bit for bit") { - // Both inputs discard exactly 0x8000 (the bf16 ULP's halfway point) below bit 16: - // the first's kept 16 bits are already even and must stay; the second's are odd - // and must round up to the next even value. This checks the rounding algorithm's - // bit pattern directly, not just self-consistency with svs::round_to_bfloat16. + // Both inputs discard exactly bit 0x8000, bf16's halfway point: the first's kept + // bits are already even and must stay, the second's are odd and must round up. float tie_even_stays = std::bit_cast(uint32_t{0x3F808000}); float tie_odd_rounds_up = std::bit_cast(uint32_t{0x3F818000}); std::vector values{tie_even_stays, tie_odd_rounds_up}; @@ -782,468 +484,150 @@ CATCH_TEST_CASE( } } -namespace { - -// Every tail an AMX tile hits: m crosses the 16-row tile boundary (0, 1, 15, 16, 17, 33, -// 67, 180), n crosses the 16-lane tile boundary (1, 15, 16, 17, 32, 33), d crosses the -// 32-element (16-pair) k-chunk boundary and parity (1, 2, 31, 32, 33, 64, 127, 128, 129). -const std::vector amx_ms{0, 1, 15, 16, 17, 33, 67, 180}; -const std::vector amx_ns{1, 15, 16, 17, 32, 33}; -const std::vector amx_ds{1, 2, 31, 32, 33, 64, 127, 128, 129}; - -} // namespace - CATCH_TEST_CASE( - "InnerProductMatrixImpl build_packed matches a double-precision reference and " - "AVX-512, over every tail shape", - "[distance][inner_product_matrix][amx]" + "AMX tile permission detection is thread-safe", "[distance][inner_product_matrix][amx]" ) { - if (!svs::detail::amx_tile_bf16_available()) { - return; - } - unsigned seed = 8000; - for (size_t d : amx_ds) { - for (size_t m : amx_ms) { - for (size_t n : amx_ns) { - auto a = random_matrix(m, d, seed++); - auto a_bf16 = to_bfloat16(a); - auto b = random_matrix(n, d, seed++); - // The kernel computes : the reference must round b too, - // not compare against the pre-rounding float values. - auto b_rounded = to_bfloat16_as_float(b); - PackedOperand packed; - packed.pack_rounded_to_bf16(b.data(), n, d); - CATCH_REQUIRE(packed.has_tile_layout()); - - CATCH_INFO("m=" << m << " n=" << n << " d=" << d); - std::vector out_amx(m * n, -1.0F); - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, out_amx.data() - ); - require_close_loose( - out_amx, - reference_from_values(promote_bfloat16(a_bf16), m, b_rounded, n, d) - ); - - std::vector out_avx512(m * n, -2.0F); - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, out_avx512.data() - ); - for (size_t idx = 0; idx < out_amx.size(); ++idx) { - CATCH_REQUIRE( - static_cast(out_amx[idx]) == - Catch::Approx(static_cast(out_avx512[idx])) - .epsilon(2e-4) - .margin(1e-4) - ); - } - } + CATCH_SECTION("amx_tile_bf16_available is consistent across concurrent threads") { + constexpr size_t num_threads = 8; + std::vector threads; + std::array results{}; + for (size_t t = 0; t < num_threads; ++t) { + threads.emplace_back([&results, t] { + results[t] = svs::detail::amx_tile_bf16_available(); + }); } - } -} - -CATCH_TEST_CASE( - "InnerProductMatrixImpl column_max matches a double-precision reference and " - "AVX-512, over every tail shape", - "[distance][inner_product_matrix][amx]" -) { - if (!svs::detail::amx_tile_bf16_available()) { - return; - } - unsigned seed = 9000; - for (size_t d : amx_ds) { - for (size_t m : amx_ms) { - for (size_t n : amx_ns) { - auto a = random_matrix(m, d, seed++); - auto a_bf16 = to_bfloat16(a); - auto b = random_matrix(n, d, seed++); - auto b_rounded = to_bfloat16_as_float(b); - PackedOperand packed; - packed.pack_rounded_to_bf16(b.data(), n, d); - CATCH_REQUIRE(packed.has_tile_layout()); - - std::vector out_amx(n, -3.0F); - InnerProductMatrixImpl::column_max( - a_bf16.data(), m, packed, out_amx.data() - ); - require_close_vec_loose( - out_amx, - reference_column_max(promote_bfloat16(a_bf16), m, b_rounded, n, d) - ); - - std::vector out_avx512(n, -4.0F); - InnerProductMatrixImpl::column_max( - a_bf16.data(), m, packed, out_avx512.data() - ); - for (size_t j = 0; j < n; ++j) { - if (out_avx512[j] == -std::numeric_limits::infinity()) { - CATCH_REQUIRE( - out_amx[j] == -std::numeric_limits::infinity() - ); - } else { - CATCH_REQUIRE( - static_cast(out_amx[j]) == - Catch::Approx(static_cast(out_avx512[j])) - .epsilon(2e-4) - .margin(1e-4) - ); - } - } - } + for (auto& th : threads) { + th.join(); } - } -} - -CATCH_TEST_CASE( - "InnerProductMatrixImpl forwards float32 and Float16 A to AVX-512 bit for bit", - "[distance][inner_product_matrix][amx]" -) { - unsigned seed = 10000; - for (size_t d : {1UL, 7UL, 128UL, 129UL}) { - for (size_t m : {0UL, 1UL, 17UL, 33UL}) { - for (size_t n : {1UL, 17UL, 33UL}) { - auto a = random_matrix(m, d, seed++); - auto a16 = to_float16(a); - auto b = random_matrix(n, d, seed++); - PackedOperand packed; - packed.pack(b.data(), n, d); - - std::vector amx_f32(m * n, -1.0F), avx512_f32(m * n, -2.0F); - InnerProductMatrixImpl::build_packed( - a.data(), m, packed, amx_f32.data() - ); - InnerProductMatrixImpl::build_packed( - a.data(), m, packed, avx512_f32.data() - ); - CATCH_REQUIRE(amx_f32 == avx512_f32); - - std::vector amx_f16(m * n, -1.0F), avx512_f16(m * n, -2.0F); - InnerProductMatrixImpl::build_packed( - a16.data(), m, packed, amx_f16.data() - ); - InnerProductMatrixImpl::build_packed( - a16.data(), m, packed, avx512_f16.data() - ); - CATCH_REQUIRE(amx_f16 == avx512_f16); - - std::vector amx_cm(n, -1.0F), avx512_cm(n, -2.0F); - InnerProductMatrixImpl::column_max( - a.data(), m, packed, amx_cm.data() - ); - InnerProductMatrixImpl::column_max( - a.data(), m, packed, avx512_cm.data() - ); - CATCH_REQUIRE(amx_cm == avx512_cm); - - std::vector via_build_amx(m * n, -1.0F), - via_build_avx512(m * n, -2.0F); - InnerProductMatrixImpl::build( - a.data(), m, b.data(), n, d, via_build_amx.data() - ); - InnerProductMatrixImpl::build( - a.data(), m, b.data(), n, d, via_build_avx512.data() - ); - CATCH_REQUIRE(via_build_amx == via_build_avx512); - } + for (bool r : results) { + CATCH_REQUIRE(r == results[0]); } } -} - -CATCH_TEST_CASE( - "InnerProductMatrixImpl falls back to AVX-512 bit for bit when B was not " - "packed with the tile layout", - "[distance][inner_product_matrix][amx]" -) { - // PackedOperand::pack (not pack_rounded_to_bf16) never fills the tile layout, even - // when AMX is compiled in and available: it is the caller's signal "B is not - // rounded to bf16," and the AMX kernel only ever computes . - unsigned seed = 11000; - for (size_t d : {1UL, 7UL, 128UL, 129UL}) { - for (size_t m : {0UL, 1UL, 17UL, 33UL}) { - for (size_t n : {1UL, 17UL, 33UL}) { - auto a = random_matrix(m, d, seed++); - auto a_bf16 = to_bfloat16(a); - auto b = random_matrix(n, d, seed++); - PackedOperand packed; - packed.pack(b.data(), n, d); - CATCH_REQUIRE(!packed.has_tile_layout()); - - std::vector amx_out(m * n, -1.0F), avx512_out(m * n, -2.0F); - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, amx_out.data() - ); - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, avx512_out.data() - ); - CATCH_REQUIRE(amx_out == avx512_out); - std::vector amx_cm(n, -1.0F), avx512_cm(n, -2.0F); - InnerProductMatrixImpl::column_max( - a_bf16.data(), m, packed, amx_cm.data() - ); - InnerProductMatrixImpl::column_max( - a_bf16.data(), m, packed, avx512_cm.data() - ); - CATCH_REQUIRE(amx_cm == avx512_cm); - } + // A worker that never calls amx_tile_bf16_available() itself (as SVS's thread pool + // doesn't) must still get correct AMX results (see fix_argument_decoded in gem.h). + CATCH_SECTION("A thread started before permission is requested can still run AMX") { + if (!svs::detail::amx_tile_bf16_available()) { + return; } - } -} - -CATCH_TEST_CASE( - "svs::detail::amx_tile_bf16_available is consistent across concurrent threads", - "[distance][inner_product_matrix][amx]" -) { - constexpr size_t num_threads = 8; - std::vector threads; - std::array results{}; - for (size_t t = 0; t < num_threads; ++t) { - threads.emplace_back([&results, t] { - results[t] = svs::detail::amx_tile_bf16_available(); - }); - } - for (auto& th : threads) { - th.join(); - } - bool first = results[0]; - for (bool r : results) { - CATCH_REQUIRE(r == first); - } - CATCH_REQUIRE(first == svs::detail::amx_tile_bf16_available()); -} - -CATCH_TEST_CASE( - "A thread started before AMX permission is ever requested can still run the AMX " - "kernel", - "[distance][inner_product_matrix][amx]" -) { - if (!svs::detail::amx_tile_bf16_available()) { - return; - } - // This test only demonstrates the property if it runs before any other test case in - // this binary has already cached svs::detail::amx_tile_bf16_available()'s answer; - // Catch2 runs test cases in a fixed but unspecified order, so instead this spawns a - // thread that calls straight into the AMX kernel without itself calling - // amx_tile_bf16_available() first, which is the actual shape of the hazard: a worker - // thread in SVS's thread pool that never calls the detection function at all, relying - // only on whichever thread called it first (see fix_argument_decoded in gem.h). - size_t m = 17, n = 17, d = 70; - auto a = random_matrix(m, d, 12000); - auto a_bf16 = to_bfloat16(a); - auto b = random_matrix(n, d, 12001); - auto b_rounded = to_bfloat16_as_float(b); - PackedOperand packed; - packed.pack_rounded_to_bf16(b.data(), n, d); + size_t m = 17, n = 17, d = 70; + auto a_bf16 = to_type(random_matrix(m, d, 12000)); + auto b = random_matrix(n, d, 12001); + auto b_rounded = to_bfloat16_as_float(b); + PackedOperand packed; + packed.pack_rounded_to_bf16(b.data(), n, d); - std::vector out(m * n, -1.0F); - std::thread worker([&] { - InnerProductMatrixImpl::build_packed( - a_bf16.data(), m, packed, out.data() + std::vector out(m * n, -1.0F); + std::thread([&] { + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, out.data() + ); + }).join(); + require_close( + out, + reference_from_values(promote_row(a_bf16), m, b_rounded, n, d), + NARROW_REL_EPS, + NARROW_ABS_EPS ); - }); - worker.join(); - require_close_loose( - out, reference_from_values(promote_bfloat16(a_bf16), m, b_rounded, n, d) - ); + } } #if SVS_TEST_HAS_MMAP namespace { -// Owns one `mmap` region sized to guard exactly `usable_bytes` of real storage against -// one neighbouring `PROT_NONE` page: `end_guarded` places the usable region so its last -// byte is the last byte before the guard page (catches an overrun), `start_guarded` -// places it at the first byte after a leading guard page (catches an underrun). A -// kernel that reads or writes one byte past what the test tells it to touch faults -// (SIGSEGV), which kills the test binary; that crash, not a Catch2 assertion, is this -// test's failure signal. -class GuardedMapping { +// RAII mmap guard: `bytes` of usable storage plus one neighbouring PROT_NONE page; +// `at_end` guards after the usable region (catches overrun), else before it (underrun). +// An out-of-bounds kernel access segfaults, which is this test's failure signal. +class Guarded { public: - static GuardedMapping end_guarded(size_t usable_bytes) { - size_t page = page_size(); - size_t usable_pages = (usable_bytes + page - 1) / page; - size_t total = (usable_pages + 1) * page; - void* base = map(total); - guard(static_cast(base) + usable_pages * page, page); - void* ptr = static_cast(base) + usable_pages * page - usable_bytes; - return GuardedMapping(base, total, ptr); - } - - static GuardedMapping start_guarded(size_t usable_bytes) { - size_t page = page_size(); - size_t usable_pages = (usable_bytes + page - 1) / page; - size_t total = (usable_pages + 1) * page; - void* base = map(total); - guard(base, page); - void* ptr = static_cast(base) + page; - return GuardedMapping(base, total, ptr); - } - - GuardedMapping(const GuardedMapping&) = delete; - GuardedMapping& operator=(const GuardedMapping&) = delete; - GuardedMapping(GuardedMapping&& other) noexcept - : base_{other.base_} - , total_{other.total_} - , ptr_{other.ptr_} { - other.base_ = nullptr; - } - GuardedMapping& operator=(GuardedMapping&&) = delete; - - ~GuardedMapping() { - if (base_ != nullptr) { - munmap(base_, total_); - } - } - + Guarded(size_t bytes, bool at_end) { + size_t page = static_cast(sysconf(_SC_PAGESIZE)); + size_t pages = (bytes + page - 1) / page; + total_ = (pages + 1) * page; + base_ = mmap( + nullptr, total_, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0 + ); + CATCH_REQUIRE(base_ != MAP_FAILED); + char* guard_page = static_cast(base_) + (at_end ? pages * page : 0); + CATCH_REQUIRE(mprotect(guard_page, page, PROT_NONE) == 0); + ptr_ = at_end ? guard_page - bytes : static_cast(base_) + page; + } + Guarded(const Guarded&) = delete; + Guarded& operator=(const Guarded&) = delete; + ~Guarded() { munmap(base_, total_); } void* data() const { return ptr_; } private: - GuardedMapping(void* base, size_t total, void* ptr) - : base_{base} - , total_{total} - , ptr_{ptr} {} - - static size_t page_size() { return static_cast(sysconf(_SC_PAGESIZE)); } - - static void* map(size_t total) { - void* base = mmap( - nullptr, total, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0 - ); - CATCH_REQUIRE(base != MAP_FAILED); - return base; - } - - static void guard(void* addr, size_t len) { - CATCH_REQUIRE(mprotect(addr, len, PROT_NONE) == 0); - } - void* base_; size_t total_; void* ptr_; }; -// Every tail shape the table and column-max kernels' SIMD/tile paths mask: m and n each -// cross the 16-wide tile/lane boundary, d crosses the 32-element (16-pair) AMX k-chunk -// boundary and both parities, matching the AMX tail sweep above but written out -// explicitly per the task's shape list rather than reusing `amx_ms`/`amx_ns`/`amx_ds`. const std::vector guard_ms{1, 15, 16, 17, 33}; const std::vector guard_ns{1, 15, 16, 17, 33}; const std::vector guard_ds{1, 2, 31, 32, 33, 127, 128, 129}; -// Runs both kernel forms (table and column-max) for one (level, element type) pair over -// every (m, n, d) in the guard sweep, with A and `out` each placed, in turn, so a -// one-element overrun or underrun would fault. `kernel` dispatches to -// InnerProductMatrixImpl::build_packed/column_max for whichever `T` the caller -// closed over; `pack_b` chooses `PackedOperand::pack` vs. `pack_rounded_to_bf16` the same -// way MaxSim/InnerProductMatrix's own callers do. -template -void run_guarded_sweep( - bool pack_b_rounded, BuildPacked build_packed, ColumnMax column_max -) { +// Runs build_packed and column_max for one (level, type) pair over every guard shape, +// with A and `out` placed at a guard boundary in turn so a one-element over/underrun +// faults. BFloat16 packs B rounded, matching every real caller of a bf16 A; the rest +// pack plain. +template void run_guarded_level() { + unsigned seed = 0; for (size_t d : guard_ds) { for (size_t m : guard_ms) { for (size_t n : guard_ns) { - auto a_src = - random_matrix(m, d, static_cast(m * 997 + n * 31 + d)); - std::vector a_host(m * d); - for (size_t i = 0; i < a_host.size(); ++i) { - a_host[i] = T(a_src[i]); - } - auto b = random_matrix(n, d, static_cast(d * 997 + m * 31 + n)); + auto a_src = random_matrix(m, d, seed++); + auto a_host = to_type(a_src); + auto b = random_matrix(n, d, seed++); PackedOperand packed; - if (pack_b_rounded) { + if constexpr (std::is_same_v) { packed.pack_rounded_to_bf16(b.data(), n, d); } else { packed.pack(b.data(), n, d); } - // (1) A ends exactly at a guard page: an overrun past A's last element - // faults. - { - GuardedMapping guard = GuardedMapping::end_guarded(m * d * sizeof(T)); - std::memcpy(guard.data(), a_host.data(), m * d * sizeof(T)); - const T* a = static_cast(guard.data()); - std::vector table_out(m * n, 0.0F); - build_packed(a, m, packed, table_out.data()); - std::vector cm_out(n, 0.0F); - column_max(a, m, packed, cm_out.data()); - } - - // (2) `out` ends exactly at a guard page, for both forms. - { - GuardedMapping guard = - GuardedMapping::end_guarded(m * n * sizeof(float)); - build_packed( - a_host.data(), m, packed, static_cast(guard.data()) + for (bool at_end : {true, false}) { + Guarded a_guard(m * d * sizeof(T), at_end); + std::memcpy(a_guard.data(), a_host.data(), m * d * sizeof(T)); + const T* a = static_cast(a_guard.data()); + std::vector table_out(m * n, 0.0F), cm_out(n, 0.0F); + InnerProductMatrixImpl::build_packed( + a, m, packed, table_out.data() ); + InnerProductMatrixImpl::column_max(a, m, packed, cm_out.data()); } - { - GuardedMapping guard = GuardedMapping::end_guarded(n * sizeof(float)); - column_max(a_host.data(), m, packed, static_cast(guard.data())); - } - - // (3) A starts at the first byte after a guard page: an underrun before - // A's first element faults. - { - GuardedMapping guard = GuardedMapping::start_guarded(m * d * sizeof(T)); - std::memcpy(guard.data(), a_host.data(), m * d * sizeof(T)); - const T* a = static_cast(guard.data()); - std::vector table_out(m * n, 0.0F); - build_packed(a, m, packed, table_out.data()); - std::vector cm_out(n, 0.0F); - column_max(a, m, packed, cm_out.data()); - } + Guarded table_guard(m * n * sizeof(float), true); + InnerProductMatrixImpl::build_packed( + a_host.data(), m, packed, static_cast(table_guard.data()) + ); + Guarded cm_guard(n * sizeof(float), true); + InnerProductMatrixImpl::column_max( + a_host.data(), m, packed, static_cast(cm_guard.data()) + ); } } } } -template void run_guarded_level() { - run_guarded_sweep( - std::same_as, - [](const T* a, size_t m, const PackedOperand& b, float* out) { - InnerProductMatrixImpl::build_packed(a, m, b, out); - }, - [](const T* a, size_t m, const PackedOperand& b, float* out) { - InnerProductMatrixImpl::column_max(a, m, b, out); - } - ); -} - } // namespace -// The AMX tile kernels (amx_table_bf16 / amx_column_max_bf16, reached only for a bfloat16 -// A against a tile-packed B) copy A's tail through a zero-padded stack buffer precisely -// to avoid reading past A's own allocation (see PackedOperand::fill_tile_layout's doc -// comment); this test is the memory-safety check for that claim, and for the AVX-512/AVX2 -// masked-tail kernels' equivalent claim. A violation is a SIGSEGV that kills this test -// binary, which is the failure signal, not a CATCH_REQUIRE. +// The AMX tile kernels copy A's tail through a zero-padded stack buffer to avoid reading +// past A's own allocation (PackedOperand::fill_tile_layout); a violation segfaults. CATCH_TEST_CASE( "InnerProductMatrixImpl kernels never read or write outside A or out, including " "their masked tails", "[distance][inner_product_matrix][amx]" ) { - CATCH_SECTION("AMX bfloat16 tile kernel") { - if (!svs::detail::amx_tile_bf16_available()) { - return; - } + if (svs::detail::amx_tile_bf16_available()) { run_guarded_level(); } - - CATCH_SECTION("AVX-512 kernels") { - if (!svs::detail::avx_runtime_flags.is_avx512f_supported()) { - return; - } + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { run_guarded_level(); run_guarded_level(); run_guarded_level(); } - - CATCH_SECTION("AVX2 kernels") { - if (!svs::detail::avx_runtime_flags.is_avx2_supported()) { - return; - } + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { run_guarded_level(); run_guarded_level(); run_guarded_level(); diff --git a/tests/svs/core/distances/maxsim.cpp b/tests/svs/core/distances/maxsim.cpp index d091a9eb5..b3aa6af33 100644 --- a/tests/svs/core/distances/maxsim.cpp +++ b/tests/svs/core/distances/maxsim.cpp @@ -40,8 +40,8 @@ namespace { // Minimal svs::data::ContiguousVectorSet model: a flat, row-major buffer of `cardinality` -// tokens of `dim` elements each. Flat (rather than a vector of per-token vectors) because -// MaxSim's packed-operand kernel needs `data()` to return one contiguous block. +// tokens of `dim` elements, since MaxSim's packed-operand kernel needs one contiguous +// block. template struct TestVectorSet { using element_type = T; std::vector flat; @@ -65,47 +65,34 @@ TestVectorSet make_set(std::initializer_list> tokens return set; } +// Same tokens as `make_set`, converting each float literal to T via T's constructor, so a +// test case can write one literal shape and run it against several document types. +template +TestVectorSet make_doc(std::initializer_list> tokens) { + TestVectorSet set; + set.dim = tokens.begin()->size(); + for (const auto& token : tokens) { + for (float v : token) { + set.flat.push_back(T(v)); + } + } + return set; +} + template double promote(T x) { return static_cast(x); } -// Namespace-scope concept (rather than a function-local requires-expression) so a -// substitution failure inside MaxSim::compute's requires-clause, which -// mentions the enclosing class template's parameter, stays in the immediate context for -// SFINAE purposes. +// Namespace-scope (not a function-local requires-expression) so a substitution failure +// inside MaxSim::compute's requires-clause stays in the immediate context. template concept CanComputeMaxSim = requires(svs::distance::MaxSim& dist, const D& d) { dist.compute(0, d); }; // Naive double-precision oracle: sum over query tokens of the max inner product to any -// document token. -template -double maxsim_reference(const TestVectorSet& query, const TestVectorSet& document) { - if (document.cardinality() == 0) { - return -std::numeric_limits::infinity(); - } - double total = 0.0; - for (size_t i = 0; i < query.cardinality(); ++i) { - auto q_token = query.token(i); - double best = -std::numeric_limits::infinity(); - for (size_t j = 0; j < document.cardinality(); ++j) { - auto d_token = document.token(j); - double dot = 0.0; - for (size_t d = 0; d < query.dimensions(); ++d) { - dot += promote(q_token[d]) * promote(d_token[d]); - } - best = std::max(best, dot); - } - total += best; - } - return total; -} - -// Same oracle as `maxsim_reference`, but both the query and the document values are -// first rounded to bfloat16 precision: matches what MaxSim actually -// computes, since fix_argument rounds the query (pack_rounded_to_bf16) and the document -// is already held at bf16 precision. -double maxsim_reference_bf16( - const TestVectorSet& query, const TestVectorSet& document -) { +// document token. Rounds the query to bf16 first for a BFloat16 document, since +// fix_argument does. +template +double +maxsim_reference(const TestVectorSet& query, const TestVectorSet& document) { if (document.cardinality() == 0) { return -std::numeric_limits::infinity(); } @@ -117,9 +104,11 @@ double maxsim_reference_bf16( auto d_token = document.token(j); double dot = 0.0; for (size_t d = 0; d < query.dimensions(); ++d) { - double qv = promote(svs::round_to_bfloat16(q_token[d])); - double dv = promote(d_token[d]); - dot += qv * dv; + double qv = promote(q_token[d]); + if constexpr (std::is_same_v) { + qv = promote(svs::round_to_bfloat16(q_token[d])); + } + dot += qv * promote(d_token[d]); } best = std::max(best, dot); } @@ -128,25 +117,21 @@ double maxsim_reference_bf16( return total; } -TestVectorSet -make_random_bf16_set(size_t cardinality, size_t dim, std::mt19937& rng) { - std::uniform_real_distribution dist(-1.0F, 1.0F); - TestVectorSet set; - set.dim = dim; - set.flat.resize(cardinality * dim); - for (auto& v : set.flat) { - v = svs::round_to_bfloat16(dist(rng)); - } - return set; -} - -TestVectorSet make_random_set(size_t cardinality, size_t dim, std::mt19937& rng) { +// Random query/document generator shared by every element type: BFloat16 rounds +// ties-to-even, float stores the draw unrounded. +template +TestVectorSet make_random_set(size_t cardinality, size_t dim, std::mt19937& rng) { std::uniform_real_distribution dist(-1.0F, 1.0F); - TestVectorSet set; + TestVectorSet set; set.dim = dim; set.flat.resize(cardinality * dim); for (auto& v : set.flat) { - v = dist(rng); + float f = dist(rng); + if constexpr (std::is_same_v) { + v = svs::round_to_bfloat16(f); + } else { + v = T(f); + } } return set; } @@ -162,30 +147,49 @@ float maxsim_score(const TestVectorSet& query, const D& document) { constexpr double MAXSIM_EPSILON = 1e-4; +template void check_oracle_on_random_sets(std::mt19937& rng) { + std::vector cardinalities{1, 2, 3, 7, 9, 17}; + std::vector dims{128, 37}; + for (size_t dim : dims) { + for (size_t nq : cardinalities) { + for (size_t nd : cardinalities) { + auto query = make_random_set(nq, dim, rng); + auto document = make_random_set(nd, dim, rng); + double expected = maxsim_reference(query, document); + float actual = maxsim_score(query, document); + CATCH_REQUIRE( + static_cast(actual) == + Catch::Approx(expected).epsilon(MAXSIM_EPSILON).margin(MAXSIM_EPSILON) + ); + } + } + } +} + } // namespace CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { - CATCH_SECTION("Trait members") { + CATCH_SECTION("Traits and compile-time document-element-type check") { CATCH_REQUIRE(std::is_same_v::compare, std::greater<>>); CATCH_REQUIRE_FALSE(svs::distance::MaxSim<>::implicit_broadcast); CATCH_REQUIRE(svs::distance::MaxSim<>::must_fix_argument); + // A mismatch is a compile error (the `requires` clause on `compute`), not a + // run-time branch. + static_assert(CanComputeMaxSim>); + static_assert(!CanComputeMaxSim>); + static_assert(CanComputeMaxSim>); + static_assert(!CanComputeMaxSim>); } - CATCH_SECTION("Hand-computed tiny case") { - // Query has two orthonormal tokens; document has one token aligned with the - // first and one aligned with neither, so the per-query maxima are 1 and 0. - auto query = make_set({{1.0F, 0.0F}, {0.0F, 1.0F}}); - auto document = make_set({{1.0F, 0.0F}, {0.0F, 0.0F}}); - float score = maxsim_score(query, document); - CATCH_REQUIRE(score == Catch::Approx(1.0F).epsilon(MAXSIM_EPSILON)); - } - + // The n_doc == 0 check is one branch in MaxSim::compute, shared by every document + // element type; float exercises it. CATCH_SECTION("Empty document set gives -inf") { auto query = make_set({{1.0F, 0.0F}}); TestVectorSet document; document.dim = 2; - float score = maxsim_score(query, document); - CATCH_REQUIRE(score == -std::numeric_limits::infinity()); + CATCH_REQUIRE( + maxsim_score(query, document) == -std::numeric_limits::infinity() + ); } CATCH_SECTION("Asymmetry") { @@ -198,44 +202,31 @@ CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { CATCH_REQUIRE(score_ab != Catch::Approx(score_ba)); } - CATCH_SECTION("Exact oracle on random sets") { + CATCH_SECTION("Exact oracle on random sets, float document") { std::mt19937 rng(0xC0FFEE); - std::vector cardinalities{1, 2, 3, 7, 9, 17}; - std::vector dims{128, 37}; - for (size_t dim : dims) { - for (size_t nq : cardinalities) { - for (size_t nd : cardinalities) { - auto query = make_random_set(nq, dim, rng); - auto document = make_random_set(nd, dim, rng); - double expected = maxsim_reference(query, document); - float actual = maxsim_score(query, document); - CATCH_REQUIRE( - static_cast(actual) == Catch::Approx(expected) - .epsilon(MAXSIM_EPSILON) - .margin(MAXSIM_EPSILON) - ); - } - } - } + check_oracle_on_random_sets(rng); + } + + CATCH_SECTION("Exact oracle on random sets, bfloat16 document, rounded query") { + std::mt19937 rng(0xB10A7); + check_oracle_on_random_sets(rng); } CATCH_SECTION("Mixed element types") { - // float query against Float16 document. + // float query against Float16 document: token 0 of query matches document token + // 0 exactly (dot = 1), token 1 has dot 0 with both document tokens. auto query = make_set({{1.0F, 0.0F, 0.0F}, {0.0F, 1.0F, 0.0F}}); - auto document = make_set( - {{svs::Float16(1.0F), svs::Float16(0.0F), svs::Float16(0.0F)}, - {svs::Float16(0.0F), svs::Float16(0.0F), svs::Float16(1.0F)}} - ); + auto document = make_doc({{1.0F, 0.0F, 0.0F}, {0.0F, 0.0F, 1.0F}}); float score = maxsim_score(query, document); - // Token 0 of query matches document token 0 exactly (dot = 1); token 1 of query - // has dot 0 with both document tokens. CATCH_REQUIRE(score == Catch::Approx(1.0F).epsilon(MAXSIM_EPSILON)); } + // The dimension check is one branch in MaxSim::compute, shared by every document + // element type; float exercises it. CATCH_SECTION("Dimension mismatch throws") { auto query = make_set({{1.0F, 0.0F, 0.0F}}); auto document = make_set({{1.0F, 0.0F}}); - svs::distance::MaxSim dist{}; + svs::distance::MaxSim dist{}; svs::distance::maybe_fix_argument(dist, query); CATCH_REQUIRE_THROWS_AS( svs::distance::compute(dist, query, document), svs::lib::ANNException @@ -244,11 +235,11 @@ CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { CATCH_SECTION("Repeated compute calls reuse one fix_argument") { std::mt19937 rng(7); - auto query = make_random_set(5, 32, rng); + auto query = make_random_set(5, 32, rng); svs::distance::MaxSim dist{}; svs::distance::maybe_fix_argument(dist, query); for (size_t i = 0; i < 4; ++i) { - auto document = make_random_set(i + 1, 32, rng); + auto document = make_random_set(i + 1, 32, rng); float direct = svs::distance::compute(dist, query, document); float expected = maxsim_reference(query, document); CATCH_REQUIRE( @@ -257,69 +248,4 @@ CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { ); } } - - CATCH_SECTION("Hand-computed bfloat16 document case") { - auto query = make_set({{1.0F, 0.0F, 0.0F}, {0.0F, 1.0F, 0.0F}}); - auto document = make_set( - {{svs::BFloat16(1.0F), svs::BFloat16(0.0F), svs::BFloat16(0.0F)}, - {svs::BFloat16(0.0F), svs::BFloat16(0.0F), svs::BFloat16(1.0F)}} - ); - float score = maxsim_score(query, document); - // Both values are exactly representable in bfloat16, so rounding changes nothing - // here: token 0 of query matches document token 0 exactly (dot = 1); token 1 has - // dot 0 with both document tokens. - CATCH_REQUIRE(score == Catch::Approx(1.0F).epsilon(MAXSIM_EPSILON)); - } - - CATCH_SECTION("Empty bfloat16 document set gives -inf") { - auto query = make_set({{1.0F, 0.0F}}); - TestVectorSet document; - document.dim = 2; - float score = maxsim_score(query, document); - CATCH_REQUIRE(score == -std::numeric_limits::infinity()); - } - - CATCH_SECTION("Bfloat16 dimension mismatch throws") { - auto query = make_set({{1.0F, 0.0F, 0.0F}}); - auto document = - make_set({{svs::BFloat16(1.0F), svs::BFloat16(0.0F)}}); - svs::distance::MaxSim dist{}; - svs::distance::maybe_fix_argument(dist, query); - CATCH_REQUIRE_THROWS_AS( - svs::distance::compute(dist, query, document), svs::lib::ANNException - ); - } - - CATCH_SECTION("Bfloat16 document exact oracle on random sets, rounded query") { - std::mt19937 rng(0xB10A7); - std::vector cardinalities{1, 2, 3, 7, 9, 17}; - std::vector dims{128, 37}; - for (size_t dim : dims) { - for (size_t nq : cardinalities) { - for (size_t nd : cardinalities) { - auto query = make_random_set(nq, dim, rng); - auto document = make_random_bf16_set(nd, dim, rng); - double expected = maxsim_reference_bf16(query, document); - float actual = maxsim_score(query, document); - CATCH_REQUIRE( - static_cast(actual) == Catch::Approx(expected) - .epsilon(MAXSIM_EPSILON) - .margin(MAXSIM_EPSILON) - ); - } - } - } - } - - CATCH_SECTION("Document element type is checked at compile time") { - using FloatDoc = TestVectorSet; - using BF16Doc = TestVectorSet; - // A MaxSim accepts a float document and rejects a bfloat16 one: the - // document's element type is part of the functor's type, so a mismatch is a - // compile error (the `requires` clause on `compute`), not a run-time branch. - static_assert(CanComputeMaxSim); - static_assert(!CanComputeMaxSim); - static_assert(CanComputeMaxSim); - static_assert(!CanComputeMaxSim); - } } diff --git a/tests/svs/core/kmeans_training.cpp b/tests/svs/core/kmeans_training.cpp index c2a01ba6a..61aebec9a 100644 --- a/tests/svs/core/kmeans_training.cpp +++ b/tests/svs/core/kmeans_training.cpp @@ -70,146 +70,139 @@ size_t brute_force_nearest( return best; } -} // namespace - -CATCH_TEST_CASE("Shared Kmeans Training - L2 Centroid Update", "[core][kmeans]") { +// `batch_size` sizes the preallocated matmul_results buffer (minibatch_size), which need +// not equal data.size(). +template +auto run_kmeans( + const svs::data::SimpleData& data, + Distance distance, + svs::data::SimpleData& centroids, + size_t num_iter, + size_t batch_size, + unsigned seed +) { namespace kmeans = svs::kmeans; - - // Two well-separated pairs of points, one per cluster, for an unambiguous assignment. - auto data = svs::data::SimpleData(4, 2); - data.set_datum(0, std::vector{0.0F, 0.0F}); - data.set_datum(1, std::vector{0.0F, 2.0F}); - data.set_datum(2, std::vector{10.0F, 0.0F}); - data.set_datum(3, std::vector{10.0F, 2.0F}); - - auto centroids = svs::data::SimpleData(2, 2); - centroids.set_datum(0, std::vector{0.0F, 0.0F}); - centroids.set_datum(1, std::vector{10.0F, 0.0F}); - - auto matmul_results = svs::data::SimpleData(10, 2); - auto rng = std::mt19937(0xc0ffee); + auto matmul_results = svs::data::SimpleData(batch_size, centroids.size()); + auto rng = std::mt19937(seed); auto threadpool = svs::threads::as_threadpool(2); auto timer = svs::lib::Timer(); - auto distance = svs::DistanceL2(); - - // centroid_adjustment divides by (count + 1), not count, so this is not a plain mean. - auto params = kmeans::KMeansTrainingParameters{1, 10}; - auto result = kmeans::kmeans_training( + auto params = kmeans::KMeansTrainingParameters{num_iter, batch_size}; + return kmeans::kmeans_training( params, data, distance, centroids, matmul_results, rng, threadpool, timer ); - - CATCH_REQUIRE(result.size() == 2); - - auto c0 = result.get_datum(0); - auto c1 = result.get_datum(1); - constexpr float tolerance = 1e-4F; - - CATCH_REQUIRE(std::abs(c0[0] - 0.0F) < tolerance); - CATCH_REQUIRE(std::abs(c0[1] - (2.0F / 3.0F)) < tolerance); - - CATCH_REQUIRE(std::abs(c1[0] - (30.0F / 3.0F)) < tolerance); - CATCH_REQUIRE(std::abs(c1[1] - (2.0F / 3.0F)) < tolerance); - - // L2 training does not normalize centroids, so they are not unit-norm in general. - float norm0 = std::sqrt(c0[0] * c0[0] + c0[1] * c0[1]); - float norm1 = std::sqrt(c1[0] * c1[0] + c1[1] * c1[1]); - CATCH_REQUIRE(std::abs(norm0 - 1.0F) > tolerance); - CATCH_REQUIRE(std::abs(norm1 - 1.0F) > tolerance); } -CATCH_TEST_CASE("Shared Kmeans Training - IP Centroids Are Unit Norm", "[core][kmeans]") { - namespace kmeans = svs::kmeans; - - auto data = svs::data::SimpleData(6, 3); - data.set_datum(0, std::vector{1.0F, 2.0F, 3.0F}); - data.set_datum(1, std::vector{2.0F, 1.0F, 4.0F}); - data.set_datum(2, std::vector{3.0F, 3.0F, 1.0F}); - data.set_datum(3, std::vector{-1.0F, -2.0F, 0.5F}); - data.set_datum(4, std::vector{-2.0F, -1.0F, 1.5F}); - data.set_datum(5, std::vector{-3.0F, -0.5F, 2.0F}); - - // Non-unit-norm initial centroids, so normalization is confirmed to happen inside. - auto centroids = svs::data::SimpleData(2, 3); - centroids.set_datum(0, data.get_datum(0)); - centroids.set_datum(1, data.get_datum(3)); - - auto matmul_results = svs::data::SimpleData(6, 2); - auto rng = std::mt19937(42); - auto threadpool = svs::threads::as_threadpool(2); - auto timer = svs::lib::Timer(); - auto distance = svs::DistanceIP(); +} // namespace - auto params = kmeans::KMeansTrainingParameters{3, 6}; - auto result = kmeans::kmeans_training( - params, data, distance, centroids, matmul_results, rng, threadpool, timer - ); +CATCH_TEST_CASE("Shared Kmeans Training", "[core][kmeans]") { + CATCH_SECTION("L2 centroid update divides by count + 1, and does not normalize") { + auto data = svs::data::SimpleData(4, 2); + data.set_datum(0, std::vector{0.0F, 0.0F}); + data.set_datum(1, std::vector{0.0F, 2.0F}); + data.set_datum(2, std::vector{10.0F, 0.0F}); + data.set_datum(3, std::vector{10.0F, 2.0F}); + + auto centroids = svs::data::SimpleData(2, 2); + centroids.set_datum(0, std::vector{0.0F, 0.0F}); + centroids.set_datum(1, std::vector{10.0F, 0.0F}); + + auto result = run_kmeans(data, svs::DistanceL2(), centroids, 1, 10, 0xc0ffee); + CATCH_REQUIRE(result.size() == 2); + + auto c0 = result.get_datum(0); + auto c1 = result.get_datum(1); + constexpr float tolerance = 1e-4F; + CATCH_REQUIRE(std::abs(c0[0] - 0.0F) < tolerance); + CATCH_REQUIRE(std::abs(c0[1] - (2.0F / 3.0F)) < tolerance); + CATCH_REQUIRE(std::abs(c1[0] - (30.0F / 3.0F)) < tolerance); + CATCH_REQUIRE(std::abs(c1[1] - (2.0F / 3.0F)) < tolerance); + + float norm0 = std::sqrt(c0[0] * c0[0] + c0[1] * c0[1]); + float norm1 = std::sqrt(c1[0] * c1[0] + c1[1] * c1[1]); + CATCH_REQUIRE(std::abs(norm0 - 1.0F) > tolerance); + CATCH_REQUIRE(std::abs(norm1 - 1.0F) > tolerance); + } - constexpr float tolerance = 1e-5F; - for (size_t i = 0; i < result.size(); ++i) { - auto c = result.get_datum(i); - float norm_sq = 0; - for (size_t d = 0; d < c.size(); ++d) { - norm_sq += c[d] * c[d]; + CATCH_SECTION("IP centroids come out unit-norm from non-unit-norm initial centroids") { + auto data = svs::data::SimpleData(6, 3); + data.set_datum(0, std::vector{1.0F, 2.0F, 3.0F}); + data.set_datum(1, std::vector{2.0F, 1.0F, 4.0F}); + data.set_datum(2, std::vector{3.0F, 3.0F, 1.0F}); + data.set_datum(3, std::vector{-1.0F, -2.0F, 0.5F}); + data.set_datum(4, std::vector{-2.0F, -1.0F, 1.5F}); + data.set_datum(5, std::vector{-3.0F, -0.5F, 2.0F}); + + auto centroids = svs::data::SimpleData(2, 3); + centroids.set_datum(0, data.get_datum(0)); + centroids.set_datum(1, data.get_datum(3)); + + auto result = run_kmeans(data, svs::DistanceIP(), centroids, 3, 6, 42); + + constexpr float tolerance = 1e-5F; + for (size_t i = 0; i < result.size(); ++i) { + auto c = result.get_datum(i); + float norm_sq = 0; + for (size_t d = 0; d < c.size(); ++d) { + norm_sq += c[d] * c[d]; + } + CATCH_REQUIRE(std::abs(std::sqrt(norm_sq) - 1.0F) < tolerance); } - CATCH_REQUIRE(std::abs(std::sqrt(norm_sq) - 1.0F) < tolerance); } -} -CATCH_TEST_CASE( - "Shared Kmeans Training - centroid_assignment Agrees With Brute Force", "[core][kmeans]" -) { - namespace kmeans = svs::kmeans; + CATCH_SECTION("centroid_assignment agrees with brute force, for L2 and IP") { + namespace kmeans = svs::kmeans; + constexpr size_t num_points = 30; + constexpr size_t num_centroids = 4; + constexpr size_t dims = 5; - constexpr size_t num_points = 30; - constexpr size_t num_centroids = 4; - constexpr size_t dims = 5; - - auto data = svs::data::SimpleData(num_points, dims); - for (size_t i = 0; i < num_points; ++i) { - auto datum = data.get_datum(i); - for (size_t d = 0; d < dims; ++d) { - // A deterministic, non-trivial pattern; no symmetry that would create ties. - datum[d] = static_cast((i * 7 + d * 3) % 11) - 5.0F; + auto data = svs::data::SimpleData(num_points, dims); + for (size_t i = 0; i < num_points; ++i) { + auto datum = data.get_datum(i); + for (size_t d = 0; d < dims; ++d) { + // A deterministic, non-trivial pattern; no symmetry that creates ties. + datum[d] = static_cast((i * 7 + d * 3) % 11) - 5.0F; + } } - } - auto centroids = svs::data::SimpleData(num_centroids, dims); - for (size_t i = 0; i < num_centroids; ++i) { - auto c = centroids.get_datum(i); - for (size_t d = 0; d < dims; ++d) { - c[d] = static_cast((i * 5 + d * 2) % 7) - 3.0F; + auto centroids = svs::data::SimpleData(num_centroids, dims); + for (size_t i = 0; i < num_centroids; ++i) { + auto c = centroids.get_datum(i); + for (size_t d = 0; d < dims; ++d) { + c[d] = static_cast((i * 5 + d * 2) % 7) - 3.0F; + } } - } - - auto threadpool = svs::threads::as_threadpool(2); - auto timer = svs::lib::Timer(); - auto batch_range = svs::threads::UnitRange{0, num_points}; - auto matmul_results = svs::data::SimpleData(num_points, num_centroids); - - auto check_distance = [&](Distance distance) { - auto data_norm = kmeans::maybe_compute_norms(data, threadpool); - auto centroids_norm = kmeans::maybe_compute_norms(centroids, threadpool); - auto assignments = std::vector(num_points); - - kmeans::centroid_assignment( - data, - data_norm, - batch_range, - distance, - centroids, - centroids_norm, - assignments, - matmul_results, - threadpool, - timer - ); - for (size_t i = 0; i < num_points; ++i) { - size_t expected = brute_force_nearest(data, i, centroids); - CATCH_REQUIRE(assignments[i] == expected); - } - }; + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto batch_range = svs::threads::UnitRange{0, num_points}; + auto matmul_results = svs::data::SimpleData(num_points, num_centroids); + + auto check_distance = [&](Distance distance) { + auto data_norm = kmeans::maybe_compute_norms(data, threadpool); + auto centroids_norm = + kmeans::maybe_compute_norms(centroids, threadpool); + auto assignments = std::vector(num_points); + + kmeans::centroid_assignment( + data, + data_norm, + batch_range, + distance, + centroids, + centroids_norm, + assignments, + matmul_results, + threadpool, + timer + ); + + for (size_t i = 0; i < num_points; ++i) { + size_t expected = brute_force_nearest(data, i, centroids); + CATCH_REQUIRE(assignments[i] == expected); + } + }; - check_distance(svs::DistanceL2()); - check_distance(svs::DistanceIP()); + check_distance(svs::DistanceL2()); + check_distance(svs::DistanceIP()); + } } From f6b7549ab4c448da5bb01c559c24e34f99f4b6be Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 17 Nov 2026 07:54:38 -0800 Subject: [PATCH 49/68] test(distance): consolidate the GEM distance and transport solver tests Replaces scattered hand-computed cases with one random oracle per distance (QEmd/QChamfer/QChamferExactQuery) checked against an independent naive reference, each covering duplicate ids and, for QChamferExactQuery, bf16 non-representable values. Consolidates the three per-distance "generic dispatch" sections and the three "dimensions() != 1 throws" sections into one test case each. Compacts reference_transport_cost to a standard min-cost-flow edge-list SPFA and merges the "many equal costs" and "uniform-weight tiny-cost" transport-solver families into one parameterized loop, with lower trial counts (fixed seeds). Lines: 798 -> 725 (9% cut). Short of the requested 50%/400-line target: the transport-solver section (independent reference + the four problem families x per-ISA cross-check, all explicitly requested to stay) is ~195 lines on its own, and the from_centroids tiling test, traits, and Vamana extension test cases are already near-minimal. Getting under 400 would mean dropping one of those explicitly-requested items, which this PR does not do unilaterally. One real coverage loss, restored: QEmd's "identical sets give 0" invariant is not exercised by the random oracle (which never draws a == b), so it stays as its own hand case. Two consolidations are not drop-in equivalent in form (only in the branches they exercise): the old hand-picked worked examples for QChamfer/QEmd are gone, replaced by randomized reference checks, so a reader can no longer eyeball one fixed expected number. Co-Authored-By: Claude Opus 5.5 --- tests/svs/core/distances/gem.cpp | 722 ++++++++++++++----------------- 1 file changed, 325 insertions(+), 397 deletions(-) diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp index f7c7554e6..1d0dce024 100644 --- a/tests/svs/core/distances/gem.cpp +++ b/tests/svs/core/distances/gem.cpp @@ -32,9 +32,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -50,8 +52,8 @@ using svs::distance::QChamfer; using svs::distance::QChamferExactQuery; using svs::distance::QEmd; -// Four unit vectors in 2D: (1,0), (0,1), (-1,0), (0,-1). Their pairwise dot products are -// all 0, 1 or -1, so both ground distances come out to round numbers by hand. +// Four unit vectors in 2D: (1,0), (0,1), (-1,0), (0,-1), so their pairwise ground +// distances come out to round numbers by hand. std::shared_ptr make_table(Ground ground) { std::vector centroids{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F, 0.0F, -1.0F}; return std::make_shared( @@ -59,8 +61,7 @@ std::shared_ptr make_table(Ground ground) { ); } -// K unit vectors of width dim, row-major: deterministic per (k, dim, seed) so the -// from_centroids tiling test below is reproducible across runs. +// K random unit vectors of width dim, row-major, deterministic per (k, dim, seed). std::vector random_unit_centroids(size_t k, size_t dim, uint32_t seed) { std::mt19937 rng(seed); std::normal_distribution gauss(0.0F, 1.0F); @@ -80,20 +81,8 @@ std::vector random_unit_centroids(size_t k, size_t dim, uint32_t seed) { return flat; } -// from_centroids now rounds every centroid component to bfloat16 (round to nearest, ties -// to even) before computing, so a reference built from the unrounded input would differ -// by bf16's ~0.4% rounding error, not just summation order; round here too so the -// pre-existing 1e-4 margin still bounds only the latter. -std::vector round_to_bf16_vec(const std::vector& flat) { - std::vector out(flat.size()); - for (size_t i = 0; i < flat.size(); ++i) { - out[i] = static_cast(svs::round_to_bfloat16(flat[i])); - } - return out; -} - -// Reference ground-cost table computed with double-precision dot products and the same -// formula from_centroids applies, independent of InnerProductMatrix/tiling. +// Reference ground-cost table via double-precision dot products over bf16-rounded +// inputs (from_centroids rounds internally too), independent of InnerProductMatrix/tiling. std::vector naive_ground_table(const std::vector& flat, size_t k, size_t dim, Ground ground) { std::vector t(k * k, 0.0); @@ -104,7 +93,8 @@ naive_ground_table(const std::vector& flat, size_t k, size_t dim, Ground } double dot = 0.0; for (size_t d = 0; d < dim; ++d) { - dot += static_cast(flat[i * dim + d]) * flat[j * dim + d]; + dot += static_cast(svs::round_to_bfloat16(flat[i * dim + d])) * + static_cast(svs::round_to_bfloat16(flat[j * dim + d])); } t[i * k + j] = ground == Ground::Cos ? std::max(0.0, 1.0 - dot) @@ -114,8 +104,8 @@ naive_ground_table(const std::vector& flat, size_t k, size_t dim, Ground return t; } -// Owns a 1-D id vector set contiguously, for exercising QEmd/QChamfer/QChamferExactQuery's -// only (vector-set) compute() overloads. +// Owns a 1-D id vector set contiguously, for QEmd/QChamfer/QChamferExactQuery's only +// (vector-set) compute() overloads. struct IdSet { std::vector ids; @@ -142,33 +132,94 @@ struct RawQuerySet { }; // Compile-time guard: the removed fixed-width padded-row layout must not come back. -// QChamfer's only compute() overload requires ContiguousVectorSet, so a type that fails -// that concept (a bare span, the old layout's carrier) cannot satisfy it either. static_assert(!svs::data::ContiguousVectorSet>); -} // namespace - -CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { - CATCH_SECTION("Cosine ground") { - auto table = make_table(Ground::Cos); - CATCH_REQUIRE((*table)(0, 0) == Catch::Approx(0.0F)); - CATCH_REQUIRE((*table)(0, 1) == Catch::Approx(1.0F)); - CATCH_REQUIRE((*table)(0, 2) == Catch::Approx(2.0F)); - CATCH_REQUIRE((*table)(0, 3) == Catch::Approx(1.0F)); - CATCH_REQUIRE((*table)(1, 2) == Catch::Approx(1.0F)); - CATCH_REQUIRE((*table)(1, 3) == Catch::Approx(2.0F)); - CATCH_REQUIRE((*table)(2, 3) == Catch::Approx(1.0F)); +// Independent reference: successive shortest augmenting paths via SPFA over the standard +// min-cost-flow edge-list graph. Deliberately a different algorithm from TransportSimplex. +double reference_transport_cost( + size_t ua, + const std::vector& supply, + size_t ub, + const std::vector& demand, + const std::vector& cost // ua * ub, row-major +) { + constexpr double inf = std::numeric_limits::infinity(); + size_t s = ua + ub, t = ua + ub + 1, n = ua + ub + 2; + std::vector eto; + std::vector ecap, ecost; + std::vector> adj(n); + auto add_edge = [&](size_t from, size_t to, double cap, double c) { + adj[from].push_back(eto.size()); + eto.push_back(static_cast(to)); + ecap.push_back(cap); + ecost.push_back(c); + adj[to].push_back(eto.size()); + eto.push_back(static_cast(from)); + ecap.push_back(0.0); + ecost.push_back(-c); + }; + for (size_t i = 0; i < ua; ++i) { + add_edge(s, i, supply[i], 0.0); + } + for (size_t j = 0; j < ub; ++j) { + add_edge(ua + j, t, demand[j], 0.0); + } + for (size_t i = 0; i < ua; ++i) { + for (size_t j = 0; j < ub; ++j) { + add_edge(i, ua + j, inf, cost[i * ub + j]); + } } - CATCH_SECTION("L2 ground") { - auto table = make_table(Ground::L2); - CATCH_REQUIRE((*table)(0, 0) == Catch::Approx(0.0F)); - CATCH_REQUIRE((*table)(0, 1) == Catch::Approx(std::sqrt(2.0F))); - CATCH_REQUIRE((*table)(0, 2) == Catch::Approx(2.0F)); - CATCH_REQUIRE((*table)(1, 3) == Catch::Approx(2.0F)); - CATCH_REQUIRE((*table)(2, 3) == Catch::Approx(std::sqrt(2.0F))); + double total = 0.0; + for (;;) { + std::vector dist(n, inf); + std::vector pred_edge(n, -1); + std::vector in_queue(n, false); + std::deque queue{s}; + dist[s] = 0.0; + in_queue[s] = true; + while (!queue.empty()) { + size_t u = queue.front(); + queue.pop_front(); + in_queue[u] = false; + for (size_t e : adj[u]) { + if (ecap[e] <= 1e-12) { + continue; + } + size_t v = static_cast(eto[e]); + if (dist[u] + ecost[e] < dist[v] - 1e-12) { + dist[v] = dist[u] + ecost[e]; + pred_edge[v] = static_cast(e); + if (!in_queue[v]) { + queue.push_back(v); + in_queue[v] = true; + } + } + } + } + if (dist[t] == inf) { + break; + } + std::vector path; + for (size_t v = t; v != s; v = static_cast(eto[path.back() ^ 1])) { + path.push_back(static_cast(pred_edge[v])); + } + double bottleneck = inf; + for (size_t e : path) { + bottleneck = std::min(bottleneck, ecap[e]); + } + for (size_t e : path) { + ecap[e] -= bottleneck; + ecap[e ^ 1] += bottleneck; + total += bottleneck * ecost[e]; + } } + return total; +} + +} // namespace +CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { CATCH_SECTION("FineCentroids holds raw rows, not a pairwise table") { std::vector flat{1.0F, 0.0F, 0.0F, 1.0F}; FineCentroids fc{2, 2, flat.data()}; @@ -183,8 +234,7 @@ CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { std::decay_t().data[0])>, svs::BFloat16>); // None of these are exactly representable in bfloat16: the test only holds if - // fc.data actually rounds (matching svs::round_to_bfloat16), not merely stores - // float32 unchanged or truncates like BFloat16's own constructor would. + // fc.data actually rounds, not merely stores float32 unchanged. std::vector flat{0.1F, -0.1F, 1.0F / 3.0F, -2.0F / 3.0F}; FineCentroids fc{2, 2, flat.data()}; for (size_t i = 0; i < flat.size(); ++i) { @@ -215,9 +265,8 @@ CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { } CATCH_TEST_CASE("CentroidTable::from_centroids tiling", "[distance][gem]") { - // K values exercising a single partial tile (1, 2, 7), a single full tile (255 off - // by one, 256 exact, 257 off by one), and several tiles (600); at a small (3) and a - // production (128) dimension, both ground distances, built with 1 and 4 threads. + // K spans a partial tile, a tile boundary (255/256/257), and several tiles (600), at + // both dims/grounds/thread-counts; the Gaussian input also exercises bf16 rounding. for (size_t k : {size_t{1}, size_t{2}, @@ -235,12 +284,7 @@ CATCH_TEST_CASE("CentroidTable::from_centroids tiling", "[distance][gem]") { k * 1009 + dim * 31 + (ground == Ground::Cos ? 1 : 2) ) ); - // from_centroids rounds internally; round the reference's input the same - // way so the comparison below isolates summation-order error, not bf16 - // rounding error. - auto reference = - naive_ground_table(round_to_bf16_vec(flat), k, dim, ground); - + auto reference = naive_ground_table(flat, k, dim, ground); auto one_thread = CentroidTable::from_centroids(flat.data(), k, dim, ground, 1); auto four_threads = @@ -254,9 +298,8 @@ CATCH_TEST_CASE("CentroidTable::from_centroids tiling", "[distance][gem]") { for (size_t j = 0; j < k; ++j) { auto ui = static_cast(i); auto uj = static_cast(j); - // Thread-count independence: bit for bit, not approximate. + // Thread-count independence and symmetry: bit for bit. CATCH_REQUIRE(one_thread(ui, uj) == four_threads(ui, uj)); - // Exact symmetry by construction: bit for bit. CATCH_REQUIRE(one_thread(ui, uj) == one_thread(uj, ui)); // Against a double-precision reference: summation order differs, // so only approximate agreement is expected. @@ -272,235 +315,125 @@ CATCH_TEST_CASE("CentroidTable::from_centroids tiling", "[distance][gem]") { } CATCH_TEST_CASE("QChamfer", "[distance][gem]") { - auto table = make_table(Ground::Cos); - QChamfer dist{table}; - - CATCH_SECTION("Hand-computed: min over doc per query token") { - // query ids {0,1}, doc ids {2,3}: table(0,2)=2,table(0,3)=1 -> min 1; - // table(1,2)=1,table(1,3)=2 -> min 1. total = 1 + 1 = 2. - auto q = make_id_set({0, 1}); - auto d = make_id_set({2, 3}); - float score = dist.compute(q.view(), d.view()); - CATCH_REQUIRE(score == Catch::Approx(2.0F)); - } - - CATCH_SECTION("Duplicate ids are counted, not merged") { - // query {0,0} against doc {1}: each of the two query tokens pays table(0,1)=1. - auto q = make_id_set({0, 0}); - auto d = make_id_set({1}); - float score = dist.compute(q.view(), d.view()); - CATCH_REQUIRE(score == Catch::Approx(2.0F)); - } - - CATCH_SECTION("Cardinality 1") { - auto q = make_id_set({0}); - auto d = make_id_set({2}); - float score = dist.compute(q.view(), d.view()); - CATCH_REQUIRE(score == Catch::Approx(2.0F)); - } - - CATCH_SECTION("Different cardinalities") { - // query {0,0,1} vs doc {2}: table(0,2)=2 (twice, one per query token) + - // table(1,2)=1 = 5. - auto q = make_id_set({0, 0, 1}); - auto d = make_id_set({2}); - float score = dist.compute(q.view(), d.view()); - CATCH_REQUIRE(score == Catch::Approx(5.0F)); - } + CATCH_SECTION("Random oracle: naive min-over-doc sum, duplicate ids included") { + std::mt19937 rng(0xC4A4FE2u); + std::uniform_int_distribution n_dist(1, 8); + std::uniform_int_distribution id_dist(0, 5); + std::uniform_real_distribution cost_dist(0.0F, 5.0F); + size_t k = 6; + for (int trial = 0; trial < 30; ++trial) { + std::vector table_flat(k * k); + for (auto& c : table_flat) { + c = cost_dist(rng); + } + auto table = std::make_shared(k, table_flat); + QChamfer dist{table}; - CATCH_SECTION("dimensions() != 1 throws") { - std::vector raw{0, 1, 2, 3}; - auto bad = - svs::data::ConstVectorSet(std::span(raw), 2, 2); - auto good = make_id_set({0}); - CATCH_REQUIRE_THROWS_AS(dist.compute(bad, good.view()), std::invalid_argument); - CATCH_REQUIRE_THROWS_AS(dist.compute(good.view(), bad), std::invalid_argument); - } + std::vector q(n_dist(rng)), d(n_dist(rng)); + for (auto& id : q) { + id = id_dist(rng); + } + for (auto& id : d) { + id = id_dist(rng); + } + double reference = 0.0; + for (auto qi : q) { + float best = std::numeric_limits::infinity(); + for (auto di : d) { + best = std::min(best, (*table)(qi, di)); + } + reference += best; + } - CATCH_SECTION("Generic svs::distance::compute dispatches to the vector-set overload") { - auto q = make_id_set({0, 1}); - auto d = make_id_set({2, 3}); - float via_generic = svs::distance::compute(dist, q.view(), d.view()); - float direct = dist.compute(q.view(), d.view()); - CATCH_REQUIRE(via_generic == Catch::Approx(direct)); + float score = dist.compute(make_id_set(q).view(), make_id_set(d).view()); + CATCH_REQUIRE( + static_cast(score) == Catch::Approx(reference).margin(1e-4) + ); + } } } CATCH_TEST_CASE("QEmd", "[distance][gem]") { - auto table = make_table(Ground::Cos); - QEmd dist{table}; + QEmd dist{make_table(Ground::Cos)}; - CATCH_SECTION("1 vs 1") { - auto a = make_id_set({0}); - auto b = make_id_set({1}); - float score = dist.compute(a.view(), b.view()); + CATCH_SECTION("1 vs 1 takes the direct table lookup, bypassing the solver") { + float score = dist.compute(make_id_set({0}).view(), make_id_set({1}).view()); CATCH_REQUIRE(score == Catch::Approx(1.0F)); // table(0,1) } - CATCH_SECTION("1 vs 2") { - // One source of mass 1 split 0.5/0.5 to two distinct sinks: 0.5*table(0,1) + - // 0.5*table(0,2) = 0.5*1 + 0.5*2 = 1.5. - auto a = make_id_set({0}); - auto b = make_id_set({1, 2}); - float score = dist.compute(a.view(), b.view()); - CATCH_REQUIRE(score == Catch::Approx(1.5F)); - } - - CATCH_SECTION("2 vs 2 with obvious optimal matching") { - // Masses 0.5/0.5 on each side; matching 0->3, 1->2 costs 0.5*1+0.5*1=1, strictly - // less than the cross matching 0->2, 1->3 (0.5*2+0.5*2=2). - auto a = make_id_set({0, 1}); - auto b = make_id_set({2, 3}); - float score = dist.compute(a.view(), b.view()); - CATCH_REQUIRE(score == Catch::Approx(1.0F)); - } - CATCH_SECTION("Duplicates merge to a single point with full mass") { - // {0,0} merges to one point of mass 1 at id 0, same as the plain "1 vs 1" case. - auto a = make_id_set({0, 0}); - auto b = make_id_set({1}); - float score = dist.compute(a.view(), b.view()); - CATCH_REQUIRE(score == Catch::Approx(1.0F)); + float score = dist.compute(make_id_set({0, 0}).view(), make_id_set({1}).view()); + CATCH_REQUIRE(score == Catch::Approx(1.0F)); // same as plain 1 vs 1 } + // Not exercised by the random oracle below, which never draws a == b. CATCH_SECTION("Identical sets give 0") { auto a = make_id_set({0, 1}); - float score = dist.compute(a.view(), a.view()); - CATCH_REQUIRE(score == Catch::Approx(0.0F)); - } - - CATCH_SECTION("Symmetry") { - auto a = make_id_set({0}); - auto b = make_id_set({1, 2}); - float ab = dist.compute(a.view(), b.view()); - float ba = dist.compute(b.view(), a.view()); - CATCH_REQUIRE(ab == Catch::Approx(ba)); - } - - CATCH_SECTION("dimensions() != 1 throws") { - std::vector raw{0, 1, 2, 3}; - auto bad = - svs::data::ConstVectorSet(std::span(raw), 2, 2); - auto good = make_id_set({0}); - CATCH_REQUIRE_THROWS_AS(dist.compute(bad, good.view()), std::invalid_argument); - CATCH_REQUIRE_THROWS_AS(dist.compute(good.view(), bad), std::invalid_argument); - } - - CATCH_SECTION("Generic svs::distance::compute dispatches to the vector-set overload") { - auto a = make_id_set({0}); - auto b = make_id_set({1, 2}); - float via_generic = svs::distance::compute(dist, a.view(), b.view()); - float direct = dist.compute(a.view(), b.view()); - CATCH_REQUIRE(via_generic == Catch::Approx(direct)); - } -} - -namespace { + CATCH_REQUIRE(dist.compute(a.view(), a.view()) == Catch::Approx(0.0F)); + } + + // Covers compute() end to end (merge_duplicates, runtime ISA dispatch, the solver) + // against the independent reference over manually-merged weights, plus symmetry. + CATCH_SECTION("Random oracle against the independent reference, with duplicate ids") { + std::mt19937 rng(0x5151u); + std::uniform_int_distribution n_dist(2, 10); + std::uniform_int_distribution id_dist(0, 5); + std::uniform_real_distribution cost_dist(0.0F, 5.0F); + size_t k = 6; + auto merge = [](const std::vector& ids) { + std::map w; + for (auto id : ids) { + w[id] += 1.0 / static_cast(ids.size()); + } + std::vector uids; + std::vector uw; + for (auto [id, weight] : w) { + uids.push_back(id); + uw.push_back(weight); + } + return std::pair{uids, uw}; + }; -// Independent reference for the balanced transportation problem: successive shortest -// augmenting paths (Ahuja-Magnanti-Orlin ch. 9) over the explicit source/sink residual -// graph, valid because every qEMD ground cost is nonnegative. Deliberately a different -// algorithm from TransportSimplex, not a second copy of it. -double reference_transport_cost( - size_t ua, - const std::vector& supply, - size_t ub, - const std::vector& demand, - const std::vector& cost // ua * ub, row-major -) { - constexpr double inf = std::numeric_limits::infinity(); - size_t s_node = ua + ub, t_node = ua + ub + 1, n = ua + ub + 2; - std::vector rem_s(supply), rem_d(demand), flow(ua * ub, 0.0); - double total = 0.0; - for (;;) { - std::vector dist(n, inf); - std::vector pred(n, -1); - std::vector kind(n, 0); // 0: S->src, 1: src->sink, 2: sink->src, 3: sink->T - dist[s_node] = 0.0; - for (size_t pass = 0; pass < n; ++pass) { - for (size_t i = 0; i < ua; ++i) { - if (rem_s[i] > 1e-12 && dist[s_node] < dist[i]) { - dist[i] = dist[s_node]; - pred[i] = static_cast(s_node); - kind[i] = 0; - } + for (int trial = 0; trial < 20; ++trial) { + std::vector table_flat(k * k); + for (auto& c : table_flat) { + c = cost_dist(rng); } - for (size_t i = 0; i < ua; ++i) { - if (dist[i] == inf) { - continue; - } - for (size_t j = 0; j < ub; ++j) { - double nd = dist[i] + cost[i * ub + j]; - if (nd < dist[ua + j] - 1e-12) { - dist[ua + j] = nd; - pred[ua + j] = static_cast(i); - kind[ua + j] = 1; - } + for (size_t i = 0; i < k; ++i) { + table_flat[i * k + i] = 0.0F; + for (size_t j = 0; j < i; ++j) { + table_flat[i * k + j] = table_flat[j * k + i]; } } - for (size_t j = 0; j < ub; ++j) { - if (dist[ua + j] == inf) { - continue; - } - for (size_t i = 0; i < ua; ++i) { - if (flow[i * ub + j] > 1e-12) { - double nd = dist[ua + j] - cost[i * ub + j]; - if (nd < dist[i] - 1e-12) { - dist[i] = nd; - pred[i] = static_cast(ua + j); - kind[i] = 2; - } - } - } - if (rem_d[j] > 1e-12 && dist[ua + j] < dist[t_node] - 1e-12) { - dist[t_node] = dist[ua + j]; - pred[t_node] = static_cast(ua + j); - kind[t_node] = 3; - } + auto table = std::make_shared(k, table_flat); + QEmd dist{table}; + + std::vector ids_a(n_dist(rng)), ids_b(n_dist(rng)); + for (auto& id : ids_a) { + id = id_dist(rng); } - } - if (dist[t_node] == inf) { - break; - } - double bottleneck = inf; - for (int64_t cur = static_cast(t_node); - cur != static_cast(s_node);) { - int64_t p = pred[static_cast(cur)]; - if (kind[static_cast(cur)] == 0) { - bottleneck = std::min(bottleneck, rem_s[static_cast(cur)]); - } else if (kind[static_cast(cur)] == 2) { - bottleneck = std::min( - bottleneck, - flow[static_cast(cur) * ub + static_cast(p) - ua] - ); - } else if (kind[static_cast(cur)] == 3) { - bottleneck = std::min(bottleneck, rem_d[static_cast(p) - ua]); + for (auto& id : ids_b) { + id = id_dist(rng); } - cur = p; - } - for (int64_t cur = static_cast(t_node); - cur != static_cast(s_node);) { - int64_t p = pred[static_cast(cur)]; - size_t curu = static_cast(cur), pu = static_cast(p); - if (kind[curu] == 0) { - rem_s[curu] -= bottleneck; - } else if (kind[curu] == 1) { - flow[pu * ub + (curu - ua)] += bottleneck; - total += bottleneck * cost[pu * ub + (curu - ua)]; - } else if (kind[curu] == 2) { - flow[curu * ub + (pu - ua)] -= bottleneck; - total -= bottleneck * cost[curu * ub + (pu - ua)]; - } else { - rem_d[pu - ua] -= bottleneck; + auto [uids_a, uw_a] = merge(ids_a); + auto [uids_b, uw_b] = merge(ids_b); + std::vector cost(uids_a.size() * uids_b.size()); + for (size_t i = 0; i < uids_a.size(); ++i) { + for (size_t j = 0; j < uids_b.size(); ++j) { + cost[i * uids_b.size() + j] = (*table)(uids_a[i], uids_b[j]); + } } - cur = p; + double reference = + reference_transport_cost(uids_a.size(), uw_a, uids_b.size(), uw_b, cost); + + float ab = dist.compute(make_id_set(ids_a).view(), make_id_set(ids_b).view()); + float ba = dist.compute(make_id_set(ids_b).view(), make_id_set(ids_a).view()); + CATCH_REQUIRE(static_cast(ab) == Catch::Approx(reference).margin(1e-5)); + CATCH_REQUIRE(ba == Catch::Approx(ab)); } } - return total; } -} // namespace - CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][gem]") { using svs::distance::AVX_AVAILABILITY; using svs::distance::QEmdImpl; @@ -509,16 +442,10 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge std::uniform_int_distribution size_dist(1, 40); std::uniform_real_distribution cost_dist(0.0, 10.0); - // Builds one random balanced problem, decoded ids_a/ids_b plus merged weights that - // already sum to 1 (matching QEmdImpl::compute_decoded's real inputs), and checks the - // NONE path against the independent reference and every compiled ISA path against NONE. - // `tiny_cost_range >= 0` draws integer costs from [0, tiny_cost_range] instead of a - // continuous spread -- the highly degenerate case (many tied reduced costs) that the - // block-search entering rule plus the strongly-feasible-tree leaving rule need to - // handle without stalling. `uniform_weights` skips random weights for exact 1/n ties. + // Checks compute_decoded's NONE path against the reference and every ISA path against + // NONE. `tiny_cost_range >= 0`/`uniform_weights` force the degenerate (tied) case. auto run_case = [&](size_t ua, size_t ub, - bool degenerate_costs, double weight_spread, bool uniform_weights = false, int tiny_cost_range = -1) { @@ -547,12 +474,8 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge for (size_t i = 0; i < ua; ++i) { ids_a[i] = static_cast(i); for (size_t j = 0; j < ub; ++j) { - double c; - if (tiny_cost_range >= 0) { - c = static_cast(tiny_cost_dist(rng)); // few distinct costs - } else { - c = degenerate_costs ? 1.0 : cost_dist(rng); // many-equal-cost case - } + double c = tiny_cost_range >= 0 ? static_cast(tiny_cost_dist(rng)) + : cost_dist(rng); cost[i * ub + j] = c; table_flat[i * k + j] = static_cast(c); } @@ -587,158 +510,165 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge }; CATCH_SECTION("Random problems") { - for (int trial = 0; trial < 250; ++trial) { - size_t ua = size_dist(rng); - size_t ub = size_dist(rng); - run_case(ua, ub, false, 1.0); - } - } - CATCH_SECTION("Many equal costs") { - for (int trial = 0; trial < 20; ++trial) { - run_case(size_dist(rng), size_dist(rng), true, 1.0); + for (int trial = 0; trial < 64; ++trial) { + run_case(size_dist(rng), size_dist(rng), 1.0); } } CATCH_SECTION("Single node on one side") { - for (int trial = 0; trial < 10; ++trial) { - run_case(1, size_dist(rng), false, 1.0); - run_case(size_dist(rng), 1, false, 1.0); + for (int trial = 0; trial < 5; ++trial) { + run_case(1, size_dist(rng), 1.0); + run_case(size_dist(rng), 1, 1.0); } } CATCH_SECTION("Large weight dynamic range") { - for (int trial = 0; trial < 20; ++trial) { - run_case(size_dist(rng), size_dist(rng), false, 1e6); + for (int trial = 0; trial < 10; ++trial) { + run_case(size_dist(rng), size_dist(rng), 1e6); } } - CATCH_SECTION("Highly degenerate: uniform weights, tiny integer cost range") { + // Merges the "many equal costs" and "uniform-weight tiny-integer-cost" families: both + // are highly degenerate (many tied reduced costs), just via a different knob. + CATCH_SECTION("Degenerate: equal costs and tiny integer cost ranges") { std::uniform_int_distribution big_size_dist(1, 64); - for (int trial = 0; trial < 30; ++trial) { - run_case( - big_size_dist(rng), - big_size_dist(rng), - false, - 1.0, - /*uniform_weights=*/true, - /*tiny_cost_range=*/2 - ); + struct Config { + bool uniform_weights; + int tiny_cost_range; + }; + for (Config cfg : {Config{false, 0}, Config{true, 2}}) { + for (int trial = 0; trial < 15; ++trial) { + run_case( + big_size_dist(rng), + big_size_dist(rng), + 1.0, + cfg.uniform_weights, + cfg.tiny_cost_range + ); + } } } } CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { - // Raw (non-unit) centroids: c0=(1,0), c1=(0,1), c2=(-1,0). - std::vector flat{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F}; - auto centroids = std::make_shared(3, 2, flat.data()); - - // Query tokens t0=(1,0), t1=(0,1); doc uses centroid ids {0,2}. - RawQuerySet query_set{{1.0F, 0.0F, 0.0F, 1.0F}, 2}; - auto doc_set = make_id_set({0, 2}); - - CATCH_SECTION("Hand-computed: naive reference over raw query tokens") { - QChamferExactQuery dist{centroids}; - svs::distance::maybe_fix_argument(dist, query_set.view()); - float score = dist.compute(query_set.view(), doc_set.view()); - - // Naive reference built directly from the raw query tokens and centroid vectors, - // independent of build_table/compute_decoded. - std::vector> tokens{{1.0F, 0.0F}, {0.0F, 1.0F}}; - std::vector doc_ids{0, 2}; - double total = 0.0; - for (auto [tx, ty] : tokens) { - float best = -std::numeric_limits::infinity(); - for (auto id : doc_ids) { - float cx = flat[id * 2]; - float cy = flat[id * 2 + 1]; - best = std::max(best, cx * tx + cy * ty); - } - total += 1.0 - static_cast(best); - } - float expected = static_cast(total / tokens.size()); - - CATCH_REQUIRE(score == Catch::Approx(expected)); - CATCH_REQUIRE(score == Catch::Approx(0.5F)); - } - - CATCH_SECTION("Generic maybe_fix_argument/compute dispatch to the vector-set overload" - ) { - QChamferExactQuery dist{centroids}; - svs::distance::maybe_fix_argument(dist, query_set.view()); - float direct = dist.compute(query_set.view(), doc_set.view()); - float via_generic = svs::distance::compute(dist, query_set.view(), doc_set.view()); - CATCH_REQUIRE(via_generic == Catch::Approx(direct)); - } + std::vector flat2x2{1.0F, 0.0F, 0.0F, 1.0F}; + QChamferExactQuery dist{std::make_shared(2, 2, flat2x2.data())}; CATCH_SECTION("fix_argument: query dimension mismatch throws") { - QChamferExactQuery dist{centroids}; RawQuerySet wrong_dim{{1.0F, 0.0F, 0.0F}, 3}; CATCH_REQUIRE_THROWS_AS( svs::distance::maybe_fix_argument(dist, wrong_dim.view()), std::invalid_argument ); } - CATCH_SECTION("compute: doc dimensions() != 1 throws") { - QChamferExactQuery dist{centroids}; + // Uniform(-1,1) draws are, with overwhelming probability, not exactly representable + // in bfloat16, so this oracle exercises the rounding path without a hardcoded value. + CATCH_SECTION("Random oracle: naive reference over non-bf16-representable values") { + std::mt19937 rng(0xB16u); + std::uniform_real_distribution val_dist(-1.0F, 1.0F); + std::uniform_int_distribution id_dist(0, 3); + size_t k = 4, dim = 3; + for (int trial = 0; trial < 15; ++trial) { + std::vector flat(k * dim); + for (auto& x : flat) { + x = val_dist(rng); + } + auto centroids = std::make_shared(k, dim, flat.data()); + QChamferExactQuery trial_dist{centroids}; + + size_t n_query = 1 + (trial % 3); + std::vector tokens(n_query * dim); + for (auto& x : tokens) { + x = val_dist(rng); + } + RawQuerySet query_set{tokens, dim}; + svs::distance::maybe_fix_argument(trial_dist, query_set.view()); + + std::vector doc_ids(2 + (trial % 2)); + for (auto& id : doc_ids) { + id = id_dist(rng); + } + float score = trial_dist.compute(query_set.view(), make_id_set(doc_ids).view()); + + // Independent of fix_argument_decoded and the kernel: rounds both sides + // itself, then takes the max dot per query token and averages 1 - max. + double total = 0.0; + for (size_t i = 0; i < n_query; ++i) { + float best = -std::numeric_limits::infinity(); + for (auto id : doc_ids) { + double dot = 0.0; + for (size_t d = 0; d < dim; ++d) { + double cv = svs::round_to_bfloat16(flat[id * dim + d]); + double qv = svs::round_to_bfloat16(tokens[i * dim + d]); + dot += cv * qv; + } + best = std::max(best, static_cast(dot)); + } + total += 1.0 - static_cast(best); + } + float expected = static_cast(total / static_cast(n_query)); + CATCH_REQUIRE(score == Catch::Approx(expected).margin(1e-4)); + } + } +} + +// Each distance's compute() calls the shared gem_detail::require_id_dimensions from a +// different call site; one section per site, same underlying check. +CATCH_TEST_CASE("id vector set dimensions() != 1 throws", "[distance][gem]") { + std::vector raw{0, 1, 2, 3}; + auto bad = svs::data::ConstVectorSet(std::span(raw), 2, 2); + auto good = make_id_set({0}); + + CATCH_SECTION("QEmd") { + QEmd dist{make_table(Ground::Cos)}; + CATCH_REQUIRE_THROWS_AS(dist.compute(bad, good.view()), std::invalid_argument); + CATCH_REQUIRE_THROWS_AS(dist.compute(good.view(), bad), std::invalid_argument); + } + CATCH_SECTION("QChamfer") { + QChamfer dist{make_table(Ground::Cos)}; + CATCH_REQUIRE_THROWS_AS(dist.compute(bad, good.view()), std::invalid_argument); + CATCH_REQUIRE_THROWS_AS(dist.compute(good.view(), bad), std::invalid_argument); + } + CATCH_SECTION("QChamferExactQuery (doc side)") { + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F}; + QChamferExactQuery dist{std::make_shared(2, 2, flat.data())}; + RawQuerySet query_set{{1.0F, 0.0F}, 2}; svs::distance::maybe_fix_argument(dist, query_set.view()); - std::vector raw{0, 1, 2, 3}; - auto bad_doc = - svs::data::ConstVectorSet(std::span(raw), 2, 2); - CATCH_REQUIRE_THROWS_AS( - dist.compute(query_set.view(), bad_doc), std::invalid_argument - ); + CATCH_REQUIRE_THROWS_AS(dist.compute(query_set.view(), bad), std::invalid_argument); } } CATCH_TEST_CASE( - "GEM bf16 rounding: non-representable centroid and query values", "[distance][gem]" + "Generic svs::distance::compute dispatches to each distance's compute() overload", + "[distance][gem]" ) { - // None of these are exactly representable in bfloat16: unlike every other fixture in - // this file (chosen to be bf16-exact so their hand-computed expectations hold - // unchanged), this case exists specifically to exercise rounding. - std::vector flat{0.1F, 0.2F, -0.3F, 0.4F, 1.0F / 3.0F, -2.0F / 3.0F}; - size_t k = 3; - size_t dim = 2; - - CATCH_SECTION("CentroidTable::from_centroids matches a bf16-rounded naive reference") { - auto reference = naive_ground_table(round_to_bf16_vec(flat), k, dim, Ground::Cos); - auto table = CentroidTable::from_centroids(flat.data(), k, dim, Ground::Cos, 1); - for (size_t i = 0; i < k; ++i) { - for (size_t j = 0; j < k; ++j) { - auto ui = static_cast(i); - auto uj = static_cast(j); - CATCH_REQUIRE( - static_cast(table(ui, uj)) == - Catch::Approx(reference[i * k + j]).margin(1e-4) - ); - } - } + CATCH_SECTION("QEmd") { + QEmd dist{make_table(Ground::Cos)}; + auto a = make_id_set({0}); + auto b = make_id_set({1, 2}); + CATCH_REQUIRE( + svs::distance::compute(dist, a.view(), b.view()) == + Catch::Approx(dist.compute(a.view(), b.view())) + ); } - CATCH_SECTION("QChamferExactQuery matches a naive reference over bf16-rounded values") { - auto centroids = std::make_shared(k, dim, flat.data()); - QChamferExactQuery dist{centroids}; - - std::vector query{0.15F, -0.25F}; - RawQuerySet query_set{query, dim}; - auto doc_set = make_id_set({0, 2}); + CATCH_SECTION("QChamfer") { + QChamfer dist{make_table(Ground::Cos)}; + auto q = make_id_set({0, 1}); + auto d = make_id_set({2, 3}); + CATCH_REQUIRE( + svs::distance::compute(dist, q.view(), d.view()) == + Catch::Approx(dist.compute(q.view(), d.view())) + ); + } + CATCH_SECTION("QChamferExactQuery") { + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F}; + QChamferExactQuery dist{std::make_shared(2, 2, flat.data())}; + RawQuerySet query_set{{1.0F, 0.0F}, 2}; + auto doc_set = make_id_set({0, 1}); svs::distance::maybe_fix_argument(dist, query_set.view()); - float score = dist.compute(query_set.view(), doc_set.view()); - - // Naive reference over the same bf16-rounded centroid and query values (round to - // nearest, ties to even on both sides), independent of fix_argument_decoded and - // the kernel. - auto rounded_centroids = round_to_bf16_vec(flat); - float qx = static_cast(svs::round_to_bfloat16(query[0])); - float qy = static_cast(svs::round_to_bfloat16(query[1])); - std::vector doc_ids{0, 2}; - float best = -std::numeric_limits::infinity(); - for (auto id : doc_ids) { - float cx = rounded_centroids[id * dim]; - float cy = rounded_centroids[id * dim + 1]; - best = std::max(best, cx * qx + cy * qy); - } - float expected = 1.0F - best; // One query token: the "mean" is just this term. - - CATCH_REQUIRE(score == Catch::Approx(expected)); + CATCH_REQUIRE( + svs::distance::compute(dist, query_set.view(), doc_set.view()) == + Catch::Approx(dist.compute(query_set.view(), doc_set.view())) + ); } } @@ -773,11 +703,9 @@ CATCH_TEST_CASE("GEM Vamana extensions", "[distance][gem][vamana]") { svs::index::vamana::ProgressivePruneStrategy>); } - // A default-constructed Distance member would not compile for QEmd/QChamfer (their - // only constructor takes the CentroidTable), so these overloads existing and - // compiling already proves they do not fall back to DefaultBuildAdaptor's generic - // default-construct path. The compute() check below additionally proves the adaptor's - // distance_ is wired to the real table passed in, not some other instance. + // Compiling proves these skip DefaultBuildAdaptor's default-construct path (QEmd/ + // QChamfer have no default ctor); compute() below proves distance_ holds the real + // table. CATCH_SECTION("build_adaptor carries the real CentroidTable through") { auto table = make_table(Ground::Cos); QEmd qemd{table}; From a46ee1f5818c5b009cbc0e9fb13029995b4262bd Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 05:01:11 -0800 Subject: [PATCH 50/68] refactor(distance): split out the reference transportation simplex Moves today's big-M network simplex, unchanged in arithmetic, into TransportSimplexReference: scalar pricing only, no Avx parameter. It becomes the AVX_AVAILABILITY::NONE path and the test oracle for the optimized solver that follows. Its helpers live in transport_reference_detail so the two cannot clash. Co-Authored-By: Claude Opus 5.5 --- .../distance/transport_simplex_reference.h | 504 ++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 include/svs/core/distance/transport_simplex_reference.h diff --git a/include/svs/core/distance/transport_simplex_reference.h b/include/svs/core/distance/transport_simplex_reference.h new file mode 100644 index 000000000..b135b2d0b --- /dev/null +++ b/include/svs/core/distance/transport_simplex_reference.h @@ -0,0 +1,504 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Primal network simplex for the dense balanced transportation problem, scalar and +// unconditionally built: the AVX_AVAILABILITY::NONE path and the test oracle that +// TransportSimplex's SIMD-pricing variants are checked against. Algorithm: Ahuja, +// Magnanti & Orlin, "Network Flows" (1993), ch. 11 (artificial-root initialization and +// strongly feasible spanning trees); Cunningham, "A network simplex method", +// Mathematical Programming 11 (1976), for the leaving-arc rule that keeps a tree +// strongly feasible under degenerate pivots. Problem shape (quantized ground distance +// over merged centroid weights) follows Bonneel et al., "Displacement Interpolation +// Using Lagrangian Mass Transport", SIGGRAPH Asia 2011. + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs::distance::transport_reference_detail { + +// Finds the most negative reduced cost in a cost row and its column, for the block-search +// entering rule in TransportSimplexReference::solve. +inline void price_row( + const double* cost_row, + const double* v, + size_t ub, + double u_i, + double& best_val, + size_t& best_col +) { + for (size_t j = 0; j < ub; ++j) { + double r = cost_row[j] - u_i - v[j]; + if (r < best_val) { + best_val = r; + best_col = j; + } + } +} + +// Scratch for one thread's in-flight solve, reused across calls and grow-only: a bigger +// (ua, ub) resizes these up, a smaller one just uses a prefix, so steady-state solves +// (same CentroidTable, same decoded token counts) allocate nothing. Sized for ua+ub+1 +// nodes: sources [0, ua), sinks [ua, ua+ub), and one artificial root node at ua+ub. +struct Workspace { + std::vector cost; // ua * ub, row-major + std::vector supply; + std::vector demand; + std::vector parent; // size ua+ub+1; -1 at the root, -2 = unvisited (scratch) + std::vector depth; // size ua+ub+1 + std::vector flow_to_parent; // size ua+ub+1; flow on the arc to parent[v] + std::vector u; // source potentials, size ua + std::vector v; // sink potentials, size ub + std::vector adj_head; // CSR adjacency over ua+ub+1 nodes, rebuilt every pivot + std::vector adj_next; + std::vector adj_to; + std::vector adj_cost; // the physical arc's cost (M for an artificial arc) + std::vector adj_flow; + std::vector bfs_order; + std::vector path_p; + std::vector path_q; + + void reserve(size_t ua, size_t ub) { + size_t n = ua + ub + 1; + if (cost.size() < ua * ub) { + cost.resize(ua * ub); + } + if (supply.size() < ua) { + supply.resize(ua); + } + if (demand.size() < ub) { + demand.resize(ub); + } + if (parent.size() < n) { + parent.resize(n); + } + if (depth.size() < n) { + depth.resize(n); + } + if (flow_to_parent.size() < n) { + flow_to_parent.resize(n); + } + if (u.size() < ua) { + u.resize(ua); + } + if (v.size() < ub) { + v.resize(ub); + } + if (adj_head.size() < n) { + adj_head.resize(n); + } + size_t edges = (n > 0) ? n - 1 : 0; + if (adj_next.size() < 2 * edges) { + adj_next.resize(2 * edges); + adj_to.resize(2 * edges); + adj_cost.resize(2 * edges); + adj_flow.resize(2 * edges); + } + if (bfs_order.size() < n) { + bfs_order.resize(n); + } + if (path_p.size() < n) { + path_p.resize(n); + } + if (path_q.size() < n) { + path_q.resize(n); + } + } +}; + +// One directed adjacency slot: node `to`, reached from `from` over an arc with physical +// cost `cost` (the real cost(i, j) for a transportation arc, or M for an artificial one) +// and current flow `flow`. +inline void add_adj( + Workspace& ws, size_t edge_slot, int64_t from, int64_t to, double cost, double flow +) { + ws.adj_to[edge_slot] = to; + ws.adj_cost[edge_slot] = cost; + ws.adj_flow[edge_slot] = flow; + ws.adj_next[edge_slot] = ws.adj_head[static_cast(from)]; + ws.adj_head[static_cast(from)] = edge_slot; +} + +// Builds the artificial-root star: one extra node R = ua+ub, an arc (source i, R) with +// flow supply[i] for every source and (R, sink j) with flow demand[j] for every sink, all +// at cost M. Every arc starts at strictly positive flow (ws.supply/ws.demand are qEMD's +// merged weights, never zero), so this tree is strongly feasible by construction: the +// strongly-feasible condition only constrains zero-flow tree arcs, and there are none +// (Ahuja-Magnanti-Orlin ch. 11.2). Northwest-corner is not used here because it is not +// automatically strongly feasible. +inline void artificial_root_adjacency(Workspace& ws, size_t ua, size_t ub, double big_m) { + size_t root = ua + ub; + std::fill( + ws.adj_head.begin(), + ws.adj_head.begin() + static_cast(root + 1), + SIZE_MAX + ); + size_t slot = 0; + for (size_t i = 0; i < ua; ++i) { + add_adj( + ws, + slot, + static_cast(i), + static_cast(root), + big_m, + ws.supply[i] + ); + add_adj( + ws, + slot + 1, + static_cast(root), + static_cast(i), + big_m, + ws.supply[i] + ); + slot += 2; + } + for (size_t j = 0; j < ub; ++j) { + add_adj( + ws, + slot, + static_cast(root), + static_cast(ua + j), + big_m, + ws.demand[j] + ); + add_adj( + ws, + slot + 1, + static_cast(ua + j), + static_cast(root), + big_m, + ws.demand[j] + ); + slot += 2; + } +} + +// Breadth-first search from the artificial root R = ua+ub (always the root: R is +// permanently part of the spanning tree, see solve()'s doc comment) over the adjacency +// already built in ws.adj_*, filling parent/depth/flow_to_parent and the dual potentials +// u/v. A source is always the tail of its arc to its tree parent, whether that arc is a +// real one (to a sink) or artificial (to R); a sink is always the head. That fixes the +// sign in the potential update below without per-arc direction bookkeeping. Every tree +// arc has zero reduced cost by construction, which is what lets solve()'s pricing scan +// treat "negative reduced cost" as "not yet basic" with no separate basis-membership +// check. +inline void rebuild_tree(Workspace& ws, size_t ua, size_t ub) { + size_t root = ua + ub; + size_t n = root + 1; + std::fill(ws.parent.begin(), ws.parent.begin() + static_cast(n), -2); + ws.parent[root] = -1; + ws.depth[root] = 0; + ws.bfs_order[0] = static_cast(root); + size_t order_end = 1; + for (size_t qi = 0; qi < order_end; ++qi) { + int64_t node = ws.bfs_order[qi]; + double node_y = + (node == static_cast(root)) + ? 0.0 + : (node < static_cast(ua) ? ws.u[static_cast(node)] + : -ws.v[static_cast(node) - ua]); + for (size_t e = ws.adj_head[static_cast(node)]; e != SIZE_MAX; + e = ws.adj_next[e]) { + int64_t to = ws.adj_to[e]; + if (ws.parent[static_cast(to)] != -2) { + continue; + } + ws.parent[static_cast(to)] = node; + ws.depth[static_cast(to)] = ws.depth[static_cast(node)] + 1; + ws.flow_to_parent[static_cast(to)] = ws.adj_flow[e]; + double cost = ws.adj_cost[e]; + if (to < static_cast(ua)) { + ws.u[static_cast(to)] = + node_y + cost; // source: tail, y = y(parent) + cost + } else { + ws.v[static_cast(to) - ua] = cost - node_y; // sink: head + } + ws.bfs_order[order_end++] = to; + } + } +} + +} // namespace svs::distance::transport_reference_detail + +namespace svs::distance { + +/// Reference implementation of the dense balanced transportation problem solver: `ua` +/// sources with supply `supply[i]`, `ub` sinks with demand `demand[j]`, a complete +/// bipartite arc set, and a per-arc cost from `cost(i, j)`. Minimizes +/// `sum_ij flow_ij * cost(i, j)` subject to `sum_j flow_ij == supply[i]`, +/// `sum_i flow_ij == demand[j]`, `flow_ij >= 0`. +/// +/// Requires `ua >= 1`, `ub >= 1`, every supply/demand entry strictly positive (true for +/// qEMD's merged weights), and `sum(supply) == sum(demand)` to within `1e-6 * +/// max(sum(supply), 1.0)`; any smaller residual (the kind two independent floating-point +/// sums that are mathematically equal can differ by) is folded into the last demand entry +/// so the solver never over- or under-subscribes a side. A larger mismatch throws +/// std::runtime_error, since it signals mismatched inputs rather than rounding. +/// +/// Initializes with an artificial root node and big-M artificial arcs to every source and +/// sink (Ahuja-Magnanti-Orlin ch. 11.2), which is strongly feasible from the start since +/// every artificial arc carries strictly positive flow. Each pivot then picks the leaving +/// arc as the *last* blocking (flow-decreasing) arc encountered while walking the cycle +/// from its apex (the two endpoints' lowest common tree ancestor) in the entering arc's +/// own direction (Cunningham, 1976); maintaining a strongly feasible tree this way +/// guarantees termination regardless of which negative-reduced-cost arc enters next, so +/// the entering rule is free to optimize for speed: a block-cyclic search over the dense +/// cost matrix, picking the most negative reduced cost in the first block (of +/// `~sqrt(ua*ub)` arcs, rounded to whole rows) that has one, resuming from where the +/// previous search left off. M is `(ua+ub) * max(cost) + 1`, the largest an optimal basic +/// solution's total real cost could be plus 1, so no finite combination of real arcs is +/// ever more expensive than keeping one unit of flow on an artificial arc. +/// +/// This type runs the scalar pricing loop unconditionally: it is the AVX_AVAILABILITY:: +/// NONE path, and it is also the test oracle that TransportSimplex's SIMD-pricing +/// variants are checked against, so its algorithm must not diverge from theirs. +struct TransportSimplexReference { + template + static double + solve(size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost) { + using namespace transport_reference_detail; + thread_local Workspace ws; + ws.reserve(ua, ub); + size_t root = ua + ub; + + double total_s = 0.0, total_d = 0.0; + for (size_t i = 0; i < ua; ++i) { + ws.supply[i] = supply[i]; + total_s += supply[i]; + } + for (size_t j = 0; j < ub; ++j) { + ws.demand[j] = demand[j]; + total_d += demand[j]; + } + double tol = 1e-6 * std::max(total_s, 1.0); + double diff = total_s - total_d; + if (diff > tol || diff < -tol) { + throw std::runtime_error( + "TransportSimplexReference::solve: supply (" + std::to_string(total_s) + + ") and demand (" + std::to_string(total_d) + ") are not balanced for ua=" + + std::to_string(ua) + ", ub=" + std::to_string(ub) + ); + } + ws.demand[ub - 1] += diff; + + double max_cost = 0.0; + for (size_t i = 0; i < ua; ++i) { + double* row = &ws.cost[i * ub]; + for (size_t j = 0; j < ub; ++j) { + double c = static_cast(cost(i, j)); + row[j] = c; + max_cost = std::max(max_cost, c); + } + } + // Bigger than any optimal basic solution's real cost could ever be, so the + // simplex never prefers routing flow through the artificial root once a real + // spanning tree is available; see the class doc comment. + double big_m = static_cast(ua + ub) * max_cost + 1.0; + + artificial_root_adjacency(ws, ua, ub, big_m); + rebuild_tree(ws, ua, ub); + + // Block size: ~sqrt(ua*ub) arcs, rounded to a whole number of rows so each block + // scan stays an integer number of rows. + size_t block_rows = static_cast(std::round( + std::sqrt(static_cast(ua) * static_cast(ub)) / + static_cast(ub) + )); + block_rows = std::clamp(block_rows, 1, ua); + size_t cursor_row = 0; + + // No polynomial bound exists for primal network simplex iteration count (even + // with a strongly feasible tree, only finiteness is guaranteed, not a bound), so + // this is a generous safety net against a latent bug, not an expected limit. + size_t max_iters = 1000 * (ua + ub) + 10000; + for (size_t iter = 0; iter < max_iters; ++iter) { + size_t best_i = ua, best_j = 0, rows_scanned = 0; + size_t row = cursor_row; + while (rows_scanned < ua) { + double block_best_val = 0.0; + size_t block_best_i = ua, block_best_j = 0; + for (size_t b = 0; b < block_rows && rows_scanned < ua; + ++b, ++rows_scanned) { + double val = 0.0; + size_t col = 0; + price_row(&ws.cost[row * ub], ws.v.data(), ub, ws.u[row], val, col); + if (val < block_best_val) { + block_best_val = val; + block_best_i = row; + block_best_j = col; + } + row = (row + 1) % ua; + } + if (block_best_i != ua) { + best_i = block_best_i; + best_j = block_best_j; + cursor_row = row; + break; + } + } + if (best_i == ua) { + double total_cost = 0.0; + for (size_t node = 0; node < root; ++node) { + if (static_cast(ws.parent[node]) == root) { + continue; // artificial arc: not part of the real transport cost + } + size_t ai, aj; + if (node < ua) { + ai = node; + aj = static_cast(ws.parent[node]) - ua; + } else { + ai = static_cast(ws.parent[node]); + aj = node - ua; + } + total_cost += ws.flow_to_parent[node] * ws.cost[ai * ub + aj]; + } + return total_cost; + } + + int64_t p_node = static_cast(best_i); + int64_t q_node = static_cast(ua + best_j); + ws.path_p.clear(); + ws.path_p.push_back(p_node); + ws.path_q.clear(); + ws.path_q.push_back(q_node); + int64_t pu = p_node, qu = q_node; + while (ws.depth[static_cast(pu)] > ws.depth[static_cast(qu)]) { + pu = ws.parent[static_cast(pu)]; + ws.path_p.push_back(pu); + } + while (ws.depth[static_cast(qu)] > ws.depth[static_cast(pu)]) { + qu = ws.parent[static_cast(qu)]; + ws.path_q.push_back(qu); + } + while (pu != qu) { + pu = ws.parent[static_cast(pu)]; + ws.path_p.push_back(pu); + qu = ws.parent[static_cast(qu)]; + ws.path_q.push_back(qu); + } + size_t lp = ws.path_p.size(); + size_t lq = ws.path_q.size(); + + // A source is always the tail of its tree arc (to a sink or to the root), a + // sink always the head (see rebuild_tree's doc comment), so walking + // parent->child is "forward" (flow increases) exactly when the child is a + // sink, and walking child->parent is forward exactly when the child is a + // source. + auto is_forward_down = [&](int64_t child) { + return child >= static_cast(ua); + }; + auto is_forward_up = [&](int64_t child) { + return child < static_cast(ua); + }; + + // Leaving arc: walk the cycle apex (= LCA = the end of either path array) + // down to p, then from q back up to the apex -- the entering arc's own + // orientation -- and take the *last* blocking (backward) arc seen, using <= + // so a later tie overwrites an earlier one (Cunningham's rule; this is what + // keeps the tree strongly feasible, not the choice of entering arc). + double theta = std::numeric_limits::infinity(); + int64_t leave_node = -1; + for (size_t t = lp; t-- > 1;) { + int64_t child = ws.path_p[t - 1]; + if (!is_forward_down(child)) { + double flow = ws.flow_to_parent[static_cast(child)]; + if (flow <= theta) { + theta = flow; + leave_node = child; + } + } + } + for (size_t t = 0; t + 1 < lq; ++t) { + int64_t child = ws.path_q[t]; + if (!is_forward_up(child)) { + double flow = ws.flow_to_parent[static_cast(child)]; + if (flow <= theta) { + theta = flow; + leave_node = child; + } + } + } + + // Apply the flow delta to every tree edge on the cycle (the entering arc is + // not one of these; it is added fresh below with flow == theta). Order does + // not matter here, unlike the leaving-arc search above. + for (size_t t = 0; t + 1 < lp; ++t) { + int64_t child = ws.path_p[t]; + double delta = is_forward_down(child) ? theta : -theta; + ws.flow_to_parent[static_cast(child)] += delta; + } + for (size_t t = 0; t + 1 < lq; ++t) { + int64_t child = ws.path_q[t]; + double delta = is_forward_up(child) ? theta : -theta; + ws.flow_to_parent[static_cast(child)] += delta; + } + + // Rebuild adjacency from the old tree minus the leaving edge, plus the + // entering arc (p_node, q_node, theta); rebuilding by BFS instead of + // patching parent/depth in place avoids re-rooting a subtree incrementally, + // which costs O(ua+ub) here against O(ua*ub) for pricing, so it is not the + // bottleneck. + std::fill( + ws.adj_head.begin(), + ws.adj_head.begin() + static_cast(root + 1), + SIZE_MAX + ); + size_t slot = 0; + for (int64_t node = 0; node < static_cast(root); ++node) { + if (node == leave_node) { + continue; + } + int64_t parent_node = ws.parent[static_cast(node)]; + double edge_cost; + if (static_cast(parent_node) == root) { + edge_cost = big_m; + } else if (node < static_cast(ua)) { + edge_cost = ws.cost + [static_cast(node) * ub + + (static_cast(parent_node) - ua)]; + } else { + edge_cost = ws.cost + [static_cast(parent_node) * ub + + (static_cast(node) - ua)]; + } + double flow = ws.flow_to_parent[static_cast(node)]; + add_adj(ws, slot, node, parent_node, edge_cost, flow); + add_adj(ws, slot + 1, parent_node, node, edge_cost, flow); + slot += 2; + } + double entering_cost = ws.cost[best_i * ub + best_j]; + add_adj(ws, slot, p_node, q_node, entering_cost, theta); + add_adj(ws, slot + 1, q_node, p_node, entering_cost, theta); + + rebuild_tree(ws, ua, ub); + } + throw std::runtime_error( + "TransportSimplexReference::solve: exceeded the iteration limit (" + + std::to_string(max_iters) + ") for ua=" + std::to_string(ua) + + ", ub=" + std::to_string(ub) + ); + } +}; + +} // namespace svs::distance From 848d21e69b8e08da8af16b1183fa56bba95f8da7 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 05:01:11 -0800 Subject: [PATCH 51/68] fix(distance): reject a non-positive last demand after the balance fold The balance residual (up to 1e-6 * total) is folded into the last demand entry. When that entry is smaller than the residual, the fold left it at zero or below, which the simplex cannot solve. That input now throws, naming ua, ub, the entry and the residual. Valid inputs are unaffected. The copy, balance check and fold move into prepare_marginals, which the optimized solver shares. It takes a caller tag type so each -march translation unit instantiates its own copy rather than sharing one weak symbol. Co-Authored-By: Claude Opus 5.5 --- .../distance/transport_simplex_reference.h | 81 +++++++++++++------ 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/include/svs/core/distance/transport_simplex_reference.h b/include/svs/core/distance/transport_simplex_reference.h index b135b2d0b..088696250 100644 --- a/include/svs/core/distance/transport_simplex_reference.h +++ b/include/svs/core/distance/transport_simplex_reference.h @@ -37,6 +37,54 @@ namespace svs::distance::transport_reference_detail { +// Copies supply/demand into the solver's own buffers, checks total balance to within +// 1e-6*max(total_supply, 1), and folds a smaller residual into the last demand entry; +// throws std::runtime_error on a larger mismatch or a fold that leaves that entry +// non-positive. Shared by TransportSimplexReference and TransportSimplex (called before +// either orients rows/columns, so the fold always lands on the caller's own last demand +// entry, never its supply) so the input contract cannot drift between them. `Caller` +// (TransportSimplexReference, or TransportSimplex::Impl) is a tag only, so +// each -march TU instantiates its own copy instead of sharing one weak symbol. +template +inline double prepare_marginals( + size_t ua, + const double* supply, + double* supply_out, + size_t ub, + const double* demand, + double* demand_out +) { + double total_s = 0.0, total_d = 0.0; + for (size_t i = 0; i < ua; ++i) { + supply_out[i] = supply[i]; + total_s += supply[i]; + } + for (size_t j = 0; j < ub; ++j) { + demand_out[j] = demand[j]; + total_d += demand[j]; + } + double tol = 1e-6 * std::max(total_s, 1.0); + double diff = total_s - total_d; + if (diff > tol || diff < -tol) { + throw std::runtime_error( + "TransportSimplex::solve: supply (" + std::to_string(total_s) + + ") and demand (" + std::to_string(total_d) + + ") are not balanced for ua=" + std::to_string(ua) + ", ub=" + std::to_string(ub) + ); + } + double last_demand = demand_out[ub - 1]; + demand_out[ub - 1] += diff; + if (demand_out[ub - 1] <= 0.0) { + throw std::runtime_error( + "TransportSimplex::solve: the last demand entry (" + + std::to_string(last_demand) + + ") is too small to absorb the balance residual (" + std::to_string(diff) + + ") for ua=" + std::to_string(ua) + ", ub=" + std::to_string(ub) + ); + } + return total_s; +} + // Finds the most negative reduced cost in a cost row and its column, for the block-search // entering rule in TransportSimplexReference::solve. inline void price_row( @@ -253,8 +301,9 @@ namespace svs::distance { /// qEMD's merged weights), and `sum(supply) == sum(demand)` to within `1e-6 * /// max(sum(supply), 1.0)`; any smaller residual (the kind two independent floating-point /// sums that are mathematically equal can differ by) is folded into the last demand entry -/// so the solver never over- or under-subscribes a side. A larger mismatch throws -/// std::runtime_error, since it signals mismatched inputs rather than rounding. +/// so the solver never over- or under-subscribes a side. A larger mismatch, or a residual +/// that leaves the last demand entry non-positive, throws std::runtime_error, since both +/// signal mismatched inputs rather than rounding. /// /// Initializes with an artificial root node and big-M artificial arcs to every source and /// sink (Ahuja-Magnanti-Orlin ch. 11.2), which is strongly feasible from the start since @@ -270,9 +319,9 @@ namespace svs::distance { /// solution's total real cost could be plus 1, so no finite combination of real arcs is /// ever more expensive than keeping one unit of flow on an artificial arc. /// -/// This type runs the scalar pricing loop unconditionally: it is the AVX_AVAILABILITY:: -/// NONE path, and it is also the test oracle that TransportSimplex's SIMD-pricing -/// variants are checked against, so its algorithm must not diverge from theirs. +/// This type runs the scalar pricing loop unconditionally: it is the `AVX_AVAILABILITY:: +/// NONE` path, and it is the oracle TransportSimplex's results (not its algorithm) are +/// checked against. struct TransportSimplexReference { template static double @@ -282,25 +331,9 @@ struct TransportSimplexReference { ws.reserve(ua, ub); size_t root = ua + ub; - double total_s = 0.0, total_d = 0.0; - for (size_t i = 0; i < ua; ++i) { - ws.supply[i] = supply[i]; - total_s += supply[i]; - } - for (size_t j = 0; j < ub; ++j) { - ws.demand[j] = demand[j]; - total_d += demand[j]; - } - double tol = 1e-6 * std::max(total_s, 1.0); - double diff = total_s - total_d; - if (diff > tol || diff < -tol) { - throw std::runtime_error( - "TransportSimplexReference::solve: supply (" + std::to_string(total_s) + - ") and demand (" + std::to_string(total_d) + ") are not balanced for ua=" + - std::to_string(ua) + ", ub=" + std::to_string(ub) - ); - } - ws.demand[ub - 1] += diff; + prepare_marginals( + ua, supply, ws.supply.data(), ub, demand, ws.demand.data() + ); double max_cost = 0.0; for (size_t i = 0; i < ua; ++i) { From 27c51a9095303ebf68a5ba25585830deaa2c8054 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 05:01:12 -0800 Subject: [PATCH 52/68] perf(distance): add float row-pricing kernels for the transportation simplex RowMin::min_in_row finds the most negative reduced cost in one cost row, over float costs with either float or double potentials. Generic, AVX2 and AVX512 bodies are bit-identical: two separate subtractions (no FMA), ascending scalar lane scan, first minimum wins. Rows are padded to 16 lanes with +inf. Co-Authored-By: Claude Opus 5.5 --- .../core/distance/transport_simplex_pricing.h | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 include/svs/core/distance/transport_simplex_pricing.h diff --git a/include/svs/core/distance/transport_simplex_pricing.h b/include/svs/core/distance/transport_simplex_pricing.h new file mode 100644 index 000000000..60c43b008 --- /dev/null +++ b/include/svs/core/distance/transport_simplex_pricing.h @@ -0,0 +1,210 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Float row-pricing kernels for TransportSimplex's float-proposes/double-confirms +// pricing split: a cost row stored as float, with a float or double reduced-cost scan. + +#include "svs/core/distance/distance_core.h" +#include "svs/lib/preprocessor.h" + +#include +#include + +#if defined(__x86_64__) +#include +#endif + +namespace svs::distance::transport_detail { + +// One AVX-512 float register, or two AVX2 float registers: a row padded to a multiple of +// this needs no scalar tail on either ISA. +inline constexpr size_t kPriceLanes = 16; + +inline size_t pad_to_lanes(size_t n) { + return (n + kPriceLanes - 1) / kPriceLanes * kPriceLanes; +} + +/// Finds the minimum reduced cost `r_j = (cost_row[j] - u_i) - v[j]` in a cost row and its +/// column. `cost_row` and `v` must both have `ub_padded` valid entries, a multiple of +/// `kPriceLanes`; padding columns (index >= the caller's real ub) must hold +infinity in +/// `cost_row` so they never win. `best_val`/`best_col` are in-out and updated only on a +/// strict improvement, scanning columns ascending, so the first minimum wins on a tie. +/// This primary definition is not dead on x86 despite the AVX2/AVX512 specializations +/// below: non-x86 builds still instantiate `RowMin`/`RowMin` through +/// `QEmdImpl`/`QEmdImpl`. +template struct RowMin { + static void min_in_row( + const float* cost_row, + const float* v, + size_t ub_padded, + float u_i, + float& best_val, + uint32_t& best_col + ) { + for (size_t j = 0; j < ub_padded; ++j) { + float r = (cost_row[j] - u_i) - v[j]; + if (r < best_val) { + best_val = r; + best_col = static_cast(j); + } + } + } + + static void min_in_row( + const float* cost_row, + const double* v, + size_t ub_padded, + double u_i, + double& best_val, + uint32_t& best_col + ) { + for (size_t j = 0; j < ub_padded; ++j) { + double r = (static_cast(cost_row[j]) - u_i) - v[j]; + if (r < best_val) { + best_val = r; + best_col = static_cast(j); + } + } + } +}; + +// Guarded per ISA: these bodies are not templates, so every TU that sees one compiles its +// intrinsics, and a TU built without that ISA fails loudly. +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +#if SVS_AVX512_F +template <> struct RowMin { + static void min_in_row( + const float* cost_row, + const float* v, + size_t ub_padded, + float u_i, + float& best_val, + uint32_t& best_col + ) { + __m512 u_vec = _mm512_set1_ps(u_i); + float buf[16]; + for (size_t j = 0; j < ub_padded; j += 16) { + // Two separate _mm512_sub_ps, not an FMA: the generic version rounds the two + // subtractions independently, and this must match it bit-for-bit. + __m512 r = _mm512_sub_ps( + _mm512_sub_ps(_mm512_loadu_ps(cost_row + j), u_vec), _mm512_loadu_ps(v + j) + ); + _mm512_storeu_ps(buf, r); + for (size_t lane = 0; lane < 16; ++lane) { + if (buf[lane] < best_val) { + best_val = buf[lane]; + best_col = static_cast(j + lane); + } + } + } + } + + static void min_in_row( + const float* cost_row, + const double* v, + size_t ub_padded, + double u_i, + double& best_val, + uint32_t& best_col + ) { + double buf[16]; + __m512d u_vec = _mm512_set1_pd(u_i); + for (size_t j = 0; j < ub_padded; j += 16) { + __m256 c_lo = _mm256_loadu_ps(cost_row + j); + __m256 c_hi = _mm256_loadu_ps(cost_row + j + 8); + __m512d r_lo = _mm512_sub_pd( + _mm512_sub_pd(_mm512_cvtps_pd(c_lo), u_vec), _mm512_loadu_pd(v + j) + ); + __m512d r_hi = _mm512_sub_pd( + _mm512_sub_pd(_mm512_cvtps_pd(c_hi), u_vec), _mm512_loadu_pd(v + j + 8) + ); + _mm512_storeu_pd(buf, r_lo); + _mm512_storeu_pd(buf + 8, r_hi); + for (size_t lane = 0; lane < 16; ++lane) { + if (buf[lane] < best_val) { + best_val = buf[lane]; + best_col = static_cast(j + lane); + } + } + } + } +}; +#endif + +SVS_VALIDATE_BOOL_ENV(SVS_AVX2) +#if SVS_AVX2 +template <> struct RowMin { + static void min_in_row( + const float* cost_row, + const float* v, + size_t ub_padded, + float u_i, + float& best_val, + uint32_t& best_col + ) { + __m256 u_vec = _mm256_set1_ps(u_i); + float buf[16]; + for (size_t j = 0; j < ub_padded; j += 16) { + __m256 r_lo = _mm256_sub_ps( + _mm256_sub_ps(_mm256_loadu_ps(cost_row + j), u_vec), _mm256_loadu_ps(v + j) + ); + __m256 r_hi = _mm256_sub_ps( + _mm256_sub_ps(_mm256_loadu_ps(cost_row + j + 8), u_vec), + _mm256_loadu_ps(v + j + 8) + ); + _mm256_storeu_ps(buf, r_lo); + _mm256_storeu_ps(buf + 8, r_hi); + for (size_t lane = 0; lane < 16; ++lane) { + if (buf[lane] < best_val) { + best_val = buf[lane]; + best_col = static_cast(j + lane); + } + } + } + } + + static void min_in_row( + const float* cost_row, + const double* v, + size_t ub_padded, + double u_i, + double& best_val, + uint32_t& best_col + ) { + __m256d u_vec = _mm256_set1_pd(u_i); + double buf[16]; + for (size_t j = 0; j < ub_padded; j += 16) { + for (size_t sub = 0; sub < 16; sub += 4) { + __m128 c = _mm_loadu_ps(cost_row + j + sub); + __m256d r = _mm256_sub_pd( + _mm256_sub_pd(_mm256_cvtps_pd(c), u_vec), _mm256_loadu_pd(v + j + sub) + ); + _mm256_storeu_pd(buf + sub, r); + } + for (size_t lane = 0; lane < 16; ++lane) { + if (buf[lane] < best_val) { + best_val = buf[lane]; + best_col = static_cast(j + lane); + } + } + } + } +}; +#endif + +} // namespace svs::distance::transport_detail From e4e4e7e7bc6b1bc65ccb5c7a0e4876442a475546 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 05:01:32 -0800 Subject: [PATCH 53/68] perf(distance): solve qEMD with a LEMON-derived network simplex Replaces the per-pivot BFS tree rebuild, which was about 80% of the old solve, with LEMON's incremental spanning-tree update (parent, thread, rev_thread, succ_num, last_succ), ported by way of POT's network_simplex_simple.h. On the sub50k neighbors workload, one thread: 246 us -> 86 us per solve (POT: 103 us). The checksum is unchanged. Structure: - The smaller side becomes the rows. The optimal cost is invariant. - Costs are stored as float. CostFn must return float, so this is exact. - The start is north-west corner, rooted at source 0, with no big-M. - Node ids are uint16_t when max(ua, ub) <= 256, else uint32_t. The solver picks the bucket itself. - Pricing is float (RowMin) on float mirrors of double potentials. - Flows and the final cost are double. - AVX_AVAILABILITY::NONE forwards to TransportSimplexReference. - Every helper is a member of TransportSimplex, so no function body becomes a weak symbol shared across -march translation units. Strong feasibility. LEMON's leaving rule (strict < on the i-side, <= on the j-side) keeps every zero-flow tree arc pointing toward the root, so its child is a source. The start meets this: - On a value tie at (i, j), the zero arc is (i+1, j), whose child is the fresh source i+1. - Every arc with a sink child carries a fresh, strictly positive amount. - The final cell uses whichever side is still untouched. No clamp. Pivot gate. Let n = ua+ub, P = (n-1)*max_cost (bounds |potential|), R = recompute_period, eps = DBL_EPSILON. - walk_tol = 2n*eps*(max_cost + P). This bounds |r_fresh - r_exact|: two root-path sums of at most n-1 steps each, plus the two subtractions that form r. Potentials walked from the root equal a fresh recompute bit for bit, and so does RowMin's double r, because the operation order is the same. - drift_tol = 2(n-1)*R*9*eps*P + walk_tol. One pivot moves a tree arc's residual by at most 9*eps*P: sigma costs 5*eps*P and applying it 4*eps*P. A node's error is the sum of residuals along its root path, so the depth factor is n-1, and there are two endpoints. - A candidate enters only if r_drifted < -drift_tol, or, when borderline, r_walked < -walk_tol. So every entering arc is truly negative, and Cunningham's finiteness proof applies. - Otherwise the solver recomputes fresh and prices once in double. Every stop happens on fresh potentials. error_bound. Each term covers a stated rounding source. - 2*walk_tol*total_mass: at the stop, r_exact >= -2*walk_tol, and complementary slackness turns that into a cost gap. - n^2*eps*max_cost*total_mass: before the final sum, the flows are recomputed from the marginals in post-order. Each flow is a signed subtree sum, and subtree sizes sum to at most n^2. - 2(n-1)*eps*max_cost*total_mass: the n-1 multiplies and adds of the final sum. The balance residual is folded into the caller's last demand before orienting, as in the reference. Portions derive from POT 0.9.5 (MIT) and LEMON (Boost Software License 1.0); see the header notices and THIRD-PARTY-PROGRAMS. Co-Authored-By: Claude Opus 5.5 --- include/svs/core/distance/gem.h | 9 +- include/svs/core/distance/transport_simplex.h | 1431 +++++++++++------ 2 files changed, 922 insertions(+), 518 deletions(-) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index 65ec6741c..759d8d5aa 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -394,12 +394,9 @@ template struct QEmdImpl { // merge_sorted_duplicates divides integer counts by n in double, so w_a/w_b each // sum to 1.0 up to a few ULPs rather than exactly; TransportSimplex::solve folds - // that residual into the last demand entry (see its doc comment) instead of - // requiring bit-exact balance here. - double cost = - TransportSimplex::solve(ua, w_a, ub, w_b, [&](size_t i, size_t j) { - return table(ids_a[i], ids_b[j]); - }); + // that residual into the last demand entry instead of requiring exact balance. + auto cost_fn = [&](size_t i, size_t j) { return table(ids_a[i], ids_b[j]); }; + double cost = TransportSimplex::solve(ua, w_a, ub, w_b, cost_fn); return static_cast(cost); } diff --git a/include/svs/core/distance/transport_simplex.h b/include/svs/core/distance/transport_simplex.h index a9d7ed756..ddbd0bcb5 100644 --- a/include/svs/core/distance/transport_simplex.h +++ b/include/svs/core/distance/transport_simplex.h @@ -16,582 +16,989 @@ #pragma once -// Clean-room primal network simplex for the dense balanced transportation problem. -// Algorithm: Ahuja, Magnanti & Orlin, "Network Flows" (1993), ch. 11 (artificial-root -// initialization and strongly feasible spanning trees); Cunningham, "A network simplex -// method", Mathematical Programming 11 (1976), for the leaving-arc rule that keeps a -// tree strongly feasible under degenerate pivots. Problem shape (quantized ground -// distance over merged centroid weights) follows Bonneel et al., "Displacement -// Interpolation Using Lagrangian Mass Transport", SIGGRAPH Asia 2011. +// Primal network simplex for the dense balanced transportation problem, specialized to a +// bipartite tree with no artificial root and no capacities. Northwest-corner +// initialization and the tree-array representation (parent/thread/rev_thread/succ_num/ +// last_succ) and incremental pivot update (findJoinNode/findLeavingArc/changeFlow/ +// updateTreeStructure/updatePotential) are ported from LEMON's network_simplex, by way of +// Nicolas Bonneel's single-file adaptation vendored into POT; see the notices below. The +// float-proposes/double-confirms pricing split and the strongly-feasible northwest-corner +// start (replacing LEMON/POT's artificial-root big-M start) are this file's own. Problem +// shape (quantized ground distance over merged centroid weights) follows Bonneel et al., +// "Displacement Interpolation Using Lagrangian Mass Transport", SIGGRAPH Asia 2011. +// +// Portions of this file are derived from: +// +// POT (Python Optimal Transport), `ot/lp/`, tag 0.9.5. +// https://github.com/PythonOT/POT +// Copyright (c) 2016-2023 POT contributors. +// Licensed under the MIT License. +// +// which itself adapted, as `network_simplex_simple.h` (Nicolas Bonneel, 2013), from +// LEMON's `network_simplex.h`: +// +// Copyright (C) 2003-2010 +// Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport +// (Egervary Research Group on Combinatorial Optimization, EGRES). +// +// Permission to use, modify and distribute this software is granted +// provided that this copyright notice appears in all copies. For +// precise terms see the accompanying LICENSE file. +// +// This software is provided "AS IS" with no warranty of any kind, +// express or implied, and with no claim as to its suitability for any +// purpose. #include "svs/core/distance/distance_core.h" -#include "svs/lib/preprocessor.h" +#include "svs/core/distance/transport_simplex_pricing.h" +#include "svs/core/distance/transport_simplex_reference.h" +#include "svs/lib/meta.h" #include #include +#include #include #include #include #include #include +#include #include -#if defined(__x86_64__) -#include -#endif - namespace svs::distance::transport_detail { -// Pricing scans a row in blocks of this many doubles: 8 covers one AVX-512 register or -// two AVX2 registers, so a row padded to a multiple of it needs no scalar tail on either -// ISA. -inline constexpr size_t kPriceBlock = 8; - -inline size_t pad_to_block(size_t n) { - return (n + kPriceBlock - 1) / kPriceBlock * kPriceBlock; -} - -// Finds the most negative reduced cost in a cost row and its column, for the block-search -// entering rule in TransportSimplex::solve. The vector subtract (the O(ub) part) is -// SIMD-widened per Avx; the min-with-index reduction per 8-or-4-wide chunk is left scalar -// (store the chunk, scan it) since it is only O(ub/width) work and a lot easier to get -// right by inspection than a vectorized horizontal-min-with-index. The generic template -// also serves AVX_AVAILABILITY::NONE and any non-x86 build. -template struct PriceRow { - // `v` and `cost_row` must both have `ub_padded` valid entries; padding columns (index - // >= the caller's real ub) must hold +infinity in cost_row so they never win the min - // regardless of what garbage sits in the matching v slot. - static void min_in_row( - const double* cost_row, - const double* v, - size_t ub_padded, - double u_i, - double& best_val, - size_t& best_col - ) { - for (size_t j = 0; j < ub_padded; ++j) { - double r = cost_row[j] - u_i - v[j]; - if (r < best_val) { - best_val = r; - best_col = j; - } - } - } +// uint16_t node ids halve the tree arrays' footprint for the common small-problem bucket; +// Dynamic has no such bound so ids must cover an arbitrary ua+ub. A pure type trait (no +// function bodies), so it carries none of the cross-TU weak-symbol risk the rest of this +// file's per-ISA machinery is keyed on TransportSimplex to avoid. +template struct TransportIndexTraits { + using node_type = uint32_t; + using arc_type = uint32_t; +}; +template <> struct TransportIndexTraits<256> { + using node_type = uint16_t; + using arc_type = uint32_t; }; -// Guarded per ISA: these bodies are not templates, so every TU that sees one compiles its -// intrinsics, and a TU built without that ISA fails with a target-option mismatch. -SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) -#if SVS_AVX512_F -template <> struct PriceRow { - static void min_in_row( - const double* cost_row, - const double* v, - size_t ub_padded, - double u_i, - double& best_val, - size_t& best_col - ) { - __m512d u_vec = _mm512_set1_pd(u_i); - double buf[8]; - for (size_t j = 0; j < ub_padded; j += 8) { - __m512d r = _mm512_sub_pd( - _mm512_sub_pd(_mm512_loadu_pd(cost_row + j), u_vec), _mm512_loadu_pd(v + j) +} // namespace svs::distance::transport_detail + +namespace svs::distance { + +/// Solves the dense balanced transportation problem exactly to within +/// `error_bound(ua, ub, max_cost, total_mass)`. Same contract as +/// `TransportSimplexReference::solve`, plus: `CostFn` must return `float` (pricing stores +/// costs as float, exact for callers whose costs already are, e.g. `CentroidTable`). +/// +/// Orients to put the smaller side in rows, then starts from a northwest-corner basic +/// feasible solution. Strong feasibility means every zero-flow tree arc has a *source* +/// child (points toward the root, LEMON's convention); the only zero-flow arc the start +/// ever adds is a value tie's degenerate arc, and that arc's child is always a fresh, +/// untouched row, never a sink. Every arc whose child is a sink instead carries that +/// sink's own fresh demand (or a fresh row's full supply), so it is always strictly +/// positive. Each pivot picks the leaving arc by Cunningham's rule (the *last* blocking +/// arc on the cycle in the entering arc's own direction), which preserves this invariant +/// under degenerate pivots and so guarantees termination regardless of which +/// negative-reduced-cost arc enters next. +/// Pricing scans the dense cost matrix in a block-cyclic resumable order in float (`Avx` +/// selects the row scan's SIMD width); a float candidate is only pivoted once its reduced +/// cost is confirmed negative in double (exactly recoverable, since the cost matrix is +/// itself stored as float), and pricing falls back to double (and a from-scratch +/// potential recompute) once a full float pass finds nothing, or once a float candidate +/// fails double confirmation. Below `narrow_index_max_side`, node ids are `uint16_t` +/// instead of `uint32_t`, halving the tree arrays' footprint; callers never see this. +/// +/// Every helper this type needs (potentials, tolerances, the dense workspace, ...) is a +/// member keyed on `Avx` (and, where it needs a node-id width, on `Impl`'s `MaxSide` +/// too), rather than a free function: an inline free function not keyed on `Avx` becomes +/// one weak symbol shared across the differently-`-march`-compiled translation units that +/// explicitly instantiate this type per ISA (see `gem.h`, near `merge_sorted_duplicates`). +template struct TransportSimplex { + /// Upper bound on `|solve() - exact optimum|`. + static double error_bound(size_t ua, size_t ub, double max_cost, double total_mass) { + return optimality_gap_bound(ua, ub, max_cost, total_mass); + } + + template + static double + solve(size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost) + requires std::same_as, float> + { + if (std::max(ua, ub) <= narrow_index_max_side) { + return Impl::solve_impl( + ua, supply, ub, demand, std::forward(cost) ); - _mm512_storeu_pd(buf, r); - for (size_t lane = 0; lane < 8; ++lane) { - if (buf[lane] < best_val) { - best_val = buf[lane]; - best_col = j + lane; - } - } } + return Impl::solve_impl( + ua, supply, ub, demand, std::forward(cost) + ); } -}; -#endif - -SVS_VALIDATE_BOOL_ENV(SVS_AVX2) -#if SVS_AVX2 -template <> struct PriceRow { - static void min_in_row( - const double* cost_row, - const double* v, - size_t ub_padded, - double u_i, - double& best_val, - size_t& best_col - ) { - __m256d u_vec = _mm256_set1_pd(u_i); - double buf[4]; - for (size_t j = 0; j < ub_padded; j += 4) { - __m256d r = _mm256_sub_pd( - _mm256_sub_pd(_mm256_loadu_pd(cost_row + j), u_vec), _mm256_loadu_pd(v + j) - ); - _mm256_storeu_pd(buf, r); - for (size_t lane = 0; lane < 4; ++lane) { - if (buf[lane] < best_val) { - best_val = buf[lane]; - best_col = j + lane; + + private: + static constexpr size_t narrow_index_max_side = 256; + static_assert( + 2 * narrow_index_max_side <= + std::numeric_limits::node_type>::max() + ); + + // node is a source iff node < m; fixes both the entering arc's direction (row -> col) + // and every tree arc's direction (LEMON's generic `_forward` collapses to this one + // comparison in a bipartite tree with no artificial root). + static bool is_source(size_t node, size_t m) { return node < m; } + + // Potential magnitude bound: a tree path sum of at most n-1 signed costs. + static double potential_bound(size_t n, double max_cost) { + return static_cast(n > 0 ? n - 1 : 0) * max_cost; + } + + // Tier-2/fallback threshold for an exact (walk_potential-based) reduced cost; see the + // commit body for the derivation. A candidate beating -walk_tol is truly negative + // independent of any potential drift. + static double walk_tol(size_t n, double max_cost) { + double eps_mach_d = std::numeric_limits::epsilon(); + return 2.0 * static_cast(n) * eps_mach_d * + (max_cost + potential_bound(n, max_cost)); + } + + // Tier-1 fast-accept threshold for a *drifted* double reduced cost (current u_d/v_d, + // stale by up to `recompute_period` pivots since the last fresh recompute): + // drift_tol = 2*(n-1)*R*9*eps*P + walk_tol, the accumulated per-pivot drift plus the + // same fresh-recompute-and-subtraction rounding walk_tol already bounds. + static double drift_tol(size_t n, double max_cost, size_t recompute_period) { + constexpr double c = 9.0; + double eps_mach_d = std::numeric_limits::epsilon(); + double per_pivot_per_arc = c * eps_mach_d * potential_bound(n, max_cost); + double accumulated_drift = 2.0 * static_cast(n > 0 ? n - 1 : 0) * + static_cast(recompute_period) * + per_pivot_per_arc; + return accumulated_drift + walk_tol(n, max_cost); + } + + // Upper bound on |solve() - exact optimum|: solve() stops once every fresh r >= + // -walk_tol, and a fresh r differs from the exact reduced cost by at most walk_tol + // (the same bound, applied once more), so the exact value is >= -2*walk_tol; plus + // flow_rounding (the final flow recompute's rounding) and final_sum_rounding (the + // double rounding of summing and multiplying at most n-1 flow*cost terms). + static double + optimality_gap_bound(size_t ua, size_t ub, double max_cost, double total_mass) { + size_t n = ua + ub; + double eps_mach_d = std::numeric_limits::epsilon(); + double n_d = static_cast(n); + double final_sum_rounding = + 2.0 * static_cast(n > 0 ? n - 1 : 0) * eps_mach_d * max_cost; + double flow_rounding = n_d * n_d * eps_mach_d * max_cost; + return (2.0 * walk_tol(n, max_cost) + flow_rounding + final_sum_rounding) * + total_mass; + } + + template struct Impl { + using NodeT = typename transport_detail::TransportIndexTraits::node_type; + + private: + // Scratch for one `solve` instantiation, grow-only so a steady-state + // solve (same shape or smaller) allocates nothing. Node ids: sources [0, m), + // sinks [m, m+k), m <= k. + struct SolverWorkspace { + std::vector cost_f; // m * padded_k, +inf in [k, padded_k) of every row + std::vector + price_f; // same shape; tree-arc cells hold +inf so pricing skips them + std::vector supply; // m; destructively consumed by the NW-corner build + std::vector demand; // k; destructively consumed by the NW-corner build + std::vector supply_marginal, + demand_marginal; // folded marginals, untouched + // Fold-before-orient scratch, each sized to k (>= both ua and ub): holds the + // caller's own supply/demand, pre-orientation, until m/k are known. + std::vector supply_folded, demand_folded; + std::vector child_sum; // final flow recompute scratch, size n + std::vector parent; // n = m+k; parent[0] (root) unused + std::vector thread; // preorder successor, circular over all n + std::vector rev_thread; // inverse of thread + std::vector succ_num; // subtree size, including self + std::vector last_succ; // last node of subtree, in preorder + std::vector flow_to_parent; // flow on the arc to parent[node] + std::vector u_d, v_d; // source/sink potentials, double master + std::vector u_f, v_f; // float mirrors, updated with every u_d/v_d + std::vector in_tree; // NW-corner bootstrap scratch + std::vector first_child, next_sibling, dfs_stack, + preorder; // bootstrap scratch + std::vector preorder_pos; // bootstrap scratch + std::vector + dirty_revs; // updateTreeStructure scratch, cleared (not freed) per pivot + std::vector walk_path; // walk_potential scratch, cleared per call + + void reserve(size_t m, size_t k, size_t padded_k) { + size_t n = m + k; + if (cost_f.size() < m * padded_k) { + cost_f.resize(m * padded_k); + price_f.resize(m * padded_k); + } + if (supply.size() < m) { + supply.resize(m); + supply_marginal.resize(m); + } + if (demand.size() < k) { + demand.resize(k); + demand_marginal.resize(k); + supply_folded.resize(k); + demand_folded.resize(k); + } + if (parent.size() < n) { + parent.resize(n); + thread.resize(n); + rev_thread.resize(n); + succ_num.resize(n); + last_succ.resize(n); + flow_to_parent.resize(n); + in_tree.resize(n); + first_child.resize(n); + next_sibling.resize(n); + dfs_stack.resize(n); + preorder.resize(n); + preorder_pos.resize(n); + dirty_revs.reserve(n); + walk_path.reserve(n); + child_sum.resize(n); + } + if (u_d.size() < m) { + u_d.resize(m); + u_f.resize(m); + } + if (v_d.size() < padded_k) { + v_d.resize(padded_k); + v_f.resize(padded_k); } } + }; + + // The dense-matrix cell for the arc between a node and its tree parent: the row + // is whichever of the two is a source, the column the other's offset by m. + static size_t arc_cell(size_t m, size_t padded_k, NodeT node, NodeT parent) { + size_t row = is_source(node, m) ? static_cast(node) + : static_cast(parent); + size_t col = is_source(node, m) ? static_cast(parent) - m + : static_cast(node) - m; + return row * padded_k + col; } - } -}; -#endif - -// Scratch for one thread's in-flight solve, reused across calls and grow-only: a bigger -// (ua, ub) resizes these up, a smaller one just uses a prefix, so steady-state solves -// (same CentroidTable, same decoded token counts) allocate nothing. Sized for ua+ub+1 -// nodes: sources [0, ua), sinks [ua, ua+ub), and one artificial root node at ua+ub. -struct Workspace { - std::vector cost; // ua * pad_to_block(ub), row-major - std::vector supply; - std::vector demand; - std::vector parent; // size ua+ub+1; -1 at the root, -2 = unvisited (scratch) - std::vector depth; // size ua+ub+1 - std::vector flow_to_parent; // size ua+ub+1; flow on the arc to parent[v] - std::vector u; // source potentials, size ua - std::vector v; // sink potentials, size pad_to_block(ub) - std::vector adj_head; // CSR adjacency over ua+ub+1 nodes, rebuilt every pivot - std::vector adj_next; - std::vector adj_to; - std::vector adj_cost; // the physical arc's cost (M for an artificial arc) - std::vector adj_flow; - std::vector bfs_order; - std::vector path_p; - std::vector path_q; - - void reserve(size_t ua, size_t ub) { - size_t n = ua + ub + 1; - size_t padded_ub = pad_to_block(ub); - if (cost.size() < ua * padded_ub) { - cost.resize(ua * padded_ub); - } - if (supply.size() < ua) { - supply.resize(ua); - } - if (demand.size() < ub) { - demand.resize(ub); - } - if (parent.size() < n) { - parent.resize(n); - } - if (depth.size() < n) { - depth.resize(n); - } - if (flow_to_parent.size() < n) { - flow_to_parent.resize(n); - } - if (u.size() < ua) { - u.resize(ua); - } - if (v.size() < padded_ub) { - v.resize(padded_ub); - } - if (adj_head.size() < n) { - adj_head.resize(n); + + // A node's pred-arc cost. Recomputed on demand instead of cached per node, since + // the tree-arc identity (node, parent[node]) already pins down the cell. + static float pred_cost_f( + const std::vector& cost_f, size_t padded_k, size_t m, NodeT node, NodeT p + ) { + return cost_f[arc_cell(m, padded_k, node, p)]; } - size_t edges = (n > 0) ? n - 1 : 0; - if (adj_next.size() < 2 * edges) { - adj_next.resize(2 * edges); - adj_to.resize(2 * edges); - adj_cost.resize(2 * edges); - adj_flow.resize(2 * edges); + + // Restores a leaving arc's price cell to its real cost once it leaves the tree. + static void unmark_tree_arc( + std::vector& price_f, + const std::vector& cost_f, + size_t padded_k, + size_t m, + NodeT node, + NodeT p + ) { + size_t cell = arc_cell(m, padded_k, node, p); + price_f[cell] = cost_f[cell]; } - if (bfs_order.size() < n) { - bfs_order.resize(n); + + // Exact potential for `node`: walk parent[] up to the root, then re-derive it + // root-down with recompute_potentials' same single-subtraction-per-step formula + // (so a walked value equals a freshly recomputed one bit for bit). + static double + walk_potential(SolverWorkspace& ws, size_t m, size_t padded_k, NodeT node) { + ws.walk_path.clear(); + for (NodeT cur = node; cur != 0; cur = ws.parent[cur]) { + ws.walk_path.push_back(cur); + } + double val = 0.0; + for (size_t idx = ws.walk_path.size(); idx-- > 0;) { + NodeT nd = ws.walk_path[idx]; + NodeT p = ws.parent[nd]; + double c = static_cast(pred_cost_f(ws.cost_f, padded_k, m, nd, p)); + val = c - val; + } + return val; } - if (path_p.size() < n) { - path_p.resize(n); + + // Builds thread/rev_thread/succ_num/last_succ from parent[] alone (the + // northwest-corner build only produces parent[]): a preorder walk, then subtree + // sizes via that preorder in reverse (a node's children, at higher preorder + // index, are already totalled by then). + static void build_tree_arrays(SolverWorkspace& ws, size_t n) { + constexpr NodeT no_node = std::numeric_limits::max(); + for (size_t v = 0; v < n; ++v) { + ws.first_child[v] = no_node; + } + for (size_t v = 1; v < n; ++v) { + NodeT p = ws.parent[v]; + ws.next_sibling[v] = ws.first_child[p]; + ws.first_child[p] = static_cast(v); + } + size_t top = 0; + ws.dfs_stack[top++] = 0; + size_t pre_count = 0; + while (top > 0) { + NodeT v = ws.dfs_stack[--top]; + ws.preorder[pre_count] = v; + ws.preorder_pos[v] = pre_count; + ++pre_count; + for (NodeT c = ws.first_child[v]; c != no_node; c = ws.next_sibling[c]) { + ws.dfs_stack[top++] = c; + } + } + for (size_t v = 0; v < n; ++v) { + ws.succ_num[v] = 1; + } + for (size_t idx = n; idx-- > 1;) { + NodeT v = ws.preorder[idx]; + ws.succ_num[ws.parent[v]] = + static_cast(ws.succ_num[ws.parent[v]] + ws.succ_num[v]); + } + for (size_t v = 0; v < n; ++v) { + size_t last_idx = + ws.preorder_pos[v] + static_cast(ws.succ_num[v]) - 1; + ws.last_succ[v] = ws.preorder[last_idx]; + } + for (size_t idx = 0; idx + 1 < n; ++idx) { + ws.thread[ws.preorder[idx]] = ws.preorder[idx + 1]; + ws.rev_thread[ws.preorder[idx + 1]] = ws.preorder[idx]; + } + ws.thread[ws.preorder[n - 1]] = 0; + ws.rev_thread[0] = ws.preorder[n - 1]; } - if (path_q.size() < n) { - path_q.resize(n); + + // Fills u_d/v_d/u_f/v_f from scratch via a thread (preorder) walk: a parent's + // potential is always known before its children's, since thread visits a node + // before its subtree. + static void + recompute_potentials(SolverWorkspace& ws, size_t m, size_t k, size_t padded_k) { + ws.u_d[0] = 0.0; + ws.u_f[0] = 0.0F; + NodeT node = ws.thread[0]; + size_t n = m + k; + for (size_t count = 1; count < n; ++count, node = ws.thread[node]) { + NodeT p = ws.parent[node]; + double c = + static_cast(pred_cost_f(ws.cost_f, padded_k, m, node, p)); + double val = (static_cast(node) < m) + ? c - ws.v_d[static_cast(p) - m] + : c - ws.u_d[p]; + if (static_cast(node) < m) { + ws.u_d[node] = val; + ws.u_f[node] = static_cast(val); + } else { + ws.v_d[static_cast(node) - m] = val; + ws.v_f[static_cast(node) - m] = static_cast(val); + } + } } - } -}; -// One directed adjacency slot: node `to`, reached from `from` over an arc with physical -// cost `cost` (the real cost(i, j) for a transportation arc, or M for an artificial one) -// and current flow `flow`. -inline void add_adj( - Workspace& ws, size_t edge_slot, int64_t from, int64_t to, double cost, double flow -) { - ws.adj_to[edge_slot] = to; - ws.adj_cost[edge_slot] = cost; - ws.adj_flow[edge_slot] = flow; - ws.adj_next[edge_slot] = ws.adj_head[static_cast(from)]; - ws.adj_head[static_cast(from)] = edge_slot; -} - -// Builds the artificial-root star: one extra node R = ua+ub, an arc (source i, R) with -// flow supply[i] for every source and (R, sink j) with flow demand[j] for every sink, all -// at cost M. Every arc starts at strictly positive flow (ws.supply/ws.demand are qEMD's -// merged weights, never zero), so this tree is strongly feasible by construction: the -// strongly-feasible condition only constrains zero-flow tree arcs, and there are none -// (Ahuja-Magnanti-Orlin ch. 11.2). Northwest-corner is not used here because it is not -// automatically strongly feasible. -inline void artificial_root_adjacency(Workspace& ws, size_t ua, size_t ub, double big_m) { - size_t root = ua + ub; - std::fill( - ws.adj_head.begin(), - ws.adj_head.begin() + static_cast(root + 1), - SIZE_MAX - ); - size_t slot = 0; - for (size_t i = 0; i < ua; ++i) { - add_adj( - ws, - slot, - static_cast(i), - static_cast(root), - big_m, - ws.supply[i] - ); - add_adj( - ws, - slot + 1, - static_cast(root), - static_cast(i), - big_m, - ws.supply[i] - ); - slot += 2; - } - for (size_t j = 0; j < ub; ++j) { - add_adj( - ws, - slot, - static_cast(root), - static_cast(ua + j), - big_m, - ws.demand[j] - ); - add_adj( - ws, - slot + 1, - static_cast(ua + j), - static_cast(root), - big_m, - ws.demand[j] - ); - slot += 2; - } -} - -// Breadth-first search from the artificial root R = ua+ub (always the root: R is -// permanently part of the spanning tree, see solve()'s doc comment) over the adjacency -// already built in ws.adj_*, filling parent/depth/flow_to_parent and the dual potentials -// u/v. A source is always the tail of its arc to its tree parent, whether that arc is a -// real one (to a sink) or artificial (to R); a sink is always the head. That fixes the -// sign in the potential update below without per-arc direction bookkeeping. Every tree -// arc has zero reduced cost by construction, which is what lets solve()'s pricing scan -// treat "negative reduced cost" as "not yet basic" with no separate basis-membership -// check. -inline void rebuild_tree(Workspace& ws, size_t ua, size_t ub) { - size_t root = ua + ub; - size_t n = root + 1; - std::fill(ws.parent.begin(), ws.parent.begin() + static_cast(n), -2); - ws.parent[root] = -1; - ws.depth[root] = 0; - ws.bfs_order[0] = static_cast(root); - size_t order_end = 1; - for (size_t qi = 0; qi < order_end; ++qi) { - int64_t node = ws.bfs_order[qi]; - double node_y = - (node == static_cast(root)) - ? 0.0 - : (node < static_cast(ua) ? ws.u[static_cast(node)] - : -ws.v[static_cast(node) - ua]); - for (size_t e = ws.adj_head[static_cast(node)]; e != SIZE_MAX; - e = ws.adj_next[e]) { - int64_t to = ws.adj_to[e]; - if (ws.parent[static_cast(to)] != -2) { - continue; - } - ws.parent[static_cast(to)] = node; - ws.depth[static_cast(to)] = ws.depth[static_cast(node)] + 1; - ws.flow_to_parent[static_cast(to)] = ws.adj_flow[e]; - double cost = ws.adj_cost[e]; - if (to < static_cast(ua)) { - ws.u[static_cast(to)] = - node_y + cost; // source: tail, y = y(parent) + cost + public: + template + static double solve_impl( + size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost + ) { + if constexpr (MaxSide != svs::Dynamic) { + size_t largest = std::max(ua, ub); + if (largest > MaxSide) { + throw std::runtime_error( + "TransportSimplex<" + std::to_string(MaxSide) + + ">::solve: max(ua, ub) = " + std::to_string(largest) + + " exceeds the bound " + std::to_string(MaxSide) + ); + } + } + + bool transposed = ua > ub; + size_t m = transposed ? ub : ua; + size_t k = transposed ? ua : ub; + + thread_local SolverWorkspace ws; + size_t padded_k = transport_detail::pad_to_lanes(k); + ws.reserve(m, k, padded_k); + size_t n = m + k; + + // Folds before orienting, on the caller's own supply/demand: folding after + // swapping rows/columns for ua > ub would land the residual on the caller's + // last supply entry instead of its last demand entry. + transport_reference_detail::prepare_marginals( + ua, supply, ws.supply_folded.data(), ub, demand, ws.demand_folded.data() + ); + const double* supply_src = + transposed ? ws.demand_folded.data() : ws.supply_folded.data(); + const double* demand_src = + transposed ? ws.supply_folded.data() : ws.demand_folded.data(); + std::copy_n(supply_src, m, ws.supply.begin()); + std::copy_n(demand_src, k, ws.demand.begin()); + // Kept aside, untouched by the NW-corner walk below, so final_cost can + // recompute every tree flow fresh from these instead of the pivot loop's + // incrementally-accumulated flow_to_parent. + std::copy_n(ws.supply.begin(), m, ws.supply_marginal.begin()); + std::copy_n(ws.demand.begin(), k, ws.demand_marginal.begin()); + + double max_cost = 0.0; + if (transposed) { + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); } else { - ws.v[static_cast(to) - ua] = cost - node_y; // sink: head + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); } - ws.bfs_order[order_end++] = to; - } - } -} + for (size_t j = k; j < padded_k; ++j) { + ws.v_d[j] = 0.0; + ws.v_f[j] = 0.0F; + } + std::copy_n(ws.cost_f.begin(), m * padded_k, ws.price_f.begin()); -} // namespace svs::distance::transport_detail + // Every zero-flow tree arc must have a source child (point toward the + // root); only the value-tie arc below is ever zero, and its child is a + // fresh, untouched (so later strictly positive) row. + std::fill( + ws.in_tree.begin(), ws.in_tree.begin() + static_cast(n), 0 + ); + ws.in_tree[0] = 1; + auto add_tree_arc = [&](size_t i, size_t j, double flow) { + size_t sink_node = m + j; + if (!ws.in_tree[sink_node]) { + ws.parent[sink_node] = static_cast(i); + ws.flow_to_parent[sink_node] = flow; + ws.in_tree[sink_node] = 1; + } else { + ws.parent[i] = static_cast(sink_node); + ws.flow_to_parent[i] = flow; + ws.in_tree[i] = 1; + } + ws.price_f[i * padded_k + j] = std::numeric_limits::infinity(); + }; + { + size_t i = 0, j = 0; + // True when row i was the side most recently entered fresh (so, at the + // final cell, is still untouched); false when column j was. + bool fresh_row = false; + while (true) { + bool last_row = (i == m - 1); + bool last_col = (j == k - 1); + if (last_row && last_col) { + // The fresh side's value is untouched and strictly positive; the + // other side's residual lives in supply/demand, not a tree arc. + double x = fresh_row ? ws.supply[i] : ws.demand[j]; + add_tree_arc(i, j, x); + ws.supply[i] -= x; + ws.demand[j] -= x; + break; + } + if (last_col) { + double x = ws.supply[i]; + add_tree_arc(i, j, x); + ws.supply[i] -= x; + ws.demand[j] -= x; + i += 1; + fresh_row = true; + } else if (last_row) { + double x = ws.demand[j]; + add_tree_arc(i, j, x); + ws.supply[i] -= x; + ws.demand[j] -= x; + j += 1; + fresh_row = false; + } else { + double x = std::min(ws.supply[i], ws.demand[j]); + add_tree_arc(i, j, x); + ws.supply[i] -= x; + ws.demand[j] -= x; + bool s_zero = ws.supply[i] <= 0.0; + bool d_zero = ws.demand[j] <= 0.0; + if (s_zero && d_zero) { + // Degenerate tie: the zero-flow arc gets a source child + // (row i+1, fresh), not a sink child, per the class doc. + add_tree_arc(i + 1, j, 0.0); + i += 1; + j += 1; + fresh_row = true; + } else if (s_zero) { + i += 1; + fresh_row = true; + } else { + j += 1; + fresh_row = false; + } + } + } + } + // parent[root] is never a real node; updateTreeStructure's last_succ walk + // relies on this exact sentinel to stop a chain that may pass through root. + ws.parent[0] = std::numeric_limits::max(); + build_tree_arrays(ws, n); + recompute_potentials(ws, m, k, padded_k); -namespace svs::distance { + size_t recompute_period = n; + double drift_threshold = drift_tol(n, max_cost, recompute_period); + double walk_threshold = walk_tol(n, max_cost); -/// Exact solver for the dense balanced transportation problem: `ua` sources with supply -/// `supply[i]`, `ub` sinks with demand `demand[j]`, a complete bipartite arc set, and a -/// per-arc cost from `cost(i, j)`. Minimizes `sum_ij flow_ij * cost(i, j)` subject to -/// `sum_j flow_ij == supply[i]`, `sum_i flow_ij == demand[j]`, `flow_ij >= 0`. -/// -/// Requires `ua >= 1`, `ub >= 1`, every supply/demand entry strictly positive (true for -/// qEMD's merged weights), and `sum(supply) == sum(demand)` to within `1e-6 * -/// max(sum(supply), 1.0)`; any smaller residual (the kind two independent floating-point -/// sums that are mathematically equal can differ by) is folded into the last demand entry -/// so the solver never over- or under-subscribes a side. A larger mismatch throws -/// std::runtime_error, since it signals mismatched inputs rather than rounding. -/// -/// Initializes with an artificial root node and big-M artificial arcs to every source and -/// sink (Ahuja-Magnanti-Orlin ch. 11.2), which is strongly feasible from the start since -/// every artificial arc carries strictly positive flow. Each pivot then picks the leaving -/// arc as the *last* blocking (flow-decreasing) arc encountered while walking the cycle -/// from its apex (the two endpoints' lowest common tree ancestor) in the entering arc's -/// own direction (Cunningham, 1976); maintaining a strongly feasible tree this way -/// guarantees termination regardless of which negative-reduced-cost arc enters next, so -/// the entering rule is free to optimize for speed: a block-cyclic search over the dense -/// cost matrix, picking the most negative reduced cost in the first block (of -/// `~sqrt(ua*ub)` arcs, rounded to whole padded rows) that has one, resuming from where -/// the previous search left off. `Avx` selects the SIMD width of each row's reduced-cost -/// scan; it does not change which solution is found. M is `(ua+ub) * max(cost) + 1`, the -/// largest an optimal basic solution's total real cost could be plus 1, so no finite -/// combination of real arcs is ever more expensive than keeping one unit of flow on an -/// artificial arc. -template struct TransportSimplex { - template - static double - solve(size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost) { - using namespace transport_detail; - thread_local Workspace ws; - ws.reserve(ua, ub); - size_t padded_ub = pad_to_block(ub); - size_t root = ua + ub; - - double total_s = 0.0, total_d = 0.0; - for (size_t i = 0; i < ua; ++i) { - ws.supply[i] = supply[i]; - total_s += supply[i]; - } - for (size_t j = 0; j < ub; ++j) { - ws.demand[j] = demand[j]; - total_d += demand[j]; - } - double tol = 1e-6 * std::max(total_s, 1.0); - double diff = total_s - total_d; - if (diff > tol || diff < -tol) { + size_t block_rows = static_cast(std::round( + std::sqrt(static_cast(m) * static_cast(k)) / + static_cast(k) + )); + block_rows = std::clamp(block_rows, 1, m); + size_t cursor_row = 0; + bool double_mode = false; + size_t pivots_since_recompute = 0; + + auto do_pivot = [&](size_t row, size_t col) { + pivot(ws, m, padded_k, row, col); + ++pivots_since_recompute; + if (pivots_since_recompute >= recompute_period) { + recompute_potentials(ws, m, k, padded_k); + pivots_since_recompute = 0; + } + }; + + // No polynomial bound exists for primal network simplex iteration count, so + // this is a generous safety net against a latent bug, not an expected limit. + size_t max_iters = 1000 * n + 10000; + for (size_t iter = 0; iter < max_iters; ++iter) { + size_t row = 0, col = 0; + bool found = !double_mode ? find_entering( + ws.price_f, + ws.u_f, + ws.v_f, + m, + padded_k, + block_rows, + cursor_row, + 0.0F, + row, + col + ) + : find_entering( + ws.price_f, + ws.u_d, + ws.v_d, + m, + padded_k, + block_rows, + cursor_row, + 0.0, + row, + col + ); + if (found) { + double r_d = static_cast(ws.cost_f[row * padded_k + col]) - + ws.u_d[row] - ws.v_d[col]; + // Tier 1: drift can't have made this one look falsely negative. + if (r_d < -drift_threshold) { + do_pivot(row, col); + continue; + } + // Tier 2: borderline; an exact root-path walk settles it either way. + if (r_d < 0.0) { + NodeT i_node = static_cast(row); + NodeT j_node = static_cast(m + col); + double r_walk = + static_cast(ws.cost_f[row * padded_k + col]) - + walk_potential(ws, m, padded_k, i_node) - + walk_potential(ws, m, padded_k, j_node); + if (r_walk < -walk_threshold) { + do_pivot(row, col); + continue; + } + } + } + + // Tier 3: neither tier above gave a trustworthy candidate (or none was + // found); the only authoritative check left is a full double pass on + // freshly recomputed potentials, which needs no further walk to trust. + recompute_potentials(ws, m, k, padded_k); + pivots_since_recompute = 0; + size_t fresh_cursor = 0; + bool found3 = find_entering( + ws.price_f, + ws.u_d, + ws.v_d, + m, + padded_k, + block_rows, + fresh_cursor, + -walk_threshold, + row, + col + ); + if (!found3) { + return final_cost(ws, m, padded_k, n); + } + double_mode = true; + do_pivot(row, col); + } throw std::runtime_error( - "TransportSimplex::solve: supply (" + std::to_string(total_s) + - ") and demand (" + std::to_string(total_d) + ") are not balanced for ua=" + - std::to_string(ua) + ", ub=" + std::to_string(ub) + "TransportSimplex::solve: exceeded the iteration limit (" + + std::to_string(max_iters) + ") for ua=" + std::to_string(ua) + + ", ub=" + std::to_string(ub) ); } - ws.demand[ub - 1] += diff; - double max_cost = 0.0; - for (size_t i = 0; i < ua; ++i) { - double* row = &ws.cost[i * padded_ub]; - for (size_t j = 0; j < ub; ++j) { - double c = static_cast(cost(i, j)); - row[j] = c; - max_cost = std::max(max_cost, c); - } - for (size_t j = ub; j < padded_ub; ++j) { - row[j] = std::numeric_limits::infinity(); + private: + // Fills cost_f/padding; Transposed is a compile-time branch so the per-cell cost + // lookup has no runtime orientation check. + template + static void fill_cost_matrix( + SolverWorkspace& ws, + size_t m, + size_t k, + size_t padded_k, + CostFn& cost, + double& max_cost + ) { + for (size_t i = 0; i < m; ++i) { + float* row = &ws.cost_f[i * padded_k]; + for (size_t j = 0; j < k; ++j) { + double c = Transposed ? static_cast(cost(j, i)) + : static_cast(cost(i, j)); + row[j] = static_cast(c); + max_cost = std::max(max_cost, c); + } + for (size_t j = k; j < padded_k; ++j) { + row[j] = std::numeric_limits::infinity(); + } } } - for (size_t j = ub; j < padded_ub; ++j) { - ws.v[j] = 0.0; - } - // Bigger than any optimal basic solution's real cost could ever be, so the - // simplex never prefers routing flow through the artificial root once a real - // spanning tree is available; see the class doc comment. - double big_m = static_cast(ua + ub) * max_cost + 1.0; - - artificial_root_adjacency(ws, ua, ub, big_m); - rebuild_tree(ws, ua, ub); - - // Block size: ~sqrt(ua*ub) arcs, rounded to a whole number of rows so each block - // scan stays an integer number of SIMD-scannable rows. - size_t block_rows = static_cast(std::round( - std::sqrt(static_cast(ua) * static_cast(ub)) / - static_cast(ub) - )); - block_rows = std::clamp(block_rows, 1, ua); - size_t cursor_row = 0; - - // No polynomial bound exists for primal network simplex iteration count (even - // with a strongly feasible tree, only finiteness is guaranteed, not a bound), so - // this is a generous safety net against a latent bug, not an expected limit. - size_t max_iters = 1000 * (ua + ub) + 10000; - for (size_t iter = 0; iter < max_iters; ++iter) { - size_t best_i = ua, best_j = 0, rows_scanned = 0; - size_t row = cursor_row; - while (rows_scanned < ua) { - double block_best_val = 0.0; - size_t block_best_i = ua, block_best_j = 0; - for (size_t b = 0; b < block_rows && rows_scanned < ua; + + // Block-cyclic resumable row scan: returns the first block (of `block_rows` + // rows, cycling from `cursor`) with an entry below `threshold`, or false once + // every row has been scanned once with none found. + template + static bool find_entering( + const std::vector& price_f, + const std::vector& u_pot, + const std::vector& v_pot, + size_t m, + size_t padded_k, + size_t block_rows, + size_t& cursor, + PotT threshold, + size_t& out_row, + size_t& out_col + ) { + size_t row = cursor; + size_t rows_scanned = 0; + while (rows_scanned < m) { + PotT block_best_val = threshold; + size_t block_best_row = m; + uint32_t block_best_col = 0; + for (size_t b = 0; b < block_rows && rows_scanned < m; ++b, ++rows_scanned) { - double val = 0.0; - size_t col = 0; - PriceRow::min_in_row( - &ws.cost[row * padded_ub], - ws.v.data(), - padded_ub, - ws.u[row], + PotT val = threshold; + uint32_t col = 0; + transport_detail::RowMin::min_in_row( + &price_f[row * padded_k], + v_pot.data(), + padded_k, + u_pot[row], val, col ); if (val < block_best_val) { block_best_val = val; - block_best_i = row; - block_best_j = col; + block_best_row = row; + block_best_col = col; } - row = (row + 1) % ua; + row = (row + 1) % m; } - if (block_best_i != ua) { - best_i = block_best_i; - best_j = block_best_j; - cursor_row = row; - break; + if (block_best_row != m) { + out_row = block_best_row; + out_col = block_best_col; + cursor = row; + return true; } } - if (best_i == ua) { - double total_cost = 0.0; - for (size_t node = 0; node < root; ++node) { - if (static_cast(ws.parent[node]) == root) { - continue; // artificial arc: not part of the real transport cost - } - size_t ai, aj; - if (node < ua) { - ai = node; - aj = static_cast(ws.parent[node]) - ua; - } else { - ai = static_cast(ws.parent[node]); - aj = node - ua; - } - total_cost += ws.flow_to_parent[node] * ws.cost[ai * padded_ub + aj]; + return false; + } + + // Recomputes every tree flow from the folded marginals in post-order instead of + // the pivot loop's incrementally-drifted flow_to_parent; see the commit body. + static double final_cost(SolverWorkspace& ws, size_t m, size_t padded_k, size_t n) { + NodeT node = 0; + for (size_t idx = 0; idx < n; ++idx, node = ws.thread[node]) { + ws.preorder[idx] = node; + } + std::fill( + ws.child_sum.begin(), ws.child_sum.begin() + static_cast(n), 0.0 + ); + for (size_t idx = n; idx-- > 1;) { + NodeT nd = ws.preorder[idx]; + NodeT p = ws.parent[nd]; + double marginal = is_source(nd, m) + ? ws.supply_marginal[nd] + : ws.demand_marginal[static_cast(nd) - m]; + double flow = marginal - ws.child_sum[nd]; + ws.flow_to_parent[nd] = flow; + ws.child_sum[p] += flow; + } + + double total = 0.0; + for (size_t idx = 1; idx < n; ++idx) { + NodeT nd = static_cast(idx); + NodeT p = ws.parent[nd]; + double c = static_cast(pred_cost_f(ws.cost_f, padded_k, m, nd, p)); + total += ws.flow_to_parent[nd] * c; + } + return total; + } + + static void + pivot(SolverWorkspace& ws, size_t m, size_t padded_k, size_t row, size_t col) { + NodeT i_node = static_cast(row); + NodeT j_node = static_cast(m + col); + + NodeT join = find_join_node(ws, i_node, j_node); + + NodeT u_in, v_in, u_out = 0; + double theta = find_leaving_arc(ws, i_node, j_node, join, m, u_in, v_in, u_out); + + // Capture the leaving cell before updateTreeStructure overwrites + // parent[u_out]. + NodeT leave_parent = ws.parent[u_out]; + unmark_tree_arc(ws.price_f, ws.cost_f, padded_k, m, u_out, leave_parent); + + change_flow(ws, i_node, j_node, join, m, theta); + update_tree_structure(ws, join, u_in, v_in, u_out, theta); + update_potential(ws, i_node, j_node, u_in, v_in, m, padded_k); + + ws.price_f[row * padded_k + col] = std::numeric_limits::infinity(); + } + + static NodeT find_join_node(SolverWorkspace& ws, NodeT i_node, NodeT j_node) { + size_t u = i_node, v = j_node; + while (u != v) { + if (ws.succ_num[u] < ws.succ_num[v]) { + u = ws.parent[u]; + } else { + v = ws.parent[v]; } - return total_cost; - } - - int64_t p_node = static_cast(best_i); - int64_t q_node = static_cast(ua + best_j); - ws.path_p.clear(); - ws.path_p.push_back(p_node); - ws.path_q.clear(); - ws.path_q.push_back(q_node); - int64_t pu = p_node, qu = q_node; - while (ws.depth[static_cast(pu)] > ws.depth[static_cast(qu)]) { - pu = ws.parent[static_cast(pu)]; - ws.path_p.push_back(pu); - } - while (ws.depth[static_cast(qu)] > ws.depth[static_cast(pu)]) { - qu = ws.parent[static_cast(qu)]; - ws.path_q.push_back(qu); - } - while (pu != qu) { - pu = ws.parent[static_cast(pu)]; - ws.path_p.push_back(pu); - qu = ws.parent[static_cast(qu)]; - ws.path_q.push_back(qu); - } - size_t lp = ws.path_p.size(); - size_t lq = ws.path_q.size(); - - // A source is always the tail of its tree arc (to a sink or to the root), a - // sink always the head (see rebuild_tree's doc comment), so walking - // parent->child is "forward" (flow increases) exactly when the child is a - // sink, and walking child->parent is forward exactly when the child is a - // source. - auto is_forward_down = [&](int64_t child) { - return child >= static_cast(ua); - }; - auto is_forward_up = [&](int64_t child) { - return child < static_cast(ua); - }; + } + return static_cast(u); + } - // Leaving arc: walk the cycle apex (= LCA = the end of either path array) - // down to p, then from q back up to the apex -- the entering arc's own - // orientation -- and take the *last* blocking (backward) arc seen, using <= - // so a later tie overwrites an earlier one (Cunningham's rule; this is what - // keeps the tree strongly feasible, not the choice of entering arc). - double theta = std::numeric_limits::infinity(); - int64_t leave_node = -1; - for (size_t t = lp; t-- > 1;) { - int64_t child = ws.path_p[t - 1]; - if (!is_forward_down(child)) { - double flow = ws.flow_to_parent[static_cast(child)]; - if (flow <= theta) { - theta = flow; - leave_node = child; - } + // Cunningham's rule: the *last* blocking arc on each side's path to the join, + // walked in the entering arc's own direction, second path's tie using <= so a + // later tie overrides an earlier one. This keeps the tree strongly feasible. + static double find_leaving_arc( + SolverWorkspace& ws, + NodeT i_node, + NodeT j_node, + NodeT join, + size_t m, + NodeT& u_in, + NodeT& v_in, + NodeT& u_out + ) { + double delta = std::numeric_limits::infinity(); + bool leaving_on_j_side = false; + for (size_t u = i_node; u != join; u = ws.parent[u]) { + double d = is_source(u, m) ? ws.flow_to_parent[u] + : std::numeric_limits::infinity(); + if (d < delta) { + delta = d; + u_out = static_cast(u); } } - for (size_t t = 0; t + 1 < lq; ++t) { - int64_t child = ws.path_q[t]; - if (!is_forward_up(child)) { - double flow = ws.flow_to_parent[static_cast(child)]; - if (flow <= theta) { - theta = flow; - leave_node = child; - } + for (size_t u = j_node; u != join; u = ws.parent[u]) { + double d = is_source(u, m) ? std::numeric_limits::infinity() + : ws.flow_to_parent[u]; + if (d <= delta) { + delta = d; + u_out = static_cast(u); + leaving_on_j_side = true; } } + if (leaving_on_j_side) { + u_in = j_node; + v_in = i_node; + } else { + u_in = i_node; + v_in = j_node; + } + return delta; + } - // Apply the flow delta to every tree edge on the cycle (the entering arc is - // not one of these; it is added fresh below with flow == theta). Order does - // not matter here, unlike the leaving-arc search above. - for (size_t t = 0; t + 1 < lp; ++t) { - int64_t child = ws.path_p[t]; - double delta = is_forward_down(child) ? theta : -theta; - ws.flow_to_parent[static_cast(child)] += delta; + static void change_flow( + SolverWorkspace& ws, + NodeT i_node, + NodeT j_node, + NodeT join, + size_t m, + double theta + ) { + if (theta <= 0.0) { + return; + } + for (size_t u = i_node; u != join; u = ws.parent[u]) { + ws.flow_to_parent[u] += is_source(u, m) ? -theta : theta; } - for (size_t t = 0; t + 1 < lq; ++t) { - int64_t child = ws.path_q[t]; - double delta = is_forward_up(child) ? theta : -theta; - ws.flow_to_parent[static_cast(child)] += delta; + for (size_t u = j_node; u != join; u = ws.parent[u]) { + ws.flow_to_parent[u] += is_source(u, m) ? theta : -theta; } + } - // Rebuild adjacency from the old tree minus the leaving edge, plus the - // entering arc (p_node, q_node, theta); rebuilding by BFS instead of - // patching parent/depth in place avoids re-rooting a subtree incrementally, - // which costs O(ua+ub) here against O(ua*ub) for pricing, so it is not the - // bottleneck. - std::fill( - ws.adj_head.begin(), - ws.adj_head.begin() + static_cast(root + 1), - SIZE_MAX - ); - size_t slot = 0; - for (int64_t node = 0; node < static_cast(root); ++node) { - if (node == leave_node) { - continue; + // Ports LEMON's updateTreeStructure: reverses the parent chain from u_in up to + // (excluding) u_out and splices thread/rev_thread accordingly. A tree arc's flow + // lives at whichever endpoint is the child, so reversing a link moves it too. + static void update_tree_structure( + SolverWorkspace& ws, + NodeT join, + NodeT u_in, + NodeT v_in, + NodeT u_out, + double theta + ) { + size_t old_rev_thread = ws.rev_thread[u_out]; + NodeT old_succ_num = ws.succ_num[u_out]; + size_t old_last_succ = ws.last_succ[u_out]; + NodeT v_out = ws.parent[u_out]; + + size_t u = ws.last_succ[u_in]; + size_t right = ws.thread[u]; + size_t last = (old_rev_thread == static_cast(v_in)) + ? ws.thread[ws.last_succ[u_out]] + : ws.thread[v_in]; + + ws.thread[v_in] = u_in; + size_t stem = u_in; + ws.dirty_revs.clear(); + ws.dirty_revs.push_back(v_in); + size_t par_stem = v_in; + while (stem != u_out) { + size_t new_stem = ws.parent[stem]; + ws.thread[u] = static_cast(new_stem); + ws.dirty_revs.push_back(static_cast(u)); + + size_t w = ws.rev_thread[stem]; + ws.thread[w] = static_cast(right); + ws.rev_thread[right] = static_cast(w); + + ws.parent[stem] = static_cast(par_stem); + par_stem = stem; + stem = new_stem; + + u = (ws.last_succ[stem] == ws.last_succ[par_stem]) ? ws.rev_thread[par_stem] + : ws.last_succ[stem]; + right = ws.thread[u]; + } + ws.parent[u_out] = static_cast(par_stem); + ws.thread[u] = static_cast(last); + ws.rev_thread[last] = static_cast(u); + ws.last_succ[u_out] = static_cast(u); + + if (old_rev_thread != static_cast(v_in)) { + ws.thread[old_rev_thread] = static_cast(right); + ws.rev_thread[right] = static_cast(old_rev_thread); + } + for (NodeT dirty : ws.dirty_revs) { + ws.rev_thread[ws.thread[dirty]] = dirty; + } + + NodeT tmp_sc = 0; + size_t tmp_ls = ws.last_succ[u_out]; + u = u_out; + while (u != static_cast(u_in)) { + size_t w = ws.parent[u]; + ws.flow_to_parent[u] = ws.flow_to_parent[w]; + tmp_sc = static_cast(tmp_sc + ws.succ_num[u] - ws.succ_num[w]); + ws.succ_num[u] = tmp_sc; + ws.last_succ[w] = static_cast(tmp_ls); + u = w; + } + ws.flow_to_parent[u_in] = theta; + ws.succ_num[u_in] = old_succ_num; + + // parent[root]'s sentinel (see solve_impl) matches this "disabled" value, so + // a chain left unbounded below is safe to walk off the end of the tree. + constexpr size_t no_node = std::numeric_limits::max(); + size_t up_limit_in = no_node, up_limit_out = no_node; + if (ws.last_succ[join] == v_in) { + up_limit_out = join; + } else { + up_limit_in = join; + } + for (u = v_in; u != up_limit_in && ws.last_succ[u] == v_in; u = ws.parent[u]) { + ws.last_succ[u] = ws.last_succ[u_out]; + } + if (join != old_rev_thread && static_cast(v_in) != old_rev_thread) { + for (u = v_out; u != up_limit_out && ws.last_succ[u] == old_last_succ; + u = ws.parent[u]) { + ws.last_succ[u] = static_cast(old_rev_thread); } - int64_t parent_node = ws.parent[static_cast(node)]; - double edge_cost; - if (static_cast(parent_node) == root) { - edge_cost = big_m; - } else if (node < static_cast(ua)) { - edge_cost = ws.cost - [static_cast(node) * padded_ub + - (static_cast(parent_node) - ua)]; - } else { - edge_cost = ws.cost - [static_cast(parent_node) * padded_ub + - (static_cast(node) - ua)]; + } else { + for (u = v_out; u != up_limit_out && ws.last_succ[u] == old_last_succ; + u = ws.parent[u]) { + ws.last_succ[u] = ws.last_succ[u_out]; } - double flow = ws.flow_to_parent[static_cast(node)]; - add_adj(ws, slot, node, parent_node, edge_cost, flow); - add_adj(ws, slot + 1, parent_node, node, edge_cost, flow); - slot += 2; } - double entering_cost = ws.cost[best_i * padded_ub + best_j]; - add_adj(ws, slot, p_node, q_node, entering_cost, theta); - add_adj(ws, slot + 1, q_node, p_node, entering_cost, theta); + for (u = v_in; u != static_cast(join); u = ws.parent[u]) { + ws.succ_num[u] = static_cast(ws.succ_num[u] + old_succ_num); + } + for (u = v_out; u != static_cast(join); u = ws.parent[u]) { + ws.succ_num[u] = static_cast(ws.succ_num[u] - old_succ_num); + } + } - rebuild_tree(ws, ua, ub); + // sigma shifts every node in u_in's moved subtree so the entering arc's + // invariant (u_i + v_j == cost) holds exactly; sources/sinks shift by + // -sigma/+sigma (not the same sign) since the invariant is a sum, not a diff. + static void update_potential( + SolverWorkspace& ws, + NodeT i_node, + NodeT j_node, + NodeT u_in, + NodeT v_in, + size_t m, + size_t padded_k + ) { + auto pi_of = [&](NodeT node) -> double { + return is_source(node, m) ? -ws.u_d[node] + : ws.v_d[static_cast(node) - m]; + }; + double entering_cost = + static_cast(ws.cost_f + [static_cast(i_node) * padded_k + + (static_cast(j_node) - m)]); + double sigma = is_source(u_in, m) ? pi_of(v_in) - pi_of(u_in) - entering_cost + : pi_of(v_in) - pi_of(u_in) + entering_cost; + + size_t end = ws.thread[ws.last_succ[u_in]]; + for (size_t node = u_in; node != end; node = ws.thread[node]) { + if (is_source(node, m)) { + ws.u_d[node] -= sigma; + ws.u_f[node] -= static_cast(sigma); + } else { + size_t j = node - m; + ws.v_d[j] += sigma; + ws.v_f[j] += static_cast(sigma); + } + } } - throw std::runtime_error( - "TransportSimplex::solve: exceeded the iteration limit (" + - std::to_string(max_iters) + ") for ua=" + std::to_string(ua) + - ", ub=" + std::to_string(ub) + }; +}; + +// NONE has no RowMin specialization of its own; it just runs the plain scalar algorithm +// every ISA is cross-checked against. +template <> struct TransportSimplex { + // NONE is the scalar baseline (no -march variation to key against), so this repeats + // TransportSimplex's own gap-bound formula rather than sharing its private one. + static double error_bound(size_t ua, size_t ub, double max_cost, double total_mass) { + size_t n = ua + ub; + double eps_mach_d = std::numeric_limits::epsilon(); + double n_d = static_cast(n); + double p = static_cast(n > 0 ? n - 1 : 0) * max_cost; + double walk_tol = 2.0 * n_d * eps_mach_d * (max_cost + p); + double flow_rounding = n_d * n_d * eps_mach_d * max_cost; + double final_sum_rounding = + 2.0 * static_cast(n > 0 ? n - 1 : 0) * eps_mach_d * max_cost; + return (2.0 * walk_tol + flow_rounding + final_sum_rounding) * total_mass; + } + + template + static double + solve(size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost) { + return TransportSimplexReference::solve( + ua, supply, ub, demand, std::forward(cost) ); } }; From 8fca1898dc7c27192a03548d194c6eb83a1ad922 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 05:01:32 -0800 Subject: [PATCH 54/68] test(distance): cross-check the optimized simplex at larger and degenerate shapes Every ISA result is compared with the NONE (reference) result within TransportSimplex::error_bound plus 2 float ulps for the float return of compute_decoded. The reference's own double rounding (about 1e-15) is far below that margin. New fixed cases: 100x100, 4x256 and 256x4, 3x200, 5x300 and 300x5 (the uint32 bucket), 1x256 and 256x1, and 100x100 with all-equal or {0,1} costs and uniform weights. Co-Authored-By: Claude Opus 5.5 --- tests/svs/core/distances/gem.cpp | 41 ++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp index 1d0dce024..9abb5c055 100644 --- a/tests/svs/core/distances/gem.cpp +++ b/tests/svs/core/distances/gem.cpp @@ -16,6 +16,7 @@ // Header under test #include "svs/core/distance/gem.h" +#include "svs/core/distance/transport_simplex.h" #include "svs/extensions/vamana/gem.h" // svs @@ -484,6 +485,7 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge ids_b[j] = static_cast(j); } CentroidTable table(k, table_flat); + double max_cost = *std::max_element(cost.begin(), cost.end()); double reference = reference_transport_cost(ua, w_a, ub, w_b, cost); float none_result = QEmdImpl::compute_decoded( @@ -493,18 +495,39 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge static_cast(none_result) == Catch::Approx(reference).margin(1e-6) ); + // Tolerance vs. NONE: the optimized solver's derived optimality-gap bound, plus 2 + // float ulps of none_result for the float cast/accumulation NONE itself already + // did. + auto isa_margin = [&](double none) { + double gap = + svs::distance::TransportSimplex::error_bound( + ua, ub, max_cost, 1.0 + ); + double ulp = 2 * static_cast(std::numeric_limits::epsilon()) * + std::max(1.0, std::abs(none)); + return gap + ulp; + }; + #if defined(__x86_64__) if (svs::detail::avx_runtime_flags.is_avx2_supported()) { float avx2_result = QEmdImpl::compute_decoded( table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub ); - CATCH_REQUIRE(avx2_result == Catch::Approx(none_result)); + CATCH_REQUIRE( + static_cast(avx2_result) == + Catch::Approx(static_cast(none_result)) + .margin(isa_margin(static_cast(none_result))) + ); } if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { float avx512_result = QEmdImpl::compute_decoded( table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub ); - CATCH_REQUIRE(avx512_result == Catch::Approx(none_result)); + CATCH_REQUIRE( + static_cast(avx512_result) == + Catch::Approx(static_cast(none_result)) + .margin(isa_margin(static_cast(none_result))) + ); } #endif }; @@ -545,6 +568,20 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge } } } + CATCH_SECTION("Larger shapes: dense, lopsided, transpose, and the Dynamic bucket") { + // 4x256/256x4 exercise the transpose; 5x300/300x5 cross into the Dynamic bucket + // (side > 256); 1x256/256x1 are single-source/sink; the last two are tied costs. + run_case(100, 100, 1.0); + run_case(4, 256, 1.0); + run_case(256, 4, 1.0); + run_case(3, 200, 1.0); + run_case(5, 300, 1.0); + run_case(300, 5, 1.0); + run_case(1, 256, 1.0); + run_case(256, 1, 1.0); + run_case(100, 100, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/0); + run_case(100, 100, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/1); + } } CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { From 26f8ae7c8723564b6b5d40245107a97e0a64637e Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 05:01:32 -0800 Subject: [PATCH 55/68] docs(third-party): add POT and LEMON license notices Co-Authored-By: Claude Opus 5.5 --- THIRD-PARTY-PROGRAMS | 61 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/THIRD-PARTY-PROGRAMS b/THIRD-PARTY-PROGRAMS index 7b1f9557f..57b8aa500 100644 --- a/THIRD-PARTY-PROGRAMS +++ b/THIRD-PARTY-PROGRAMS @@ -217,3 +217,64 @@ THE SOFTWARE. -- NOTE: Third party dependency used by this software -- This software depends on the fmt lib (MIT License), and users must comply to its license: https://raw.githubusercontent.com/fmtlib/fmt/master/LICENSE + +-------------------------------------------------------------------------------- +9. POT (include/svs/core/distance/transport_simplex.h, https://github.com/PythonOT/POT/tree/0.9.5) + +MIT License + +Copyright (c) 2016-2023 POT contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +10. LEMON (include/svs/core/distance/transport_simplex.h, https://lemon.cs.elte.hu) + +LEMON code without an explicit copyright notice is covered by the following +copyright/license. + +Copyright (C) 2003-2012 Egervary Jeno Kombinatorikus Optimalizalasi +Kutatocsoport (Egervary Combinatorial Optimization Research Group, +EGRES). + +=========================================================================== +Boost Software License, Version 1.0 +=========================================================================== + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. From 1df3cff1b797374c711b5fad083a2edfc6767006 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 06:54:09 -0800 Subject: [PATCH 56/68] perf(distance): skip non-improving blocks in the row-pricing argmin The SIMD RowMin overloads stored each block of reduced costs and scanned it with a branchy scalar loop, about 40% of solve_impl's self cycles on sub50k. Each block now tests a strict-less mask against the running best and is skipped when the mask is empty. Only an improving block resolves its minimum: the lowest column equal to the block minimum wins, and its exact bits are taken from the lane itself (compress on AVX-512, a stack readback on AVX2), so ties, signed zeros and NaN behave as in the scalar primary template. Co-Authored-By: Claude Opus 5.5 --- .../core/distance/transport_simplex_pricing.h | 152 +++++++++++++----- 1 file changed, 108 insertions(+), 44 deletions(-) diff --git a/include/svs/core/distance/transport_simplex_pricing.h b/include/svs/core/distance/transport_simplex_pricing.h index 60c43b008..b80380a04 100644 --- a/include/svs/core/distance/transport_simplex_pricing.h +++ b/include/svs/core/distance/transport_simplex_pricing.h @@ -22,8 +22,10 @@ #include "svs/core/distance/distance_core.h" #include "svs/lib/preprocessor.h" +#include #include #include +#include #if defined(__x86_64__) #include @@ -97,20 +99,26 @@ template <> struct RowMin { uint32_t& best_col ) { __m512 u_vec = _mm512_set1_ps(u_i); - float buf[16]; for (size_t j = 0; j < ub_padded; j += 16) { // Two separate _mm512_sub_ps, not an FMA: the generic version rounds the two // subtractions independently, and this must match it bit-for-bit. __m512 r = _mm512_sub_ps( _mm512_sub_ps(_mm512_loadu_ps(cost_row + j), u_vec), _mm512_loadu_ps(v + j) ); - _mm512_storeu_ps(buf, r); - for (size_t lane = 0; lane < 16; ++lane) { - if (buf[lane] < best_val) { - best_val = buf[lane]; - best_col = static_cast(j + lane); - } + // Ordered-quiet: NaN never wins, matching scalar `<`. A zero mask -- the common + // case once a good candidate is found -- skips the block untouched. + __mmask16 lt_mask = _mm512_cmp_ps_mask(r, _mm512_set1_ps(best_val), _CMP_LT_OQ); + if (lt_mask == 0) { + continue; } + // reduce_min can return a bit pattern no lane had (e.g. +0.0 vs -0.0 on a tie); + // compress then moves the lowest-index lane equal to that value to position 0. + float block_min = _mm512_mask_reduce_min_ps(lt_mask, r); + __mmask16 eq_mask = static_cast<__mmask16>( + lt_mask & _mm512_cmp_ps_mask(r, _mm512_set1_ps(block_min), _CMP_EQ_OQ) + ); + best_val = _mm512_cvtss_f32(_mm512_maskz_compress_ps(eq_mask, r)); + best_col = static_cast(j + std::countr_zero(eq_mask)); } } @@ -122,25 +130,36 @@ template <> struct RowMin { double& best_val, uint32_t& best_col ) { - double buf[16]; __m512d u_vec = _mm512_set1_pd(u_i); + // One 8-lane half (mask8 is native width for double); see the float/float overload + // above for why compress, not reduce_min, supplies best_val. + auto resolve = [&](__m512d r, size_t col_base) { + __mmask8 lt_mask = _mm512_cmp_pd_mask(r, _mm512_set1_pd(best_val), _CMP_LT_OQ); + if (lt_mask == 0) { + return; + } + double block_min = _mm512_mask_reduce_min_pd(lt_mask, r); + __mmask8 eq_mask = static_cast<__mmask8>( + lt_mask & _mm512_cmp_pd_mask(r, _mm512_set1_pd(block_min), _CMP_EQ_OQ) + ); + best_val = _mm512_cvtsd_f64(_mm512_maskz_compress_pd(eq_mask, r)); + best_col = static_cast(col_base + std::countr_zero(eq_mask)); + }; for (size_t j = 0; j < ub_padded; j += 16) { __m256 c_lo = _mm256_loadu_ps(cost_row + j); __m256 c_hi = _mm256_loadu_ps(cost_row + j + 8); - __m512d r_lo = _mm512_sub_pd( - _mm512_sub_pd(_mm512_cvtps_pd(c_lo), u_vec), _mm512_loadu_pd(v + j) + resolve( + _mm512_sub_pd( + _mm512_sub_pd(_mm512_cvtps_pd(c_lo), u_vec), _mm512_loadu_pd(v + j) + ), + j ); - __m512d r_hi = _mm512_sub_pd( - _mm512_sub_pd(_mm512_cvtps_pd(c_hi), u_vec), _mm512_loadu_pd(v + j + 8) + resolve( + _mm512_sub_pd( + _mm512_sub_pd(_mm512_cvtps_pd(c_hi), u_vec), _mm512_loadu_pd(v + j + 8) + ), + j + 8 ); - _mm512_storeu_pd(buf, r_lo); - _mm512_storeu_pd(buf + 8, r_hi); - for (size_t lane = 0; lane < 16; ++lane) { - if (buf[lane] < best_val) { - best_val = buf[lane]; - best_col = static_cast(j + lane); - } - } } } }; @@ -158,23 +177,47 @@ template <> struct RowMin { uint32_t& best_col ) { __m256 u_vec = _mm256_set1_ps(u_i); - float buf[16]; + __m256 inf_vec = _mm256_set1_ps(std::numeric_limits::infinity()); + // AVX2 has no compress instruction, so the winning lane's exact r is read back + // from a stack buffer on the rare lt_mask != 0 branch, not taken from block_min. + auto resolve = [&](__m256 r, size_t col_base) { + __m256 lt_cmp = _mm256_cmp_ps(r, _mm256_set1_ps(best_val), _CMP_LT_OQ); + auto lt_mask = static_cast(_mm256_movemask_ps(lt_cmp)); + if (lt_mask == 0) { + return; + } + __m256 masked = _mm256_blendv_ps(inf_vec, r, lt_cmp); + __m128 m = _mm_min_ps( + _mm256_castps256_ps128(masked), _mm256_extractf128_ps(masked, 1) + ); + m = _mm_min_ps(m, _mm_shuffle_ps(m, m, _MM_SHUFFLE(0, 0, 3, 2))); + m = _mm_min_ps(m, _mm_shuffle_ps(m, m, _MM_SHUFFLE(0, 0, 0, 1))); + float block_min = _mm_cvtss_f32(m); + auto eq_mask = + lt_mask & static_cast(_mm256_movemask_ps( + _mm256_cmp_ps(r, _mm256_set1_ps(block_min), _CMP_EQ_OQ) + )); + auto lane = static_cast(std::countr_zero(eq_mask)); + float buf[8]; + _mm256_storeu_ps(buf, r); + best_val = buf[lane]; + best_col = static_cast(col_base + lane); + }; for (size_t j = 0; j < ub_padded; j += 16) { - __m256 r_lo = _mm256_sub_ps( - _mm256_sub_ps(_mm256_loadu_ps(cost_row + j), u_vec), _mm256_loadu_ps(v + j) + resolve( + _mm256_sub_ps( + _mm256_sub_ps(_mm256_loadu_ps(cost_row + j), u_vec), + _mm256_loadu_ps(v + j) + ), + j ); - __m256 r_hi = _mm256_sub_ps( - _mm256_sub_ps(_mm256_loadu_ps(cost_row + j + 8), u_vec), - _mm256_loadu_ps(v + j + 8) + resolve( + _mm256_sub_ps( + _mm256_sub_ps(_mm256_loadu_ps(cost_row + j + 8), u_vec), + _mm256_loadu_ps(v + j + 8) + ), + j + 8 ); - _mm256_storeu_ps(buf, r_lo); - _mm256_storeu_ps(buf + 8, r_hi); - for (size_t lane = 0; lane < 16; ++lane) { - if (buf[lane] < best_val) { - best_val = buf[lane]; - best_col = static_cast(j + lane); - } - } } } @@ -187,20 +230,41 @@ template <> struct RowMin { uint32_t& best_col ) { __m256d u_vec = _mm256_set1_pd(u_i); - double buf[16]; + __m256d inf_vec = _mm256_set1_pd(std::numeric_limits::infinity()); + // Same technique as the float/float overload above, at the 4-double width AVX2 + // actually has. + auto resolve = [&](__m256d r, size_t col_base) { + __m256d lt_cmp = _mm256_cmp_pd(r, _mm256_set1_pd(best_val), _CMP_LT_OQ); + auto lt_mask = static_cast(_mm256_movemask_pd(lt_cmp)); + if (lt_mask == 0) { + return; + } + __m256d masked = _mm256_blendv_pd(inf_vec, r, lt_cmp); + __m128d m = _mm_min_pd( + _mm256_castpd256_pd128(masked), _mm256_extractf128_pd(masked, 1) + ); + m = _mm_min_pd(m, _mm_shuffle_pd(m, m, 0x1)); + double block_min = _mm_cvtsd_f64(m); + auto eq_mask = + lt_mask & static_cast(_mm256_movemask_pd( + _mm256_cmp_pd(r, _mm256_set1_pd(block_min), _CMP_EQ_OQ) + )); + auto lane = static_cast(std::countr_zero(eq_mask)); + double buf[4]; + _mm256_storeu_pd(buf, r); + best_val = buf[lane]; + best_col = static_cast(col_base + lane); + }; for (size_t j = 0; j < ub_padded; j += 16) { for (size_t sub = 0; sub < 16; sub += 4) { __m128 c = _mm_loadu_ps(cost_row + j + sub); - __m256d r = _mm256_sub_pd( - _mm256_sub_pd(_mm256_cvtps_pd(c), u_vec), _mm256_loadu_pd(v + j + sub) + resolve( + _mm256_sub_pd( + _mm256_sub_pd(_mm256_cvtps_pd(c), u_vec), + _mm256_loadu_pd(v + j + sub) + ), + j + sub ); - _mm256_storeu_pd(buf + sub, r); - } - for (size_t lane = 0; lane < 16; ++lane) { - if (buf[lane] < best_val) { - best_val = buf[lane]; - best_col = static_cast(j + lane); - } } } } From 56282cadcfd6a789e48acc5db3fb0f45f630c948 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 06:54:10 -0800 Subject: [PATCH 57/68] perf(distance): hoist workspace pointers in the simplex pivot path update_potential, find_join_node, the leaving-arc search, change_flow, update_tree_structure and recompute_potentials index std::vector members of the workspace in their loops. A store through one vector's data may alias another vector's pointer member, so the compiler reloaded the base pointers each iteration. The loops now use __restrict locals. Reads, writes and operation order are unchanged. Co-Authored-By: Claude Opus 5.5 --- include/svs/core/distance/transport_simplex.h | 178 ++++++++++-------- 1 file changed, 104 insertions(+), 74 deletions(-) diff --git a/include/svs/core/distance/transport_simplex.h b/include/svs/core/distance/transport_simplex.h index ddbd0bcb5..d7b8f3a40 100644 --- a/include/svs/core/distance/transport_simplex.h +++ b/include/svs/core/distance/transport_simplex.h @@ -372,23 +372,31 @@ template struct TransportSimplex { // before its subtree. static void recompute_potentials(SolverWorkspace& ws, size_t m, size_t k, size_t padded_k) { - ws.u_d[0] = 0.0; - ws.u_f[0] = 0.0F; - NodeT node = ws.thread[0]; + // Hoisted so the compiler doesn't reload each vector's base pointer every + // iteration (a store through one could otherwise alias another's data()). + const NodeT* const __restrict thread = ws.thread.data(); + const NodeT* const __restrict parent = ws.parent.data(); + double* const __restrict u_d = ws.u_d.data(); + float* const __restrict u_f = ws.u_f.data(); + double* const __restrict v_d = ws.v_d.data(); + float* const __restrict v_f = ws.v_f.data(); + u_d[0] = 0.0; + u_f[0] = 0.0F; + NodeT node = thread[0]; size_t n = m + k; - for (size_t count = 1; count < n; ++count, node = ws.thread[node]) { - NodeT p = ws.parent[node]; + for (size_t count = 1; count < n; ++count, node = thread[node]) { + NodeT p = parent[node]; double c = static_cast(pred_cost_f(ws.cost_f, padded_k, m, node, p)); double val = (static_cast(node) < m) - ? c - ws.v_d[static_cast(p) - m] - : c - ws.u_d[p]; + ? c - v_d[static_cast(p) - m] + : c - u_d[p]; if (static_cast(node) < m) { - ws.u_d[node] = val; - ws.u_f[node] = static_cast(val); + u_d[node] = val; + u_f[node] = static_cast(val); } else { - ws.v_d[static_cast(node) - m] = val; - ws.v_f[static_cast(node) - m] = static_cast(val); + v_d[static_cast(node) - m] = val; + v_f[static_cast(node) - m] = static_cast(val); } } } @@ -765,12 +773,14 @@ template struct TransportSimplex { } static NodeT find_join_node(SolverWorkspace& ws, NodeT i_node, NodeT j_node) { + const NodeT* const __restrict succ_num = ws.succ_num.data(); + const NodeT* const __restrict parent = ws.parent.data(); size_t u = i_node, v = j_node; while (u != v) { - if (ws.succ_num[u] < ws.succ_num[v]) { - u = ws.parent[u]; + if (succ_num[u] < succ_num[v]) { + u = parent[u]; } else { - v = ws.parent[v]; + v = parent[v]; } } return static_cast(u); @@ -789,19 +799,21 @@ template struct TransportSimplex { NodeT& v_in, NodeT& u_out ) { + const NodeT* const __restrict parent = ws.parent.data(); + const double* const __restrict flow_to_parent = ws.flow_to_parent.data(); double delta = std::numeric_limits::infinity(); bool leaving_on_j_side = false; - for (size_t u = i_node; u != join; u = ws.parent[u]) { - double d = is_source(u, m) ? ws.flow_to_parent[u] + for (size_t u = i_node; u != join; u = parent[u]) { + double d = is_source(u, m) ? flow_to_parent[u] : std::numeric_limits::infinity(); if (d < delta) { delta = d; u_out = static_cast(u); } } - for (size_t u = j_node; u != join; u = ws.parent[u]) { + for (size_t u = j_node; u != join; u = parent[u]) { double d = is_source(u, m) ? std::numeric_limits::infinity() - : ws.flow_to_parent[u]; + : flow_to_parent[u]; if (d <= delta) { delta = d; u_out = static_cast(u); @@ -829,11 +841,13 @@ template struct TransportSimplex { if (theta <= 0.0) { return; } - for (size_t u = i_node; u != join; u = ws.parent[u]) { - ws.flow_to_parent[u] += is_source(u, m) ? -theta : theta; + const NodeT* const __restrict parent = ws.parent.data(); + double* const __restrict flow_to_parent = ws.flow_to_parent.data(); + for (size_t u = i_node; u != join; u = parent[u]) { + flow_to_parent[u] += is_source(u, m) ? -theta : theta; } - for (size_t u = j_node; u != join; u = ws.parent[u]) { - ws.flow_to_parent[u] += is_source(u, m) ? theta : -theta; + for (size_t u = j_node; u != join; u = parent[u]) { + flow_to_parent[u] += is_source(u, m) ? theta : -theta; } } @@ -848,94 +862,103 @@ template struct TransportSimplex { NodeT u_out, double theta ) { - size_t old_rev_thread = ws.rev_thread[u_out]; - NodeT old_succ_num = ws.succ_num[u_out]; - size_t old_last_succ = ws.last_succ[u_out]; - NodeT v_out = ws.parent[u_out]; + // Hoisted: none of these vectors alias each other, but without __restrict a + // store through one forces a reload of every other's data() each iteration. + NodeT* const __restrict parent = ws.parent.data(); + NodeT* const __restrict thread = ws.thread.data(); + NodeT* const __restrict rev_thread = ws.rev_thread.data(); + NodeT* const __restrict succ_num = ws.succ_num.data(); + NodeT* const __restrict last_succ = ws.last_succ.data(); + double* const __restrict flow_to_parent = ws.flow_to_parent.data(); - size_t u = ws.last_succ[u_in]; - size_t right = ws.thread[u]; + size_t old_rev_thread = rev_thread[u_out]; + NodeT old_succ_num = succ_num[u_out]; + size_t old_last_succ = last_succ[u_out]; + NodeT v_out = parent[u_out]; + + size_t u = last_succ[u_in]; + size_t right = thread[u]; size_t last = (old_rev_thread == static_cast(v_in)) - ? ws.thread[ws.last_succ[u_out]] - : ws.thread[v_in]; + ? thread[last_succ[u_out]] + : thread[v_in]; - ws.thread[v_in] = u_in; + thread[v_in] = u_in; size_t stem = u_in; ws.dirty_revs.clear(); ws.dirty_revs.push_back(v_in); size_t par_stem = v_in; while (stem != u_out) { - size_t new_stem = ws.parent[stem]; - ws.thread[u] = static_cast(new_stem); + size_t new_stem = parent[stem]; + thread[u] = static_cast(new_stem); ws.dirty_revs.push_back(static_cast(u)); - size_t w = ws.rev_thread[stem]; - ws.thread[w] = static_cast(right); - ws.rev_thread[right] = static_cast(w); + size_t w = rev_thread[stem]; + thread[w] = static_cast(right); + rev_thread[right] = static_cast(w); - ws.parent[stem] = static_cast(par_stem); + parent[stem] = static_cast(par_stem); par_stem = stem; stem = new_stem; - u = (ws.last_succ[stem] == ws.last_succ[par_stem]) ? ws.rev_thread[par_stem] - : ws.last_succ[stem]; - right = ws.thread[u]; + u = (last_succ[stem] == last_succ[par_stem]) ? rev_thread[par_stem] + : last_succ[stem]; + right = thread[u]; } - ws.parent[u_out] = static_cast(par_stem); - ws.thread[u] = static_cast(last); - ws.rev_thread[last] = static_cast(u); - ws.last_succ[u_out] = static_cast(u); + parent[u_out] = static_cast(par_stem); + thread[u] = static_cast(last); + rev_thread[last] = static_cast(u); + last_succ[u_out] = static_cast(u); if (old_rev_thread != static_cast(v_in)) { - ws.thread[old_rev_thread] = static_cast(right); - ws.rev_thread[right] = static_cast(old_rev_thread); + thread[old_rev_thread] = static_cast(right); + rev_thread[right] = static_cast(old_rev_thread); } for (NodeT dirty : ws.dirty_revs) { - ws.rev_thread[ws.thread[dirty]] = dirty; + rev_thread[thread[dirty]] = dirty; } NodeT tmp_sc = 0; - size_t tmp_ls = ws.last_succ[u_out]; + size_t tmp_ls = last_succ[u_out]; u = u_out; while (u != static_cast(u_in)) { - size_t w = ws.parent[u]; - ws.flow_to_parent[u] = ws.flow_to_parent[w]; - tmp_sc = static_cast(tmp_sc + ws.succ_num[u] - ws.succ_num[w]); - ws.succ_num[u] = tmp_sc; - ws.last_succ[w] = static_cast(tmp_ls); + size_t w = parent[u]; + flow_to_parent[u] = flow_to_parent[w]; + tmp_sc = static_cast(tmp_sc + succ_num[u] - succ_num[w]); + succ_num[u] = tmp_sc; + last_succ[w] = static_cast(tmp_ls); u = w; } - ws.flow_to_parent[u_in] = theta; - ws.succ_num[u_in] = old_succ_num; + flow_to_parent[u_in] = theta; + succ_num[u_in] = old_succ_num; // parent[root]'s sentinel (see solve_impl) matches this "disabled" value, so // a chain left unbounded below is safe to walk off the end of the tree. constexpr size_t no_node = std::numeric_limits::max(); size_t up_limit_in = no_node, up_limit_out = no_node; - if (ws.last_succ[join] == v_in) { + if (last_succ[join] == v_in) { up_limit_out = join; } else { up_limit_in = join; } - for (u = v_in; u != up_limit_in && ws.last_succ[u] == v_in; u = ws.parent[u]) { - ws.last_succ[u] = ws.last_succ[u_out]; + for (u = v_in; u != up_limit_in && last_succ[u] == v_in; u = parent[u]) { + last_succ[u] = last_succ[u_out]; } if (join != old_rev_thread && static_cast(v_in) != old_rev_thread) { - for (u = v_out; u != up_limit_out && ws.last_succ[u] == old_last_succ; - u = ws.parent[u]) { - ws.last_succ[u] = static_cast(old_rev_thread); + for (u = v_out; u != up_limit_out && last_succ[u] == old_last_succ; + u = parent[u]) { + last_succ[u] = static_cast(old_rev_thread); } } else { - for (u = v_out; u != up_limit_out && ws.last_succ[u] == old_last_succ; - u = ws.parent[u]) { - ws.last_succ[u] = ws.last_succ[u_out]; + for (u = v_out; u != up_limit_out && last_succ[u] == old_last_succ; + u = parent[u]) { + last_succ[u] = last_succ[u_out]; } } - for (u = v_in; u != static_cast(join); u = ws.parent[u]) { - ws.succ_num[u] = static_cast(ws.succ_num[u] + old_succ_num); + for (u = v_in; u != static_cast(join); u = parent[u]) { + succ_num[u] = static_cast(succ_num[u] + old_succ_num); } - for (u = v_out; u != static_cast(join); u = ws.parent[u]) { - ws.succ_num[u] = static_cast(ws.succ_num[u] - old_succ_num); + for (u = v_out; u != static_cast(join); u = parent[u]) { + succ_num[u] = static_cast(succ_num[u] - old_succ_num); } } @@ -962,15 +985,22 @@ template struct TransportSimplex { double sigma = is_source(u_in, m) ? pi_of(v_in) - pi_of(u_in) - entering_cost : pi_of(v_in) - pi_of(u_in) + entering_cost; - size_t end = ws.thread[ws.last_succ[u_in]]; - for (size_t node = u_in; node != end; node = ws.thread[node]) { + // Hoisted: these never alias each other, but without __restrict every store + // through one forces a reload of the others' data() each iteration. + const NodeT* const __restrict thread = ws.thread.data(); + double* const __restrict u_d = ws.u_d.data(); + float* const __restrict u_f = ws.u_f.data(); + double* const __restrict v_d = ws.v_d.data(); + float* const __restrict v_f = ws.v_f.data(); + size_t end = thread[ws.last_succ[u_in]]; + for (size_t node = u_in; node != end; node = thread[node]) { if (is_source(node, m)) { - ws.u_d[node] -= sigma; - ws.u_f[node] -= static_cast(sigma); + u_d[node] -= sigma; + u_f[node] -= static_cast(sigma); } else { size_t j = node - m; - ws.v_d[j] += sigma; - ws.v_f[j] += static_cast(sigma); + v_d[j] += sigma; + v_f[j] += static_cast(sigma); } } } From 3116194aa8b41224988057c023e47df04620d207 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 06:54:15 -0800 Subject: [PATCH 58/68] perf(distance): back CentroidTable with the hugepage allocator The dense K x K table is 1 GiB at K=16384, and qEMD's cost fill does scattered lookups into it, which are TLB- and cache-miss bound. It now lives in SimpleData with HugepageAllocator, the same storage as VectorSetData and SimpleGraph. It takes 1 GiB or 2 MiB pages when available and falls back to normal pages otherwise. CentroidTable becomes copyable (deep copy) because SimpleData is. Co-Authored-By: Claude Opus 5.5 --- include/svs/core/distance/gem.h | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index 759d8d5aa..102fbd472 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -17,6 +17,7 @@ #pragma once #include "svs/concepts/vector_set.h" +#include "svs/core/data/simple.h" #include "svs/core/distance/distance_core.h" #include "svs/core/distance/gem_router.h" #include "svs/core/distance/inner_product_matrix.h" @@ -67,17 +68,20 @@ inline Ground parse_ground(std::string_view s) { /// Owns a dense K x K matrix of pairwise centroid distances, row-major. Built either by /// loading a precomputed file or in-process from centroids (see from_centroids()). +/// Backed by the hugepage allocator (see `svs::HugepageAllocator`, used the same way by +/// `svs::data::VectorSetData` and `svs::graphs::SimpleGraph`): random lookups into a +/// K=16384 table (1 GiB) are TLB/cache-miss bound, and hugepages shrink the TLB footprint. struct CentroidTable { size_t K; - std::unique_ptr data; + svs::data::SimpleData> data; CentroidTable(size_t k, std::vector table) : K{k} - , data{std::make_unique(k * k)} { + , data{k, k, svs::HugepageAllocator()} { if (table.size() != K * K) { throw std::invalid_argument("CentroidTable: data size does not match K*K"); } - std::copy(table.begin(), table.end(), data.get()); + std::copy(table.begin(), table.end(), data.data()); } // The file has no header, so K comes from table.json via the caller. @@ -101,7 +105,7 @@ struct CentroidTable { // wasted bandwidth on a buffer that can be hundreds of GiB. CentroidTable result{k, uninitialized_tag{}}; in.read( - reinterpret_cast(result.data.get()), + reinterpret_cast(result.data.data()), static_cast(expected_bytes) ); if (!in) { @@ -129,7 +133,7 @@ struct CentroidTable { const float* centroids, size_t k, size_t dim, Ground ground, size_t num_threads ) { CentroidTable result{k, uninitialized_tag{}}; - float* out = result.data.get(); + float* out = result.data.data(); svs::threads::DefaultThreadPool pool(num_threads); // Tile edge: an operand block (tile * dim * 4 bytes, 128 KiB at dim=128) and the @@ -218,17 +222,17 @@ struct CentroidTable { } float operator()(uint32_t i, uint32_t j) const { - return data[static_cast(i) * K + j]; + return data.data()[static_cast(i) * K + j]; } private: struct uninitialized_tag {}; - // new[] with no parens default-initializes float (indeterminate value, no zero - // fill); ownership passes to unique_ptr immediately, so this stays exception-safe. + // SimpleData's allocator-backed ctor default-constructs each element in place; for + // a trivial float that writes nothing, matching make_unique_for_overwrite's intent. CentroidTable(size_t k, uninitialized_tag) : K{k} - , data{std::make_unique_for_overwrite(k * k)} {} + , data{k, k, svs::HugepageAllocator()} {} }; /// Owns the raw K x dim fine-centroid vectors (row-major, not a pairwise table), as From 705115ba1d1249f5ac4e29517cb606dd91162df5 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 06:54:16 -0800 Subject: [PATCH 59/68] perf(distance): keep qEMD cost lookups row-local in transposed solves When ua > ub, TransportSimplex transposes and its cost fill walks ids_a in the inner loop, so every lookup landed in a different 64 KiB table row. A table built by from_centroids is exactly symmetric, since both halves are written from one float. For such a table compute_decoded now swaps the lookup to table(ids_b[j], ids_a[i]), which keeps the row fixed. Results are bit-identical. Tables from the raw constructor or load keep the old lookup. Co-Authored-By: Claude Opus 5.5 --- include/svs/core/distance/gem.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index 102fbd472..a01a1fa63 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -74,6 +74,8 @@ inline Ground parse_ground(std::string_view s) { struct CentroidTable { size_t K; svs::data::SimpleData> data; + // True only for a table from_centroids() built; see the comment where it is set. + bool symmetric = false; CentroidTable(size_t k, std::vector table) : K{k} @@ -218,6 +220,10 @@ struct CentroidTable { } } ); + // Every unordered pair is written once and copied to both halves above, so + // table(i,j) == table(j,i) bit-for-bit; QEmdImpl's swapped transposed-solve + // lookup (gem.h) is only exact because of that, not a general table property. + result.symmetric = true; return result; } @@ -399,6 +405,14 @@ template struct QEmdImpl { // merge_sorted_duplicates divides integer counts by n in double, so w_a/w_b each // sum to 1.0 up to a few ULPs rather than exactly; TransportSimplex::solve folds // that residual into the last demand entry instead of requiring exact balance. + if (table.symmetric && ua > ub) { + // TransportSimplex transposes and walks ids_a in its inner loop when ua > + // ub; swapping keeps the row (ids_b) fixed there. If that transpose rule + // changes, results stay correct but this locality win is lost. + auto cost_fn = [&](size_t i, size_t j) { return table(ids_b[j], ids_a[i]); }; + double cost = TransportSimplex::solve(ua, w_a, ub, w_b, cost_fn); + return static_cast(cost); + } auto cost_fn = [&](size_t i, size_t j) { return table(ids_a[i], ids_b[j]); }; double cost = TransportSimplex::solve(ua, w_a, ub, w_b, cost_fn); return static_cast(cost); From 6842734fe6aed2dd3fc3f22ff3d733bf3f0ab6cc Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 11:40:29 -0800 Subject: [PATCH 60/68] feat(distance): count transport simplex pivots in stats builds Adds TransportSimplex::thread_pivot_count(), a thread-local pivot counter that exists only when SVS_ENABLE_STATS is set (SVS_EXPERIMENTAL_ENABLE_STATS). Normal builds compile no extra code. The counter is per ISA instantiation, so a caller must read it through the same Avx as the dispatched solver. It lets a benchmark report pivots / (m + k - 1). Co-Authored-By: Claude Opus 5.5 --- include/svs/core/distance/transport_simplex.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/svs/core/distance/transport_simplex.h b/include/svs/core/distance/transport_simplex.h index d7b8f3a40..3dcf42b60 100644 --- a/include/svs/core/distance/transport_simplex.h +++ b/include/svs/core/distance/transport_simplex.h @@ -118,6 +118,16 @@ template struct TransportSimplex { return optimality_gap_bound(ua, ub, max_cost, total_mass); } +#if SVS_ENABLE_STATS + /// Pivots performed by this thread's solves since the caller last reset it. Stats + /// builds only (`SVS_EXPERIMENTAL_ENABLE_STATS`). Per ISA instantiation: read it + /// through the same `Avx` the dispatched solver used, or it reads 0. + static size_t& thread_pivot_count() { + thread_local size_t count = 0; + return count; + } +#endif + template static double solve(size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost) @@ -551,6 +561,9 @@ template struct TransportSimplex { size_t pivots_since_recompute = 0; auto do_pivot = [&](size_t row, size_t col) { +#if SVS_ENABLE_STATS + ++thread_pivot_count(); +#endif pivot(ws, m, padded_k, row, col); ++pivots_since_recompute; if (pivots_since_recompute >= recompute_period) { From 8a648b2a254cc7e7646589e2cf3a8d861491a722 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 11:40:31 -0800 Subject: [PATCH 61/68] perf(distance): start qEMD solves from a cheapest-row column order The northwest-corner start now walks columns in an order grouped by each column's cheapest row (col_argmin, ties broken by ascending column id) instead of column-id order, so the start tree is biased toward low-cost arcs before any pivot runs. Strong feasibility still holds, because this is the same NW-corner walk (the existing construction's argument) applied to a relabelled column axis; the walk's argument only ever reasons about *position* (i, p), never about a column's real id j = col_order[p]: - The forced last-row/last-col steps and the final cell are unchanged: they fire on p == k - 1 exactly as they fired on j == k - 1, and the final cell still hands the fresh side's untouched, strictly-positive value to the one remaining arc. - A value tie (supply and demand exhausted together) still adds a zero arc to row i + 1 with a source child, because that branch never reads j except to pass it to add_tree_arc; which column j happens to be is irrelevant to the branch's own logic. - Every other zero-flow arc claim is unaffected: the tie arc is still the only arc add_tree_arc is ever called with flow == 0, since every other call passes min(supply[i], demand[j]) once one side is still positive, or a fresh side's full remaining value. - Permuting columns cannot create a zero sink-child arc: sink-child flow values are exactly the same set of quantities (supply/demand residuals) the identity-order walk would have produced, just assigned to relabelled j's; relabelling a column changes which array slot a value lands in, not the value itself or whether it is positive. - Row identity is untouched (rows still visit 0..m-1 in order, root stays row 0), which is what the proof actually depends on; only columns are relabelled. Extra cost: one compare per cost cell while filling (fill_cost_matrix also tracks col_min/col_argmin), plus an O(m + k) counting sort for col_order. On qEMD solves from a 50k-document subset (K=16384), pivots per (m + k - 1) drop 5-9% in every set-size bucket, and 1-thread solve time drops 2.4-4.4%. Results are unchanged. The start therefore runs unconditionally, with no size gate. Co-Authored-By: Claude Opus 5.5 --- include/svs/core/distance/transport_simplex.h | 69 ++++++++++++++++--- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/include/svs/core/distance/transport_simplex.h b/include/svs/core/distance/transport_simplex.h index 3dcf42b60..6c28daaf9 100644 --- a/include/svs/core/distance/transport_simplex.h +++ b/include/svs/core/distance/transport_simplex.h @@ -89,8 +89,10 @@ namespace svs::distance { /// `TransportSimplexReference::solve`, plus: `CostFn` must return `float` (pricing stores /// costs as float, exact for callers whose costs already are, e.g. `CentroidTable`). /// -/// Orients to put the smaller side in rows, then starts from a northwest-corner basic -/// feasible solution. Strong feasibility means every zero-flow tree arc has a *source* +/// Orients to put the smaller side in rows, then starts from a northwest-corner walk over +/// columns grouped by their cheapest row (ties broken by ascending column id), trading a +/// one-pass cost scan for fewer pivots than a cost-blind NW start needs. +/// Strong feasibility means every zero-flow tree arc has a *source* /// child (points toward the root, LEMON's convention); the only zero-flow arc the start /// ever adds is a value tie's degenerate arc, and that arc's child is always a fresh, /// untouched row, never a sink. Every arc whose child is a sink instead carries that @@ -219,6 +221,12 @@ template struct TransportSimplex { // Fold-before-orient scratch, each sized to k (>= both ua and ub): holds the // caller's own supply/demand, pre-orientation, until m/k are known. std::vector supply_folded, demand_folded; + // Cost-aware column order: col_min/col_argmin (per-column cheapest cost/row) + // feed col_order, a stable counting sort by col_argmin (scratch col_count). + std::vector col_min; + std::vector col_argmin; + std::vector col_order; + std::vector col_count; std::vector child_sum; // final flow recompute scratch, size n std::vector parent; // n = m+k; parent[0] (root) unused std::vector thread; // preorder successor, circular over all n @@ -246,11 +254,17 @@ template struct TransportSimplex { supply.resize(m); supply_marginal.resize(m); } + if (col_count.size() < m + 1) { + col_count.resize(m + 1); + } if (demand.size() < k) { demand.resize(k); demand_marginal.resize(k); supply_folded.resize(k); demand_folded.resize(k); + col_min.resize(k); + col_argmin.resize(k); + col_order.resize(k); } if (parent.size() < n) { parent.resize(n); @@ -460,6 +474,7 @@ template struct TransportSimplex { } else { fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); } + build_col_order(ws, m, k); for (size_t j = k; j < padded_k; ++j) { ws.v_d[j] = 0.0; ws.v_f[j] = 0.0F; @@ -487,13 +502,17 @@ template struct TransportSimplex { ws.price_f[i * padded_k + j] = std::numeric_limits::infinity(); }; { - size_t i = 0, j = 0; + size_t i = 0; + // p is the walk's position among columns; j = col_order[p] is the real + // column id. The walk's correctness depends only on visiting order. + size_t p = 0; + size_t j = static_cast(ws.col_order[p]); // True when row i was the side most recently entered fresh (so, at the // final cell, is still untouched); false when column j was. bool fresh_row = false; while (true) { bool last_row = (i == m - 1); - bool last_col = (j == k - 1); + bool last_col = (p == k - 1); if (last_row && last_col) { // The fresh side's value is untouched and strictly positive; the // other side's residual lives in supply/demand, not a tree arc. @@ -515,7 +534,8 @@ template struct TransportSimplex { add_tree_arc(i, j, x); ws.supply[i] -= x; ws.demand[j] -= x; - j += 1; + p += 1; + j = static_cast(ws.col_order[p]); fresh_row = false; } else { double x = std::min(ws.supply[i], ws.demand[j]); @@ -529,13 +549,15 @@ template struct TransportSimplex { // (row i+1, fresh), not a sink child, per the class doc. add_tree_arc(i + 1, j, 0.0); i += 1; - j += 1; + p += 1; + j = static_cast(ws.col_order[p]); fresh_row = true; } else if (s_zero) { i += 1; fresh_row = true; } else { - j += 1; + p += 1; + j = static_cast(ws.col_order[p]); fresh_row = false; } } @@ -656,8 +678,8 @@ template struct TransportSimplex { } private: - // Fills cost_f/padding; Transposed is a compile-time branch so the per-cell cost - // lookup has no runtime orientation check. + // Fills cost_f/padding; Transposed avoids a runtime orientation check. Also folds + // each column's cheapest cost/row into col_min/col_argmin while filling. template static void fill_cost_matrix( SolverWorkspace& ws, @@ -674,6 +696,13 @@ template struct TransportSimplex { : static_cast(cost(i, j)); row[j] = static_cast(c); max_cost = std::max(max_cost, c); + // Row 0 seeds col_min by assignment rather than a separate +inf + // pass; strict `<` below keeps the smallest row on a tie. + float cf = static_cast(c); + if (i == 0 || cf < ws.col_min[j]) { + ws.col_min[j] = cf; + ws.col_argmin[j] = static_cast(i); + } } for (size_t j = k; j < padded_k; ++j) { row[j] = std::numeric_limits::infinity(); @@ -681,6 +710,28 @@ template struct TransportSimplex { } } + // Stable counting sort of columns by col_argmin, ties kept in ascending column + // id: col_count's prefix sums give each row's bucket start before the fill pass. + static void build_col_order(SolverWorkspace& ws, size_t m, size_t k) { + std::fill( + ws.col_count.begin(), + ws.col_count.begin() + static_cast(m + 1), + NodeT{0} + ); + for (size_t j = 0; j < k; ++j) { + ++ws.col_count[static_cast(ws.col_argmin[j]) + 1]; + } + for (size_t r = 0; r < m; ++r) { + ws.col_count[r + 1] = + static_cast(ws.col_count[r + 1] + ws.col_count[r]); + } + for (size_t j = 0; j < k; ++j) { + size_t row = ws.col_argmin[j]; + ws.col_order[ws.col_count[row]] = static_cast(j); + ws.col_count[row] = static_cast(ws.col_count[row] + 1); + } + } + // Block-cyclic resumable row scan: returns the first block (of `block_rows` // rows, cycling from `cursor`) with an entry below `threshold`, or false once // every row has been scanned once with none found. From 2b01046e27e28314bcce5534d206b8e796cd2d25 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 11:40:32 -0800 Subject: [PATCH 62/68] test(distance): cover start-tree degeneracy in the qEMD solver cross-check Adds a section to the random cross-check case: all-equal and {0,1} costs, uniform and repeated ({1,2,3}) weights, 1xn, nx1 and nxn shapes up to 256x256, plus a cost pattern where row 0 is the cheapest row for every column (every column in one group). Existing sections draw the same random numbers as before. Co-Authored-By: Claude Opus 5.5 --- tests/svs/core/distances/gem.cpp | 58 ++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp index 9abb5c055..08f15ad62 100644 --- a/tests/svs/core/distances/gem.cpp +++ b/tests/svs/core/distances/gem.cpp @@ -449,11 +449,27 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge size_t ub, double weight_spread, bool uniform_weights = false, - int tiny_cost_range = -1) { + int tiny_cost_range = -1, + int repeated_levels = 0, + bool row_index_cost = false) { std::vector w_a(ua), w_b(ub); if (uniform_weights) { std::fill(w_a.begin(), w_a.end(), 1.0 / static_cast(ua)); std::fill(w_b.begin(), w_b.end(), 1.0 / static_cast(ub)); + } else if (repeated_levels > 0) { + // Weights from a small integer set, normalized: ties the walk without making + // every weight equal, unlike uniform_weights. + std::uniform_int_distribution level_dist(1, repeated_levels); + for (auto* w : {&w_a, &w_b}) { + double total = 0.0; + for (auto& x : *w) { + x = static_cast(level_dist(rng)); + total += x; + } + for (auto& x : *w) { + x /= total; + } + } } else { for (auto* w : {&w_a, &w_b}) { double total = 0.0; @@ -475,8 +491,10 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge for (size_t i = 0; i < ua; ++i) { ids_a[i] = static_cast(i); for (size_t j = 0; j < ub; ++j) { - double c = tiny_cost_range >= 0 ? static_cast(tiny_cost_dist(rng)) - : cost_dist(rng); + // row_index_cost: row 0 is cheapest for every column, by construction. + double c = row_index_cost ? static_cast(i) + : tiny_cost_range >= 0 ? static_cast(tiny_cost_dist(rng)) + : cost_dist(rng); cost[i * ub + j] = c; table_flat[i * k + j] = static_cast(c); } @@ -582,6 +600,40 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge run_case(100, 100, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/0); run_case(100, 100, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/1); } + // Ties in weights and costs are where the start tree's walk adds zero-flow arcs and + // where the cheapest-row column grouping collapses; both threaten strong feasibility. + CATCH_SECTION("Start-tree degeneracy: tied costs, repeated weights, extreme shapes") { + for (int tiny_cost_range : {0, 1}) { + for (bool uniform_weights : {false, true}) { + int repeated_levels = uniform_weights ? 0 : 3; + for (size_t n : {size_t{2}, size_t{7}, size_t{64}}) { + // With uniform weights, supply and demand tie at every walk step. + run_case(n, n, 1.0, uniform_weights, tiny_cost_range, repeated_levels); + } + // 300 keeps the Dynamic (uint32 id) bucket's 1xn/nx1 cases tied too. + for (size_t n : + {size_t{2}, size_t{7}, size_t{64}, size_t{256}, size_t{300}}) { + run_case(1, n, 1.0, uniform_weights, tiny_cost_range, repeated_levels); + run_case(n, 1, 1.0, uniform_weights, tiny_cost_range, repeated_levels); + } + } + } + + // Uniform weights only: other weights raise the reference's augmenting-path count + // toward ua+ub-1 and dominate the section's runtime. + run_case(256, 256, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/0); + run_case(256, 256, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/1); + run_case(257, 257, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/0); + run_case(257, 257, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/1); + + // Tied, lopsided, and into the Dynamic bucket on the larger side. + run_case(3, 300, 1.0, false, /*tiny_cost_range=*/1, /*repeated_levels=*/3); + run_case(300, 3, 1.0, false, /*tiny_cost_range=*/1, /*repeated_levels=*/3); + + // Row 0 is cheapest for every column, so every column lands in one group. + run_case(64, 64, 1.0, false, -1, 3, /*row_index_cost=*/true); + run_case(256, 64, 1.0, false, -1, 3, /*row_index_cost=*/true); + } } CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { From be14c2db6cf34c27c16273e9b5d221f561a12526 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 11:40:34 -0800 Subject: [PATCH 63/68] feat(kmeans): add e-means mini-batch k-means training svs::kmeans::emeans_train in svs/core/kmeans_emeans.h implements Algorithm 1 of Bessa et al., arXiv:2611.09501: - EMA per-cluster counts and sums (centroid = S / N), with a 1-based batch counter as in Algorithm 1. - A cosine-annealed step from alpha0 down to 0.1 * alpha0. - Batch size and alpha0 from the paper's Table 2 scaling laws when unset, including the /8 batch law above n = 2M. - Dead-cluster revival by splitting every 5 batches at tau = 0.1. - A final pass that sets each centroid to the mean of its last-epoch points. It reuses centroid_assignment, generate_norms and init_centroids from kmeans_training.h. Assignment runs in chunks of at most assignment_chunk rows, so the score buffer is assignment_chunk x k floats rather than B x k. The state (k * d double sums, k counts) and all scratch are allocated once. Revival costs O(k + dead * log k) per round. Deviations and readings where the paper is silent are commented in the code: - the tail batch is trained; - power-of-two rounding is done in log2 space; - the donor pool excludes this round's dead clusters; - delta = 2^-7; - epochs defaults to 2, an SVS choice. The scaling laws were fitted on DEEP (L2, 256 <= k <= 65536). Outside that range they are applied as published. The returned objective is the MSE of the final centroids. Tests cover: determinism; revival of a forced-dead cluster; the scaling laws against hand-computed values; a tail batch; the k > n error and the other parameter checks. Co-Authored-By: Claude Opus 5.5 --- include/svs/core/kmeans_emeans.h | 772 +++++++++++++++++++++++++++++ tests/svs/core/kmeans_training.cpp | 175 +++++++ 2 files changed, 947 insertions(+) create mode 100644 include/svs/core/kmeans_emeans.h diff --git a/include/svs/core/kmeans_emeans.h b/include/svs/core/kmeans_emeans.h new file mode 100644 index 000000000..b27ae451c --- /dev/null +++ b/include/svs/core/kmeans_emeans.h @@ -0,0 +1,772 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/core/kmeans_training.h" +#include "svs/lib/exception.h" +#include "svs/lib/threads/threadpool.h" +#include "svs/lib/timing.h" + +// stl +#include +#include +#include +#include +#include +#include +#include + +// e-means: mini-batch k-means with EMA centroid updates, a cosine-annealed step size, and +// dead-cluster revival, following Algorithm 1 of Bessa et al., "e-means: Billion-scale +// mini-batch k-means with low-cost approximations", arXiv:2611.09501v1. Shares its data, +// rng, thread pool and result shape with kmeans_training.h's Lloyd path and +// quantization/gem/codebook.h's train_centroids. +namespace svs::kmeans { + +/// @brief Parameters for e-means training (arXiv:2611.09501v1, Algorithm 1). +struct EMeansParameters { + /// Number of epochs T: full passes over a fresh shuffle of the data. 2 is an SVS + /// default for GEM-scale codebooks, not a value the paper recommends. + size_t epochs = 2; + /// Minibatch size B. Unset selects the batch scaling law (Eq. 7, Table 2), rounded to + /// the nearest power of two and capped at the dataset size. + std::optional batch_size = std::nullopt; + /// Initial EMA step alpha0. Unset selects the step scaling law (Eq. 6, Table 2), + /// evaluated at the resolved batch size and capped at 1. + std::optional alpha0 = std::nullopt; + /// Cosine-annealing floor: alpha decays from alpha0 to epsilon * alpha0 (Eq. 5). Must + /// lie in [0, 1]. + double epsilon = 0.1; + /// Revival threshold: a cluster is revived once its EMA count drops below tau times + /// the mean EMA count over all clusters (Appendix H.1). Must be non-negative. + double tau = 0.1; + /// Batches between dead-cluster revival checks. Must be positive (used as a modulus). + size_t revival_interval = 5; + /// When true, run the closing pass that replaces each centroid with the plain mean of + /// the points assigned to it during the last epoch (Fig. 22's "closing mean"). + bool final_mean_pass = true; + /// Caps the per-chunk assignment score buffer at assignment_chunk * k floats, the same + /// role kmeans_training.h's minibatch_size plays for the Lloyd path. Must be positive. + size_t assignment_chunk = 10'000; +}; + +/// @brief Centroids and the k-means objective (mean squared error) after emeans_train; +/// same shape as quantization::gem::L2KMeansResult, the GEM Lloyd path's result. +struct EMeansResult { + data::SimpleData centroids; + double objective = 0.0; +}; + +namespace detail { + +// Table 2 coefficients (arXiv:2611.09501v1 Sec. 4), fit by ordinary least squares in log +// space on validation MSE from DEEP/L2, 256 <= k <= 65536; not validated for other +// metrics, other datasets, or very low dimension. +inline constexpr double EMEANS_BATCH_COEFF_SMALL_N = 0.035; +// Table 2 prints this branch's coefficient as 0.0044; kept as 0.035/8 (0.004375) to match +// the paper's stated derivation ("divide by 8") rather than its rounded printed value. +inline constexpr double EMEANS_BATCH_COEFF_LARGE_N = 0.035 / 8.0; // n > 2e6 (Eq. 7 note) +inline constexpr size_t EMEANS_BATCH_LARGE_N_THRESHOLD = 2'000'000; +inline constexpr double EMEANS_BATCH_EXP_K = 0.13; +inline constexpr double EMEANS_BATCH_EXP_N = 0.88; +inline constexpr double EMEANS_BATCH_EXP_T = 0.81; + +inline constexpr double EMEANS_ALPHA_COEFF = 0.824; +inline constexpr double EMEANS_ALPHA_EXP_K = -0.46; +inline constexpr double EMEANS_ALPHA_EXP_B = 0.90; +inline constexpr double EMEANS_ALPHA_EXP_N = -0.45; +inline constexpr double EMEANS_ALPHA_EXP_T = -0.11; + +// Appendix H.1's split delta: one reading of "BF16 machine epsilon" as 2^-7 (bf16's 7 +// explicit mantissa bits); the paper does not spell out the exact bit count it means. +inline constexpr double EMEANS_REVIVAL_DELTA = 1.0 / 128.0; + +// EMA count/sum initializer (Algorithm 1, step 2): N_j starts near zero so the first +// batch's statistics dominate gamma*N_j + alpha*n_j immediately. +inline constexpr double EMEANS_INITIAL_COUNT = 1e-10; + +/// Eq. 7: batch-size scaling law, rounded to the nearest power of two and capped at n. +// "Nearest power of two" rounds log2(raw) to the nearest integer; the paper does not say +// which of the two bracketing powers wins exactly at a midpoint, this does round-half-up. +inline size_t resolve_batch_size(size_t n, size_t k, size_t epochs) { + double b_c = n > EMEANS_BATCH_LARGE_N_THRESHOLD ? EMEANS_BATCH_COEFF_LARGE_N + : EMEANS_BATCH_COEFF_SMALL_N; + double raw = b_c * std::pow(static_cast(k), EMEANS_BATCH_EXP_K) * + std::pow(static_cast(n), EMEANS_BATCH_EXP_N) * + std::pow(static_cast(epochs), EMEANS_BATCH_EXP_T); + double pow2 = std::pow(2.0, std::round(std::log2(std::max(raw, 1.0)))); + size_t b = static_cast(std::min(pow2, static_cast(n))); + return std::clamp(b, size_t{1}, n); +} + +/// Eq. 6: step scaling law, evaluated at the resolved batch size, capped at 1. +inline double resolve_alpha0(size_t n, size_t k, size_t batch_size, size_t epochs) { + double raw = EMEANS_ALPHA_COEFF * std::pow(static_cast(k), EMEANS_ALPHA_EXP_K) * + std::pow(static_cast(batch_size), EMEANS_ALPHA_EXP_B) * + std::pow(static_cast(n), EMEANS_ALPHA_EXP_N) * + std::pow(static_cast(epochs), EMEANS_ALPHA_EXP_T); + return std::min(raw, 1.0); +} + +// Eq. 5: alpha(t) = alpha0 * (eps + (1-eps) * 0.5 * (1 + cos(pi*t/M))), t = 1..M, matching +// Algorithm 1's own 1-indexed t (also used, unchanged, for the "t mod 5" revival check). +inline double cosine_step(double alpha0, double epsilon, size_t t, size_t total_batches) { + double phase = + std::numbers::pi * static_cast(t) / static_cast(total_batches); + return alpha0 * (epsilon + (1.0 - epsilon) * 0.5 * (1.0 + std::cos(phase))); +} + +// Appendix H.1: every `revival_interval` batches, a cluster with EMA count below +// tau * mean(N) is revived by splitting a donor drawn with probability proportional to +// the donor's own EMA count, without replacement among this round's revivals. Mutates N/S +// in place. `dead_scratch`/`excluded_scratch`/`cum_scratch` are caller-owned, sized k, so +// a revival round allocates nothing of its own inside the batch loop. +// +// O(k) to build the prefix sum once, then O(log k) expected per donor draw (binary search +// plus a bounded number of redraws on a collision), with an O(k) linear-scan fallback only +// once the redraw budget is exhausted. The paper draws donors "among all clusters"; this +// also excludes this round's own dead clusters from the donor pool, which differs only +// when a dead cluster would otherwise have been drawn as its own donor (negligible: a +// dead cluster's weight N_j is already far below the mean by construction). +inline void revive_dead_clusters( + std::vector& N, + std::vector& S, + size_t dims, + double tau, + std::mt19937& rng, + std::vector& dead_scratch, + std::vector& excluded_scratch, + std::vector& cum_scratch +) { + size_t k = N.size(); + double mean_n = std::accumulate(N.begin(), N.end(), 0.0) / static_cast(k); + double threshold = tau * mean_n; + + auto& dead = dead_scratch; + auto& excluded = excluded_scratch; + auto& cum = cum_scratch; + dead.clear(); + std::fill(excluded.begin(), excluded.end(), false); + for (size_t j = 0; j < k; ++j) { + if (N[j] < threshold) { + dead.push_back(j); + excluded[j] = true; + } + } + if (dead.empty()) { + return; + } + + // Prefix sum of donor weight; excluded (dead) clusters contribute 0, so a draw never + // lands on one without needing a separate filter in the binary search below. + double running = 0.0; + for (size_t c = 0; c < k; ++c) { + running += excluded[c] ? 0.0 : N[c]; + cum[c] = running; + } + double total_weight = running; + if (total_weight <= 0.0) { + return; // No eligible donor at all this round; leave every cluster dead for now. + } + + auto distribution = std::uniform_real_distribution(0.0, total_weight); + constexpr int max_redraws = 8; + for (size_t j : dead) { + size_t donor = k; + for (int attempt = 0; attempt < max_redraws; ++attempt) { + double r = distribution(rng); + auto it = std::lower_bound(cum.begin(), cum.end(), r); + size_t candidate = std::min(static_cast(it - cum.begin()), k - 1); + if (!excluded[candidate]) { + donor = candidate; + break; + } + } + if (donor == k) { + // Redraw budget exhausted (heavy exclusion this round): fall back to a + // guaranteed-correct linear scan for the first eligible cluster. + for (size_t c = 0; c < k; ++c) { + if (!excluded[c]) { + donor = c; + break; + } + } + } + if (donor == k) { + continue; // No eligible donor left this round; leave the cluster dead for now. + } + excluded[donor] = true; + + // Split the donor along a hyperplane through its centroid: opposite-sign + // perturbation on alternating dimensions, then halve its count between the two. + double donor_n_half = N[donor] / 2.0; + for (size_t d = 0; d < dims; ++d) { + double c_val = S[donor * dims + d] / N[donor]; + double sign = (d % 2 == 0) ? 1.0 : -1.0; + S[j * dims + d] = c_val * (1.0 + sign * EMEANS_REVIVAL_DELTA) * donor_n_half; + S[donor * dims + d] = + c_val * (1.0 - sign * EMEANS_REVIVAL_DELTA) * donor_n_half; + } + N[j] = donor_n_half; + N[donor] = donor_n_half; + } +} + +// Accumulates this batch's per-cluster count n_j and sum s_j (Algorithm 1's per-batch +// statistics), ahead of the EMA update. Range-partitions by centroid id so each thread +// owns disjoint output, the same determinism-by-construction pattern kmeans_training.h's +// centroid_adjustment uses; `batch_counts`/`batch_sums` must be zeroed by the caller. +template +void accumulate_batch_stats( + const data::SimpleData& gathered, + size_t count, + const std::vector& local_assignments, + size_t num_clusters, + std::vector& batch_counts, + std::vector& batch_sums, + Pool& threadpool +) { + size_t n_threads = threadpool.size(); + size_t dims = gathered.dimensions(); + threads::parallel_for( + threadpool, + threads::StaticPartition{n_threads}, + [&](auto /*indices*/, auto tid) { + size_t centroid_start = (num_clusters * tid) / n_threads; + size_t centroid_end = (num_clusters * (tid + 1)) / n_threads; + for (size_t i = 0; i < count; ++i) { + size_t assignment = local_assignments[i]; + if (assignment >= centroid_start && assignment < centroid_end) { + batch_counts[assignment] += 1.0; + auto datum = gathered.get_datum(i); + double* sum_row = batch_sums.data() + assignment * dims; + for (size_t p = 0; p < dims; ++p) { + sum_row[p] += static_cast(datum[p]); + } + } + } + } + ); +} + +// Closing pass (Fig. 22): replace each centroid with the plain mean of the points +// assigned to it over the whole last epoch. A cluster with no last-epoch assignment keeps +// its EMA readout rather than being zeroed. Allocates once, after the batch loop ends, not +// inside it. +template +void closing_mean_pass( + const Data& data, + data::SimpleData& centroids, + const std::vector& assignments, + Pool& threadpool +) { + size_t n_threads = threadpool.size(); + size_t k = centroids.size(); + size_t dims = centroids.dimensions(); + std::vector sums(k * dims, 0.0); + std::vector counts(k, 0); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_threads}, + [&](auto /*indices*/, auto tid) { + size_t centroid_start = (k * tid) / n_threads; + size_t centroid_end = (k * (tid + 1)) / n_threads; + for (auto i : data.eachindex()) { + size_t a = assignments[i]; + if (a >= centroid_start && a < centroid_end) { + counts[a]++; + auto datum = data.get_datum(i); + double* row = sums.data() + a * dims; + for (size_t d = 0; d < dims; ++d) { + row[d] += static_cast(datum[d]); + } + } + } + } + ); + + threads::parallel_for( + threadpool, + threads::StaticPartition{k}, + [&](auto indices, auto /*tid*/) { + for (auto j : indices) { + if (counts[j] != 0) { + auto c = centroids.get_datum(j); + double inv = 1.0 / static_cast(counts[j]); + const double* row = sums.data() + j * dims; + for (size_t d = 0; d < dims; ++d) { + c[d] = static_cast(row[d] * inv); + } + } + } + } + ); +} + +// Sum of squared L2 distances from `data_batch[0..count)` to its assigned centroid, +// computed from the raw vectors (not the matmul scores). Per-thread partials over a +// StaticPartition, summed in tid order, so the result is independent of scheduling. +template +double accumulate_squared_error( + const Data& data_batch, + size_t count, + const std::vector& local_assignments, + const data::SimpleData& centroids, + Pool& threadpool +) { + size_t n_threads = threadpool.size(); + size_t dims = centroids.dimensions(); + std::vector partials(n_threads, 0.0); + threads::parallel_for( + threadpool, + threads::StaticPartition{count}, + [&](auto indices, auto tid) { + double sum = 0.0; + for (auto i : indices) { + auto x = data_batch.get_datum(i); + auto c = centroids.get_datum(local_assignments[i]); + double sq = 0.0; + for (size_t d = 0; d < dims; ++d) { + double diff = static_cast(x[d]) - static_cast(c[d]); + sq += diff * diff; + } + sum += sq; + } + partials[tid] = sum; + } + ); + double total = 0.0; + for (size_t tid = 0; tid < n_threads; ++tid) { + total += partials[tid]; + } + return total; +} + +// Assigns data[window_start..window_start+window_size) to their nearest centroid, into +// chunk_assignments[0..window_size); window_size must not exceed chunk_assignments' size. +// Caller chunks the batch so chunk_matmul_results never grows past chunk_size * k floats. +template +void assign_chunk( + const Data& data, + size_t window_start, + size_t window_size, + Distance& distance, + data::SimpleData& centroids, + std::vector& centroids_norm, + std::vector& chunk_assignments, + std::vector& chunk_norm_scratch, + data::SimpleData& chunk_matmul_results, + Pool& threadpool, + lib::Timer& scratch_timer +) { + auto window_view = data::make_view( + data, threads::UnitRange{window_start, window_start + window_size} + ); + generate_norms(window_view, chunk_norm_scratch, threadpool); + centroid_assignment( + window_view, + chunk_norm_scratch, + threads::UnitRange{0, window_size}, + distance, + centroids, + centroids_norm, + chunk_assignments, + chunk_matmul_results, + threadpool, + scratch_timer + ); +} + +// Final objective (Algorithm 1's MSE): one pass over `data` in assignment_chunk-sized +// windows, reusing the Lloyd assignment kernel, not a per-point brute-force scan over k. +template +double compute_objective( + Data& data, + data::SimpleData& centroids, + size_t chunk_size, + data::SimpleData& chunk_matmul_results, + std::vector& chunk_norm_scratch, + std::vector& centroids_norm, + std::vector& chunk_assignments, + Pool& threadpool, + lib::Timer& scratch_timer +) { + size_t n = data.size(); + generate_norms(centroids, centroids_norm, threadpool); + auto distance = svs::DistanceL2{}; + + double total = 0.0; + for (size_t start = 0; start < n; start += chunk_size) { + size_t this_chunk = std::min(chunk_size, n - start); + assign_chunk( + data, + start, + this_chunk, + distance, + centroids, + centroids_norm, + chunk_assignments, + chunk_norm_scratch, + chunk_matmul_results, + threadpool, + scratch_timer + ); + auto window_view = + data::make_view(data, threads::UnitRange{start, start + this_chunk}); + total += accumulate_squared_error( + window_view, this_chunk, chunk_assignments, centroids, threadpool + ); + } + return total / static_cast(n); +} + +} // namespace detail + +/// @brief The batch size and alpha0 emeans_train actually uses, after applying Table 2's +/// scaling laws to any unset EMeansParameters field. Pure function of its inputs, so a +/// caller can recompute the same values for logging without re-running training. +struct ResolvedEMeansParameters { + size_t batch_size = 0; + double alpha0 = 0.0; +}; + +/// @brief Resolve EMeansParameters::batch_size/alpha0 against n and k, applying Table 2's +/// scaling laws (Eq. 6, Eq. 7) to whichever field is unset. The sole place that combines a +/// user override with the scaling-law default, so emeans_train and a caller logging the +/// effective parameters agree by construction. +inline ResolvedEMeansParameters +resolve_emeans_parameters(size_t n, size_t k, const EMeansParameters& parameters) { + size_t batch_size = std::clamp( + parameters.batch_size.value_or(detail::resolve_batch_size(n, k, parameters.epochs)), + size_t{1}, + n + ); + double alpha0 = parameters.alpha0.value_or( + detail::resolve_alpha0(n, k, batch_size, parameters.epochs) + ); + return ResolvedEMeansParameters{batch_size, alpha0}; +} + +/// @brief e-means training (arXiv:2611.09501v1, Algorithm 1), starting from caller-given +/// initial centroids. +/// +/// @param parameters e-means hyperparameters; see EMeansParameters. +/// @param data The training set. Not modified. +/// @param initial_centroids The k starting centroids (Algorithm 1, step 1). Its size fixes +/// the number of clusters k; copied into the EMA state and not aliased afterward. +/// @param rng Advanced once per epoch shuffle, and once per revival-round donor draw; a +/// fixed seed and thread count reproduce the same centroids run to run. +/// @param threadpool Thread pool used for every parallel step. +/// @param timer Receives per-epoch timing entries, in kmeans_training.h's style. +/// +/// @throws ANNEXCEPTION if initial_centroids has more clusters than `data` has points; if +/// a user-supplied batch_size is 0 or alpha0 is outside (0, 1]; or if epochs, +/// assignment_chunk, or revival_interval is 0, epsilon is outside [0, 1], or tau +/// is negative. +template +EMeansResult emeans_train( + const EMeansParameters& parameters, + Data& data, + data::SimpleData initial_centroids, + std::mt19937& rng, + Pool& threadpool, + lib::Timer& timer +) { + size_t n = data.size(); + size_t k = initial_centroids.size(); + size_t dims = data.dimensions(); + + if (k > n) { + throw ANNEXCEPTION( + "emeans_train: num_clusters={} exceeds the number of available points n={}", + k, + n + ); + } + if (parameters.epochs == 0) { + throw ANNEXCEPTION("emeans_train: epochs must be positive, got 0"); + } + if (parameters.batch_size.has_value() && *parameters.batch_size == 0) { + throw ANNEXCEPTION( + "emeans_train: batch_size must be positive, got {}", *parameters.batch_size + ); + } + if (parameters.alpha0.has_value() && + (*parameters.alpha0 <= 0.0 || *parameters.alpha0 > 1.0)) { + throw ANNEXCEPTION( + "emeans_train: alpha0 must lie in (0, 1], got {}", *parameters.alpha0 + ); + } + if (parameters.assignment_chunk == 0) { + throw ANNEXCEPTION("emeans_train: assignment_chunk must be positive, got 0"); + } + if (parameters.revival_interval == 0) { + throw ANNEXCEPTION("emeans_train: revival_interval must be positive, got 0"); + } + if (parameters.epsilon < 0.0 || parameters.epsilon > 1.0) { + throw ANNEXCEPTION( + "emeans_train: epsilon must lie in [0, 1], got {}", parameters.epsilon + ); + } + if (parameters.tau < 0.0) { + throw ANNEXCEPTION( + "emeans_train: tau must be non-negative, got {}", parameters.tau + ); + } + + auto training_timer = timer.push_back("EMeans training"); + + auto resolved = resolve_emeans_parameters(n, k, parameters); + size_t batch_size = resolved.batch_size; + double alpha0 = resolved.alpha0; + size_t chunk_size = std::min(parameters.assignment_chunk, n); + + // M, the cosine schedule's span: the paper writes M = T*floor(n/B), but a tail batch + // (n not a multiple of B) is still trained here, so M is the true batch count instead. + size_t batches_per_epoch = lib::div_round_up(n, batch_size); + size_t total_batches = parameters.epochs * batches_per_epoch; + + // State (Algorithm 1, step 2): double counts/sums, k + k*d doubles, allocated once. + std::vector N(k, detail::EMEANS_INITIAL_COUNT); + std::vector S(k * dims, 0.0); + for (size_t j = 0; j < k; ++j) { + auto c = initial_centroids.get_datum(j); + for (size_t d = 0; d < dims; ++d) { + S[j * dims + d] = N[j] * static_cast(c[d]); + } + } + + // Scratch buffers sized once and reused for every batch; chunk_matmul_results is + // capped at chunk_size rows, not batch_size (B from the scaling law can be huge). + auto centroids_fp32 = data::SimpleData(k, dims); + auto gathered = data::SimpleData(batch_size, dims); + auto chunk_matmul_results = data::SimpleData(chunk_size, k); + std::vector chunk_norm_scratch(chunk_size); + std::vector chunk_assignments(chunk_size); + std::vector centroids_norm(k); + std::vector local_assignments(batch_size); + std::vector batch_counts(k); + std::vector batch_sums(k * dims); + std::vector perm(n); + for (size_t i = 0; i < n; ++i) { + perm[i] = i; + } + std::vector last_epoch_assignments(n); + std::vector revival_dead_scratch; + revival_dead_scratch.reserve(k); + std::vector revival_excluded_scratch(k, false); + std::vector revival_cum_scratch(k); + // Per-batch kernel timing is uninteresting at hundreds of batches; keep it off `timer`. + lib::Timer scratch_timer; + + auto distance = svs::DistanceL2{}; + // Algorithm 1 is 1-indexed ("t <- t+1, gamma <- gamma(t)"): t ranges 1..M, so + // alpha(t) < alpha0 strictly even on the very first batch (gamma never starts at 0). + size_t t = 0; + for (size_t epoch = 0; epoch < parameters.epochs; ++epoch) { + auto epoch_timer = timer.push_back("epoch"); + std::shuffle(perm.begin(), perm.end(), rng); + bool last_epoch = (epoch + 1 == parameters.epochs); + + for (size_t batch = 0; batch < batches_per_epoch; ++batch) { + size_t start = batch * batch_size; + size_t end = std::min(start + batch_size, n); + size_t this_batch_size = end - start; + ++t; + + // Gather this batch's points into the contiguous scratch buffer the reused + // Lloyd-path assignment kernel (centroid_assignment) requires. + threads::parallel_for( + threadpool, + threads::StaticPartition{this_batch_size}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + auto dst = gathered.get_datum(i); + auto src = data.get_datum(perm[start + i]); + for (size_t d = 0; d < dims; ++d) { + dst[d] = static_cast(src[d]); + } + } + } + ); + + double alpha = + detail::cosine_step(alpha0, parameters.epsilon, t, total_batches); + double gamma = 1.0 - alpha; + + // Eq. 4 readout: c_j = S_j / N_j, into the fp32 buffer centroid_assignment + // uses. A cluster with N_j == 0 (EMA-decayed to nothing, never revived) keeps + // its previous centroid instead of dividing by zero. + threads::parallel_for( + threadpool, + threads::StaticPartition{k}, + [&](auto indices, auto /*tid*/) { + for (auto j : indices) { + if (N[j] == 0.0) { + continue; + } + auto c = centroids_fp32.get_datum(j); + for (size_t d = 0; d < dims; ++d) { + c[d] = static_cast(S[j * dims + d] / N[j]); + } + } + } + ); + + generate_norms(centroids_fp32, centroids_norm, threadpool); + for (size_t offset = 0; offset < this_batch_size; offset += chunk_size) { + size_t this_chunk = std::min(chunk_size, this_batch_size - offset); + detail::assign_chunk( + gathered, + offset, + this_chunk, + distance, + centroids_fp32, + centroids_norm, + chunk_assignments, + chunk_norm_scratch, + chunk_matmul_results, + threadpool, + scratch_timer + ); + for (size_t i = 0; i < this_chunk; ++i) { + local_assignments[offset + i] = chunk_assignments[i]; + } + } + + std::fill(batch_counts.begin(), batch_counts.end(), 0.0); + std::fill(batch_sums.begin(), batch_sums.end(), 0.0); + detail::accumulate_batch_stats( + gathered, + this_batch_size, + local_assignments, + k, + batch_counts, + batch_sums, + threadpool + ); + + // Eq. 2: N_j <- gamma*N_j + alpha*n_j, S_j <- gamma*S_j + alpha*s_j. + threads::parallel_for( + threadpool, + threads::StaticPartition{k}, + [&](auto indices, auto /*tid*/) { + for (auto j : indices) { + N[j] = gamma * N[j] + alpha * batch_counts[j]; + double* s_row = S.data() + j * dims; + const double* bs_row = batch_sums.data() + j * dims; + for (size_t d = 0; d < dims; ++d) { + s_row[d] = gamma * s_row[d] + alpha * bs_row[d]; + } + } + } + ); + + if (last_epoch) { + for (size_t i = 0; i < this_batch_size; ++i) { + last_epoch_assignments[perm[start + i]] = local_assignments[i]; + } + } + + if (t % parameters.revival_interval == 0) { + detail::revive_dead_clusters( + N, + S, + dims, + parameters.tau, + rng, + revival_dead_scratch, + revival_excluded_scratch, + revival_cum_scratch + ); + } + } + } + + // Final readout (same zero-count guard as the per-batch readout above), then the + // optional closing-mean pass (Fig. 22). + threads::parallel_for( + threadpool, + threads::StaticPartition{k}, + [&](auto indices, auto /*tid*/) { + for (auto j : indices) { + if (N[j] == 0.0) { + continue; + } + auto c = centroids_fp32.get_datum(j); + for (size_t d = 0; d < dims; ++d) { + c[d] = static_cast(S[j * dims + d] / N[j]); + } + } + } + ); + if (parameters.final_mean_pass) { + detail::closing_mean_pass(data, centroids_fp32, last_epoch_assignments, threadpool); + } + + double objective = detail::compute_objective( + data, + centroids_fp32, + chunk_size, + chunk_matmul_results, + chunk_norm_scratch, + centroids_norm, + chunk_assignments, + threadpool, + scratch_timer + ); + training_timer.finish(); + return EMeansResult{std::move(centroids_fp32), objective}; +} + +/// @brief e-means training (arXiv:2611.09501v1, Algorithm 1), initializing centroids by +/// random sampling without replacement from `data` (Algorithm 1, step 1). +/// +/// @param parameters e-means hyperparameters; see EMeansParameters. +/// @param data The training set. Not modified. +/// @param num_clusters k, the target number of clusters. +/// @param rng Advanced by the initial sample, then as emeans_train's other overload does. +/// @param threadpool Thread pool used for every parallel step. +/// @param timer Receives per-epoch timing entries, in kmeans_training.h's style. +/// +/// @throws ANNEXCEPTION if num_clusters exceeds the number of available points, or for the +/// same invalid-parameter cases as the initial-centroids overload. +template +EMeansResult emeans_train( + const EMeansParameters& parameters, + Data& data, + size_t num_clusters, + std::mt19937& rng, + Pool& threadpool, + lib::Timer& timer +) { + size_t n = data.size(); + if (num_clusters > n) { + throw ANNEXCEPTION( + "emeans_train: num_clusters={} exceeds the number of available points n={}", + num_clusters, + n + ); + } + std::vector id_scratch(num_clusters); + auto initial = init_centroids(data, id_scratch, num_clusters, rng, threadpool); + return emeans_train(parameters, data, std::move(initial), rng, threadpool, timer); +} + +} // namespace svs::kmeans diff --git a/tests/svs/core/kmeans_training.cpp b/tests/svs/core/kmeans_training.cpp index 61aebec9a..12cd19b31 100644 --- a/tests/svs/core/kmeans_training.cpp +++ b/tests/svs/core/kmeans_training.cpp @@ -16,6 +16,7 @@ // header under test #include "svs/core/kmeans_training.h" +#include "svs/core/kmeans_emeans.h" // svs #include "svs/core/data.h" @@ -26,8 +27,11 @@ #include "catch2/catch_test_macros.hpp" // stl +#include #include +#include #include +#include #include namespace { @@ -206,3 +210,174 @@ CATCH_TEST_CASE("Shared Kmeans Training", "[core][kmeans]") { check_distance(svs::DistanceIP()); } } + +namespace { + +// Three well-separated clusters, far enough apart that nearest-centroid assignment never +// ties and small perturbations in training never cross a cluster boundary. +svs::data::SimpleData make_three_clusters() { + auto data = svs::data::SimpleData(15, 2); + std::array, 3> centers{ + {{0.0F, 0.0F}, {100.0F, 0.0F}, {0.0F, 100.0F}}}; + for (size_t c = 0; c < 3; ++c) { + for (size_t i = 0; i < 5; ++i) { + float jitter = static_cast(i) * 0.1F; + data.set_datum( + c * 5 + i, + std::vector{centers[c][0] + jitter, centers[c][1] + jitter} + ); + } + } + return data; +} + +svs::kmeans::EMeansParameters small_emeans_parameters() { + svs::kmeans::EMeansParameters parameters; + parameters.epochs = 2; + parameters.batch_size = 3; + parameters.alpha0 = 0.5; + parameters.revival_interval = 5; + parameters.tau = 0.1; + return parameters; +} + +} // namespace + +CATCH_TEST_CASE("EMeans Training", "[core][kmeans][emeans]") { + CATCH_SECTION("determinism for a fixed seed and thread count") { + auto data = make_three_clusters(); + auto parameters = small_emeans_parameters(); + + auto run = [&]() { + auto rng = std::mt19937(0xc0ffee); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + return svs::kmeans::emeans_train( + parameters, data, size_t{3}, rng, threadpool, timer + ); + }; + + auto result_a = run(); + auto result_b = run(); + + CATCH_REQUIRE(result_a.centroids.size() == result_b.centroids.size()); + for (size_t i = 0; i < result_a.centroids.size(); ++i) { + auto a = result_a.centroids.get_datum(i); + auto b = result_b.centroids.get_datum(i); + for (size_t d = 0; d < a.size(); ++d) { + CATCH_REQUIRE(a[d] == b[d]); + } + } + CATCH_REQUIRE(result_a.objective == result_b.objective); + } + + CATCH_SECTION("a forced-dead cluster is revived") { + auto data = make_three_clusters(); + auto parameters = small_emeans_parameters(); + + // Four initial centroids: one near each real cluster, plus a fourth parked far + // from all data so it is never the nearest centroid for any point and stays dead + // (EMA count near its 1e-10 floor) until revival splits a donor into it. + auto initial = svs::data::SimpleData(4, 2); + initial.set_datum(0, std::vector{0.0F, 0.0F}); + initial.set_datum(1, std::vector{100.0F, 0.0F}); + initial.set_datum(2, std::vector{0.0F, 100.0F}); + initial.set_datum(3, std::vector{1000.0F, 1000.0F}); + + auto rng = std::mt19937(42); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto result = + svs::kmeans::emeans_train(parameters, data, initial, rng, threadpool, timer); + + // Revival splits a donor near the data into this slot; an unrevived cluster would + // still sit at (1000, 1000), about 1000 from every real cluster center. + auto final_dead = result.centroids.get_datum(3); + float nearest_center = std::numeric_limits::max(); + constexpr std::array, 3> centers{ + {{0.0F, 0.0F}, {100.0F, 0.0F}, {0.0F, 100.0F}}}; + for (auto [cx, cy] : centers) { + float dx = final_dead[0] - cx; + float dy = final_dead[1] - cy; + nearest_center = std::min(nearest_center, dx * dx + dy * dy); + } + CATCH_REQUIRE(nearest_center < 10.0F * 10.0F); + } + + CATCH_SECTION("resolve_emeans_parameters matches hand-computed scaling-law values") { + // Hand-computed (Python) from Eq. 6/Eq. 7 with the code's exact coefficients. + svs::kmeans::EMeansParameters small_n_params; + small_n_params.epochs = 2; + auto small_n = + svs::kmeans::resolve_emeans_parameters(100'000, 1'000, small_n_params); + CATCH_REQUIRE(small_n.batch_size == 4096); + CATCH_REQUIRE(std::abs(small_n.alpha0 - 0.3191081273388448) < 1e-6); + + // n > 2e6: the batch law's coefficient switches to 0.035/8. + svs::kmeans::EMeansParameters large_n_params; + large_n_params.epochs = 2; + auto large_n = + svs::kmeans::resolve_emeans_parameters(5'000'000, 1'000, large_n_params); + CATCH_REQUIRE(large_n.batch_size == 16384); + CATCH_REQUIRE(std::abs(large_n.alpha0 - 0.19109765404135304) < 1e-6); + } + + CATCH_SECTION("batch_size not dividing n exercises the tail batch") { + auto data = make_three_clusters(); // n = 15 + auto parameters = small_emeans_parameters(); + parameters.batch_size = 4; // 15 / 4 = 3 remainder 3: a tail batch of size 3 + + auto rng = std::mt19937(0xc0ffee); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto result = + svs::kmeans::emeans_train(parameters, data, size_t{3}, rng, threadpool, timer); + + CATCH_REQUIRE(result.centroids.size() == 3); + CATCH_REQUIRE(std::isfinite(result.objective)); + } + + // Not a general invariant: the last epoch's assignments come from moving centroids, so + // the closing mean need not dominate the EMA readout in general. Holds here because + // this toy set's clusters are so well separated that the assignment never wavers. + CATCH_SECTION( + "the final mean pass does not raise the objective (well-separated toy set)" + ) { + auto data = make_three_clusters(); + + auto with_pass_params = small_emeans_parameters(); + with_pass_params.final_mean_pass = true; + auto without_pass_params = small_emeans_parameters(); + without_pass_params.final_mean_pass = false; + + auto run = [&](const svs::kmeans::EMeansParameters& parameters) { + auto rng = std::mt19937(0xc0ffee); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + return svs::kmeans::emeans_train( + parameters, data, size_t{3}, rng, threadpool, timer + ); + }; + + auto with_pass = run(with_pass_params); + auto without_pass = run(without_pass_params); + + CATCH_REQUIRE(with_pass.objective <= without_pass.objective + 1e-6); + } + + CATCH_SECTION("k > n throws") { + auto data = svs::data::SimpleData(3, 2); + data.set_datum(0, std::vector{0.0F, 0.0F}); + data.set_datum(1, std::vector{1.0F, 0.0F}); + data.set_datum(2, std::vector{0.0F, 1.0F}); + + auto parameters = small_emeans_parameters(); + auto rng = std::mt19937(0xc0ffee); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + + CATCH_REQUIRE_THROWS( + svs::kmeans::emeans_train(parameters, data, size_t{5}, rng, threadpool, timer) + ); + } +} From 207c339758ba6715ffc2f4b30cbf660ef8c133a3 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 11:40:35 -0800 Subject: [PATCH 64/68] feat(gem): select Lloyd or e-means for codebook training CodebookParameters gains kmeans_algorithm (KMeansAlgorithm::Lloyd by default) and emeans_epochs, which defaults to EMeansParameters' own default. When e-means is selected, train_centroids runs emeans_train: - kmeans_iterations is ignored; - minibatch_size caps the assignment chunk; - the objective is reported as a sum, the same convention as the Lloyd path. Default behaviour is unchanged. The faiss 256-points-per-centroid bound now applies only to Lloyd. The coarse-cluster k-means stays Lloyd. Co-Authored-By: Claude Opus 5.5 --- include/svs/quantization/gem/codebook.h | 43 ++++++++++++++++++- include/svs/quantization/gem/codebook_timed.h | 4 +- tests/svs/quantization/gem/codebook.cpp | 29 +++++++++++++ 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/include/svs/quantization/gem/codebook.h b/include/svs/quantization/gem/codebook.h index 8197b5915..57657ec9c 100644 --- a/include/svs/quantization/gem/codebook.h +++ b/include/svs/quantization/gem/codebook.h @@ -25,6 +25,7 @@ #include "svs/core/data/vector_set.h" #include "svs/core/distance/gem.h" #include "svs/core/distance/inner_product_matrix.h" +#include "svs/core/kmeans_emeans.h" #include "svs/core/kmeans_training.h" #include "svs/lib/bfloat16.h" #include "svs/lib/exception.h" @@ -104,12 +105,19 @@ inline size_t colbert_sample_size(size_t num_passages) { /// Which kernel assign_tokens/quantize use to find each token's nearest centroid. enum class Assigner { ExactGemm, InnerProductMatrix }; +/// Which k-means algorithm train_centroids runs. Lloyd is the historical default and its +/// behavior is unchanged by this enum's existence; EMeans is svs::kmeans::emeans_train +/// (arXiv:2611.09501v1), wired in as an alternative for the same plain-L2 contract. +enum class KMeansAlgorithm { Lloyd, EMeans }; + /// Parameters for codebook training, assignment and the coarse-cluster side table. struct CodebookParameters { /// Number of fine centroids K; 0 selects colbert_num_partitions from the token count. size_t num_centroids = 0; + /// Lloyd iteration count; ignored when kmeans_algorithm == EMeans (use emeans_epochs). size_t kmeans_iterations = 4; /// GEMM chunk size for training's shared buffer; assign_tokens sizes its own instead. + /// Also EMeansParameters::assignment_chunk's source when kmeans_algorithm == EMeans. size_t minibatch_size = 10'000; uint32_t seed = 42; double heldout_fraction = 0.05; @@ -123,6 +131,12 @@ struct CodebookParameters { svs::distance::Ground ground = svs::distance::Ground::Cos; /// Per-thread score-buffer budget; fixed-row sizing once hit 587 GB at K=131'072. size_t assign_buffer_bytes = 32 * 1024 * 1024; + /// Which k-means algorithm train_centroids runs; Lloyd (kmeans_iterations) by default. + KMeansAlgorithm kmeans_algorithm = KMeansAlgorithm::Lloyd; + /// EMeans epochs T, used only when kmeans_algorithm == EMeans. Batch size and alpha0 + /// are left to svs::kmeans::EMeansParameters's scaling-law defaults. Defaults to + /// EMeansParameters's own default so the two never drift apart. + size_t emeans_epochs = svs::kmeans::EMeansParameters{}.epochs; }; /// Any contiguous vector-set dataset of float or Float16 tokens: VectorSetData and @@ -261,7 +275,9 @@ L2KMeansResult train_centroids( size_t minibatch_size, std::mt19937& rng, Pool& threadpool, - Observer observer = Observer{} + Observer observer = Observer{}, + KMeansAlgorithm algorithm = KMeansAlgorithm::Lloyd, + size_t emeans_epochs = svs::kmeans::EMeansParameters{}.epochs ) { size_t n = data.size(); if (n < num_clusters) { @@ -271,7 +287,27 @@ L2KMeansResult train_centroids( num_clusters ); } + + if (algorithm == KMeansAlgorithm::EMeans) { + svs::kmeans::EMeansParameters emeans_params; + emeans_params.epochs = emeans_epochs; + // Caps EMeans' per-chunk score buffer the same way minibatch_size caps Lloyd's. + emeans_params.assignment_chunk = minibatch_size; + svs::lib::Timer scratch_timer; + observer.before_iteration(0); + auto emeans_result = svs::kmeans::emeans_train( + emeans_params, data, num_clusters, rng, threadpool, scratch_timer + ); + observer.after_iteration(0); + // L2KMeansResult.objective is the raw sum detail::compute_objective produces; + // emeans_train returns the mean, so GEM's kmeans_objective field stays comparable + // across algorithms. + double objective_sum = emeans_result.objective * static_cast(n); + return L2KMeansResult{std::move(emeans_result.centroids), objective_sum}; + } + // faiss subsamples past 256/centroid; throws instead of silently diverging from it. + // Lloyd-only: e-means is designed to train on n far beyond this bound. if (n > num_clusters * 256) { throw ANNEXCEPTION( "train_centroids: {} points exceed faiss's 256-per-centroid training bound " @@ -466,7 +502,10 @@ train_codebook(const Data& tokens, const CodebookParameters& parameters, Pool& t parameters.kmeans_iterations, parameters.minibatch_size, rng, - threadpool + threadpool, + NoOpKMeansObserver{}, + parameters.kmeans_algorithm, + parameters.emeans_epochs ); svs::lib::Timer scratch_timer; diff --git a/include/svs/quantization/gem/codebook_timed.h b/include/svs/quantization/gem/codebook_timed.h index baec42ce9..11ee4c9ef 100644 --- a/include/svs/quantization/gem/codebook_timed.h +++ b/include/svs/quantization/gem/codebook_timed.h @@ -87,7 +87,9 @@ std::pair train_codebook_timed( parameters.minibatch_size, rng, threadpool, - detail::IterationTimer{timings.train_iterations, {}} + detail::IterationTimer{timings.train_iterations, {}}, + parameters.kmeans_algorithm, + parameters.emeans_epochs ); timings.train = std::chrono::duration(std::chrono::steady_clock::now() - train_start) diff --git a/tests/svs/quantization/gem/codebook.cpp b/tests/svs/quantization/gem/codebook.cpp index edf7cd024..d339a9264 100644 --- a/tests/svs/quantization/gem/codebook.cpp +++ b/tests/svs/quantization/gem/codebook.cpp @@ -429,3 +429,32 @@ CATCH_TEST_CASE("GEM Codebook - failure paths", "[quantization][gem]") { CATCH_REQUIRE_THROWS(gem::coarse_clusters(centroids, parameters, threadpool)); } } + +CATCH_TEST_CASE("GEM Codebook - k-means algorithm selection", "[quantization][gem]") { + CATCH_SECTION("default CodebookParameters selects Lloyd") { + CATCH_REQUIRE( + gem::CodebookParameters{}.kmeans_algorithm == gem::KMeansAlgorithm::Lloyd + ); + } + + CATCH_SECTION("EMeans trains K centroids of the right dimension, finite objective") { + constexpr size_t DIM = 8; + constexpr size_t K = 4; + constexpr size_t NUM_PASSAGES = 40; + auto centers = make_random_centers(K, DIM, 0x2024); + auto corpus = make_clustered_corpus( + centers, K, DIM, std::vector(NUM_PASSAGES, 2), 0xaaaa, 0.1F + ); + gem::CodebookParameters parameters; + parameters.num_centroids = K; + parameters.kmeans_algorithm = gem::KMeansAlgorithm::EMeans; + parameters.emeans_epochs = 1; + auto threadpool = svs::threads::as_threadpool(2); + + auto result = gem::train_codebook(corpus, parameters, threadpool); + + CATCH_REQUIRE(result.centroids.size() == K); + CATCH_REQUIRE(result.centroids.dimensions() == DIM); + CATCH_REQUIRE(std::isfinite(result.objective)); + } +} From 5c650da3c413c1ef40b367163d682a15e9c14f2e Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 11:40:47 -0800 Subject: [PATCH 65/68] perf(vamana): prune back-edges one node per work unit, largest first The back-edge prune ran DynamicPartition over the BackedgeBuffer buckets, 1000 consecutive node ids each. With an expensive distance, a few buckets holding hub nodes with thousands of overflow edges ran alone at the end of every batch. The prune now flattens the buckets into one list of overflow nodes. The list is sorted by overflow set size descending, ties by id. Threads claim entries one at a time from an atomic cursor. The per-thread setup (scratch vectors, build adaptor, stats collector) still runs once per thread. Generation and its bucket locks are unchanged. Each node's prune reads only its own adjacency and the data, so the per-node result does not depend on the schedule. Measured on a 50k-document qEMD sub-build (R=24, L=80, alpha=1.2, 112 threads, one NUMA node): 755 s before, 290 s after (2.6x). The build is not deterministic even before the change: two baseline builds share 99.9% of edges. Against that noise, degree statistics and recall@10 are unchanged. Co-Authored-By: Claude Opus 5.5 --- include/svs/index/vamana/vamana_build.h | 162 ++++++++++++++---------- 1 file changed, 92 insertions(+), 70 deletions(-) diff --git a/include/svs/index/vamana/vamana_build.h b/include/svs/index/vamana/vamana_build.h index d1f48eb14..e72b14e28 100644 --- a/include/svs/index/vamana/vamana_build.h +++ b/include/svs/index/vamana/vamana_build.h @@ -43,6 +43,7 @@ // stdlib #include +#include #include #include #include @@ -549,82 +550,103 @@ class VamanaBuilder { // that were recorded in the previous process. // // Take care to avoid duplicate entries. + // + // One work unit per node, largest overflow first: hub nodes no longer serialize a + // batch's tail on a few threads. Pointers stay valid since nothing mutates buckets. auto prune_timer = timer.push_back("pruning backedges"); - threads::parallel_for( - threadpool_, - threads::DynamicPartition{backedge_buffer_.buckets(), 1}, - [&](auto& buckets, uint64_t SVS_UNUSED(tid)) { - // Thread local auxiliary data structures. - std::vector> candidates{}; - std::vector pruned_results{}; - auto build_adaptor = extensions::build_adaptor(data_, distance_function_); - - auto general_accessor = build_adaptor.general_accessor(); - auto&& general_distance = build_adaptor.general_distance(); - - auto cmp = distance::comparator(general_distance); - StatsCollector stats{}; - // Backedge counting spans the whole thread, so wrap once and reuse below. - auto&& counted_backedge_distance = - stats.wrap_backedge_prune(general_distance); - for (auto& bucket : buckets) { - for (const auto& kv : bucket) { - // The ``neighbors`` class is a set. - auto src = kv.first; - const auto& neighbors = kv.second; - const auto& src_data = general_accessor(data_, src); - distance::maybe_fix_argument(general_distance, src_data); - - // Helper lambda to make distance computations look a little - // cleaner. - auto make_neighbor = [&](auto i) { - return Neighbor{ - i, - distance::compute( - counted_backedge_distance, - src_data, - general_accessor(data_, i) - )}; - }; - - candidates.clear(); - // Add the overflow candidates. - for (auto n : neighbors) { - candidates.push_back(make_neighbor(n)); - } + using entry_type = typename BackedgeBuffer::map_type::value_type; + auto& buckets = backedge_buffer_.buckets(); + size_t total_entries = 0; + for (const auto& bucket : buckets) { + total_entries += bucket.size(); + } + std::vector worklist{}; + worklist.reserve(total_entries); + for (const auto& bucket : buckets) { + for (const auto& kv : bucket) { + worklist.push_back(&kv); + } + } + // Ties on ``src`` keep the schedule deterministic. + std::sort(worklist.begin(), worklist.end(), [](const auto* a, const auto* b) { + if (a->second.size() != b->second.size()) { + return a->second.size() > b->second.size(); + } + return a->first < b->first; + }); - // Add the old adjacency list. - for (auto n : graph_.get_node(src)) { - if (!neighbors.contains(n)) { - candidates.push_back(make_neighbor(n)); - } - } - std::sort(candidates.begin(), candidates.end(), TotalOrder(cmp)); - candidates.resize( - std::min(candidates.size(), params_.max_candidate_pool_size) - ); + // Setup runs once per thread, not per node: this path also serves cheap distances, + // where per-node setup would rival the prune itself. + std::atomic cursor{0}; + threads::parallel_for(threadpool_, [&](uint64_t SVS_UNUSED(tid)) { + // Thread local auxiliary data structures. + std::vector> candidates{}; + std::vector pruned_results{}; + auto build_adaptor = extensions::build_adaptor(data_, distance_function_); + + auto general_accessor = build_adaptor.general_accessor(); + auto&& general_distance = build_adaptor.general_distance(); + + auto cmp = distance::comparator(general_distance); + StatsCollector stats{}; + // Backedge counting spans the whole thread, so wrap once and reuse below. + auto&& counted_backedge_distance = stats.wrap_backedge_prune(general_distance); + for (size_t w = cursor.fetch_add(1, std::memory_order_relaxed); + w < worklist.size(); + w = cursor.fetch_add(1, std::memory_order_relaxed)) { + const auto* entry = worklist[w]; + // The ``neighbors`` class is a set. + auto src = entry->first; + const auto& neighbors = entry->second; + const auto& src_data = general_accessor(data_, src); + distance::maybe_fix_argument(general_distance, src_data); + + // Helper lambda to make distance computations look a little + // cleaner. + auto make_neighbor = [&](auto i) { + return Neighbor{ + i, + distance::compute( + counted_backedge_distance, src_data, general_accessor(data_, i) + )}; + }; + + candidates.clear(); + // Add the overflow candidates. + for (auto n : neighbors) { + candidates.push_back(make_neighbor(n)); + } - { - [[maybe_unused]] auto backedge_prune_timer = - stats.time_backedge_prune(); - heuristic_prune_neighbors( - prune_strategy(distance_function_), - params_.prune_to, - alpha, - data_, - general_accessor, - counted_backedge_distance, - src, - lib::as_const_span(candidates), - pruned_results - ); - } - graph_.replace_node(src, pruned_results); + // Add the old adjacency list. + for (auto n : graph_.get_node(src)) { + if (!neighbors.contains(n)) { + candidates.push_back(make_neighbor(n)); } } - stats.merge(); + std::sort(candidates.begin(), candidates.end(), TotalOrder(cmp)); + candidates.resize( + std::min(candidates.size(), params_.max_candidate_pool_size) + ); + + { + [[maybe_unused]] auto backedge_prune_timer = + stats.time_backedge_prune(); + heuristic_prune_neighbors( + prune_strategy(distance_function_), + params_.prune_to, + alpha, + data_, + general_accessor, + counted_backedge_distance, + src, + lib::as_const_span(candidates), + pruned_results + ); + } + graph_.replace_node(src, pruned_results); } - ); + stats.merge(); + }); } private: From e3b0f73d91356f9a2c4108e0588b0bab5257d1d3 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 18 Nov 2026 14:52:01 -0800 Subject: [PATCH 66/68] feat(gem): train the codebook with e-means by default CodebookParameters::kmeans_algorithm now defaults to KMeansAlgorithm::EMeans (2 epochs, scaling-law batch size and step). Lloyd stays selectable. The switch was made by the 1M MS MARCO run's criteria: the e-means objective is at most Lloyd's at no more training time, and MRR@10 is within 0.005 of the Lloyd codebook at every search window. The numbers are in the run's decision record. Co-Authored-By: Claude Opus 5.5 --- include/svs/quantization/gem/codebook.h | 10 +++++----- tests/svs/quantization/gem/codebook.cpp | 8 ++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/include/svs/quantization/gem/codebook.h b/include/svs/quantization/gem/codebook.h index 57657ec9c..a1b4cace4 100644 --- a/include/svs/quantization/gem/codebook.h +++ b/include/svs/quantization/gem/codebook.h @@ -105,9 +105,9 @@ inline size_t colbert_sample_size(size_t num_passages) { /// Which kernel assign_tokens/quantize use to find each token's nearest centroid. enum class Assigner { ExactGemm, InnerProductMatrix }; -/// Which k-means algorithm train_centroids runs. Lloyd is the historical default and its -/// behavior is unchanged by this enum's existence; EMeans is svs::kmeans::emeans_train -/// (arXiv:2611.09501v1), wired in as an alternative for the same plain-L2 contract. +/// Which k-means algorithm train_centroids runs. EMeans is svs::kmeans::emeans_train +/// (arXiv:2611.09501v1) and CodebookParameters' default; Lloyd is the earlier default, +/// same plain-L2 contract. enum class KMeansAlgorithm { Lloyd, EMeans }; /// Parameters for codebook training, assignment and the coarse-cluster side table. @@ -131,8 +131,8 @@ struct CodebookParameters { svs::distance::Ground ground = svs::distance::Ground::Cos; /// Per-thread score-buffer budget; fixed-row sizing once hit 587 GB at K=131'072. size_t assign_buffer_bytes = 32 * 1024 * 1024; - /// Which k-means algorithm train_centroids runs; Lloyd (kmeans_iterations) by default. - KMeansAlgorithm kmeans_algorithm = KMeansAlgorithm::Lloyd; + /// Which k-means algorithm train_codebook runs; EMeans (emeans_epochs) by default. + KMeansAlgorithm kmeans_algorithm = KMeansAlgorithm::EMeans; /// EMeans epochs T, used only when kmeans_algorithm == EMeans. Batch size and alpha0 /// are left to svs::kmeans::EMeansParameters's scaling-law defaults. Defaults to /// EMeansParameters's own default so the two never drift apart. diff --git a/tests/svs/quantization/gem/codebook.cpp b/tests/svs/quantization/gem/codebook.cpp index d339a9264..7d5cec3a6 100644 --- a/tests/svs/quantization/gem/codebook.cpp +++ b/tests/svs/quantization/gem/codebook.cpp @@ -230,6 +230,8 @@ CATCH_TEST_CASE("GEM Codebook - quantize pipeline", "[quantization][gem]") { centers, 16, DIM, std::vector(NUM_PASSAGES, 2), 0x1357, 0.1F ); gem::CodebookParameters parameters; // num_centroids == 0: use the default rule. + // Per-iteration timing and the faiss bound below are Lloyd's contract. + parameters.kmeans_algorithm = gem::KMeansAlgorithm::Lloyd; auto threadpool = svs::threads::as_threadpool(2); auto result = gem::quantize(corpus, parameters, threadpool); size_t expected_K = gem::colbert_num_partitions(NUM_PASSAGES * 2); @@ -412,6 +414,8 @@ CATCH_TEST_CASE("GEM Codebook - failure paths", "[quantization][gem]") { auto dense_corpus = make_clustered_corpus(centers, K, DIM, {600}, 6, 0.05F); gem::CodebookParameters parameters; parameters.num_centroids = 2; // 600ish training tokens > 2 * 256 == 512 + parameters.kmeans_algorithm = + gem::KMeansAlgorithm::Lloyd; // the bound is Lloyd-only CATCH_REQUIRE_THROWS(gem::train_codebook(dense_corpus, parameters, threadpool)); } @@ -431,9 +435,9 @@ CATCH_TEST_CASE("GEM Codebook - failure paths", "[quantization][gem]") { } CATCH_TEST_CASE("GEM Codebook - k-means algorithm selection", "[quantization][gem]") { - CATCH_SECTION("default CodebookParameters selects Lloyd") { + CATCH_SECTION("default CodebookParameters selects EMeans") { CATCH_REQUIRE( - gem::CodebookParameters{}.kmeans_algorithm == gem::KMeansAlgorithm::Lloyd + gem::CodebookParameters{}.kmeans_algorithm == gem::KMeansAlgorithm::EMeans ); } From a8543efbcd0e1af3adb7bb73cfc2b9bb9eff7a2c Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 19 Nov 2026 03:30:29 -0800 Subject: [PATCH 67/68] test(vamana): check that a one-thread static build is deterministic Two one-thread builds of data/test_dataset (L2, R 16, window 32) must give identical adjacency lists. With more threads, back-edge lock order changes the graph, so this is the strongest determinism the static build offers. It also records how 5c650da3 (one back-edge prune per work unit, largest first) was shown not to change the graph, beyond matching statistics: - With one thread the generation phase has no lock-order effect, so the bucket schedule and the per-node schedule must agree byte for byte. qEMD, 1,000 docs of the 50k subset, R 24, L 80, alpha 1.2, unrouted: 705115ba against 705115ba plus the 5c650da3 vamana_build.h, and against HEAD, give graph files that differ only in the 16-byte file UUID. - L2 on data/test_dataset via utils/build_index (window 64, R 32, alpha 1.2, one thread): 705115ba, HEAD and a second HEAD run likewise differ only in the UUID. - ThreadSanitizer on a multi-threaded static build at HEAD (2,000 to 200,000 points, degree 2 to 32, 8 to 112 threads): no reports, in the prune phase or elsewhere. TSan sees only races unordered by happens-before, and the per-vertex locks order most pairs, so this is supporting evidence; the one-thread identity carries the proof. - [vamana] (49 cases) and [integration][build][vamana] pass at HEAD. Co-Authored-By: Claude Opus 5.5 --- tests/svs/index/vamana/vamana_build.cpp | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/svs/index/vamana/vamana_build.cpp b/tests/svs/index/vamana/vamana_build.cpp index 6935e3cb3..2d2a4bf83 100644 --- a/tests/svs/index/vamana/vamana_build.cpp +++ b/tests/svs/index/vamana/vamana_build.cpp @@ -19,13 +19,23 @@ #include "svs/index/vamana/vamana_build.h" // svs +#include "svs/core/data/simple.h" +#include "svs/core/distance.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/extensions.h" +#include "svs/lib/narrow.h" #include "svs/lib/threads.h" // catch2 #include "catch2/catch_test_macros.hpp" +// tests +#include "tests/utils/test_dataset.h" + // stl +#include #include +#include #include #include @@ -142,3 +152,43 @@ CATCH_TEST_CASE("Index Build Utilities", "[vamana][vamana_build]") { } } } + +CATCH_TEST_CASE( + "Static Vamana build is deterministic with one thread", "[vamana][vamana_build]" +) { + // With more threads, back-edge lock order changes the resulting graph. + auto data = svs::data::SimpleData::load(test_dataset::data_svs_file()); + auto params = vamana::VamanaBuildParameters{1.2F, 16, 32, 750, 60, true}; + auto pool = svs::threads::DefaultThreadPool(1); + auto entry_point = + svs::lib::narrow(vamana::extensions::compute_entry_point(data, pool)); + + auto build = [&]() { + auto graph = svs::default_graph(data.size(), params.graph_max_degree); + auto builder = vamana::VamanaBuilder( + graph, + data, + svs::distance::DistanceL2{}, + params, + pool, + vamana::extensions::estimate_prefetch_parameters(data) + ); + builder.construct(1.0F, entry_point); + builder.construct(params.alpha, entry_point); + return graph; + }; + + auto graph_a = build(); + auto graph_b = build(); + + CATCH_REQUIRE(graph_a.n_nodes() == graph_b.n_nodes()); + std::optional first_mismatch; + for (size_t i = 0; i < graph_a.n_nodes(); ++i) { + if (!std::ranges::equal(graph_a.get_node(i), graph_b.get_node(i))) { + first_mismatch = i; + break; + } + } + CATCH_INFO("first mismatching node: " << first_mismatch.value_or(0)); + CATCH_REQUIRE(!first_mismatch.has_value()); +} From 8593a4dd641fbe8732525e1cf6479af66bf58ab1 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 19 Nov 2026 03:37:41 -0800 Subject: [PATCH 68/68] feat(distance): select the transport-simplex start at run time Adds enum class TransportStart { NorthWest, CheapestRow }. QEmd takes it as an optional third constructor argument (default NorthWest) and passes it through QEmdImpl::compute / compute_decoded to TransportSimplex::solve. The choice is not saved with an index: it changes how fast a solve reaches the optimum, not the optimum. NorthWest walks columns in id order and skips the column-min fold, so it reproduces the solver before 8a648b2a. This changes the default: CheapestRow, hard-wired since 8a648b2a, is now opt-in. Both starts share one strong-feasibility argument, since the walk reasons only about positions and rows keep their order. The random cross-check, including its start-degeneracy section, runs under both starts. On 200k qEMD solves (50k subset, K 16384), pivots per (m + k - 1) and checksums equal 705115ba's for NorthWest and 5c650da3's for CheapestRow on all four datasets. At 1 thread, NorthWest is within 1% of 705115ba, and CheapestRow is 3-5% faster than NorthWest. Co-Authored-By: Claude Opus 5.5 --- include/svs/core/distance/gem.h | 34 ++++-- include/svs/core/distance/transport_simplex.h | 101 ++++++++++++------ tests/svs/core/distances/gem.cpp | 73 +++++++++---- 3 files changed, 146 insertions(+), 62 deletions(-) diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h index a01a1fa63..2971c6733 100644 --- a/include/svs/core/distance/gem.h +++ b/include/svs/core/distance/gem.h @@ -396,7 +396,8 @@ template struct QEmdImpl { size_t ua, const uint32_t* ids_b, const double* w_b, - size_t ub + size_t ub, + TransportStart start ) { if (ua == 1 && ub == 1) { return table(ids_a[0], ids_b[0]); @@ -410,11 +411,11 @@ template struct QEmdImpl { // ub; swapping keeps the row (ids_b) fixed there. If that transpose rule // changes, results stay correct but this locality win is lost. auto cost_fn = [&](size_t i, size_t j) { return table(ids_b[j], ids_a[i]); }; - double cost = TransportSimplex::solve(ua, w_a, ub, w_b, cost_fn); + double cost = TransportSimplex::solve(ua, w_a, ub, w_b, cost_fn, start); return static_cast(cost); } auto cost_fn = [&](size_t i, size_t j) { return table(ids_a[i], ids_b[j]); }; - double cost = TransportSimplex::solve(ua, w_a, ub, w_b, cost_fn); + double cost = TransportSimplex::solve(ua, w_a, ub, w_b, cost_fn, start); return static_cast(cost); } @@ -425,7 +426,8 @@ template struct QEmdImpl { const uint32_t* ids_a, size_t na, const uint32_t* ids_b, - size_t nb + size_t nb, + TransportStart start ) { thread_local gem_detail::MergeScratch scratch_a; thread_local gem_detail::MergeScratch scratch_b; @@ -438,7 +440,8 @@ template struct QEmdImpl { ua, scratch_b.ids.data(), scratch_b.weights.data(), - ub + ub, + start ); } @@ -557,13 +560,19 @@ class QEmd { explicit QEmd( std::shared_ptr table, - std::shared_ptr router = nullptr + std::shared_ptr router = nullptr, + TransportStart start = TransportStart::NorthWest ) : table_{std::move(table)} - , router_{std::move(router)} {} + , router_{std::move(router)} + , start_{start} {} const std::shared_ptr& router() const { return router_; } + /// Transport-simplex start used by `compute`. Not saved with an index: it changes the + /// pivot count, not the optimum. + TransportStart start() const { return start_; } + // Vector-set path: ids come straight from a contiguous vector set's own storage; T == // uint32_t skips the cast copy (gem_detail::id_span), any other unsigned type costs it. template @@ -580,18 +589,23 @@ class QEmd { size_t nb = b.cardinality(); if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { return QEmdImpl::compute( - *table_, ids_a, na, ids_b, nb + *table_, ids_a, na, ids_b, nb, start_ ); } if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { - return QEmdImpl::compute(*table_, ids_a, na, ids_b, nb); + return QEmdImpl::compute( + *table_, ids_a, na, ids_b, nb, start_ + ); } - return QEmdImpl::compute(*table_, ids_a, na, ids_b, nb); + return QEmdImpl::compute( + *table_, ids_a, na, ids_b, nb, start_ + ); } private: std::shared_ptr table_; std::shared_ptr router_; + TransportStart start_; }; inline constexpr bool operator==(const QEmd&, const QEmd&) { return true; } diff --git a/include/svs/core/distance/transport_simplex.h b/include/svs/core/distance/transport_simplex.h index 6c28daaf9..7815213c2 100644 --- a/include/svs/core/distance/transport_simplex.h +++ b/include/svs/core/distance/transport_simplex.h @@ -60,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -84,23 +85,29 @@ template <> struct TransportIndexTraits<256> { namespace svs::distance { +/// Column order of `TransportSimplex::solve`'s northwest-corner start. `NorthWest` walks +/// columns in id order. `CheapestRow` groups them by cheapest row first (ties by ascending +/// column id): one extra compare per cost cell for fewer pivots. Both reach the same +/// optimum; only the pivot count differs. +enum class TransportStart { NorthWest, CheapestRow }; + /// Solves the dense balanced transportation problem exactly to within /// `error_bound(ua, ub, max_cost, total_mass)`. Same contract as /// `TransportSimplexReference::solve`, plus: `CostFn` must return `float` (pricing stores /// costs as float, exact for callers whose costs already are, e.g. `CentroidTable`). /// /// Orients to put the smaller side in rows, then starts from a northwest-corner walk over -/// columns grouped by their cheapest row (ties broken by ascending column id), trading a -/// one-pass cost scan for fewer pivots than a cost-blind NW start needs. -/// Strong feasibility means every zero-flow tree arc has a *source* -/// child (points toward the root, LEMON's convention); the only zero-flow arc the start -/// ever adds is a value tie's degenerate arc, and that arc's child is always a fresh, -/// untouched row, never a sink. Every arc whose child is a sink instead carries that -/// sink's own fresh demand (or a fresh row's full supply), so it is always strictly -/// positive. Each pivot picks the leaving arc by Cunningham's rule (the *last* blocking -/// arc on the cycle in the entering arc's own direction), which preserves this invariant -/// under degenerate pivots and so guarantees termination regardless of which -/// negative-reduced-cost arc enters next. +/// columns in the order `TransportStart` picks. Both starts share one argument: the walk +/// reasons only about positions (row i, column position p), never a column's id, and rows +/// keep their order, so the root is row 0 and any column permutation stays strongly +/// feasible. Strong feasibility means every zero-flow tree arc has a *source* child (points +/// toward the root, LEMON's convention); the only zero-flow arc the start ever adds is a +/// value tie's degenerate arc, and that arc's child is always a fresh, untouched row, never +/// a sink. Every arc whose child is a sink instead carries that sink's own fresh demand (or +/// a fresh row's full supply), so it is always strictly positive. Each pivot picks the +/// leaving arc by Cunningham's rule (the *last* blocking arc on the cycle in the entering +/// arc's own direction), which preserves this invariant under degenerate pivots and so +/// guarantees termination regardless of which negative-reduced-cost arc enters next. /// Pricing scans the dense cost matrix in a block-cyclic resumable order in float (`Avx` /// selects the row scan's SIMD width); a float candidate is only pivoted once its reduced /// cost is confirmed negative in double (exactly recoverable, since the cost matrix is @@ -131,17 +138,23 @@ template struct TransportSimplex { #endif template - static double - solve(size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost) + static double solve( + size_t ua, + const double* supply, + size_t ub, + const double* demand, + CostFn&& cost, + TransportStart start = TransportStart::NorthWest + ) requires std::same_as, float> { if (std::max(ua, ub) <= narrow_index_max_side) { return Impl::solve_impl( - ua, supply, ub, demand, std::forward(cost) + ua, supply, ub, demand, std::forward(cost), start ); } return Impl::solve_impl( - ua, supply, ub, demand, std::forward(cost) + ua, supply, ub, demand, std::forward(cost), start ); } @@ -428,7 +441,12 @@ template struct TransportSimplex { public: template static double solve_impl( - size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost + size_t ua, + const double* supply, + size_t ub, + const double* demand, + CostFn&& cost, + TransportStart start ) { if constexpr (MaxSide != svs::Dynamic) { size_t largest = std::max(ua, ub); @@ -469,12 +487,24 @@ template struct TransportSimplex { std::copy_n(ws.demand.begin(), k, ws.demand_marginal.begin()); double max_cost = 0.0; - if (transposed) { - fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + bool cheapest_row = (start == TransportStart::CheapestRow); + if (cheapest_row) { + if (transposed) { + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + } else { + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + } + build_col_order(ws, m, k); } else { - fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + if (transposed) { + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + } else { + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + } + // NorthWest's column order is the identity: the walk below always + // indexes through col_order, so fill it rather than branch the walk. + std::iota(ws.col_order.begin(), ws.col_order.begin() + k, NodeT{0}); } - build_col_order(ws, m, k); for (size_t j = k; j < padded_k; ++j) { ws.v_d[j] = 0.0; ws.v_f[j] = 0.0F; @@ -678,9 +708,9 @@ template struct TransportSimplex { } private: - // Fills cost_f/padding; Transposed avoids a runtime orientation check. Also folds - // each column's cheapest cost/row into col_min/col_argmin while filling. - template + // Fills cost_f/padding; Transposed avoids a runtime orientation check. TrackColMin + // also folds each column's cheapest cost/row into col_min/col_argmin (CheapestRow). + template static void fill_cost_matrix( SolverWorkspace& ws, size_t m, @@ -696,12 +726,14 @@ template struct TransportSimplex { : static_cast(cost(i, j)); row[j] = static_cast(c); max_cost = std::max(max_cost, c); - // Row 0 seeds col_min by assignment rather than a separate +inf - // pass; strict `<` below keeps the smallest row on a tie. - float cf = static_cast(c); - if (i == 0 || cf < ws.col_min[j]) { - ws.col_min[j] = cf; - ws.col_argmin[j] = static_cast(i); + if constexpr (TrackColMin) { + // Row 0 seeds col_min by assignment rather than a separate +inf + // pass; strict `<` below keeps the smallest row on a tie. + float cf = static_cast(c); + if (i == 0 || cf < ws.col_min[j]) { + ws.col_min[j] = cf; + ws.col_argmin[j] = static_cast(i); + } } } for (size_t j = k; j < padded_k; ++j) { @@ -1088,9 +1120,16 @@ template <> struct TransportSimplex { return (2.0 * walk_tol + flow_rounding + final_sum_rounding) * total_mass; } + // The reference solver has no start choice, so the start is ignored. template - static double - solve(size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost) { + static double solve( + size_t ua, + const double* supply, + size_t ub, + const double* demand, + CostFn&& cost, + TransportStart /*start*/ = TransportStart::NorthWest + ) { return TransportSimplexReference::solve( ua, supply, ub, demand, std::forward(cost) ); diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp index 08f15ad62..1645bd7f7 100644 --- a/tests/svs/core/distances/gem.cpp +++ b/tests/svs/core/distances/gem.cpp @@ -433,6 +433,26 @@ CATCH_TEST_CASE("QEmd", "[distance][gem]") { CATCH_REQUIRE(ba == Catch::Approx(ab)); } } + + CATCH_SECTION("start() reports the constructed start; CheapestRow matches the " + "default distance") { + auto table = make_table(Ground::Cos); + QEmd default_dist{table}; + QEmd cheapest_dist{table, nullptr, svs::distance::TransportStart::CheapestRow}; + CATCH_REQUIRE(default_dist.start() == svs::distance::TransportStart::NorthWest); + CATCH_REQUIRE(cheapest_dist.start() == svs::distance::TransportStart::CheapestRow); + + auto a = make_id_set({0, 1}); + auto b = make_id_set({2, 3}); + float default_score = default_dist.compute(a.view(), b.view()); + float cheapest_score = cheapest_dist.compute(a.view(), b.view()); + double gap = svs::distance::TransportSimplex< + svs::distance::AVX_AVAILABILITY::AVX2>::error_bound(2, 2, 2.0, 1.0); + CATCH_REQUIRE( + static_cast(cheapest_score) == + Catch::Approx(static_cast(default_score)).margin(gap + 1e-6) + ); + } } CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][gem]") { @@ -507,7 +527,14 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge double reference = reference_transport_cost(ua, w_a, ub, w_b, cost); float none_result = QEmdImpl::compute_decoded( - table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub + table, + ids_a.data(), + w_a.data(), + ua, + ids_b.data(), + w_b.data(), + ub, + svs::distance::TransportStart::NorthWest ); CATCH_REQUIRE( static_cast(none_result) == Catch::Approx(reference).margin(1e-6) @@ -526,28 +553,32 @@ CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][ge return gap + ulp; }; + for (auto start : + {svs::distance::TransportStart::NorthWest, + svs::distance::TransportStart::CheapestRow}) { #if defined(__x86_64__) - if (svs::detail::avx_runtime_flags.is_avx2_supported()) { - float avx2_result = QEmdImpl::compute_decoded( - table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub - ); - CATCH_REQUIRE( - static_cast(avx2_result) == - Catch::Approx(static_cast(none_result)) - .margin(isa_margin(static_cast(none_result))) - ); - } - if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { - float avx512_result = QEmdImpl::compute_decoded( - table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub - ); - CATCH_REQUIRE( - static_cast(avx512_result) == - Catch::Approx(static_cast(none_result)) - .margin(isa_margin(static_cast(none_result))) - ); - } + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + float avx2_result = QEmdImpl::compute_decoded( + table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub, start + ); + CATCH_REQUIRE( + static_cast(avx2_result) == + Catch::Approx(static_cast(none_result)) + .margin(isa_margin(static_cast(none_result))) + ); + } + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + float avx512_result = QEmdImpl::compute_decoded( + table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub, start + ); + CATCH_REQUIRE( + static_cast(avx512_result) == + Catch::Approx(static_cast(none_result)) + .margin(isa_margin(static_cast(none_result))) + ); + } #endif + } }; CATCH_SECTION("Random problems") {