diff --git a/CMakeLists.txt b/CMakeLists.txt index 810652815..5393416e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,8 +77,8 @@ include("cmake/fmt.cmake") include("cmake/spdlog.cmake") include("cmake/toml.cmake") -# IVF requires Intel(R) MKL support -if(SVS_EXPERIMENTAL_ENABLE_IVF) +# IVF and multi-vector both train k-means codebooks, which need Intel(R) MKL support. +if(SVS_EXPERIMENTAL_ENABLE_IVF OR SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) include("cmake/mkl.cmake") target_compile_options(${SVS_LIB} INTERFACE "-DSVS_HAVE_MKL=1") endif() diff --git a/THIRD-PARTY-PROGRAMS b/THIRD-PARTY-PROGRAMS index 7b1f9557f..57b8aa500 100644 --- a/THIRD-PARTY-PROGRAMS +++ b/THIRD-PARTY-PROGRAMS @@ -217,3 +217,64 @@ THE SOFTWARE. -- NOTE: Third party dependency used by this software -- This software depends on the fmt lib (MIT License), and users must comply to its license: https://raw.githubusercontent.com/fmtlib/fmt/master/LICENSE + +-------------------------------------------------------------------------------- +9. POT (include/svs/core/distance/transport_simplex.h, https://github.com/PythonOT/POT/tree/0.9.5) + +MIT License + +Copyright (c) 2016-2023 POT contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +10. LEMON (include/svs/core/distance/transport_simplex.h, https://lemon.cs.elte.hu) + +LEMON code without an explicit copyright notice is covered by the following +copyright/license. + +Copyright (C) 2003-2012 Egervary Jeno Kombinatorikus Optimalizalasi +Kutatocsoport (Egervary Combinatorial Optimization Research Group, +EGRES). + +=========================================================================== +Boost Software License, Version 1.0 +=========================================================================== + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/cmake/mkl.cmake b/cmake/mkl.cmake index e0ef5c991..b84fd439d 100644 --- a/cmake/mkl.cmake +++ b/cmake/mkl.cmake @@ -45,7 +45,7 @@ if (SVS_EXPERIMENTAL_BUILD_CUSTOM_MKL) NO_DEFAULT_PATH REQUIRED ) - if (SVS_EXPERIMENTAL_ENABLE_IVF) + if (SVS_EXPERIMENTAL_ENABLE_IVF OR SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) set(mkl_functions_file ${CMAKE_CURRENT_LIST_DIR}/mkl_functions_ivf) else() set(mkl_functions_file ${CMAKE_CURRENT_LIST_DIR}/mkl_functions) diff --git a/cmake/multi-arch.cmake b/cmake/multi-arch.cmake index aeb81e693..06422f33e 100644 --- a/cmake/multi-arch.cmake +++ b/cmake/multi-arch.cmake @@ -13,21 +13,54 @@ # limitations under the License. set(SVS_X86_SRC_DIR "${PROJECT_SOURCE_DIR}/include/svs/multi-arch/x86") +# Tuple format: source,name,arch[,extra_option,extra_option,...]. Fields beyond the +# first three are optional and additive: an existing tuple (no 4th+ field) still gets +# exactly -march=${arch} -mtune=${arch}, unchanged; a tuple that adds fields gets those +# two flags plus every extra option, appended verbatim. amx.cpp uses this to compile at +# exactly the AVX-512 level's instruction set (arch=cascadelake, matching the avx512 +# tuple below) plus the two AMX extensions, rather than at -march=sapphirerapids: that +# also enables AVX512-FP16/BF16/AVX-VNNI/etc, which the runtime probe +# (amx_tile_bf16_available()) never tests and a hypervisor could expose AMX without, so +# the compiler must not be free to use them when auto-vectorizing ordinary code in that +# file. set(SVS_X86 "${SVS_X86_SRC_DIR}/avx2.cpp,avx2,haswell" "${SVS_X86_SRC_DIR}/avx512.cpp,avx512,cascadelake" ) +# AMX (tile load/store/config plus the bf16 dot-product tile instruction) is optional: a +# compiler that rejects -mamx-tile/-mamx-bf16 (too old, or a non-x86 cross build) must +# still produce a working library with the level compiled out, so amx.cpp is added to +# SVS_X86 only when this check passes, and SVS_AMX_TILE_BF16_SUPPORTED (read by +# include/svs/lib/preprocessor.h) is defined only then. +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-mamx-tile -mamx-bf16" SVS_COMPILER_SUPPORTS_AMX) +if(SVS_COMPILER_SUPPORTS_AMX) + list(APPEND SVS_X86 "${SVS_X86_SRC_DIR}/amx.cpp,amx,cascadelake,-mamx-tile,-mamx-bf16") +endif() + set(SVS_X86_OBJECT_FILES) foreach(x86_info IN LISTS SVS_X86) string(REPLACE "," ";" x86_info "${x86_info}") + list(LENGTH x86_info x86_info_len) list(GET x86_info 0 src) list(GET x86_info 1 avx) list(GET x86_info 2 arch) set(lib_name "svs_x86_${avx}") add_library(${lib_name} INTERFACE) target_compile_options(${lib_name} INTERFACE -march=${arch} -mtune=${arch}) - set(obj_name ${arch}_obj) + # Object target names key off `arch` by default, but the amx tuple above + # intentionally reuses cascadelake's `arch` (same instruction set as the avx512 + # tuple, see the comment above SVS_X86): when extra options are present, the object + # target is named from `avx` instead, so it doesn't collide with that other tuple's + # target of the same name. + if(x86_info_len GREATER 3) + list(SUBLIST x86_info 3 -1 extra_options) + target_compile_options(${lib_name} INTERFACE ${extra_options}) + set(obj_name ${avx}_obj) + else() + set(obj_name ${arch}_obj) + endif() add_library(${obj_name} OBJECT ${src}) target_link_libraries(${obj_name} PRIVATE ${SVS_LIB} svs::compile_options fmt::fmt ${lib_name}) @@ -37,6 +70,14 @@ endforeach() add_library(svs_x86_objects STATIC ${SVS_X86_OBJECT_FILES}) target_link_libraries(svs_export INTERFACE svs_x86_objects) +if(SVS_COMPILER_SUPPORTS_AMX) + # Uniform across every consumer of svs_export, unlike the per-file -march flags above: + # PackedOperand (inner_product_matrix.h) decides whether to fill the AMX tile layout + # from this, and it is included by TUs that never pass -march=sapphirerapids + # themselves (gem.h, maxsim.h, tests). + target_compile_definitions(svs_export INTERFACE SVS_AMX_TILE_BF16_SUPPORTED) +endif() + install( TARGETS svs_x86_objects EXPORT svs-targets diff --git a/cmake/options.cmake b/cmake/options.cmake index bfc3ebf71..908afa0c4 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -110,6 +110,16 @@ option(SVS_EXPERIMENTAL_ENABLE_IVF OFF # disabled by default ) +option(SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR + "Enable multi-vector (late interaction) support. Requires Intel(R) MKL support" + OFF # disabled by default +) + +option(SVS_EXPERIMENTAL_ENABLE_STATS + "Enable build and search measurement counters" + OFF # disabled by default +) + ##### ##### svsbenchmark ##### @@ -163,6 +173,18 @@ else() target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_IVF=0) endif() +if (SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_MULTI_VECTOR=1) +else() + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_MULTI_VECTOR=0) +endif() + +if (SVS_EXPERIMENTAL_ENABLE_STATS) + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_STATS=1) +else() + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_STATS=0) +endif() + ##### ##### Helper target to apply relevant compiler optimizations. ##### diff --git a/include/svs/concepts/vector_set.h b/include/svs/concepts/vector_set.h new file mode 100644 index 000000000..3f94fa42a --- /dev/null +++ b/include/svs/concepts/vector_set.h @@ -0,0 +1,109 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/// +/// @ingroup concepts +/// @defgroup vector_set_concept_entry Concepts modeling late-interaction vector-set +/// datasets. +/// + +#include +#include +#include + +namespace svs { +namespace data { + +// clang-format off + +/// +/// @ingroup vector_set_concept_entry +/// @brief A variable-cardinality set of token vectors, as used by late-interaction +/// retrieval (e.g. one set per document, one token vector per word-piece embedding). +/// +/// @code{.cpp} +/// template +/// concept VectorSet = requires(const S& s, size_t i) { +/// // The element type of each token vector. +/// typename S::element_type; +/// +/// // The number of token vectors in the set. +/// { s.cardinality() } -> std::convertible_to; +/// +/// // The dimensionality shared by every token vector in the set. +/// { s.dimensions() } -> std::convertible_to; +/// +/// // The token vector at index ``i``, with ``0 <= i < cardinality()``. +/// { s.token(i) } -> std::convertible_to>; +/// }; +/// @endcode +/// +template +concept VectorSet = requires(const S& s, size_t i) { + typename S::element_type; + { s.cardinality() } -> std::convertible_to; + { s.dimensions() } -> std::convertible_to; + { s.token(i) } -> std::convertible_to>; +}; + +/// +/// @ingroup vector_set_concept_entry +/// @brief Compatibility interface for a dataset whose elements are vector sets. +/// +/// @code{.cpp} +/// template +/// concept ImmutableVectorSetDataset = requires(const D& d, size_t i) { +/// // The value returned by ``get_datum`` must itself model ``VectorSet``. +/// requires VectorSet; +/// +/// // The number of vector sets in the dataset. +/// { d.size() } -> std::convertible_to; +/// +/// // The dimensionality shared by every token vector in the dataset. +/// { d.dimensions() } -> std::convertible_to; +/// +/// // A constant handle to the vector set at index ``i``. +/// { d.get_datum(i) } -> std::convertible_to; +/// +/// // Performance optimization. Prefetch the vector set at index ``i``. +/// d.prefetch(i); +/// }; +/// @endcode +/// +template +concept ImmutableVectorSetDataset = requires(const D& d, size_t i) { + requires VectorSet; + { d.size() } -> std::convertible_to; + { d.dimensions() } -> std::convertible_to; + { d.get_datum(i) } -> std::convertible_to; + d.prefetch(i); +}; + +/// +/// @ingroup vector_set_concept_entry +/// @brief A ``VectorSet`` whose tokens are stored row-major and contiguous, reachable +/// from ``data()``. +/// +template +concept ContiguousVectorSet = VectorSet && requires(const S& s) { + { s.data() } -> std::convertible_to; +}; +// clang-format on + +} // namespace data +} // namespace svs diff --git a/include/svs/core/data/vector_set.h b/include/svs/core/data/vector_set.h new file mode 100644 index 000000000..85704df5f --- /dev/null +++ b/include/svs/core/data/vector_set.h @@ -0,0 +1,335 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/data.h" +#include "svs/concepts/vector_set.h" +#include "svs/core/data/simple.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/lib/exception.h" +#include "svs/lib/file.h" +#include "svs/lib/memory.h" +#include "svs/lib/narrow.h" +#include "svs/lib/saveload.h" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs { +namespace data { + +/// +/// @brief Owning dataset whose elements are vector sets: documents made of token +/// vectors, stored CSR-style exactly as ``ConstVectorSetView`` reads them. +/// +/// The token matrix (every token vector of every document, concatenated row-major) is +/// held in a ``SimpleData``, so SVS's allocators (including the +/// hugepage allocator), alignment, and binary file format are reused as-is. A +/// ``std::vector`` of length ``size() + 1`` holds the CSR offsets, counted in +/// token vectors. +/// +/// @tparam T The element type of each token vector. +/// @tparam Extent The compile-time dimensionality shared by every token vector, or +/// ``svs::Dynamic`` to resolve it at run time. Default: ``svs::Dynamic``. +/// @tparam Alloc The allocator used for the token matrix. Must not be a ``Blocked`` +/// allocator; see the ``static_assert`` below. +/// +template > +class VectorSetData { + // A document's tokens must land in one contiguous run (`get_datum` returns a single + // span into the token matrix); the Blocked allocator splits storage into separate + // blocks, which would tear a document's tokens across blocks. + static_assert( + !is_blocked_v, + "VectorSetData requires contiguous per-document token storage; a Blocked " + "allocator is not supported." + ); + + public: + /// The element type of each token vector. + using element_type = T; + /// The type returned by ``get_datum``: a constant handle to one document's tokens. + using value_type = ConstVectorSet; + /// @copydoc value_type + using const_value_type = ConstVectorSet; + /// The allocator type backing the token matrix. + using allocator_type = Alloc; + /// The concrete type of the token-matrix member. + using tokens_type = SimpleData; + // VamanaIndex reads Data::extent in its class body (index/vamana/index.h); without + // it this type cannot be used as an index's dataset. + static constexpr size_t extent = Extent; + + static constexpr lib::Version save_version = lib::Version(0, 0, 0); + static constexpr std::string_view serialization_schema = "vector_set_data"; + + /// + /// @brief Take ownership of an already-built token matrix and offsets array. + /// + /// @param tokens The row-major token matrix: ``total_tokens`` rows (the sum of every + /// document's cardinality) by ``dimensions()`` columns, where ``total_tokens`` + /// is derived from ``offsets.back()``. + /// @param offsets CSR-style offsets counted in token vectors: length + /// ``size() + 1``, non-decreasing, ``offsets.front() == 0``, and + /// ``offsets.back()`` equal to the number of rows in ``tokens``. + /// + /// Both arguments are consumed by move into this container; no payload copy is + /// made. Pass ``std::move(tokens)`` and ``std::move(offsets)`` to hand over a + /// caller's own storage. A caller that wants to keep an independent copy must copy + /// explicitly before calling this constructor. + /// + VectorSetData(tokens_type tokens, std::vector offsets) + : tokens_{std::move(tokens)} + , offsets_{std::move(offsets)} { + detail::validate_vector_set_layout( + offsets_, tokens_.size() * tokens_.dimensions(), tokens_.dimensions() + ); + } + + /// + /// @brief Copy a borrowed view into a new, owned ``VectorSetData``. + /// + /// @param view The borrowed vector-set view to copy from. + /// @param allocator The allocator used for the new token matrix. Default: + /// a default-constructed ``Alloc``. + /// + /// For the common case where a caller only has third-party, borrowed memory and + /// wants SVS to own a copy. The copy is performed document by document, not in + /// parallel; callers with large data who can hand over ownership instead should + /// prefer the move constructor above. + /// + static VectorSetData + copy_from(ConstVectorSetView view, const Alloc& allocator = {}) { + size_t dims = view.dimensions(); + auto tokens = tokens_type(view.total_tokens(), dims, allocator); + auto offsets = std::vector(view.size() + 1); + size_t running = 0; + for (size_t i = 0; i < view.size(); ++i) { + offsets[i] = running; + auto set = view.get_datum(i); + for (size_t j = 0, jmax = set.cardinality(); j < jmax; ++j) { + auto src = set.token(j); + auto dst = tokens.get_datum(running + j); + std::copy(src.begin(), src.end(), dst.begin()); + } + running += set.cardinality(); + } + offsets[view.size()] = running; + return VectorSetData(std::move(tokens), std::move(offsets)); + } + + /// Return the number of documents (vector sets) in the dataset. + size_t size() const { return offsets_.size() - 1; } + /// Return the dimensionality shared by every token vector. + size_t dimensions() const { return tokens_.dimensions(); } + /// Return the number of token vectors in document ``i``. + /// + /// **Preconditions:** ``0 <= i < size()``. + size_t cardinality(size_t i) const { + assert(i < size()); + return lib::narrow(offsets_[i + 1] - offsets_[i]); + } + /// Return the total number of token vectors across every document. + size_t total_tokens() const { return tokens_.size(); } + + /// + /// @brief Return a constant handle to the token vectors of document ``i``. + /// + /// **Preconditions:** ``0 <= i < size()``. + /// + const_value_type get_datum(size_t i) const { + size_t set_cardinality = cardinality(i); + size_t begin = lib::narrow(offsets_[i]); + size_t dims = dimensions(); + return ConstVectorSet( + std::span(tokens_.data() + begin * dims, set_cardinality * dims), + set_cardinality, + dims + ); + } + + /// Prefetch the first token vector of document ``i``; see + /// ``ConstVectorSetView::prefetch`` for why only the first token is prefetched. + void prefetch(size_t i) const { + if (cardinality(i) == 0) { + return; + } + tokens_.prefetch(lib::narrow(offsets_[i])); + } + + /// Return the token matrix backing this dataset. + const tokens_type& tokens() const { return tokens_; } + /// Return the CSR offsets backing this dataset, counted in token vectors. + std::span offsets() const { return offsets_; } + + /// + /// @brief Return a non-owning ``ConstVectorSetView`` over this container's storage. + /// + /// The returned view is invalidated if this container is moved from or destroyed. + /// + ConstVectorSetView view() const { + return ConstVectorSetView( + std::span(tokens_.data(), tokens_.size() * dimensions()), + offsets(), + dimensions() + ); + } + + ///// IO + + static bool check_load_compatibility(std::string_view schema, lib::Version version) { + return schema == serialization_schema && version == save_version; + } + + /// + /// @brief Save this dataset to disk. + /// + /// The token matrix is saved by delegating to its own ``save``, nested as a + /// sub-table under the ``"tokens"`` key. The offsets are written to a standalone + /// binary file (raw ``uint64_t``, no header) named through ``ctx`` and referenced by + /// the ``"offsets_file"``/``"num_offsets"`` metadata keys. + /// + lib::SaveTable save(const lib::SaveContext& ctx) const { + auto offsets_path = ctx.generate_name("vector_set_offsets", "bin"); + { + auto io = lib::open_write(offsets_path); + io.write( + reinterpret_cast(offsets_.data()), + lib::narrow(offsets_.size() * sizeof(uint64_t)) + ); + } + return lib::SaveTable( + serialization_schema, + save_version, + {{"tokens", lib::save(tokens_, ctx)}, + {"offsets_file", lib::save(offsets_path.filename())}, + {"num_offsets", lib::save(offsets_.size())}} + ); + } + + /// + /// @brief Reload a previously saved dataset. + /// + /// @param table The table containing saved hyper parameters, as produced by + /// ``save(const lib::SaveContext&)``. + /// @param allocator The allocator used for the reloaded token matrix. Default: a + /// default-constructed ``Alloc``. + /// + /// Validation is performed by the constructor through the same shared layout check + /// used by ``ConstVectorSetView``, so a corrupted offsets file or mismatched token + /// count fails with the same diagnostics as an in-memory construction would. + /// + static VectorSetData load(const lib::LoadTable& table, const Alloc& allocator = {}) { + auto tokens = lib::load_at(table, "tokens", allocator); + size_t num_offsets = lib::load_at(table, "num_offsets"); + auto offsets_path = table.resolve_at("offsets_file"); + + size_t expected_bytes = num_offsets * sizeof(uint64_t); + size_t actual_bytes = std::filesystem::file_size(offsets_path); + if (actual_bytes != expected_bytes) { + throw ANNEXCEPTION( + "Vector set offsets file {} has {} bytes but {} offsets ({} bytes) were " + "expected", + offsets_path, + actual_bytes, + num_offsets, + expected_bytes + ); + } + + auto offsets = std::vector(num_offsets); + { + auto io = lib::open_read(offsets_path); + io.read( + reinterpret_cast(offsets.data()), + lib::narrow(expected_bytes) + ); + } + return VectorSetData(std::move(tokens), std::move(offsets)); + } + + /// + /// @brief Reload a dataset previously saved to ``path`` via + /// ``lib::save_to_disk`` or ``VamanaIndex::save``. + /// + /// @param path The directory the dataset was saved into. + /// @param allocator The allocator used for the reloaded token matrix. Default: a + /// default-constructed ``Alloc``. + /// + static VectorSetData + load(const std::filesystem::path& path, const Alloc& allocator = {}) { + return lib::load_from_disk(path, allocator); + } + + private: + tokens_type tokens_; + std::vector offsets_; +}; + +static_assert(ImmutableMemoryDataset>); +static_assert(ImmutableVectorSetDataset>); + +} // namespace data + +/// +/// @brief Loader for owning vector-set datasets, mirroring ``svs::VectorDataLoader``. +/// +/// @tparam T The element type of each token vector. +/// @tparam Extent The compile-time dimensionality of the token vectors. Default: +/// ``svs::Dynamic``. +/// @tparam Allocator The allocator to use for the memory backing the data when loaded. +/// Default: ``HugepageAllocator``. +/// +/// Satisfies the ``f.load()`` shape ``svs::detail::dispatch_load`` looks for, so an +/// instance may be passed anywhere a Vamana ``DataProto`` is expected, e.g. to +/// ``svs::index::vamana::auto_assemble``. +/// +template > +class VectorSetDataLoader { + public: + /// @brief The full type of the loaded dataset. + using return_type = data::VectorSetData; + + /// + /// @brief Construct a new VectorSetDataLoader. + /// + /// @param path The directory previously used to save a ``VectorSetData`` (via + /// ``lib::save_to_disk`` or ``VamanaIndex::save``). + /// @param allocator The allocator to use when loading. Default: a + /// default-constructed ``Allocator``. + /// + VectorSetDataLoader(const std::filesystem::path& path, const Allocator& allocator = {}) + : path_{path} + , allocator_{allocator} {} + + /// @brief Load the dataset from disk. + return_type load() const { return return_type::load(path_, allocator_); } + + ///// Members + std::filesystem::path path_ = {}; + Allocator allocator_ = {}; +}; + +} // namespace svs diff --git a/include/svs/core/data/vector_set_view.h b/include/svs/core/data/vector_set_view.h new file mode 100644 index 000000000..9daeace62 --- /dev/null +++ b/include/svs/core/data/vector_set_view.h @@ -0,0 +1,208 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/data.h" +#include "svs/concepts/vector_set.h" +#include "svs/lib/exception.h" +#include "svs/lib/misc.h" +#include "svs/lib/narrow.h" +#include "svs/lib/prefetch.h" + +// stl +#include +#include +#include +#include +#include + +namespace svs { +namespace data { + +namespace detail { + +/// +/// @brief Validate a CSR-style vector-set offsets array against the token storage it +/// indexes into. +/// +/// Shared by ``ConstVectorSetView`` and ``VectorSetData`` so the validation rules for the +/// two cannot drift apart: ``offsets`` must be non-empty, start at 0, be non-decreasing, +/// and its last entry (the total token count) must satisfy +/// ``offsets.back() * dimensions == token_elements`` without overflowing ``size_t``. If +/// ``Extent`` is static, ``dimensions`` must equal it. +/// +template +void validate_vector_set_layout( + std::span offsets, size_t token_elements, size_t dimensions +) { + if (offsets.empty()) { + throw ANNEXCEPTION("Vector set offsets must be non-empty, got an empty span"); + } + if (offsets.front() != 0) { + throw ANNEXCEPTION("Vector set offsets must start at 0, got {}", offsets.front()); + } + for (size_t i = 1; i < offsets.size(); ++i) { + if (offsets[i] < offsets[i - 1]) { + throw ANNEXCEPTION( + "Vector set offsets must be non-decreasing, got {} after {}", + offsets[i], + offsets[i - 1] + ); + } + } + size_t total_tokens = lib::narrow(offsets.back()); + if (dimensions != 0 && total_tokens > std::numeric_limits::max() / dimensions) { + throw ANNEXCEPTION( + "Token count {} times dimensions {} overflows size_t!", total_tokens, dimensions + ); + } + size_t expected_tokens = total_tokens * dimensions; + if (expected_tokens != token_elements) { + throw ANNEXCEPTION( + "Vector set offsets imply {} token elements but got {} token elements", + expected_tokens, + token_elements + ); + } + if constexpr (Extent != Dynamic) { + if (dimensions != Extent) { + throw ANNEXCEPTION( + "Vector set view has static extent {} but got dimension {}", + Extent, + dimensions + ); + } + } +} + +} // namespace detail + +/// +/// @brief Non-owning view of one vector set: a row-major, contiguous run of token +/// vectors sharing a dimensionality. +/// +template class ConstVectorSet { + public: + using element_type = T; + + ConstVectorSet(std::span data, size_t cardinality, size_t dimensions) + : data_{data} + , cardinality_{cardinality} + , dimensions_{dimensions} {} + + const T* data() const { return data_.data(); } + size_t cardinality() const { return cardinality_; } + constexpr size_t dimensions() const { + if constexpr (Extent != Dynamic) { + return Extent; + } else { + return dimensions_; + } + } + bool empty() const { return cardinality_ == 0; } + + std::span token(size_t i) const { + assert(i < cardinality_); + const T* base = data_.data() + i * dimensions(); + if constexpr (Extent != Dynamic) { + return std::span(base, Extent); + } else { + return std::span(base, dimensions_); + } + } + + private: + std::span data_; + size_t cardinality_; + size_t dimensions_; +}; + +static_assert(VectorSet>); +static_assert(ContiguousVectorSet>); + +/// +/// @brief Non-owning CSR-style view of a dataset whose elements are vector sets. +/// +/// ``offsets`` counts token vectors (not elements), has length ``size() + 1``, is +/// non-decreasing, and ``offsets[0] == 0``; ``offsets[i + 1] - offsets[i]`` is the +/// cardinality of set ``i``. +/// +template class ConstVectorSetView { + public: + using element_type = T; + using value_type = ConstVectorSet; + using const_value_type = ConstVectorSet; + // VamanaIndex reads Data::extent in its class body (index/vamana/index.h); without + // it this view cannot be used as an index's dataset. + static constexpr size_t extent = Extent; + + ConstVectorSetView( + std::span tokens, std::span offsets, size_t dimensions + ) + : tokens_{tokens} + , offsets_{offsets} + , dimensions_{dimensions} { + detail::validate_vector_set_layout(offsets_, tokens_.size(), dimensions_); + } + + size_t size() const { return offsets_.size() - 1; } + constexpr size_t dimensions() const { + if constexpr (Extent != Dynamic) { + return Extent; + } else { + return dimensions_; + } + } + size_t cardinality(size_t i) const { + assert(i < size()); + return lib::narrow(offsets_[i + 1] - offsets_[i]); + } + size_t total_tokens() const { return lib::narrow(offsets_.back()); } + + const_value_type get_datum(size_t i) const { + size_t set_cardinality = cardinality(i); + size_t begin = lib::narrow(offsets_[i]) * dimensions(); + size_t count = set_cardinality * dimensions(); + return ConstVectorSet( + tokens_.subspan(begin, count), set_cardinality, dimensions() + ); + } + + // Only the first token vector of the set is close enough to be worth prefetching; + // a large set still has to stream the rest from memory. + void prefetch(size_t i) const { + if (cardinality(i) == 0) { + return; + } + size_t begin = lib::narrow(offsets_[i]) * dimensions(); + lib::prefetch(tokens_.subspan(begin, dimensions())); + } + + private: + std::span tokens_; + std::span offsets_; + size_t dimensions_; +}; + +static_assert(ImmutableVectorSetDataset>); +// Vamana build and batch search take the view as dataset and as query set through this +// concept. +static_assert(ImmutableMemoryDataset>); + +} // namespace data +} // namespace svs diff --git a/include/svs/core/data/with_vector_sets.h b/include/svs/core/data/with_vector_sets.h new file mode 100644 index 000000000..4eeddbb9d --- /dev/null +++ b/include/svs/core/data/with_vector_sets.h @@ -0,0 +1,123 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/vector_set.h" +#include "svs/lib/exception.h" +#include "svs/lib/saveload.h" + +// stl +#include +#include +#include +#include + +namespace svs::data { + +/// +/// @brief Pairs a primary dataset (one row per entry, used for graph traversal) with a +/// vector-set dataset (one token-vector set per entry, used for re-ranking). +/// +/// Models ``ImmutableMemoryDataset`` by forwarding every member to ``primary()``, so +/// graph search over a ``WithVectorSets`` is identical to graph search over the primary +/// dataset alone. ``get_vector_set(i)`` is the extra accessor a Vamana re-rank extension +/// uses after the graph search has produced a candidate pool. +/// +template class WithVectorSets { + public: + using value_type = typename Primary::value_type; + using const_value_type = typename Primary::const_value_type; + using element_type = typename Primary::element_type; + + // VamanaIndex reads `Data::extent` unconditionally, outside the ImmutableMemoryDataset + // concept; omitting this breaks every VamanaIndex instantiation over this type. + static constexpr size_t extent = Primary::extent; + + WithVectorSets(Primary primary, Sets sets) + : primary_{std::move(primary)} + , sets_{std::move(sets)} { + if (primary_.size() != sets_.size()) { + throw ANNEXCEPTION( + "Primary dataset has {} entries but vector-set dataset has {} entries", + primary_.size(), + sets_.size() + ); + } + } + + size_t size() const { return primary_.size(); } + size_t dimensions() const { return primary_.dimensions(); } + const_value_type get_datum(size_t i) const { return primary_.get_datum(i); } + void prefetch(size_t i) const { primary_.prefetch(i); } + + const Primary& primary() const { return primary_; } + const Sets& vector_sets() const { return sets_; } + + /// The candidate's token-vector set, for re-ranking after graph search. + typename Sets::const_value_type get_vector_set(size_t i) const { + return sets_.get_datum(i); + } + + ///// IO + // + // Available only when both `Primary` and `Sets` are themselves saveable/loadable; + // a non-owning `Sets` (e.g. `ConstVectorSetView`) lacks `save`/`load`, and the + // requires-clauses below make that a silent non-participant rather than a hard + // error, so `WithVectorSets` over a view still compiles as a dataset member. + + static constexpr lib::Version save_version = lib::Version(0, 0, 0); + static constexpr std::string_view serialization_schema = "with_vector_sets"; + + static bool check_load_compatibility(std::string_view schema, lib::Version version) { + return schema == serialization_schema && version == save_version; + } + + /// @brief Save both members, nested as sub-tables under the ``"primary"`` and + /// ``"vector_sets"`` keys. + lib::SaveTable save(const lib::SaveContext& ctx) const + requires requires(const Primary& p, const Sets& s, const lib::SaveContext& c) { + p.save(c); + s.save(c); + } + { + return lib::SaveTable( + serialization_schema, + save_version, + {{"primary", lib::save(primary_, ctx)}, {"vector_sets", lib::save(sets_, ctx)}} + ); + } + + /// @brief Reload both members with their default-constructed allocators. + static WithVectorSets load(const lib::LoadTable& table) + requires requires(const lib::LoadTable& t) { + { Primary::load(t) } -> std::same_as; + { Sets::load(t) } -> std::same_as; + } + { + return WithVectorSets( + lib::load_at(table, "primary"), + lib::load_at(table, "vector_sets") + ); + } + + private: + Primary primary_; + Sets sets_; +}; + +} // namespace svs::data diff --git a/include/svs/core/distance/distance_core.h b/include/svs/core/distance/distance_core.h index 4f59f9de0..5022991bd 100644 --- a/include/svs/core/distance/distance_core.h +++ b/include/svs/core/distance/distance_core.h @@ -26,7 +26,12 @@ namespace svs::distance { -enum class AVX_AVAILABILITY { NONE, AVX2, AVX512 }; +// Explicit values, not contiguous: nothing in this codebase reads AVX_AVAILABILITY's +// ordinal or compares it, only switches/templates on the enumerator itself (checked by +// grep before adding AMX), so the gap between AVX512 and AMX is free to reserve for a +// parallel split of AVX512 into two enumerators (e.g. AVX512F, AVX512VNNI) without +// renumbering AMX. +enum class AVX_AVAILABILITY { NONE = 0, AVX2 = 1, AVX512 = 2, AMX = 10 }; constexpr std::array supported_dim_list{ 64, 96, 100, 128, 160, 200, 512, 768, svs::Dynamic}; diff --git a/include/svs/core/distance/gem.h b/include/svs/core/distance/gem.h new file mode 100644 index 000000000..2971c6733 --- /dev/null +++ b/include/svs/core/distance/gem.h @@ -0,0 +1,786 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/concepts/vector_set.h" +#include "svs/core/data/simple.h" +#include "svs/core/distance/distance_core.h" +#include "svs/core/distance/gem_router.h" +#include "svs/core/distance/inner_product_matrix.h" +#include "svs/core/distance/transport_simplex.h" +#include "svs/lib/avx_detection.h" +#include "svs/lib/bfloat16.h" +#include "svs/lib/misc.h" +#include "svs/lib/preprocessor.h" +#include "svs/lib/threads.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__x86_64__) +#include +#endif + +namespace svs::distance { + +/// Ground distance between two unit centroids the from_centroids() table is built from: +/// Cos is `max(0, 1 - )` (GEM Eq. 11), L2 is `sqrt(max(0, 2 - 2))`. +enum class Ground { Cos, L2 }; + +inline Ground parse_ground(std::string_view s) { + if (s == "cos") { + return Ground::Cos; + } + if (s == "l2") { + return Ground::L2; + } + throw std::invalid_argument( + "parse_ground: unsupported ground distance '" + std::string(s) + + "'; expected cos or l2" + ); +} + +/// Owns a dense K x K matrix of pairwise centroid distances, row-major. Built either by +/// loading a precomputed file or in-process from centroids (see from_centroids()). +/// Backed by the hugepage allocator (see `svs::HugepageAllocator`, used the same way by +/// `svs::data::VectorSetData` and `svs::graphs::SimpleGraph`): random lookups into a +/// K=16384 table (1 GiB) are TLB/cache-miss bound, and hugepages shrink the TLB footprint. +struct CentroidTable { + size_t K; + svs::data::SimpleData> data; + // True only for a table from_centroids() built; see the comment where it is set. + bool symmetric = false; + + CentroidTable(size_t k, std::vector table) + : K{k} + , data{k, k, svs::HugepageAllocator()} { + if (table.size() != K * K) { + throw std::invalid_argument("CentroidTable: data size does not match K*K"); + } + std::copy(table.begin(), table.end(), data.data()); + } + + // The file has no header, so K comes from table.json via the caller. + static CentroidTable load(const std::filesystem::path& path, size_t k) { + auto expected_bytes = k * k * sizeof(float); + std::ifstream in(path, std::ios::binary); + if (!in) { + throw std::runtime_error("CentroidTable::load: cannot open " + path.string()); + } + in.seekg(0, std::ios::end); + auto actual_bytes = static_cast(in.tellg()); + if (actual_bytes != expected_bytes) { + throw std::runtime_error( + "CentroidTable::load: " + path.string() + " has " + + std::to_string(actual_bytes) + " bytes, expected " + + std::to_string(expected_bytes) + " for K=" + std::to_string(k) + ); + } + in.seekg(0, std::ios::beg); + // Uninitialized: the read below fills every element, so zero-filling first is + // wasted bandwidth on a buffer that can be hundreds of GiB. + CentroidTable result{k, uninitialized_tag{}}; + in.read( + reinterpret_cast(result.data.data()), + static_cast(expected_bytes) + ); + if (!in) { + throw std::runtime_error("CentroidTable::load: short read on " + path.string()); + } + return result; + } + + /// Computes T[i][j] = ground(bf16(c_i), bf16(c_j)) from K row-major `dim`-wide + /// centroids, T[i][i] = 0. `centroids` is float32 (the on-disk/in-memory format is + /// unchanged); each tile rounds its own row and column block to bfloat16 (round to + /// nearest, ties to even, matching FineCentroids) before computing, so this table and + /// FineCentroids agree on the same rounded centroid values. Symmetric by + /// construction: each unordered pair {i, j} is computed once, by the tile that owns + /// it, and the same float is stored at T[i][j] and T[j][i] (never two + /// independently-computed dot products); that holds regardless of which operand role + /// bf16-rounding was applied to, since both row and column blocks are rounded + /// identically before either is used. Parallelized over tiles on `num_threads` + /// threads, dynamically scheduled so no two tiles ever write the same cell; the + /// result is bit-for-bit identical regardless of `num_threads`. Uses an uninitialized + /// allocation (see the private constructor below) and writes every cell exactly once + /// (off-diagonal tiles) or twice with the same value (diagonal tiles' upper/lower + /// halves), so no thread does a sequential full-matrix pass. + static CentroidTable from_centroids( + const float* centroids, size_t k, size_t dim, Ground ground, size_t num_threads + ) { + CentroidTable result{k, uninitialized_tag{}}; + float* out = result.data.data(); + svs::threads::DefaultThreadPool pool(num_threads); + + // Tile edge: an operand block (tile * dim * 4 bytes, 128 KiB at dim=128) and the + // tile buffer (tile * tile * 4 bytes, 256 KiB) together stay well inside a + // typical >=1 MiB per-core L2. + constexpr size_t tile = 256; + // Tiles per dynamic-partition steal: a few tiles amortize the atomic-counter + // fetch_add without starving the small tiles near the far column blocks. + constexpr size_t grain = 4; + + size_t num_blocks = svs::lib::div_round_up(k, tile); + + // Upper triangle of blocks, diagonal included: tile (bi, bj), bi <= bj, owns + // every unordered pair whose row falls in block bi and column in block bj. Built + // once so the uneven per-row-block tile count (row block 0 has num_blocks tiles, + // the last has one) can be load-balanced by a dynamic partition over this list + // rather than a static partition over rows. + struct BlockPair { + uint32_t bi; + uint32_t bj; + }; + std::vector tiles; + tiles.reserve(num_blocks * (num_blocks + 1) / 2); + for (uint32_t bi = 0; bi < num_blocks; ++bi) { + for (uint32_t bj = bi; bj < num_blocks; ++bj) { + tiles.push_back({bi, bj}); + } + } + + auto ground_cost = [ground](float dot) { + return ground == Ground::Cos ? std::max(0.0F, 1.0F - dot) + : std::sqrt(std::max(0.0F, 2.0F - 2.0F * dot)); + }; + + svs::threads::parallel_for( + pool, + svs::threads::DynamicPartition{tiles.size(), grain}, + [&](const auto& tile_ids, uint64_t /*tid*/) { + // Reused across this thread's tiles: `buf` holds the tile output, + // `rows_bf16` the row block rounded to bfloat16, `cols_packed` the column + // block packed and rounded by the same bfloat16 rounding. Both operands end + // up rounded to the same precision this table is meant to agree with + // FineCentroids on, so the whole method sees one (bf16-rounded) set of + // centroid values. + thread_local std::vector buf; + thread_local std::vector rows_bf16; + thread_local PackedOperand cols_packed; + buf.resize(tile * tile); + for (size_t t : tile_ids) { + auto [bi, bj] = tiles[t]; + size_t row0 = static_cast(bi) * tile; + size_t col0 = static_cast(bj) * tile; + size_t rows = std::min(tile, k - row0); + size_t cols = std::min(tile, k - col0); + rows_bf16.resize(rows * dim); + for (size_t idx = 0; idx < rows * dim; ++idx) { + rows_bf16[idx] = + svs::round_to_bfloat16(centroids[row0 * dim + idx]); + } + cols_packed.pack_rounded_to_bf16(centroids + col0 * dim, cols, dim); + InnerProductMatrix::compute_packed( + rows_bf16.data(), rows, cols_packed, buf.data() + ); + bool diag = (bi == bj); + for (size_t r = 0; r < rows; ++r) { + size_t i = row0 + r; + // Off-diagonal tiles (row and column blocks disjoint) own every + // (r, c); diagonal tiles start at c == r to skip the lower half, + // already owned by this same tile's (r, c) with roles swapped. + size_t c_start = diag ? r : 0; + for (size_t c = c_start; c < cols; ++c) { + size_t j = col0 + c; + if (diag && c == r) { + out[i * k + j] = 0.0F; + continue; + } + float value = ground_cost(buf[r * cols + c]); + out[i * k + j] = value; + out[j * k + i] = value; + } + } + } + } + ); + // Every unordered pair is written once and copied to both halves above, so + // table(i,j) == table(j,i) bit-for-bit; QEmdImpl's swapped transposed-solve + // lookup (gem.h) is only exact because of that, not a general table property. + result.symmetric = true; + return result; + } + + float operator()(uint32_t i, uint32_t j) const { + return data.data()[static_cast(i) * K + j]; + } + + private: + struct uninitialized_tag {}; + + // SimpleData's allocator-backed ctor default-constructs each element in place; for + // a trivial float that writes nothing, matching make_unique_for_overwrite's intent. + CentroidTable(size_t k, uninitialized_tag) + : K{k} + , data{k, k, svs::HugepageAllocator()} {} +}; + +/// Owns the raw K x dim fine-centroid vectors (row-major, not a pairwise table), as +/// bfloat16: every value is the float32 input rounded to nearest, ties to even (matching +/// `svs::round_to_bfloat16`, not `BFloat16`'s own truncating conversion). One +/// representation, no float32 copy kept once construction/`load` returns, so every ISA +/// level (including a future one that multiplies bf16 natively) computes on identical +/// values. Used by QChamferExactQuery to build a per-query (K x query_tokens) dot table +/// against the exact float query, mirroring GEM's score_fine_centers +/// (src/search/routing.cpp:55-61), which never quantizes the query side. +struct FineCentroids { + size_t K; + size_t dim; + std::unique_ptr data; + + /// Rounds `flat` (K rows of `dim_` float32 each, row-major; the on-disk/in-memory + /// input format is unchanged) to bfloat16, in parallel over row blocks on + /// `num_threads` threads. Peak memory is `flat`'s own K * dim_ * 4 bytes plus this + /// object's K * dim_ * 2 bytes: no second float32 buffer is allocated here. + FineCentroids(size_t k, size_t dim_, const float* flat, size_t num_threads = 1) + : K{k} + , dim{dim_} + , data{std::make_unique_for_overwrite(k * dim_)} { + round_rows(flat, data.get(), k * dim_, num_threads); + } + + // Raw file of K rows of `dim` floats with no header, so K comes from the file size. + // Reads and rounds in row-block chunks rather than slurping the whole file into a + // second float32 buffer: peak memory is this object's K * dim * 2 bytes (the bf16 + // result, allocated once, uninitialized) plus one chunk's + // min(K, kLoadChunkRows) * dim * 4 bytes (4 MiB at dim=128), not the full K * dim + // float32 matrix held alongside the bf16 one. + static FineCentroids + load(const std::filesystem::path& path, size_t dim, size_t num_threads = 1) { + std::ifstream in(path, std::ios::binary); + if (!in) { + throw std::runtime_error("FineCentroids::load: cannot open " + path.string()); + } + in.seekg(0, std::ios::end); + auto bytes = static_cast(in.tellg()); + auto row_bytes = dim * sizeof(float); + if (bytes % row_bytes != 0) { + throw std::runtime_error( + "FineCentroids::load: " + path.string() + " has " + std::to_string(bytes) + + " bytes, not a multiple of dim=" + std::to_string(dim) + " * 4 bytes" + ); + } + auto k = bytes / row_bytes; + in.seekg(0, std::ios::beg); + + FineCentroids result{k, dim, uninitialized_tag{}}; + constexpr size_t kLoadChunkRows = 8192; + std::vector chunk(std::min(k, kLoadChunkRows) * dim); + for (size_t row0 = 0; row0 < k; row0 += kLoadChunkRows) { + size_t rows = std::min(kLoadChunkRows, k - row0); + size_t count = rows * dim; + in.read( + reinterpret_cast(chunk.data()), + static_cast(count * sizeof(float)) + ); + if (!in) { + throw std::runtime_error( + "FineCentroids::load: short read on " + path.string() + ); + } + round_rows(chunk.data(), result.data.get() + row0 * dim, count, num_threads); + } + return result; + } + + private: + struct uninitialized_tag {}; + + // new[] with no parens default-initializes BFloat16 (indeterminate value, matching + // CentroidTable's uninitialized constructor); every element is overwritten by + // round_rows before this object becomes visible to a caller. + FineCentroids(size_t k, size_t dim_, uninitialized_tag) + : K{k} + , dim{dim_} + , data{std::make_unique_for_overwrite(k * dim_)} {} + + // Rounds `count` contiguous float32 values to bfloat16, statically partitioned + // across `num_threads` threads; shared by the constructor and the per-chunk path in + // `load` so there is exactly one place that performs this conversion. + static void + round_rows(const float* flat, svs::BFloat16* out, size_t count, size_t num_threads) { + svs::threads::DefaultThreadPool pool(num_threads); + svs::threads::parallel_for( + pool, + svs::threads::StaticPartition{count}, + [&](const auto& ids, uint64_t /*tid*/) { + for (size_t i : ids) { + out[i] = svs::round_to_bfloat16(flat[i]); + } + } + ); + } +}; + +namespace gem_detail { + +// Thread-local because the functors are shared across threads (implicit_broadcast). +struct MergeScratch { + std::vector ids; + std::vector weights; +}; + +// Casts a vector set's centroid ids to uint32_t with no merge; qCH counts duplicates. +// Only instantiated for element types other than uint32_t (id_span's cast-copy branch). +template +inline void decode_ids(const T* ids, size_t n, std::vector& out) { + out.resize(n); + for (size_t i = 0; i < n; ++i) { + out[i] = static_cast(ids[i]); + } +} + +// Returns a contiguous uint32_t id span for a 1-D id vector set with element type T: its +// own storage when T is already uint32_t (no copy), else a cast copy into `scratch`. +template +inline const uint32_t* id_span(const T* ids, size_t n, std::vector& scratch) { + if constexpr (std::is_same_v) { + return ids; + } else { + decode_ids(ids, n, scratch); + return scratch.data(); + } +} + +// An id vector set stores one centroid id per token, so dimensions() must be 1. A +// concept-constrained vector set has no static Extent to check at compile time, so this +// is the vector-set path's only dimension check. +inline void require_id_dimensions(size_t dims) { + if (dims != 1) { + throw std::invalid_argument( + "id vector set must have dimensions() == 1 (one centroid id per " + "token), got " + + std::to_string(dims) + ); + } +} + +} // namespace gem_detail + +// Instantiated per ISA in svs/multi-arch/x86; without SVS_NOINLINE the caller inlines the +// body at baseline flags. compute_decoded's pricing loop is explicitly vectorized per Avx +// (TransportSimplex); compute_decoded takes already-decoded (deduplicated, weighted) +// centroid ids, keeping the merge and the transportation solve as separately testable +// steps. +template struct QEmdImpl { + SVS_NOINLINE static float compute_decoded( + const CentroidTable& table, + const uint32_t* ids_a, + const double* w_a, + size_t ua, + const uint32_t* ids_b, + const double* w_b, + size_t ub, + TransportStart start + ) { + if (ua == 1 && ub == 1) { + return table(ids_a[0], ids_b[0]); + } + + // merge_sorted_duplicates divides integer counts by n in double, so w_a/w_b each + // sum to 1.0 up to a few ULPs rather than exactly; TransportSimplex::solve folds + // that residual into the last demand entry instead of requiring exact balance. + if (table.symmetric && ua > ub) { + // TransportSimplex transposes and walks ids_a in its inner loop when ua > + // ub; swapping keeps the row (ids_b) fixed there. If that transpose rule + // changes, results stay correct but this locality win is lost. + auto cost_fn = [&](size_t i, size_t j) { return table(ids_b[j], ids_a[i]); }; + double cost = TransportSimplex::solve(ua, w_a, ub, w_b, cost_fn, start); + return static_cast(cost); + } + auto cost_fn = [&](size_t i, size_t j) { return table(ids_a[i], ids_b[j]); }; + double cost = TransportSimplex::solve(ua, w_a, ub, w_b, cost_fn, start); + return static_cast(cost); + } + + // Decodes both ids-sets' own uint32_t storage, merges each set's duplicates into + // weighted unique ids, then solves the transportation problem over the result. + SVS_NOINLINE static float compute( + const CentroidTable& table, + const uint32_t* ids_a, + size_t na, + const uint32_t* ids_b, + size_t nb, + TransportStart start + ) { + thread_local gem_detail::MergeScratch scratch_a; + thread_local gem_detail::MergeScratch scratch_b; + size_t ua = merge_duplicates(ids_a, na, scratch_a); + size_t ub = merge_duplicates(ids_b, nb, scratch_b); + return compute_decoded( + table, + scratch_a.ids.data(), + scratch_a.weights.data(), + ua, + scratch_b.ids.data(), + scratch_b.weights.data(), + ub, + start + ); + } + + private: + // Overloads, not a free function template: QEmdImpl is explicitly instantiated + // per ISA in avx2.cpp/avx512.cpp, and a free template's inline symbol is shared + // (and ODR-violated under different -m flags) across those translation units. + static size_t merge_sorted_duplicates(size_t n, gem_detail::MergeScratch& scratch) { + std::sort(scratch.ids.begin(), scratch.ids.end()); + scratch.weights.clear(); + size_t unique_count = 0; + size_t i = 0; + while (i < n) { + size_t j = i; + while (j < n && scratch.ids[j] == scratch.ids[i]) { + ++j; + } + scratch.ids[unique_count] = scratch.ids[i]; + scratch.weights.push_back(static_cast(j - i) / static_cast(n)); + ++unique_count; + i = j; + } + scratch.ids.resize(unique_count); + return unique_count; + } + + static size_t + merge_duplicates(const uint32_t* ids, size_t n, gem_detail::MergeScratch& scratch) { + scratch.ids.assign(ids, ids + n); + return merge_sorted_duplicates(n, scratch); + } +}; + +// Dispatched like QEmdImpl. compute_decoded takes centroid id arrays already cast to +// uint32_t from the vector set's own storage by QChamfer::compute. +template struct QChamferImpl { + SVS_NOINLINE static float compute_decoded( + const CentroidTable& table, + const uint32_t* q, + size_t nq, + const uint32_t* d, + size_t nd + ) { + double total = 0.0; + for (size_t i = 0; i < nq; ++i) { + float best = std::numeric_limits::infinity(); + for (size_t j = 0; j < nd; ++j) { + best = std::min(best, table(q[i], d[j])); + } + total += best; + } + return static_cast(total); + } +}; + +// GEM never quantizes query tokens: QChamferExactQuery scores them via InnerProductMatrix. +// compute_decoded reduces that table; no intrinsics, but still instantiated per ISA (not +// specialized) like QEmdImpl, so no weak inline symbol is shared across -m flags. +template struct QChamferExactImpl { + // GEM's code_distance (src/distance/maxsim.cpp:9-45): for each query token, the max + // dot product to any of the document's (possibly repeated) centroid ids; final score + // is `1 - mean_q max_doc dot`, i.e. 1 minus the mean of those per-token maxima. + SVS_NOINLINE static float compute_decoded( + const float* table, size_t n_query, const uint32_t* doc_ids, size_t n_doc + ) { + thread_local std::vector maxima; + maxima.assign(n_query, -std::numeric_limits::infinity()); + for (size_t j = 0; j < n_doc; ++j) { + const float* row = table + static_cast(doc_ids[j]) * n_query; + for (size_t i = 0; i < n_query; ++i) { + maxima[i] = std::max(maxima[i], row[i]); + } + } + double total = 0.0; + for (size_t i = 0; i < n_query; ++i) { + total += 1.0 - static_cast(maxima[i]); + } + return static_cast(total / static_cast(n_query)); + } +}; + +#if defined(__x86_64__) +extern template struct QEmdImpl; +extern template struct QEmdImpl; +extern template struct QChamferImpl; +extern template struct QChamferImpl; +extern template struct QChamferExactImpl; +extern template struct QChamferExactImpl; +#endif + +/// Reports which ISA path QEmd/QChamfer dispatch to on this process, for diagnostics. +inline std::string_view gem_dispatch_isa() { + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + return "avx512"; + } + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + return "avx2"; + } + return "none"; +} + +/// Build-time distance: quantized Earth Mover's Distance (qEMD) between two token sets, +/// solved exactly as a balanced transportation problem over merged centroid weights. +/// Smaller is better. +/// +/// Both arguments are id vector sets (`svs::data::ContiguousVectorSet` with an unsigned +/// integer element type and `dimensions() == 1`): one centroid id per token, each id in +/// `[0, table.K)`. Holds a shared, read-only `CentroidTable` the caller owns and keeps +/// alive for the lifetime of this object. +class QEmd { + public: + using compare = std::less<>; + + static constexpr bool implicit_broadcast = true; + static constexpr std::string_view name = "gem_qemd"; + + explicit QEmd( + std::shared_ptr table, + std::shared_ptr router = nullptr, + TransportStart start = TransportStart::NorthWest + ) + : table_{std::move(table)} + , router_{std::move(router)} + , start_{start} {} + + const std::shared_ptr& router() const { return router_; } + + /// Transport-simplex start used by `compute`. Not saved with an index: it changes the + /// pivot count, not the optimum. + TransportStart start() const { return start_; } + + // Vector-set path: ids come straight from a contiguous vector set's own storage; T == + // uint32_t skips the cast copy (gem_detail::id_span), any other unsigned type costs it. + template + requires std::unsigned_integral && + std::unsigned_integral + float compute(const Sa& a, const Sb& b) const { + gem_detail::require_id_dimensions(a.dimensions()); + gem_detail::require_id_dimensions(b.dimensions()); + thread_local std::vector buf_a; + thread_local std::vector buf_b; + const uint32_t* ids_a = gem_detail::id_span(a.data(), a.cardinality(), buf_a); + const uint32_t* ids_b = gem_detail::id_span(b.data(), b.cardinality(), buf_b); + size_t na = a.cardinality(); + size_t nb = b.cardinality(); + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { + return QEmdImpl::compute( + *table_, ids_a, na, ids_b, nb, start_ + ); + } + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { + return QEmdImpl::compute( + *table_, ids_a, na, ids_b, nb, start_ + ); + } + return QEmdImpl::compute( + *table_, ids_a, na, ids_b, nb, start_ + ); + } + + private: + std::shared_ptr table_; + std::shared_ptr router_; + TransportStart start_; +}; + +inline constexpr bool operator==(const QEmd&, const QEmd&) { return true; } + +/// Search-time distance: quantized Chamfer distance (qCH, GEM Eq. 16). For every query +/// token (duplicates counted), takes the min centroid distance to any doc token, and +/// sums. Smaller is better. The query is the left-hand argument. +/// +/// Both arguments are id vector sets (`svs::data::ContiguousVectorSet` with an unsigned +/// integer element type and `dimensions() == 1`): one centroid id per token, each id in +/// `[0, table.K)`. Holds a shared, read-only `CentroidTable` the caller owns and keeps +/// alive for the lifetime of this object. +class QChamfer { + public: + using compare = std::less<>; + static constexpr bool implicit_broadcast = true; + static constexpr std::string_view name = "gem_qchamfer"; + + explicit QChamfer(std::shared_ptr table) + : table_{std::move(table)} {} + + // Vector-set path: compute_decoded's signature already matches an id vector set's + // own (ids pointer, cardinality) storage, so T == uint32_t needs no decode copy. + template + requires std::unsigned_integral && + std::unsigned_integral + float compute(const Sq& query, const Sd& doc) const { + gem_detail::require_id_dimensions(query.dimensions()); + gem_detail::require_id_dimensions(doc.dimensions()); + thread_local std::vector buf_q; + thread_local std::vector buf_d; + const uint32_t* q_ids = + gem_detail::id_span(query.data(), query.cardinality(), buf_q); + const uint32_t* d_ids = gem_detail::id_span(doc.data(), doc.cardinality(), buf_d); + size_t nq = query.cardinality(); + size_t nd = doc.cardinality(); + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { + return QChamferImpl::compute_decoded( + *table_, q_ids, nq, d_ids, nd + ); + } + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { + return QChamferImpl::compute_decoded( + *table_, q_ids, nq, d_ids, nd + ); + } + return QChamferImpl::compute_decoded( + *table_, q_ids, nq, d_ids, nd + ); + } + + private: + std::shared_ptr table_; +}; + +inline constexpr bool operator==(const QChamfer&, const QChamfer&) { return true; } + +/// Search-time distance: GEM's actual in-graph qCH (GEM_REFERENCE.md "search-time distance: +/// is the query quantized?" row). Unlike QChamfer, the query is never id-quantized: it +/// stays as raw float tokens, and fix_argument() builds a dense (K x query_tokens) dot +/// table against every fine centroid once per query (score_fine_centers, src/search/ +/// routing.cpp:55-61), both sides rounded to bfloat16 so the table's values do not depend +/// on which ISA level computed them (see FineCentroids); compute() then reduces that +/// table against the document's quantized centroid ids (code_distance, +/// src/distance/maxsim.cpp:9-45). Smaller is better. +/// +/// Holds a per-query table as ordinary (non-shared) state, so each search thread's +/// shallow_copy of this functor gets its own table; implicit_broadcast is false and +/// must_fix_argument is true to force a fix_argument() call before any compute(). +/// +/// The query (fix_argument's argument) is a vector set of raw (unquantized) fp32 tokens +/// with dimensions() equal to the centroid dimension. The doc (compute's second argument) +/// is an id vector set (unsigned integer element type, dimensions() == 1): one centroid +/// id per token, each id in [0, K). compute's query argument is unused; fix_argument +/// already consumed the query into this object's table. +class QChamferExactQuery { + public: + using compare = std::less<>; + static constexpr bool implicit_broadcast = false; + static constexpr bool must_fix_argument = true; + static constexpr std::string_view name = "gem_qchamfer_exact_query"; + + explicit QChamferExactQuery( + std::shared_ptr centroids, + std::shared_ptr router = nullptr + ) + : centroids_{std::move(centroids)} + , router_{std::move(router)} { + if (router_ && router_->num_fine() != centroids_->K) { + throw std::invalid_argument( + "QChamferExactQuery: router has " + std::to_string(router_->num_fine()) + + " fine centroids, centroids has K = " + std::to_string(centroids_->K) + ); + } + } + + const std::shared_ptr& router() const { return router_; } + + /// The per-query (K x n_query) dot table built by the last fix_argument() call. + const float* table() const { return table_.data(); } + + /// Number of query tokens the last fix_argument() call built table() for. + size_t query_tokens() const { return n_query_; } + + // Vector-set path: a query set of raw (unquantized) fp32 token vectors, contiguous + // in the set's own storage, with dimensions() equal to the centroid dimension. + template + requires std::same_as + void fix_argument(const S& query) { + if (query.dimensions() != centroids_->dim) { + throw std::invalid_argument( + "QChamferExactQuery::fix_argument: query dimension " + + std::to_string(query.dimensions()) + " does not match centroid dimension " + + std::to_string(centroids_->dim) + ); + } + fix_argument_decoded(query.data(), query.cardinality()); + } + + // `query` is unused: fix_argument() already consumed it into table_/n_query_, and + // every compute() call in a search shares that one query's table. `doc` is an id + // vector set, decoded with no copy when its element type is uint32_t. + template + requires std::unsigned_integral + float compute(const Query& /*query*/, const Sd& doc) const { + gem_detail::require_id_dimensions(doc.dimensions()); + thread_local std::vector buf_d; + const uint32_t* doc_ids = gem_detail::id_span(doc.data(), doc.cardinality(), buf_d); + return compute_decoded(doc_ids, doc.cardinality()); + } + + private: + // Builds the (K x n_query) dot table once per query: table[f * n_query + i] = + // dot(bf16(centroid_f), bf16(query_token_i)), matching maxsim.cpp's + // Eigen::Map(table, query_tokens, fine_centers) column-major read of the + // same buffer (scores.col(f)[i] == table[f*n_query+i]). That is exactly + // InnerProductMatrix::compute_packed's A x B^T with A = the (already bf16) centroids + // (K rows) and B = query tokens (n_query rows), rounded to bf16 by this call rather + // than the kernel, so every ISA level sees the same rounded values on both operands. + void fix_argument_decoded(const float* tokens, size_t n_query) { + n_query_ = n_query; + table_.resize(centroids_->K * n_query_); + packed_query_.pack_rounded_to_bf16(tokens, n_query_, centroids_->dim); + InnerProductMatrix::compute_packed( + centroids_->data.get(), centroids_->K, packed_query_, table_.data() + ); + } + + // Reduces the per-query table against a doc's decoded ids, dispatched to the ISA the + // per-ISA compute_decoded was built for. + float compute_decoded(const uint32_t* doc_ids, size_t n_doc) const { + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx512f_supported(), 1)) { + return QChamferExactImpl::compute_decoded( + table_.data(), n_query_, doc_ids, n_doc + ); + } + if (__builtin_expect(svs::detail::avx_runtime_flags.is_avx2_supported(), 1)) { + return QChamferExactImpl::compute_decoded( + table_.data(), n_query_, doc_ids, n_doc + ); + } + return QChamferExactImpl::compute_decoded( + table_.data(), n_query_, doc_ids, n_doc + ); + } + + std::shared_ptr centroids_; + std::shared_ptr router_; + PackedOperand packed_query_; + std::vector table_; + size_t n_query_ = 0; +}; + +inline constexpr bool operator==(const QChamferExactQuery&, const QChamferExactQuery&) { + return true; +} + +} // namespace svs::distance diff --git a/include/svs/core/distance/gem_router.h b/include/svs/core/distance/gem_router.h new file mode 100644 index 000000000..1627901e0 --- /dev/null +++ b/include/svs/core/distance/gem_router.h @@ -0,0 +1,410 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/concepts/vector_set.h" +#include "svs/lib/exception.h" +#include "svs/lib/threads.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs::distance { + +/// Groups GEM fine centroids into coarse clusters, each with a representative document, +/// so a Vamana search can start from multiple entry points chosen by query-token score. +/// See `svs::index::vamana::extensions::GemRoutedInitializer` (extensions/vamana/gem.h). +struct GemRouter { + static constexpr uint32_t no_representative = std::numeric_limits::max(); + + // coarse_of_fine[f] is the coarse cluster of fine centroid f, < num_coarse(). + std::vector coarse_of_fine; + // representative[c] is a document id, or no_representative if no document touches c. + std::vector representative; + // Coarse clusters probed per query token in route_query. + size_t nprobe = 4; + + size_t num_coarse() const { return representative.size(); } + size_t num_fine() const { return coarse_of_fine.size(); } + + /// Scratch for route_query, reused across calls so a search thread allocates nothing + /// once every buffer has grown to its steady-state size. + struct QueryScratch { + std::vector coarse_score; + std::vector> candidates; + std::vector picked; + }; + + /// Scratch for route_ids, reused the same way as QueryScratch. + struct IdScratch { + std::vector> counts; + }; + + /// Builds representatives from `docs` (one centroid id per token, dimensions() == 1) + /// given a fine-to-coarse assignment. Deterministic regardless of `pool` size: shares + /// are reduced in ascending document-id order, so ties always pick the lowest id. + template + requires std::unsigned_integral + static GemRouter build( + std::vector coarse_of_fine, + size_t num_coarse, + const IdSets& docs, + Pool& pool, + size_t nprobe = 4 + ) { + size_t k = coarse_of_fine.size(); + for (size_t f = 0; f < k; ++f) { + if (coarse_of_fine[f] >= num_coarse) { + throw ANNEXCEPTION( + "GemRouter::build: coarse_of_fine[{}] = {} is not < num_coarse = {}", + f, + coarse_of_fine[f], + num_coarse + ); + } + } + + // (doc id, cluster, share) claims; disjoint per thread, so concatenated not merged. + struct Claim { + uint32_t doc_id; + uint32_t cluster; + double share; + }; + threads::SequentialTLS> claims(pool.size()); + + // Errors are recorded, not thrown: parallel_for doesn't guarantee propagation. + std::atomic bad_id{false}; + std::atomic bad_value{0}; + std::atomic bad_dims{false}; + std::atomic bad_dims_doc{0}; + std::atomic bad_dims_value{0}; + + threads::parallel_for( + pool, + threads::StaticPartition{docs.size()}, + [&](const auto& ids, uint64_t tid) { + auto& out = claims.at(tid); + std::vector> local_counts; + for (size_t i : ids) { + auto doc = docs.get_datum(i); + if (doc.dimensions() != 1) { + bad_dims.store(true, std::memory_order_relaxed); + bad_dims_doc.store(i, std::memory_order_relaxed); + bad_dims_value.store(doc.dimensions(), std::memory_order_relaxed); + continue; + } + size_t card = doc.cardinality(); + if (card == 0) { + continue; + } + local_counts.clear(); + for (size_t t = 0; t < card; ++t) { + uint64_t fine_id = static_cast(doc.token(t)[0]); + if (fine_id >= k) { + bad_id.store(true, std::memory_order_relaxed); + bad_value.store(fine_id, std::memory_order_relaxed); + continue; + } + local_counts.emplace_back(coarse_of_fine[fine_id], uint32_t{1}); + } + std::sort(local_counts.begin(), local_counts.end()); + size_t w = 0; + size_t n = local_counts.size(); + for (size_t j = 0; j < n;) { + size_t j2 = j; + uint32_t total = 0; + while (j2 < n && local_counts[j2].first == local_counts[j].first) { + total += local_counts[j2].second; + ++j2; + } + local_counts[w++] = {local_counts[j].first, total}; + j = j2; + } + for (size_t j = 0; j < w; ++j) { + out.push_back( + {static_cast(i), + local_counts[j].first, + static_cast(local_counts[j].second) / + static_cast(card)} + ); + } + } + } + ); + + if (bad_dims.load()) { + throw ANNEXCEPTION( + "GemRouter::build: document {} has dimensions() = {}, expected 1 (one " + "centroid id per token)", + bad_dims_doc.load(), + bad_dims_value.load() + ); + } + if (bad_id.load()) { + throw ANNEXCEPTION( + "GemRouter::build: document centroid id {} is not < K = {}", + bad_value.load(), + k + ); + } + + std::vector all; + claims.visit([&](const std::vector& part) { + all.insert(all.end(), part.begin(), part.end()); + }); + std::sort(all.begin(), all.end(), [](const Claim& a, const Claim& b) { + return a.doc_id < b.doc_id; + }); + + GemRouter result; + result.coarse_of_fine = std::move(coarse_of_fine); + result.representative.assign(num_coarse, no_representative); + result.nprobe = nprobe; + std::vector best_share(num_coarse, -1.0); + for (const auto& claim : all) { + if (claim.share > best_share[claim.cluster]) { + best_share[claim.cluster] = claim.share; + result.representative[claim.cluster] = claim.doc_id; + } + } + return result; + } + + /// Thread-count convenience overload: builds a `DefaultThreadPool` of + /// `num_threads` threads for the duration of the call. + template + requires std::unsigned_integral + static GemRouter build( + std::vector coarse_of_fine, + size_t num_coarse, + const IdSets& docs, + size_t num_threads, + size_t nprobe = 4 + ) { + auto pool = threads::DefaultThreadPool(num_threads); + return build(std::move(coarse_of_fine), num_coarse, docs, pool, nprobe); + } + + /// Scores cluster c for query token i as the max of `table[f * n_query + i]` over its + /// fine centroids f, picks the `nprobe` best clusters per token (ties to lower id), + /// and writes their representatives to `out`, deduplicated and sorted ascending. + void route_query( + const float* table, + size_t n_query, + std::vector& out, + QueryScratch& scratch + ) const { + size_t nc = num_coarse(); + scratch.coarse_score.assign(nc * n_query, -std::numeric_limits::infinity()); + size_t k = num_fine(); + for (size_t f = 0; f < k; ++f) { + uint32_t c = coarse_of_fine[f]; + const float* row = table + f * n_query; + float* dst = scratch.coarse_score.data() + static_cast(c) * n_query; + for (size_t i = 0; i < n_query; ++i) { + dst[i] = std::max(dst[i], row[i]); + } + } + + scratch.picked.clear(); + size_t take = std::min(nprobe, nc); + for (size_t i = 0; i < n_query; ++i) { + scratch.candidates.clear(); + scratch.candidates.reserve(nc); + for (size_t c = 0; c < nc; ++c) { + scratch.candidates.emplace_back( + scratch.coarse_score[c * n_query + i], static_cast(c) + ); + } + std::partial_sort( + scratch.candidates.begin(), + scratch.candidates.begin() + static_cast(take), + scratch.candidates.end(), + [](const auto& a, const auto& b) { + if (a.first != b.first) { + return a.first > b.first; + } + return a.second < b.second; + } + ); + for (size_t j = 0; j < take; ++j) { + uint32_t rep = representative[scratch.candidates[j].second]; + if (rep != no_representative) { + scratch.picked.push_back(rep); + } + } + } + + out.assign(scratch.picked.begin(), scratch.picked.end()); + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + } + + /// Ranks `ids`' coarse clusters by token count (ties to the lower cluster id), takes + /// the top `max_clusters`, and writes their representatives (deduplicated, sorted + /// ascending, `no_representative` skipped) to `out`. Lookups only, no scoring. + void route_ids( + std::span ids, + size_t max_clusters, + std::vector& out, + IdScratch& scratch + ) const { + scratch.counts.clear(); + for (uint32_t id : ids) { + if (id >= num_fine()) { + throw ANNEXCEPTION( + "GemRouter::route_ids: centroid id {} is not < num_fine() = {}", + id, + num_fine() + ); + } + scratch.counts.emplace_back(coarse_of_fine[id], uint32_t{1}); + } + std::sort(scratch.counts.begin(), scratch.counts.end()); + size_t w = 0; + size_t n = scratch.counts.size(); + for (size_t i = 0; i < n;) { + size_t j = i; + uint32_t total = 0; + while (j < n && scratch.counts[j].first == scratch.counts[i].first) { + total += scratch.counts[j].second; + ++j; + } + scratch.counts[w++] = {scratch.counts[i].first, total}; + i = j; + } + scratch.counts.resize(w); + + size_t take = std::min(max_clusters, scratch.counts.size()); + std::partial_sort( + scratch.counts.begin(), + scratch.counts.begin() + static_cast(take), + scratch.counts.end(), + [](const auto& a, const auto& b) { + if (a.second != b.second) { + return a.second > b.second; + } + return a.first < b.first; + } + ); + + out.clear(); + for (size_t i = 0; i < take; ++i) { + uint32_t rep = representative[scratch.counts[i].first]; + if (rep != no_representative) { + out.push_back(rep); + } + } + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + } + + void save(const std::filesystem::path& path) const { + std::ofstream out(path, std::ios::binary); + if (!out) { + throw ANNEXCEPTION("GemRouter::save: cannot open {}", path.string()); + } + Header header{}; + header.k = num_fine(); + header.num_coarse_val = num_coarse(); + header.nprobe_val = nprobe; + out.write(reinterpret_cast(&header), sizeof(header)); + out.write( + reinterpret_cast(coarse_of_fine.data()), + static_cast(coarse_of_fine.size() * sizeof(uint32_t)) + ); + out.write( + reinterpret_cast(representative.data()), + static_cast(representative.size() * sizeof(uint32_t)) + ); + if (!out) { + throw ANNEXCEPTION("GemRouter::save: short write to {}", path.string()); + } + } + + static GemRouter load(const std::filesystem::path& path) { + std::ifstream in(path, std::ios::binary); + if (!in) { + throw ANNEXCEPTION("GemRouter::load: cannot open {}", path.string()); + } + Header header{}; + in.read(reinterpret_cast(&header), sizeof(header)); + if (!in) { + throw ANNEXCEPTION( + "GemRouter::load: {} is too short to hold a GemRouter header", path.string() + ); + } + if (header.magic != kMagic) { + throw ANNEXCEPTION( + "GemRouter::load: {} has magic {:#x}, expected {:#x}", + path.string(), + header.magic, + kMagic + ); + } + if (header.version != kVersion) { + throw ANNEXCEPTION( + "GemRouter::load: {} has version {}, expected {}", + path.string(), + header.version, + kVersion + ); + } + + GemRouter result; + result.coarse_of_fine.resize(header.k); + result.representative.resize(header.num_coarse_val); + result.nprobe = header.nprobe_val; + in.read( + reinterpret_cast(result.coarse_of_fine.data()), + static_cast(header.k * sizeof(uint32_t)) + ); + in.read( + reinterpret_cast(result.representative.data()), + static_cast(header.num_coarse_val * sizeof(uint32_t)) + ); + if (!in) { + throw ANNEXCEPTION("GemRouter::load: {} is truncated", path.string()); + } + return result; + } + + private: + static constexpr uint64_t kMagic = 0x474d52545231ULL; // "GMRTR1" in ASCII hex-ish. + static constexpr uint32_t kVersion = 1; + + // Trivially copyable on-disk header; field order is the file layout. + struct Header { + uint64_t magic = kMagic; + uint32_t version = kVersion; + uint64_t k = 0; + uint64_t num_coarse_val = 0; + uint64_t nprobe_val = 0; + }; +}; + +} // namespace svs::distance diff --git a/include/svs/core/distance/inner_product_matrix.h b/include/svs/core/distance/inner_product_matrix.h new file mode 100644 index 000000000..7acfb1e41 --- /dev/null +++ b/include/svs/core/distance/inner_product_matrix.h @@ -0,0 +1,1516 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/core/distance/distance_core.h" +#include "svs/core/distance/simd_utils.h" +#include "svs/lib/avx_detection.h" +#include "svs/lib/bfloat16.h" +#include "svs/lib/float16.h" +#include "svs/lib/preprocessor.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace svs::distance { + +// One accumulator per (A-row, B-lane-block), no horizontal reduction: measured on this +// machine, 8 independent accumulators is where row-blocking stops buying throughput, +// matching ~4-cycle FMA latency x 2 ports (see the PR description for the sweep). +inline constexpr size_t inner_product_matrix_row_block = 8; + +/// Owns B's packed operand: B transposed into a [d][n] float32 buffer, so every row-block +/// of A can stream through it with a plain `loadu` instead of a gather. Packing is split +/// from use (`pack` vs. the kernel entry points below) because the table form repacks B +/// once per call (B is as cheap as the table it produces), while MaxSim packs one query +/// and reuses it against ~1000 documents: repacking per document would cost as much as +/// scoring them. +class PackedOperand { + public: + /// Repacks `b` (`n` rows of `d` floats each, row-major) into this object's transposed + /// buffer. Storage grows only when `n * d` exceeds the current buffer's capacity + /// (`std::vector::resize` does not reallocate when shrinking or staying within + /// capacity), so calling this with the same or a smaller shape across repeated calls + /// costs no allocation. + void pack(const float* b, size_t n, size_t d) { + n_ = n; + d_ = d; + buffer_.resize(n * d); + for (size_t j = 0; j < n; ++j) { + const float* bj = b + j * d; + for (size_t k = 0; k < d; ++k) { + buffer_[k * n + j] = bj[k]; + } + } + } + + /// Same contract and layout as `pack`, except every value is first rounded to + /// bfloat16 precision (round to nearest, ties to even) and stored back as float32. + /// The kernel itself only ever computes on the values it is given, + /// unrounded; rounding B to the precision A is held at (bf16, see FineCentroids) is + /// the caller's choice of precision, made here rather than inside the kernel, so a + /// future caller that wants a different precision for B is never blocked by this one. + /// A scalar loop (not vectorized): at GEM's production shape this rounds ~4096 + /// values per query, negligible next to the O(K * d) table build that follows. + void pack_rounded_to_bf16(const float* b, size_t n, size_t d) { + n_ = n; + d_ = d; + buffer_.resize(n * d); + for (size_t j = 0; j < n; ++j) { + const float* bj = b + j * d; + for (size_t k = 0; k < d; ++k) { + buffer_[k * n + j] = static_cast(svs::round_to_bfloat16(bj[k])); + } + } +#if SVS_AMX_COMPILED_IN + // Only worth building when some TU in this binary can consume it (amx.cpp + // exists, SVS_AMX_COMPILED_IN) and the CPU/OS actually grant tile use; otherwise + // this would cost a few KiB and a scalar pass on every call for nothing. + if (svs::detail::amx_tile_bf16_available()) { + fill_tile_layout(b, n, d); + } else { + has_tile_layout_ = false; + } +#endif + } + + size_t n() const { return n_; } + size_t d() const { return d_; } + const float* data() const { return buffer_.data(); } + + /// True only right after `pack_rounded_to_bf16` filled the AMX tile layout below; + /// `pack` never sets this, since its values are not rounded and the AMX kernel only + /// ever computes . + bool has_tile_layout() const { return has_tile_layout_; } + + /// Pair-interleaved, transposed bf16 tile block for query-lane tile `nt` (16 lanes, + /// zero-padded past `n`) and d/2-pair tile `kt` (16 pairs, zero-padded past + /// ceil(d/2)): row `r`'s dword at lane `c` packs + /// (bf16(b[nt*16+c][2*(kt*16+r)]), bf16(b[nt*16+c][2*(kt*16+r)+1])) low/high, i.e. + /// exactly the `b` tile operand _tile_dpbf16ps expects (see maxsim.h's contract). + /// Valid only when `has_tile_layout()`; caller (amx.cpp) owns interpreting it. + const uint32_t* tile_block(size_t nt, size_t kt) const { + return tile_buffer_.data() + (nt * kp_tiles_ + kt) * kTileElements; + } + size_t n_tiles() const { return n_tiles_; } + size_t kp_tiles() const { return kp_tiles_; } + + private: + static constexpr size_t kTileElements = 16 * 16; + + // Builds the tile-layout buffer described by tile_block() above. Zero-padding both + // the lane tail (n not a multiple of 16) and the pair tail (d odd, or not a multiple + // of 32) is safe here because B's padding only ever multiplies against A's real + // values and contributes 0; A's own out-of-bounds tail is handled separately, by the + // AMX kernel copying A's last partial k-chunk through a zero-padded stack buffer + // rather than reading past A's allocation. + void fill_tile_layout(const float* b, size_t n, size_t d) { + size_t kp = (d + 1) / 2; + n_tiles_ = (n + 15) / 16; + kp_tiles_ = (kp + 15) / 16; + tile_buffer_.assign(n_tiles_ * kp_tiles_ * kTileElements, uint32_t{0}); + for (size_t j = 0; j < n; ++j) { + const float* bj = b + j * d; + size_t nt = j / 16; + size_t lane = j % 16; + for (size_t k = 0; k < d; ++k) { + size_t kp_idx = k / 2; + size_t kt = kp_idx / 16; + size_t row = kp_idx % 16; + uint32_t bits = static_cast(svs::round_to_bfloat16(bj[k]).raw()); + uint32_t* dword = + &tile_buffer_[(nt * kp_tiles_ + kt) * kTileElements + row * 16 + lane]; + if (k % 2 == 0) { + *dword = (*dword & 0xFFFF0000u) | bits; + } else { + *dword = (*dword & 0x0000FFFFu) | (bits << 16); + } + } + } + has_tile_layout_ = true; + } + + std::vector buffer_; + size_t n_ = 0; + size_t d_ = 0; + std::vector tile_buffer_; + size_t n_tiles_ = 0; + size_t kp_tiles_ = 0; + bool has_tile_layout_ = false; +}; + +/// Per-ISA kernel family for a large row-major A (m x d) against B, packed once into a +/// `PackedOperand` by the caller. This primary template is the scalar, compiler +/// auto-vectorized generic implementation: both the fallback when no SIMD ISA is +/// available and the reference the ISA-specific specializations below are checked +/// against in tests. Those specializations are declared here and defined in avx2.cpp / +/// avx512.cpp so their intrinsics only compile under their own -march flags, mirroring +/// QChamferExactImpl in gem.h. +/// +/// Two output forms share every row of A they stream over: +/// * `build`/`build_packed`: the dense table, out[i * n + j] = . +/// * `column_max`: out[j] = max over i of , i.e. the column-wise maximum of +/// that same table, computed with no table ever materialized. +/// +/// Both forms are additionally overloaded on A's element type (float32, svs::Float16 and +/// svs::BFloat16); the narrow-type overloads convert one row-block of A at a time into a +/// float32 scratch and run the identical float32 path, so no element type is ever branched +/// on per row. BFloat16's widening is exact (it is float32's high 16 bits, zero-extended), +/// so unlike Float16 there is no rounding question on A's side; B's precision for a +/// BFloat16 A is the caller's choice, made by packing with +/// `PackedOperand::pack_rounded_to_bf16` instead of `pack`. +template struct InnerProductMatrixImpl { + // Shared by every element type A can hold: float32 needs no conversion (the + // static_cast below is a no-op then), Float16/BFloat16 each widen losslessly to + // float32 per element. One shape, never a per-row branch on T. + template + static void + build_packed_generic(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + const float* bt = b.data(); + for (size_t i = 0; i < m; ++i) { + const T* ai = a + i * d; + float* row = out + i * n; + for (size_t j = 0; j < n; ++j) { + float dot = 0.0F; + for (size_t k = 0; k < d; ++k) { + dot += static_cast(ai[k]) * bt[k * n + j]; + } + row[j] = dot; + } + } + } + + SVS_NOINLINE static void + build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + build_packed_generic(a, m, b, out); + } + + SVS_NOINLINE static void + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + build_packed_generic(a, m, b, out); + } + + SVS_NOINLINE static void + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + build_packed_generic(a, m, b, out); + } + + // Packs B into a reused thread_local buffer so this keeps its no-allocation-per-call + // property: InnerProductMatrix::compute (GEM's fix_argument path) calls this once per + // query against all K centroids, not once per document, so repacking here is cheap. + SVS_NOINLINE static void + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); + } + + template + static void + column_max_generic(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + const float* bt = b.data(); + std::fill(out, out + n, -std::numeric_limits::infinity()); + for (size_t i = 0; i < m; ++i) { + const T* ai = a + i * d; + for (size_t j = 0; j < n; ++j) { + float dot = 0.0F; + for (size_t k = 0; k < d; ++k) { + dot += static_cast(ai[k]) * bt[k * n + j]; + } + out[j] = std::max(out[j], dot); + } + } + } + + SVS_NOINLINE static void + column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + column_max_generic(a, m, b, out); + } + + SVS_NOINLINE static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + column_max_generic(a, m, b, out); + } + + SVS_NOINLINE static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + column_max_generic(a, m, b, out); + } +}; + +#if defined(__x86_64__) + +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +#if SVS_AVX512_F + +namespace inner_product_matrix_avx512_detail { + +constexpr size_t kRowBlock = inner_product_matrix_row_block; + +// Accumulates kRowBlock rows of A (padded by repeating the last valid row when the +// document has fewer rows left, see callers) against one full 16-lane block of packed B. +// Shared by the table and column-max output forms so this FMA chain is written once. +SVS_FORCE_INLINE void accumulate_full( + const float* const* arow, const float* bt, size_t n, size_t d, size_t j, __m512* acc +) { + for (size_t k = 0; k < d; ++k) { + __m512 bv = _mm512_loadu_ps(bt + k * n + j); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +// Same as accumulate_full for the trailing lane-block narrower than 16 lanes. +SVS_FORCE_INLINE void accumulate_tail( + const float* const* arow, + const float* bt, + size_t n, + size_t d, + size_t j, + __mmask16 mask, + __m512* acc +) { + for (size_t k = 0; k < d; ++k) { + __m512 bv = _mm512_maskz_loadu_ps(mask, bt + k * n + j); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_fmadd_ps(_mm512_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +// Pairwise-max reduction of the kRowBlock accumulators into their lane-wise maximum. +// Padding slots (see accumulate_full's comment) repeat a value already present in a real +// slot, so folding them into the tree does not change the result. +SVS_FORCE_INLINE __m512 reduce_max(const __m512* acc) { + __m512 m01 = _mm512_max_ps(acc[0], acc[1]); + __m512 m23 = _mm512_max_ps(acc[2], acc[3]); + __m512 m45 = _mm512_max_ps(acc[4], acc[5]); + __m512 m67 = _mm512_max_ps(acc[6], acc[7]); + return _mm512_max_ps(_mm512_max_ps(m01, m23), _mm512_max_ps(m45, m67)); +} + +// Widens one 16-lane block of Float16 to float32 with AVX-512F's native half->single. +SVS_FORCE_INLINE __m512 widen16(const svs::Float16* p) { + return _mm512_cvtph_ps(_mm256_loadu_epi16(p)); +} + +// Widens one 16-lane block of BFloat16 to float32: bfloat16 is already float32's high 16 +// bits, so this is a lossless zero-extend (cvtepu16_epi32, AVX-512F) into the top half of +// each 32-bit lane (slli by 16), not a rounding conversion. +SVS_FORCE_INLINE __m512 widen16(const svs::BFloat16* p) { + __m256i halfwords = _mm256_loadu_si256(reinterpret_cast(p)); + return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(halfwords), 16)); +} + +// Tail (< 16 lanes) widening for either 16-bit type: zero-pads through a stack buffer +// sized by sizeof(T) (2 bytes for both Float16 and BFloat16), then reuses widen16. Uses a +// plain memcpy rather than a masked load so BFloat16's tail needs nothing beyond +// AVX-512F (Float16's masked load above already relies on AVX-512BW+VL for +// _mm256_loadu_epi16, a preexisting requirement this does not widen). +template SVS_FORCE_INLINE __m512 widen16_tail(const T* p, size_t count) { + __m256i buffer = _mm256_setzero_si256(); + std::memcpy(&buffer, p, count * sizeof(T)); + return widen16(reinterpret_cast(&buffer)); +} + +// Converts kRowBlock rows of `a` (T is Float16 or BFloat16) starting at row `i` into a +// contiguous float32 scratch, 16 lanes at a time; shared by the table and column-max +// forms below so either narrow type is widened in exactly one place. +template +SVS_FORCE_INLINE void +convert_row_block(const T* a, size_t i, size_t rows, size_t d, float* scratch) { + for (size_t r = 0; r < rows; ++r) { + const T* src = a + (i + r) * d; + float* dst = scratch + r * d; + size_t k = 0; + for (; k + 16 <= d; k += 16) { + _mm512_storeu_ps(dst + k, widen16(src + k)); + } + if (k < d) { + size_t tail = d - k; + __mmask16 dmask = svs::create_mask<16>(svs::lib::MaybeStatic(tail)); + _mm512_mask_storeu_ps(dst + k, dmask, widen16_tail(src + k, tail)); + } + } +} + +// Table form for a narrow-element A (Float16 or BFloat16): identical register-blocked +// loop to the float32 build_packed above, sourced from a row-block converted to float32 +// by convert_row_block instead of `a` directly. +template +inline void build_packed_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_mask_storeu_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } +} + +// Column-max form for a narrow-element A (Float16 or BFloat16): same structure as +// build_packed_narrow above, reduced through reduce_max instead of stored per row. +template +inline void column_max_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m512 current = _mm512_loadu_ps(out + j); + _mm512_storeu_ps(out + j, _mm512_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m512 current = _mm512_maskz_loadu_ps(tail_mask, out + j); + _mm512_mask_storeu_ps( + out + j, tail_mask, _mm512_max_ps(reduce_max(acc), current) + ); + } + } +} + +} // namespace inner_product_matrix_avx512_detail + +// Partial specializations on a dummy parameter: a full one's explicit instantiation emits +// no inline members, and GCC 11 emits nothing for a second constrained one with equal +// arguments. +template struct InnerProductMatrixImpl { + // Register-blocked micro-kernel: inner_product_matrix_row_block rows of A share one + // pass over a lane-block of B, giving that many independent FMA accumulator chains + // with no horizontal reduction (each lane is one output), so the dependency-chain + // latency of a single accumulator is hidden behind the others instead of + // serializing the loop over d. + SVS_NOINLINE static void + build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + using namespace inner_product_matrix_avx512_detail; + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm512_mask_storeu_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } + } + + // Table form for a Float16 A: see convert_row_block above for the shared widening. + SVS_NOINLINE static void + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx512_detail::build_packed_narrow(a, m, b, out); + } + + // Table form for a BFloat16 A: see convert_row_block above for the shared widening. + SVS_NOINLINE static void + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx512_detail::build_packed_narrow(a, m, b, out); + } + + // Packs B into a reused thread_local buffer: the table form repacks on every call + // (one query against all centroids), unlike MaxSim's column_max below which packs + // once per query and calls into this ISA's kernels directly with the already-packed + // operand. + SVS_NOINLINE static void + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + if (m == 0 || n == 0) { + return; + } + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); + } + + // Column-wise maximum of the same table build_packed computes, without ever + // materializing it: each row-block's accumulators are reduced to a per-lane maximum + // and folded into a running maximum held in `out` itself, so one reduction serves + // the whole document for the price of one load/max/store per lane-block per + // row-block. + SVS_NOINLINE static void + column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + using namespace inner_product_matrix_avx512_detail; + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 16; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + __mmask16 tail_mask = svs::create_mask(svs::lib::MaybeStatic(n)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m512 current = _mm512_loadu_ps(out + j); + _mm512_storeu_ps(out + j, _mm512_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m512 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm512_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m512 current = _mm512_maskz_loadu_ps(tail_mask, out + j); + _mm512_mask_storeu_ps( + out + j, tail_mask, _mm512_max_ps(reduce_max(acc), current) + ); + } + } + } + + // Float16 overload: see convert_row_block above for the shared widening (AVX-512F's + // native 16-wide _mm512_cvtph_ps). + SVS_NOINLINE static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx512_detail::column_max_narrow(a, m, b, out); + } + + // BFloat16 overload: see convert_row_block above for the shared (exact) widening. + SVS_NOINLINE static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx512_detail::column_max_narrow(a, m, b, out); + } +}; +#endif + +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +SVS_VALIDATE_BOOL_ENV(SVS_AVX2) +#if !SVS_AVX512_F && SVS_AVX2 + +namespace inner_product_matrix_avx2_detail { + +constexpr size_t kRowBlock = inner_product_matrix_row_block; + +// Same role as avx512.cpp's accumulate_full/accumulate_tail, at AVX2's 8-wide registers; +// AVX2 has no masked load/store, so the tail lane-block uses _mm256_maskload_ps with an +// all-lanes blend mask built from the bit pattern simd_utils.h's ConvertToFloat<8> uses. +SVS_FORCE_INLINE void accumulate_full( + const float* const* arow, const float* bt, size_t n, size_t d, size_t j, __m256* acc +) { + for (size_t k = 0; k < d; ++k) { + __m256 bv = _mm256_loadu_ps(bt + k * n + j); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +SVS_FORCE_INLINE void accumulate_tail( + const float* const* arow, + const float* bt, + size_t n, + size_t d, + size_t j, + __m256i mask, + __m256* acc +) { + for (size_t k = 0; k < d; ++k) { + __m256 bv = _mm256_maskload_ps(bt + k * n + j, mask); + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_fmadd_ps(_mm256_set1_ps(arow[r][k]), bv, acc[r]); + } + } +} + +// Pairwise-max reduction; see avx512_detail::reduce_max for why padding slots are safe +// to include. +SVS_FORCE_INLINE __m256 reduce_max(const __m256* acc) { + __m256 m01 = _mm256_max_ps(acc[0], acc[1]); + __m256 m23 = _mm256_max_ps(acc[2], acc[3]); + __m256 m45 = _mm256_max_ps(acc[4], acc[5]); + __m256 m67 = _mm256_max_ps(acc[6], acc[7]); + return _mm256_max_ps(_mm256_max_ps(m01, m23), _mm256_max_ps(m45, m67)); +} + +// Widens one 8-lane block of Float16 to float32 with F16C's native half->single. +SVS_FORCE_INLINE __m256 widen8(const svs::Float16* p) { + return _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(p))); +} + +// Widens one 8-lane block of BFloat16 to float32: bfloat16 is already float32's high 16 +// bits, so this is a lossless zero-extend (cvtepu16_epi32) into the top half of each +// 32-bit lane (slli by 16), not a rounding conversion. +SVS_FORCE_INLINE __m256 widen8(const svs::BFloat16* p) { + __m128i halfwords = _mm_loadu_si128(reinterpret_cast(p)); + return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(halfwords), 16)); +} + +// Tail (< 8 lanes) widening for either 16-bit type: zero-pads through a stack buffer +// sized by sizeof(T) (2 bytes for both Float16 and BFloat16), then reuses widen8. +template SVS_FORCE_INLINE __m256 widen8_tail(const T* p, uint8_t mask_bits) { + __m128i buffer = _mm_setzero_si128(); + std::memcpy(&buffer, p, __builtin_popcount(mask_bits) * sizeof(T)); + return widen8(reinterpret_cast(&buffer)); +} + +// Converts kRowBlock rows of `a` (T is Float16 or BFloat16) starting at row `i` into a +// contiguous float32 scratch, 8 lanes at a time; shared by the table and column-max forms +// below so either narrow type is widened in exactly one place. +template +SVS_FORCE_INLINE void +convert_row_block(const T* a, size_t i, size_t rows, size_t d, float* scratch) { + for (size_t r = 0; r < rows; ++r) { + const T* src = a + (i + r) * d; + float* dst = scratch + r * d; + size_t k = 0; + for (; k + 8 <= d; k += 8) { + _mm256_storeu_ps(dst + k, widen8(src + k)); + } + if (k < d) { + uint8_t dbits = svs::create_mask<8>(svs::lib::MaybeStatic(d - k)); + __m256i dmask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(dbits)); + _mm256_maskstore_ps(dst + k, dmask, widen8_tail(src + k, dbits)); + } + } +} + +// Table form for a narrow-element A (Float16 or BFloat16): identical register-blocked +// loop to the float32 build_packed above, sourced from a row-block converted to float32 +// by convert_row_block instead of `a` directly. +template +inline void build_packed_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_maskstore_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } +} + +// Column-max form for a narrow-element A (Float16 or BFloat16): same structure as +// build_packed_narrow above, reduced through reduce_max instead of stored per row. +template +inline void column_max_narrow(const T* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + thread_local std::vector scratch; + scratch.resize(kRowBlock * d); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + convert_row_block(a, i, rows, d, scratch.data()); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = scratch.data() + std::min(r, rows - 1) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m256 current = _mm256_loadu_ps(out + j); + _mm256_storeu_ps(out + j, _mm256_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m256 current = _mm256_maskload_ps(out + j, tail_mask); + _mm256_maskstore_ps( + out + j, tail_mask, _mm256_max_ps(reduce_max(acc), current) + ); + } + } +} + +} // namespace inner_product_matrix_avx2_detail + +template struct InnerProductMatrixImpl { + // Same register-blocking as InnerProductMatrixImpl::build_packed, at AVX2's + // 8-wide registers. + SVS_NOINLINE static void + build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + using namespace inner_product_matrix_avx2_detail; + size_t n = b.n(); + size_t d = b.d(); + if (m == 0 || n == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = + _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + float* orow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + for (size_t r = 0; r < rows; ++r) { + orow[r] = out + (i + r) * n; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_storeu_ps(orow[r] + j, acc[r]); + } + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + for (size_t r = 0; r < rows; ++r) { + _mm256_maskstore_ps(orow[r] + j, tail_mask, acc[r]); + } + } + } + } + + // Table form for a Float16 A: see convert_row_block above for the shared widening. + SVS_NOINLINE static void + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx2_detail::build_packed_narrow(a, m, b, out); + } + + // Table form for a BFloat16 A: see convert_row_block above for the shared widening. + SVS_NOINLINE static void + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx2_detail::build_packed_narrow(a, m, b, out); + } + + // See InnerProductMatrixImpl::build for why this packs into a thread_local + // buffer rather than taking a PackedOperand from the caller. + SVS_NOINLINE static void + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + if (m == 0 || n == 0) { + return; + } + thread_local PackedOperand packed; + packed.pack(b, n, d); + build_packed(a, m, packed, out); + } + + // See InnerProductMatrixImpl::column_max for the algorithm; this is the same + // reduction at AVX2's 8-wide registers. + SVS_NOINLINE static void + column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + using namespace inner_product_matrix_avx2_detail; + size_t n = b.n(); + size_t d = b.d(); + if (n == 0) { + return; + } + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + constexpr size_t vl = 8; + const float* bt = b.data(); + size_t full_lanes = n - n % vl; + uint8_t tail_bits = svs::create_mask(svs::lib::MaybeStatic(n)); + __m256i tail_mask = + _mm256_castps_si256(svs::simd::create_blend_mask_avx2(tail_bits)); + + for (size_t i = 0; i < m; i += kRowBlock) { + size_t rows = std::min(kRowBlock, m - i); + const float* arow[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + arow[r] = a + (i + std::min(r, rows - 1)) * d; + } + + size_t j = 0; + for (; j < full_lanes; j += vl) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_full(arow, bt, n, d, j, acc); + __m256 current = _mm256_loadu_ps(out + j); + _mm256_storeu_ps(out + j, _mm256_max_ps(reduce_max(acc), current)); + } + if (j < n) { + __m256 acc[kRowBlock]; + for (size_t r = 0; r < kRowBlock; ++r) { + acc[r] = _mm256_setzero_ps(); + } + accumulate_tail(arow, bt, n, d, j, tail_mask, acc); + __m256 current = _mm256_maskload_ps(out + j, tail_mask); + _mm256_maskstore_ps( + out + j, tail_mask, _mm256_max_ps(reduce_max(acc), current) + ); + } + } + } + + // Float16 overload: see convert_row_block above for the shared widening (F16C's + // 8-wide _mm256_cvtph_ps, available because avx2.cpp is compiled for the "haswell" + // arch in cmake/multi-arch.cmake, and Haswell carries F16C alongside AVX2). + SVS_NOINLINE static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx2_detail::column_max_narrow(a, m, b, out); + } + + // BFloat16 overload: see convert_row_block above for the shared (exact) widening. + SVS_NOINLINE static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + inner_product_matrix_avx2_detail::column_max_narrow(a, m, b, out); + } +}; +#endif + +// The scalar-distance kernels (L2/IP/cosine) and QEmd/QChamfer/QChamferExactImpl dispatch +// through a runtime if-chain on svs::detail::avx_runtime_flags (euclidean.h, cosine.h, +// inner_product.h, gem.h), never through a switch or template argument naming +// AVX_AVAILABILITY::AMX; adding the AMX enumerator needs no new instantiation of any of +// them (checked: `git grep -n AVX_AVAILABILITY` finds no exhaustive switch or `AMX` +// branch to add outside InnerProductMatrix). AMX only applies to the InnerProductMatrix +// family below, whose public dispatch is the one exhaustive switch this change extends. + +// Declared (full member set, matching every other level) only when SVS_AMX_TILE_BF16 +// (__AMX_TILE__ && __AMX_BF16__) is set for the current TU's own compiler flags: only +// amx.cpp is compiled with -mamx-tile -mamx-bf16, same per-TU pattern as +// SVS_AVX512_F/SVS_AVX2 above, not the build-wide SVS_AMX_COMPILED_IN that PackedOperand +// reads. build_packed/column_max for float32 and Float16 A, and for a BFloat16 A whose B +// was not packed with the tile layout, forward to the AVX512 implementation (explicit +// forwarding below, one source of truth); only a BFloat16 A against a tile-packed B runs +// the real tile kernel. +SVS_VALIDATE_BOOL_ENV(SVS_AMX_TILE_BF16) +#if SVS_AMX_TILE_BF16 + +namespace inner_product_matrix_amx_detail { + +// 64-byte tile-config struct the palette-1 AMX instructions read (Intel SDM Vol 1, +// 3.3.7): colsb/rows per tmm register 0-15, reserved bytes must stay zero. +struct TileConfig { + uint8_t palette_id = 0; + uint8_t start_row = 0; + uint8_t reserved[14] = {}; + uint16_t colsb[16] = {}; + uint8_t rows[16] = {}; +}; + +// Releases tile state on every exit path (normal return or exception): leaving tile +// state live costs later context switches and ordinary SSE/AVX code extra save/restore +// work, per Intel's optimization guidance for AMX. +struct TileGuard { + TileGuard() = default; + ~TileGuard() { _tile_release(); } + TileGuard(const TileGuard&) = delete; + TileGuard& operator=(const TileGuard&) = delete; +}; + +// tmm register assignment: tmm0 = A's current k-chunk (16 rows x 32 bf16, i.e. 16 +// pairs); tmm1/tmm2 = B's two n-lane-tiles for the pair-chunk (pair-interleaved +// transposed bf16, see PackedOperand::tile_block); tmm3/tmm4 = the matching fp32 +// accumulators. Two n-tiles in flight so their dpbf16ps chains are independent of each +// other (both read tmm0, but write distinct accumulators), per the task's "several +// accumulator tiles overlap" requirement. Named here for the tile-config array indexing +// below (ordinary C++, any expression is fine); every `_tile_*` call past this point +// uses the literal 0/1/2/3/4 directly, because gcc's tile intrinsics are macros that +// stringify their register argument (`#dst`), so a named constant used there would +// stringify to its identifier text, not a digit, and fail to assemble. +enum Tmm : int { kTmmA = 0, kTmmB0 = 1, kTmmB1 = 2, kTmmAcc0 = 3, kTmmAcc1 = 4 }; + +// gcc 11's _tile_loadd/_tile_stored/_tile_loadconfig macros expand to inline asm with no +// memory operand and no clobber, so the optimizer believes they neither read nor write +// memory: a plain C++ store just before a load, or a plain C++ read just after a store, +// through the same address has no ordering guarantee and can be reordered around the +// tile instruction (sunk store -> stale/garbage read; hoisted read -> stale/garbage +// value), independent of any `volatile` on the asm itself (that only orders the asm +// relative to *other* volatile operations, not to ordinary memory accesses). Every tile +// memory access in this file goes through one of the wrappers below instead of the bare +// intrinsic, so the dependency is a real operand or clobber the compiler understands +// rather than an accident of how today's code generation happens to schedule things. +// `_tile_zero`/`_tile_dpbf16ps`/`_tile_release` touch only tile *registers*, never +// memory, so they keep using the compiler's intrinsics unwrapped. + +// Loads the 64-byte tile configuration from `cfg`, naming the whole struct as the +// input memory this reads (not just a pointer-sized slice of it, which is what gcc's own +// _tile_loadconfig macro's cast effectively does): every `cfg.rows[]`/`cfg.colsb[]` store +// before this call is now visibly a dependency, not an accident of scheduling. +inline void tile_loadconfig_safe(const TileConfig& cfg) { + __asm__ volatile("ldtilecfg\t%0" ::"m"(cfg)); +} + +// One generated function per literal tile number actually used: the mnemonic's register +// is spelled into the asm text (`#tmm`), which gcc's own tile intrinsics require to be a +// literal token, so a runtime tile-number parameter is not an option. Kept local to this +// namespace: the macro only generates definitions used a few lines below it, nothing +// external depends on it (include/svs/multi-arch/AGENTS.md's rule is about a macro whose +// defining header is absent from a TU that uses it, which does not apply to a macro +// defined and consumed in the same file). +// +// _strided takes a runtime stride (A's row pitch, which varies with `d`) and so cannot +// name a compile-time memory extent; its "memory" clobber is a full barrier instead, +// forcing every pending store to complete before the load and every cached value to be +// re-read after it. +#define SVS_AMX_DEFINE_TILE_LOADD_STRIDED(tmm) \ + inline void tile_loadd_tmm##tmm##_strided(const void* base, long stride) { \ + __asm__ volatile("{tileloadd\t(%0,%1,1), %%tmm" #tmm "|tileloadd\t%%tmm" #tmm \ + ", [%0+%1*1]}" ::"r"(base), \ + "r"(stride) \ + : "memory"); \ + } + +// _fixed is for a source whose extent is a compile-time constant 1024 bytes (B's tile +// blocks, always a full 16x64-byte tile; the local zero-padded tail buffer, likewise +// 16x64 bytes): the dummy "m" operand (unused in the asm text) names that exact memory as +// read, so the compiler orders this correctly against its last write without needing a +// blanket barrier. +#define SVS_AMX_DEFINE_TILE_LOADD_FIXED(tmm) \ + inline void tile_loadd_tmm##tmm##_fixed(const void* base) { \ + __asm__ volatile( \ + "{tileloadd\t(%0,%1,1), %%tmm" #tmm "|tileloadd\t%%tmm" #tmm \ + ", [%0+%1*1]}" ::"r"(base), \ + "r"(64L), \ + "m"(*reinterpret_cast(base)) \ + ); \ + } + +// Stores are always to a caller-owned 16x16 fp32 buffer (1024 bytes, compile-time +// constant): the dummy "=m" output operand tells the compiler this asm writes that +// memory, so a later plain read of the same buffer is ordered after the store instead of +// possibly being hoisted before it. +#define SVS_AMX_DEFINE_TILE_STORED_FIXED(tmm) \ + inline void tile_stored_tmm##tmm##_fixed(void* base) { \ + __asm__ volatile("{tilestored\t%%tmm" #tmm \ + ", (%1,%2,1)|tilestored\t[%1+%2*1], %%tmm" #tmm "}" \ + : "=m"(*reinterpret_cast(base)) \ + : "r"(base), "r"(64L)); \ + } + +SVS_AMX_DEFINE_TILE_LOADD_STRIDED(0) +SVS_AMX_DEFINE_TILE_LOADD_FIXED(0) +SVS_AMX_DEFINE_TILE_LOADD_FIXED(1) +SVS_AMX_DEFINE_TILE_LOADD_FIXED(2) +SVS_AMX_DEFINE_TILE_STORED_FIXED(3) +SVS_AMX_DEFINE_TILE_STORED_FIXED(4) + +#undef SVS_AMX_DEFINE_TILE_LOADD_STRIDED +#undef SVS_AMX_DEFINE_TILE_LOADD_FIXED +#undef SVS_AMX_DEFINE_TILE_STORED_FIXED + +inline void configure_tiles(uint8_t a_acc_rows) { + TileConfig cfg; + cfg.palette_id = 1; + cfg.rows[kTmmA] = a_acc_rows; + cfg.colsb[kTmmA] = 64; + cfg.rows[kTmmAcc0] = a_acc_rows; + cfg.colsb[kTmmAcc0] = 64; + cfg.rows[kTmmAcc1] = a_acc_rows; + cfg.colsb[kTmmAcc1] = 64; + // B's tiles are always a full 16 (pair-)rows: PackedOperand::tile_block zero-pads + // every tile block to 16x16, so there is no partial-row case to configure for them. + cfg.rows[kTmmB0] = 16; + cfg.colsb[kTmmB0] = 64; + cfg.rows[kTmmB1] = 16; + cfg.colsb[kTmmB1] = 64; + tile_loadconfig_safe(cfg); +} + +// Computes the dot products of up to 16 rows of A (starting at `a_row0`, already known +// not to read past `d` columns of any row via the tail copy below) against B's n-tiles +// `nt0` (and `nt1` when `has_nt1`) for every k-chunk, leaving the result in tmm3 (and +// tmm4). Tiles must already be configured by configure_tiles() for this call's row count. +// +// A's tail k-chunk (the last one, when d is not a multiple of 32) is copied through a +// zero-padded stack buffer rather than loaded directly: loading straight from `a_row0` +// would read up to 31 bf16 elements past column d-1 of the block's last real row, which +// is caller memory this kernel does not own past `d`. +inline void accumulate_group( + const svs::BFloat16* a_row0, + size_t rows, + size_t stride_bytes, + size_t full_k_chunks, + size_t k_tail, + const PackedOperand& b, + size_t nt0, + bool has_nt1, + size_t nt1 +) { + _tile_zero(3); + if (has_nt1) { + _tile_zero(4); + } + const auto* base = reinterpret_cast(a_row0); + for (size_t kt = 0; kt < full_k_chunks; ++kt) { + tile_loadd_tmm0_strided(base + kt * 64, static_cast(stride_bytes)); + tile_loadd_tmm1_fixed(b.tile_block(nt0, kt)); + _tile_dpbf16ps(3, 0, 1); + if (has_nt1) { + tile_loadd_tmm2_fixed(b.tile_block(nt1, kt)); + _tile_dpbf16ps(4, 0, 2); + } + } + if (k_tail > 0) { + alignas(64) uint16_t tail[16][32] = {}; + for (size_t r = 0; r < rows; ++r) { + std::memcpy(tail[r], base + r * stride_bytes + full_k_chunks * 64, k_tail * 2); + } + tile_loadd_tmm0_fixed(tail); + tile_loadd_tmm1_fixed(b.tile_block(nt0, full_k_chunks)); + _tile_dpbf16ps(3, 0, 1); + if (has_nt1) { + tile_loadd_tmm2_fixed(b.tile_block(nt1, full_k_chunks)); + _tile_dpbf16ps(4, 0, 2); + } + } +} + +// Shared row-block/n-tile-group driver for both output forms below: walks A 16 rows at a +// time (reconfiguring tiles once more only for a final row-block narrower than 16, never +// per row-block in the steady state) and B 32 lanes (two n-tiles) at a time, handing each +// block's two 16x16 fp32 results to `emit(row0, rows, nt0, has_nt1, nt1, c0, c1)`. +template +inline void for_each_block( + const svs::BFloat16* a, size_t m, size_t d, const PackedOperand& b, Emit&& emit +) { + size_t n_tiles = b.n_tiles(); + size_t full_k_chunks = d / 32; + size_t k_tail = d % 32; + size_t stride_bytes = d * 2; + + TileGuard guard; + configure_tiles(16); + bool configured_full = true; + + for (size_t i = 0; i < m; i += 16) { + size_t rows = std::min(16, m - i); + if (rows < 16 && configured_full) { + configure_tiles(static_cast(rows)); + configured_full = false; + } + const svs::BFloat16* row0 = a + i * d; + for (size_t nt0 = 0; nt0 < n_tiles; nt0 += 2) { + bool has_nt1 = nt0 + 1 < n_tiles; + size_t nt1 = has_nt1 ? nt0 + 1 : nt0; + alignas(64) float c0[16][16]; + alignas(64) float c1[16][16]; + accumulate_group( + row0, rows, stride_bytes, full_k_chunks, k_tail, b, nt0, has_nt1, nt1 + ); + tile_stored_tmm3_fixed(c0); + if (has_nt1) { + tile_stored_tmm4_fixed(c1); + } + emit(i, rows, nt0, has_nt1, nt1, c0, c1); + } + } +} + +// AMX table kernel for a BFloat16 A against a tile-packed B: writes out[i*n+j] exactly +// like the generic/AVX-512 forms, with the same values (summed in a +// different order, hence InnerProductMatrix's tolerance-based tests, not a bit-exact +// one). +inline void +amx_table_bf16(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + for_each_block( + a, + m, + b.d(), + b, + [&](size_t row0, + size_t rows, + size_t nt0, + bool has_nt1, + size_t nt1, + auto& c0, + auto& c1) { + size_t lanes0 = std::min(16, n - nt0 * 16); + for (size_t r = 0; r < rows; ++r) { + float* dst = out + (row0 + r) * n + nt0 * 16; + std::memcpy(dst, c0[r], lanes0 * sizeof(float)); + } + if (has_nt1) { + size_t lanes1 = std::min(16, n - nt1 * 16); + for (size_t r = 0; r < rows; ++r) { + float* dst = out + (row0 + r) * n + nt1 * 16; + std::memcpy(dst, c1[r], lanes1 * sizeof(float)); + } + } + } + ); +} + +// AMX column-max kernel for a BFloat16 A against a tile-packed B: folds each block's two +// 16x16 tiles into the running per-lane maximum with AVX-512 (the AMX level implies +// AVX-512F), reusing the same row-then-lane reduction shape as +// InnerProductMatrixImpl::column_max above. +inline void +amx_column_max_bf16(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + size_t n = b.n(); + std::fill(out, out + n, -std::numeric_limits::infinity()); + if (m == 0) { + return; + } + for_each_block( + a, + m, + b.d(), + b, + [&](size_t /*row0*/, + size_t rows, + size_t nt0, + bool has_nt1, + size_t nt1, + auto& c0, + auto& c1) { + auto fold = [&](size_t nt, auto& c) { + size_t lanes = std::min(16, n - nt * 16); + alignas(64) float colmax[16] = {}; + for (size_t j = 0; j < 16; ++j) { + float best = -std::numeric_limits::infinity(); + for (size_t r = 0; r < rows; ++r) { + best = std::max(best, c[r][j]); + } + colmax[j] = best; + } + for (size_t j = 0; j < lanes; ++j) { + out[nt * 16 + j] = std::max(out[nt * 16 + j], colmax[j]); + } + }; + fold(nt0, c0); + if (has_nt1) { + fold(nt1, c1); + } + } + ); +} + +} // namespace inner_product_matrix_amx_detail + +template struct InnerProductMatrixImpl { + // Never the AMX precision case (A is float32, not bf16): one source of truth is the + // AVX-512 kernel. + SVS_NOINLINE static void + build_packed(const float* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + } + + SVS_NOINLINE static void + build_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + } + + SVS_NOINLINE static void + build_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + if (m == 0 || b.n() == 0) { + return; + } + if (!b.has_tile_layout()) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + return; + } + inner_product_matrix_amx_detail::amx_table_bf16(a, m, b, out); + } + + // `a` is always float32 here (InnerProductMatrix::compute's only entry point to this + // overload): the AMX precision case never applies, so this is a plain forward. + SVS_NOINLINE static void + build(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + InnerProductMatrixImpl::build(a, m, b, n, d, out); + } + + SVS_NOINLINE static void + column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::column_max(a, m, b, out); + } + + SVS_NOINLINE static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + InnerProductMatrixImpl::column_max(a, m, b, out); + } + + SVS_NOINLINE static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + if (!b.has_tile_layout()) { + InnerProductMatrixImpl::column_max(a, m, b, out); + return; + } + inner_product_matrix_amx_detail::amx_column_max_bf16(a, m, b, out); + } +}; +#endif + +#endif // defined(__x86_64__) + +#if defined(__x86_64__) +extern template struct InnerProductMatrixImpl; +extern template struct InnerProductMatrixImpl; +#endif +// Gated on the build-wide SVS_AMX_COMPILED_IN, not __x86_64__ alone: amx.cpp is only built +// when the compiler accepts -mamx-tile -mamx-bf16 (cmake/multi-arch.cmake's +// SVS_COMPILER_SUPPORTS_AMX check), compiled at -march=cascadelake plus those two flags, +// not at -march=sapphirerapids; when that check fails, no TU anywhere provides +// `template struct InnerProductMatrixImpl;`'s definition, and an +// unconditional `extern template` here would then be an unresolved symbol at link time +// the moment anything calls InnerProductMatrix, not just when AMX runs. +#if SVS_AMX_COMPILED_IN +extern template struct InnerProductMatrixImpl; +#endif + +/// Public entry points for the InnerProductMatrixImpl kernel family: the only place that +/// picks an ISA for this kernel. A caller that needs this kernel's output names the form +/// it wants (`compute` for the table, `column_max` for the column-wise maximum) and never +/// names `InnerProductMatrixImpl` or an `AVX_AVAILABILITY` enumerator itself; the layer +/// that owns the ISA choice (this class) is the only one that repeats it. +class InnerProductMatrix { + public: + /// Dense inner-product table between a large row-major matrix and a small row-major + /// matrix, both float32: out[i * n + j] = for i in [0, m), j in [0, n). + /// + /// @param a Row-major "A" operand, m rows of d floats each; the large, streamed + /// operand (e.g. all centroids). + /// @param m Number of rows of `a`. 0 is valid and leaves `out` untouched. + /// @param b Row-major "B" operand, n rows of d floats each; the small operand (e.g. + /// one query's tokens). Packed once per call into a thread_local PackedOperand + /// this class owns, so `b` itself is read once and not retained. + /// @param n Number of rows of `b`. 0 is valid and leaves `out` untouched. + /// @param d Row width shared by `a` and `b`. + /// @param out Row-major output buffer, caller-owned, at least m * n floats: + /// out[i * n + j] holds . Untouched when m == 0 or n == 0. + static void + compute(const float* a, size_t m, const float* b, size_t n, size_t d, float* out) { + if (m == 0 || n == 0) { + return; + } + dispatch([&](auto avx) { + InnerProductMatrixImpl::build(a, m, b, n, d, out); + }); + } + + /// Column-wise maximum of the same table `compute` would produce, for a float32 A, + /// against an already-packed B: out[j] = max over i of , for j in [0, n). + /// No table is ever materialized. + /// + /// @param a Row-major "A" operand, m rows of `b.d()` floats each; the large, + /// streamed operand (e.g. one document's tokens). + /// @param m Number of rows of `a`. 0 is valid: every lane of `out` is set to + /// -infinity, matching an empty A having no row to contribute a maximum. + /// @param b A `PackedOperand` already packed by the caller (e.g. once per query), + /// read but not modified. + /// @param out Caller-owned output buffer of exactly `b.n()` floats, overwritten in + /// full (every lane, including when m == 0): out[j] holds the column maximum + /// for j in [0, b.n()). + static void column_max(const float* a, size_t m, const PackedOperand& b, float* out) { + dispatch([&](auto avx) { + InnerProductMatrixImpl::column_max(a, m, b, out); + }); + } + + /// Same contract as the float32 `column_max` above, for a svs::Float16 A: converts + /// one row-block of `a` at a time into a float32 scratch and reduces identically. + static void + column_max(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + dispatch([&](auto avx) { + InnerProductMatrixImpl::column_max(a, m, b, out); + }); + } + + /// Same contract as the float32 `column_max` above, for a svs::BFloat16 A. + static void + column_max(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + dispatch([&](auto avx) { + InnerProductMatrixImpl::column_max(a, m, b, out); + }); + } + + /// Dense inner-product table (same layout as `compute`) between a large row-major A + /// and an already-packed B, for an A element type other than float32. Unlike + /// `compute`, this takes a `PackedOperand` the caller packed itself, so the caller + /// controls B's precision (e.g. `PackedOperand::pack_rounded_to_bf16` for a bf16 A, so + /// every ISA level sees the same rounded values on both operands). + /// + /// @param a Row-major "A" operand, m rows of `b.d()` elements each. + /// @param m Number of rows of `a`. 0 is valid and leaves `out` untouched. + /// @param b A `PackedOperand` already packed by the caller, read but not modified. + /// @param out Row-major output buffer, caller-owned, at least m * b.n() floats. + static void + compute_packed(const svs::BFloat16* a, size_t m, const PackedOperand& b, float* out) { + if (m == 0 || b.n() == 0) { + return; + } + dispatch([&](auto avx) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + }); + } + + /// Same contract as `compute_packed` above, for a svs::Float16 A. + static void + compute_packed(const svs::Float16* a, size_t m, const PackedOperand& b, float* out) { + if (m == 0 || b.n() == 0) { + return; + } + dispatch([&](auto avx) { + InnerProductMatrixImpl::build_packed(a, m, b, out); + }); + } + + private: + // Picks the ISA once per call and invokes `f` with that choice as a compile-time + // constant (an std::integral_constant, so `InnerProductMatrixImpl` below is a valid template argument), so every public entry point above + // writes this exhaustive switch zero times of its own. + template static void dispatch(F&& f) { + switch (select_isa()) { + case AVX_AVAILABILITY::AMX: + // select_isa() only returns AMX when SVS_AMX_COMPILED_IN (amx.cpp was + // built), so the symbol this case names exists only then; this case + // itself must stay unconditional so -Wswitch under -Werror still catches + // a 5th enumerator added later. Without SVS_AMX_COMPILED_IN this case is + // unreachable (select_isa() never returns it) and falls back to AVX512, + // never naming a specialization that cannot exist in this build. +#if SVS_AMX_COMPILED_IN + f(std::integral_constant{}); +#else + f(std::integral_constant{}); +#endif + return; + case AVX_AVAILABILITY::AVX512: + f(std::integral_constant{}); + return; + case AVX_AVAILABILITY::AVX2: + f(std::integral_constant{}); + return; + case AVX_AVAILABILITY::NONE: + f(std::integral_constant{}); + return; + } + // Exhaustive switch over AVX_AVAILABILITY's enumerators, no default: -Wswitch + // under -Werror turns a 5th enumerator added later into a compile error here, + // rather than this call silently keeping today's dispatch. + __builtin_unreachable(); + } + + // AMX is checked first regardless of A's element type or B's packing: every + // InnerProductMatrixImpl member handles "not actually usable for this call" + // itself (forwarding to AVX512), so this one-branch-per-call choice never needs to + // know the call's operand types. Gated on SVS_AMX_COMPILED_IN so a build that never + // compiled amx.cpp never returns an enumerator dispatch() cannot safely name. + static AVX_AVAILABILITY select_isa() { +#if SVS_AMX_COMPILED_IN + if (svs::detail::amx_tile_bf16_available()) { + return AVX_AVAILABILITY::AMX; + } +#endif + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + return AVX_AVAILABILITY::AVX512; + } + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + return AVX_AVAILABILITY::AVX2; + } + return AVX_AVAILABILITY::NONE; + } +}; + +} // namespace svs::distance diff --git a/include/svs/core/distance/maxsim.h b/include/svs/core/distance/maxsim.h new file mode 100644 index 000000000..8290f6f5d --- /dev/null +++ b/include/svs/core/distance/maxsim.h @@ -0,0 +1,143 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/vector_set.h" +#include "svs/core/distance/inner_product_matrix.h" +#include "svs/lib/bfloat16.h" +#include "svs/lib/exception.h" +#include "svs/lib/float16.h" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs::distance { + +/// Sum over query tokens of the max inner product over document tokens. Asymmetric; +/// larger is better; inputs are not normalised here. +/// +/// Per-query state, following the library's fix_argument protocol (like +/// QChamferExactQuery in gem.h): `fix_argument(query)` packs the query's tokens once into +/// a `PackedOperand` and sizes a member maxima buffer to the query's cardinality; every +/// `compute(document)` call that follows scores one document against that packed query +/// through `InnerProductMatrix::column_max`, writing into the member buffer rather than a +/// thread_local one, so compute performs no thread_local lookup per document. That is +/// safe precisely because `implicit_broadcast` is false: the packed query and the maxima +/// buffer are both ordinary (non-shared) state, so each search thread's shallow_copy of +/// this functor owns its own copy of both rather than sharing either one. +/// `must_fix_argument` is set because `compute` is meaningless without a prior +/// `fix_argument`. +/// +/// `DocElement` (default `float`) is the element type of the documents this instance +/// scores: `float` and `svs::Float16` documents are packed unrounded, bit for bit as +/// before this type gained a template parameter; `svs::BFloat16` documents are packed +/// through `PackedOperand::pack_rounded_to_bf16`, which both matches the bf16 precision +/// those documents are already held at and is the one packing `InnerProductMatrix`'s AMX +/// level requires to route a bf16 A through the tile kernel (see inner_product_matrix.h). +/// Baking the document element type into the type, rather than branching on it inside +/// `fix_argument`, means a document of the wrong element type fails to compile (`compute`'s +/// `requires` clause below) instead of silently mismatching rounding at run time, and +/// means `fix_argument` packs the query exactly once regardless of `DocElement` rather than +/// packing twice to keep both a rounded and an unrounded copy on hand. The non-template +/// spelling `MaxSim` keeps meaning `MaxSim` for existing callers. +/// +/// The query (fix_argument's argument) is a `svs::data::ContiguousVectorSet` of float32 +/// tokens. The document (compute's second argument) is a `svs::data::ContiguousVectorSet` +/// of `DocElement` tokens with the same dimensionality as the query; a mismatch throws +/// (checked once per document, which is negligible next to the kernel it guards). An +/// empty document scores -infinity, strictly worse than any real score. +template class MaxSim { + static_assert( + std::same_as || std::same_as || + std::same_as, + "MaxSim's document element type must be float, svs::Float16 or svs::BFloat16" + ); + + public: + using compare = std::greater<>; + + static constexpr bool implicit_broadcast = false; + static constexpr bool must_fix_argument = true; + static constexpr std::string_view name = "maxsim"; + + /// Packs `query`'s tokens (float32, contiguous) into this object's `PackedOperand`, + /// reusing its storage across calls. `query.cardinality() == 0` is a caller error, + /// asserted only: a MaxSim score over zero query tokens is undefined, not merely + /// small, so there is nothing for `compute` to usefully return. + template + requires std::same_as + void fix_argument(const S& query) { + assert(query.cardinality() > 0); + if constexpr (std::same_as) { + packed_.pack_rounded_to_bf16( + query.data(), query.cardinality(), query.dimensions() + ); + } else { + packed_.pack(query.data(), query.cardinality(), query.dimensions()); + } + maxima_.resize(packed_.n()); + } + + /// `query` is unused: `fix_argument` already consumed it into `packed_`, and every + /// `compute` call in a search shares that one query's packing, matching the two-arg + /// overload `distance::compute` dispatches to for a `must_fix_argument` functor + /// (see QChamferExactQuery::compute in gem.h for the same shape). Non-const: writes + /// into the member maxima buffer `fix_argument` sized. + template + requires std::same_as + float compute(const Query& /*query*/, const D& document) { + if (document.dimensions() != packed_.d()) { + throw ANNEXCEPTION( + "MaxSim::compute: document dimension {} does not match the fixed " + "query's dimension {}", + document.dimensions(), + packed_.d() + ); + } + size_t n_doc = document.cardinality(); + if (n_doc == 0) { + return -std::numeric_limits::infinity(); + } + + InnerProductMatrix::column_max(document.data(), n_doc, packed_, maxima_.data()); + + double total = 0.0; + for (float v : maxima_) { + total += static_cast(v); + } + return static_cast(total); + } + + private: + PackedOperand packed_; + std::vector maxima_; +}; + +template +inline constexpr bool operator==(const MaxSim&, const MaxSim&) { + return true; +} + +} // namespace svs::distance diff --git a/include/svs/core/distance/transport_simplex.h b/include/svs/core/distance/transport_simplex.h new file mode 100644 index 000000000..7815213c2 --- /dev/null +++ b/include/svs/core/distance/transport_simplex.h @@ -0,0 +1,1139 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Primal network simplex for the dense balanced transportation problem, specialized to a +// bipartite tree with no artificial root and no capacities. Northwest-corner +// initialization and the tree-array representation (parent/thread/rev_thread/succ_num/ +// last_succ) and incremental pivot update (findJoinNode/findLeavingArc/changeFlow/ +// updateTreeStructure/updatePotential) are ported from LEMON's network_simplex, by way of +// Nicolas Bonneel's single-file adaptation vendored into POT; see the notices below. The +// float-proposes/double-confirms pricing split and the strongly-feasible northwest-corner +// start (replacing LEMON/POT's artificial-root big-M start) are this file's own. Problem +// shape (quantized ground distance over merged centroid weights) follows Bonneel et al., +// "Displacement Interpolation Using Lagrangian Mass Transport", SIGGRAPH Asia 2011. +// +// Portions of this file are derived from: +// +// POT (Python Optimal Transport), `ot/lp/`, tag 0.9.5. +// https://github.com/PythonOT/POT +// Copyright (c) 2016-2023 POT contributors. +// Licensed under the MIT License. +// +// which itself adapted, as `network_simplex_simple.h` (Nicolas Bonneel, 2013), from +// LEMON's `network_simplex.h`: +// +// Copyright (C) 2003-2010 +// Egervary Jeno Kombinatorikus Optimalizalasi Kutatocsoport +// (Egervary Research Group on Combinatorial Optimization, EGRES). +// +// Permission to use, modify and distribute this software is granted +// provided that this copyright notice appears in all copies. For +// precise terms see the accompanying LICENSE file. +// +// This software is provided "AS IS" with no warranty of any kind, +// express or implied, and with no claim as to its suitability for any +// purpose. + +#include "svs/core/distance/distance_core.h" +#include "svs/core/distance/transport_simplex_pricing.h" +#include "svs/core/distance/transport_simplex_reference.h" +#include "svs/lib/meta.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs::distance::transport_detail { + +// uint16_t node ids halve the tree arrays' footprint for the common small-problem bucket; +// Dynamic has no such bound so ids must cover an arbitrary ua+ub. A pure type trait (no +// function bodies), so it carries none of the cross-TU weak-symbol risk the rest of this +// file's per-ISA machinery is keyed on TransportSimplex to avoid. +template struct TransportIndexTraits { + using node_type = uint32_t; + using arc_type = uint32_t; +}; +template <> struct TransportIndexTraits<256> { + using node_type = uint16_t; + using arc_type = uint32_t; +}; + +} // namespace svs::distance::transport_detail + +namespace svs::distance { + +/// Column order of `TransportSimplex::solve`'s northwest-corner start. `NorthWest` walks +/// columns in id order. `CheapestRow` groups them by cheapest row first (ties by ascending +/// column id): one extra compare per cost cell for fewer pivots. Both reach the same +/// optimum; only the pivot count differs. +enum class TransportStart { NorthWest, CheapestRow }; + +/// Solves the dense balanced transportation problem exactly to within +/// `error_bound(ua, ub, max_cost, total_mass)`. Same contract as +/// `TransportSimplexReference::solve`, plus: `CostFn` must return `float` (pricing stores +/// costs as float, exact for callers whose costs already are, e.g. `CentroidTable`). +/// +/// Orients to put the smaller side in rows, then starts from a northwest-corner walk over +/// columns in the order `TransportStart` picks. Both starts share one argument: the walk +/// reasons only about positions (row i, column position p), never a column's id, and rows +/// keep their order, so the root is row 0 and any column permutation stays strongly +/// feasible. Strong feasibility means every zero-flow tree arc has a *source* child (points +/// toward the root, LEMON's convention); the only zero-flow arc the start ever adds is a +/// value tie's degenerate arc, and that arc's child is always a fresh, untouched row, never +/// a sink. Every arc whose child is a sink instead carries that sink's own fresh demand (or +/// a fresh row's full supply), so it is always strictly positive. Each pivot picks the +/// leaving arc by Cunningham's rule (the *last* blocking arc on the cycle in the entering +/// arc's own direction), which preserves this invariant under degenerate pivots and so +/// guarantees termination regardless of which negative-reduced-cost arc enters next. +/// Pricing scans the dense cost matrix in a block-cyclic resumable order in float (`Avx` +/// selects the row scan's SIMD width); a float candidate is only pivoted once its reduced +/// cost is confirmed negative in double (exactly recoverable, since the cost matrix is +/// itself stored as float), and pricing falls back to double (and a from-scratch +/// potential recompute) once a full float pass finds nothing, or once a float candidate +/// fails double confirmation. Below `narrow_index_max_side`, node ids are `uint16_t` +/// instead of `uint32_t`, halving the tree arrays' footprint; callers never see this. +/// +/// Every helper this type needs (potentials, tolerances, the dense workspace, ...) is a +/// member keyed on `Avx` (and, where it needs a node-id width, on `Impl`'s `MaxSide` +/// too), rather than a free function: an inline free function not keyed on `Avx` becomes +/// one weak symbol shared across the differently-`-march`-compiled translation units that +/// explicitly instantiate this type per ISA (see `gem.h`, near `merge_sorted_duplicates`). +template struct TransportSimplex { + /// Upper bound on `|solve() - exact optimum|`. + static double error_bound(size_t ua, size_t ub, double max_cost, double total_mass) { + return optimality_gap_bound(ua, ub, max_cost, total_mass); + } + +#if SVS_ENABLE_STATS + /// Pivots performed by this thread's solves since the caller last reset it. Stats + /// builds only (`SVS_EXPERIMENTAL_ENABLE_STATS`). Per ISA instantiation: read it + /// through the same `Avx` the dispatched solver used, or it reads 0. + static size_t& thread_pivot_count() { + thread_local size_t count = 0; + return count; + } +#endif + + template + static double solve( + size_t ua, + const double* supply, + size_t ub, + const double* demand, + CostFn&& cost, + TransportStart start = TransportStart::NorthWest + ) + requires std::same_as, float> + { + if (std::max(ua, ub) <= narrow_index_max_side) { + return Impl::solve_impl( + ua, supply, ub, demand, std::forward(cost), start + ); + } + return Impl::solve_impl( + ua, supply, ub, demand, std::forward(cost), start + ); + } + + private: + static constexpr size_t narrow_index_max_side = 256; + static_assert( + 2 * narrow_index_max_side <= + std::numeric_limits::node_type>::max() + ); + + // node is a source iff node < m; fixes both the entering arc's direction (row -> col) + // and every tree arc's direction (LEMON's generic `_forward` collapses to this one + // comparison in a bipartite tree with no artificial root). + static bool is_source(size_t node, size_t m) { return node < m; } + + // Potential magnitude bound: a tree path sum of at most n-1 signed costs. + static double potential_bound(size_t n, double max_cost) { + return static_cast(n > 0 ? n - 1 : 0) * max_cost; + } + + // Tier-2/fallback threshold for an exact (walk_potential-based) reduced cost; see the + // commit body for the derivation. A candidate beating -walk_tol is truly negative + // independent of any potential drift. + static double walk_tol(size_t n, double max_cost) { + double eps_mach_d = std::numeric_limits::epsilon(); + return 2.0 * static_cast(n) * eps_mach_d * + (max_cost + potential_bound(n, max_cost)); + } + + // Tier-1 fast-accept threshold for a *drifted* double reduced cost (current u_d/v_d, + // stale by up to `recompute_period` pivots since the last fresh recompute): + // drift_tol = 2*(n-1)*R*9*eps*P + walk_tol, the accumulated per-pivot drift plus the + // same fresh-recompute-and-subtraction rounding walk_tol already bounds. + static double drift_tol(size_t n, double max_cost, size_t recompute_period) { + constexpr double c = 9.0; + double eps_mach_d = std::numeric_limits::epsilon(); + double per_pivot_per_arc = c * eps_mach_d * potential_bound(n, max_cost); + double accumulated_drift = 2.0 * static_cast(n > 0 ? n - 1 : 0) * + static_cast(recompute_period) * + per_pivot_per_arc; + return accumulated_drift + walk_tol(n, max_cost); + } + + // Upper bound on |solve() - exact optimum|: solve() stops once every fresh r >= + // -walk_tol, and a fresh r differs from the exact reduced cost by at most walk_tol + // (the same bound, applied once more), so the exact value is >= -2*walk_tol; plus + // flow_rounding (the final flow recompute's rounding) and final_sum_rounding (the + // double rounding of summing and multiplying at most n-1 flow*cost terms). + static double + optimality_gap_bound(size_t ua, size_t ub, double max_cost, double total_mass) { + size_t n = ua + ub; + double eps_mach_d = std::numeric_limits::epsilon(); + double n_d = static_cast(n); + double final_sum_rounding = + 2.0 * static_cast(n > 0 ? n - 1 : 0) * eps_mach_d * max_cost; + double flow_rounding = n_d * n_d * eps_mach_d * max_cost; + return (2.0 * walk_tol(n, max_cost) + flow_rounding + final_sum_rounding) * + total_mass; + } + + template struct Impl { + using NodeT = typename transport_detail::TransportIndexTraits::node_type; + + private: + // Scratch for one `solve` instantiation, grow-only so a steady-state + // solve (same shape or smaller) allocates nothing. Node ids: sources [0, m), + // sinks [m, m+k), m <= k. + struct SolverWorkspace { + std::vector cost_f; // m * padded_k, +inf in [k, padded_k) of every row + std::vector + price_f; // same shape; tree-arc cells hold +inf so pricing skips them + std::vector supply; // m; destructively consumed by the NW-corner build + std::vector demand; // k; destructively consumed by the NW-corner build + std::vector supply_marginal, + demand_marginal; // folded marginals, untouched + // Fold-before-orient scratch, each sized to k (>= both ua and ub): holds the + // caller's own supply/demand, pre-orientation, until m/k are known. + std::vector supply_folded, demand_folded; + // Cost-aware column order: col_min/col_argmin (per-column cheapest cost/row) + // feed col_order, a stable counting sort by col_argmin (scratch col_count). + std::vector col_min; + std::vector col_argmin; + std::vector col_order; + std::vector col_count; + std::vector child_sum; // final flow recompute scratch, size n + std::vector parent; // n = m+k; parent[0] (root) unused + std::vector thread; // preorder successor, circular over all n + std::vector rev_thread; // inverse of thread + std::vector succ_num; // subtree size, including self + std::vector last_succ; // last node of subtree, in preorder + std::vector flow_to_parent; // flow on the arc to parent[node] + std::vector u_d, v_d; // source/sink potentials, double master + std::vector u_f, v_f; // float mirrors, updated with every u_d/v_d + std::vector in_tree; // NW-corner bootstrap scratch + std::vector first_child, next_sibling, dfs_stack, + preorder; // bootstrap scratch + std::vector preorder_pos; // bootstrap scratch + std::vector + dirty_revs; // updateTreeStructure scratch, cleared (not freed) per pivot + std::vector walk_path; // walk_potential scratch, cleared per call + + void reserve(size_t m, size_t k, size_t padded_k) { + size_t n = m + k; + if (cost_f.size() < m * padded_k) { + cost_f.resize(m * padded_k); + price_f.resize(m * padded_k); + } + if (supply.size() < m) { + supply.resize(m); + supply_marginal.resize(m); + } + if (col_count.size() < m + 1) { + col_count.resize(m + 1); + } + if (demand.size() < k) { + demand.resize(k); + demand_marginal.resize(k); + supply_folded.resize(k); + demand_folded.resize(k); + col_min.resize(k); + col_argmin.resize(k); + col_order.resize(k); + } + if (parent.size() < n) { + parent.resize(n); + thread.resize(n); + rev_thread.resize(n); + succ_num.resize(n); + last_succ.resize(n); + flow_to_parent.resize(n); + in_tree.resize(n); + first_child.resize(n); + next_sibling.resize(n); + dfs_stack.resize(n); + preorder.resize(n); + preorder_pos.resize(n); + dirty_revs.reserve(n); + walk_path.reserve(n); + child_sum.resize(n); + } + if (u_d.size() < m) { + u_d.resize(m); + u_f.resize(m); + } + if (v_d.size() < padded_k) { + v_d.resize(padded_k); + v_f.resize(padded_k); + } + } + }; + + // The dense-matrix cell for the arc between a node and its tree parent: the row + // is whichever of the two is a source, the column the other's offset by m. + static size_t arc_cell(size_t m, size_t padded_k, NodeT node, NodeT parent) { + size_t row = is_source(node, m) ? static_cast(node) + : static_cast(parent); + size_t col = is_source(node, m) ? static_cast(parent) - m + : static_cast(node) - m; + return row * padded_k + col; + } + + // A node's pred-arc cost. Recomputed on demand instead of cached per node, since + // the tree-arc identity (node, parent[node]) already pins down the cell. + static float pred_cost_f( + const std::vector& cost_f, size_t padded_k, size_t m, NodeT node, NodeT p + ) { + return cost_f[arc_cell(m, padded_k, node, p)]; + } + + // Restores a leaving arc's price cell to its real cost once it leaves the tree. + static void unmark_tree_arc( + std::vector& price_f, + const std::vector& cost_f, + size_t padded_k, + size_t m, + NodeT node, + NodeT p + ) { + size_t cell = arc_cell(m, padded_k, node, p); + price_f[cell] = cost_f[cell]; + } + + // Exact potential for `node`: walk parent[] up to the root, then re-derive it + // root-down with recompute_potentials' same single-subtraction-per-step formula + // (so a walked value equals a freshly recomputed one bit for bit). + static double + walk_potential(SolverWorkspace& ws, size_t m, size_t padded_k, NodeT node) { + ws.walk_path.clear(); + for (NodeT cur = node; cur != 0; cur = ws.parent[cur]) { + ws.walk_path.push_back(cur); + } + double val = 0.0; + for (size_t idx = ws.walk_path.size(); idx-- > 0;) { + NodeT nd = ws.walk_path[idx]; + NodeT p = ws.parent[nd]; + double c = static_cast(pred_cost_f(ws.cost_f, padded_k, m, nd, p)); + val = c - val; + } + return val; + } + + // Builds thread/rev_thread/succ_num/last_succ from parent[] alone (the + // northwest-corner build only produces parent[]): a preorder walk, then subtree + // sizes via that preorder in reverse (a node's children, at higher preorder + // index, are already totalled by then). + static void build_tree_arrays(SolverWorkspace& ws, size_t n) { + constexpr NodeT no_node = std::numeric_limits::max(); + for (size_t v = 0; v < n; ++v) { + ws.first_child[v] = no_node; + } + for (size_t v = 1; v < n; ++v) { + NodeT p = ws.parent[v]; + ws.next_sibling[v] = ws.first_child[p]; + ws.first_child[p] = static_cast(v); + } + size_t top = 0; + ws.dfs_stack[top++] = 0; + size_t pre_count = 0; + while (top > 0) { + NodeT v = ws.dfs_stack[--top]; + ws.preorder[pre_count] = v; + ws.preorder_pos[v] = pre_count; + ++pre_count; + for (NodeT c = ws.first_child[v]; c != no_node; c = ws.next_sibling[c]) { + ws.dfs_stack[top++] = c; + } + } + for (size_t v = 0; v < n; ++v) { + ws.succ_num[v] = 1; + } + for (size_t idx = n; idx-- > 1;) { + NodeT v = ws.preorder[idx]; + ws.succ_num[ws.parent[v]] = + static_cast(ws.succ_num[ws.parent[v]] + ws.succ_num[v]); + } + for (size_t v = 0; v < n; ++v) { + size_t last_idx = + ws.preorder_pos[v] + static_cast(ws.succ_num[v]) - 1; + ws.last_succ[v] = ws.preorder[last_idx]; + } + for (size_t idx = 0; idx + 1 < n; ++idx) { + ws.thread[ws.preorder[idx]] = ws.preorder[idx + 1]; + ws.rev_thread[ws.preorder[idx + 1]] = ws.preorder[idx]; + } + ws.thread[ws.preorder[n - 1]] = 0; + ws.rev_thread[0] = ws.preorder[n - 1]; + } + + // Fills u_d/v_d/u_f/v_f from scratch via a thread (preorder) walk: a parent's + // potential is always known before its children's, since thread visits a node + // before its subtree. + static void + recompute_potentials(SolverWorkspace& ws, size_t m, size_t k, size_t padded_k) { + // Hoisted so the compiler doesn't reload each vector's base pointer every + // iteration (a store through one could otherwise alias another's data()). + const NodeT* const __restrict thread = ws.thread.data(); + const NodeT* const __restrict parent = ws.parent.data(); + double* const __restrict u_d = ws.u_d.data(); + float* const __restrict u_f = ws.u_f.data(); + double* const __restrict v_d = ws.v_d.data(); + float* const __restrict v_f = ws.v_f.data(); + u_d[0] = 0.0; + u_f[0] = 0.0F; + NodeT node = thread[0]; + size_t n = m + k; + for (size_t count = 1; count < n; ++count, node = thread[node]) { + NodeT p = parent[node]; + double c = + static_cast(pred_cost_f(ws.cost_f, padded_k, m, node, p)); + double val = (static_cast(node) < m) + ? c - v_d[static_cast(p) - m] + : c - u_d[p]; + if (static_cast(node) < m) { + u_d[node] = val; + u_f[node] = static_cast(val); + } else { + v_d[static_cast(node) - m] = val; + v_f[static_cast(node) - m] = static_cast(val); + } + } + } + + public: + template + static double solve_impl( + size_t ua, + const double* supply, + size_t ub, + const double* demand, + CostFn&& cost, + TransportStart start + ) { + if constexpr (MaxSide != svs::Dynamic) { + size_t largest = std::max(ua, ub); + if (largest > MaxSide) { + throw std::runtime_error( + "TransportSimplex<" + std::to_string(MaxSide) + + ">::solve: max(ua, ub) = " + std::to_string(largest) + + " exceeds the bound " + std::to_string(MaxSide) + ); + } + } + + bool transposed = ua > ub; + size_t m = transposed ? ub : ua; + size_t k = transposed ? ua : ub; + + thread_local SolverWorkspace ws; + size_t padded_k = transport_detail::pad_to_lanes(k); + ws.reserve(m, k, padded_k); + size_t n = m + k; + + // Folds before orienting, on the caller's own supply/demand: folding after + // swapping rows/columns for ua > ub would land the residual on the caller's + // last supply entry instead of its last demand entry. + transport_reference_detail::prepare_marginals( + ua, supply, ws.supply_folded.data(), ub, demand, ws.demand_folded.data() + ); + const double* supply_src = + transposed ? ws.demand_folded.data() : ws.supply_folded.data(); + const double* demand_src = + transposed ? ws.supply_folded.data() : ws.demand_folded.data(); + std::copy_n(supply_src, m, ws.supply.begin()); + std::copy_n(demand_src, k, ws.demand.begin()); + // Kept aside, untouched by the NW-corner walk below, so final_cost can + // recompute every tree flow fresh from these instead of the pivot loop's + // incrementally-accumulated flow_to_parent. + std::copy_n(ws.supply.begin(), m, ws.supply_marginal.begin()); + std::copy_n(ws.demand.begin(), k, ws.demand_marginal.begin()); + + double max_cost = 0.0; + bool cheapest_row = (start == TransportStart::CheapestRow); + if (cheapest_row) { + if (transposed) { + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + } else { + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + } + build_col_order(ws, m, k); + } else { + if (transposed) { + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + } else { + fill_cost_matrix(ws, m, k, padded_k, cost, max_cost); + } + // NorthWest's column order is the identity: the walk below always + // indexes through col_order, so fill it rather than branch the walk. + std::iota(ws.col_order.begin(), ws.col_order.begin() + k, NodeT{0}); + } + for (size_t j = k; j < padded_k; ++j) { + ws.v_d[j] = 0.0; + ws.v_f[j] = 0.0F; + } + std::copy_n(ws.cost_f.begin(), m * padded_k, ws.price_f.begin()); + + // Every zero-flow tree arc must have a source child (point toward the + // root); only the value-tie arc below is ever zero, and its child is a + // fresh, untouched (so later strictly positive) row. + std::fill( + ws.in_tree.begin(), ws.in_tree.begin() + static_cast(n), 0 + ); + ws.in_tree[0] = 1; + auto add_tree_arc = [&](size_t i, size_t j, double flow) { + size_t sink_node = m + j; + if (!ws.in_tree[sink_node]) { + ws.parent[sink_node] = static_cast(i); + ws.flow_to_parent[sink_node] = flow; + ws.in_tree[sink_node] = 1; + } else { + ws.parent[i] = static_cast(sink_node); + ws.flow_to_parent[i] = flow; + ws.in_tree[i] = 1; + } + ws.price_f[i * padded_k + j] = std::numeric_limits::infinity(); + }; + { + size_t i = 0; + // p is the walk's position among columns; j = col_order[p] is the real + // column id. The walk's correctness depends only on visiting order. + size_t p = 0; + size_t j = static_cast(ws.col_order[p]); + // True when row i was the side most recently entered fresh (so, at the + // final cell, is still untouched); false when column j was. + bool fresh_row = false; + while (true) { + bool last_row = (i == m - 1); + bool last_col = (p == k - 1); + if (last_row && last_col) { + // The fresh side's value is untouched and strictly positive; the + // other side's residual lives in supply/demand, not a tree arc. + double x = fresh_row ? ws.supply[i] : ws.demand[j]; + add_tree_arc(i, j, x); + ws.supply[i] -= x; + ws.demand[j] -= x; + break; + } + if (last_col) { + double x = ws.supply[i]; + add_tree_arc(i, j, x); + ws.supply[i] -= x; + ws.demand[j] -= x; + i += 1; + fresh_row = true; + } else if (last_row) { + double x = ws.demand[j]; + add_tree_arc(i, j, x); + ws.supply[i] -= x; + ws.demand[j] -= x; + p += 1; + j = static_cast(ws.col_order[p]); + fresh_row = false; + } else { + double x = std::min(ws.supply[i], ws.demand[j]); + add_tree_arc(i, j, x); + ws.supply[i] -= x; + ws.demand[j] -= x; + bool s_zero = ws.supply[i] <= 0.0; + bool d_zero = ws.demand[j] <= 0.0; + if (s_zero && d_zero) { + // Degenerate tie: the zero-flow arc gets a source child + // (row i+1, fresh), not a sink child, per the class doc. + add_tree_arc(i + 1, j, 0.0); + i += 1; + p += 1; + j = static_cast(ws.col_order[p]); + fresh_row = true; + } else if (s_zero) { + i += 1; + fresh_row = true; + } else { + p += 1; + j = static_cast(ws.col_order[p]); + fresh_row = false; + } + } + } + } + // parent[root] is never a real node; updateTreeStructure's last_succ walk + // relies on this exact sentinel to stop a chain that may pass through root. + ws.parent[0] = std::numeric_limits::max(); + build_tree_arrays(ws, n); + recompute_potentials(ws, m, k, padded_k); + + size_t recompute_period = n; + double drift_threshold = drift_tol(n, max_cost, recompute_period); + double walk_threshold = walk_tol(n, max_cost); + + size_t block_rows = static_cast(std::round( + std::sqrt(static_cast(m) * static_cast(k)) / + static_cast(k) + )); + block_rows = std::clamp(block_rows, 1, m); + size_t cursor_row = 0; + bool double_mode = false; + size_t pivots_since_recompute = 0; + + auto do_pivot = [&](size_t row, size_t col) { +#if SVS_ENABLE_STATS + ++thread_pivot_count(); +#endif + pivot(ws, m, padded_k, row, col); + ++pivots_since_recompute; + if (pivots_since_recompute >= recompute_period) { + recompute_potentials(ws, m, k, padded_k); + pivots_since_recompute = 0; + } + }; + + // No polynomial bound exists for primal network simplex iteration count, so + // this is a generous safety net against a latent bug, not an expected limit. + size_t max_iters = 1000 * n + 10000; + for (size_t iter = 0; iter < max_iters; ++iter) { + size_t row = 0, col = 0; + bool found = !double_mode ? find_entering( + ws.price_f, + ws.u_f, + ws.v_f, + m, + padded_k, + block_rows, + cursor_row, + 0.0F, + row, + col + ) + : find_entering( + ws.price_f, + ws.u_d, + ws.v_d, + m, + padded_k, + block_rows, + cursor_row, + 0.0, + row, + col + ); + if (found) { + double r_d = static_cast(ws.cost_f[row * padded_k + col]) - + ws.u_d[row] - ws.v_d[col]; + // Tier 1: drift can't have made this one look falsely negative. + if (r_d < -drift_threshold) { + do_pivot(row, col); + continue; + } + // Tier 2: borderline; an exact root-path walk settles it either way. + if (r_d < 0.0) { + NodeT i_node = static_cast(row); + NodeT j_node = static_cast(m + col); + double r_walk = + static_cast(ws.cost_f[row * padded_k + col]) - + walk_potential(ws, m, padded_k, i_node) - + walk_potential(ws, m, padded_k, j_node); + if (r_walk < -walk_threshold) { + do_pivot(row, col); + continue; + } + } + } + + // Tier 3: neither tier above gave a trustworthy candidate (or none was + // found); the only authoritative check left is a full double pass on + // freshly recomputed potentials, which needs no further walk to trust. + recompute_potentials(ws, m, k, padded_k); + pivots_since_recompute = 0; + size_t fresh_cursor = 0; + bool found3 = find_entering( + ws.price_f, + ws.u_d, + ws.v_d, + m, + padded_k, + block_rows, + fresh_cursor, + -walk_threshold, + row, + col + ); + if (!found3) { + return final_cost(ws, m, padded_k, n); + } + double_mode = true; + do_pivot(row, col); + } + throw std::runtime_error( + "TransportSimplex::solve: exceeded the iteration limit (" + + std::to_string(max_iters) + ") for ua=" + std::to_string(ua) + + ", ub=" + std::to_string(ub) + ); + } + + private: + // Fills cost_f/padding; Transposed avoids a runtime orientation check. TrackColMin + // also folds each column's cheapest cost/row into col_min/col_argmin (CheapestRow). + template + static void fill_cost_matrix( + SolverWorkspace& ws, + size_t m, + size_t k, + size_t padded_k, + CostFn& cost, + double& max_cost + ) { + for (size_t i = 0; i < m; ++i) { + float* row = &ws.cost_f[i * padded_k]; + for (size_t j = 0; j < k; ++j) { + double c = Transposed ? static_cast(cost(j, i)) + : static_cast(cost(i, j)); + row[j] = static_cast(c); + max_cost = std::max(max_cost, c); + if constexpr (TrackColMin) { + // Row 0 seeds col_min by assignment rather than a separate +inf + // pass; strict `<` below keeps the smallest row on a tie. + float cf = static_cast(c); + if (i == 0 || cf < ws.col_min[j]) { + ws.col_min[j] = cf; + ws.col_argmin[j] = static_cast(i); + } + } + } + for (size_t j = k; j < padded_k; ++j) { + row[j] = std::numeric_limits::infinity(); + } + } + } + + // Stable counting sort of columns by col_argmin, ties kept in ascending column + // id: col_count's prefix sums give each row's bucket start before the fill pass. + static void build_col_order(SolverWorkspace& ws, size_t m, size_t k) { + std::fill( + ws.col_count.begin(), + ws.col_count.begin() + static_cast(m + 1), + NodeT{0} + ); + for (size_t j = 0; j < k; ++j) { + ++ws.col_count[static_cast(ws.col_argmin[j]) + 1]; + } + for (size_t r = 0; r < m; ++r) { + ws.col_count[r + 1] = + static_cast(ws.col_count[r + 1] + ws.col_count[r]); + } + for (size_t j = 0; j < k; ++j) { + size_t row = ws.col_argmin[j]; + ws.col_order[ws.col_count[row]] = static_cast(j); + ws.col_count[row] = static_cast(ws.col_count[row] + 1); + } + } + + // Block-cyclic resumable row scan: returns the first block (of `block_rows` + // rows, cycling from `cursor`) with an entry below `threshold`, or false once + // every row has been scanned once with none found. + template + static bool find_entering( + const std::vector& price_f, + const std::vector& u_pot, + const std::vector& v_pot, + size_t m, + size_t padded_k, + size_t block_rows, + size_t& cursor, + PotT threshold, + size_t& out_row, + size_t& out_col + ) { + size_t row = cursor; + size_t rows_scanned = 0; + while (rows_scanned < m) { + PotT block_best_val = threshold; + size_t block_best_row = m; + uint32_t block_best_col = 0; + for (size_t b = 0; b < block_rows && rows_scanned < m; + ++b, ++rows_scanned) { + PotT val = threshold; + uint32_t col = 0; + transport_detail::RowMin::min_in_row( + &price_f[row * padded_k], + v_pot.data(), + padded_k, + u_pot[row], + val, + col + ); + if (val < block_best_val) { + block_best_val = val; + block_best_row = row; + block_best_col = col; + } + row = (row + 1) % m; + } + if (block_best_row != m) { + out_row = block_best_row; + out_col = block_best_col; + cursor = row; + return true; + } + } + return false; + } + + // Recomputes every tree flow from the folded marginals in post-order instead of + // the pivot loop's incrementally-drifted flow_to_parent; see the commit body. + static double final_cost(SolverWorkspace& ws, size_t m, size_t padded_k, size_t n) { + NodeT node = 0; + for (size_t idx = 0; idx < n; ++idx, node = ws.thread[node]) { + ws.preorder[idx] = node; + } + std::fill( + ws.child_sum.begin(), ws.child_sum.begin() + static_cast(n), 0.0 + ); + for (size_t idx = n; idx-- > 1;) { + NodeT nd = ws.preorder[idx]; + NodeT p = ws.parent[nd]; + double marginal = is_source(nd, m) + ? ws.supply_marginal[nd] + : ws.demand_marginal[static_cast(nd) - m]; + double flow = marginal - ws.child_sum[nd]; + ws.flow_to_parent[nd] = flow; + ws.child_sum[p] += flow; + } + + double total = 0.0; + for (size_t idx = 1; idx < n; ++idx) { + NodeT nd = static_cast(idx); + NodeT p = ws.parent[nd]; + double c = static_cast(pred_cost_f(ws.cost_f, padded_k, m, nd, p)); + total += ws.flow_to_parent[nd] * c; + } + return total; + } + + static void + pivot(SolverWorkspace& ws, size_t m, size_t padded_k, size_t row, size_t col) { + NodeT i_node = static_cast(row); + NodeT j_node = static_cast(m + col); + + NodeT join = find_join_node(ws, i_node, j_node); + + NodeT u_in, v_in, u_out = 0; + double theta = find_leaving_arc(ws, i_node, j_node, join, m, u_in, v_in, u_out); + + // Capture the leaving cell before updateTreeStructure overwrites + // parent[u_out]. + NodeT leave_parent = ws.parent[u_out]; + unmark_tree_arc(ws.price_f, ws.cost_f, padded_k, m, u_out, leave_parent); + + change_flow(ws, i_node, j_node, join, m, theta); + update_tree_structure(ws, join, u_in, v_in, u_out, theta); + update_potential(ws, i_node, j_node, u_in, v_in, m, padded_k); + + ws.price_f[row * padded_k + col] = std::numeric_limits::infinity(); + } + + static NodeT find_join_node(SolverWorkspace& ws, NodeT i_node, NodeT j_node) { + const NodeT* const __restrict succ_num = ws.succ_num.data(); + const NodeT* const __restrict parent = ws.parent.data(); + size_t u = i_node, v = j_node; + while (u != v) { + if (succ_num[u] < succ_num[v]) { + u = parent[u]; + } else { + v = parent[v]; + } + } + return static_cast(u); + } + + // Cunningham's rule: the *last* blocking arc on each side's path to the join, + // walked in the entering arc's own direction, second path's tie using <= so a + // later tie overrides an earlier one. This keeps the tree strongly feasible. + static double find_leaving_arc( + SolverWorkspace& ws, + NodeT i_node, + NodeT j_node, + NodeT join, + size_t m, + NodeT& u_in, + NodeT& v_in, + NodeT& u_out + ) { + const NodeT* const __restrict parent = ws.parent.data(); + const double* const __restrict flow_to_parent = ws.flow_to_parent.data(); + double delta = std::numeric_limits::infinity(); + bool leaving_on_j_side = false; + for (size_t u = i_node; u != join; u = parent[u]) { + double d = is_source(u, m) ? flow_to_parent[u] + : std::numeric_limits::infinity(); + if (d < delta) { + delta = d; + u_out = static_cast(u); + } + } + for (size_t u = j_node; u != join; u = parent[u]) { + double d = is_source(u, m) ? std::numeric_limits::infinity() + : flow_to_parent[u]; + if (d <= delta) { + delta = d; + u_out = static_cast(u); + leaving_on_j_side = true; + } + } + if (leaving_on_j_side) { + u_in = j_node; + v_in = i_node; + } else { + u_in = i_node; + v_in = j_node; + } + return delta; + } + + static void change_flow( + SolverWorkspace& ws, + NodeT i_node, + NodeT j_node, + NodeT join, + size_t m, + double theta + ) { + if (theta <= 0.0) { + return; + } + const NodeT* const __restrict parent = ws.parent.data(); + double* const __restrict flow_to_parent = ws.flow_to_parent.data(); + for (size_t u = i_node; u != join; u = parent[u]) { + flow_to_parent[u] += is_source(u, m) ? -theta : theta; + } + for (size_t u = j_node; u != join; u = parent[u]) { + flow_to_parent[u] += is_source(u, m) ? theta : -theta; + } + } + + // Ports LEMON's updateTreeStructure: reverses the parent chain from u_in up to + // (excluding) u_out and splices thread/rev_thread accordingly. A tree arc's flow + // lives at whichever endpoint is the child, so reversing a link moves it too. + static void update_tree_structure( + SolverWorkspace& ws, + NodeT join, + NodeT u_in, + NodeT v_in, + NodeT u_out, + double theta + ) { + // Hoisted: none of these vectors alias each other, but without __restrict a + // store through one forces a reload of every other's data() each iteration. + NodeT* const __restrict parent = ws.parent.data(); + NodeT* const __restrict thread = ws.thread.data(); + NodeT* const __restrict rev_thread = ws.rev_thread.data(); + NodeT* const __restrict succ_num = ws.succ_num.data(); + NodeT* const __restrict last_succ = ws.last_succ.data(); + double* const __restrict flow_to_parent = ws.flow_to_parent.data(); + + size_t old_rev_thread = rev_thread[u_out]; + NodeT old_succ_num = succ_num[u_out]; + size_t old_last_succ = last_succ[u_out]; + NodeT v_out = parent[u_out]; + + size_t u = last_succ[u_in]; + size_t right = thread[u]; + size_t last = (old_rev_thread == static_cast(v_in)) + ? thread[last_succ[u_out]] + : thread[v_in]; + + thread[v_in] = u_in; + size_t stem = u_in; + ws.dirty_revs.clear(); + ws.dirty_revs.push_back(v_in); + size_t par_stem = v_in; + while (stem != u_out) { + size_t new_stem = parent[stem]; + thread[u] = static_cast(new_stem); + ws.dirty_revs.push_back(static_cast(u)); + + size_t w = rev_thread[stem]; + thread[w] = static_cast(right); + rev_thread[right] = static_cast(w); + + parent[stem] = static_cast(par_stem); + par_stem = stem; + stem = new_stem; + + u = (last_succ[stem] == last_succ[par_stem]) ? rev_thread[par_stem] + : last_succ[stem]; + right = thread[u]; + } + parent[u_out] = static_cast(par_stem); + thread[u] = static_cast(last); + rev_thread[last] = static_cast(u); + last_succ[u_out] = static_cast(u); + + if (old_rev_thread != static_cast(v_in)) { + thread[old_rev_thread] = static_cast(right); + rev_thread[right] = static_cast(old_rev_thread); + } + for (NodeT dirty : ws.dirty_revs) { + rev_thread[thread[dirty]] = dirty; + } + + NodeT tmp_sc = 0; + size_t tmp_ls = last_succ[u_out]; + u = u_out; + while (u != static_cast(u_in)) { + size_t w = parent[u]; + flow_to_parent[u] = flow_to_parent[w]; + tmp_sc = static_cast(tmp_sc + succ_num[u] - succ_num[w]); + succ_num[u] = tmp_sc; + last_succ[w] = static_cast(tmp_ls); + u = w; + } + flow_to_parent[u_in] = theta; + succ_num[u_in] = old_succ_num; + + // parent[root]'s sentinel (see solve_impl) matches this "disabled" value, so + // a chain left unbounded below is safe to walk off the end of the tree. + constexpr size_t no_node = std::numeric_limits::max(); + size_t up_limit_in = no_node, up_limit_out = no_node; + if (last_succ[join] == v_in) { + up_limit_out = join; + } else { + up_limit_in = join; + } + for (u = v_in; u != up_limit_in && last_succ[u] == v_in; u = parent[u]) { + last_succ[u] = last_succ[u_out]; + } + if (join != old_rev_thread && static_cast(v_in) != old_rev_thread) { + for (u = v_out; u != up_limit_out && last_succ[u] == old_last_succ; + u = parent[u]) { + last_succ[u] = static_cast(old_rev_thread); + } + } else { + for (u = v_out; u != up_limit_out && last_succ[u] == old_last_succ; + u = parent[u]) { + last_succ[u] = last_succ[u_out]; + } + } + for (u = v_in; u != static_cast(join); u = parent[u]) { + succ_num[u] = static_cast(succ_num[u] + old_succ_num); + } + for (u = v_out; u != static_cast(join); u = parent[u]) { + succ_num[u] = static_cast(succ_num[u] - old_succ_num); + } + } + + // sigma shifts every node in u_in's moved subtree so the entering arc's + // invariant (u_i + v_j == cost) holds exactly; sources/sinks shift by + // -sigma/+sigma (not the same sign) since the invariant is a sum, not a diff. + static void update_potential( + SolverWorkspace& ws, + NodeT i_node, + NodeT j_node, + NodeT u_in, + NodeT v_in, + size_t m, + size_t padded_k + ) { + auto pi_of = [&](NodeT node) -> double { + return is_source(node, m) ? -ws.u_d[node] + : ws.v_d[static_cast(node) - m]; + }; + double entering_cost = + static_cast(ws.cost_f + [static_cast(i_node) * padded_k + + (static_cast(j_node) - m)]); + double sigma = is_source(u_in, m) ? pi_of(v_in) - pi_of(u_in) - entering_cost + : pi_of(v_in) - pi_of(u_in) + entering_cost; + + // Hoisted: these never alias each other, but without __restrict every store + // through one forces a reload of the others' data() each iteration. + const NodeT* const __restrict thread = ws.thread.data(); + double* const __restrict u_d = ws.u_d.data(); + float* const __restrict u_f = ws.u_f.data(); + double* const __restrict v_d = ws.v_d.data(); + float* const __restrict v_f = ws.v_f.data(); + size_t end = thread[ws.last_succ[u_in]]; + for (size_t node = u_in; node != end; node = thread[node]) { + if (is_source(node, m)) { + u_d[node] -= sigma; + u_f[node] -= static_cast(sigma); + } else { + size_t j = node - m; + v_d[j] += sigma; + v_f[j] += static_cast(sigma); + } + } + } + }; +}; + +// NONE has no RowMin specialization of its own; it just runs the plain scalar algorithm +// every ISA is cross-checked against. +template <> struct TransportSimplex { + // NONE is the scalar baseline (no -march variation to key against), so this repeats + // TransportSimplex's own gap-bound formula rather than sharing its private one. + static double error_bound(size_t ua, size_t ub, double max_cost, double total_mass) { + size_t n = ua + ub; + double eps_mach_d = std::numeric_limits::epsilon(); + double n_d = static_cast(n); + double p = static_cast(n > 0 ? n - 1 : 0) * max_cost; + double walk_tol = 2.0 * n_d * eps_mach_d * (max_cost + p); + double flow_rounding = n_d * n_d * eps_mach_d * max_cost; + double final_sum_rounding = + 2.0 * static_cast(n > 0 ? n - 1 : 0) * eps_mach_d * max_cost; + return (2.0 * walk_tol + flow_rounding + final_sum_rounding) * total_mass; + } + + // The reference solver has no start choice, so the start is ignored. + template + static double solve( + size_t ua, + const double* supply, + size_t ub, + const double* demand, + CostFn&& cost, + TransportStart /*start*/ = TransportStart::NorthWest + ) { + return TransportSimplexReference::solve( + ua, supply, ub, demand, std::forward(cost) + ); + } +}; + +} // namespace svs::distance diff --git a/include/svs/core/distance/transport_simplex_pricing.h b/include/svs/core/distance/transport_simplex_pricing.h new file mode 100644 index 000000000..b80380a04 --- /dev/null +++ b/include/svs/core/distance/transport_simplex_pricing.h @@ -0,0 +1,274 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Float row-pricing kernels for TransportSimplex's float-proposes/double-confirms +// pricing split: a cost row stored as float, with a float or double reduced-cost scan. + +#include "svs/core/distance/distance_core.h" +#include "svs/lib/preprocessor.h" + +#include +#include +#include +#include + +#if defined(__x86_64__) +#include +#endif + +namespace svs::distance::transport_detail { + +// One AVX-512 float register, or two AVX2 float registers: a row padded to a multiple of +// this needs no scalar tail on either ISA. +inline constexpr size_t kPriceLanes = 16; + +inline size_t pad_to_lanes(size_t n) { + return (n + kPriceLanes - 1) / kPriceLanes * kPriceLanes; +} + +/// Finds the minimum reduced cost `r_j = (cost_row[j] - u_i) - v[j]` in a cost row and its +/// column. `cost_row` and `v` must both have `ub_padded` valid entries, a multiple of +/// `kPriceLanes`; padding columns (index >= the caller's real ub) must hold +infinity in +/// `cost_row` so they never win. `best_val`/`best_col` are in-out and updated only on a +/// strict improvement, scanning columns ascending, so the first minimum wins on a tie. +/// This primary definition is not dead on x86 despite the AVX2/AVX512 specializations +/// below: non-x86 builds still instantiate `RowMin`/`RowMin` through +/// `QEmdImpl`/`QEmdImpl`. +template struct RowMin { + static void min_in_row( + const float* cost_row, + const float* v, + size_t ub_padded, + float u_i, + float& best_val, + uint32_t& best_col + ) { + for (size_t j = 0; j < ub_padded; ++j) { + float r = (cost_row[j] - u_i) - v[j]; + if (r < best_val) { + best_val = r; + best_col = static_cast(j); + } + } + } + + static void min_in_row( + const float* cost_row, + const double* v, + size_t ub_padded, + double u_i, + double& best_val, + uint32_t& best_col + ) { + for (size_t j = 0; j < ub_padded; ++j) { + double r = (static_cast(cost_row[j]) - u_i) - v[j]; + if (r < best_val) { + best_val = r; + best_col = static_cast(j); + } + } + } +}; + +// Guarded per ISA: these bodies are not templates, so every TU that sees one compiles its +// intrinsics, and a TU built without that ISA fails loudly. +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +#if SVS_AVX512_F +template <> struct RowMin { + static void min_in_row( + const float* cost_row, + const float* v, + size_t ub_padded, + float u_i, + float& best_val, + uint32_t& best_col + ) { + __m512 u_vec = _mm512_set1_ps(u_i); + for (size_t j = 0; j < ub_padded; j += 16) { + // Two separate _mm512_sub_ps, not an FMA: the generic version rounds the two + // subtractions independently, and this must match it bit-for-bit. + __m512 r = _mm512_sub_ps( + _mm512_sub_ps(_mm512_loadu_ps(cost_row + j), u_vec), _mm512_loadu_ps(v + j) + ); + // Ordered-quiet: NaN never wins, matching scalar `<`. A zero mask -- the common + // case once a good candidate is found -- skips the block untouched. + __mmask16 lt_mask = _mm512_cmp_ps_mask(r, _mm512_set1_ps(best_val), _CMP_LT_OQ); + if (lt_mask == 0) { + continue; + } + // reduce_min can return a bit pattern no lane had (e.g. +0.0 vs -0.0 on a tie); + // compress then moves the lowest-index lane equal to that value to position 0. + float block_min = _mm512_mask_reduce_min_ps(lt_mask, r); + __mmask16 eq_mask = static_cast<__mmask16>( + lt_mask & _mm512_cmp_ps_mask(r, _mm512_set1_ps(block_min), _CMP_EQ_OQ) + ); + best_val = _mm512_cvtss_f32(_mm512_maskz_compress_ps(eq_mask, r)); + best_col = static_cast(j + std::countr_zero(eq_mask)); + } + } + + static void min_in_row( + const float* cost_row, + const double* v, + size_t ub_padded, + double u_i, + double& best_val, + uint32_t& best_col + ) { + __m512d u_vec = _mm512_set1_pd(u_i); + // One 8-lane half (mask8 is native width for double); see the float/float overload + // above for why compress, not reduce_min, supplies best_val. + auto resolve = [&](__m512d r, size_t col_base) { + __mmask8 lt_mask = _mm512_cmp_pd_mask(r, _mm512_set1_pd(best_val), _CMP_LT_OQ); + if (lt_mask == 0) { + return; + } + double block_min = _mm512_mask_reduce_min_pd(lt_mask, r); + __mmask8 eq_mask = static_cast<__mmask8>( + lt_mask & _mm512_cmp_pd_mask(r, _mm512_set1_pd(block_min), _CMP_EQ_OQ) + ); + best_val = _mm512_cvtsd_f64(_mm512_maskz_compress_pd(eq_mask, r)); + best_col = static_cast(col_base + std::countr_zero(eq_mask)); + }; + for (size_t j = 0; j < ub_padded; j += 16) { + __m256 c_lo = _mm256_loadu_ps(cost_row + j); + __m256 c_hi = _mm256_loadu_ps(cost_row + j + 8); + resolve( + _mm512_sub_pd( + _mm512_sub_pd(_mm512_cvtps_pd(c_lo), u_vec), _mm512_loadu_pd(v + j) + ), + j + ); + resolve( + _mm512_sub_pd( + _mm512_sub_pd(_mm512_cvtps_pd(c_hi), u_vec), _mm512_loadu_pd(v + j + 8) + ), + j + 8 + ); + } + } +}; +#endif + +SVS_VALIDATE_BOOL_ENV(SVS_AVX2) +#if SVS_AVX2 +template <> struct RowMin { + static void min_in_row( + const float* cost_row, + const float* v, + size_t ub_padded, + float u_i, + float& best_val, + uint32_t& best_col + ) { + __m256 u_vec = _mm256_set1_ps(u_i); + __m256 inf_vec = _mm256_set1_ps(std::numeric_limits::infinity()); + // AVX2 has no compress instruction, so the winning lane's exact r is read back + // from a stack buffer on the rare lt_mask != 0 branch, not taken from block_min. + auto resolve = [&](__m256 r, size_t col_base) { + __m256 lt_cmp = _mm256_cmp_ps(r, _mm256_set1_ps(best_val), _CMP_LT_OQ); + auto lt_mask = static_cast(_mm256_movemask_ps(lt_cmp)); + if (lt_mask == 0) { + return; + } + __m256 masked = _mm256_blendv_ps(inf_vec, r, lt_cmp); + __m128 m = _mm_min_ps( + _mm256_castps256_ps128(masked), _mm256_extractf128_ps(masked, 1) + ); + m = _mm_min_ps(m, _mm_shuffle_ps(m, m, _MM_SHUFFLE(0, 0, 3, 2))); + m = _mm_min_ps(m, _mm_shuffle_ps(m, m, _MM_SHUFFLE(0, 0, 0, 1))); + float block_min = _mm_cvtss_f32(m); + auto eq_mask = + lt_mask & static_cast(_mm256_movemask_ps( + _mm256_cmp_ps(r, _mm256_set1_ps(block_min), _CMP_EQ_OQ) + )); + auto lane = static_cast(std::countr_zero(eq_mask)); + float buf[8]; + _mm256_storeu_ps(buf, r); + best_val = buf[lane]; + best_col = static_cast(col_base + lane); + }; + for (size_t j = 0; j < ub_padded; j += 16) { + resolve( + _mm256_sub_ps( + _mm256_sub_ps(_mm256_loadu_ps(cost_row + j), u_vec), + _mm256_loadu_ps(v + j) + ), + j + ); + resolve( + _mm256_sub_ps( + _mm256_sub_ps(_mm256_loadu_ps(cost_row + j + 8), u_vec), + _mm256_loadu_ps(v + j + 8) + ), + j + 8 + ); + } + } + + static void min_in_row( + const float* cost_row, + const double* v, + size_t ub_padded, + double u_i, + double& best_val, + uint32_t& best_col + ) { + __m256d u_vec = _mm256_set1_pd(u_i); + __m256d inf_vec = _mm256_set1_pd(std::numeric_limits::infinity()); + // Same technique as the float/float overload above, at the 4-double width AVX2 + // actually has. + auto resolve = [&](__m256d r, size_t col_base) { + __m256d lt_cmp = _mm256_cmp_pd(r, _mm256_set1_pd(best_val), _CMP_LT_OQ); + auto lt_mask = static_cast(_mm256_movemask_pd(lt_cmp)); + if (lt_mask == 0) { + return; + } + __m256d masked = _mm256_blendv_pd(inf_vec, r, lt_cmp); + __m128d m = _mm_min_pd( + _mm256_castpd256_pd128(masked), _mm256_extractf128_pd(masked, 1) + ); + m = _mm_min_pd(m, _mm_shuffle_pd(m, m, 0x1)); + double block_min = _mm_cvtsd_f64(m); + auto eq_mask = + lt_mask & static_cast(_mm256_movemask_pd( + _mm256_cmp_pd(r, _mm256_set1_pd(block_min), _CMP_EQ_OQ) + )); + auto lane = static_cast(std::countr_zero(eq_mask)); + double buf[4]; + _mm256_storeu_pd(buf, r); + best_val = buf[lane]; + best_col = static_cast(col_base + lane); + }; + for (size_t j = 0; j < ub_padded; j += 16) { + for (size_t sub = 0; sub < 16; sub += 4) { + __m128 c = _mm_loadu_ps(cost_row + j + sub); + resolve( + _mm256_sub_pd( + _mm256_sub_pd(_mm256_cvtps_pd(c), u_vec), + _mm256_loadu_pd(v + j + sub) + ), + j + sub + ); + } + } + } +}; +#endif + +} // namespace svs::distance::transport_detail diff --git a/include/svs/core/distance/transport_simplex_reference.h b/include/svs/core/distance/transport_simplex_reference.h new file mode 100644 index 000000000..088696250 --- /dev/null +++ b/include/svs/core/distance/transport_simplex_reference.h @@ -0,0 +1,537 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Primal network simplex for the dense balanced transportation problem, scalar and +// unconditionally built: the AVX_AVAILABILITY::NONE path and the test oracle that +// TransportSimplex's SIMD-pricing variants are checked against. Algorithm: Ahuja, +// Magnanti & Orlin, "Network Flows" (1993), ch. 11 (artificial-root initialization and +// strongly feasible spanning trees); Cunningham, "A network simplex method", +// Mathematical Programming 11 (1976), for the leaving-arc rule that keeps a tree +// strongly feasible under degenerate pivots. Problem shape (quantized ground distance +// over merged centroid weights) follows Bonneel et al., "Displacement Interpolation +// Using Lagrangian Mass Transport", SIGGRAPH Asia 2011. + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs::distance::transport_reference_detail { + +// Copies supply/demand into the solver's own buffers, checks total balance to within +// 1e-6*max(total_supply, 1), and folds a smaller residual into the last demand entry; +// throws std::runtime_error on a larger mismatch or a fold that leaves that entry +// non-positive. Shared by TransportSimplexReference and TransportSimplex (called before +// either orients rows/columns, so the fold always lands on the caller's own last demand +// entry, never its supply) so the input contract cannot drift between them. `Caller` +// (TransportSimplexReference, or TransportSimplex::Impl) is a tag only, so +// each -march TU instantiates its own copy instead of sharing one weak symbol. +template +inline double prepare_marginals( + size_t ua, + const double* supply, + double* supply_out, + size_t ub, + const double* demand, + double* demand_out +) { + double total_s = 0.0, total_d = 0.0; + for (size_t i = 0; i < ua; ++i) { + supply_out[i] = supply[i]; + total_s += supply[i]; + } + for (size_t j = 0; j < ub; ++j) { + demand_out[j] = demand[j]; + total_d += demand[j]; + } + double tol = 1e-6 * std::max(total_s, 1.0); + double diff = total_s - total_d; + if (diff > tol || diff < -tol) { + throw std::runtime_error( + "TransportSimplex::solve: supply (" + std::to_string(total_s) + + ") and demand (" + std::to_string(total_d) + + ") are not balanced for ua=" + std::to_string(ua) + ", ub=" + std::to_string(ub) + ); + } + double last_demand = demand_out[ub - 1]; + demand_out[ub - 1] += diff; + if (demand_out[ub - 1] <= 0.0) { + throw std::runtime_error( + "TransportSimplex::solve: the last demand entry (" + + std::to_string(last_demand) + + ") is too small to absorb the balance residual (" + std::to_string(diff) + + ") for ua=" + std::to_string(ua) + ", ub=" + std::to_string(ub) + ); + } + return total_s; +} + +// Finds the most negative reduced cost in a cost row and its column, for the block-search +// entering rule in TransportSimplexReference::solve. +inline void price_row( + const double* cost_row, + const double* v, + size_t ub, + double u_i, + double& best_val, + size_t& best_col +) { + for (size_t j = 0; j < ub; ++j) { + double r = cost_row[j] - u_i - v[j]; + if (r < best_val) { + best_val = r; + best_col = j; + } + } +} + +// Scratch for one thread's in-flight solve, reused across calls and grow-only: a bigger +// (ua, ub) resizes these up, a smaller one just uses a prefix, so steady-state solves +// (same CentroidTable, same decoded token counts) allocate nothing. Sized for ua+ub+1 +// nodes: sources [0, ua), sinks [ua, ua+ub), and one artificial root node at ua+ub. +struct Workspace { + std::vector cost; // ua * ub, row-major + std::vector supply; + std::vector demand; + std::vector parent; // size ua+ub+1; -1 at the root, -2 = unvisited (scratch) + std::vector depth; // size ua+ub+1 + std::vector flow_to_parent; // size ua+ub+1; flow on the arc to parent[v] + std::vector u; // source potentials, size ua + std::vector v; // sink potentials, size ub + std::vector adj_head; // CSR adjacency over ua+ub+1 nodes, rebuilt every pivot + std::vector adj_next; + std::vector adj_to; + std::vector adj_cost; // the physical arc's cost (M for an artificial arc) + std::vector adj_flow; + std::vector bfs_order; + std::vector path_p; + std::vector path_q; + + void reserve(size_t ua, size_t ub) { + size_t n = ua + ub + 1; + if (cost.size() < ua * ub) { + cost.resize(ua * ub); + } + if (supply.size() < ua) { + supply.resize(ua); + } + if (demand.size() < ub) { + demand.resize(ub); + } + if (parent.size() < n) { + parent.resize(n); + } + if (depth.size() < n) { + depth.resize(n); + } + if (flow_to_parent.size() < n) { + flow_to_parent.resize(n); + } + if (u.size() < ua) { + u.resize(ua); + } + if (v.size() < ub) { + v.resize(ub); + } + if (adj_head.size() < n) { + adj_head.resize(n); + } + size_t edges = (n > 0) ? n - 1 : 0; + if (adj_next.size() < 2 * edges) { + adj_next.resize(2 * edges); + adj_to.resize(2 * edges); + adj_cost.resize(2 * edges); + adj_flow.resize(2 * edges); + } + if (bfs_order.size() < n) { + bfs_order.resize(n); + } + if (path_p.size() < n) { + path_p.resize(n); + } + if (path_q.size() < n) { + path_q.resize(n); + } + } +}; + +// One directed adjacency slot: node `to`, reached from `from` over an arc with physical +// cost `cost` (the real cost(i, j) for a transportation arc, or M for an artificial one) +// and current flow `flow`. +inline void add_adj( + Workspace& ws, size_t edge_slot, int64_t from, int64_t to, double cost, double flow +) { + ws.adj_to[edge_slot] = to; + ws.adj_cost[edge_slot] = cost; + ws.adj_flow[edge_slot] = flow; + ws.adj_next[edge_slot] = ws.adj_head[static_cast(from)]; + ws.adj_head[static_cast(from)] = edge_slot; +} + +// Builds the artificial-root star: one extra node R = ua+ub, an arc (source i, R) with +// flow supply[i] for every source and (R, sink j) with flow demand[j] for every sink, all +// at cost M. Every arc starts at strictly positive flow (ws.supply/ws.demand are qEMD's +// merged weights, never zero), so this tree is strongly feasible by construction: the +// strongly-feasible condition only constrains zero-flow tree arcs, and there are none +// (Ahuja-Magnanti-Orlin ch. 11.2). Northwest-corner is not used here because it is not +// automatically strongly feasible. +inline void artificial_root_adjacency(Workspace& ws, size_t ua, size_t ub, double big_m) { + size_t root = ua + ub; + std::fill( + ws.adj_head.begin(), + ws.adj_head.begin() + static_cast(root + 1), + SIZE_MAX + ); + size_t slot = 0; + for (size_t i = 0; i < ua; ++i) { + add_adj( + ws, + slot, + static_cast(i), + static_cast(root), + big_m, + ws.supply[i] + ); + add_adj( + ws, + slot + 1, + static_cast(root), + static_cast(i), + big_m, + ws.supply[i] + ); + slot += 2; + } + for (size_t j = 0; j < ub; ++j) { + add_adj( + ws, + slot, + static_cast(root), + static_cast(ua + j), + big_m, + ws.demand[j] + ); + add_adj( + ws, + slot + 1, + static_cast(ua + j), + static_cast(root), + big_m, + ws.demand[j] + ); + slot += 2; + } +} + +// Breadth-first search from the artificial root R = ua+ub (always the root: R is +// permanently part of the spanning tree, see solve()'s doc comment) over the adjacency +// already built in ws.adj_*, filling parent/depth/flow_to_parent and the dual potentials +// u/v. A source is always the tail of its arc to its tree parent, whether that arc is a +// real one (to a sink) or artificial (to R); a sink is always the head. That fixes the +// sign in the potential update below without per-arc direction bookkeeping. Every tree +// arc has zero reduced cost by construction, which is what lets solve()'s pricing scan +// treat "negative reduced cost" as "not yet basic" with no separate basis-membership +// check. +inline void rebuild_tree(Workspace& ws, size_t ua, size_t ub) { + size_t root = ua + ub; + size_t n = root + 1; + std::fill(ws.parent.begin(), ws.parent.begin() + static_cast(n), -2); + ws.parent[root] = -1; + ws.depth[root] = 0; + ws.bfs_order[0] = static_cast(root); + size_t order_end = 1; + for (size_t qi = 0; qi < order_end; ++qi) { + int64_t node = ws.bfs_order[qi]; + double node_y = + (node == static_cast(root)) + ? 0.0 + : (node < static_cast(ua) ? ws.u[static_cast(node)] + : -ws.v[static_cast(node) - ua]); + for (size_t e = ws.adj_head[static_cast(node)]; e != SIZE_MAX; + e = ws.adj_next[e]) { + int64_t to = ws.adj_to[e]; + if (ws.parent[static_cast(to)] != -2) { + continue; + } + ws.parent[static_cast(to)] = node; + ws.depth[static_cast(to)] = ws.depth[static_cast(node)] + 1; + ws.flow_to_parent[static_cast(to)] = ws.adj_flow[e]; + double cost = ws.adj_cost[e]; + if (to < static_cast(ua)) { + ws.u[static_cast(to)] = + node_y + cost; // source: tail, y = y(parent) + cost + } else { + ws.v[static_cast(to) - ua] = cost - node_y; // sink: head + } + ws.bfs_order[order_end++] = to; + } + } +} + +} // namespace svs::distance::transport_reference_detail + +namespace svs::distance { + +/// Reference implementation of the dense balanced transportation problem solver: `ua` +/// sources with supply `supply[i]`, `ub` sinks with demand `demand[j]`, a complete +/// bipartite arc set, and a per-arc cost from `cost(i, j)`. Minimizes +/// `sum_ij flow_ij * cost(i, j)` subject to `sum_j flow_ij == supply[i]`, +/// `sum_i flow_ij == demand[j]`, `flow_ij >= 0`. +/// +/// Requires `ua >= 1`, `ub >= 1`, every supply/demand entry strictly positive (true for +/// qEMD's merged weights), and `sum(supply) == sum(demand)` to within `1e-6 * +/// max(sum(supply), 1.0)`; any smaller residual (the kind two independent floating-point +/// sums that are mathematically equal can differ by) is folded into the last demand entry +/// so the solver never over- or under-subscribes a side. A larger mismatch, or a residual +/// that leaves the last demand entry non-positive, throws std::runtime_error, since both +/// signal mismatched inputs rather than rounding. +/// +/// Initializes with an artificial root node and big-M artificial arcs to every source and +/// sink (Ahuja-Magnanti-Orlin ch. 11.2), which is strongly feasible from the start since +/// every artificial arc carries strictly positive flow. Each pivot then picks the leaving +/// arc as the *last* blocking (flow-decreasing) arc encountered while walking the cycle +/// from its apex (the two endpoints' lowest common tree ancestor) in the entering arc's +/// own direction (Cunningham, 1976); maintaining a strongly feasible tree this way +/// guarantees termination regardless of which negative-reduced-cost arc enters next, so +/// the entering rule is free to optimize for speed: a block-cyclic search over the dense +/// cost matrix, picking the most negative reduced cost in the first block (of +/// `~sqrt(ua*ub)` arcs, rounded to whole rows) that has one, resuming from where the +/// previous search left off. M is `(ua+ub) * max(cost) + 1`, the largest an optimal basic +/// solution's total real cost could be plus 1, so no finite combination of real arcs is +/// ever more expensive than keeping one unit of flow on an artificial arc. +/// +/// This type runs the scalar pricing loop unconditionally: it is the `AVX_AVAILABILITY:: +/// NONE` path, and it is the oracle TransportSimplex's results (not its algorithm) are +/// checked against. +struct TransportSimplexReference { + template + static double + solve(size_t ua, const double* supply, size_t ub, const double* demand, CostFn&& cost) { + using namespace transport_reference_detail; + thread_local Workspace ws; + ws.reserve(ua, ub); + size_t root = ua + ub; + + prepare_marginals( + ua, supply, ws.supply.data(), ub, demand, ws.demand.data() + ); + + double max_cost = 0.0; + for (size_t i = 0; i < ua; ++i) { + double* row = &ws.cost[i * ub]; + for (size_t j = 0; j < ub; ++j) { + double c = static_cast(cost(i, j)); + row[j] = c; + max_cost = std::max(max_cost, c); + } + } + // Bigger than any optimal basic solution's real cost could ever be, so the + // simplex never prefers routing flow through the artificial root once a real + // spanning tree is available; see the class doc comment. + double big_m = static_cast(ua + ub) * max_cost + 1.0; + + artificial_root_adjacency(ws, ua, ub, big_m); + rebuild_tree(ws, ua, ub); + + // Block size: ~sqrt(ua*ub) arcs, rounded to a whole number of rows so each block + // scan stays an integer number of rows. + size_t block_rows = static_cast(std::round( + std::sqrt(static_cast(ua) * static_cast(ub)) / + static_cast(ub) + )); + block_rows = std::clamp(block_rows, 1, ua); + size_t cursor_row = 0; + + // No polynomial bound exists for primal network simplex iteration count (even + // with a strongly feasible tree, only finiteness is guaranteed, not a bound), so + // this is a generous safety net against a latent bug, not an expected limit. + size_t max_iters = 1000 * (ua + ub) + 10000; + for (size_t iter = 0; iter < max_iters; ++iter) { + size_t best_i = ua, best_j = 0, rows_scanned = 0; + size_t row = cursor_row; + while (rows_scanned < ua) { + double block_best_val = 0.0; + size_t block_best_i = ua, block_best_j = 0; + for (size_t b = 0; b < block_rows && rows_scanned < ua; + ++b, ++rows_scanned) { + double val = 0.0; + size_t col = 0; + price_row(&ws.cost[row * ub], ws.v.data(), ub, ws.u[row], val, col); + if (val < block_best_val) { + block_best_val = val; + block_best_i = row; + block_best_j = col; + } + row = (row + 1) % ua; + } + if (block_best_i != ua) { + best_i = block_best_i; + best_j = block_best_j; + cursor_row = row; + break; + } + } + if (best_i == ua) { + double total_cost = 0.0; + for (size_t node = 0; node < root; ++node) { + if (static_cast(ws.parent[node]) == root) { + continue; // artificial arc: not part of the real transport cost + } + size_t ai, aj; + if (node < ua) { + ai = node; + aj = static_cast(ws.parent[node]) - ua; + } else { + ai = static_cast(ws.parent[node]); + aj = node - ua; + } + total_cost += ws.flow_to_parent[node] * ws.cost[ai * ub + aj]; + } + return total_cost; + } + + int64_t p_node = static_cast(best_i); + int64_t q_node = static_cast(ua + best_j); + ws.path_p.clear(); + ws.path_p.push_back(p_node); + ws.path_q.clear(); + ws.path_q.push_back(q_node); + int64_t pu = p_node, qu = q_node; + while (ws.depth[static_cast(pu)] > ws.depth[static_cast(qu)]) { + pu = ws.parent[static_cast(pu)]; + ws.path_p.push_back(pu); + } + while (ws.depth[static_cast(qu)] > ws.depth[static_cast(pu)]) { + qu = ws.parent[static_cast(qu)]; + ws.path_q.push_back(qu); + } + while (pu != qu) { + pu = ws.parent[static_cast(pu)]; + ws.path_p.push_back(pu); + qu = ws.parent[static_cast(qu)]; + ws.path_q.push_back(qu); + } + size_t lp = ws.path_p.size(); + size_t lq = ws.path_q.size(); + + // A source is always the tail of its tree arc (to a sink or to the root), a + // sink always the head (see rebuild_tree's doc comment), so walking + // parent->child is "forward" (flow increases) exactly when the child is a + // sink, and walking child->parent is forward exactly when the child is a + // source. + auto is_forward_down = [&](int64_t child) { + return child >= static_cast(ua); + }; + auto is_forward_up = [&](int64_t child) { + return child < static_cast(ua); + }; + + // Leaving arc: walk the cycle apex (= LCA = the end of either path array) + // down to p, then from q back up to the apex -- the entering arc's own + // orientation -- and take the *last* blocking (backward) arc seen, using <= + // so a later tie overwrites an earlier one (Cunningham's rule; this is what + // keeps the tree strongly feasible, not the choice of entering arc). + double theta = std::numeric_limits::infinity(); + int64_t leave_node = -1; + for (size_t t = lp; t-- > 1;) { + int64_t child = ws.path_p[t - 1]; + if (!is_forward_down(child)) { + double flow = ws.flow_to_parent[static_cast(child)]; + if (flow <= theta) { + theta = flow; + leave_node = child; + } + } + } + for (size_t t = 0; t + 1 < lq; ++t) { + int64_t child = ws.path_q[t]; + if (!is_forward_up(child)) { + double flow = ws.flow_to_parent[static_cast(child)]; + if (flow <= theta) { + theta = flow; + leave_node = child; + } + } + } + + // Apply the flow delta to every tree edge on the cycle (the entering arc is + // not one of these; it is added fresh below with flow == theta). Order does + // not matter here, unlike the leaving-arc search above. + for (size_t t = 0; t + 1 < lp; ++t) { + int64_t child = ws.path_p[t]; + double delta = is_forward_down(child) ? theta : -theta; + ws.flow_to_parent[static_cast(child)] += delta; + } + for (size_t t = 0; t + 1 < lq; ++t) { + int64_t child = ws.path_q[t]; + double delta = is_forward_up(child) ? theta : -theta; + ws.flow_to_parent[static_cast(child)] += delta; + } + + // Rebuild adjacency from the old tree minus the leaving edge, plus the + // entering arc (p_node, q_node, theta); rebuilding by BFS instead of + // patching parent/depth in place avoids re-rooting a subtree incrementally, + // which costs O(ua+ub) here against O(ua*ub) for pricing, so it is not the + // bottleneck. + std::fill( + ws.adj_head.begin(), + ws.adj_head.begin() + static_cast(root + 1), + SIZE_MAX + ); + size_t slot = 0; + for (int64_t node = 0; node < static_cast(root); ++node) { + if (node == leave_node) { + continue; + } + int64_t parent_node = ws.parent[static_cast(node)]; + double edge_cost; + if (static_cast(parent_node) == root) { + edge_cost = big_m; + } else if (node < static_cast(ua)) { + edge_cost = ws.cost + [static_cast(node) * ub + + (static_cast(parent_node) - ua)]; + } else { + edge_cost = ws.cost + [static_cast(parent_node) * ub + + (static_cast(node) - ua)]; + } + double flow = ws.flow_to_parent[static_cast(node)]; + add_adj(ws, slot, node, parent_node, edge_cost, flow); + add_adj(ws, slot + 1, parent_node, node, edge_cost, flow); + slot += 2; + } + double entering_cost = ws.cost[best_i * ub + best_j]; + add_adj(ws, slot, p_node, q_node, entering_cost, theta); + add_adj(ws, slot + 1, q_node, p_node, entering_cost, theta); + + rebuild_tree(ws, ua, ub); + } + throw std::runtime_error( + "TransportSimplexReference::solve: exceeded the iteration limit (" + + std::to_string(max_iters) + ") for ua=" + std::to_string(ua) + + ", ub=" + std::to_string(ub) + ); + } +}; + +} // namespace svs::distance diff --git a/include/svs/core/kmeans_emeans.h b/include/svs/core/kmeans_emeans.h new file mode 100644 index 000000000..b27ae451c --- /dev/null +++ b/include/svs/core/kmeans_emeans.h @@ -0,0 +1,772 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/core/kmeans_training.h" +#include "svs/lib/exception.h" +#include "svs/lib/threads/threadpool.h" +#include "svs/lib/timing.h" + +// stl +#include +#include +#include +#include +#include +#include +#include + +// e-means: mini-batch k-means with EMA centroid updates, a cosine-annealed step size, and +// dead-cluster revival, following Algorithm 1 of Bessa et al., "e-means: Billion-scale +// mini-batch k-means with low-cost approximations", arXiv:2611.09501v1. Shares its data, +// rng, thread pool and result shape with kmeans_training.h's Lloyd path and +// quantization/gem/codebook.h's train_centroids. +namespace svs::kmeans { + +/// @brief Parameters for e-means training (arXiv:2611.09501v1, Algorithm 1). +struct EMeansParameters { + /// Number of epochs T: full passes over a fresh shuffle of the data. 2 is an SVS + /// default for GEM-scale codebooks, not a value the paper recommends. + size_t epochs = 2; + /// Minibatch size B. Unset selects the batch scaling law (Eq. 7, Table 2), rounded to + /// the nearest power of two and capped at the dataset size. + std::optional batch_size = std::nullopt; + /// Initial EMA step alpha0. Unset selects the step scaling law (Eq. 6, Table 2), + /// evaluated at the resolved batch size and capped at 1. + std::optional alpha0 = std::nullopt; + /// Cosine-annealing floor: alpha decays from alpha0 to epsilon * alpha0 (Eq. 5). Must + /// lie in [0, 1]. + double epsilon = 0.1; + /// Revival threshold: a cluster is revived once its EMA count drops below tau times + /// the mean EMA count over all clusters (Appendix H.1). Must be non-negative. + double tau = 0.1; + /// Batches between dead-cluster revival checks. Must be positive (used as a modulus). + size_t revival_interval = 5; + /// When true, run the closing pass that replaces each centroid with the plain mean of + /// the points assigned to it during the last epoch (Fig. 22's "closing mean"). + bool final_mean_pass = true; + /// Caps the per-chunk assignment score buffer at assignment_chunk * k floats, the same + /// role kmeans_training.h's minibatch_size plays for the Lloyd path. Must be positive. + size_t assignment_chunk = 10'000; +}; + +/// @brief Centroids and the k-means objective (mean squared error) after emeans_train; +/// same shape as quantization::gem::L2KMeansResult, the GEM Lloyd path's result. +struct EMeansResult { + data::SimpleData centroids; + double objective = 0.0; +}; + +namespace detail { + +// Table 2 coefficients (arXiv:2611.09501v1 Sec. 4), fit by ordinary least squares in log +// space on validation MSE from DEEP/L2, 256 <= k <= 65536; not validated for other +// metrics, other datasets, or very low dimension. +inline constexpr double EMEANS_BATCH_COEFF_SMALL_N = 0.035; +// Table 2 prints this branch's coefficient as 0.0044; kept as 0.035/8 (0.004375) to match +// the paper's stated derivation ("divide by 8") rather than its rounded printed value. +inline constexpr double EMEANS_BATCH_COEFF_LARGE_N = 0.035 / 8.0; // n > 2e6 (Eq. 7 note) +inline constexpr size_t EMEANS_BATCH_LARGE_N_THRESHOLD = 2'000'000; +inline constexpr double EMEANS_BATCH_EXP_K = 0.13; +inline constexpr double EMEANS_BATCH_EXP_N = 0.88; +inline constexpr double EMEANS_BATCH_EXP_T = 0.81; + +inline constexpr double EMEANS_ALPHA_COEFF = 0.824; +inline constexpr double EMEANS_ALPHA_EXP_K = -0.46; +inline constexpr double EMEANS_ALPHA_EXP_B = 0.90; +inline constexpr double EMEANS_ALPHA_EXP_N = -0.45; +inline constexpr double EMEANS_ALPHA_EXP_T = -0.11; + +// Appendix H.1's split delta: one reading of "BF16 machine epsilon" as 2^-7 (bf16's 7 +// explicit mantissa bits); the paper does not spell out the exact bit count it means. +inline constexpr double EMEANS_REVIVAL_DELTA = 1.0 / 128.0; + +// EMA count/sum initializer (Algorithm 1, step 2): N_j starts near zero so the first +// batch's statistics dominate gamma*N_j + alpha*n_j immediately. +inline constexpr double EMEANS_INITIAL_COUNT = 1e-10; + +/// Eq. 7: batch-size scaling law, rounded to the nearest power of two and capped at n. +// "Nearest power of two" rounds log2(raw) to the nearest integer; the paper does not say +// which of the two bracketing powers wins exactly at a midpoint, this does round-half-up. +inline size_t resolve_batch_size(size_t n, size_t k, size_t epochs) { + double b_c = n > EMEANS_BATCH_LARGE_N_THRESHOLD ? EMEANS_BATCH_COEFF_LARGE_N + : EMEANS_BATCH_COEFF_SMALL_N; + double raw = b_c * std::pow(static_cast(k), EMEANS_BATCH_EXP_K) * + std::pow(static_cast(n), EMEANS_BATCH_EXP_N) * + std::pow(static_cast(epochs), EMEANS_BATCH_EXP_T); + double pow2 = std::pow(2.0, std::round(std::log2(std::max(raw, 1.0)))); + size_t b = static_cast(std::min(pow2, static_cast(n))); + return std::clamp(b, size_t{1}, n); +} + +/// Eq. 6: step scaling law, evaluated at the resolved batch size, capped at 1. +inline double resolve_alpha0(size_t n, size_t k, size_t batch_size, size_t epochs) { + double raw = EMEANS_ALPHA_COEFF * std::pow(static_cast(k), EMEANS_ALPHA_EXP_K) * + std::pow(static_cast(batch_size), EMEANS_ALPHA_EXP_B) * + std::pow(static_cast(n), EMEANS_ALPHA_EXP_N) * + std::pow(static_cast(epochs), EMEANS_ALPHA_EXP_T); + return std::min(raw, 1.0); +} + +// Eq. 5: alpha(t) = alpha0 * (eps + (1-eps) * 0.5 * (1 + cos(pi*t/M))), t = 1..M, matching +// Algorithm 1's own 1-indexed t (also used, unchanged, for the "t mod 5" revival check). +inline double cosine_step(double alpha0, double epsilon, size_t t, size_t total_batches) { + double phase = + std::numbers::pi * static_cast(t) / static_cast(total_batches); + return alpha0 * (epsilon + (1.0 - epsilon) * 0.5 * (1.0 + std::cos(phase))); +} + +// Appendix H.1: every `revival_interval` batches, a cluster with EMA count below +// tau * mean(N) is revived by splitting a donor drawn with probability proportional to +// the donor's own EMA count, without replacement among this round's revivals. Mutates N/S +// in place. `dead_scratch`/`excluded_scratch`/`cum_scratch` are caller-owned, sized k, so +// a revival round allocates nothing of its own inside the batch loop. +// +// O(k) to build the prefix sum once, then O(log k) expected per donor draw (binary search +// plus a bounded number of redraws on a collision), with an O(k) linear-scan fallback only +// once the redraw budget is exhausted. The paper draws donors "among all clusters"; this +// also excludes this round's own dead clusters from the donor pool, which differs only +// when a dead cluster would otherwise have been drawn as its own donor (negligible: a +// dead cluster's weight N_j is already far below the mean by construction). +inline void revive_dead_clusters( + std::vector& N, + std::vector& S, + size_t dims, + double tau, + std::mt19937& rng, + std::vector& dead_scratch, + std::vector& excluded_scratch, + std::vector& cum_scratch +) { + size_t k = N.size(); + double mean_n = std::accumulate(N.begin(), N.end(), 0.0) / static_cast(k); + double threshold = tau * mean_n; + + auto& dead = dead_scratch; + auto& excluded = excluded_scratch; + auto& cum = cum_scratch; + dead.clear(); + std::fill(excluded.begin(), excluded.end(), false); + for (size_t j = 0; j < k; ++j) { + if (N[j] < threshold) { + dead.push_back(j); + excluded[j] = true; + } + } + if (dead.empty()) { + return; + } + + // Prefix sum of donor weight; excluded (dead) clusters contribute 0, so a draw never + // lands on one without needing a separate filter in the binary search below. + double running = 0.0; + for (size_t c = 0; c < k; ++c) { + running += excluded[c] ? 0.0 : N[c]; + cum[c] = running; + } + double total_weight = running; + if (total_weight <= 0.0) { + return; // No eligible donor at all this round; leave every cluster dead for now. + } + + auto distribution = std::uniform_real_distribution(0.0, total_weight); + constexpr int max_redraws = 8; + for (size_t j : dead) { + size_t donor = k; + for (int attempt = 0; attempt < max_redraws; ++attempt) { + double r = distribution(rng); + auto it = std::lower_bound(cum.begin(), cum.end(), r); + size_t candidate = std::min(static_cast(it - cum.begin()), k - 1); + if (!excluded[candidate]) { + donor = candidate; + break; + } + } + if (donor == k) { + // Redraw budget exhausted (heavy exclusion this round): fall back to a + // guaranteed-correct linear scan for the first eligible cluster. + for (size_t c = 0; c < k; ++c) { + if (!excluded[c]) { + donor = c; + break; + } + } + } + if (donor == k) { + continue; // No eligible donor left this round; leave the cluster dead for now. + } + excluded[donor] = true; + + // Split the donor along a hyperplane through its centroid: opposite-sign + // perturbation on alternating dimensions, then halve its count between the two. + double donor_n_half = N[donor] / 2.0; + for (size_t d = 0; d < dims; ++d) { + double c_val = S[donor * dims + d] / N[donor]; + double sign = (d % 2 == 0) ? 1.0 : -1.0; + S[j * dims + d] = c_val * (1.0 + sign * EMEANS_REVIVAL_DELTA) * donor_n_half; + S[donor * dims + d] = + c_val * (1.0 - sign * EMEANS_REVIVAL_DELTA) * donor_n_half; + } + N[j] = donor_n_half; + N[donor] = donor_n_half; + } +} + +// Accumulates this batch's per-cluster count n_j and sum s_j (Algorithm 1's per-batch +// statistics), ahead of the EMA update. Range-partitions by centroid id so each thread +// owns disjoint output, the same determinism-by-construction pattern kmeans_training.h's +// centroid_adjustment uses; `batch_counts`/`batch_sums` must be zeroed by the caller. +template +void accumulate_batch_stats( + const data::SimpleData& gathered, + size_t count, + const std::vector& local_assignments, + size_t num_clusters, + std::vector& batch_counts, + std::vector& batch_sums, + Pool& threadpool +) { + size_t n_threads = threadpool.size(); + size_t dims = gathered.dimensions(); + threads::parallel_for( + threadpool, + threads::StaticPartition{n_threads}, + [&](auto /*indices*/, auto tid) { + size_t centroid_start = (num_clusters * tid) / n_threads; + size_t centroid_end = (num_clusters * (tid + 1)) / n_threads; + for (size_t i = 0; i < count; ++i) { + size_t assignment = local_assignments[i]; + if (assignment >= centroid_start && assignment < centroid_end) { + batch_counts[assignment] += 1.0; + auto datum = gathered.get_datum(i); + double* sum_row = batch_sums.data() + assignment * dims; + for (size_t p = 0; p < dims; ++p) { + sum_row[p] += static_cast(datum[p]); + } + } + } + } + ); +} + +// Closing pass (Fig. 22): replace each centroid with the plain mean of the points +// assigned to it over the whole last epoch. A cluster with no last-epoch assignment keeps +// its EMA readout rather than being zeroed. Allocates once, after the batch loop ends, not +// inside it. +template +void closing_mean_pass( + const Data& data, + data::SimpleData& centroids, + const std::vector& assignments, + Pool& threadpool +) { + size_t n_threads = threadpool.size(); + size_t k = centroids.size(); + size_t dims = centroids.dimensions(); + std::vector sums(k * dims, 0.0); + std::vector counts(k, 0); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_threads}, + [&](auto /*indices*/, auto tid) { + size_t centroid_start = (k * tid) / n_threads; + size_t centroid_end = (k * (tid + 1)) / n_threads; + for (auto i : data.eachindex()) { + size_t a = assignments[i]; + if (a >= centroid_start && a < centroid_end) { + counts[a]++; + auto datum = data.get_datum(i); + double* row = sums.data() + a * dims; + for (size_t d = 0; d < dims; ++d) { + row[d] += static_cast(datum[d]); + } + } + } + } + ); + + threads::parallel_for( + threadpool, + threads::StaticPartition{k}, + [&](auto indices, auto /*tid*/) { + for (auto j : indices) { + if (counts[j] != 0) { + auto c = centroids.get_datum(j); + double inv = 1.0 / static_cast(counts[j]); + const double* row = sums.data() + j * dims; + for (size_t d = 0; d < dims; ++d) { + c[d] = static_cast(row[d] * inv); + } + } + } + } + ); +} + +// Sum of squared L2 distances from `data_batch[0..count)` to its assigned centroid, +// computed from the raw vectors (not the matmul scores). Per-thread partials over a +// StaticPartition, summed in tid order, so the result is independent of scheduling. +template +double accumulate_squared_error( + const Data& data_batch, + size_t count, + const std::vector& local_assignments, + const data::SimpleData& centroids, + Pool& threadpool +) { + size_t n_threads = threadpool.size(); + size_t dims = centroids.dimensions(); + std::vector partials(n_threads, 0.0); + threads::parallel_for( + threadpool, + threads::StaticPartition{count}, + [&](auto indices, auto tid) { + double sum = 0.0; + for (auto i : indices) { + auto x = data_batch.get_datum(i); + auto c = centroids.get_datum(local_assignments[i]); + double sq = 0.0; + for (size_t d = 0; d < dims; ++d) { + double diff = static_cast(x[d]) - static_cast(c[d]); + sq += diff * diff; + } + sum += sq; + } + partials[tid] = sum; + } + ); + double total = 0.0; + for (size_t tid = 0; tid < n_threads; ++tid) { + total += partials[tid]; + } + return total; +} + +// Assigns data[window_start..window_start+window_size) to their nearest centroid, into +// chunk_assignments[0..window_size); window_size must not exceed chunk_assignments' size. +// Caller chunks the batch so chunk_matmul_results never grows past chunk_size * k floats. +template +void assign_chunk( + const Data& data, + size_t window_start, + size_t window_size, + Distance& distance, + data::SimpleData& centroids, + std::vector& centroids_norm, + std::vector& chunk_assignments, + std::vector& chunk_norm_scratch, + data::SimpleData& chunk_matmul_results, + Pool& threadpool, + lib::Timer& scratch_timer +) { + auto window_view = data::make_view( + data, threads::UnitRange{window_start, window_start + window_size} + ); + generate_norms(window_view, chunk_norm_scratch, threadpool); + centroid_assignment( + window_view, + chunk_norm_scratch, + threads::UnitRange{0, window_size}, + distance, + centroids, + centroids_norm, + chunk_assignments, + chunk_matmul_results, + threadpool, + scratch_timer + ); +} + +// Final objective (Algorithm 1's MSE): one pass over `data` in assignment_chunk-sized +// windows, reusing the Lloyd assignment kernel, not a per-point brute-force scan over k. +template +double compute_objective( + Data& data, + data::SimpleData& centroids, + size_t chunk_size, + data::SimpleData& chunk_matmul_results, + std::vector& chunk_norm_scratch, + std::vector& centroids_norm, + std::vector& chunk_assignments, + Pool& threadpool, + lib::Timer& scratch_timer +) { + size_t n = data.size(); + generate_norms(centroids, centroids_norm, threadpool); + auto distance = svs::DistanceL2{}; + + double total = 0.0; + for (size_t start = 0; start < n; start += chunk_size) { + size_t this_chunk = std::min(chunk_size, n - start); + assign_chunk( + data, + start, + this_chunk, + distance, + centroids, + centroids_norm, + chunk_assignments, + chunk_norm_scratch, + chunk_matmul_results, + threadpool, + scratch_timer + ); + auto window_view = + data::make_view(data, threads::UnitRange{start, start + this_chunk}); + total += accumulate_squared_error( + window_view, this_chunk, chunk_assignments, centroids, threadpool + ); + } + return total / static_cast(n); +} + +} // namespace detail + +/// @brief The batch size and alpha0 emeans_train actually uses, after applying Table 2's +/// scaling laws to any unset EMeansParameters field. Pure function of its inputs, so a +/// caller can recompute the same values for logging without re-running training. +struct ResolvedEMeansParameters { + size_t batch_size = 0; + double alpha0 = 0.0; +}; + +/// @brief Resolve EMeansParameters::batch_size/alpha0 against n and k, applying Table 2's +/// scaling laws (Eq. 6, Eq. 7) to whichever field is unset. The sole place that combines a +/// user override with the scaling-law default, so emeans_train and a caller logging the +/// effective parameters agree by construction. +inline ResolvedEMeansParameters +resolve_emeans_parameters(size_t n, size_t k, const EMeansParameters& parameters) { + size_t batch_size = std::clamp( + parameters.batch_size.value_or(detail::resolve_batch_size(n, k, parameters.epochs)), + size_t{1}, + n + ); + double alpha0 = parameters.alpha0.value_or( + detail::resolve_alpha0(n, k, batch_size, parameters.epochs) + ); + return ResolvedEMeansParameters{batch_size, alpha0}; +} + +/// @brief e-means training (arXiv:2611.09501v1, Algorithm 1), starting from caller-given +/// initial centroids. +/// +/// @param parameters e-means hyperparameters; see EMeansParameters. +/// @param data The training set. Not modified. +/// @param initial_centroids The k starting centroids (Algorithm 1, step 1). Its size fixes +/// the number of clusters k; copied into the EMA state and not aliased afterward. +/// @param rng Advanced once per epoch shuffle, and once per revival-round donor draw; a +/// fixed seed and thread count reproduce the same centroids run to run. +/// @param threadpool Thread pool used for every parallel step. +/// @param timer Receives per-epoch timing entries, in kmeans_training.h's style. +/// +/// @throws ANNEXCEPTION if initial_centroids has more clusters than `data` has points; if +/// a user-supplied batch_size is 0 or alpha0 is outside (0, 1]; or if epochs, +/// assignment_chunk, or revival_interval is 0, epsilon is outside [0, 1], or tau +/// is negative. +template +EMeansResult emeans_train( + const EMeansParameters& parameters, + Data& data, + data::SimpleData initial_centroids, + std::mt19937& rng, + Pool& threadpool, + lib::Timer& timer +) { + size_t n = data.size(); + size_t k = initial_centroids.size(); + size_t dims = data.dimensions(); + + if (k > n) { + throw ANNEXCEPTION( + "emeans_train: num_clusters={} exceeds the number of available points n={}", + k, + n + ); + } + if (parameters.epochs == 0) { + throw ANNEXCEPTION("emeans_train: epochs must be positive, got 0"); + } + if (parameters.batch_size.has_value() && *parameters.batch_size == 0) { + throw ANNEXCEPTION( + "emeans_train: batch_size must be positive, got {}", *parameters.batch_size + ); + } + if (parameters.alpha0.has_value() && + (*parameters.alpha0 <= 0.0 || *parameters.alpha0 > 1.0)) { + throw ANNEXCEPTION( + "emeans_train: alpha0 must lie in (0, 1], got {}", *parameters.alpha0 + ); + } + if (parameters.assignment_chunk == 0) { + throw ANNEXCEPTION("emeans_train: assignment_chunk must be positive, got 0"); + } + if (parameters.revival_interval == 0) { + throw ANNEXCEPTION("emeans_train: revival_interval must be positive, got 0"); + } + if (parameters.epsilon < 0.0 || parameters.epsilon > 1.0) { + throw ANNEXCEPTION( + "emeans_train: epsilon must lie in [0, 1], got {}", parameters.epsilon + ); + } + if (parameters.tau < 0.0) { + throw ANNEXCEPTION( + "emeans_train: tau must be non-negative, got {}", parameters.tau + ); + } + + auto training_timer = timer.push_back("EMeans training"); + + auto resolved = resolve_emeans_parameters(n, k, parameters); + size_t batch_size = resolved.batch_size; + double alpha0 = resolved.alpha0; + size_t chunk_size = std::min(parameters.assignment_chunk, n); + + // M, the cosine schedule's span: the paper writes M = T*floor(n/B), but a tail batch + // (n not a multiple of B) is still trained here, so M is the true batch count instead. + size_t batches_per_epoch = lib::div_round_up(n, batch_size); + size_t total_batches = parameters.epochs * batches_per_epoch; + + // State (Algorithm 1, step 2): double counts/sums, k + k*d doubles, allocated once. + std::vector N(k, detail::EMEANS_INITIAL_COUNT); + std::vector S(k * dims, 0.0); + for (size_t j = 0; j < k; ++j) { + auto c = initial_centroids.get_datum(j); + for (size_t d = 0; d < dims; ++d) { + S[j * dims + d] = N[j] * static_cast(c[d]); + } + } + + // Scratch buffers sized once and reused for every batch; chunk_matmul_results is + // capped at chunk_size rows, not batch_size (B from the scaling law can be huge). + auto centroids_fp32 = data::SimpleData(k, dims); + auto gathered = data::SimpleData(batch_size, dims); + auto chunk_matmul_results = data::SimpleData(chunk_size, k); + std::vector chunk_norm_scratch(chunk_size); + std::vector chunk_assignments(chunk_size); + std::vector centroids_norm(k); + std::vector local_assignments(batch_size); + std::vector batch_counts(k); + std::vector batch_sums(k * dims); + std::vector perm(n); + for (size_t i = 0; i < n; ++i) { + perm[i] = i; + } + std::vector last_epoch_assignments(n); + std::vector revival_dead_scratch; + revival_dead_scratch.reserve(k); + std::vector revival_excluded_scratch(k, false); + std::vector revival_cum_scratch(k); + // Per-batch kernel timing is uninteresting at hundreds of batches; keep it off `timer`. + lib::Timer scratch_timer; + + auto distance = svs::DistanceL2{}; + // Algorithm 1 is 1-indexed ("t <- t+1, gamma <- gamma(t)"): t ranges 1..M, so + // alpha(t) < alpha0 strictly even on the very first batch (gamma never starts at 0). + size_t t = 0; + for (size_t epoch = 0; epoch < parameters.epochs; ++epoch) { + auto epoch_timer = timer.push_back("epoch"); + std::shuffle(perm.begin(), perm.end(), rng); + bool last_epoch = (epoch + 1 == parameters.epochs); + + for (size_t batch = 0; batch < batches_per_epoch; ++batch) { + size_t start = batch * batch_size; + size_t end = std::min(start + batch_size, n); + size_t this_batch_size = end - start; + ++t; + + // Gather this batch's points into the contiguous scratch buffer the reused + // Lloyd-path assignment kernel (centroid_assignment) requires. + threads::parallel_for( + threadpool, + threads::StaticPartition{this_batch_size}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + auto dst = gathered.get_datum(i); + auto src = data.get_datum(perm[start + i]); + for (size_t d = 0; d < dims; ++d) { + dst[d] = static_cast(src[d]); + } + } + } + ); + + double alpha = + detail::cosine_step(alpha0, parameters.epsilon, t, total_batches); + double gamma = 1.0 - alpha; + + // Eq. 4 readout: c_j = S_j / N_j, into the fp32 buffer centroid_assignment + // uses. A cluster with N_j == 0 (EMA-decayed to nothing, never revived) keeps + // its previous centroid instead of dividing by zero. + threads::parallel_for( + threadpool, + threads::StaticPartition{k}, + [&](auto indices, auto /*tid*/) { + for (auto j : indices) { + if (N[j] == 0.0) { + continue; + } + auto c = centroids_fp32.get_datum(j); + for (size_t d = 0; d < dims; ++d) { + c[d] = static_cast(S[j * dims + d] / N[j]); + } + } + } + ); + + generate_norms(centroids_fp32, centroids_norm, threadpool); + for (size_t offset = 0; offset < this_batch_size; offset += chunk_size) { + size_t this_chunk = std::min(chunk_size, this_batch_size - offset); + detail::assign_chunk( + gathered, + offset, + this_chunk, + distance, + centroids_fp32, + centroids_norm, + chunk_assignments, + chunk_norm_scratch, + chunk_matmul_results, + threadpool, + scratch_timer + ); + for (size_t i = 0; i < this_chunk; ++i) { + local_assignments[offset + i] = chunk_assignments[i]; + } + } + + std::fill(batch_counts.begin(), batch_counts.end(), 0.0); + std::fill(batch_sums.begin(), batch_sums.end(), 0.0); + detail::accumulate_batch_stats( + gathered, + this_batch_size, + local_assignments, + k, + batch_counts, + batch_sums, + threadpool + ); + + // Eq. 2: N_j <- gamma*N_j + alpha*n_j, S_j <- gamma*S_j + alpha*s_j. + threads::parallel_for( + threadpool, + threads::StaticPartition{k}, + [&](auto indices, auto /*tid*/) { + for (auto j : indices) { + N[j] = gamma * N[j] + alpha * batch_counts[j]; + double* s_row = S.data() + j * dims; + const double* bs_row = batch_sums.data() + j * dims; + for (size_t d = 0; d < dims; ++d) { + s_row[d] = gamma * s_row[d] + alpha * bs_row[d]; + } + } + } + ); + + if (last_epoch) { + for (size_t i = 0; i < this_batch_size; ++i) { + last_epoch_assignments[perm[start + i]] = local_assignments[i]; + } + } + + if (t % parameters.revival_interval == 0) { + detail::revive_dead_clusters( + N, + S, + dims, + parameters.tau, + rng, + revival_dead_scratch, + revival_excluded_scratch, + revival_cum_scratch + ); + } + } + } + + // Final readout (same zero-count guard as the per-batch readout above), then the + // optional closing-mean pass (Fig. 22). + threads::parallel_for( + threadpool, + threads::StaticPartition{k}, + [&](auto indices, auto /*tid*/) { + for (auto j : indices) { + if (N[j] == 0.0) { + continue; + } + auto c = centroids_fp32.get_datum(j); + for (size_t d = 0; d < dims; ++d) { + c[d] = static_cast(S[j * dims + d] / N[j]); + } + } + } + ); + if (parameters.final_mean_pass) { + detail::closing_mean_pass(data, centroids_fp32, last_epoch_assignments, threadpool); + } + + double objective = detail::compute_objective( + data, + centroids_fp32, + chunk_size, + chunk_matmul_results, + chunk_norm_scratch, + centroids_norm, + chunk_assignments, + threadpool, + scratch_timer + ); + training_timer.finish(); + return EMeansResult{std::move(centroids_fp32), objective}; +} + +/// @brief e-means training (arXiv:2611.09501v1, Algorithm 1), initializing centroids by +/// random sampling without replacement from `data` (Algorithm 1, step 1). +/// +/// @param parameters e-means hyperparameters; see EMeansParameters. +/// @param data The training set. Not modified. +/// @param num_clusters k, the target number of clusters. +/// @param rng Advanced by the initial sample, then as emeans_train's other overload does. +/// @param threadpool Thread pool used for every parallel step. +/// @param timer Receives per-epoch timing entries, in kmeans_training.h's style. +/// +/// @throws ANNEXCEPTION if num_clusters exceeds the number of available points, or for the +/// same invalid-parameter cases as the initial-centroids overload. +template +EMeansResult emeans_train( + const EMeansParameters& parameters, + Data& data, + size_t num_clusters, + std::mt19937& rng, + Pool& threadpool, + lib::Timer& timer +) { + size_t n = data.size(); + if (num_clusters > n) { + throw ANNEXCEPTION( + "emeans_train: num_clusters={} exceeds the number of available points n={}", + num_clusters, + n + ); + } + std::vector id_scratch(num_clusters); + auto initial = init_centroids(data, id_scratch, num_clusters, rng, threadpool); + return emeans_train(parameters, data, std::move(initial), rng, threadpool, timer); +} + +} // namespace svs::kmeans diff --git a/include/svs/core/kmeans_training.h b/include/svs/core/kmeans_training.h new file mode 100644 index 000000000..7c5297c6f --- /dev/null +++ b/include/svs/core/kmeans_training.h @@ -0,0 +1,573 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/data.h" +#include "svs/concepts/distance.h" +#include "svs/core/data/simple.h" +#include "svs/core/data/view.h" +#include "svs/core/distance.h" +#include "svs/lib/exception.h" +#include "svs/lib/neighbor.h" +#include "svs/lib/threads/threadpool.h" +#include "svs/lib/timing.h" +#include "svs/lib/type_traits.h" + +// external +#include "tsl/robin_set.h" + +// Intel(R) MKL +#include + +// stl +#include + +// Generic k-means training shared by IVF and multi-vector codebook training. +namespace svs::kmeans { + +/// Helper trait to check if a distance type behaves like IP (inner product) +template +inline constexpr bool is_ip_like_v = + std::is_same_v, distance::DistanceIP> || + std::is_same_v, distance::DistanceCosineSimilarity>; + +/// Helper trait to check if a distance type is L2 +template +inline constexpr bool is_l2_v = + std::is_same_v, distance::DistanceL2>; + +// Small epsilon value used for floating-point comparisons to avoid precision +// issues. The value 1/1024 (approximately 0.0009765625) is chosen as a reasonable +// threshold for numerical stability in algorithms such as k-means clustering, where exact +constexpr double EPSILON = 1.0 / 1024.0; + +/// @brief Parameters controlling the k-means training loop. +struct KMeansTrainingParameters { + /// The number of iterations used in kmeans training. + size_t num_iterations; + /// The size of each minibatch. + size_t minibatch_size; +}; + +// Helper functions to convert data from one type to another using threadpool +template +void convert_data(D1& src, D2& dst, Pool& threadpool) { + // Note: Destination size can be bigger than the source as we preallocate a bigger + // buffer and reuse it to reduce the cost of frequent allocations + if (src.size() > dst.size() || src.dimensions() != dst.dimensions()) { + throw ANNEXCEPTION( + "Unexpected data shapes sizes: {}, {}; dims: {}, {}!", + src.size(), + dst.size(), + src.dimensions(), + dst.dimensions() + ); + } + + threads::parallel_for( + threadpool, + threads::StaticPartition{src.size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + dst.set_datum(i, src.get_datum(i)); + } + } + ); +} + +template +auto convert_data(Data& src, Pool& threadpool) { + auto dst = svs::data::SimpleData(src.size(), src.dimensions()); + convert_data(src, dst, threadpool); + return dst; +} + +// Partial specialization to preserve the dimensionality and Allocation type +template +auto convert_data(svs::data::SimpleData& src, Pool& threadpool) { + using allocator_type = svs::lib::rebind_allocator_t; + allocator_type rebound_alloctor = {}; + + auto dst = svs::data::SimpleData( + src.size(), src.dimensions(), rebound_alloctor + ); + convert_data(src, dst, threadpool); + return dst; +} + +template auto convert_data(Data& src) { + auto dst = svs::data::SimpleData(src.size(), src.dimensions()); + auto threadpool = threads::as_threadpool(1); + convert_data(src, dst, threadpool); + return dst; +} + +template +void compute_matmul( + const T* data, const T* centroids, float* results, size_t m, size_t n, size_t k +) { + // Early return for zero dimensions. + // Calling Intel MKL functions with zero dimensions may result in undefined behavior + // or runtime errors. This check ensures we avoid such cases. + if (m == 0 || n == 0 || k == 0) { + return; // Nothing to compute + } + + // Check for integer overflow when casting to int (MKL requirement) + constexpr size_t max_int = static_cast(std::numeric_limits::max()); + if (m > max_int || n > max_int || k > max_int) { + throw ANNEXCEPTION( + "Matrix dimensions too large for Intel MKL GEMM: m={}, n={}, k={}, max={}", + m, + n, + k, + max_int + ); + } + + // Cast size_t parameters to int for MKL GEMM functions + int m_int = static_cast(m); + int n_int = static_cast(n); + int k_int = static_cast(k); + if constexpr (std::is_same_v) { + cblas_sgemm( + CblasRowMajor, // CBLAS_LAYOUT layout + CblasNoTrans, // CBLAS_TRANSPOSE TransA + CblasTrans, // CBLAS_TRANSPOSE TransB + m_int, // const int M + n_int, // const int N + k_int, // const int K + 1.0F, // float alpha + data, // const float* A + k_int, // const int lda + centroids, // const float* B + k_int, // const int ldb + 0.0F, // const float beta + results, // float* c + n_int // const int ldc + ); + } else if constexpr (std::is_same_v) { + cblas_gemm_bf16bf16f32( + CblasRowMajor, // CBLAS_LAYOUT layout + CblasNoTrans, // CBLAS_TRANSPOSE TransA + CblasTrans, // CBLAS_TRANSPOSE TransB + m_int, // const int M + n_int, // const int N + k_int, // const int K + 1.0F, // float alpha + (const uint16_t*)data, // const *uint16_t A + k_int, // const int lda + (const uint16_t*)centroids, // const uint16_t* B + k_int, // const int ldb + 0.0F, // const float beta + results, // float* c + n_int // const int ldc + ); + } else if constexpr (std::is_same_v) { + cblas_gemm_f16f16f32( + CblasRowMajor, // CBLAS_LAYOUT layout + CblasNoTrans, // CBLAS_TRANSPOSE TransA + CblasTrans, // CBLAS_TRANSPOSE TransB + m_int, // const int M + n_int, // const int N + k_int, // const int K + 1.0F, // float alpha + (const uint16_t*)data, // const *uint16_t A + k_int, // const int lda + (const uint16_t*)centroids, // const uint16_t* B + k_int, // const int ldb + 0.0F, // const float beta + results, // float* c + n_int // const int ldc + ); + } else { + throw ANNEXCEPTION("GEMM type not supported!"); + } +} + +inline static void +generate_unique_ids(std::vector& ids, size_t id_range, std::mt19937& rng) { + size_t n = ids.size(); + tsl::robin_set seen; + seen.reserve(n); + + while (seen.size() < n) { + auto j = rng() % id_range; + seen.insert(j); + } + + size_t i = 0; + for (auto it = seen.begin(); it != seen.end(); ++it, ++i) { + ids[i] = *it; + } +} + +template +void normalize_centroids( + data::SimpleData& centroids, Pool& threadpool, lib::Timer& timer +) { + auto normalize_centroids_t = timer.push_back("normalize centroids"); + threads::parallel_for( + threadpool, + threads::StaticPartition{centroids.size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + auto datum = centroids.get_datum(i); + float norm = distance::norm(datum); + if (norm != 0.0) { + float norm_inv = 1.0F / norm; + for (size_t j = 0; j < datum.size(); j++) { + datum[j] = datum[j] * norm_inv; + } + } + } + } + ); + normalize_centroids_t.finish(); +} + +template < + data::ImmutableMemoryDataset Data, + typename T, + typename Distance, + threads::ThreadPool Pool> +void centroid_assignment( + const Data& data, + std::vector& data_norm, + threads::UnitRange batch_range, + Distance& SVS_UNUSED(distance), + data::SimpleData& centroids, + std::vector& centroids_norm, + std::vector& assignments, + data::SimpleData& matmul_results, + Pool& threadpool, + lib::Timer& timer +) { + using DataType = typename Data::element_type; + using CentroidType = T; + + // Convert data to match centroid type if necessary, otherwise use original data + [[maybe_unused]] data::SimpleData data_conv; + if constexpr (!std::is_same_v) { + data_conv = convert_data(data, threadpool); + } + const auto& matmul_data = [&]() -> const auto& { + if constexpr (!std::is_same_v) { + return data_conv; + } else { + return data; + } + } + (); + + auto generate_assignments = timer.push_back("generate assignments"); + threads::parallel_for( + threadpool, + threads::StaticPartition{batch_range.size()}, + [&](auto indices, auto /*tid*/) { + auto range = threads::UnitRange(indices); + compute_matmul( + matmul_data.get_datum(range.start()).data(), + centroids.data(), + matmul_results.get_datum(range.start()).data(), + range.size(), + centroids.size(), + matmul_data.dimensions() + ); + if constexpr (is_ip_like_v) { + for (auto i : indices) { + auto nearest = + type_traits::sentinel_v, std::greater<>>; + auto dists = matmul_results.get_datum(i); + for (size_t j = 0; j < centroids.size(); j++) { + nearest = std::max(nearest, Neighbor(j, dists[j])); + } + assignments[batch_range.start() + i] = nearest.id(); + } + } else if constexpr (is_l2_v) { + for (auto i : indices) { + auto nearest = type_traits::sentinel_v, std::less<>>; + auto dists = matmul_results.get_datum(i); + for (size_t j = 0; j < centroids.size(); j++) { + auto dist = data_norm[batch_range.start() + i] + centroids_norm[j] - + (2 * dists[j]); + nearest = std::min(nearest, Neighbor(j, dist)); + } + assignments[batch_range.start() + i] = nearest.id(); + } + } else { + // Compile-time error for unsupported distance types + static_assert( + sizeof(Distance) == 0, + "Only L2, MIP, and Cosine distances are supported in IVF build!" + ); + } + } + ); + generate_assignments.finish(); +} + +template +void centroid_adjustment( + Data& data, + data::SimpleData& centroids, + std::vector& assignments, + std::vector& counts, + Pool& threadpool, + lib::Timer& timer +) { + auto adjust_centroids = timer.push_back("adjust centroids"); + size_t n_threads = threadpool.size(); + size_t n_centroids = centroids.size(); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_threads}, + [&](auto /*indices*/, auto tid) { + size_t centroid_start = (n_centroids * tid) / n_threads; + size_t centroid_end = (n_centroids * (tid + 1)) / n_threads; + for (auto i : data.eachindex()) { + auto assignment = assignments[i]; + if (assignment >= centroid_start && assignment < centroid_end) { + counts.at(assignment)++; + auto datum = data.get_datum(i); + auto this_centroid = centroids.get_datum(assignment); + for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { + this_centroid[p] += lib::narrow_cast(datum[p]); + } + } + } + } + ); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_centroids}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + if (counts.at(i) != 0) { + auto this_centroid = centroids.get_datum(i); + float norm = 1.0 / (counts.at(i) + 1); + // float norm = 1.0 / counts.at(i); + for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { + this_centroid[p] *= norm; + } + } + } + } + ); + adjust_centroids.finish(); +} + +template +void centroid_split( + Data& data, + data::SimpleData& centroids, + std::vector& counts, + std::mt19937& rng, + Pool& SVS_UNUSED(threadpool), + lib::Timer& timer +) { + auto split_centroids = timer.push_back("split centroids"); + + auto num_centroids = centroids.size(); + auto dims = centroids.dimensions(); + auto num_data = data.size(); + + auto distribution = std::uniform_real_distribution(0.0, 1.0); + for (size_t i = 0; i < num_centroids; i++) { + if (counts.at(i) == 0) { + size_t j; + for (j = 0; true; j = (j + 1) % num_centroids) { + if (counts.at(j) == 0) { + continue; + } + float p = static_cast(counts.at(j)) / static_cast(num_data); + float r = distribution(rng); + if (r < p) { + break; + } + } + centroids.set_datum(i, centroids.get_datum(j)); + for (size_t k = 0; k < dims; k++) { + if (k % 2 == 0) { + centroids.get_datum(i)[k] *= 1 + EPSILON; + centroids.get_datum(j)[k] *= 1 - EPSILON; + } else { + centroids.get_datum(i)[k] *= 1 - EPSILON; + centroids.get_datum(j)[k] *= 1 + EPSILON; + } + } + counts.at(i) = counts.at(j) / 2; + counts.at(j) -= counts.at(i); + } + } + split_centroids.finish(); +} + +template +void generate_norms(Data& data, std::vector& norms, Pool& threadpool) { + norms.resize(data.size()); + threads::parallel_for( + threadpool, + threads::StaticPartition{data.size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + norms[i] = distance::norm_square(data.get_datum(i)); + } + } + ); +} + +template < + data::ImmutableMemoryDataset Data, + typename T, + typename Distance, + threads::ThreadPool Pool> +auto kmeans_training( + const KMeansTrainingParameters& parameters, + Data& data, + Distance& distance, + data::SimpleData& centroids, + data::SimpleData& matmul_results, + std::mt19937& rng, + Pool& threadpool, + lib::Timer& timer +) { + auto training_timer = timer.push_back("Kmeans training"); + data::SimpleData centroids_fp32 = convert_data(centroids, threadpool); + + if constexpr (is_ip_like_v) { + normalize_centroids(centroids_fp32, threadpool, timer); + } + + auto assignments = std::vector(data.size()); + std::vector data_norm; + if constexpr (is_l2_v) { + generate_norms(data, data_norm, threadpool); + } + std::vector centroids_norm; + + for (size_t iter = 0; iter < parameters.num_iterations; ++iter) { + auto iter_timer = timer.push_back("iteration"); + auto batchsize = parameters.minibatch_size; + auto num_batches = lib::div_round_up(data.size(), batchsize); + if constexpr (is_l2_v) { + generate_norms(centroids_fp32, centroids_norm, threadpool); + } + + // Convert from fp32 to fp16/bf16 + convert_data(centroids_fp32, centroids, threadpool); + + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = threads::UnitRange{ + batch * batchsize, std::min((batch + 1) * batchsize, data.size())}; + auto data_batch = data::make_view(data, this_batch); + centroid_assignment( + data_batch, + data_norm, + this_batch, + distance, + centroids, + centroids_norm, + assignments, + matmul_results, + threadpool, + timer + ); + } + + // Convert back to fp32 + convert_data(centroids, centroids_fp32, threadpool); + + auto counts = std::vector(centroids.size()); + centroid_adjustment(data, centroids_fp32, assignments, counts, threadpool, timer); + + centroid_split(data, centroids_fp32, counts, rng, threadpool, timer); + + if constexpr (is_ip_like_v) { + normalize_centroids(centroids_fp32, threadpool, timer); + } + } + + // Finally call the conversion to get the updated centroids + // after adjustment and split + convert_data(centroids_fp32, centroids, threadpool); + training_timer.finish(); + return centroids_fp32; +} + +/// @brief Generate a random subset of data for training +template +data::SimpleData make_training_set( + const Data& data, + std::vector& ids, + size_t num_training, + std::mt19937& rng, + Pool& threadpool +) { + data::SimpleData trainset( + num_training, data.dimensions() + ); + generate_unique_ids(ids, data.size(), rng); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_training}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + trainset.set_datum(i, data.get_datum(ids[i])); + } + } + ); + return trainset; +} + +/// @brief Initialize centroids randomly from the training set +template +data::SimpleData init_centroids( + const Data& trainset, + std::vector& ids, + size_t num_centroids, + std::mt19937& rng, + Pool& threadpool +) { + data::SimpleData centroids(num_centroids, trainset.dimensions()); + generate_unique_ids(ids, trainset.size(), rng); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_centroids}, + [&](auto indices, auto) { + for (auto i : indices) { + centroids.set_datum(i, trainset.get_datum(ids[i])); + } + } + ); + return centroids; +} + +/// @brief Compute norms for L2 distance if needed +template +std::vector maybe_compute_norms(const Data& data, Pool& threadpool) { + std::vector norms; + if constexpr (is_l2_v) { + generate_norms(data, norms, threadpool); + } + return norms; +} + +} // namespace svs::kmeans diff --git a/include/svs/extensions/vamana/gem.h b/include/svs/extensions/vamana/gem.h new file mode 100644 index 000000000..463828544 --- /dev/null +++ b/include/svs/extensions/vamana/gem.h @@ -0,0 +1,264 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// index/vamana depends on core/distance, not the reverse, so these overrides live here +// rather than in core/distance/gem.h. +#include "svs/concepts/vector_set.h" +#include "svs/core/data/simple.h" +#include "svs/core/distance/gem.h" +#include "svs/index/vamana/extensions.h" +#include "svs/index/vamana/prune.h" +#include "svs/lib/exception.h" + +// stl +#include +#include +#include +#include +#include +#include + +namespace svs::index::vamana { + +// qEMD is "smaller is better" like L2 (std::less<>), so share its MRNG pruning +// bookkeeping rather than IP/cosine's. +template <> struct PruneStrategy { + using type = ProgressivePruneStrategy; +}; + +// qCH is also std::less<>; the MVG ablation (--metric qch) builds the graph with QChamfer +// directly, so it needs the same pruning strategy as QEmd. +template <> struct PruneStrategy { + using type = ProgressivePruneStrategy; +}; + +/// Starts from the index's fixed entry points plus, with a GemRouter, the routed +/// representatives for the query's table; falls back to EntryPointInitializer's +/// behavior exactly when there is no router. +template struct GemRoutedInitializer { + template < + typename Buffer, + typename Computer, + graphs::ImmutableMemoryGraph Graph, + typename Builder, + typename Tracker> + void operator()( + Buffer& buffer, + const Computer& computer, + const Graph& graph, + const Builder& builder, + Tracker& tracker + ) const { + using Idx = typename Buffer::index_type; + buffer.clear(); + + auto push = [&](I id) { + auto dist = computer(id); + buffer.push_back(builder(id, dist)); + graph.prefetch_node(id); + tracker.visited(Neighbor{id, dist}, 1); + }; + + const auto& router = distance_->router(); + if (!router) { + for (I id : entry_points_) { + push(id); + } + buffer.sort(); + return; + } + + // thread_local: no per-query allocation once scratch reaches its steady-state size. + thread_local svs::distance::GemRouter::QueryScratch route_scratch; + thread_local std::vector routed; + router->route_query( + distance_->table(), distance_->query_tokens(), routed, route_scratch + ); + + thread_local std::vector merged; + merged.assign(entry_points_.begin(), entry_points_.end()); + for (uint32_t id : routed) { + merged.push_back(static_cast(id)); + } + std::sort(merged.begin(), merged.end()); + merged.erase(std::unique(merged.begin(), merged.end()), merged.end()); + + for (I id : merged) { + push(id); + } + buffer.sort(); + } + + ///// Members + const svs::distance::QChamferExactQuery* distance_; + std::span entry_points_; +}; + +namespace extensions { + +// DefaultBuildAdaptor default-constructs its distance_ member, which drops the +// CentroidTable svs::distance::QEmd holds; forward the real functor through aggregate-init +// instead. +inline DefaultBuildAdaptor svs_invoke( + svs::tag_t, const auto& /*data*/, const svs::distance::QEmd& distance +) { + return DefaultBuildAdaptor{distance}; +} + +// Same reason as the QEmd overload above: QChamfer also holds a CentroidTable that a +// default-constructed distance_ would drop. +inline DefaultBuildAdaptor svs_invoke( + svs::tag_t, const auto& /*data*/, const svs::distance::QChamfer& distance +) { + return DefaultBuildAdaptor{distance}; +} + +// Routes a node's own insert like GemRoutedInitializer routes a query; no-op if unrouted. +template + requires std::unsigned_integral +std::span svs_invoke( + svs::tag_t, + const Data& data, + const svs::distance::QEmd& distance, + Idx node_id, + std::span entry_points, + std::vector& scratch +) { + const auto& router = distance.router(); + if (!router) { + return entry_points; + } + + auto doc = data.get_datum(node_id); + svs::distance::gem_detail::require_id_dimensions(doc.dimensions()); + thread_local std::vector id_buf; + const uint32_t* ids = + svs::distance::gem_detail::id_span(doc.data(), doc.cardinality(), id_buf); + + thread_local svs::distance::GemRouter::IdScratch route_scratch; + thread_local std::vector routed; + router->route_ids( + std::span(ids, doc.cardinality()), + router->nprobe, + routed, + route_scratch + ); + + scratch.assign(entry_points.begin(), entry_points.end()); + for (uint32_t id : routed) { + scratch.push_back(static_cast(id)); + } + std::sort(scratch.begin(), scratch.end()); + scratch.erase(std::unique(scratch.begin(), scratch.end()), scratch.end()); + scratch.erase(std::remove(scratch.begin(), scratch.end(), node_id), scratch.end()); + + if (scratch.empty()) { + return entry_points; + } + return std::span(scratch); +} + +// Routes a QChamferExactQuery search; no-op to EntryPointInitializer when unrouted. +template +GemRoutedInitializer svs_invoke( + svs::tag_t, + const Data& /*data*/, + const svs::distance::QChamferExactQuery& distance, + std::span entry_points +) { + return GemRoutedInitializer{&distance, entry_points}; +} + +// L2 medoid of each document's pooled fine-centroid mean; a real entry point for raw ids. +template + requires std::unsigned_integral +size_t gem_pooled_entry_point( + const IdSets& docs, const svs::distance::FineCentroids& centroids, Pool& pool +) { + size_t n = docs.size(); + size_t dim = centroids.dim; + auto pooled = svs::data::SimpleData(n, dim); + + threads::parallel_for( + pool, + threads::StaticPartition{n}, + [&](const auto& ids, uint64_t /*tid*/) { + std::vector accum(dim); + for (size_t i : ids) { + auto doc = docs.get_datum(i); + if (doc.dimensions() != 1) { + throw ANNEXCEPTION( + "gem_pooled_entry_point: document {} has dimensions() = {}, " + "expected 1 (one centroid id per token)", + i, + doc.dimensions() + ); + } + size_t card = doc.cardinality(); + std::fill(accum.begin(), accum.end(), 0.0); + for (size_t t = 0; t < card; ++t) { + uint64_t cid = static_cast(doc.token(t)[0]); + if (cid >= centroids.K) { + throw ANNEXCEPTION( + "gem_pooled_entry_point: document {} has centroid id {}, " + "not < K = {}", + i, + cid, + centroids.K + ); + } + const svs::BFloat16* row = centroids.data.get() + cid * dim; + for (size_t d = 0; d < dim; ++d) { + accum[d] += static_cast(row[d]); + } + } + auto out = pooled.get_datum(i); + for (size_t d = 0; d < dim; ++d) { + out[d] = card > 0 + ? static_cast(accum[d] / static_cast(card)) + : 0.0F; + } + } + } + ); + + return utils::find_medioid(pooled, pool); +} + +// Turns an opaque medioid.h compile failure on centroid-id sets into a clear diagnostic. +template + requires svs::data::ImmutableVectorSetDataset && + std::unsigned_integral +size_t svs_invoke( + svs::tag_t, + const Data& /*dataset*/, + Pool& /*threadpool*/, + Predicate&& /*predicate*/ +) { + static_assert( + !std::is_same_v, + "compute_entry_point has no L2 medoid for centroid-id vector sets; build one " + "with svs::index::vamana::extensions::gem_pooled_entry_point and pass it to the " + "VamanaIndex constructor directly instead of relying on automatic entry-point " + "computation" + ); + return 0; +} + +} // namespace extensions +} // namespace svs::index::vamana diff --git a/include/svs/extensions/vamana/maxsim.h b/include/svs/extensions/vamana/maxsim.h new file mode 100644 index 000000000..1b82cd67e --- /dev/null +++ b/include/svs/extensions/vamana/maxsim.h @@ -0,0 +1,233 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/core/data/with_vector_sets.h" +#include "svs/core/distance/maxsim.h" +#include "svs/index/vamana/extensions.h" +#include "svs/lib/exception.h" +#include "svs/lib/preprocessor.h" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs::index::vamana::extensions { + +/// Process-wide re-rank timing accumulated since the last `maxsim_rerank_stats_reset()`. +struct MaxSimRerankStats { + uint64_t queries; + uint64_t search_ns; + uint64_t rerank_ns; +}; + +namespace stats_detail { +// Process-wide, not per-index: simpler than plumbing counters through every thread. +inline std::atomic maxsim_queries{0}; +inline std::atomic maxsim_search_ns{0}; +inline std::atomic maxsim_rerank_ns{0}; +} // namespace stats_detail + +inline MaxSimRerankStats maxsim_rerank_stats_snapshot() { + return MaxSimRerankStats{ + stats_detail::maxsim_queries.load(std::memory_order_relaxed), + stats_detail::maxsim_search_ns.load(std::memory_order_relaxed), + stats_detail::maxsim_rerank_ns.load(std::memory_order_relaxed)}; +} + +inline void maxsim_rerank_stats_reset() { + stats_detail::maxsim_queries.store(0, std::memory_order_relaxed); + stats_detail::maxsim_search_ns.store(0, std::memory_order_relaxed); + stats_detail::maxsim_rerank_ns.store(0, std::memory_order_relaxed); +} + +/// No-op re-rank timer: both methods are empty, so binding one into the re-rank hot path +/// costs nothing. Mirrors `CountingMaxSimRerankStats`'s interface exactly. +class NullMaxSimRerankStats { + public: + void mark_search_done() {} + void finish() {} +}; + +/// Re-rank timer that actually records: `mark_search_done()` splits the search and +/// re-rank durations, `finish()` folds both plus one query into the global accumulator. +class CountingMaxSimRerankStats { + public: + void mark_search_done() { search_done_ = std::chrono::steady_clock::now(); } + + void finish() { + auto rerank_done = std::chrono::steady_clock::now(); + auto ns = [](auto from, auto to) { + return static_cast( + std::chrono::duration_cast(to - from).count() + ); + }; + stats_detail::maxsim_queries.fetch_add(1, std::memory_order_relaxed); + stats_detail::maxsim_search_ns.fetch_add( + ns(start_, search_done_), std::memory_order_relaxed + ); + stats_detail::maxsim_rerank_ns.fetch_add( + ns(search_done_, rerank_done), std::memory_order_relaxed + ); + } + + private: + std::chrono::steady_clock::time_point start_ = std::chrono::steady_clock::now(); + std::chrono::steady_clock::time_point search_done_ = start_; +}; + +/// The timer `single_search` uses by default. Both timers above always compile, so only +/// this selection depends on the build-time flag. +using DefaultMaxSimRerankStats = std::conditional_t< + static_cast(SVS_ENABLE_STATS), + CountingMaxSimRerankStats, + NullMaxSimRerankStats>; + +namespace detail { + +// The intended caller's graph-search distance already consumes raw query tokens, so the +// primary distance must accept the query set directly against a primary datum. +template +concept AcceptsVectorSetQuery = + svs::data::VectorSet && + requires(Distance& dist, const Query& query, const PrimaryDatum& datum) { + { svs::distance::compute(dist, query, datum) } -> std::convertible_to; + }; + +// Re-sort by MaxSim score (larger is better). The buffer's own comparator is fixed at +// construction to the primary distance's direction, which may disagree with MaxSim's, so +// sort a local copy and write the order back rather than reuse `search_buffer.sort()`. +// `compare` does not depend on MaxSim's document-element template parameter, so the +// default instantiation's comparator applies regardless of which one scored the buffer. +template void resort_by_maxsim(SearchBuffer& search_buffer) { + size_t valid = search_buffer.size(); + using value_type = typename SearchBuffer::value_type; + std::vector reordered(search_buffer.begin(), search_buffer.end()); + std::sort( + reordered.begin(), + reordered.end(), + [](const value_type& a, const value_type& b) { + return svs::distance::MaxSim<>::compare{}(a.distance(), b.distance()); + } + ); + for (size_t j = 0; j < valid; ++j) { + search_buffer[j] = reordered[j]; + } +} + +} // namespace detail + +/// Native re-rank: run the ordinary graph search with the primary distance, then +/// overwrite every candidate still in the pool with its MaxSim score against the query +/// set, and restore best-first order by that score. +template < + typename Primary, + svs::data::ImmutableVectorSetDataset Sets, + typename SearchBuffer, + typename Distance, + typename Query, + typename Search, + typename Index> + requires detail:: + AcceptsVectorSetQuery + void svs_invoke( + svs::tag_t, + const svs::data::WithVectorSets& dataset, + SearchBuffer& search_buffer, + Distance& distance, + const Query& query, + const Search& search, + const Index& SVS_UNUSED(index), + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + ) { + // `check_and_supplement_search_buffer` inserts candidates scored with the primary + // distance into a buffer this overload has already rescored to MaxSim; mixing the two + // would sort as if every entry were a MaxSim score. + static_assert( + !Index::needs_id_translation, + "MaxSim re-rank does not support indexes that need id translation" + ); + + if (cancel()) { + return; + } + + if (query.dimensions() != dataset.vector_sets().dimensions()) { + throw ANNEXCEPTION( + "Query token dimension {} does not match dataset token dimension {}", + query.dimensions(), + dataset.vector_sets().dimensions() + ); + } + + auto accessor = data::GetDatumAccessor(); + DefaultMaxSimRerankStats rerank_stats{}; + search(query, accessor, distance, search_buffer); + rerank_stats.mark_search_done(); + + if (cancel()) { + return; + } + + // Packs the query once (must_fix_argument) and reuses that packing for every + // candidate still in the pool: repacking per candidate would cost as much as + // scoring them, which is exactly what PackedOperand/MaxSim::fix_argument avoids. + // The document element type is the vector-set dataset's own element type, so a + // bfloat16 store automatically gets the bf16 (and AMX, where available) kernel path + // while float/Float16 stores keep today's unrounded behaviour. + using DocElement = typename Sets::const_value_type::element_type; + svs::distance::MaxSim maxsim{}; + svs::distance::maybe_fix_argument(maxsim, query); + + size_t valid = search_buffer.size(); + for (size_t j = 0; j < valid; ++j) { + auto& neighbor = search_buffer[j]; + float score = + svs::distance::compute(maxsim, query, dataset.get_vector_set(neighbor.id())); + neighbor.set_distance(score); + } + + detail::resort_by_maxsim(search_buffer); + + rerank_stats.finish(); +} + +// The default `per_thread_batch_search` (extensions.h) dispatches per query through the +// `single_search` customization point above, which already leaves the buffer sorted +// best-first by MaxSim; no override is needed for the batch path. + +// `UsesReranking` is left at its default of `false`: the iterator's reranking path +// (include/svs/index/vamana/iterator.h) recomputes distances via `data.get_primary(id)` +// against the base distance, which this dataset does not expose and which would not be a +// MaxSim score regardless. + +template +auto svs_invoke( + svs::tag_t, + const svs::data::WithVectorSets& dataset +) { + return reconstruct_accessor(dataset.primary()); +} + +} // namespace svs::index::vamana::extensions diff --git a/include/svs/index/ivf/common.h b/include/svs/index/ivf/common.h index feedddadc..3948a8ddc 100644 --- a/include/svs/index/ivf/common.h +++ b/include/svs/index/ivf/common.h @@ -22,6 +22,7 @@ #include "svs/core/data/simple.h" #include "svs/core/data/view.h" #include "svs/core/distance.h" +#include "svs/core/kmeans_training.h" #include "svs/core/logging.h" #include "svs/index/ivf/sorted_buffer.h" #include "svs/lib/exception.h" @@ -42,21 +43,21 @@ // Common definitions. namespace svs::index::ivf { -/// Helper trait to check if a distance type behaves like IP (inner product) -template -inline constexpr bool is_ip_like_v = - std::is_same_v, distance::DistanceIP> || - std::is_same_v, distance::DistanceCosineSimilarity>; - -/// Helper trait to check if a distance type is L2 -template -inline constexpr bool is_l2_v = - std::is_same_v, distance::DistanceL2>; - -// Small epsilon value used for floating-point comparisons to avoid precision -// issues. The value 1/1024 (approximately 0.0009765625) is chosen as a reasonable -// threshold for numerical stability in algorithms such as k-means clustering, where exact -constexpr double EPSILON = 1.0 / 1024.0; +// Re-exports the generic k-means names so existing svs::index::ivf::* call sites compile. +using svs::kmeans::centroid_adjustment; +using svs::kmeans::centroid_assignment; +using svs::kmeans::centroid_split; +using svs::kmeans::compute_matmul; +using svs::kmeans::convert_data; +using svs::kmeans::EPSILON; +using svs::kmeans::generate_norms; +using svs::kmeans::generate_unique_ids; +using svs::kmeans::init_centroids; +using svs::kmeans::is_ip_like_v; +using svs::kmeans::is_l2_v; +using svs::kmeans::make_training_set; +using svs::kmeans::maybe_compute_norms; +using svs::kmeans::normalize_centroids; /// Minimum training sample multiplier for clustering algorithms. /// When training data size is small relative to the number of clusters, we ensure @@ -185,376 +186,8 @@ struct IVFSearchParameters { operator==(const IVFSearchParameters&, const IVFSearchParameters&) = default; }; -// Helper functions to convert data from one type to another using threadpool -template -void convert_data(D1& src, D2& dst, Pool& threadpool) { - // Note: Destination size can be bigger than the source as we preallocate a bigger - // buffer and reuse it to reduce the cost of frequent allocations - if (src.size() > dst.size() || src.dimensions() != dst.dimensions()) { - throw ANNEXCEPTION( - "Unexpected data shapes sizes: {}, {}; dims: {}, {}!", - src.size(), - dst.size(), - src.dimensions(), - dst.dimensions() - ); - } - - threads::parallel_for( - threadpool, - threads::StaticPartition{src.size()}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - dst.set_datum(i, src.get_datum(i)); - } - } - ); -} - -template -auto convert_data(Data& src, Pool& threadpool) { - auto dst = svs::data::SimpleData(src.size(), src.dimensions()); - convert_data(src, dst, threadpool); - return dst; -} - -// Partial specialization to preserve the dimensionality and Allocation type -template -auto convert_data(svs::data::SimpleData& src, Pool& threadpool) { - using allocator_type = svs::lib::rebind_allocator_t; - allocator_type rebound_alloctor = {}; - - auto dst = svs::data::SimpleData( - src.size(), src.dimensions(), rebound_alloctor - ); - convert_data(src, dst, threadpool); - return dst; -} - -template auto convert_data(Data& src) { - auto dst = svs::data::SimpleData(src.size(), src.dimensions()); - auto threadpool = threads::as_threadpool(1); - convert_data(src, dst, threadpool); - return dst; -} - -template -void compute_matmul( - const T* data, const T* centroids, float* results, size_t m, size_t n, size_t k -) { - // Early return for zero dimensions. - // Calling Intel MKL functions with zero dimensions may result in undefined behavior - // or runtime errors. This check ensures we avoid such cases. - if (m == 0 || n == 0 || k == 0) { - return; // Nothing to compute - } - - // Check for integer overflow when casting to int (MKL requirement) - constexpr size_t max_int = static_cast(std::numeric_limits::max()); - if (m > max_int || n > max_int || k > max_int) { - throw ANNEXCEPTION( - "Matrix dimensions too large for Intel MKL GEMM: m={}, n={}, k={}, max={}", - m, - n, - k, - max_int - ); - } - - // Cast size_t parameters to int for MKL GEMM functions - int m_int = static_cast(m); - int n_int = static_cast(n); - int k_int = static_cast(k); - if constexpr (std::is_same_v) { - cblas_sgemm( - CblasRowMajor, // CBLAS_LAYOUT layout - CblasNoTrans, // CBLAS_TRANSPOSE TransA - CblasTrans, // CBLAS_TRANSPOSE TransB - m_int, // const int M - n_int, // const int N - k_int, // const int K - 1.0F, // float alpha - data, // const float* A - k_int, // const int lda - centroids, // const float* B - k_int, // const int ldb - 0.0F, // const float beta - results, // float* c - n_int // const int ldc - ); - } else if constexpr (std::is_same_v) { - cblas_gemm_bf16bf16f32( - CblasRowMajor, // CBLAS_LAYOUT layout - CblasNoTrans, // CBLAS_TRANSPOSE TransA - CblasTrans, // CBLAS_TRANSPOSE TransB - m_int, // const int M - n_int, // const int N - k_int, // const int K - 1.0F, // float alpha - (const uint16_t*)data, // const *uint16_t A - k_int, // const int lda - (const uint16_t*)centroids, // const uint16_t* B - k_int, // const int ldb - 0.0F, // const float beta - results, // float* c - n_int // const int ldc - ); - } else if constexpr (std::is_same_v) { - cblas_gemm_f16f16f32( - CblasRowMajor, // CBLAS_LAYOUT layout - CblasNoTrans, // CBLAS_TRANSPOSE TransA - CblasTrans, // CBLAS_TRANSPOSE TransB - m_int, // const int M - n_int, // const int N - k_int, // const int K - 1.0F, // float alpha - (const uint16_t*)data, // const *uint16_t A - k_int, // const int lda - (const uint16_t*)centroids, // const uint16_t* B - k_int, // const int ldb - 0.0F, // const float beta - results, // float* c - n_int // const int ldc - ); - } else { - throw ANNEXCEPTION("GEMM type not supported!"); - } -} - -inline static void -generate_unique_ids(std::vector& ids, size_t id_range, std::mt19937& rng) { - size_t n = ids.size(); - tsl::robin_set seen; - seen.reserve(n); - - while (seen.size() < n) { - auto j = rng() % id_range; - seen.insert(j); - } - - size_t i = 0; - for (auto it = seen.begin(); it != seen.end(); ++it, ++i) { - ids[i] = *it; - } -} - -template -void normalize_centroids( - data::SimpleData& centroids, Pool& threadpool, lib::Timer& timer -) { - auto normalize_centroids_t = timer.push_back("normalize centroids"); - threads::parallel_for( - threadpool, - threads::StaticPartition{centroids.size()}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - auto datum = centroids.get_datum(i); - float norm = distance::norm(datum); - if (norm != 0.0) { - float norm_inv = 1.0F / norm; - for (size_t j = 0; j < datum.size(); j++) { - datum[j] = datum[j] * norm_inv; - } - } - } - } - ); - normalize_centroids_t.finish(); -} - -template < - data::ImmutableMemoryDataset Data, - typename T, - typename Distance, - threads::ThreadPool Pool> -void centroid_assignment( - const Data& data, - std::vector& data_norm, - threads::UnitRange batch_range, - Distance& SVS_UNUSED(distance), - data::SimpleData& centroids, - std::vector& centroids_norm, - std::vector& assignments, - data::SimpleData& matmul_results, - Pool& threadpool, - lib::Timer& timer -) { - using DataType = typename Data::element_type; - using CentroidType = T; - - // Convert data to match centroid type if necessary, otherwise use original data - [[maybe_unused]] data::SimpleData data_conv; - if constexpr (!std::is_same_v) { - data_conv = convert_data(data, threadpool); - } - const auto& matmul_data = [&]() -> const auto& { - if constexpr (!std::is_same_v) { - return data_conv; - } else { - return data; - } - } - (); - - auto generate_assignments = timer.push_back("generate assignments"); - threads::parallel_for( - threadpool, - threads::StaticPartition{batch_range.size()}, - [&](auto indices, auto /*tid*/) { - auto range = threads::UnitRange(indices); - compute_matmul( - matmul_data.get_datum(range.start()).data(), - centroids.data(), - matmul_results.get_datum(range.start()).data(), - range.size(), - centroids.size(), - matmul_data.dimensions() - ); - if constexpr (is_ip_like_v) { - for (auto i : indices) { - auto nearest = - type_traits::sentinel_v, std::greater<>>; - auto dists = matmul_results.get_datum(i); - for (size_t j = 0; j < centroids.size(); j++) { - nearest = std::max(nearest, Neighbor(j, dists[j])); - } - assignments[batch_range.start() + i] = nearest.id(); - } - } else if constexpr (is_l2_v) { - for (auto i : indices) { - auto nearest = type_traits::sentinel_v, std::less<>>; - auto dists = matmul_results.get_datum(i); - for (size_t j = 0; j < centroids.size(); j++) { - auto dist = data_norm[batch_range.start() + i] + centroids_norm[j] - - (2 * dists[j]); - nearest = std::min(nearest, Neighbor(j, dist)); - } - assignments[batch_range.start() + i] = nearest.id(); - } - } else { - // Compile-time error for unsupported distance types - static_assert( - sizeof(Distance) == 0, - "Only L2, MIP, and Cosine distances are supported in IVF build!" - ); - } - } - ); - generate_assignments.finish(); -} - -template -void centroid_adjustment( - Data& data, - data::SimpleData& centroids, - std::vector& assignments, - std::vector& counts, - Pool& threadpool, - lib::Timer& timer -) { - auto adjust_centroids = timer.push_back("adjust centroids"); - size_t n_threads = threadpool.size(); - size_t n_centroids = centroids.size(); - - threads::parallel_for( - threadpool, - threads::StaticPartition{n_threads}, - [&](auto /*indices*/, auto tid) { - size_t centroid_start = (n_centroids * tid) / n_threads; - size_t centroid_end = (n_centroids * (tid + 1)) / n_threads; - for (auto i : data.eachindex()) { - auto assignment = assignments[i]; - if (assignment >= centroid_start && assignment < centroid_end) { - counts.at(assignment)++; - auto datum = data.get_datum(i); - auto this_centroid = centroids.get_datum(assignment); - for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { - this_centroid[p] += lib::narrow_cast(datum[p]); - } - } - } - } - ); - - threads::parallel_for( - threadpool, - threads::StaticPartition{n_centroids}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - if (counts.at(i) != 0) { - auto this_centroid = centroids.get_datum(i); - float norm = 1.0 / (counts.at(i) + 1); - // float norm = 1.0 / counts.at(i); - for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { - this_centroid[p] *= norm; - } - } - } - } - ); - adjust_centroids.finish(); -} - -template -void centroid_split( - Data& data, - data::SimpleData& centroids, - std::vector& counts, - std::mt19937& rng, - Pool& SVS_UNUSED(threadpool), - lib::Timer& timer -) { - auto split_centroids = timer.push_back("split centroids"); - - auto num_centroids = centroids.size(); - auto dims = centroids.dimensions(); - auto num_data = data.size(); - - auto distribution = std::uniform_real_distribution(0.0, 1.0); - for (size_t i = 0; i < num_centroids; i++) { - if (counts.at(i) == 0) { - size_t j; - for (j = 0; true; j = (j + 1) % num_centroids) { - if (counts.at(j) == 0) { - continue; - } - float p = static_cast(counts.at(j)) / static_cast(num_data); - float r = distribution(rng); - if (r < p) { - break; - } - } - centroids.set_datum(i, centroids.get_datum(j)); - for (size_t k = 0; k < dims; k++) { - if (k % 2 == 0) { - centroids.get_datum(i)[k] *= 1 + EPSILON; - centroids.get_datum(j)[k] *= 1 - EPSILON; - } else { - centroids.get_datum(i)[k] *= 1 - EPSILON; - centroids.get_datum(j)[k] *= 1 + EPSILON; - } - } - counts.at(i) = counts.at(j) / 2; - counts.at(j) -= counts.at(i); - } - } - split_centroids.finish(); -} - -template -void generate_norms(Data& data, std::vector& norms, Pool& threadpool) { - norms.resize(data.size()); - threads::parallel_for( - threadpool, - threads::StaticPartition{data.size()}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - norms[i] = distance::norm_square(data.get_datum(i)); - } - } - ); -} - +/// @brief Forward to svs::kmeans::kmeans_training using the IVF build parameters this +/// index has always taken; kept so existing IVF call sites need no change. template < data::ImmutableMemoryDataset Data, typename T, @@ -570,67 +203,11 @@ auto kmeans_training( Pool& threadpool, lib::Timer& timer ) { - auto training_timer = timer.push_back("Kmeans training"); - data::SimpleData centroids_fp32 = convert_data(centroids, threadpool); - - if constexpr (is_ip_like_v) { - normalize_centroids(centroids_fp32, threadpool, timer); - } - - auto assignments = std::vector(data.size()); - std::vector data_norm; - if constexpr (is_l2_v) { - generate_norms(data, data_norm, threadpool); - } - std::vector centroids_norm; - - for (size_t iter = 0; iter < parameters.num_iterations_; ++iter) { - auto iter_timer = timer.push_back("iteration"); - auto batchsize = parameters.minibatch_size_; - auto num_batches = lib::div_round_up(data.size(), batchsize); - if constexpr (is_l2_v) { - generate_norms(centroids_fp32, centroids_norm, threadpool); - } - - // Convert from fp32 to fp16/bf16 - convert_data(centroids_fp32, centroids, threadpool); - - for (size_t batch = 0; batch < num_batches; ++batch) { - auto this_batch = threads::UnitRange{ - batch * batchsize, std::min((batch + 1) * batchsize, data.size())}; - auto data_batch = data::make_view(data, this_batch); - centroid_assignment( - data_batch, - data_norm, - this_batch, - distance, - centroids, - centroids_norm, - assignments, - matmul_results, - threadpool, - timer - ); - } - - // Convert back to fp32 - convert_data(centroids, centroids_fp32, threadpool); - - auto counts = std::vector(centroids.size()); - centroid_adjustment(data, centroids_fp32, assignments, counts, threadpool, timer); - - centroid_split(data, centroids_fp32, counts, rng, threadpool, timer); - - if constexpr (is_ip_like_v) { - normalize_centroids(centroids_fp32, threadpool, timer); - } - } - - // Finally call the conversion to get the updated centroids - // after adjustment and split - convert_data(centroids_fp32, centroids, threadpool); - training_timer.finish(); - return centroids_fp32; + auto kmeans_parameters = svs::kmeans::KMeansTrainingParameters{ + parameters.num_iterations_, parameters.minibatch_size_}; + return svs::kmeans::kmeans_training( + kmeans_parameters, data, distance, centroids, matmul_results, rng, threadpool, timer + ); } template < @@ -683,64 +260,6 @@ void compute_centroid_distances( ); } -/// @brief Generate a random subset of data for training -template -data::SimpleData make_training_set( - const Data& data, - std::vector& ids, - size_t num_training, - std::mt19937& rng, - Pool& threadpool -) { - data::SimpleData trainset( - num_training, data.dimensions() - ); - generate_unique_ids(ids, data.size(), rng); - threads::parallel_for( - threadpool, - threads::StaticPartition{num_training}, - [&](auto indices, auto /*tid*/) { - for (auto i : indices) { - trainset.set_datum(i, data.get_datum(ids[i])); - } - } - ); - return trainset; -} - -/// @brief Initialize centroids randomly from the training set -template -data::SimpleData init_centroids( - const Data& trainset, - std::vector& ids, - size_t num_centroids, - std::mt19937& rng, - Pool& threadpool -) { - data::SimpleData centroids(num_centroids, trainset.dimensions()); - generate_unique_ids(ids, trainset.size(), rng); - threads::parallel_for( - threadpool, - threads::StaticPartition{num_centroids}, - [&](auto indices, auto) { - for (auto i : indices) { - centroids.set_datum(i, trainset.get_datum(ids[i])); - } - } - ); - return centroids; -} - -/// @brief Compute norms for L2 distance if needed -template -std::vector maybe_compute_norms(const Data& data, Pool& threadpool) { - std::vector norms; - if constexpr (is_l2_v) { - generate_norms(data, norms, threadpool); - } - return norms; -} - /// @brief Assign all points to clusters according to assignments template std::vector> group_assignments( diff --git a/include/svs/index/vamana/build_stats.h b/include/svs/index/vamana/build_stats.h new file mode 100644 index 000000000..3bfaeac7b --- /dev/null +++ b/include/svs/index/vamana/build_stats.h @@ -0,0 +1,226 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/concepts/distance.h" + +#include +#include +#include +#include +#include + +namespace svs::index::vamana { + +/// Build-time measurement counters. All distance counts and durations are summed over +/// every worker thread; `max_search_distances_per_insert` is a max over inserts instead. +struct BuildStats { + uint64_t inserts = 0; + uint64_t search_distances = 0; + uint64_t pool_distances = 0; + uint64_t prune_distances = 0; + uint64_t backedge_prune_distances = 0; + uint64_t search_ns = 0; + uint64_t prune_ns = 0; + uint64_t backedge_prune_ns = 0; + uint64_t max_search_distances_per_insert = 0; + + BuildStats& operator+=(const BuildStats& other) { + inserts += other.inserts; + search_distances += other.search_distances; + pool_distances += other.pool_distances; + prune_distances += other.prune_distances; + backedge_prune_distances += other.backedge_prune_distances; + search_ns += other.search_ns; + prune_ns += other.prune_ns; + backedge_prune_ns += other.backedge_prune_ns; + max_search_distances_per_insert = std::max( + max_search_distances_per_insert, other.max_search_distances_per_insert + ); + return *this; + } +}; + +namespace detail { +inline std::mutex& build_stats_mutex() { + static std::mutex mutex; + return mutex; +} + +inline BuildStats& build_stats_accumulator() { + static BuildStats stats{}; + return stats; +} +} // namespace detail + +/// Return a copy of the process-wide accumulator. +inline BuildStats build_stats_snapshot() { + std::lock_guard lock(detail::build_stats_mutex()); + return detail::build_stats_accumulator(); +} + +/// Zero the process-wide accumulator. +inline void build_stats_reset() { + std::lock_guard lock(detail::build_stats_mutex()); + detail::build_stats_accumulator() = BuildStats{}; +} + +/// Fold a thread's local counters into the process-wide accumulator. +/// Call once per thread, after that thread's `parallel_for` body finishes, so the hot +/// path itself touches no shared state. +inline void build_stats_merge(const BuildStats& stats) { + std::lock_guard lock(detail::build_stats_mutex()); + detail::build_stats_accumulator() += stats; +} + +/// Nanoseconds elapsed since `start`, for summing into a `BuildStats` duration field. +inline uint64_t build_stats_elapsed_ns(std::chrono::steady_clock::time_point start) { + auto elapsed = std::chrono::steady_clock::now() - start; + return static_cast( + std::chrono::duration_cast(elapsed).count() + ); +} + +/// Wraps `Dist` by reference, counting each distance::compute call into an unsynchronized +/// uint64_t& counter; forwards compare/implicit_broadcast/must_fix_argument from Dist. +/// Select PruneStrategy from the unwrapped Dist: it has no specialization for this type. +template class CountingDistance { + public: + using compare = distance::compare_t; + static constexpr bool implicit_broadcast = distance::implicitly_broadcastable(); + static constexpr bool must_fix_argument = distance::fix_argument_mandated(); + + CountingDistance(Dist& distance, uint64_t& counter) + : distance_{distance} + , counter_{counter} {} + + template float compute(A a, B b) { + ++counter_; + return distance::compute(distance_, a, b); + } + + // Constrained to Dist's own fix_argument support, so ShouldFix matches Dist exactly. + template + requires requires(Dist& d, A a) { d.fix_argument(a); } + void fix_argument(A a) { + distance_.fix_argument(a); + } + + auto comparator() const { return distance::comparator(distance_); } + + private: + Dist& distance_; + uint64_t& counter_; +}; + +/// No-op build-stats collector: every member is empty, so binding it into the build hot +/// path costs nothing. Mirrors `CountingBuildStats`'s interface exactly. +class NullBuildStats { + public: + struct ScopedTimer {}; + + ScopedTimer time_search() { return {}; } + ScopedTimer time_prune() { return {}; } + ScopedTimer time_backedge_prune() { return {}; } + + template Dist& wrap_search(Dist& dist) { return dist; } + template Dist& wrap_pool(Dist& dist) { return dist; } + template Dist& wrap_prune(Dist& dist) { return dist; } + template Dist& wrap_backedge_prune(Dist& dist) { return dist; } + + void finish_search() {} + void finish_pool() {} + void finish_prune() {} + void merge() {} +}; + +/// Build-stats collector that actually counts: one instance per worker thread, merged +/// into the process-wide accumulator once that thread's work is done. +class CountingBuildStats { + public: + // Adds its own elapsed time into `target` on destruction; never copied or moved, so + // that add happens exactly once per timed scope. + class ScopedTimer { + public: + explicit ScopedTimer(uint64_t& target) + : target_{&target} + , start_{std::chrono::steady_clock::now()} {} + ScopedTimer(const ScopedTimer&) = delete; + ScopedTimer& operator=(const ScopedTimer&) = delete; + ScopedTimer(ScopedTimer&&) = delete; + ScopedTimer& operator=(ScopedTimer&&) = delete; + ~ScopedTimer() { *target_ += build_stats_elapsed_ns(start_); } + + private: + uint64_t* target_; + std::chrono::steady_clock::time_point start_; + }; + + ScopedTimer time_search() { return ScopedTimer{current_.search_ns}; } + ScopedTimer time_prune() { return ScopedTimer{current_.prune_ns}; } + ScopedTimer time_backedge_prune() { return ScopedTimer{current_.backedge_prune_ns}; } + + // Resets its own per-call counter, so the caller must invoke this once per + // search/pool/prune step rather than reusing a previously returned wrapper. + template CountingDistance wrap_search(Dist& dist) { + search_count_ = 0; + return CountingDistance{dist, search_count_}; + } + template CountingDistance wrap_pool(Dist& dist) { + pool_count_ = 0; + return CountingDistance{dist, pool_count_}; + } + template CountingDistance wrap_prune(Dist& dist) { + prune_count_ = 0; + return CountingDistance{dist, prune_count_}; + } + + // Backedge counting spans the whole thread's work, not one node, so this is called + // once per thread and the returned wrapper is reused across every source vertex. + template CountingDistance wrap_backedge_prune(Dist& dist) { + return CountingDistance{dist, backedge_count_}; + } + + void finish_search() { + current_.search_distances += search_count_; + current_.max_search_distances_per_insert = + std::max(current_.max_search_distances_per_insert, search_count_); + ++current_.inserts; + } + void finish_pool() { current_.pool_distances += pool_count_; } + void finish_prune() { current_.prune_distances += prune_count_; } + + /// Call once per thread, after that thread's loop over nodes finishes. + void merge() { + current_.backedge_prune_distances = backedge_count_; + build_stats_merge(current_); + } + + private: + BuildStats current_{}; + uint64_t search_count_ = 0; + uint64_t pool_count_ = 0; + uint64_t prune_count_ = 0; + uint64_t backedge_count_ = 0; +}; + +/// The collector `VamanaBuilder` uses by default. Both collectors above always compile, +/// so only this selection depends on the build-time flag. +using DefaultBuildStats = std:: + conditional_t(SVS_ENABLE_STATS), CountingBuildStats, NullBuildStats>; + +} // namespace svs::index::vamana diff --git a/include/svs/index/vamana/connectivity.h b/include/svs/index/vamana/connectivity.h new file mode 100644 index 000000000..c68821d8d --- /dev/null +++ b/include/svs/index/vamana/connectivity.h @@ -0,0 +1,355 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Repairs graph connectivity a build can break by skipping most nodes (e.g. GEM starts). +#include "svs/concepts/data.h" +#include "svs/concepts/graph.h" +#include "svs/index/vamana/build_params.h" +#include "svs/index/vamana/extensions.h" +#include "svs/index/vamana/greedy_search.h" +#include "svs/index/vamana/prune.h" +#include "svs/index/vamana/search_buffer.h" +#include "svs/lib/misc.h" +#include "svs/lib/narrow.h" +#include "svs/lib/neighbor.h" +#include "svs/lib/threads.h" + +// stl +#include +#include +#include +#include + +namespace svs::index::vamana { + +/// Breadth-first search over out-edges from `roots`; returns the node ids `roots` cannot +/// reach, ascending. +template +std::vector +unreachable_nodes(const Graph& graph, std::span roots) { + using Idx = typename Graph::index_type; + size_t n = graph.n_nodes(); + std::vector visited(n, false); + + std::vector queue; + queue.reserve(n); + for (Idx root : roots) { + if (!visited[root]) { + visited[root] = true; + queue.push_back(root); + } + } + for (size_t head = 0; head < queue.size(); ++head) { + for (Idx v : graph.get_node(queue[head])) { + if (!visited[v]) { + visited[v] = true; + queue.push_back(v); + } + } + } + + std::vector result; + for (size_t i = 0; i < n; ++i) { + if (!visited[i]) { + result.push_back(lib::narrow_cast(i)); + } + } + return result; +} + +/// Outcome of `repair_connectivity`. edges_forced_unsafe, unlike the other edge counts, +/// has no safety guarantee and can orphan the node it displaced. `stalled` means a round +/// made no progress, which stops the repair before `max_rounds`. +struct ConnectivityReport { + size_t rounds = 0; + size_t unreachable_before = 0; + size_t unreachable_after = 0; + size_t edges_appended = 0; + size_t edges_accepted_through_prune = 0; + size_t edges_forced = 0; + size_t edges_forced_unsafe = 0; + bool stalled = false; +}; + +/// Reconnects every node `unreachable_nodes` reports, round by round, until none remain +/// or `max_rounds` is spent. Candidates are searched in parallel per round, but applied +/// serially in ascending node-id order since accepting v for one node changes deg(v). +template < + graphs::MemoryGraph Graph, + data::ImmutableMemoryDataset Data, + typename Dist, + threads::ThreadPool Pool> +ConnectivityReport repair_connectivity( + Graph& graph, + const Data& data, + const Dist& distance, + const VamanaBuildParameters& params, + std::span roots, + Pool& threadpool, + size_t max_rounds = 4 +) { + using Idx = typename Graph::index_type; + + std::vector is_root(graph.n_nodes(), false); + for (Idx root : roots) { + is_root[root] = true; + } + + ConnectivityReport report{}; + auto unreachable = unreachable_nodes(graph, roots); + report.unreachable_before = unreachable.size(); + + for (size_t round = 0; round < max_rounds && !unreachable.empty(); ++round) { + ++report.rounds; + + // Reachability snapshot at round start: proposals below are filtered against it. + std::vector reachable(graph.n_nodes(), true); + for (Idx u : unreachable) { + reachable[u] = false; + } + + // Kept current as edges change, so the forced step can detect a stranding drop. + std::vector reachable_in_degree(graph.n_nodes(), 0); + for (size_t x = 0; x < graph.n_nodes(); ++x) { + if (reachable[x]) { + for (Idx y : graph.get_node(lib::narrow_cast(x))) { + ++reachable_in_degree[y]; + } + } + } + + // Parallel search only; Phase B is serial since deg(v) changes within a round. + std::vector> proposals(unreachable.size()); + threads::parallel_for( + threadpool, + threads::StaticPartition{unreachable.size()}, + [&](const auto& is, uint64_t /*tid*/) { + auto build_adaptor = extensions::build_adaptor(data, distance); + auto accessor = build_adaptor.graph_search_accessor(); + auto&& search_distance = build_adaptor.graph_search_distance(); + auto search_buffer = + SearchBuffer>{params.window_size}; + auto tracker = NullTracker{}; + + for (size_t k : is) { + Idx u = unreachable[k]; + search_buffer.clear(); + const auto& query = + build_adaptor.access_query_for_graph_search(data, u); + greedy_search( + graph, + data, + accessor, + query, + search_distance, + search_buffer, + EntryPointInitializer{roots}, + NeighborBuilder(), + tracker + ); + search_buffer.sort(); + + auto& candidates = proposals[k]; + for (size_t i = 0, imax = search_buffer.size(); i < imax; ++i) { + Idx id = search_buffer[i].id(); + if (reachable[id]) { + candidates.push_back(id); + } + } + } + } + ); + + // Phase B (serial, ascending u). + auto build_adaptor = extensions::build_adaptor(data, distance); + auto accessor = build_adaptor.general_accessor(); + auto&& general_distance = build_adaptor.general_distance(); + auto cmp = distance::comparator(general_distance); + + std::vector order(unreachable.size()); + for (size_t k = 0; k < order.size(); ++k) { + order[k] = k; + } + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { + return unreachable[a] < unreachable[b]; + }); + + // Safe to drop: a root (always reachable) or has another reachable in-edge. + auto is_safe_to_drop = [&](Idx n) { + return is_root[n] || reachable_in_degree[n] >= 2; + }; + + std::vector> pool; + std::vector pruned; + std::vector old_neighbors; + for (size_t k : order) { + Idx u = unreachable[k]; + const auto& candidates = proposals[k]; + bool accepted = false; + + // Pass 1: append if room, else re-prune and accept only if all drops are safe. + for (Idx v : candidates) { + if (graph.get_node_degree(v) < params.graph_max_degree) { + graph.add_edge(v, u); + ++reachable_in_degree[u]; + ++report.edges_appended; + accepted = true; + break; + } + + const auto& v_data = accessor(data, v); + distance::maybe_fix_argument(general_distance, v_data); + pool.clear(); + old_neighbors.clear(); + for (Idx n : graph.get_node(v)) { + old_neighbors.push_back(n); + pool.emplace_back( + n, distance::compute(general_distance, v_data, accessor(data, n)) + ); + } + pool.emplace_back( + u, distance::compute(general_distance, v_data, accessor(data, u)) + ); + std::sort(pool.begin(), pool.end(), TotalOrder(cmp)); + + heuristic_prune_neighbors( + prune_strategy(distance), + params.graph_max_degree, + params.alpha, + data, + accessor, + general_distance, + v, + lib::as_const_span(pool), + pruned + ); + + if (std::find(pruned.begin(), pruned.end(), u) == pruned.end()) { + continue; + } + + bool all_drops_safe = true; + for (Idx old_id : old_neighbors) { + bool dropped = + std::find(pruned.begin(), pruned.end(), old_id) == pruned.end(); + if (dropped && !is_safe_to_drop(old_id)) { + all_drops_safe = false; + break; + } + } + if (!all_drops_safe) { + continue; + } + + graph.replace_node(v, pruned); + for (Idx old_id : old_neighbors) { + if (std::find(pruned.begin(), pruned.end(), old_id) == pruned.end()) { + --reachable_in_degree[old_id]; + } + } + ++reachable_in_degree[u]; + ++report.edges_accepted_through_prune; + accepted = true; + break; + } + + if (accepted) { + continue; + } + + // Pass 2: forced replacement of a safe neighbor; candidates is never empty. + bool forced_safe = false; + Idx safe_v{}; + Idx safe_w{}; + for (Idx v : candidates) { + const auto& v_data = accessor(data, v); + distance::maybe_fix_argument(general_distance, v_data); + + std::vector> safe_candidates; + for (Idx n : graph.get_node(v)) { + if (is_safe_to_drop(n)) { + safe_candidates.emplace_back( + n, + distance::compute(general_distance, v_data, accessor(data, n)) + ); + } + } + if (!safe_candidates.empty()) { + auto farthest = std::max_element( + safe_candidates.begin(), safe_candidates.end(), TotalOrder(cmp) + ); + safe_v = v; + safe_w = farthest->id(); + forced_safe = true; + break; + } + } + + // Pass 3: forces the nearest candidate regardless of safety, orphaning `w`. + Idx v{}; + Idx w{}; + if (forced_safe) { + v = safe_v; + w = safe_w; + } else { + v = candidates.front(); + const auto& v_data = accessor(data, v); + distance::maybe_fix_argument(general_distance, v_data); + + std::vector> ranked; + for (Idx n : graph.get_node(v)) { + ranked.emplace_back( + n, distance::compute(general_distance, v_data, accessor(data, n)) + ); + } + auto farthest = + std::max_element(ranked.begin(), ranked.end(), TotalOrder(cmp)); + w = farthest->id(); + } + + std::vector new_neighbors; + for (Idx n : graph.get_node(v)) { + if (n != w) { + new_neighbors.push_back(n); + } + } + new_neighbors.push_back(u); + graph.replace_node(v, new_neighbors); + + --reachable_in_degree[w]; + ++reachable_in_degree[u]; + + if (forced_safe) { + ++report.edges_forced; + } else { + ++report.edges_forced_unsafe; + } + } + + size_t unreachable_before_round = unreachable.size(); + unreachable = unreachable_nodes(graph, roots); + if (unreachable.size() >= unreachable_before_round) { + report.stalled = true; + break; + } + } + report.unreachable_after = unreachable.size(); + return report; +} + +} // namespace svs::index::vamana diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 96ffeaa06..10b3bdc8a 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -27,6 +27,11 @@ #include "svs/lib/preprocessor.h" #include "svs/lib/threads.h" +// stl +#include +#include +#include + namespace svs::index::vamana::extensions { ///// @@ -281,10 +286,66 @@ DefaultBuildAdaptor svs_invoke( return DefaultBuildAdaptor{}; } +/// Builds the per-insert start nodes `generate_neighbors` uses. Default: entry_points +/// unchanged. An extension may route to per-node starts instead (GEM, gem.h); `scratch` +/// is per-thread storage it may fill and return a span over, avoiding allocation. +struct BuildEntryPointsType { + template + std::span operator()( + const Data& data, + const Distance& distance, + Idx node_id, + std::span entry_points, + std::vector& scratch + ) const { + return svs::svs_invoke(*this, data, distance, node_id, entry_points, scratch); + } +}; + +inline constexpr BuildEntryPointsType build_entry_points{}; + +// Default Implementation. +template +std::span svs_invoke( + svs::tag_t, + const Data& SVS_UNUSED(data), + const Distance& SVS_UNUSED(distance), + Idx SVS_UNUSED(node_id), + std::span entry_points, + std::vector& SVS_UNUSED(scratch) +) { + return entry_points; +} + ///// ///// SEARCH EXTENSIONS ///// +/// Builds the greedy-search initializer for a query. Default: EntryPointInitializer over +/// the index's fixed entry points; an extension may start per-query instead (GEM, gem.h). +/// Invoked inside greedy_search after fix_argument, so it may read per-query state. +struct SearchInitializerType { + template + auto operator()( + const Data& data, const Distance& distance, std::span entry_points + ) const { + return svs::svs_invoke(*this, data, distance, entry_points); + } +}; + +inline constexpr SearchInitializerType search_initializer{}; + +// Default Implementation. +template +vamana::EntryPointInitializer svs_invoke( + svs::tag_t, + const Data& SVS_UNUSED(data), + const Distance& SVS_UNUSED(distance), + std::span entry_points +) { + return vamana::EntryPointInitializer{entry_points}; +} + /// Preprartion steps for single search. struct VamanaSingleSearchSetupType { template diff --git a/include/svs/index/vamana/gem_build.h b/include/svs/index/vamana/gem_build.h new file mode 100644 index 000000000..4b0438157 --- /dev/null +++ b/include/svs/index/vamana/gem_build.h @@ -0,0 +1,162 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Mirrors index.h's build constructor, which hard-codes an L2/IP/Cosine check QEmd fails. +#include "svs/core/distance/gem.h" +#include "svs/core/distance/gem_router.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/build_params.h" +#include "svs/index/vamana/connectivity.h" +#include "svs/index/vamana/extensions.h" +#include "svs/index/vamana/vamana_build.h" +#include "svs/lib/threads.h" + +// stl +#include +#include +#include +#include +#include + +namespace svs::index::vamana { + +namespace gem_build_detail { +// Shared with gem_insert_order so the two never disagree on a valid representative. +template +std::vector valid_representatives(const svs::distance::GemRouter& router) { + std::vector reps = router.representative; + std::sort(reps.begin(), reps.end()); + reps.erase(std::unique(reps.begin(), reps.end()), reps.end()); + + std::vector out; + out.reserve(reps.size()); + for (uint32_t rep : reps) { + if (rep != svs::distance::GemRouter::no_representative) { + out.push_back(static_cast(rep)); + } + } + return out; +} +} // namespace gem_build_detail + +/// Insert order for a GEM build: every valid representative of `router` (deduplicated, +/// ascending), then every other document ascending. Every document in +/// `[0, num_documents)` appears exactly once. +template +std::vector +gem_insert_order(const svs::distance::GemRouter& router, size_t num_documents) { + std::vector order; + order.reserve(num_documents); + std::vector inserted(num_documents, false); + + for (Idx rep : gem_build_detail::valid_representatives(router)) { + inserted.at(rep) = true; + order.push_back(rep); + } + for (size_t doc = 0; doc < num_documents; ++doc) { + if (!inserted[doc]) { + order.push_back(static_cast(doc)); + } + } + return order; +} + +/// Outcome of `build_gem_graph`. `edges_added` is the safe total; `edges_forced_unsafe` +/// and `stalled` pass through from the connectivity report so a caller can see whether +/// the repair orphaned a node, or gave up early after making no more progress. +struct GemBuildReport { + size_t documents = 0; + size_t representatives_inserted_first = 0; + size_t unreachable_before_repair = 0; + size_t unreachable_after_repair = 0; + size_t repair_rounds = 0; + size_t edges_added = 0; + size_t edges_forced_unsafe = 0; + bool stalled = false; +}; + +/// Builds a Vamana graph over `data` (centroid-id sets) with `distance` (QEmd). With a +/// router, `gem_insert_order` picks the insert order; otherwise the ascending full range. +/// Repair roots are `entry_point` plus every valid representative. +template < + data::ImmutableMemoryDataset IdSets, + threads::ThreadPool Pool, + typename Allocator = HugepageAllocator> +std::pair, GemBuildReport> build_gem_graph( + const IdSets& data, + const svs::distance::QEmd& distance, + const VamanaBuildParameters& params, + Pool& threadpool, + uint32_t entry_point, + const Allocator& graph_allocator = {}, + svs::logging::logger_ptr logger = svs::logging::get() +) { + using Idx = uint32_t; + size_t num_documents = data.size(); + + GemBuildReport report{}; + report.documents = num_documents; + + auto graph = + default_graph(num_documents, params.graph_max_degree, graph_allocator); + + auto router = distance.router(); + std::vector roots{entry_point}; + if (router) { + auto reps = gem_build_detail::valid_representatives(*router); + report.representatives_inserted_first = reps.size(); + roots.insert(roots.end(), reps.begin(), reps.end()); + std::sort(roots.begin(), roots.end()); + roots.erase(std::unique(roots.begin(), roots.end()), roots.end()); + } + + auto builder = VamanaBuilder( + graph, + data, + distance, + params, + threadpool, + extensions::estimate_prefetch_parameters(data), + logger + ); + + if (router) { + auto order = gem_insert_order(*router, num_documents); + builder.construct(1.0F, entry_point, order, logging::Level::Trace, logger); + builder.construct(params.alpha, entry_point, order, logging::Level::Trace, logger); + } else { + builder.construct(1.0F, entry_point, logging::Level::Trace, logger); + builder.construct(params.alpha, entry_point, logging::Level::Trace, logger); + } + + auto connectivity = repair_connectivity( + graph, data, distance, params, lib::as_const_span(roots), threadpool + ); + report.unreachable_before_repair = connectivity.unreachable_before; + report.unreachable_after_repair = connectivity.unreachable_after; + report.repair_rounds = connectivity.rounds; + report.edges_added = connectivity.edges_appended + + connectivity.edges_accepted_through_prune + + connectivity.edges_forced; + report.edges_forced_unsafe = connectivity.edges_forced_unsafe; + report.stalled = connectivity.stalled; + + return {std::move(graph), report}; +} + +} // namespace svs::index::vamana diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index 4d4a47f09..843d75209 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -499,7 +499,9 @@ class VamanaIndex { query, distance, buffer, - vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, + extensions::search_initializer( + data_, distance, lib::as_const_span(entry_point_) + ), internal_search_builder(), prefetch_parameters, cancel diff --git a/include/svs/index/vamana/vamana_build.h b/include/svs/index/vamana/vamana_build.h index f658a974f..e72b14e28 100644 --- a/include/svs/index/vamana/vamana_build.h +++ b/include/svs/index/vamana/vamana_build.h @@ -21,6 +21,7 @@ #include "svs/concepts/distance.h" #include "svs/core/logging.h" #include "svs/index/vamana/build_params.h" +#include "svs/index/vamana/build_stats.h" #include "svs/index/vamana/extensions.h" #include "svs/index/vamana/greedy_search.h" #include "svs/index/vamana/prune.h" @@ -42,11 +43,13 @@ // stdlib #include +#include #include #include #include #include #include +#include #include namespace svs::index::vamana { @@ -166,7 +169,8 @@ template < graphs::MemoryGraph Graph, data::ImmutableMemoryDataset Data, typename Dist, - threads::ThreadPool Pool> + threads::ThreadPool Pool, + typename StatsCollector = DefaultBuildStats> class VamanaBuilder { public: // Type Aliases @@ -357,6 +361,7 @@ class VamanaBuilder { // Scratch space. std::vector> pool{}; + std::vector entry_point_scratch{}; auto search_buffer = search_buffer_type{params_.window_size}; // Enable use of the visited filter of the search buffer. @@ -370,6 +375,7 @@ class VamanaBuilder { auto&& graph_search_distance = build_adaptor.graph_search_distance(); auto&& general_distance = build_adaptor.general_distance(); auto general_accessor = build_adaptor.general_accessor(); + StatsCollector stats{}; for (auto node_id : local_indices) { pool.clear(); @@ -382,21 +388,33 @@ class VamanaBuilder { // Perform the greedy search. // The search tracker will be used if it is enabled. + auto&& counted_graph_search_distance = + stats.wrap_search(graph_search_distance); { + [[maybe_unused]] auto search_timer = stats.time_search(); + // local_indices' element type varies; narrow to avoid two Idx's. + auto node_entry_points = extensions::build_entry_points( + data_, + distance_function_, + lib::narrow_cast(node_id), + lib::as_const_span(entry_points), + entry_point_scratch + ); auto accessor = build_adaptor.graph_search_accessor(); greedy_search( graph_, data_, accessor, graph_search_query, - graph_search_distance, + counted_graph_search_distance, search_buffer, - vamana::EntryPointInitializer{lib::as_const_span(entry_points)}, + vamana::EntryPointInitializer{node_entry_points}, NeighborBuilder(), tracker, prefetch_hint_ ); } + stats.finish_search(); const auto& post_search_query = build_adaptor.modify_post_search_query( data_, node_id, graph_search_query @@ -435,6 +453,7 @@ class VamanaBuilder { } // Add neighbors of the query that are not part of `visited`. + auto&& counted_pool_distance = stats.wrap_pool(general_distance); for (auto id : graph_.get_node(node_id)) { assert(id != node_id); // Try to emplace the node id into the visited set. @@ -445,13 +464,14 @@ class VamanaBuilder { pool.emplace_back( id, distance::compute( - general_distance, + counted_pool_distance, post_search_query, general_accessor(data_, id) ) ); } } + stats.finish_pool(); std::sort( pool.begin(), @@ -463,18 +483,24 @@ class VamanaBuilder { // Prune and wait for an update. thread_local_updates.emplace_back(node_id, std::vector{}); auto& pruned_results = thread_local_updates.back().second; - heuristic_prune_neighbors( - prune_strategy(distance_function_), - params_.graph_max_degree, - alpha, - data_, - general_accessor, - general_distance, - node_id, - lib::as_const_span(pool), - pruned_results - ); + auto&& counted_prune_distance = stats.wrap_prune(general_distance); + { + [[maybe_unused]] auto prune_timer = stats.time_prune(); + heuristic_prune_neighbors( + prune_strategy(distance_function_), + params_.graph_max_degree, + alpha, + data_, + general_accessor, + counted_prune_distance, + node_id, + lib::as_const_span(pool), + pruned_results + ); + } + stats.finish_prune(); } + stats.merge(); } ); @@ -524,71 +550,103 @@ class VamanaBuilder { // that were recorded in the previous process. // // Take care to avoid duplicate entries. + // + // One work unit per node, largest overflow first: hub nodes no longer serialize a + // batch's tail on a few threads. Pointers stay valid since nothing mutates buckets. auto prune_timer = timer.push_back("pruning backedges"); - threads::parallel_for( - threadpool_, - threads::DynamicPartition{backedge_buffer_.buckets(), 1}, - [&](auto& buckets, uint64_t SVS_UNUSED(tid)) { - // Thread local auxiliary data structures. - std::vector> candidates{}; - std::vector pruned_results{}; - auto build_adaptor = extensions::build_adaptor(data_, distance_function_); - - auto general_accessor = build_adaptor.general_accessor(); - auto&& general_distance = build_adaptor.general_distance(); - - auto cmp = distance::comparator(general_distance); - for (auto& bucket : buckets) { - for (const auto& kv : bucket) { - // The ``neighbors`` class is a set. - auto src = kv.first; - const auto& neighbors = kv.second; - const auto& src_data = general_accessor(data_, src); - distance::maybe_fix_argument(general_distance, src_data); - - // Helper lambda to make distance computations look a little - // cleaner. - auto make_neighbor = [&](auto i) { - return Neighbor{ - i, - distance::compute( - general_distance, src_data, general_accessor(data_, i) - )}; - }; - - candidates.clear(); - // Add the overflow candidates. - for (auto n : neighbors) { - candidates.push_back(make_neighbor(n)); - } + using entry_type = typename BackedgeBuffer::map_type::value_type; + auto& buckets = backedge_buffer_.buckets(); + size_t total_entries = 0; + for (const auto& bucket : buckets) { + total_entries += bucket.size(); + } + std::vector worklist{}; + worklist.reserve(total_entries); + for (const auto& bucket : buckets) { + for (const auto& kv : bucket) { + worklist.push_back(&kv); + } + } + // Ties on ``src`` keep the schedule deterministic. + std::sort(worklist.begin(), worklist.end(), [](const auto* a, const auto* b) { + if (a->second.size() != b->second.size()) { + return a->second.size() > b->second.size(); + } + return a->first < b->first; + }); - // Add the old adjacency list. - for (auto n : graph_.get_node(src)) { - if (!neighbors.contains(n)) { - candidates.push_back(make_neighbor(n)); - } - } - std::sort(candidates.begin(), candidates.end(), TotalOrder(cmp)); - candidates.resize( - std::min(candidates.size(), params_.max_candidate_pool_size) - ); + // Setup runs once per thread, not per node: this path also serves cheap distances, + // where per-node setup would rival the prune itself. + std::atomic cursor{0}; + threads::parallel_for(threadpool_, [&](uint64_t SVS_UNUSED(tid)) { + // Thread local auxiliary data structures. + std::vector> candidates{}; + std::vector pruned_results{}; + auto build_adaptor = extensions::build_adaptor(data_, distance_function_); + + auto general_accessor = build_adaptor.general_accessor(); + auto&& general_distance = build_adaptor.general_distance(); + + auto cmp = distance::comparator(general_distance); + StatsCollector stats{}; + // Backedge counting spans the whole thread, so wrap once and reuse below. + auto&& counted_backedge_distance = stats.wrap_backedge_prune(general_distance); + for (size_t w = cursor.fetch_add(1, std::memory_order_relaxed); + w < worklist.size(); + w = cursor.fetch_add(1, std::memory_order_relaxed)) { + const auto* entry = worklist[w]; + // The ``neighbors`` class is a set. + auto src = entry->first; + const auto& neighbors = entry->second; + const auto& src_data = general_accessor(data_, src); + distance::maybe_fix_argument(general_distance, src_data); + + // Helper lambda to make distance computations look a little + // cleaner. + auto make_neighbor = [&](auto i) { + return Neighbor{ + i, + distance::compute( + counted_backedge_distance, src_data, general_accessor(data_, i) + )}; + }; + + candidates.clear(); + // Add the overflow candidates. + for (auto n : neighbors) { + candidates.push_back(make_neighbor(n)); + } - heuristic_prune_neighbors( - prune_strategy(distance_function_), - params_.prune_to, - alpha, - data_, - general_accessor, - general_distance, - src, - lib::as_const_span(candidates), - pruned_results - ); - graph_.replace_node(src, pruned_results); + // Add the old adjacency list. + for (auto n : graph_.get_node(src)) { + if (!neighbors.contains(n)) { + candidates.push_back(make_neighbor(n)); } } + std::sort(candidates.begin(), candidates.end(), TotalOrder(cmp)); + candidates.resize( + std::min(candidates.size(), params_.max_candidate_pool_size) + ); + + { + [[maybe_unused]] auto backedge_prune_timer = + stats.time_backedge_prune(); + heuristic_prune_neighbors( + prune_strategy(distance_function_), + params_.prune_to, + alpha, + data_, + general_accessor, + counted_backedge_distance, + src, + lib::as_const_span(candidates), + pruned_results + ); + } + graph_.replace_node(src, pruned_results); } - ); + stats.merge(); + }); } private: diff --git a/include/svs/lib/avx_detection.h b/include/svs/lib/avx_detection.h index fc9b246f9..0c9c07212 100644 --- a/include/svs/lib/avx_detection.h +++ b/include/svs/lib/avx_detection.h @@ -16,8 +16,15 @@ #pragma once +#include #include +#if defined(__x86_64__) && defined(__linux__) +#include +#include +#include +#endif + namespace svs::detail { #ifdef __x86_64__ @@ -36,6 +43,8 @@ struct AVXRuntimeFlags { avx2 = extended_features && ((ebx & (1 << 5)) != 0); avx512f = extended_features && ((ebx & (1 << 16)) != 0); avx512vnni = extended_features && ((ecx & (1 << 11)) != 0); + amx_tile_bf16_cpu = + extended_features && ((edx & (1 << 24)) != 0) && ((edx & (1 << 22)) != 0); } bool is_avx2_supported() const noexcept { return avx2; } @@ -45,15 +54,83 @@ struct AVXRuntimeFlags { bool avx2; bool avx512f; bool avx512vnni; + bool amx_tile_bf16_cpu; }; #else struct AVXRuntimeFlags { bool is_avx2_supported() const noexcept { return false; } bool is_avx512f_supported() const noexcept { return false; } bool is_avx512vnni_supported() const noexcept { return false; } + + bool amx_tile_bf16_cpu = false; }; #endif inline const AVXRuntimeFlags avx_runtime_flags = {}; +#if defined(__x86_64__) && defined(__linux__) +// Kernel XFEATURE_XTILEDATA index for ARCH_REQ_XCOMP_PERM; see amx_tile_bf16_available's +// documentation below for where this number comes from and how it was confirmed. +inline constexpr unsigned long SVS_XFEATURE_XTILEDATA = 18; + +/// Returns true the first time, and every time after, that this process may run AMX +/// tile instructions without SIGILL: the CPU has AMX-TILE and AMX-BF16, and the one-time +/// `ARCH_REQ_XCOMP_PERM` request below succeeded. Any "no" here is silent, not an error: +/// dispatch simply does not pick AMX. +/// +/// Requests AMX tile-data OS permission exactly once per process, the first time any +/// thread calls this function. Requesting at library load would run in whatever process +/// SVS is linked into before that process has decided it wants AMX at all; requesting +/// per-call would repeat a syscall this function's caching exists to avoid. The request +/// must run unconditionally, every time this initializes: XCR0 (read via `xgetbv`) +/// reports which xstate components the CPU/OS architecturally support, not whether this +/// process has been granted permission to use tile data, so an "already looks enabled" +/// shortcut on XCR0 would skip the one call that actually grants it (confirmed by +/// experiment: a process that never calls `ARCH_REQ_XCOMP_PERM` still reads XCR0 bits +/// 17/18 as set, then SIGILLs on the first real tile instruction). +/// +/// Thread-safety and the once-ness come from the function-local static's initializer +/// guarantee (C++ [stmt.dcl]): concurrent first callers block on the same guard variable +/// and only one of them runs the body; every call after that is the "one predictable +/// branch on a cached value" the dispatch call site needs. +/// +/// The feature number passed to `ARCH_REQ_XCOMP_PERM` (18, `SVS_XFEATURE_XTILEDATA` +/// above) is not in any header on this machine: glibc's `asm/prctl.h` defines +/// `ARCH_REQ_XCOMP_PERM` itself but not the per-feature indices it takes. 18 is +/// `XFEATURE_XTILEDATA` per the kernel's xstate ABI (`Documentation/arch/x86/xstate.rst` +/// in the Linux source tree); confirmed by experiment on this machine (Linux, x86-64, +/// see `/tmp/amx-study/t2c_detect.c`) that requesting permission with this value is what +/// lets tile instructions run without SIGILL, not merely a documented-but-untested +/// number. +/// +/// This syscall is process-wide, not just for the calling thread: once granted, tile +/// state (the eight tmm registers and the tile-config register) becomes part of every +/// thread's extended processor state in this process, saved and restored across context +/// switches like any other xstate component from then on. A process running under a +/// seccomp policy that kills on an unlisted syscall will be terminated right here, at +/// this call, if `arch_prctl` is not on that policy's allow list. Taken from kernel +/// documentation, not verified on this machine: the kernel may refuse the request if an +/// installed alternate signal stack (`sigaltstack`) is smaller than the now-larger +/// xstate save area requires. +inline bool amx_tile_bf16_available() { + static const bool available = [] { + // amx.cpp is compiled at exactly the AVX-512 level's instruction set (plus the + // two AMX extensions, see cmake/multi-arch.cmake) and its non-tile code path + // forwards to that level's kernels directly, so AMX is only usable where AVX-512 + // already is: a CPU could in principle expose the AMX bits without AVX-512F + // (e.g. a hypervisor masking features independently), and that combination is + // not one this file's code was built for. + if (!avx_runtime_flags.amx_tile_bf16_cpu || + !avx_runtime_flags.is_avx512f_supported()) { + return false; + } + long rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, SVS_XFEATURE_XTILEDATA); + return rc == 0; + }(); + return available; +} +#else +inline bool amx_tile_bf16_available() { return false; } +#endif + } // namespace svs::detail diff --git a/include/svs/lib/bfloat16.h b/include/svs/lib/bfloat16.h index a1c257a62..1a39aee40 100644 --- a/include/svs/lib/bfloat16.h +++ b/include/svs/lib/bfloat16.h @@ -58,6 +58,19 @@ inline uint16_t float_to_bfloat16_untyped(const float x) { return lib::narrow(u >> 16); } +// Round-to-nearest-even truncation to bfloat16, as a uint16_t of the result's high half. +// NaN is kept as NaN (never promoted to infinity by the rounding add) with its quiet bit +// forced on, since x86 NaN payloads are not required to survive this truncation anyway. +inline uint16_t float_to_bfloat16_rne_untyped(const float x) { + uint32_t u = bitcast_float_to_uint32(x); + if ((u & 0x7fffffffU) > 0x7f800000U) { + return lib::narrow((u | 0x00400000U) >> 16); + } + uint32_t lsb = (u >> 16) & 1U; + u += 0x7fffU + lsb; + return lib::narrow(u >> 16); +} + } // namespace detail // On GCC, we need to add this attribute so that BFloat16 members can appear inside @@ -107,6 +120,14 @@ static_assert(std::is_standard_layout_v); // like signed zeros. inline bool operator==(BFloat16 x, BFloat16 y) { return float{x} == float{y}; } +// BFloat16's own constructor (above) truncates: it keeps the high 16 bits of `x` and +// discards the rest with no rounding. This rounds to nearest, ties to even, instead; use +// it wherever more than one caller needs the two to agree bit-for-bit (e.g. a kernel +// operand rounded on one side and a centroid matrix rounded on the other). +inline BFloat16 round_to_bfloat16(float x) { + return BFloat16::from_raw(detail::float_to_bfloat16_rne_untyped(x)); +} + ///// ///// Pretty Printing ///// @@ -114,6 +135,7 @@ inline bool operator==(BFloat16 x, BFloat16 y) { return float{x} == float{y}; } } // namespace bfloat16 using BFloat16 = bfloat16::BFloat16; +using bfloat16::round_to_bfloat16; // SVS local arithmetric trait. template <> inline constexpr bool is_arithmetic_v = true; diff --git a/include/svs/lib/preprocessor.h b/include/svs/lib/preprocessor.h index 8b1a8a85a..518180ae1 100644 --- a/include/svs/lib/preprocessor.h +++ b/include/svs/lib/preprocessor.h @@ -114,6 +114,17 @@ consteval bool is_one_or_zero(const char* ptr) { #define SVS_OMP 0 #endif +// Set uniformly by cmake/multi-arch.cmake on every consumer of the library, unlike +// SVS_AVX512_F/SVS_AMX_TILE_BF16 below which read the *current* TU's own -march flags: +// PackedOperand::pack_rounded_to_bf16 runs in TUs that never pass -mamx-tile themselves +// (gem.h, maxsim.h, tests), so it needs a build-wide answer to "does amx.cpp exist in +// this build at all" to decide whether filling the AMX tile layout is worth its cost. +#if defined(SVS_AMX_TILE_BF16_SUPPORTED) +#define SVS_AMX_COMPILED_IN 1 +#else +#define SVS_AMX_COMPILED_IN 0 +#endif + ///// ///// Intel(R) AVX extensions ///// @@ -170,6 +181,17 @@ inline constexpr bool have_avx512_avx2 = true; inline constexpr bool have_avx512_avx2 = true; #endif +// AMX tile load/store/config and the bf16 dot-product tile instruction. Both gate the +// same translation unit (amx.cpp is compiled with -mamx-tile -mamx-bf16 together, see +// cmake/multi-arch.cmake), so one guard covers both rather than two that could disagree. +#if defined(__AMX_TILE__) && defined(__AMX_BF16__) +#define SVS_AMX_TILE_BF16 1 +inline constexpr bool have_amx_tile_bf16 = true; +#else +#define SVS_AMX_TILE_BF16 0 +inline constexpr bool have_amx_tile_bf16 = false; +#endif + } // namespace svs::arch namespace svs { diff --git a/include/svs/multi-arch/x86/amx.cpp b/include/svs/multi-arch/x86/amx.cpp new file mode 100644 index 000000000..25a9096ae --- /dev/null +++ b/include/svs/multi-arch/x86/amx.cpp @@ -0,0 +1,29 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(__x86_64__) +#include "svs/core/distance/inner_product_matrix.h" + +namespace svs::distance { + +SVS_VALIDATE_BOOL_ENV(SVS_AMX_TILE_BF16) +#if SVS_AMX_TILE_BF16 +template struct InnerProductMatrixImpl; +#endif + +} // namespace svs::distance + +#endif diff --git a/include/svs/multi-arch/x86/avx2.cpp b/include/svs/multi-arch/x86/avx2.cpp index bff53ae10..f4fe3b91f 100644 --- a/include/svs/multi-arch/x86/avx2.cpp +++ b/include/svs/multi-arch/x86/avx2.cpp @@ -17,7 +17,9 @@ #if defined(__x86_64__) #include "svs/core/distance/cosine.h" #include "svs/core/distance/euclidean.h" +#include "svs/core/distance/gem.h" #include "svs/core/distance/inner_product.h" +#include "svs/core/distance/inner_product_matrix.h" namespace svs::distance { @@ -52,6 +54,11 @@ DISTANCE_CS_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX2); DISTANCE_CS_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX2); DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); +template struct QEmdImpl; +template struct QChamferImpl; +template struct QChamferExactImpl; +template struct InnerProductMatrixImpl; + } // namespace svs::distance #endif diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index bee150d75..753710398 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -17,7 +17,9 @@ #if defined(__x86_64__) #include "svs/core/distance/cosine.h" #include "svs/core/distance/euclidean.h" +#include "svs/core/distance/gem.h" #include "svs/core/distance/inner_product.h" +#include "svs/core/distance/inner_product_matrix.h" namespace svs::distance { @@ -52,6 +54,11 @@ DISTANCE_CS_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512); DISTANCE_CS_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512); DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); +template struct QEmdImpl; +template struct QChamferImpl; +template struct QChamferExactImpl; +template struct InnerProductMatrixImpl; + } // namespace svs::distance #endif diff --git a/include/svs/quantization/gem/codebook.h b/include/svs/quantization/gem/codebook.h new file mode 100644 index 000000000..a1b4cace4 --- /dev/null +++ b/include/svs/quantization/gem/codebook.h @@ -0,0 +1,878 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if SVS_ENABLE_MULTI_VECTOR + +// svs +#include "svs/concepts/data.h" +#include "svs/concepts/vector_set.h" +#include "svs/core/data/simple.h" +#include "svs/core/data/vector_set.h" +#include "svs/core/distance/gem.h" +#include "svs/core/distance/inner_product_matrix.h" +#include "svs/core/kmeans_emeans.h" +#include "svs/core/kmeans_training.h" +#include "svs/lib/bfloat16.h" +#include "svs/lib/exception.h" +#include "svs/lib/float16.h" +#include "svs/lib/misc.h" +#include "svs/lib/narrow.h" +#include "svs/lib/threads.h" +#include "svs/lib/timing.h" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Trains the codebook GEM assigns tokens to, matching faiss's ColBERT k-means indexer. +namespace svs::quantization::gem { + +namespace detail { + +// Exact floor(sqrt(x)): std::sqrt seeds it, then local search corrects any libm rounding. +inline size_t isqrt_floor(size_t x) { + if (x == 0) { + return 0; + } + size_t r = static_cast(std::sqrt(static_cast(x))); + while (r > 0 && r > x / r) { + --r; + } + while ((r + 1) <= x / (r + 1)) { + ++r; + } + return r; +} + +} // namespace detail + +/// K = 2^floor(log2(16 * sqrt(num_tokens))), ColBERT's indexer rule. Uses std::frexp's +/// exact mantissa/exponent split rather than std::log2, since log2 of an exact power of +/// two is not guaranteed correctly rounded on every libm. +inline size_t colbert_num_partitions(size_t num_tokens) { + if (num_tokens == 0) { + throw ANNEXCEPTION("colbert_num_partitions: num_tokens must be positive, got 0"); + } + double raw = 16.0 * std::sqrt(static_cast(num_tokens)); + int exponent = 0; + std::frexp(raw, &exponent); + // mantissa in [0.5,1) means floor(log2(raw)) == exponent - 1 exactly. + if (exponent < 1) { + throw ANNEXCEPTION( + "colbert_num_partitions: num_tokens={} is too small to produce a positive " + "centroid count", + num_tokens + ); + } + return size_t{1} << (exponent - 1); +} + +/// num_sampled_passages = min(1 + floor(16 * sqrt(120 * num_passages)), num_passages). +/// 16 * sqrt(120 * n) == sqrt(30720 * n) exactly, so the floor uses integer sqrt +/// (detail::isqrt_floor) instead of floating log2/sqrt, avoiding rounding ambiguity. +inline size_t colbert_sample_size(size_t num_passages) { + if (num_passages == 0) { + throw ANNEXCEPTION("colbert_sample_size: num_passages must be positive, got 0"); + } + size_t raw = detail::isqrt_floor(30720ULL * num_passages); + return std::min(size_t{1} + raw, num_passages); +} + +/// Which kernel assign_tokens/quantize use to find each token's nearest centroid. +enum class Assigner { ExactGemm, InnerProductMatrix }; + +/// Which k-means algorithm train_centroids runs. EMeans is svs::kmeans::emeans_train +/// (arXiv:2611.09501v1) and CodebookParameters' default; Lloyd is the earlier default, +/// same plain-L2 contract. +enum class KMeansAlgorithm { Lloyd, EMeans }; + +/// Parameters for codebook training, assignment and the coarse-cluster side table. +struct CodebookParameters { + /// Number of fine centroids K; 0 selects colbert_num_partitions from the token count. + size_t num_centroids = 0; + /// Lloyd iteration count; ignored when kmeans_algorithm == EMeans (use emeans_epochs). + size_t kmeans_iterations = 4; + /// GEMM chunk size for training's shared buffer; assign_tokens sizes its own instead. + /// Also EMeansParameters::assignment_chunk's source when kmeans_algorithm == EMeans. + size_t minibatch_size = 10'000; + uint32_t seed = 42; + double heldout_fraction = 0.05; + size_t heldout_cap = 50'000; + /// Number of coarse clusters. 0 selects max(1, num_centroids * 5 / 32). + size_t num_coarse = 0; + size_t coarse_iterations = 4; + /// Default InnerProductMatrix measured faster than ExactGemm and matched it exactly. + Assigner assigner = Assigner::InnerProductMatrix; + size_t assign_candidates = 8; + svs::distance::Ground ground = svs::distance::Ground::Cos; + /// Per-thread score-buffer budget; fixed-row sizing once hit 587 GB at K=131'072. + size_t assign_buffer_bytes = 32 * 1024 * 1024; + /// Which k-means algorithm train_codebook runs; EMeans (emeans_epochs) by default. + KMeansAlgorithm kmeans_algorithm = KMeansAlgorithm::EMeans; + /// EMeans epochs T, used only when kmeans_algorithm == EMeans. Batch size and alpha0 + /// are left to svs::kmeans::EMeansParameters's scaling-law defaults. Defaults to + /// EMeansParameters's own default so the two never drift apart. + size_t emeans_epochs = svs::kmeans::EMeansParameters{}.epochs; +}; + +/// Any contiguous vector-set dataset of float or Float16 tokens: VectorSetData and +/// ConstVectorSetView both satisfy this without copying their token storage. +template +concept TokenVectorStore = + svs::data::ImmutableVectorSetDataset && + svs::data::ContiguousVectorSet && + (std::is_same_v || + std::is_same_v) && + requires(const Data& d) { + { d.total_tokens() } -> std::convertible_to; + { d.cardinality(size_t{0}) } -> std::convertible_to; + }; + +namespace detail { + +// Sum of squared L2 distances to assigned centroids, accumulated in double; O(n*dim). +template +double compute_objective( + const Data& data, + const svs::data::SimpleData& centroids, + const std::vector& assignments +) { + double total = 0.0; + for (auto i : data.eachindex()) { + auto datum = data.get_datum(i); + auto c = centroids.get_datum(assignments[i]); + double sq = 0.0; + for (size_t d = 0, dmax = datum.size(); d < dmax; ++d) { + double diff = static_cast(datum[d]) - static_cast(c[d]); + sq += diff * diff; + } + total += sq; + } + return total; +} + +// Plain mean, matching faiss, not svs::kmeans::centroid_adjustment's IVF-owned formula. +template +void plain_mean_update( + const Data& data, + svs::data::SimpleData& centroids, + const std::vector& assignments, + std::vector& counts, + Pool& threadpool +) { + size_t n_threads = threadpool.size(); + size_t n_centroids = centroids.size(); + std::fill(counts.begin(), counts.end(), size_t{0}); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_centroids}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + auto c = centroids.get_datum(i); + std::fill(c.begin(), c.end(), 0.0F); + } + } + ); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_threads}, + [&](auto /*indices*/, auto tid) { + size_t centroid_start = (n_centroids * tid) / n_threads; + size_t centroid_end = (n_centroids * (tid + 1)) / n_threads; + for (auto i : data.eachindex()) { + auto assignment = assignments[i]; + if (assignment >= centroid_start && assignment < centroid_end) { + counts.at(assignment)++; + auto datum = data.get_datum(i); + auto c = centroids.get_datum(assignment); + for (size_t p = 0, pmax = c.size(); p < pmax; ++p) { + c[p] += static_cast(datum[p]); + } + } + } + } + ); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_centroids}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + if (counts.at(i) != 0) { + auto c = centroids.get_datum(i); + float norm = 1.0F / static_cast(counts.at(i)); + for (size_t p = 0, pmax = c.size(); p < pmax; ++p) { + c[p] *= norm; + } + } + } + } + ); +} + +} // namespace detail + +/// Centroids and the k-means objective after train_centroids; the observer (if any) owns +/// per-iteration diagnostics instead, so this result does not carry them. +struct L2KMeansResult { + svs::data::SimpleData centroids; + double objective = 0.0; +}; + +/// Default k-means observer: both hooks are empty, so an optimizer can elide the calls +/// entirely and a caller pays nothing for not timing iterations. +struct NoOpKMeansObserver { + void before_iteration(size_t /*iter*/) {} + void after_iteration(size_t /*iter*/) {} +}; + +/// Satisfied by NoOpKMeansObserver and by any type exposing the same two hooks, e.g. one +/// that times the interval between them. +template +concept KMeansObserver = requires(T observer, size_t iter) { + observer.before_iteration(iter); + observer.after_iteration(iter); + }; + +/// Shared L2/plain-mean trainer; takes Data by non-const ref, so callers pass a local +/// copy. `observer`'s before_/after_iteration are called once per iteration, with no +/// other effect on the result, so a caller can measure iterations without this function +/// depending on any timing facility itself. +template < + svs::data::ImmutableMemoryDataset Data, + threads::ThreadPool Pool, + KMeansObserver Observer = NoOpKMeansObserver> +L2KMeansResult train_centroids( + Data& data, + size_t num_clusters, + size_t iterations, + size_t minibatch_size, + std::mt19937& rng, + Pool& threadpool, + Observer observer = Observer{}, + KMeansAlgorithm algorithm = KMeansAlgorithm::Lloyd, + size_t emeans_epochs = svs::kmeans::EMeansParameters{}.epochs +) { + size_t n = data.size(); + if (n < num_clusters) { + throw ANNEXCEPTION( + "train_centroids: only {} points available, need at least num_clusters={}", + n, + num_clusters + ); + } + + if (algorithm == KMeansAlgorithm::EMeans) { + svs::kmeans::EMeansParameters emeans_params; + emeans_params.epochs = emeans_epochs; + // Caps EMeans' per-chunk score buffer the same way minibatch_size caps Lloyd's. + emeans_params.assignment_chunk = minibatch_size; + svs::lib::Timer scratch_timer; + observer.before_iteration(0); + auto emeans_result = svs::kmeans::emeans_train( + emeans_params, data, num_clusters, rng, threadpool, scratch_timer + ); + observer.after_iteration(0); + // L2KMeansResult.objective is the raw sum detail::compute_objective produces; + // emeans_train returns the mean, so GEM's kmeans_objective field stays comparable + // across algorithms. + double objective_sum = emeans_result.objective * static_cast(n); + return L2KMeansResult{std::move(emeans_result.centroids), objective_sum}; + } + + // faiss subsamples past 256/centroid; throws instead of silently diverging from it. + // Lloyd-only: e-means is designed to train on n far beyond this bound. + if (n > num_clusters * 256) { + throw ANNEXCEPTION( + "train_centroids: {} points exceed faiss's 256-per-centroid training bound " + "for num_clusters={} (max {}); subsampling instead of throwing would silently " + "differ from the reference training procedure", + n, + num_clusters, + num_clusters * 256 + ); + } + + std::vector id_scratch(num_clusters); + auto centroids = + svs::kmeans::init_centroids(data, id_scratch, num_clusters, rng, threadpool); + + size_t batch_cap = std::min(minibatch_size, n); + auto matmul_results = svs::data::SimpleData(batch_cap, num_clusters); + std::vector data_norm; + svs::kmeans::generate_norms(data, data_norm, threadpool); + std::vector centroids_norm; + std::vector assignments(n); + std::vector counts(num_clusters); + auto distance = svs::DistanceL2(); + svs::lib::Timer scratch_timer; + + double objective = 0.0; + for (size_t iter = 0; iter < iterations; ++iter) { + observer.before_iteration(iter); + svs::kmeans::generate_norms(centroids, centroids_norm, threadpool); + + size_t num_batches = svs::lib::div_round_up(n, batch_cap); + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = threads::UnitRange{ + batch * batch_cap, std::min((batch + 1) * batch_cap, n)}; + auto data_batch = data::make_view(data, this_batch); + svs::kmeans::centroid_assignment( + data_batch, + data_norm, + this_batch, + distance, + centroids, + centroids_norm, + assignments, + matmul_results, + threadpool, + scratch_timer + ); + } + + // Objective is defined on the last pass, against the centroids before its update. + if (iter + 1 == iterations) { + objective = detail::compute_objective(data, centroids, assignments); + } + + detail::plain_mean_update(data, centroids, assignments, counts, threadpool); + svs::kmeans::centroid_split( + data, centroids, counts, rng, threadpool, scratch_timer + ); + observer.after_iteration(iter); + } + + return L2KMeansResult{std::move(centroids), objective}; +} + +/// sample_training_tokens's output: the float32 tokens train_centroids trains on, plus +/// the resolved centroid count and sampling sizes train_codebook reports. +struct SampledTrainingTokens { + svs::data::SimpleData trainset; + size_t K = 0; + size_t num_sampled_passages = 0; + size_t num_training_tokens = 0; +}; + +/// Steps 1-3: resolve K, sample passages, then gather/shuffle/hold out their tokens into a +/// float32 trainset. Only sampled tokens convert to float32, so a memory-mapped Float16 +/// store is never copied in full. Advances `rng`, so a caller chaining into +/// train_centroids with the same `rng` continues its stream rather than restarting it. +template +SampledTrainingTokens sample_training_tokens( + const Data& tokens, + const CodebookParameters& parameters, + std::mt19937& rng, + Pool& threadpool +) { + size_t dim = tokens.dimensions(); + size_t num_passages = tokens.size(); + size_t num_total_tokens = tokens.total_tokens(); + if (num_passages == 0 || num_total_tokens == 0) { + throw ANNEXCEPTION( + "sample_training_tokens: tokens dataset is empty ({} passages, {} tokens)", + num_passages, + num_total_tokens + ); + } + + size_t K = parameters.num_centroids == 0 ? colbert_num_partitions(num_total_tokens) + : parameters.num_centroids; + constexpr size_t max_id = static_cast(std::numeric_limits::max()); + if (K > max_id) { + throw ANNEXCEPTION( + "sample_training_tokens: K={} exceeds the uint32_t centroid-id range (max {})", + K, + max_id + ); + } + + size_t sample_size = colbert_sample_size(num_passages); + + std::vector passage_ids(sample_size); + svs::kmeans::generate_unique_ids(passage_ids, num_passages, rng); + std::sort(passage_ids.begin(), passage_ids.end()); + + size_t total_sample_tokens = 0; + for (size_t pid : passage_ids) { + total_sample_tokens += tokens.cardinality(pid); + } + + struct TokenRef { + uint32_t sample_idx; + uint32_t token_idx; + }; + std::vector refs; + refs.reserve(total_sample_tokens); + for (size_t s = 0; s < sample_size; ++s) { + size_t card = tokens.cardinality(passage_ids[s]); + for (size_t t = 0; t < card; ++t) { + refs.push_back( + {svs::lib::narrow_cast(s), svs::lib::narrow_cast(t)} + ); + } + } + std::shuffle(refs.begin(), refs.end(), rng); + + size_t heldout = std::min( + parameters.heldout_cap, + static_cast( + parameters.heldout_fraction * static_cast(total_sample_tokens) + ) + ); + size_t num_training = total_sample_tokens - heldout; + + if (num_training < K) { + throw ANNEXCEPTION( + "sample_training_tokens: only {} training tokens sampled (after a {} token " + "held-out split), need at least K={}", + num_training, + heldout, + K + ); + } + + auto trainset = svs::data::SimpleData(num_training, dim); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_training}, + [&](auto indices, auto /*tid*/) { + std::vector row(dim); + for (auto i : indices) { + auto ref = refs[i]; + size_t pid = passage_ids[ref.sample_idx]; + auto token = tokens.get_datum(pid).token(ref.token_idx); + for (size_t d = 0; d < dim; ++d) { + row[d] = static_cast(token[d]); + } + trainset.set_datum(i, row); + } + } + ); + + return SampledTrainingTokens{std::move(trainset), K, sample_size, num_training}; +} + +/// Result of train_codebook: the normalized float32 centroids plus training diagnostics. +struct TrainedCodebook { + svs::data::SimpleData centroids; + double objective = 0.0; + size_t num_sampled_passages = 0; + size_t num_training_tokens = 0; +}; + +/// Composes sample_training_tokens and train_centroids, then normalizes once: the +/// pipeline a caller wanting plain results (no phase timing) runs end to end. +template +TrainedCodebook +train_codebook(const Data& tokens, const CodebookParameters& parameters, Pool& threadpool) { + std::mt19937 rng(parameters.seed); + auto sampled = sample_training_tokens(tokens, parameters, rng, threadpool); + + auto kmeans_result = train_centroids( + sampled.trainset, + sampled.K, + parameters.kmeans_iterations, + parameters.minibatch_size, + rng, + threadpool, + NoOpKMeansObserver{}, + parameters.kmeans_algorithm, + parameters.emeans_epochs + ); + + svs::lib::Timer scratch_timer; + svs::kmeans::normalize_centroids(kmeans_result.centroids, threadpool, scratch_timer); + + return TrainedCodebook{ + std::move(kmeans_result.centroids), + kmeans_result.objective, + sampled.num_sampled_passages, + sampled.num_training_tokens}; +} + +/// Step 5: assign every document token to its nearest (argmax inner product) normalized +/// centroid, lowest id breaking ties. Threads own disjoint passage ranges and write only +/// their own tokens' ids, so the result never depends on thread count. +template +svs::data::VectorSetData assign_tokens( + const Data& tokens, + const svs::data::SimpleData& centroids, + const CodebookParameters& parameters, + Pool& threadpool +) { + size_t dim = tokens.dimensions(); + if (centroids.dimensions() != dim) { + throw ANNEXCEPTION( + "assign_tokens: token dimension {} does not match centroid dimension {}", + dim, + centroids.dimensions() + ); + } + size_t K = centroids.size(); + constexpr size_t max_id = static_cast(std::numeric_limits::max()); + if (K > max_id) { + throw ANNEXCEPTION( + "assign_tokens: K={} exceeds the uint32_t centroid-id range (max {})", K, max_id + ); + } + + size_t num_passages = tokens.size(); + std::vector offsets(num_passages + 1, 0); + for (size_t p = 0; p < num_passages; ++p) { + offsets[p + 1] = offsets[p] + tokens.cardinality(p); + } + size_t total_tokens = lib::narrow(offsets.back()); + + auto id_storage = svs::data::SimpleData(total_tokens, 1); + + svs::distance::PackedOperand packed_centroids; + if (parameters.assigner == Assigner::InnerProductMatrix) { + packed_centroids.pack_rounded_to_bf16(centroids.data(), K, dim); + } + + size_t candidates = std::max(size_t{1}, std::min(parameters.assign_candidates, K)); + + // scores is batch_cap * K floats; derive batch_cap from assign_buffer_bytes, not K. + constexpr size_t max_assign_batch_rows = 4096; + size_t bytes_per_row = K * sizeof(float); + size_t rows_by_budget = + std::max(size_t{1}, parameters.assign_buffer_bytes / bytes_per_row); + size_t batch_cap = std::min(rows_by_budget, max_assign_batch_rows); + batch_cap = std::min(batch_cap, std::max(size_t{1}, total_tokens)); + + threads::parallel_for( + threadpool, + threads::StaticPartition{num_passages}, + [&](auto indices, auto /*tid*/) { + std::vector batch_floats(batch_cap * dim); + std::vector batch_global_idx(batch_cap); + std::vector scores(batch_cap * K); + std::vector batch_bf16; + std::vector shortlist(K); + if (parameters.assigner == Assigner::InnerProductMatrix) { + batch_bf16.resize(batch_cap * dim); + } + size_t count = 0; + + auto flush = [&]() { + if (count == 0) { + return; + } + if (parameters.assigner == Assigner::ExactGemm) { + svs::kmeans::compute_matmul( + batch_floats.data(), centroids.data(), scores.data(), count, K, dim + ); + for (size_t r = 0; r < count; ++r) { + const float* row = scores.data() + r * K; + uint32_t best = 0; + float best_score = row[0]; + for (size_t j = 1; j < K; ++j) { + if (row[j] > best_score) { + best_score = row[j]; + best = svs::lib::narrow_cast(j); + } + } + id_storage.get_datum(batch_global_idx[r])[0] = best; + } + } else { + for (size_t r = 0; r < count; ++r) { + const float* src = batch_floats.data() + r * dim; + svs::BFloat16* dst = batch_bf16.data() + r * dim; + for (size_t d = 0; d < dim; ++d) { + dst[d] = svs::round_to_bfloat16(src[d]); + } + } + svs::distance::InnerProductMatrix::compute_packed( + batch_bf16.data(), count, packed_centroids, scores.data() + ); + for (size_t r = 0; r < count; ++r) { + const float* row = scores.data() + r * K; + for (size_t j = 0; j < K; ++j) { + shortlist[j] = svs::lib::narrow_cast(j); + } + std::partial_sort( + shortlist.begin(), + shortlist.begin() + static_cast(candidates), + shortlist.end(), + [&](uint32_t a, uint32_t b) { return row[a] > row[b]; } + ); + // Ascending by id so the rescore below ties like ExactGemm does. + std::sort( + shortlist.begin(), + shortlist.begin() + static_cast(candidates) + ); + + const float* token_row = batch_floats.data() + r * dim; + uint32_t best = shortlist[0]; + float best_dot = -std::numeric_limits::infinity(); + for (size_t c = 0; c < candidates; ++c) { + uint32_t cid = shortlist[c]; + auto crow = centroids.get_datum(cid); + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += token_row[d] * crow[d]; + } + if (dot > best_dot) { + best_dot = dot; + best = cid; + } + } + id_storage.get_datum(batch_global_idx[r])[0] = best; + } + } + count = 0; + }; + + for (auto p : indices) { + auto doc = tokens.get_datum(p); + size_t card = doc.cardinality(); + for (size_t t = 0; t < card; ++t) { + auto token = doc.token(t); + float* dst = batch_floats.data() + count * dim; + for (size_t d = 0; d < dim; ++d) { + dst[d] = static_cast(token[d]); + } + batch_global_idx[count] = offsets[p] + t; + ++count; + if (count == batch_cap) { + flush(); + } + } + } + flush(); + } + ); + + return svs::data::VectorSetData(std::move(id_storage), offsets); +} + +/// Mismatches between the stored ids and a brute-force exact float32 argmax, on a random +/// sample of tokens. Uses plain loops only, sharing no kernel with either assigner, so it +/// serves as an independent check on both. +struct MismatchReport { + size_t mismatches = 0; + size_t sample_size = 0; +}; + +template +MismatchReport count_assignment_mismatches( + const Data& tokens, + const svs::data::SimpleData& centroids, + const svs::data::VectorSetData& ids, + size_t sample_size, + uint32_t seed +) { + size_t dim = tokens.dimensions(); + size_t K = centroids.size(); + auto offsets = ids.offsets(); + size_t total_tokens = lib::narrow(offsets.back()); + sample_size = std::min(sample_size, total_tokens); + + std::mt19937 rng(seed); + std::uniform_int_distribution dist(0, total_tokens == 0 ? 0 : total_tokens - 1); + + size_t mismatches = 0; + for (size_t s = 0; s < sample_size; ++s) { + size_t global = dist(rng); + size_t passage = static_cast( + std::upper_bound(offsets.begin(), offsets.end(), global) - offsets.begin() - 1 + ); + size_t local = global - lib::narrow(offsets[passage]); + + auto token = tokens.get_datum(passage).token(local); + uint32_t best = 0; + float best_dot = -std::numeric_limits::infinity(); + for (size_t j = 0; j < K; ++j) { + auto c = centroids.get_datum(j); + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += static_cast(token[d]) * c[d]; + } + if (dot > best_dot) { + best_dot = dot; + best = svs::lib::narrow_cast(j); + } + } + + uint32_t stored = ids.get_datum(passage).token(local)[0]; + if (stored != best) { + ++mismatches; + } + } + return MismatchReport{mismatches, sample_size}; +} + +/// Clusters fine centroids into coarse groups via the same L2/plain-mean trainer, then +/// an exact nearest-coarse assignment. `centroids` is read-only; a local copy is +/// clustered since train_centroids takes Data by non-const ref. +struct CoarseClusters { + std::vector coarse_of_fine; + size_t num_coarse = 0; +}; + +template +CoarseClusters coarse_clusters( + const svs::data::SimpleData& centroids, + const CodebookParameters& parameters, + Pool& threadpool +) { + size_t K = centroids.size(); + size_t dim = centroids.dimensions(); + size_t num_coarse = parameters.num_coarse == 0 ? std::max(1, K * 5 / 32) + : parameters.num_coarse; + if (num_coarse > K) { + throw ANNEXCEPTION( + "coarse_clusters: num_coarse={} exceeds the number of fine centroids {}", + num_coarse, + K + ); + } + + // Offset from the fine-training seed so the two never share a random stream. + std::mt19937 rng(parameters.seed + 1); + auto working_copy = centroids; + auto kmeans_result = train_centroids( + working_copy, + num_coarse, + parameters.coarse_iterations, + parameters.minibatch_size, + rng, + threadpool + ); + svs::lib::Timer scratch_timer; + svs::kmeans::normalize_centroids(kmeans_result.centroids, threadpool, scratch_timer); + + std::vector coarse_of_fine(K); + threads::parallel_for( + threadpool, + threads::StaticPartition{K}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + auto c = centroids.get_datum(i); + uint32_t best = 0; + float best_dot = -std::numeric_limits::infinity(); + for (size_t j = 0; j < num_coarse; ++j) { + auto cc = kmeans_result.centroids.get_datum(j); + float dot = 0.0F; + for (size_t d = 0; d < dim; ++d) { + dot += c[d] * cc[d]; + } + if (dot > best_dot) { + best_dot = dot; + best = svs::lib::narrow_cast(j); + } + } + coarse_of_fine[i] = best; + } + } + ); + + return CoarseClusters{std::move(coarse_of_fine), num_coarse}; +} + +/// build_ground_structures's output: the bf16 FineCentroids and pairwise CentroidTable +/// GEM's distances need, built from the same trained centroids. +struct GroundStructures { + std::shared_ptr fine_centroids; + std::shared_ptr centroid_table; +}; + +/// Builds the bf16 FineCentroids and the pairwise CentroidTable from trained centroids; +/// the pair quantize's distances need but neither centroid_table nor FineCentroids' +/// constructor derives from the other, so they are built together here once. +inline GroundStructures build_ground_structures( + const svs::data::SimpleData& centroids, + const CodebookParameters& parameters, + size_t num_threads +) { + size_t K = centroids.size(); + size_t dim = centroids.dimensions(); + auto fine_centroids = std::make_shared( + K, dim, centroids.data(), num_threads + ); + auto centroid_table = std::make_shared( + svs::distance::CentroidTable::from_centroids( + centroids.data(), K, dim, parameters.ground, num_threads + ) + ); + return GroundStructures{std::move(fine_centroids), std::move(centroid_table)}; +} + +/// Full pipeline: train, assign, build the bf16 FineCentroids and the pairwise +/// CentroidTable, and cluster the fine centroids themselves. The single entry point a +/// driver calls to turn a raw token store into everything GEM's distances need. +struct Quantized { + svs::data::SimpleData centroids; + std::shared_ptr fine_centroids; + std::shared_ptr centroid_table; + svs::data::VectorSetData ids; + std::vector coarse_of_fine; + size_t num_coarse = 0; + size_t K = 0; + double objective = 0.0; + size_t num_sampled_passages = 0; + size_t num_training_tokens = 0; +}; + +/// Composes train_codebook, assign_tokens, build_ground_structures and coarse_clusters: +/// the pipeline a caller wanting plain results (no phase timing) runs end to end. +template +Quantized +quantize(const Data& tokens, const CodebookParameters& parameters, Pool& threadpool) { + auto trained = train_codebook(tokens, parameters, threadpool); + auto ids = assign_tokens(tokens, trained.centroids, parameters, threadpool); + + size_t K = trained.centroids.size(); + size_t num_threads = threadpool.size(); + auto ground = build_ground_structures(trained.centroids, parameters, num_threads); + auto coarse = coarse_clusters(trained.centroids, parameters, threadpool); + + return Quantized{ + std::move(trained.centroids), + std::move(ground.fine_centroids), + std::move(ground.centroid_table), + std::move(ids), + std::move(coarse.coarse_of_fine), + coarse.num_coarse, + K, + trained.objective, + trained.num_sampled_passages, + trained.num_training_tokens}; +} + +} // namespace svs::quantization::gem + +#else + +#error \ + "svs/quantization/gem/codebook.h needs SVS_ENABLE_MULTI_VECTOR=1 (cmake: SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR=YES); it pulls in Intel MKL through kmeans_training.h." + +#endif // SVS_ENABLE_MULTI_VECTOR diff --git a/include/svs/quantization/gem/codebook_timed.h b/include/svs/quantization/gem/codebook_timed.h new file mode 100644 index 000000000..11ee4c9ef --- /dev/null +++ b/include/svs/quantization/gem/codebook_timed.h @@ -0,0 +1,173 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if SVS_ENABLE_MULTI_VECTOR + +// svs +#include "svs/lib/threads.h" +#include "svs/lib/timing.h" +#include "svs/quantization/gem/codebook.h" + +// stl +#include +#include +#include +#include + +// Opt-in wall-clock timing for svs::quantization::gem::quantize/train_codebook. Nothing +// in the library includes this header; a caller that wants phase timings does. +namespace svs::quantization::gem { + +/// Wall-clock seconds for each phase, measured with std::chrono::steady_clock. +struct QuantizeTimings { + double sample = 0.0; + double train = 0.0; + double normalize = 0.0; + double assign = 0.0; + double ground_table = 0.0; + double coarse = 0.0; + double total = 0.0; + std::vector train_iterations; +}; + +namespace detail { + +// Times the interval between before_/after_iteration and appends it to `seconds`, which +// outlives the observer; satisfies KMeansObserver. +struct IterationTimer { + std::vector& seconds; + std::chrono::steady_clock::time_point start; + + void before_iteration(size_t /*iter*/) { start = std::chrono::steady_clock::now(); } + void after_iteration(size_t /*iter*/) { + seconds.push_back( + std::chrono::duration(std::chrono::steady_clock::now() - start).count() + ); + } +}; + +} // namespace detail + +/// train_codebook, plus a QuantizeTimings for sample/train/normalize/train_iterations. +template +std::pair train_codebook_timed( + const Data& tokens, const CodebookParameters& parameters, Pool& threadpool +) { + auto total_start = std::chrono::steady_clock::now(); + QuantizeTimings timings; + + std::mt19937 rng(parameters.seed); + + auto sample_start = std::chrono::steady_clock::now(); + auto sampled = sample_training_tokens(tokens, parameters, rng, threadpool); + timings.sample = + std::chrono::duration(std::chrono::steady_clock::now() - sample_start) + .count(); + + auto train_start = std::chrono::steady_clock::now(); + auto kmeans_result = train_centroids( + sampled.trainset, + sampled.K, + parameters.kmeans_iterations, + parameters.minibatch_size, + rng, + threadpool, + detail::IterationTimer{timings.train_iterations, {}}, + parameters.kmeans_algorithm, + parameters.emeans_epochs + ); + timings.train = + std::chrono::duration(std::chrono::steady_clock::now() - train_start) + .count(); + + auto normalize_start = std::chrono::steady_clock::now(); + svs::lib::Timer scratch_timer; + svs::kmeans::normalize_centroids(kmeans_result.centroids, threadpool, scratch_timer); + timings.normalize = + std::chrono::duration(std::chrono::steady_clock::now() - normalize_start) + .count(); + + timings.total = + std::chrono::duration(std::chrono::steady_clock::now() - total_start) + .count(); + + auto trained = TrainedCodebook{ + std::move(kmeans_result.centroids), + kmeans_result.objective, + sampled.num_sampled_passages, + sampled.num_training_tokens}; + return {std::move(trained), std::move(timings)}; +} + +/// quantize, plus the full QuantizeTimings (train_codebook_timed's phases, then +/// assign/ground_table/coarse/total). Identical result to quantize on the same input. +template +std::pair +quantize_timed(const Data& tokens, const CodebookParameters& parameters, Pool& threadpool) { + auto total_start = std::chrono::steady_clock::now(); + + auto [trained, timings] = train_codebook_timed(tokens, parameters, threadpool); + + auto assign_start = std::chrono::steady_clock::now(); + auto ids = assign_tokens(tokens, trained.centroids, parameters, threadpool); + timings.assign = + std::chrono::duration(std::chrono::steady_clock::now() - assign_start) + .count(); + + size_t K = trained.centroids.size(); + size_t num_threads = threadpool.size(); + + // Timed with the pairwise table build; neither has its own QuantizeTimings field. + auto ground_start = std::chrono::steady_clock::now(); + auto ground = build_ground_structures(trained.centroids, parameters, num_threads); + timings.ground_table = + std::chrono::duration(std::chrono::steady_clock::now() - ground_start) + .count(); + + auto coarse_start = std::chrono::steady_clock::now(); + auto coarse = coarse_clusters(trained.centroids, parameters, threadpool); + timings.coarse = + std::chrono::duration(std::chrono::steady_clock::now() - coarse_start) + .count(); + + timings.total = + std::chrono::duration(std::chrono::steady_clock::now() - total_start) + .count(); + + auto quantized = Quantized{ + std::move(trained.centroids), + std::move(ground.fine_centroids), + std::move(ground.centroid_table), + std::move(ids), + std::move(coarse.coarse_of_fine), + coarse.num_coarse, + K, + trained.objective, + trained.num_sampled_passages, + trained.num_training_tokens}; + return {std::move(quantized), std::move(timings)}; +} + +} // namespace svs::quantization::gem + +#else + +#error \ + "svs/quantization/gem/codebook_timed.h needs SVS_ENABLE_MULTI_VECTOR=1 (cmake: SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR=YES); it pulls in Intel MKL through codebook.h." + +#endif // SVS_ENABLE_MULTI_VECTOR diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 88a9ae412..0a5ad0d01 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -109,11 +109,17 @@ set(TEST_SOURCES ${TEST_DIR}/svs/core/data/simple.cpp ${TEST_DIR}/svs/core/data/serialization.cpp ${TEST_DIR}/svs/core/data/view.cpp + ${TEST_DIR}/svs/core/data/vector_set_view.cpp + ${TEST_DIR}/svs/core/data/vector_set.cpp ${TEST_DIR}/svs/core/distance.cpp ${TEST_DIR}/svs/core/distances/simd_utils.cpp ${TEST_DIR}/svs/core/distances/distance_euclidean.cpp ${TEST_DIR}/svs/core/distances/inner_product.cpp + ${TEST_DIR}/svs/core/distances/inner_product_matrix.cpp ${TEST_DIR}/svs/core/distances/cosine.cpp + ${TEST_DIR}/svs/core/distances/gem.cpp + ${TEST_DIR}/svs/core/distances/gem_router.cpp + ${TEST_DIR}/svs/core/distances/maxsim.cpp ${TEST_DIR}/svs/core/graph.cpp ${TEST_DIR}/svs/core/io/vecs.cpp ${TEST_DIR}/svs/core/io/binary.cpp @@ -132,9 +138,15 @@ set(TEST_SOURCES ${TEST_DIR}/svs/index/flat/flat.cpp ${TEST_DIR}/svs/index/flat/dynamic_flat.cpp ${TEST_DIR}/svs/index/vamana/build_parameters.cpp + ${TEST_DIR}/svs/index/vamana/build_stats.cpp + ${TEST_DIR}/svs/index/vamana/connectivity.cpp ${TEST_DIR}/svs/index/vamana/consolidate.cpp ${TEST_DIR}/svs/index/vamana/filter.cpp + ${TEST_DIR}/svs/index/vamana/gem_build.cpp + ${TEST_DIR}/svs/index/vamana/gem_routing.cpp + ${TEST_DIR}/svs/index/vamana/gem_vector_set.cpp ${TEST_DIR}/svs/index/vamana/index.cpp + ${TEST_DIR}/svs/index/vamana/maxsim_rerank.cpp ${TEST_DIR}/svs/index/vamana/prune.cpp ${TEST_DIR}/svs/index/vamana/search_buffer.cpp ${TEST_DIR}/svs/index/vamana/search_parameters.cpp @@ -232,6 +244,20 @@ if (SVS_EXPERIMENTAL_ENABLE_IVF) ) endif() +if (SVS_EXPERIMENTAL_ENABLE_IVF OR SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) + message("Enabling shared k-means tests!") + list(APPEND TEST_SOURCES + ${TEST_DIR}/svs/core/kmeans_training.cpp + ) +endif() + +if (SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR) + message("Enabling GEM codebook tests!") + list(APPEND TEST_SOURCES + ${TEST_DIR}/svs/quantization/gem/codebook.cpp + ) +endif() + add_executable(tests ${TEST_SOURCES}) # Path to the test dataset. diff --git a/tests/svs/core/data/vector_set.cpp b/tests/svs/core/data/vector_set.cpp new file mode 100644 index 000000000..5cb0846dc --- /dev/null +++ b/tests/svs/core/data/vector_set.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/core/data/vector_set.h" + +// svs +#include "svs/concepts/vector_set.h" +#include "svs/core/allocator.h" +#include "svs/core/data/simple.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/lib/float16.h" +#include "svs/lib/saveload.h" + +// tests +#include "tests/utils/utils.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include + +namespace { +// 3 documents with cardinalities 2, 0, 3 over dimension 2, matching the fixture in +// vector_set_view.cpp so both headers are checked against the same layout. +svs::data::SimpleData make_tokens() { + auto tokens = svs::data::SimpleData(5, 2); + float values[5][2] = { + {1.0f, 2.0f}, {3.0f, 4.0f}, {5.0f, 6.0f}, {7.0f, 8.0f}, {9.0f, 10.0f}}; + for (size_t i = 0; i < 5; ++i) { + tokens.set_datum(i, std::span(values[i], 2)); + } + return tokens; +} + +std::vector make_offsets() { return {0, 2, 2, 5}; } + +// Every document's tokens, element-wise, must match between two equivalent datasets. +template void require_same_contents(const A& a, const B& b) { + CATCH_REQUIRE(a.size() == b.size()); + for (size_t i = 0; i < a.size(); ++i) { + auto da = a.get_datum(i); + auto db = b.get_datum(i); + CATCH_REQUIRE(da.cardinality() == db.cardinality()); + for (size_t j = 0; j < da.cardinality(); ++j) { + auto ta = da.token(j); + auto tb = db.token(j); + CATCH_REQUIRE(std::equal(ta.begin(), ta.end(), tb.begin())); + } + } +} +} // namespace + +CATCH_TEST_CASE("Vector Set Data", "[core][data][vector_set]") { + CATCH_SECTION("Construction by move is a no-copy takeover") { + auto tokens = make_tokens(); + const float* original_ptr = tokens.data(); + + auto container = svs::data::VectorSetData(std::move(tokens), make_offsets()); + CATCH_REQUIRE(container.tokens().data() == original_ptr); + + // The moved-from source is left in a state that is safe to inspect (even if its + // contents are unspecified); this should not throw or crash. + CATCH_REQUIRE_NOTHROW(tokens.get_allocator()); + } + + CATCH_SECTION("Accessors") { + auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); + CATCH_REQUIRE(container.size() == 3); + CATCH_REQUIRE(container.dimensions() == 2); + CATCH_REQUIRE(container.total_tokens() == 5); + auto offsets = container.offsets(); + CATCH_REQUIRE(offsets.size() == 4); + CATCH_REQUIRE(offsets[3] == 5); + + // cardinality() and get_datum() against the ground truth, one pass. + std::vector>> expected{ + {{1.0f, 2.0f}, {3.0f, 4.0f}}, {}, {{5.0f, 6.0f}, {7.0f, 8.0f}, {9.0f, 10.0f}}}; + for (size_t i = 0; i < expected.size(); ++i) { + auto set = container.get_datum(i); + CATCH_REQUIRE(set.cardinality() == expected[i].size()); + CATCH_REQUIRE(set.empty() == expected[i].empty()); + for (size_t j = 0; j < expected[i].size(); ++j) { + auto t = set.token(j); + CATCH_REQUIRE(t[0] == expected[i][j][0]); + CATCH_REQUIRE(t[1] == expected[i][j][1]); + } + container.prefetch(i); // exercised for the empty document (i==1) too + } + } + + CATCH_SECTION("view() matches the container element-wise") { + auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); + auto view = container.view(); + CATCH_REQUIRE(view.dimensions() == container.dimensions()); + require_same_contents(container, view); + } + + CATCH_SECTION("copy_from takes ownership of a copy of a borrowed view") { + auto tokens = std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + auto offsets = std::vector{0, 1, 3}; + auto view = svs::data::ConstVectorSetView(tokens, offsets, 2); + + auto container = svs::data::VectorSetData::copy_from(view); + CATCH_REQUIRE(container.tokens().data() != tokens.data()); + CATCH_REQUIRE(container.cardinality(0) == 1); + CATCH_REQUIRE(container.cardinality(1) == 2); + CATCH_REQUIRE(container.get_datum(1).token(1)[0] == 5.0f); + require_same_contents(container, view); + } + + CATCH_SECTION("Constructor delegates to the shared offsets validator") { + // The constructor adds no validation beyond calling validate_vector_set_layout, + // whose branches are already covered against ConstVectorSetView in + // vector_set_view.cpp. + CATCH_REQUIRE_THROWS_AS( + svs::data::VectorSetData( + svs::data::SimpleData(2, 2), std::vector{} + ), + svs::ANNException + ); + } + + CATCH_SECTION("Static extent, integer element type") { + auto tokens = svs::data::SimpleData(3, 1); + tokens.set_datum(0, std::vector{7}); + tokens.set_datum(1, std::vector{8}); + tokens.set_datum(2, std::vector{9}); + + auto container = svs::data::VectorSetData( + std::move(tokens), std::vector{0, 2, 3} + ); + CATCH_REQUIRE(container.dimensions() == 1); + CATCH_REQUIRE(container.get_datum(0).token(1)[0] == 8); + static_assert(svs::data::ImmutableVectorSetDataset); + } + + CATCH_SECTION("Float16 element type") { + auto tokens = svs::data::SimpleData(2, 3); + tokens.set_datum(0, std::vector{1.0f, 2.0f, 3.0f}); + tokens.set_datum(1, std::vector{4.0f, 5.0f, 6.0f}); + + auto container = svs::data::VectorSetData( + std::move(tokens), std::vector{0, 1, 2} + ); + CATCH_REQUIRE(container.size() == 2); + CATCH_REQUIRE(float(container.get_datum(1).token(0)[0]) == 4.0f); + } + + CATCH_SECTION("Blocked allocator is rejected") { + // `VectorSetData` static_asserts against a Blocked allocator (its per-document + // tokens would no longer be contiguous), so this checks the gating condition + // without instantiating `VectorSetData>`, which + // would be a hard compile error rather than a test failure. + using BlockedAlloc = svs::data::Blocked>; + static_assert(svs::data::is_blocked_v); + } + + CATCH_SECTION("Save/load") { + auto temp_directory = svs_test::prepare_temp_directory_v2(); + auto container = svs::data::VectorSetData(make_tokens(), make_offsets()); + svs::lib::save_to_disk(container, temp_directory); + + CATCH_SECTION("Round trip") { + auto reloaded = + svs::lib::load_from_disk>(temp_directory); + CATCH_REQUIRE(reloaded.dimensions() == container.dimensions()); + CATCH_REQUIRE(reloaded.total_tokens() == container.total_tokens()); + auto a = container.offsets(); + auto b = reloaded.offsets(); + CATCH_REQUIRE(std::equal(a.begin(), a.end(), b.begin())); + require_same_contents(container, reloaded); + } + + CATCH_SECTION("A corrupted offsets file throws") { + bool found_offsets_file = false; + for (auto const& entry : std::filesystem::directory_iterator(temp_directory)) { + if (entry.path().filename().string().starts_with("vector_set_offsets")) { + found_offsets_file = true; + // Truncate so its size no longer matches the saved metadata. + std::filesystem::resize_file(entry.path(), 1); + } + } + CATCH_REQUIRE(found_offsets_file); + CATCH_REQUIRE_THROWS_AS( + svs::lib::load_from_disk>(temp_directory), + svs::ANNException + ); + } + } + + CATCH_SECTION("Loading with the wrong schema throws") { + auto temp_directory = svs_test::prepare_temp_directory_v2(); + // SimpleData has a different, incompatible schema ("uncompressed_data"). + svs::lib::save_to_disk(svs::data::SimpleData(2, 2), temp_directory); + CATCH_REQUIRE_THROWS( + svs::lib::load_from_disk>(temp_directory) + ); + } +} diff --git a/tests/svs/core/data/vector_set_view.cpp b/tests/svs/core/data/vector_set_view.cpp new file mode 100644 index 000000000..5a8002cbd --- /dev/null +++ b/tests/svs/core/data/vector_set_view.cpp @@ -0,0 +1,115 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/core/data/vector_set_view.h" + +// other misc SVS stuff +#include "svs/concepts/vector_set.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include + +CATCH_TEST_CASE("Vector Set View", "[core][data][vector_set_view]") { + // 3 sets with cardinalities 2, 0, 3 over dimension 2. + auto tokens = + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f}; + auto offsets = std::vector{0, 2, 2, 5}; + + CATCH_SECTION("Construction and contents") { + auto view = svs::data::ConstVectorSetView(tokens, offsets, 2); + CATCH_REQUIRE(view.size() == 3); + CATCH_REQUIRE(view.dimensions() == 2); + CATCH_REQUIRE(view.total_tokens() == 5); + + // size(), cardinality() and get_datum() against the ground truth, one pass. + std::vector>> expected{ + {{1.0f, 2.0f}, {3.0f, 4.0f}}, {}, {{5.0f, 6.0f}, {7.0f, 8.0f}, {9.0f, 10.0f}}}; + for (size_t i = 0; i < expected.size(); ++i) { + auto set = view.get_datum(i); + CATCH_REQUIRE(set.cardinality() == expected[i].size()); + CATCH_REQUIRE(set.dimensions() == 2); + CATCH_REQUIRE(set.empty() == expected[i].empty()); + for (size_t j = 0; j < expected[i].size(); ++j) { + auto t = set.token(j); + CATCH_REQUIRE(t[0] == expected[i][j][0]); + CATCH_REQUIRE(t[1] == expected[i][j][1]); + } + view.prefetch(i); // exercised for the empty set (i==1) too + } + } + + CATCH_SECTION("Validation failures: one per throw in validate_vector_set_layout") { + CATCH_REQUIRE_THROWS_AS( + (svs::data::ConstVectorSetView( + std::span(), std::span(), 2 + )), + svs::ANNException + ); // offsets.empty() + CATCH_REQUIRE_THROWS_AS( + svs::data::ConstVectorSetView(tokens, std::vector{1, 2}, 2), + svs::ANNException + ); // offsets.front() != 0 + CATCH_REQUIRE_THROWS_AS( + svs::data::ConstVectorSetView(tokens, std::vector{0, 2, 1}, 2), + svs::ANNException + ); // offsets not non-decreasing + CATCH_REQUIRE_THROWS_AS( + svs::data::ConstVectorSetView(tokens, std::vector{0, 1}, 2), + svs::ANNException + ); // offsets.back() * dimensions != token_elements + CATCH_REQUIRE_THROWS_AS( + svs::data::ConstVectorSetView( + std::span(), std::vector{0, uint64_t{1} << 63}, 2 + ), + svs::ANNException + ); // offsets.back() * dimensions overflows size_t + } + + CATCH_SECTION("Static extent") { + auto static_tokens = std::vector{1.0f, 2.0f, 3.0f, 4.0f}; + auto static_offsets = std::vector{0, 1, 2}; + + auto view = + svs::data::ConstVectorSetView(static_tokens, static_offsets, 2); + CATCH_REQUIRE(view.dimensions() == 2); + CATCH_REQUIRE(view.get_datum(0).token(0)[0] == 1.0f); + + CATCH_REQUIRE_THROWS_AS( + (svs::data::ConstVectorSetView(static_tokens, static_offsets, 2)), + svs::ANNException + ); + } + + CATCH_SECTION("Non-float element type, concept checks") { + auto byte_tokens = std::vector{1, 2, 3, 4, 5, 6}; + auto byte_offsets = std::vector{0, 1, 3}; + auto view = svs::data::ConstVectorSetView(byte_tokens, byte_offsets, 2); + + CATCH_REQUIRE(view.size() == 2); + CATCH_REQUIRE(view.cardinality(0) == 1); + CATCH_REQUIRE(view.cardinality(1) == 2); + CATCH_REQUIRE(view.get_datum(1).token(1)[1] == 6); + + static_assert(svs::data::VectorSet); + static_assert(svs::data::ImmutableVectorSetDataset); + } +} diff --git a/tests/svs/core/distances/gem.cpp b/tests/svs/core/distances/gem.cpp new file mode 100644 index 000000000..1645bd7f7 --- /dev/null +++ b/tests/svs/core/distances/gem.cpp @@ -0,0 +1,846 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test +#include "svs/core/distance/gem.h" +#include "svs/core/distance/transport_simplex.h" +#include "svs/extensions/vamana/gem.h" + +// svs +#include "svs/concepts/distance.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/index/vamana/prune.h" +#include "svs/lib/bfloat16.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stdlib +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using svs::distance::CentroidTable; +using svs::distance::FineCentroids; +using svs::distance::Ground; +using svs::distance::QChamfer; +using svs::distance::QChamferExactQuery; +using svs::distance::QEmd; + +// Four unit vectors in 2D: (1,0), (0,1), (-1,0), (0,-1), so their pairwise ground +// distances come out to round numbers by hand. +std::shared_ptr make_table(Ground ground) { + std::vector centroids{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F, 0.0F, -1.0F}; + return std::make_shared( + CentroidTable::from_centroids(centroids.data(), 4, 2, ground, 1) + ); +} + +// K random unit vectors of width dim, row-major, deterministic per (k, dim, seed). +std::vector random_unit_centroids(size_t k, size_t dim, uint32_t seed) { + std::mt19937 rng(seed); + std::normal_distribution gauss(0.0F, 1.0F); + std::vector flat(k * dim); + for (size_t i = 0; i < k; ++i) { + float* row = flat.data() + i * dim; + double norm2 = 0.0; + for (size_t d = 0; d < dim; ++d) { + row[d] = gauss(rng); + norm2 += static_cast(row[d]) * row[d]; + } + double norm = std::sqrt(norm2); + for (size_t d = 0; d < dim; ++d) { + row[d] = static_cast(row[d] / norm); + } + } + return flat; +} + +// Reference ground-cost table via double-precision dot products over bf16-rounded +// inputs (from_centroids rounds internally too), independent of InnerProductMatrix/tiling. +std::vector +naive_ground_table(const std::vector& flat, size_t k, size_t dim, Ground ground) { + std::vector t(k * k, 0.0); + for (size_t i = 0; i < k; ++i) { + for (size_t j = 0; j < k; ++j) { + if (i == j) { + continue; + } + double dot = 0.0; + for (size_t d = 0; d < dim; ++d) { + dot += static_cast(svs::round_to_bfloat16(flat[i * dim + d])) * + static_cast(svs::round_to_bfloat16(flat[j * dim + d])); + } + t[i * k + j] = ground == Ground::Cos + ? std::max(0.0, 1.0 - dot) + : std::sqrt(std::max(0.0, 2.0 - 2.0 * dot)); + } + } + return t; +} + +// Owns a 1-D id vector set contiguously, for QEmd/QChamfer/QChamferExactQuery's only +// (vector-set) compute() overloads. +struct IdSet { + std::vector ids; + + svs::data::ConstVectorSet view() const { + return svs::data::ConstVectorSet( + std::span(ids), ids.size(), 1 + ); + } +}; + +IdSet make_id_set(std::vector ids) { return IdSet{std::move(ids)}; } + +// Owns a raw-query vector set for QChamferExactQuery: `n` raw fp32 tokens of width `dim`, +// row-major, contiguously. +struct RawQuerySet { + std::vector tokens; + size_t dim; + + svs::data::ConstVectorSet view() const { + return svs::data::ConstVectorSet( + std::span(tokens), tokens.size() / dim, dim + ); + } +}; + +// Compile-time guard: the removed fixed-width padded-row layout must not come back. +static_assert(!svs::data::ContiguousVectorSet>); + +// Independent reference: successive shortest augmenting paths via SPFA over the standard +// min-cost-flow edge-list graph. Deliberately a different algorithm from TransportSimplex. +double reference_transport_cost( + size_t ua, + const std::vector& supply, + size_t ub, + const std::vector& demand, + const std::vector& cost // ua * ub, row-major +) { + constexpr double inf = std::numeric_limits::infinity(); + size_t s = ua + ub, t = ua + ub + 1, n = ua + ub + 2; + std::vector eto; + std::vector ecap, ecost; + std::vector> adj(n); + auto add_edge = [&](size_t from, size_t to, double cap, double c) { + adj[from].push_back(eto.size()); + eto.push_back(static_cast(to)); + ecap.push_back(cap); + ecost.push_back(c); + adj[to].push_back(eto.size()); + eto.push_back(static_cast(from)); + ecap.push_back(0.0); + ecost.push_back(-c); + }; + for (size_t i = 0; i < ua; ++i) { + add_edge(s, i, supply[i], 0.0); + } + for (size_t j = 0; j < ub; ++j) { + add_edge(ua + j, t, demand[j], 0.0); + } + for (size_t i = 0; i < ua; ++i) { + for (size_t j = 0; j < ub; ++j) { + add_edge(i, ua + j, inf, cost[i * ub + j]); + } + } + + double total = 0.0; + for (;;) { + std::vector dist(n, inf); + std::vector pred_edge(n, -1); + std::vector in_queue(n, false); + std::deque queue{s}; + dist[s] = 0.0; + in_queue[s] = true; + while (!queue.empty()) { + size_t u = queue.front(); + queue.pop_front(); + in_queue[u] = false; + for (size_t e : adj[u]) { + if (ecap[e] <= 1e-12) { + continue; + } + size_t v = static_cast(eto[e]); + if (dist[u] + ecost[e] < dist[v] - 1e-12) { + dist[v] = dist[u] + ecost[e]; + pred_edge[v] = static_cast(e); + if (!in_queue[v]) { + queue.push_back(v); + in_queue[v] = true; + } + } + } + } + if (dist[t] == inf) { + break; + } + std::vector path; + for (size_t v = t; v != s; v = static_cast(eto[path.back() ^ 1])) { + path.push_back(static_cast(pred_edge[v])); + } + double bottleneck = inf; + for (size_t e : path) { + bottleneck = std::min(bottleneck, ecap[e]); + } + for (size_t e : path) { + ecap[e] -= bottleneck; + ecap[e ^ 1] += bottleneck; + total += bottleneck * ecost[e]; + } + } + return total; +} + +} // namespace + +CATCH_TEST_CASE("CentroidTable / FineCentroids", "[distance][gem]") { + CATCH_SECTION("FineCentroids holds raw rows, not a pairwise table") { + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F}; + FineCentroids fc{2, 2, flat.data()}; + CATCH_REQUIRE(fc.K == 2); + CATCH_REQUIRE(fc.dim == 2); + CATCH_REQUIRE(fc.data[0] == 1.0F); + CATCH_REQUIRE(fc.data[3] == 1.0F); + } + + CATCH_SECTION("FineCentroids rounds to bfloat16, round to nearest even") { + CATCH_STATIC_REQUIRE(std::is_same_v< + std::decay_t().data[0])>, + svs::BFloat16>); + // None of these are exactly representable in bfloat16: the test only holds if + // fc.data actually rounds, not merely stores float32 unchanged. + std::vector flat{0.1F, -0.1F, 1.0F / 3.0F, -2.0F / 3.0F}; + FineCentroids fc{2, 2, flat.data()}; + for (size_t i = 0; i < flat.size(); ++i) { + CATCH_REQUIRE(fc.data[i] == svs::round_to_bfloat16(flat[i])); + } + } + + CATCH_SECTION("FineCentroids::load round-trips through the unchanged float32 file " + "format") { + std::vector flat{0.1F, 0.2F, 0.3F, -0.4F, 2.5F, -3.5F}; + auto path = + std::filesystem::temp_directory_path() / "svs_gem_fine_centroids_test.bin"; + { + std::ofstream out(path, std::ios::binary); + out.write( + reinterpret_cast(flat.data()), + static_cast(flat.size() * sizeof(float)) + ); + } + auto fc = FineCentroids::load(path, 2); + std::filesystem::remove(path); + CATCH_REQUIRE(fc.K == 3); + CATCH_REQUIRE(fc.dim == 2); + for (size_t i = 0; i < flat.size(); ++i) { + CATCH_REQUIRE(fc.data[i] == svs::round_to_bfloat16(flat[i])); + } + } +} + +CATCH_TEST_CASE("CentroidTable::from_centroids tiling", "[distance][gem]") { + // K spans a partial tile, a tile boundary (255/256/257), and several tiles (600), at + // both dims/grounds/thread-counts; the Gaussian input also exercises bf16 rounding. + for (size_t k : + {size_t{1}, + size_t{2}, + size_t{7}, + size_t{255}, + size_t{256}, + size_t{257}, + size_t{600}}) { + for (size_t dim : {size_t{3}, size_t{128}}) { + for (Ground ground : {Ground::Cos, Ground::L2}) { + auto flat = random_unit_centroids( + k, + dim, + static_cast( + k * 1009 + dim * 31 + (ground == Ground::Cos ? 1 : 2) + ) + ); + auto reference = naive_ground_table(flat, k, dim, ground); + auto one_thread = + CentroidTable::from_centroids(flat.data(), k, dim, ground, 1); + auto four_threads = + CentroidTable::from_centroids(flat.data(), k, dim, ground, 4); + + for (size_t i = 0; i < k; ++i) { + CATCH_REQUIRE( + one_thread(static_cast(i), static_cast(i)) == + 0.0F + ); + for (size_t j = 0; j < k; ++j) { + auto ui = static_cast(i); + auto uj = static_cast(j); + // Thread-count independence and symmetry: bit for bit. + CATCH_REQUIRE(one_thread(ui, uj) == four_threads(ui, uj)); + CATCH_REQUIRE(one_thread(ui, uj) == one_thread(uj, ui)); + // Against a double-precision reference: summation order differs, + // so only approximate agreement is expected. + CATCH_REQUIRE( + static_cast(one_thread(ui, uj)) == + Catch::Approx(reference[i * k + j]).margin(1e-4) + ); + } + } + } + } + } +} + +CATCH_TEST_CASE("QChamfer", "[distance][gem]") { + CATCH_SECTION("Random oracle: naive min-over-doc sum, duplicate ids included") { + std::mt19937 rng(0xC4A4FE2u); + std::uniform_int_distribution n_dist(1, 8); + std::uniform_int_distribution id_dist(0, 5); + std::uniform_real_distribution cost_dist(0.0F, 5.0F); + size_t k = 6; + for (int trial = 0; trial < 30; ++trial) { + std::vector table_flat(k * k); + for (auto& c : table_flat) { + c = cost_dist(rng); + } + auto table = std::make_shared(k, table_flat); + QChamfer dist{table}; + + std::vector q(n_dist(rng)), d(n_dist(rng)); + for (auto& id : q) { + id = id_dist(rng); + } + for (auto& id : d) { + id = id_dist(rng); + } + double reference = 0.0; + for (auto qi : q) { + float best = std::numeric_limits::infinity(); + for (auto di : d) { + best = std::min(best, (*table)(qi, di)); + } + reference += best; + } + + float score = dist.compute(make_id_set(q).view(), make_id_set(d).view()); + CATCH_REQUIRE( + static_cast(score) == Catch::Approx(reference).margin(1e-4) + ); + } + } +} + +CATCH_TEST_CASE("QEmd", "[distance][gem]") { + QEmd dist{make_table(Ground::Cos)}; + + CATCH_SECTION("1 vs 1 takes the direct table lookup, bypassing the solver") { + float score = dist.compute(make_id_set({0}).view(), make_id_set({1}).view()); + CATCH_REQUIRE(score == Catch::Approx(1.0F)); // table(0,1) + } + + CATCH_SECTION("Duplicates merge to a single point with full mass") { + float score = dist.compute(make_id_set({0, 0}).view(), make_id_set({1}).view()); + CATCH_REQUIRE(score == Catch::Approx(1.0F)); // same as plain 1 vs 1 + } + + // Not exercised by the random oracle below, which never draws a == b. + CATCH_SECTION("Identical sets give 0") { + auto a = make_id_set({0, 1}); + CATCH_REQUIRE(dist.compute(a.view(), a.view()) == Catch::Approx(0.0F)); + } + + // Covers compute() end to end (merge_duplicates, runtime ISA dispatch, the solver) + // against the independent reference over manually-merged weights, plus symmetry. + CATCH_SECTION("Random oracle against the independent reference, with duplicate ids") { + std::mt19937 rng(0x5151u); + std::uniform_int_distribution n_dist(2, 10); + std::uniform_int_distribution id_dist(0, 5); + std::uniform_real_distribution cost_dist(0.0F, 5.0F); + size_t k = 6; + auto merge = [](const std::vector& ids) { + std::map w; + for (auto id : ids) { + w[id] += 1.0 / static_cast(ids.size()); + } + std::vector uids; + std::vector uw; + for (auto [id, weight] : w) { + uids.push_back(id); + uw.push_back(weight); + } + return std::pair{uids, uw}; + }; + + for (int trial = 0; trial < 20; ++trial) { + std::vector table_flat(k * k); + for (auto& c : table_flat) { + c = cost_dist(rng); + } + for (size_t i = 0; i < k; ++i) { + table_flat[i * k + i] = 0.0F; + for (size_t j = 0; j < i; ++j) { + table_flat[i * k + j] = table_flat[j * k + i]; + } + } + auto table = std::make_shared(k, table_flat); + QEmd dist{table}; + + std::vector ids_a(n_dist(rng)), ids_b(n_dist(rng)); + for (auto& id : ids_a) { + id = id_dist(rng); + } + for (auto& id : ids_b) { + id = id_dist(rng); + } + auto [uids_a, uw_a] = merge(ids_a); + auto [uids_b, uw_b] = merge(ids_b); + std::vector cost(uids_a.size() * uids_b.size()); + for (size_t i = 0; i < uids_a.size(); ++i) { + for (size_t j = 0; j < uids_b.size(); ++j) { + cost[i * uids_b.size() + j] = (*table)(uids_a[i], uids_b[j]); + } + } + double reference = + reference_transport_cost(uids_a.size(), uw_a, uids_b.size(), uw_b, cost); + + float ab = dist.compute(make_id_set(ids_a).view(), make_id_set(ids_b).view()); + float ba = dist.compute(make_id_set(ids_b).view(), make_id_set(ids_a).view()); + CATCH_REQUIRE(static_cast(ab) == Catch::Approx(reference).margin(1e-5)); + CATCH_REQUIRE(ba == Catch::Approx(ab)); + } + } + + CATCH_SECTION("start() reports the constructed start; CheapestRow matches the " + "default distance") { + auto table = make_table(Ground::Cos); + QEmd default_dist{table}; + QEmd cheapest_dist{table, nullptr, svs::distance::TransportStart::CheapestRow}; + CATCH_REQUIRE(default_dist.start() == svs::distance::TransportStart::NorthWest); + CATCH_REQUIRE(cheapest_dist.start() == svs::distance::TransportStart::CheapestRow); + + auto a = make_id_set({0, 1}); + auto b = make_id_set({2, 3}); + float default_score = default_dist.compute(a.view(), b.view()); + float cheapest_score = cheapest_dist.compute(a.view(), b.view()); + double gap = svs::distance::TransportSimplex< + svs::distance::AVX_AVAILABILITY::AVX2>::error_bound(2, 2, 2.0, 1.0); + CATCH_REQUIRE( + static_cast(cheapest_score) == + Catch::Approx(static_cast(default_score)).margin(gap + 1e-6) + ); + } +} + +CATCH_TEST_CASE("QEmd transportation solve: random cross-checks", "[distance][gem]") { + using svs::distance::AVX_AVAILABILITY; + using svs::distance::QEmdImpl; + + std::mt19937 rng(0x7EA5CADEu); + std::uniform_int_distribution size_dist(1, 40); + std::uniform_real_distribution cost_dist(0.0, 10.0); + + // Checks compute_decoded's NONE path against the reference and every ISA path against + // NONE. `tiny_cost_range >= 0`/`uniform_weights` force the degenerate (tied) case. + auto run_case = [&](size_t ua, + size_t ub, + double weight_spread, + bool uniform_weights = false, + int tiny_cost_range = -1, + int repeated_levels = 0, + bool row_index_cost = false) { + std::vector w_a(ua), w_b(ub); + if (uniform_weights) { + std::fill(w_a.begin(), w_a.end(), 1.0 / static_cast(ua)); + std::fill(w_b.begin(), w_b.end(), 1.0 / static_cast(ub)); + } else if (repeated_levels > 0) { + // Weights from a small integer set, normalized: ties the walk without making + // every weight equal, unlike uniform_weights. + std::uniform_int_distribution level_dist(1, repeated_levels); + for (auto* w : {&w_a, &w_b}) { + double total = 0.0; + for (auto& x : *w) { + x = static_cast(level_dist(rng)); + total += x; + } + for (auto& x : *w) { + x /= total; + } + } + } else { + for (auto* w : {&w_a, &w_b}) { + double total = 0.0; + std::uniform_real_distribution wd(0.01, weight_spread); + for (auto& x : *w) { + x = wd(rng); + total += x; + } + for (auto& x : *w) { + x /= total; // Same normalization merge_sorted_duplicates relies on. + } + } + } + size_t k = std::max(ua, ub); + std::vector ids_a(ua), ids_b(ub); + std::vector cost(ua * ub); + std::vector table_flat(k * k, 0.0F); + std::uniform_int_distribution tiny_cost_dist(0, std::max(tiny_cost_range, 0)); + for (size_t i = 0; i < ua; ++i) { + ids_a[i] = static_cast(i); + for (size_t j = 0; j < ub; ++j) { + // row_index_cost: row 0 is cheapest for every column, by construction. + double c = row_index_cost ? static_cast(i) + : tiny_cost_range >= 0 ? static_cast(tiny_cost_dist(rng)) + : cost_dist(rng); + cost[i * ub + j] = c; + table_flat[i * k + j] = static_cast(c); + } + } + for (size_t j = 0; j < ub; ++j) { + ids_b[j] = static_cast(j); + } + CentroidTable table(k, table_flat); + double max_cost = *std::max_element(cost.begin(), cost.end()); + + double reference = reference_transport_cost(ua, w_a, ub, w_b, cost); + float none_result = QEmdImpl::compute_decoded( + table, + ids_a.data(), + w_a.data(), + ua, + ids_b.data(), + w_b.data(), + ub, + svs::distance::TransportStart::NorthWest + ); + CATCH_REQUIRE( + static_cast(none_result) == Catch::Approx(reference).margin(1e-6) + ); + + // Tolerance vs. NONE: the optimized solver's derived optimality-gap bound, plus 2 + // float ulps of none_result for the float cast/accumulation NONE itself already + // did. + auto isa_margin = [&](double none) { + double gap = + svs::distance::TransportSimplex::error_bound( + ua, ub, max_cost, 1.0 + ); + double ulp = 2 * static_cast(std::numeric_limits::epsilon()) * + std::max(1.0, std::abs(none)); + return gap + ulp; + }; + + for (auto start : + {svs::distance::TransportStart::NorthWest, + svs::distance::TransportStart::CheapestRow}) { +#if defined(__x86_64__) + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + float avx2_result = QEmdImpl::compute_decoded( + table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub, start + ); + CATCH_REQUIRE( + static_cast(avx2_result) == + Catch::Approx(static_cast(none_result)) + .margin(isa_margin(static_cast(none_result))) + ); + } + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + float avx512_result = QEmdImpl::compute_decoded( + table, ids_a.data(), w_a.data(), ua, ids_b.data(), w_b.data(), ub, start + ); + CATCH_REQUIRE( + static_cast(avx512_result) == + Catch::Approx(static_cast(none_result)) + .margin(isa_margin(static_cast(none_result))) + ); + } +#endif + } + }; + + CATCH_SECTION("Random problems") { + for (int trial = 0; trial < 64; ++trial) { + run_case(size_dist(rng), size_dist(rng), 1.0); + } + } + CATCH_SECTION("Single node on one side") { + for (int trial = 0; trial < 5; ++trial) { + run_case(1, size_dist(rng), 1.0); + run_case(size_dist(rng), 1, 1.0); + } + } + CATCH_SECTION("Large weight dynamic range") { + for (int trial = 0; trial < 10; ++trial) { + run_case(size_dist(rng), size_dist(rng), 1e6); + } + } + // Merges the "many equal costs" and "uniform-weight tiny-integer-cost" families: both + // are highly degenerate (many tied reduced costs), just via a different knob. + CATCH_SECTION("Degenerate: equal costs and tiny integer cost ranges") { + std::uniform_int_distribution big_size_dist(1, 64); + struct Config { + bool uniform_weights; + int tiny_cost_range; + }; + for (Config cfg : {Config{false, 0}, Config{true, 2}}) { + for (int trial = 0; trial < 15; ++trial) { + run_case( + big_size_dist(rng), + big_size_dist(rng), + 1.0, + cfg.uniform_weights, + cfg.tiny_cost_range + ); + } + } + } + CATCH_SECTION("Larger shapes: dense, lopsided, transpose, and the Dynamic bucket") { + // 4x256/256x4 exercise the transpose; 5x300/300x5 cross into the Dynamic bucket + // (side > 256); 1x256/256x1 are single-source/sink; the last two are tied costs. + run_case(100, 100, 1.0); + run_case(4, 256, 1.0); + run_case(256, 4, 1.0); + run_case(3, 200, 1.0); + run_case(5, 300, 1.0); + run_case(300, 5, 1.0); + run_case(1, 256, 1.0); + run_case(256, 1, 1.0); + run_case(100, 100, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/0); + run_case(100, 100, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/1); + } + // Ties in weights and costs are where the start tree's walk adds zero-flow arcs and + // where the cheapest-row column grouping collapses; both threaten strong feasibility. + CATCH_SECTION("Start-tree degeneracy: tied costs, repeated weights, extreme shapes") { + for (int tiny_cost_range : {0, 1}) { + for (bool uniform_weights : {false, true}) { + int repeated_levels = uniform_weights ? 0 : 3; + for (size_t n : {size_t{2}, size_t{7}, size_t{64}}) { + // With uniform weights, supply and demand tie at every walk step. + run_case(n, n, 1.0, uniform_weights, tiny_cost_range, repeated_levels); + } + // 300 keeps the Dynamic (uint32 id) bucket's 1xn/nx1 cases tied too. + for (size_t n : + {size_t{2}, size_t{7}, size_t{64}, size_t{256}, size_t{300}}) { + run_case(1, n, 1.0, uniform_weights, tiny_cost_range, repeated_levels); + run_case(n, 1, 1.0, uniform_weights, tiny_cost_range, repeated_levels); + } + } + } + + // Uniform weights only: other weights raise the reference's augmenting-path count + // toward ua+ub-1 and dominate the section's runtime. + run_case(256, 256, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/0); + run_case(256, 256, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/1); + run_case(257, 257, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/0); + run_case(257, 257, 1.0, /*uniform_weights=*/true, /*tiny_cost_range=*/1); + + // Tied, lopsided, and into the Dynamic bucket on the larger side. + run_case(3, 300, 1.0, false, /*tiny_cost_range=*/1, /*repeated_levels=*/3); + run_case(300, 3, 1.0, false, /*tiny_cost_range=*/1, /*repeated_levels=*/3); + + // Row 0 is cheapest for every column, so every column lands in one group. + run_case(64, 64, 1.0, false, -1, 3, /*row_index_cost=*/true); + run_case(256, 64, 1.0, false, -1, 3, /*row_index_cost=*/true); + } +} + +CATCH_TEST_CASE("QChamferExactQuery", "[distance][gem]") { + std::vector flat2x2{1.0F, 0.0F, 0.0F, 1.0F}; + QChamferExactQuery dist{std::make_shared(2, 2, flat2x2.data())}; + + CATCH_SECTION("fix_argument: query dimension mismatch throws") { + RawQuerySet wrong_dim{{1.0F, 0.0F, 0.0F}, 3}; + CATCH_REQUIRE_THROWS_AS( + svs::distance::maybe_fix_argument(dist, wrong_dim.view()), std::invalid_argument + ); + } + + // Uniform(-1,1) draws are, with overwhelming probability, not exactly representable + // in bfloat16, so this oracle exercises the rounding path without a hardcoded value. + CATCH_SECTION("Random oracle: naive reference over non-bf16-representable values") { + std::mt19937 rng(0xB16u); + std::uniform_real_distribution val_dist(-1.0F, 1.0F); + std::uniform_int_distribution id_dist(0, 3); + size_t k = 4, dim = 3; + for (int trial = 0; trial < 15; ++trial) { + std::vector flat(k * dim); + for (auto& x : flat) { + x = val_dist(rng); + } + auto centroids = std::make_shared(k, dim, flat.data()); + QChamferExactQuery trial_dist{centroids}; + + size_t n_query = 1 + (trial % 3); + std::vector tokens(n_query * dim); + for (auto& x : tokens) { + x = val_dist(rng); + } + RawQuerySet query_set{tokens, dim}; + svs::distance::maybe_fix_argument(trial_dist, query_set.view()); + + std::vector doc_ids(2 + (trial % 2)); + for (auto& id : doc_ids) { + id = id_dist(rng); + } + float score = trial_dist.compute(query_set.view(), make_id_set(doc_ids).view()); + + // Independent of fix_argument_decoded and the kernel: rounds both sides + // itself, then takes the max dot per query token and averages 1 - max. + double total = 0.0; + for (size_t i = 0; i < n_query; ++i) { + float best = -std::numeric_limits::infinity(); + for (auto id : doc_ids) { + double dot = 0.0; + for (size_t d = 0; d < dim; ++d) { + double cv = svs::round_to_bfloat16(flat[id * dim + d]); + double qv = svs::round_to_bfloat16(tokens[i * dim + d]); + dot += cv * qv; + } + best = std::max(best, static_cast(dot)); + } + total += 1.0 - static_cast(best); + } + float expected = static_cast(total / static_cast(n_query)); + CATCH_REQUIRE(score == Catch::Approx(expected).margin(1e-4)); + } + } +} + +// Each distance's compute() calls the shared gem_detail::require_id_dimensions from a +// different call site; one section per site, same underlying check. +CATCH_TEST_CASE("id vector set dimensions() != 1 throws", "[distance][gem]") { + std::vector raw{0, 1, 2, 3}; + auto bad = svs::data::ConstVectorSet(std::span(raw), 2, 2); + auto good = make_id_set({0}); + + CATCH_SECTION("QEmd") { + QEmd dist{make_table(Ground::Cos)}; + CATCH_REQUIRE_THROWS_AS(dist.compute(bad, good.view()), std::invalid_argument); + CATCH_REQUIRE_THROWS_AS(dist.compute(good.view(), bad), std::invalid_argument); + } + CATCH_SECTION("QChamfer") { + QChamfer dist{make_table(Ground::Cos)}; + CATCH_REQUIRE_THROWS_AS(dist.compute(bad, good.view()), std::invalid_argument); + CATCH_REQUIRE_THROWS_AS(dist.compute(good.view(), bad), std::invalid_argument); + } + CATCH_SECTION("QChamferExactQuery (doc side)") { + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F}; + QChamferExactQuery dist{std::make_shared(2, 2, flat.data())}; + RawQuerySet query_set{{1.0F, 0.0F}, 2}; + svs::distance::maybe_fix_argument(dist, query_set.view()); + CATCH_REQUIRE_THROWS_AS(dist.compute(query_set.view(), bad), std::invalid_argument); + } +} + +CATCH_TEST_CASE( + "Generic svs::distance::compute dispatches to each distance's compute() overload", + "[distance][gem]" +) { + CATCH_SECTION("QEmd") { + QEmd dist{make_table(Ground::Cos)}; + auto a = make_id_set({0}); + auto b = make_id_set({1, 2}); + CATCH_REQUIRE( + svs::distance::compute(dist, a.view(), b.view()) == + Catch::Approx(dist.compute(a.view(), b.view())) + ); + } + + CATCH_SECTION("QChamfer") { + QChamfer dist{make_table(Ground::Cos)}; + auto q = make_id_set({0, 1}); + auto d = make_id_set({2, 3}); + CATCH_REQUIRE( + svs::distance::compute(dist, q.view(), d.view()) == + Catch::Approx(dist.compute(q.view(), d.view())) + ); + } + + CATCH_SECTION("QChamferExactQuery") { + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F}; + QChamferExactQuery dist{std::make_shared(2, 2, flat.data())}; + RawQuerySet query_set{{1.0F, 0.0F}, 2}; + auto doc_set = make_id_set({0, 1}); + svs::distance::maybe_fix_argument(dist, query_set.view()); + CATCH_REQUIRE( + svs::distance::compute(dist, query_set.view(), doc_set.view()) == + Catch::Approx(dist.compute(query_set.view(), doc_set.view())) + ); + } +} + +CATCH_TEST_CASE("GEM distance traits", "[distance][gem]") { + CATCH_STATIC_REQUIRE(std::is_same_v>); + CATCH_STATIC_REQUIRE(std::is_same_v>); + CATCH_STATIC_REQUIRE(std::is_same_v< + svs::distance::QChamferExactQuery::compare, + std::less<>>); + + CATCH_STATIC_REQUIRE(svs::distance::implicitly_broadcastable()); + CATCH_STATIC_REQUIRE(svs::distance::implicitly_broadcastable() + ); + CATCH_STATIC_REQUIRE( + !svs::distance::implicitly_broadcastable() + ); + + CATCH_STATIC_REQUIRE(!svs::distance::fix_argument_mandated()); + CATCH_STATIC_REQUIRE(!svs::distance::fix_argument_mandated()); + CATCH_STATIC_REQUIRE( + svs::distance::fix_argument_mandated() + ); +} + +CATCH_TEST_CASE("GEM Vamana extensions", "[distance][gem][vamana]") { + CATCH_SECTION("Prune strategy") { + static_assert(std::is_same_v< + svs::index::vamana::prune_strategy_t, + svs::index::vamana::ProgressivePruneStrategy>); + static_assert(std::is_same_v< + svs::index::vamana::prune_strategy_t, + svs::index::vamana::ProgressivePruneStrategy>); + } + + // Compiling proves these skip DefaultBuildAdaptor's default-construct path (QEmd/ + // QChamfer have no default ctor); compute() below proves distance_ holds the real + // table. + CATCH_SECTION("build_adaptor carries the real CentroidTable through") { + auto table = make_table(Ground::Cos); + QEmd qemd{table}; + int dummy_data = 0; + auto adaptor = svs::index::vamana::extensions::build_adaptor(dummy_data, qemd); + auto a = make_id_set({0}); + auto b = make_id_set({1}); + float score = adaptor.general_distance().compute(a.view(), b.view()); + CATCH_REQUIRE(score == Catch::Approx(1.0F)); // table(0,1), same as the QEmd test. + + QChamfer qchamfer{table}; + auto adaptor2 = svs::index::vamana::extensions::build_adaptor(dummy_data, qchamfer); + auto q = make_id_set({0, 1}); + auto d = make_id_set({2, 3}); + float score2 = adaptor2.general_distance().compute(q.view(), d.view()); + CATCH_REQUIRE(score2 == Catch::Approx(2.0F)); + } +} diff --git a/tests/svs/core/distances/gem_router.cpp b/tests/svs/core/distances/gem_router.cpp new file mode 100644 index 000000000..6b2d73951 --- /dev/null +++ b/tests/svs/core/distances/gem_router.cpp @@ -0,0 +1,239 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test +#include "svs/core/distance/gem_router.h" + +// svs +#include "svs/core/data/vector_set_view.h" +#include "svs/lib/threads.h" + +// tests +#include "tests/utils/utils.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include + +namespace { +using svs::distance::GemRouter; + +constexpr uint32_t NO_REP = GemRouter::no_representative; + +svs::data::ConstVectorSetView +view_of(const std::vector& ids, const std::vector& offsets) { + return svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); +} + +// 6 fine centroids / 4 coarse clusters (cluster 3 empty); doc3/doc4 tie for cluster 2 and +// doc3 (lower id) must win. +std::vector hand_coarse_of_fine() { return {0, 0, 1, 1, 2, 2}; } +std::vector hand_ids() { return {0, 0, 2, 0, 1, 1, 2, 3, 4, 4}; } +std::vector hand_offsets() { return {0, 3, 6, 8, 9, 10}; } + +std::vector random_ids(size_t n, size_t k, std::vector& offsets_out) { + std::mt19937 rng(0xC0FFEE); + std::uniform_int_distribution id_dist(0, static_cast(k - 1)); + std::vector ids; + offsets_out.assign(n + 1, 0); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 1 + (i % 7); + offsets_out[i + 1] = offsets_out[i] + cardinality; + for (size_t t = 0; t < cardinality; ++t) { + ids.push_back(id_dist(rng)); + } + } + return ids; +} +} // namespace + +CATCH_TEST_CASE("GemRouter build", "[distance][gem][gem_router]") { + auto ids = hand_ids(); + auto offsets = hand_offsets(); + auto docs = view_of(ids, offsets); + + CATCH_SECTION("Representative rule: tie to lower doc id, untouched cluster is NO_REP") { + auto router = + GemRouter::build(hand_coarse_of_fine(), /*num_coarse=*/4, docs, size_t{1}); + CATCH_REQUIRE(router.num_fine() == 6); + CATCH_REQUIRE(router.num_coarse() == 4); + CATCH_REQUIRE(router.representative[0] == 1); + CATCH_REQUIRE(router.representative[1] == 2); + CATCH_REQUIRE(router.representative[2] == 3); // tie broken to the lower doc id + CATCH_REQUIRE(router.representative[3] == NO_REP); // untouched cluster + } + + CATCH_SECTION("Out-of-range coarse id is rejected") { + auto bad = hand_coarse_of_fine(); + bad[2] = 99; // out of [0, num_coarse) + CATCH_REQUIRE_THROWS(GemRouter::build(bad, 4, docs, size_t{1})); + } + + CATCH_SECTION("Result is independent of thread count") { + constexpr size_t k = 24; + constexpr size_t num_coarse = 6; + std::vector random_offsets; + auto random_id_list = random_ids(400, k, random_offsets); + auto random_docs = view_of(random_id_list, random_offsets); + + std::vector coarse_of_fine(k); + std::mt19937 rng(0xFEED); + std::uniform_int_distribution cluster_dist(0, num_coarse - 1); + for (auto& c : coarse_of_fine) { + c = cluster_dist(rng); + } + + auto router_1 = + GemRouter::build(coarse_of_fine, num_coarse, random_docs, size_t{1}); + auto router_4 = + GemRouter::build(coarse_of_fine, num_coarse, random_docs, size_t{4}); + CATCH_REQUIRE(router_1.representative == router_4.representative); + CATCH_REQUIRE(router_1.coarse_of_fine == router_4.coarse_of_fine); + } +} + +// Covers route_query's whole spec at once (nprobe cap, NO_REP skip, dedup across tokens, +// tie-breaking to the lower cluster id): any hand-picked case is a strict instance of it. +CATCH_TEST_CASE( + "GemRouter route_query matches a brute-force reference", "[distance][gem][gem_router]" +) { + constexpr size_t k = 20; + constexpr size_t num_coarse = 5; + constexpr size_t n_query = 6; + constexpr size_t nprobe = 2; + + std::mt19937 rng(0xABCDEF); + std::uniform_int_distribution cluster_dist(0, num_coarse - 1); + std::uniform_real_distribution score_dist(-5.0F, 5.0F); + std::uniform_int_distribution rep_dist(0, 99); + std::bernoulli_distribution has_rep(0.7); // some clusters get NO_REP + + GemRouter router; + router.coarse_of_fine.resize(k); + for (auto& c : router.coarse_of_fine) { + c = cluster_dist(rng); + } + router.representative.resize(num_coarse); + for (auto& r : router.representative) { + r = has_rep(rng) ? rep_dist(rng) : NO_REP; + } + router.nprobe = nprobe; + + std::vector table(k * n_query); // table[f * n_query + i] + for (auto& v : table) { + v = score_dist(rng); + } + + GemRouter::QueryScratch scratch; + std::vector out; + router.route_query(table.data(), n_query, out, scratch); + + // Independent reference implementation of the same spec. + std::vector expected; + for (size_t i = 0; i < n_query; ++i) { + std::vector coarse_score( + num_coarse, -std::numeric_limits::infinity() + ); + for (size_t f = 0; f < k; ++f) { + coarse_score[router.coarse_of_fine[f]] = + std::max(coarse_score[router.coarse_of_fine[f]], table[f * n_query + i]); + } + std::vector order(num_coarse); + for (size_t c = 0; c < num_coarse; ++c) { + order[c] = static_cast(c); + } + std::sort(order.begin(), order.end(), [&](uint32_t a, uint32_t b) { + if (coarse_score[a] != coarse_score[b]) { + return coarse_score[a] > coarse_score[b]; + } + return a < b; + }); + for (size_t j = 0; j < std::min(nprobe, num_coarse); ++j) { + uint32_t rep = router.representative[order[j]]; + if (rep != NO_REP) { + expected.push_back(rep); + } + } + } + std::sort(expected.begin(), expected.end()); + expected.erase(std::unique(expected.begin(), expected.end()), expected.end()); + + CATCH_REQUIRE(out == expected); + CATCH_REQUIRE(std::is_sorted(out.begin(), out.end())); + CATCH_REQUIRE(std::adjacent_find(out.begin(), out.end()) == out.end()); +} + +CATCH_TEST_CASE("GemRouter route_ids", "[distance][gem][gem_router]") { + GemRouter router; + router.coarse_of_fine = {0, 0, 1, 1, 2, 2}; + router.representative = {100, 200, 300}; + GemRouter::IdScratch scratch; + std::vector out; + + CATCH_SECTION("Ranked by count desc, ties to the lower cluster id") { + std::vector ids{0, 0, 2, 4}; + router.route_ids(ids, 2, out, scratch); + CATCH_REQUIRE(out == std::vector{100, 200}); + + router.route_ids(ids, 3, out, scratch); + CATCH_REQUIRE(out == std::vector{100, 200, 300}); + } + + CATCH_SECTION("no_representative clusters are skipped") { + router.representative = {100, NO_REP, 300}; + std::vector ids{2, 3}; // both tokens map to coarse1 + router.route_ids(ids, 5, out, scratch); + CATCH_REQUIRE(out.empty()); + } +} + +CATCH_TEST_CASE("GemRouter save/load", "[distance][gem][gem_router]") { + svs_test::prepare_temp_directory(); + auto path = svs_test::temp_directory() / "gem_router.bin"; + + GemRouter router; + router.coarse_of_fine = hand_coarse_of_fine(); + router.representative = {1, 2, 3, NO_REP}; + router.nprobe = 7; + router.save(path); + + auto loaded = GemRouter::load(path); + CATCH_REQUIRE(loaded.coarse_of_fine == router.coarse_of_fine); + CATCH_REQUIRE(loaded.representative == router.representative); + CATCH_REQUIRE(loaded.nprobe == router.nprobe); + + CATCH_SECTION("A corrupted file is rejected") { + // Flip a byte inside the magic field. + std::fstream f(path, std::ios::binary | std::ios::in | std::ios::out); + char byte = 0; + f.read(&byte, 1); + f.seekp(0); + byte = static_cast(~byte); + f.write(&byte, 1); + f.close(); + CATCH_REQUIRE_THROWS(GemRouter::load(path)); + } +} diff --git a/tests/svs/core/distances/inner_product_matrix.cpp b/tests/svs/core/distances/inner_product_matrix.cpp new file mode 100644 index 000000000..7427ef55c --- /dev/null +++ b/tests/svs/core/distances/inner_product_matrix.cpp @@ -0,0 +1,637 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test +#include "svs/core/distance/inner_product_matrix.h" + +// svs +#include "svs/lib/avx_detection.h" +#include "svs/lib/bfloat16.h" +#include "svs/lib/float16.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stdlib +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// POSIX, guarded: the guard-page test below needs mmap/mprotect/munmap, which this file +// must still compile without on a non-POSIX target. +#if defined(__unix__) || defined(__APPLE__) +#define SVS_TEST_HAS_MMAP 1 +#include +#include +#else +#define SVS_TEST_HAS_MMAP 0 +#endif + +namespace { + +using svs::distance::AVX_AVAILABILITY; +using svs::distance::InnerProductMatrix; +using svs::distance::InnerProductMatrixImpl; +using svs::distance::PackedOperand; + +// float32 tolerance: summation order differs from the SIMD kernels, not just ULP +// rounding. Float16/BFloat16/AMX reorder further still, needing ~2x that. +constexpr double REL_EPS = 1e-4; +constexpr double ABS_EPS = 1e-5; +constexpr double NARROW_REL_EPS = 2e-4; +constexpr double NARROW_ABS_EPS = 1e-4; + +std::vector random_matrix(size_t rows, size_t d, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution dist(-1.0F, 1.0F); + std::vector out(rows * d); + for (auto& v : out) { + v = dist(rng); + } + return out; +} + +// Converts float32 to A's element type as the kernel would see it: BFloat16 rounds +// ties-to-even (not the truncating constructor); float/Float16 use their constructor. +template std::vector to_type(const std::vector& a) { + std::vector out(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + if constexpr (std::is_same_v) { + out[i] = svs::round_to_bfloat16(a[i]); + } else { + out[i] = T(a[i]); + } + } + return out; +} + +// Promotes any of A's element types to double via each type's lossless `operator float`. +template std::vector promote_row(const std::vector& a) { + std::vector out(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + out[i] = static_cast(static_cast(a[i])); + } + return out; +} + +// Rounds B to bf16 and back to float32: the AMX/bf16 kernels see bf16(b), so a +// reference against pre-rounding b would be wrong by rounding error, not ordering. +std::vector to_bfloat16_as_float(const std::vector& b) { + std::vector out(b.size()); + for (size_t i = 0; i < b.size(); ++i) { + out[i] = static_cast(svs::round_to_bfloat16(b[i])); + } + return out; +} + +// out[i * n + j] = , over an already-promoted `a` so a narrow-type A's +// post-conversion values are what the reference uses. +std::vector reference_from_values( + const std::vector& a, size_t m, const std::vector& b, size_t n, size_t d +) { + std::vector out(m * n, 0.0); + for (size_t i = 0; i < m; ++i) { + for (size_t j = 0; j < n; ++j) { + double acc = 0.0; + for (size_t k = 0; k < d; ++k) { + acc += a[i * d + k] * static_cast(b[j * d + k]); + } + out[i * n + j] = acc; + } + } + return out; +} + +// out[j] = max over i of , or -infinity if m == 0. +std::vector reference_column_max( + const std::vector& a, size_t m, const std::vector& b, size_t n, size_t d +) { + std::vector out(n, -std::numeric_limits::infinity()); + for (size_t i = 0; i < m; ++i) { + for (size_t j = 0; j < n; ++j) { + double acc = 0.0; + for (size_t k = 0; k < d; ++k) { + acc += a[i * d + k] * static_cast(b[j * d + k]); + } + out[j] = std::max(out[j], acc); + } + } + return out; +} + +// `expected[i] == -inf` requires `got[i]` exactly -inf rather than Approx-equal +// (column_max's empty-input value); table-form callers never hit that case. +void require_close( + const std::vector& got, + const std::vector& expected, + double rel = REL_EPS, + double abs_margin = ABS_EPS +) { + CATCH_REQUIRE(got.size() == expected.size()); + for (size_t i = 0; i < got.size(); ++i) { + if (expected[i] == -std::numeric_limits::infinity()) { + CATCH_REQUIRE(got[i] == -std::numeric_limits::infinity()); + } else { + CATCH_REQUIRE( + static_cast(got[i]) == + Catch::Approx(expected[i]).epsilon(rel).margin(abs_margin) + ); + } + } +} + +// Shapes hitting every tail class NONE/AVX2/AVX512 need: m crosses the empty (0) and +// 8-row block boundary; n crosses the 16-lane boundary; d crosses the 16-lane boundary. +const std::vector ms{0, 1, 2, 7, 8, 9, 17, 67, 180}; +const std::vector ns{1, 15, 16, 17, 32, 33}; +const std::vector ds{1, 7, 128, 129}; + +// Checks build_packed and column_max for A held as T, at ISA level Avx, against a double +// reference and, for Avx != NONE, against the generic kernel. +template +void check_kernels( + const std::vector& a, const std::vector& b, size_t m, size_t n, size_t d +) { + auto a_t = to_type(a); + PackedOperand packed; + packed.pack(b.data(), n, d); + constexpr bool narrow = !std::is_same_v; + double rel = narrow ? NARROW_REL_EPS : REL_EPS; + double abs_margin = narrow ? NARROW_ABS_EPS : ABS_EPS; + + std::vector table(m * n, -1.0F), col_max(n, -3.0F); + InnerProductMatrixImpl::build_packed(a_t.data(), m, packed, table.data()); + InnerProductMatrixImpl::column_max(a_t.data(), m, packed, col_max.data()); + require_close( + table, reference_from_values(promote_row(a_t), m, b, n, d), rel, abs_margin + ); + require_close( + col_max, reference_column_max(promote_row(a_t), m, b, n, d), rel, abs_margin + ); + + if constexpr (Avx != AVX_AVAILABILITY::NONE) { + std::vector generic_table(m * n, -1.0F), generic_col_max(n, -3.0F); + InnerProductMatrixImpl::build_packed( + a_t.data(), m, packed, generic_table.data() + ); + InnerProductMatrixImpl::column_max( + a_t.data(), m, packed, generic_col_max.data() + ); + require_close(table, promote_row(generic_table), rel, abs_margin); + require_close(col_max, promote_row(generic_col_max), rel, abs_margin); + } +} + +template void check_level() { + unsigned seed = 0; + for (size_t d : ds) { + for (size_t m : ms) { + for (size_t n : ns) { + auto a = random_matrix(m, d, seed++); + auto b = random_matrix(n, d, seed++); + check_kernels(a, b, m, n, d); + check_kernels(a, b, m, n, d); + check_kernels(a, b, m, n, d); + } + } + } +} + +} // namespace + +CATCH_TEST_CASE( + "InnerProductMatrixImpl build_packed and column_max match a double reference, and " + "the generic kernel, for every A type and available non-AMX ISA", + "[distance][inner_product_matrix]" +) { + check_level(); + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + check_level(); + } + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + check_level(); + } +} + +namespace { + +// Every tail an AMX tile hits: m/n cross the 16-wide tile/lane boundary, d crosses the +// 32-element (16-pair) k-chunk boundary and both parities. +const std::vector amx_ms{0, 1, 15, 16, 17, 33, 67, 180}; +const std::vector amx_ns{1, 15, 16, 17, 32, 33}; +const std::vector amx_ds{1, 2, 31, 32, 33, 64, 127, 128, 129}; + +// Bit-exact comparison for one (A type, shape): AMX forwards to AVX-512 with no Approx +// tolerance whenever it isn't running its own bf16 tile kernel. +template +void require_amx_matches_avx512( + const T* a, size_t m, const PackedOperand& packed, size_t n +) { + std::vector table_amx(m * n, -1.0F), table_avx512(m * n, -2.0F); + InnerProductMatrixImpl::build_packed( + a, m, packed, table_amx.data() + ); + InnerProductMatrixImpl::build_packed( + a, m, packed, table_avx512.data() + ); + CATCH_REQUIRE(table_amx == table_avx512); + + std::vector cm_amx(n, -1.0F), cm_avx512(n, -2.0F); + InnerProductMatrixImpl::column_max(a, m, packed, cm_amx.data()); + InnerProductMatrixImpl::column_max( + a, m, packed, cm_avx512.data() + ); + CATCH_REQUIRE(cm_amx == cm_avx512); +} + +} // namespace + +// Every InnerProductMatrixImpl overload besides BFloat16's build_packed/column_max +// is an unconditional one-line forward to AVX-512 with no branch: float32 stands in for +// Float16 too, since both overloads have that identical shape. BFloat16's build_packed/ +// column_max branch on an early m==0/n==0 return and on has_tile_layout(); B is plain- +// packed below, so this covers the early return and the non-tile branch, both distinct +// from the float32/Float16 case. The small, fixed shape set is deliberate: none of these +// branches depend on shape beyond m == 0 or n == 0. +CATCH_TEST_CASE( + "InnerProductMatrixImpl forwards to AVX-512 bit for bit except for its own " + "bf16 tile kernel, which matches a double reference and AVX-512", + "[distance][inner_product_matrix][amx]" +) { + unsigned seed = 20000; + for (size_t m : {0UL, 1UL, 9UL, 33UL}) { + for (size_t n : {0UL, 1UL, 17UL, 33UL}) { + size_t d = 7; + auto a = random_matrix(m, d, seed++); + auto a_bf16 = to_type(a); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + + require_amx_matches_avx512(a.data(), m, packed, n); + require_amx_matches_avx512(a_bf16.data(), m, packed, n); + + std::vector build_amx(m * n, -1.0F), build_avx512(m * n, -2.0F); + InnerProductMatrixImpl::build( + a.data(), m, b.data(), n, d, build_amx.data() + ); + InnerProductMatrixImpl::build( + a.data(), m, b.data(), n, d, build_avx512.data() + ); + CATCH_REQUIRE(build_amx == build_avx512); + } + } + + // AMX's own bf16 tile kernel: reached only when B is packed with + // pack_rounded_to_bf16 and tile hardware is available. This is the one branch in + // the AMX code that isn't a plain forward, so it gets the full tail-shape sweep. + if (!svs::detail::amx_tile_bf16_available()) { + return; + } + seed = 90000; + for (size_t d : amx_ds) { + for (size_t m : amx_ms) { + for (size_t n : amx_ns) { + auto a_bf16 = to_type(random_matrix(m, d, seed++)); + auto b = random_matrix(n, d, seed++); + auto b_rounded = to_bfloat16_as_float(b); + PackedOperand packed; + packed.pack_rounded_to_bf16(b.data(), n, d); + CATCH_REQUIRE(packed.has_tile_layout()); + + std::vector table_amx(m * n, -1.0F), table_avx512(m * n, -2.0F); + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, table_amx.data() + ); + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, table_avx512.data() + ); + require_close( + table_amx, + reference_from_values(promote_row(a_bf16), m, b_rounded, n, d), + NARROW_REL_EPS, + NARROW_ABS_EPS + ); + require_close( + table_amx, promote_row(table_avx512), NARROW_REL_EPS, NARROW_ABS_EPS + ); + + std::vector cm_amx(n, -3.0F), cm_avx512(n, -4.0F); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, cm_amx.data() + ); + InnerProductMatrixImpl::column_max( + a_bf16.data(), m, packed, cm_avx512.data() + ); + require_close( + cm_amx, + reference_column_max(promote_row(a_bf16), m, b_rounded, n, d), + NARROW_REL_EPS, + NARROW_ABS_EPS + ); + require_close( + cm_amx, promote_row(cm_avx512), NARROW_REL_EPS, NARROW_ABS_EPS + ); + } + } + } +} + +CATCH_TEST_CASE( + "InnerProductMatrix degenerate shapes leave `out` untouched", + "[distance][inner_product_matrix]" +) { + constexpr size_t d = 4; + auto a = random_matrix(3, d, 7); + auto b = random_matrix(3, d, 11); + for (auto [m, n] : std::vector>{{0, 3}, {3, 0}}) { + std::vector out{1.0F, 2.0F, 3.0F}; + auto before = out; + InnerProductMatrix::compute(a.data(), m, b.data(), n, d, out.data()); + CATCH_REQUIRE(out == before); + } +} + +namespace { + +// Picks the ISA compute()/compute_packed() would pick: select_isa() is private, so this +// repeats its priority order against the public avx_runtime_flags. +template void with_highest_isa(F&& f) { + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + f(std::integral_constant{}); + } else if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + f(std::integral_constant{}); + } else { + f(std::integral_constant{}); + } +} + +} // namespace + +// compute_packed's BFloat16 and Float16 overloads share one dispatch call, unconditional +// on T, so BFloat16 alone exercises that dispatch; Float16 would hit the identical branch. +CATCH_TEST_CASE( + "InnerProductMatrix's public entry points equal the reference and pick the same ISA " + "result as InnerProductMatrixImpl, for float32 and BFloat16 rows", + "[distance][inner_product_matrix]" +) { + unsigned seed = 5000; + for (size_t d : {1UL, 7UL, 128UL}) { + for (size_t m : {0UL, 1UL, 9UL, 33UL}) { + for (size_t n : {1UL, 17UL, 33UL}) { + auto a = random_matrix(m, d, seed++); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + packed.pack(b.data(), n, d); + + std::vector via_build(m * n, -1.0F); + InnerProductMatrix::compute(a.data(), m, b.data(), n, d, via_build.data()); + require_close(via_build, reference_from_values(promote_row(a), m, b, n, d)); + + std::vector via_packed(m * n, -2.0F); + with_highest_isa([&](auto avx_c) { + InnerProductMatrixImpl::build_packed( + a.data(), m, packed, via_packed.data() + ); + }); + CATCH_REQUIRE(via_build == via_packed); + + auto a_bf16 = to_type(a); + std::vector via_public(m * n, -1.0F); + InnerProductMatrix::compute_packed( + a_bf16.data(), m, packed, via_public.data() + ); + std::vector via_direct(m * n, -2.0F); + with_highest_isa([&](auto avx_c) { + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, via_direct.data() + ); + }); + CATCH_REQUIRE(via_public == via_direct); + } + } + } +} + +CATCH_TEST_CASE( + "PackedOperand::pack_rounded_to_bf16 rounds to nearest, ties to even", + "[distance][inner_product_matrix]" +) { + CATCH_SECTION("Matches svs::round_to_bfloat16 over representative values") { + // Exact bf16 value, round-down, round-up, zeros of both signs, and the largest + // finite float32 (whose rounding overflows to bf16 infinity, exercised below). + std::vector values{ + 1.0F, + 0.1F, + -0.1F, + 0.0F, + -0.0F, + 100.25F, + -100.25F, + std::numeric_limits::max(), + -std::numeric_limits::max(), + }; + PackedOperand packed; + packed.pack_rounded_to_bf16(values.data(), values.size(), 1); + for (size_t i = 0; i < values.size(); ++i) { + float expected = static_cast(svs::round_to_bfloat16(values[i])); + CATCH_REQUIRE(packed.data()[i] == expected); + } + } + + CATCH_SECTION("Exact halfway cases round to the even neighbor, checked bit for bit") { + // Both inputs discard exactly bit 0x8000, bf16's halfway point: the first's kept + // bits are already even and must stay, the second's are odd and must round up. + float tie_even_stays = std::bit_cast(uint32_t{0x3F808000}); + float tie_odd_rounds_up = std::bit_cast(uint32_t{0x3F818000}); + std::vector values{tie_even_stays, tie_odd_rounds_up}; + PackedOperand packed; + packed.pack_rounded_to_bf16(values.data(), values.size(), 1); + CATCH_REQUIRE(std::bit_cast(packed.data()[0]) == uint32_t{0x3F800000}); + CATCH_REQUIRE(std::bit_cast(packed.data()[1]) == uint32_t{0x3F820000}); + } + + CATCH_SECTION("Rounding float32's max overflows to signed infinity") { + std::vector values{ + std::numeric_limits::max(), -std::numeric_limits::max()}; + PackedOperand packed; + packed.pack_rounded_to_bf16(values.data(), values.size(), 1); + CATCH_REQUIRE(packed.data()[0] == std::numeric_limits::infinity()); + CATCH_REQUIRE(packed.data()[1] == -std::numeric_limits::infinity()); + } +} + +CATCH_TEST_CASE( + "AMX tile permission detection is thread-safe", "[distance][inner_product_matrix][amx]" +) { + CATCH_SECTION("amx_tile_bf16_available is consistent across concurrent threads") { + constexpr size_t num_threads = 8; + std::vector threads; + std::array results{}; + for (size_t t = 0; t < num_threads; ++t) { + threads.emplace_back([&results, t] { + results[t] = svs::detail::amx_tile_bf16_available(); + }); + } + for (auto& th : threads) { + th.join(); + } + for (bool r : results) { + CATCH_REQUIRE(r == results[0]); + } + } + + // A worker that never calls amx_tile_bf16_available() itself (as SVS's thread pool + // doesn't) must still get correct AMX results (see fix_argument_decoded in gem.h). + CATCH_SECTION("A thread started before permission is requested can still run AMX") { + if (!svs::detail::amx_tile_bf16_available()) { + return; + } + size_t m = 17, n = 17, d = 70; + auto a_bf16 = to_type(random_matrix(m, d, 12000)); + auto b = random_matrix(n, d, 12001); + auto b_rounded = to_bfloat16_as_float(b); + PackedOperand packed; + packed.pack_rounded_to_bf16(b.data(), n, d); + + std::vector out(m * n, -1.0F); + std::thread([&] { + InnerProductMatrixImpl::build_packed( + a_bf16.data(), m, packed, out.data() + ); + }).join(); + require_close( + out, + reference_from_values(promote_row(a_bf16), m, b_rounded, n, d), + NARROW_REL_EPS, + NARROW_ABS_EPS + ); + } +} + +#if SVS_TEST_HAS_MMAP + +namespace { + +// RAII mmap guard: `bytes` of usable storage plus one neighbouring PROT_NONE page; +// `at_end` guards after the usable region (catches overrun), else before it (underrun). +// An out-of-bounds kernel access segfaults, which is this test's failure signal. +class Guarded { + public: + Guarded(size_t bytes, bool at_end) { + size_t page = static_cast(sysconf(_SC_PAGESIZE)); + size_t pages = (bytes + page - 1) / page; + total_ = (pages + 1) * page; + base_ = mmap( + nullptr, total_, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0 + ); + CATCH_REQUIRE(base_ != MAP_FAILED); + char* guard_page = static_cast(base_) + (at_end ? pages * page : 0); + CATCH_REQUIRE(mprotect(guard_page, page, PROT_NONE) == 0); + ptr_ = at_end ? guard_page - bytes : static_cast(base_) + page; + } + Guarded(const Guarded&) = delete; + Guarded& operator=(const Guarded&) = delete; + ~Guarded() { munmap(base_, total_); } + void* data() const { return ptr_; } + + private: + void* base_; + size_t total_; + void* ptr_; +}; + +const std::vector guard_ms{1, 15, 16, 17, 33}; +const std::vector guard_ns{1, 15, 16, 17, 33}; +const std::vector guard_ds{1, 2, 31, 32, 33, 127, 128, 129}; + +// Runs build_packed and column_max for one (level, type) pair over every guard shape, +// with A and `out` placed at a guard boundary in turn so a one-element over/underrun +// faults. BFloat16 packs B rounded, matching every real caller of a bf16 A; the rest +// pack plain. +template void run_guarded_level() { + unsigned seed = 0; + for (size_t d : guard_ds) { + for (size_t m : guard_ms) { + for (size_t n : guard_ns) { + auto a_src = random_matrix(m, d, seed++); + auto a_host = to_type(a_src); + auto b = random_matrix(n, d, seed++); + PackedOperand packed; + if constexpr (std::is_same_v) { + packed.pack_rounded_to_bf16(b.data(), n, d); + } else { + packed.pack(b.data(), n, d); + } + + for (bool at_end : {true, false}) { + Guarded a_guard(m * d * sizeof(T), at_end); + std::memcpy(a_guard.data(), a_host.data(), m * d * sizeof(T)); + const T* a = static_cast(a_guard.data()); + std::vector table_out(m * n, 0.0F), cm_out(n, 0.0F); + InnerProductMatrixImpl::build_packed( + a, m, packed, table_out.data() + ); + InnerProductMatrixImpl::column_max(a, m, packed, cm_out.data()); + } + Guarded table_guard(m * n * sizeof(float), true); + InnerProductMatrixImpl::build_packed( + a_host.data(), m, packed, static_cast(table_guard.data()) + ); + Guarded cm_guard(n * sizeof(float), true); + InnerProductMatrixImpl::column_max( + a_host.data(), m, packed, static_cast(cm_guard.data()) + ); + } + } + } +} + +} // namespace + +// The AMX tile kernels copy A's tail through a zero-padded stack buffer to avoid reading +// past A's own allocation (PackedOperand::fill_tile_layout); a violation segfaults. +CATCH_TEST_CASE( + "InnerProductMatrixImpl kernels never read or write outside A or out, including " + "their masked tails", + "[distance][inner_product_matrix][amx]" +) { + if (svs::detail::amx_tile_bf16_available()) { + run_guarded_level(); + } + if (svs::detail::avx_runtime_flags.is_avx512f_supported()) { + run_guarded_level(); + run_guarded_level(); + run_guarded_level(); + } + if (svs::detail::avx_runtime_flags.is_avx2_supported()) { + run_guarded_level(); + run_guarded_level(); + run_guarded_level(); + } +} + +#endif // SVS_TEST_HAS_MMAP diff --git a/tests/svs/core/distances/maxsim.cpp b/tests/svs/core/distances/maxsim.cpp new file mode 100644 index 000000000..b3aa6af33 --- /dev/null +++ b/tests/svs/core/distances/maxsim.cpp @@ -0,0 +1,251 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test +#include "svs/core/distance/maxsim.h" + +// svs +#include "svs/concepts/distance.h" +#include "svs/lib/bfloat16.h" +#include "svs/lib/float16.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stdlib +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// Minimal svs::data::ContiguousVectorSet model: a flat, row-major buffer of `cardinality` +// tokens of `dim` elements, since MaxSim's packed-operand kernel needs one contiguous +// block. +template struct TestVectorSet { + using element_type = T; + std::vector flat; + size_t dim = 0; + + size_t cardinality() const { return dim == 0 ? 0 : flat.size() / dim; } + size_t dimensions() const { return dim; } + std::span token(size_t i) const { + return std::span(flat.data() + i * dim, dim); + } + const T* data() const { return flat.data(); } +}; + +template +TestVectorSet make_set(std::initializer_list> tokens) { + TestVectorSet set; + set.dim = tokens.begin()->size(); + for (const auto& token : tokens) { + set.flat.insert(set.flat.end(), token.begin(), token.end()); + } + return set; +} + +// Same tokens as `make_set`, converting each float literal to T via T's constructor, so a +// test case can write one literal shape and run it against several document types. +template +TestVectorSet make_doc(std::initializer_list> tokens) { + TestVectorSet set; + set.dim = tokens.begin()->size(); + for (const auto& token : tokens) { + for (float v : token) { + set.flat.push_back(T(v)); + } + } + return set; +} + +template double promote(T x) { return static_cast(x); } + +// Namespace-scope (not a function-local requires-expression) so a substitution failure +// inside MaxSim::compute's requires-clause stays in the immediate context. +template +concept CanComputeMaxSim = + requires(svs::distance::MaxSim& dist, const D& d) { dist.compute(0, d); }; + +// Naive double-precision oracle: sum over query tokens of the max inner product to any +// document token. Rounds the query to bf16 first for a BFloat16 document, since +// fix_argument does. +template +double +maxsim_reference(const TestVectorSet& query, const TestVectorSet& document) { + if (document.cardinality() == 0) { + return -std::numeric_limits::infinity(); + } + double total = 0.0; + for (size_t i = 0; i < query.cardinality(); ++i) { + auto q_token = query.token(i); + double best = -std::numeric_limits::infinity(); + for (size_t j = 0; j < document.cardinality(); ++j) { + auto d_token = document.token(j); + double dot = 0.0; + for (size_t d = 0; d < query.dimensions(); ++d) { + double qv = promote(q_token[d]); + if constexpr (std::is_same_v) { + qv = promote(svs::round_to_bfloat16(q_token[d])); + } + dot += qv * promote(d_token[d]); + } + best = std::max(best, dot); + } + total += best; + } + return total; +} + +// Random query/document generator shared by every element type: BFloat16 rounds +// ties-to-even, float stores the draw unrounded. +template +TestVectorSet make_random_set(size_t cardinality, size_t dim, std::mt19937& rng) { + std::uniform_real_distribution dist(-1.0F, 1.0F); + TestVectorSet set; + set.dim = dim; + set.flat.resize(cardinality * dim); + for (auto& v : set.flat) { + float f = dist(rng); + if constexpr (std::is_same_v) { + v = svs::round_to_bfloat16(f); + } else { + v = T(f); + } + } + return set; +} + +// Packs `query` into a fresh MaxSim functor, through the generic maybe_fix_argument hook +// (the protocol every ShouldFix-opted-in distance goes through), then scores `document`. +template +float maxsim_score(const TestVectorSet& query, const D& document) { + svs::distance::MaxSim dist{}; + svs::distance::maybe_fix_argument(dist, query); + return svs::distance::compute(dist, query, document); +} + +constexpr double MAXSIM_EPSILON = 1e-4; + +template void check_oracle_on_random_sets(std::mt19937& rng) { + std::vector cardinalities{1, 2, 3, 7, 9, 17}; + std::vector dims{128, 37}; + for (size_t dim : dims) { + for (size_t nq : cardinalities) { + for (size_t nd : cardinalities) { + auto query = make_random_set(nq, dim, rng); + auto document = make_random_set(nd, dim, rng); + double expected = maxsim_reference(query, document); + float actual = maxsim_score(query, document); + CATCH_REQUIRE( + static_cast(actual) == + Catch::Approx(expected).epsilon(MAXSIM_EPSILON).margin(MAXSIM_EPSILON) + ); + } + } + } +} + +} // namespace + +CATCH_TEST_CASE("MaxSim distance", "[distance][maxsim]") { + CATCH_SECTION("Traits and compile-time document-element-type check") { + CATCH_REQUIRE(std::is_same_v::compare, std::greater<>>); + CATCH_REQUIRE_FALSE(svs::distance::MaxSim<>::implicit_broadcast); + CATCH_REQUIRE(svs::distance::MaxSim<>::must_fix_argument); + // A mismatch is a compile error (the `requires` clause on `compute`), not a + // run-time branch. + static_assert(CanComputeMaxSim>); + static_assert(!CanComputeMaxSim>); + static_assert(CanComputeMaxSim>); + static_assert(!CanComputeMaxSim>); + } + + // The n_doc == 0 check is one branch in MaxSim::compute, shared by every document + // element type; float exercises it. + CATCH_SECTION("Empty document set gives -inf") { + auto query = make_set({{1.0F, 0.0F}}); + TestVectorSet document; + document.dim = 2; + CATCH_REQUIRE( + maxsim_score(query, document) == -std::numeric_limits::infinity() + ); + } + + CATCH_SECTION("Asymmetry") { + // `a` has two tokens that each max out against `b`'s lone token, so swapping + // query/document sums over a different number of maxima (2 vs. 1). + auto a = make_set({{1.0F, 0.0F}, {1.0F, 0.0F}}); + auto b = make_set({{1.0F, 0.0F}}); + float score_ab = maxsim_score(a, b); + float score_ba = maxsim_score(b, a); + CATCH_REQUIRE(score_ab != Catch::Approx(score_ba)); + } + + CATCH_SECTION("Exact oracle on random sets, float document") { + std::mt19937 rng(0xC0FFEE); + check_oracle_on_random_sets(rng); + } + + CATCH_SECTION("Exact oracle on random sets, bfloat16 document, rounded query") { + std::mt19937 rng(0xB10A7); + check_oracle_on_random_sets(rng); + } + + CATCH_SECTION("Mixed element types") { + // float query against Float16 document: token 0 of query matches document token + // 0 exactly (dot = 1), token 1 has dot 0 with both document tokens. + auto query = make_set({{1.0F, 0.0F, 0.0F}, {0.0F, 1.0F, 0.0F}}); + auto document = make_doc({{1.0F, 0.0F, 0.0F}, {0.0F, 0.0F, 1.0F}}); + float score = maxsim_score(query, document); + CATCH_REQUIRE(score == Catch::Approx(1.0F).epsilon(MAXSIM_EPSILON)); + } + + // The dimension check is one branch in MaxSim::compute, shared by every document + // element type; float exercises it. + CATCH_SECTION("Dimension mismatch throws") { + auto query = make_set({{1.0F, 0.0F, 0.0F}}); + auto document = make_set({{1.0F, 0.0F}}); + svs::distance::MaxSim dist{}; + svs::distance::maybe_fix_argument(dist, query); + CATCH_REQUIRE_THROWS_AS( + svs::distance::compute(dist, query, document), svs::lib::ANNException + ); + } + + CATCH_SECTION("Repeated compute calls reuse one fix_argument") { + std::mt19937 rng(7); + auto query = make_random_set(5, 32, rng); + svs::distance::MaxSim dist{}; + svs::distance::maybe_fix_argument(dist, query); + for (size_t i = 0; i < 4; ++i) { + auto document = make_random_set(i + 1, 32, rng); + float direct = svs::distance::compute(dist, query, document); + float expected = maxsim_reference(query, document); + CATCH_REQUIRE( + static_cast(direct) == + Catch::Approx(expected).epsilon(MAXSIM_EPSILON).margin(MAXSIM_EPSILON) + ); + } + } +} diff --git a/tests/svs/core/kmeans_training.cpp b/tests/svs/core/kmeans_training.cpp new file mode 100644 index 000000000..12cd19b31 --- /dev/null +++ b/tests/svs/core/kmeans_training.cpp @@ -0,0 +1,383 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/core/kmeans_training.h" +#include "svs/core/kmeans_emeans.h" + +// svs +#include "svs/core/data.h" +#include "svs/core/distance.h" +#include "svs/lib/threads.h" + +// catch +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include + +namespace { + +// Brute-force nearest centroid, used as an oracle independent of centroid_assignment. +template +size_t brute_force_nearest( + const Data& data, size_t i, const svs::data::SimpleData& centroids +) { + auto datum = data.get_datum(i); + size_t best = 0; + if constexpr (std::is_same_v) { + float best_score = -std::numeric_limits::infinity(); + for (size_t j = 0; j < centroids.size(); ++j) { + auto c = centroids.get_datum(j); + float dot = 0; + for (size_t d = 0; d < datum.size(); ++d) { + dot += datum[d] * c[d]; + } + if (dot > best_score) { + best_score = dot; + best = j; + } + } + } else { + float best_dist = std::numeric_limits::infinity(); + for (size_t j = 0; j < centroids.size(); ++j) { + auto c = centroids.get_datum(j); + float dist = 0; + for (size_t d = 0; d < datum.size(); ++d) { + float diff = datum[d] - c[d]; + dist += diff * diff; + } + if (dist < best_dist) { + best_dist = dist; + best = j; + } + } + } + return best; +} + +// `batch_size` sizes the preallocated matmul_results buffer (minibatch_size), which need +// not equal data.size(). +template +auto run_kmeans( + const svs::data::SimpleData& data, + Distance distance, + svs::data::SimpleData& centroids, + size_t num_iter, + size_t batch_size, + unsigned seed +) { + namespace kmeans = svs::kmeans; + auto matmul_results = svs::data::SimpleData(batch_size, centroids.size()); + auto rng = std::mt19937(seed); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto params = kmeans::KMeansTrainingParameters{num_iter, batch_size}; + return kmeans::kmeans_training( + params, data, distance, centroids, matmul_results, rng, threadpool, timer + ); +} + +} // namespace + +CATCH_TEST_CASE("Shared Kmeans Training", "[core][kmeans]") { + CATCH_SECTION("L2 centroid update divides by count + 1, and does not normalize") { + auto data = svs::data::SimpleData(4, 2); + data.set_datum(0, std::vector{0.0F, 0.0F}); + data.set_datum(1, std::vector{0.0F, 2.0F}); + data.set_datum(2, std::vector{10.0F, 0.0F}); + data.set_datum(3, std::vector{10.0F, 2.0F}); + + auto centroids = svs::data::SimpleData(2, 2); + centroids.set_datum(0, std::vector{0.0F, 0.0F}); + centroids.set_datum(1, std::vector{10.0F, 0.0F}); + + auto result = run_kmeans(data, svs::DistanceL2(), centroids, 1, 10, 0xc0ffee); + CATCH_REQUIRE(result.size() == 2); + + auto c0 = result.get_datum(0); + auto c1 = result.get_datum(1); + constexpr float tolerance = 1e-4F; + CATCH_REQUIRE(std::abs(c0[0] - 0.0F) < tolerance); + CATCH_REQUIRE(std::abs(c0[1] - (2.0F / 3.0F)) < tolerance); + CATCH_REQUIRE(std::abs(c1[0] - (30.0F / 3.0F)) < tolerance); + CATCH_REQUIRE(std::abs(c1[1] - (2.0F / 3.0F)) < tolerance); + + float norm0 = std::sqrt(c0[0] * c0[0] + c0[1] * c0[1]); + float norm1 = std::sqrt(c1[0] * c1[0] + c1[1] * c1[1]); + CATCH_REQUIRE(std::abs(norm0 - 1.0F) > tolerance); + CATCH_REQUIRE(std::abs(norm1 - 1.0F) > tolerance); + } + + CATCH_SECTION("IP centroids come out unit-norm from non-unit-norm initial centroids") { + auto data = svs::data::SimpleData(6, 3); + data.set_datum(0, std::vector{1.0F, 2.0F, 3.0F}); + data.set_datum(1, std::vector{2.0F, 1.0F, 4.0F}); + data.set_datum(2, std::vector{3.0F, 3.0F, 1.0F}); + data.set_datum(3, std::vector{-1.0F, -2.0F, 0.5F}); + data.set_datum(4, std::vector{-2.0F, -1.0F, 1.5F}); + data.set_datum(5, std::vector{-3.0F, -0.5F, 2.0F}); + + auto centroids = svs::data::SimpleData(2, 3); + centroids.set_datum(0, data.get_datum(0)); + centroids.set_datum(1, data.get_datum(3)); + + auto result = run_kmeans(data, svs::DistanceIP(), centroids, 3, 6, 42); + + constexpr float tolerance = 1e-5F; + for (size_t i = 0; i < result.size(); ++i) { + auto c = result.get_datum(i); + float norm_sq = 0; + for (size_t d = 0; d < c.size(); ++d) { + norm_sq += c[d] * c[d]; + } + CATCH_REQUIRE(std::abs(std::sqrt(norm_sq) - 1.0F) < tolerance); + } + } + + CATCH_SECTION("centroid_assignment agrees with brute force, for L2 and IP") { + namespace kmeans = svs::kmeans; + constexpr size_t num_points = 30; + constexpr size_t num_centroids = 4; + constexpr size_t dims = 5; + + auto data = svs::data::SimpleData(num_points, dims); + for (size_t i = 0; i < num_points; ++i) { + auto datum = data.get_datum(i); + for (size_t d = 0; d < dims; ++d) { + // A deterministic, non-trivial pattern; no symmetry that creates ties. + datum[d] = static_cast((i * 7 + d * 3) % 11) - 5.0F; + } + } + + auto centroids = svs::data::SimpleData(num_centroids, dims); + for (size_t i = 0; i < num_centroids; ++i) { + auto c = centroids.get_datum(i); + for (size_t d = 0; d < dims; ++d) { + c[d] = static_cast((i * 5 + d * 2) % 7) - 3.0F; + } + } + + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto batch_range = svs::threads::UnitRange{0, num_points}; + auto matmul_results = svs::data::SimpleData(num_points, num_centroids); + + auto check_distance = [&](Distance distance) { + auto data_norm = kmeans::maybe_compute_norms(data, threadpool); + auto centroids_norm = + kmeans::maybe_compute_norms(centroids, threadpool); + auto assignments = std::vector(num_points); + + kmeans::centroid_assignment( + data, + data_norm, + batch_range, + distance, + centroids, + centroids_norm, + assignments, + matmul_results, + threadpool, + timer + ); + + for (size_t i = 0; i < num_points; ++i) { + size_t expected = brute_force_nearest(data, i, centroids); + CATCH_REQUIRE(assignments[i] == expected); + } + }; + + check_distance(svs::DistanceL2()); + check_distance(svs::DistanceIP()); + } +} + +namespace { + +// Three well-separated clusters, far enough apart that nearest-centroid assignment never +// ties and small perturbations in training never cross a cluster boundary. +svs::data::SimpleData make_three_clusters() { + auto data = svs::data::SimpleData(15, 2); + std::array, 3> centers{ + {{0.0F, 0.0F}, {100.0F, 0.0F}, {0.0F, 100.0F}}}; + for (size_t c = 0; c < 3; ++c) { + for (size_t i = 0; i < 5; ++i) { + float jitter = static_cast(i) * 0.1F; + data.set_datum( + c * 5 + i, + std::vector{centers[c][0] + jitter, centers[c][1] + jitter} + ); + } + } + return data; +} + +svs::kmeans::EMeansParameters small_emeans_parameters() { + svs::kmeans::EMeansParameters parameters; + parameters.epochs = 2; + parameters.batch_size = 3; + parameters.alpha0 = 0.5; + parameters.revival_interval = 5; + parameters.tau = 0.1; + return parameters; +} + +} // namespace + +CATCH_TEST_CASE("EMeans Training", "[core][kmeans][emeans]") { + CATCH_SECTION("determinism for a fixed seed and thread count") { + auto data = make_three_clusters(); + auto parameters = small_emeans_parameters(); + + auto run = [&]() { + auto rng = std::mt19937(0xc0ffee); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + return svs::kmeans::emeans_train( + parameters, data, size_t{3}, rng, threadpool, timer + ); + }; + + auto result_a = run(); + auto result_b = run(); + + CATCH_REQUIRE(result_a.centroids.size() == result_b.centroids.size()); + for (size_t i = 0; i < result_a.centroids.size(); ++i) { + auto a = result_a.centroids.get_datum(i); + auto b = result_b.centroids.get_datum(i); + for (size_t d = 0; d < a.size(); ++d) { + CATCH_REQUIRE(a[d] == b[d]); + } + } + CATCH_REQUIRE(result_a.objective == result_b.objective); + } + + CATCH_SECTION("a forced-dead cluster is revived") { + auto data = make_three_clusters(); + auto parameters = small_emeans_parameters(); + + // Four initial centroids: one near each real cluster, plus a fourth parked far + // from all data so it is never the nearest centroid for any point and stays dead + // (EMA count near its 1e-10 floor) until revival splits a donor into it. + auto initial = svs::data::SimpleData(4, 2); + initial.set_datum(0, std::vector{0.0F, 0.0F}); + initial.set_datum(1, std::vector{100.0F, 0.0F}); + initial.set_datum(2, std::vector{0.0F, 100.0F}); + initial.set_datum(3, std::vector{1000.0F, 1000.0F}); + + auto rng = std::mt19937(42); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto result = + svs::kmeans::emeans_train(parameters, data, initial, rng, threadpool, timer); + + // Revival splits a donor near the data into this slot; an unrevived cluster would + // still sit at (1000, 1000), about 1000 from every real cluster center. + auto final_dead = result.centroids.get_datum(3); + float nearest_center = std::numeric_limits::max(); + constexpr std::array, 3> centers{ + {{0.0F, 0.0F}, {100.0F, 0.0F}, {0.0F, 100.0F}}}; + for (auto [cx, cy] : centers) { + float dx = final_dead[0] - cx; + float dy = final_dead[1] - cy; + nearest_center = std::min(nearest_center, dx * dx + dy * dy); + } + CATCH_REQUIRE(nearest_center < 10.0F * 10.0F); + } + + CATCH_SECTION("resolve_emeans_parameters matches hand-computed scaling-law values") { + // Hand-computed (Python) from Eq. 6/Eq. 7 with the code's exact coefficients. + svs::kmeans::EMeansParameters small_n_params; + small_n_params.epochs = 2; + auto small_n = + svs::kmeans::resolve_emeans_parameters(100'000, 1'000, small_n_params); + CATCH_REQUIRE(small_n.batch_size == 4096); + CATCH_REQUIRE(std::abs(small_n.alpha0 - 0.3191081273388448) < 1e-6); + + // n > 2e6: the batch law's coefficient switches to 0.035/8. + svs::kmeans::EMeansParameters large_n_params; + large_n_params.epochs = 2; + auto large_n = + svs::kmeans::resolve_emeans_parameters(5'000'000, 1'000, large_n_params); + CATCH_REQUIRE(large_n.batch_size == 16384); + CATCH_REQUIRE(std::abs(large_n.alpha0 - 0.19109765404135304) < 1e-6); + } + + CATCH_SECTION("batch_size not dividing n exercises the tail batch") { + auto data = make_three_clusters(); // n = 15 + auto parameters = small_emeans_parameters(); + parameters.batch_size = 4; // 15 / 4 = 3 remainder 3: a tail batch of size 3 + + auto rng = std::mt19937(0xc0ffee); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + auto result = + svs::kmeans::emeans_train(parameters, data, size_t{3}, rng, threadpool, timer); + + CATCH_REQUIRE(result.centroids.size() == 3); + CATCH_REQUIRE(std::isfinite(result.objective)); + } + + // Not a general invariant: the last epoch's assignments come from moving centroids, so + // the closing mean need not dominate the EMA readout in general. Holds here because + // this toy set's clusters are so well separated that the assignment never wavers. + CATCH_SECTION( + "the final mean pass does not raise the objective (well-separated toy set)" + ) { + auto data = make_three_clusters(); + + auto with_pass_params = small_emeans_parameters(); + with_pass_params.final_mean_pass = true; + auto without_pass_params = small_emeans_parameters(); + without_pass_params.final_mean_pass = false; + + auto run = [&](const svs::kmeans::EMeansParameters& parameters) { + auto rng = std::mt19937(0xc0ffee); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + return svs::kmeans::emeans_train( + parameters, data, size_t{3}, rng, threadpool, timer + ); + }; + + auto with_pass = run(with_pass_params); + auto without_pass = run(without_pass_params); + + CATCH_REQUIRE(with_pass.objective <= without_pass.objective + 1e-6); + } + + CATCH_SECTION("k > n throws") { + auto data = svs::data::SimpleData(3, 2); + data.set_datum(0, std::vector{0.0F, 0.0F}); + data.set_datum(1, std::vector{1.0F, 0.0F}); + data.set_datum(2, std::vector{0.0F, 1.0F}); + + auto parameters = small_emeans_parameters(); + auto rng = std::mt19937(0xc0ffee); + auto threadpool = svs::threads::as_threadpool(2); + auto timer = svs::lib::Timer(); + + CATCH_REQUIRE_THROWS( + svs::kmeans::emeans_train(parameters, data, size_t{5}, rng, threadpool, timer) + ); + } +} diff --git a/tests/svs/index/vamana/build_stats.cpp b/tests/svs/index/vamana/build_stats.cpp new file mode 100644 index 000000000..1db53b122 --- /dev/null +++ b/tests/svs/index/vamana/build_stats.cpp @@ -0,0 +1,139 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test: the build-time measurement counters. +#include "svs/index/vamana/build_stats.h" + +// svs +#include "svs/core/data/simple.h" +#include "svs/core/distance.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/extensions.h" +#include "svs/index/vamana/prune.h" +#include "svs/index/vamana/vamana_build.h" +#include "svs/lib/narrow.h" +#include "svs/lib/threads.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include + +// Named (not anonymous) so `svs::index::vamana::PruneStrategy` below can name this type. +namespace build_stats_test { + +// Static atomic: DefaultBuildAdaptor default-constructs a fresh Distance per thread. +struct CountingTestDistance { + using compare = std::less<>; + static constexpr bool implicit_broadcast = true; + + template float compute(A a, B b) const { + counter().fetch_add(1, std::memory_order_relaxed); + return svs::distance::compute(svs::distance::DistanceL2{}, a, b); + } + + static std::atomic& counter() { + static std::atomic count{0}; + return count; + } +}; + +svs::data::SimpleData make_random_data(size_t n, size_t dims, uint64_t seed) { + auto data = svs::data::SimpleData(n, dims); + auto rng = std::mt19937_64(seed); + auto distribution = std::uniform_real_distribution(-1.0F, 1.0F); + auto buffer = std::vector(dims); + for (size_t i = 0; i < n; ++i) { + for (float& v : buffer) { + v = distribution(rng); + } + data.set_datum(i, buffer); + } + return data; +} + +} // namespace build_stats_test + +namespace svs::index::vamana { +// Shares QEmd's PruneStrategy (std::less<>), not IP/cosine's (extensions/vamana/gem.h). +template <> struct PruneStrategy { + using type = ProgressivePruneStrategy; +}; +} // namespace svs::index::vamana + +CATCH_TEST_CASE("Build Stats", "[vamana][build_stats]") { + namespace vamana = svs::index::vamana; + + const size_t n_points = 300; + const size_t dims = 16; + auto data = build_stats_test::make_random_data(n_points, dims, 0xB0105); + + // Small graph_max_degree vs window_size/pool_size forces backedge pruning to occur. + auto build_parameters = vamana::VamanaBuildParameters{1.2F, 8, 16, 32, 6, true}; + + auto entry_point_pool = svs::threads::DefaultThreadPool(2); + auto entry_point = svs::lib::narrow( + vamana::extensions::compute_entry_point(data, entry_point_pool) + ); + + auto graph = svs::default_graph(n_points, build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(2); + + build_stats_test::CountingTestDistance::counter() = 0; + vamana::build_stats_reset(); + + // Instantiate with CountingBuildStats explicitly, so this test exercises the real + // collector regardless of the ambient SVS_ENABLE_STATS build flag. + auto builder = vamana::VamanaBuilder< + decltype(graph), + decltype(data), + build_stats_test::CountingTestDistance, + decltype(build_pool), + vamana::CountingBuildStats>( + graph, + data, + build_stats_test::CountingTestDistance{}, + build_parameters, + build_pool, + vamana::extensions::estimate_prefetch_parameters(data) + ); + // Two full passes, mirroring VamanaIndex: alpha pinned to 1.0, then the real alpha. + builder.construct(1.0F, entry_point); + builder.construct(build_parameters.alpha, entry_point); + + auto stats = vamana::build_stats_snapshot(); + uint64_t total_compute_calls = build_stats_test::CountingTestDistance::counter().load(); + + CATCH_REQUIRE(stats.inserts == 2 * n_points); + CATCH_REQUIRE( + stats.search_distances + stats.pool_distances + stats.prune_distances + + stats.backedge_prune_distances == + total_compute_calls + ); + + CATCH_REQUIRE(stats.search_distances > 0); + CATCH_REQUIRE(stats.pool_distances > 0); + CATCH_REQUIRE(stats.prune_distances > 0); + CATCH_REQUIRE(stats.backedge_prune_distances > 0); + CATCH_REQUIRE(stats.search_ns > 0); + CATCH_REQUIRE(stats.prune_ns > 0); + CATCH_REQUIRE(stats.backedge_prune_ns > 0); + CATCH_REQUIRE(stats.max_search_distances_per_insert > 0); +} diff --git a/tests/svs/index/vamana/connectivity.cpp b/tests/svs/index/vamana/connectivity.cpp new file mode 100644 index 000000000..6c0d95da1 --- /dev/null +++ b/tests/svs/index/vamana/connectivity.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test: unreachable_nodes and repair_connectivity. +#include "svs/index/vamana/connectivity.h" + +// svs +#include "svs/core/data/simple.h" +#include "svs/core/distance/euclidean.h" +#include "svs/core/graph.h" +#include "svs/lib/threads.h" + +// tests +#include "tests/svs/index/vamana/gem_test_utils.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include + +namespace { + +using Idx = uint32_t; +namespace v = svs::index::vamana; + +svs::data::SimpleData make_positions(const std::vector& values) { + auto data = svs::data::SimpleData(values.size(), 1); + for (size_t i = 0; i < values.size(); ++i) { + data.set_datum(i, std::vector{values[i]}); + } + return data; +} + +svs::graphs::SimpleGraph build_test_graph( + size_t n_nodes, size_t max_degree, const std::vector>& adjacency +) { + auto graph = svs::default_graph(n_nodes, max_degree); + for (size_t i = 0; i < adjacency.size(); ++i) { + graph.replace_node(static_cast(i), adjacency[i]); + } + return graph; +} + +std::vector> adjacency_snapshot(const svs::graphs::SimpleGraph& graph +) { + std::vector> out(graph.n_nodes()); + for (size_t i = 0; i < graph.n_nodes(); ++i) { + auto node = graph.get_node(static_cast(i)); + out[i].assign(node.begin(), node.end()); + } + return out; +} + +// One row per repair_connectivity scenario. Fields left at their default (nullopt/empty) +// mirror an assertion the original cases didn't make either. +struct RepairCase { + std::string name{}; + size_t n_nodes{}; + std::vector> adjacency{}; + std::vector positions{}; + v::VamanaBuildParameters params{}; + size_t max_rounds = 4; + size_t unreachable_before{}; + size_t unreachable_after{}; + std::optional rounds{}; + bool rounds_is_min = false; + std::optional edges_total{}; + std::optional edges_forced_unsafe{}; + std::optional stalled{}; + std::vector exact_unreachable_ids{}; + bool check_unchanged = false; +}; + +} // namespace + +CATCH_TEST_CASE("unreachable_nodes", "[index][vamana][connectivity]") { + auto graph = build_test_graph(5, 3, {{1, 2}, {}, {3}}); + std::vector roots{0}; + CATCH_REQUIRE( + v::unreachable_nodes(graph, std::span(roots)) == std::vector{4} + ); + roots.push_back(4); + CATCH_REQUIRE(v::unreachable_nodes(graph, std::span(roots)).empty()); +} + +CATCH_TEST_CASE("repair_connectivity", "[index][vamana][connectivity]") { + std::vector cases = { + {.name = "leaves an already-connected graph untouched", + .n_nodes = 5, + .adjacency = {{1}, {2}, {3}, {4}}, + .positions = {0.0F, 1.0F, 2.0F, 3.0F, 4.0F}, + .params = v::VamanaBuildParameters{1.2F, 3, 10, 10, 3, true}, + .unreachable_before = 0, + .unreachable_after = 0, + .rounds = 0, + .edges_total = 0, + .check_unchanged = true}, + {.name = "reconnects a disconnected node with room to spare", + .n_nodes = 5, + .adjacency = {{1}, {2}, {3}}, + .positions = {0.0F, 1.0F, 2.0F, 3.0F, 2.1F}, + .params = v::VamanaBuildParameters{1.2F, 3, 10, 10, 3, true}, + .unreachable_before = 1, + .unreachable_after = 0, + .rounds = 1, + .rounds_is_min = true, + .edges_total = 1}, + {.name = "reconnects when the nearest reachable neighbors are all at full degree", + .n_nodes = 5, + .adjacency = {{1, 2}, {2, 3}, {3, 0}, {0, 1}}, + .positions = {0.0F, 1.0F, 2.0F, 3.0F, 0.1F}, + .params = v::VamanaBuildParameters{1.2F, 2, 10, 10, 2, true}, + .unreachable_before = 1, + .unreachable_after = 0, + .edges_total = 1, + .edges_forced_unsafe = 0, + .stalled = false}, + {.name = "falls back to an unsafe forced replacement when no candidate is safe", + .n_nodes = 4, + .adjacency = {{1}, {2}}, + .positions = {100.0F, 0.0F, 2.0F, -3.0F}, + .params = v::VamanaBuildParameters{1.0F, 1, 1, 10, 1, true}, + .max_rounds = 1, + .unreachable_before = 1, + .unreachable_after = 1, + .rounds = 1, + .edges_total = 0, + .edges_forced_unsafe = 1, + .stalled = true, + .exact_unreachable_ids = {2}}, + {.name = "stall guard stops an oscillating repair before max_rounds", + .n_nodes = 4, + .adjacency = {{1}, {2}}, + .positions = {100.0F, 0.0F, 2.0F, -3.0F}, + .params = v::VamanaBuildParameters{1.0F, 1, 1, 10, 1, true}, + .unreachable_before = 1, + .unreachable_after = 1, + .rounds = 1, + .edges_forced_unsafe = 1, + .stalled = true}, + }; + + for (const auto& c : cases) { + CATCH_SECTION(c.name) { + auto graph = + build_test_graph(c.n_nodes, c.params.graph_max_degree, c.adjacency); + auto before = c.check_unchanged ? adjacency_snapshot(graph) + : std::vector>{}; + auto data = make_positions(c.positions); + svs::distance::DistanceL2 distance{}; + auto pool = svs::threads::DefaultThreadPool(1); + std::vector roots{0}; + auto report = v::repair_connectivity( + graph, + data, + distance, + c.params, + std::span(roots), + pool, + c.max_rounds + ); + + CATCH_REQUIRE(report.unreachable_before == c.unreachable_before); + CATCH_REQUIRE(report.unreachable_after == c.unreachable_after); + if (c.rounds) { + if (c.rounds_is_min) { + CATCH_REQUIRE(report.rounds >= *c.rounds); + } else { + CATCH_REQUIRE(report.rounds == *c.rounds); + } + } + if (c.edges_total) { + CATCH_REQUIRE( + report.edges_appended + report.edges_accepted_through_prune + + report.edges_forced == + *c.edges_total + ); + } + if (c.edges_forced_unsafe) { + CATCH_REQUIRE(report.edges_forced_unsafe == *c.edges_forced_unsafe); + } + if (c.stalled) { + CATCH_REQUIRE(report.stalled == *c.stalled); + } + CATCH_REQUIRE( + v::unreachable_nodes(graph, std::span(roots)).size() == + report.unreachable_after + ); + if (!c.exact_unreachable_ids.empty()) { + CATCH_REQUIRE( + v::unreachable_nodes(graph, std::span(roots)) == + c.exact_unreachable_ids + ); + } + svs_test::require_degree_bound(graph, c.params.graph_max_degree); + if (c.check_unchanged) { + CATCH_REQUIRE(adjacency_snapshot(graph) == before); + } + } + } +} diff --git a/tests/svs/index/vamana/gem_build.cpp b/tests/svs/index/vamana/gem_build.cpp new file mode 100644 index 000000000..a61a7ab72 --- /dev/null +++ b/tests/svs/index/vamana/gem_build.cpp @@ -0,0 +1,170 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Headers under test: gem_build.h and the QEmd build_entry_points overload (gem.h). +#include "svs/index/vamana/gem_build.h" + +// svs +#include "svs/core/data.h" +#include "svs/core/data/vector_set.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/core/distance/gem.h" +#include "svs/core/distance/gem_router.h" +#include "svs/core/graph.h" +#include "svs/extensions/vamana/gem.h" +#include "svs/index/vamana/connectivity.h" +#include "svs/lib/threads.h" + +// tests +#include "tests/svs/index/vamana/gem_test_utils.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include + +namespace { + +using svs::distance::QEmd; +using svs_test::gem::make_centroid_table; +using svs_test::gem::make_centroids; +using svs_test::gem::make_doc_corpus; +using svs_test::gem::round_robin_coarse_of_fine; + +} // namespace + +// Covers the no-router / all-unrepresented fallback for GEM entry-point routing at the +// layer that owns it; gem_routing.cpp's GemRoutedInitializer mirrors it without repeating. +CATCH_TEST_CASE("gem_insert_order and build_entry_points", "[vamana][gem][gem_build]") { + namespace v = svs::index::vamana; + namespace ext = v::extensions; + + CATCH_SECTION("gem_insert_order dedups representatives ahead of the rest, ascending") { + svs::distance::GemRouter router; + router.coarse_of_fine = {0, 0, 1, 1, 2, 2}; + router.representative = {4, 2, svs::distance::GemRouter::no_representative, 2}; + router.nprobe = 2; + + auto order = v::gem_insert_order(router, 6); + auto sorted_order = order; + std::sort(sorted_order.begin(), sorted_order.end()); + CATCH_REQUIRE(sorted_order == std::vector{0, 1, 2, 3, 4, 5}); + CATCH_REQUIRE(order.size() >= 2); + CATCH_REQUIRE(order[0] == 2); + CATCH_REQUIRE(order[1] == 4); + CATCH_REQUIRE( + std::vector(order.begin() + 2, order.end()) == + std::vector{0, 1, 3, 5} + ); + } + + auto centroids = make_centroids(0xB001); + auto table = make_centroid_table(centroids); + auto doc_corpus = make_doc_corpus(20, 0xB002); + auto docs = doc_corpus.view(); + std::vector entry_points{3}; + std::vector scratch; + + CATCH_SECTION("No router returns entry_points unchanged") { + QEmd distance{table}; + auto result = ext::build_entry_points( + docs, distance, uint32_t{7}, std::span(entry_points), scratch + ); + CATCH_REQUIRE(std::vector(result.begin(), result.end()) == entry_points); + } + + CATCH_SECTION("Every cluster unrepresented falls back to entry_points") { + svs::distance::GemRouter all_norep; + all_norep.coarse_of_fine = round_robin_coarse_of_fine(4); + all_norep.representative.assign(4, svs::distance::GemRouter::no_representative); + all_norep.nprobe = 2; + auto router_ptr = std::make_shared(all_norep); + + QEmd distance{table, router_ptr}; + auto result = ext::build_entry_points( + docs, distance, uint32_t{7}, std::span(entry_points), scratch + ); + CATCH_REQUIRE(std::vector(result.begin(), result.end()) == entry_points); + } + + CATCH_SECTION("A real router returns entry_points union routed, minus node_id") { + constexpr size_t num_coarse = 4; + auto router = svs::distance::GemRouter::build( + round_robin_coarse_of_fine(num_coarse), num_coarse, docs, size_t{1}, size_t{2} + ); + auto router_ptr = std::make_shared(router); + QEmd distance{table, router_ptr}; + + uint32_t node_id = 7; + auto doc = docs.get_datum(node_id); + std::vector doc_ids(doc.data(), doc.data() + doc.cardinality()); + + std::vector routed; + svs::distance::GemRouter::IdScratch id_scratch; + router.route_ids(doc_ids, router.nprobe, routed, id_scratch); + + std::vector expected(entry_points.begin(), entry_points.end()); + expected.insert(expected.end(), routed.begin(), routed.end()); + std::sort(expected.begin(), expected.end()); + expected.erase(std::unique(expected.begin(), expected.end()), expected.end()); + expected.erase( + std::remove(expected.begin(), expected.end(), node_id), expected.end() + ); + + auto result = ext::build_entry_points( + docs, distance, node_id, std::span(entry_points), scratch + ); + CATCH_REQUIRE(std::vector(result.begin(), result.end()) == expected); + } +} + +CATCH_TEST_CASE("build_gem_graph", "[vamana][gem][gem_build]") { + namespace v = svs::index::vamana; + + auto centroids = make_centroids(0xB003); + auto table = make_centroid_table(centroids); + + const size_t n_docs = 300; + auto doc_corpus = make_doc_corpus(n_docs, 0xB004); + auto docs = doc_corpus.view(); + + auto params = v::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + const uint32_t entry_point = 0; + + constexpr size_t num_coarse = 8; + auto router = svs::distance::GemRouter::build( + round_robin_coarse_of_fine(num_coarse), num_coarse, docs, size_t{1}, size_t{3} + ); + auto router_ptr = std::make_shared(router); + + for (auto maybe_router : + {std::shared_ptr(nullptr), router_ptr}) { + QEmd distance{table, std::move(maybe_router)}; + auto pool = svs::threads::DefaultThreadPool(1); + auto [graph, report] = + v::build_gem_graph(docs, distance, params, pool, entry_point); + + CATCH_REQUIRE(report.documents == n_docs); + CATCH_REQUIRE(report.unreachable_after_repair == 0); + svs_test::require_degree_bound(graph, params.graph_max_degree); + } +} diff --git a/tests/svs/index/vamana/gem_routing.cpp b/tests/svs/index/vamana/gem_routing.cpp new file mode 100644 index 000000000..623ffcb48 --- /dev/null +++ b/tests/svs/index/vamana/gem_routing.cpp @@ -0,0 +1,245 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Headers under test: GemRoutedInitializer, search_initializer, gem_pooled_entry_point. +// No-router / all-unrepresented fallback is covered by gem_build.cpp's build_entry_points. +#include "svs/core/distance/gem.h" +#include "svs/core/distance/gem_router.h" +#include "svs/extensions/vamana/gem.h" + +// svs +#include "svs/core/data/vector_set.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/index.h" +#include "svs/lib/threads.h" + +// tests +#include "tests/svs/index/vamana/gem_test_utils.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using svs::distance::GemRouter; +using svs::distance::QChamfer; +using svs::distance::QChamferExactQuery; +using svs_test::gem::DIM; +using svs_test::gem::K; +using svs_test::gem::make_centroid_table; +using svs_test::gem::make_centroids; +using svs_test::gem::make_doc_corpus; +using svs_test::gem::make_fine_centroids; +using svs_test::gem::round_robin_coarse_of_fine; + +// Raw-float query tokens, one per centroid dimension: what QChamferExactQuery expects. +struct QueryCorpus { + std::vector tokens; + std::vector offsets; + + svs::data::ConstVectorSetView view() const { + return svs::data::ConstVectorSetView( + std::span(tokens), std::span(offsets), DIM + ); + } +}; + +// Noisy centroids, so routing/recall has real nearest-centroid structure to exploit. +QueryCorpus +make_query_corpus(const std::vector& centroids, size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution centroid_dist(0, static_cast(K - 1)); + std::normal_distribution noise(0.0F, 0.05F); + + QueryCorpus corpus; + corpus.offsets.resize(n + 1, 0); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 1 + (i % 3); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + for (size_t t = 0; t < cardinality; ++t) { + uint32_t c = centroid_dist(rng); + for (size_t d = 0; d < DIM; ++d) { + corpus.tokens.push_back(centroids[c * DIM + d] + noise(rng)); + } + } + } + return corpus; +} + +// Minimal greedy-search test double: records pushed ids, nothing else. +struct RecordingBuffer { + using index_type = uint32_t; + std::vector ids; + + void clear() { ids.clear(); } + void push_back(uint32_t id) { ids.push_back(id); } + void sort() { std::sort(ids.begin(), ids.end()); } +}; + +auto dummy_computer() { + return [](uint32_t /*id*/) { return 0.0F; }; +} + +auto id_builder() { + return [](uint32_t id, float /*dist*/) { return id; }; +} + +} // namespace + +CATCH_TEST_CASE( + "GemRoutedInitializer with a real router pushes exactly medoid union routed", + "[vamana][gem][gem_router]" +) { + auto centroids = make_centroids(0x5555); + auto fine = make_fine_centroids(centroids); + + auto doc_corpus = make_doc_corpus(100, 0x6666); + auto docs = doc_corpus.view(); + + constexpr size_t num_coarse = 4; + auto router = GemRouter::build( + round_robin_coarse_of_fine(num_coarse), num_coarse, docs, size_t{1}, size_t{2} + ); + auto router_ptr = std::make_shared(router); + + QChamferExactQuery distance{fine, router_ptr}; + + auto queries = make_query_corpus(centroids, 1, 0x7777); + auto query = queries.view().get_datum(0); + distance.fix_argument(query); + + std::vector entry_points{7}; + GemRouter::QueryScratch scratch; + std::vector routed; + router.route_query(distance.table(), distance.query_tokens(), routed, scratch); + + std::vector expected(entry_points.begin(), entry_points.end()); + expected.insert(expected.end(), routed.begin(), routed.end()); + std::sort(expected.begin(), expected.end()); + expected.erase(std::unique(expected.begin(), expected.end()), expected.end()); + + auto graph = svs::default_graph(100, 32); + auto computer = dummy_computer(); + auto builder = id_builder(); + svs::index::vamana::NullTracker tracker; + + RecordingBuffer buf; + svs::index::vamana::GemRoutedInitializer{ + &distance, + std::span(entry_points)}(buf, computer, graph, builder, tracker); + + CATCH_REQUIRE(buf.ids == expected); +} + +CATCH_TEST_CASE( + "Routed search is not worse than unrouted on a small index", "[vamana][gem][gem_router]" +) { + auto centroids = make_centroids(0x8888); + auto table = make_centroid_table(centroids); + auto fine = make_fine_centroids(centroids); + + const size_t n_docs = 150; + auto doc_corpus = make_doc_corpus(n_docs, 0x9999); + auto docs = doc_corpus.view(); + auto queries = make_query_corpus(centroids, 20, 0xAAAA); + auto query_view = queries.view(); + + QChamfer build_dist{table}; + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + const uint32_t entry_point = 0; + + auto graph = svs::default_graph(n_docs, build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + graph, + docs, + build_dist, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(docs) + ); + builder.construct(build_parameters.alpha, entry_point); + + constexpr size_t num_coarse = 6; + auto router = GemRouter::build( + round_robin_coarse_of_fine(num_coarse), num_coarse, docs, size_t{1}, size_t{3} + ); + auto router_ptr = std::make_shared(router); + + auto search_params = [](size_t window) { + return svs::index::vamana::VamanaSearchParameters{{window, window}, false, 0, 0}; + }; + + auto run_search = [&](std::shared_ptr maybe_router) { + QChamferExactQuery dist{fine, std::move(maybe_router)}; + auto pool = svs::threads::DefaultThreadPool(1); + auto graph_copy = graph; + auto index = svs::index::vamana::VamanaIndex( + std::move(graph_copy), docs, entry_point, dist, std::move(pool) + ); + auto sp = search_params(20); + auto scratch = index.scratchspace(sp); + + size_t top1_hits = 0; + for (size_t q = 0; q < query_view.size(); ++q) { + auto query = query_view.get_datum(q); + index.search(query, scratch); + + QChamferExactQuery oracle_dist{fine}; + oracle_dist.fix_argument(query); + auto oracle = svs_test::brute_force_top_k(oracle_dist, query, docs); + if (scratch.buffer[0].id() == oracle[0].first) { + ++top1_hits; + } + } + return top1_hits; + }; + + size_t hits_unrouted = run_search(nullptr); + size_t hits_routed = run_search(router_ptr); + + CATCH_REQUIRE(hits_routed >= hits_unrouted); +} + +CATCH_TEST_CASE( + "gem_pooled_entry_point picks doc0 as the medoid", "[vamana][gem][gem_router]" +) { + constexpr size_t dim = 2; + std::vector flat{1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F, 0.0F, -1.0F}; + svs::distance::FineCentroids fine(4, dim, flat.data(), 1); + + std::vector ids{0, 1, 2, 3, 0}; + std::vector offsets{0, 2, 4, 5}; + auto docs = svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); + + auto pool = svs::threads::DefaultThreadPool(1); + size_t entry = svs::index::vamana::extensions::gem_pooled_entry_point(docs, fine, pool); + CATCH_REQUIRE(entry == 0); +} diff --git a/tests/svs/index/vamana/gem_test_utils.h b/tests/svs/index/vamana/gem_test_utils.h new file mode 100644 index 000000000..cb59b6d7c --- /dev/null +++ b/tests/svs/index/vamana/gem_test_utils.h @@ -0,0 +1,150 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Shared fixtures and oracles for the vamana test directory's GEM, MaxSim and +// connectivity tests: a brute-force top-k oracle, a graph degree-bound check, and (in +// svs_test::gem) a unit-norm centroid table plus a deterministic multi-token id-corpus. + +// svs +#include "svs/core/data/vector_set_view.h" +#include "svs/core/distance.h" +#include "svs/core/distance/gem.h" +#include "svs/core/distance/gem_router.h" +#include "svs/core/graph.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs_test { + +// Every document's score against `query` under `dist`, sorted best-first by +// `Dist::compare`. `dist` must already carry any argument-fixing the distance needs. +template +std::vector> +brute_force_top_k(Dist& dist, const Query& query, const DocsView& docs) { + std::vector> scored; + scored.reserve(docs.size()); + for (uint32_t id = 0; id < docs.size(); ++id) { + scored.emplace_back(id, svs::distance::compute(dist, query, docs.get_datum(id))); + } + std::sort(scored.begin(), scored.end(), [](const auto& a, const auto& b) { + return typename Dist::compare{}(a.second, b.second); + }); + return scored; +} + +template +void require_degree_bound(const svs::graphs::SimpleGraph& graph, size_t max_degree) { + for (size_t i = 0; i < graph.n_nodes(); ++i) { + CATCH_REQUIRE(graph.get_node_degree(static_cast(i)) <= max_degree); + } +} + +} // namespace svs_test + +namespace svs_test::gem { + +inline constexpr size_t K = 16; +inline constexpr size_t DIM = 4; + +// K unit vectors in R^DIM, deterministic for a given seed. +inline std::vector make_centroids(uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution coord(-1.0F, 1.0F); + std::vector centroids(K * DIM); + for (size_t k = 0; k < K; ++k) { + double norm = 0.0; + for (size_t d = 0; d < DIM; ++d) { + float v = coord(rng); + centroids[k * DIM + d] = v; + norm += static_cast(v) * static_cast(v); + } + norm = std::sqrt(norm); + for (size_t d = 0; d < DIM; ++d) { + centroids[k * DIM + d] = static_cast(centroids[k * DIM + d] / norm); + } + } + return centroids; +} + +inline std::shared_ptr +make_centroid_table(const std::vector& centroids) { + return std::make_shared( + svs::distance::CentroidTable::from_centroids( + centroids.data(), K, DIM, svs::distance::Ground::Cos, 1 + ) + ); +} + +inline std::shared_ptr +make_fine_centroids(const std::vector& centroids) { + return std::make_shared( + K, DIM, centroids.data(), 1 + ); +} + +// A CSR-style corpus of id-token vector sets: cardinality cycles 1..6, ids uniform over K. +struct IdSetCorpus { + std::vector ids; + std::vector offsets; + + size_t size() const { return offsets.size() - 1; } + + svs::data::ConstVectorSetView view() const { + return svs::data::ConstVectorSetView( + std::span(ids), std::span(offsets), 1 + ); + } +}; + +inline IdSetCorpus make_doc_corpus(size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution id_dist(0, static_cast(K - 1)); + IdSetCorpus corpus; + corpus.offsets.resize(n + 1, 0); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 1 + (i % 6); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + for (size_t t = 0; t < cardinality; ++t) { + corpus.ids.push_back(id_dist(rng)); + } + } + return corpus; +} + +// Fine centroid f goes to coarse cluster f % num_coarse; deterministic, not realistic. +inline std::vector round_robin_coarse_of_fine(size_t num_coarse) { + std::vector out(K); + for (size_t f = 0; f < K; ++f) { + out[f] = static_cast(f % num_coarse); + } + return out; +} + +} // namespace svs_test::gem diff --git a/tests/svs/index/vamana/gem_vector_set.cpp b/tests/svs/index/vamana/gem_vector_set.cpp new file mode 100644 index 000000000..354ce9935 --- /dev/null +++ b/tests/svs/index/vamana/gem_vector_set.cpp @@ -0,0 +1,202 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Proves a VamanaIndex builds, searches, and saves/reloads over an owning +// VectorSetData of centroid-id vector sets, using QChamfer throughout. +#include "svs/core/distance/gem.h" +#include "svs/extensions/vamana/gem.h" + +// svs +#include "svs/core/data/vector_set.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/index.h" +#include "svs/lib/threads.h" + +// tests +#include "tests/svs/index/vamana/gem_test_utils.h" +#include "tests/utils/utils.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using svs_test::gem::make_centroid_table; +using svs_test::gem::make_centroids; +using svs_test::gem::make_doc_corpus; + +svs::index::vamana::VamanaSearchParameters search_params(size_t window, size_t capacity) { + return svs::index::vamana::VamanaSearchParameters{{window, capacity}, false, 0, 0}; +} + +// Builds a graph+index over `docs` through VamanaBuilder directly (not the +// VamanaIndex build-parameters constructor, which hard-codes L2/IP/Cosine and throws for +// qCH, same as qEMD). +template +auto build_index( + Docs docs, + svs::distance::QChamfer dist, + const svs::index::vamana::VamanaBuildParameters& build_parameters, + uint32_t entry_point +) { + auto graph = + svs::default_graph(docs.size(), build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + graph, + docs, + dist, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(docs) + ); + builder.construct(build_parameters.alpha, entry_point); + + auto pool = svs::threads::DefaultThreadPool(1); + return svs::index::vamana::VamanaIndex( + std::move(graph), std::move(docs), entry_point, dist, std::move(pool) + ); +} + +} // namespace + +CATCH_TEST_CASE( + "Vamana build, search and save/reload over a vector-set dataset", "[vamana][gem]" +) { + auto table = make_centroid_table(make_centroids(0x5EED)); + svs::distance::QChamfer dist{table}; + + const size_t n_docs = 200; + auto corpus = make_doc_corpus(n_docs, 0xDEADBEEF); + auto queries = make_doc_corpus(10, 0xC0FFEE); + auto query_view = queries.view(); + auto owning_docs = svs::data::VectorSetData::copy_from(corpus.view()); + + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + const uint32_t entry_point = 0; + + auto index = build_index(std::move(owning_docs), dist, build_parameters, entry_point); + const auto& docs = index.view_data(); + + CATCH_SECTION("Small-window search returns a best-first, internally consistent pool") { + const size_t capacity = 20; + auto sp = search_params(capacity, capacity); + auto scratch = index.scratchspace(sp); + + for (size_t q = 0; q < query_view.size(); ++q) { + auto query = query_view.get_datum(q); + index.search(query, scratch); + + size_t valid = scratch.buffer.size(); + CATCH_REQUIRE(valid > 0); + float previous = -std::numeric_limits::infinity(); + for (size_t j = 0; j < valid; ++j) { + const auto& neighbor = scratch.buffer[j]; + float direct = + svs::distance::compute(dist, query, docs.get_datum(neighbor.id())); + CATCH_REQUIRE( + static_cast(neighbor.distance()) == + Catch::Approx(direct).epsilon(1e-5).margin(1e-5) + ); + CATCH_REQUIRE(neighbor.distance() >= previous); + previous = neighbor.distance(); + } + } + } + + CATCH_SECTION("Exhaustive search's top-1 matches brute force") { + const size_t capacity = n_docs; + auto sp = search_params(capacity, capacity); + auto scratch = index.scratchspace(sp); + + for (size_t q = 0; q < query_view.size(); ++q) { + auto query = query_view.get_datum(q); + index.search(query, scratch); + auto oracle = svs_test::brute_force_top_k(dist, query, docs); + + // Centroid ids repeat, so several documents can tie for the best score; check + // the id against every document tied for it, not just oracle[0].first. + const auto& top1 = scratch.buffer[0]; + CATCH_REQUIRE( + static_cast(top1.distance()) == + Catch::Approx(oracle[0].second).epsilon(1e-5).margin(1e-5) + ); + bool id_ties_for_best = + std::any_of(oracle.begin(), oracle.end(), [&](const auto& entry) { + return entry.first == top1.id() && + std::abs(entry.second - oracle[0].second) < 1e-5F; + }); + CATCH_REQUIRE(id_ties_for_best); + } + } + + CATCH_SECTION("Save and reload preserve every query's result") { + // `apply` gives the index real VamanaBuildParameters (build_index's lower + // VamanaBuilder path left them at sentinels that overflow TOML save) without + // re-running the L2/IP/Cosine distance-type check. + index.apply(svs::index::vamana::VamanaIndexParameters{ + entry_point, build_parameters, index.get_search_parameters()}); + + auto temp_directory = svs_test::prepare_temp_directory_v2(); + auto config_dir = temp_directory / "config"; + auto graph_dir = temp_directory / "graph"; + auto data_dir = temp_directory / "data"; + index.save(config_dir, graph_dir, data_dir); + + auto reload_pool = svs::threads::DefaultThreadPool(1); + auto reloaded = svs::index::vamana::auto_assemble( + config_dir, + svs::GraphLoader(graph_dir), + svs::VectorSetDataLoader(data_dir), + dist, + std::move(reload_pool) + ); + + const size_t capacity = n_docs; + auto sp = search_params(capacity, capacity); + auto scratch_before = index.scratchspace(sp); + auto scratch_after = reloaded.scratchspace(sp); + + for (size_t q = 0; q < query_view.size(); ++q) { + auto query = query_view.get_datum(q); + index.search(query, scratch_before); + reloaded.search(query, scratch_after); + + CATCH_REQUIRE(scratch_before.buffer.size() == scratch_after.buffer.size()); + for (size_t j = 0; j < scratch_before.buffer.size(); ++j) { + CATCH_REQUIRE( + scratch_before.buffer[j].id() == scratch_after.buffer[j].id() + ); + CATCH_REQUIRE( + scratch_before.buffer[j].distance() == + scratch_after.buffer[j].distance() + ); + } + } + } +} diff --git a/tests/svs/index/vamana/maxsim_rerank.cpp b/tests/svs/index/vamana/maxsim_rerank.cpp new file mode 100644 index 000000000..d6bb9c8c7 --- /dev/null +++ b/tests/svs/index/vamana/maxsim_rerank.cpp @@ -0,0 +1,548 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Header under test: the Vamana re-rank extension for vector-set (MaxSim) datasets. +#include "svs/extensions/vamana/maxsim.h" + +// svs +#include "svs/concepts/vector_set.h" +#include "svs/core/data.h" +#include "svs/core/data/vector_set.h" +#include "svs/core/data/vector_set_view.h" +#include "svs/core/data/with_vector_sets.h" +#include "svs/core/distance/maxsim.h" +#include "svs/core/graph.h" +#include "svs/core/query_result.h" +#include "svs/index/vamana/index.h" +#include "svs/lib/misc.h" +#include "svs/lib/narrow.h" +#include "svs/lib/saveload.h" +#include "svs/lib/threads.h" + +// tests +#include "tests/svs/index/vamana/gem_test_utils.h" +#include "tests/utils/utils.h" + +// catch2 +#include "catch2/catch_approx.hpp" +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t DIM = 8; +constexpr double EPS = 1e-4; + +// `compare = std::less<>` is the opposite direction from MaxSim's `std::greater<>`, to +// exercise the re-rank extension's comparator-flip handling. +struct MeanNegativeInnerProduct { + using compare = std::less<>; + static constexpr bool implicit_broadcast = true; + static constexpr std::string_view name = "test_mean_negative_ip"; + + template + float compute(const Q& query, std::span primary_row) const { + size_t dim = primary_row.size(); + size_t cardinality = query.cardinality(); + std::vector mean(dim, 0.0); + for (size_t t = 0; t < cardinality; ++t) { + auto token = query.token(t); + for (size_t d = 0; d < dim; ++d) { + mean[d] += token[d]; + } + } + double dot = 0.0; + for (size_t d = 0; d < dim; ++d) { + dot += (mean[d] / static_cast(cardinality)) * primary_row[d]; + } + return static_cast(-dot); + } + + // Build compares two primary rows directly, not a query set against a row, so this + // overload serves vamana_build.h's candidate-pool and reverse-edge loops. + float compute(std::span a, std::span b) const { + double dot = 0.0; + for (size_t d = 0; d < a.size(); ++d) { + dot += static_cast(a[d]) * static_cast(b[d]); + } + return static_cast(-dot); + } +}; + +} // namespace + +// IP-flavored (no triangle inequality), so pruning needs IterativePruneStrategy like +// DistanceIP, not vamana_build.h's default L2 one. +namespace svs::index::vamana { +template <> struct PruneStrategy<::MeanNegativeInnerProduct> { + using type = IterativePruneStrategy; +}; +} // namespace svs::index::vamana + +namespace { + +using Dist = MeanNegativeInnerProduct; + +void normalize(std::span token) { + double norm = 0.0; + for (float v : token) { + norm += static_cast(v) * static_cast(v); + } + norm = std::sqrt(norm); + for (float& v : token) { + v = static_cast(static_cast(v) / norm); + } +} + +// A deterministic corpus of vector sets: cardinality cycles 1..6, tokens are unit-norm +// (MaxSim assumes unit-norm tokens), and a document's primary row is the mean of its +// tokens. +struct Corpus { + std::vector tokens; + std::vector offsets; + svs::data::SimpleData primary; + + size_t size() const { return offsets.size() - 1; } + + svs::data::ConstVectorSetView vector_sets() const { + return svs::data::ConstVectorSetView( + std::span(tokens), std::span(offsets), DIM + ); + } +}; + +Corpus make_corpus(size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution dist(-1.0F, 1.0F); + + Corpus corpus; + corpus.offsets.resize(n + 1, 0); + corpus.primary = svs::data::SimpleData(n, DIM); + + std::vector mean(DIM); + std::vector token(DIM); + for (size_t i = 0; i < n; ++i) { + size_t cardinality = 1 + (i % 6); + corpus.offsets[i + 1] = corpus.offsets[i] + cardinality; + std::fill(mean.begin(), mean.end(), 0.0F); + for (size_t t = 0; t < cardinality; ++t) { + for (size_t d = 0; d < DIM; ++d) { + token[d] = dist(rng); + } + normalize(token); + for (size_t d = 0; d < DIM; ++d) { + mean[d] += token[d]; + } + corpus.tokens.insert(corpus.tokens.end(), token.begin(), token.end()); + } + for (size_t d = 0; d < DIM; ++d) { + mean[d] /= static_cast(cardinality); + } + corpus.primary.set_datum(i, mean); + } + return corpus; +} + +svs::index::vamana::VamanaSearchParameters search_params(size_t window, size_t capacity) { + return svs::index::vamana::VamanaSearchParameters{{window, capacity}, false, 0, 0}; +} + +uint32_t local_entry_point(const svs::data::SimpleData& primary) { + auto pool = svs::threads::DefaultThreadPool(1); + return svs::lib::narrow( + svs::index::vamana::extensions::compute_entry_point(primary, pool) + ); +} + +// Builds a graph and VamanaIndex over `data` through VamanaBuilder directly (not the +// VamanaIndex build-parameters constructor, which hard-codes L2/IP/Cosine and throws for +// this test-local distance), with the two-phase construct VamanaIndex's own constructor +// runs internally (alpha pinned to 1.0, then the real alpha). +template +auto build_maxsim_index( + Data data, + const svs::index::vamana::VamanaBuildParameters& build_parameters, + uint32_t entry_point +) { + auto graph = + svs::default_graph(data.size(), build_parameters.graph_max_degree); + auto build_pool = svs::threads::DefaultThreadPool(1); + auto builder = svs::index::vamana::VamanaBuilder( + graph, + data, + Dist{}, + build_parameters, + build_pool, + svs::index::vamana::extensions::estimate_prefetch_parameters(data) + ); + builder.construct(1.0F, entry_point); + builder.construct(build_parameters.alpha, entry_point); + + auto pool = svs::threads::DefaultThreadPool(1); + return svs::index::vamana::VamanaIndex( + std::move(graph), std::move(data), entry_point, Dist{}, std::move(pool) + ); +} + +// Overwrites docs 198/199 (cardinality 1 and 2) and query 1 (cardinality 2) so doc 199's +// second token wins against the query's second token (better MaxSim score) while its +// primary mean-dot score is worse than doc 198's: a MaxSim/primary-order inversion +// witness, independent of the random seed. +void make_inversion_witness(Corpus& corpus, Corpus& query_corpus) { + constexpr size_t DOC_A = 198; + constexpr size_t DOC_B = 199; + constexpr size_t QUERY = 1; + + std::vector e1(DIM, 0.0F); + e1[0] = 1.0F; + std::vector e2(DIM, 0.0F); + e2[1] = 1.0F; + std::vector q2 = {0.3F, 0.95F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}; + normalize(q2); + + auto set_token = [](Corpus& c, size_t doc, size_t t, const std::vector& v) { + size_t base = (c.offsets[doc] + t) * DIM; + std::copy(v.begin(), v.end(), c.tokens.data() + base); + }; + + set_token(corpus, DOC_A, 0, e1); + corpus.primary.set_datum(DOC_A, e1); + + set_token(corpus, DOC_B, 0, e1); + set_token(corpus, DOC_B, 1, e2); + std::vector mean_b(DIM, 0.0F); + for (size_t d = 0; d < DIM; ++d) { + mean_b[d] = (e1[d] + e2[d]) / 2.0F; + } + corpus.primary.set_datum(DOC_B, mean_b); + + set_token(query_corpus, QUERY, 0, e1); + set_token(query_corpus, QUERY, 1, q2); +} + +// Checks the MaxSim rerank contract against the whole candidate pool a +// single-query search leaves in `scratch`: every score matches the oracle, the pool is +// sorted best-first, and its first `num_neighbors` entries are the brute-force top-k +// restricted to the pool's own ids. +template < + typename ElemT, + svs::data::VectorSet Query, + typename Index, + typename Scratch, + typename DocsView> +void check_rerank_pool( + Index& index, + Scratch& scratch, + const Query& query, + const DocsView& doc_sets, + size_t num_neighbors +) { + index.search(query, scratch); + const auto& buffer = scratch.buffer; + + svs::distance::MaxSim maxsim{}; + svs::distance::maybe_fix_argument(maxsim, query); + + std::vector pool_ids; + pool_ids.reserve(buffer.size()); + float previous_score = std::numeric_limits::infinity(); + for (const auto& neighbor : buffer) { + float expected = + svs::distance::compute(maxsim, query, doc_sets.get_datum(neighbor.id())); + CATCH_REQUIRE( + static_cast(neighbor.distance()) == + Catch::Approx(expected).epsilon(EPS).margin(EPS) + ); + CATCH_REQUIRE(neighbor.distance() <= previous_score); + previous_score = neighbor.distance(); + pool_ids.push_back(neighbor.id()); + } + // Pool strictly larger than num_neighbors: the re-rank had more than one candidate. + CATCH_REQUIRE(pool_ids.size() > num_neighbors); + + std::set pool_set(pool_ids.begin(), pool_ids.end()); + auto oracle = svs_test::brute_force_top_k(maxsim, query, doc_sets); + std::vector restricted_oracle; + for (const auto& scored : oracle) { + if (pool_set.contains(scored.first)) { + restricted_oracle.push_back(scored.first); + if (restricted_oracle.size() == num_neighbors) { + break; + } + } + } + for (size_t k = 0; k < num_neighbors; ++k) { + CATCH_REQUIRE(buffer[k].id() == restricted_oracle[k]); + } +} + +// `Data` has no `allocator_type` of its own to build a `VectorDataLoader`-style loader +// around, so a `svs::lib::Lazy` callable reloads it whole instead. +template +void check_save_reload_round_trip( + Index& index, + const svs::index::vamana::VamanaBuildParameters& build_parameters, + uint32_t entry_point, + const svs::data::ConstVectorSetView& queries +) { + using Data = typename Index::data_type; + index.apply(svs::index::vamana::VamanaIndexParameters{ + entry_point, build_parameters, index.get_search_parameters()}); + + auto temp_directory = svs_test::prepare_temp_directory_v2(); + auto config_dir = temp_directory / "config"; + auto graph_dir = temp_directory / "graph"; + auto data_dir = temp_directory / "data"; + index.save(config_dir, graph_dir, data_dir); + + auto reload_pool = svs::threads::DefaultThreadPool(1); + auto reloaded = svs::index::vamana::auto_assemble( + config_dir, + svs::GraphLoader(graph_dir), + SVS_LAZY(svs::lib::load_from_disk(data_dir)), + Dist{}, + std::move(reload_pool) + ); + + const size_t capacity = index.size(); + auto sp = search_params(capacity, capacity); + auto scratch_before = index.scratchspace(sp); + auto scratch_after = reloaded.scratchspace(sp); + + for (size_t q = 0; q < queries.size(); ++q) { + auto query = queries.get_datum(q); + index.search(query, scratch_before); + reloaded.search(query, scratch_after); + + CATCH_REQUIRE(scratch_before.buffer.size() == scratch_after.buffer.size()); + for (size_t j = 0; j < scratch_before.buffer.size(); ++j) { + CATCH_REQUIRE(scratch_before.buffer[j].id() == scratch_after.buffer[j].id()); + CATCH_REQUIRE( + scratch_before.buffer[j].distance() == scratch_after.buffer[j].distance() + ); + } + } +} + +} // namespace + +// Builds one owning WithVectorSets index and runs every behavior below as a section on +// it (Catch re-runs this setup per leaf section). +CATCH_TEST_CASE("MaxSim re-rank Vamana extension", "[vamana][maxsim_rerank]") { + using Data = svs::data:: + WithVectorSets, svs::data::VectorSetData>; + + CATCH_SECTION("Mismatched primary/sets sizes throw") { + auto primary_corpus = make_corpus(5, 1); + auto sets_corpus = make_corpus(4, 2); + CATCH_REQUIRE_THROWS(Data( + primary_corpus.primary, + svs::data::VectorSetData::copy_from(sets_corpus.vector_sets()) + )); + } + + const size_t n_docs = 200; + auto corpus = make_corpus(n_docs, 0xDEADBEEF); + auto query_corpus = make_corpus(5, 0xC0FFEE); + make_inversion_witness(corpus, query_corpus); + + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + auto entry_point = local_entry_point(corpus.primary); + + auto full_index = build_maxsim_index( + Data( + corpus.primary, svs::data::VectorSetData::copy_from(corpus.vector_sets()) + ), + build_parameters, + entry_point + ); + const auto& doc_sets = full_index.view_data().vector_sets(); + auto queries = query_corpus.vector_sets(); + + CATCH_SECTION("Query token dimension mismatch throws before the graph search") { + std::vector bad_tokens(DIM - 1, 0.1F); + std::vector bad_offsets{0, 1}; + auto bad_query_set = svs::data::ConstVectorSetView( + std::span(bad_tokens), + std::span(bad_offsets), + DIM - 1 + ); + auto bad_query = bad_query_set.get_datum(0); + + auto sp = search_params(40, 40); + auto full_scratch = full_index.scratchspace(sp); + CATCH_REQUIRE_THROWS(full_index.search(bad_query, full_scratch)); + } + + // Capacity == n_docs leaves every document in the pool: the strongest check against + // the full-corpus oracle. A tighter window only re-exercises the generic search + // buffer's eviction path, which search_buffer.cpp already owns. + CATCH_SECTION("Candidate pool is scored and ordered by MaxSim") { + const size_t num_neighbors = 10; + auto sp = search_params(n_docs, n_docs); + auto full_scratch = full_index.scratchspace(sp); + + for (size_t q = 0; q < queries.size(); ++q) { + auto query = queries.get_datum(q); + check_rerank_pool( + full_index, full_scratch, query, doc_sets, num_neighbors + ); + } + } + + CATCH_SECTION("MaxSim order of the pool can differ from the primary-distance order") { + constexpr uint32_t DOC_A = 198; + constexpr uint32_t DOC_B = 199; + auto sp = search_params(n_docs, n_docs); + auto full_scratch = full_index.scratchspace(sp); + + auto query = queries.get_datum(1); + full_index.search(query, full_scratch); + const auto& buffer = full_scratch.buffer; + + auto it_a = std::find_if(buffer.begin(), buffer.end(), [](const auto& n) { + return n.id() == DOC_A; + }); + auto it_b = std::find_if(buffer.begin(), buffer.end(), [](const auto& n) { + return n.id() == DOC_B; + }); + CATCH_REQUIRE(it_a != buffer.end()); + CATCH_REQUIRE(it_b != buffer.end()); + CATCH_REQUIRE( + std::distance(buffer.begin(), it_b) < std::distance(buffer.begin(), it_a) + ); + + // The generic `distance::compute` dispatch takes the functor by non-const + // reference, so a named lvalue is required (a temporary will not bind). + Dist dist{}; + float primary_a = + svs::distance::compute(dist, query, corpus.primary.get_datum(DOC_A)); + float primary_b = + svs::distance::compute(dist, query, corpus.primary.get_datum(DOC_B)); + CATCH_REQUIRE(Dist::compare{}(primary_a, primary_b)); + } + + CATCH_SECTION("Batch search matches the first k entries of the single-query path") { + const size_t capacity = n_docs; + const size_t num_neighbors = 8; + auto sp = search_params(capacity, capacity); + const size_t num_queries = queries.size(); + + auto full_scratch = full_index.scratchspace(sp); + using PoolEntry = typename std::decay_t::value_type; + std::vector> single_query_results; + single_query_results.reserve(num_queries); + for (size_t q = 0; q < num_queries; ++q) { + auto query = queries.get_datum(q); + full_index.search(query, full_scratch); + std::vector top_k( + full_scratch.buffer.begin(), full_scratch.buffer.begin() + num_neighbors + ); + single_query_results.push_back(std::move(top_k)); + } + + svs::QueryResult result(num_queries, num_neighbors); + full_index.search(result.view(), queries, sp); + + for (size_t q = 0; q < num_queries; ++q) { + for (size_t k = 0; k < num_neighbors; ++k) { + uint32_t id = result.index(q, k); + float score = result.distance(q, k); + CATCH_REQUIRE(id == single_query_results[q][k].id()); + CATCH_REQUIRE( + static_cast(score) == + Catch::Approx(single_query_results[q][k].distance()) + .epsilon(EPS) + .margin(EPS) + ); + } + } + } + + CATCH_SECTION("Save and reload preserve every query's result") { + check_save_reload_round_trip(full_index, build_parameters, entry_point, queries); + } +} + +namespace { + +// Rounds `corpus`'s float tokens to bfloat16 into a fresh owning `VectorSetData` +// with the same CSR offsets, so the bf16 store's documents are exactly what the float +// corpus's documents round to. +svs::data::VectorSetData round_tokens_to_bf16(const Corpus& corpus) { + size_t total_tokens = corpus.tokens.size() / DIM; + svs::data::SimpleData tokens(total_tokens, DIM); + std::vector row(DIM); + for (size_t i = 0; i < total_tokens; ++i) { + for (size_t d = 0; d < DIM; ++d) { + row[d] = svs::round_to_bfloat16(corpus.tokens[i * DIM + d]); + } + tokens.set_datum(i, row); + } + return svs::data::VectorSetData(std::move(tokens), corpus.offsets); +} + +} // namespace + +// A BFloat16 element type makes the re-rank extension pick MaxSim +// (svs/extensions/vamana/maxsim.h selects on Sets::const_value_type::element_type), +// reaching the bf16 InnerProductMatrix::column_max overload rather than the float one. +CATCH_TEST_CASE( + "MaxSim re-rank over an owning bfloat16 vector-set store", "[vamana][maxsim_rerank]" +) { + using Data = svs::data::WithVectorSets< + svs::data::SimpleData, + svs::data::VectorSetData>; + + const size_t n_docs = 80; + auto corpus = make_corpus(n_docs, 0x5EED5EED); + auto query_corpus = make_corpus(5, 0xFACE5EED); + auto data = Data(corpus.primary, round_tokens_to_bf16(corpus)); + + auto build_parameters = + svs::index::vamana::VamanaBuildParameters{1.2F, 16, 32, 64, 16, true}; + auto entry_point = local_entry_point(corpus.primary); + auto index = build_maxsim_index(std::move(data), build_parameters, entry_point); + + auto queries = query_corpus.vector_sets(); + const size_t capacity = n_docs; + const size_t num_neighbors = 10; + auto sp = search_params(capacity, capacity); + auto scratch = index.scratchspace(sp); + + for (size_t q = 0; q < queries.size(); ++q) { + auto query = queries.get_datum(q); + check_rerank_pool( + index, scratch, query, index.view_data().vector_sets(), num_neighbors + ); + } +} diff --git a/tests/svs/index/vamana/vamana_build.cpp b/tests/svs/index/vamana/vamana_build.cpp index 6935e3cb3..2d2a4bf83 100644 --- a/tests/svs/index/vamana/vamana_build.cpp +++ b/tests/svs/index/vamana/vamana_build.cpp @@ -19,13 +19,23 @@ #include "svs/index/vamana/vamana_build.h" // svs +#include "svs/core/data/simple.h" +#include "svs/core/distance.h" +#include "svs/core/graph.h" +#include "svs/index/vamana/extensions.h" +#include "svs/lib/narrow.h" #include "svs/lib/threads.h" // catch2 #include "catch2/catch_test_macros.hpp" +// tests +#include "tests/utils/test_dataset.h" + // stl +#include #include +#include #include #include @@ -142,3 +152,43 @@ CATCH_TEST_CASE("Index Build Utilities", "[vamana][vamana_build]") { } } } + +CATCH_TEST_CASE( + "Static Vamana build is deterministic with one thread", "[vamana][vamana_build]" +) { + // With more threads, back-edge lock order changes the resulting graph. + auto data = svs::data::SimpleData::load(test_dataset::data_svs_file()); + auto params = vamana::VamanaBuildParameters{1.2F, 16, 32, 750, 60, true}; + auto pool = svs::threads::DefaultThreadPool(1); + auto entry_point = + svs::lib::narrow(vamana::extensions::compute_entry_point(data, pool)); + + auto build = [&]() { + auto graph = svs::default_graph(data.size(), params.graph_max_degree); + auto builder = vamana::VamanaBuilder( + graph, + data, + svs::distance::DistanceL2{}, + params, + pool, + vamana::extensions::estimate_prefetch_parameters(data) + ); + builder.construct(1.0F, entry_point); + builder.construct(params.alpha, entry_point); + return graph; + }; + + auto graph_a = build(); + auto graph_b = build(); + + CATCH_REQUIRE(graph_a.n_nodes() == graph_b.n_nodes()); + std::optional first_mismatch; + for (size_t i = 0; i < graph_a.n_nodes(); ++i) { + if (!std::ranges::equal(graph_a.get_node(i), graph_b.get_node(i))) { + first_mismatch = i; + break; + } + } + CATCH_INFO("first mismatching node: " << first_mismatch.value_or(0)); + CATCH_REQUIRE(!first_mismatch.has_value()); +} diff --git a/tests/svs/quantization/gem/codebook.cpp b/tests/svs/quantization/gem/codebook.cpp new file mode 100644 index 000000000..7d5cec3a6 --- /dev/null +++ b/tests/svs/quantization/gem/codebook.cpp @@ -0,0 +1,464 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/quantization/gem/codebook.h" +#include "svs/quantization/gem/codebook_timed.h" + +// svs +#include "svs/core/data/simple.h" +#include "svs/core/data/vector_set.h" +#include "svs/lib/threads.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// stl +#include +#include +#include +#include +#include +#include +#include + +namespace { +namespace gem = svs::quantization::gem; +constexpr float TOL = 1e-4F; + +std::vector random_unit_vector(std::mt19937& rng, size_t dim) { + std::normal_distribution gauss(0.0F, 1.0F); + std::vector v(dim); + double norm2 = 0.0; + for (auto& x : v) { + x = gauss(rng); + norm2 += static_cast(x) * x; + } + double norm = std::sqrt(norm2); + for (auto& x : v) { + x = static_cast(static_cast(x) / norm); + } + return v; +} + +std::vector make_random_centers(size_t K, size_t dim, uint32_t seed) { + std::mt19937 rng(seed); + std::vector centers(K * dim); + for (size_t k = 0; k < K; ++k) { + auto c = random_unit_vector(rng, dim); + std::copy(c.begin(), c.end(), centers.begin() + static_cast(k * dim)); + } + return centers; +} + +// Fine centroids as a SimpleData, laid out row-major from a flat `centers` buffer. +svs::data::SimpleData +centroids_from(const std::vector& centers, size_t K, size_t dim) { + svs::data::SimpleData centroids(K, dim); + for (size_t k = 0; k < K; ++k) { + centroids.set_datum(k, std::span(centers.data() + k * dim, dim)); + } + return centroids; +} + +// Each token perturbs a center with a large margin, so even an approximate assigner agrees. +svs::data::VectorSetData make_clustered_corpus( + const std::vector& centers, + size_t K, + size_t dim, + const std::vector& cardinalities, + uint32_t seed, + float noise +) { + std::mt19937 rng(seed); + size_t num_passages = cardinalities.size(); + size_t total_tokens = 0; + for (auto c : cardinalities) { + total_tokens += c; + } + svs::data::SimpleData token_matrix(total_tokens, dim); + std::vector offsets(num_passages + 1, 0); + std::uniform_int_distribution pick(0, K - 1); + std::normal_distribution noise_dist(0.0F, noise); + + size_t global = 0; + for (size_t p = 0; p < num_passages; ++p) { + offsets[p + 1] = offsets[p] + cardinalities[p]; + for (size_t t = 0; t < cardinalities[p]; ++t) { + size_t k = pick(rng); + std::vector row(dim); + double norm2 = 0.0; + for (size_t d = 0; d < dim; ++d) { + row[d] = centers[k * dim + d] + noise_dist(rng); + norm2 += static_cast(row[d]) * row[d]; + } + double norm = std::sqrt(norm2); + for (auto& x : row) { + x = static_cast(static_cast(x) / norm); + } + token_matrix.set_datum(global, row); + ++global; + } + } + return svs::data::VectorSetData(std::move(token_matrix), std::move(offsets)); +} + +uint32_t brute_force_best( + std::span token, const svs::data::SimpleData& centroids +) { + uint32_t best = 0; + float best_dot = -std::numeric_limits::infinity(); + for (size_t j = 0, jmax = centroids.size(); j < jmax; ++j) { + auto c = centroids.get_datum(j); + float dot = 0.0F; + for (size_t d = 0, dmax = token.size(); d < dmax; ++d) { + dot += token[d] * c[d]; + } + if (dot > best_dot) { + best_dot = dot; + best = static_cast(j); + } + } + return best; +} + +std::vector flatten_ids(const svs::data::VectorSetData& ids) { + std::vector out; + out.reserve(ids.total_tokens()); + for (size_t p = 0, pmax = ids.size(); p < pmax; ++p) { + auto doc = ids.get_datum(p); + for (size_t t = 0, tmax = doc.cardinality(); t < tmax; ++t) { + out.push_back(doc.token(t)[0]); + } + } + return out; +} + +svs::data::VectorSetData +rebuild_ids(const std::vector& flat, std::vector offsets) { + svs::data::SimpleData storage(flat.size(), 1); + for (size_t i = 0; i < flat.size(); ++i) { + storage.set_datum(i, std::vector{flat[i]}); + } + return svs::data::VectorSetData(std::move(storage), std::move(offsets)); +} +} // namespace + +CATCH_TEST_CASE("GEM Codebook - sizing rules", "[quantization][gem]") { + CATCH_SECTION("colbert_num_partitions") { + // Exact powers of two: 16 * sqrt(1) == 16, 16 * sqrt(4) == 32. + CATCH_REQUIRE(gem::colbert_num_partitions(1) == 16); + CATCH_REQUIRE(gem::colbert_num_partitions(4) == 32); + // One token below/above the num_tokens=4 boundary (raw crosses exactly 32 at n=4). + CATCH_REQUIRE(gem::colbert_num_partitions(3) == 16); + CATCH_REQUIRE(gem::colbert_num_partitions(5) == 32); + CATCH_REQUIRE_THROWS(gem::colbert_num_partitions(0)); + } + + CATCH_SECTION("colbert_sample_size") { + // Small corpora: the formula's raw value exceeds num_passages, so it is capped. + CATCH_REQUIRE(gem::colbert_sample_size(1) == 1); + CATCH_REQUIRE(gem::colbert_sample_size(10) == 10); + // Large corpus: 1 + floor(16 * sqrt(120 * 1'000'000)) is well under num_passages. + size_t num_passages = 1'000'000; + size_t expected = + 1 + static_cast( + std::floor(16.0 * std::sqrt(120.0 * static_cast(num_passages))) + ); + CATCH_REQUIRE(gem::colbert_sample_size(num_passages) == expected); + CATCH_REQUIRE(expected < num_passages); + CATCH_REQUIRE_THROWS(gem::colbert_sample_size(0)); + // Perfect-square boundary (921'600 == 960^2); integer sqrt must not be off by one. + CATCH_REQUIRE(gem::detail::isqrt_floor(921'600) == 960); + CATCH_REQUIRE(gem::detail::isqrt_floor(921'599) == 959); + CATCH_REQUIRE(gem::detail::isqrt_floor(921'601) == 960); + } +} + +CATCH_TEST_CASE( + "GEM Codebook - plain mean update differs from (old + sum) / (count + 1)", + "[quantization][gem]" +) { + auto data = svs::data::SimpleData(4, 2); + data.set_datum(0, std::vector{0.0F, 0.0F}); + data.set_datum(1, std::vector{0.0F, 2.0F}); + data.set_datum(2, std::vector{10.0F, 0.0F}); + data.set_datum(3, std::vector{10.0F, 2.0F}); + auto centroids = svs::data::SimpleData(2, 2); + centroids.set_datum(0, std::vector{0.0F, 0.0F}); + centroids.set_datum(1, std::vector{10.0F, 0.0F}); + + std::vector assignments{0, 0, 1, 1}; + std::vector counts(2); + auto threadpool = svs::threads::as_threadpool(2); + gem::detail::plain_mean_update(data, centroids, assignments, counts, threadpool); + + auto c0 = centroids.get_datum(0); + auto c1 = centroids.get_datum(1); + // Plain mean: cluster 0 -> (0,1); cluster 1 -> (10,1). + CATCH_REQUIRE(std::abs(c0[0] - 0.0F) < TOL); + CATCH_REQUIRE(std::abs(c0[1] - 1.0F) < TOL); + CATCH_REQUIRE(std::abs(c1[0] - 10.0F) < TOL); + CATCH_REQUIRE(std::abs(c1[1] - 1.0F) < TOL); + // (old + sum) / (count + 1) would give (0 + 0 + 2) / 3 == 0.667, not the plain mean 1. + CATCH_REQUIRE(std::abs(c0[1] - (2.0F / 3.0F)) > TOL); +} + +// One small corpus drives the whole pipeline; each SECTION below asserts a different group +// of properties on the shared `result`/`corpus` instead of rebuilding its own input. Cases +// that genuinely need different input (thread count, byte budget) get their own SECTION +// too, still off this one corpus; only parameters/sizing that need a different seed/shape +// get their own TEST_CASE (sizing rules, plain-mean-update, failure paths). +CATCH_TEST_CASE("GEM Codebook - quantize pipeline", "[quantization][gem]") { + constexpr size_t DIM = 16; + constexpr size_t NUM_PASSAGES = 100; + auto centers = make_random_centers(16, DIM, 0x9999); + auto corpus = make_clustered_corpus( + centers, 16, DIM, std::vector(NUM_PASSAGES, 2), 0x1357, 0.1F + ); + gem::CodebookParameters parameters; // num_centroids == 0: use the default rule. + // Per-iteration timing and the faiss bound below are Lloyd's contract. + parameters.kmeans_algorithm = gem::KMeansAlgorithm::Lloyd; + auto threadpool = svs::threads::as_threadpool(2); + auto result = gem::quantize(corpus, parameters, threadpool); + size_t expected_K = gem::colbert_num_partitions(NUM_PASSAGES * 2); + + CATCH_SECTION( + "Pipeline wiring, default coarse rule, unit-norm, non-increasing objective" + ) { + CATCH_REQUIRE(result.K == expected_K); + CATCH_REQUIRE(result.centroids.size() == expected_K); + CATCH_REQUIRE(result.centroids.dimensions() == DIM); + CATCH_REQUIRE(result.fine_centroids->K == expected_K); + CATCH_REQUIRE(result.fine_centroids->dim == DIM); + CATCH_REQUIRE(result.centroid_table->K == expected_K); + CATCH_REQUIRE(result.ids.total_tokens() == NUM_PASSAGES * 2); + for (size_t p = 0; p < NUM_PASSAGES; ++p) { + CATCH_REQUIRE(result.ids.cardinality(p) == 2); + } + // Default num_coarse rule (num_coarse == 0 in `parameters`). + CATCH_REQUIRE(result.num_coarse == std::max(1, expected_K * 5 / 32)); + CATCH_REQUIRE(result.coarse_of_fine.size() == expected_K); + for (auto c : result.coarse_of_fine) { + CATCH_REQUIRE(c < result.num_coarse); + } + for (size_t i = 0; i < result.centroids.size(); ++i) { + auto c = result.centroids.get_datum(i); + float norm_sq = 0.0F; + for (size_t d = 0; d < c.size(); ++d) { + norm_sq += c[d] * c[d]; + } + CATCH_REQUIRE(std::abs(std::sqrt(norm_sq) - 1.0F) < 1e-4F); + } + + // Same seed and sampling: more k-means passes cannot raise the final objective. + auto one_iter_params = parameters; + one_iter_params.kmeans_iterations = 1; + auto one_iter = gem::train_codebook(corpus, one_iter_params, threadpool); + auto four_iter_params = parameters; + four_iter_params.kmeans_iterations = 4; + auto four_iter = gem::train_codebook(corpus, four_iter_params, threadpool); + CATCH_REQUIRE(four_iter.objective <= one_iter.objective + 1e-3); + } + + CATCH_SECTION("coarse_clusters: explicit num_coarse is honored") { + gem::CodebookParameters explicit_params = parameters; + explicit_params.num_coarse = 5; + auto explicit_coarse = + gem::coarse_clusters(result.centroids, explicit_params, threadpool); + CATCH_REQUIRE(explicit_coarse.num_coarse == 5); + for (auto c : explicit_coarse.coarse_of_fine) { + CATCH_REQUIRE(c < 5); + } + } + + CATCH_SECTION( + "assign_tokens agrees with brute force and the mismatch checker, both assigners" + ) { + for (auto assigner : + {gem::Assigner::ExactGemm, gem::Assigner::InnerProductMatrix}) { + gem::CodebookParameters assign_params = parameters; + assign_params.assigner = assigner; + auto ids = + gem::assign_tokens(corpus, result.centroids, assign_params, threadpool); + auto flat = flatten_ids(ids); + + size_t global = 0; + for (size_t p = 0; p < corpus.size(); ++p) { + auto doc = corpus.get_datum(p); + for (size_t t = 0; t < doc.cardinality(); ++t) { + CATCH_REQUIRE( + flat[global++] == brute_force_best(doc.token(t), result.centroids) + ); + } + } + + auto report = + gem::count_assignment_mismatches(corpus, result.centroids, ids, 150, 99); + CATCH_REQUIRE(report.mismatches == 0); + CATCH_REQUIRE(report.sample_size == 150); + + // Corrupt a few ids and confirm the checker notices. + auto offsets = + std::vector(ids.offsets().begin(), ids.offsets().end()); + auto corrupted = flat; + for (size_t i = 0; i < corrupted.size(); i += 37) { + corrupted[i] = static_cast((corrupted[i] + 1) % expected_K); + } + auto bad_report = gem::count_assignment_mismatches( + corpus, result.centroids, rebuild_ids(corrupted, offsets), 2000, 99 + ); + CATCH_REQUIRE(bad_report.mismatches > 0); + } + } + + // Thread partitioning and batch sizing happen before the per-assigner flush branch, so + // neither depends on `assigner`; one (the default, InnerProductMatrix) is enough. + CATCH_SECTION("assign_tokens: thread-count independence and byte-budget batching") { + auto pool1 = svs::threads::as_threadpool(1); + auto pool4 = svs::threads::as_threadpool(4); + auto flat1 = + flatten_ids(gem::assign_tokens(corpus, result.centroids, parameters, pool1)); + CATCH_REQUIRE( + flat1 == + flatten_ids(gem::assign_tokens(corpus, result.centroids, parameters, pool4)) + ); + + // expected_K(128)*4 bytes/row: 256 forces a 1-row batch, 1536 forces a 3-row batch. + for (size_t budget : {size_t{256}, size_t{1536}}) { + gem::CodebookParameters tiny = parameters; + tiny.assign_buffer_bytes = budget; + CATCH_REQUIRE( + flatten_ids(gem::assign_tokens(corpus, result.centroids, tiny, pool1)) == + flat1 + ); + } + } + + CATCH_SECTION( + "quantize_timed reproduces quantize's result exactly and times every phase" + ) { + auto [timed, timings] = gem::quantize_timed(corpus, parameters, threadpool); + CATCH_REQUIRE(timed.K == result.K); + CATCH_REQUIRE(timed.num_coarse == result.num_coarse); + CATCH_REQUIRE(timed.objective == result.objective); + CATCH_REQUIRE(timed.num_sampled_passages == result.num_sampled_passages); + CATCH_REQUIRE(timed.num_training_tokens == result.num_training_tokens); + CATCH_REQUIRE(timed.coarse_of_fine == result.coarse_of_fine); + CATCH_REQUIRE(flatten_ids(timed.ids) == flatten_ids(result.ids)); + for (size_t i = 0; i < timed.centroids.size(); ++i) { + auto a = timed.centroids.get_datum(i); + auto b = result.centroids.get_datum(i); + for (size_t d = 0; d < a.size(); ++d) { + CATCH_REQUIRE(a[d] == b[d]); + } + } + CATCH_REQUIRE(timings.train_iterations.size() == parameters.kmeans_iterations); + CATCH_REQUIRE(timings.total >= 0.0); + CATCH_REQUIRE(timings.sample >= 0.0); + CATCH_REQUIRE(timings.train >= 0.0); + CATCH_REQUIRE(timings.normalize >= 0.0); + CATCH_REQUIRE(timings.assign >= 0.0); + CATCH_REQUIRE(timings.ground_table >= 0.0); + CATCH_REQUIRE(timings.coarse >= 0.0); + } +} + +CATCH_TEST_CASE("GEM Codebook - failure paths", "[quantization][gem]") { + constexpr size_t DIM = 8; + constexpr size_t K = 4; + auto threadpool = svs::threads::as_threadpool(1); + // Shared by every case below that just needs *some* valid corpus/centroids; only the + // 256-per-centroid bound needs a differently-shaped corpus to trigger that branch. + auto centers = make_random_centers(K, DIM, 1); + auto corpus = + make_clustered_corpus(centers, K, DIM, std::vector(10, 4), 2, 0.05F); + + CATCH_SECTION("empty tokens dataset") { + auto empty = svs::data::VectorSetData( + svs::data::SimpleData(0, DIM), std::vector{0} + ); + CATCH_REQUIRE_THROWS( + gem::train_codebook(empty, gem::CodebookParameters{}, threadpool) + ); + } + + CATCH_SECTION("num_centroids above the uint32_t id range") { + gem::CodebookParameters parameters; + parameters.num_centroids = + static_cast(std::numeric_limits::max()) + 1; + CATCH_REQUIRE_THROWS(gem::train_codebook(corpus, parameters, threadpool)); + } + + CATCH_SECTION("fewer training tokens than K") { + gem::CodebookParameters parameters; + parameters.num_centroids = 1'000; // far more centroids than tokens sampled + CATCH_REQUIRE_THROWS(gem::train_codebook(corpus, parameters, threadpool)); + } + + CATCH_SECTION("training points exceed faiss's 256-per-centroid bound") { + // One passage, 600 tokens; colbert_sample_size caps at 1 passage as training set. + auto dense_corpus = make_clustered_corpus(centers, K, DIM, {600}, 6, 0.05F); + gem::CodebookParameters parameters; + parameters.num_centroids = 2; // 600ish training tokens > 2 * 256 == 512 + parameters.kmeans_algorithm = + gem::KMeansAlgorithm::Lloyd; // the bound is Lloyd-only + CATCH_REQUIRE_THROWS(gem::train_codebook(dense_corpus, parameters, threadpool)); + } + + CATCH_SECTION("assign_tokens token/centroid dimension mismatch") { + svs::data::SimpleData bad_centroids(K, DIM + 1); + CATCH_REQUIRE_THROWS( + gem::assign_tokens(corpus, bad_centroids, gem::CodebookParameters{}, threadpool) + ); + } + + CATCH_SECTION("coarse_clusters num_coarse exceeds K") { + auto centroids = centroids_from(centers, K, DIM); + gem::CodebookParameters parameters; + parameters.num_coarse = K + 1; + CATCH_REQUIRE_THROWS(gem::coarse_clusters(centroids, parameters, threadpool)); + } +} + +CATCH_TEST_CASE("GEM Codebook - k-means algorithm selection", "[quantization][gem]") { + CATCH_SECTION("default CodebookParameters selects EMeans") { + CATCH_REQUIRE( + gem::CodebookParameters{}.kmeans_algorithm == gem::KMeansAlgorithm::EMeans + ); + } + + CATCH_SECTION("EMeans trains K centroids of the right dimension, finite objective") { + constexpr size_t DIM = 8; + constexpr size_t K = 4; + constexpr size_t NUM_PASSAGES = 40; + auto centers = make_random_centers(K, DIM, 0x2024); + auto corpus = make_clustered_corpus( + centers, K, DIM, std::vector(NUM_PASSAGES, 2), 0xaaaa, 0.1F + ); + gem::CodebookParameters parameters; + parameters.num_centroids = K; + parameters.kmeans_algorithm = gem::KMeansAlgorithm::EMeans; + parameters.emeans_epochs = 1; + auto threadpool = svs::threads::as_threadpool(2); + + auto result = gem::train_codebook(corpus, parameters, threadpool); + + CATCH_REQUIRE(result.centroids.size() == K); + CATCH_REQUIRE(result.centroids.dimensions() == DIM); + CATCH_REQUIRE(std::isfinite(result.objective)); + } +}