Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
68 commits
Select commit Hold shift + click to select a range
d84131b
feat(distance): add quantized EMD and Chamfer distances for token-set…
ahuber21 Oct 30, 2026
62ae84e
feat(vamana): let the GEM distances drive Vamana builds
ahuber21 Oct 30, 2026
2fd1670
feat(data): add vector-set dataset concept and non-owning CSR view
ahuber21 Nov 12, 2026
d558cbc
feat(distance): add MaxSim score over vector sets
ahuber21 Nov 12, 2026
c91096c
test(distance): cover the GEM distances and register the new tests
ahuber21 Nov 12, 2026
a89d857
feat(vamana): re-rank the candidate pool with MaxSim
ahuber21 Nov 12, 2026
aac09b4
feat(distance): let the GEM distances read variable-length vector sets
ahuber21 Nov 12, 2026
5814b8b
fix(vamana): make the MaxSim re-rank and its tests build and pass
ahuber21 Nov 13, 2026
35feecc
fix(distance): keep QEmd duplicate merging callable from outside
ahuber21 Nov 13, 2026
20d86cb
feat(data): add an owning vector-set container with save/load
ahuber21 Nov 13, 2026
dfae7e0
feat(data): save and load WithVectorSets as one dataset
ahuber21 Nov 13, 2026
e7813fd
refactor(distance)!: drop the padded-row layout from GEM distances
ahuber21 Nov 13, 2026
b362c83
feat(distance): add a register-blocked inner-product matrix kernel
ahuber21 Nov 13, 2026
24e9220
fix(distance): stop sharing QChamferExactImpl's reduction across ISA TUs
ahuber21 Nov 13, 2026
aed9285
feat(distance): rerank MaxSim with a column-max inner-product kernel
ahuber21 Nov 13, 2026
9417576
refactor(distance): keep ISA dispatch for MaxSim's kernel inside Inne…
ahuber21 Nov 13, 2026
01adc9d
perf(distance): build CentroidTable with the inner-product kernel, ea…
ahuber21 Nov 13, 2026
99de3c4
feat(distance): add a bfloat16 A operand and bf16-rounded B packing t…
ahuber21 Nov 13, 2026
f16125e
feat(distance): store GEM's fine centroids and ground-cost table as b…
ahuber21 Nov 13, 2026
2f217c1
feat(distance): add an AMX ISA level with a gap for the AVX512 split
ahuber21 Nov 13, 2026
d24f03b
feat(distance): AMX tile kernels for InnerProductMatrix's bf16 A
ahuber21 Nov 13, 2026
5219fda
test(distance): cover the AMX kernel over every tile tail shape
ahuber21 Nov 13, 2026
0de173e
feat(distance): route bfloat16-document MaxSim through the bf16/AMX k…
ahuber21 Nov 13, 2026
0c5c189
test(distance): guard-page memory-safety test for the tile kernels' t…
ahuber21 Nov 13, 2026
b3b9a45
fix(distance): give every AMX tile memory access a real dependency
ahuber21 Nov 13, 2026
2b3e02d
fix(distance): compile amx.cpp at the AVX-512 level's flags, not sapp…
ahuber21 Nov 13, 2026
0793c54
docs(distance): document AMX permission's provenance and host impact
ahuber21 Nov 13, 2026
231b019
refactor(kmeans): move the generic k-means out of the IVF index
ahuber21 Nov 17, 2026
4acd5e6
test(kmeans): add a unit test for the shared k-means header
ahuber21 Nov 17, 2026
451c01c
build: add the multi-vector flag and build k-means under it
ahuber21 Nov 17, 2026
cbab021
feat(vamana): count build distance calls per phase behind a stats flag
ahuber21 Nov 17, 2026
e7f686c
feat(distance): add GemRouter for coarse-cluster routing
ahuber21 Nov 17, 2026
fae6fea
feat(vamana): compute search start nodes per query through an extensi…
ahuber21 Nov 17, 2026
db68f34
fix(vamana): stop taking an L2 medoid over centroid-id sets
ahuber21 Nov 17, 2026
152d320
feat(vamana): time the MaxSim re-rank apart from the graph search beh…
ahuber21 Nov 17, 2026
78b4c59
fix(distance): stop the vendored POT macros leaking out of gem.h
ahuber21 Nov 17, 2026
fc93ea3
feat(quantization): train the GEM codebook and assign tokens inside SVS
ahuber21 Nov 17, 2026
8f4b39e
feat(vamana): pick insert start nodes through an extension hook
ahuber21 Nov 17, 2026
b446a12
feat(vamana): check and repair graph connectivity after a build
ahuber21 Nov 17, 2026
e1c2dca
feat(vamana): build the GEM graph with routed starts and representati…
ahuber21 Nov 17, 2026
858fa45
refactor(quantization): move GEM quantize timings into an opt-in header
ahuber21 Nov 17, 2026
3409289
refactor(distance): define the inner-product matrix kernels in their …
ahuber21 Nov 17, 2026
4248763
refactor(vamana): collect build stats through a compile-time collector
ahuber21 Nov 17, 2026
e5eaeee
test(vamana): consolidate the GEM, MaxSim and connectivity tests
ahuber21 Nov 17, 2026
b65458d
feat(distance): solve qEMD with an in-house transportation simplex
ahuber21 Nov 17, 2026
5dc64b2
chore(third-party): drop the vendored POT network simplex
ahuber21 Nov 17, 2026
58f931e
test(gem): consolidate the router, codebook and vector-set tests
ahuber21 Nov 17, 2026
cdc17a4
test(distance): consolidate the inner-product matrix, MaxSim and k-me…
ahuber21 Nov 17, 2026
f6b7549
test(distance): consolidate the GEM distance and transport solver tests
ahuber21 Nov 17, 2026
a46ee1f
refactor(distance): split out the reference transportation simplex
ahuber21 Nov 18, 2026
848d21e
fix(distance): reject a non-positive last demand after the balance fold
ahuber21 Nov 18, 2026
27c51a9
perf(distance): add float row-pricing kernels for the transportation …
ahuber21 Nov 18, 2026
e4e4e7e
perf(distance): solve qEMD with a LEMON-derived network simplex
ahuber21 Nov 18, 2026
8fca189
test(distance): cross-check the optimized simplex at larger and degen…
ahuber21 Nov 18, 2026
26f8ae7
docs(third-party): add POT and LEMON license notices
ahuber21 Nov 18, 2026
1df3cff
perf(distance): skip non-improving blocks in the row-pricing argmin
ahuber21 Nov 18, 2026
56282ca
perf(distance): hoist workspace pointers in the simplex pivot path
ahuber21 Nov 18, 2026
3116194
perf(distance): back CentroidTable with the hugepage allocator
ahuber21 Nov 18, 2026
705115b
perf(distance): keep qEMD cost lookups row-local in transposed solves
ahuber21 Nov 18, 2026
6842734
feat(distance): count transport simplex pivots in stats builds
ahuber21 Nov 18, 2026
8a648b2
perf(distance): start qEMD solves from a cheapest-row column order
ahuber21 Nov 18, 2026
2b01046
test(distance): cover start-tree degeneracy in the qEMD solver cross-…
ahuber21 Nov 18, 2026
be14c2d
feat(kmeans): add e-means mini-batch k-means training
ahuber21 Nov 18, 2026
207c339
feat(gem): select Lloyd or e-means for codebook training
ahuber21 Nov 18, 2026
5c650da
perf(vamana): prune back-edges one node per work unit, largest first
ahuber21 Nov 18, 2026
e3b0f73
feat(gem): train the codebook with e-means by default
ahuber21 Nov 18, 2026
a8543ef
test(vamana): check that a one-thread static build is deterministic
ahuber21 Nov 19, 2026
8593a4d
feat(distance): select the transport-simplex start at run time
ahuber21 Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ include("cmake/fmt.cmake")
include("cmake/spdlog.cmake")
include("cmake/toml.cmake")

# IVF requires Intel(R) MKL support
if(SVS_EXPERIMENTAL_ENABLE_IVF)
# IVF and multi-vector both train k-means codebooks, which need Intel(R) MKL support.
if(SVS_EXPERIMENTAL_ENABLE_IVF OR SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR)
include("cmake/mkl.cmake")
target_compile_options(${SVS_LIB} INTERFACE "-DSVS_HAVE_MKL=1")
endif()
Expand Down
61 changes: 61 additions & 0 deletions THIRD-PARTY-PROGRAMS
Original file line number Diff line number Diff line change
Expand Up @@ -217,3 +217,64 @@ THE SOFTWARE.
-- NOTE: Third party dependency used by this software --
This software depends on the fmt lib (MIT License),
and users must comply to its license: https://raw.githubusercontent.com/fmtlib/fmt/master/LICENSE

--------------------------------------------------------------------------------
9. POT (include/svs/core/distance/transport_simplex.h, https://github.com/PythonOT/POT/tree/0.9.5)

MIT License

Copyright (c) 2016-2023 POT contributors

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

--------------------------------------------------------------------------------
10. LEMON (include/svs/core/distance/transport_simplex.h, https://lemon.cs.elte.hu)

LEMON code without an explicit copyright notice is covered by the following
copyright/license.

Copyright (C) 2003-2012 Egervary Jeno Kombinatorikus Optimalizalasi
Kutatocsoport (Egervary Combinatorial Optimization Research Group,
EGRES).

===========================================================================
Boost Software License, Version 1.0
===========================================================================

Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:

The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
2 changes: 1 addition & 1 deletion cmake/mkl.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ if (SVS_EXPERIMENTAL_BUILD_CUSTOM_MKL)
NO_DEFAULT_PATH
REQUIRED
)
if (SVS_EXPERIMENTAL_ENABLE_IVF)
if (SVS_EXPERIMENTAL_ENABLE_IVF OR SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR)
set(mkl_functions_file ${CMAKE_CURRENT_LIST_DIR}/mkl_functions_ivf)
else()
set(mkl_functions_file ${CMAKE_CURRENT_LIST_DIR}/mkl_functions)
Expand Down
43 changes: 42 additions & 1 deletion cmake/multi-arch.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,54 @@
# limitations under the License.

set(SVS_X86_SRC_DIR "${PROJECT_SOURCE_DIR}/include/svs/multi-arch/x86")
# Tuple format: source,name,arch[,extra_option,extra_option,...]. Fields beyond the
# first three are optional and additive: an existing tuple (no 4th+ field) still gets
# exactly -march=${arch} -mtune=${arch}, unchanged; a tuple that adds fields gets those
# two flags plus every extra option, appended verbatim. amx.cpp uses this to compile at
# exactly the AVX-512 level's instruction set (arch=cascadelake, matching the avx512
# tuple below) plus the two AMX extensions, rather than at -march=sapphirerapids: that
# also enables AVX512-FP16/BF16/AVX-VNNI/etc, which the runtime probe
# (amx_tile_bf16_available()) never tests and a hypervisor could expose AMX without, so
# the compiler must not be free to use them when auto-vectorizing ordinary code in that
# file.
set(SVS_X86
"${SVS_X86_SRC_DIR}/avx2.cpp,avx2,haswell"
"${SVS_X86_SRC_DIR}/avx512.cpp,avx512,cascadelake"
)

# AMX (tile load/store/config plus the bf16 dot-product tile instruction) is optional: a
# compiler that rejects -mamx-tile/-mamx-bf16 (too old, or a non-x86 cross build) must
# still produce a working library with the level compiled out, so amx.cpp is added to
# SVS_X86 only when this check passes, and SVS_AMX_TILE_BF16_SUPPORTED (read by
# include/svs/lib/preprocessor.h) is defined only then.
include(CheckCXXCompilerFlag)
check_cxx_compiler_flag("-mamx-tile -mamx-bf16" SVS_COMPILER_SUPPORTS_AMX)
if(SVS_COMPILER_SUPPORTS_AMX)
list(APPEND SVS_X86 "${SVS_X86_SRC_DIR}/amx.cpp,amx,cascadelake,-mamx-tile,-mamx-bf16")
endif()

set(SVS_X86_OBJECT_FILES)
foreach(x86_info IN LISTS SVS_X86)
string(REPLACE "," ";" x86_info "${x86_info}")
list(LENGTH x86_info x86_info_len)
list(GET x86_info 0 src)
list(GET x86_info 1 avx)
list(GET x86_info 2 arch)
set(lib_name "svs_x86_${avx}")
add_library(${lib_name} INTERFACE)
target_compile_options(${lib_name} INTERFACE -march=${arch} -mtune=${arch})
set(obj_name ${arch}_obj)
# Object target names key off `arch` by default, but the amx tuple above
# intentionally reuses cascadelake's `arch` (same instruction set as the avx512
# tuple, see the comment above SVS_X86): when extra options are present, the object
# target is named from `avx` instead, so it doesn't collide with that other tuple's
# target of the same name.
if(x86_info_len GREATER 3)
list(SUBLIST x86_info 3 -1 extra_options)
target_compile_options(${lib_name} INTERFACE ${extra_options})
set(obj_name ${avx}_obj)
else()
set(obj_name ${arch}_obj)
endif()

add_library(${obj_name} OBJECT ${src})
target_link_libraries(${obj_name} PRIVATE ${SVS_LIB} svs::compile_options fmt::fmt ${lib_name})
Expand All @@ -37,6 +70,14 @@ endforeach()
add_library(svs_x86_objects STATIC ${SVS_X86_OBJECT_FILES})
target_link_libraries(svs_export INTERFACE svs_x86_objects)

if(SVS_COMPILER_SUPPORTS_AMX)
# Uniform across every consumer of svs_export, unlike the per-file -march flags above:
# PackedOperand (inner_product_matrix.h) decides whether to fill the AMX tile layout
# from this, and it is included by TUs that never pass -march=sapphirerapids
# themselves (gem.h, maxsim.h, tests).
target_compile_definitions(svs_export INTERFACE SVS_AMX_TILE_BF16_SUPPORTED)
endif()

install(
TARGETS svs_x86_objects
EXPORT svs-targets
Expand Down
22 changes: 22 additions & 0 deletions cmake/options.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,16 @@ option(SVS_EXPERIMENTAL_ENABLE_IVF
OFF # disabled by default
)

option(SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR
"Enable multi-vector (late interaction) support. Requires Intel(R) MKL support"
OFF # disabled by default
)

option(SVS_EXPERIMENTAL_ENABLE_STATS
"Enable build and search measurement counters"
OFF # disabled by default
)

#####
##### svsbenchmark
#####
Expand Down Expand Up @@ -163,6 +173,18 @@ else()
target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_IVF=0)
endif()

if (SVS_EXPERIMENTAL_ENABLE_MULTI_VECTOR)
target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_MULTI_VECTOR=1)
else()
target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_MULTI_VECTOR=0)
endif()

if (SVS_EXPERIMENTAL_ENABLE_STATS)
target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_STATS=1)
else()
target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_STATS=0)
endif()

#####
##### Helper target to apply relevant compiler optimizations.
#####
Expand Down
109 changes: 109 additions & 0 deletions include/svs/concepts/vector_set.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
* Copyright 2026 Intel Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma once

///
/// @ingroup concepts
/// @defgroup vector_set_concept_entry Concepts modeling late-interaction vector-set
/// datasets.
///

#include <concepts>
#include <cstddef>
#include <span>

namespace svs {
namespace data {

// clang-format off

///
/// @ingroup vector_set_concept_entry
/// @brief A variable-cardinality set of token vectors, as used by late-interaction
/// retrieval (e.g. one set per document, one token vector per word-piece embedding).
///
/// @code{.cpp}
/// template <typename S>
/// concept VectorSet = requires(const S& s, size_t i) {
/// // The element type of each token vector.
/// typename S::element_type;
///
/// // The number of token vectors in the set.
/// { s.cardinality() } -> std::convertible_to<size_t>;
///
/// // The dimensionality shared by every token vector in the set.
/// { s.dimensions() } -> std::convertible_to<size_t>;
///
/// // The token vector at index ``i``, with ``0 <= i < cardinality()``.
/// { s.token(i) } -> std::convertible_to<std::span<const typename S::element_type>>;
/// };
/// @endcode
///
template <typename S>
concept VectorSet = requires(const S& s, size_t i) {
typename S::element_type;
{ s.cardinality() } -> std::convertible_to<size_t>;
{ s.dimensions() } -> std::convertible_to<size_t>;
{ s.token(i) } -> std::convertible_to<std::span<const typename S::element_type>>;
};

///
/// @ingroup vector_set_concept_entry
/// @brief Compatibility interface for a dataset whose elements are vector sets.
///
/// @code{.cpp}
/// template <typename D>
/// concept ImmutableVectorSetDataset = requires(const D& d, size_t i) {
/// // The value returned by ``get_datum`` must itself model ``VectorSet``.
/// requires VectorSet<typename D::const_value_type>;
///
/// // The number of vector sets in the dataset.
/// { d.size() } -> std::convertible_to<size_t>;
///
/// // The dimensionality shared by every token vector in the dataset.
/// { d.dimensions() } -> std::convertible_to<size_t>;
///
/// // A constant handle to the vector set at index ``i``.
/// { d.get_datum(i) } -> std::convertible_to<typename D::const_value_type>;
///
/// // Performance optimization. Prefetch the vector set at index ``i``.
/// d.prefetch(i);
/// };
/// @endcode
///
template <typename D>
concept ImmutableVectorSetDataset = requires(const D& d, size_t i) {
requires VectorSet<typename D::const_value_type>;
{ d.size() } -> std::convertible_to<size_t>;
{ d.dimensions() } -> std::convertible_to<size_t>;
{ d.get_datum(i) } -> std::convertible_to<typename D::const_value_type>;
d.prefetch(i);
};

///
/// @ingroup vector_set_concept_entry
/// @brief A ``VectorSet`` whose tokens are stored row-major and contiguous, reachable
/// from ``data()``.
///
template <typename S>
concept ContiguousVectorSet = VectorSet<S> && requires(const S& s) {
{ s.data() } -> std::convertible_to<const typename S::element_type*>;
};
// clang-format on

} // namespace data
} // namespace svs
Loading
Loading