From be989d4dd23a6d239f7c984979680743ee986117 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 09:23:39 +0200 Subject: [PATCH 01/11] SQ4: Add packed 4-bit scalar quantization for standalone HNSW Adds VecSimQuant_SQ4: FP32 input, IP and L2 (Cosine on pre-normalized vectors resolves to IP), optional mean centering, standalone HNSW only. Codes are packed two per byte, followed by the SQ8 metadata. Distances use the generic scalar kernels from the StorageType split; the FP32-query IP falls back to per-element reconstruction on a non-finite result. Unsupported configurations are rejected with NULL, and the existing V5 serializer stores the quant type and mean. Co-Authored-By: Claude Opus 5.5 --- src/VecSim/algorithms/hnsw/hnsw_serializer.h | 2 +- .../algorithms/hnsw/hnsw_serializer_impl.h | 4 +- .../components/components_factory.h | 21 + src/VecSim/index_factories/hnsw_factory.cpp | 56 +- src/VecSim/index_factories/tiered_factory.cpp | 3 + src/VecSim/spaces/IP/IP.cpp | 34 ++ src/VecSim/spaces/IP/IP.h | 9 + src/VecSim/spaces/IP_space.cpp | 18 + src/VecSim/spaces/IP_space.h | 6 + src/VecSim/spaces/L2/L2.cpp | 13 + src/VecSim/spaces/L2/L2.h | 4 + src/VecSim/spaces/L2_space.cpp | 18 + src/VecSim/spaces/L2_space.h | 6 + src/VecSim/spaces/computer/preprocessors.h | 27 +- src/VecSim/spaces/spaces.cpp | 35 ++ src/VecSim/types/sq4.h | 66 +++ src/VecSim/vec_sim_common.h | 4 +- tests/unit/CMakeLists.txt | 3 + tests/unit/test_hnsw_sq4.cpp | 561 ++++++++++++++++++ tests/unit/test_hnsw_sq8.cpp | 2 +- tests/unit/test_sq4_components.cpp | 317 ++++++++++ tests/unit/test_sq4_distances.cpp | 349 +++++++++++ 22 files changed, 1540 insertions(+), 18 deletions(-) create mode 100644 src/VecSim/types/sq4.h create mode 100644 tests/unit/test_hnsw_sq4.cpp create mode 100644 tests/unit/test_sq4_components.cpp create mode 100644 tests/unit/test_sq4_distances.cpp diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer.h b/src/VecSim/algorithms/hnsw/hnsw_serializer.h index dabc6f379..f27ef694c 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer.h @@ -24,7 +24,7 @@ class HNSWSerializer : public Serializer { DEPRECATED = 2, // Last deprecated version V3, V4, - V5, // SQ8 quantization fields (quantType + mean vector) + V5, // scalar quantization fields: SQ8/SQ4 (quantType + mean vector) INVALID }; diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h index e72036a81..d8ec348ce 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h @@ -263,10 +263,10 @@ void HNSWIndex::saveIndexFields(std::ofstream &output) const writeBinaryPOD(output, this->maxElements); // This will be used to restore the index initial // capacity - // V5 fields: SQ8 quantization support (between factory params and index build params) + // V5 fields: SQ8/SQ4 quantization support (between factory params and index build params) if (this->quantType != VecSimQuant_NONE) { writeBinaryPOD(output, this->quantType); - // Write mean vector (dim floats). Empty vector means no mean (zero-mean SQ8). + // Write mean vector (dim floats). Empty vector means no mean (zero-mean quantization). bool hasMean = !this->serializedMeanVector.empty(); writeBinaryPOD(output, hasMean); if (hasMean) { diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index 39fc2486a..9b97165e6 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -113,6 +113,27 @@ CreateSQ8IndexComponents(const std::shared_ptr &allocator, size mean_ptr); } +template +IndexComponents +CreateSQ4IndexComponents(const std::shared_ptr &allocator, size_t dim, + const void *mean_ptr) { + static_assert(std::is_same_v, "SQ4 supports FP32 input only"); + static_assert(Metric == VecSimMetric_IP || Metric == VecSimMetric_L2, + "SQ4 supports IP and L2 only"); + return CreateQuantizedIndexComponents( + allocator, dim, static_cast(mean_ptr)); +} + +template +IndexComponents +CreateSQ4IndexComponents(const std::shared_ptr &allocator, size_t dim, + const void *mean_ptr) { + if (mean_ptr) { + return CreateSQ4IndexComponents(allocator, dim, mean_ptr); + } + return CreateSQ4IndexComponents(allocator, dim, mean_ptr); +} + template size_t EstimateComponentsMemory(VecSimMetric metric, bool is_normalized) { size_t allocations_overhead = VecSimAllocator::getAllocationOverheadSize(); diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index c9ce8d642..bcb625548 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -14,12 +14,14 @@ #include "VecSim/algorithms/hnsw/hnsw.h" #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" +#include "VecSim/types/sq4.h" #include using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; using sq8 = vecsim_types::sq8; +using sq4 = vecsim_types::sq4; namespace HNSWFactory { @@ -56,10 +58,13 @@ template VecSimType type, VecSimMetric resolved_metric) { // Reject unknown quantizers instead of silently creating an unquantized index. - if (quant_type != VecSimQuant_SQ8) { + if (quant_type != VecSimQuant_SQ8 && quant_type != VecSimQuant_SQ4) { return false; } - // SQ8 kernels accept only FLOAT32 and FLOAT16 input. + if (quant_type == VecSimQuant_SQ4 && type != VecSimType_FLOAT32) { + return false; + } + // SQ8 additionally accepts FLOAT16 input. if (type != VecSimType_FLOAT32 && type != VecSimType_FLOAT16) { return false; } @@ -106,6 +111,15 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { return NULL; } + if (hnswParams->quantType == VecSimQuant_SQ4) { + if (metric == VecSimMetric_L2) { + return NewIndex_Quantized( + hnswParams, abstractInitParams, mean_ptr); + } + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); + } + if (hnswParams->type == VecSimType_FLOAT32) { if (metric == VecSimMetric_L2) { return NewIndex_Quantized(hnswParams, abstractInitParams, @@ -196,7 +210,12 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi ResolveQuantizedMetric(params->metric, is_normalized))) { throw std::invalid_argument("Unsupported quantization params for HNSW index"); } - // Template arguments do not affect these component sizes. + // Template arguments do not affect these component sizes. The storage type does not + // either; the asserts keep that true, so one estimate serves SQ8 and SQ4. + static_assert(sizeof(QuantPreprocessor) == + sizeof(QuantPreprocessor)); + static_assert(sizeof(QuantPreprocessor) == + sizeof(QuantPreprocessor)); if (with_mean) { est += allocations_overhead + sizeof(DistanceCalculatorWithNorm); @@ -245,7 +264,12 @@ size_t EstimateElementSize(const HNSWParams *params, bool with_mean) { size_t stored_data_size; // Preserve the element estimator's existing contract: return a size and validate in NewIndex. - if (params->quantType == VecSimQuant_SQ8) { + if (params->quantType == VecSimQuant_SQ4) { + stored_data_size = + params->metric == VecSimMetric_L2 + ? GetQuantizedStoredDataSize(params->dim, with_mean) + : GetQuantizedStoredDataSize(params->dim, with_mean); + } else if (params->quantType == VecSimQuant_SQ8) { if (params->metric == VecSimMetric_L2) { stored_data_size = GetQuantizedStoredDataSize(params->dim, with_mean); } else { @@ -323,11 +347,12 @@ static bool InitializeParams(std::ifstream &source_params, HNSWParams ¶ms, // VecSimQuantType, so it must never be loaded into the enum. std::underlying_type_t raw_quant_type; Serializer::readBinaryPOD(source_params, raw_quant_type); - if (raw_quant_type != VecSimQuant_NONE && raw_quant_type != VecSimQuant_SQ8) { + if (raw_quant_type != VecSimQuant_NONE && raw_quant_type != VecSimQuant_SQ8 && + raw_quant_type != VecSimQuant_SQ4) { return false; } params.quantType = static_cast(raw_quant_type); - if (params.quantType == VecSimQuant_SQ8) { + if (params.quantType != VecSimQuant_NONE) { bool hasMean = false; Serializer::readBinaryPOD(source_params, hasMean); if (hasMean) { @@ -379,6 +404,25 @@ VecSimIndex *NewIndex(const std::string &location, bool is_normalized) { return NULL; } + if (params.quantType == VecSimQuant_SQ4) { + abstractInitParams.isQuantized = true; + if (metric == VecSimMetric_L2) { + abstractInitParams.storedDataSize = + GetQuantizedStoredDataSize(params.dim, + mean_ptr != nullptr); + auto components = CreateSQ4IndexComponents( + abstractInitParams.allocator, params.dim, mean_ptr); + return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, + components, version, mean_ptr); + } + abstractInitParams.storedDataSize = + GetQuantizedStoredDataSize(params.dim, mean_ptr != nullptr); + auto components = CreateSQ4IndexComponents( + abstractInitParams.allocator, params.dim, mean_ptr); + return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, + components, version, mean_ptr); + } + // Override blob sizes for SQ8 storage layout. size_t dim = params.dim; abstractInitParams.isQuantized = true; diff --git a/src/VecSim/index_factories/tiered_factory.cpp b/src/VecSim/index_factories/tiered_factory.cpp index c638a8495..56d718d5f 100644 --- a/src/VecSim/index_factories/tiered_factory.cpp +++ b/src/VecSim/index_factories/tiered_factory.cpp @@ -104,6 +104,9 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) { inline size_t EstimateInitialSize(const TieredIndexParams *params) { const auto &hnsw_params = params->primaryIndexParams->algoParams.hnswParams; + if (hnsw_params.quantType == VecSimQuant_SQ4) { + throw std::invalid_argument("Tiered HNSW SQ4 is not supported"); + } const bool requires_training = RequiresQuantizationTraining(params); const bool with_mean = requires_training || hnsw_params.quantParams != nullptr; size_t est = HNSWFactory::EstimateInitialSize(&hnsw_params, true, with_mean); diff --git a/src/VecSim/spaces/IP/IP.cpp b/src/VecSim/spaces/IP/IP.cpp index 136617c62..b075764f2 100644 --- a/src/VecSim/spaces/IP/IP.cpp +++ b/src/VecSim/spaces/IP/IP.cpp @@ -11,11 +11,45 @@ #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq4.h" #include "VecSim/utils/alignment.h" +#include using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; using sq8 = vecsim_types::sq8; +using sq4 = vecsim_types::sq4; + +// Per-element reconstruction of the SQ4-FP32 inner product distance. Slower than the matched +// form, but it never forms the unscaled code dot product, which can overflow float for large query +// values. +float SQ4_FP32_InnerProductReconstructed(const void *pVect1v, const void *pVect2v, + size_t dimension) { + const auto *x = static_cast(pVect1v); + const auto *y = static_cast(pVect2v); + // Metadata follows the packed payload and is not necessarily float-aligned. + const auto *params = x + sq4::data_bytes_count(dimension); + const float min_val = load_unaligned(params + sq4::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params + sq4::DELTA * sizeof(float)); + float dot = 0.0f; + for (size_t i = 0; i < dimension; ++i) { + dot += (min_val + delta * sq4::load_code(x, i)) * y[i]; + } + return 1.0f - dot; +} + +float SQ4_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) { + const float dist = + 1.0f - spaces::quantized::FP32_InnerProduct_Impl(pVect1v, pVect2v, dimension); + if (!std::isfinite(dist)) { + return SQ4_FP32_InnerProductReconstructed(pVect1v, pVect2v, dimension); + } + return dist; +} + +float SQ4_SQ4_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) { + return 1.0f - spaces::quantized::Stored_InnerProduct_Impl(pVect1v, pVect2v, dimension); +} /* * Optimized asymmetric SQ8-FP32 inner product using algebraic identity: diff --git a/src/VecSim/spaces/IP/IP.h b/src/VecSim/spaces/IP/IP.h index 3566e8d73..8c291dbf0 100644 --- a/src/VecSim/spaces/IP/IP.h +++ b/src/VecSim/spaces/IP/IP.h @@ -10,6 +10,15 @@ #include +// Packed SQ4 storage, compared with FP32 queries or another SQ4 vector. +float SQ4_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension); +float SQ4_SQ4_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension); +// Per-element reconstruction of the SQ4-FP32 inner product distance (1 - IP). SQ4_FP32_InnerProduct +// falls back to it when the matched form is not finite; SIMD SQ4-FP32 kernels can do the same. +// Returns a distance; callers return it as is (no 1 - x). +float SQ4_FP32_InnerProductReconstructed(const void *pVect1v, const void *pVect2v, + size_t dimension); + // SQ8-FP32: Common inner product implementation that returns the raw inner product value // (not distance). Used by SQ8_FP32_InnerProduct, SQ8_FP32_Cosine, and SQ8_FP32_L2Sqr. // pVect1 is storage (SQ8): [uint8_t values (dim)] [min_val] [delta] [x_sum] [x_sum_squares (L2 diff --git a/src/VecSim/spaces/IP_space.cpp b/src/VecSim/spaces/IP_space.cpp index 7ae4da228..224e28bb6 100644 --- a/src/VecSim/spaces/IP_space.cpp +++ b/src/VecSim/spaces/IP_space.cpp @@ -39,6 +39,24 @@ using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; namespace spaces { +// SQ4-FP32: packed storage and unaligned metadata, so no alignment requirement. +dist_func_t IP_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment, + const void *arch_opt) { + if (alignment) { + *alignment = 0; + } + return SQ4_FP32_InnerProduct; +} + +// SQ4-SQ4: packed storage and unaligned metadata, so no alignment requirement. +dist_func_t IP_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment, + const void *arch_opt) { + if (alignment) { + *alignment = 0; + } + return SQ4_SQ4_InnerProduct; +} + // SQ8-FP32: asymmetric distance between SQ8 storage and FP32 query dist_func_t IP_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment, const void *arch_opt) { diff --git a/src/VecSim/spaces/IP_space.h b/src/VecSim/spaces/IP_space.h index 85022e5c9..35530cea5 100644 --- a/src/VecSim/spaces/IP_space.h +++ b/src/VecSim/spaces/IP_space.h @@ -10,6 +10,12 @@ #include "VecSim/spaces/spaces.h" namespace spaces { +// Packed SQ4 storage has no alignment requirement, including its metadata. +dist_func_t IP_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, + const void *arch_opt = nullptr); +dist_func_t IP_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, + const void *arch_opt = nullptr); + // SQ8-FP32: asymmetric distance between FP32 query and SQ8 storage dist_func_t IP_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, const void *arch_opt = nullptr); diff --git a/src/VecSim/spaces/L2/L2.cpp b/src/VecSim/spaces/L2/L2.cpp index 2bc7f2cfb..90a7caedd 100644 --- a/src/VecSim/spaces/L2/L2.cpp +++ b/src/VecSim/spaces/L2/L2.cpp @@ -12,12 +12,25 @@ #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq4.h" #include "VecSim/utils/alignment.h" #include using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; using sq8 = vecsim_types::sq8; +using sq4 = vecsim_types::sq4; + +float SQ4_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { + return spaces::quantized::FP32_L2Sqr(pVect1v, pVect2v, dimension); +} + +// SQ4-SQ4 L2 squared distance with the same identity as SQ8-SQ8: +// ||x - y||^2 = ||x||^2 + ||y||^2 - 2*IP(x, y) +// from the stored SUM_SQUARES metadata and the stored-to-stored inner product. +float SQ4_SQ4_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { + return spaces::quantized::Stored_L2Sqr(pVect1v, pVect2v, dimension); +} /* * Asymmetric SQ8-FP32 L2 squared distance, accumulated directly per component: diff --git a/src/VecSim/spaces/L2/L2.h b/src/VecSim/spaces/L2/L2.h index 9afbaeb43..bb75ccf72 100644 --- a/src/VecSim/spaces/L2/L2.h +++ b/src/VecSim/spaces/L2/L2.h @@ -10,6 +10,10 @@ #include +// Packed SQ4 storage, compared with FP32 queries or another SQ4 vector. +float SQ4_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension); +float SQ4_SQ4_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension); + // SQ8-FP32: pVect1v vector of type uint8 (SQ8) and pVect2v vector of type fp32 float SQ8_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension); diff --git a/src/VecSim/spaces/L2_space.cpp b/src/VecSim/spaces/L2_space.cpp index f1ec7f696..63e2c2d60 100644 --- a/src/VecSim/spaces/L2_space.cpp +++ b/src/VecSim/spaces/L2_space.cpp @@ -39,6 +39,24 @@ using float16 = vecsim_types::float16; namespace spaces { +// SQ4-FP32: packed storage and unaligned metadata, so no alignment requirement. +dist_func_t L2_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment, + const void *arch_opt) { + if (alignment) { + *alignment = 0; + } + return SQ4_FP32_L2Sqr; +} + +// SQ4-SQ4: packed storage and unaligned metadata, so no alignment requirement. +dist_func_t L2_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment, + const void *arch_opt) { + if (alignment) { + *alignment = 0; + } + return SQ4_SQ4_L2Sqr; +} + // SQ8-FP32: asymmetric L2 distance between SQ8 storage and FP32 query dist_func_t L2_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment, const void *arch_opt) { diff --git a/src/VecSim/spaces/L2_space.h b/src/VecSim/spaces/L2_space.h index 816644f80..eddc1b7f7 100644 --- a/src/VecSim/spaces/L2_space.h +++ b/src/VecSim/spaces/L2_space.h @@ -22,6 +22,12 @@ dist_func_t L2_INT8_GetDistFunc(size_t dim, unsigned char *alignment = nu const void *arch_opt = nullptr); dist_func_t L2_UINT8_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, const void *arch_opt = nullptr); +// SQ4-FP32: packed storage has no alignment requirement, including its metadata. +dist_func_t L2_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, + const void *arch_opt = nullptr); +// SQ4-SQ4: packed storage has no alignment requirement, including its metadata. +dist_func_t L2_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, + const void *arch_opt = nullptr); // SQ8-FP32: asymmetric L2 distance between FP32 query and SQ8 storage dist_func_t L2_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, const void *arch_opt = nullptr); diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 262474dd5..e400a7f2d 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -25,6 +25,7 @@ #include "VecSim/memory/memory_utils.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq4.h" #include "VecSim/utils/vecsim_stl.h" class PreprocessorInterface : public VecsimBaseObject { @@ -143,8 +144,8 @@ class CosinePreprocessor : public PreprocessorInterface { /* * QuantPreprocessor is a preprocessor that quantizes storage vectors from DataType to a - * lower precision representation defined by StorageType (SQ8, the current storage type, - * stores one uint8_t code per coordinate). + * lower precision representation (SQ8 stores one uint8_t code per coordinate, SQ4 packs + * two 4-bit codes per byte). * Query vectors remain as DataType for asymmetric distance computation. * * The quantized storage blob contains the quantized values along with metadata (min value, @@ -153,7 +154,7 @@ class CosinePreprocessor : public PreprocessorInterface { * and then scaling the values to fit in the storage type's code range. * * Storage layout: - * | quantized_values[dim] | min_val | delta | x_sum | (x_sum_squares for L2 only) | + * | quantized_values (packed for SQ4) | min_val | delta | x_sum | (L2 sum_squares) | * where, writing x_r[i] = min_val + delta * a[i] for the value a blob actually represents: * x_sum = Σx_r[i]: sum of the reconstructed values, * x_sum_squares = Σx_r[i]²: sum of their squares. @@ -388,8 +389,20 @@ class QuantPreprocessor : public PreprocessorInterface { d_delta * d_delta * q_sum_squares); } + if constexpr (WithNorm && Metric == VecSimMetric_IP && + std::is_same_v) { + // Correction must describe the same reconstructed vector as the packed codes. + // Read the emitted codes so this also works when input and output alias. + double reconstructed_mean_ip = 0.0; + for (size_t j = 0; j < this->dim; ++j) { + const double centered = d_min + d_delta * StorageType::load_code(quantized, j); + reconstructed_mean_ip += (centered + mean[j]) * mean[j]; + } + x_mean_ip = static_cast(reconstructed_mean_ip); + } + // Metadata uses MetadataType. Use memcpy because the metadata offset (after the - // quantized body) is not guaranteed to be sizeof(MetadataType)-aligned. + // quantized body, packed for SQ4) is not guaranteed to be sizeof(MetadataType)-aligned. void *meta_dst = quantized + StorageType::data_bytes_count(this->dim); MetadataType buf[5] = {min_val, delta, sum}; size_t n = 3; @@ -512,9 +525,9 @@ class QuantPreprocessor : public PreprocessorInterface { /** * Preprocesses the original blob into separate storage and query blobs. * - * Storage vectors are quantized to StorageType scalar codes (SQ8: one uint8_t per - * coordinate), with metadata (min, delta, sum, and sum_squares for L2) appended for - * distance reconstruction. + * Storage vectors are quantized to scalar codes (SQ8: one uint8_t per coordinate, SQ4: two + * 4-bit codes packed per byte), with metadata (min, delta, sum, and + * sum_squares for L2) appended for distance reconstruction. * * Query vectors use QueryType for asymmetric distance computation, with a precomputed * sum (for IP/Cosine) or sum of squares (for L2) appended for efficient distance calculation. diff --git a/src/VecSim/spaces/spaces.cpp b/src/VecSim/spaces/spaces.cpp index 7df458cd2..1e62c0b5c 100644 --- a/src/VecSim/spaces/spaces.cpp +++ b/src/VecSim/spaces/spaces.cpp @@ -9,6 +9,9 @@ #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq4.h" +#include "VecSim/spaces/IP/IP.h" +#include "VecSim/spaces/L2/L2.h" #include "VecSim/spaces/space_includes.h" #include "VecSim/spaces/spaces.h" #include "VecSim/spaces/IP_space.h" @@ -144,6 +147,38 @@ GetDistFunc(VecSimMetric metric throw std::invalid_argument("Invalid metric"); } +template <> +dist_func_t GetDistFunc(VecSimMetric metric, size_t dim, + unsigned char *alignment) { + if (alignment) { + *alignment = 0; + } + switch (metric) { + case VecSimMetric_IP: + return IP_SQ4_SQ4_GetDistFunc(dim, alignment); + case VecSimMetric_L2: + return L2_SQ4_SQ4_GetDistFunc(dim, alignment); + default: + throw std::invalid_argument("SQ4 supports only IP and L2 metrics"); + } +} + +template <> +dist_func_t GetDistFunc(VecSimMetric metric, size_t dim, + unsigned char *alignment) { + if (alignment) { + *alignment = 0; + } + switch (metric) { + case VecSimMetric_IP: + return IP_SQ4_FP32_GetDistFunc(dim, alignment); + case VecSimMetric_L2: + return L2_SQ4_FP32_GetDistFunc(dim, alignment); + default: + throw std::invalid_argument("SQ4 supports only IP and L2 metrics"); + } +} + template <> normalizeVector_f GetNormalizeFunc(void) { return normalizeVector_imp; diff --git a/src/VecSim/types/sq4.h b/src/VecSim/types/sq4.h new file mode 100644 index 000000000..1bb031ef1 --- /dev/null +++ b/src/VecSim/types/sq4.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include "VecSim/types/sq8.h" + +namespace vecsim_types { + +// Two scalar-quantized coordinates per byte, followed by the same metadata as SQ8. +struct sq4 : sq8 { + static constexpr uint8_t max_code = 15; + + static constexpr size_t data_bytes_count(size_t dim) { return dim / 2 + dim % 2; } + + static uint8_t load_code(const uint8_t *data, size_t index) { + return (data[index / 2] >> ((index % 2) * 4)) & max_code; + } + + // Coordinates are stored in ascending order. Starting each byte with an assignment allows + // in-place preprocessing and leaves the unused high nibble zero for odd dimensions. + static void store_code(uint8_t *data, size_t index, uint8_t code) { + if (index % 2 == 0) { + data[index / 2] = code & max_code; + } else { + data[index / 2] = (data[index / 2] & max_code) | ((code & max_code) << 4); + } + } + + // Code dot hooks bound to the packed load_code above. + static float code_dot(const uint8_t *a, const uint8_t *b, size_t dim) { + float product = 0; + for (size_t i = 0; i < dim; i++) { + product += load_code(a, i) * load_code(b, i); + } + return product; + } + + static float code_dot_fp32(const uint8_t *codes, const float *y, size_t dim) { + float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + size_t i = 0; + size_t dim4 = dim & ~size_t(3); + for (; i < dim4; i += 4) { + sum0 += static_cast(load_code(codes, i + 0)) * y[i + 0]; + sum1 += static_cast(load_code(codes, i + 1)) * y[i + 1]; + sum2 += static_cast(load_code(codes, i + 2)) * y[i + 2]; + sum3 += static_cast(load_code(codes, i + 3)) * y[i + 3]; + } + for (; i < dim; i++) { + sum0 += static_cast(load_code(codes, i)) * y[i]; + } + return (sum0 + sum1) + (sum2 + sum3); + } + + template + static constexpr size_t storage_bytes_count(size_t dim) { + return data_bytes_count(dim) + storage_metadata_count() * sizeof(float); + } +}; + +} // namespace vecsim_types diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 419aa9884..1d7cf7aa3 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -73,6 +73,8 @@ typedef enum { VecSimQuant_NONE = 0, // No quantization (default). // 8-bit scalar quantization with optional mean centering. VecSimQuant_SQ8 = 1, + // Packed 4-bit scalar quantization with optional mean centering (FP32 input). + VecSimQuant_SQ4 = 2, } VecSimQuantType; // Algorithm type/library. @@ -190,7 +192,7 @@ typedef struct { size_t efRuntime; double epsilon; VecSimQuantType quantType; // Defaults to VecSimQuant_NONE. - // SQ8 mean vector (float[dim]); NULL disables mean centering. Copied during construction. + // SQ8/SQ4 mean vector (float[dim]); NULL disables mean centering. Copied during construction. // L2 centering keeps queries in FLOAT32, including when the input type is FLOAT16. const void *quantParams; } HNSWParams; diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 67a8a00f3..fd8e18721 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -35,6 +35,7 @@ else() endif() add_executable(test_hnsw ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw.cpp test_hnsw_multi.cpp test_hnsw_tiered.cpp unit_test_utils.cpp) +add_executable(test_hnsw_sq4 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_sq4.cpp test_sq4_distances.cpp test_sq4_components.cpp unit_test_utils.cpp) add_executable(test_hnsw_sq8 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_sq8.cpp unit_test_utils.cpp) add_executable(test_bruteforce ../utils/test_main_with_timeout.cpp test_bruteforce.cpp test_bruteforce_multi.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) add_executable(test_allocator ../utils/test_main_with_timeout.cpp test_allocator.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) @@ -51,6 +52,7 @@ add_executable(test_svs ../utils/test_main_with_timeout.cpp ../utils/mock_thread target_link_libraries(test_hnsw PUBLIC gtest VectorSimilarity) target_link_libraries(test_hnsw_sq8 PUBLIC gtest VectorSimilarity) +target_link_libraries(test_hnsw_sq4 PUBLIC gtest VectorSimilarity) target_link_libraries(test_bruteforce PUBLIC gtest VectorSimilarity) target_link_libraries(test_allocator PUBLIC gtest VectorSimilarity) target_link_libraries(test_spaces PUBLIC gtest VectorSimilarity) @@ -68,6 +70,7 @@ include(GoogleTest) gtest_discover_tests(test_hnsw) gtest_discover_tests(test_hnsw_sq8) +gtest_discover_tests(test_hnsw_sq4) gtest_discover_tests(test_bruteforce) gtest_discover_tests(test_allocator) gtest_discover_tests(test_spaces) diff --git a/tests/unit/test_hnsw_sq4.cpp b/tests/unit/test_hnsw_sq4.cpp new file mode 100644 index 000000000..aeb16ac2e --- /dev/null +++ b/tests/unit/test_hnsw_sq4.cpp @@ -0,0 +1,561 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#include "gtest/gtest.h" +#include "VecSim/algorithms/hnsw/hnsw_single.h" +#include "VecSim/index_factories/hnsw_factory.h" +#include "VecSim/index_factories/tiered_factory.h" +#include "VecSim/types/sq4.h" +#include "VecSim/vec_sim.h" +#include "unit_test_utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using IndexPtr = std::unique_ptr; +using SQ4Case = std::tuple; +using Results = std::vector>; +class HNSWSQ4Test : public ::testing::TestWithParam {}; + +std::vector reconstruct(const char *blob, size_t dim, const std::vector &mean) { + float metadata[2]; + std::memcpy(metadata, blob + (dim + 1) / 2, sizeof(metadata)); + std::vector result(dim); + for (size_t d = 0; d < dim; ++d) { + auto code = (static_cast(blob[d / 2]) >> (4 * (d % 2))) & 15; + result[d] = metadata[0] + metadata[1] * code + (mean.empty() ? 0 : mean[d]); + } + return result; +} + +double distance(const std::vector &a, const std::vector &b, VecSimMetric metric) { + double sum = 0; + for (size_t d = 0; d < a.size(); ++d) { + double x = a[d], y = b[d]; + sum += metric == VecSimMetric_L2 ? (x - y) * (x - y) : x * y; + } + return metric == VecSimMetric_L2 ? sum : 1 - sum; +} + +std::vector make_mean(bool centered, size_t dim) { + std::vector mean; + if (centered) { + mean.resize(dim); + for (size_t d = 0; d < dim; ++d) + mean[d] = 0.3f * std::sin(float(d + 1)); + } + return mean; +} + +std::vector> random_vectors(size_t count, size_t dim, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution random(-1.0f, 1.0f); + std::vector> vectors(count, std::vector(dim)); + for (auto &vector : vectors) + for (auto &value : vector) + value = random(rng); + return vectors; +} + +HNSWParams sq4_params(VecSimMetric metric, bool multi, size_t dim, const std::vector &mean) { + return {.type = VecSimType_FLOAT32, + .dim = dim, + .metric = metric, + .multi = multi, + .blockSize = 32, + .M = 16, + .efConstruction = 100, + .efRuntime = 128, + .quantType = VecSimQuant_SQ4, + .quantParams = mean.empty() ? nullptr : mean.data()}; +} + +// An ef far above the index size makes the HNSW search exhaustive over the connected graph, so +// indexes whose graphs differ (the level generator is not serialized) still agree on the top k. +Results top_k(VecSimIndex *index, const float *query, size_t k) { + VecSimQueryParams query_params{}; + query_params.hnswRuntimeParams.efRuntime = 1024; + Results results; + auto *reply = VecSimIndex_TopKQuery(index, query, k, &query_params, BY_SCORE); + EXPECT_EQ(VecSimQueryReply_GetCode(reply), VecSim_QueryReply_OK); + auto *iterator = VecSimQueryReply_GetIterator(reply); + while (VecSimQueryReply_IteratorHasNext(iterator)) { + const auto *item = VecSimQueryReply_IteratorNext(iterator); + results.emplace_back(VecSimQueryResult_GetId(item), VecSimQueryResult_GetScore(item)); + } + VecSimQueryReply_IteratorFree(iterator); + VecSimQueryReply_Free(reply); + return results; +} + +std::string temp_index_path(const std::string &prefix) { + std::string case_name = ::testing::UnitTest::GetInstance()->current_test_info()->name(); + std::replace(case_name.begin(), case_name.end(), '/', '_'); + return ::testing::TempDir() + prefix + case_name + ".hnsw"; +} + +TEST_P(HNSWSQ4Test, InsertQueryMutateAndSerialize) { + const auto [metric, centered, multi, dim] = GetParam(); + std::vector mean = make_mean(centered, dim); + HNSWParams hp = sq4_params(metric, multi, dim, mean); + auto params = CreateParams(hp); + IndexPtr index(VecSimIndex_New(¶ms), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + auto *hnsw = dynamic_cast *>(index.get()); + ASSERT_NE(hnsw, nullptr); + EXPECT_EQ(EstimateInitialSize(hp), index->getAllocationSize()); + EXPECT_TRUE(hnsw->usesQuantizedStorage()); + EXPECT_EQ(hnsw->getStoredDataSize(), + (dim + 1) / 2 + sizeof(float) * (metric == VecSimMetric_L2 ? 4 : (centered ? 4 : 3))); + + constexpr size_t count = 64; + std::mt19937 rng(42); + std::uniform_real_distribution random(-1.0f, 1.0f); + std::vector> inputs(count, std::vector(dim)); + for (size_t i = 0; i < count; ++i) { + for (auto &value : inputs[i]) + value = random(rng); + ASSERT_EQ(VecSimIndex_AddVector(index.get(), inputs[i].data(), i), 1); + } + ASSERT_TRUE(hnsw->checkIntegrity().valid_state); + std::vector query(dim); + for (auto &value : query) + value = random(rng); + std::vector> expected; + for (size_t i = 0; i < count; ++i) { + auto reconstructed = reconstruct(hnsw->getDataByInternalId(i), dim, mean); + double score = distance(reconstructed, query, metric); + expected.emplace_back(score, i); + EXPECT_NEAR(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), i, query.data()), score, + 2e-5 * (1.0 + std::abs(score))); + auto other = reconstruct(hnsw->getDataByInternalId((i + 1) % count), dim, mean); + const double expected_pair = distance(reconstructed, other, metric); + EXPECT_NEAR(hnsw->calcDistance(hnsw->getDataByInternalId(i), + hnsw->getDataByInternalId((i + 1) % count)), + expected_pair, 3e-5 * (1.0 + std::abs(expected_pair))); + } + std::sort(expected.begin(), expected.end()); + auto check_results = [&](VecSimIndex *target) { + runTopKSearchTest(target, query.data(), 10, [&](size_t id, double score, size_t pos) { + EXPECT_EQ(id, expected[pos].second); + EXPECT_NEAR(score, expected[pos].first, 2e-5 * (1.0 + std::abs(expected[pos].first))); + }); + }; + check_results(index.get()); + const double radius = std::max(0.0, (expected[19].first + expected[20].first) / 2); + const size_t range_count = + std::count_if(expected.begin(), expected.end(), + [radius](const auto &result) { return result.first <= radius; }); + // A bounded HNSW range search is approximate. Validate every returned score and membership. + auto *range = VecSimIndex_RangeQuery(index.get(), query.data(), radius, nullptr, BY_SCORE); + EXPECT_EQ(VecSimQueryReply_GetCode(range), VecSim_QueryReply_OK); + EXPECT_GT(VecSimQueryReply_Len(range), 0); + EXPECT_LE(VecSimQueryReply_Len(range), range_count); + std::unordered_set range_ids; + auto *iterator = VecSimQueryReply_GetIterator(range); + while (VecSimQueryReply_IteratorHasNext(iterator)) { + const auto *item = VecSimQueryReply_IteratorNext(iterator); + const size_t id = VecSimQueryResult_GetId(item); + EXPECT_TRUE(range_ids.insert(id).second); + const double score = VecSimQueryResult_GetScore(item); + const auto match = std::find_if(expected.begin(), expected.end(), + [id](const auto &result) { return result.second == id; }); + EXPECT_NE(match, expected.end()); + if (match != expected.end()) + EXPECT_NEAR(score, match->first, 2e-5 * (1.0 + std::abs(match->first))); + EXPECT_LE(score, radius); + } + VecSimQueryReply_IteratorFree(iterator); + VecSimQueryReply_Free(range); + // A radius containing the entire connected graph verifies full traversal and scoring. + runRangeQueryTest( + index.get(), query.data(), std::max(0.0, expected.back().first) + 1.0, + [&](size_t id, double score, size_t pos) { + EXPECT_EQ(id, expected[pos].second); + EXPECT_NEAR(score, expected[pos].first, 2e-5 * (1.0 + std::abs(expected[pos].first))); + }, + count, BY_SCORE); + auto *batch = VecSimBatchIterator_New(index.get(), query.data(), nullptr); + runBatchIteratorSearchTest(batch, 10, [&](size_t id, double score, size_t pos) { + EXPECT_EQ(id, expected[pos].second); + EXPECT_NEAR(score, expected[pos].first, 2e-5 * (1.0 + std::abs(expected[pos].first))); + }); + VecSimBatchIterator_Free(batch); + const auto path = temp_index_path("sq4_roundtrip_"); + hnsw->saveIndex(path); + IndexPtr loaded(HNSWFactory::NewIndex(path), VecSimIndex_Free); + std::remove(path.c_str()); + ASSERT_NE(loaded, nullptr); + auto *restored = dynamic_cast *>(loaded.get()); + ASSERT_NE(restored, nullptr); + EXPECT_EQ(restored->quantType, VecSimQuant_SQ4); + EXPECT_EQ(restored->serializedMeanVector, mean); + ASSERT_TRUE(restored->checkIntegrity().valid_state); + for (size_t i = 0; i < count; ++i) { + EXPECT_EQ(std::memcmp(hnsw->getDataByInternalId(i), restored->getDataByInternalId(i), + hnsw->getStoredDataSize()), + 0); + } + check_results(loaded.get()); + // Exercise duplicate-label insertion and deletion through the actual index API. + EXPECT_EQ(VecSimIndex_AddVector(loaded.get(), query.data(), 0), multi ? 1 : 0); + EXPECT_EQ(VecSimIndex_IndexSize(loaded.get()), count + (multi ? 1 : 0)); + EXPECT_EQ(VecSimIndex_DeleteVector(loaded.get(), 0), multi ? 2 : 1); + EXPECT_EQ(VecSimIndex_IndexSize(loaded.get()), count - 1); + EXPECT_TRUE(restored->checkIntegrity().valid_state); +} + +// A loaded index must keep working as a writable index: new vectors are quantized with the +// restored mean and the index answers like one that never went through serialization. +TEST_P(HNSWSQ4Test, SaveLoadThenMutate) { + const auto [metric, centered, multi, dim] = GetParam(); + constexpr size_t initial = 200, added = 50, deleted = 20, queries = 10, k = 10; + const std::vector mean = make_mean(centered, dim); + const HNSWParams hp = sq4_params(metric, multi, dim, mean); + const auto vectors = random_vectors(initial + added, dim, 7); + const auto query_vectors = random_vectors(queries, dim, 8); + + IndexPtr original(HNSWFactory::NewIndex(&hp), VecSimIndex_Free); + ASSERT_NE(original, nullptr); + auto *original_hnsw = dynamic_cast *>(original.get()); + ASSERT_NE(original_hnsw, nullptr); + for (size_t i = 0; i < initial; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(original.get(), vectors[i].data(), i), 1); + } + std::vector before; + for (const auto &query : query_vectors) { + before.push_back(top_k(original.get(), query.data(), k)); + ASSERT_EQ(before.back().size(), k); + } + + const auto path = temp_index_path("sq4_save_load_mutate_"); + original_hnsw->saveIndex(path); + IndexPtr loaded(HNSWFactory::NewIndex(path), VecSimIndex_Free); + std::remove(path.c_str()); + ASSERT_NE(loaded, nullptr); + auto *loaded_hnsw = dynamic_cast *>(loaded.get()); + ASSERT_NE(loaded_hnsw, nullptr); + EXPECT_EQ(loaded_hnsw->quantType, VecSimQuant_SQ4); + EXPECT_EQ(loaded_hnsw->serializedMeanVector, mean); + ASSERT_EQ(loaded_hnsw->getStoredDataSize(), original_hnsw->getStoredDataSize()); + for (size_t q = 0; q < queries; ++q) { + EXPECT_EQ(top_k(loaded.get(), query_vectors[q].data(), k), before[q]) << "query " << q; + } + + // Each label holds one vector, so before any deletion label i has internal id i. + for (size_t i = initial; i < initial + added; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(loaded.get(), vectors[i].data(), i), 1); + const auto expected_blob = original_hnsw->preprocessForStorage(vectors[i].data()); + EXPECT_EQ(std::memcmp(loaded_hnsw->getDataByInternalId(i), expected_blob.get(), + loaded_hnsw->getStoredDataSize()), + 0) + << "label " << i; + } + for (size_t i = 0; i < deleted; ++i) { + ASSERT_EQ(VecSimIndex_DeleteVector(loaded.get(), i), 1); + } + EXPECT_EQ(VecSimIndex_IndexSize(loaded.get()), initial + added - deleted); + EXPECT_TRUE(loaded_hnsw->checkIntegrity().valid_state); + + IndexPtr fresh(HNSWFactory::NewIndex(&hp), VecSimIndex_Free); + ASSERT_NE(fresh, nullptr); + for (size_t i = 0; i < initial + added; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(fresh.get(), vectors[i].data(), i), 1); + } + for (size_t i = 0; i < deleted; ++i) { + ASSERT_EQ(VecSimIndex_DeleteVector(fresh.get(), i), 1); + } + for (size_t q = 0; q < queries; ++q) { + const Results expected = top_k(fresh.get(), query_vectors[q].data(), k); + ASSERT_EQ(expected.size(), k); + EXPECT_EQ(top_k(loaded.get(), query_vectors[q].data(), k), expected) << "query " << q; + } +} + +// Multi-value only: gtest cannot mix TEST and TEST_P in one suite, so single-value cases skip. +TEST_P(HNSWSQ4Test, DuplicateLabelsMulti) { + const auto [metric, centered, multi, dim] = GetParam(); + if (!multi) { + GTEST_SKIP() << "multi-value only"; + } + const std::vector mean = make_mean(centered, dim); + const HNSWParams hp = sq4_params(metric, multi, dim, mean); + IndexPtr index(HNSWFactory::NewIndex(&hp), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + + constexpr size_t label = 5, other_label = 9, copies = 3; + const auto vectors = random_vectors(copies + 1, dim, 11); + for (size_t i = 0; i < copies; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(index.get(), vectors[i].data(), label), 1); + } + ASSERT_EQ(VecSimIndex_AddVector(index.get(), vectors[copies].data(), other_label), 1); + EXPECT_EQ(VecSimIndex_IndexSize(index.get()), copies + 1); + EXPECT_EQ(index->indexLabelCount(), 2u); + + // KNN reports the duplicated label once, scored by its closest vector (which is also what + // the multi-value getDistanceFrom reports). + const Results results = top_k(index.get(), vectors[0].data(), 10); + ASSERT_EQ(results.size(), 2u); + EXPECT_NE(results[0].first, results[1].first); + const auto *hnsw = dynamic_cast *>(index.get()); + ASSERT_NE(hnsw, nullptr); + const auto query_blob = hnsw->preprocessQuery(vectors[0].data()); + double best = std::numeric_limits::max(); + for (size_t id = 0; id < copies; ++id) { + best = std::min( + best, hnsw->calcDistanceForQuery(hnsw->getDataByInternalId(id), query_blob.get())); + } + EXPECT_DOUBLE_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), label, vectors[0].data()), + best); + size_t label_hits = 0; + for (const auto &[id, score] : results) { + if (id == label) { + ++label_hits; + EXPECT_DOUBLE_EQ(score, best); + } + } + EXPECT_EQ(label_hits, 1u); + + EXPECT_EQ(VecSimIndex_DeleteVector(index.get(), label), static_cast(copies)); + EXPECT_EQ(VecSimIndex_IndexSize(index.get()), 1u); + EXPECT_EQ(index->indexLabelCount(), 1u); + const Results remaining = top_k(index.get(), vectors[0].data(), 10); + ASSERT_EQ(remaining.size(), 1u); + EXPECT_EQ(remaining[0].first, other_label); +} + +INSTANTIATE_TEST_SUITE_P(FP32, HNSWSQ4Test, + ::testing::Combine(::testing::Values(VecSimMetric_IP, VecSimMetric_L2), + ::testing::Bool(), ::testing::Bool(), + ::testing::Values(1u, 17u, 64u))); + +TEST(HNSWSQ4Factory, PreNormalizedCosineUsesIP) { + HNSWParams hp = {.type = VecSimType_FLOAT32, + .dim = 3, + .metric = VecSimMetric_Cosine, + .quantType = VecSimQuant_SQ4}; + VecSimParams params = CreateParams(hp); + IndexPtr index(HNSWFactory::NewIndex(¶ms, true), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + const float x[] = {1, 0, 0}; + const float y[] = {0, 1, 0}; + ASSERT_EQ(VecSimIndex_AddVector(index.get(), x, 0), 1); + EXPECT_FLOAT_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), 0, x), 0); + EXPECT_FLOAT_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), 0, y), 1); +} + +TEST(HNSWSQ4Factory, LoadRejectsUnsupportedSQ4Snapshot) { + HNSWParams hp = {.type = VecSimType_FLOAT32, + .dim = 4, + .metric = VecSimMetric_L2, + .M = 8, + .efConstruction = 40, + .quantType = VecSimQuant_SQ4}; + IndexPtr index(HNSWFactory::NewIndex(&hp), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + const float a[] = {0.25f, 0.3f, 0.35f, 0.4f}; + const float b[] = {0.75f, 0.8f, 0.85f, 0.9f}; + ASSERT_EQ(VecSimIndex_AddVector(index.get(), a, 0), 1); + ASSERT_EQ(VecSimIndex_AddVector(index.get(), b, 1), 1); + auto *hnsw = dynamic_cast *>(index.get()); + ASSERT_NE(hnsw, nullptr); + + const auto path = temp_index_path("sq4_reject_"); + // Field offsets follow HNSWSerializer::saveIndex and HNSWIndex::saveIndexFields. + constexpr std::streamoff type_offset = + sizeof(HNSWSerializer::EncodingVersion) + sizeof(VecSimAlgo) + sizeof(size_t); + constexpr std::streamoff metric_offset = type_offset + sizeof(VecSimType); + auto save_and_patch = [&](std::streamoff offset, auto value) { + hnsw->saveIndex(path); + std::fstream stream(path, std::ios::in | std::ios::out | std::ios::binary); + ASSERT_TRUE(stream.is_open()); + // Check the offset points at the expected field before overwriting it. + decltype(value) original{}; + stream.seekg(offset); + Serializer::readBinaryPOD(stream, original); + if constexpr (std::is_same_v) { + ASSERT_EQ(original, VecSimType_FLOAT32); + } else { + ASSERT_EQ(original, VecSimMetric_L2); + } + stream.seekp(offset); + Serializer::writeBinaryPOD(stream, value); + }; + auto expect_rejected = [&]() { + IndexPtr loaded(HNSWFactory::NewIndex(path), VecSimIndex_Free); + EXPECT_EQ(loaded, nullptr); + }; + + // The unmodified snapshot loads. + hnsw->saveIndex(path); + { + IndexPtr loaded(HNSWFactory::NewIndex(path), VecSimIndex_Free); + ASSERT_NE(loaded, nullptr); + } + // SQ4 accepts FP32 only, unlike SQ8. + save_and_patch(type_offset, VecSimType_FLOAT16); + expect_rejected(); + // A raw Cosine metric has no scalar quantization kernel. + save_and_patch(metric_offset, VecSimMetric_Cosine); + expect_rejected(); + std::remove(path.c_str()); +} + +// Mirrors HNSWSQ8Test.SizeEstimation, including its 1% tolerance on block growth. +TEST(HNSWSQ4Factory, EstimatesMatchAllocation) { + constexpr size_t dim = 768, block_size = 256; + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + for (bool centered : {false, true}) { + SCOPED_TRACE(std::string(VecSimMetric_ToString(metric)) + + (centered ? " centered" : " plain")); + const std::vector mean = make_mean(centered, dim); + HNSWParams hp = sq4_params(metric, false, dim, mean); + hp.blockSize = block_size; + hp.M = 64; + VecSimParams params = CreateParams(hp); + IndexPtr index(VecSimIndex_New(¶ms), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + EXPECT_EQ(VecSimIndex_EstimateInitialSize(¶ms), + VecSimIndex_StatsInfo(index.get()).memory); + + const auto vectors = random_vectors(block_size + 1, dim, 3); + for (size_t i = 0; i < block_size; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(index.get(), vectors[i].data(), i), 1); + } + + // Measure the allocation caused by growing the index by one block. + const size_t estimation = VecSimIndex_EstimateElementSize(¶ms) * block_size; + const size_t before = VecSimIndex_StatsInfo(index.get()).memory; + ASSERT_EQ(VecSimIndex_AddVector(index.get(), vectors[block_size].data(), block_size), + 1); + const size_t actual = VecSimIndex_StatsInfo(index.get()).memory - before; + + EXPECT_GE(estimation, actual * 0.99); + EXPECT_LE(estimation, actual * 1.01); + } + } +} + +// Tiered SQ4 needs mean training, which is out of scope: creation and estimation both reject it, +// and the estimator's exception is translated to SIZE_MAX at the C API. +TEST(HNSWSQ4Factory, RejectsTieredSQ4) { + constexpr size_t dim = 32; + const std::vector mean = make_mean(true, dim); + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + for (bool centered : {false, true}) { + for (size_t training_set_size : {size_t{0}, size_t{10}}) { + SCOPED_TRACE(std::string(VecSimMetric_ToString(metric)) + + (centered ? " centered" : " plain") + " training " + + std::to_string(training_set_size)); + HNSWParams hp = {.type = VecSimType_FLOAT32, + .dim = dim, + .metric = metric, + .quantType = VecSimQuant_SQ4, + .quantParams = centered ? mean.data() : nullptr}; + VecSimParams primary = CreateParams(hp); + TieredIndexParams tiered = {.primaryIndexParams = &primary, + .specificParams = {TieredHNSWParams{ + .QuantNormalizationSetSize = training_set_size}}}; + VecSimParams params = CreateParams(tiered); + + VecSimIndex *direct = TieredFactory::NewIndex(&tiered); + EXPECT_EQ(direct, nullptr); + if (direct) { + VecSimIndex_Free(direct); + } + VecSimIndex *created = VecSimIndex_New(¶ms); + EXPECT_EQ(created, nullptr); + if (created) { + VecSimIndex_Free(created); + } + EXPECT_THROW(TieredFactory::EstimateInitialSize(&tiered), std::invalid_argument); + size_t estimate = 0; + EXPECT_NO_THROW(estimate = VecSimIndex_EstimateInitialSize(¶ms)); + EXPECT_EQ(estimate, SIZE_MAX); + } + } + } +} + +// Every rejection must happen in parameter validation, before components are built, so no +// exception reaches the C API and the factory returns NULL rather than throwing. +TEST(HNSWSQ4Factory, RejectsInvalidParams) { + auto expect_rejected = [](const HNSWParams &hp, bool is_normalized) { + VecSimIndex *direct = nullptr; + EXPECT_NO_THROW(direct = HNSWFactory::NewIndex(&hp, is_normalized)); + EXPECT_EQ(direct, nullptr); + if (direct) { + VecSimIndex_Free(direct); + } + EXPECT_THROW(HNSWFactory::EstimateInitialSize(&hp, is_normalized), std::invalid_argument); + if (is_normalized) { + // The C API never declares input normalized; that path is covered above. + return; + } + VecSimParams params = CreateParams(hp); + VecSimIndex *created = nullptr; + EXPECT_NO_THROW(created = VecSimIndex_New(¶ms)); + EXPECT_EQ(created, nullptr); + if (created) { + VecSimIndex_Free(created); + } + size_t estimate = 0; + EXPECT_NO_THROW(estimate = VecSimIndex_EstimateInitialSize(¶ms)); + EXPECT_EQ(estimate, SIZE_MAX); + }; + + constexpr size_t dim = 32; + const std::vector mean = make_mean(true, dim); + for (const float *quant_params : {static_cast(nullptr), mean.data()}) { + SCOPED_TRACE(quant_params ? "centered" : "plain"); + for (auto type : {VecSimType_FLOAT16, VecSimType_BFLOAT16, VecSimType_FLOAT64, + VecSimType_INT8, VecSimType_UINT8}) { + SCOPED_TRACE(VecSimType_ToString(type)); + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + expect_rejected({.type = type, + .dim = dim, + .metric = metric, + .quantType = VecSimQuant_SQ4, + .quantParams = quant_params}, + false); + } + } + + // Raw Cosine has no SQ4 kernel; only pre-normalized Cosine (resolved to IP) is accepted. + expect_rejected({.type = VecSimType_FLOAT32, + .dim = dim, + .metric = VecSimMetric_Cosine, + .quantType = VecSimQuant_SQ4, + .quantParams = quant_params}, + false); + + // Value 3 is not a VecSimQuantType enumerator. + for (bool is_normalized : {false, true}) { + expect_rejected({.type = VecSimType_FLOAT32, + .dim = dim, + .metric = VecSimMetric_IP, + .quantType = static_cast(3), + .quantParams = quant_params}, + is_normalized); + } + } +} +} // namespace diff --git a/tests/unit/test_hnsw_sq8.cpp b/tests/unit/test_hnsw_sq8.cpp index 10d177d1c..8708c1f1c 100644 --- a/tests/unit/test_hnsw_sq8.cpp +++ b/tests/unit/test_hnsw_sq8.cpp @@ -632,7 +632,7 @@ TYPED_TEST(HNSWSQ8Test, LoadRejectsUnsupportedQuantizedSnapshot) { } // Written as raw bytes: an out-of-range value is not a valid VecSimQuantType. - save_and_patch(quant_type_offset, static_cast(VecSimQuant_SQ8 + 1)); + save_and_patch(quant_type_offset, std::numeric_limits::max()); expect_rejected(); for (VecSimType unsupported_type : diff --git a/tests/unit/test_sq4_components.cpp b/tests/unit/test_sq4_components.cpp new file mode 100644 index 000000000..0842ab3f4 --- /dev/null +++ b/tests/unit/test_sq4_components.cpp @@ -0,0 +1,317 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "VecSim/index_factories/components/components_factory.h" +#include "VecSim/spaces/L2_space.h" +#include "VecSim/types/sq4.h" + +namespace { + +using vecsim_types::sq4; + +float metadata(const void *blob, size_t body_bytes, size_t index) { + return load_unaligned(static_cast(blob) + body_bytes + + index * sizeof(float)); +} + +std::vector reconstruct(const void *blob, size_t dim, const std::vector &mean) { + const auto *codes = static_cast(blob); + const double min = metadata(blob, sq4::data_bytes_count(dim), sq4::MIN_VAL); + const double delta = metadata(blob, sq4::data_bytes_count(dim), sq4::DELTA); + std::vector result(dim); + for (size_t i = 0; i < dim; ++i) { + // Decode directly so the oracle also checks the production nibble order. + const unsigned code = (codes[i / 2] >> (4 * (i % 2))) & 15; + result[i] = min + delta * code + (mean.empty() ? 0.0 : mean[i]); + } + return result; +} + +template +double referenceDistance(const Left &left, const Right &right) { + double result = Metric == VecSimMetric_IP ? 1.0 : 0.0; + for (size_t i = 0; i < left.size(); ++i) { + if constexpr (Metric == VecSimMetric_IP) { + result -= double(left[i]) * double(right[i]); + } else { + const double difference = double(left[i]) - double(right[i]); + result += difference * difference; + } + } + return result; +} + +template +void checkComponents(size_t dim, bool constant) { + SCOPED_TRACE(::testing::Message() << "metric=" << Metric << " mean=" << WithMean + << " dim=" << dim << " constant=" << constant); + auto allocator = VecSimAllocator::newVecsimAllocator(); + std::vector x(dim), y(dim), mean(WithMean ? dim : 0); + for (size_t i = 0; i < dim; ++i) { + if constexpr (WithMean) { + mean[i] = 0.7f * (int(i % 5) - 2); + } + const float offset = WithMean ? mean[i] : 0.0f; + x[i] = offset + (constant ? 1.25f : 0.31f * (int((i * 7) % 19) - 9)); + y[i] = offset + (constant ? -0.75f : 0.23f * (int((i * 11) % 17) - 8)); + } + auto components = + CreateSQ4IndexComponents(allocator, dim, WithMean ? mean.data() : nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + const size_t input_size = dim * sizeof(float); + const size_t storage_size = sq4::storage_bytes_count(dim); + auto *container = dynamic_cast *>(preprocessors.get()); + ASSERT_NE(container, nullptr); + auto *quantizer = dynamic_cast *>( + container->getPreprocessors()[0]); + ASSERT_NE(quantizer, nullptr); + auto x_blob = preprocessors->preprocessForStorage(x.data(), input_size); + auto y_blob = preprocessors->preprocessForStorage(y.data(), input_size); + auto query = preprocessors->preprocessQuery(y.data(), input_size); + auto combined = preprocessors->preprocess(y.data(), input_size); + const size_t query_size = + input_size + sq4::query_metadata_count() * sizeof(float); + EXPECT_EQ(std::memcmp(combined.getStorageBlob(), y_blob.get(), storage_size), 0); + EXPECT_EQ(std::memcmp(combined.getQueryBlob(), query.get(), query_size), 0); + const auto rx = reconstruct(x_blob.get(), dim, mean); + const auto ry = reconstruct(y_blob.get(), dim, mean); + const double symmetric = referenceDistance(rx, ry); + const double asymmetric = referenceDistance(rx, y); + const double sym_tolerance = 2e-5 * (1.0 + std::abs(symmetric)); + const double asym_tolerance = 2e-5 * (1.0 + std::abs(asymmetric)); + EXPECT_NEAR(calculator->calcDistance(x_blob.get(), y_blob.get(), dim), symmetric, + sym_tolerance); + EXPECT_NEAR(calculator->calcDistanceForQuery(x_blob.get(), query.get(), dim), asymmetric, + asym_tolerance); + auto stored_dispatch = calculator->getDistanceDispatch(DistanceMode::StoredToStored); + auto query_dispatch = calculator->getDistanceDispatch(DistanceMode::StoredToQuery); + ASSERT_TRUE(stored_dispatch.isValid()); + ASSERT_TRUE(query_dispatch.isValid()); + EXPECT_NEAR(stored_dispatch(x_blob.get(), y_blob.get(), dim), symmetric, sym_tolerance); + EXPECT_NEAR(query_dispatch(x_blob.get(), query.get(), dim), asymmetric, asym_tolerance); + + if (dim % 2) { + EXPECT_EQ(static_cast(x_blob.get())[dim / 2] & 0xf0, 0); + EXPECT_EQ(static_cast(y_blob.get())[dim / 2] & 0xf0, 0); + } + // The in-place allocation must hold both the original input and the processed blob; + // metadata makes the processed blob larger for tiny dimensions. + const size_t buffer_size = std::max(input_size, storage_size); + std::vector in_place((buffer_size + sizeof(float) - 1) / sizeof(float) + 4); + std::memset(in_place.data(), 0xa5, in_place.size() * sizeof(float)); + std::memcpy(in_place.data(), x.data(), input_size); + preprocessors->preprocessStorageInPlace(in_place.data(), buffer_size); + EXPECT_EQ(std::memcmp(in_place.data(), x_blob.get(), storage_size), 0); + const auto *bytes = reinterpret_cast(in_place.data()); + for (size_t i = buffer_size; i < in_place.size() * sizeof(float); ++i) { + EXPECT_EQ(bytes[i], 0xa5); + } + + if constexpr (WithMean && Metric == VecSimMetric_IP) { + double correction = 0.0; + for (size_t i = 0; i < dim; ++i) { + correction += rx[i] * mean[i]; + } + EXPECT_NEAR( + metadata(x_blob.get(), sq4::data_bytes_count(dim), sq4::mean_ip_index()), + correction, 2e-6 * (1.0 + std::abs(correction))); + } +} + +template +void checkDimensions() { + for (size_t dim : {1u, 2u, 3u, 4u, 5u, 7u, 16u, 37u, 128u}) { + for (bool constant : {false, true}) { + checkComponents(dim, constant); + checkComponents(dim, constant); + } + } +} + +} // namespace + +TEST(SQ4Components, L2MatchesReconstruction) { checkDimensions(); } + +TEST(SQ4Components, IPMatchesReconstruction) { checkDimensions(); } + +TEST(SQ4Components, PackedCodesAndMetadataSizes) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + const float input[] = {0, 4, 8, 11, 15}; + QuantPreprocessor preprocessor(allocator, 5); + void *storage = nullptr; + size_t size = sizeof(input); + preprocessor.preprocessForStorage(input, storage, size, 0); + ASSERT_EQ(size, 3 + 4 * sizeof(float)); + const auto *bytes = static_cast(storage); + EXPECT_EQ(bytes[0], 0x40); + EXPECT_EQ(bytes[1], 0xb8); + EXPECT_EQ(bytes[2], 0x0f); + EXPECT_FLOAT_EQ(metadata(storage, 3, sq4::MIN_VAL), 0); + EXPECT_FLOAT_EQ(metadata(storage, 3, sq4::DELTA), 1.0f); + allocator->free_allocation(storage); +} + +TEST(SQ4Components, OddDimTailNibbleIsZero) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + for (size_t dim : {1u, 15u, 17u, 767u}) { + SCOPED_TRACE(::testing::Message() << "dim=" << dim); + std::vector input(dim); + for (size_t i = 0; i < dim; ++i) { + input[i] = 0.37f * (int((i * 13) % 23) - 11); + } + // The last coordinate holds the maximum, so its code is 15 (for dim > 1) and a misplaced + // nibble shows. + input[dim - 1] = 100.0f; + auto check = [&](auto metric_tag) { + constexpr VecSimMetric Metric = decltype(metric_tag)::value; + auto components = CreateSQ4IndexComponents(allocator, dim, nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); + const auto *bytes = static_cast(blob.get()); + const size_t metadata_count = Metric == VecSimMetric_L2 ? 4 : 3; + EXPECT_EQ((sq4::storage_bytes_count(dim)), + dim / 2 + dim % 2 + metadata_count * sizeof(float)); + // A single coordinate is a constant range, which encodes as code 0. + EXPECT_EQ(sq4::load_code(bytes, dim - 1), dim == 1 ? 0 : 15); + if (dim % 2) { + EXPECT_EQ(bytes[dim / 2] & 0xf0, 0); + } + }; + check(std::integral_constant{}); + check(std::integral_constant{}); + } +} + +TEST(SQ4Components, NonFiniteInputEncodesInBounds) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t dim = 17; + const float inf = std::numeric_limits::infinity(); + std::vector input(dim); + for (size_t i = 0; i < dim; ++i) { + input[i] = 0.5f * (int(i % 7) - 3); + } + input[2] = std::numeric_limits::quiet_NaN(); + input[5] = inf; + input[9] = -inf; + input[13] = 1e30f; + std::vector mean(dim); + for (size_t i = 0; i < dim; ++i) { + mean[i] = 0.25f * (int(i % 3) - 1); + } + + auto check = [&](auto metric_tag, auto with_mean_tag) { + constexpr VecSimMetric Metric = decltype(metric_tag)::value; + constexpr bool WithMean = decltype(with_mean_tag)::value; + SCOPED_TRACE(::testing::Message() << "metric=" << Metric << " mean=" << WithMean); + auto components = CreateSQ4IndexComponents(allocator, dim, + WithMean ? mean.data() : nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + const size_t storage_size = sq4::storage_bytes_count(dim); + + auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); + for (size_t i = 0; i < dim; ++i) { + EXPECT_LE(sq4::load_code(static_cast(blob.get()), i), 15); + } + + // In place, with a sentinel tail: nothing past the stored blob may be written. The input + // is larger than the stored blob here, so the bytes between them keep the input. + const size_t input_size = dim * sizeof(float); + ASSERT_GT(input_size, storage_size); + std::vector buffer(input_size + 16, 0xa5); + std::memcpy(buffer.data(), input.data(), input_size); + preprocessors->preprocessStorageInPlace(buffer.data(), input_size); + EXPECT_EQ(std::memcmp(buffer.data(), blob.get(), storage_size), 0); + EXPECT_EQ(std::memcmp(buffer.data() + storage_size, + reinterpret_cast(input.data()) + storage_size, + input_size - storage_size), + 0); + for (size_t i = input_size; i < buffer.size(); ++i) { + EXPECT_EQ(buffer[i], 0xa5); + } + // Odd dim: the unused high nibble stays zero even for non-finite input. + EXPECT_EQ(buffer[dim / 2] & 0xf0, 0); + }; + check(std::integral_constant{}, std::false_type{}); + check(std::integral_constant{}, std::false_type{}); + check(std::integral_constant{}, std::true_type{}); + check(std::integral_constant{}, std::true_type{}); +} + +TEST(SQ4Components, CenteredIPBlobSize768) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t dim = 768; + static_assert(sq4::storage_bytes_count(dim) == 400); + std::vector input(dim), mean(dim); + for (size_t i = 0; i < dim; ++i) { + input[i] = 0.01f * (int(i % 41) - 20); + mean[i] = 0.002f * (int(i % 9) - 4); + } + auto components = CreateSQ4IndexComponents(allocator, dim, mean.data()); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto *container = dynamic_cast *>(preprocessors.get()); + ASSERT_NE(container, nullptr); + auto *quantizer = container->getPreprocessors()[0]; + ASSERT_NE((dynamic_cast *>(quantizer)), + nullptr); + void *storage = nullptr; + size_t size = dim * sizeof(float); + quantizer->preprocessForStorage(input.data(), storage, size, 0); + EXPECT_EQ(size, 400u); + allocator->free_allocation(storage); +} + +// SQ4 components are distinct types from the SQ8 ones that applySQ8Mean and +// HNSWIndex::setQuantizationMean dynamic_cast to, so the SQ8 mean path cannot apply to SQ4. +TEST(SQ4Components, SQ4ComponentsAreNotSQ8Types) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t dim = 8; + std::vector mean(dim, 0.5f); + auto components = CreateSQ4IndexComponents(allocator, dim, mean.data()); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto *container = dynamic_cast *>(preprocessors.get()); + ASSERT_NE(container, nullptr); + EXPECT_EQ((dynamic_cast *>( + container->getPreprocessors()[0])), + nullptr); + EXPECT_EQ((dynamic_cast *>( + calculator.get())), + nullptr); + EXPECT_NE((dynamic_cast *>( + calculator.get())), + nullptr); +} + +// Centered and plain SQ4 L2 indexes compare stored vectors with the same kernel, as SQ8 does. +TEST(SQ4Components, StoredL2UsesOneKernel) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t dim = 64; + std::vector zero_mean(dim, 0.0f); + for (bool centered : {true, false}) { + SCOPED_TRACE(centered); + auto components = CreateSQ4IndexComponents( + allocator, dim, centered ? zero_mean.data() : nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + EXPECT_EQ(calculator->getDistanceDispatch(DistanceMode::StoredToStored).stateless_func, + spaces::L2_SQ4_SQ4_GetDistFunc(dim)); + } +} diff --git a/tests/unit/test_sq4_distances.cpp b/tests/unit/test_sq4_distances.cpp new file mode 100644 index 000000000..39ee83fcf --- /dev/null +++ b/tests/unit/test_sq4_distances.cpp @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#include "gtest/gtest.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/spaces/IP/IP.h" +#include "VecSim/spaces/L2/L2.h" +#include "VecSim/spaces/IP_space.h" +#include "VecSim/spaces/L2_space.h" +#include "VecSim/types/sq4.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using vecsim_types::sq4; +using vecsim_types::sq8; + +static_assert(sq8::max_code == 255); +static_assert(sq8::data_bytes_count(7) == 7); +static_assert(sq8::storage_bytes_count(7) == 19); +static_assert(sq8::storage_bytes_count(7) == 23); +static_assert(sq4::data_bytes_count(0) == 0); +static_assert(sq4::data_bytes_count(7) == 4); +static_assert(sq4::storage_bytes_count(7) == 20); +static_assert(sq4::storage_bytes_count(7) == 20); + +// Offset the blob by one byte to exercise unaligned metadata reads as well as odd payload sizes. +struct PackedVector { + std::vector bytes; + std::vector reconstructed; + + PackedVector(const std::vector &codes, float minimum, float delta) + : bytes(1 + sq4::storage_bytes_count(codes.size()), 0xff), + reconstructed(codes.size()) { + float metadata[4] = {minimum, delta, 0.0f, 0.0f}; + for (size_t i = 0; i < codes.size(); ++i) { + sq4::store_code(data(), i, codes[i]); + reconstructed[i] = minimum + delta * codes[i]; + metadata[sq4::SUM] += reconstructed[i]; + metadata[sq4::SUM_SQUARES] += reconstructed[i] * reconstructed[i]; + } + std::memcpy(data() + sq4::data_bytes_count(codes.size()), metadata, sizeof(metadata)); + } + + uint8_t *data() { return bytes.data() + 1; } +}; + +float referenceDistance(VecSimMetric metric, const std::vector &x, + const std::vector &y) { + float result = 0.0f; + for (size_t i = 0; i < x.size(); ++i) { + if (metric == VecSimMetric_IP) { + result += x[i] * y[i]; + } else { + const float diff = x[i] - y[i]; + result += diff * diff; + } + } + return metric == VecSimMetric_IP ? 1.0f - result : result; +} +} // namespace + +TEST(SQ4Distances, PackedLayoutAndSQ8Compatibility) { + uint8_t packed[8]; + std::memset(packed, 0xff, sizeof(packed)); + for (size_t i = 0; i < 16; ++i) { + sq4::store_code(packed, i, i); + if (i % 2 == 0) { + EXPECT_EQ(packed[i / 2] & 0xf0, 0); + } + } + for (size_t i = 0; i < 16; ++i) { + EXPECT_EQ(sq4::load_code(packed, i), i); + if (i % 2 == 0) { + EXPECT_EQ(packed[i / 2], i | ((i + 1) << 4)); + } + } + uint8_t unpacked[256]; + for (size_t i = 0; i < 256; ++i) { + sq8::store_code(unpacked, i, i); + EXPECT_EQ(sq8::load_code(unpacked, i), i); + EXPECT_EQ(unpacked[i], i); + } +} + +TEST(SQ4Distances, DispatchMatchesExplicitReconstruction) { + for (size_t dim : {0u, 1u, 2u, 3u, 4u, 7u, 16u, 31u, 32u, 65u}) { + SCOPED_TRACE(dim); + std::vector x_codes(dim), y_codes(dim); + std::vector query(dim + sq4::query_metadata_count(), 0.0f); + std::vector query_values(dim); + for (size_t i = 0; i < dim; ++i) { + x_codes[i] = (i * 7) % 16; + y_codes[i] = (i * 3 + 15) % 16; + query[i] = query_values[i] = static_cast(static_cast(i % 13) - 6) / 8; + query[dim + sq4::SUM_QUERY] += query[i]; + query[dim + sq4::SUM_SQUARES_QUERY] += query[i] * query[i]; + } + PackedVector x(x_codes, -1.25f, 0.125f); + PackedVector y(y_codes, 0.75f, 0.0625f); + if (dim % 2) { + EXPECT_EQ(x.data()[sq4::data_bytes_count(dim) - 1] & 0xf0, 0); + EXPECT_EQ(y.data()[sq4::data_bytes_count(dim) - 1] & 0xf0, 0); + } + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + unsigned char alignment = 255; + auto symmetric = spaces::GetDistFunc(metric, dim, &alignment); + EXPECT_EQ(alignment, 0); + auto asymmetric = spaces::GetDistFunc(metric, dim, &alignment); + EXPECT_EQ(alignment, 0); + EXPECT_FLOAT_EQ(symmetric(x.data(), y.data(), dim), + referenceDistance(metric, x.reconstructed, y.reconstructed)); + EXPECT_FLOAT_EQ(asymmetric(x.data(), query.data(), dim), + referenceDistance(metric, x.reconstructed, query_values)); + EXPECT_EQ(symmetric, (spaces::GetDistFunc(metric, dim, nullptr))); + EXPECT_EQ(asymmetric, (spaces::GetDistFunc(metric, dim, nullptr))); + if (metric == VecSimMetric_L2) { + EXPECT_FLOAT_EQ(symmetric(x.data(), x.data(), dim), 0.0f); + EXPECT_FLOAT_EQ(symmetric(x.data(), y.data(), dim), + symmetric(y.data(), x.data(), dim)); + } + } + } +} + +TEST(SQ4Distances, ConstantVectorsAndOddPadding) { + std::vector codes(5, 0); + PackedVector x(codes, 2.0f, 0.0f); + PackedVector y(codes, -3.0f, 0.0f); + std::vector query(5 + sq4::query_metadata_count(), -3.0f); + query[5 + sq4::SUM_QUERY] = -15.0f; + query[5 + sq4::SUM_SQUARES_QUERY] = 45.0f; + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + auto symmetric = spaces::GetDistFunc(metric, 5, nullptr); + auto asymmetric = spaces::GetDistFunc(metric, 5, nullptr); + const float expected = metric == VecSimMetric_IP ? 31.0f : 125.0f; + EXPECT_FLOAT_EQ(symmetric(x.data(), y.data(), 5), expected); + EXPECT_FLOAT_EQ(asymmetric(x.data(), query.data(), 5), expected); + // The unused nibble is outside the vector, even if a foreign writer leaves it nonzero. + x.data()[sq4::data_bytes_count(5) - 1] |= 0xf0; + EXPECT_FLOAT_EQ(symmetric(x.data(), y.data(), 5), expected); + EXPECT_FLOAT_EQ(asymmetric(x.data(), query.data(), 5), expected); + } +} + +TEST(SQ4Distances, FP32L2PreservesResidualsAtLargeCommonOffset) { + PackedVector x({0, 1, 2}, 1048576.0f, 0.03125f); + PackedVector y({0, 0, 0}, 1048576.0f, 0.0f); + std::vector query(3 + sq4::query_metadata_count(), 1048576.0f); + query[3 + sq4::SUM_QUERY] = 3.0f * 1048576.0f; + query[3 + sq4::SUM_SQUARES_QUERY] = 3.0f * 1048576.0f * 1048576.0f; + const float expected = 5.0f * 0.03125f * 0.03125f; + EXPECT_FLOAT_EQ(SQ4_FP32_L2Sqr(x.data(), query.data(), 3), expected); +} + +TEST(SQ4Distances, RejectsUnsupportedMetric) { + EXPECT_THROW((spaces::GetDistFunc(VecSimMetric_Cosine, 3, nullptr)), + std::invalid_argument); + EXPECT_THROW((spaces::GetDistFunc(VecSimMetric_Cosine, 3, nullptr)), + std::invalid_argument); +} + +namespace { +// Double-precision reference over the decoded vectors. +double refDistance(VecSimMetric metric, const std::vector &x, + const std::vector &y) { + double result = 0.0; + for (size_t i = 0; i < x.size(); ++i) { + result += metric == VecSimMetric_IP ? x[i] * y[i] : (x[i] - y[i]) * (x[i] - y[i]); + } + return metric == VecSimMetric_IP ? 1.0 - result : result; +} + +std::vector makeQuery(const std::vector &values) { + const size_t dim = values.size(); + std::vector query(dim + sq4::query_metadata_count(), 0.0f); + for (size_t i = 0; i < dim; ++i) { + query[i] = values[i]; + query[dim + sq4::SUM_QUERY] += values[i]; + query[dim + sq4::SUM_SQUARES_QUERY] += values[i] * values[i]; + } + return query; +} + +// Stored metadata slot of a packed vector, as the distance kernels read it. +double storedMeta(const uint8_t *blob, size_t dim, size_t slot) { + float value; + std::memcpy(&value, blob + sq4::data_bytes_count(dim) + slot * sizeof(float), sizeof(value)); + return value; +} + +// Stored-to-stored IP identity over the stored metadata, evaluated in double: +// IP = min_x * sum_y + min_y * sum_x - dim * min_x * min_y + delta_x * delta_y * sum(qx * qy) +double identityIP(const uint8_t *x, const uint8_t *y, size_t dim) { + double code_dot = 0.0; + for (size_t i = 0; i < dim; ++i) { + code_dot += double(sq4::load_code(x, i)) * sq4::load_code(y, i); + } + const double min_x = storedMeta(x, dim, sq4::MIN_VAL), dx = storedMeta(x, dim, sq4::DELTA); + const double min_y = storedMeta(y, dim, sq4::MIN_VAL), dy = storedMeta(y, dim, sq4::DELTA); + return min_x * storedMeta(y, dim, sq4::SUM) + min_y * storedMeta(x, dim, sq4::SUM) - + double(dim) * min_x * min_y + dx * dy * code_dot; +} + +// Per-element reconstruction of the SQ4-FP32 inner product distance. +float reconstructedIP(const uint8_t *x, const float *y, size_t dim, float min_val, float delta) { + float dot = 0.0f; + for (size_t i = 0; i < dim; ++i) { + dot += (min_val + delta * sq4::load_code(x, i)) * y[i]; + } + return 1.0f - dot; +} +} // namespace + +TEST(SQ4Distances, OddDimMatchesReference) { + std::mt19937 rng(1234); + std::uniform_int_distribution code_dist(0, 15); + std::uniform_real_distribution val_dist(-1.0f, 1.0f); + for (size_t dim : {1u, 15u, 16u, 17u, 64u, 767u, 768u}) { + SCOPED_TRACE(dim); + std::vector x_codes(dim), y_codes(dim); + std::vector values(dim); + for (size_t i = 0; i < dim; ++i) { + x_codes[i] = code_dist(rng); + y_codes[i] = code_dist(rng); + values[i] = val_dist(rng); + } + PackedVector x(x_codes, -0.5f, 1.0f / 15); + PackedVector y(y_codes, 0.25f, 1.0f / 30); + auto query = makeQuery(values); + std::vector xd(dim), yd(dim), qd(dim); + for (size_t i = 0; i < dim; ++i) { + qd[i] = values[i]; + } + // Decode with the exact float min/delta stored in the blobs, then compute in double. + const float minx = -0.5f, dx = 1.0f / 15, miny = 0.25f, dy = 1.0f / 30; + for (size_t i = 0; i < dim; ++i) { + xd[i] = double(minx) + double(dx) * sq4::load_code(x.data(), i); + yd[i] = double(miny) + double(dy) * sq4::load_code(y.data(), i); + } + auto check = [](float actual, double ref) { + EXPECT_NEAR(actual, ref, 1e-4 * std::max(1.0, std::abs(ref))); + }; + check(SQ4_SQ4_InnerProduct(x.data(), y.data(), dim), + 1.0 - identityIP(x.data(), y.data(), dim)); + check(SQ4_SQ4_L2Sqr(x.data(), y.data(), dim), refDistance(VecSimMetric_L2, xd, yd)); + check(SQ4_FP32_InnerProduct(x.data(), query.data(), dim), + refDistance(VecSimMetric_IP, xd, qd)); + check(SQ4_FP32_L2Sqr(x.data(), query.data(), dim), refDistance(VecSimMetric_L2, xd, qd)); + } +} + +TEST(SQ4Distances, MatchedIPEqualsReconstruction) { + constexpr size_t dim = 768; + std::mt19937 rng(99); + std::uniform_int_distribution code_dist(0, 15); + std::uniform_real_distribution val_dist(-1.0f, 1.0f); + std::vector codes(dim); + std::vector values(dim); + for (size_t i = 0; i < dim; ++i) { + codes[i] = code_dist(rng); + values[i] = val_dist(rng); + } + const float min_val = -0.75f, delta = 0.1f; + PackedVector x(codes, min_val, delta); + auto query = makeQuery(values); + const float ref = reconstructedIP(x.data(), query.data(), dim, min_val, delta); + EXPECT_NEAR(SQ4_FP32_InnerProduct(x.data(), query.data(), dim), ref, + 1e-5 * std::max(1.0f, std::abs(ref))); +} + +TEST(SQ4Distances, ReconstructedFallbackIsExported) { + for (size_t dim : {1u, 17u, 768u}) { + SCOPED_TRACE(dim); + std::mt19937 rng(7); + std::uniform_int_distribution code_dist(0, 15); + std::uniform_real_distribution val_dist(-1.0f, 1.0f); + std::vector codes(dim); + std::vector values(dim); + for (size_t i = 0; i < dim; ++i) { + codes[i] = code_dist(rng); + values[i] = val_dist(rng); + } + const float min_val = -0.75f, delta = 0.1f; + PackedVector x(codes, min_val, delta); + auto query = makeQuery(values); + double dot = 0.0; + for (size_t i = 0; i < dim; ++i) { + dot += (double(min_val) + double(delta) * codes[i]) * values[i]; + } + const double ref = 1.0 - dot; + EXPECT_NEAR(SQ4_FP32_InnerProductReconstructed(x.data(), query.data(), dim), ref, + 1e-5 * std::max(1.0, std::abs(ref))); + } +} + +TEST(SQ4Distances, MatchedIPOverflowFallsBack) { + constexpr size_t dim = 16; + // codes {0, 15 x 15} with min 0 and delta 1/15 (what storing [0, 1 x 15] produces). + std::vector codes(dim, 15); + codes[0] = 0; + const float min_val = 0.0f, delta = 1.0f / 15; + PackedVector x(codes, min_val, delta); + auto query = makeQuery(std::vector(dim, std::numeric_limits::max() / 32)); + ASSERT_TRUE(std::isfinite(query[dim + sq4::SUM_QUERY])); + // The unscaled code dot overflows float, so the matched form is non-finite. + float code_dot = 0.0f; + for (size_t i = 0; i < dim; ++i) { + code_dot += sq4::load_code(x.data(), i) * query[i]; + } + ASSERT_TRUE(std::isinf(code_dot)); + const float expected = reconstructedIP(x.data(), query.data(), dim, min_val, delta); + ASSERT_TRUE(std::isfinite(expected)); + const float scalar = SQ4_FP32_InnerProduct(x.data(), query.data(), dim); + EXPECT_TRUE(std::isfinite(scalar)); + EXPECT_FLOAT_EQ(scalar, expected); + auto dispatched = spaces::GetDistFunc(VecSimMetric_IP, dim, nullptr); + EXPECT_FLOAT_EQ(dispatched(x.data(), query.data(), dim), expected); +} + +TEST(SQ4Distances, DispatchIsScalarWithZeroAlignment) { + for (size_t dim : {7u, 8u, 15u, 16u, 63u, 64u, 768u}) { + SCOPED_TRACE(dim); + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + SCOPED_TRACE(metric); + const bool ip = metric == VecSimMetric_IP; + unsigned char alignment = 255; + EXPECT_EQ((spaces::GetDistFunc(metric, dim, &alignment)), + ip ? SQ4_FP32_InnerProduct : SQ4_FP32_L2Sqr); + EXPECT_EQ(alignment, 0); + alignment = 255; + EXPECT_EQ((spaces::GetDistFunc(metric, dim, &alignment)), + ip ? SQ4_SQ4_InnerProduct : SQ4_SQ4_L2Sqr); + EXPECT_EQ(alignment, 0); + } + } +} From 3672c754ee97f43fafd6f51805587c8402dcca7d Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 09:23:39 +0200 Subject: [PATCH 02/11] SQ4: Expose SQ4 in the Python bindings Co-Authored-By: Claude Opus 5.5 --- src/python_bindings/bindings.cpp | 6 ++- tests/flow/test_hnsw_sq4.py | 66 ++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 tests/flow/test_hnsw_sq4.py diff --git a/src/python_bindings/bindings.cpp b/src/python_bindings/bindings.cpp index ab46ffb14..edd5fbd12 100644 --- a/src/python_bindings/bindings.cpp +++ b/src/python_bindings/bindings.cpp @@ -41,8 +41,9 @@ std::optional validateQuantizationMean(const py::object &quant if (quantization_mean.is_none()) { return std::nullopt; } - if (params.quantType != VecSimQuant_SQ8) { - throw py::value_error("quantization_mean requires quantType VecSimQuant_SQ8"); + if (params.quantType != VecSimQuant_SQ8 && params.quantType != VecSimQuant_SQ4) { + throw py::value_error( + "quantization_mean requires quantType VecSimQuant_SQ8 or VecSimQuant_SQ4"); } QuantizationMean mean(quantization_mean); @@ -812,6 +813,7 @@ PYBIND11_MODULE(VecSim, m) { py::enum_(m, "VecSimQuantType") .value("VecSimQuant_NONE", VecSimQuant_NONE) .value("VecSimQuant_SQ8", VecSimQuant_SQ8) + .value("VecSimQuant_SQ4", VecSimQuant_SQ4) .export_values(); py::enum_(m, "VecSimOptionMode") diff --git a/tests/flow/test_hnsw_sq4.py b/tests/flow/test_hnsw_sq4.py new file mode 100644 index 000000000..a5b0212c3 --- /dev/null +++ b/tests/flow/test_hnsw_sq4.py @@ -0,0 +1,66 @@ +# Copyright (c) 2006-Present, Redis Ltd. +# All rights reserved. +# +# Licensed under your choice of the Redis Source Available License 2.0 +# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the +# GNU Affero General Public License v3 (AGPLv3). +import pytest +from common import * + +DIM = 33 +NUM_VECTORS = 500 +NUM_QUERIES = 20 +K = 10 +MIN_RECALL = 0.7 + + +def make_sq4_params(metric, data_type=VecSimType_FLOAT32): + params = create_hnsw_params(dim=DIM, num_elements=NUM_VECTORS, metric=metric, + data_type=data_type, ef_construction=200, m=16, ef_runtime=200) + params.quantType = VecSimQuant_SQ4 + return params + + +def brute_force_labels(data, queries, metric): + if metric == VecSimMetric_L2: + dists = ((queries[:, None, :] - data[None, :, :]) ** 2).sum(axis=2) + else: + dists = 1.0 - queries @ data.T + return np.argsort(dists, axis=1)[:, :K] + + +@pytest.mark.parametrize("metric", [ + pytest.param(VecSimMetric_IP, id="ip"), + pytest.param(VecSimMetric_L2, id="l2"), +]) +@pytest.mark.parametrize("centered", [False, True], ids=["plain", "centered"]) +def test_sq4_hnsw_plain_and_centered(metric, centered): + rng = np.random.default_rng(42) + data = rng.random((NUM_VECTORS, DIM), dtype=np.float32) + queries = rng.random((NUM_QUERIES, DIM), dtype=np.float32) + if metric == VecSimMetric_IP: + data /= np.linalg.norm(data, axis=1, keepdims=True) + queries /= np.linalg.norm(queries, axis=1, keepdims=True) + + params = make_sq4_params(metric) + mean = data.mean(axis=0).astype(np.float32) if centered else None + index = HNSWIndex(params, quantization_mean=mean) + for label, vec in enumerate(data): + index.add_vector(vec, label) + assert index.index_size() == NUM_VECTORS + + expected = brute_force_labels(data, queries, metric) + hits = 0 + for i, query in enumerate(queries): + labels, distances = index.knn_query(query, K) + assert np.isfinite(distances).all(), distances + hits += len(set(labels[0]) & set(expected[i])) + recall = hits / (NUM_QUERIES * K) + assert recall >= MIN_RECALL, recall + + +def test_sq4_rejects_fp16_and_cosine(): + with pytest.raises(ValueError, match="Unsupported vector index parameters"): + HNSWIndex(make_sq4_params(VecSimMetric_L2, VecSimType_FLOAT16)) + with pytest.raises(ValueError, match="Unsupported vector index parameters"): + HNSWIndex(make_sq4_params(VecSimMetric_Cosine)) From f9726022c3c3f9b2a1f9b9a10f1482c76fc1c56b Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 10:43:26 +0200 Subject: [PATCH 03/11] SQ4: Apply review decisions Build SQ4 components through the generic CreateQuantizedIndexComponents wrapper and drop the SQ4-specific factory overloads. Remove the SQ4-FP32 inner product overflow fallback so SQ4 returns the plain distance like SQ8. Give sq4 packed code dot hooks that read each byte once, and name the nibble shift bits_per_code. Co-Authored-By: Claude Opus 5.5 --- .../components/components_factory.h | 21 -------- src/VecSim/index_factories/hnsw_factory.cpp | 4 +- src/VecSim/spaces/IP/IP.cpp | 26 +--------- src/VecSim/spaces/IP/IP.h | 5 -- src/VecSim/types/sq4.h | 47 ++++++++++------- tests/unit/test_sq4_components.cpp | 19 ++++--- tests/unit/test_sq4_distances.cpp | 52 +------------------ 7 files changed, 44 insertions(+), 130 deletions(-) diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index 9b97165e6..39fc2486a 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -113,27 +113,6 @@ CreateSQ8IndexComponents(const std::shared_ptr &allocator, size mean_ptr); } -template -IndexComponents -CreateSQ4IndexComponents(const std::shared_ptr &allocator, size_t dim, - const void *mean_ptr) { - static_assert(std::is_same_v, "SQ4 supports FP32 input only"); - static_assert(Metric == VecSimMetric_IP || Metric == VecSimMetric_L2, - "SQ4 supports IP and L2 only"); - return CreateQuantizedIndexComponents( - allocator, dim, static_cast(mean_ptr)); -} - -template -IndexComponents -CreateSQ4IndexComponents(const std::shared_ptr &allocator, size_t dim, - const void *mean_ptr) { - if (mean_ptr) { - return CreateSQ4IndexComponents(allocator, dim, mean_ptr); - } - return CreateSQ4IndexComponents(allocator, dim, mean_ptr); -} - template size_t EstimateComponentsMemory(VecSimMetric metric, bool is_normalized) { size_t allocations_overhead = VecSimAllocator::getAllocationOverheadSize(); diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index bcb625548..76dd6c535 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -410,14 +410,14 @@ VecSimIndex *NewIndex(const std::string &location, bool is_normalized) { abstractInitParams.storedDataSize = GetQuantizedStoredDataSize(params.dim, mean_ptr != nullptr); - auto components = CreateSQ4IndexComponents( + auto components = CreateQuantizedIndexComponents( abstractInitParams.allocator, params.dim, mean_ptr); return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, components, version, mean_ptr); } abstractInitParams.storedDataSize = GetQuantizedStoredDataSize(params.dim, mean_ptr != nullptr); - auto components = CreateSQ4IndexComponents( + auto components = CreateQuantizedIndexComponents( abstractInitParams.allocator, params.dim, mean_ptr); return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, components, version, mean_ptr); diff --git a/src/VecSim/spaces/IP/IP.cpp b/src/VecSim/spaces/IP/IP.cpp index b075764f2..c95f878bb 100644 --- a/src/VecSim/spaces/IP/IP.cpp +++ b/src/VecSim/spaces/IP/IP.cpp @@ -13,38 +13,14 @@ #include "VecSim/types/sq8.h" #include "VecSim/types/sq4.h" #include "VecSim/utils/alignment.h" -#include using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; using sq8 = vecsim_types::sq8; using sq4 = vecsim_types::sq4; -// Per-element reconstruction of the SQ4-FP32 inner product distance. Slower than the matched -// form, but it never forms the unscaled code dot product, which can overflow float for large query -// values. -float SQ4_FP32_InnerProductReconstructed(const void *pVect1v, const void *pVect2v, - size_t dimension) { - const auto *x = static_cast(pVect1v); - const auto *y = static_cast(pVect2v); - // Metadata follows the packed payload and is not necessarily float-aligned. - const auto *params = x + sq4::data_bytes_count(dimension); - const float min_val = load_unaligned(params + sq4::MIN_VAL * sizeof(float)); - const float delta = load_unaligned(params + sq4::DELTA * sizeof(float)); - float dot = 0.0f; - for (size_t i = 0; i < dimension; ++i) { - dot += (min_val + delta * sq4::load_code(x, i)) * y[i]; - } - return 1.0f - dot; -} - float SQ4_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) { - const float dist = - 1.0f - spaces::quantized::FP32_InnerProduct_Impl(pVect1v, pVect2v, dimension); - if (!std::isfinite(dist)) { - return SQ4_FP32_InnerProductReconstructed(pVect1v, pVect2v, dimension); - } - return dist; + return 1.0f - spaces::quantized::FP32_InnerProduct_Impl(pVect1v, pVect2v, dimension); } float SQ4_SQ4_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) { diff --git a/src/VecSim/spaces/IP/IP.h b/src/VecSim/spaces/IP/IP.h index 8c291dbf0..0f5deb683 100644 --- a/src/VecSim/spaces/IP/IP.h +++ b/src/VecSim/spaces/IP/IP.h @@ -13,11 +13,6 @@ // Packed SQ4 storage, compared with FP32 queries or another SQ4 vector. float SQ4_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension); float SQ4_SQ4_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension); -// Per-element reconstruction of the SQ4-FP32 inner product distance (1 - IP). SQ4_FP32_InnerProduct -// falls back to it when the matched form is not finite; SIMD SQ4-FP32 kernels can do the same. -// Returns a distance; callers return it as is (no 1 - x). -float SQ4_FP32_InnerProductReconstructed(const void *pVect1v, const void *pVect2v, - size_t dimension); // SQ8-FP32: Common inner product implementation that returns the raw inner product value // (not distance). Used by SQ8_FP32_InnerProduct, SQ8_FP32_Cosine, and SQ8_FP32_L2Sqr. diff --git a/src/VecSim/types/sq4.h b/src/VecSim/types/sq4.h index 1bb031ef1..d62d805b5 100644 --- a/src/VecSim/types/sq4.h +++ b/src/VecSim/types/sq4.h @@ -15,11 +15,12 @@ namespace vecsim_types { // Two scalar-quantized coordinates per byte, followed by the same metadata as SQ8. struct sq4 : sq8 { static constexpr uint8_t max_code = 15; + static constexpr unsigned bits_per_code = 4; static constexpr size_t data_bytes_count(size_t dim) { return dim / 2 + dim % 2; } static uint8_t load_code(const uint8_t *data, size_t index) { - return (data[index / 2] >> ((index % 2) * 4)) & max_code; + return (data[index / 2] >> ((index % 2) * bits_per_code)) & max_code; } // Coordinates are stored in ascending order. Starting each byte with an assignment allows @@ -28,33 +29,43 @@ struct sq4 : sq8 { if (index % 2 == 0) { data[index / 2] = code & max_code; } else { - data[index / 2] = (data[index / 2] & max_code) | ((code & max_code) << 4); + data[index / 2] = (data[index / 2] & max_code) | ((code & max_code) << bits_per_code); } } - // Code dot hooks bound to the packed load_code above. + // Dot product of two packed code arrays: sum(q1_i * q2_i). Each byte is loaded once and both + // nibbles are used. The 64-bit integer sum is exact for any dimension. static float code_dot(const uint8_t *a, const uint8_t *b, size_t dim) { - float product = 0; - for (size_t i = 0; i < dim; i++) { - product += load_code(a, i) * load_code(b, i); + const size_t full_bytes = dim / 2; + uint64_t sum = 0; + for (size_t j = 0; j < full_bytes; j++) { + const uint32_t a_byte = a[j], b_byte = b[j]; + sum += (a_byte & max_code) * (b_byte & max_code) + + (a_byte >> bits_per_code) * (b_byte >> bits_per_code); } - return product; + if (dim % 2) { + // Odd dim: the last byte holds only a low nibble. + sum += uint32_t(a[full_bytes] & max_code) * uint32_t(b[full_bytes] & max_code); + } + return static_cast(sum); } + // Dot product of a packed code array with FP32 values: sum(q_i * y_i). Each byte is loaded + // once; the low nibble pairs with y[2j] and the high nibble with y[2j + 1], in two independent + // accumulators for instruction-level parallelism. static float code_dot_fp32(const uint8_t *codes, const float *y, size_t dim) { - float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; - size_t i = 0; - size_t dim4 = dim & ~size_t(3); - for (; i < dim4; i += 4) { - sum0 += static_cast(load_code(codes, i + 0)) * y[i + 0]; - sum1 += static_cast(load_code(codes, i + 1)) * y[i + 1]; - sum2 += static_cast(load_code(codes, i + 2)) * y[i + 2]; - sum3 += static_cast(load_code(codes, i + 3)) * y[i + 3]; + const size_t full_bytes = dim / 2; + float sum_low = 0, sum_high = 0; + for (size_t j = 0; j < full_bytes; j++) { + const uint8_t byte = codes[j]; + sum_low += static_cast(byte & max_code) * y[2 * j]; + sum_high += static_cast(byte >> bits_per_code) * y[2 * j + 1]; } - for (; i < dim; i++) { - sum0 += static_cast(load_code(codes, i)) * y[i]; + if (dim % 2) { + // Odd dim: the last byte holds only a low nibble. + sum_low += static_cast(codes[full_bytes] & max_code) * y[dim - 1]; } - return (sum0 + sum1) + (sum2 + sum3); + return sum_low + sum_high; } template diff --git a/tests/unit/test_sq4_components.cpp b/tests/unit/test_sq4_components.cpp index 0842ab3f4..bb748b751 100644 --- a/tests/unit/test_sq4_components.cpp +++ b/tests/unit/test_sq4_components.cpp @@ -69,8 +69,8 @@ void checkComponents(size_t dim, bool constant) { x[i] = offset + (constant ? 1.25f : 0.31f * (int((i * 7) % 19) - 9)); y[i] = offset + (constant ? -0.75f : 0.23f * (int((i * 11) % 17) - 8)); } - auto components = - CreateSQ4IndexComponents(allocator, dim, WithMean ? mean.data() : nullptr); + auto components = CreateQuantizedIndexComponents( + allocator, dim, WithMean ? mean.data() : nullptr); std::unique_ptr> calculator(components.indexCalculator); std::unique_ptr preprocessors(components.preprocessors); const size_t input_size = dim * sizeof(float); @@ -179,7 +179,8 @@ TEST(SQ4Components, OddDimTailNibbleIsZero) { input[dim - 1] = 100.0f; auto check = [&](auto metric_tag) { constexpr VecSimMetric Metric = decltype(metric_tag)::value; - auto components = CreateSQ4IndexComponents(allocator, dim, nullptr); + auto components = CreateQuantizedIndexComponents( + allocator, dim, nullptr); std::unique_ptr> calculator(components.indexCalculator); std::unique_ptr preprocessors(components.preprocessors); auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); @@ -219,8 +220,8 @@ TEST(SQ4Components, NonFiniteInputEncodesInBounds) { constexpr VecSimMetric Metric = decltype(metric_tag)::value; constexpr bool WithMean = decltype(with_mean_tag)::value; SCOPED_TRACE(::testing::Message() << "metric=" << Metric << " mean=" << WithMean); - auto components = CreateSQ4IndexComponents(allocator, dim, - WithMean ? mean.data() : nullptr); + auto components = CreateQuantizedIndexComponents( + allocator, dim, WithMean ? mean.data() : nullptr); std::unique_ptr> calculator(components.indexCalculator); std::unique_ptr preprocessors(components.preprocessors); const size_t storage_size = sq4::storage_bytes_count(dim); @@ -263,7 +264,8 @@ TEST(SQ4Components, CenteredIPBlobSize768) { input[i] = 0.01f * (int(i % 41) - 20); mean[i] = 0.002f * (int(i % 9) - 4); } - auto components = CreateSQ4IndexComponents(allocator, dim, mean.data()); + auto components = CreateQuantizedIndexComponents( + allocator, dim, mean.data()); std::unique_ptr> calculator(components.indexCalculator); std::unique_ptr preprocessors(components.preprocessors); auto *container = dynamic_cast *>(preprocessors.get()); @@ -284,7 +286,8 @@ TEST(SQ4Components, SQ4ComponentsAreNotSQ8Types) { auto allocator = VecSimAllocator::newVecsimAllocator(); constexpr size_t dim = 8; std::vector mean(dim, 0.5f); - auto components = CreateSQ4IndexComponents(allocator, dim, mean.data()); + auto components = CreateQuantizedIndexComponents( + allocator, dim, mean.data()); std::unique_ptr> calculator(components.indexCalculator); std::unique_ptr preprocessors(components.preprocessors); auto *container = dynamic_cast *>(preprocessors.get()); @@ -307,7 +310,7 @@ TEST(SQ4Components, StoredL2UsesOneKernel) { std::vector zero_mean(dim, 0.0f); for (bool centered : {true, false}) { SCOPED_TRACE(centered); - auto components = CreateSQ4IndexComponents( + auto components = CreateQuantizedIndexComponents( allocator, dim, centered ? zero_mean.data() : nullptr); std::unique_ptr> calculator(components.indexCalculator); std::unique_ptr preprocessors(components.preprocessors); diff --git a/tests/unit/test_sq4_distances.cpp b/tests/unit/test_sq4_distances.cpp index 39ee83fcf..4ac414990 100644 --- a/tests/unit/test_sq4_distances.cpp +++ b/tests/unit/test_sq4_distances.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -262,7 +261,7 @@ TEST(SQ4Distances, OddDimMatchesReference) { } } -TEST(SQ4Distances, MatchedIPEqualsReconstruction) { +TEST(SQ4Distances, FP32IPEqualsReconstruction) { constexpr size_t dim = 768; std::mt19937 rng(99); std::uniform_int_distribution code_dist(0, 15); @@ -281,55 +280,6 @@ TEST(SQ4Distances, MatchedIPEqualsReconstruction) { 1e-5 * std::max(1.0f, std::abs(ref))); } -TEST(SQ4Distances, ReconstructedFallbackIsExported) { - for (size_t dim : {1u, 17u, 768u}) { - SCOPED_TRACE(dim); - std::mt19937 rng(7); - std::uniform_int_distribution code_dist(0, 15); - std::uniform_real_distribution val_dist(-1.0f, 1.0f); - std::vector codes(dim); - std::vector values(dim); - for (size_t i = 0; i < dim; ++i) { - codes[i] = code_dist(rng); - values[i] = val_dist(rng); - } - const float min_val = -0.75f, delta = 0.1f; - PackedVector x(codes, min_val, delta); - auto query = makeQuery(values); - double dot = 0.0; - for (size_t i = 0; i < dim; ++i) { - dot += (double(min_val) + double(delta) * codes[i]) * values[i]; - } - const double ref = 1.0 - dot; - EXPECT_NEAR(SQ4_FP32_InnerProductReconstructed(x.data(), query.data(), dim), ref, - 1e-5 * std::max(1.0, std::abs(ref))); - } -} - -TEST(SQ4Distances, MatchedIPOverflowFallsBack) { - constexpr size_t dim = 16; - // codes {0, 15 x 15} with min 0 and delta 1/15 (what storing [0, 1 x 15] produces). - std::vector codes(dim, 15); - codes[0] = 0; - const float min_val = 0.0f, delta = 1.0f / 15; - PackedVector x(codes, min_val, delta); - auto query = makeQuery(std::vector(dim, std::numeric_limits::max() / 32)); - ASSERT_TRUE(std::isfinite(query[dim + sq4::SUM_QUERY])); - // The unscaled code dot overflows float, so the matched form is non-finite. - float code_dot = 0.0f; - for (size_t i = 0; i < dim; ++i) { - code_dot += sq4::load_code(x.data(), i) * query[i]; - } - ASSERT_TRUE(std::isinf(code_dot)); - const float expected = reconstructedIP(x.data(), query.data(), dim, min_val, delta); - ASSERT_TRUE(std::isfinite(expected)); - const float scalar = SQ4_FP32_InnerProduct(x.data(), query.data(), dim); - EXPECT_TRUE(std::isfinite(scalar)); - EXPECT_FLOAT_EQ(scalar, expected); - auto dispatched = spaces::GetDistFunc(VecSimMetric_IP, dim, nullptr); - EXPECT_FLOAT_EQ(dispatched(x.data(), query.data(), dim), expected); -} - TEST(SQ4Distances, DispatchIsScalarWithZeroAlignment) { for (size_t dim : {7u, 8u, 15u, 16u, 63u, 64u, 768u}) { SCOPED_TRACE(dim); From 2c6c28f6375a5fbb1d5c6cee45248a3d16c644b3 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 12:13:54 +0200 Subject: [PATCH 04/11] SQ4: Accumulate the centered IP correction while quantizing The SQ4 mean correction x_mean_ip is summed from each code as it is generated, in coordinate order with one double accumulator, instead of in a second pass that re-reads and decodes the packed codes. The stored bytes and the correction are unchanged. For SQ4 the input-side sum in find_min_max, which was overwritten, is skipped. Co-Authored-By: Claude Opus 5.5 --- src/VecSim/spaces/computer/preprocessors.h | 37 +++++++++++++++------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index e400a7f2d..469d6d979 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -269,6 +269,11 @@ class QuantPreprocessor : public PreprocessorInterface { using OUTPUT_TYPE = uint8_t; using MetadataType = float; // Quantized storage metadata is always FP32. + // SQ4's centered IP correction x_mean_ip describes the reconstructed vector, like the other + // stored sums, so it is accumulated from the generated codes instead of the input values. + static constexpr bool kMeanIpFromCodes = + WithNorm && Metric == VecSimMetric_IP && std::is_same_v; + static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP || Metric == VecSimMetric_Cosine, "QuantPreprocessor only supports L2, IP and Cosine metrics"); @@ -322,6 +327,17 @@ class QuantPreprocessor : public PreprocessorInterface { // the metadata rather than round it. This is the write path, once per vector. uint64_t q0{}, q1{}, q2{}, q3{}; // only used for L2 + // For centered SQ4 IP, the mean correction is accumulated from the codes as they are + // generated (see kMeanIpFromCodes), in coordinate order with one accumulator. + const double d_min = min_val, d_delta = delta; + [[maybe_unused]] double reconstructed_mean_ip = 0.0; + [[maybe_unused]] const auto add_mean_ip = [&](size_t idx, OUTPUT_TYPE code) { + if constexpr (kMeanIpFromCodes) { + const double centered = d_min + d_delta * code; + reconstructed_mean_ip += (centered + mean[idx]) * mean[idx]; + } + }; + size_t i = 0; // round dim down to the nearest multiple of 4 size_t dim_round_down = this->dim & ~size_t(3); @@ -344,6 +360,10 @@ class QuantPreprocessor : public PreprocessorInterface { StorageType::store_code(quantized, i + 1, a1); StorageType::store_code(quantized, i + 2, a2); StorageType::store_code(quantized, i + 3, a3); + add_mean_ip(i, a0); + add_mean_ip(i + 1, a1); + add_mean_ip(i + 2, a2); + add_mean_ip(i + 3, a3); s0 += a0; s1 += a1; @@ -369,6 +389,7 @@ class QuantPreprocessor : public PreprocessorInterface { const float x = transformed_value(input, i); const OUTPUT_TYPE a = to_byte((x - min_val) * inv_delta); StorageType::store_code(quantized, i, a); + add_mean_ip(i, a); q_sum += a; if constexpr (Metric == VecSimMetric_L2) { q_sum_squares += uint64_t{a} * a; @@ -380,7 +401,7 @@ class QuantPreprocessor : public PreprocessorInterface { // unchanged; only what the numbers describe changes. // sum = sum(x_r[i]) = dim*min + delta*q_sum // sum_squares = sum(x_r[i]^2) = dim*min^2 + 2*min*delta*q_sum + delta^2*q_sum_squares - const double d_min = min_val, d_delta = delta, d_dim = static_cast(this->dim); + const double d_dim = static_cast(this->dim); const MetadataType sum = static_cast(d_dim * d_min + d_delta * q_sum); MetadataType sum_squares{}; if constexpr (Metric == VecSimMetric_L2) { @@ -389,15 +410,7 @@ class QuantPreprocessor : public PreprocessorInterface { d_delta * d_delta * q_sum_squares); } - if constexpr (WithNorm && Metric == VecSimMetric_IP && - std::is_same_v) { - // Correction must describe the same reconstructed vector as the packed codes. - // Read the emitted codes so this also works when input and output alias. - double reconstructed_mean_ip = 0.0; - for (size_t j = 0; j < this->dim; ++j) { - const double centered = d_min + d_delta * StorageType::load_code(quantized, j); - reconstructed_mean_ip += (centered + mean[j]) * mean[j]; - } + if constexpr (kMeanIpFromCodes) { x_mean_ip = static_cast(reconstructed_mean_ip); } @@ -653,7 +666,7 @@ class QuantPreprocessor : public PreprocessorInterface { float max_val = value; // x_mean_ip only feeds the IP correction term (see DistanceCalculatorWithNorm); the // normalized L2 correction cancels the mean terms exactly, so skip the extra work. - if constexpr (Metric == VecSimMetric_IP) + if constexpr (Metric == VecSimMetric_IP && !kMeanIpFromCodes) x_mean_ip = first_input_value * mean[0]; for (size_t i = 1; i < dim; ++i) { @@ -661,7 +674,7 @@ class QuantPreprocessor : public PreprocessorInterface { value = input_value - mean[i]; min_val = std::min(min_val, value); max_val = std::max(max_val, value); - if constexpr (Metric == VecSimMetric_IP) + if constexpr (Metric == VecSimMetric_IP && !kMeanIpFromCodes) x_mean_ip += input_value * mean[i]; } return {min_val, max_val}; From a202b78cc29f5c001fb971fe2aff2f72bd7ac78c Mon Sep 17 00:00:00 2001 From: dor-forer Date: Sun, 8 Nov 2026 15:00:31 +0200 Subject: [PATCH 05/11] SQ4: Skip unused L2 query reductions Keep two initialized FP32 padding slots for SIMD tails and update the query metadata contract and component checks. Validation on dorer-intel: SQ4 81 passed, 12 expected skips; SQ8 209 passed. --- src/VecSim/spaces/computer/preprocessors.h | 19 +++++++++++++++---- src/VecSim/spaces/quantized_kernels.h | 1 + tests/unit/test_sq4_components.cpp | 10 ++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 469d6d979..4746458d2 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -179,11 +179,13 @@ class CosinePreprocessor : public PreprocessorInterface { * store metric-specific FP32 values from the query body to accelerate asymmetric distance * computation: * - For IP/Cosine: y_sum = Σy_i (sum of query values) - * - For L2: y_sum = Σy_i (sum of query values), y_sum_squares = Σy_i² (sum of squared query values) + * - For SQ8 L2: y_sum = Σy_i, y_sum_squares = Σy_i² + * - For SQ4 L2: two zero-initialized reserved FP32 slots; direct residual kernels need no sums. * * Query blob layout: * - For IP/Cosine: | query_values[dim] | y_sum | * - For L2: | query_values[dim] | y_sum | y_sum_squares | + * SQ4 L2 keeps the same layout, with both trailing slots reserved and zero-initialized. * * Query metadata is always FP32. The query blob size is: * - For IP/Cosine: dim * sizeof(DataType) + 1 * sizeof(float) @@ -202,6 +204,7 @@ class CosinePreprocessor : public PreprocessorInterface { * where y_sum = Σy_i is precomputed and stored in the query blob. * * For L2: + * The identity below explains the metadata; direct residual query kernels do not consume it. * ||x_r - y||² = Σx_r[i]² - 2*Σ(x_r[i] * y_i) + Σy_i² * = x_sum_squares - 2 * IP(x_r, y) + y_sum_squares * where: @@ -601,12 +604,13 @@ class QuantPreprocessor : public PreprocessorInterface { * the original query values. * * - For IP/Cosine: y_sum = Σy_i (sum of query values) - * - For L2: y_sum = Σy_i (sum of query values), y_sum_squares = Σy_i² (sum of squared query - * values) + * - For SQ8 L2: y_sum = Σy_i, y_sum_squares = Σy_i² + * - For SQ4 L2: two zero-initialized reserved FP32 slots; no query reductions are needed. * * Query blob layout: * - For IP/Cosine: | query_values[dim] | y_sum | * - For L2: | query_values[dim] | y_sum | y_sum_squares | + * SQ4 L2 reserves and zero-initializes both trailing slots instead of computing sums. * * Query blob size: * - For IP/Cosine: dim * sizeof(QueryType) + 1 * sizeof(float) @@ -632,7 +636,14 @@ class QuantPreprocessor : public PreprocessorInterface { // Compute and write FP32 query metadata after the query body. The metadata offset is // body_bytes, which is not guaranteed to be 4-byte aligned for FP16 query bodies. void *metadata_dst = static_cast(blob) + body_bytes; - assign_query_metadata(query_values, input, metadata_dst); + if constexpr (std::is_same_v && Metric == VecSimMetric_L2) { + // SQ4 L2 computes residuals directly, so query sums are unused. Keep initialized + // padding for SIMD tails that load a final query pair when dim is odd. + const MetadataType padding[2] = {}; + memcpy(metadata_dst, padding, sizeof(padding)); + } else { + assign_query_metadata(query_values, input, metadata_dst); + } query_blob_size = this->query_bytes_count; } diff --git a/src/VecSim/spaces/quantized_kernels.h b/src/VecSim/spaces/quantized_kernels.h index 6cad11929..bc9b47ed9 100644 --- a/src/VecSim/spaces/quantized_kernels.h +++ b/src/VecSim/spaces/quantized_kernels.h @@ -22,6 +22,7 @@ // // Stored layout: [codes (data_bytes_count(dim) bytes)] [min_val] [delta] [sum] [sum_squares (L2)] // FP32 query layout: [float values (dim)] [y_sum] [y_sum_squares (L2 only)] +// SQ4 L2 uses only the query body; its two trailing FP32 slots are reserved and zero-initialized. // // The operand order in these bodies is load-bearing and relies on FP addition not being // reassociated; they must not be built with -ffast-math / -Ofast. diff --git a/tests/unit/test_sq4_components.cpp b/tests/unit/test_sq4_components.cpp index bb748b751..2b72620de 100644 --- a/tests/unit/test_sq4_components.cpp +++ b/tests/unit/test_sq4_components.cpp @@ -88,6 +88,16 @@ void checkComponents(size_t dim, bool constant) { input_size + sq4::query_metadata_count() * sizeof(float); EXPECT_EQ(std::memcmp(combined.getStorageBlob(), y_blob.get(), storage_size), 0); EXPECT_EQ(std::memcmp(combined.getQueryBlob(), query.get(), query_size), 0); + if constexpr (Metric == VecSimMetric_L2) { + // Direct residual kernels need no query sums, but SIMD odd tails may read padding. + EXPECT_FLOAT_EQ(metadata(query.get(), input_size, sq4::SUM_QUERY), 0.0f); + EXPECT_FLOAT_EQ(metadata(query.get(), input_size, sq4::SUM_SQUARES_QUERY), 0.0f); + const auto *query_values = static_cast(query.get()); + for (size_t i = 0; i < dim; ++i) { + const float expected = WithMean ? y[i] - mean[i] : y[i]; + EXPECT_FLOAT_EQ(query_values[i], expected); + } + } const auto rx = reconstruct(x_blob.get(), dim, mean); const auto ry = reconstruct(y_blob.get(), dim, mean); const double symmetric = referenceDistance(rx, ry); From 9f3f5b9226bb769d570dde54251ee17f23ff100c Mon Sep 17 00:00:00 2001 From: dor-forer Date: Sun, 8 Nov 2026 15:12:34 +0200 Subject: [PATCH 06/11] SQ4: Reuse each packed byte in scalar FP32 L2 Decode both nibbles from one load, preserve the residual and coordinate accumulation order, and handle odd tails explicitly. Verify bitwise agreement with the previous scalar arithmetic across odd/even dimensions, nonzero padding, and large offsets. Validation on dorer-intel: SQ4 82 passed with 12 expected skips; SQ8 209 passed. GCC 13.3 production assembly has one byte load per pair. --- src/VecSim/spaces/L2/L2.cpp | 25 ++++++++++++++++++++++++- tests/unit/test_sq4_distances.cpp | 28 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/VecSim/spaces/L2/L2.cpp b/src/VecSim/spaces/L2/L2.cpp index 90a7caedd..8fc0173d5 100644 --- a/src/VecSim/spaces/L2/L2.cpp +++ b/src/VecSim/spaces/L2/L2.cpp @@ -22,7 +22,30 @@ using sq8 = vecsim_types::sq8; using sq4 = vecsim_types::sq4; float SQ4_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { - return spaces::quantized::FP32_L2Sqr(pVect1v, pVect2v, dimension); + const auto *codes = static_cast(pVect1v); + const auto *query = static_cast(pVect2v); + const auto *params = codes + sq4::data_bytes_count(dimension); + const float min_val = load_unaligned(params + sq4::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params + sq4::DELTA * sizeof(float)); + + float result = 0.0f; + const size_t full_bytes = dimension / 2; + for (size_t j = 0; j < full_bytes; ++j) { + // Load once and use both nibbles. Keep min - query before adding delta * code so + // rounding stays at the residual's scale, and accumulate in coordinate order. + const uint8_t byte = codes[j]; + const float even = delta * float(byte & sq4::max_code) + (min_val - query[2 * j]); + const float odd = delta * float(byte >> sq4::bits_per_code) + (min_val - query[2 * j + 1]); + result += even * even; + result += odd * odd; + } + if (dimension % 2) { + // Only the low nibble is a coordinate; never read a query value for the padding. + const float last = + delta * float(codes[full_bytes] & sq4::max_code) + (min_val - query[dimension - 1]); + result += last * last; + } + return result; } // SQ4-SQ4 L2 squared distance with the same identity as SQ8-SQ8: diff --git a/tests/unit/test_sq4_distances.cpp b/tests/unit/test_sq4_distances.cpp index 4ac414990..590836cd4 100644 --- a/tests/unit/test_sq4_distances.cpp +++ b/tests/unit/test_sq4_distances.cpp @@ -15,6 +15,7 @@ #include "VecSim/types/sq4.h" #include +#include #include #include #include @@ -164,6 +165,33 @@ TEST(SQ4Distances, FP32L2PreservesResidualsAtLargeCommonOffset) { EXPECT_FLOAT_EQ(SQ4_FP32_L2Sqr(x.data(), query.data(), 3), expected); } +TEST(SQ4Distances, FP32L2PackedPairsPreserveCoordinateOrder) { + for (size_t dim : {0u, 1u, 2u, 3u, 7u, 16u, 31u, 32u, 33u, 65u, 767u, 768u, 769u}) { + for (const auto &[minimum, delta] : + {std::pair{-0.75f, 0.1f}, std::pair{1048576.0f, 0.03125f}}) { + SCOPED_TRACE(::testing::Message() << "dim=" << dim << " min=" << minimum); + std::vector codes(dim); + std::vector query(dim); + float expected = 0.0f; + for (size_t i = 0; i < dim; ++i) { + codes[i] = (i * 7 + 3) % 16; + query[i] = minimum + 0.13f * (int(i % 17) - 8); + // Use the original codes, independently of the packed blob, and the previous + // scalar kernel's operand and reduction order. + const float diff = delta * float(codes[i]) + (minimum - query[i]); + expected += diff * diff; + } + PackedVector stored(codes, minimum, delta); + if (dim % 2) { + // A foreign writer's nonzero padding must not become another coordinate. + stored.data()[dim / 2] |= 0xf0; + } + const float actual = SQ4_FP32_L2Sqr(stored.data(), query.data(), dim); + EXPECT_EQ(std::bit_cast(actual), std::bit_cast(expected)); + } + } +} + TEST(SQ4Distances, RejectsUnsupportedMetric) { EXPECT_THROW((spaces::GetDistFunc(VecSimMetric_Cosine, 3, nullptr)), std::invalid_argument); From d6dee30cee7d6eb8b1cc7b8a1c3b24ae0c099fe9 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 15:50:44 +0200 Subject: [PATCH 07/11] SQ4: Compute centered IP correction from original input --- src/VecSim/spaces/computer/preprocessors.h | 35 +++--------------- tests/unit/test_hnsw_sq4.cpp | 11 +++++- tests/unit/test_sq4_components.cpp | 43 ++++++++++++++++++++-- 3 files changed, 54 insertions(+), 35 deletions(-) diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 4746458d2..0732026e5 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -272,11 +272,6 @@ class QuantPreprocessor : public PreprocessorInterface { using OUTPUT_TYPE = uint8_t; using MetadataType = float; // Quantized storage metadata is always FP32. - // SQ4's centered IP correction x_mean_ip describes the reconstructed vector, like the other - // stored sums, so it is accumulated from the generated codes instead of the input values. - static constexpr bool kMeanIpFromCodes = - WithNorm && Metric == VecSimMetric_IP && std::is_same_v; - static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP || Metric == VecSimMetric_Cosine, "QuantPreprocessor only supports L2, IP and Cosine metrics"); @@ -330,17 +325,6 @@ class QuantPreprocessor : public PreprocessorInterface { // the metadata rather than round it. This is the write path, once per vector. uint64_t q0{}, q1{}, q2{}, q3{}; // only used for L2 - // For centered SQ4 IP, the mean correction is accumulated from the codes as they are - // generated (see kMeanIpFromCodes), in coordinate order with one accumulator. - const double d_min = min_val, d_delta = delta; - [[maybe_unused]] double reconstructed_mean_ip = 0.0; - [[maybe_unused]] const auto add_mean_ip = [&](size_t idx, OUTPUT_TYPE code) { - if constexpr (kMeanIpFromCodes) { - const double centered = d_min + d_delta * code; - reconstructed_mean_ip += (centered + mean[idx]) * mean[idx]; - } - }; - size_t i = 0; // round dim down to the nearest multiple of 4 size_t dim_round_down = this->dim & ~size_t(3); @@ -363,10 +347,6 @@ class QuantPreprocessor : public PreprocessorInterface { StorageType::store_code(quantized, i + 1, a1); StorageType::store_code(quantized, i + 2, a2); StorageType::store_code(quantized, i + 3, a3); - add_mean_ip(i, a0); - add_mean_ip(i + 1, a1); - add_mean_ip(i + 2, a2); - add_mean_ip(i + 3, a3); s0 += a0; s1 += a1; @@ -392,7 +372,6 @@ class QuantPreprocessor : public PreprocessorInterface { const float x = transformed_value(input, i); const OUTPUT_TYPE a = to_byte((x - min_val) * inv_delta); StorageType::store_code(quantized, i, a); - add_mean_ip(i, a); q_sum += a; if constexpr (Metric == VecSimMetric_L2) { q_sum_squares += uint64_t{a} * a; @@ -404,7 +383,7 @@ class QuantPreprocessor : public PreprocessorInterface { // unchanged; only what the numbers describe changes. // sum = sum(x_r[i]) = dim*min + delta*q_sum // sum_squares = sum(x_r[i]^2) = dim*min^2 + 2*min*delta*q_sum + delta^2*q_sum_squares - const double d_dim = static_cast(this->dim); + const double d_min = min_val, d_delta = delta, d_dim = static_cast(this->dim); const MetadataType sum = static_cast(d_dim * d_min + d_delta * q_sum); MetadataType sum_squares{}; if constexpr (Metric == VecSimMetric_L2) { @@ -413,10 +392,6 @@ class QuantPreprocessor : public PreprocessorInterface { d_delta * d_delta * q_sum_squares); } - if constexpr (kMeanIpFromCodes) { - x_mean_ip = static_cast(reconstructed_mean_ip); - } - // Metadata uses MetadataType. Use memcpy because the metadata offset (after the // quantized body, packed for SQ4) is not guaranteed to be sizeof(MetadataType)-aligned. void *meta_dst = quantized + StorageType::data_bytes_count(this->dim); @@ -675,9 +650,9 @@ class QuantPreprocessor : public PreprocessorInterface { float value = first_input_value - mean[0]; float min_val = value; float max_val = value; - // x_mean_ip only feeds the IP correction term (see DistanceCalculatorWithNorm); the - // normalized L2 correction cancels the mean terms exactly, so skip the extra work. - if constexpr (Metric == VecSimMetric_IP && !kMeanIpFromCodes) + // Use the original input for the IP correction: a correction from reconstructed + // values would also multiply quantization error by the mean. L2 needs no correction. + if constexpr (Metric == VecSimMetric_IP) x_mean_ip = first_input_value * mean[0]; for (size_t i = 1; i < dim; ++i) { @@ -685,7 +660,7 @@ class QuantPreprocessor : public PreprocessorInterface { value = input_value - mean[i]; min_val = std::min(min_val, value); max_val = std::max(max_val, value); - if constexpr (Metric == VecSimMetric_IP && !kMeanIpFromCodes) + if constexpr (Metric == VecSimMetric_IP) x_mean_ip += input_value * mean[i]; } return {min_val, max_val}; diff --git a/tests/unit/test_hnsw_sq4.cpp b/tests/unit/test_hnsw_sq4.cpp index aeb16ac2e..62f17ca4f 100644 --- a/tests/unit/test_hnsw_sq4.cpp +++ b/tests/unit/test_hnsw_sq4.cpp @@ -147,7 +147,16 @@ TEST_P(HNSWSQ4Test, InsertQueryMutateAndSerialize) { EXPECT_NEAR(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), i, query.data()), score, 2e-5 * (1.0 + std::abs(score))); auto other = reconstruct(hnsw->getDataByInternalId((i + 1) % count), dim, mean); - const double expected_pair = distance(reconstructed, other, metric); + double expected_pair = distance(reconstructed, other, metric); + if (centered && metric == VecSimMetric_IP) { + expected_pair = 1.0; + for (size_t j = 0; j < dim; ++j) { + expected_pair -= + (double(reconstructed[j]) - mean[j]) * (double(other[j]) - mean[j]) + + double(inputs[i][j]) * mean[j] + double(inputs[(i + 1) % count][j]) * mean[j] - + double(mean[j]) * mean[j]; + } + } EXPECT_NEAR(hnsw->calcDistance(hnsw->getDataByInternalId(i), hnsw->getDataByInternalId((i + 1) % count)), expected_pair, 3e-5 * (1.0 + std::abs(expected_pair))); diff --git a/tests/unit/test_sq4_components.cpp b/tests/unit/test_sq4_components.cpp index 2b72620de..1e243ce6d 100644 --- a/tests/unit/test_sq4_components.cpp +++ b/tests/unit/test_sq4_components.cpp @@ -100,7 +100,16 @@ void checkComponents(size_t dim, bool constant) { } const auto rx = reconstruct(x_blob.get(), dim, mean); const auto ry = reconstruct(y_blob.get(), dim, mean); - const double symmetric = referenceDistance(rx, ry); + double symmetric = referenceDistance(rx, ry); + if constexpr (WithMean && Metric == VecSimMetric_IP) { + // The kernel uses reconstructed centered values, but the mean correction uses the + // original inputs to avoid multiplying quantization error by the mean. + symmetric = 1.0; + for (size_t i = 0; i < dim; ++i) { + symmetric -= (rx[i] - mean[i]) * (ry[i] - mean[i]) + double(x[i]) * mean[i] + + double(y[i]) * mean[i] - double(mean[i]) * mean[i]; + } + } const double asymmetric = referenceDistance(rx, y); const double sym_tolerance = 2e-5 * (1.0 + std::abs(symmetric)); const double asym_tolerance = 2e-5 * (1.0 + std::abs(asymmetric)); @@ -133,9 +142,9 @@ void checkComponents(size_t dim, bool constant) { } if constexpr (WithMean && Metric == VecSimMetric_IP) { - double correction = 0.0; + float correction = 0.0f; for (size_t i = 0; i < dim; ++i) { - correction += rx[i] * mean[i]; + correction += x[i] * mean[i]; } EXPECT_NEAR( metadata(x_blob.get(), sq4::data_bytes_count(dim), sq4::mean_ip_index()), @@ -157,7 +166,33 @@ void checkDimensions() { TEST(SQ4Components, L2MatchesReconstruction) { checkDimensions(); } -TEST(SQ4Components, IPMatchesReconstruction) { checkDimensions(); } +TEST(SQ4Components, IPMatchesQuantizedKernelAndOriginalMeanCorrection) { + checkDimensions(); +} + +TEST(SQ4Components, CenteredIPCorrectionAvoidsMeanAmplifiedQuantizationError) { + constexpr size_t dim = 3; + const std::vector mean(dim, 10.0f); + const std::vector x = {10.0f, 13.0f, 12.1f}; + const std::vector y = {10.0f, 13.0f, 11.0f}; + auto allocator = VecSimAllocator::newVecsimAllocator(); + auto components = + CreateQuantizedIndexComponents(allocator, dim, mean.data()); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto xb = preprocessors->preprocessForStorage(x.data(), dim * sizeof(float)); + auto yb = preprocessors->preprocessForStorage(y.data(), dim * sizeof(float)); + const auto rx = reconstruct(xb.get(), dim, mean); + const auto ry = reconstruct(yb.get(), dim, mean); + const double truth = referenceDistance(x, y); + const double decoded = referenceDistance(rx, ry); + const float actual = calculator->calcDistance(xb.get(), yb.get(), dim); + EXPECT_FLOAT_EQ( + metadata(xb.get(), sq4::data_bytes_count(dim), sq4::mean_ip_index()), + 351.0f); + EXPECT_LT(std::abs(actual - truth), 0.11); + EXPECT_GT(std::abs(decoded - truth), 1.0); +} TEST(SQ4Components, PackedCodesAndMetadataSizes) { auto allocator = VecSimAllocator::newVecsimAllocator(); From 4860441bdd9dbf291ce5b3ebbde3594abf3a29e6 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 17:52:50 +0200 Subject: [PATCH 08/11] SQ4: Multiply code_dot nibbles in 64-bit The products are at most 225 and never overflowed, but multiplying in the 64-bit accumulator type keeps static analysis from flagging a widening after a 32-bit multiply. Co-Authored-By: Claude Opus 5.5 --- src/VecSim/types/sq4.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/VecSim/types/sq4.h b/src/VecSim/types/sq4.h index d62d805b5..30e164297 100644 --- a/src/VecSim/types/sq4.h +++ b/src/VecSim/types/sq4.h @@ -39,13 +39,13 @@ struct sq4 : sq8 { const size_t full_bytes = dim / 2; uint64_t sum = 0; for (size_t j = 0; j < full_bytes; j++) { - const uint32_t a_byte = a[j], b_byte = b[j]; + const uint64_t a_byte = a[j], b_byte = b[j]; sum += (a_byte & max_code) * (b_byte & max_code) + (a_byte >> bits_per_code) * (b_byte >> bits_per_code); } if (dim % 2) { // Odd dim: the last byte holds only a low nibble. - sum += uint32_t(a[full_bytes] & max_code) * uint32_t(b[full_bytes] & max_code); + sum += uint64_t(a[full_bytes] & max_code) * uint64_t(b[full_bytes] & max_code); } return static_cast(sum); } From bf9f9935a2f56d16b40f0f1a28cc8d92b384b45e Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Mon, 9 Nov 2026 16:23:37 +0200 Subject: [PATCH 09/11] Use shared quantization metadata directly for SQ4 --- src/VecSim/types/sq4.h | 6 ++++-- tests/unit/test_sq4_distances.cpp | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/VecSim/types/sq4.h b/src/VecSim/types/sq4.h index 30e164297..660ed4424 100644 --- a/src/VecSim/types/sq4.h +++ b/src/VecSim/types/sq4.h @@ -8,12 +8,14 @@ */ #pragma once -#include "VecSim/types/sq8.h" +#include +#include "VecSim/types/quantized_metadata.h" namespace vecsim_types { // Two scalar-quantized coordinates per byte, followed by the same metadata as SQ8. -struct sq4 : sq8 { +struct sq4 : quantized_metadata { + using value_type = uint8_t; static constexpr uint8_t max_code = 15; static constexpr unsigned bits_per_code = 4; diff --git a/tests/unit/test_sq4_distances.cpp b/tests/unit/test_sq4_distances.cpp index 590836cd4..2c0d51f3f 100644 --- a/tests/unit/test_sq4_distances.cpp +++ b/tests/unit/test_sq4_distances.cpp @@ -13,6 +13,7 @@ #include "VecSim/spaces/IP_space.h" #include "VecSim/spaces/L2_space.h" #include "VecSim/types/sq4.h" +#include "VecSim/types/sq8.h" #include #include From b8ad85297e285ec8ba7a164ba114421817b206fa Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Mon, 9 Nov 2026 16:33:23 +0200 Subject: [PATCH 10/11] Rename SQ4 code accessors to get_code and set_code --- src/VecSim/types/sq4.h | 4 ++-- tests/unit/test_sq4_components.cpp | 4 ++-- tests/unit/test_sq4_distances.cpp | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/VecSim/types/sq4.h b/src/VecSim/types/sq4.h index 660ed4424..58d6e131c 100644 --- a/src/VecSim/types/sq4.h +++ b/src/VecSim/types/sq4.h @@ -21,13 +21,13 @@ struct sq4 : quantized_metadata { static constexpr size_t data_bytes_count(size_t dim) { return dim / 2 + dim % 2; } - static uint8_t load_code(const uint8_t *data, size_t index) { + static uint8_t get_code(const uint8_t *data, size_t index) { return (data[index / 2] >> ((index % 2) * bits_per_code)) & max_code; } // Coordinates are stored in ascending order. Starting each byte with an assignment allows // in-place preprocessing and leaves the unused high nibble zero for odd dimensions. - static void store_code(uint8_t *data, size_t index, uint8_t code) { + static void set_code(uint8_t *data, size_t index, uint8_t code) { if (index % 2 == 0) { data[index / 2] = code & max_code; } else { diff --git a/tests/unit/test_sq4_components.cpp b/tests/unit/test_sq4_components.cpp index 1e243ce6d..b7db8e16e 100644 --- a/tests/unit/test_sq4_components.cpp +++ b/tests/unit/test_sq4_components.cpp @@ -234,7 +234,7 @@ TEST(SQ4Components, OddDimTailNibbleIsZero) { EXPECT_EQ((sq4::storage_bytes_count(dim)), dim / 2 + dim % 2 + metadata_count * sizeof(float)); // A single coordinate is a constant range, which encodes as code 0. - EXPECT_EQ(sq4::load_code(bytes, dim - 1), dim == 1 ? 0 : 15); + EXPECT_EQ(sq4::get_code(bytes, dim - 1), dim == 1 ? 0 : 15); if (dim % 2) { EXPECT_EQ(bytes[dim / 2] & 0xf0, 0); } @@ -273,7 +273,7 @@ TEST(SQ4Components, NonFiniteInputEncodesInBounds) { auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); for (size_t i = 0; i < dim; ++i) { - EXPECT_LE(sq4::load_code(static_cast(blob.get()), i), 15); + EXPECT_LE(sq4::get_code(static_cast(blob.get()), i), 15); } // In place, with a sentinel tail: nothing past the stored blob may be written. The input diff --git a/tests/unit/test_sq4_distances.cpp b/tests/unit/test_sq4_distances.cpp index 2c0d51f3f..345ffde8e 100644 --- a/tests/unit/test_sq4_distances.cpp +++ b/tests/unit/test_sq4_distances.cpp @@ -47,7 +47,7 @@ struct PackedVector { reconstructed(codes.size()) { float metadata[4] = {minimum, delta, 0.0f, 0.0f}; for (size_t i = 0; i < codes.size(); ++i) { - sq4::store_code(data(), i, codes[i]); + sq4::set_code(data(), i, codes[i]); reconstructed[i] = minimum + delta * codes[i]; metadata[sq4::SUM] += reconstructed[i]; metadata[sq4::SUM_SQUARES] += reconstructed[i] * reconstructed[i]; @@ -77,21 +77,21 @@ TEST(SQ4Distances, PackedLayoutAndSQ8Compatibility) { uint8_t packed[8]; std::memset(packed, 0xff, sizeof(packed)); for (size_t i = 0; i < 16; ++i) { - sq4::store_code(packed, i, i); + sq4::set_code(packed, i, i); if (i % 2 == 0) { EXPECT_EQ(packed[i / 2] & 0xf0, 0); } } for (size_t i = 0; i < 16; ++i) { - EXPECT_EQ(sq4::load_code(packed, i), i); + EXPECT_EQ(sq4::get_code(packed, i), i); if (i % 2 == 0) { EXPECT_EQ(packed[i / 2], i | ((i + 1) << 4)); } } uint8_t unpacked[256]; for (size_t i = 0; i < 256; ++i) { - sq8::store_code(unpacked, i, i); - EXPECT_EQ(sq8::load_code(unpacked, i), i); + sq8::set_code(unpacked, i, i); + EXPECT_EQ(sq8::get_code(unpacked, i), i); EXPECT_EQ(unpacked[i], i); } } @@ -234,7 +234,7 @@ double storedMeta(const uint8_t *blob, size_t dim, size_t slot) { double identityIP(const uint8_t *x, const uint8_t *y, size_t dim) { double code_dot = 0.0; for (size_t i = 0; i < dim; ++i) { - code_dot += double(sq4::load_code(x, i)) * sq4::load_code(y, i); + code_dot += double(sq4::get_code(x, i)) * sq4::get_code(y, i); } const double min_x = storedMeta(x, dim, sq4::MIN_VAL), dx = storedMeta(x, dim, sq4::DELTA); const double min_y = storedMeta(y, dim, sq4::MIN_VAL), dy = storedMeta(y, dim, sq4::DELTA); @@ -246,7 +246,7 @@ double identityIP(const uint8_t *x, const uint8_t *y, size_t dim) { float reconstructedIP(const uint8_t *x, const float *y, size_t dim, float min_val, float delta) { float dot = 0.0f; for (size_t i = 0; i < dim; ++i) { - dot += (min_val + delta * sq4::load_code(x, i)) * y[i]; + dot += (min_val + delta * sq4::get_code(x, i)) * y[i]; } return 1.0f - dot; } @@ -275,8 +275,8 @@ TEST(SQ4Distances, OddDimMatchesReference) { // Decode with the exact float min/delta stored in the blobs, then compute in double. const float minx = -0.5f, dx = 1.0f / 15, miny = 0.25f, dy = 1.0f / 30; for (size_t i = 0; i < dim; ++i) { - xd[i] = double(minx) + double(dx) * sq4::load_code(x.data(), i); - yd[i] = double(miny) + double(dy) * sq4::load_code(y.data(), i); + xd[i] = double(minx) + double(dx) * sq4::get_code(x.data(), i); + yd[i] = double(miny) + double(dy) * sq4::get_code(y.data(), i); } auto check = [](float actual, double ref) { EXPECT_NEAR(actual, ref, 1e-4 * std::max(1.0, std::abs(ref))); From a9efe8c792b1ebf9fbfee2f98e34000fbc291fed Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Mon, 9 Nov 2026 16:49:48 +0200 Subject: [PATCH 11/11] Document scalar SQ4 code dot-product callers --- src/VecSim/types/sq4.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/VecSim/types/sq4.h b/src/VecSim/types/sq4.h index 58d6e131c..6427b19ca 100644 --- a/src/VecSim/types/sq4.h +++ b/src/VecSim/types/sq4.h @@ -37,6 +37,7 @@ struct sq4 : quantized_metadata { // Dot product of two packed code arrays: sum(q1_i * q2_i). Each byte is loaded once and both // nibbles are used. The 64-bit integer sum is exact for any dimension. + // Used by scalar SQ4-to-SQ4 IP and L2 kernels; SIMD kernels compute the dot product separately. static float code_dot(const uint8_t *a, const uint8_t *b, size_t dim) { const size_t full_bytes = dim / 2; uint64_t sum = 0;