diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index 15a7d54f3..2d03fc233 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -14,12 +14,14 @@ #include "VecSim/algorithms/hnsw/hnsw.h" #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" +#include "VecSim/types/sq4.h" #include using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; using sq8 = vecsim_types::sq8; +using sq4 = vecsim_types::sq4; namespace HNSWFactory { @@ -56,10 +58,13 @@ template VecSimType type, VecSimMetric resolved_metric) { // Reject unknown quantizers instead of silently creating an unquantized index. - if (quant_type != VecSimQuant_SQ8) { + if (quant_type != VecSimQuant_SQ8 && quant_type != VecSimQuant_SQ4) { return false; } - // SQ8 kernels accept only FLOAT32 and FLOAT16 input. + if (quant_type == VecSimQuant_SQ4 && type != VecSimType_FLOAT32) { + return false; + } + // SQ8 additionally accepts FLOAT16 input. if (type != VecSimType_FLOAT32 && type != VecSimType_FLOAT16) { return false; } @@ -106,6 +111,15 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { return NULL; } + if (hnswParams->quantType == VecSimQuant_SQ4) { + if (metric == VecSimMetric_L2) { + return NewIndex_Quantized( + hnswParams, abstractInitParams, mean_ptr); + } + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); + } + if (hnswParams->type == VecSimType_FLOAT32) { if (metric == VecSimMetric_L2) { return NewIndex_Quantized(hnswParams, abstractInitParams, @@ -196,7 +210,12 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi ResolveQuantizedMetric(params->metric, is_normalized))) { throw std::invalid_argument("Unsupported quantization params for HNSW index"); } - // Template arguments do not affect these component sizes. + // Template arguments do not affect these component sizes. The storage type does not + // either; the asserts keep that true, so one estimate serves SQ8 and SQ4. + static_assert(sizeof(QuantPreprocessor) == + sizeof(QuantPreprocessor)); + static_assert(sizeof(QuantPreprocessor) == + sizeof(QuantPreprocessor)); if (with_mean) { est += allocations_overhead + sizeof(DistanceCalculatorWithNorm); @@ -245,7 +264,12 @@ size_t EstimateElementSize(const HNSWParams *params, bool with_mean) { size_t stored_data_size; // Preserve the element estimator's existing contract: return a size and validate in NewIndex. - if (params->quantType == VecSimQuant_SQ8) { + if (params->quantType == VecSimQuant_SQ4) { + stored_data_size = + params->metric == VecSimMetric_L2 + ? GetQuantizedStoredDataSize(params->dim, with_mean) + : GetQuantizedStoredDataSize(params->dim, with_mean); + } else if (params->quantType == VecSimQuant_SQ8) { if (params->metric == VecSimMetric_L2) { stored_data_size = GetQuantizedStoredDataSize(params->dim, with_mean); } else { @@ -336,7 +360,8 @@ static bool InitializeParams(std::ifstream &source_params, HNSWParams ¶ms, // VecSimQuantType, so it must never be loaded into the enum. std::underlying_type_t raw_quant_type; Serializer::readBinaryPOD(source_params, raw_quant_type); - if (raw_quant_type != VecSimQuant_NONE && raw_quant_type != VecSimQuant_SQ8) { + if (raw_quant_type != VecSimQuant_NONE && raw_quant_type != VecSimQuant_SQ8 && + raw_quant_type != VecSimQuant_SQ4) { return false; } params.quantType = static_cast(raw_quant_type); @@ -392,6 +417,15 @@ VecSimIndex *NewIndex(const std::string &location, bool is_normalized) { return NULL; } + if (params.quantType == VecSimQuant_SQ4) { + if (metric == VecSimMetric_L2) { + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); + } + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); + } + if (params.type == VecSimType_FLOAT32) { if (metric == VecSimMetric_L2) { return LoadIndex_Quantized( diff --git a/src/VecSim/index_factories/tiered_factory.cpp b/src/VecSim/index_factories/tiered_factory.cpp index c638a8495..56d718d5f 100644 --- a/src/VecSim/index_factories/tiered_factory.cpp +++ b/src/VecSim/index_factories/tiered_factory.cpp @@ -104,6 +104,9 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) { inline size_t EstimateInitialSize(const TieredIndexParams *params) { const auto &hnsw_params = params->primaryIndexParams->algoParams.hnswParams; + if (hnsw_params.quantType == VecSimQuant_SQ4) { + throw std::invalid_argument("Tiered HNSW SQ4 is not supported"); + } const bool requires_training = RequiresQuantizationTraining(params); const bool with_mean = requires_training || hnsw_params.quantParams != nullptr; size_t est = HNSWFactory::EstimateInitialSize(&hnsw_params, true, with_mean); diff --git a/src/VecSim/spaces/IP/IP.cpp b/src/VecSim/spaces/IP/IP.cpp index 136617c62..c95f878bb 100644 --- a/src/VecSim/spaces/IP/IP.cpp +++ b/src/VecSim/spaces/IP/IP.cpp @@ -11,11 +11,21 @@ #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq4.h" #include "VecSim/utils/alignment.h" using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; using sq8 = vecsim_types::sq8; +using sq4 = vecsim_types::sq4; + +float SQ4_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) { + return 1.0f - spaces::quantized::FP32_InnerProduct_Impl(pVect1v, pVect2v, dimension); +} + +float SQ4_SQ4_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) { + return 1.0f - spaces::quantized::Stored_InnerProduct_Impl(pVect1v, pVect2v, dimension); +} /* * Optimized asymmetric SQ8-FP32 inner product using algebraic identity: diff --git a/src/VecSim/spaces/IP/IP.h b/src/VecSim/spaces/IP/IP.h index 3566e8d73..0f5deb683 100644 --- a/src/VecSim/spaces/IP/IP.h +++ b/src/VecSim/spaces/IP/IP.h @@ -10,6 +10,10 @@ #include +// Packed SQ4 storage, compared with FP32 queries or another SQ4 vector. +float SQ4_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension); +float SQ4_SQ4_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension); + // SQ8-FP32: Common inner product implementation that returns the raw inner product value // (not distance). Used by SQ8_FP32_InnerProduct, SQ8_FP32_Cosine, and SQ8_FP32_L2Sqr. // pVect1 is storage (SQ8): [uint8_t values (dim)] [min_val] [delta] [x_sum] [x_sum_squares (L2 diff --git a/src/VecSim/spaces/IP_space.cpp b/src/VecSim/spaces/IP_space.cpp index 7ae4da228..224e28bb6 100644 --- a/src/VecSim/spaces/IP_space.cpp +++ b/src/VecSim/spaces/IP_space.cpp @@ -39,6 +39,24 @@ using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; namespace spaces { +// SQ4-FP32: packed storage and unaligned metadata, so no alignment requirement. +dist_func_t IP_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment, + const void *arch_opt) { + if (alignment) { + *alignment = 0; + } + return SQ4_FP32_InnerProduct; +} + +// SQ4-SQ4: packed storage and unaligned metadata, so no alignment requirement. +dist_func_t IP_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment, + const void *arch_opt) { + if (alignment) { + *alignment = 0; + } + return SQ4_SQ4_InnerProduct; +} + // SQ8-FP32: asymmetric distance between SQ8 storage and FP32 query dist_func_t IP_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment, const void *arch_opt) { diff --git a/src/VecSim/spaces/IP_space.h b/src/VecSim/spaces/IP_space.h index 85022e5c9..35530cea5 100644 --- a/src/VecSim/spaces/IP_space.h +++ b/src/VecSim/spaces/IP_space.h @@ -10,6 +10,12 @@ #include "VecSim/spaces/spaces.h" namespace spaces { +// Packed SQ4 storage has no alignment requirement, including its metadata. +dist_func_t IP_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, + const void *arch_opt = nullptr); +dist_func_t IP_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, + const void *arch_opt = nullptr); + // SQ8-FP32: asymmetric distance between FP32 query and SQ8 storage dist_func_t IP_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, const void *arch_opt = nullptr); diff --git a/src/VecSim/spaces/L2/L2.cpp b/src/VecSim/spaces/L2/L2.cpp index 2bc7f2cfb..8fc0173d5 100644 --- a/src/VecSim/spaces/L2/L2.cpp +++ b/src/VecSim/spaces/L2/L2.cpp @@ -12,12 +12,48 @@ #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq4.h" #include "VecSim/utils/alignment.h" #include using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; using sq8 = vecsim_types::sq8; +using sq4 = vecsim_types::sq4; + +float SQ4_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { + const auto *codes = static_cast(pVect1v); + const auto *query = static_cast(pVect2v); + const auto *params = codes + sq4::data_bytes_count(dimension); + const float min_val = load_unaligned(params + sq4::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params + sq4::DELTA * sizeof(float)); + + float result = 0.0f; + const size_t full_bytes = dimension / 2; + for (size_t j = 0; j < full_bytes; ++j) { + // Load once and use both nibbles. Keep min - query before adding delta * code so + // rounding stays at the residual's scale, and accumulate in coordinate order. + const uint8_t byte = codes[j]; + const float even = delta * float(byte & sq4::max_code) + (min_val - query[2 * j]); + const float odd = delta * float(byte >> sq4::bits_per_code) + (min_val - query[2 * j + 1]); + result += even * even; + result += odd * odd; + } + if (dimension % 2) { + // Only the low nibble is a coordinate; never read a query value for the padding. + const float last = + delta * float(codes[full_bytes] & sq4::max_code) + (min_val - query[dimension - 1]); + result += last * last; + } + return result; +} + +// SQ4-SQ4 L2 squared distance with the same identity as SQ8-SQ8: +// ||x - y||^2 = ||x||^2 + ||y||^2 - 2*IP(x, y) +// from the stored SUM_SQUARES metadata and the stored-to-stored inner product. +float SQ4_SQ4_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { + return spaces::quantized::Stored_L2Sqr(pVect1v, pVect2v, dimension); +} /* * Asymmetric SQ8-FP32 L2 squared distance, accumulated directly per component: diff --git a/src/VecSim/spaces/L2/L2.h b/src/VecSim/spaces/L2/L2.h index 9afbaeb43..bb75ccf72 100644 --- a/src/VecSim/spaces/L2/L2.h +++ b/src/VecSim/spaces/L2/L2.h @@ -10,6 +10,10 @@ #include +// Packed SQ4 storage, compared with FP32 queries or another SQ4 vector. +float SQ4_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension); +float SQ4_SQ4_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension); + // SQ8-FP32: pVect1v vector of type uint8 (SQ8) and pVect2v vector of type fp32 float SQ8_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension); diff --git a/src/VecSim/spaces/L2_space.cpp b/src/VecSim/spaces/L2_space.cpp index f1ec7f696..63e2c2d60 100644 --- a/src/VecSim/spaces/L2_space.cpp +++ b/src/VecSim/spaces/L2_space.cpp @@ -39,6 +39,24 @@ using float16 = vecsim_types::float16; namespace spaces { +// SQ4-FP32: packed storage and unaligned metadata, so no alignment requirement. +dist_func_t L2_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment, + const void *arch_opt) { + if (alignment) { + *alignment = 0; + } + return SQ4_FP32_L2Sqr; +} + +// SQ4-SQ4: packed storage and unaligned metadata, so no alignment requirement. +dist_func_t L2_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment, + const void *arch_opt) { + if (alignment) { + *alignment = 0; + } + return SQ4_SQ4_L2Sqr; +} + // SQ8-FP32: asymmetric L2 distance between SQ8 storage and FP32 query dist_func_t L2_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment, const void *arch_opt) { diff --git a/src/VecSim/spaces/L2_space.h b/src/VecSim/spaces/L2_space.h index 816644f80..eddc1b7f7 100644 --- a/src/VecSim/spaces/L2_space.h +++ b/src/VecSim/spaces/L2_space.h @@ -22,6 +22,12 @@ dist_func_t L2_INT8_GetDistFunc(size_t dim, unsigned char *alignment = nu const void *arch_opt = nullptr); dist_func_t L2_UINT8_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, const void *arch_opt = nullptr); +// SQ4-FP32: packed storage has no alignment requirement, including its metadata. +dist_func_t L2_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, + const void *arch_opt = nullptr); +// SQ4-SQ4: packed storage has no alignment requirement, including its metadata. +dist_func_t L2_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, + const void *arch_opt = nullptr); // SQ8-FP32: asymmetric L2 distance between FP32 query and SQ8 storage dist_func_t L2_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, const void *arch_opt = nullptr); diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 3f46e2d48..1dbcd2aef 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -25,6 +25,7 @@ #include "VecSim/memory/memory_utils.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq4.h" #include "VecSim/utils/vecsim_stl.h" class PreprocessorInterface : public VecsimBaseObject { @@ -143,8 +144,8 @@ class CosinePreprocessor : public PreprocessorInterface { /* * QuantPreprocessor is a preprocessor that quantizes storage vectors from DataType to a - * lower precision representation defined by StorageType (SQ8, the current storage type, - * stores one uint8_t code per coordinate). + * lower precision representation (SQ8 stores one uint8_t code per coordinate, SQ4 packs + * two 4-bit codes per byte). * Query vectors remain as DataType for asymmetric distance computation. * * The quantized storage blob contains the quantized values along with metadata (min value, @@ -153,7 +154,7 @@ class CosinePreprocessor : public PreprocessorInterface { * and then scaling the values to fit in the storage type's code range. * * Storage layout: - * | quantized_values[dim] | min_val | delta | x_sum | (x_sum_squares for L2 only) | + * | quantized_values (packed for SQ4) | min_val | delta | x_sum | (L2 sum_squares) | * where, writing x_r[i] = min_val + delta * a[i] for the value a blob actually represents: * x_sum = Σx_r[i]: sum of the reconstructed values, * x_sum_squares = Σx_r[i]²: sum of their squares. @@ -178,11 +179,13 @@ class CosinePreprocessor : public PreprocessorInterface { * store metric-specific FP32 values from the query body to accelerate asymmetric distance * computation: * - For IP/Cosine: y_sum = Σy_i (sum of query values) - * - For L2: y_sum = Σy_i (sum of query values), y_sum_squares = Σy_i² (sum of squared query values) + * - For SQ8 L2: y_sum = Σy_i, y_sum_squares = Σy_i² + * - For SQ4 L2: two zero-initialized reserved FP32 slots; direct residual kernels need no sums. * * Query blob layout: * - For IP/Cosine: | query_values[dim] | y_sum | * - For L2: | query_values[dim] | y_sum | y_sum_squares | + * SQ4 L2 keeps the same layout, with both trailing slots reserved and zero-initialized. * * Query metadata is always FP32. The query blob size is: * - For IP/Cosine: dim * sizeof(DataType) + 1 * sizeof(float) @@ -201,6 +204,7 @@ class CosinePreprocessor : public PreprocessorInterface { * where y_sum = Σy_i is precomputed and stored in the query blob. * * For L2: + * The identity below explains the metadata; direct residual query kernels do not consume it. * ||x_r - y||² = Σx_r[i]² - 2*Σ(x_r[i] * y_i) + Σy_i² * = x_sum_squares - 2 * IP(x_r, y) + y_sum_squares * where: @@ -389,7 +393,7 @@ class QuantPreprocessor : public PreprocessorInterface { } // Metadata uses MetadataType. Use memcpy because the metadata offset (after the - // quantized body) is not guaranteed to be sizeof(MetadataType)-aligned. + // quantized body, packed for SQ4) is not guaranteed to be sizeof(MetadataType)-aligned. void *meta_dst = quantized + StorageType::data_bytes_count(this->dim); MetadataType buf[5] = {min_val, delta, sum}; size_t n = 3; @@ -512,9 +516,9 @@ class QuantPreprocessor : public PreprocessorInterface { /** * Preprocesses the original blob into separate storage and query blobs. * - * Storage vectors are quantized to StorageType scalar codes (SQ8: one uint8_t per - * coordinate), with metadata (min, delta, sum, and sum_squares for L2) appended for - * distance reconstruction. + * Storage vectors are quantized to scalar codes (SQ8: one uint8_t per coordinate, SQ4: two + * 4-bit codes packed per byte), with metadata (min, delta, sum, and + * sum_squares for L2) appended for distance reconstruction. * * Query vectors use QueryType for asymmetric distance computation, with a precomputed * sum (for IP/Cosine) or sum of squares (for L2) appended for efficient distance calculation. @@ -575,12 +579,13 @@ class QuantPreprocessor : public PreprocessorInterface { * the original query values. * * - For IP/Cosine: y_sum = Σy_i (sum of query values) - * - For L2: y_sum = Σy_i (sum of query values), y_sum_squares = Σy_i² (sum of squared query - * values) + * - For SQ8 L2: y_sum = Σy_i, y_sum_squares = Σy_i² + * - For SQ4 L2: two zero-initialized reserved FP32 slots; no query reductions are needed. * * Query blob layout: * - For IP/Cosine: | query_values[dim] | y_sum | * - For L2: | query_values[dim] | y_sum | y_sum_squares | + * SQ4 L2 reserves and zero-initializes both trailing slots instead of computing sums. * * Query blob size: * - For IP/Cosine: dim * sizeof(QueryType) + 1 * sizeof(float) @@ -606,7 +611,14 @@ class QuantPreprocessor : public PreprocessorInterface { // Compute and write FP32 query metadata after the query body. The metadata offset is // body_bytes, which is not guaranteed to be 4-byte aligned for FP16 query bodies. void *metadata_dst = static_cast(blob) + body_bytes; - assign_query_metadata(query_values, input, metadata_dst); + if constexpr (std::is_same_v && Metric == VecSimMetric_L2) { + // SQ4 L2 computes residuals directly, so query sums are unused. Keep initialized + // padding for SIMD tails that load a final query pair when dim is odd. + const MetadataType padding[2] = {}; + memcpy(metadata_dst, padding, sizeof(padding)); + } else { + assign_query_metadata(query_values, input, metadata_dst); + } query_blob_size = this->query_bytes_count; } @@ -638,8 +650,8 @@ class QuantPreprocessor : public PreprocessorInterface { float value = first_input_value - mean[0]; float min_val = value; float max_val = value; - // x_mean_ip only feeds the IP correction term (see DistanceCalculatorWithNorm); the - // normalized L2 correction cancels the mean terms exactly, so skip the extra work. + // Use the original input for the IP correction: a correction from reconstructed + // values would also multiply quantization error by the mean. L2 needs no correction. if constexpr (Metric == VecSimMetric_IP) x_mean_ip = first_input_value * mean[0]; diff --git a/src/VecSim/spaces/quantized_kernels.h b/src/VecSim/spaces/quantized_kernels.h index c84dd0e03..26246f40c 100644 --- a/src/VecSim/spaces/quantized_kernels.h +++ b/src/VecSim/spaces/quantized_kernels.h @@ -22,6 +22,7 @@ // // Stored layout: [codes (data_bytes_count(dim) bytes)] [min_val] [delta] [sum] [sum_squares (L2)] // FP32 query layout: [float values (dim)] [y_sum] [y_sum_squares (L2 only)] +// SQ4 L2 uses only the query body; its two trailing FP32 slots are reserved and zero-initialized. // // The operand order in these bodies is load-bearing and relies on FP addition not being // reassociated; they must not be built with -ffast-math / -Ofast. diff --git a/src/VecSim/spaces/spaces.cpp b/src/VecSim/spaces/spaces.cpp index 7df458cd2..1e62c0b5c 100644 --- a/src/VecSim/spaces/spaces.cpp +++ b/src/VecSim/spaces/spaces.cpp @@ -9,6 +9,9 @@ #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq4.h" +#include "VecSim/spaces/IP/IP.h" +#include "VecSim/spaces/L2/L2.h" #include "VecSim/spaces/space_includes.h" #include "VecSim/spaces/spaces.h" #include "VecSim/spaces/IP_space.h" @@ -144,6 +147,38 @@ GetDistFunc(VecSimMetric metric throw std::invalid_argument("Invalid metric"); } +template <> +dist_func_t GetDistFunc(VecSimMetric metric, size_t dim, + unsigned char *alignment) { + if (alignment) { + *alignment = 0; + } + switch (metric) { + case VecSimMetric_IP: + return IP_SQ4_SQ4_GetDistFunc(dim, alignment); + case VecSimMetric_L2: + return L2_SQ4_SQ4_GetDistFunc(dim, alignment); + default: + throw std::invalid_argument("SQ4 supports only IP and L2 metrics"); + } +} + +template <> +dist_func_t GetDistFunc(VecSimMetric metric, size_t dim, + unsigned char *alignment) { + if (alignment) { + *alignment = 0; + } + switch (metric) { + case VecSimMetric_IP: + return IP_SQ4_FP32_GetDistFunc(dim, alignment); + case VecSimMetric_L2: + return L2_SQ4_FP32_GetDistFunc(dim, alignment); + default: + throw std::invalid_argument("SQ4 supports only IP and L2 metrics"); + } +} + template <> normalizeVector_f GetNormalizeFunc(void) { return normalizeVector_imp; diff --git a/src/VecSim/types/sq4.h b/src/VecSim/types/sq4.h new file mode 100644 index 000000000..6427b19ca --- /dev/null +++ b/src/VecSim/types/sq4.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include "VecSim/types/quantized_metadata.h" + +namespace vecsim_types { + +// Two scalar-quantized coordinates per byte, followed by the same metadata as SQ8. +struct sq4 : quantized_metadata { + using value_type = uint8_t; + static constexpr uint8_t max_code = 15; + static constexpr unsigned bits_per_code = 4; + + static constexpr size_t data_bytes_count(size_t dim) { return dim / 2 + dim % 2; } + + static uint8_t get_code(const uint8_t *data, size_t index) { + return (data[index / 2] >> ((index % 2) * bits_per_code)) & max_code; + } + + // Coordinates are stored in ascending order. Starting each byte with an assignment allows + // in-place preprocessing and leaves the unused high nibble zero for odd dimensions. + static void set_code(uint8_t *data, size_t index, uint8_t code) { + if (index % 2 == 0) { + data[index / 2] = code & max_code; + } else { + data[index / 2] = (data[index / 2] & max_code) | ((code & max_code) << bits_per_code); + } + } + + // Dot product of two packed code arrays: sum(q1_i * q2_i). Each byte is loaded once and both + // nibbles are used. The 64-bit integer sum is exact for any dimension. + // Used by scalar SQ4-to-SQ4 IP and L2 kernels; SIMD kernels compute the dot product separately. + static float code_dot(const uint8_t *a, const uint8_t *b, size_t dim) { + const size_t full_bytes = dim / 2; + uint64_t sum = 0; + for (size_t j = 0; j < full_bytes; j++) { + const uint64_t a_byte = a[j], b_byte = b[j]; + sum += (a_byte & max_code) * (b_byte & max_code) + + (a_byte >> bits_per_code) * (b_byte >> bits_per_code); + } + if (dim % 2) { + // Odd dim: the last byte holds only a low nibble. + sum += uint64_t(a[full_bytes] & max_code) * uint64_t(b[full_bytes] & max_code); + } + return static_cast(sum); + } + + // Dot product of a packed code array with FP32 values: sum(q_i * y_i). Each byte is loaded + // once; the low nibble pairs with y[2j] and the high nibble with y[2j + 1], in two independent + // accumulators for instruction-level parallelism. + static float code_dot_fp32(const uint8_t *codes, const float *y, size_t dim) { + const size_t full_bytes = dim / 2; + float sum_low = 0, sum_high = 0; + for (size_t j = 0; j < full_bytes; j++) { + const uint8_t byte = codes[j]; + sum_low += static_cast(byte & max_code) * y[2 * j]; + sum_high += static_cast(byte >> bits_per_code) * y[2 * j + 1]; + } + if (dim % 2) { + // Odd dim: the last byte holds only a low nibble. + sum_low += static_cast(codes[full_bytes] & max_code) * y[dim - 1]; + } + return sum_low + sum_high; + } + + template + static constexpr size_t storage_bytes_count(size_t dim) { + return data_bytes_count(dim) + storage_metadata_count() * sizeof(float); + } +}; + +} // namespace vecsim_types diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 419aa9884..1d7cf7aa3 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -73,6 +73,8 @@ typedef enum { VecSimQuant_NONE = 0, // No quantization (default). // 8-bit scalar quantization with optional mean centering. VecSimQuant_SQ8 = 1, + // Packed 4-bit scalar quantization with optional mean centering (FP32 input). + VecSimQuant_SQ4 = 2, } VecSimQuantType; // Algorithm type/library. @@ -190,7 +192,7 @@ typedef struct { size_t efRuntime; double epsilon; VecSimQuantType quantType; // Defaults to VecSimQuant_NONE. - // SQ8 mean vector (float[dim]); NULL disables mean centering. Copied during construction. + // SQ8/SQ4 mean vector (float[dim]); NULL disables mean centering. Copied during construction. // L2 centering keeps queries in FLOAT32, including when the input type is FLOAT16. const void *quantParams; } HNSWParams; diff --git a/src/python_bindings/bindings.cpp b/src/python_bindings/bindings.cpp index ab46ffb14..edd5fbd12 100644 --- a/src/python_bindings/bindings.cpp +++ b/src/python_bindings/bindings.cpp @@ -41,8 +41,9 @@ std::optional validateQuantizationMean(const py::object &quant if (quantization_mean.is_none()) { return std::nullopt; } - if (params.quantType != VecSimQuant_SQ8) { - throw py::value_error("quantization_mean requires quantType VecSimQuant_SQ8"); + if (params.quantType != VecSimQuant_SQ8 && params.quantType != VecSimQuant_SQ4) { + throw py::value_error( + "quantization_mean requires quantType VecSimQuant_SQ8 or VecSimQuant_SQ4"); } QuantizationMean mean(quantization_mean); @@ -812,6 +813,7 @@ PYBIND11_MODULE(VecSim, m) { py::enum_(m, "VecSimQuantType") .value("VecSimQuant_NONE", VecSimQuant_NONE) .value("VecSimQuant_SQ8", VecSimQuant_SQ8) + .value("VecSimQuant_SQ4", VecSimQuant_SQ4) .export_values(); py::enum_(m, "VecSimOptionMode") diff --git a/tests/flow/test_hnsw_sq4.py b/tests/flow/test_hnsw_sq4.py new file mode 100644 index 000000000..a5b0212c3 --- /dev/null +++ b/tests/flow/test_hnsw_sq4.py @@ -0,0 +1,66 @@ +# Copyright (c) 2006-Present, Redis Ltd. +# All rights reserved. +# +# Licensed under your choice of the Redis Source Available License 2.0 +# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the +# GNU Affero General Public License v3 (AGPLv3). +import pytest +from common import * + +DIM = 33 +NUM_VECTORS = 500 +NUM_QUERIES = 20 +K = 10 +MIN_RECALL = 0.7 + + +def make_sq4_params(metric, data_type=VecSimType_FLOAT32): + params = create_hnsw_params(dim=DIM, num_elements=NUM_VECTORS, metric=metric, + data_type=data_type, ef_construction=200, m=16, ef_runtime=200) + params.quantType = VecSimQuant_SQ4 + return params + + +def brute_force_labels(data, queries, metric): + if metric == VecSimMetric_L2: + dists = ((queries[:, None, :] - data[None, :, :]) ** 2).sum(axis=2) + else: + dists = 1.0 - queries @ data.T + return np.argsort(dists, axis=1)[:, :K] + + +@pytest.mark.parametrize("metric", [ + pytest.param(VecSimMetric_IP, id="ip"), + pytest.param(VecSimMetric_L2, id="l2"), +]) +@pytest.mark.parametrize("centered", [False, True], ids=["plain", "centered"]) +def test_sq4_hnsw_plain_and_centered(metric, centered): + rng = np.random.default_rng(42) + data = rng.random((NUM_VECTORS, DIM), dtype=np.float32) + queries = rng.random((NUM_QUERIES, DIM), dtype=np.float32) + if metric == VecSimMetric_IP: + data /= np.linalg.norm(data, axis=1, keepdims=True) + queries /= np.linalg.norm(queries, axis=1, keepdims=True) + + params = make_sq4_params(metric) + mean = data.mean(axis=0).astype(np.float32) if centered else None + index = HNSWIndex(params, quantization_mean=mean) + for label, vec in enumerate(data): + index.add_vector(vec, label) + assert index.index_size() == NUM_VECTORS + + expected = brute_force_labels(data, queries, metric) + hits = 0 + for i, query in enumerate(queries): + labels, distances = index.knn_query(query, K) + assert np.isfinite(distances).all(), distances + hits += len(set(labels[0]) & set(expected[i])) + recall = hits / (NUM_QUERIES * K) + assert recall >= MIN_RECALL, recall + + +def test_sq4_rejects_fp16_and_cosine(): + with pytest.raises(ValueError, match="Unsupported vector index parameters"): + HNSWIndex(make_sq4_params(VecSimMetric_L2, VecSimType_FLOAT16)) + with pytest.raises(ValueError, match="Unsupported vector index parameters"): + HNSWIndex(make_sq4_params(VecSimMetric_Cosine)) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 67a8a00f3..fd8e18721 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -35,6 +35,7 @@ else() endif() add_executable(test_hnsw ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw.cpp test_hnsw_multi.cpp test_hnsw_tiered.cpp unit_test_utils.cpp) +add_executable(test_hnsw_sq4 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_sq4.cpp test_sq4_distances.cpp test_sq4_components.cpp unit_test_utils.cpp) add_executable(test_hnsw_sq8 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_sq8.cpp unit_test_utils.cpp) add_executable(test_bruteforce ../utils/test_main_with_timeout.cpp test_bruteforce.cpp test_bruteforce_multi.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) add_executable(test_allocator ../utils/test_main_with_timeout.cpp test_allocator.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) @@ -51,6 +52,7 @@ add_executable(test_svs ../utils/test_main_with_timeout.cpp ../utils/mock_thread target_link_libraries(test_hnsw PUBLIC gtest VectorSimilarity) target_link_libraries(test_hnsw_sq8 PUBLIC gtest VectorSimilarity) +target_link_libraries(test_hnsw_sq4 PUBLIC gtest VectorSimilarity) target_link_libraries(test_bruteforce PUBLIC gtest VectorSimilarity) target_link_libraries(test_allocator PUBLIC gtest VectorSimilarity) target_link_libraries(test_spaces PUBLIC gtest VectorSimilarity) @@ -68,6 +70,7 @@ include(GoogleTest) gtest_discover_tests(test_hnsw) gtest_discover_tests(test_hnsw_sq8) +gtest_discover_tests(test_hnsw_sq4) gtest_discover_tests(test_bruteforce) gtest_discover_tests(test_allocator) gtest_discover_tests(test_spaces) diff --git a/tests/unit/test_hnsw_sq4.cpp b/tests/unit/test_hnsw_sq4.cpp new file mode 100644 index 000000000..62f17ca4f --- /dev/null +++ b/tests/unit/test_hnsw_sq4.cpp @@ -0,0 +1,570 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#include "gtest/gtest.h" +#include "VecSim/algorithms/hnsw/hnsw_single.h" +#include "VecSim/index_factories/hnsw_factory.h" +#include "VecSim/index_factories/tiered_factory.h" +#include "VecSim/types/sq4.h" +#include "VecSim/vec_sim.h" +#include "unit_test_utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using IndexPtr = std::unique_ptr; +using SQ4Case = std::tuple; +using Results = std::vector>; +class HNSWSQ4Test : public ::testing::TestWithParam {}; + +std::vector reconstruct(const char *blob, size_t dim, const std::vector &mean) { + float metadata[2]; + std::memcpy(metadata, blob + (dim + 1) / 2, sizeof(metadata)); + std::vector result(dim); + for (size_t d = 0; d < dim; ++d) { + auto code = (static_cast(blob[d / 2]) >> (4 * (d % 2))) & 15; + result[d] = metadata[0] + metadata[1] * code + (mean.empty() ? 0 : mean[d]); + } + return result; +} + +double distance(const std::vector &a, const std::vector &b, VecSimMetric metric) { + double sum = 0; + for (size_t d = 0; d < a.size(); ++d) { + double x = a[d], y = b[d]; + sum += metric == VecSimMetric_L2 ? (x - y) * (x - y) : x * y; + } + return metric == VecSimMetric_L2 ? sum : 1 - sum; +} + +std::vector make_mean(bool centered, size_t dim) { + std::vector mean; + if (centered) { + mean.resize(dim); + for (size_t d = 0; d < dim; ++d) + mean[d] = 0.3f * std::sin(float(d + 1)); + } + return mean; +} + +std::vector> random_vectors(size_t count, size_t dim, unsigned seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution random(-1.0f, 1.0f); + std::vector> vectors(count, std::vector(dim)); + for (auto &vector : vectors) + for (auto &value : vector) + value = random(rng); + return vectors; +} + +HNSWParams sq4_params(VecSimMetric metric, bool multi, size_t dim, const std::vector &mean) { + return {.type = VecSimType_FLOAT32, + .dim = dim, + .metric = metric, + .multi = multi, + .blockSize = 32, + .M = 16, + .efConstruction = 100, + .efRuntime = 128, + .quantType = VecSimQuant_SQ4, + .quantParams = mean.empty() ? nullptr : mean.data()}; +} + +// An ef far above the index size makes the HNSW search exhaustive over the connected graph, so +// indexes whose graphs differ (the level generator is not serialized) still agree on the top k. +Results top_k(VecSimIndex *index, const float *query, size_t k) { + VecSimQueryParams query_params{}; + query_params.hnswRuntimeParams.efRuntime = 1024; + Results results; + auto *reply = VecSimIndex_TopKQuery(index, query, k, &query_params, BY_SCORE); + EXPECT_EQ(VecSimQueryReply_GetCode(reply), VecSim_QueryReply_OK); + auto *iterator = VecSimQueryReply_GetIterator(reply); + while (VecSimQueryReply_IteratorHasNext(iterator)) { + const auto *item = VecSimQueryReply_IteratorNext(iterator); + results.emplace_back(VecSimQueryResult_GetId(item), VecSimQueryResult_GetScore(item)); + } + VecSimQueryReply_IteratorFree(iterator); + VecSimQueryReply_Free(reply); + return results; +} + +std::string temp_index_path(const std::string &prefix) { + std::string case_name = ::testing::UnitTest::GetInstance()->current_test_info()->name(); + std::replace(case_name.begin(), case_name.end(), '/', '_'); + return ::testing::TempDir() + prefix + case_name + ".hnsw"; +} + +TEST_P(HNSWSQ4Test, InsertQueryMutateAndSerialize) { + const auto [metric, centered, multi, dim] = GetParam(); + std::vector mean = make_mean(centered, dim); + HNSWParams hp = sq4_params(metric, multi, dim, mean); + auto params = CreateParams(hp); + IndexPtr index(VecSimIndex_New(¶ms), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + auto *hnsw = dynamic_cast *>(index.get()); + ASSERT_NE(hnsw, nullptr); + EXPECT_EQ(EstimateInitialSize(hp), index->getAllocationSize()); + EXPECT_TRUE(hnsw->usesQuantizedStorage()); + EXPECT_EQ(hnsw->getStoredDataSize(), + (dim + 1) / 2 + sizeof(float) * (metric == VecSimMetric_L2 ? 4 : (centered ? 4 : 3))); + + constexpr size_t count = 64; + std::mt19937 rng(42); + std::uniform_real_distribution random(-1.0f, 1.0f); + std::vector> inputs(count, std::vector(dim)); + for (size_t i = 0; i < count; ++i) { + for (auto &value : inputs[i]) + value = random(rng); + ASSERT_EQ(VecSimIndex_AddVector(index.get(), inputs[i].data(), i), 1); + } + ASSERT_TRUE(hnsw->checkIntegrity().valid_state); + std::vector query(dim); + for (auto &value : query) + value = random(rng); + std::vector> expected; + for (size_t i = 0; i < count; ++i) { + auto reconstructed = reconstruct(hnsw->getDataByInternalId(i), dim, mean); + double score = distance(reconstructed, query, metric); + expected.emplace_back(score, i); + EXPECT_NEAR(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), i, query.data()), score, + 2e-5 * (1.0 + std::abs(score))); + auto other = reconstruct(hnsw->getDataByInternalId((i + 1) % count), dim, mean); + double expected_pair = distance(reconstructed, other, metric); + if (centered && metric == VecSimMetric_IP) { + expected_pair = 1.0; + for (size_t j = 0; j < dim; ++j) { + expected_pair -= + (double(reconstructed[j]) - mean[j]) * (double(other[j]) - mean[j]) + + double(inputs[i][j]) * mean[j] + double(inputs[(i + 1) % count][j]) * mean[j] - + double(mean[j]) * mean[j]; + } + } + EXPECT_NEAR(hnsw->calcDistance(hnsw->getDataByInternalId(i), + hnsw->getDataByInternalId((i + 1) % count)), + expected_pair, 3e-5 * (1.0 + std::abs(expected_pair))); + } + std::sort(expected.begin(), expected.end()); + auto check_results = [&](VecSimIndex *target) { + runTopKSearchTest(target, query.data(), 10, [&](size_t id, double score, size_t pos) { + EXPECT_EQ(id, expected[pos].second); + EXPECT_NEAR(score, expected[pos].first, 2e-5 * (1.0 + std::abs(expected[pos].first))); + }); + }; + check_results(index.get()); + const double radius = std::max(0.0, (expected[19].first + expected[20].first) / 2); + const size_t range_count = + std::count_if(expected.begin(), expected.end(), + [radius](const auto &result) { return result.first <= radius; }); + // A bounded HNSW range search is approximate. Validate every returned score and membership. + auto *range = VecSimIndex_RangeQuery(index.get(), query.data(), radius, nullptr, BY_SCORE); + EXPECT_EQ(VecSimQueryReply_GetCode(range), VecSim_QueryReply_OK); + EXPECT_GT(VecSimQueryReply_Len(range), 0); + EXPECT_LE(VecSimQueryReply_Len(range), range_count); + std::unordered_set range_ids; + auto *iterator = VecSimQueryReply_GetIterator(range); + while (VecSimQueryReply_IteratorHasNext(iterator)) { + const auto *item = VecSimQueryReply_IteratorNext(iterator); + const size_t id = VecSimQueryResult_GetId(item); + EXPECT_TRUE(range_ids.insert(id).second); + const double score = VecSimQueryResult_GetScore(item); + const auto match = std::find_if(expected.begin(), expected.end(), + [id](const auto &result) { return result.second == id; }); + EXPECT_NE(match, expected.end()); + if (match != expected.end()) + EXPECT_NEAR(score, match->first, 2e-5 * (1.0 + std::abs(match->first))); + EXPECT_LE(score, radius); + } + VecSimQueryReply_IteratorFree(iterator); + VecSimQueryReply_Free(range); + // A radius containing the entire connected graph verifies full traversal and scoring. + runRangeQueryTest( + index.get(), query.data(), std::max(0.0, expected.back().first) + 1.0, + [&](size_t id, double score, size_t pos) { + EXPECT_EQ(id, expected[pos].second); + EXPECT_NEAR(score, expected[pos].first, 2e-5 * (1.0 + std::abs(expected[pos].first))); + }, + count, BY_SCORE); + auto *batch = VecSimBatchIterator_New(index.get(), query.data(), nullptr); + runBatchIteratorSearchTest(batch, 10, [&](size_t id, double score, size_t pos) { + EXPECT_EQ(id, expected[pos].second); + EXPECT_NEAR(score, expected[pos].first, 2e-5 * (1.0 + std::abs(expected[pos].first))); + }); + VecSimBatchIterator_Free(batch); + const auto path = temp_index_path("sq4_roundtrip_"); + hnsw->saveIndex(path); + IndexPtr loaded(HNSWFactory::NewIndex(path), VecSimIndex_Free); + std::remove(path.c_str()); + ASSERT_NE(loaded, nullptr); + auto *restored = dynamic_cast *>(loaded.get()); + ASSERT_NE(restored, nullptr); + EXPECT_EQ(restored->quantType, VecSimQuant_SQ4); + EXPECT_EQ(restored->serializedMeanVector, mean); + ASSERT_TRUE(restored->checkIntegrity().valid_state); + for (size_t i = 0; i < count; ++i) { + EXPECT_EQ(std::memcmp(hnsw->getDataByInternalId(i), restored->getDataByInternalId(i), + hnsw->getStoredDataSize()), + 0); + } + check_results(loaded.get()); + // Exercise duplicate-label insertion and deletion through the actual index API. + EXPECT_EQ(VecSimIndex_AddVector(loaded.get(), query.data(), 0), multi ? 1 : 0); + EXPECT_EQ(VecSimIndex_IndexSize(loaded.get()), count + (multi ? 1 : 0)); + EXPECT_EQ(VecSimIndex_DeleteVector(loaded.get(), 0), multi ? 2 : 1); + EXPECT_EQ(VecSimIndex_IndexSize(loaded.get()), count - 1); + EXPECT_TRUE(restored->checkIntegrity().valid_state); +} + +// A loaded index must keep working as a writable index: new vectors are quantized with the +// restored mean and the index answers like one that never went through serialization. +TEST_P(HNSWSQ4Test, SaveLoadThenMutate) { + const auto [metric, centered, multi, dim] = GetParam(); + constexpr size_t initial = 200, added = 50, deleted = 20, queries = 10, k = 10; + const std::vector mean = make_mean(centered, dim); + const HNSWParams hp = sq4_params(metric, multi, dim, mean); + const auto vectors = random_vectors(initial + added, dim, 7); + const auto query_vectors = random_vectors(queries, dim, 8); + + IndexPtr original(HNSWFactory::NewIndex(&hp), VecSimIndex_Free); + ASSERT_NE(original, nullptr); + auto *original_hnsw = dynamic_cast *>(original.get()); + ASSERT_NE(original_hnsw, nullptr); + for (size_t i = 0; i < initial; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(original.get(), vectors[i].data(), i), 1); + } + std::vector before; + for (const auto &query : query_vectors) { + before.push_back(top_k(original.get(), query.data(), k)); + ASSERT_EQ(before.back().size(), k); + } + + const auto path = temp_index_path("sq4_save_load_mutate_"); + original_hnsw->saveIndex(path); + IndexPtr loaded(HNSWFactory::NewIndex(path), VecSimIndex_Free); + std::remove(path.c_str()); + ASSERT_NE(loaded, nullptr); + auto *loaded_hnsw = dynamic_cast *>(loaded.get()); + ASSERT_NE(loaded_hnsw, nullptr); + EXPECT_EQ(loaded_hnsw->quantType, VecSimQuant_SQ4); + EXPECT_EQ(loaded_hnsw->serializedMeanVector, mean); + ASSERT_EQ(loaded_hnsw->getStoredDataSize(), original_hnsw->getStoredDataSize()); + for (size_t q = 0; q < queries; ++q) { + EXPECT_EQ(top_k(loaded.get(), query_vectors[q].data(), k), before[q]) << "query " << q; + } + + // Each label holds one vector, so before any deletion label i has internal id i. + for (size_t i = initial; i < initial + added; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(loaded.get(), vectors[i].data(), i), 1); + const auto expected_blob = original_hnsw->preprocessForStorage(vectors[i].data()); + EXPECT_EQ(std::memcmp(loaded_hnsw->getDataByInternalId(i), expected_blob.get(), + loaded_hnsw->getStoredDataSize()), + 0) + << "label " << i; + } + for (size_t i = 0; i < deleted; ++i) { + ASSERT_EQ(VecSimIndex_DeleteVector(loaded.get(), i), 1); + } + EXPECT_EQ(VecSimIndex_IndexSize(loaded.get()), initial + added - deleted); + EXPECT_TRUE(loaded_hnsw->checkIntegrity().valid_state); + + IndexPtr fresh(HNSWFactory::NewIndex(&hp), VecSimIndex_Free); + ASSERT_NE(fresh, nullptr); + for (size_t i = 0; i < initial + added; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(fresh.get(), vectors[i].data(), i), 1); + } + for (size_t i = 0; i < deleted; ++i) { + ASSERT_EQ(VecSimIndex_DeleteVector(fresh.get(), i), 1); + } + for (size_t q = 0; q < queries; ++q) { + const Results expected = top_k(fresh.get(), query_vectors[q].data(), k); + ASSERT_EQ(expected.size(), k); + EXPECT_EQ(top_k(loaded.get(), query_vectors[q].data(), k), expected) << "query " << q; + } +} + +// Multi-value only: gtest cannot mix TEST and TEST_P in one suite, so single-value cases skip. +TEST_P(HNSWSQ4Test, DuplicateLabelsMulti) { + const auto [metric, centered, multi, dim] = GetParam(); + if (!multi) { + GTEST_SKIP() << "multi-value only"; + } + const std::vector mean = make_mean(centered, dim); + const HNSWParams hp = sq4_params(metric, multi, dim, mean); + IndexPtr index(HNSWFactory::NewIndex(&hp), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + + constexpr size_t label = 5, other_label = 9, copies = 3; + const auto vectors = random_vectors(copies + 1, dim, 11); + for (size_t i = 0; i < copies; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(index.get(), vectors[i].data(), label), 1); + } + ASSERT_EQ(VecSimIndex_AddVector(index.get(), vectors[copies].data(), other_label), 1); + EXPECT_EQ(VecSimIndex_IndexSize(index.get()), copies + 1); + EXPECT_EQ(index->indexLabelCount(), 2u); + + // KNN reports the duplicated label once, scored by its closest vector (which is also what + // the multi-value getDistanceFrom reports). + const Results results = top_k(index.get(), vectors[0].data(), 10); + ASSERT_EQ(results.size(), 2u); + EXPECT_NE(results[0].first, results[1].first); + const auto *hnsw = dynamic_cast *>(index.get()); + ASSERT_NE(hnsw, nullptr); + const auto query_blob = hnsw->preprocessQuery(vectors[0].data()); + double best = std::numeric_limits::max(); + for (size_t id = 0; id < copies; ++id) { + best = std::min( + best, hnsw->calcDistanceForQuery(hnsw->getDataByInternalId(id), query_blob.get())); + } + EXPECT_DOUBLE_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), label, vectors[0].data()), + best); + size_t label_hits = 0; + for (const auto &[id, score] : results) { + if (id == label) { + ++label_hits; + EXPECT_DOUBLE_EQ(score, best); + } + } + EXPECT_EQ(label_hits, 1u); + + EXPECT_EQ(VecSimIndex_DeleteVector(index.get(), label), static_cast(copies)); + EXPECT_EQ(VecSimIndex_IndexSize(index.get()), 1u); + EXPECT_EQ(index->indexLabelCount(), 1u); + const Results remaining = top_k(index.get(), vectors[0].data(), 10); + ASSERT_EQ(remaining.size(), 1u); + EXPECT_EQ(remaining[0].first, other_label); +} + +INSTANTIATE_TEST_SUITE_P(FP32, HNSWSQ4Test, + ::testing::Combine(::testing::Values(VecSimMetric_IP, VecSimMetric_L2), + ::testing::Bool(), ::testing::Bool(), + ::testing::Values(1u, 17u, 64u))); + +TEST(HNSWSQ4Factory, PreNormalizedCosineUsesIP) { + HNSWParams hp = {.type = VecSimType_FLOAT32, + .dim = 3, + .metric = VecSimMetric_Cosine, + .quantType = VecSimQuant_SQ4}; + VecSimParams params = CreateParams(hp); + IndexPtr index(HNSWFactory::NewIndex(¶ms, true), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + const float x[] = {1, 0, 0}; + const float y[] = {0, 1, 0}; + ASSERT_EQ(VecSimIndex_AddVector(index.get(), x, 0), 1); + EXPECT_FLOAT_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), 0, x), 0); + EXPECT_FLOAT_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), 0, y), 1); +} + +TEST(HNSWSQ4Factory, LoadRejectsUnsupportedSQ4Snapshot) { + HNSWParams hp = {.type = VecSimType_FLOAT32, + .dim = 4, + .metric = VecSimMetric_L2, + .M = 8, + .efConstruction = 40, + .quantType = VecSimQuant_SQ4}; + IndexPtr index(HNSWFactory::NewIndex(&hp), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + const float a[] = {0.25f, 0.3f, 0.35f, 0.4f}; + const float b[] = {0.75f, 0.8f, 0.85f, 0.9f}; + ASSERT_EQ(VecSimIndex_AddVector(index.get(), a, 0), 1); + ASSERT_EQ(VecSimIndex_AddVector(index.get(), b, 1), 1); + auto *hnsw = dynamic_cast *>(index.get()); + ASSERT_NE(hnsw, nullptr); + + const auto path = temp_index_path("sq4_reject_"); + // Field offsets follow HNSWSerializer::saveIndex and HNSWIndex::saveIndexFields. + constexpr std::streamoff type_offset = + sizeof(HNSWSerializer::EncodingVersion) + sizeof(VecSimAlgo) + sizeof(size_t); + constexpr std::streamoff metric_offset = type_offset + sizeof(VecSimType); + auto save_and_patch = [&](std::streamoff offset, auto value) { + hnsw->saveIndex(path); + std::fstream stream(path, std::ios::in | std::ios::out | std::ios::binary); + ASSERT_TRUE(stream.is_open()); + // Check the offset points at the expected field before overwriting it. + decltype(value) original{}; + stream.seekg(offset); + Serializer::readBinaryPOD(stream, original); + if constexpr (std::is_same_v) { + ASSERT_EQ(original, VecSimType_FLOAT32); + } else { + ASSERT_EQ(original, VecSimMetric_L2); + } + stream.seekp(offset); + Serializer::writeBinaryPOD(stream, value); + }; + auto expect_rejected = [&]() { + IndexPtr loaded(HNSWFactory::NewIndex(path), VecSimIndex_Free); + EXPECT_EQ(loaded, nullptr); + }; + + // The unmodified snapshot loads. + hnsw->saveIndex(path); + { + IndexPtr loaded(HNSWFactory::NewIndex(path), VecSimIndex_Free); + ASSERT_NE(loaded, nullptr); + } + // SQ4 accepts FP32 only, unlike SQ8. + save_and_patch(type_offset, VecSimType_FLOAT16); + expect_rejected(); + // A raw Cosine metric has no scalar quantization kernel. + save_and_patch(metric_offset, VecSimMetric_Cosine); + expect_rejected(); + std::remove(path.c_str()); +} + +// Mirrors HNSWSQ8Test.SizeEstimation, including its 1% tolerance on block growth. +TEST(HNSWSQ4Factory, EstimatesMatchAllocation) { + constexpr size_t dim = 768, block_size = 256; + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + for (bool centered : {false, true}) { + SCOPED_TRACE(std::string(VecSimMetric_ToString(metric)) + + (centered ? " centered" : " plain")); + const std::vector mean = make_mean(centered, dim); + HNSWParams hp = sq4_params(metric, false, dim, mean); + hp.blockSize = block_size; + hp.M = 64; + VecSimParams params = CreateParams(hp); + IndexPtr index(VecSimIndex_New(¶ms), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + EXPECT_EQ(VecSimIndex_EstimateInitialSize(¶ms), + VecSimIndex_StatsInfo(index.get()).memory); + + const auto vectors = random_vectors(block_size + 1, dim, 3); + for (size_t i = 0; i < block_size; ++i) { + ASSERT_EQ(VecSimIndex_AddVector(index.get(), vectors[i].data(), i), 1); + } + + // Measure the allocation caused by growing the index by one block. + const size_t estimation = VecSimIndex_EstimateElementSize(¶ms) * block_size; + const size_t before = VecSimIndex_StatsInfo(index.get()).memory; + ASSERT_EQ(VecSimIndex_AddVector(index.get(), vectors[block_size].data(), block_size), + 1); + const size_t actual = VecSimIndex_StatsInfo(index.get()).memory - before; + + EXPECT_GE(estimation, actual * 0.99); + EXPECT_LE(estimation, actual * 1.01); + } + } +} + +// Tiered SQ4 needs mean training, which is out of scope: creation and estimation both reject it, +// and the estimator's exception is translated to SIZE_MAX at the C API. +TEST(HNSWSQ4Factory, RejectsTieredSQ4) { + constexpr size_t dim = 32; + const std::vector mean = make_mean(true, dim); + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + for (bool centered : {false, true}) { + for (size_t training_set_size : {size_t{0}, size_t{10}}) { + SCOPED_TRACE(std::string(VecSimMetric_ToString(metric)) + + (centered ? " centered" : " plain") + " training " + + std::to_string(training_set_size)); + HNSWParams hp = {.type = VecSimType_FLOAT32, + .dim = dim, + .metric = metric, + .quantType = VecSimQuant_SQ4, + .quantParams = centered ? mean.data() : nullptr}; + VecSimParams primary = CreateParams(hp); + TieredIndexParams tiered = {.primaryIndexParams = &primary, + .specificParams = {TieredHNSWParams{ + .QuantNormalizationSetSize = training_set_size}}}; + VecSimParams params = CreateParams(tiered); + + VecSimIndex *direct = TieredFactory::NewIndex(&tiered); + EXPECT_EQ(direct, nullptr); + if (direct) { + VecSimIndex_Free(direct); + } + VecSimIndex *created = VecSimIndex_New(¶ms); + EXPECT_EQ(created, nullptr); + if (created) { + VecSimIndex_Free(created); + } + EXPECT_THROW(TieredFactory::EstimateInitialSize(&tiered), std::invalid_argument); + size_t estimate = 0; + EXPECT_NO_THROW(estimate = VecSimIndex_EstimateInitialSize(¶ms)); + EXPECT_EQ(estimate, SIZE_MAX); + } + } + } +} + +// Every rejection must happen in parameter validation, before components are built, so no +// exception reaches the C API and the factory returns NULL rather than throwing. +TEST(HNSWSQ4Factory, RejectsInvalidParams) { + auto expect_rejected = [](const HNSWParams &hp, bool is_normalized) { + VecSimIndex *direct = nullptr; + EXPECT_NO_THROW(direct = HNSWFactory::NewIndex(&hp, is_normalized)); + EXPECT_EQ(direct, nullptr); + if (direct) { + VecSimIndex_Free(direct); + } + EXPECT_THROW(HNSWFactory::EstimateInitialSize(&hp, is_normalized), std::invalid_argument); + if (is_normalized) { + // The C API never declares input normalized; that path is covered above. + return; + } + VecSimParams params = CreateParams(hp); + VecSimIndex *created = nullptr; + EXPECT_NO_THROW(created = VecSimIndex_New(¶ms)); + EXPECT_EQ(created, nullptr); + if (created) { + VecSimIndex_Free(created); + } + size_t estimate = 0; + EXPECT_NO_THROW(estimate = VecSimIndex_EstimateInitialSize(¶ms)); + EXPECT_EQ(estimate, SIZE_MAX); + }; + + constexpr size_t dim = 32; + const std::vector mean = make_mean(true, dim); + for (const float *quant_params : {static_cast(nullptr), mean.data()}) { + SCOPED_TRACE(quant_params ? "centered" : "plain"); + for (auto type : {VecSimType_FLOAT16, VecSimType_BFLOAT16, VecSimType_FLOAT64, + VecSimType_INT8, VecSimType_UINT8}) { + SCOPED_TRACE(VecSimType_ToString(type)); + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + expect_rejected({.type = type, + .dim = dim, + .metric = metric, + .quantType = VecSimQuant_SQ4, + .quantParams = quant_params}, + false); + } + } + + // Raw Cosine has no SQ4 kernel; only pre-normalized Cosine (resolved to IP) is accepted. + expect_rejected({.type = VecSimType_FLOAT32, + .dim = dim, + .metric = VecSimMetric_Cosine, + .quantType = VecSimQuant_SQ4, + .quantParams = quant_params}, + false); + + // Value 3 is not a VecSimQuantType enumerator. + for (bool is_normalized : {false, true}) { + expect_rejected({.type = VecSimType_FLOAT32, + .dim = dim, + .metric = VecSimMetric_IP, + .quantType = static_cast(3), + .quantParams = quant_params}, + is_normalized); + } + } +} +} // namespace diff --git a/tests/unit/test_hnsw_sq8.cpp b/tests/unit/test_hnsw_sq8.cpp index 10d177d1c..8708c1f1c 100644 --- a/tests/unit/test_hnsw_sq8.cpp +++ b/tests/unit/test_hnsw_sq8.cpp @@ -632,7 +632,7 @@ TYPED_TEST(HNSWSQ8Test, LoadRejectsUnsupportedQuantizedSnapshot) { } // Written as raw bytes: an out-of-range value is not a valid VecSimQuantType. - save_and_patch(quant_type_offset, static_cast(VecSimQuant_SQ8 + 1)); + save_and_patch(quant_type_offset, std::numeric_limits::max()); expect_rejected(); for (VecSimType unsupported_type : diff --git a/tests/unit/test_sq4_components.cpp b/tests/unit/test_sq4_components.cpp new file mode 100644 index 000000000..b7db8e16e --- /dev/null +++ b/tests/unit/test_sq4_components.cpp @@ -0,0 +1,365 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "VecSim/index_factories/components/components_factory.h" +#include "VecSim/spaces/L2_space.h" +#include "VecSim/types/sq4.h" + +namespace { + +using vecsim_types::sq4; + +float metadata(const void *blob, size_t body_bytes, size_t index) { + return load_unaligned(static_cast(blob) + body_bytes + + index * sizeof(float)); +} + +std::vector reconstruct(const void *blob, size_t dim, const std::vector &mean) { + const auto *codes = static_cast(blob); + const double min = metadata(blob, sq4::data_bytes_count(dim), sq4::MIN_VAL); + const double delta = metadata(blob, sq4::data_bytes_count(dim), sq4::DELTA); + std::vector result(dim); + for (size_t i = 0; i < dim; ++i) { + // Decode directly so the oracle also checks the production nibble order. + const unsigned code = (codes[i / 2] >> (4 * (i % 2))) & 15; + result[i] = min + delta * code + (mean.empty() ? 0.0 : mean[i]); + } + return result; +} + +template +double referenceDistance(const Left &left, const Right &right) { + double result = Metric == VecSimMetric_IP ? 1.0 : 0.0; + for (size_t i = 0; i < left.size(); ++i) { + if constexpr (Metric == VecSimMetric_IP) { + result -= double(left[i]) * double(right[i]); + } else { + const double difference = double(left[i]) - double(right[i]); + result += difference * difference; + } + } + return result; +} + +template +void checkComponents(size_t dim, bool constant) { + SCOPED_TRACE(::testing::Message() << "metric=" << Metric << " mean=" << WithMean + << " dim=" << dim << " constant=" << constant); + auto allocator = VecSimAllocator::newVecsimAllocator(); + std::vector x(dim), y(dim), mean(WithMean ? dim : 0); + for (size_t i = 0; i < dim; ++i) { + if constexpr (WithMean) { + mean[i] = 0.7f * (int(i % 5) - 2); + } + const float offset = WithMean ? mean[i] : 0.0f; + x[i] = offset + (constant ? 1.25f : 0.31f * (int((i * 7) % 19) - 9)); + y[i] = offset + (constant ? -0.75f : 0.23f * (int((i * 11) % 17) - 8)); + } + auto components = CreateQuantizedIndexComponents( + allocator, dim, WithMean ? mean.data() : nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + const size_t input_size = dim * sizeof(float); + const size_t storage_size = sq4::storage_bytes_count(dim); + auto *container = dynamic_cast *>(preprocessors.get()); + ASSERT_NE(container, nullptr); + auto *quantizer = dynamic_cast *>( + container->getPreprocessors()[0]); + ASSERT_NE(quantizer, nullptr); + auto x_blob = preprocessors->preprocessForStorage(x.data(), input_size); + auto y_blob = preprocessors->preprocessForStorage(y.data(), input_size); + auto query = preprocessors->preprocessQuery(y.data(), input_size); + auto combined = preprocessors->preprocess(y.data(), input_size); + const size_t query_size = + input_size + sq4::query_metadata_count() * sizeof(float); + EXPECT_EQ(std::memcmp(combined.getStorageBlob(), y_blob.get(), storage_size), 0); + EXPECT_EQ(std::memcmp(combined.getQueryBlob(), query.get(), query_size), 0); + if constexpr (Metric == VecSimMetric_L2) { + // Direct residual kernels need no query sums, but SIMD odd tails may read padding. + EXPECT_FLOAT_EQ(metadata(query.get(), input_size, sq4::SUM_QUERY), 0.0f); + EXPECT_FLOAT_EQ(metadata(query.get(), input_size, sq4::SUM_SQUARES_QUERY), 0.0f); + const auto *query_values = static_cast(query.get()); + for (size_t i = 0; i < dim; ++i) { + const float expected = WithMean ? y[i] - mean[i] : y[i]; + EXPECT_FLOAT_EQ(query_values[i], expected); + } + } + const auto rx = reconstruct(x_blob.get(), dim, mean); + const auto ry = reconstruct(y_blob.get(), dim, mean); + double symmetric = referenceDistance(rx, ry); + if constexpr (WithMean && Metric == VecSimMetric_IP) { + // The kernel uses reconstructed centered values, but the mean correction uses the + // original inputs to avoid multiplying quantization error by the mean. + symmetric = 1.0; + for (size_t i = 0; i < dim; ++i) { + symmetric -= (rx[i] - mean[i]) * (ry[i] - mean[i]) + double(x[i]) * mean[i] + + double(y[i]) * mean[i] - double(mean[i]) * mean[i]; + } + } + const double asymmetric = referenceDistance(rx, y); + const double sym_tolerance = 2e-5 * (1.0 + std::abs(symmetric)); + const double asym_tolerance = 2e-5 * (1.0 + std::abs(asymmetric)); + EXPECT_NEAR(calculator->calcDistance(x_blob.get(), y_blob.get(), dim), symmetric, + sym_tolerance); + EXPECT_NEAR(calculator->calcDistanceForQuery(x_blob.get(), query.get(), dim), asymmetric, + asym_tolerance); + auto stored_dispatch = calculator->getDistanceDispatch(DistanceMode::StoredToStored); + auto query_dispatch = calculator->getDistanceDispatch(DistanceMode::StoredToQuery); + ASSERT_TRUE(stored_dispatch.isValid()); + ASSERT_TRUE(query_dispatch.isValid()); + EXPECT_NEAR(stored_dispatch(x_blob.get(), y_blob.get(), dim), symmetric, sym_tolerance); + EXPECT_NEAR(query_dispatch(x_blob.get(), query.get(), dim), asymmetric, asym_tolerance); + + if (dim % 2) { + EXPECT_EQ(static_cast(x_blob.get())[dim / 2] & 0xf0, 0); + EXPECT_EQ(static_cast(y_blob.get())[dim / 2] & 0xf0, 0); + } + // The in-place allocation must hold both the original input and the processed blob; + // metadata makes the processed blob larger for tiny dimensions. + const size_t buffer_size = std::max(input_size, storage_size); + std::vector in_place((buffer_size + sizeof(float) - 1) / sizeof(float) + 4); + std::memset(in_place.data(), 0xa5, in_place.size() * sizeof(float)); + std::memcpy(in_place.data(), x.data(), input_size); + preprocessors->preprocessStorageInPlace(in_place.data(), buffer_size); + EXPECT_EQ(std::memcmp(in_place.data(), x_blob.get(), storage_size), 0); + const auto *bytes = reinterpret_cast(in_place.data()); + for (size_t i = buffer_size; i < in_place.size() * sizeof(float); ++i) { + EXPECT_EQ(bytes[i], 0xa5); + } + + if constexpr (WithMean && Metric == VecSimMetric_IP) { + float correction = 0.0f; + for (size_t i = 0; i < dim; ++i) { + correction += x[i] * mean[i]; + } + EXPECT_NEAR( + metadata(x_blob.get(), sq4::data_bytes_count(dim), sq4::mean_ip_index()), + correction, 2e-6 * (1.0 + std::abs(correction))); + } +} + +template +void checkDimensions() { + for (size_t dim : {1u, 2u, 3u, 4u, 5u, 7u, 16u, 37u, 128u}) { + for (bool constant : {false, true}) { + checkComponents(dim, constant); + checkComponents(dim, constant); + } + } +} + +} // namespace + +TEST(SQ4Components, L2MatchesReconstruction) { checkDimensions(); } + +TEST(SQ4Components, IPMatchesQuantizedKernelAndOriginalMeanCorrection) { + checkDimensions(); +} + +TEST(SQ4Components, CenteredIPCorrectionAvoidsMeanAmplifiedQuantizationError) { + constexpr size_t dim = 3; + const std::vector mean(dim, 10.0f); + const std::vector x = {10.0f, 13.0f, 12.1f}; + const std::vector y = {10.0f, 13.0f, 11.0f}; + auto allocator = VecSimAllocator::newVecsimAllocator(); + auto components = + CreateQuantizedIndexComponents(allocator, dim, mean.data()); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto xb = preprocessors->preprocessForStorage(x.data(), dim * sizeof(float)); + auto yb = preprocessors->preprocessForStorage(y.data(), dim * sizeof(float)); + const auto rx = reconstruct(xb.get(), dim, mean); + const auto ry = reconstruct(yb.get(), dim, mean); + const double truth = referenceDistance(x, y); + const double decoded = referenceDistance(rx, ry); + const float actual = calculator->calcDistance(xb.get(), yb.get(), dim); + EXPECT_FLOAT_EQ( + metadata(xb.get(), sq4::data_bytes_count(dim), sq4::mean_ip_index()), + 351.0f); + EXPECT_LT(std::abs(actual - truth), 0.11); + EXPECT_GT(std::abs(decoded - truth), 1.0); +} + +TEST(SQ4Components, PackedCodesAndMetadataSizes) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + const float input[] = {0, 4, 8, 11, 15}; + QuantPreprocessor preprocessor(allocator, 5); + void *storage = nullptr; + size_t size = sizeof(input); + preprocessor.preprocessForStorage(input, storage, size, 0); + ASSERT_EQ(size, 3 + 4 * sizeof(float)); + const auto *bytes = static_cast(storage); + EXPECT_EQ(bytes[0], 0x40); + EXPECT_EQ(bytes[1], 0xb8); + EXPECT_EQ(bytes[2], 0x0f); + EXPECT_FLOAT_EQ(metadata(storage, 3, sq4::MIN_VAL), 0); + EXPECT_FLOAT_EQ(metadata(storage, 3, sq4::DELTA), 1.0f); + allocator->free_allocation(storage); +} + +TEST(SQ4Components, OddDimTailNibbleIsZero) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + for (size_t dim : {1u, 15u, 17u, 767u}) { + SCOPED_TRACE(::testing::Message() << "dim=" << dim); + std::vector input(dim); + for (size_t i = 0; i < dim; ++i) { + input[i] = 0.37f * (int((i * 13) % 23) - 11); + } + // The last coordinate holds the maximum, so its code is 15 (for dim > 1) and a misplaced + // nibble shows. + input[dim - 1] = 100.0f; + auto check = [&](auto metric_tag) { + constexpr VecSimMetric Metric = decltype(metric_tag)::value; + auto components = CreateQuantizedIndexComponents( + allocator, dim, nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); + const auto *bytes = static_cast(blob.get()); + const size_t metadata_count = Metric == VecSimMetric_L2 ? 4 : 3; + EXPECT_EQ((sq4::storage_bytes_count(dim)), + dim / 2 + dim % 2 + metadata_count * sizeof(float)); + // A single coordinate is a constant range, which encodes as code 0. + EXPECT_EQ(sq4::get_code(bytes, dim - 1), dim == 1 ? 0 : 15); + if (dim % 2) { + EXPECT_EQ(bytes[dim / 2] & 0xf0, 0); + } + }; + check(std::integral_constant{}); + check(std::integral_constant{}); + } +} + +TEST(SQ4Components, NonFiniteInputEncodesInBounds) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t dim = 17; + const float inf = std::numeric_limits::infinity(); + std::vector input(dim); + for (size_t i = 0; i < dim; ++i) { + input[i] = 0.5f * (int(i % 7) - 3); + } + input[2] = std::numeric_limits::quiet_NaN(); + input[5] = inf; + input[9] = -inf; + input[13] = 1e30f; + std::vector mean(dim); + for (size_t i = 0; i < dim; ++i) { + mean[i] = 0.25f * (int(i % 3) - 1); + } + + auto check = [&](auto metric_tag, auto with_mean_tag) { + constexpr VecSimMetric Metric = decltype(metric_tag)::value; + constexpr bool WithMean = decltype(with_mean_tag)::value; + SCOPED_TRACE(::testing::Message() << "metric=" << Metric << " mean=" << WithMean); + auto components = CreateQuantizedIndexComponents( + allocator, dim, WithMean ? mean.data() : nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + const size_t storage_size = sq4::storage_bytes_count(dim); + + auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); + for (size_t i = 0; i < dim; ++i) { + EXPECT_LE(sq4::get_code(static_cast(blob.get()), i), 15); + } + + // In place, with a sentinel tail: nothing past the stored blob may be written. The input + // is larger than the stored blob here, so the bytes between them keep the input. + const size_t input_size = dim * sizeof(float); + ASSERT_GT(input_size, storage_size); + std::vector buffer(input_size + 16, 0xa5); + std::memcpy(buffer.data(), input.data(), input_size); + preprocessors->preprocessStorageInPlace(buffer.data(), input_size); + EXPECT_EQ(std::memcmp(buffer.data(), blob.get(), storage_size), 0); + EXPECT_EQ(std::memcmp(buffer.data() + storage_size, + reinterpret_cast(input.data()) + storage_size, + input_size - storage_size), + 0); + for (size_t i = input_size; i < buffer.size(); ++i) { + EXPECT_EQ(buffer[i], 0xa5); + } + // Odd dim: the unused high nibble stays zero even for non-finite input. + EXPECT_EQ(buffer[dim / 2] & 0xf0, 0); + }; + check(std::integral_constant{}, std::false_type{}); + check(std::integral_constant{}, std::false_type{}); + check(std::integral_constant{}, std::true_type{}); + check(std::integral_constant{}, std::true_type{}); +} + +TEST(SQ4Components, CenteredIPBlobSize768) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t dim = 768; + static_assert(sq4::storage_bytes_count(dim) == 400); + std::vector input(dim), mean(dim); + for (size_t i = 0; i < dim; ++i) { + input[i] = 0.01f * (int(i % 41) - 20); + mean[i] = 0.002f * (int(i % 9) - 4); + } + auto components = CreateQuantizedIndexComponents( + allocator, dim, mean.data()); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto *container = dynamic_cast *>(preprocessors.get()); + ASSERT_NE(container, nullptr); + auto *quantizer = container->getPreprocessors()[0]; + ASSERT_NE((dynamic_cast *>(quantizer)), + nullptr); + void *storage = nullptr; + size_t size = dim * sizeof(float); + quantizer->preprocessForStorage(input.data(), storage, size, 0); + EXPECT_EQ(size, 400u); + allocator->free_allocation(storage); +} + +// SQ4 components are distinct types from the SQ8 ones that applySQ8Mean and +// HNSWIndex::setQuantizationMean dynamic_cast to, so the SQ8 mean path cannot apply to SQ4. +TEST(SQ4Components, SQ4ComponentsAreNotSQ8Types) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t dim = 8; + std::vector mean(dim, 0.5f); + auto components = CreateQuantizedIndexComponents( + allocator, dim, mean.data()); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto *container = dynamic_cast *>(preprocessors.get()); + ASSERT_NE(container, nullptr); + EXPECT_EQ((dynamic_cast *>( + container->getPreprocessors()[0])), + nullptr); + EXPECT_EQ((dynamic_cast *>( + calculator.get())), + nullptr); + EXPECT_NE((dynamic_cast *>( + calculator.get())), + nullptr); +} + +// Centered and plain SQ4 L2 indexes compare stored vectors with the same kernel, as SQ8 does. +TEST(SQ4Components, StoredL2UsesOneKernel) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t dim = 64; + std::vector zero_mean(dim, 0.0f); + for (bool centered : {true, false}) { + SCOPED_TRACE(centered); + auto components = CreateQuantizedIndexComponents( + allocator, dim, centered ? zero_mean.data() : nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + EXPECT_EQ(calculator->getDistanceDispatch(DistanceMode::StoredToStored).stateless_func, + spaces::L2_SQ4_SQ4_GetDistFunc(dim)); + } +} diff --git a/tests/unit/test_sq4_distances.cpp b/tests/unit/test_sq4_distances.cpp new file mode 100644 index 000000000..345ffde8e --- /dev/null +++ b/tests/unit/test_sq4_distances.cpp @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#include "gtest/gtest.h" +#include "VecSim/spaces/spaces.h" +#include "VecSim/spaces/IP/IP.h" +#include "VecSim/spaces/L2/L2.h" +#include "VecSim/spaces/IP_space.h" +#include "VecSim/spaces/L2_space.h" +#include "VecSim/types/sq4.h" +#include "VecSim/types/sq8.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using vecsim_types::sq4; +using vecsim_types::sq8; + +static_assert(sq8::max_code == 255); +static_assert(sq8::data_bytes_count(7) == 7); +static_assert(sq8::storage_bytes_count(7) == 19); +static_assert(sq8::storage_bytes_count(7) == 23); +static_assert(sq4::data_bytes_count(0) == 0); +static_assert(sq4::data_bytes_count(7) == 4); +static_assert(sq4::storage_bytes_count(7) == 20); +static_assert(sq4::storage_bytes_count(7) == 20); + +// Offset the blob by one byte to exercise unaligned metadata reads as well as odd payload sizes. +struct PackedVector { + std::vector bytes; + std::vector reconstructed; + + PackedVector(const std::vector &codes, float minimum, float delta) + : bytes(1 + sq4::storage_bytes_count(codes.size()), 0xff), + reconstructed(codes.size()) { + float metadata[4] = {minimum, delta, 0.0f, 0.0f}; + for (size_t i = 0; i < codes.size(); ++i) { + sq4::set_code(data(), i, codes[i]); + reconstructed[i] = minimum + delta * codes[i]; + metadata[sq4::SUM] += reconstructed[i]; + metadata[sq4::SUM_SQUARES] += reconstructed[i] * reconstructed[i]; + } + std::memcpy(data() + sq4::data_bytes_count(codes.size()), metadata, sizeof(metadata)); + } + + uint8_t *data() { return bytes.data() + 1; } +}; + +float referenceDistance(VecSimMetric metric, const std::vector &x, + const std::vector &y) { + float result = 0.0f; + for (size_t i = 0; i < x.size(); ++i) { + if (metric == VecSimMetric_IP) { + result += x[i] * y[i]; + } else { + const float diff = x[i] - y[i]; + result += diff * diff; + } + } + return metric == VecSimMetric_IP ? 1.0f - result : result; +} +} // namespace + +TEST(SQ4Distances, PackedLayoutAndSQ8Compatibility) { + uint8_t packed[8]; + std::memset(packed, 0xff, sizeof(packed)); + for (size_t i = 0; i < 16; ++i) { + sq4::set_code(packed, i, i); + if (i % 2 == 0) { + EXPECT_EQ(packed[i / 2] & 0xf0, 0); + } + } + for (size_t i = 0; i < 16; ++i) { + EXPECT_EQ(sq4::get_code(packed, i), i); + if (i % 2 == 0) { + EXPECT_EQ(packed[i / 2], i | ((i + 1) << 4)); + } + } + uint8_t unpacked[256]; + for (size_t i = 0; i < 256; ++i) { + sq8::set_code(unpacked, i, i); + EXPECT_EQ(sq8::get_code(unpacked, i), i); + EXPECT_EQ(unpacked[i], i); + } +} + +TEST(SQ4Distances, DispatchMatchesExplicitReconstruction) { + for (size_t dim : {0u, 1u, 2u, 3u, 4u, 7u, 16u, 31u, 32u, 65u}) { + SCOPED_TRACE(dim); + std::vector x_codes(dim), y_codes(dim); + std::vector query(dim + sq4::query_metadata_count(), 0.0f); + std::vector query_values(dim); + for (size_t i = 0; i < dim; ++i) { + x_codes[i] = (i * 7) % 16; + y_codes[i] = (i * 3 + 15) % 16; + query[i] = query_values[i] = static_cast(static_cast(i % 13) - 6) / 8; + query[dim + sq4::SUM_QUERY] += query[i]; + query[dim + sq4::SUM_SQUARES_QUERY] += query[i] * query[i]; + } + PackedVector x(x_codes, -1.25f, 0.125f); + PackedVector y(y_codes, 0.75f, 0.0625f); + if (dim % 2) { + EXPECT_EQ(x.data()[sq4::data_bytes_count(dim) - 1] & 0xf0, 0); + EXPECT_EQ(y.data()[sq4::data_bytes_count(dim) - 1] & 0xf0, 0); + } + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + unsigned char alignment = 255; + auto symmetric = spaces::GetDistFunc(metric, dim, &alignment); + EXPECT_EQ(alignment, 0); + auto asymmetric = spaces::GetDistFunc(metric, dim, &alignment); + EXPECT_EQ(alignment, 0); + EXPECT_FLOAT_EQ(symmetric(x.data(), y.data(), dim), + referenceDistance(metric, x.reconstructed, y.reconstructed)); + EXPECT_FLOAT_EQ(asymmetric(x.data(), query.data(), dim), + referenceDistance(metric, x.reconstructed, query_values)); + EXPECT_EQ(symmetric, (spaces::GetDistFunc(metric, dim, nullptr))); + EXPECT_EQ(asymmetric, (spaces::GetDistFunc(metric, dim, nullptr))); + if (metric == VecSimMetric_L2) { + EXPECT_FLOAT_EQ(symmetric(x.data(), x.data(), dim), 0.0f); + EXPECT_FLOAT_EQ(symmetric(x.data(), y.data(), dim), + symmetric(y.data(), x.data(), dim)); + } + } + } +} + +TEST(SQ4Distances, ConstantVectorsAndOddPadding) { + std::vector codes(5, 0); + PackedVector x(codes, 2.0f, 0.0f); + PackedVector y(codes, -3.0f, 0.0f); + std::vector query(5 + sq4::query_metadata_count(), -3.0f); + query[5 + sq4::SUM_QUERY] = -15.0f; + query[5 + sq4::SUM_SQUARES_QUERY] = 45.0f; + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + auto symmetric = spaces::GetDistFunc(metric, 5, nullptr); + auto asymmetric = spaces::GetDistFunc(metric, 5, nullptr); + const float expected = metric == VecSimMetric_IP ? 31.0f : 125.0f; + EXPECT_FLOAT_EQ(symmetric(x.data(), y.data(), 5), expected); + EXPECT_FLOAT_EQ(asymmetric(x.data(), query.data(), 5), expected); + // The unused nibble is outside the vector, even if a foreign writer leaves it nonzero. + x.data()[sq4::data_bytes_count(5) - 1] |= 0xf0; + EXPECT_FLOAT_EQ(symmetric(x.data(), y.data(), 5), expected); + EXPECT_FLOAT_EQ(asymmetric(x.data(), query.data(), 5), expected); + } +} + +TEST(SQ4Distances, FP32L2PreservesResidualsAtLargeCommonOffset) { + PackedVector x({0, 1, 2}, 1048576.0f, 0.03125f); + PackedVector y({0, 0, 0}, 1048576.0f, 0.0f); + std::vector query(3 + sq4::query_metadata_count(), 1048576.0f); + query[3 + sq4::SUM_QUERY] = 3.0f * 1048576.0f; + query[3 + sq4::SUM_SQUARES_QUERY] = 3.0f * 1048576.0f * 1048576.0f; + const float expected = 5.0f * 0.03125f * 0.03125f; + EXPECT_FLOAT_EQ(SQ4_FP32_L2Sqr(x.data(), query.data(), 3), expected); +} + +TEST(SQ4Distances, FP32L2PackedPairsPreserveCoordinateOrder) { + for (size_t dim : {0u, 1u, 2u, 3u, 7u, 16u, 31u, 32u, 33u, 65u, 767u, 768u, 769u}) { + for (const auto &[minimum, delta] : + {std::pair{-0.75f, 0.1f}, std::pair{1048576.0f, 0.03125f}}) { + SCOPED_TRACE(::testing::Message() << "dim=" << dim << " min=" << minimum); + std::vector codes(dim); + std::vector query(dim); + float expected = 0.0f; + for (size_t i = 0; i < dim; ++i) { + codes[i] = (i * 7 + 3) % 16; + query[i] = minimum + 0.13f * (int(i % 17) - 8); + // Use the original codes, independently of the packed blob, and the previous + // scalar kernel's operand and reduction order. + const float diff = delta * float(codes[i]) + (minimum - query[i]); + expected += diff * diff; + } + PackedVector stored(codes, minimum, delta); + if (dim % 2) { + // A foreign writer's nonzero padding must not become another coordinate. + stored.data()[dim / 2] |= 0xf0; + } + const float actual = SQ4_FP32_L2Sqr(stored.data(), query.data(), dim); + EXPECT_EQ(std::bit_cast(actual), std::bit_cast(expected)); + } + } +} + +TEST(SQ4Distances, RejectsUnsupportedMetric) { + EXPECT_THROW((spaces::GetDistFunc(VecSimMetric_Cosine, 3, nullptr)), + std::invalid_argument); + EXPECT_THROW((spaces::GetDistFunc(VecSimMetric_Cosine, 3, nullptr)), + std::invalid_argument); +} + +namespace { +// Double-precision reference over the decoded vectors. +double refDistance(VecSimMetric metric, const std::vector &x, + const std::vector &y) { + double result = 0.0; + for (size_t i = 0; i < x.size(); ++i) { + result += metric == VecSimMetric_IP ? x[i] * y[i] : (x[i] - y[i]) * (x[i] - y[i]); + } + return metric == VecSimMetric_IP ? 1.0 - result : result; +} + +std::vector makeQuery(const std::vector &values) { + const size_t dim = values.size(); + std::vector query(dim + sq4::query_metadata_count(), 0.0f); + for (size_t i = 0; i < dim; ++i) { + query[i] = values[i]; + query[dim + sq4::SUM_QUERY] += values[i]; + query[dim + sq4::SUM_SQUARES_QUERY] += values[i] * values[i]; + } + return query; +} + +// Stored metadata slot of a packed vector, as the distance kernels read it. +double storedMeta(const uint8_t *blob, size_t dim, size_t slot) { + float value; + std::memcpy(&value, blob + sq4::data_bytes_count(dim) + slot * sizeof(float), sizeof(value)); + return value; +} + +// Stored-to-stored IP identity over the stored metadata, evaluated in double: +// IP = min_x * sum_y + min_y * sum_x - dim * min_x * min_y + delta_x * delta_y * sum(qx * qy) +double identityIP(const uint8_t *x, const uint8_t *y, size_t dim) { + double code_dot = 0.0; + for (size_t i = 0; i < dim; ++i) { + code_dot += double(sq4::get_code(x, i)) * sq4::get_code(y, i); + } + const double min_x = storedMeta(x, dim, sq4::MIN_VAL), dx = storedMeta(x, dim, sq4::DELTA); + const double min_y = storedMeta(y, dim, sq4::MIN_VAL), dy = storedMeta(y, dim, sq4::DELTA); + return min_x * storedMeta(y, dim, sq4::SUM) + min_y * storedMeta(x, dim, sq4::SUM) - + double(dim) * min_x * min_y + dx * dy * code_dot; +} + +// Per-element reconstruction of the SQ4-FP32 inner product distance. +float reconstructedIP(const uint8_t *x, const float *y, size_t dim, float min_val, float delta) { + float dot = 0.0f; + for (size_t i = 0; i < dim; ++i) { + dot += (min_val + delta * sq4::get_code(x, i)) * y[i]; + } + return 1.0f - dot; +} +} // namespace + +TEST(SQ4Distances, OddDimMatchesReference) { + std::mt19937 rng(1234); + std::uniform_int_distribution code_dist(0, 15); + std::uniform_real_distribution val_dist(-1.0f, 1.0f); + for (size_t dim : {1u, 15u, 16u, 17u, 64u, 767u, 768u}) { + SCOPED_TRACE(dim); + std::vector x_codes(dim), y_codes(dim); + std::vector values(dim); + for (size_t i = 0; i < dim; ++i) { + x_codes[i] = code_dist(rng); + y_codes[i] = code_dist(rng); + values[i] = val_dist(rng); + } + PackedVector x(x_codes, -0.5f, 1.0f / 15); + PackedVector y(y_codes, 0.25f, 1.0f / 30); + auto query = makeQuery(values); + std::vector xd(dim), yd(dim), qd(dim); + for (size_t i = 0; i < dim; ++i) { + qd[i] = values[i]; + } + // Decode with the exact float min/delta stored in the blobs, then compute in double. + const float minx = -0.5f, dx = 1.0f / 15, miny = 0.25f, dy = 1.0f / 30; + for (size_t i = 0; i < dim; ++i) { + xd[i] = double(minx) + double(dx) * sq4::get_code(x.data(), i); + yd[i] = double(miny) + double(dy) * sq4::get_code(y.data(), i); + } + auto check = [](float actual, double ref) { + EXPECT_NEAR(actual, ref, 1e-4 * std::max(1.0, std::abs(ref))); + }; + check(SQ4_SQ4_InnerProduct(x.data(), y.data(), dim), + 1.0 - identityIP(x.data(), y.data(), dim)); + check(SQ4_SQ4_L2Sqr(x.data(), y.data(), dim), refDistance(VecSimMetric_L2, xd, yd)); + check(SQ4_FP32_InnerProduct(x.data(), query.data(), dim), + refDistance(VecSimMetric_IP, xd, qd)); + check(SQ4_FP32_L2Sqr(x.data(), query.data(), dim), refDistance(VecSimMetric_L2, xd, qd)); + } +} + +TEST(SQ4Distances, FP32IPEqualsReconstruction) { + constexpr size_t dim = 768; + std::mt19937 rng(99); + std::uniform_int_distribution code_dist(0, 15); + std::uniform_real_distribution val_dist(-1.0f, 1.0f); + std::vector codes(dim); + std::vector values(dim); + for (size_t i = 0; i < dim; ++i) { + codes[i] = code_dist(rng); + values[i] = val_dist(rng); + } + const float min_val = -0.75f, delta = 0.1f; + PackedVector x(codes, min_val, delta); + auto query = makeQuery(values); + const float ref = reconstructedIP(x.data(), query.data(), dim, min_val, delta); + EXPECT_NEAR(SQ4_FP32_InnerProduct(x.data(), query.data(), dim), ref, + 1e-5 * std::max(1.0f, std::abs(ref))); +} + +TEST(SQ4Distances, DispatchIsScalarWithZeroAlignment) { + for (size_t dim : {7u, 8u, 15u, 16u, 63u, 64u, 768u}) { + SCOPED_TRACE(dim); + for (auto metric : {VecSimMetric_IP, VecSimMetric_L2}) { + SCOPED_TRACE(metric); + const bool ip = metric == VecSimMetric_IP; + unsigned char alignment = 255; + EXPECT_EQ((spaces::GetDistFunc(metric, dim, &alignment)), + ip ? SQ4_FP32_InnerProduct : SQ4_FP32_L2Sqr); + EXPECT_EQ(alignment, 0); + alignment = 255; + EXPECT_EQ((spaces::GetDistFunc(metric, dim, &alignment)), + ip ? SQ4_SQ4_InnerProduct : SQ4_SQ4_L2Sqr); + EXPECT_EQ(alignment, 0); + } + } +}