diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer.h b/src/VecSim/algorithms/hnsw/hnsw_serializer.h index dabc6f379..de223c0aa 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer.h @@ -24,7 +24,7 @@ class HNSWSerializer : public Serializer { DEPRECATED = 2, // Last deprecated version V3, V4, - V5, // SQ8 quantization fields (quantType + mean vector) + V5, // scalar quantization fields (quantType + mean vector) INVALID }; diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h index e72036a81..2865c5d81 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h @@ -263,10 +263,10 @@ void HNSWIndex::saveIndexFields(std::ofstream &output) const writeBinaryPOD(output, this->maxElements); // This will be used to restore the index initial // capacity - // V5 fields: SQ8 quantization support (between factory params and index build params) + // V5 fields: scalar quantization support (between factory params and index build params) if (this->quantType != VecSimQuant_NONE) { writeBinaryPOD(output, this->quantType); - // Write mean vector (dim floats). Empty vector means no mean (zero-mean SQ8). + // Write mean vector (dim floats). Empty vector means no mean (zero-mean quantization). bool hasMean = !this->serializedMeanVector.empty(); writeBinaryPOD(output, hasMean); if (hasMean) { diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index 9b77b9b15..39fc2486a 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -40,19 +40,19 @@ template return alignment; } -template +template IndexComponents -CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, - const float *mean_ptr) { - using Preprocessor = QuantPreprocessor; +CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, + const float *mean_ptr) { + using Preprocessor = QuantPreprocessor; using QueryType = typename Preprocessor::QueryType; unsigned char storage_alignment = 0, asym_storage_alignment = 0; - // Graph construction compares two stored SQ8 blobs; search compares a stored blob with a + // Graph construction compares two stored quantized blobs; search compares a stored blob with a // QueryType query. Both dispatchers report alignment for the stored operand. - auto sym_func = spaces::GetDistFunc(Metric, dim, &storage_alignment); - auto asym_func = spaces::GetDistFunc( - Metric, dim, &asym_storage_alignment); + auto sym_func = spaces::GetDistFunc(Metric, dim, &storage_alignment); + auto asym_func = + spaces::GetDistFunc(Metric, dim, &asym_storage_alignment); storage_alignment = spaces::combineAlignments(storage_alignment, asym_storage_alignment); const unsigned char query_alignment = GetQueryAlignment(Metric, dim); @@ -69,7 +69,7 @@ CreateSQ8IndexComponents(const std::shared_ptr &allocator, size } pp = new (allocator) Preprocessor(allocator, dim, mean_vec); - calc = new (allocator) DistanceCalculatorWithNorm( + calc = new (allocator) DistanceCalculatorWithNorm( allocator, asym_func, sym_func, mean_sum_squares); } else { pp = new (allocator) Preprocessor(allocator, dim); @@ -79,19 +79,38 @@ CreateSQ8IndexComponents(const std::shared_ptr &allocator, size auto *container = new (allocator) MultiPreprocessorsContainer(allocator, query_alignment, storage_alignment); [[maybe_unused]] const int ret = container->addPreprocessor(pp); - assert(ret != -1 && "SQ8 preprocessor was not added correctly"); + assert(ret != -1 && "Quantization preprocessor was not added correctly"); return {calc, container}; } +// Run-time overload: selects WithMean from whether mean_ptr is set. +template +IndexComponents +CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, + const float *mean_ptr) { + if (mean_ptr) { + return CreateQuantizedIndexComponents(allocator, dim, + mean_ptr); + } + return CreateQuantizedIndexComponents(allocator, dim, + mean_ptr); +} + +template +IndexComponents +CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, + const float *mean_ptr) { + return CreateQuantizedIndexComponents( + allocator, dim, mean_ptr); +} + template IndexComponents CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, const float *mean_ptr) { - if (mean_ptr) { - return CreateSQ8IndexComponents(allocator, dim, mean_ptr); - } - return CreateSQ8IndexComponents(allocator, dim, mean_ptr); + return CreateQuantizedIndexComponents(allocator, dim, + mean_ptr); } template diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index 2e553146f..15a7d54f3 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -37,42 +37,50 @@ NewIndex_ChooseMultiOrSingle(const HNSWParams *params, HNSWIndex_Single(params, abstractInitParams, components); } -template -[[nodiscard]] constexpr size_t GetSQ8StoredDataSize(size_t dim, bool with_norm) { +template +[[nodiscard]] constexpr size_t GetQuantizedStoredDataSize(size_t dim, bool with_norm) { static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP); - return with_norm ? sq8::storage_bytes_count(dim) - : sq8::storage_bytes_count(dim); + return with_norm ? StorageType::template storage_bytes_count(dim) + : StorageType::template storage_bytes_count(dim); } // Cosine over pre-normalized vectors is computed as inner product. -[[nodiscard]] constexpr VecSimMetric ResolveSQ8Metric(VecSimMetric metric, bool is_normalized) { +[[nodiscard]] constexpr VecSimMetric ResolveQuantizedMetric(VecSimMetric metric, + bool is_normalized) { return (is_normalized && metric == VecSimMetric_Cosine) ? VecSimMetric_IP : metric; } // Keep construction and initial-size validation in sync. -[[nodiscard]] constexpr bool SQ8ParamsSupported(VecSimType type, VecSimMetric resolved_metric) { +[[nodiscard]] constexpr bool QuantizationParamsSupported(VecSimQuantType quant_type, + VecSimType type, + VecSimMetric resolved_metric) { + // Reject unknown quantizers instead of silently creating an unquantized index. + if (quant_type != VecSimQuant_SQ8) { + return false; + } // SQ8 kernels accept only FLOAT32 and FLOAT16 input. if (type != VecSimType_FLOAT32 && type != VecSimType_FLOAT16) { return false; } - // Only L2 and inner product have SQ8 kernels. + // Only L2 and inner product have scalar quantization kernels. if (resolved_metric != VecSimMetric_L2 && resolved_metric != VecSimMetric_IP) { return false; } return true; } -template -VecSimIndex *NewIndex_SQ8(const HNSWParams *hnswParams, AbstractIndexInitParams abstractInitParams, - const float *mean_ptr) { +template +VecSimIndex *NewIndex_Quantized(const HNSWParams *hnswParams, + AbstractIndexInitParams abstractInitParams, const float *mean_ptr) { const bool with_norm = mean_ptr != nullptr; abstractInitParams.storedDataSize = - GetSQ8StoredDataSize(abstractInitParams.dim, with_norm); + GetQuantizedStoredDataSize(abstractInitParams.dim, with_norm); abstractInitParams.isQuantized = true; - IndexComponents components = CreateSQ8IndexComponents( - abstractInitParams.allocator, abstractInitParams.dim, mean_ptr); + IndexComponents components = + CreateQuantizedIndexComponents( + abstractInitParams.allocator, abstractInitParams.dim, mean_ptr); auto index = NewIndex_ChooseMultiOrSingle(hnswParams, abstractInitParams, components); #ifdef BUILD_TESTS @@ -91,38 +99,33 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { VecSimFactory::NewAbstractInitParams(hnswParams, params->logCtx, is_normalized); if (hnswParams->quantType != VecSimQuant_NONE) { - // Reject unknown quantizers instead of silently creating an unquantized index. - if (hnswParams->quantType != VecSimQuant_SQ8) { - return NULL; - } - - const VecSimMetric metric = ResolveSQ8Metric(hnswParams->metric, is_normalized); + const VecSimMetric metric = ResolveQuantizedMetric(hnswParams->metric, is_normalized); const float *mean_ptr = static_cast(hnswParams->quantParams); - if (!SQ8ParamsSupported(hnswParams->type, metric)) { + if (!QuantizationParamsSupported(hnswParams->quantType, hnswParams->type, metric)) { return NULL; } if (hnswParams->type == VecSimType_FLOAT32) { if (metric == VecSimMetric_L2) { - return NewIndex_SQ8(hnswParams, abstractInitParams, - mean_ptr); + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); } else if (metric == VecSimMetric_IP) { - return NewIndex_SQ8(hnswParams, abstractInitParams, - mean_ptr); + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); } } else if (hnswParams->type == VecSimType_FLOAT16) { if (metric == VecSimMetric_L2) { - return NewIndex_SQ8(hnswParams, abstractInitParams, - mean_ptr); + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); } else if (metric == VecSimMetric_IP) { - return NewIndex_SQ8(hnswParams, abstractInitParams, - mean_ptr); + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); } } // Keep the return for release builds so future dispatch changes cannot fall through. - assert(false && "unhandled SQ8 data type and metric combination"); + assert(false && "unhandled quantized data type and metric combination"); return NULL; } @@ -189,8 +192,8 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi if (params->quantType != VecSimQuant_NONE) { // Keep construction and initial-size validation in sync. - if (params->quantType != VecSimQuant_SQ8 || - !SQ8ParamsSupported(params->type, ResolveSQ8Metric(params->metric, is_normalized))) { + if (!QuantizationParamsSupported(params->quantType, params->type, + ResolveQuantizedMetric(params->metric, is_normalized))) { throw std::invalid_argument("Unsupported quantization params for HNSW index"); } // Template arguments do not affect these component sizes. @@ -244,9 +247,9 @@ size_t EstimateElementSize(const HNSWParams *params, bool with_mean) { // Preserve the element estimator's existing contract: return a size and validate in NewIndex. if (params->quantType == VecSimQuant_SQ8) { if (params->metric == VecSimMetric_L2) { - stored_data_size = GetSQ8StoredDataSize(params->dim, with_mean); + stored_data_size = GetQuantizedStoredDataSize(params->dim, with_mean); } else { - stored_data_size = GetSQ8StoredDataSize(params->dim, with_mean); + stored_data_size = GetQuantizedStoredDataSize(params->dim, with_mean); } } else { stored_data_size = @@ -302,6 +305,19 @@ inline VecSimIndex *NewIndex_ChooseMultiOrSingle(std::ifstream &input, const HNS return index; } +template +VecSimIndex *LoadIndex_Quantized(std::ifstream &input, const HNSWParams ¶ms, + AbstractIndexInitParams abstractInitParams, + HNSWSerializer::EncodingVersion version, const float *mean_ptr) { + abstractInitParams.isQuantized = true; + abstractInitParams.storedDataSize = + GetQuantizedStoredDataSize(params.dim, mean_ptr != nullptr); + auto components = CreateQuantizedIndexComponents( + abstractInitParams.allocator, params.dim, mean_ptr); + return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, + components, version, mean_ptr); +} + // Initialize @params from file for V3+. For V5+ also reads quantType and mean vector. // Returns false if the file holds a quantizer value this version does not know. static bool InitializeParams(std::ifstream &source_params, HNSWParams ¶ms, @@ -324,7 +340,7 @@ static bool InitializeParams(std::ifstream &source_params, HNSWParams ¶ms, return false; } params.quantType = static_cast(raw_quant_type); - if (params.quantType == VecSimQuant_SQ8) { + if (params.quantType != VecSimQuant_NONE) { bool hasMean = false; Serializer::readBinaryPOD(source_params, hasMean); if (hasMean) { @@ -370,46 +386,27 @@ VecSimIndex *NewIndex(const std::string &location, bool is_normalized) { if (params.quantType != VecSimQuant_NONE) { const float *mean_ptr = meanVector.empty() ? nullptr : meanVector.data(); - const VecSimMetric metric = ResolveSQ8Metric(params.metric, is_normalized); + const VecSimMetric metric = ResolveQuantizedMetric(params.metric, is_normalized); - if (!SQ8ParamsSupported(params.type, metric)) { + if (!QuantizationParamsSupported(params.quantType, params.type, metric)) { return NULL; } - // Override blob sizes for SQ8 storage layout. - size_t dim = params.dim; - abstractInitParams.isQuantized = true; - if (metric == VecSimMetric_L2) { - abstractInitParams.storedDataSize = - GetSQ8StoredDataSize(dim, mean_ptr != nullptr); - } else { - abstractInitParams.storedDataSize = - GetSQ8StoredDataSize(dim, mean_ptr != nullptr); - } - if (params.type == VecSimType_FLOAT32) { if (metric == VecSimMetric_L2) { - auto components = CreateSQ8IndexComponents( - abstractInitParams.allocator, dim, mean_ptr); - return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, - components, version, mean_ptr); + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); } else { - auto components = CreateSQ8IndexComponents( - abstractInitParams.allocator, dim, mean_ptr); - return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, - components, version, mean_ptr); + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); } } else if (params.type == VecSimType_FLOAT16) { if (metric == VecSimMetric_L2) { - auto components = CreateSQ8IndexComponents( - abstractInitParams.allocator, dim, mean_ptr); - return NewIndex_ChooseMultiOrSingle( - input, ¶ms, abstractInitParams, components, version, mean_ptr); + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); } else { - auto components = CreateSQ8IndexComponents( - abstractInitParams.allocator, dim, mean_ptr); - return NewIndex_ChooseMultiOrSingle( - input, ¶ms, abstractInitParams, components, version, mean_ptr); + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); } } } diff --git a/src/VecSim/spaces/IP/IP.cpp b/src/VecSim/spaces/IP/IP.cpp index 8cefa1c8f..136617c62 100644 --- a/src/VecSim/spaces/IP/IP.cpp +++ b/src/VecSim/spaces/IP/IP.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "IP.h" +#include "VecSim/spaces/quantized_kernels.h" #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" @@ -32,40 +33,7 @@ using sq8 = vecsim_types::sq8; * SQ8_FP32_L2Sqr. */ float SQ8_FP32_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, size_t dimension) { - const auto *pVect1 = static_cast(pVect1v); - const auto *pVect2 = static_cast(pVect2v); - - // Use 4 accumulators for instruction-level parallelism - float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; - - // Main loop: process 4 elements per iteration - size_t i = 0; - size_t dim4 = dimension & ~size_t(3); // dim4 is a multiple of 4 - for (; i < dim4; i += 4) { - sum0 += static_cast(pVect1[i + 0]) * pVect2[i + 0]; - sum1 += static_cast(pVect1[i + 1]) * pVect2[i + 1]; - sum2 += static_cast(pVect1[i + 2]) * pVect2[i + 2]; - sum3 += static_cast(pVect1[i + 3]) * pVect2[i + 3]; - } - - // Handle remainder (0-3 elements) - for (; i < dimension; i++) { - sum0 += static_cast(pVect1[i]) * pVect2[i]; - } - - // Combine accumulators - float quantized_dot = (sum0 + sum1) + (sum2 + sum3); - - // Storage metadata follows a byte payload and is not necessarily float-aligned. - const auto *params = pVect1 + dimension; - const float min_val = load_unaligned(params + sq8::MIN_VAL * sizeof(float)); - const float delta = load_unaligned(params + sq8::DELTA * sizeof(float)); - - // Get precomputed y_sum from query blob (pVect2 is FP32, stored after the dim floats) - const float y_sum = pVect2[dimension + sq8::SUM_QUERY]; - - // Apply formula: IP = min * y_sum + delta * Σ(q_i * y_i) - return min_val * y_sum + delta * quantized_dot; + return spaces::quantized::FP32_InnerProduct_Impl(pVect1v, pVect2v, dimension); } float SQ8_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) { @@ -143,31 +111,7 @@ float SQ8_FP16_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension // (not distance). Used by both SQ8_SQ8_InnerProduct, SQ8_SQ8_Cosine, and SQ8_SQ8_L2Sqr. // Vector layout: [uint8_t values (dim)] [min_val (float)] [delta (float)] [sum (float)] float SQ8_SQ8_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, size_t dimension) { - const auto *pVect1 = static_cast(pVect1v); - const auto *pVect2 = static_cast(pVect2v); - - // Compute inner product of quantized values: Σ(q1[i]*q2[i]) - float product = 0; - for (size_t i = 0; i < dimension; i++) { - product += pVect1[i] * pVect2[i]; - } - - // Metadata follows byte payloads and is not necessarily float-aligned. - const auto *params1 = pVect1 + dimension; - const float min_val1 = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); - const float delta1 = load_unaligned(params1 + sq8::DELTA * sizeof(float)); - const float sum1 = load_unaligned(params1 + sq8::SUM * sizeof(float)); - - // Get quantization parameters from pVect2 - const auto *params2 = pVect2 + dimension; - const float min_val2 = load_unaligned(params2 + sq8::MIN_VAL * sizeof(float)); - const float delta2 = load_unaligned(params2 + sq8::DELTA * sizeof(float)); - const float sum2 = load_unaligned(params2 + sq8::SUM * sizeof(float)); - - // Apply the algebraic formula using precomputed sums: - // IP = min1*sum2 + min2*sum1 + delta1*delta2*Σ(q1[i]*q2[i]) - dim*min1*min2 - return min_val1 * sum2 + min_val2 * sum1 - static_cast(dimension) * min_val1 * min_val2 + - delta1 * delta2 * product; + return spaces::quantized::Stored_InnerProduct_Impl(pVect1v, pVect2v, dimension); } // SQ8-to-SQ8: Both vectors are uint8 quantized with precomputed sum diff --git a/src/VecSim/spaces/L2/L2.cpp b/src/VecSim/spaces/L2/L2.cpp index da0c25971..2bc7f2cfb 100644 --- a/src/VecSim/spaces/L2/L2.cpp +++ b/src/VecSim/spaces/L2/L2.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "L2.h" +#include "VecSim/spaces/quantized_kernels.h" #include "VecSim/spaces/IP/IP.h" #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" @@ -34,25 +35,7 @@ using sq8 = vecsim_types::sq8; * pVect2 is query (FP32): [float values (dim)] [y_sum] [y_sum_squares] */ float SQ8_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { - // Storage metadata follows a byte payload and is not necessarily float-aligned. - const auto *pVect1 = static_cast(pVect1v); - const auto *pVect2 = static_cast(pVect2v); - - const auto *params1 = pVect1 + dimension; - const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); - const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); - - float res = 0; - for (size_t i = 0; i < dimension; i++) { - // Order matters. When the stored range sits far from zero, min_val and y_i are within a - // factor of two of each other, so min_val - y_i is exact (Sterbenz) and rounding happens - // only at the scale of the residual. Forming (min_val + delta*q_i) first would round at - // the scale of min_val and wipe out small residuals. When min_val and y_i are not close, - // the residual is large and either order is fine. - float diff = delta * static_cast(pVect1[i]) + (min_val - pVect2[i]); - res += diff * diff; - } - return res; + return spaces::quantized::FP32_L2Sqr(pVect1v, pVect2v, dimension); } /* @@ -194,19 +177,5 @@ float UINT8_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { // - IP(x, y) is computed using SQ8_SQ8_InnerProduct_Impl float SQ8_SQ8_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { - const auto *pVect1 = static_cast(pVect1v); - const auto *pVect2 = static_cast(pVect2v); - - // Get precomputed sum of squares from both vectors - // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] - const float sum_sq_1 = - load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); - const float sum_sq_2 = - load_unaligned(pVect2 + dimension + sq8::SUM_SQUARES * sizeof(float)); - - // Use the common inner product implementation - const float ip = SQ8_SQ8_InnerProduct_Impl(pVect1v, pVect2v, dimension); - - // L2² = ||x||² + ||y||² - 2*IP(x, y) - return sum_sq_1 + sum_sq_2 - 2.0f * ip; + return spaces::quantized::Stored_L2Sqr(pVect1v, pVect2v, dimension); } diff --git a/src/VecSim/spaces/computer/calculator.h b/src/VecSim/spaces/computer/calculator.h index 793402ad1..230042ab2 100644 --- a/src/VecSim/spaces/computer/calculator.h +++ b/src/VecSim/spaces/computer/calculator.h @@ -125,21 +125,21 @@ class DistanceCalculatorCommon }; /** - * Distance calculator for mean-normalized SQ8 indices. + * Distance calculator for mean-normalized quantized indices. * - * Stored vectors are SQ8-quantized from x' = x - mean. IP queries remain unshifted and use + * Stored vectors are quantized from x' = x - mean. IP queries remain unshifted and use * analytical correction terms derived from the per-vector x_mean_ip / y_mean_ip metadata. * L2 queries are centered once during preprocessing, allowing the base kernel to compute the * original distance directly without cancellation-prone correction terms. * * Correction formulas (expressed in terms of distance semantics): * - * Asymmetric (stored vector x as SQ8 of x'): + * Asymmetric (stored vector x as quantization of x'): * IP distance: dist(x,y) = asym_func(x_blob, y_blob) - y_mean_ip * L2 distance: dist(x,y) = asym_func(x_blob, centered_y_blob) * where centered_y = y - mean * - * Symmetric (both x and y as SQ8 blobs): + * Symmetric (both x and y as quantized blobs): * IP distance: dist(x,y) = sym_func(x_blob, y_blob) - x_mean_ip - y_mean_ip * + mean_sum_squares * L2 distance: dist(x,y) = sym_func(x_blob, y_blob) (mean terms cancel exactly) @@ -150,14 +150,13 @@ class DistanceCalculatorCommon * DataType is float or float16 * DistType is float */ -template +template class DistanceCalculatorWithNorm : public IndexCalculatorInterface { static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP, "DistanceCalculatorWithNorm only supports L2 and IP metrics"); private: - using sq8 = vecsim_types::sq8; - struct WithNormDistanceContext { spaces::dist_func_t stored_func; spaces::dist_func_t query_func; @@ -175,12 +174,12 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { // base = 1 - IP(x', y'). We want 1 - IP(x, y). // IP(x, y) = IP(x', y') + x_mean_ip + y_mean_ip - mean_sum_squares // => 1 - IP(x, y) = base - x_mean_ip - y_mean_ip + mean_sum_squares - float x_mean_ip = - load_unaligned(static_cast(v1) + dim + - sq8::template mean_ip_index() * sizeof(float)); - float y_mean_ip = - load_unaligned(static_cast(v2) + dim + - sq8::template mean_ip_index() * sizeof(float)); + float x_mean_ip = load_unaligned( + static_cast(v1) + StorageType::data_bytes_count(dim) + + StorageType::template mean_ip_index() * sizeof(float)); + float y_mean_ip = load_unaligned( + static_cast(v2) + StorageType::data_bytes_count(dim) + + StorageType::template mean_ip_index() * sizeof(float)); return base - x_mean_ip - y_mean_ip + context->mean_sum_squares; } else { // L2: ||x - y||² = ||x' - y'||² (mean terms cancel exactly) return base; @@ -192,9 +191,9 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { const auto *context = static_cast(opaque_context); DistType base = context->query_func(candidate, query, dim); if constexpr (Metric == VecSimMetric_IP) { - float y_mean_ip = - load_unaligned(static_cast(query) + dim * sizeof(DataType) + - sq8::template query_mean_ip_index() * sizeof(float)); + float y_mean_ip = load_unaligned( + static_cast(query) + dim * sizeof(DataType) + + StorageType::template query_mean_ip_index() * sizeof(float)); // base = 1 - IP(x', y). We want 1 - IP(x, y). // IP(x, y) = IP(x', y) + y_mean_ip // => 1 - IP(x, y) = base - y_mean_ip @@ -226,13 +225,13 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { } } - // Symmetric: both v1 and v2 are stored SQ8-of-x' blobs. + // Symmetric: both v1 and v2 are stored quantized-x' blobs. DistType calcDistance(const void *v1, const void *v2, size_t dim) const override { return calcStoredWithContext(&context_, v1, v2, dim); } // Asymmetric: IP queries are raw; L2 queries are centered during preprocessing. The candidate - // is a stored SQ8-of-x' blob. + // is a stored quantized-x' blob. // Note: query_dist_func expects (storage, query) argument order. DistType calcDistanceForQuery(const void *candidate, const void *query, size_t dim) const override { diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 1be9b06b8..3f46e2d48 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -143,13 +143,14 @@ class CosinePreprocessor : public PreprocessorInterface { /* * QuantPreprocessor is a preprocessor that quantizes storage vectors from DataType to a - * lower precision representation using OUTPUT_TYPE (uint8_t). + * lower precision representation defined by StorageType (SQ8, the current storage type, + * stores one uint8_t code per coordinate). * Query vectors remain as DataType for asymmetric distance computation. * * The quantized storage blob contains the quantized values along with metadata (min value, * scaling factor, and precomputed sums for reconstruction) in a single contiguous blob. * The quantization is done by finding the minimum and maximum values of the input vector, - * and then scaling the values to fit in the range of [0, 255]. + * and then scaling the values to fit in the storage type's code range. * * Storage layout: * | quantized_values[dim] | min_val | delta | x_sum | (x_sum_squares for L2 only) | @@ -164,8 +165,8 @@ class CosinePreprocessor : public PreprocessorInterface { * * Storage metadata is always FP32 (independent of DataType) to match the asymmetric distance * kernels. The quantized blob size is: - * - For L2: dim * sizeof(OUTPUT_TYPE) + 4 * sizeof(float) - * - For IP/Cosine: dim * sizeof(OUTPUT_TYPE) + 3 * sizeof(float) + * - For L2: StorageType::data_bytes_count(dim) + 4 * sizeof(float) + * - For IP/Cosine: StorageType::data_bytes_count(dim) + 3 * sizeof(float) * * Reconstruction formulas: * Given quantized value q_i, the original value is reconstructed as: @@ -256,7 +257,8 @@ static inline T from_fp32(float x) { } } -template +template class QuantPreprocessor : public PreprocessorInterface { public: // Center L2 queries in FP32 even when the original input is FP16. @@ -264,8 +266,7 @@ class QuantPreprocessor : public PreprocessorInterface { private: using OUTPUT_TYPE = uint8_t; - using MetadataType = float; // SQ8 metadata is always FP32 (see class doc). - using sq8 = vecsim_types::sq8; + using MetadataType = float; // Quantized storage metadata is always FP32. static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP || Metric == VecSimMetric_Cosine, @@ -283,7 +284,8 @@ class QuantPreprocessor : public PreprocessorInterface { // Calculate scaling factor (typed as MetadataType because they end up as metadata). const MetadataType diff = (max_val - min_val); - const MetadataType delta = (diff == 0.0f) ? MetadataType{1} : diff / MetadataType{255}; + const MetadataType delta = + (diff == 0.0f) ? MetadataType{1} : diff / MetadataType{StorageType::max_code}; const MetadataType inv_delta = MetadataType{1} / delta; // Saturating conversion. Casting a float to uint8_t is undefined when the truncated value @@ -299,8 +301,8 @@ class QuantPreprocessor : public PreprocessorInterface { if (!(scaled > MetadataType{0})) { return OUTPUT_TYPE{0}; // zero, negative, -inf, or NaN } - if (scaled >= MetadataType{255}) { - return OUTPUT_TYPE{255}; // includes +inf + if (scaled >= MetadataType{StorageType::max_code}) { + return OUTPUT_TYPE{StorageType::max_code}; // includes +inf } return static_cast(scaled + MetadataType{0.5}); }; @@ -337,10 +339,10 @@ class QuantPreprocessor : public PreprocessorInterface { const OUTPUT_TYPE a1 = to_byte((x1 - min_val) * inv_delta); const OUTPUT_TYPE a2 = to_byte((x2 - min_val) * inv_delta); const OUTPUT_TYPE a3 = to_byte((x3 - min_val) * inv_delta); - quantized[i] = a0; - quantized[i + 1] = a1; - quantized[i + 2] = a2; - quantized[i + 3] = a3; + StorageType::set_code(quantized, i, a0); + StorageType::set_code(quantized, i + 1, a1); + StorageType::set_code(quantized, i + 2, a2); + StorageType::set_code(quantized, i + 3, a3); s0 += a0; s1 += a1; @@ -365,7 +367,7 @@ class QuantPreprocessor : public PreprocessorInterface { for (; i < this->dim; ++i) { const float x = transformed_value(input, i); const OUTPUT_TYPE a = to_byte((x - min_val) * inv_delta); - quantized[i] = a; + StorageType::set_code(quantized, i, a); q_sum += a; if constexpr (Metric == VecSimMetric_L2) { q_sum_squares += uint64_t{a} * a; @@ -386,9 +388,9 @@ class QuantPreprocessor : public PreprocessorInterface { d_delta * d_delta * q_sum_squares); } - // Metadata uses MetadataType. Use memcpy because the metadata offset - // (dim * sizeof(uint8_t)) is not guaranteed to be sizeof(MetadataType)-aligned. - void *meta_dst = quantized + this->dim; + // Metadata uses MetadataType. Use memcpy because the metadata offset (after the + // quantized body) is not guaranteed to be sizeof(MetadataType)-aligned. + void *meta_dst = quantized + StorageType::data_bytes_count(this->dim); MetadataType buf[5] = {min_val, delta, sum}; size_t n = 3; if constexpr (Metric == VecSimMetric_L2) @@ -490,26 +492,29 @@ class QuantPreprocessor : public PreprocessorInterface { QuantPreprocessor(std::shared_ptr allocator, size_t dim) requires(!WithNorm) : PreprocessorInterface(allocator), dim(dim), - storage_bytes_count(sq8::storage_bytes_count(dim)), + storage_bytes_count(StorageType::template storage_bytes_count(dim)), query_bytes_count(dim * sizeof(QueryType) + - sq8::query_metadata_count() * sizeof(MetadataType)) {} + StorageType::template query_metadata_count() * + sizeof(MetadataType)) {} // WithNorm constructor: accepts a pre-computed mean vector (FP32, length == dim). QuantPreprocessor(std::shared_ptr allocator, size_t dim, const vecsim_stl::vector &mean_vec) requires(WithNorm) : PreprocessorInterface(allocator), mean(mean_vec), dim(dim), - storage_bytes_count(sq8::storage_bytes_count(dim)), + storage_bytes_count(StorageType::template storage_bytes_count(dim)), query_bytes_count(dim * sizeof(QueryType) + - sq8::query_metadata_count() * sizeof(MetadataType)) { + StorageType::template query_metadata_count() * + sizeof(MetadataType)) { assert(this->mean.size() == dim && "mean vector size must equal dim"); } /** * Preprocesses the original blob into separate storage and query blobs. * - * Storage vectors are quantized to uint8_t values, with metadata (min, delta, sum, and - * sum_squares for L2) appended for distance reconstruction. + * Storage vectors are quantized to StorageType scalar codes (SQ8: one uint8_t per + * coordinate), with metadata (min, delta, sum, and sum_squares for L2) appended for + * distance reconstruction. * * Query vectors use QueryType for asymmetric distance computation, with a precomputed * sum (for IP/Cosine) or sum of squares (for L2) appended for efficient distance calculation. diff --git a/src/VecSim/spaces/quantized_kernels.h b/src/VecSim/spaces/quantized_kernels.h new file mode 100644 index 000000000..c84dd0e03 --- /dev/null +++ b/src/VecSim/spaces/quantized_kernels.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include +#include "VecSim/utils/alignment.h" + +// SIMD translation units (functions/*.cpp) must not include this header; they call the exported +// scalar functions (e.g. SQ8_FP32_L2Sqr) instead. +// +// Scalar kernels for scalar quantized storage, generic over a StorageType that provides +// get_code(data, i), data_bytes_count(dim) and the MIN_VAL / DELTA / SUM / SUM_SQUARES metadata +// indices (stored after the codes) and SUM_QUERY (stored after the FP32 query values). +// It also provides the code dot hooks code_dot(a, b, dim) and code_dot_fp32(codes, y, dim). +// +// Stored layout: [codes (data_bytes_count(dim) bytes)] [min_val] [delta] [sum] [sum_squares (L2)] +// FP32 query layout: [float values (dim)] [y_sum] [y_sum_squares (L2 only)] +// +// The operand order in these bodies is load-bearing and relies on FP addition not being +// reassociated; they must not be built with -ffast-math / -Ofast. +namespace spaces { +namespace quantized { + +/* + * Asymmetric stored-FP32 inner product using the algebraic identity: + * IP(x, y) = sum((min + delta * q_i) * y_i) = min * y_sum + delta * sum(q_i * y_i) + * + * sum(q_i * y_i) comes from StorageType::code_dot_fp32. + * Returns the raw inner product value (not distance). + */ +template +static inline float FP32_InnerProduct_Impl(const void *stored, const void *query, size_t dim) { + const auto *pVect1 = static_cast(stored); + const auto *pVect2 = static_cast(query); + + // Compute inner product of quantized values with the query: sum(q_i * y_i) + const float quantized_dot = StorageType::code_dot_fp32(pVect1, pVect2, dim); + + // Storage metadata follows a byte payload and is not necessarily float-aligned. + const auto *params = pVect1 + StorageType::data_bytes_count(dim); + const float min_val = load_unaligned(params + StorageType::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params + StorageType::DELTA * sizeof(float)); + + // Get precomputed y_sum from query blob (stored after the dim floats) + const float y_sum = pVect2[dim + StorageType::SUM_QUERY]; + + // Apply formula: IP = min * y_sum + delta * sum(q_i * y_i) + return min_val * y_sum + delta * quantized_dot; +} + +/* + * Asymmetric stored-FP32 L2 squared distance, accumulated directly per component: + * ||x - y||^2 = sum((dequant(x_i) - y_i)^2), where dequant(x_i) = min_val + delta * q_i + * + * Accumulating residuals keeps each component's rounding at the scale of |x_i - y_i|. Expanding + * to ||x||^2 + ||y||^2 - 2*IP(x, y) would round at the scale of the norms, which loses the + * distance when x and y share a large common offset relative to their spread. + */ +template +static inline float FP32_L2Sqr(const void *stored, const void *query, size_t dim) { + // Storage metadata follows a byte payload and is not necessarily float-aligned. + const auto *pVect1 = static_cast(stored); + const auto *pVect2 = static_cast(query); + + const auto *params1 = pVect1 + StorageType::data_bytes_count(dim); + const float min_val = load_unaligned(params1 + StorageType::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + StorageType::DELTA * sizeof(float)); + + float res = 0; + for (size_t i = 0; i < dim; i++) { + // Order matters. When the stored range sits far from zero, min_val and y_i are within a + // factor of two of each other, so min_val - y_i is exact (Sterbenz) and rounding happens + // only at the scale of the residual. Forming (min_val + delta*q_i) first would round at + // the scale of min_val and wipe out small residuals. When min_val and y_i are not close, + // the residual is large and either order is fine. + float diff = + delta * static_cast(StorageType::get_code(pVect1, i)) + (min_val - pVect2[i]); + res += diff * diff; + } + return res; +} + +/* + * Symmetric stored-stored inner product using precomputed sums: + * IP = min1*sum2 + min2*sum1 - dim*min1*min2 + delta1*delta2*sum(q1_i*q2_i) + * Returns the raw inner product value (not distance). + */ +template +static inline float Stored_InnerProduct_Impl(const void *a, const void *b, size_t dim) { + const auto *pVect1 = static_cast(a); + const auto *pVect2 = static_cast(b); + + // Compute inner product of quantized values: sum(q1[i]*q2[i]) + const float product = StorageType::code_dot(pVect1, pVect2, dim); + + // Metadata follows byte payloads and is not necessarily float-aligned. + const auto *params1 = pVect1 + StorageType::data_bytes_count(dim); + const float min_val1 = load_unaligned(params1 + StorageType::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(params1 + StorageType::DELTA * sizeof(float)); + const float sum1 = load_unaligned(params1 + StorageType::SUM * sizeof(float)); + + // Get quantization parameters from pVect2 + const auto *params2 = pVect2 + StorageType::data_bytes_count(dim); + const float min_val2 = load_unaligned(params2 + StorageType::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(params2 + StorageType::DELTA * sizeof(float)); + const float sum2 = load_unaligned(params2 + StorageType::SUM * sizeof(float)); + + // Apply the algebraic formula using precomputed sums: + // IP = min1*sum2 + min2*sum1 + delta1*delta2*sum(q1[i]*q2[i]) - dim*min1*min2 + return min_val1 * sum2 + min_val2 * sum1 - static_cast(dim) * min_val1 * min_val2 + + delta1 * delta2 * product; +} + +/* + * Symmetric stored-stored L2 squared distance: + * ||x - y||^2 = ||x||^2 + ||y||^2 - 2*IP(x, y) + * where ||x||^2 and ||y||^2 are the precomputed SUM_SQUARES metadata and IP(x, y) is computed by + * Stored_InnerProduct_Impl. + */ +template +static inline float Stored_L2Sqr(const void *a, const void *b, size_t dim) { + const auto *pVect1 = static_cast(a); + const auto *pVect2 = static_cast(b); + + // Get precomputed sum of squares from both vectors + const size_t data_bytes = StorageType::data_bytes_count(dim); + const float sum_sq_1 = + load_unaligned(pVect1 + data_bytes + StorageType::SUM_SQUARES * sizeof(float)); + const float sum_sq_2 = + load_unaligned(pVect2 + data_bytes + StorageType::SUM_SQUARES * sizeof(float)); + + // Use the common inner product implementation + const float ip = Stored_InnerProduct_Impl(a, b, dim); + + // L2^2 = ||x||^2 + ||y||^2 - 2*IP(x, y) + return sum_sq_1 + sum_sq_2 - 2.0f * ip; +} + +} // namespace quantized +} // namespace spaces diff --git a/src/VecSim/types/quantized_metadata.h b/src/VecSim/types/quantized_metadata.h new file mode 100644 index 000000000..7d726b5c7 --- /dev/null +++ b/src/VecSim/types/quantized_metadata.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include "VecSim/vec_sim_common.h" + +namespace vecsim_types { + +// FP32 reconstruction metadata shared by scalar-quantized storage formats. +struct quantized_metadata { + // Metadata layout indices (stored after quantized values) + enum MetadataIndex : size_t { + MIN_VAL = 0, + DELTA = 1, + SUM = 2, + SUM_SQUARES = 3 // Only for L2 + }; + + enum QueryMetadataIndex : size_t { + SUM_QUERY = 0, + SUM_SQUARES_QUERY = 1 // Only for L2 + }; + + // Template on Metric and WithNorm — compile-time constants + // WithNorm: one extra metadata slot for x_mean_ip / y_mean_ip. Only needed for IP: the + // normalized L2 correction cancels the mean terms exactly during centering and never reads + // mean_ip (see DistanceCalculatorWithNorm), so no extra slot is allocated for L2. + template + static constexpr size_t storage_metadata_count() { + return ((Metric == VecSimMetric_L2) ? 4 : 3) + + ((WithNorm && Metric == VecSimMetric_IP) ? 1 : 0); + } + + template + static constexpr size_t query_metadata_count() { + return ((Metric == VecSimMetric_L2) ? 2 : 1) + + ((WithNorm && Metric == VecSimMetric_IP) ? 1 : 0); + } + + // Index of x_mean_ip / y_mean_ip in the last slot in metadata array + template + static constexpr size_t mean_ip_index() { + return storage_metadata_count() - 1; + } + + template + static constexpr size_t query_mean_ip_index() { + return query_metadata_count() - 1; + } +}; + +} // namespace vecsim_types diff --git a/src/VecSim/types/sq8.h b/src/VecSim/types/sq8.h index ac2446807..55adccf74 100644 --- a/src/VecSim/types/sq8.h +++ b/src/VecSim/types/sq8.h @@ -10,59 +10,59 @@ #include #include -#include "VecSim/vec_sim_common.h" +#include +#include "VecSim/types/quantized_metadata.h" namespace vecsim_types { // Represents a scalar-quantized 8-bit blob with reconstruction metadata -struct sq8 { +struct sq8 : quantized_metadata { using value_type = uint8_t; + static constexpr uint8_t max_code = std::numeric_limits::max(); - // Metadata layout indices (stored after quantized values) - enum MetadataIndex : size_t { - MIN_VAL = 0, - DELTA = 1, - SUM = 2, - SUM_SQUARES = 3 // Only for L2 - }; + static constexpr size_t data_bytes_count(size_t dim) { return dim; } - enum QueryMetadataIndex : size_t { - SUM_QUERY = 0, - SUM_SQUARES_QUERY = 1 // Only for L2 - }; + static uint8_t get_code(const uint8_t *data, size_t index) { return data[index]; } - // Template on Metric and WithNorm — compile-time constants - // WithNorm: one extra metadata slot for x_mean_ip / y_mean_ip. Only needed for IP: the - // normalized L2 correction cancels the mean terms exactly during centering and never reads - // mean_ip (see DistanceCalculatorWithNorm), so no extra slot is allocated for L2. - template - static constexpr size_t storage_metadata_count() { - return ((Metric == VecSimMetric_L2) ? 4 : 3) + - ((WithNorm && Metric == VecSimMetric_IP) ? 1 : 0); + static void set_code(uint8_t *data, size_t index, uint8_t code) { data[index] = code; } + + // Dot product of two code arrays: sum(q1_i * q2_i). + static float code_dot(const uint8_t *a, const uint8_t *b, size_t dim) { + float product = 0; + for (size_t i = 0; i < dim; i++) { + product += get_code(a, i) * get_code(b, i); + } + return product; } - template - static constexpr size_t query_metadata_count() { - return ((Metric == VecSimMetric_L2) ? 2 : 1) + - ((WithNorm && Metric == VecSimMetric_IP) ? 1 : 0); + // Dot product of a code array with FP32 values: sum(q_i * y_i). Uses 4 accumulators for + // instruction-level parallelism. + static float code_dot_fp32(const uint8_t *codes, const float *y, size_t dim) { + float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + + // Main loop: process 4 elements per iteration + size_t i = 0; + size_t dim4 = dim & ~size_t(3); // dim4 is a multiple of 4 + for (; i < dim4; i += 4) { + sum0 += static_cast(get_code(codes, i + 0)) * y[i + 0]; + sum1 += static_cast(get_code(codes, i + 1)) * y[i + 1]; + sum2 += static_cast(get_code(codes, i + 2)) * y[i + 2]; + sum3 += static_cast(get_code(codes, i + 3)) * y[i + 3]; + } + + // Handle remainder (0-3 elements) + for (; i < dim; i++) { + sum0 += static_cast(get_code(codes, i)) * y[i]; + } + + // Combine accumulators + return (sum0 + sum1) + (sum2 + sum3); } // Stored layout: one quantized byte per dimension, followed by metric-specific FP32 metadata. template static constexpr size_t storage_bytes_count(size_t dim) { - return dim * sizeof(value_type) + - storage_metadata_count() * sizeof(float); - } - - // Index of x_mean_ip / y_mean_ip in the last slot in metadata array - template - static constexpr size_t mean_ip_index() { - return storage_metadata_count() - 1; - } - - template - static constexpr size_t query_mean_ip_index() { - return query_metadata_count() - 1; + return data_bytes_count(dim) + storage_metadata_count() * sizeof(float); } }; diff --git a/tests/unit/test_components.cpp b/tests/unit/test_components.cpp index 375ee52b1..6744756b3 100644 --- a/tests/unit/test_components.cpp +++ b/tests/unit/test_components.cpp @@ -10,12 +10,16 @@ #include #include #include +#include +#include +#include #include "gtest/gtest.h" #include "VecSim/vec_sim.h" #include "VecSim/quantization/sq8_training_state.h" #include "VecSim/spaces/computer/preprocessor_container.h" #include "VecSim/spaces/computer/preprocessors.h" #include "VecSim/spaces/computer/calculator.h" +#include "VecSim/index_factories/components/components_factory.h" #include "VecSim/spaces/IP_space.h" #include "VecSim/spaces/L2_space.h" #include "unit_test_utils.h" @@ -1443,6 +1447,121 @@ TEST(PreprocessorsTest, QuantizationTestAllEntriesEqual) { delete quant_preprocessor; } +namespace { + +template +DataType fromFloat(float value) { + if constexpr (std::is_same_v) { + return vecsim_types::FP32_to_FP16(value); + } else { + return value; + } +} + +// SQ8 byte-identity guard. Every input lies on a 0.25 grid spanning exactly 255 steps, so delta +// is exactly 0.25, every code is an exact integer and every metadata value is exactly +// representable. The expected blob is therefore derived below with exact arithmetic, without +// calling QuantPreprocessor, and any change to the SQ8 encoding shows up as a byte mismatch. +template +void checkSQ8Bytes(size_t dim) { + constexpr bool kFP16 = std::is_same_v; + SCOPED_TRACE(::testing::Message() << "fp16=" << kFP16 << " metric=" << Metric + << " mean=" << WithMean << " dim=" << dim); + constexpr double kMin = -8.0, kStep = 0.25; + auto allocator = VecSimAllocator::newVecsimAllocator(); + std::vector codes(dim); + std::vector mean(dim, 0.0f), input_fp32(dim); + std::vector input(dim); + for (size_t i = 0; i < dim; ++i) { + // Codes 0 and 255 pin the range; the rest exercise every byte position. + codes[i] = i == 0 ? 0 : i == 1 ? 255 : uint8_t((i * 37 + 11) % 256); + if constexpr (WithMean) { + mean[i] = 0.5f * (int(i % 5) - 2); + } + input_fp32[i] = float(kMin + kStep * codes[i] + mean[i]); + input[i] = fromFloat(input_fp32[i]); + } + + double sum = 0, sum_squares = 0, mean_ip = 0; + for (size_t i = 0; i < dim; ++i) { + const double reconstructed = kMin + kStep * codes[i]; + sum += reconstructed; + sum_squares += reconstructed * reconstructed; + mean_ip += double(input_fp32[i]) * mean[i]; + } + std::vector expected_metadata = {float(kMin), float(kStep), float(sum)}; + if constexpr (Metric == VecSimMetric_L2) { + expected_metadata.push_back(float(sum_squares)); + } + if constexpr (WithMean && Metric == VecSimMetric_IP) { + expected_metadata.push_back(float(mean_ip)); + } + std::vector expected(codes); + expected.resize(dim + expected_metadata.size() * sizeof(float)); + std::memcpy(expected.data() + dim, expected_metadata.data(), + expected_metadata.size() * sizeof(float)); + ASSERT_EQ(expected.size(), (sq8::storage_bytes_count(dim))); + + auto encode = [&](const QuantPreprocessor &preprocessor) { + void *storage = nullptr; + size_t size = dim * sizeof(DataType); + preprocessor.preprocessForStorage(input.data(), storage, size, 0); + const auto *bytes = static_cast(storage); + std::vector result(bytes, bytes + size); + allocator->free_allocation(storage); + return result; + }; + if constexpr (WithMean) { + vecsim_stl::vector mean_vec(allocator); + mean_vec.assign(mean.begin(), mean.end()); + EXPECT_EQ(encode(QuantPreprocessor(allocator, dim, mean_vec)), + expected); + } else { + EXPECT_EQ(encode(QuantPreprocessor(allocator, dim)), expected); + } + + // The SQ8 factory wrapper must produce the same bytes and keep naming the SQ8 component + // types that applySQ8Mean dynamic_casts to. + if constexpr (std::is_same_v && Metric != VecSimMetric_Cosine) { + auto components = CreateSQ8IndexComponents(allocator, dim, + WithMean ? mean.data() : nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto *container = + dynamic_cast *>(preprocessors.get()); + ASSERT_NE(container, nullptr); + EXPECT_NE((dynamic_cast *>( + container->getPreprocessors()[0])), + nullptr); + if constexpr (WithMean) { + EXPECT_NE((dynamic_cast *>( + calculator.get())), + nullptr); + } + auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); + const auto *factory_bytes = static_cast(blob.get()); + EXPECT_EQ(std::vector(factory_bytes, factory_bytes + expected.size()), expected); + } +} + +template +void checkSQ8BytesAllModes() { + for (size_t dim : {2u, 5u, 37u, 128u}) { + checkSQ8Bytes(dim); + checkSQ8Bytes(dim); + checkSQ8Bytes(dim); + checkSQ8Bytes(dim); + checkSQ8Bytes(dim); + } +} + +} // namespace + +TEST(PreprocessorsTest, QuantizedStorageSQ8BytesUnchanged) { + checkSQ8BytesAllModes(); + checkSQ8BytesAllModes(); +} + // Parameterized test class for QuantPreprocessor with different metrics class QuantPreprocessorMetricTest : public testing::TestWithParam { protected: diff --git a/tests/unit/test_spaces.cpp b/tests/unit/test_spaces.cpp index 923bb7c1b..69d6c84d9 100644 --- a/tests/unit/test_spaces.cpp +++ b/tests/unit/test_spaces.cpp @@ -27,6 +27,7 @@ #include "VecSim/spaces/L2_space.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/spaces/quantized_kernels.h" #include "VecSim/spaces/functions/AVX512F.h" #include "VecSim/spaces/functions/AVX.h" #include "VecSim/spaces/functions/SSE.h" @@ -411,6 +412,171 @@ TEST_F(SpacesTest, SQ8_FP32_odd_dim_unaligned_metadata_test) { } } +/* ============ Generic scalar quantized kernels: SQ8 bit-identity ============ + * Verbatim copies of the four scalar SQ8 kernel bodies as they were on origin/main before they + * became wrappers over spaces::quantized. Only the function names differ. They pin the original + * results so the test does not compare the wrappers against the templates they call. + */ +namespace { + +float OriginalSQ8_FP32_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, + size_t dimension) { + const auto *pVect1 = static_cast(pVect1v); + const auto *pVect2 = static_cast(pVect2v); + + // Use 4 accumulators for instruction-level parallelism + float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + + // Main loop: process 4 elements per iteration + size_t i = 0; + size_t dim4 = dimension & ~size_t(3); // dim4 is a multiple of 4 + for (; i < dim4; i += 4) { + sum0 += static_cast(pVect1[i + 0]) * pVect2[i + 0]; + sum1 += static_cast(pVect1[i + 1]) * pVect2[i + 1]; + sum2 += static_cast(pVect1[i + 2]) * pVect2[i + 2]; + sum3 += static_cast(pVect1[i + 3]) * pVect2[i + 3]; + } + + // Handle remainder (0-3 elements) + for (; i < dimension; i++) { + sum0 += static_cast(pVect1[i]) * pVect2[i]; + } + + // Combine accumulators + float quantized_dot = (sum0 + sum1) + (sum2 + sum3); + + // Storage metadata follows a byte payload and is not necessarily float-aligned. + const auto *params = pVect1 + dimension; + const float min_val = load_unaligned(params + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params + sq8::DELTA * sizeof(float)); + + // Get precomputed y_sum from query blob (pVect2 is FP32, stored after the dim floats) + const float y_sum = pVect2[dimension + sq8::SUM_QUERY]; + + // Apply formula: IP = min * y_sum + delta * Σ(q_i * y_i) + return min_val * y_sum + delta * quantized_dot; +} + +float OriginalSQ8_SQ8_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, + size_t dimension) { + const auto *pVect1 = static_cast(pVect1v); + const auto *pVect2 = static_cast(pVect2v); + + // Compute inner product of quantized values: Σ(q1[i]*q2[i]) + float product = 0; + for (size_t i = 0; i < dimension; i++) { + product += pVect1[i] * pVect2[i]; + } + + // Metadata follows byte payloads and is not necessarily float-aligned. + const auto *params1 = pVect1 + dimension; + const float min_val1 = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(params1 + sq8::DELTA * sizeof(float)); + const float sum1 = load_unaligned(params1 + sq8::SUM * sizeof(float)); + + // Get quantization parameters from pVect2 + const auto *params2 = pVect2 + dimension; + const float min_val2 = load_unaligned(params2 + sq8::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(params2 + sq8::DELTA * sizeof(float)); + const float sum2 = load_unaligned(params2 + sq8::SUM * sizeof(float)); + + // Apply the algebraic formula using precomputed sums: + // IP = min1*sum2 + min2*sum1 + delta1*delta2*Σ(q1[i]*q2[i]) - dim*min1*min2 + return min_val1 * sum2 + min_val2 * sum1 - static_cast(dimension) * min_val1 * min_val2 + + delta1 * delta2 * product; +} + +float OriginalSQ8_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { + // Storage metadata follows a byte payload and is not necessarily float-aligned. + const auto *pVect1 = static_cast(pVect1v); + const auto *pVect2 = static_cast(pVect2v); + + const auto *params1 = pVect1 + dimension; + const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); + + float res = 0; + for (size_t i = 0; i < dimension; i++) { + // Order matters. When the stored range sits far from zero, min_val and y_i are within a + // factor of two of each other, so min_val - y_i is exact (Sterbenz) and rounding happens + // only at the scale of the residual. Forming (min_val + delta*q_i) first would round at + // the scale of min_val and wipe out small residuals. When min_val and y_i are not close, + // the residual is large and either order is fine. + float diff = delta * static_cast(pVect1[i]) + (min_val - pVect2[i]); + res += diff * diff; + } + return res; +} + +float OriginalSQ8_SQ8_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { + const auto *pVect1 = static_cast(pVect1v); + const auto *pVect2 = static_cast(pVect2v); + + // Get precomputed sum of squares from both vectors + // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] + const float sum_sq_1 = + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); + const float sum_sq_2 = + load_unaligned(pVect2 + dimension + sq8::SUM_SQUARES * sizeof(float)); + + // Use the common inner product implementation + const float ip = OriginalSQ8_SQ8_InnerProduct_Impl(pVect1v, pVect2v, dimension); + + // L2² = ||x||² + ||y||² - 2*IP(x, y) + return sum_sq_1 + sum_sq_2 - 2.0f * ip; +} + +void ExpectBitIdentical(float expected, float actual, const char *what, size_t dim, + bool normalized) { + EXPECT_EQ(std::memcmp(&expected, &actual, sizeof(float)), 0) + << what << " differs at dim " << dim << " normalized " << normalized << ": expected " + << expected << " actual " << actual; +} + +} // namespace + +TEST_F(SpacesTest, SQ8GenericScalarKernelsBitIdentical) { + for (size_t dim : {1, 2, 3, 4, 5, 16, 17, 63, 64, 65, 66, 767, 768}) { + for (bool normalized : {false, true}) { + const size_t storage_size = sq8::storage_bytes_count(dim); + std::vector storage1(storage_size); + std::vector storage2(storage_size); + std::vector query(dim + sq8::query_metadata_count()); + test_utils::populate_float_vec_to_sq8_with_metadata(storage1.data(), dim, normalized, + 1234); + test_utils::populate_float_vec_to_sq8_with_metadata(storage2.data(), dim, normalized, + 5678); + test_utils::populate_sq8_fp32_query(query.data(), dim, normalized, 4321); + const uint8_t *x = storage1.data(); + const uint8_t *y = storage2.data(); + const float *q = query.data(); + + const float fp32_ip = 1.0f - OriginalSQ8_FP32_InnerProduct_Impl(x, q, dim); + ExpectBitIdentical(fp32_ip, SQ8_FP32_InnerProduct(x, q, dim), "SQ8_FP32_InnerProduct", + dim, normalized); + ExpectBitIdentical(fp32_ip, 1.0f - quantized::FP32_InnerProduct_Impl(x, q, dim), + "quantized::FP32_InnerProduct_Impl", dim, normalized); + + const float fp32_l2 = OriginalSQ8_FP32_L2Sqr(x, q, dim); + ExpectBitIdentical(fp32_l2, SQ8_FP32_L2Sqr(x, q, dim), "SQ8_FP32_L2Sqr", dim, + normalized); + ExpectBitIdentical(fp32_l2, quantized::FP32_L2Sqr(x, q, dim), + "quantized::FP32_L2Sqr", dim, normalized); + + const float sq8_ip = 1.0f - OriginalSQ8_SQ8_InnerProduct_Impl(x, y, dim); + ExpectBitIdentical(sq8_ip, SQ8_SQ8_InnerProduct(x, y, dim), "SQ8_SQ8_InnerProduct", dim, + normalized); + ExpectBitIdentical(sq8_ip, 1.0f - quantized::Stored_InnerProduct_Impl(x, y, dim), + "quantized::Stored_InnerProduct_Impl", dim, normalized); + + const float sq8_l2 = OriginalSQ8_SQ8_L2Sqr(x, y, dim); + ExpectBitIdentical(sq8_l2, SQ8_SQ8_L2Sqr(x, y, dim), "SQ8_SQ8_L2Sqr", dim, normalized); + ExpectBitIdentical(sq8_l2, quantized::Stored_L2Sqr(x, y, dim), + "quantized::Stored_L2Sqr", dim, normalized); + } + } +} + /* ==================== MOD-17526 regression tests ==================== * The SQ8-FP32 L2 kernels used to compute L2^2 via the ||x||^2 + ||y||^2 - 2*IP identity, which * cancels catastrophically in FP32 when both vectors share a large common offset.