From e95a27ee4949f208d135075bb3078218127e5a30 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Wed, 4 Nov 2026 10:56:32 +0200 Subject: [PATCH 01/10] Generalize the scalar quantization pipeline over a StorageType No behavior change: SQ8 stays bit-for-bit identical. QuantPreprocessor and DistanceCalculatorWithNorm take a StorageType parameter that defaults to sq8, and sq8 gains the hooks the pipeline now goes through (max_code, data_bytes_count, load_code, store_code). CreateQuantizedIndexComponents holds the shared factory body and CreateSQ8IndexComponents wraps it. This prepares the pipeline for additional storage formats. PreprocessorsTest.QuantizedStorageSQ8BytesUnchanged checks SQ8 blobs against bytes derived with exact arithmetic, independent of the template. Co-Authored-By: Claude Opus 5.5 --- .../components/components_factory.h | 28 +++-- src/VecSim/spaces/computer/calculator.h | 35 +++--- src/VecSim/spaces/computer/preprocessors.h | 53 ++++---- src/VecSim/types/sq8.h | 10 +- tests/unit/test_components.cpp | 119 ++++++++++++++++++ 5 files changed, 191 insertions(+), 54 deletions(-) diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index 9b77b9b15..bca77eddf 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -40,19 +40,19 @@ template return alignment; } -template +template IndexComponents -CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, - const float *mean_ptr) { - using Preprocessor = QuantPreprocessor; +CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, + const float *mean_ptr) { + using Preprocessor = QuantPreprocessor; using QueryType = typename Preprocessor::QueryType; unsigned char storage_alignment = 0, asym_storage_alignment = 0; - // Graph construction compares two stored SQ8 blobs; search compares a stored blob with a + // Graph construction compares two stored quantized blobs; search compares a stored blob with a // QueryType query. Both dispatchers report alignment for the stored operand. - auto sym_func = spaces::GetDistFunc(Metric, dim, &storage_alignment); - auto asym_func = spaces::GetDistFunc( - Metric, dim, &asym_storage_alignment); + auto sym_func = spaces::GetDistFunc(Metric, dim, &storage_alignment); + auto asym_func = + spaces::GetDistFunc(Metric, dim, &asym_storage_alignment); storage_alignment = spaces::combineAlignments(storage_alignment, asym_storage_alignment); const unsigned char query_alignment = GetQueryAlignment(Metric, dim); @@ -69,7 +69,7 @@ CreateSQ8IndexComponents(const std::shared_ptr &allocator, size } pp = new (allocator) Preprocessor(allocator, dim, mean_vec); - calc = new (allocator) DistanceCalculatorWithNorm( + calc = new (allocator) DistanceCalculatorWithNorm( allocator, asym_func, sym_func, mean_sum_squares); } else { pp = new (allocator) Preprocessor(allocator, dim); @@ -79,11 +79,19 @@ CreateSQ8IndexComponents(const std::shared_ptr &allocator, size auto *container = new (allocator) MultiPreprocessorsContainer(allocator, query_alignment, storage_alignment); [[maybe_unused]] const int ret = container->addPreprocessor(pp); - assert(ret != -1 && "SQ8 preprocessor was not added correctly"); + assert(ret != -1 && "Quantization preprocessor was not added correctly"); return {calc, container}; } +template +IndexComponents +CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, + const float *mean_ptr) { + return CreateQuantizedIndexComponents( + allocator, dim, mean_ptr); +} + template IndexComponents CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, diff --git a/src/VecSim/spaces/computer/calculator.h b/src/VecSim/spaces/computer/calculator.h index 793402ad1..230042ab2 100644 --- a/src/VecSim/spaces/computer/calculator.h +++ b/src/VecSim/spaces/computer/calculator.h @@ -125,21 +125,21 @@ class DistanceCalculatorCommon }; /** - * Distance calculator for mean-normalized SQ8 indices. + * Distance calculator for mean-normalized quantized indices. * - * Stored vectors are SQ8-quantized from x' = x - mean. IP queries remain unshifted and use + * Stored vectors are quantized from x' = x - mean. IP queries remain unshifted and use * analytical correction terms derived from the per-vector x_mean_ip / y_mean_ip metadata. * L2 queries are centered once during preprocessing, allowing the base kernel to compute the * original distance directly without cancellation-prone correction terms. * * Correction formulas (expressed in terms of distance semantics): * - * Asymmetric (stored vector x as SQ8 of x'): + * Asymmetric (stored vector x as quantization of x'): * IP distance: dist(x,y) = asym_func(x_blob, y_blob) - y_mean_ip * L2 distance: dist(x,y) = asym_func(x_blob, centered_y_blob) * where centered_y = y - mean * - * Symmetric (both x and y as SQ8 blobs): + * Symmetric (both x and y as quantized blobs): * IP distance: dist(x,y) = sym_func(x_blob, y_blob) - x_mean_ip - y_mean_ip * + mean_sum_squares * L2 distance: dist(x,y) = sym_func(x_blob, y_blob) (mean terms cancel exactly) @@ -150,14 +150,13 @@ class DistanceCalculatorCommon * DataType is float or float16 * DistType is float */ -template +template class DistanceCalculatorWithNorm : public IndexCalculatorInterface { static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP, "DistanceCalculatorWithNorm only supports L2 and IP metrics"); private: - using sq8 = vecsim_types::sq8; - struct WithNormDistanceContext { spaces::dist_func_t stored_func; spaces::dist_func_t query_func; @@ -175,12 +174,12 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { // base = 1 - IP(x', y'). We want 1 - IP(x, y). // IP(x, y) = IP(x', y') + x_mean_ip + y_mean_ip - mean_sum_squares // => 1 - IP(x, y) = base - x_mean_ip - y_mean_ip + mean_sum_squares - float x_mean_ip = - load_unaligned(static_cast(v1) + dim + - sq8::template mean_ip_index() * sizeof(float)); - float y_mean_ip = - load_unaligned(static_cast(v2) + dim + - sq8::template mean_ip_index() * sizeof(float)); + float x_mean_ip = load_unaligned( + static_cast(v1) + StorageType::data_bytes_count(dim) + + StorageType::template mean_ip_index() * sizeof(float)); + float y_mean_ip = load_unaligned( + static_cast(v2) + StorageType::data_bytes_count(dim) + + StorageType::template mean_ip_index() * sizeof(float)); return base - x_mean_ip - y_mean_ip + context->mean_sum_squares; } else { // L2: ||x - y||² = ||x' - y'||² (mean terms cancel exactly) return base; @@ -192,9 +191,9 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { const auto *context = static_cast(opaque_context); DistType base = context->query_func(candidate, query, dim); if constexpr (Metric == VecSimMetric_IP) { - float y_mean_ip = - load_unaligned(static_cast(query) + dim * sizeof(DataType) + - sq8::template query_mean_ip_index() * sizeof(float)); + float y_mean_ip = load_unaligned( + static_cast(query) + dim * sizeof(DataType) + + StorageType::template query_mean_ip_index() * sizeof(float)); // base = 1 - IP(x', y). We want 1 - IP(x, y). // IP(x, y) = IP(x', y) + y_mean_ip // => 1 - IP(x, y) = base - y_mean_ip @@ -226,13 +225,13 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { } } - // Symmetric: both v1 and v2 are stored SQ8-of-x' blobs. + // Symmetric: both v1 and v2 are stored quantized-x' blobs. DistType calcDistance(const void *v1, const void *v2, size_t dim) const override { return calcStoredWithContext(&context_, v1, v2, dim); } // Asymmetric: IP queries are raw; L2 queries are centered during preprocessing. The candidate - // is a stored SQ8-of-x' blob. + // is a stored quantized-x' blob. // Note: query_dist_func expects (storage, query) argument order. DistType calcDistanceForQuery(const void *candidate, const void *query, size_t dim) const override { diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 1be9b06b8..262474dd5 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -143,13 +143,14 @@ class CosinePreprocessor : public PreprocessorInterface { /* * QuantPreprocessor is a preprocessor that quantizes storage vectors from DataType to a - * lower precision representation using OUTPUT_TYPE (uint8_t). + * lower precision representation defined by StorageType (SQ8, the current storage type, + * stores one uint8_t code per coordinate). * Query vectors remain as DataType for asymmetric distance computation. * * The quantized storage blob contains the quantized values along with metadata (min value, * scaling factor, and precomputed sums for reconstruction) in a single contiguous blob. * The quantization is done by finding the minimum and maximum values of the input vector, - * and then scaling the values to fit in the range of [0, 255]. + * and then scaling the values to fit in the storage type's code range. * * Storage layout: * | quantized_values[dim] | min_val | delta | x_sum | (x_sum_squares for L2 only) | @@ -164,8 +165,8 @@ class CosinePreprocessor : public PreprocessorInterface { * * Storage metadata is always FP32 (independent of DataType) to match the asymmetric distance * kernels. The quantized blob size is: - * - For L2: dim * sizeof(OUTPUT_TYPE) + 4 * sizeof(float) - * - For IP/Cosine: dim * sizeof(OUTPUT_TYPE) + 3 * sizeof(float) + * - For L2: StorageType::data_bytes_count(dim) + 4 * sizeof(float) + * - For IP/Cosine: StorageType::data_bytes_count(dim) + 3 * sizeof(float) * * Reconstruction formulas: * Given quantized value q_i, the original value is reconstructed as: @@ -256,7 +257,8 @@ static inline T from_fp32(float x) { } } -template +template class QuantPreprocessor : public PreprocessorInterface { public: // Center L2 queries in FP32 even when the original input is FP16. @@ -264,8 +266,7 @@ class QuantPreprocessor : public PreprocessorInterface { private: using OUTPUT_TYPE = uint8_t; - using MetadataType = float; // SQ8 metadata is always FP32 (see class doc). - using sq8 = vecsim_types::sq8; + using MetadataType = float; // Quantized storage metadata is always FP32. static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP || Metric == VecSimMetric_Cosine, @@ -283,7 +284,8 @@ class QuantPreprocessor : public PreprocessorInterface { // Calculate scaling factor (typed as MetadataType because they end up as metadata). const MetadataType diff = (max_val - min_val); - const MetadataType delta = (diff == 0.0f) ? MetadataType{1} : diff / MetadataType{255}; + const MetadataType delta = + (diff == 0.0f) ? MetadataType{1} : diff / MetadataType{StorageType::max_code}; const MetadataType inv_delta = MetadataType{1} / delta; // Saturating conversion. Casting a float to uint8_t is undefined when the truncated value @@ -299,8 +301,8 @@ class QuantPreprocessor : public PreprocessorInterface { if (!(scaled > MetadataType{0})) { return OUTPUT_TYPE{0}; // zero, negative, -inf, or NaN } - if (scaled >= MetadataType{255}) { - return OUTPUT_TYPE{255}; // includes +inf + if (scaled >= MetadataType{StorageType::max_code}) { + return OUTPUT_TYPE{StorageType::max_code}; // includes +inf } return static_cast(scaled + MetadataType{0.5}); }; @@ -337,10 +339,10 @@ class QuantPreprocessor : public PreprocessorInterface { const OUTPUT_TYPE a1 = to_byte((x1 - min_val) * inv_delta); const OUTPUT_TYPE a2 = to_byte((x2 - min_val) * inv_delta); const OUTPUT_TYPE a3 = to_byte((x3 - min_val) * inv_delta); - quantized[i] = a0; - quantized[i + 1] = a1; - quantized[i + 2] = a2; - quantized[i + 3] = a3; + StorageType::store_code(quantized, i, a0); + StorageType::store_code(quantized, i + 1, a1); + StorageType::store_code(quantized, i + 2, a2); + StorageType::store_code(quantized, i + 3, a3); s0 += a0; s1 += a1; @@ -365,7 +367,7 @@ class QuantPreprocessor : public PreprocessorInterface { for (; i < this->dim; ++i) { const float x = transformed_value(input, i); const OUTPUT_TYPE a = to_byte((x - min_val) * inv_delta); - quantized[i] = a; + StorageType::store_code(quantized, i, a); q_sum += a; if constexpr (Metric == VecSimMetric_L2) { q_sum_squares += uint64_t{a} * a; @@ -386,9 +388,9 @@ class QuantPreprocessor : public PreprocessorInterface { d_delta * d_delta * q_sum_squares); } - // Metadata uses MetadataType. Use memcpy because the metadata offset - // (dim * sizeof(uint8_t)) is not guaranteed to be sizeof(MetadataType)-aligned. - void *meta_dst = quantized + this->dim; + // Metadata uses MetadataType. Use memcpy because the metadata offset (after the + // quantized body) is not guaranteed to be sizeof(MetadataType)-aligned. + void *meta_dst = quantized + StorageType::data_bytes_count(this->dim); MetadataType buf[5] = {min_val, delta, sum}; size_t n = 3; if constexpr (Metric == VecSimMetric_L2) @@ -490,26 +492,29 @@ class QuantPreprocessor : public PreprocessorInterface { QuantPreprocessor(std::shared_ptr allocator, size_t dim) requires(!WithNorm) : PreprocessorInterface(allocator), dim(dim), - storage_bytes_count(sq8::storage_bytes_count(dim)), + storage_bytes_count(StorageType::template storage_bytes_count(dim)), query_bytes_count(dim * sizeof(QueryType) + - sq8::query_metadata_count() * sizeof(MetadataType)) {} + StorageType::template query_metadata_count() * + sizeof(MetadataType)) {} // WithNorm constructor: accepts a pre-computed mean vector (FP32, length == dim). QuantPreprocessor(std::shared_ptr allocator, size_t dim, const vecsim_stl::vector &mean_vec) requires(WithNorm) : PreprocessorInterface(allocator), mean(mean_vec), dim(dim), - storage_bytes_count(sq8::storage_bytes_count(dim)), + storage_bytes_count(StorageType::template storage_bytes_count(dim)), query_bytes_count(dim * sizeof(QueryType) + - sq8::query_metadata_count() * sizeof(MetadataType)) { + StorageType::template query_metadata_count() * + sizeof(MetadataType)) { assert(this->mean.size() == dim && "mean vector size must equal dim"); } /** * Preprocesses the original blob into separate storage and query blobs. * - * Storage vectors are quantized to uint8_t values, with metadata (min, delta, sum, and - * sum_squares for L2) appended for distance reconstruction. + * Storage vectors are quantized to StorageType scalar codes (SQ8: one uint8_t per + * coordinate), with metadata (min, delta, sum, and sum_squares for L2) appended for + * distance reconstruction. * * Query vectors use QueryType for asymmetric distance computation, with a precomputed * sum (for IP/Cosine) or sum of squares (for L2) appended for efficient distance calculation. diff --git a/src/VecSim/types/sq8.h b/src/VecSim/types/sq8.h index ac2446807..f40e37cca 100644 --- a/src/VecSim/types/sq8.h +++ b/src/VecSim/types/sq8.h @@ -17,6 +17,13 @@ namespace vecsim_types { // Represents a scalar-quantized 8-bit blob with reconstruction metadata struct sq8 { using value_type = uint8_t; + static constexpr uint8_t max_code = 255; + + static constexpr size_t data_bytes_count(size_t dim) { return dim; } + + static uint8_t load_code(const uint8_t *data, size_t index) { return data[index]; } + + static void store_code(uint8_t *data, size_t index, uint8_t code) { data[index] = code; } // Metadata layout indices (stored after quantized values) enum MetadataIndex : size_t { @@ -50,8 +57,7 @@ struct sq8 { // Stored layout: one quantized byte per dimension, followed by metric-specific FP32 metadata. template static constexpr size_t storage_bytes_count(size_t dim) { - return dim * sizeof(value_type) + - storage_metadata_count() * sizeof(float); + return data_bytes_count(dim) + storage_metadata_count() * sizeof(float); } // Index of x_mean_ip / y_mean_ip in the last slot in metadata array diff --git a/tests/unit/test_components.cpp b/tests/unit/test_components.cpp index 375ee52b1..6744756b3 100644 --- a/tests/unit/test_components.cpp +++ b/tests/unit/test_components.cpp @@ -10,12 +10,16 @@ #include #include #include +#include +#include +#include #include "gtest/gtest.h" #include "VecSim/vec_sim.h" #include "VecSim/quantization/sq8_training_state.h" #include "VecSim/spaces/computer/preprocessor_container.h" #include "VecSim/spaces/computer/preprocessors.h" #include "VecSim/spaces/computer/calculator.h" +#include "VecSim/index_factories/components/components_factory.h" #include "VecSim/spaces/IP_space.h" #include "VecSim/spaces/L2_space.h" #include "unit_test_utils.h" @@ -1443,6 +1447,121 @@ TEST(PreprocessorsTest, QuantizationTestAllEntriesEqual) { delete quant_preprocessor; } +namespace { + +template +DataType fromFloat(float value) { + if constexpr (std::is_same_v) { + return vecsim_types::FP32_to_FP16(value); + } else { + return value; + } +} + +// SQ8 byte-identity guard. Every input lies on a 0.25 grid spanning exactly 255 steps, so delta +// is exactly 0.25, every code is an exact integer and every metadata value is exactly +// representable. The expected blob is therefore derived below with exact arithmetic, without +// calling QuantPreprocessor, and any change to the SQ8 encoding shows up as a byte mismatch. +template +void checkSQ8Bytes(size_t dim) { + constexpr bool kFP16 = std::is_same_v; + SCOPED_TRACE(::testing::Message() << "fp16=" << kFP16 << " metric=" << Metric + << " mean=" << WithMean << " dim=" << dim); + constexpr double kMin = -8.0, kStep = 0.25; + auto allocator = VecSimAllocator::newVecsimAllocator(); + std::vector codes(dim); + std::vector mean(dim, 0.0f), input_fp32(dim); + std::vector input(dim); + for (size_t i = 0; i < dim; ++i) { + // Codes 0 and 255 pin the range; the rest exercise every byte position. + codes[i] = i == 0 ? 0 : i == 1 ? 255 : uint8_t((i * 37 + 11) % 256); + if constexpr (WithMean) { + mean[i] = 0.5f * (int(i % 5) - 2); + } + input_fp32[i] = float(kMin + kStep * codes[i] + mean[i]); + input[i] = fromFloat(input_fp32[i]); + } + + double sum = 0, sum_squares = 0, mean_ip = 0; + for (size_t i = 0; i < dim; ++i) { + const double reconstructed = kMin + kStep * codes[i]; + sum += reconstructed; + sum_squares += reconstructed * reconstructed; + mean_ip += double(input_fp32[i]) * mean[i]; + } + std::vector expected_metadata = {float(kMin), float(kStep), float(sum)}; + if constexpr (Metric == VecSimMetric_L2) { + expected_metadata.push_back(float(sum_squares)); + } + if constexpr (WithMean && Metric == VecSimMetric_IP) { + expected_metadata.push_back(float(mean_ip)); + } + std::vector expected(codes); + expected.resize(dim + expected_metadata.size() * sizeof(float)); + std::memcpy(expected.data() + dim, expected_metadata.data(), + expected_metadata.size() * sizeof(float)); + ASSERT_EQ(expected.size(), (sq8::storage_bytes_count(dim))); + + auto encode = [&](const QuantPreprocessor &preprocessor) { + void *storage = nullptr; + size_t size = dim * sizeof(DataType); + preprocessor.preprocessForStorage(input.data(), storage, size, 0); + const auto *bytes = static_cast(storage); + std::vector result(bytes, bytes + size); + allocator->free_allocation(storage); + return result; + }; + if constexpr (WithMean) { + vecsim_stl::vector mean_vec(allocator); + mean_vec.assign(mean.begin(), mean.end()); + EXPECT_EQ(encode(QuantPreprocessor(allocator, dim, mean_vec)), + expected); + } else { + EXPECT_EQ(encode(QuantPreprocessor(allocator, dim)), expected); + } + + // The SQ8 factory wrapper must produce the same bytes and keep naming the SQ8 component + // types that applySQ8Mean dynamic_casts to. + if constexpr (std::is_same_v && Metric != VecSimMetric_Cosine) { + auto components = CreateSQ8IndexComponents(allocator, dim, + WithMean ? mean.data() : nullptr); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + auto *container = + dynamic_cast *>(preprocessors.get()); + ASSERT_NE(container, nullptr); + EXPECT_NE((dynamic_cast *>( + container->getPreprocessors()[0])), + nullptr); + if constexpr (WithMean) { + EXPECT_NE((dynamic_cast *>( + calculator.get())), + nullptr); + } + auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); + const auto *factory_bytes = static_cast(blob.get()); + EXPECT_EQ(std::vector(factory_bytes, factory_bytes + expected.size()), expected); + } +} + +template +void checkSQ8BytesAllModes() { + for (size_t dim : {2u, 5u, 37u, 128u}) { + checkSQ8Bytes(dim); + checkSQ8Bytes(dim); + checkSQ8Bytes(dim); + checkSQ8Bytes(dim); + checkSQ8Bytes(dim); + } +} + +} // namespace + +TEST(PreprocessorsTest, QuantizedStorageSQ8BytesUnchanged) { + checkSQ8BytesAllModes(); + checkSQ8BytesAllModes(); +} + // Parameterized test class for QuantPreprocessor with different metrics class QuantPreprocessorMetricTest : public testing::TestWithParam { protected: From 07c2e9573f0c48d088c0b7e54eaea74dae288408 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Wed, 4 Nov 2026 11:41:54 +0200 Subject: [PATCH 02/10] Make the SQ8 scalar kernels generic over StorageType Move the scalar SQ8-FP32 and SQ8-SQ8 inner product and L2 kernel bodies into spaces::quantized templates in quantized_kernels.h. Codes are read through StorageType::load_code and metadata is located with data_bytes_count, so a future storage type can reuse them. The SQ8 functions become thin wrappers. A new test pins bit identity against verbatim copies of the original kernels. Co-Authored-By: Claude Opus 5.5 --- src/VecSim/spaces/IP/IP.cpp | 62 +--------- src/VecSim/spaces/L2/L2.cpp | 37 +----- src/VecSim/spaces/quantized_kernels.h | 167 ++++++++++++++++++++++++++ tests/unit/test_spaces.cpp | 166 +++++++++++++++++++++++++ 4 files changed, 339 insertions(+), 93 deletions(-) create mode 100644 src/VecSim/spaces/quantized_kernels.h diff --git a/src/VecSim/spaces/IP/IP.cpp b/src/VecSim/spaces/IP/IP.cpp index 8cefa1c8f..136617c62 100644 --- a/src/VecSim/spaces/IP/IP.cpp +++ b/src/VecSim/spaces/IP/IP.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "IP.h" +#include "VecSim/spaces/quantized_kernels.h" #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" @@ -32,40 +33,7 @@ using sq8 = vecsim_types::sq8; * SQ8_FP32_L2Sqr. */ float SQ8_FP32_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, size_t dimension) { - const auto *pVect1 = static_cast(pVect1v); - const auto *pVect2 = static_cast(pVect2v); - - // Use 4 accumulators for instruction-level parallelism - float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; - - // Main loop: process 4 elements per iteration - size_t i = 0; - size_t dim4 = dimension & ~size_t(3); // dim4 is a multiple of 4 - for (; i < dim4; i += 4) { - sum0 += static_cast(pVect1[i + 0]) * pVect2[i + 0]; - sum1 += static_cast(pVect1[i + 1]) * pVect2[i + 1]; - sum2 += static_cast(pVect1[i + 2]) * pVect2[i + 2]; - sum3 += static_cast(pVect1[i + 3]) * pVect2[i + 3]; - } - - // Handle remainder (0-3 elements) - for (; i < dimension; i++) { - sum0 += static_cast(pVect1[i]) * pVect2[i]; - } - - // Combine accumulators - float quantized_dot = (sum0 + sum1) + (sum2 + sum3); - - // Storage metadata follows a byte payload and is not necessarily float-aligned. - const auto *params = pVect1 + dimension; - const float min_val = load_unaligned(params + sq8::MIN_VAL * sizeof(float)); - const float delta = load_unaligned(params + sq8::DELTA * sizeof(float)); - - // Get precomputed y_sum from query blob (pVect2 is FP32, stored after the dim floats) - const float y_sum = pVect2[dimension + sq8::SUM_QUERY]; - - // Apply formula: IP = min * y_sum + delta * Σ(q_i * y_i) - return min_val * y_sum + delta * quantized_dot; + return spaces::quantized::FP32_InnerProduct_Impl(pVect1v, pVect2v, dimension); } float SQ8_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) { @@ -143,31 +111,7 @@ float SQ8_FP16_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension // (not distance). Used by both SQ8_SQ8_InnerProduct, SQ8_SQ8_Cosine, and SQ8_SQ8_L2Sqr. // Vector layout: [uint8_t values (dim)] [min_val (float)] [delta (float)] [sum (float)] float SQ8_SQ8_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, size_t dimension) { - const auto *pVect1 = static_cast(pVect1v); - const auto *pVect2 = static_cast(pVect2v); - - // Compute inner product of quantized values: Σ(q1[i]*q2[i]) - float product = 0; - for (size_t i = 0; i < dimension; i++) { - product += pVect1[i] * pVect2[i]; - } - - // Metadata follows byte payloads and is not necessarily float-aligned. - const auto *params1 = pVect1 + dimension; - const float min_val1 = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); - const float delta1 = load_unaligned(params1 + sq8::DELTA * sizeof(float)); - const float sum1 = load_unaligned(params1 + sq8::SUM * sizeof(float)); - - // Get quantization parameters from pVect2 - const auto *params2 = pVect2 + dimension; - const float min_val2 = load_unaligned(params2 + sq8::MIN_VAL * sizeof(float)); - const float delta2 = load_unaligned(params2 + sq8::DELTA * sizeof(float)); - const float sum2 = load_unaligned(params2 + sq8::SUM * sizeof(float)); - - // Apply the algebraic formula using precomputed sums: - // IP = min1*sum2 + min2*sum1 + delta1*delta2*Σ(q1[i]*q2[i]) - dim*min1*min2 - return min_val1 * sum2 + min_val2 * sum1 - static_cast(dimension) * min_val1 * min_val2 + - delta1 * delta2 * product; + return spaces::quantized::Stored_InnerProduct_Impl(pVect1v, pVect2v, dimension); } // SQ8-to-SQ8: Both vectors are uint8 quantized with precomputed sum diff --git a/src/VecSim/spaces/L2/L2.cpp b/src/VecSim/spaces/L2/L2.cpp index da0c25971..2bc7f2cfb 100644 --- a/src/VecSim/spaces/L2/L2.cpp +++ b/src/VecSim/spaces/L2/L2.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "L2.h" +#include "VecSim/spaces/quantized_kernels.h" #include "VecSim/spaces/IP/IP.h" #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" @@ -34,25 +35,7 @@ using sq8 = vecsim_types::sq8; * pVect2 is query (FP32): [float values (dim)] [y_sum] [y_sum_squares] */ float SQ8_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { - // Storage metadata follows a byte payload and is not necessarily float-aligned. - const auto *pVect1 = static_cast(pVect1v); - const auto *pVect2 = static_cast(pVect2v); - - const auto *params1 = pVect1 + dimension; - const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); - const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); - - float res = 0; - for (size_t i = 0; i < dimension; i++) { - // Order matters. When the stored range sits far from zero, min_val and y_i are within a - // factor of two of each other, so min_val - y_i is exact (Sterbenz) and rounding happens - // only at the scale of the residual. Forming (min_val + delta*q_i) first would round at - // the scale of min_val and wipe out small residuals. When min_val and y_i are not close, - // the residual is large and either order is fine. - float diff = delta * static_cast(pVect1[i]) + (min_val - pVect2[i]); - res += diff * diff; - } - return res; + return spaces::quantized::FP32_L2Sqr(pVect1v, pVect2v, dimension); } /* @@ -194,19 +177,5 @@ float UINT8_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { // - IP(x, y) is computed using SQ8_SQ8_InnerProduct_Impl float SQ8_SQ8_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { - const auto *pVect1 = static_cast(pVect1v); - const auto *pVect2 = static_cast(pVect2v); - - // Get precomputed sum of squares from both vectors - // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] - const float sum_sq_1 = - load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); - const float sum_sq_2 = - load_unaligned(pVect2 + dimension + sq8::SUM_SQUARES * sizeof(float)); - - // Use the common inner product implementation - const float ip = SQ8_SQ8_InnerProduct_Impl(pVect1v, pVect2v, dimension); - - // L2² = ||x||² + ||y||² - 2*IP(x, y) - return sum_sq_1 + sum_sq_2 - 2.0f * ip; + return spaces::quantized::Stored_L2Sqr(pVect1v, pVect2v, dimension); } diff --git a/src/VecSim/spaces/quantized_kernels.h b/src/VecSim/spaces/quantized_kernels.h new file mode 100644 index 000000000..c6770f0fa --- /dev/null +++ b/src/VecSim/spaces/quantized_kernels.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include +#include "VecSim/utils/alignment.h" + +// SIMD translation units (functions/*.cpp) must not include this header; they call the exported +// scalar functions (e.g. SQ8_FP32_L2Sqr) instead. +// +// Scalar kernels for scalar quantized storage, generic over a StorageType that provides +// load_code(data, i), data_bytes_count(dim) and the MIN_VAL / DELTA / SUM / SUM_SQUARES metadata +// indices (stored after the codes) and SUM_QUERY (stored after the FP32 query values). +// +// Stored layout: [codes (data_bytes_count(dim) bytes)] [min_val] [delta] [sum] [sum_squares (L2)] +// FP32 query layout: [float values (dim)] [y_sum] [y_sum_squares (L2 only)] +// +// The operand order in these bodies is load-bearing and relies on FP addition not being +// reassociated; they must not be built with -ffast-math / -Ofast. +namespace spaces { +namespace quantized { + +/* + * Asymmetric stored-FP32 inner product using the algebraic identity: + * IP(x, y) = sum((min + delta * q_i) * y_i) = min * y_sum + delta * sum(q_i * y_i) + * + * Uses 4x loop unrolling with multiple accumulators for ILP. + * Returns the raw inner product value (not distance). + */ +template +static inline float FP32_InnerProduct_Impl(const void *stored, const void *query, size_t dim) { + const auto *pVect1 = static_cast(stored); + const auto *pVect2 = static_cast(query); + + // Use 4 accumulators for instruction-level parallelism + float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + + // Main loop: process 4 elements per iteration + size_t i = 0; + size_t dim4 = dim & ~size_t(3); // dim4 is a multiple of 4 + for (; i < dim4; i += 4) { + sum0 += static_cast(StorageType::load_code(pVect1, i + 0)) * pVect2[i + 0]; + sum1 += static_cast(StorageType::load_code(pVect1, i + 1)) * pVect2[i + 1]; + sum2 += static_cast(StorageType::load_code(pVect1, i + 2)) * pVect2[i + 2]; + sum3 += static_cast(StorageType::load_code(pVect1, i + 3)) * pVect2[i + 3]; + } + + // Handle remainder (0-3 elements) + for (; i < dim; i++) { + sum0 += static_cast(StorageType::load_code(pVect1, i)) * pVect2[i]; + } + + // Combine accumulators + float quantized_dot = (sum0 + sum1) + (sum2 + sum3); + + // Storage metadata follows a byte payload and is not necessarily float-aligned. + const auto *params = pVect1 + StorageType::data_bytes_count(dim); + const float min_val = load_unaligned(params + StorageType::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params + StorageType::DELTA * sizeof(float)); + + // Get precomputed y_sum from query blob (stored after the dim floats) + const float y_sum = pVect2[dim + StorageType::SUM_QUERY]; + + // Apply formula: IP = min * y_sum + delta * sum(q_i * y_i) + return min_val * y_sum + delta * quantized_dot; +} + +/* + * Asymmetric stored-FP32 L2 squared distance, accumulated directly per component: + * ||x - y||^2 = sum((dequant(x_i) - y_i)^2), where dequant(x_i) = min_val + delta * q_i + * + * Accumulating residuals keeps each component's rounding at the scale of |x_i - y_i|. Expanding + * to ||x||^2 + ||y||^2 - 2*IP(x, y) would round at the scale of the norms, which loses the + * distance when x and y share a large common offset relative to their spread. + */ +template +static inline float FP32_L2Sqr(const void *stored, const void *query, size_t dim) { + // Storage metadata follows a byte payload and is not necessarily float-aligned. + const auto *pVect1 = static_cast(stored); + const auto *pVect2 = static_cast(query); + + const auto *params1 = pVect1 + StorageType::data_bytes_count(dim); + const float min_val = load_unaligned(params1 + StorageType::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + StorageType::DELTA * sizeof(float)); + + float res = 0; + for (size_t i = 0; i < dim; i++) { + // Order matters. When the stored range sits far from zero, min_val and y_i are within a + // factor of two of each other, so min_val - y_i is exact (Sterbenz) and rounding happens + // only at the scale of the residual. Forming (min_val + delta*q_i) first would round at + // the scale of min_val and wipe out small residuals. When min_val and y_i are not close, + // the residual is large and either order is fine. + float diff = + delta * static_cast(StorageType::load_code(pVect1, i)) + (min_val - pVect2[i]); + res += diff * diff; + } + return res; +} + +/* + * Symmetric stored-stored inner product using precomputed sums: + * IP = min1*sum2 + min2*sum1 - dim*min1*min2 + delta1*delta2*sum(q1_i*q2_i) + * Returns the raw inner product value (not distance). + */ +template +static inline float Stored_InnerProduct_Impl(const void *a, const void *b, size_t dim) { + const auto *pVect1 = static_cast(a); + const auto *pVect2 = static_cast(b); + + // Compute inner product of quantized values: sum(q1[i]*q2[i]) + float product = 0; + for (size_t i = 0; i < dim; i++) { + product += StorageType::load_code(pVect1, i) * StorageType::load_code(pVect2, i); + } + + // Metadata follows byte payloads and is not necessarily float-aligned. + const auto *params1 = pVect1 + StorageType::data_bytes_count(dim); + const float min_val1 = load_unaligned(params1 + StorageType::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(params1 + StorageType::DELTA * sizeof(float)); + const float sum1 = load_unaligned(params1 + StorageType::SUM * sizeof(float)); + + // Get quantization parameters from pVect2 + const auto *params2 = pVect2 + StorageType::data_bytes_count(dim); + const float min_val2 = load_unaligned(params2 + StorageType::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(params2 + StorageType::DELTA * sizeof(float)); + const float sum2 = load_unaligned(params2 + StorageType::SUM * sizeof(float)); + + // Apply the algebraic formula using precomputed sums: + // IP = min1*sum2 + min2*sum1 + delta1*delta2*sum(q1[i]*q2[i]) - dim*min1*min2 + return min_val1 * sum2 + min_val2 * sum1 - static_cast(dim) * min_val1 * min_val2 + + delta1 * delta2 * product; +} + +/* + * Symmetric stored-stored L2 squared distance: + * ||x - y||^2 = ||x||^2 + ||y||^2 - 2*IP(x, y) + * where ||x||^2 and ||y||^2 are the precomputed SUM_SQUARES metadata and IP(x, y) is computed by + * Stored_InnerProduct_Impl. + */ +template +static inline float Stored_L2Sqr(const void *a, const void *b, size_t dim) { + const auto *pVect1 = static_cast(a); + const auto *pVect2 = static_cast(b); + + // Get precomputed sum of squares from both vectors + const size_t data_bytes = StorageType::data_bytes_count(dim); + const float sum_sq_1 = + load_unaligned(pVect1 + data_bytes + StorageType::SUM_SQUARES * sizeof(float)); + const float sum_sq_2 = + load_unaligned(pVect2 + data_bytes + StorageType::SUM_SQUARES * sizeof(float)); + + // Use the common inner product implementation + const float ip = Stored_InnerProduct_Impl(a, b, dim); + + // L2^2 = ||x||^2 + ||y||^2 - 2*IP(x, y) + return sum_sq_1 + sum_sq_2 - 2.0f * ip; +} + +} // namespace quantized +} // namespace spaces diff --git a/tests/unit/test_spaces.cpp b/tests/unit/test_spaces.cpp index 923bb7c1b..69d6c84d9 100644 --- a/tests/unit/test_spaces.cpp +++ b/tests/unit/test_spaces.cpp @@ -27,6 +27,7 @@ #include "VecSim/spaces/L2_space.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/spaces/quantized_kernels.h" #include "VecSim/spaces/functions/AVX512F.h" #include "VecSim/spaces/functions/AVX.h" #include "VecSim/spaces/functions/SSE.h" @@ -411,6 +412,171 @@ TEST_F(SpacesTest, SQ8_FP32_odd_dim_unaligned_metadata_test) { } } +/* ============ Generic scalar quantized kernels: SQ8 bit-identity ============ + * Verbatim copies of the four scalar SQ8 kernel bodies as they were on origin/main before they + * became wrappers over spaces::quantized. Only the function names differ. They pin the original + * results so the test does not compare the wrappers against the templates they call. + */ +namespace { + +float OriginalSQ8_FP32_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, + size_t dimension) { + const auto *pVect1 = static_cast(pVect1v); + const auto *pVect2 = static_cast(pVect2v); + + // Use 4 accumulators for instruction-level parallelism + float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + + // Main loop: process 4 elements per iteration + size_t i = 0; + size_t dim4 = dimension & ~size_t(3); // dim4 is a multiple of 4 + for (; i < dim4; i += 4) { + sum0 += static_cast(pVect1[i + 0]) * pVect2[i + 0]; + sum1 += static_cast(pVect1[i + 1]) * pVect2[i + 1]; + sum2 += static_cast(pVect1[i + 2]) * pVect2[i + 2]; + sum3 += static_cast(pVect1[i + 3]) * pVect2[i + 3]; + } + + // Handle remainder (0-3 elements) + for (; i < dimension; i++) { + sum0 += static_cast(pVect1[i]) * pVect2[i]; + } + + // Combine accumulators + float quantized_dot = (sum0 + sum1) + (sum2 + sum3); + + // Storage metadata follows a byte payload and is not necessarily float-aligned. + const auto *params = pVect1 + dimension; + const float min_val = load_unaligned(params + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params + sq8::DELTA * sizeof(float)); + + // Get precomputed y_sum from query blob (pVect2 is FP32, stored after the dim floats) + const float y_sum = pVect2[dimension + sq8::SUM_QUERY]; + + // Apply formula: IP = min * y_sum + delta * Σ(q_i * y_i) + return min_val * y_sum + delta * quantized_dot; +} + +float OriginalSQ8_SQ8_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, + size_t dimension) { + const auto *pVect1 = static_cast(pVect1v); + const auto *pVect2 = static_cast(pVect2v); + + // Compute inner product of quantized values: Σ(q1[i]*q2[i]) + float product = 0; + for (size_t i = 0; i < dimension; i++) { + product += pVect1[i] * pVect2[i]; + } + + // Metadata follows byte payloads and is not necessarily float-aligned. + const auto *params1 = pVect1 + dimension; + const float min_val1 = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta1 = load_unaligned(params1 + sq8::DELTA * sizeof(float)); + const float sum1 = load_unaligned(params1 + sq8::SUM * sizeof(float)); + + // Get quantization parameters from pVect2 + const auto *params2 = pVect2 + dimension; + const float min_val2 = load_unaligned(params2 + sq8::MIN_VAL * sizeof(float)); + const float delta2 = load_unaligned(params2 + sq8::DELTA * sizeof(float)); + const float sum2 = load_unaligned(params2 + sq8::SUM * sizeof(float)); + + // Apply the algebraic formula using precomputed sums: + // IP = min1*sum2 + min2*sum1 + delta1*delta2*Σ(q1[i]*q2[i]) - dim*min1*min2 + return min_val1 * sum2 + min_val2 * sum1 - static_cast(dimension) * min_val1 * min_val2 + + delta1 * delta2 * product; +} + +float OriginalSQ8_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { + // Storage metadata follows a byte payload and is not necessarily float-aligned. + const auto *pVect1 = static_cast(pVect1v); + const auto *pVect2 = static_cast(pVect2v); + + const auto *params1 = pVect1 + dimension; + const float min_val = load_unaligned(params1 + sq8::MIN_VAL * sizeof(float)); + const float delta = load_unaligned(params1 + sq8::DELTA * sizeof(float)); + + float res = 0; + for (size_t i = 0; i < dimension; i++) { + // Order matters. When the stored range sits far from zero, min_val and y_i are within a + // factor of two of each other, so min_val - y_i is exact (Sterbenz) and rounding happens + // only at the scale of the residual. Forming (min_val + delta*q_i) first would round at + // the scale of min_val and wipe out small residuals. When min_val and y_i are not close, + // the residual is large and either order is fine. + float diff = delta * static_cast(pVect1[i]) + (min_val - pVect2[i]); + res += diff * diff; + } + return res; +} + +float OriginalSQ8_SQ8_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) { + const auto *pVect1 = static_cast(pVect1v); + const auto *pVect2 = static_cast(pVect2v); + + // Get precomputed sum of squares from both vectors + // Layout: [uint8_t values (dim)] [min_val] [delta] [sum] [sum_of_squares] + const float sum_sq_1 = + load_unaligned(pVect1 + dimension + sq8::SUM_SQUARES * sizeof(float)); + const float sum_sq_2 = + load_unaligned(pVect2 + dimension + sq8::SUM_SQUARES * sizeof(float)); + + // Use the common inner product implementation + const float ip = OriginalSQ8_SQ8_InnerProduct_Impl(pVect1v, pVect2v, dimension); + + // L2² = ||x||² + ||y||² - 2*IP(x, y) + return sum_sq_1 + sum_sq_2 - 2.0f * ip; +} + +void ExpectBitIdentical(float expected, float actual, const char *what, size_t dim, + bool normalized) { + EXPECT_EQ(std::memcmp(&expected, &actual, sizeof(float)), 0) + << what << " differs at dim " << dim << " normalized " << normalized << ": expected " + << expected << " actual " << actual; +} + +} // namespace + +TEST_F(SpacesTest, SQ8GenericScalarKernelsBitIdentical) { + for (size_t dim : {1, 2, 3, 4, 5, 16, 17, 63, 64, 65, 66, 767, 768}) { + for (bool normalized : {false, true}) { + const size_t storage_size = sq8::storage_bytes_count(dim); + std::vector storage1(storage_size); + std::vector storage2(storage_size); + std::vector query(dim + sq8::query_metadata_count()); + test_utils::populate_float_vec_to_sq8_with_metadata(storage1.data(), dim, normalized, + 1234); + test_utils::populate_float_vec_to_sq8_with_metadata(storage2.data(), dim, normalized, + 5678); + test_utils::populate_sq8_fp32_query(query.data(), dim, normalized, 4321); + const uint8_t *x = storage1.data(); + const uint8_t *y = storage2.data(); + const float *q = query.data(); + + const float fp32_ip = 1.0f - OriginalSQ8_FP32_InnerProduct_Impl(x, q, dim); + ExpectBitIdentical(fp32_ip, SQ8_FP32_InnerProduct(x, q, dim), "SQ8_FP32_InnerProduct", + dim, normalized); + ExpectBitIdentical(fp32_ip, 1.0f - quantized::FP32_InnerProduct_Impl(x, q, dim), + "quantized::FP32_InnerProduct_Impl", dim, normalized); + + const float fp32_l2 = OriginalSQ8_FP32_L2Sqr(x, q, dim); + ExpectBitIdentical(fp32_l2, SQ8_FP32_L2Sqr(x, q, dim), "SQ8_FP32_L2Sqr", dim, + normalized); + ExpectBitIdentical(fp32_l2, quantized::FP32_L2Sqr(x, q, dim), + "quantized::FP32_L2Sqr", dim, normalized); + + const float sq8_ip = 1.0f - OriginalSQ8_SQ8_InnerProduct_Impl(x, y, dim); + ExpectBitIdentical(sq8_ip, SQ8_SQ8_InnerProduct(x, y, dim), "SQ8_SQ8_InnerProduct", dim, + normalized); + ExpectBitIdentical(sq8_ip, 1.0f - quantized::Stored_InnerProduct_Impl(x, y, dim), + "quantized::Stored_InnerProduct_Impl", dim, normalized); + + const float sq8_l2 = OriginalSQ8_SQ8_L2Sqr(x, y, dim); + ExpectBitIdentical(sq8_l2, SQ8_SQ8_L2Sqr(x, y, dim), "SQ8_SQ8_L2Sqr", dim, normalized); + ExpectBitIdentical(sq8_l2, quantized::Stored_L2Sqr(x, y, dim), + "quantized::Stored_L2Sqr", dim, normalized); + } + } +} + /* ==================== MOD-17526 regression tests ==================== * The SQ8-FP32 L2 kernels used to compute L2^2 via the ||x||^2 + ||y||^2 - 2*IP identity, which * cancels catastrophically in FP32 when both vectors share a large common offset. From 15c0fa6c4e15a37366a5c3b02c4c944a071e5ea7 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Thu, 5 Nov 2026 18:39:20 +0200 Subject: [PATCH 03/10] Use numeric_limits for the SQ8 max code Co-Authored-By: Claude Opus 5.5 --- src/VecSim/types/sq8.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/VecSim/types/sq8.h b/src/VecSim/types/sq8.h index f40e37cca..38fe1e7c7 100644 --- a/src/VecSim/types/sq8.h +++ b/src/VecSim/types/sq8.h @@ -10,6 +10,7 @@ #include #include +#include #include "VecSim/vec_sim_common.h" namespace vecsim_types { @@ -17,7 +18,7 @@ namespace vecsim_types { // Represents a scalar-quantized 8-bit blob with reconstruction metadata struct sq8 { using value_type = uint8_t; - static constexpr uint8_t max_code = 255; + static constexpr uint8_t max_code = std::numeric_limits::max(); static constexpr size_t data_bytes_count(size_t dim) { return dim; } From 2c95db85abb338bd008be12d6739c3f7c0a0c828 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 09:32:40 +0200 Subject: [PATCH 04/10] Generalize the HNSW quantization factory helpers over a StorageType Renames the SQ8-only factory helpers (stored size, metric resolution, parameter validation, index construction) to quantization-generic names and gives the stored-size helper a StorageType parameter (default sq8). The unknown-quantizer check moves into the shared validation. SQ8 is still the only accepted quantizer, so behavior is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/VecSim/index_factories/hnsw_factory.cpp | 70 +++++++++++---------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index 2e553146f..dd271d9f3 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -37,26 +37,33 @@ NewIndex_ChooseMultiOrSingle(const HNSWParams *params, HNSWIndex_Single(params, abstractInitParams, components); } -template -[[nodiscard]] constexpr size_t GetSQ8StoredDataSize(size_t dim, bool with_norm) { +template +[[nodiscard]] constexpr size_t GetQuantizedStoredDataSize(size_t dim, bool with_norm) { static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP); - return with_norm ? sq8::storage_bytes_count(dim) - : sq8::storage_bytes_count(dim); + return with_norm ? StorageType::template storage_bytes_count(dim) + : StorageType::template storage_bytes_count(dim); } // Cosine over pre-normalized vectors is computed as inner product. -[[nodiscard]] constexpr VecSimMetric ResolveSQ8Metric(VecSimMetric metric, bool is_normalized) { +[[nodiscard]] constexpr VecSimMetric ResolveQuantizedMetric(VecSimMetric metric, + bool is_normalized) { return (is_normalized && metric == VecSimMetric_Cosine) ? VecSimMetric_IP : metric; } // Keep construction and initial-size validation in sync. -[[nodiscard]] constexpr bool SQ8ParamsSupported(VecSimType type, VecSimMetric resolved_metric) { +[[nodiscard]] constexpr bool QuantizationParamsSupported(VecSimQuantType quant_type, + VecSimType type, + VecSimMetric resolved_metric) { + // Reject unknown quantizers instead of silently creating an unquantized index. + if (quant_type != VecSimQuant_SQ8) { + return false; + } // SQ8 kernels accept only FLOAT32 and FLOAT16 input. if (type != VecSimType_FLOAT32 && type != VecSimType_FLOAT16) { return false; } - // Only L2 and inner product have SQ8 kernels. + // Only L2 and inner product have scalar quantization kernels. if (resolved_metric != VecSimMetric_L2 && resolved_metric != VecSimMetric_IP) { return false; } @@ -64,11 +71,11 @@ template } template -VecSimIndex *NewIndex_SQ8(const HNSWParams *hnswParams, AbstractIndexInitParams abstractInitParams, - const float *mean_ptr) { +VecSimIndex *NewIndex_Quantized(const HNSWParams *hnswParams, + AbstractIndexInitParams abstractInitParams, const float *mean_ptr) { const bool with_norm = mean_ptr != nullptr; abstractInitParams.storedDataSize = - GetSQ8StoredDataSize(abstractInitParams.dim, with_norm); + GetQuantizedStoredDataSize(abstractInitParams.dim, with_norm); abstractInitParams.isQuantized = true; IndexComponents components = CreateSQ8IndexComponents( @@ -91,38 +98,33 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { VecSimFactory::NewAbstractInitParams(hnswParams, params->logCtx, is_normalized); if (hnswParams->quantType != VecSimQuant_NONE) { - // Reject unknown quantizers instead of silently creating an unquantized index. - if (hnswParams->quantType != VecSimQuant_SQ8) { - return NULL; - } - - const VecSimMetric metric = ResolveSQ8Metric(hnswParams->metric, is_normalized); + const VecSimMetric metric = ResolveQuantizedMetric(hnswParams->metric, is_normalized); const float *mean_ptr = static_cast(hnswParams->quantParams); - if (!SQ8ParamsSupported(hnswParams->type, metric)) { + if (!QuantizationParamsSupported(hnswParams->quantType, hnswParams->type, metric)) { return NULL; } if (hnswParams->type == VecSimType_FLOAT32) { if (metric == VecSimMetric_L2) { - return NewIndex_SQ8(hnswParams, abstractInitParams, - mean_ptr); + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); } else if (metric == VecSimMetric_IP) { - return NewIndex_SQ8(hnswParams, abstractInitParams, - mean_ptr); + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); } } else if (hnswParams->type == VecSimType_FLOAT16) { if (metric == VecSimMetric_L2) { - return NewIndex_SQ8(hnswParams, abstractInitParams, - mean_ptr); + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); } else if (metric == VecSimMetric_IP) { - return NewIndex_SQ8(hnswParams, abstractInitParams, - mean_ptr); + return NewIndex_Quantized(hnswParams, abstractInitParams, + mean_ptr); } } // Keep the return for release builds so future dispatch changes cannot fall through. - assert(false && "unhandled SQ8 data type and metric combination"); + assert(false && "unhandled quantized data type and metric combination"); return NULL; } @@ -189,8 +191,8 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi if (params->quantType != VecSimQuant_NONE) { // Keep construction and initial-size validation in sync. - if (params->quantType != VecSimQuant_SQ8 || - !SQ8ParamsSupported(params->type, ResolveSQ8Metric(params->metric, is_normalized))) { + if (!QuantizationParamsSupported(params->quantType, params->type, + ResolveQuantizedMetric(params->metric, is_normalized))) { throw std::invalid_argument("Unsupported quantization params for HNSW index"); } // Template arguments do not affect these component sizes. @@ -244,9 +246,9 @@ size_t EstimateElementSize(const HNSWParams *params, bool with_mean) { // Preserve the element estimator's existing contract: return a size and validate in NewIndex. if (params->quantType == VecSimQuant_SQ8) { if (params->metric == VecSimMetric_L2) { - stored_data_size = GetSQ8StoredDataSize(params->dim, with_mean); + stored_data_size = GetQuantizedStoredDataSize(params->dim, with_mean); } else { - stored_data_size = GetSQ8StoredDataSize(params->dim, with_mean); + stored_data_size = GetQuantizedStoredDataSize(params->dim, with_mean); } } else { stored_data_size = @@ -370,9 +372,9 @@ VecSimIndex *NewIndex(const std::string &location, bool is_normalized) { if (params.quantType != VecSimQuant_NONE) { const float *mean_ptr = meanVector.empty() ? nullptr : meanVector.data(); - const VecSimMetric metric = ResolveSQ8Metric(params.metric, is_normalized); + const VecSimMetric metric = ResolveQuantizedMetric(params.metric, is_normalized); - if (!SQ8ParamsSupported(params.type, metric)) { + if (!QuantizationParamsSupported(params.quantType, params.type, metric)) { return NULL; } @@ -381,10 +383,10 @@ VecSimIndex *NewIndex(const std::string &location, bool is_normalized) { abstractInitParams.isQuantized = true; if (metric == VecSimMetric_L2) { abstractInitParams.storedDataSize = - GetSQ8StoredDataSize(dim, mean_ptr != nullptr); + GetQuantizedStoredDataSize(dim, mean_ptr != nullptr); } else { abstractInitParams.storedDataSize = - GetSQ8StoredDataSize(dim, mean_ptr != nullptr); + GetQuantizedStoredDataSize(dim, mean_ptr != nullptr); } if (params.type == VecSimType_FLOAT32) { From bdc5d1394b152a8325e7c1a4c185c34aa3dd5a70 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 10:41:38 +0200 Subject: [PATCH 05/10] Add per-type code dot hooks and a generic components wrapper Move the SQ8 code dot loops into sq8::code_dot and sq8::code_dot_fp32 so the generic scalar kernels call per-type hooks with unchanged arithmetic. Add a run-time CreateQuantizedIndexComponents overload that picks WithMean from mean_ptr, and make NewIndex_Quantized generic over StorageType. Co-Authored-By: Claude Opus 5.5 --- .../components/components_factory.h | 19 ++++++++--- src/VecSim/index_factories/hnsw_factory.cpp | 9 ++--- src/VecSim/spaces/quantized_kernels.h | 30 +++-------------- src/VecSim/types/sq8.h | 33 +++++++++++++++++++ 4 files changed, 58 insertions(+), 33 deletions(-) diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index bca77eddf..39fc2486a 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -84,6 +84,19 @@ CreateQuantizedIndexComponents(const std::shared_ptr &allocator return {calc, container}; } +// Run-time overload: selects WithMean from whether mean_ptr is set. +template +IndexComponents +CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, + const float *mean_ptr) { + if (mean_ptr) { + return CreateQuantizedIndexComponents(allocator, dim, + mean_ptr); + } + return CreateQuantizedIndexComponents(allocator, dim, + mean_ptr); +} + template IndexComponents CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, @@ -96,10 +109,8 @@ template IndexComponents CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, const float *mean_ptr) { - if (mean_ptr) { - return CreateSQ8IndexComponents(allocator, dim, mean_ptr); - } - return CreateSQ8IndexComponents(allocator, dim, mean_ptr); + return CreateQuantizedIndexComponents(allocator, dim, + mean_ptr); } template diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index dd271d9f3..c9ce8d642 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -70,16 +70,17 @@ template return true; } -template +template VecSimIndex *NewIndex_Quantized(const HNSWParams *hnswParams, AbstractIndexInitParams abstractInitParams, const float *mean_ptr) { const bool with_norm = mean_ptr != nullptr; abstractInitParams.storedDataSize = - GetQuantizedStoredDataSize(abstractInitParams.dim, with_norm); + GetQuantizedStoredDataSize(abstractInitParams.dim, with_norm); abstractInitParams.isQuantized = true; - IndexComponents components = CreateSQ8IndexComponents( - abstractInitParams.allocator, abstractInitParams.dim, mean_ptr); + IndexComponents components = + CreateQuantizedIndexComponents( + abstractInitParams.allocator, abstractInitParams.dim, mean_ptr); auto index = NewIndex_ChooseMultiOrSingle(hnswParams, abstractInitParams, components); #ifdef BUILD_TESTS diff --git a/src/VecSim/spaces/quantized_kernels.h b/src/VecSim/spaces/quantized_kernels.h index c6770f0fa..6cad11929 100644 --- a/src/VecSim/spaces/quantized_kernels.h +++ b/src/VecSim/spaces/quantized_kernels.h @@ -18,6 +18,7 @@ // Scalar kernels for scalar quantized storage, generic over a StorageType that provides // load_code(data, i), data_bytes_count(dim) and the MIN_VAL / DELTA / SUM / SUM_SQUARES metadata // indices (stored after the codes) and SUM_QUERY (stored after the FP32 query values). +// It also provides the code dot hooks code_dot(a, b, dim) and code_dot_fp32(codes, y, dim). // // Stored layout: [codes (data_bytes_count(dim) bytes)] [min_val] [delta] [sum] [sum_squares (L2)] // FP32 query layout: [float values (dim)] [y_sum] [y_sum_squares (L2 only)] @@ -31,7 +32,7 @@ namespace quantized { * Asymmetric stored-FP32 inner product using the algebraic identity: * IP(x, y) = sum((min + delta * q_i) * y_i) = min * y_sum + delta * sum(q_i * y_i) * - * Uses 4x loop unrolling with multiple accumulators for ILP. + * sum(q_i * y_i) comes from StorageType::code_dot_fp32. * Returns the raw inner product value (not distance). */ template @@ -39,26 +40,8 @@ static inline float FP32_InnerProduct_Impl(const void *stored, const void *query const auto *pVect1 = static_cast(stored); const auto *pVect2 = static_cast(query); - // Use 4 accumulators for instruction-level parallelism - float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; - - // Main loop: process 4 elements per iteration - size_t i = 0; - size_t dim4 = dim & ~size_t(3); // dim4 is a multiple of 4 - for (; i < dim4; i += 4) { - sum0 += static_cast(StorageType::load_code(pVect1, i + 0)) * pVect2[i + 0]; - sum1 += static_cast(StorageType::load_code(pVect1, i + 1)) * pVect2[i + 1]; - sum2 += static_cast(StorageType::load_code(pVect1, i + 2)) * pVect2[i + 2]; - sum3 += static_cast(StorageType::load_code(pVect1, i + 3)) * pVect2[i + 3]; - } - - // Handle remainder (0-3 elements) - for (; i < dim; i++) { - sum0 += static_cast(StorageType::load_code(pVect1, i)) * pVect2[i]; - } - - // Combine accumulators - float quantized_dot = (sum0 + sum1) + (sum2 + sum3); + // Compute inner product of quantized values with the query: sum(q_i * y_i) + const float quantized_dot = StorageType::code_dot_fp32(pVect1, pVect2, dim); // Storage metadata follows a byte payload and is not necessarily float-aligned. const auto *params = pVect1 + StorageType::data_bytes_count(dim); @@ -115,10 +98,7 @@ static inline float Stored_InnerProduct_Impl(const void *a, const void *b, size_ const auto *pVect2 = static_cast(b); // Compute inner product of quantized values: sum(q1[i]*q2[i]) - float product = 0; - for (size_t i = 0; i < dim; i++) { - product += StorageType::load_code(pVect1, i) * StorageType::load_code(pVect2, i); - } + const float product = StorageType::code_dot(pVect1, pVect2, dim); // Metadata follows byte payloads and is not necessarily float-aligned. const auto *params1 = pVect1 + StorageType::data_bytes_count(dim); diff --git a/src/VecSim/types/sq8.h b/src/VecSim/types/sq8.h index 38fe1e7c7..27636ac91 100644 --- a/src/VecSim/types/sq8.h +++ b/src/VecSim/types/sq8.h @@ -26,6 +26,39 @@ struct sq8 { static void store_code(uint8_t *data, size_t index, uint8_t code) { data[index] = code; } + // Dot product of two code arrays: sum(q1_i * q2_i). + static float code_dot(const uint8_t *a, const uint8_t *b, size_t dim) { + float product = 0; + for (size_t i = 0; i < dim; i++) { + product += load_code(a, i) * load_code(b, i); + } + return product; + } + + // Dot product of a code array with FP32 values: sum(q_i * y_i). Uses 4 accumulators for + // instruction-level parallelism. + static float code_dot_fp32(const uint8_t *codes, const float *y, size_t dim) { + float sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + + // Main loop: process 4 elements per iteration + size_t i = 0; + size_t dim4 = dim & ~size_t(3); // dim4 is a multiple of 4 + for (; i < dim4; i += 4) { + sum0 += static_cast(load_code(codes, i + 0)) * y[i + 0]; + sum1 += static_cast(load_code(codes, i + 1)) * y[i + 1]; + sum2 += static_cast(load_code(codes, i + 2)) * y[i + 2]; + sum3 += static_cast(load_code(codes, i + 3)) * y[i + 3]; + } + + // Handle remainder (0-3 elements) + for (; i < dim; i++) { + sum0 += static_cast(load_code(codes, i)) * y[i]; + } + + // Combine accumulators + return (sum0 + sum1) + (sum2 + sum3); + } + // Metadata layout indices (stored after quantized values) enum MetadataIndex : size_t { MIN_VAL = 0, From 217026b84aa123ae3bc0f483998a0a85b9d47b25 Mon Sep 17 00:00:00 2001 From: dor-forer Date: Sun, 8 Nov 2026 11:40:57 +0200 Subject: [PATCH 06/10] Generalize quantized HNSW snapshot loading --- src/VecSim/algorithms/hnsw/hnsw_serializer.h | 2 +- .../algorithms/hnsw/hnsw_serializer_impl.h | 4 +- src/VecSim/index_factories/hnsw_factory.cpp | 50 ++++++++----------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer.h b/src/VecSim/algorithms/hnsw/hnsw_serializer.h index dabc6f379..de223c0aa 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer.h @@ -24,7 +24,7 @@ class HNSWSerializer : public Serializer { DEPRECATED = 2, // Last deprecated version V3, V4, - V5, // SQ8 quantization fields (quantType + mean vector) + V5, // scalar quantization fields (quantType + mean vector) INVALID }; diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h index e72036a81..2865c5d81 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h @@ -263,10 +263,10 @@ void HNSWIndex::saveIndexFields(std::ofstream &output) const writeBinaryPOD(output, this->maxElements); // This will be used to restore the index initial // capacity - // V5 fields: SQ8 quantization support (between factory params and index build params) + // V5 fields: scalar quantization support (between factory params and index build params) if (this->quantType != VecSimQuant_NONE) { writeBinaryPOD(output, this->quantType); - // Write mean vector (dim floats). Empty vector means no mean (zero-mean SQ8). + // Write mean vector (dim floats). Empty vector means no mean (zero-mean quantization). bool hasMean = !this->serializedMeanVector.empty(); writeBinaryPOD(output, hasMean); if (hasMean) { diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index c9ce8d642..15a7d54f3 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -305,6 +305,19 @@ inline VecSimIndex *NewIndex_ChooseMultiOrSingle(std::ifstream &input, const HNS return index; } +template +VecSimIndex *LoadIndex_Quantized(std::ifstream &input, const HNSWParams ¶ms, + AbstractIndexInitParams abstractInitParams, + HNSWSerializer::EncodingVersion version, const float *mean_ptr) { + abstractInitParams.isQuantized = true; + abstractInitParams.storedDataSize = + GetQuantizedStoredDataSize(params.dim, mean_ptr != nullptr); + auto components = CreateQuantizedIndexComponents( + abstractInitParams.allocator, params.dim, mean_ptr); + return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, + components, version, mean_ptr); +} + // Initialize @params from file for V3+. For V5+ also reads quantType and mean vector. // Returns false if the file holds a quantizer value this version does not know. static bool InitializeParams(std::ifstream &source_params, HNSWParams ¶ms, @@ -327,7 +340,7 @@ static bool InitializeParams(std::ifstream &source_params, HNSWParams ¶ms, return false; } params.quantType = static_cast(raw_quant_type); - if (params.quantType == VecSimQuant_SQ8) { + if (params.quantType != VecSimQuant_NONE) { bool hasMean = false; Serializer::readBinaryPOD(source_params, hasMean); if (hasMean) { @@ -379,40 +392,21 @@ VecSimIndex *NewIndex(const std::string &location, bool is_normalized) { return NULL; } - // Override blob sizes for SQ8 storage layout. - size_t dim = params.dim; - abstractInitParams.isQuantized = true; - if (metric == VecSimMetric_L2) { - abstractInitParams.storedDataSize = - GetQuantizedStoredDataSize(dim, mean_ptr != nullptr); - } else { - abstractInitParams.storedDataSize = - GetQuantizedStoredDataSize(dim, mean_ptr != nullptr); - } - if (params.type == VecSimType_FLOAT32) { if (metric == VecSimMetric_L2) { - auto components = CreateSQ8IndexComponents( - abstractInitParams.allocator, dim, mean_ptr); - return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, - components, version, mean_ptr); + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); } else { - auto components = CreateSQ8IndexComponents( - abstractInitParams.allocator, dim, mean_ptr); - return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, - components, version, mean_ptr); + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); } } else if (params.type == VecSimType_FLOAT16) { if (metric == VecSimMetric_L2) { - auto components = CreateSQ8IndexComponents( - abstractInitParams.allocator, dim, mean_ptr); - return NewIndex_ChooseMultiOrSingle( - input, ¶ms, abstractInitParams, components, version, mean_ptr); + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); } else { - auto components = CreateSQ8IndexComponents( - abstractInitParams.allocator, dim, mean_ptr); - return NewIndex_ChooseMultiOrSingle( - input, ¶ms, abstractInitParams, components, version, mean_ptr); + return LoadIndex_Quantized( + input, params, abstractInitParams, version, mean_ptr); } } } From 6a60e4087e8d31c8cdd78ad06c260fed252c9279 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 15:23:14 +0200 Subject: [PATCH 07/10] Generalize quantized components around storage contexts --- research/storage-context/README.md | 30 ++++ src/VecSim/algorithms/hnsw/hnsw.h | 71 +++++--- .../algorithms/hnsw/hnsw_serializer_impl.h | 17 +- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 4 + .../components/components_factory.h | 150 ++++++++++++----- src/VecSim/index_factories/hnsw_factory.cpp | 47 +++--- src/VecSim/index_factories/hnsw_factory.h | 6 +- src/VecSim/index_factories/tiered_factory.cpp | 32 +--- .../quantization/quantization_runtime.h | 44 +++++ src/VecSim/quantization/sq8_mean.h | 31 ---- src/VecSim/quantization/storage_context.h | 154 ++++++++++++++++++ src/VecSim/spaces/computer/calculator.h | 26 +++ .../spaces/computer/preprocessor_container.h | 106 +++++++----- src/VecSim/spaces/computer/preprocessors.h | 132 +++++++++++++-- src/VecSim/types/sq8.h | 1 + src/VecSim/vec_sim_common.h | 9 +- src/VecSim/vec_sim_index.h | 20 ++- tests/unit/test_components.cpp | 82 +++++++++- 18 files changed, 727 insertions(+), 235 deletions(-) create mode 100644 research/storage-context/README.md create mode 100644 src/VecSim/quantization/quantization_runtime.h delete mode 100644 src/VecSim/quantization/sq8_mean.h create mode 100644 src/VecSim/quantization/storage_context.h diff --git a/research/storage-context/README.md b/research/storage-context/README.md new file mode 100644 index 000000000..3a291cbfa --- /dev/null +++ b/research/storage-context/README.md @@ -0,0 +1,30 @@ +# Shared quantized storage contexts + +StorageType selects a typed context. The same QuantPreprocessor, QuantizedCalculator, +component factory and backend lifecycle serve scalar and transformed quantizers. +The scalar-family default context retains SQ8 layout hooks and existing encoding, +correction arithmetic and optimized kernel selection. A non-scalar storage type +can supply `template using Context`. + +The context owns encoding, prepared-query layout, dispatch, mean installation, +validation and fitted-state persistence. It has a stable address. Component wrappers +share ownership through VecSimAllocator-backed shared_ptrs; the index keeps a non-owning +runtime alias. A scoring call uses the existing cached dispatch, without a new +per-candidate virtual call. The shared control allocation and mean buffer are included +in the component estimate; shared allocation size is measured before context +construction, without fitting data or allocating a rotation. + +Tiered construction requests a backend prepared for the regular training phase. +It does not interpret storage-specific parameters. Mean installation occurs with +queries excluded and before the first stored vector. Failed preparation retains +previous fitted state. Input validation precedes destructive replacements. + +This foundation introduces no new accepted quantization mode. Existing scalar V5 +payloads and nonquantized V4 payloads retain their layouts. Serialization remains +available only with test helpers. Existing SQ4 layout/code traits can use the scalar +default context; TQ2/TQ3 are supplied separately by the stacked draft. + +Regression coverage checks SQ8 stored bytes against independent expected bytes, +legacy distance results and stateless function pointers, common component allocation +estimates, combined preprocessing failure cleanup and allocated buffer lifetime. +See the stacked TQ draft's validation receipt for current ARM execution evidence. diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h index 11b4247ef..5e1ee2787 100644 --- a/src/VecSim/algorithms/hnsw/hnsw.h +++ b/src/VecSim/algorithms/hnsw/hnsw.h @@ -21,7 +21,6 @@ #include "VecSim/vec_sim_common.h" #include "VecSim/vec_sim_index.h" #include "VecSim/spaces/computer/preprocessors.h" -#include "VecSim/quantization/sq8_mean.h" #include #include "VecSim/tombstone_interface.h" @@ -92,35 +91,22 @@ class HNSWIndex : public VecSimIndexAbstract, public HNSWSerializer #endif { -private: - template - void setSQ8Mean(std::span mean) noexcept { - applySQ8Mean(*this->getPreprocessorsContainer(), - *this->getIndexCalculator(), mean); - } - public: - // Writer-only operation: install the mean once, before any stored vector, under the - // exclusive tiered main lock and before submitting insertion jobs. The backend must have - // its final SQ8 WithNorm components; the caller owns and enforces the one-time transition. - // A quantizer that learns storedDataSize from data needs deferred backend creation. - void setQuantizationMean(std::span mean) noexcept { + // Install the mean before storing any vector, with queries excluded by the caller. + // The storage context prepares all fallible state before publishing the new mean; + // cached scoring callbacks keep referring to the same context throughout. + void setQuantizationMean(std::span mean) { assert(this->isQuantized && curElementCount == 0 && mean.size() == this->dim); - if constexpr (QuantInput && std::is_same_v) { - if (this->metric == VecSimMetric_L2) { - setSQ8Mean(mean); - } else { - // Tiered cosine inputs are already normalized, and use IP SQ8 components. - assert(this->metric == VecSimMetric_IP || this->metric == VecSimMetric_Cosine); - setSQ8Mean(mean); - } + assert(this->getQuantizationRuntime()); + this->getQuantizationRuntime()->setMean(mean); #ifdef BUILD_TESTS - assert(serializedMeanVector.size() == this->dim); + // Compatibility view for existing scalar serialization tests. State is saved by + // the context; only an already allocated view is updated here. + if (!serializedMeanVector.empty()) { + assert(serializedMeanVector.size() == mean.size()); std::copy(mean.begin(), mean.end(), serializedMeanVector.begin()); -#endif - } else { - assert(false && "Unsupported SQ8 data type"); } +#endif } protected: @@ -647,6 +633,7 @@ VecSimUpdateCode HNSWIndex::updateVectors(labelType label, return VecSimUpdate_OK; } + this->validateQuantizationInput(new_blobs, n, this->getInputBlobSize(), this->metric); this->deleteVector(label); const char *blob = static_cast(new_blobs); for (size_t i = 0; i < n; i++) { @@ -2459,7 +2446,7 @@ VecSimQueryReply *HNSWIndex::rangeQuery(const void *query_da template VecSimIndexDebugInfo HNSWIndex::debugInfo() const { - VecSimIndexDebugInfo info; + VecSimIndexDebugInfo info{}; info.commonInfo = this->getCommonInfo(); auto [ep_id, max_level] = this->safeGetEntryPointState(); @@ -2472,6 +2459,14 @@ VecSimIndexDebugInfo HNSWIndex::debugInfo() const { info.hnswInfo.entrypoint = ep_id != INVALID_ID ? getExternalLabel(ep_id) : INVALID_LABEL; info.hnswInfo.visitedNodesPoolSize = this->visitedNodesHandlerPool.getPoolSize(); info.hnswInfo.numberOfMarkedDeletedNodes = this->getNumMarkedDeleted(); + if (const auto *context = this->getQuantizationRuntime()) { + const auto quant = context->info(); + info.hnswInfo.quantizationType = quant.type; + info.hnswInfo.quantizedVectorBytes = quant.vectorBytes; + info.hnswInfo.quantizerSharedBytes = quant.sharedBytes; + info.hnswInfo.quantizerQueryKernel = quant.queryKernel; + info.hnswInfo.quantizerGraphKernel = quant.graphKernel; + } return info; } @@ -2538,6 +2533,30 @@ VecSimDebugInfoIterator *HNSWIndex::debugInfoIterator() cons .fieldType = INFOFIELD_UINT64, .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.numberOfMarkedDeletedNodes}}}); + if (const auto *context = this->getQuantizationRuntime(); context && context->info().name) { + const auto quant = context->info(); + auto integer = [&](const char *name, uint64_t value) { + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = name, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = value}}}); + }; + auto string = [&](const char *name, const char *value) { + if (value) + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = name, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = value}}}); + }; + string("QUANTIZATION", quant.name); + string("QUANT_SCALE", quant.scale); + string("QUANT_BIAS", quant.bias); + integer("QUANT_VECTOR_BYTES", quant.vectorBytes); + integer("QUANT_SHARED_BYTES", quant.sharedBytes); + integer("QUANT_QUERY_BYTES", quant.queryBytes); + string("QUANT_QUERY_KERNEL", quant.queryKernel); + string("QUANT_GRAPH_KERNEL", quant.graphKernel); + } return infoIterator; } diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h index 2865c5d81..bee8a7391 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h @@ -43,8 +43,9 @@ HNSWIndex::HNSWIndex(std::ifstream &input, const HNSWParams template HNSWSerializer::EncodingVersion HNSWIndex::getWriteVersion() const { - return this->isQuantized ? HNSWSerializer::EncodingVersion::V5 - : HNSWSerializer::EncodingVersion::V4; + const auto *context = this->getQuantizationRuntime(); + return context ? static_cast(context->encodingVersion()) + : HNSWSerializer::EncodingVersion::V4; } template @@ -263,16 +264,10 @@ void HNSWIndex::saveIndexFields(std::ofstream &output) const writeBinaryPOD(output, this->maxElements); // This will be used to restore the index initial // capacity - // V5 fields: scalar quantization support (between factory params and index build params) - if (this->quantType != VecSimQuant_NONE) { + // The graph owns framing; the storage context owns its fitted-state payload. + if (const auto *context = this->getQuantizationRuntime()) { writeBinaryPOD(output, this->quantType); - // Write mean vector (dim floats). Empty vector means no mean (zero-mean quantization). - bool hasMean = !this->serializedMeanVector.empty(); - writeBinaryPOD(output, hasMean); - if (hasMean) { - output.write(reinterpret_cast(this->serializedMeanVector.data()), - this->dim * sizeof(float)); - } + context->saveState(output); } // Save index build parameters diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 7f8fea28a..90f80e5df 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -946,6 +946,8 @@ HNSWInsertJob *TieredHNSWIndex::addVectorToFlat(const void * // parameters. template int TieredHNSWIndex::addVector(const void *blob, labelType label) { + getHNSWIndex()->validateQuantizationInput(blob, 1, this->frontendIndex->getInputBlobSize(), + this->frontendIndex->getMetric()); if (sqAccumulationState) { return addVectorDuringAccumulation(blob, label); } @@ -1105,6 +1107,8 @@ VecSimUpdateCode TieredHNSWIndex::updateVectors(labelType la return updateVectorSingleValue(label, new_blobs, n); // -- Multi Value -- + getHNSWIndex()->validateQuantizationInput(new_blobs, n, this->frontendIndex->getInputBlobSize(), + this->frontendIndex->getMetric()); this->deleteVector(label); const char *blob = static_cast(new_blobs); for (size_t i = 0; i < n; i++) { diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index 39fc2486a..220265552 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -40,61 +40,127 @@ template return alignment; } +// A prebuilt context allows callers with fitted typed state to use the common wrappers. +template +IndexComponents CreateQuantizedIndexComponents( + const std::shared_ptr &allocator, size_t, + std::shared_ptr> context) { + using Preprocessor = QuantPreprocessor; + using Context = typename Preprocessor::Context; + auto calculator = std::unique_ptr>( + new (allocator) QuantizedCalculator(allocator, context)); + auto preprocessor = + std::unique_ptr(new (allocator) Preprocessor(allocator, context)); + auto container = std::unique_ptr>( + new (allocator) MultiPreprocessorsContainer( + allocator, context->queryAlignment(), context->storageAlignment())); + [[maybe_unused]] const int ret = container->addPreprocessor(preprocessor.get()); + assert(ret != -1 && "Quantization preprocessor was not added correctly"); + preprocessor.release(); + return {calculator.release(), container.release(), context.get()}; +} + +template +IndexComponents +CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, + const QuantizationInit &init) { + using Context = StorageContextFor; + auto context = + std::allocate_shared(VecsimSTLAllocator(allocator), allocator, dim, init); + return CreateQuantizedIndexComponents( + allocator, dim, std::move(context)); +} + template IndexComponents CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, const float *mean_ptr) { - using Preprocessor = QuantPreprocessor; - using QueryType = typename Preprocessor::QueryType; - unsigned char storage_alignment = 0, asym_storage_alignment = 0; - - // Graph construction compares two stored quantized blobs; search compares a stored blob with a - // QueryType query. Both dispatchers report alignment for the stored operand. - auto sym_func = spaces::GetDistFunc(Metric, dim, &storage_alignment); - auto asym_func = - spaces::GetDistFunc(Metric, dim, &asym_storage_alignment); - storage_alignment = spaces::combineAlignments(storage_alignment, asym_storage_alignment); - const unsigned char query_alignment = GetQueryAlignment(Metric, dim); - - PreprocessorInterface *pp = nullptr; - IndexCalculatorInterface *calc = nullptr; - - if constexpr (WithMean) { - vecsim_stl::vector mean_vec(allocator); - mean_vec.assign(mean_ptr, mean_ptr + dim); - - float mean_sum_squares = 0.0f; - for (float v : mean_vec) { - mean_sum_squares += v * v; - } + return CreateQuantizedIndexComponents( + allocator, dim, QuantizationInit{.parameters = mean_ptr}); +} - pp = new (allocator) Preprocessor(allocator, dim, mean_vec); - calc = new (allocator) DistanceCalculatorWithNorm( - allocator, asym_func, sym_func, mean_sum_squares); - } else { - pp = new (allocator) Preprocessor(allocator, dim); - calc = new (allocator) DistanceCalculatorCommon(allocator, sym_func, asym_func); +// The context interprets parameters; this boundary only selects the mean-aware type. +template +IndexComponents +CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, + const QuantizationInit &init) { + if (init.parameters || init.prepareForTraining) { + return CreateQuantizedIndexComponents(allocator, dim, + init); } - - auto *container = new (allocator) - MultiPreprocessorsContainer(allocator, query_alignment, storage_alignment); - [[maybe_unused]] const int ret = container->addPreprocessor(pp); - assert(ret != -1 && "Quantization preprocessor was not added correctly"); - - return {calc, container}; + return CreateQuantizedIndexComponents(allocator, dim, + init); } -// Run-time overload: selects WithMean from whether mean_ptr is set. template IndexComponents CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, const float *mean_ptr) { - if (mean_ptr) { - return CreateQuantizedIndexComponents(allocator, dim, - mean_ptr); + return CreateQuantizedIndexComponents( + allocator, dim, QuantizationInit{.parameters = mean_ptr}); +} + +namespace quantization_detail { + +struct AllocationMeasured {}; +inline size_t &sharedAllocationBytes() { + static thread_local size_t bytes = 0; + return bytes; +} + +// The control block layout is implementation-defined. Use the same allocator state size and +// alignment as VecsimSTLAllocator so its rebound allocate_shared block has the same layout. +// Stop at allocation: estimating never constructs a context or fits quantization state. +template +struct SharedAllocationProbe { + using value_type = T; + std::shared_ptr allocator; + SharedAllocationProbe() = default; + template + SharedAllocationProbe(const SharedAllocationProbe &other) : allocator(other.allocator) {} + T *allocate(size_t count) { + sharedAllocationBytes() = count * sizeof(T); + throw AllocationMeasured{}; } - return CreateQuantizedIndexComponents(allocator, dim, - mean_ptr); + void deallocate(T *, size_t) {} +}; + +template +size_t contextControlBytes() { + static_assert(sizeof(SharedAllocationProbe) == sizeof(VecsimSTLAllocator)); + static_assert(alignof(SharedAllocationProbe) == alignof(VecsimSTLAllocator)); + static const size_t bytes = [] { + try { + std::allocate_shared(SharedAllocationProbe{}, + std::shared_ptr{}, size_t{1}, + QuantizationInit{}); + } catch (const AllocationMeasured &) { + return sharedAllocationBytes() - sizeof(Context); + } + std::terminate(); + }(); + return bytes; +} + +} // namespace quantization_detail + +template +size_t EstimateQuantizedComponentsMemory(size_t dim, const QuantizationInit &init) { + using Context = StorageContextFor; + const size_t overhead = VecSimAllocator::getAllocationOverheadSize(); + return Context::estimateMemory(dim, init) + + quantization_detail::contextControlBytes() + overhead + + sizeof(QuantPreprocessor) + overhead + + sizeof(QuantizedCalculator) + overhead + + sizeof(MultiPreprocessorsContainer) + overhead; +} + +template +size_t EstimateQuantizedComponentsMemory(size_t dim, const QuantizationInit &init) { + if (init.parameters || init.prepareForTraining) { + return EstimateQuantizedComponentsMemory(dim, init); + } + return EstimateQuantizedComponentsMemory(dim, init); } template diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index 15a7d54f3..4f80a1537 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -72,35 +72,34 @@ template template VecSimIndex *NewIndex_Quantized(const HNSWParams *hnswParams, - AbstractIndexInitParams abstractInitParams, const float *mean_ptr) { - const bool with_norm = mean_ptr != nullptr; - abstractInitParams.storedDataSize = - GetQuantizedStoredDataSize(abstractInitParams.dim, with_norm); + AbstractIndexInitParams abstractInitParams, + const QuantizationInit &init) { abstractInitParams.isQuantized = true; IndexComponents components = - CreateQuantizedIndexComponents( - abstractInitParams.allocator, abstractInitParams.dim, mean_ptr); + CreateQuantizedIndexComponents(abstractInitParams.allocator, + abstractInitParams.dim, init); + abstractInitParams.storedDataSize = components.quantization->storageBytes(); auto index = NewIndex_ChooseMultiOrSingle(hnswParams, abstractInitParams, components); #ifdef BUILD_TESTS // Store quantization metadata for re-serialization. index->quantType = hnswParams->quantType; - if (mean_ptr != nullptr) { - index->serializedMeanVector.assign(mean_ptr, mean_ptr + abstractInitParams.dim); - } + const auto mean = components.quantization->mean(); + index->serializedMeanVector.assign(mean.begin(), mean.end()); #endif return index; } -VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { +VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized, bool prepareForTraining) { const HNSWParams *hnswParams = ¶ms->algoParams.hnswParams; AbstractIndexInitParams abstractInitParams = VecSimFactory::NewAbstractInitParams(hnswParams, params->logCtx, is_normalized); if (hnswParams->quantType != VecSimQuant_NONE) { const VecSimMetric metric = ResolveQuantizedMetric(hnswParams->metric, is_normalized); - const float *mean_ptr = static_cast(hnswParams->quantParams); + const QuantizationInit init{.parameters = hnswParams->quantParams, + .prepareForTraining = prepareForTraining}; if (!QuantizationParamsSupported(hnswParams->quantType, hnswParams->type, metric)) { return NULL; @@ -109,18 +108,18 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { if (hnswParams->type == VecSimType_FLOAT32) { if (metric == VecSimMetric_L2) { return NewIndex_Quantized(hnswParams, abstractInitParams, - mean_ptr); + init); } else if (metric == VecSimMetric_IP) { return NewIndex_Quantized(hnswParams, abstractInitParams, - mean_ptr); + init); } } else if (hnswParams->type == VecSimType_FLOAT16) { if (metric == VecSimMetric_L2) { return NewIndex_Quantized(hnswParams, abstractInitParams, - mean_ptr); + init); } else if (metric == VecSimMetric_IP) { return NewIndex_Quantized(hnswParams, abstractInitParams, - mean_ptr); + init); } } @@ -166,9 +165,9 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { return NULL; } -VecSimIndex *NewIndex(const HNSWParams *params, bool is_normalized) { +VecSimIndex *NewIndex(const HNSWParams *params, bool is_normalized, bool prepareForTraining) { VecSimParams vecSimParams = {.algoParams = {.hnswParams = HNSWParams{*params}}}; - return NewIndex(&vecSimParams); + return NewIndex(&vecSimParams, is_normalized, prepareForTraining); } template @@ -196,17 +195,13 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi ResolveQuantizedMetric(params->metric, is_normalized))) { throw std::invalid_argument("Unsupported quantization params for HNSW index"); } - // Template arguments do not affect these component sizes. + const QuantizationInit init{.prepareForTraining = with_mean}; if (with_mean) { - est += allocations_overhead + - sizeof(DistanceCalculatorWithNorm); - est += allocations_overhead + sizeof(MultiPreprocessorsContainer); - est += allocations_overhead + sizeof(QuantPreprocessor); - est += allocations_overhead + params->dim * sizeof(float); + est += EstimateQuantizedComponentsMemory(params->dim, + init); } else { - est += allocations_overhead + sizeof(DistanceCalculatorCommon); - est += allocations_overhead + sizeof(MultiPreprocessorsContainer); - est += allocations_overhead + sizeof(QuantPreprocessor); + est += EstimateQuantizedComponentsMemory( + params->dim, init); } est += EstimateInitialSize_ChooseMultiOrSingle(params->multi); } else if (params->type == VecSimType_FLOAT32) { diff --git a/src/VecSim/index_factories/hnsw_factory.h b/src/VecSim/index_factories/hnsw_factory.h index 41f2399cc..4cb86fd57 100644 --- a/src/VecSim/index_factories/hnsw_factory.h +++ b/src/VecSim/index_factories/hnsw_factory.h @@ -22,8 +22,10 @@ namespace HNSWFactory { * the blobs sent to the index are already normalized. For example, in case it's a tiered index, * where the blobs are normalized by the frontend index. */ -VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized = false); -VecSimIndex *NewIndex(const HNSWParams *params, bool is_normalized = false); +VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized = false, + bool prepareForTraining = false); +VecSimIndex *NewIndex(const HNSWParams *params, bool is_normalized = false, + bool prepareForTraining = false); size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized = false); size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool with_mean); size_t EstimateElementSize(const HNSWParams *params); diff --git a/src/VecSim/index_factories/tiered_factory.cpp b/src/VecSim/index_factories/tiered_factory.cpp index c638a8495..3307ee687 100644 --- a/src/VecSim/index_factories/tiered_factory.cpp +++ b/src/VecSim/index_factories/tiered_factory.cpp @@ -40,42 +40,16 @@ static inline BFParams NewBFParams(const TieredIndexParams *params) { } static inline bool RequiresQuantizationTraining(const TieredIndexParams *params) { - return params->primaryIndexParams->algoParams.hnswParams.quantType == VecSimQuant_SQ8 && + return params->primaryIndexParams->algoParams.hnswParams.quantType != VecSimQuant_NONE && params->specificParams.tieredHnswParams.QuantNormalizationSetSize > 0; } -template -static inline bool IsQuantizationSupported(const TieredIndexParams *params) { - const auto &hnsw_params = params->primaryIndexParams->algoParams.hnswParams; - if (hnsw_params.quantType == VecSimQuant_NONE) { - return true; - } - - if constexpr (!QuantInput || !std::is_same_v) { - return false; - } else { - return hnsw_params.quantType == VecSimQuant_SQ8; - } -} - template inline VecSimIndex *NewIndex(const TieredIndexParams *params) { - if (!IsQuantizationSupported(params)) { - return nullptr; - } - const auto &hnsw_params = params->primaryIndexParams->algoParams.hnswParams; auto management_layer_allocator = VecSimAllocator::newVecsimAllocator(); - VecSimParams backend_params = *params->primaryIndexParams; - vecsim_stl::vector initial_mean(management_layer_allocator); - if (RequiresQuantizationTraining(params)) { - // Allocate the final SQ8 WithMean layout now. The factory copies this temporary mean; - // tiered keeps the graph empty until it installs the mean accumulated from FLAT. - initial_mean.resize(hnsw_params.dim, 0.0f); - backend_params.algoParams.hnswParams.quantParams = initial_mean.data(); - } - auto *hnsw_index = - static_cast *>(HNSWFactory::NewIndex(&backend_params, true)); + auto *hnsw_index = static_cast *>(HNSWFactory::NewIndex( + params->primaryIndexParams, true, RequiresQuantizationTraining(params))); if (!hnsw_index) { return nullptr; } diff --git a/src/VecSim/quantization/quantization_runtime.h b/src/VecSim/quantization/quantization_runtime.h new file mode 100644 index 000000000..d5e817c1d --- /dev/null +++ b/src/VecSim/quantization/quantization_runtime.h @@ -0,0 +1,44 @@ +#pragma once +#include +#include +#include +#include "VecSim/vec_sim_common.h" + +struct QuantizationInit { + const void *parameters = nullptr; + bool prepareForTraining = false; +#ifdef BUILD_TESTS + std::istream *fittedState = nullptr; +#endif +}; +struct QuantizationInfo { + VecSimQuantType type = VecSimQuant_NONE; + const char *name = nullptr; + const char *scale = nullptr; + const char *bias = nullptr; + size_t vectorBytes = 0; + size_t queryBytes = 0; + size_t sharedBytes = 0; + const char *queryKernel = nullptr; + const char *graphKernel = nullptr; +}; +// Cold-path lifecycle for a storage context. The component wrappers own the context; +// indexes retain a non-owning alias and never delete it separately. +class QuantizationRuntime { +public: + virtual ~QuantizationRuntime() = default; + virtual void validateBatch(const void *rows, size_t count, size_t rowBytes, + VecSimMetric inputMetric) const = 0; + // Caller excludes queries and has not stored any vectors. A failed preparation + // leaves all previous fitted state intact; successful publication keeps dispatch addresses. + virtual void setMean(std::span mean) = 0; + virtual std::span mean() const = 0; + virtual size_t storageBytes() const = 0; + virtual size_t queryBytes() const = 0; + virtual QuantizationInfo info() const = 0; +#ifdef BUILD_TESTS + virtual unsigned encodingVersion() const = 0; + virtual void saveState(std::ostream &output) const = 0; + virtual void validateStoredRecord(const void *record) const = 0; +#endif +}; diff --git a/src/VecSim/quantization/sq8_mean.h b/src/VecSim/quantization/sq8_mean.h deleted file mode 100644 index b0dfe4a7f..000000000 --- a/src/VecSim/quantization/sq8_mean.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2006-Present, Redis Ltd. - * All rights reserved. - * - * Licensed under your choice of the Redis Source Available License 2.0 - * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the - * GNU Affero General Public License v3 (AGPLv3). - */ -#pragma once - -#include -#include - -#include "VecSim/spaces/computer/calculator.h" -#include "VecSim/spaces/computer/preprocessor_container.h" - -// Install before storing vectors, with queries excluded by the caller. Requires exactly one -// SQ8 WithNorm preprocessor in slot 0; cosine normalization must be performed by the frontend. -template -void applySQ8Mean(PreprocessorsContainerAbstract &preprocessors, - IndexCalculatorInterface &index_calculator, - std::span mean) noexcept { - auto &container = static_cast &>(preprocessors); - auto *preprocessor = - dynamic_cast *>(container.getPreprocessors()[0]); - auto *calculator = - dynamic_cast *>(&index_calculator); - assert(preprocessor && calculator); - preprocessor->setMean(mean); - calculator->setMeanSumSquares(mean); -} diff --git a/src/VecSim/quantization/storage_context.h b/src/VecSim/quantization/storage_context.h new file mode 100644 index 000000000..f28357145 --- /dev/null +++ b/src/VecSim/quantization/storage_context.h @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#ifdef BUILD_TESTS +#include +#endif + +#include "VecSim/quantization/quantization_runtime.h" +#include "VecSim/spaces/computer/calculator.h" +#include "VecSim/spaces/computer/preprocessors.h" + +// Scalar storage policies supply layout/code hooks. Other families can select their own +// Context through StorageType::Context without changing the component wrappers. +template +class ScalarStorageContext : public QuantizationRuntime { + using Engine = ScalarQuantPreprocessor; + using Calculator = + std::conditional_t, + DistanceCalculatorCommon>; + +public: + using QueryType = typename Engine::QueryType; + + ScalarStorageContext(const std::shared_ptr &allocator, size_t dim, + const QuantizationInit &init) + : dim_(dim), engine_(makeEngine(allocator, dim, init)), + calculator_(makeCalculator(allocator, dim)), + stored_dispatch_(calculator_.getDistanceDispatch(DistanceMode::StoredToStored)), + query_dispatch_(calculator_.getDistanceDispatch(DistanceMode::StoredToQuery)) {} + + ScalarStorageContext(const ScalarStorageContext &) = delete; + ScalarStorageContext &operator=(const ScalarStorageContext &) = delete; + + void encodeStorage(const void *input, void *output) const { + engine_.encodeStorage(input, output); + } + void prepareQuery(const void *input, void *output) const { + engine_.prepareQuery(input, output); + } + void encodeStorageInPlace(void *input, size_t inputBytes) const { + engine_.preprocessStorageInPlace(input, inputBytes); + } + unsigned char storageAlignment() const { return storage_alignment_; } + unsigned char queryAlignment() const { return query_alignment_; } + DistanceDispatch getDistanceDispatch(DistanceMode mode) const { + return mode == DistanceMode::StoredToStored ? stored_dispatch_ : query_dispatch_; + } + + void validateBatch(const void *, size_t, size_t, VecSimMetric) const override {} + + // Call only before storing vectors and with concurrent queries excluded. Validation precedes + // publication; once it succeeds neither the engine nor the cached calculator update can throw. + void setMean(std::span values) override { + if constexpr (WithMean) { + if (values.size() != dim_) { + throw std::invalid_argument("Mean vector size must equal dimension"); + } + engine_.setMean(values); + calculator_.setMeanSumSquares(values); + } else { + if (!values.empty()) { + throw std::invalid_argument("Storage context does not use a mean"); + } + } + } + std::span mean() const override { return engine_.getMean(); } + size_t storageBytes() const override { + return StorageType::template storage_bytes_count(dim_); + } + size_t queryBytes() const override { + return dim_ * sizeof(QueryType) + + StorageType::template query_metadata_count() * sizeof(float); + } + QuantizationInfo info() const override { + QuantizationInfo result; + if constexpr (requires { StorageType::quantization_type; }) { + result.type = StorageType::quantization_type; + } + result.vectorBytes = storageBytes(); + result.queryBytes = queryBytes(); + result.sharedBytes = mean().size_bytes(); + return result; + } + + // Includes owned dynamic state; the common estimator separately counts allocate_shared's + // implementation-defined control block and the component wrappers. + static size_t estimateMemory(size_t dim, const QuantizationInit &) { + return sizeof(ScalarStorageContext) + + (WithMean ? dim * sizeof(float) + VecSimAllocator::getAllocationOverheadSize() : 0); + } +#ifdef BUILD_TESTS + unsigned encodingVersion() const override { return 5; } + void saveState(std::ostream &output) const override { + const bool has_mean = WithMean; + output.write(reinterpret_cast(&has_mean), sizeof(has_mean)); + if constexpr (WithMean) { + const auto values = mean(); + output.write(reinterpret_cast(values.data()), values.size_bytes()); + } + } + void validateStoredRecord(const void *) const override {} +#endif + +private: + static Engine makeEngine(const std::shared_ptr &allocator, size_t dim, + const QuantizationInit &init) { + if constexpr (WithMean) { + vecsim_stl::vector values(dim, 0.0f, allocator); + if (!init.prepareForTraining) { + const auto *mean = static_cast(init.parameters); + if (!mean) { + throw std::invalid_argument("Mean-normalized storage requires a mean"); + } + std::copy_n(mean, dim, values.begin()); + } + return Engine(allocator, dim, values); + } else { + return Engine(allocator, dim); + } + } + Calculator makeCalculator(const std::shared_ptr &allocator, size_t dim) { + unsigned char asym_alignment = 0; + auto sym = spaces::GetDistFunc(Metric, dim, &storage_alignment_); + auto asym = + spaces::GetDistFunc(Metric, dim, &asym_alignment); + storage_alignment_ = spaces::combineAlignments(storage_alignment_, asym_alignment); + spaces::GetDistFunc(Metric, dim, &query_alignment_); + if constexpr (WithMean) { + float sum_squares = 0.0f; + for (float value : engine_.getMean()) { + sum_squares += value * value; + } + return Calculator(allocator, asym, sym, sum_squares); + } else { + return Calculator(allocator, sym, asym); + } + } + + const size_t dim_; + unsigned char storage_alignment_ = 0; + unsigned char query_alignment_ = 0; + Engine engine_; + Calculator calculator_; + DistanceDispatch stored_dispatch_; + DistanceDispatch query_dispatch_; +}; diff --git a/src/VecSim/spaces/computer/calculator.h b/src/VecSim/spaces/computer/calculator.h index 230042ab2..02b88912b 100644 --- a/src/VecSim/spaces/computer/calculator.h +++ b/src/VecSim/spaces/computer/calculator.h @@ -250,3 +250,29 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { return DistanceDispatch::stateful(&context_, calcQueryWithContext); } }; + +// Context dispatches are selected once; the index caches these same callbacks and addresses. +template +class QuantizedCalculator : public IndexCalculatorInterface { +public: + QuantizedCalculator(std::shared_ptr allocator, + std::shared_ptr context) + : IndexCalculatorInterface(allocator), context_(std::move(context)), + stored_(context_->getDistanceDispatch(DistanceMode::StoredToStored)), + query_(context_->getDistanceDispatch(DistanceMode::StoredToQuery)) {} + + float calcDistance(const void *lhs, const void *rhs, size_t dim) const override { + return stored_(lhs, rhs, dim); + } + float calcDistanceForQuery(const void *stored, const void *query, size_t dim) const override { + return query_(stored, query, dim); + } + DistanceDispatch getDistanceDispatch(DistanceMode mode) const override { + return mode == DistanceMode::StoredToStored ? stored_ : query_; + } + +private: + std::shared_ptr context_; + DistanceDispatch stored_; + DistanceDispatch query_; +}; diff --git a/src/VecSim/spaces/computer/preprocessor_container.h b/src/VecSim/spaces/computer/preprocessor_container.h index 5f0e74d4b..81543d114 100644 --- a/src/VecSim/spaces/computer/preprocessor_container.h +++ b/src/VecSim/spaces/computer/preprocessor_container.h @@ -56,7 +56,7 @@ class PreprocessorsContainerAbstract : public VecsimBaseObject { MemoryUtils::unique_blob wrapAllocated(void *blob) const { return MemoryUtils::unique_blob( - blob, [this](void *ptr) { this->allocator->free_allocation(ptr); }); + blob, [allocator = this->allocator](void *ptr) { allocator->free_allocation(ptr); }); } static MemoryUtils::unique_blob wrapWithDummyDeleter(void *ptr) { @@ -186,35 +186,33 @@ MultiPreprocessorsContainer::preprocess(const void *o size_t storage_blob_size = input_blob_size; size_t query_blob_size = input_blob_size; - for (auto pp : preprocessors) { - if (!pp) - break; - pp->preprocess(original_blob, storage_blob, query_blob, storage_blob_size, query_blob_size, - this->storage_alignment, this->query_alignment); - } - // At least one blob was allocated. - - // If they point to the same memory, we need to free only one of them. - if (storage_blob == query_blob) { - return ProcessedBlobs(std::move(this->wrapAllocated(storage_blob)), - std::move(Base::wrapWithDummyDeleter(storage_blob))); - } - - if (storage_blob == nullptr) { // we processed only the query - return ProcessedBlobs( - std::move(Base::wrapWithDummyDeleter(const_cast(original_blob))), - std::move(this->wrapAllocated(query_blob))); - } - - if (query_blob == nullptr) { // we processed only the storage - // query might need to be aligned - auto query_ptr = this->maybeCopyToAlignedMem(original_blob, input_blob_size); - return ProcessedBlobs(std::move(this->wrapAllocated(storage_blob)), std::move(query_ptr)); + try { + for (auto pp : preprocessors) { + if (!pp) + break; + pp->preprocess(original_blob, storage_blob, query_blob, storage_blob_size, + query_blob_size, this->storage_alignment, this->query_alignment); + } + } catch (...) { + if (query_blob && query_blob != storage_blob) + this->allocator->free_allocation(query_blob); + if (storage_blob) + this->allocator->free_allocation(storage_blob); + throw; } - - // Else, both were allocated separately, we need to release both. - return ProcessedBlobs(std::move(this->wrapAllocated(storage_blob)), - std::move(this->wrapAllocated(query_blob))); + // Keep both allocations owned while constructing the returned deleters or copying a query. + auto free = [allocator = this->allocator](void *ptr) { allocator->free_allocation(ptr); }; + std::unique_ptr storage_guard(storage_blob, free); + std::unique_ptr query_guard( + query_blob != storage_blob ? query_blob : nullptr, free); + auto storage = storage_blob ? this->wrapAllocated(storage_blob) + : Base::wrapWithDummyDeleter(const_cast(original_blob)); + storage_guard.release(); + auto query = query_blob == storage_blob && query_blob ? Base::wrapWithDummyDeleter(query_blob) + : query_blob ? this->wrapAllocated(query_blob) + : this->maybeCopyToAlignedMem(original_blob, input_blob_size); + query_guard.release(); + return ProcessedBlobs(std::move(storage), std::move(query)); } template @@ -223,15 +221,25 @@ MultiPreprocessorsContainer::preprocessForStorage( const void *original_blob, size_t input_blob_size) const { void *storage_blob = nullptr; - for (auto pp : preprocessors) { - if (!pp) - break; - pp->preprocessForStorage(original_blob, storage_blob, input_blob_size, - this->storage_alignment); + try { + for (auto pp : preprocessors) { + if (!pp) + break; + pp->preprocessForStorage(original_blob, storage_blob, input_blob_size, + this->storage_alignment); + } + } catch (...) { + if (storage_blob) + this->allocator->free_allocation(storage_blob); + throw; } - return storage_blob ? std::move(this->wrapAllocated(storage_blob)) - : std::move(Base::wrapWithDummyDeleter(const_cast(original_blob))); + auto free = [allocator = this->allocator](void *ptr) { allocator->free_allocation(ptr); }; + std::unique_ptr guard(storage_blob, free); + auto result = storage_blob ? this->wrapAllocated(storage_blob) + : Base::wrapWithDummyDeleter(const_cast(original_blob)); + guard.release(); + return result; } template @@ -239,15 +247,25 @@ MemoryUtils::unique_blob MultiPreprocessorsContainer: const void *original_blob, size_t input_blob_size, bool force_copy) const { void *query_blob = nullptr; - for (auto pp : preprocessors) { - if (!pp) - break; - // modifies the memory in place - pp->preprocessQuery(original_blob, query_blob, input_blob_size, this->query_alignment); + try { + for (auto pp : preprocessors) { + if (!pp) + break; + // modifies the memory in place + pp->preprocessQuery(original_blob, query_blob, input_blob_size, this->query_alignment); + } + } catch (...) { + if (query_blob) + this->allocator->free_allocation(query_blob); + throw; } - return query_blob - ? std::move(this->wrapAllocated(query_blob)) - : std::move(this->maybeCopyToAlignedMem(original_blob, input_blob_size, force_copy)); + auto free = [allocator = this->allocator](void *ptr) { allocator->free_allocation(ptr); }; + std::unique_ptr guard(query_blob, free); + auto result = query_blob + ? this->wrapAllocated(query_blob) + : this->maybeCopyToAlignedMem(original_blob, input_blob_size, force_copy); + guard.release(); + return result; } template diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 262474dd5..e6a9c7a34 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -142,7 +142,7 @@ class CosinePreprocessor : public PreprocessorInterface { }; /* - * QuantPreprocessor is a preprocessor that quantizes storage vectors from DataType to a + * ScalarQuantPreprocessor is a preprocessor that quantizes storage vectors from DataType to a * lower precision representation defined by StorageType (SQ8, the current storage type, * stores one uint8_t code per coordinate). * Query vectors remain as DataType for asymmetric distance computation. @@ -232,7 +232,7 @@ class CosinePreprocessor : public PreprocessorInterface { * ||x - y||² = sum_sq_x + sum_sq_y - 2 * IP(x, y) * where sum_sq_x, sum_sq_y are precomputed sums of the squared represented values. */ -// Input types accepted by QuantPreprocessor. Opt-in via std::same_as so unrelated types +// Input types accepted by ScalarQuantPreprocessor. Opt-in via std::same_as so unrelated types // (e.g. integers, double, bfloat16) are rejected at the template head with a named constraint. template concept QuantInput = std::same_as || std::same_as; @@ -259,7 +259,7 @@ static inline T from_fp32(float x) { template -class QuantPreprocessor : public PreprocessorInterface { +class ScalarQuantPreprocessor : public PreprocessorInterface { public: // Center L2 queries in FP32 even when the original input is FP16. using QueryType = std::conditional_t; @@ -270,7 +270,7 @@ class QuantPreprocessor : public PreprocessorInterface { static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP || Metric == VecSimMetric_Cosine, - "QuantPreprocessor only supports L2, IP and Cosine metrics"); + "ScalarQuantPreprocessor only supports L2, IP and Cosine metrics"); static_assert(!WithNorm || Metric != VecSimMetric_Cosine, "WithNorm does not support Cosine metric."); @@ -489,7 +489,7 @@ class QuantPreprocessor : public PreprocessorInterface { } // Standard constructor (WithNorm == false): no mean vector. - QuantPreprocessor(std::shared_ptr allocator, size_t dim) + ScalarQuantPreprocessor(std::shared_ptr allocator, size_t dim) requires(!WithNorm) : PreprocessorInterface(allocator), dim(dim), storage_bytes_count(StorageType::template storage_bytes_count(dim)), @@ -498,8 +498,8 @@ class QuantPreprocessor : public PreprocessorInterface { sizeof(MetadataType)) {} // WithNorm constructor: accepts a pre-computed mean vector (FP32, length == dim). - QuantPreprocessor(std::shared_ptr allocator, size_t dim, - const vecsim_stl::vector &mean_vec) + ScalarQuantPreprocessor(std::shared_ptr allocator, size_t dim, + const vecsim_stl::vector &mean_vec) requires(WithNorm) : PreprocessorInterface(allocator), mean(mean_vec), dim(dim), storage_bytes_count(StorageType::template storage_bytes_count(dim)), @@ -558,10 +558,7 @@ class QuantPreprocessor : public PreprocessorInterface { assert(!blob && "storage_blob must be nullptr"); blob = this->allocator->allocate_aligned(storage_bytes_count, storage_alignment); - // Cast to appropriate types - const DataType *input = static_cast(original_blob); - OUTPUT_TYPE *quantized = static_cast(blob); - quantize(input, quantized); + encodeStorage(original_blob, blob); input_blob_size = storage_bytes_count; } @@ -592,6 +589,15 @@ class QuantPreprocessor : public PreprocessorInterface { // Allocate aligned memory for the query blob blob = this->allocator->allocate_aligned(this->query_bytes_count, alignment); + prepareQuery(original_blob, blob); + query_blob_size = this->query_bytes_count; + } + + void encodeStorage(const void *input, void *output) const { + quantize(static_cast(input), static_cast(output)); + } + + void prepareQuery(const void *original_blob, void *blob) const { const size_t body_bytes = this->dim * sizeof(QueryType); const DataType *input = static_cast(original_blob); QueryType *query_values = static_cast(blob); @@ -607,8 +613,13 @@ class QuantPreprocessor : public PreprocessorInterface { // body_bytes, which is not guaranteed to be 4-byte aligned for FP16 query bodies. void *metadata_dst = static_cast(blob) + body_bytes; assign_query_metadata(query_values, input, metadata_dst); + } - query_blob_size = this->query_bytes_count; + std::span getMean() const { + if constexpr (WithNorm) { + return {mean.data(), mean.size()}; + } + return {}; } void preprocessStorageInPlace(void *original_blob, size_t input_blob_size) const override { @@ -663,3 +674,100 @@ class QuantPreprocessor : public PreprocessorInterface { const size_t storage_bytes_count; const size_t query_bytes_count; }; + +// Storage policies may replace the scalar family's context while retaining the same wrappers. +template +class ScalarStorageContext; + +template +struct StorageContextSelector { + using type = ScalarStorageContext; +}; + +template +struct StorageContextSelector< + StorageType, Input, Metric, WithMean, + std::void_t>> { + using type = typename StorageType::template Context; +}; + +template +using StorageContextFor = + typename StorageContextSelector::type; + +#include "VecSim/quantization/quantization_runtime.h" + +template +class QuantPreprocessor : public PreprocessorInterface { +public: + using Context = StorageContextFor; + using QueryType = typename Context::QueryType; + + QuantPreprocessor(std::shared_ptr allocator, std::shared_ptr context) + : PreprocessorInterface(allocator), context_(std::move(context)) {} + + QuantPreprocessor(std::shared_ptr allocator, size_t dim) + requires(!WithMean) + : QuantPreprocessor(allocator, + std::allocate_shared(VecsimSTLAllocator(allocator), + allocator, dim, QuantizationInit{})) {} + + QuantPreprocessor(std::shared_ptr allocator, size_t dim, + const vecsim_stl::vector &mean) + requires(WithMean) + : QuantPreprocessor(allocator, std::allocate_shared( + VecsimSTLAllocator(allocator), allocator, dim, + QuantizationInit{.parameters = mean.data()})) { + assert(mean.size() == dim); + } + + void setMean(std::span mean) + requires(WithMean) + { + context_->setMean(mean); + } + + void preprocess(const void *input, void *&storage, void *&query, size_t &storageBytes, + size_t &queryBytes, unsigned char storageAlignment, + unsigned char queryAlignment) const override { + assert(!storage && !query); + assert(&storageBytes != &queryBytes); + auto storage_output = + this->allocator->allocate_aligned_unique(context_->storageBytes(), storageAlignment); + auto query_output = + this->allocator->allocate_aligned_unique(context_->queryBytes(), queryAlignment); + context_->encodeStorage(input, storage_output.get()); + context_->prepareQuery(input, query_output.get()); + storage = storage_output.release(); + query = query_output.release(); + storageBytes = context_->storageBytes(); + queryBytes = context_->queryBytes(); + } + void preprocessForStorage(const void *input, void *&output, size_t &bytes, + unsigned char alignment) const override { + assert(!output); + auto prepared = + this->allocator->allocate_aligned_unique(context_->storageBytes(), alignment); + context_->encodeStorage(input, prepared.get()); + output = prepared.release(); + bytes = context_->storageBytes(); + } + void preprocessQuery(const void *input, void *&output, size_t &bytes, + unsigned char alignment) const override { + assert(!output); + auto prepared = this->allocator->allocate_aligned_unique(context_->queryBytes(), alignment); + context_->prepareQuery(input, prepared.get()); + output = prepared.release(); + bytes = context_->queryBytes(); + } + void preprocessStorageInPlace(void *input, size_t bytes) const override { + context_->encodeStorageInPlace(input, bytes); + } + +private: + std::shared_ptr context_; +}; + +#include "VecSim/quantization/storage_context.h" diff --git a/src/VecSim/types/sq8.h b/src/VecSim/types/sq8.h index 27636ac91..1d4165bba 100644 --- a/src/VecSim/types/sq8.h +++ b/src/VecSim/types/sq8.h @@ -18,6 +18,7 @@ namespace vecsim_types { // Represents a scalar-quantized 8-bit blob with reconstruction metadata struct sq8 { using value_type = uint8_t; + static constexpr VecSimQuantType quantization_type = VecSimQuant_SQ8; static constexpr uint8_t max_code = std::numeric_limits::max(); static constexpr size_t data_bytes_count(size_t dim) { return dim; } diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 419aa9884..8767bb915 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -428,7 +428,12 @@ typedef struct { size_t max_level; // Number of graph levels. size_t entrypoint; // Entrypoint vector label. size_t visitedNodesPoolSize; // The max number of parallel graph scans so far. - size_t numberOfMarkedDeletedNodes; // The number of nodes that are marked as deleted. + size_t numberOfMarkedDeletedNodes; + VecSimQuantType quantizationType; + size_t quantizedVectorBytes; + size_t quantizerSharedBytes; + const char *quantizerQueryKernel; + const char *quantizerGraphKernel; } hnswInfoStruct; typedef struct { @@ -447,7 +452,7 @@ typedef struct { size_t numThreads; // Maximum number of threads to be used by svs for ingestion. size_t lastReservedThreads; // Number of threads that were successfully reserved by the last // ingestion operation. - size_t numberOfMarkedDeletedNodes; // The number of nodes that are marked as deleted. + size_t numberOfMarkedDeletedNodes; size_t searchWindowSize; // Search window size for Vamana graph accuracy/latency tune. size_t searchBufferCapacity; // Search buffer capacity for Vamana graph accuracy/latency tune. size_t leanvecDim; // Leanvec dimension to use when LeanVec is enabled. diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index fbdab3a67..64e884057 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -15,6 +15,7 @@ #include "VecSim/utils/vec_utils.h" #include "VecSim/spaces/spaces.h" #include "VecSim/spaces/computer/calculator.h" +#include "VecSim/quantization/quantization_runtime.h" #include "VecSim/spaces/computer/preprocessor_container.h" #include "info_iterator_struct.h" #include "containers/data_blocks_container.h" @@ -66,6 +67,8 @@ template struct IndexComponents { IndexCalculatorInterface *indexCalculator; PreprocessorsContainerAbstract *preprocessors; + // Alias owned by the component wrappers, never deleted separately. + QuantizationRuntime *quantization = nullptr; }; /** @@ -91,6 +94,7 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { DistanceDispatch storedDistanceDispatch; DistanceDispatch queryDistanceDispatch; PreprocessorsContainerAbstract *preprocessors; // Storage and query preprocessors. + QuantizationRuntime *quantization; // Non-owning alias into the component context. size_t inputBlobSize; // The size of input vectors/queries blob in bytes. May differ from dim * // sizeof(vecType) when vectors have been externally preprocessed (e.g., @@ -138,8 +142,8 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { components.indexCalculator ? components.indexCalculator->getDistanceDispatch(DistanceMode::StoredToQuery) : DistanceDispatch{}), - preprocessors(components.preprocessors), inputBlobSize(params.inputBlobSize), - storedDataSize(params.storedDataSize) { + preprocessors(components.preprocessors), quantization(components.quantization), + inputBlobSize(params.inputBlobSize), storedDataSize(params.storedDataSize) { assert(VecSimType_sizeof(vecType)); assert(storedDataSize); assert(inputBlobSize); @@ -228,6 +232,14 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { inline void setLastSearchMode(VecSearchMode mode) override { this->lastMode = mode; } inline bool isMultiValue() const { return isMulti; } inline bool usesQuantizedStorage() const { return isQuantized; } + QuantizationRuntime *getQuantizationRuntime() const { return quantization; } + + // Validate the complete input before a replacement can remove existing rows. + void validateQuantizationInput(const void *rows, size_t count, size_t rowBytes, + VecSimMetric inputMetric) const { + if (quantization) + quantization->validateBatch(rows, count, rowBytes, inputMetric); + } inline VecSimType getType() const { return vecType; } inline VecSimMetric getMetric() const { return metric; } inline size_t getStoredDataSize() const { return storedDataSize; } @@ -365,7 +377,9 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { } IndexComponents get_components() const { - return {.indexCalculator = this->indexCalculator, .preprocessors = this->preprocessors}; + return {.indexCalculator = this->indexCalculator, + .preprocessors = this->preprocessors, + .quantization = this->quantization}; } // Test-only: override the disk flag to exercise disk-specific param resolution diff --git a/tests/unit/test_components.cpp b/tests/unit/test_components.cpp index 6744756b3..5bc83a6c5 100644 --- a/tests/unit/test_components.cpp +++ b/tests/unit/test_components.cpp @@ -1520,8 +1520,7 @@ void checkSQ8Bytes(size_t dim) { EXPECT_EQ(encode(QuantPreprocessor(allocator, dim)), expected); } - // The SQ8 factory wrapper must produce the same bytes and keep naming the SQ8 component - // types that applySQ8Mean dynamic_casts to. + // The SQ8 factory must preserve scalar bytes and the selected cached distance kernels. if constexpr (std::is_same_v && Metric != VecSimMetric_Cosine) { auto components = CreateSQ8IndexComponents(allocator, dim, WithMean ? mean.data() : nullptr); @@ -1533,14 +1532,31 @@ void checkSQ8Bytes(size_t dim) { EXPECT_NE((dynamic_cast *>( container->getPreprocessors()[0])), nullptr); - if constexpr (WithMean) { - EXPECT_NE((dynamic_cast *>( - calculator.get())), - nullptr); - } auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); const auto *factory_bytes = static_cast(blob.get()); EXPECT_EQ(std::vector(factory_bytes, factory_bytes + expected.size()), expected); + auto query = preprocessors->preprocessQuery(input.data(), dim * sizeof(float)); + auto stored_dispatch = calculator->getDistanceDispatch(DistanceMode::StoredToStored); + auto query_dispatch = calculator->getDistanceDispatch(DistanceMode::StoredToQuery); + unsigned char sym_alignment = 0, asym_alignment = 0; + auto sym = spaces::GetDistFunc(Metric, dim, &sym_alignment); + using QueryType = typename QuantPreprocessor::QueryType; + auto asym = spaces::GetDistFunc(Metric, dim, &asym_alignment); + if constexpr (WithMean) { + float mean_sum_squares = 0.0f; + for (float value : mean) + mean_sum_squares += value * value; + DistanceCalculatorWithNorm legacy(allocator, asym, sym, + mean_sum_squares); + EXPECT_FLOAT_EQ(stored_dispatch(blob.get(), blob.get(), dim), + legacy.calcDistance(blob.get(), blob.get(), dim)); + EXPECT_FLOAT_EQ(query_dispatch(blob.get(), query.get(), dim), + legacy.calcDistanceForQuery(blob.get(), query.get(), dim)); + } + if constexpr (!WithMean || Metric == VecSimMetric_L2) { + EXPECT_EQ(stored_dispatch.stateless_func, sym); + EXPECT_EQ(query_dispatch.stateless_func, asym); + } } } @@ -1562,6 +1578,58 @@ TEST(PreprocessorsTest, QuantizedStorageSQ8BytesUnchanged) { checkSQ8BytesAllModes(); } +TEST(PreprocessorsTest, QuantizedBlobsOutliveComponents) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto baseline = allocator->getAllocationSize(); + constexpr size_t dim = 8; + const float input[dim] = {1, 2, 3, 4, 5, 6, 7, 8}; + auto components = CreateSQ8IndexComponents(allocator, dim, nullptr); + EXPECT_EQ(allocator->getAllocationSize() - baseline, + (EstimateQuantizedComponentsMemory( + dim, QuantizationInit{}))); + auto blobs = components.preprocessors->preprocess(input, sizeof(input)); + delete components.preprocessors; + delete components.indexCalculator; + EXPECT_EQ(std::memcmp(blobs.getQueryBlob(), input, sizeof(input)), 0); + blobs = ProcessedBlobs{}; + EXPECT_EQ(allocator->getAllocationSize(), baseline); +} + +namespace { + +template +class ThrowingQueryContext + : public ScalarStorageContext { + using Base = ScalarStorageContext; + +public: + using Base::Base; + void prepareQuery(const void *, void *) const { throw std::runtime_error("Query preparation"); } +}; + +struct ThrowingQueryStorage { + template + using Context = ThrowingQueryContext; +}; + +} // namespace + +TEST(PreprocessorsTest, QuantizedCombinedPreparationFailureReleasesBothOutputs) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto baseline = allocator->getAllocationSize(); + constexpr size_t dim = 8; + const float input[dim] = {1, 2, 3, 4, 5, 6, 7, 8}; + auto components = + CreateQuantizedIndexComponents( + allocator, dim, QuantizationInit{}); + const auto component_bytes = allocator->getAllocationSize(); + EXPECT_THROW(components.preprocessors->preprocess(input, sizeof(input)), std::runtime_error); + EXPECT_EQ(allocator->getAllocationSize(), component_bytes); + delete components.preprocessors; + delete components.indexCalculator; + EXPECT_EQ(allocator->getAllocationSize(), baseline); +} + // Parameterized test class for QuantPreprocessor with different metrics class QuantPreprocessorMetricTest : public testing::TestWithParam { protected: From 2a15413211d250b249fc08099a2169c20443ca1c Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 8 Nov 2026 15:41:18 +0200 Subject: [PATCH 08/10] Revert "Generalize quantized components around storage contexts" This reverts commit 6a60e4087e8d31c8cdd78ad06c260fed252c9279. --- research/storage-context/README.md | 30 ---- src/VecSim/algorithms/hnsw/hnsw.h | 71 +++----- .../algorithms/hnsw/hnsw_serializer_impl.h | 17 +- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 4 - .../components/components_factory.h | 150 +++++------------ src/VecSim/index_factories/hnsw_factory.cpp | 47 +++--- src/VecSim/index_factories/hnsw_factory.h | 6 +- src/VecSim/index_factories/tiered_factory.cpp | 32 +++- .../quantization/quantization_runtime.h | 44 ----- src/VecSim/quantization/sq8_mean.h | 31 ++++ src/VecSim/quantization/storage_context.h | 154 ------------------ src/VecSim/spaces/computer/calculator.h | 26 --- .../spaces/computer/preprocessor_container.h | 106 +++++------- src/VecSim/spaces/computer/preprocessors.h | 132 ++------------- src/VecSim/types/sq8.h | 1 - src/VecSim/vec_sim_common.h | 9 +- src/VecSim/vec_sim_index.h | 20 +-- tests/unit/test_components.cpp | 82 +--------- 18 files changed, 235 insertions(+), 727 deletions(-) delete mode 100644 research/storage-context/README.md delete mode 100644 src/VecSim/quantization/quantization_runtime.h create mode 100644 src/VecSim/quantization/sq8_mean.h delete mode 100644 src/VecSim/quantization/storage_context.h diff --git a/research/storage-context/README.md b/research/storage-context/README.md deleted file mode 100644 index 3a291cbfa..000000000 --- a/research/storage-context/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Shared quantized storage contexts - -StorageType selects a typed context. The same QuantPreprocessor, QuantizedCalculator, -component factory and backend lifecycle serve scalar and transformed quantizers. -The scalar-family default context retains SQ8 layout hooks and existing encoding, -correction arithmetic and optimized kernel selection. A non-scalar storage type -can supply `template using Context`. - -The context owns encoding, prepared-query layout, dispatch, mean installation, -validation and fitted-state persistence. It has a stable address. Component wrappers -share ownership through VecSimAllocator-backed shared_ptrs; the index keeps a non-owning -runtime alias. A scoring call uses the existing cached dispatch, without a new -per-candidate virtual call. The shared control allocation and mean buffer are included -in the component estimate; shared allocation size is measured before context -construction, without fitting data or allocating a rotation. - -Tiered construction requests a backend prepared for the regular training phase. -It does not interpret storage-specific parameters. Mean installation occurs with -queries excluded and before the first stored vector. Failed preparation retains -previous fitted state. Input validation precedes destructive replacements. - -This foundation introduces no new accepted quantization mode. Existing scalar V5 -payloads and nonquantized V4 payloads retain their layouts. Serialization remains -available only with test helpers. Existing SQ4 layout/code traits can use the scalar -default context; TQ2/TQ3 are supplied separately by the stacked draft. - -Regression coverage checks SQ8 stored bytes against independent expected bytes, -legacy distance results and stateless function pointers, common component allocation -estimates, combined preprocessing failure cleanup and allocated buffer lifetime. -See the stacked TQ draft's validation receipt for current ARM execution evidence. diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h index 5e1ee2787..11b4247ef 100644 --- a/src/VecSim/algorithms/hnsw/hnsw.h +++ b/src/VecSim/algorithms/hnsw/hnsw.h @@ -21,6 +21,7 @@ #include "VecSim/vec_sim_common.h" #include "VecSim/vec_sim_index.h" #include "VecSim/spaces/computer/preprocessors.h" +#include "VecSim/quantization/sq8_mean.h" #include #include "VecSim/tombstone_interface.h" @@ -91,22 +92,35 @@ class HNSWIndex : public VecSimIndexAbstract, public HNSWSerializer #endif { +private: + template + void setSQ8Mean(std::span mean) noexcept { + applySQ8Mean(*this->getPreprocessorsContainer(), + *this->getIndexCalculator(), mean); + } + public: - // Install the mean before storing any vector, with queries excluded by the caller. - // The storage context prepares all fallible state before publishing the new mean; - // cached scoring callbacks keep referring to the same context throughout. - void setQuantizationMean(std::span mean) { + // Writer-only operation: install the mean once, before any stored vector, under the + // exclusive tiered main lock and before submitting insertion jobs. The backend must have + // its final SQ8 WithNorm components; the caller owns and enforces the one-time transition. + // A quantizer that learns storedDataSize from data needs deferred backend creation. + void setQuantizationMean(std::span mean) noexcept { assert(this->isQuantized && curElementCount == 0 && mean.size() == this->dim); - assert(this->getQuantizationRuntime()); - this->getQuantizationRuntime()->setMean(mean); + if constexpr (QuantInput && std::is_same_v) { + if (this->metric == VecSimMetric_L2) { + setSQ8Mean(mean); + } else { + // Tiered cosine inputs are already normalized, and use IP SQ8 components. + assert(this->metric == VecSimMetric_IP || this->metric == VecSimMetric_Cosine); + setSQ8Mean(mean); + } #ifdef BUILD_TESTS - // Compatibility view for existing scalar serialization tests. State is saved by - // the context; only an already allocated view is updated here. - if (!serializedMeanVector.empty()) { - assert(serializedMeanVector.size() == mean.size()); + assert(serializedMeanVector.size() == this->dim); std::copy(mean.begin(), mean.end(), serializedMeanVector.begin()); - } #endif + } else { + assert(false && "Unsupported SQ8 data type"); + } } protected: @@ -633,7 +647,6 @@ VecSimUpdateCode HNSWIndex::updateVectors(labelType label, return VecSimUpdate_OK; } - this->validateQuantizationInput(new_blobs, n, this->getInputBlobSize(), this->metric); this->deleteVector(label); const char *blob = static_cast(new_blobs); for (size_t i = 0; i < n; i++) { @@ -2446,7 +2459,7 @@ VecSimQueryReply *HNSWIndex::rangeQuery(const void *query_da template VecSimIndexDebugInfo HNSWIndex::debugInfo() const { - VecSimIndexDebugInfo info{}; + VecSimIndexDebugInfo info; info.commonInfo = this->getCommonInfo(); auto [ep_id, max_level] = this->safeGetEntryPointState(); @@ -2459,14 +2472,6 @@ VecSimIndexDebugInfo HNSWIndex::debugInfo() const { info.hnswInfo.entrypoint = ep_id != INVALID_ID ? getExternalLabel(ep_id) : INVALID_LABEL; info.hnswInfo.visitedNodesPoolSize = this->visitedNodesHandlerPool.getPoolSize(); info.hnswInfo.numberOfMarkedDeletedNodes = this->getNumMarkedDeleted(); - if (const auto *context = this->getQuantizationRuntime()) { - const auto quant = context->info(); - info.hnswInfo.quantizationType = quant.type; - info.hnswInfo.quantizedVectorBytes = quant.vectorBytes; - info.hnswInfo.quantizerSharedBytes = quant.sharedBytes; - info.hnswInfo.quantizerQueryKernel = quant.queryKernel; - info.hnswInfo.quantizerGraphKernel = quant.graphKernel; - } return info; } @@ -2533,30 +2538,6 @@ VecSimDebugInfoIterator *HNSWIndex::debugInfoIterator() cons .fieldType = INFOFIELD_UINT64, .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.numberOfMarkedDeletedNodes}}}); - if (const auto *context = this->getQuantizationRuntime(); context && context->info().name) { - const auto quant = context->info(); - auto integer = [&](const char *name, uint64_t value) { - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = name, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = value}}}); - }; - auto string = [&](const char *name, const char *value) { - if (value) - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = name, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = value}}}); - }; - string("QUANTIZATION", quant.name); - string("QUANT_SCALE", quant.scale); - string("QUANT_BIAS", quant.bias); - integer("QUANT_VECTOR_BYTES", quant.vectorBytes); - integer("QUANT_SHARED_BYTES", quant.sharedBytes); - integer("QUANT_QUERY_BYTES", quant.queryBytes); - string("QUANT_QUERY_KERNEL", quant.queryKernel); - string("QUANT_GRAPH_KERNEL", quant.graphKernel); - } return infoIterator; } diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h index bee8a7391..2865c5d81 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h @@ -43,9 +43,8 @@ HNSWIndex::HNSWIndex(std::ifstream &input, const HNSWParams template HNSWSerializer::EncodingVersion HNSWIndex::getWriteVersion() const { - const auto *context = this->getQuantizationRuntime(); - return context ? static_cast(context->encodingVersion()) - : HNSWSerializer::EncodingVersion::V4; + return this->isQuantized ? HNSWSerializer::EncodingVersion::V5 + : HNSWSerializer::EncodingVersion::V4; } template @@ -264,10 +263,16 @@ void HNSWIndex::saveIndexFields(std::ofstream &output) const writeBinaryPOD(output, this->maxElements); // This will be used to restore the index initial // capacity - // The graph owns framing; the storage context owns its fitted-state payload. - if (const auto *context = this->getQuantizationRuntime()) { + // V5 fields: scalar quantization support (between factory params and index build params) + if (this->quantType != VecSimQuant_NONE) { writeBinaryPOD(output, this->quantType); - context->saveState(output); + // Write mean vector (dim floats). Empty vector means no mean (zero-mean quantization). + bool hasMean = !this->serializedMeanVector.empty(); + writeBinaryPOD(output, hasMean); + if (hasMean) { + output.write(reinterpret_cast(this->serializedMeanVector.data()), + this->dim * sizeof(float)); + } } // Save index build parameters diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 90f80e5df..7f8fea28a 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -946,8 +946,6 @@ HNSWInsertJob *TieredHNSWIndex::addVectorToFlat(const void * // parameters. template int TieredHNSWIndex::addVector(const void *blob, labelType label) { - getHNSWIndex()->validateQuantizationInput(blob, 1, this->frontendIndex->getInputBlobSize(), - this->frontendIndex->getMetric()); if (sqAccumulationState) { return addVectorDuringAccumulation(blob, label); } @@ -1107,8 +1105,6 @@ VecSimUpdateCode TieredHNSWIndex::updateVectors(labelType la return updateVectorSingleValue(label, new_blobs, n); // -- Multi Value -- - getHNSWIndex()->validateQuantizationInput(new_blobs, n, this->frontendIndex->getInputBlobSize(), - this->frontendIndex->getMetric()); this->deleteVector(label); const char *blob = static_cast(new_blobs); for (size_t i = 0; i < n; i++) { diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index 220265552..39fc2486a 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -40,127 +40,61 @@ template return alignment; } -// A prebuilt context allows callers with fitted typed state to use the common wrappers. -template -IndexComponents CreateQuantizedIndexComponents( - const std::shared_ptr &allocator, size_t, - std::shared_ptr> context) { - using Preprocessor = QuantPreprocessor; - using Context = typename Preprocessor::Context; - auto calculator = std::unique_ptr>( - new (allocator) QuantizedCalculator(allocator, context)); - auto preprocessor = - std::unique_ptr(new (allocator) Preprocessor(allocator, context)); - auto container = std::unique_ptr>( - new (allocator) MultiPreprocessorsContainer( - allocator, context->queryAlignment(), context->storageAlignment())); - [[maybe_unused]] const int ret = container->addPreprocessor(preprocessor.get()); - assert(ret != -1 && "Quantization preprocessor was not added correctly"); - preprocessor.release(); - return {calculator.release(), container.release(), context.get()}; -} - -template -IndexComponents -CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, - const QuantizationInit &init) { - using Context = StorageContextFor; - auto context = - std::allocate_shared(VecsimSTLAllocator(allocator), allocator, dim, init); - return CreateQuantizedIndexComponents( - allocator, dim, std::move(context)); -} - template IndexComponents CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, const float *mean_ptr) { - return CreateQuantizedIndexComponents( - allocator, dim, QuantizationInit{.parameters = mean_ptr}); -} + using Preprocessor = QuantPreprocessor; + using QueryType = typename Preprocessor::QueryType; + unsigned char storage_alignment = 0, asym_storage_alignment = 0; + + // Graph construction compares two stored quantized blobs; search compares a stored blob with a + // QueryType query. Both dispatchers report alignment for the stored operand. + auto sym_func = spaces::GetDistFunc(Metric, dim, &storage_alignment); + auto asym_func = + spaces::GetDistFunc(Metric, dim, &asym_storage_alignment); + storage_alignment = spaces::combineAlignments(storage_alignment, asym_storage_alignment); + const unsigned char query_alignment = GetQueryAlignment(Metric, dim); + + PreprocessorInterface *pp = nullptr; + IndexCalculatorInterface *calc = nullptr; + + if constexpr (WithMean) { + vecsim_stl::vector mean_vec(allocator); + mean_vec.assign(mean_ptr, mean_ptr + dim); + + float mean_sum_squares = 0.0f; + for (float v : mean_vec) { + mean_sum_squares += v * v; + } -// The context interprets parameters; this boundary only selects the mean-aware type. -template -IndexComponents -CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, - const QuantizationInit &init) { - if (init.parameters || init.prepareForTraining) { - return CreateQuantizedIndexComponents(allocator, dim, - init); + pp = new (allocator) Preprocessor(allocator, dim, mean_vec); + calc = new (allocator) DistanceCalculatorWithNorm( + allocator, asym_func, sym_func, mean_sum_squares); + } else { + pp = new (allocator) Preprocessor(allocator, dim); + calc = new (allocator) DistanceCalculatorCommon(allocator, sym_func, asym_func); } - return CreateQuantizedIndexComponents(allocator, dim, - init); + + auto *container = new (allocator) + MultiPreprocessorsContainer(allocator, query_alignment, storage_alignment); + [[maybe_unused]] const int ret = container->addPreprocessor(pp); + assert(ret != -1 && "Quantization preprocessor was not added correctly"); + + return {calc, container}; } +// Run-time overload: selects WithMean from whether mean_ptr is set. template IndexComponents CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, const float *mean_ptr) { - return CreateQuantizedIndexComponents( - allocator, dim, QuantizationInit{.parameters = mean_ptr}); -} - -namespace quantization_detail { - -struct AllocationMeasured {}; -inline size_t &sharedAllocationBytes() { - static thread_local size_t bytes = 0; - return bytes; -} - -// The control block layout is implementation-defined. Use the same allocator state size and -// alignment as VecsimSTLAllocator so its rebound allocate_shared block has the same layout. -// Stop at allocation: estimating never constructs a context or fits quantization state. -template -struct SharedAllocationProbe { - using value_type = T; - std::shared_ptr allocator; - SharedAllocationProbe() = default; - template - SharedAllocationProbe(const SharedAllocationProbe &other) : allocator(other.allocator) {} - T *allocate(size_t count) { - sharedAllocationBytes() = count * sizeof(T); - throw AllocationMeasured{}; - } - void deallocate(T *, size_t) {} -}; - -template -size_t contextControlBytes() { - static_assert(sizeof(SharedAllocationProbe) == sizeof(VecsimSTLAllocator)); - static_assert(alignof(SharedAllocationProbe) == alignof(VecsimSTLAllocator)); - static const size_t bytes = [] { - try { - std::allocate_shared(SharedAllocationProbe{}, - std::shared_ptr{}, size_t{1}, - QuantizationInit{}); - } catch (const AllocationMeasured &) { - return sharedAllocationBytes() - sizeof(Context); - } - std::terminate(); - }(); - return bytes; -} - -} // namespace quantization_detail - -template -size_t EstimateQuantizedComponentsMemory(size_t dim, const QuantizationInit &init) { - using Context = StorageContextFor; - const size_t overhead = VecSimAllocator::getAllocationOverheadSize(); - return Context::estimateMemory(dim, init) + - quantization_detail::contextControlBytes() + overhead + - sizeof(QuantPreprocessor) + overhead + - sizeof(QuantizedCalculator) + overhead + - sizeof(MultiPreprocessorsContainer) + overhead; -} - -template -size_t EstimateQuantizedComponentsMemory(size_t dim, const QuantizationInit &init) { - if (init.parameters || init.prepareForTraining) { - return EstimateQuantizedComponentsMemory(dim, init); + if (mean_ptr) { + return CreateQuantizedIndexComponents(allocator, dim, + mean_ptr); } - return EstimateQuantizedComponentsMemory(dim, init); + return CreateQuantizedIndexComponents(allocator, dim, + mean_ptr); } template diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index 4f80a1537..15a7d54f3 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -72,34 +72,35 @@ template template VecSimIndex *NewIndex_Quantized(const HNSWParams *hnswParams, - AbstractIndexInitParams abstractInitParams, - const QuantizationInit &init) { + AbstractIndexInitParams abstractInitParams, const float *mean_ptr) { + const bool with_norm = mean_ptr != nullptr; + abstractInitParams.storedDataSize = + GetQuantizedStoredDataSize(abstractInitParams.dim, with_norm); abstractInitParams.isQuantized = true; IndexComponents components = - CreateQuantizedIndexComponents(abstractInitParams.allocator, - abstractInitParams.dim, init); - abstractInitParams.storedDataSize = components.quantization->storageBytes(); + CreateQuantizedIndexComponents( + abstractInitParams.allocator, abstractInitParams.dim, mean_ptr); auto index = NewIndex_ChooseMultiOrSingle(hnswParams, abstractInitParams, components); #ifdef BUILD_TESTS // Store quantization metadata for re-serialization. index->quantType = hnswParams->quantType; - const auto mean = components.quantization->mean(); - index->serializedMeanVector.assign(mean.begin(), mean.end()); + if (mean_ptr != nullptr) { + index->serializedMeanVector.assign(mean_ptr, mean_ptr + abstractInitParams.dim); + } #endif return index; } -VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized, bool prepareForTraining) { +VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { const HNSWParams *hnswParams = ¶ms->algoParams.hnswParams; AbstractIndexInitParams abstractInitParams = VecSimFactory::NewAbstractInitParams(hnswParams, params->logCtx, is_normalized); if (hnswParams->quantType != VecSimQuant_NONE) { const VecSimMetric metric = ResolveQuantizedMetric(hnswParams->metric, is_normalized); - const QuantizationInit init{.parameters = hnswParams->quantParams, - .prepareForTraining = prepareForTraining}; + const float *mean_ptr = static_cast(hnswParams->quantParams); if (!QuantizationParamsSupported(hnswParams->quantType, hnswParams->type, metric)) { return NULL; @@ -108,18 +109,18 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized, bool prepa if (hnswParams->type == VecSimType_FLOAT32) { if (metric == VecSimMetric_L2) { return NewIndex_Quantized(hnswParams, abstractInitParams, - init); + mean_ptr); } else if (metric == VecSimMetric_IP) { return NewIndex_Quantized(hnswParams, abstractInitParams, - init); + mean_ptr); } } else if (hnswParams->type == VecSimType_FLOAT16) { if (metric == VecSimMetric_L2) { return NewIndex_Quantized(hnswParams, abstractInitParams, - init); + mean_ptr); } else if (metric == VecSimMetric_IP) { return NewIndex_Quantized(hnswParams, abstractInitParams, - init); + mean_ptr); } } @@ -165,9 +166,9 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized, bool prepa return NULL; } -VecSimIndex *NewIndex(const HNSWParams *params, bool is_normalized, bool prepareForTraining) { +VecSimIndex *NewIndex(const HNSWParams *params, bool is_normalized) { VecSimParams vecSimParams = {.algoParams = {.hnswParams = HNSWParams{*params}}}; - return NewIndex(&vecSimParams, is_normalized, prepareForTraining); + return NewIndex(&vecSimParams); } template @@ -195,13 +196,17 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi ResolveQuantizedMetric(params->metric, is_normalized))) { throw std::invalid_argument("Unsupported quantization params for HNSW index"); } - const QuantizationInit init{.prepareForTraining = with_mean}; + // Template arguments do not affect these component sizes. if (with_mean) { - est += EstimateQuantizedComponentsMemory(params->dim, - init); + est += allocations_overhead + + sizeof(DistanceCalculatorWithNorm); + est += allocations_overhead + sizeof(MultiPreprocessorsContainer); + est += allocations_overhead + sizeof(QuantPreprocessor); + est += allocations_overhead + params->dim * sizeof(float); } else { - est += EstimateQuantizedComponentsMemory( - params->dim, init); + est += allocations_overhead + sizeof(DistanceCalculatorCommon); + est += allocations_overhead + sizeof(MultiPreprocessorsContainer); + est += allocations_overhead + sizeof(QuantPreprocessor); } est += EstimateInitialSize_ChooseMultiOrSingle(params->multi); } else if (params->type == VecSimType_FLOAT32) { diff --git a/src/VecSim/index_factories/hnsw_factory.h b/src/VecSim/index_factories/hnsw_factory.h index 4cb86fd57..41f2399cc 100644 --- a/src/VecSim/index_factories/hnsw_factory.h +++ b/src/VecSim/index_factories/hnsw_factory.h @@ -22,10 +22,8 @@ namespace HNSWFactory { * the blobs sent to the index are already normalized. For example, in case it's a tiered index, * where the blobs are normalized by the frontend index. */ -VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized = false, - bool prepareForTraining = false); -VecSimIndex *NewIndex(const HNSWParams *params, bool is_normalized = false, - bool prepareForTraining = false); +VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized = false); +VecSimIndex *NewIndex(const HNSWParams *params, bool is_normalized = false); size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized = false); size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool with_mean); size_t EstimateElementSize(const HNSWParams *params); diff --git a/src/VecSim/index_factories/tiered_factory.cpp b/src/VecSim/index_factories/tiered_factory.cpp index 3307ee687..c638a8495 100644 --- a/src/VecSim/index_factories/tiered_factory.cpp +++ b/src/VecSim/index_factories/tiered_factory.cpp @@ -40,16 +40,42 @@ static inline BFParams NewBFParams(const TieredIndexParams *params) { } static inline bool RequiresQuantizationTraining(const TieredIndexParams *params) { - return params->primaryIndexParams->algoParams.hnswParams.quantType != VecSimQuant_NONE && + return params->primaryIndexParams->algoParams.hnswParams.quantType == VecSimQuant_SQ8 && params->specificParams.tieredHnswParams.QuantNormalizationSetSize > 0; } +template +static inline bool IsQuantizationSupported(const TieredIndexParams *params) { + const auto &hnsw_params = params->primaryIndexParams->algoParams.hnswParams; + if (hnsw_params.quantType == VecSimQuant_NONE) { + return true; + } + + if constexpr (!QuantInput || !std::is_same_v) { + return false; + } else { + return hnsw_params.quantType == VecSimQuant_SQ8; + } +} + template inline VecSimIndex *NewIndex(const TieredIndexParams *params) { + if (!IsQuantizationSupported(params)) { + return nullptr; + } + const auto &hnsw_params = params->primaryIndexParams->algoParams.hnswParams; auto management_layer_allocator = VecSimAllocator::newVecsimAllocator(); - auto *hnsw_index = static_cast *>(HNSWFactory::NewIndex( - params->primaryIndexParams, true, RequiresQuantizationTraining(params))); + VecSimParams backend_params = *params->primaryIndexParams; + vecsim_stl::vector initial_mean(management_layer_allocator); + if (RequiresQuantizationTraining(params)) { + // Allocate the final SQ8 WithMean layout now. The factory copies this temporary mean; + // tiered keeps the graph empty until it installs the mean accumulated from FLAT. + initial_mean.resize(hnsw_params.dim, 0.0f); + backend_params.algoParams.hnswParams.quantParams = initial_mean.data(); + } + auto *hnsw_index = + static_cast *>(HNSWFactory::NewIndex(&backend_params, true)); if (!hnsw_index) { return nullptr; } diff --git a/src/VecSim/quantization/quantization_runtime.h b/src/VecSim/quantization/quantization_runtime.h deleted file mode 100644 index d5e817c1d..000000000 --- a/src/VecSim/quantization/quantization_runtime.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once -#include -#include -#include -#include "VecSim/vec_sim_common.h" - -struct QuantizationInit { - const void *parameters = nullptr; - bool prepareForTraining = false; -#ifdef BUILD_TESTS - std::istream *fittedState = nullptr; -#endif -}; -struct QuantizationInfo { - VecSimQuantType type = VecSimQuant_NONE; - const char *name = nullptr; - const char *scale = nullptr; - const char *bias = nullptr; - size_t vectorBytes = 0; - size_t queryBytes = 0; - size_t sharedBytes = 0; - const char *queryKernel = nullptr; - const char *graphKernel = nullptr; -}; -// Cold-path lifecycle for a storage context. The component wrappers own the context; -// indexes retain a non-owning alias and never delete it separately. -class QuantizationRuntime { -public: - virtual ~QuantizationRuntime() = default; - virtual void validateBatch(const void *rows, size_t count, size_t rowBytes, - VecSimMetric inputMetric) const = 0; - // Caller excludes queries and has not stored any vectors. A failed preparation - // leaves all previous fitted state intact; successful publication keeps dispatch addresses. - virtual void setMean(std::span mean) = 0; - virtual std::span mean() const = 0; - virtual size_t storageBytes() const = 0; - virtual size_t queryBytes() const = 0; - virtual QuantizationInfo info() const = 0; -#ifdef BUILD_TESTS - virtual unsigned encodingVersion() const = 0; - virtual void saveState(std::ostream &output) const = 0; - virtual void validateStoredRecord(const void *record) const = 0; -#endif -}; diff --git a/src/VecSim/quantization/sq8_mean.h b/src/VecSim/quantization/sq8_mean.h new file mode 100644 index 000000000..b0dfe4a7f --- /dev/null +++ b/src/VecSim/quantization/sq8_mean.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include + +#include "VecSim/spaces/computer/calculator.h" +#include "VecSim/spaces/computer/preprocessor_container.h" + +// Install before storing vectors, with queries excluded by the caller. Requires exactly one +// SQ8 WithNorm preprocessor in slot 0; cosine normalization must be performed by the frontend. +template +void applySQ8Mean(PreprocessorsContainerAbstract &preprocessors, + IndexCalculatorInterface &index_calculator, + std::span mean) noexcept { + auto &container = static_cast &>(preprocessors); + auto *preprocessor = + dynamic_cast *>(container.getPreprocessors()[0]); + auto *calculator = + dynamic_cast *>(&index_calculator); + assert(preprocessor && calculator); + preprocessor->setMean(mean); + calculator->setMeanSumSquares(mean); +} diff --git a/src/VecSim/quantization/storage_context.h b/src/VecSim/quantization/storage_context.h deleted file mode 100644 index f28357145..000000000 --- a/src/VecSim/quantization/storage_context.h +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright (c) 2006-Present, Redis Ltd. - * All rights reserved. - * - * Licensed under your choice of the Redis Source Available License 2.0 - * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the - * GNU Affero General Public License v3 (AGPLv3). - */ -#pragma once - -#include -#ifdef BUILD_TESTS -#include -#endif - -#include "VecSim/quantization/quantization_runtime.h" -#include "VecSim/spaces/computer/calculator.h" -#include "VecSim/spaces/computer/preprocessors.h" - -// Scalar storage policies supply layout/code hooks. Other families can select their own -// Context through StorageType::Context without changing the component wrappers. -template -class ScalarStorageContext : public QuantizationRuntime { - using Engine = ScalarQuantPreprocessor; - using Calculator = - std::conditional_t, - DistanceCalculatorCommon>; - -public: - using QueryType = typename Engine::QueryType; - - ScalarStorageContext(const std::shared_ptr &allocator, size_t dim, - const QuantizationInit &init) - : dim_(dim), engine_(makeEngine(allocator, dim, init)), - calculator_(makeCalculator(allocator, dim)), - stored_dispatch_(calculator_.getDistanceDispatch(DistanceMode::StoredToStored)), - query_dispatch_(calculator_.getDistanceDispatch(DistanceMode::StoredToQuery)) {} - - ScalarStorageContext(const ScalarStorageContext &) = delete; - ScalarStorageContext &operator=(const ScalarStorageContext &) = delete; - - void encodeStorage(const void *input, void *output) const { - engine_.encodeStorage(input, output); - } - void prepareQuery(const void *input, void *output) const { - engine_.prepareQuery(input, output); - } - void encodeStorageInPlace(void *input, size_t inputBytes) const { - engine_.preprocessStorageInPlace(input, inputBytes); - } - unsigned char storageAlignment() const { return storage_alignment_; } - unsigned char queryAlignment() const { return query_alignment_; } - DistanceDispatch getDistanceDispatch(DistanceMode mode) const { - return mode == DistanceMode::StoredToStored ? stored_dispatch_ : query_dispatch_; - } - - void validateBatch(const void *, size_t, size_t, VecSimMetric) const override {} - - // Call only before storing vectors and with concurrent queries excluded. Validation precedes - // publication; once it succeeds neither the engine nor the cached calculator update can throw. - void setMean(std::span values) override { - if constexpr (WithMean) { - if (values.size() != dim_) { - throw std::invalid_argument("Mean vector size must equal dimension"); - } - engine_.setMean(values); - calculator_.setMeanSumSquares(values); - } else { - if (!values.empty()) { - throw std::invalid_argument("Storage context does not use a mean"); - } - } - } - std::span mean() const override { return engine_.getMean(); } - size_t storageBytes() const override { - return StorageType::template storage_bytes_count(dim_); - } - size_t queryBytes() const override { - return dim_ * sizeof(QueryType) + - StorageType::template query_metadata_count() * sizeof(float); - } - QuantizationInfo info() const override { - QuantizationInfo result; - if constexpr (requires { StorageType::quantization_type; }) { - result.type = StorageType::quantization_type; - } - result.vectorBytes = storageBytes(); - result.queryBytes = queryBytes(); - result.sharedBytes = mean().size_bytes(); - return result; - } - - // Includes owned dynamic state; the common estimator separately counts allocate_shared's - // implementation-defined control block and the component wrappers. - static size_t estimateMemory(size_t dim, const QuantizationInit &) { - return sizeof(ScalarStorageContext) + - (WithMean ? dim * sizeof(float) + VecSimAllocator::getAllocationOverheadSize() : 0); - } -#ifdef BUILD_TESTS - unsigned encodingVersion() const override { return 5; } - void saveState(std::ostream &output) const override { - const bool has_mean = WithMean; - output.write(reinterpret_cast(&has_mean), sizeof(has_mean)); - if constexpr (WithMean) { - const auto values = mean(); - output.write(reinterpret_cast(values.data()), values.size_bytes()); - } - } - void validateStoredRecord(const void *) const override {} -#endif - -private: - static Engine makeEngine(const std::shared_ptr &allocator, size_t dim, - const QuantizationInit &init) { - if constexpr (WithMean) { - vecsim_stl::vector values(dim, 0.0f, allocator); - if (!init.prepareForTraining) { - const auto *mean = static_cast(init.parameters); - if (!mean) { - throw std::invalid_argument("Mean-normalized storage requires a mean"); - } - std::copy_n(mean, dim, values.begin()); - } - return Engine(allocator, dim, values); - } else { - return Engine(allocator, dim); - } - } - Calculator makeCalculator(const std::shared_ptr &allocator, size_t dim) { - unsigned char asym_alignment = 0; - auto sym = spaces::GetDistFunc(Metric, dim, &storage_alignment_); - auto asym = - spaces::GetDistFunc(Metric, dim, &asym_alignment); - storage_alignment_ = spaces::combineAlignments(storage_alignment_, asym_alignment); - spaces::GetDistFunc(Metric, dim, &query_alignment_); - if constexpr (WithMean) { - float sum_squares = 0.0f; - for (float value : engine_.getMean()) { - sum_squares += value * value; - } - return Calculator(allocator, asym, sym, sum_squares); - } else { - return Calculator(allocator, sym, asym); - } - } - - const size_t dim_; - unsigned char storage_alignment_ = 0; - unsigned char query_alignment_ = 0; - Engine engine_; - Calculator calculator_; - DistanceDispatch stored_dispatch_; - DistanceDispatch query_dispatch_; -}; diff --git a/src/VecSim/spaces/computer/calculator.h b/src/VecSim/spaces/computer/calculator.h index 02b88912b..230042ab2 100644 --- a/src/VecSim/spaces/computer/calculator.h +++ b/src/VecSim/spaces/computer/calculator.h @@ -250,29 +250,3 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { return DistanceDispatch::stateful(&context_, calcQueryWithContext); } }; - -// Context dispatches are selected once; the index caches these same callbacks and addresses. -template -class QuantizedCalculator : public IndexCalculatorInterface { -public: - QuantizedCalculator(std::shared_ptr allocator, - std::shared_ptr context) - : IndexCalculatorInterface(allocator), context_(std::move(context)), - stored_(context_->getDistanceDispatch(DistanceMode::StoredToStored)), - query_(context_->getDistanceDispatch(DistanceMode::StoredToQuery)) {} - - float calcDistance(const void *lhs, const void *rhs, size_t dim) const override { - return stored_(lhs, rhs, dim); - } - float calcDistanceForQuery(const void *stored, const void *query, size_t dim) const override { - return query_(stored, query, dim); - } - DistanceDispatch getDistanceDispatch(DistanceMode mode) const override { - return mode == DistanceMode::StoredToStored ? stored_ : query_; - } - -private: - std::shared_ptr context_; - DistanceDispatch stored_; - DistanceDispatch query_; -}; diff --git a/src/VecSim/spaces/computer/preprocessor_container.h b/src/VecSim/spaces/computer/preprocessor_container.h index 81543d114..5f0e74d4b 100644 --- a/src/VecSim/spaces/computer/preprocessor_container.h +++ b/src/VecSim/spaces/computer/preprocessor_container.h @@ -56,7 +56,7 @@ class PreprocessorsContainerAbstract : public VecsimBaseObject { MemoryUtils::unique_blob wrapAllocated(void *blob) const { return MemoryUtils::unique_blob( - blob, [allocator = this->allocator](void *ptr) { allocator->free_allocation(ptr); }); + blob, [this](void *ptr) { this->allocator->free_allocation(ptr); }); } static MemoryUtils::unique_blob wrapWithDummyDeleter(void *ptr) { @@ -186,33 +186,35 @@ MultiPreprocessorsContainer::preprocess(const void *o size_t storage_blob_size = input_blob_size; size_t query_blob_size = input_blob_size; - try { - for (auto pp : preprocessors) { - if (!pp) - break; - pp->preprocess(original_blob, storage_blob, query_blob, storage_blob_size, - query_blob_size, this->storage_alignment, this->query_alignment); - } - } catch (...) { - if (query_blob && query_blob != storage_blob) - this->allocator->free_allocation(query_blob); - if (storage_blob) - this->allocator->free_allocation(storage_blob); - throw; + for (auto pp : preprocessors) { + if (!pp) + break; + pp->preprocess(original_blob, storage_blob, query_blob, storage_blob_size, query_blob_size, + this->storage_alignment, this->query_alignment); + } + // At least one blob was allocated. + + // If they point to the same memory, we need to free only one of them. + if (storage_blob == query_blob) { + return ProcessedBlobs(std::move(this->wrapAllocated(storage_blob)), + std::move(Base::wrapWithDummyDeleter(storage_blob))); + } + + if (storage_blob == nullptr) { // we processed only the query + return ProcessedBlobs( + std::move(Base::wrapWithDummyDeleter(const_cast(original_blob))), + std::move(this->wrapAllocated(query_blob))); + } + + if (query_blob == nullptr) { // we processed only the storage + // query might need to be aligned + auto query_ptr = this->maybeCopyToAlignedMem(original_blob, input_blob_size); + return ProcessedBlobs(std::move(this->wrapAllocated(storage_blob)), std::move(query_ptr)); } - // Keep both allocations owned while constructing the returned deleters or copying a query. - auto free = [allocator = this->allocator](void *ptr) { allocator->free_allocation(ptr); }; - std::unique_ptr storage_guard(storage_blob, free); - std::unique_ptr query_guard( - query_blob != storage_blob ? query_blob : nullptr, free); - auto storage = storage_blob ? this->wrapAllocated(storage_blob) - : Base::wrapWithDummyDeleter(const_cast(original_blob)); - storage_guard.release(); - auto query = query_blob == storage_blob && query_blob ? Base::wrapWithDummyDeleter(query_blob) - : query_blob ? this->wrapAllocated(query_blob) - : this->maybeCopyToAlignedMem(original_blob, input_blob_size); - query_guard.release(); - return ProcessedBlobs(std::move(storage), std::move(query)); + + // Else, both were allocated separately, we need to release both. + return ProcessedBlobs(std::move(this->wrapAllocated(storage_blob)), + std::move(this->wrapAllocated(query_blob))); } template @@ -221,25 +223,15 @@ MultiPreprocessorsContainer::preprocessForStorage( const void *original_blob, size_t input_blob_size) const { void *storage_blob = nullptr; - try { - for (auto pp : preprocessors) { - if (!pp) - break; - pp->preprocessForStorage(original_blob, storage_blob, input_blob_size, - this->storage_alignment); - } - } catch (...) { - if (storage_blob) - this->allocator->free_allocation(storage_blob); - throw; + for (auto pp : preprocessors) { + if (!pp) + break; + pp->preprocessForStorage(original_blob, storage_blob, input_blob_size, + this->storage_alignment); } - auto free = [allocator = this->allocator](void *ptr) { allocator->free_allocation(ptr); }; - std::unique_ptr guard(storage_blob, free); - auto result = storage_blob ? this->wrapAllocated(storage_blob) - : Base::wrapWithDummyDeleter(const_cast(original_blob)); - guard.release(); - return result; + return storage_blob ? std::move(this->wrapAllocated(storage_blob)) + : std::move(Base::wrapWithDummyDeleter(const_cast(original_blob))); } template @@ -247,25 +239,15 @@ MemoryUtils::unique_blob MultiPreprocessorsContainer: const void *original_blob, size_t input_blob_size, bool force_copy) const { void *query_blob = nullptr; - try { - for (auto pp : preprocessors) { - if (!pp) - break; - // modifies the memory in place - pp->preprocessQuery(original_blob, query_blob, input_blob_size, this->query_alignment); - } - } catch (...) { - if (query_blob) - this->allocator->free_allocation(query_blob); - throw; + for (auto pp : preprocessors) { + if (!pp) + break; + // modifies the memory in place + pp->preprocessQuery(original_blob, query_blob, input_blob_size, this->query_alignment); } - auto free = [allocator = this->allocator](void *ptr) { allocator->free_allocation(ptr); }; - std::unique_ptr guard(query_blob, free); - auto result = query_blob - ? this->wrapAllocated(query_blob) - : this->maybeCopyToAlignedMem(original_blob, input_blob_size, force_copy); - guard.release(); - return result; + return query_blob + ? std::move(this->wrapAllocated(query_blob)) + : std::move(this->maybeCopyToAlignedMem(original_blob, input_blob_size, force_copy)); } template diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index e6a9c7a34..262474dd5 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -142,7 +142,7 @@ class CosinePreprocessor : public PreprocessorInterface { }; /* - * ScalarQuantPreprocessor is a preprocessor that quantizes storage vectors from DataType to a + * QuantPreprocessor is a preprocessor that quantizes storage vectors from DataType to a * lower precision representation defined by StorageType (SQ8, the current storage type, * stores one uint8_t code per coordinate). * Query vectors remain as DataType for asymmetric distance computation. @@ -232,7 +232,7 @@ class CosinePreprocessor : public PreprocessorInterface { * ||x - y||² = sum_sq_x + sum_sq_y - 2 * IP(x, y) * where sum_sq_x, sum_sq_y are precomputed sums of the squared represented values. */ -// Input types accepted by ScalarQuantPreprocessor. Opt-in via std::same_as so unrelated types +// Input types accepted by QuantPreprocessor. Opt-in via std::same_as so unrelated types // (e.g. integers, double, bfloat16) are rejected at the template head with a named constraint. template concept QuantInput = std::same_as || std::same_as; @@ -259,7 +259,7 @@ static inline T from_fp32(float x) { template -class ScalarQuantPreprocessor : public PreprocessorInterface { +class QuantPreprocessor : public PreprocessorInterface { public: // Center L2 queries in FP32 even when the original input is FP16. using QueryType = std::conditional_t; @@ -270,7 +270,7 @@ class ScalarQuantPreprocessor : public PreprocessorInterface { static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP || Metric == VecSimMetric_Cosine, - "ScalarQuantPreprocessor only supports L2, IP and Cosine metrics"); + "QuantPreprocessor only supports L2, IP and Cosine metrics"); static_assert(!WithNorm || Metric != VecSimMetric_Cosine, "WithNorm does not support Cosine metric."); @@ -489,7 +489,7 @@ class ScalarQuantPreprocessor : public PreprocessorInterface { } // Standard constructor (WithNorm == false): no mean vector. - ScalarQuantPreprocessor(std::shared_ptr allocator, size_t dim) + QuantPreprocessor(std::shared_ptr allocator, size_t dim) requires(!WithNorm) : PreprocessorInterface(allocator), dim(dim), storage_bytes_count(StorageType::template storage_bytes_count(dim)), @@ -498,8 +498,8 @@ class ScalarQuantPreprocessor : public PreprocessorInterface { sizeof(MetadataType)) {} // WithNorm constructor: accepts a pre-computed mean vector (FP32, length == dim). - ScalarQuantPreprocessor(std::shared_ptr allocator, size_t dim, - const vecsim_stl::vector &mean_vec) + QuantPreprocessor(std::shared_ptr allocator, size_t dim, + const vecsim_stl::vector &mean_vec) requires(WithNorm) : PreprocessorInterface(allocator), mean(mean_vec), dim(dim), storage_bytes_count(StorageType::template storage_bytes_count(dim)), @@ -558,7 +558,10 @@ class ScalarQuantPreprocessor : public PreprocessorInterface { assert(!blob && "storage_blob must be nullptr"); blob = this->allocator->allocate_aligned(storage_bytes_count, storage_alignment); - encodeStorage(original_blob, blob); + // Cast to appropriate types + const DataType *input = static_cast(original_blob); + OUTPUT_TYPE *quantized = static_cast(blob); + quantize(input, quantized); input_blob_size = storage_bytes_count; } @@ -589,15 +592,6 @@ class ScalarQuantPreprocessor : public PreprocessorInterface { // Allocate aligned memory for the query blob blob = this->allocator->allocate_aligned(this->query_bytes_count, alignment); - prepareQuery(original_blob, blob); - query_blob_size = this->query_bytes_count; - } - - void encodeStorage(const void *input, void *output) const { - quantize(static_cast(input), static_cast(output)); - } - - void prepareQuery(const void *original_blob, void *blob) const { const size_t body_bytes = this->dim * sizeof(QueryType); const DataType *input = static_cast(original_blob); QueryType *query_values = static_cast(blob); @@ -613,13 +607,8 @@ class ScalarQuantPreprocessor : public PreprocessorInterface { // body_bytes, which is not guaranteed to be 4-byte aligned for FP16 query bodies. void *metadata_dst = static_cast(blob) + body_bytes; assign_query_metadata(query_values, input, metadata_dst); - } - std::span getMean() const { - if constexpr (WithNorm) { - return {mean.data(), mean.size()}; - } - return {}; + query_blob_size = this->query_bytes_count; } void preprocessStorageInPlace(void *original_blob, size_t input_blob_size) const override { @@ -674,100 +663,3 @@ class ScalarQuantPreprocessor : public PreprocessorInterface { const size_t storage_bytes_count; const size_t query_bytes_count; }; - -// Storage policies may replace the scalar family's context while retaining the same wrappers. -template -class ScalarStorageContext; - -template -struct StorageContextSelector { - using type = ScalarStorageContext; -}; - -template -struct StorageContextSelector< - StorageType, Input, Metric, WithMean, - std::void_t>> { - using type = typename StorageType::template Context; -}; - -template -using StorageContextFor = - typename StorageContextSelector::type; - -#include "VecSim/quantization/quantization_runtime.h" - -template -class QuantPreprocessor : public PreprocessorInterface { -public: - using Context = StorageContextFor; - using QueryType = typename Context::QueryType; - - QuantPreprocessor(std::shared_ptr allocator, std::shared_ptr context) - : PreprocessorInterface(allocator), context_(std::move(context)) {} - - QuantPreprocessor(std::shared_ptr allocator, size_t dim) - requires(!WithMean) - : QuantPreprocessor(allocator, - std::allocate_shared(VecsimSTLAllocator(allocator), - allocator, dim, QuantizationInit{})) {} - - QuantPreprocessor(std::shared_ptr allocator, size_t dim, - const vecsim_stl::vector &mean) - requires(WithMean) - : QuantPreprocessor(allocator, std::allocate_shared( - VecsimSTLAllocator(allocator), allocator, dim, - QuantizationInit{.parameters = mean.data()})) { - assert(mean.size() == dim); - } - - void setMean(std::span mean) - requires(WithMean) - { - context_->setMean(mean); - } - - void preprocess(const void *input, void *&storage, void *&query, size_t &storageBytes, - size_t &queryBytes, unsigned char storageAlignment, - unsigned char queryAlignment) const override { - assert(!storage && !query); - assert(&storageBytes != &queryBytes); - auto storage_output = - this->allocator->allocate_aligned_unique(context_->storageBytes(), storageAlignment); - auto query_output = - this->allocator->allocate_aligned_unique(context_->queryBytes(), queryAlignment); - context_->encodeStorage(input, storage_output.get()); - context_->prepareQuery(input, query_output.get()); - storage = storage_output.release(); - query = query_output.release(); - storageBytes = context_->storageBytes(); - queryBytes = context_->queryBytes(); - } - void preprocessForStorage(const void *input, void *&output, size_t &bytes, - unsigned char alignment) const override { - assert(!output); - auto prepared = - this->allocator->allocate_aligned_unique(context_->storageBytes(), alignment); - context_->encodeStorage(input, prepared.get()); - output = prepared.release(); - bytes = context_->storageBytes(); - } - void preprocessQuery(const void *input, void *&output, size_t &bytes, - unsigned char alignment) const override { - assert(!output); - auto prepared = this->allocator->allocate_aligned_unique(context_->queryBytes(), alignment); - context_->prepareQuery(input, prepared.get()); - output = prepared.release(); - bytes = context_->queryBytes(); - } - void preprocessStorageInPlace(void *input, size_t bytes) const override { - context_->encodeStorageInPlace(input, bytes); - } - -private: - std::shared_ptr context_; -}; - -#include "VecSim/quantization/storage_context.h" diff --git a/src/VecSim/types/sq8.h b/src/VecSim/types/sq8.h index 1d4165bba..27636ac91 100644 --- a/src/VecSim/types/sq8.h +++ b/src/VecSim/types/sq8.h @@ -18,7 +18,6 @@ namespace vecsim_types { // Represents a scalar-quantized 8-bit blob with reconstruction metadata struct sq8 { using value_type = uint8_t; - static constexpr VecSimQuantType quantization_type = VecSimQuant_SQ8; static constexpr uint8_t max_code = std::numeric_limits::max(); static constexpr size_t data_bytes_count(size_t dim) { return dim; } diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 8767bb915..419aa9884 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -428,12 +428,7 @@ typedef struct { size_t max_level; // Number of graph levels. size_t entrypoint; // Entrypoint vector label. size_t visitedNodesPoolSize; // The max number of parallel graph scans so far. - size_t numberOfMarkedDeletedNodes; - VecSimQuantType quantizationType; - size_t quantizedVectorBytes; - size_t quantizerSharedBytes; - const char *quantizerQueryKernel; - const char *quantizerGraphKernel; + size_t numberOfMarkedDeletedNodes; // The number of nodes that are marked as deleted. } hnswInfoStruct; typedef struct { @@ -452,7 +447,7 @@ typedef struct { size_t numThreads; // Maximum number of threads to be used by svs for ingestion. size_t lastReservedThreads; // Number of threads that were successfully reserved by the last // ingestion operation. - size_t numberOfMarkedDeletedNodes; + size_t numberOfMarkedDeletedNodes; // The number of nodes that are marked as deleted. size_t searchWindowSize; // Search window size for Vamana graph accuracy/latency tune. size_t searchBufferCapacity; // Search buffer capacity for Vamana graph accuracy/latency tune. size_t leanvecDim; // Leanvec dimension to use when LeanVec is enabled. diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index 64e884057..fbdab3a67 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -15,7 +15,6 @@ #include "VecSim/utils/vec_utils.h" #include "VecSim/spaces/spaces.h" #include "VecSim/spaces/computer/calculator.h" -#include "VecSim/quantization/quantization_runtime.h" #include "VecSim/spaces/computer/preprocessor_container.h" #include "info_iterator_struct.h" #include "containers/data_blocks_container.h" @@ -67,8 +66,6 @@ template struct IndexComponents { IndexCalculatorInterface *indexCalculator; PreprocessorsContainerAbstract *preprocessors; - // Alias owned by the component wrappers, never deleted separately. - QuantizationRuntime *quantization = nullptr; }; /** @@ -94,7 +91,6 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { DistanceDispatch storedDistanceDispatch; DistanceDispatch queryDistanceDispatch; PreprocessorsContainerAbstract *preprocessors; // Storage and query preprocessors. - QuantizationRuntime *quantization; // Non-owning alias into the component context. size_t inputBlobSize; // The size of input vectors/queries blob in bytes. May differ from dim * // sizeof(vecType) when vectors have been externally preprocessed (e.g., @@ -142,8 +138,8 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { components.indexCalculator ? components.indexCalculator->getDistanceDispatch(DistanceMode::StoredToQuery) : DistanceDispatch{}), - preprocessors(components.preprocessors), quantization(components.quantization), - inputBlobSize(params.inputBlobSize), storedDataSize(params.storedDataSize) { + preprocessors(components.preprocessors), inputBlobSize(params.inputBlobSize), + storedDataSize(params.storedDataSize) { assert(VecSimType_sizeof(vecType)); assert(storedDataSize); assert(inputBlobSize); @@ -232,14 +228,6 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { inline void setLastSearchMode(VecSearchMode mode) override { this->lastMode = mode; } inline bool isMultiValue() const { return isMulti; } inline bool usesQuantizedStorage() const { return isQuantized; } - QuantizationRuntime *getQuantizationRuntime() const { return quantization; } - - // Validate the complete input before a replacement can remove existing rows. - void validateQuantizationInput(const void *rows, size_t count, size_t rowBytes, - VecSimMetric inputMetric) const { - if (quantization) - quantization->validateBatch(rows, count, rowBytes, inputMetric); - } inline VecSimType getType() const { return vecType; } inline VecSimMetric getMetric() const { return metric; } inline size_t getStoredDataSize() const { return storedDataSize; } @@ -377,9 +365,7 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { } IndexComponents get_components() const { - return {.indexCalculator = this->indexCalculator, - .preprocessors = this->preprocessors, - .quantization = this->quantization}; + return {.indexCalculator = this->indexCalculator, .preprocessors = this->preprocessors}; } // Test-only: override the disk flag to exercise disk-specific param resolution diff --git a/tests/unit/test_components.cpp b/tests/unit/test_components.cpp index 5bc83a6c5..6744756b3 100644 --- a/tests/unit/test_components.cpp +++ b/tests/unit/test_components.cpp @@ -1520,7 +1520,8 @@ void checkSQ8Bytes(size_t dim) { EXPECT_EQ(encode(QuantPreprocessor(allocator, dim)), expected); } - // The SQ8 factory must preserve scalar bytes and the selected cached distance kernels. + // The SQ8 factory wrapper must produce the same bytes and keep naming the SQ8 component + // types that applySQ8Mean dynamic_casts to. if constexpr (std::is_same_v && Metric != VecSimMetric_Cosine) { auto components = CreateSQ8IndexComponents(allocator, dim, WithMean ? mean.data() : nullptr); @@ -1532,31 +1533,14 @@ void checkSQ8Bytes(size_t dim) { EXPECT_NE((dynamic_cast *>( container->getPreprocessors()[0])), nullptr); + if constexpr (WithMean) { + EXPECT_NE((dynamic_cast *>( + calculator.get())), + nullptr); + } auto blob = preprocessors->preprocessForStorage(input.data(), dim * sizeof(float)); const auto *factory_bytes = static_cast(blob.get()); EXPECT_EQ(std::vector(factory_bytes, factory_bytes + expected.size()), expected); - auto query = preprocessors->preprocessQuery(input.data(), dim * sizeof(float)); - auto stored_dispatch = calculator->getDistanceDispatch(DistanceMode::StoredToStored); - auto query_dispatch = calculator->getDistanceDispatch(DistanceMode::StoredToQuery); - unsigned char sym_alignment = 0, asym_alignment = 0; - auto sym = spaces::GetDistFunc(Metric, dim, &sym_alignment); - using QueryType = typename QuantPreprocessor::QueryType; - auto asym = spaces::GetDistFunc(Metric, dim, &asym_alignment); - if constexpr (WithMean) { - float mean_sum_squares = 0.0f; - for (float value : mean) - mean_sum_squares += value * value; - DistanceCalculatorWithNorm legacy(allocator, asym, sym, - mean_sum_squares); - EXPECT_FLOAT_EQ(stored_dispatch(blob.get(), blob.get(), dim), - legacy.calcDistance(blob.get(), blob.get(), dim)); - EXPECT_FLOAT_EQ(query_dispatch(blob.get(), query.get(), dim), - legacy.calcDistanceForQuery(blob.get(), query.get(), dim)); - } - if constexpr (!WithMean || Metric == VecSimMetric_L2) { - EXPECT_EQ(stored_dispatch.stateless_func, sym); - EXPECT_EQ(query_dispatch.stateless_func, asym); - } } } @@ -1578,58 +1562,6 @@ TEST(PreprocessorsTest, QuantizedStorageSQ8BytesUnchanged) { checkSQ8BytesAllModes(); } -TEST(PreprocessorsTest, QuantizedBlobsOutliveComponents) { - auto allocator = VecSimAllocator::newVecsimAllocator(); - const auto baseline = allocator->getAllocationSize(); - constexpr size_t dim = 8; - const float input[dim] = {1, 2, 3, 4, 5, 6, 7, 8}; - auto components = CreateSQ8IndexComponents(allocator, dim, nullptr); - EXPECT_EQ(allocator->getAllocationSize() - baseline, - (EstimateQuantizedComponentsMemory( - dim, QuantizationInit{}))); - auto blobs = components.preprocessors->preprocess(input, sizeof(input)); - delete components.preprocessors; - delete components.indexCalculator; - EXPECT_EQ(std::memcmp(blobs.getQueryBlob(), input, sizeof(input)), 0); - blobs = ProcessedBlobs{}; - EXPECT_EQ(allocator->getAllocationSize(), baseline); -} - -namespace { - -template -class ThrowingQueryContext - : public ScalarStorageContext { - using Base = ScalarStorageContext; - -public: - using Base::Base; - void prepareQuery(const void *, void *) const { throw std::runtime_error("Query preparation"); } -}; - -struct ThrowingQueryStorage { - template - using Context = ThrowingQueryContext; -}; - -} // namespace - -TEST(PreprocessorsTest, QuantizedCombinedPreparationFailureReleasesBothOutputs) { - auto allocator = VecSimAllocator::newVecsimAllocator(); - const auto baseline = allocator->getAllocationSize(); - constexpr size_t dim = 8; - const float input[dim] = {1, 2, 3, 4, 5, 6, 7, 8}; - auto components = - CreateQuantizedIndexComponents( - allocator, dim, QuantizationInit{}); - const auto component_bytes = allocator->getAllocationSize(); - EXPECT_THROW(components.preprocessors->preprocess(input, sizeof(input)), std::runtime_error); - EXPECT_EQ(allocator->getAllocationSize(), component_bytes); - delete components.preprocessors; - delete components.indexCalculator; - EXPECT_EQ(allocator->getAllocationSize(), baseline); -} - // Parameterized test class for QuantPreprocessor with different metrics class QuantPreprocessorMetricTest : public testing::TestWithParam { protected: From 492b5a8a1e2fa33c46839b15978cbcdf6e9e6c07 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Mon, 9 Nov 2026 16:23:18 +0200 Subject: [PATCH 09/10] Extract shared scalar quantization metadata --- src/VecSim/types/quantized_metadata.h | 59 +++++++++++++++++++++++++++ src/VecSim/types/sq8.h | 44 +------------------- 2 files changed, 61 insertions(+), 42 deletions(-) create mode 100644 src/VecSim/types/quantized_metadata.h diff --git a/src/VecSim/types/quantized_metadata.h b/src/VecSim/types/quantized_metadata.h new file mode 100644 index 000000000..7d726b5c7 --- /dev/null +++ b/src/VecSim/types/quantized_metadata.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include "VecSim/vec_sim_common.h" + +namespace vecsim_types { + +// FP32 reconstruction metadata shared by scalar-quantized storage formats. +struct quantized_metadata { + // Metadata layout indices (stored after quantized values) + enum MetadataIndex : size_t { + MIN_VAL = 0, + DELTA = 1, + SUM = 2, + SUM_SQUARES = 3 // Only for L2 + }; + + enum QueryMetadataIndex : size_t { + SUM_QUERY = 0, + SUM_SQUARES_QUERY = 1 // Only for L2 + }; + + // Template on Metric and WithNorm — compile-time constants + // WithNorm: one extra metadata slot for x_mean_ip / y_mean_ip. Only needed for IP: the + // normalized L2 correction cancels the mean terms exactly during centering and never reads + // mean_ip (see DistanceCalculatorWithNorm), so no extra slot is allocated for L2. + template + static constexpr size_t storage_metadata_count() { + return ((Metric == VecSimMetric_L2) ? 4 : 3) + + ((WithNorm && Metric == VecSimMetric_IP) ? 1 : 0); + } + + template + static constexpr size_t query_metadata_count() { + return ((Metric == VecSimMetric_L2) ? 2 : 1) + + ((WithNorm && Metric == VecSimMetric_IP) ? 1 : 0); + } + + // Index of x_mean_ip / y_mean_ip in the last slot in metadata array + template + static constexpr size_t mean_ip_index() { + return storage_metadata_count() - 1; + } + + template + static constexpr size_t query_mean_ip_index() { + return query_metadata_count() - 1; + } +}; + +} // namespace vecsim_types diff --git a/src/VecSim/types/sq8.h b/src/VecSim/types/sq8.h index 27636ac91..00338af92 100644 --- a/src/VecSim/types/sq8.h +++ b/src/VecSim/types/sq8.h @@ -11,12 +11,12 @@ #include #include #include -#include "VecSim/vec_sim_common.h" +#include "VecSim/types/quantized_metadata.h" namespace vecsim_types { // Represents a scalar-quantized 8-bit blob with reconstruction metadata -struct sq8 { +struct sq8 : quantized_metadata { using value_type = uint8_t; static constexpr uint8_t max_code = std::numeric_limits::max(); @@ -59,51 +59,11 @@ struct sq8 { return (sum0 + sum1) + (sum2 + sum3); } - // Metadata layout indices (stored after quantized values) - enum MetadataIndex : size_t { - MIN_VAL = 0, - DELTA = 1, - SUM = 2, - SUM_SQUARES = 3 // Only for L2 - }; - - enum QueryMetadataIndex : size_t { - SUM_QUERY = 0, - SUM_SQUARES_QUERY = 1 // Only for L2 - }; - - // Template on Metric and WithNorm — compile-time constants - // WithNorm: one extra metadata slot for x_mean_ip / y_mean_ip. Only needed for IP: the - // normalized L2 correction cancels the mean terms exactly during centering and never reads - // mean_ip (see DistanceCalculatorWithNorm), so no extra slot is allocated for L2. - template - static constexpr size_t storage_metadata_count() { - return ((Metric == VecSimMetric_L2) ? 4 : 3) + - ((WithNorm && Metric == VecSimMetric_IP) ? 1 : 0); - } - - template - static constexpr size_t query_metadata_count() { - return ((Metric == VecSimMetric_L2) ? 2 : 1) + - ((WithNorm && Metric == VecSimMetric_IP) ? 1 : 0); - } - // Stored layout: one quantized byte per dimension, followed by metric-specific FP32 metadata. template static constexpr size_t storage_bytes_count(size_t dim) { return data_bytes_count(dim) + storage_metadata_count() * sizeof(float); } - - // Index of x_mean_ip / y_mean_ip in the last slot in metadata array - template - static constexpr size_t mean_ip_index() { - return storage_metadata_count() - 1; - } - - template - static constexpr size_t query_mean_ip_index() { - return query_metadata_count() - 1; - } }; } // namespace vecsim_types From c6620f6b04917fc2c68e78d6196961c1bd7acce9 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Mon, 9 Nov 2026 16:31:49 +0200 Subject: [PATCH 10/10] Rename quantized code accessors to get_code and set_code --- src/VecSim/spaces/computer/preprocessors.h | 10 +++++----- src/VecSim/spaces/quantized_kernels.h | 4 ++-- src/VecSim/types/sq8.h | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 262474dd5..3f46e2d48 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -339,10 +339,10 @@ class QuantPreprocessor : public PreprocessorInterface { const OUTPUT_TYPE a1 = to_byte((x1 - min_val) * inv_delta); const OUTPUT_TYPE a2 = to_byte((x2 - min_val) * inv_delta); const OUTPUT_TYPE a3 = to_byte((x3 - min_val) * inv_delta); - StorageType::store_code(quantized, i, a0); - StorageType::store_code(quantized, i + 1, a1); - StorageType::store_code(quantized, i + 2, a2); - StorageType::store_code(quantized, i + 3, a3); + StorageType::set_code(quantized, i, a0); + StorageType::set_code(quantized, i + 1, a1); + StorageType::set_code(quantized, i + 2, a2); + StorageType::set_code(quantized, i + 3, a3); s0 += a0; s1 += a1; @@ -367,7 +367,7 @@ class QuantPreprocessor : public PreprocessorInterface { for (; i < this->dim; ++i) { const float x = transformed_value(input, i); const OUTPUT_TYPE a = to_byte((x - min_val) * inv_delta); - StorageType::store_code(quantized, i, a); + StorageType::set_code(quantized, i, a); q_sum += a; if constexpr (Metric == VecSimMetric_L2) { q_sum_squares += uint64_t{a} * a; diff --git a/src/VecSim/spaces/quantized_kernels.h b/src/VecSim/spaces/quantized_kernels.h index 6cad11929..c84dd0e03 100644 --- a/src/VecSim/spaces/quantized_kernels.h +++ b/src/VecSim/spaces/quantized_kernels.h @@ -16,7 +16,7 @@ // scalar functions (e.g. SQ8_FP32_L2Sqr) instead. // // Scalar kernels for scalar quantized storage, generic over a StorageType that provides -// load_code(data, i), data_bytes_count(dim) and the MIN_VAL / DELTA / SUM / SUM_SQUARES metadata +// get_code(data, i), data_bytes_count(dim) and the MIN_VAL / DELTA / SUM / SUM_SQUARES metadata // indices (stored after the codes) and SUM_QUERY (stored after the FP32 query values). // It also provides the code dot hooks code_dot(a, b, dim) and code_dot_fp32(codes, y, dim). // @@ -81,7 +81,7 @@ static inline float FP32_L2Sqr(const void *stored, const void *query, size_t dim // the scale of min_val and wipe out small residuals. When min_val and y_i are not close, // the residual is large and either order is fine. float diff = - delta * static_cast(StorageType::load_code(pVect1, i)) + (min_val - pVect2[i]); + delta * static_cast(StorageType::get_code(pVect1, i)) + (min_val - pVect2[i]); res += diff * diff; } return res; diff --git a/src/VecSim/types/sq8.h b/src/VecSim/types/sq8.h index 00338af92..55adccf74 100644 --- a/src/VecSim/types/sq8.h +++ b/src/VecSim/types/sq8.h @@ -22,15 +22,15 @@ struct sq8 : quantized_metadata { static constexpr size_t data_bytes_count(size_t dim) { return dim; } - static uint8_t load_code(const uint8_t *data, size_t index) { return data[index]; } + static uint8_t get_code(const uint8_t *data, size_t index) { return data[index]; } - static void store_code(uint8_t *data, size_t index, uint8_t code) { data[index] = code; } + static void set_code(uint8_t *data, size_t index, uint8_t code) { data[index] = code; } // Dot product of two code arrays: sum(q1_i * q2_i). static float code_dot(const uint8_t *a, const uint8_t *b, size_t dim) { float product = 0; for (size_t i = 0; i < dim; i++) { - product += load_code(a, i) * load_code(b, i); + product += get_code(a, i) * get_code(b, i); } return product; } @@ -44,15 +44,15 @@ struct sq8 : quantized_metadata { size_t i = 0; size_t dim4 = dim & ~size_t(3); // dim4 is a multiple of 4 for (; i < dim4; i += 4) { - sum0 += static_cast(load_code(codes, i + 0)) * y[i + 0]; - sum1 += static_cast(load_code(codes, i + 1)) * y[i + 1]; - sum2 += static_cast(load_code(codes, i + 2)) * y[i + 2]; - sum3 += static_cast(load_code(codes, i + 3)) * y[i + 3]; + sum0 += static_cast(get_code(codes, i + 0)) * y[i + 0]; + sum1 += static_cast(get_code(codes, i + 1)) * y[i + 1]; + sum2 += static_cast(get_code(codes, i + 2)) * y[i + 2]; + sum3 += static_cast(get_code(codes, i + 3)) * y[i + 3]; } // Handle remainder (0-3 elements) for (; i < dim; i++) { - sum0 += static_cast(load_code(codes, i)) * y[i]; + sum0 += static_cast(get_code(codes, i)) * y[i]; } // Combine accumulators