From bc5ed8001d87b203d081429d5f1cac8be48a25fa Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Sun, 15 Nov 2026 14:06:38 +0200 Subject: [PATCH] Add TQ2/TQ3 as quantized StorageTypes Add TurboQuant 2-bit and 3-bit MSE quantization for FP32 inner-product and normalized cosine HNSW indexes, standalone and tiered. - VecSimQuant_TQ2/TQ3 and an optional VecSimTurboQuantParams (mean, seed; default seed 42 and a zero mean). - A dense-rotation codec with projection-match scaling and a mean-error correction. Encoding and scoring never throw; only construction and loading validate. - TurboQuantPreprocessor and TurboQuantCalculator share one codec; the calculator returns a stateful dispatch bound to it. - Tiered training reuses SQ8TrainingState and installs the trained mean through HNSWIndex::setQuantizationMean. - Serialization V5 writes the fitted state (including the full rotation) after quantType. SQ8 and SQ4 bytes are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/VecSim/algorithms/hnsw/hnsw.h | 5 + .../algorithms/hnsw/hnsw_base_tests_friends.h | 1 + src/VecSim/algorithms/hnsw/hnsw_serializer.h | 2 +- .../algorithms/hnsw/hnsw_serializer_impl.h | 18 +- .../hnsw/hnsw_tiered_tests_friends.h | 1 + src/VecSim/index_factories/hnsw_factory.cpp | 62 ++- src/VecSim/index_factories/tiered_factory.cpp | 16 +- src/VecSim/quantization/turboquant.h | 433 +++++++++++++++ .../quantization/turboquant_components.h | 149 +++++ src/VecSim/quantization/turboquant_kernels.h | 214 ++++++++ src/VecSim/types/turboquant.h | 28 + src/VecSim/vec_sim_common.h | 12 + tests/unit/CMakeLists.txt | 3 + tests/unit/test_hnsw_sq4.cpp | 4 +- tests/unit/test_hnsw_turboquant.cpp | 512 ++++++++++++++++++ 15 files changed, 1445 insertions(+), 15 deletions(-) create mode 100644 src/VecSim/quantization/turboquant.h create mode 100644 src/VecSim/quantization/turboquant_components.h create mode 100644 src/VecSim/quantization/turboquant_kernels.h create mode 100644 src/VecSim/types/turboquant.h create mode 100644 tests/unit/test_hnsw_turboquant.cpp diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h index 11b4247ef..ba6b7ad8b 100644 --- a/src/VecSim/algorithms/hnsw/hnsw.h +++ b/src/VecSim/algorithms/hnsw/hnsw.h @@ -22,6 +22,7 @@ #include "VecSim/vec_sim_index.h" #include "VecSim/spaces/computer/preprocessors.h" #include "VecSim/quantization/sq8_mean.h" +#include "VecSim/quantization/turboquant_components.h" #include #include "VecSim/tombstone_interface.h" @@ -106,6 +107,10 @@ class HNSWIndex : public VecSimIndexAbstract, // A quantizer that learns storedDataSize from data needs deferred backend creation. void setQuantizationMean(std::span mean) noexcept { assert(this->isQuantized && curElementCount == 0 && mean.size() == this->dim); + if (auto *codec = vecsim::quantization::GetTurboQuantCodec(this->getIndexCalculator())) { + codec->setMean(mean); + return; + } if constexpr (QuantInput && std::is_same_v) { if (this->metric == VecSimMetric_L2) { setSQ8Mean(mean); diff --git a/src/VecSim/algorithms/hnsw/hnsw_base_tests_friends.h b/src/VecSim/algorithms/hnsw/hnsw_base_tests_friends.h index 37566aa9f..77cb0251e 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_base_tests_friends.h +++ b/src/VecSim/algorithms/hnsw/hnsw_base_tests_friends.h @@ -47,3 +47,4 @@ INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_updateVectorsTwiceBeforeIngesti INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_updateVectorsAsync_Test) INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_addRelabelAndUpdateTogetherUnderLoad_Test) INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_queryWhileUpdating_Test) +friend class TurboQuantTest; diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer.h b/src/VecSim/algorithms/hnsw/hnsw_serializer.h index de223c0aa..1ea882769 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer.h @@ -24,7 +24,7 @@ class HNSWSerializer : public Serializer { DEPRECATED = 2, // Last deprecated version V3, V4, - V5, // scalar quantization fields (quantType + mean vector) + V5, // quantization fields (quantType + mean vector, or TQ2/TQ3 fitted state) INVALID }; diff --git a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h index 2865c5d81..af436b238 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h +++ b/src/VecSim/algorithms/hnsw/hnsw_serializer_impl.h @@ -266,12 +266,18 @@ void HNSWIndex::saveIndexFields(std::ofstream &output) const // V5 fields: scalar quantization support (between factory params and index build params) if (this->quantType != VecSimQuant_NONE) { writeBinaryPOD(output, this->quantType); - // Write mean vector (dim floats). Empty vector means no mean (zero-mean quantization). - bool hasMean = !this->serializedMeanVector.empty(); - writeBinaryPOD(output, hasMean); - if (hasMean) { - output.write(reinterpret_cast(this->serializedMeanVector.data()), - this->dim * sizeof(float)); + if (const auto *codec = + vecsim::quantization::GetTurboQuantCodec(this->getIndexCalculator())) { + // TQ2/TQ3 write their fitted state (including the full rotation) instead of a mean. + codec->saveFitted(output); + } else { + // Write mean vector (dim floats). Empty vector means no mean (zero-mean quantization). + bool hasMean = !this->serializedMeanVector.empty(); + writeBinaryPOD(output, hasMean); + if (hasMean) { + output.write(reinterpret_cast(this->serializedMeanVector.data()), + this->dim * sizeof(float)); + } } } diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered_tests_friends.h b/src/VecSim/algorithms/hnsw/hnsw_tiered_tests_friends.h index bdd1c5c93..0b863a122 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered_tests_friends.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered_tests_friends.h @@ -97,3 +97,4 @@ INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_updateVectorsTwiceBeforeIngesti INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_updateVectorsAsync_Test) INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_addRelabelAndUpdateTogetherUnderLoad_Test) INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTestBasic_queryWhileUpdating_Test) +friend class TurboQuantTest; diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index 2d03fc233..b6a0adf88 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -15,6 +15,8 @@ #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq4.h" +#include "VecSim/types/turboquant.h" +#include "VecSim/quantization/turboquant_components.h" #include @@ -22,6 +24,8 @@ using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; using sq8 = vecsim_types::sq8; using sq4 = vecsim_types::sq4; +using vecsim::quantization::isTurboQuant; +using vecsim::quantization::turboQuantBits; namespace HNSWFactory { @@ -57,6 +61,10 @@ template [[nodiscard]] constexpr bool QuantizationParamsSupported(VecSimQuantType quant_type, VecSimType type, VecSimMetric resolved_metric) { + // TurboQuant has FP32 inner-product kernels only; cosine requires normalized input. + if (quant_type == VecSimQuant_TQ2 || quant_type == VecSimQuant_TQ3) { + return type == VecSimType_FLOAT32 && resolved_metric == VecSimMetric_IP; + } // Reject unknown quantizers instead of silently creating an unquantized index. if (quant_type != VecSimQuant_SQ8 && quant_type != VecSimQuant_SQ4) { return false; @@ -98,6 +106,23 @@ VecSimIndex *NewIndex_Quantized(const HNSWParams *hnswParams, return index; } +// The codec constructor rejects an unsupported dim or a nonfinite mean. +static VecSimIndex *NewIndex_TurboQuant(const HNSWParams *hnswParams, + AbstractIndexInitParams abstractInitParams) { + auto components = vecsim::quantization::CreateTurboQuantIndexComponents( + abstractInitParams.allocator, abstractInitParams.dim, turboQuantBits(hnswParams->quantType), + static_cast(hnswParams->quantParams)); + abstractInitParams.storedDataSize = + vecsim::quantization::GetTurboQuantCodec(components.indexCalculator)->storageBytes(); + abstractInitParams.isQuantized = true; + auto index = + NewIndex_ChooseMultiOrSingle(hnswParams, abstractInitParams, components); +#ifdef BUILD_TESTS + index->quantType = hnswParams->quantType; +#endif + return index; +} + VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { const HNSWParams *hnswParams = ¶ms->algoParams.hnswParams; AbstractIndexInitParams abstractInitParams = @@ -111,6 +136,10 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) { return NULL; } + if (isTurboQuant(hnswParams->quantType)) { + return NewIndex_TurboQuant(hnswParams, abstractInitParams); + } + if (hnswParams->quantType == VecSimQuant_SQ4) { if (metric == VecSimMetric_L2) { return NewIndex_Quantized( @@ -216,7 +245,10 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi sizeof(QuantPreprocessor)); static_assert(sizeof(QuantPreprocessor) == sizeof(QuantPreprocessor)); - if (with_mean) { + if (isTurboQuant(params->quantType)) { + est += vecsim::quantization::EstimateTurboQuantComponentsMemory( + params->dim, turboQuantBits(params->quantType)); + } else if (with_mean) { est += allocations_overhead + sizeof(DistanceCalculatorWithNorm); est += allocations_overhead + sizeof(MultiPreprocessorsContainer); @@ -264,7 +296,16 @@ size_t EstimateElementSize(const HNSWParams *params, bool with_mean) { size_t stored_data_size; // Preserve the element estimator's existing contract: return a size and validate in NewIndex. - if (params->quantType == VecSimQuant_SQ4) { + if (isTurboQuant(params->quantType)) { + if (params->type != VecSimType_FLOAT32 || + (params->metric != VecSimMetric_IP && params->metric != VecSimMetric_Cosine) || + !vecsim::quantization::TurboQuant::dimSupported(params->dim)) { + return SIZE_MAX; + } + stored_data_size = params->quantType == VecSimQuant_TQ2 + ? vecsim_types::tq2::storage_bytes_count(params->dim) + : vecsim_types::tq3::storage_bytes_count(params->dim); + } else if (params->quantType == VecSimQuant_SQ4) { stored_data_size = params->metric == VecSimMetric_L2 ? GetQuantizedStoredDataSize(params->dim, with_mean) @@ -361,11 +402,13 @@ static bool InitializeParams(std::ifstream &source_params, HNSWParams ¶ms, std::underlying_type_t raw_quant_type; Serializer::readBinaryPOD(source_params, raw_quant_type); if (raw_quant_type != VecSimQuant_NONE && raw_quant_type != VecSimQuant_SQ8 && - raw_quant_type != VecSimQuant_SQ4) { + raw_quant_type != VecSimQuant_SQ4 && raw_quant_type != VecSimQuant_TQ2 && + raw_quant_type != VecSimQuant_TQ3) { return false; } params.quantType = static_cast(raw_quant_type); - if (params.quantType != VecSimQuant_NONE) { + // TQ2/TQ3 store fitted state here instead; the loader reads it with the codec. + if (params.quantType != VecSimQuant_NONE && !isTurboQuant(params.quantType)) { bool hasMean = false; Serializer::readBinaryPOD(source_params, hasMean); if (hasMean) { @@ -417,6 +460,17 @@ VecSimIndex *NewIndex(const std::string &location, bool is_normalized) { return NULL; } + if (isTurboQuant(params.quantType)) { + auto codec = vecsim::quantization::TurboQuant::loadFitted( + input, abstractInitParams.allocator, params.dim, turboQuantBits(params.quantType)); + abstractInitParams.isQuantized = true; + abstractInitParams.storedDataSize = codec->storageBytes(); + auto components = vecsim::quantization::CreateTurboQuantIndexComponents( + abstractInitParams.allocator, std::move(codec)); + return NewIndex_ChooseMultiOrSingle(input, ¶ms, abstractInitParams, + components, version); + } + if (params.quantType == VecSimQuant_SQ4) { if (metric == VecSimMetric_L2) { return LoadIndex_Quantized( diff --git a/src/VecSim/index_factories/tiered_factory.cpp b/src/VecSim/index_factories/tiered_factory.cpp index 56d718d5f..d0dadae13 100644 --- a/src/VecSim/index_factories/tiered_factory.cpp +++ b/src/VecSim/index_factories/tiered_factory.cpp @@ -23,6 +23,7 @@ using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; +using vecsim::quantization::isTurboQuant; namespace TieredFactory { @@ -40,7 +41,8 @@ static inline BFParams NewBFParams(const TieredIndexParams *params) { } static inline bool RequiresQuantizationTraining(const TieredIndexParams *params) { - return params->primaryIndexParams->algoParams.hnswParams.quantType == VecSimQuant_SQ8 && + const auto type = params->primaryIndexParams->algoParams.hnswParams.quantType; + return (type == VecSimQuant_SQ8 || isTurboQuant(type)) && params->specificParams.tieredHnswParams.QuantNormalizationSetSize > 0; } @@ -53,6 +55,8 @@ static inline bool IsQuantizationSupported(const TieredIndexParams *params) { if constexpr (!QuantInput || !std::is_same_v) { return false; + } else if constexpr (std::is_same_v) { + return hnsw_params.quantType == VecSimQuant_SQ8 || isTurboQuant(hnsw_params.quantType); } else { return hnsw_params.quantType == VecSimQuant_SQ8; } @@ -68,7 +72,15 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) { auto management_layer_allocator = VecSimAllocator::newVecsimAllocator(); VecSimParams backend_params = *params->primaryIndexParams; vecsim_stl::vector initial_mean(management_layer_allocator); - if (RequiresQuantizationTraining(params)) { + VecSimTurboQuantParams tq_params{.mean = nullptr, .seed = VECSIM_TURBOQUANT_DEFAULT_SEED}; + if (RequiresQuantizationTraining(params) && isTurboQuant(hnsw_params.quantType)) { + // TQ2/TQ3 always carry a mean. Start from zero; tiered installs the trained mean. + if (const auto *config = + static_cast(hnsw_params.quantParams)) { + tq_params.seed = config->seed; + } + backend_params.algoParams.hnswParams.quantParams = &tq_params; + } else if (RequiresQuantizationTraining(params)) { // Allocate the final SQ8 WithMean layout now. The factory copies this temporary mean; // tiered keeps the graph empty until it installs the mean accumulated from FLAT. initial_mean.resize(hnsw_params.dim, 0.0f); diff --git a/src/VecSim/quantization/turboquant.h b/src/VecSim/quantization/turboquant.h new file mode 100644 index 000000000..b6ae58075 --- /dev/null +++ b/src/VecSim/quantization/turboquant.h @@ -0,0 +1,433 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include "VecSim/quantization/turboquant_kernels.h" +#include "VecSim/utils/vecsim_stl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef BUILD_TESTS +#include +#include +#endif + +namespace vecsim::quantization { + +// Dense MSE TurboQuant estimator for 1 - inner product, with a projection-match scale and a +// mean-error correction. Stored vectors encode r = x - mean as +// [ceil(d*b/8) packed codes][scale][delta], where n = ||r||, u = R*r/n, c = the centroid decode, +// scale = n/(u.c) and delta = scale*(c.R*mean) - r.mean. Queries hold [R*q: d floats][q.mean]. +// Only construction and loading validate and throw. Encoding allocates scratch, so it can +// throw std::bad_alloc; query preparation and scoring never throw. +class TurboQuant { + std::shared_ptr allocator_; + size_t dim_; + unsigned bits_; + uint64_t seed_; + turboquant_detail::PackedDot packed_dot_ = nullptr; + turboquant_detail::RowDot row_dot_ = nullptr; + turboquant_detail::PackedPairDot stored_pair_dot_ = nullptr; + size_t code_bytes_; + vecsim_stl::vector rotation_; + vecsim_stl::vector centroids_; + vecsim_stl::vector bias_mean_; + vecsim_stl::vector bias_rotated_mean_; + float bias_mean_norm_squared_ = 0; + + static size_t validatedDim(const std::shared_ptr &allocator, size_t dim, + unsigned bits, std::span mean) { + if (!allocator || !dimSupported(dim) || (bits != 2 && bits != 3) || mean.size() != dim) + throw std::invalid_argument("TurboQuant requires dim >= 2, 2/3 bits and a dim mean"); + for (float value : mean) { + if (!std::isfinite(value)) + throw std::invalid_argument("TurboQuant bias mean must be finite"); + } + return dim; + } + + static uint64_t splitSeed(uint64_t seed) { + seed += UINT64_C(0x9e3779b97f4a7c15); + seed = (seed ^ (seed >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); + seed = (seed ^ (seed >> 27)) * UINT64_C(0x94d049bb133111eb); + return seed ^ (seed >> 31); + } + + void initializeRotation(uint64_t seed) { + std::mt19937_64 rotation_rng(splitSeed(seed)); + std::normal_distribution gaussian; + vecsim_stl::vector basis(dim_ * dim_, allocator_); + // Row-wise Gaussian QR with positive diagonal convention gives Haar orthogonal Pi. + // Reorthogonalization and double workspace keep the initialization stable at large d. + for (size_t row = 0; row < dim_; ++row) { + double *current = basis.data() + row * dim_; + for (size_t col = 0; col < dim_; ++col) { + current[col] = gaussian(rotation_rng); + } + for (unsigned pass = 0; pass < 2; ++pass) { + for (size_t previous = 0; previous < row; ++previous) { + const double *other = basis.data() + previous * dim_; + double dot = 0; + for (size_t col = 0; col < dim_; ++col) { + dot += current[col] * other[col]; + } + for (size_t col = 0; col < dim_; ++col) { + current[col] -= dot * other[col]; + } + } + } + double norm = 0; + for (size_t col = 0; col < dim_; ++col) { + norm += current[col] * current[col]; + } + norm = std::sqrt(norm); + if (!(norm > 0) || !std::isfinite(norm)) { + throw std::runtime_error("TurboQuant Gaussian QR is degenerate"); + } + for (size_t col = 0; col < dim_; ++col) { + current[col] /= norm; + rotation_[row * dim_ + col] = static_cast(current[col]); + } + } + } + + void initializeCentroids() { + // The exact finite-d sphere marginal is (1-z*z)^((d-3)/2). With z=sin(theta), + // integration weight becomes cos(theta)^(d-2), including the singular d=2 case. + // Midpoint quadrature approximates its Lloyd-Max solution; it is not a Gaussian codebook. + constexpr size_t samples = 32768; + vecsim_stl::vector coordinates(samples, allocator_); + vecsim_stl::vector weights(samples, allocator_); + const size_t levels = centroids_.size(); + vecsim_stl::vector centers(levels, allocator_); + vecsim_stl::vector masses(levels, allocator_); + vecsim_stl::vector sums(levels, allocator_); + for (size_t sample = 0; sample < samples; ++sample) { + const double theta = std::numbers::pi * ((sample + 0.5) / samples - 0.5); + coordinates[sample] = std::sin(theta); + weights[sample] = std::pow(std::cos(theta), static_cast(dim_ - 2)); + } + for (size_t level = 0; level < levels; ++level) { + centers[level] = (2.0 * level + 1.0 - levels) / levels * + std::min(1.0, 2.5 / std::sqrt(static_cast(dim_))); + } + bool converged = false; + for (unsigned iteration = 0; iteration < 1024; ++iteration) { + std::fill(masses.begin(), masses.end(), 0); + std::fill(sums.begin(), sums.end(), 0); + size_t level = 0; + for (size_t sample = 0; sample < samples; ++sample) { + while (level + 1 < levels && + coordinates[sample] > (centers[level] + centers[level + 1]) * 0.5) { + ++level; + } + masses[level] += weights[sample]; + sums[level] += weights[sample] * coordinates[sample]; + } + double movement = 0; + for (size_t level = 0; level < levels; ++level) { + if (!(masses[level] > 0)) { + throw std::runtime_error("TurboQuant sphere codebook has an empty cell"); + } + const double updated = sums[level] / masses[level]; + movement = std::max(movement, std::abs(updated - centers[level])); + centers[level] = updated; + } + if (movement < 1e-12) { + converged = true; + break; + } + } + if (!converged) { + throw std::runtime_error("TurboQuant sphere codebook did not converge"); + } + for (size_t level = 0; level < levels; ++level) { + centroids_[level] = static_cast(centers[level]); + } + } + + // Derived mean terms: ||mean||^2 and R*mean, both accumulated in double. + void computeMeanTerms(std::span mean, float *rotated, + float &norm_squared) const noexcept { + double norm = 0; + for (float value : mean) + norm += double(value) * value; + norm_squared = static_cast(norm); + for (size_t row = 0; row < dim_; ++row) { + double dot = 0; + for (size_t col = 0; col < dim_; ++col) + dot += double(rotation_[row * dim_ + col]) * mean[col]; + rotated[row] = static_cast(dot); + } + } + + void selectKernels() { + packed_dot_ = + bits_ == 2 ? turboquant_detail::packedDot<2> : turboquant_detail::packedDot<3>; + stored_pair_dot_ = + bits_ == 2 ? turboquant_detail::packedPairDot<2> : turboquant_detail::packedPairDot<3>; +#if defined(CPU_FEATURES_ARCH_AARCH64) && defined(__ARM_NEON) + if (cpu_features::GetAarch64Info().features.asimd) { + if (dim_ >= 4) { + row_dot_ = turboquant_detail::rowDotNEON; + } + if (dim_ >= 8) { + packed_dot_ = bits_ == 2 ? turboquant_detail::packedDotNEON<2> + : turboquant_detail::packedDotNEON<3>; + stored_pair_dot_ = bits_ == 2 ? turboquant_detail::packedPairDotNEON<2> + : turboquant_detail::packedPairDotNEON<3>; + } + } +#endif + } + + static float readFloat(const unsigned char *bytes) { + float value; + std::memcpy(&value, bytes, sizeof(value)); + return value; + } + static void writeFloat(unsigned char *bytes, float value) { + std::memcpy(bytes, &value, sizeof(value)); + } + void writeCode(unsigned char *bytes, size_t coordinate, unsigned code) const { + const size_t start = coordinate * bits_; + for (unsigned bit = 0; bit < bits_; ++bit) { + bytes[(start + bit) / 8] |= ((code >> bit) & 1U) << ((start + bit) % 8); + } + } + float biasOffset(const void *stored) const { + return readFloat(static_cast(stored) + code_bytes_ + sizeof(float)); + } + +public: + // Bound both permanent matrices and the double-precision QR workspace before multiplying. + static bool dimSupported(size_t dim) { + constexpr size_t limit = std::numeric_limits::max() / (4 * sizeof(double)); + return dim >= 2 && dim <= limit / dim; + } + + // `initialize = false` leaves the rotation and centroids for loadFitted to fill. + TurboQuant(std::shared_ptr allocator, size_t dim, unsigned bits, + uint64_t seed, std::span mean, bool initialize = true) + : allocator_(std::move(allocator)), dim_(validatedDim(allocator_, dim, bits, mean)), + bits_(bits), seed_(seed), code_bytes_((dim_ * bits_ + 7) / 8), + rotation_(dim_ * dim_, allocator_), centroids_(size_t{1} << bits_, allocator_), + bias_mean_(dim_, allocator_), bias_rotated_mean_(dim_, allocator_) { + std::copy(mean.begin(), mean.end(), bias_mean_.begin()); + if (initialize) { + initializeRotation(seed); + initializeCentroids(); + } + computeMeanTerms(biasMean(), bias_rotated_mean_.data(), bias_mean_norm_squared_); + selectKernels(); + } + + // Writer-only: install a fitted mean before any vector is stored, with readers excluded. + // Updates the buffers in place, so the codec address and cached dispatches remain valid. + void setMean(std::span mean) noexcept { + assert(mean.size() == dim_); + std::copy(mean.begin(), mean.end(), bias_mean_.begin()); + computeMeanTerms(biasMean(), bias_rotated_mean_.data(), bias_mean_norm_squared_); + } + +#ifdef BUILD_TESTS + // Fitted state, schema 1. The full rotation is stored; the seed is metadata only. + void saveFitted(std::ostream &out) const { + const uint32_t header[] = {1, bits_, 1, 3, 1}; // schema, MSE, projection-match, mean-error + out.write(reinterpret_cast(header), sizeof(header)); + out.write(reinterpret_cast(&seed_), sizeof(seed_)); + auto write = [&](const auto &values) { + uint64_t count = values.size(); + out.write(reinterpret_cast(&count), sizeof(count)); + out.write(reinterpret_cast(values.data()), count * sizeof(float)); + }; + write(rotation_); + write(centroids_); + write(bias_mean_); + write(bias_rotated_mean_); + out.write(reinterpret_cast(&bias_mean_norm_squared_), sizeof(float)); + } + static std::shared_ptr loadFitted(std::istream &in, + const std::shared_ptr &allocator, + size_t dim, unsigned bits) { + // Check the dimension and complete payload before allocating based on file contents. + if (!dimSupported(dim) || (bits != 2 && bits != 3)) + throw std::runtime_error("Invalid TurboQuant fitted dimension"); + uint32_t header[5]{}; + uint64_t seed = 0; + in.read(reinterpret_cast(header), sizeof(header)); + in.read(reinterpret_cast(&seed), sizeof(seed)); + if (!in || header[0] != 1 || header[1] != bits || header[2] != 1 || header[3] != 3 || + header[4] != 1) + throw std::runtime_error("Unsupported TurboQuant fitted policy"); + const auto position = in.tellg(); + in.seekg(0, std::ios::end); + const auto end = in.tellg(); + in.seekg(position); + const uint64_t minimum = + 4 * sizeof(uint64_t) + sizeof(float) + + (uint64_t(dim) * dim + (uint64_t(1) << bits) + 2 * dim) * sizeof(float); + if (!in || position < 0 || end < position || uint64_t(end - position) < minimum) + throw std::runtime_error("Truncated TurboQuant fitted state"); + vecsim_stl::vector zero(dim, 0.0f, allocator); + auto codec = std::allocate_shared( + VecsimSTLAllocator(allocator), allocator, dim, bits, seed, + std::span(zero.data(), zero.size()), false); + auto read = [&](auto &values) { + uint64_t count = 0; + in.read(reinterpret_cast(&count), sizeof(count)); + if (!in || count != values.size()) + throw std::runtime_error("TurboQuant fitted length mismatch"); + in.read(reinterpret_cast(values.data()), count * sizeof(float)); + if (!in) + throw std::runtime_error("Truncated TurboQuant fitted state"); + for (float value : values) + if (!std::isfinite(value)) + throw std::runtime_error("Nonfinite TurboQuant fitted state"); + }; + read(codec->rotation_); + read(codec->centroids_); + read(codec->bias_mean_); + read(codec->bias_rotated_mean_); + in.read(reinterpret_cast(&codec->bias_mean_norm_squared_), sizeof(float)); + if (!in || !std::isfinite(codec->bias_mean_norm_squared_)) + throw std::runtime_error("Invalid TurboQuant fitted mean norm"); + for (size_t i = 0; i < codec->centroids_.size(); ++i) + if (std::abs(codec->centroids_[i]) > 1 || + (i && codec->centroids_[i] <= codec->centroids_[i - 1])) + throw std::runtime_error("Invalid TurboQuant fitted centroids"); + // Validate the relationship between persisted derived terms and the exact stored mean. + vecsim_stl::vector rotated(dim, allocator); + float norm_squared = 0; + codec->computeMeanTerms(codec->biasMean(), rotated.data(), norm_squared); + if (norm_squared != codec->bias_mean_norm_squared_ || + !std::equal(rotated.begin(), rotated.end(), codec->bias_rotated_mean_.begin())) + throw std::runtime_error("Inconsistent TurboQuant fitted mean terms"); + for (size_t row = 0; row < dim; ++row) { + double squared = 0; + for (size_t col = 0; col < dim; ++col) { + const double value = codec->rotation_[row * dim + col]; + squared += value * value; + } + if (std::abs(squared - 1.0) > 1e-4) + throw std::runtime_error("Invalid TurboQuant fitted rotation"); + } + return codec; + } +#endif + + size_t dim() const { return dim_; } + unsigned bits() const { return bits_; } + uint64_t seed() const { return seed_; } + std::span rotation() const { return {rotation_.data(), rotation_.size()}; } + std::span centroids() const { return {centroids_.data(), centroids_.size()}; } + std::span biasMean() const { return {bias_mean_.data(), bias_mean_.size()}; } + std::span biasRotatedMean() const { + return {bias_rotated_mean_.data(), bias_rotated_mean_.size()}; + } + size_t storageBytes() const { return code_bytes_ + 2 * sizeof(float); } + size_t queryBytes() const { return (dim_ + 1) * sizeof(float); } + // Rotation, centroids, mean and rotated mean, each in its own allocation. + static size_t sharedDataBytes(size_t dim, unsigned bits) { + return (dim * dim + (size_t(1) << bits) + 2 * dim) * sizeof(float); + } + + // Encodes x - mean. Reads all of `input` before writing `stored`, so they may alias. + void encode(const float *input, void *stored) const { + vecsim_stl::vector unit(dim_, allocator_); + float norm = 0; + double residual_mean_dot = 0; + for (size_t i = 0; i < dim_; ++i) { + unit[i] = input[i] - bias_mean_[i]; + norm = std::hypot(norm, unit[i]); + residual_mean_dot += double(unit[i]) * bias_mean_[i]; + } + auto *bytes = static_cast(stored); + std::memset(bytes, 0, storageBytes()); + if (norm == 0) { + return; + } + for (size_t i = 0; i < dim_; ++i) { + unit[i] /= norm; + } + double unit_code_dot = 0, code_mean_dot = 0; + for (size_t row = 0; row < dim_; ++row) { + float rotated = 0; + for (size_t col = 0; col < dim_; ++col) { + rotated += rotation_[row * dim_ + col] * unit[col]; + } + unsigned code = 0; + while (code + 1 < centroids_.size() && + rotated > (centroids_[code] + centroids_[code + 1]) * 0.5f) { + ++code; + } + writeCode(bytes, row, code); + code_mean_dot += double(centroids_[code]) * bias_rotated_mean_[row]; + unit_code_dot += double(rotated) * centroids_[code]; + } + const float scale = static_cast(double(norm) / unit_code_dot); + writeFloat(bytes + code_bytes_, scale); + writeFloat(bytes + code_bytes_ + sizeof(float), + static_cast(double(scale) * code_mean_dot - residual_mean_dot)); + } + + // Writes [R*q][q.mean] into an FP32-aligned buffer that does not alias `input`. + void prepareQuery(const float *input, void *query) const { + auto *output = static_cast(query); + for (size_t row = 0; row < dim_; ++row) { + float value = 0; + if (row_dot_) { + value = row_dot_(rotation_.data() + row * dim_, input, dim_); + } else { + for (size_t col = 0; col < dim_; ++col) { + value += rotation_[row * dim_ + col] * input[col]; + } + } + output[row] = value; + } + float mean_ip = 0; + for (size_t i = 0; i < dim_; ++i) + mean_ip += bias_mean_[i] * input[i]; + output[dim_] = mean_ip; + } + + // Stored-to-query estimate of 1 - x.q. Stored buffers may be unaligned. + float queryDistance(const void *stored, const void *query) const { + const auto *bytes = static_cast(stored); + const auto *query_bytes = static_cast(query); + const float mean_ip = readFloat(query_bytes + dim_ * sizeof(float)); + const float norm = readFloat(bytes + code_bytes_); + if (norm == 0) { + return 1.0f - mean_ip; + } + const float score = 1.0f - norm * packed_dot_(bytes, query_bytes, centroids_.data(), dim_); + return (score + biasOffset(stored)) - mean_ip; + } + + // Symmetric graph estimate of 1 - x.y. Codes represent x - mean, so both cross terms and + // the original-coordinate mean norm belong in the inner product. + float storedDistance(const void *a, const void *b) const { + const auto *bytes_a = static_cast(a); + const auto *bytes_b = static_cast(b); + const float norm_a = readFloat(bytes_a + code_bytes_); + const float norm_b = readFloat(bytes_b + code_bytes_); + const float dot = stored_pair_dot_(bytes_a, bytes_b, centroids_.data(), dim_, norm_a, + norm_b, bias_rotated_mean_.data()); + const float score = 1.0f - (dot + bias_mean_norm_squared_); + return score + (biasOffset(a) + biasOffset(b)); + } +}; + +} // namespace vecsim::quantization diff --git a/src/VecSim/quantization/turboquant_components.h b/src/VecSim/quantization/turboquant_components.h new file mode 100644 index 000000000..0117ab19a --- /dev/null +++ b/src/VecSim/quantization/turboquant_components.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include +#include + +#include "VecSim/quantization/turboquant.h" +#include "VecSim/spaces/computer/calculator.h" +#include "VecSim/spaces/computer/preprocessor_container.h" +#include "VecSim/vec_sim_index.h" + +namespace vecsim::quantization { + +inline bool isTurboQuant(VecSimQuantType type) { + return type == VecSimQuant_TQ2 || type == VecSimQuant_TQ3; +} +inline unsigned turboQuantBits(VecSimQuantType type) { + assert(isTurboQuant(type)); + return type == VecSimQuant_TQ2 ? 2 : 3; +} + +// FP32 input only. Storage holds the encoded x - mean; queries hold [R*q][q.mean]. +class TurboQuantPreprocessor : public PreprocessorInterface { + std::shared_ptr codec_; + +public: + TurboQuantPreprocessor(std::shared_ptr allocator, + std::shared_ptr codec) + : PreprocessorInterface(allocator), codec_(std::move(codec)) {} + + void preprocess(const void *original_blob, void *&storage_blob, void *&query_blob, + size_t &storage_blob_size, size_t &query_blob_size, + unsigned char storage_alignment, unsigned char query_alignment) const override { + assert(!storage_blob && !query_blob && &storage_blob_size != &query_blob_size); + preprocessForStorage(original_blob, storage_blob, storage_blob_size, storage_alignment); + preprocessQuery(original_blob, query_blob, query_blob_size, query_alignment); + } + void preprocessForStorage(const void *original_blob, void *&blob, size_t &input_blob_size, + unsigned char alignment) const override { + assert(!blob); + blob = this->allocator->allocate_aligned(codec_->storageBytes(), alignment); + codec_->encode(static_cast(original_blob), blob); + input_blob_size = codec_->storageBytes(); + } + void preprocessQuery(const void *original_blob, void *&blob, size_t &input_blob_size, + unsigned char alignment) const override { + assert(!blob); + blob = this->allocator->allocate_aligned(codec_->queryBytes(), alignment); + codec_->prepareQuery(static_cast(original_blob), blob); + input_blob_size = codec_->queryBytes(); + } + void preprocessStorageInPlace(void *original_blob, size_t input_blob_size) const override { + assert(input_blob_size >= codec_->storageBytes()); + codec_->encode(static_cast(original_blob), original_blob); + } +}; + +// Shares the codec with the preprocessor. Dispatches capture the codec address, which stays +// stable when the tiered index installs its fitted mean. +class TurboQuantCalculator : public IndexCalculatorInterface { + std::shared_ptr codec_; + + static float stored(const void *codec, const void *a, const void *b, size_t) { + return static_cast(codec)->storedDistance(a, b); + } + static float query(const void *codec, const void *stored, const void *query, size_t) { + return static_cast(codec)->queryDistance(stored, query); + } + +public: + TurboQuantCalculator(std::shared_ptr allocator, + std::shared_ptr codec) + : IndexCalculatorInterface(allocator), codec_(std::move(codec)) {} + + TurboQuant &codec() const { return *codec_; } + + float calcDistance(const void *v1, const void *v2, size_t dim) const override { + return stored(codec_.get(), v1, v2, dim); + } + float calcDistanceForQuery(const void *candidate, const void *query_blob, + size_t dim) const override { + return query(codec_.get(), candidate, query_blob, dim); + } + DistanceDispatch getDistanceDispatch(DistanceMode mode) const override { + return DistanceDispatch::stateful( + codec_.get(), mode == DistanceMode::StoredToStored ? stored : query); + } +}; + +// Returns the codec of a TQ2/TQ3 index, or nullptr for any other calculator. +template +TurboQuant *GetTurboQuantCodec(const IndexCalculatorInterface *calculator) { + if constexpr (std::is_same_v) { + const auto *tq = dynamic_cast(calculator); + return tq ? &tq->codec() : nullptr; + } + return nullptr; +} + +inline IndexComponents +CreateTurboQuantIndexComponents(const std::shared_ptr &allocator, + std::shared_ptr codec) { + auto *calculator = new (allocator) TurboQuantCalculator(allocator, codec); + auto *container = new (allocator) + MultiPreprocessorsContainer(allocator, alignof(float), /*storage*/ 0); + [[maybe_unused]] const int ret = + container->addPreprocessor(new (allocator) TurboQuantPreprocessor(allocator, codec)); + assert(ret != -1 && "TurboQuant preprocessor was not added correctly"); + return {calculator, container}; +} + +// NULL params select a zero mean and the default seed. Throws on an unsupported dim or a +// nonfinite mean; the index factory converts that into a NULL index. +inline IndexComponents +CreateTurboQuantIndexComponents(const std::shared_ptr &allocator, size_t dim, + unsigned bits, const VecSimTurboQuantParams *params) { + vecsim_stl::vector mean(dim, 0.0f, allocator); + if (params && params->mean) { + std::copy(params->mean, params->mean + dim, mean.begin()); + } + auto codec = std::allocate_shared( + VecsimSTLAllocator(allocator), allocator, dim, bits, + params ? params->seed : VECSIM_TURBOQUANT_DEFAULT_SEED, + std::span(mean.data(), mean.size())); + return CreateTurboQuantIndexComponents(allocator, std::move(codec)); +} + +inline size_t EstimateTurboQuantComponentsMemory(size_t dim, unsigned bits) { + if (!TurboQuant::dimSupported(dim)) { + throw std::invalid_argument("Unsupported TurboQuant dimension"); + } + const size_t overhead = VecSimAllocator::getAllocationOverheadSize(); + // Calculator, container and preprocessor objects; the shared codec block (object plus + // control block); and its four float buffers. + return sizeof(TurboQuantCalculator) + sizeof(MultiPreprocessorsContainer) + + sizeof(TurboQuantPreprocessor) + sizeof(TurboQuant) + + sizeof(VecsimSTLAllocator) + 4 * sizeof(void *) + 8 * overhead + + TurboQuant::sharedDataBytes(dim, bits); +} + +} // namespace vecsim::quantization diff --git a/src/VecSim/quantization/turboquant_kernels.h b/src/VecSim/quantization/turboquant_kernels.h new file mode 100644 index 000000000..e0b48c910 --- /dev/null +++ b/src/VecSim/quantization/turboquant_kernels.h @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include "cpu_features_macros.h" +#include +#include +#include +#if defined(CPU_FEATURES_ARCH_AARCH64) && defined(__ARM_NEON) +#include "cpuinfo_aarch64.h" +#include +#endif + +namespace vecsim::quantization::turboquant_detail { + +using PackedDot = float (*)(const unsigned char *, const unsigned char *, const float *, size_t); +using RowDot = float (*)(const float *, const float *, size_t); +using PackedPairDot = float (*)(const unsigned char *, const unsigned char *, const float *, size_t, + float, float, const float *); + +inline float loadFloat(const unsigned char *bytes) { + float value; + std::memcpy(&value, bytes, sizeof(value)); + return value; +} + +template +inline unsigned loadPackedBlock(const unsigned char *bytes) { + static_assert(Bits == 2 || Bits == 3); + unsigned packed = unsigned(bytes[0]) | (unsigned(bytes[1]) << 8); + if constexpr (Bits == 3) { + packed |= unsigned(bytes[2]) << 16; + } + return packed; +} + +template +inline unsigned loadCode(const unsigned char *bytes, size_t coordinate) { + const size_t start = coordinate * Bits; + const unsigned shift = start % 8; + unsigned packed = bytes[start / 8]; + if (shift + Bits > 8) { + packed |= unsigned(bytes[start / 8 + 1]) << 8; + } + return (packed >> shift) & ((1U << Bits) - 1); +} + +template +inline float packedDot(const unsigned char *codes, const unsigned char *query, + const float *centroids, size_t dim) { + float sum = 0; + size_t i = 0; + for (; i + 8 <= dim; i += 8) { + const unsigned packed = loadPackedBlock(codes + i / 8 * Bits); + for (unsigned lane = 0; lane < 8; ++lane) { + const unsigned code = (packed >> (lane * Bits)) & ((1U << Bits) - 1); + sum += loadFloat(query + (i + lane) * sizeof(float)) * centroids[code]; + } + } + for (; i < dim; ++i) { + sum += loadFloat(query + i * sizeof(float)) * centroids[loadCode(codes, i)]; + } + return sum; +} + +// Both inputs remain packed. A null mean denotes the uncentered estimator. +template +inline float packedPairDot(const unsigned char *a, const unsigned char *b, const float *centroids, + size_t dim, float norm_a, float norm_b, const float *rotated_mean) { + float sum = 0; + size_t i = 0; + for (; i + 8 <= dim; i += 8) { + const unsigned packed_a = loadPackedBlock(a + i / 8 * Bits); + const unsigned packed_b = loadPackedBlock(b + i / 8 * Bits); + for (unsigned lane = 0; lane < 8; ++lane) { + const unsigned shift = lane * Bits; + const float za = norm_a * centroids[(packed_a >> shift) & ((1U << Bits) - 1)]; + const float zb = norm_b * centroids[(packed_b >> shift) & ((1U << Bits) - 1)]; + sum += za * zb; + if (rotated_mean) + sum += (za + zb) * rotated_mean[i + lane]; + } + } + for (; i < dim; ++i) { + const float za = norm_a * centroids[loadCode(a, i)]; + const float zb = norm_b * centroids[loadCode(b, i)]; + sum += za * zb; + if (rotated_mean) + sum += (za + zb) * rotated_mean[i]; + } + return sum; +} + +#if defined(CPU_FEATURES_ARCH_AARCH64) && defined(__ARM_NEON) +inline float32x4_t loadQuery4(const unsigned char *bytes) { + float32x4_t value; + std::memcpy(&value, bytes, sizeof(value)); + return value; +} + +template +inline float32x4_t lookupCentroids(uint32x4_t codes, const uint8x16x2_t &table) { + // Each code selects the four consecutive bytes of its FP32 centroid. + const auto indices = + vreinterpretq_u8_u32(vaddq_u32(vmulq_n_u32(codes, 0x04040404U), vdupq_n_u32(0x03020100U))); + if constexpr (Bits == 2) { + return vreinterpretq_f32_u8(vqtbl1q_u8(table.val[0], indices)); + } else { + return vreinterpretq_f32_u8(vqtbl2q_u8(table, indices)); + } +} + +template +inline float packedDotNEON(const unsigned char *codes, const unsigned char *query, + const float *centroids, size_t dim) { + uint8x16x2_t table; + table.val[0] = vreinterpretq_u8_f32(vld1q_f32(centroids)); + table.val[1] = vdupq_n_u8(0); + if constexpr (Bits == 3) { + table.val[1] = vreinterpretq_u8_f32(vld1q_f32(centroids + 4)); + } + const int32_t shifts0[] = {0, -int(Bits), -int(2 * Bits), -int(3 * Bits)}; + const int32_t shifts1[] = {-int(4 * Bits), -int(5 * Bits), -int(6 * Bits), -int(7 * Bits)}; + const int32x4_t shift0 = vld1q_s32(shifts0), shift1 = vld1q_s32(shifts1); + const uint32x4_t mask = vdupq_n_u32((1U << Bits) - 1); + float32x4_t sum0 = vdupq_n_f32(0), sum1 = vdupq_n_f32(0); + size_t i = 0; + for (; i + 8 <= dim; i += 8) { + const uint32x4_t packed = vdupq_n_u32(loadPackedBlock(codes + i / 8 * Bits)); + const auto c0 = lookupCentroids(vandq_u32(vshlq_u32(packed, shift0), mask), table); + const auto c1 = lookupCentroids(vandq_u32(vshlq_u32(packed, shift1), mask), table); + // Make the validated Release FMA arithmetic explicit in both native kernels. + sum0 = vfmaq_f32(sum0, loadQuery4(query + i * sizeof(float)), c0); + sum1 = vfmaq_f32(sum1, loadQuery4(query + (i + 4) * sizeof(float)), c1); + } + float sum = vaddvq_f32(vaddq_f32(sum0, sum1)); + for (; i < dim; ++i) { + sum += loadFloat(query + i * sizeof(float)) * centroids[loadCode(codes, i)]; + } + return sum; +} + +template +inline float packedPairDotNEON(const unsigned char *a, const unsigned char *b, + const float *centroids, size_t dim, float norm_a, float norm_b, + const float *rotated_mean) { + uint8x16x2_t table; + table.val[0] = vreinterpretq_u8_f32(vld1q_f32(centroids)); + table.val[1] = vdupq_n_u8(0); + if constexpr (Bits == 3) + table.val[1] = vreinterpretq_u8_f32(vld1q_f32(centroids + 4)); + const int32_t shifts0[] = {0, -int(Bits), -int(2 * Bits), -int(3 * Bits)}; + const int32_t shifts1[] = {-int(4 * Bits), -int(5 * Bits), -int(6 * Bits), -int(7 * Bits)}; + const auto shift0 = vld1q_s32(shifts0), shift1 = vld1q_s32(shifts1); + const auto mask = vdupq_n_u32((1U << Bits) - 1); + float32x4_t sum0 = vdupq_n_f32(0), sum1 = vdupq_n_f32(0); + size_t i = 0; + for (; i + 8 <= dim; i += 8) { + const auto packed_a = vdupq_n_u32(loadPackedBlock(a + i / 8 * Bits)); + const auto packed_b = vdupq_n_u32(loadPackedBlock(b + i / 8 * Bits)); + const auto a0 = vmulq_n_f32( + lookupCentroids(vandq_u32(vshlq_u32(packed_a, shift0), mask), table), norm_a); + const auto a1 = vmulq_n_f32( + lookupCentroids(vandq_u32(vshlq_u32(packed_a, shift1), mask), table), norm_a); + const auto b0 = vmulq_n_f32( + lookupCentroids(vandq_u32(vshlq_u32(packed_b, shift0), mask), table), norm_b); + const auto b1 = vmulq_n_f32( + lookupCentroids(vandq_u32(vshlq_u32(packed_b, shift1), mask), table), norm_b); + sum0 = vaddq_f32(sum0, vmulq_f32(a0, b0)); + sum1 = vaddq_f32(sum1, vmulq_f32(a1, b1)); + if (rotated_mean) { + sum0 = vaddq_f32(sum0, vmulq_f32(vaddq_f32(a0, b0), vld1q_f32(rotated_mean + i))); + sum1 = vaddq_f32(sum1, vmulq_f32(vaddq_f32(a1, b1), vld1q_f32(rotated_mean + i + 4))); + } + } + float sum = vaddvq_f32(vaddq_f32(sum0, sum1)); + for (; i < dim; ++i) { + const float za = norm_a * centroids[loadCode(a, i)]; + const float zb = norm_b * centroids[loadCode(b, i)]; + sum += za * zb; + if (rotated_mean) + sum += (za + zb) * rotated_mean[i]; + } + return sum; +} + +inline float rowDotNEON(const float *row, const float *query, size_t dim) { + float32x4_t sum0 = vdupq_n_f32(0), sum1 = vdupq_n_f32(0); + float32x4_t sum2 = vdupq_n_f32(0), sum3 = vdupq_n_f32(0); + size_t i = 0; + for (; i + 16 <= dim; i += 16) { + sum0 = vaddq_f32(sum0, vmulq_f32(vld1q_f32(row + i), vld1q_f32(query + i))); + sum1 = vaddq_f32(sum1, vmulq_f32(vld1q_f32(row + i + 4), vld1q_f32(query + i + 4))); + sum2 = vaddq_f32(sum2, vmulq_f32(vld1q_f32(row + i + 8), vld1q_f32(query + i + 8))); + sum3 = vaddq_f32(sum3, vmulq_f32(vld1q_f32(row + i + 12), vld1q_f32(query + i + 12))); + } + for (; i + 4 <= dim; i += 4) { + sum0 = vaddq_f32(sum0, vmulq_f32(vld1q_f32(row + i), vld1q_f32(query + i))); + } + float sum = vaddvq_f32(vaddq_f32(vaddq_f32(sum0, sum1), vaddq_f32(sum2, sum3))); + for (; i < dim; ++i) { + sum += row[i] * query[i]; + } + return sum; +} +#endif + +} // namespace vecsim::quantization::turboquant_detail diff --git a/src/VecSim/types/turboquant.h b/src/VecSim/types/turboquant.h new file mode 100644 index 000000000..7215eb5f9 --- /dev/null +++ b/src/VecSim/types/turboquant.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include + +namespace vecsim_types { + +// Stored layout: packed Bits-wide codes, then the FP32 scale and the FP32 mean-error delta. +template +struct tq { + static_assert(Bits == 2 || Bits == 3); + static constexpr size_t data_bytes_count(size_t dim) { return (dim * Bits + 7) / 8; } + static constexpr size_t storage_bytes_count(size_t dim) { + return data_bytes_count(dim) + 2 * sizeof(float); + } +}; + +using tq2 = tq<2>; +using tq3 = tq<3>; + +} // namespace vecsim_types diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 1d7cf7aa3..f9f163417 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -75,8 +75,19 @@ typedef enum { VecSimQuant_SQ8 = 1, // Packed 4-bit scalar quantization with optional mean centering (FP32 input). VecSimQuant_SQ4 = 2, + VecSimQuant_TQ2 = 3, // FP32 MSE TurboQuant, 2-bit codes. + VecSimQuant_TQ3 = 4, // FP32 MSE TurboQuant, 3-bit codes. } VecSimQuantType; +// Optional configuration for TQ2/TQ3. The factory copies the mean during construction. +// NULL quantParams selects seed 42 and a zero mean. A supplied config uses its seed exactly +// (including 0). Tiered indexes with a positive QuantNormalizationSetSize train the mean. +typedef struct { + const float *mean; // Optional float[dim]; NULL selects a zero mean. + uint64_t seed; +} VecSimTurboQuantParams; +#define VECSIM_TURBOQUANT_DEFAULT_SEED UINT64_C(42) + // Algorithm type/library. typedef enum { VecSimAlgo_BF, VecSimAlgo_HNSWLIB, VecSimAlgo_TIERED, VecSimAlgo_SVS } VecSimAlgo; @@ -193,6 +204,7 @@ typedef struct { double epsilon; VecSimQuantType quantType; // Defaults to VecSimQuant_NONE. // SQ8/SQ4 mean vector (float[dim]); NULL disables mean centering. Copied during construction. + // TQ2/TQ3: VecSimTurboQuantParams*; NULL uses a zero mean and seed 42. // L2 centering keeps queries in FLOAT32, including when the input type is FLOAT16. const void *quantParams; } HNSWParams; diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 96226f2ff..61c3b692a 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -36,6 +36,7 @@ endif() add_executable(test_hnsw ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw.cpp test_hnsw_multi.cpp test_hnsw_tiered.cpp unit_test_utils.cpp) add_executable(test_hnsw_sq4 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_sq4.cpp test_sq4_distances.cpp test_sq4_components.cpp unit_test_utils.cpp) +add_executable(test_hnsw_turboquant ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_turboquant.cpp unit_test_utils.cpp) add_executable(test_hnsw_sq8 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_sq8.cpp unit_test_utils.cpp) add_executable(test_bruteforce ../utils/test_main_with_timeout.cpp test_bruteforce.cpp test_bruteforce_multi.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) add_executable(test_allocator ../utils/test_main_with_timeout.cpp test_allocator.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) @@ -52,6 +53,7 @@ add_executable(test_svs ../utils/test_main_with_timeout.cpp ../utils/mock_thread target_link_libraries(test_hnsw PUBLIC gtest VectorSimilarity) target_link_libraries(test_hnsw_sq8 PUBLIC gtest VectorSimilarity) +target_link_libraries(test_hnsw_turboquant PUBLIC gtest VectorSimilarity) target_link_libraries(test_hnsw_sq4 PUBLIC gtest VectorSimilarity) target_link_libraries(test_bruteforce PUBLIC gtest VectorSimilarity) target_link_libraries(test_allocator PUBLIC gtest VectorSimilarity) @@ -75,6 +77,7 @@ gtest_discover_tests(test_allocation_transfer) gtest_discover_tests(test_hnsw) gtest_discover_tests(test_hnsw_sq8) gtest_discover_tests(test_hnsw_sq4) +gtest_discover_tests(test_hnsw_turboquant) gtest_discover_tests(test_bruteforce) gtest_discover_tests(test_allocator) gtest_discover_tests(test_spaces) diff --git a/tests/unit/test_hnsw_sq4.cpp b/tests/unit/test_hnsw_sq4.cpp index 62f17ca4f..7448a89cf 100644 --- a/tests/unit/test_hnsw_sq4.cpp +++ b/tests/unit/test_hnsw_sq4.cpp @@ -556,12 +556,12 @@ TEST(HNSWSQ4Factory, RejectsInvalidParams) { .quantParams = quant_params}, false); - // Value 3 is not a VecSimQuantType enumerator. + // Value 5 is not a VecSimQuantType enumerator. for (bool is_normalized : {false, true}) { expect_rejected({.type = VecSimType_FLOAT32, .dim = dim, .metric = VecSimMetric_IP, - .quantType = static_cast(3), + .quantType = static_cast(5), .quantParams = quant_params}, is_normalized); } diff --git a/tests/unit/test_hnsw_turboquant.cpp b/tests/unit/test_hnsw_turboquant.cpp new file mode 100644 index 000000000..671949d51 --- /dev/null +++ b/tests/unit/test_hnsw_turboquant.cpp @@ -0,0 +1,512 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#include "gtest/gtest.h" +#include "VecSim/algorithms/hnsw/hnsw_single.h" +#include "VecSim/algorithms/hnsw/hnsw_tiered.h" +#include "VecSim/index_factories/hnsw_factory.h" +#include "VecSim/index_factories/tiered_factory.h" +#include "VecSim/quantization/turboquant_components.h" +#include "VecSim/vec_sim.h" +#include "mock_thread_pool.h" +#include "unit_test_utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace vecsim::quantization; + +// Friend of HNSWIndex and TieredHNSWIndex, for typed inspection in the tests below. +class TurboQuantTest : public ::testing::TestWithParam { +protected: + static const TurboQuant &codecOf(const HNSWIndex *index) { + const auto *codec = GetTurboQuantCodec(index->getIndexCalculator()); + if (!codec) + throw std::runtime_error("Expected a TurboQuant index"); + return *codec; + } + static HNSWIndex *backendOf(VecSimIndex *index) { + return dynamic_cast *>(index)->getHNSWIndex(); + } +}; + +namespace { +using IndexOwner = std::unique_ptr; + +HNSWParams parameters(unsigned bits, size_t dim, const VecSimTurboQuantParams *config = nullptr) { + return {.type = VecSimType_FLOAT32, .dim = dim, .metric = VecSimMetric_IP, .multi = false, + .blockSize = 8, .M = 8, .efConstruction = 64, .efRuntime = 128, + .quantType = bits == 2 ? VecSimQuant_TQ2 : VecSimQuant_TQ3, .quantParams = config}; +} +float unalignedFloat(const void *data, size_t offset) { + float value; + std::memcpy(&value, static_cast(data) + offset, sizeof(value)); + return value; +} +// An independent bit reader and decoded-score reference, including the full mean error. +std::vector decoded(const TurboQuant &quantizer, const void *blob) { + const auto *bytes = static_cast(blob); + std::vector values(quantizer.dim()); + const float scale = unalignedFloat(blob, (quantizer.dim() * quantizer.bits() + 7) / 8); + for (size_t i = 0; i < values.size(); ++i) { + unsigned code = 0; + for (unsigned b = 0; b < quantizer.bits(); ++b) { + const size_t bit = i * quantizer.bits() + b; + code |= ((bytes[bit / 8] >> (bit % 8)) & 1) << b; + } + values[i] = double(scale) * quantizer.centroids()[code]; + } + return values; +} +double queryReference(const TurboQuant &quantizer, const void *blob, const float *query) { + const auto values = decoded(quantizer, blob); + double score = 1 + unalignedFloat(blob, quantizer.storageBytes() - sizeof(float)); + for (size_t i = 0; i < quantizer.dim(); ++i) { + double rotated = 0; + for (size_t j = 0; j < quantizer.dim(); ++j) + rotated += double(quantizer.rotation()[i * quantizer.dim() + j]) * query[j]; + score -= values[i] * rotated; + score -= double(quantizer.biasMean()[i]) * query[i]; + } + return score; +} +double graphReference(const TurboQuant &quantizer, const void *a, const void *b) { + const auto x = decoded(quantizer, a), y = decoded(quantizer, b); + double inner = 0, mean_squared = 0; + for (size_t i = 0; i < quantizer.dim(); ++i) { + double rotated_mean = 0; + for (size_t j = 0; j < quantizer.dim(); ++j) + rotated_mean += + double(quantizer.rotation()[i * quantizer.dim() + j]) * quantizer.biasMean()[j]; + inner += x[i] * y[i] + (x[i] + y[i]) * rotated_mean; + mean_squared += double(quantizer.biasMean()[i]) * quantizer.biasMean()[i]; + } + return 1 - (inner + mean_squared) + + unalignedFloat(a, quantizer.storageBytes() - sizeof(float)) + + unalignedFloat(b, quantizer.storageBytes() - sizeof(float)); +} +std::string path(unsigned bits, const char *suffix) { + return ::testing::TempDir() + "vecsim_tq" + std::to_string(bits) + "_" + suffix + ".hnsw"; +} +struct WriteModeGuard { + VecSimWriteMode previous = VecSimIndexInterface::asyncWriteMode; + ~WriteModeGuard() { VecSim_SetWriteMode(previous); } +}; +struct TestPool : tieredIndexMock { + TestPool() : tieredIndexMock(0) {} + ~TestPool() { + while (!jobQ.empty()) + thread_iteration(); + reset_ctx(); // The mock destructor otherwise dereferences an empty index context. + } +}; +VecSimIndex *newTiered(TestPool &pool, VecSimParams &primary, size_t training_size) { + TieredIndexParams params{.jobQueue = &pool.jobQ, .jobQueueCtx = pool.ctx, + .submitCb = tieredIndexMock::submit_callback, .flatBufferLimit = 1, + .primaryIndexParams = &primary, + .specificParams = {TieredHNSWParams{ + .QuantNormalizationSetSize = training_size}}}; + auto *index = TieredFactory::NewIndex(¶ms); + pool.ctx->index_strong_ref = std::shared_ptr(index, VecSimIndex_Free); + return index; +} +} // namespace + +TEST_P(TurboQuantTest, IndependentEncodingAndScoreReferencesWithUnalignedTails) { + const unsigned bits = GetParam(); + for (size_t dim : {2, 7, 8, 9, 17}) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + std::vector mean(dim), x(dim), y(dim), query(dim), residual(dim); + for (size_t i = 0; i < dim; ++i) { + mean[i] = 0.12f * std::sin(float(i + 1)); + x[i] = 0.3f * std::cos(float(i + 2)); + y[i] = 0.2f * std::sin(float(i + 3)); + query[i] = 0.1f * std::cos(float(i + 4)); + residual[i] = x[i] - mean[i]; + } + const VecSimTurboQuantParams config{.mean = mean.data(), .seed = 42}; + auto parts = CreateTurboQuantIndexComponents(allocator, dim, bits, &config); + std::unique_ptr pp(parts.preprocessors); + std::unique_ptr> calculator(parts.indexCalculator); + const TurboQuant &native = *GetTurboQuantCodec(calculator.get()); + auto a = pp->preprocessForStorage(x.data(), dim * sizeof(float)); + auto b = pp->preprocessForStorage(y.data(), dim * sizeof(float)); + auto q = pp->preprocessQuery(query.data(), dim * sizeof(float)); + EXPECT_EQ(native.storageBytes(), (dim * bits + 7) / 8 + 8); + EXPECT_EQ(native.queryBytes(), 4 * dim + 4); + // Independently recover centroid choices and projection-match scale from raw residual. + float norm = 0; + for (float value : residual) + norm = std::hypot(norm, value); + double projection = 0, code_mean = 0, residual_mean = 0; + std::vector expected_codes(dim); + for (size_t i = 0; i < dim; ++i) { + float rotated = 0; + for (size_t j = 0; j < dim; ++j) + rotated += native.rotation()[i * dim + j] * (residual[j] / norm); + unsigned expected_code = 0; + for (size_t c = 1; c < native.centroids().size(); ++c) + if (rotated > (native.centroids()[c - 1] + native.centroids()[c]) * 0.5f) + expected_code = c; + expected_codes[i] = expected_code; + projection += double(rotated) * native.centroids()[expected_code]; + code_mean += double(native.centroids()[expected_code]) * native.biasRotatedMean()[i]; + residual_mean += double(residual[i]) * mean[i]; + } + const size_t code_bytes = (dim * bits + 7) / 8; + const float expected_scale = float(double(norm) / projection); + EXPECT_FLOAT_EQ(unalignedFloat(a.get(), code_bytes), expected_scale); + EXPECT_FLOAT_EQ(unalignedFloat(a.get(), code_bytes + 4), + float(double(expected_scale) * code_mean - residual_mean)); + const auto decoded_a = decoded(native, a.get()); + for (size_t i = 0; i < dim; ++i) + EXPECT_DOUBLE_EQ(decoded_a[i], + double(expected_scale) * native.centroids()[expected_codes[i]]); + // A one-byte offset exercises the real unaligned record layout and full stride. + std::vector unaligned(native.storageBytes() + 1); + std::memcpy(unaligned.data() + 1, a.get(), native.storageBytes()); + EXPECT_NEAR(calculator->calcDistanceForQuery(unaligned.data() + 1, q.get(), dim), + queryReference(native, a.get(), query.data()), 3e-6); + EXPECT_NEAR(calculator->calcDistance(a.get(), b.get(), dim), + graphReference(native, a.get(), b.get()), 3e-6); + // Cached dispatches use the same scoring as the virtual calls. + const auto cached = calculator->getDistanceDispatch(DistanceMode::StoredToQuery); + EXPECT_EQ(cached(a.get(), q.get(), dim), + calculator->calcDistanceForQuery(a.get(), q.get(), dim)); + auto zero_residual = pp->preprocessForStorage(mean.data(), dim * sizeof(float)); + EXPECT_EQ(unalignedFloat(zero_residual.get(), code_bytes), 0); + EXPECT_NEAR(calculator->calcDistanceForQuery(zero_residual.get(), q.get(), dim), + queryReference(native, zero_residual.get(), query.data()), 3e-6); + } +} + +TEST_P(TurboQuantTest, PublicFactoryMetadataRangePersistenceAndMutation) { + const unsigned bits = GetParam(); + constexpr size_t dim = 9; + std::vector mean(dim, 0.1f), query(dim, 0.05f); + VecSimTurboQuantParams config{.mean = mean.data(), .seed = 0}; + auto params = CreateParams(parameters(bits, dim, &config)); + IndexOwner index(VecSimIndex_New(¶ms), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + auto *hnsw = dynamic_cast *>(index.get()); + ASSERT_NE(hnsw, nullptr); + const TurboQuant &codec = codecOf(hnsw); + EXPECT_EQ(codec.seed(), 0); + EXPECT_EQ(hnsw->getStoredDataSize(), (dim * bits + 7) / 8 + 8); + EXPECT_EQ(index->debugInfo().commonInfo.memory, hnsw->getAllocator()->getAllocationSize()); + EXPECT_GE(VecSimIndex_EstimateInitialSize(¶ms), index->getAllocationSize()); + EXPECT_GE(VecSimIndex_EstimateElementSize(¶ms), hnsw->getStoredDataSize()); + std::array, 12> rows{}; + for (size_t i = 0; i < rows.size(); ++i) { + for (size_t j = 0; j < dim; ++j) + rows[i][j] = 0.2f * std::sin(float(i * dim + j + 1)); + EXPECT_EQ(VecSimIndex_AddVector(index.get(), rows[i].data(), i), 1); + } + const double expected = queryReference(codec, hnsw->getDataByInternalId(0), query.data()); + EXPECT_NEAR(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), 0, query.data()), expected, 3e-6); + for (double radius : {expected - 1e-3, expected + 1e-3}) { + auto *reply = VecSimIndex_RangeQuery(index.get(), query.data(), radius, nullptr, BY_SCORE); + auto *it = VecSimQueryReply_GetIterator(reply); + bool found = false; + while (VecSimQueryReply_IteratorHasNext(it)) { + const auto *item = VecSimQueryReply_IteratorNext(it); + if (VecSimQueryResult_GetId(item) == 0) + found = true; + EXPECT_LE(VecSimQueryResult_GetScore(item), radius + 1e-6); + } + EXPECT_EQ(found, radius > expected); + VecSimQueryReply_IteratorFree(it); + VecSimQueryReply_Free(reply); + } + const auto filename = path(bits, "roundtrip"); + hnsw->saveIndex(filename); + { + std::ifstream in(filename, std::ios::binary); + EXPECT_EQ(HNSWSerializer::ReadVersion(in), HNSWSerializer::EncodingVersion::V5); + } + IndexOwner restored(HNSWFactory::NewIndex(filename), VecSimIndex_Free); + ASSERT_NE(restored, nullptr); + auto *loaded = dynamic_cast *>(restored.get()); + ASSERT_NE(loaded, nullptr); + for (size_t i = 0; i < rows.size(); ++i) { + EXPECT_EQ(std::memcmp(hnsw->getDataByInternalId(i), loaded->getDataByInternalId(i), + hnsw->getStoredDataSize()), + 0); + EXPECT_FLOAT_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), i, query.data()), + VecSimIndex_GetDistanceFrom_Unsafe(restored.get(), i, query.data())); + } + // The fitted state, including the full rotation, round-trips exactly. + for (auto getter : {&TurboQuant::rotation, &TurboQuant::centroids, &TurboQuant::biasMean, + &TurboQuant::biasRotatedMean}) { + const auto lhs = (codec.*getter)(); + const auto rhs = (codecOf(loaded).*getter)(); + ASSERT_EQ(lhs.size(), rhs.size()); + EXPECT_TRUE(std::equal(lhs.begin(), lhs.end(), rhs.begin())); + } + EXPECT_EQ(VecSimIndex_AddVector(restored.get(), rows[0].data(), 99), 1); + EXPECT_EQ(restored->relabelVector(99, 100), VecSimRelabel_OK); + EXPECT_EQ(VecSimIndex_DeleteVector(restored.get(), 100), 1); + EXPECT_TRUE(loaded->checkIntegrity().valid_state); + std::remove(filename.c_str()); +} + +TEST_P(TurboQuantTest, CosinePersistenceRequiresNormalizedInputOptIn) { + const unsigned bits = GetParam(); + constexpr size_t dim = 9; + auto hp = parameters(bits, dim); + hp.metric = VecSimMetric_Cosine; + auto params = CreateParams(hp); + EXPECT_EQ(VecSimIndex_New(¶ms), nullptr); + IndexOwner index(HNSWFactory::NewIndex(¶ms, true), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + auto *backend = dynamic_cast *>(index.get()); + ASSERT_NE(backend, nullptr); + std::array row{3, 4}, query{4, 3}; + VecSim_Normalize(row.data(), dim, VecSimType_FLOAT32); + VecSim_Normalize(query.data(), dim, VecSimType_FLOAT32); + ASSERT_EQ(VecSimIndex_AddVector(index.get(), row.data(), 0), 1); + const auto filename = path(bits, "cosine"); + backend->saveIndex(filename); + IndexOwner raw_loaded(HNSWFactory::NewIndex(filename, false), VecSimIndex_Free); + EXPECT_EQ(raw_loaded, nullptr); + IndexOwner restored(HNSWFactory::NewIndex(filename, true), VecSimIndex_Free); + ASSERT_NE(restored, nullptr); + auto *loaded = dynamic_cast *>(restored.get()); + ASSERT_NE(loaded, nullptr); + // Opting in promises normalized queries and inserts; the backend normalizes neither. + row = {4, -3}; + VecSim_Normalize(row.data(), dim, VecSimType_FLOAT32); + ASSERT_EQ(VecSimIndex_AddVector(index.get(), row.data(), 1), 1); + ASSERT_EQ(VecSimIndex_AddVector(restored.get(), row.data(), 1), 1); + for (size_t label : {0U, 1U}) { + EXPECT_EQ(std::memcmp(backend->getDataByInternalId(label), + loaded->getDataByInternalId(label), backend->getStoredDataSize()), + 0); + EXPECT_FLOAT_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index.get(), label, query.data()), + VecSimIndex_GetDistanceFrom_Unsafe(restored.get(), label, query.data())); + } + const auto *stored = loaded->getDataByInternalId(0); + EXPECT_NEAR(VecSimIndex_GetDistanceFrom_Unsafe(restored.get(), 0, query.data()), + queryReference(codecOf(loaded), stored, query.data()), 3e-6); + EXPECT_TRUE(loaded->checkIntegrity().valid_state); + std::remove(filename.c_str()); +} + +TEST_P(TurboQuantTest, UnsupportedConfigurationsAreRejected) { + auto hp = parameters(GetParam(), 9); + for (auto type : {VecSimType_FLOAT16, VecSimType_FLOAT64, VecSimType_INT8}) { + hp.type = type; + auto params = CreateParams(hp); + EXPECT_EQ(VecSimIndex_New(¶ms), nullptr); + EXPECT_EQ(VecSimIndex_EstimateInitialSize(¶ms), SIZE_MAX); + EXPECT_EQ(VecSimIndex_EstimateElementSize(¶ms), SIZE_MAX); + } + hp.type = VecSimType_FLOAT32; + hp.metric = VecSimMetric_L2; + auto l2 = CreateParams(hp); + EXPECT_EQ(VecSimIndex_New(&l2), nullptr); + EXPECT_EQ(VecSimIndex_EstimateInitialSize(&l2), SIZE_MAX); + EXPECT_EQ(VecSimIndex_EstimateElementSize(&l2), SIZE_MAX); + hp.metric = VecSimMetric_IP; + hp.dim = 1; + auto too_small = CreateParams(hp); + EXPECT_EQ(VecSimIndex_New(&too_small), nullptr); + EXPECT_EQ(VecSimIndex_EstimateInitialSize(&too_small), SIZE_MAX); + EXPECT_EQ(VecSimIndex_EstimateElementSize(&too_small), SIZE_MAX); +} + +TEST_P(TurboQuantTest, TieredLiveMeanTrainingRetriesAndMigratesInBothWriteModes) { + const unsigned bits = GetParam(); + WriteModeGuard mode_guard; + for (auto mode : {VecSim_WriteAsync, VecSim_WriteInPlace}) { + for (auto metric : {VecSimMetric_IP, VecSimMetric_Cosine}) { + VecSim_SetWriteMode(mode); + TestPool pool; + constexpr size_t dim = 9; + auto hp = parameters(bits, dim); + hp.metric = metric; + auto primary = CreateParams(hp); + auto *base = newTiered(pool, primary, 4); + ASSERT_NE(base, nullptr); + auto *tiered = dynamic_cast *>(base); + auto *backend = backendOf(base); + std::array, 5> rows{}; + for (size_t i = 0; i < rows.size(); ++i) + for (size_t j = 0; j < dim; ++j) + rows[i][j] = 0.3f * std::sin(float(i * dim + j + 1)); + VecSimIndex_AddVector(base, rows[0].data(), 10); + VecSimIndex_AddVector(base, rows[1].data(), 20); + EXPECT_EQ(VecSimIndex_AddVector(base, rows[2].data(), 10), 0); + EXPECT_EQ(VecSimIndex_DeleteVector(base, 20), 1); + EXPECT_EQ(base->relabelVector(10, 11), VecSimRelabel_OK); + EXPECT_EQ(base->statisticInfo().flatBufferSize, 1); + EXPECT_EQ(backend->indexSize(), 0); + EXPECT_TRUE(pool.jobQ.empty()); + VecSimIndex_AddVector(base, rows[1].data(), 20); + VecSimIndex_AddVector(base, rows[3].data(), 30); + tiered->setBeforeQuantizationFinalizationHook([] { throw std::bad_alloc(); }); + EXPECT_THROW(VecSimIndex_AddVector(base, rows[4].data(), 40), std::bad_alloc); + EXPECT_EQ(base->statisticInfo().flatBufferSize, 4); + EXPECT_EQ(backend->indexSize(), 0); + for (float value : codecOf(backend).biasMean()) + EXPECT_EQ(value, 0); + // A failed finalization attempt is retried by the next writer. + tiered->setBeforeQuantizationFinalizationHook({}); + EXPECT_EQ(VecSimIndex_AddVector(base, rows[4].data(), 40), 0); + if (mode == VecSim_WriteAsync) { + EXPECT_EQ(pool.jobQ.size(), 4); + while (!pool.jobQ.empty()) + pool.thread_iteration(); + } else { + EXPECT_TRUE(pool.jobQ.empty()); + } + ASSERT_EQ(backend->indexSize(), 4); + EXPECT_EQ(base->statisticInfo().flatBufferSize, 0); + std::vector expected_mean(dim); + for (size_t i : {size_t(1), size_t(2), size_t(3), size_t(4)}) { + auto row = rows[i]; + if (metric == VecSimMetric_Cosine) + VecSim_Normalize(row.data(), dim, VecSimType_FLOAT32); + for (size_t j = 0; j < dim; ++j) + expected_mean[j] += double(row[j]) / 4; + } + for (size_t j = 0; j < dim; ++j) + EXPECT_NEAR(codecOf(backend).biasMean()[j], expected_mean[j], 3e-8); + EXPECT_TRUE(backend->checkIntegrity().valid_state); + const auto saved = codecOf(backend).biasMean(); + const std::vector saved_mean(saved.begin(), saved.end()); + // Removing every row and adding again never restarts a fitted backend. + for (size_t label : {11, 20, 30, 40}) + VecSimIndex_DeleteVector(base, label); + while (!pool.jobQ.empty()) + pool.thread_iteration(); + VecSimIndex_AddVector(base, rows[0].data(), 50); + while (!pool.jobQ.empty()) + pool.thread_iteration(); + EXPECT_TRUE(std::equal(saved_mean.begin(), saved_mean.end(), + codecOf(backend).biasMean().begin())); + } + } +} + +TEST_P(TurboQuantTest, MultiValueTrainingAndSubsequentUpdatesKeepOneFittedMean) { + WriteModeGuard guard; + VecSim_SetWriteMode(VecSim_WriteInPlace); + TestPool pool; + constexpr size_t dim = 7; + auto hp = parameters(GetParam(), dim); + hp.multi = true; + auto primary = CreateParams(hp); + auto *base = newTiered(pool, primary, 3); + ASSERT_NE(base, nullptr); + auto *backend = backendOf(base); + std::array a{}, b{}, c{}; + for (size_t i = 0; i < dim; ++i) { + a[i] = 0.1f * i; + b[i] = -0.1f * i; + c[i] = 0.03f * i; + } + VecSimIndex_AddVector(base, a.data(), 10); + VecSimIndex_AddVector(base, b.data(), 10); + EXPECT_EQ(VecSimIndex_DeleteVector(base, 10), 2); + EXPECT_EQ(base->statisticInfo().flatBufferSize, 0); + VecSimIndex_AddVector(base, a.data(), 11); + std::array replacement{}; + std::copy(b.begin(), b.end(), replacement.begin()); + std::copy(c.begin(), c.end(), replacement.begin() + dim); + EXPECT_EQ(VecSimIndex_UpdateVectors(base, 11, replacement.data(), 2), VecSimUpdate_OK); + EXPECT_EQ(base->statisticInfo().flatBufferSize, 2); + EXPECT_EQ(backend->indexSize(), 0); + VecSimIndex_AddVector(base, a.data(), 12); + EXPECT_EQ(base->statisticInfo().flatBufferSize, 0); + EXPECT_EQ(backend->indexSize(), 3); + for (size_t i = 0; i < dim; ++i) + EXPECT_NEAR(codecOf(backend).biasMean()[i], (double(a[i]) + b[i] + c[i]) / 3, 2e-8); + EXPECT_EQ(VecSimIndex_UpdateVectors(base, 11, a.data(), 1), VecSimUpdate_OK); + EXPECT_EQ(base->indexSize(), 2); + EXPECT_TRUE(backend->checkIntegrity().valid_state); +} + +TEST_P(TurboQuantTest, RejectsTruncatedOrCorruptFittedState) { + const unsigned bits = GetParam(); + auto params = CreateParams(parameters(bits, 9)); + IndexOwner index(VecSimIndex_New(¶ms), VecSimIndex_Free); + auto *backend = dynamic_cast *>(index.get()); + ASSERT_NE(backend, nullptr); + std::array row{}; + row[0] = 0.5f; + VecSimIndex_AddVector(index.get(), row.data(), 1); + const auto filename = path(bits, "corrupt"); + backend->saveIndex(filename); + std::string bytes; + { + std::ifstream source(filename, std::ios::binary); + bytes.assign(std::istreambuf_iterator(source), std::istreambuf_iterator()); + } + // V5 writes the fitted state right after quantType. + const size_t fitted = sizeof(HNSWSerializer::EncodingVersion) + sizeof(VecSimAlgo) + + sizeof(size_t) + sizeof(VecSimType) + sizeof(VecSimMetric) + + sizeof(size_t) + sizeof(bool) + sizeof(size_t) + sizeof(VecSimQuantType); + auto reject = [&](const std::string &corrupt) { + { + std::ofstream out(filename, std::ios::binary | std::ios::trunc); + out.write(corrupt.data(), corrupt.size()); + } + EXPECT_THROW({ IndexOwner invalid(HNSWFactory::NewIndex(filename), VecSimIndex_Free); }, + std::runtime_error); + }; + for (size_t length : {size_t(0), fitted, fitted + 16, fitted + 200}) + reject(bytes.substr(0, length)); + auto unknown = bytes; + const uint32_t bad_policy = 999; + std::memcpy(unknown.data() + fitted + 3 * sizeof(uint32_t), &bad_policy, sizeof(bad_policy)); + reject(unknown); + auto bad_length = bytes; + const uint64_t huge = UINT64_MAX; + std::memcpy(bad_length.data() + fitted + 5 * sizeof(uint32_t) + sizeof(uint64_t), &huge, + sizeof(huge)); + reject(bad_length); + auto nonfinite = bytes; + const float nan = std::numeric_limits::quiet_NaN(); + std::memcpy(nonfinite.data() + fitted + 5 * sizeof(uint32_t) + 2 * sizeof(uint64_t), &nan, + sizeof(nan)); + reject(nonfinite); + std::remove(filename.c_str()); +} + +TEST_P(TurboQuantTest, ExistingWriterVersionsRemainUnchanged) { + for (auto type : {VecSimQuant_NONE, VecSimQuant_SQ8, VecSimQuant_SQ4}) { + auto hp = parameters(GetParam(), 9); + hp.quantType = type; + auto params = CreateParams(hp); + IndexOwner index(VecSimIndex_New(¶ms), VecSimIndex_Free); + ASSERT_NE(index, nullptr); + auto *backend = dynamic_cast *>(index.get()); + ASSERT_NE(backend, nullptr); + EXPECT_THROW(codecOf(backend), std::runtime_error); + const auto filename = path(unsigned(type), "legacy"); + backend->saveIndex(filename); + { + std::ifstream in(filename, std::ios::binary); + EXPECT_EQ(HNSWSerializer::ReadVersion(in), type == VecSimQuant_NONE + ? HNSWSerializer::EncodingVersion::V4 + : HNSWSerializer::EncodingVersion::V5); + } + IndexOwner restored(HNSWFactory::NewIndex(filename), VecSimIndex_Free); + EXPECT_NE(restored, nullptr); + std::remove(filename.c_str()); + } +} + +INSTANTIATE_TEST_SUITE_P(TQ2AndTQ3, TurboQuantTest, ::testing::Values(2U, 3U));