From cb143f6e40b9b7e5e1dabb0fa1aea85ee2396225 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Thu, 12 Nov 2026 10:26:39 +0200 Subject: [PATCH 1/3] MOD-19169: Add the A8 SQ8 inner product kernel for 8-bit quantized queries An SQ8 IP or Cosine query can be quantized once to signed 8-bit codes on its own grid, so each distance becomes one uint8 x int8 integer dot product plus a short double epilogue that folds the query's rounding error into per-query constants. - types/sq8_a8.h: the query blob layout (int8 codes, zero padding to 8 bytes, QD, QM, QS, QK and Y_MEAN_IP as double, P as int64). - spaces/sq8_a8_query_encoder.h: the scalar encoder for FP32 and FP16 queries. - quantized_distance_helpers.h: the shared double epilogue, with explicit fma so every tier rounds the same way. - quantized_kernels.h, IP.h/.cpp: the scalar SQ8_A8_InnerProduct reference. - IP_AVX512F_BW_VL_VNNI_SQ8_A8.h: the vpdpbusd kernel with 4 accumulators and a masked tail, dispatched on dim % 64. - IP_SQ8_A8_GetDistFunc returns nullptr unless AVX-512 F/BW/VL/VNNI is available and SQ8_A8_MIN_DIM (768) <= dim <= SQ8_A8_MAX_EXACT_SIMD_DIM (65,793, where the int32 code dot stops being exact). Cosine indexes resolve to IP, so there is no Cosine chooser. - test_spaces.cpp: encoder and scalar kernel against definitions in long double, the AVX-512 kernel against the scalar one (exact integer dot, bit-identical floats) for every residual and the edge dims, and the chooser gates via arch_opt. Nothing calls the chooser yet. Co-Authored-By: Claude Opus 5.5 --- src/VecSim/spaces/IP/IP.cpp | 4 + src/VecSim/spaces/IP/IP.h | 6 + .../spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_A8.h | 96 ++++ src/VecSim/spaces/IP_space.cpp | 20 + src/VecSim/spaces/IP_space.h | 5 + .../spaces/functions/AVX512F_BW_VL_VNNI.cpp | 9 + .../spaces/functions/AVX512F_BW_VL_VNNI.h | 3 + .../spaces/quantized_distance_helpers.h | 32 ++ src/VecSim/spaces/quantized_kernels.h | 16 + src/VecSim/spaces/spaces.h | 15 + src/VecSim/spaces/sq8_a8_query_encoder.h | 123 +++++ src/VecSim/types/sq8_a8.h | 50 ++ tests/unit/test_spaces.cpp | 480 ++++++++++++++++++ 13 files changed, 859 insertions(+) create mode 100644 src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_A8.h create mode 100644 src/VecSim/spaces/sq8_a8_query_encoder.h create mode 100644 src/VecSim/types/sq8_a8.h diff --git a/src/VecSim/spaces/IP/IP.cpp b/src/VecSim/spaces/IP/IP.cpp index c95f878bb..54a4ee03e 100644 --- a/src/VecSim/spaces/IP/IP.cpp +++ b/src/VecSim/spaces/IP/IP.cpp @@ -54,6 +54,10 @@ float SQ8_FP32_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension return SQ8_FP32_InnerProduct(pVect1v, pVect2v, dimension); } +float SQ8_A8_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) { + return spaces::quantized::SQ8_A8_IPDistance(pVect1v, pVect2v, dimension); +} + /* * Optimized asymmetric SQ8-FP16 inner product using algebraic identity: * IP(x, y) = Σ(x_i * y_i) diff --git a/src/VecSim/spaces/IP/IP.h b/src/VecSim/spaces/IP/IP.h index 0f5deb683..25e358a37 100644 --- a/src/VecSim/spaces/IP/IP.h +++ b/src/VecSim/spaces/IP/IP.h @@ -53,6 +53,12 @@ float SQ8_SQ8_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dime // Vector layout: [uint8_t values (dim)] [min_val (float)] [delta (float)] [sum (float)] float SQ8_SQ8_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension); +// SQ8 storage vs an A8 query, the scalar reference of the A8 kernels. It is not selected by the +// dispatcher; IP_SQ8_A8_GetDistFunc returns only SIMD kernels. +// pVect1 is storage (SQ8): [uint8_t values (dim)] [min_val] [delta] [sum] +// pVect2 is the A8 query (vecsim_types::sq8_a8_query): [int8 codes (dim)] [pad] [constants] +float SQ8_A8_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension); + float FP32_InnerProduct(const void *pVect1, const void *pVect2, size_t dimension); double FP64_InnerProduct(const void *pVect1, const void *pVect2, size_t dimension); diff --git a/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_A8.h b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_A8.h new file mode 100644 index 000000000..448c21895 --- /dev/null +++ b/src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_A8.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/quantized_distance_helpers.h" +#include + +namespace { + +/* + * SQ8 storage vs an A8 query (vecsim_types::sq8_a8_query). The stored uint8 codes and the int8 + * query codes are exactly the unsigned-by-signed operands of vpdpbusd, which adds four products + * into each int32 lane without saturation. The rest of the distance is the shared double + * epilogue, so this kernel returns the scalar reference's float for the same code dot. + * + * Four independent accumulators take 256 bytes per iteration, up to three remaining 64-byte + * chunks go to their own accumulators, and one masked load of each operand reads the last + * `residual` bytes, so nothing past the codes is read. + * + * Exactness: |a_i * p_i| <= 255 * 128, and every lane and every partial sum of the reduction adds + * at most dim products, so all of them are exact in int32 while 32,640 * dim fits INT32_MAX + * (spaces::SQ8_A8_MAX_EXACT_SIMD_DIM). + */ + +// A distinct zero for each accumulator. When the accumulators start from one shared zero register, +// GCC 13 copies each tied vpdpbusd accumulator twice per loop iteration; the empty asm makes every +// initial zero its own value, which removes the copies. It changes no arithmetic. +static inline __m512i SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI() { + __m512i zero = _mm512_setzero_si512(); + __asm__ volatile("" : "+v"(zero)); + return zero; +} + +template // 0..63 +static inline int32_t SQ8_A8_CodeDot_AVX512F_BW_VL_VNNI(const uint8_t *codes, + const int8_t *query_codes, size_t dim) { + __m512i acc0 = SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI(); + __m512i acc1 = SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI(); + __m512i acc2 = SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI(); + __m512i acc3 = SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI(); + const size_t complete = dim - residual; + size_t i = 0; + for (; i + 256 <= complete; i += 256) { + acc0 = _mm512_dpbusd_epi32(acc0, _mm512_loadu_si512(codes + i), + _mm512_loadu_si512(query_codes + i)); + acc1 = _mm512_dpbusd_epi32(acc1, _mm512_loadu_si512(codes + i + 64), + _mm512_loadu_si512(query_codes + i + 64)); + acc2 = _mm512_dpbusd_epi32(acc2, _mm512_loadu_si512(codes + i + 128), + _mm512_loadu_si512(query_codes + i + 128)); + acc3 = _mm512_dpbusd_epi32(acc3, _mm512_loadu_si512(codes + i + 192), + _mm512_loadu_si512(query_codes + i + 192)); + } + if (i < complete) { + acc0 = _mm512_dpbusd_epi32(acc0, _mm512_loadu_si512(codes + i), + _mm512_loadu_si512(query_codes + i)); + i += 64; + if (i < complete) { + acc1 = _mm512_dpbusd_epi32(acc1, _mm512_loadu_si512(codes + i), + _mm512_loadu_si512(query_codes + i)); + i += 64; + if (i < complete) { + acc2 = _mm512_dpbusd_epi32(acc2, _mm512_loadu_si512(codes + i), + _mm512_loadu_si512(query_codes + i)); + i += 64; + } + } + } + __m512i acc = _mm512_add_epi32(_mm512_add_epi32(acc0, acc1), _mm512_add_epi32(acc2, acc3)); + // The tail goes into the summed accumulator: accumulating it into one of the loop's + // accumulators makes GCC 13 copy that accumulator twice per loop iteration. + if constexpr (residual > 0) { + constexpr __mmask64 mask = (uint64_t(1) << residual) - 1; + acc = _mm512_dpbusd_epi32(acc, _mm512_maskz_loadu_epi8(mask, codes + i), + _mm512_maskz_loadu_epi8(mask, query_codes + i)); + } + return _mm512_reduce_add_epi32(acc); +} + +// Returns the distance 1 - IP; Cosine on normalized vectors uses the same kernel. +template // 0..63 +float SQ8_A8_InnerProductSIMD64_AVX512F_BW_VL_VNNI(const void *pVect1v, const void *pVect2v, + size_t dimension) { + const auto *stored = static_cast(pVect1v); + const auto *query = static_cast(pVect2v); + const int32_t code_dot = SQ8_A8_CodeDot_AVX512F_BW_VL_VNNI( + stored, reinterpret_cast(query), dimension); + return spaces::quantized::SQ8_A8_InnerProduct_FromCodeDot(stored, query, dimension, code_dot); +} + +} // namespace diff --git a/src/VecSim/spaces/IP_space.cpp b/src/VecSim/spaces/IP_space.cpp index ba5a5f0a8..cd86e5919 100644 --- a/src/VecSim/spaces/IP_space.cpp +++ b/src/VecSim/spaces/IP_space.cpp @@ -536,6 +536,26 @@ dist_func_t Cosine_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignme return ret_dist_func; } +// SQ8 storage vs an A8 query. There is no scalar default: the scalar SQ8_A8_InnerProduct is only +// the tests' reference, so outside the gates the caller gets nullptr and keeps its existing query +// kernel. +dist_func_t IP_SQ8_A8_GetDistFunc(size_t dim, unsigned char *alignment, + const void *arch_opt) { + if (dim < spaces::SQ8_A8_MIN_DIM || dim > spaces::SQ8_A8_MAX_EXACT_SIMD_DIM) { + return nullptr; + } + [[maybe_unused]] auto features = getCpuOptimizationFeatures(arch_opt); + +#ifdef CPU_FEATURES_ARCH_X86_64 +#ifdef OPT_AVX512_F_BW_VL_VNNI + if (features.avx512f && features.avx512bw && features.avx512vl && features.avx512vnni) { + return Choose_SQ8_A8_IP_implementation_AVX512F_BW_VL_VNNI(dim); + } +#endif +#endif // __x86_64__ + return nullptr; +} + dist_func_t IP_FP32_GetDistFunc(size_t dim, unsigned char *alignment, const void *arch_opt) { unsigned char dummy_alignment; if (alignment == nullptr) { diff --git a/src/VecSim/spaces/IP_space.h b/src/VecSim/spaces/IP_space.h index 35530cea5..3c6f0d8a2 100644 --- a/src/VecSim/spaces/IP_space.h +++ b/src/VecSim/spaces/IP_space.h @@ -50,4 +50,9 @@ dist_func_t IP_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignment = const void *arch_opt = nullptr); dist_func_t Cosine_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, const void *arch_opt = nullptr); +// SQ8 storage vs an A8 query (vecsim_types::sq8_a8_query). Returns nullptr when the A8 path is off +// for this dim and CPU; the caller then keeps its existing query path. The kernels need no +// alignment, so the hint is left unchanged. +dist_func_t IP_SQ8_A8_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, + const void *arch_opt = nullptr); } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.cpp b/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.cpp index 75c5e423e..a0791d51a 100644 --- a/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.cpp +++ b/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.cpp @@ -20,6 +20,8 @@ #include "VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_SQ8.h" #include "VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ8_SQ8.h" +#include "VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_A8.h" + #include "VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ4_FP32.h" #include "VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_SQ4_FP32.h" @@ -121,6 +123,13 @@ dist_func_t Choose_SQ8_SQ8_L2_implementation_AVX512F_BW_VL_VNNI(size_t di return ret_dist_func; } +// SQ8 storage vs an A8 query (int8 query codes plus per-query constants) +dist_func_t Choose_SQ8_A8_IP_implementation_AVX512F_BW_VL_VNNI(size_t dim) { + dist_func_t ret_dist_func; + CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_A8_InnerProductSIMD64_AVX512F_BW_VL_VNNI); + return ret_dist_func; +} + #include "implementation_chooser_cleanup.h" } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.h b/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.h index f0022cee3..68689d22d 100644 --- a/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.h +++ b/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.h @@ -38,4 +38,7 @@ dist_func_t Choose_SQ8_SQ8_IP_implementation_AVX512F_BW_VL_VNNI(size_t di dist_func_t Choose_SQ8_SQ8_Cosine_implementation_AVX512F_BW_VL_VNNI(size_t dim); dist_func_t Choose_SQ8_SQ8_L2_implementation_AVX512F_BW_VL_VNNI(size_t dim); +// SQ8 storage vs an A8 query (int8 query codes plus per-query constants); also serves Cosine +dist_func_t Choose_SQ8_A8_IP_implementation_AVX512F_BW_VL_VNNI(size_t dim); + } // namespace spaces diff --git a/src/VecSim/spaces/quantized_distance_helpers.h b/src/VecSim/spaces/quantized_distance_helpers.h index dddaf0ceb..66be4efa6 100644 --- a/src/VecSim/spaces/quantized_distance_helpers.h +++ b/src/VecSim/spaces/quantized_distance_helpers.h @@ -8,9 +8,12 @@ */ #pragma once +#include #include #include #include "VecSim/utils/alignment.h" +#include "VecSim/types/sq8.h" +#include "VecSim/types/sq8_a8.h" namespace spaces { namespace quantized { @@ -37,5 +40,34 @@ static inline float Stored_InnerProduct_FromCodeDot(const uint8_t *a, const uint delta1 * delta2 * code_dot; } +// Complete the SQ8 storage vs A8 query inner product distance from the integer code dot +// sum(a_i * p_i) (see vecsim_types::sq8_a8_query), in double: +// distance = 1 - [(delta * QD) * dot + min * QM + SUM * QS + QK] +// Every tier calls this helper, so all tiers return the same float for the same dot. The fused +// steps are written as std::fma: the SIMD translation units are built with FMA-capable ISA flags +// and the compiler's default contraction, which would fuse a written a * b + c there but not in +// the scalar translation unit, and the tiers would then round differently. +static inline float SQ8_A8_InnerProduct_FromCodeDot(const uint8_t *stored, const uint8_t *query, + size_t dim, int64_t code_dot) { + using sq8 = vecsim_types::sq8; + using a8 = vecsim_types::sq8_a8_query; + // Stored metadata follows the codes and is not necessarily float-aligned. + const auto *params = stored + sq8::data_bytes_count(dim); + const double min_val = load_unaligned(params + sq8::MIN_VAL * sizeof(float)); + const double delta = load_unaligned(params + sq8::DELTA * sizeof(float)); + const double sum = load_unaligned(params + sq8::SUM * sizeof(float)); + const auto *consts = query + a8::consts_offset(dim); + const double qd = load_unaligned(consts + a8::QD * sizeof(double)); + const double qm = load_unaligned(consts + a8::QM * sizeof(double)); + const double qs = load_unaligned(consts + a8::QS * sizeof(double)); + const double qk = load_unaligned(consts + a8::QK * sizeof(double)); + + double ip = delta * qd * static_cast(code_dot); + ip = std::fma(min_val, qm, ip); + ip = std::fma(sum, qs, ip); + ip += qk; + return static_cast(1.0 - ip); +} + } // namespace quantized } // namespace spaces diff --git a/src/VecSim/spaces/quantized_kernels.h b/src/VecSim/spaces/quantized_kernels.h index 245a8fa47..bb3428761 100644 --- a/src/VecSim/spaces/quantized_kernels.h +++ b/src/VecSim/spaces/quantized_kernels.h @@ -130,5 +130,21 @@ static inline float Stored_L2Sqr(const void *a, const void *b, size_t dim) { return sum_sq_1 + sum_sq_2 - 2.0f * ip; } +/* + * SQ8 storage vs an A8 query (int8 codes plus per-query constants, see + * vecsim_types::sq8_a8_query): the integer code dot sum(a_i * p_i), accumulated in int64 so it is + * exact at any dimension, completed by the epilogue every tier shares. Returns the distance. + */ +static inline float SQ8_A8_IPDistance(const void *stored, const void *query, size_t dim) { + const auto *codes = static_cast(stored); + const auto *query_codes = static_cast(query); + int64_t code_dot = 0; + for (size_t i = 0; i < dim; i++) { + code_dot += static_cast(codes[i]) * static_cast(query_codes[i]); + } + return SQ8_A8_InnerProduct_FromCodeDot(codes, static_cast(query), dim, + code_dot); +} + } // namespace quantized } // namespace spaces diff --git a/src/VecSim/spaces/spaces.h b/src/VecSim/spaces/spaces.h index 2ab04116c..c00529123 100644 --- a/src/VecSim/spaces/spaces.h +++ b/src/VecSim/spaces/spaces.h @@ -65,6 +65,21 @@ static constexpr size_t UINT8_MAX_EXACT_SIMD_DIM = std::numeric_limits::max() / (std::numeric_limits::max() * std::numeric_limits::max()); +// The SQ8 kernels for an A8 query (int8 codes, vecsim_types::sq8_a8_query) multiply uint8 stored +// codes by int8 query codes, so each product is at most 255 * 128 = 32,640 in magnitude, and they +// hold the code dot in a signed 32-bit int. That makes 65,793 the largest dimension they are exact +// at: 32,640 * 65,793 is 2,147,483,520 and fits INT32_MAX, and one dimension more does not. The +// bound is wider than UINT8_MAX_EXACT_SIMD_DIM, so between the two an index scores queries with +// SIMD A8 kernels while its stored-to-stored distances use the scalar kernel. +static constexpr size_t SQ8_A8_MAX_EXACT_SIMD_DIM = + std::numeric_limits::max() / + (std::numeric_limits::max() * -static_cast(std::numeric_limits::min())); + +// The A8 query path is used from this dimension up. Below it, encoding the query and the +// per-distance epilogue take a larger share of each distance, and the measured gains were mixed, +// so smaller indexes keep their existing query kernels. +static constexpr size_t SQ8_A8_MIN_DIM = 768; + // The SQ4-SQ4 inner product kernels accumulate the integer code dot sum(q1 * q2), each product at // most 15 * 15 = 225, in unsigned 32-bit lanes (vpadalq_u16 / vdotq_u32 / svdot_u32), and NEON // reduces those lanes with vaddvq_u32, whose result is also a uint32. The narrowest accumulator is diff --git a/src/VecSim/spaces/sq8_a8_query_encoder.h b/src/VecSim/spaces/sq8_a8_query_encoder.h new file mode 100644 index 000000000..34f31f81c --- /dev/null +++ b/src/VecSim/spaces/sq8_a8_query_encoder.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include "VecSim/types/float16.h" +#include "VecSim/types/sq8_a8.h" + +// Scalar encoder of the A8 query blob (vecsim_types::sq8_a8_query). It is ISA-neutral, with +// internal linkage, so a vectorized encoder can reuse the per-element step for its tail and must +// produce the same codes bit for bit. The codes depend only on double operations that do not +// change under FP contraction (128 * s is exact), so they match across translation units; the +// constants are plain sums and may differ in the last bits between encoders. +namespace spaces { +namespace quantized { + +// The query's step s and offset c from its range. A constant query gets s = 1, which still +// reconstructs it exactly (every code is -128 and every rounding error is 0). +struct SQ8_A8_QueryGrid { + double step; + double offset; +}; + +static inline SQ8_A8_QueryGrid SQ8_A8_QueryGridFromRange(float min_val, float max_val) { + // In double, so the range of a query spanning most of the float range stays finite. + const double step = max_val > min_val ? (static_cast(max_val) - min_val) / 255.0 : 1.0; + return {step, static_cast(min_val) + 128.0 * step}; +} + +// Running sums of the encoding: P = sum p_i, E = sum e_i and PE = sum p_i * e_i, where +// e_i = v_i - s * p_i is the rounding error in the frame centered on c (v_i = y_i - c is exact). +struct SQ8_A8_QuerySums { + int64_t P = 0; + double E = 0; + double PE = 0; +}; + +// Encodes one element: p = clamp(floor((y - c) / s + 0.5), -128, 127). The clamp is written as +// max then min with the bound second, which maps NaN to -128 like the SIMD max / min do. +static inline int8_t SQ8_A8_EncodeElement(float y, const SQ8_A8_QueryGrid &grid, + SQ8_A8_QuerySums &sums) { + const double v = static_cast(y) - grid.offset; + double code = std::floor(v / grid.step + 0.5); + code = code > -128.0 ? code : -128.0; + code = code < 127.0 ? code : 127.0; + const double err = v - grid.step * code; + sums.P += static_cast(code); + sums.E += err; + sums.PE += code * err; + return static_cast(code); +} + +// Zeroes the padding after the codes and writes the constants. K = s * sum (p_i - P / dim) * e_i +// is formed in one pass as s * (PE - (P / dim) * E). Y_MEAN_IP is written as 0; a mean-centered +// index overwrites it. +static inline void SQ8_A8_FinishQuery(const SQ8_A8_QueryGrid &grid, const SQ8_A8_QuerySums &sums, + size_t dim, void *blob) { + using a8 = vecsim_types::sq8_a8_query; + auto *out = static_cast(blob); + const size_t consts_offset = a8::consts_offset(dim); + std::memset(out + dim, 0, consts_offset - dim); + + const double dim_d = static_cast(dim); + const double p_sum = static_cast(sums.P); + const double consts[a8::P] = { + grid.step, // QD + grid.step * p_sum, // QM + grid.offset + sums.E / dim_d, // QS + grid.step * (sums.PE - (p_sum / dim_d) * sums.E), // QK + 0.0, // Y_MEAN_IP + }; + auto *consts_out = out + consts_offset; + std::memcpy(consts_out, consts, sizeof(consts)); + std::memcpy(consts_out + a8::P * sizeof(double), &sums.P, sizeof(int64_t)); +} + +// Writes the full A8 query blob (vecsim_types::sq8_a8_query::blob_bytes_count(dim) bytes) for a +// float or float16 query. FP16 values are widened to FP32 first, so an FP16 query encodes exactly +// as its FP32 widening. +template +static inline void SQ8_A8_EncodeQuery(const InputType *query, size_t dim, void *blob) { + static_assert(std::is_same_v || + std::is_same_v, + "The A8 query encoder takes float or float16 input"); + assert(dim > 0); + const auto value = [query](size_t i) -> float { + if constexpr (std::is_same_v) { + return vecsim_types::FP16_to_FP32(query[i]); + } else { + return query[i]; + } + }; + + float min_val = value(0); + float max_val = min_val; + for (size_t i = 1; i < dim; i++) { + const float y = value(i); + min_val = y < min_val ? y : min_val; + max_val = y > max_val ? y : max_val; + } + const SQ8_A8_QueryGrid grid = SQ8_A8_QueryGridFromRange(min_val, max_val); + + auto *codes = static_cast(blob); + SQ8_A8_QuerySums sums; + for (size_t i = 0; i < dim; i++) { + codes[i] = SQ8_A8_EncodeElement(value(i), grid, sums); + } + SQ8_A8_FinishQuery(grid, sums, dim, blob); +} + +} // namespace quantized +} // namespace spaces diff --git a/src/VecSim/types/sq8_a8.h b/src/VecSim/types/sq8_a8.h new file mode 100644 index 000000000..58bfd30a1 --- /dev/null +++ b/src/VecSim/types/sq8_a8.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include + +namespace vecsim_types { + +// Query blob of an SQ8 IP / Cosine index whose query is quantized once to signed 8-bit codes +// ("A8"), so that each distance is one integer dot product against the stored uint8 codes. +// +// The query y is encoded on its own grid yh_i = c + s * p_i, with s = (max y - min y) / 255, +// c = min y + 128 * s and codes p_i in [-128, 127]. With e_i = y_i - yh_i, E = sum e_i, +// P = sum p_i and K = s * sum (p_i - P / dim) * e_i, the inner product with a stored vector +// (min m, delta d, SUM) is estimated as +// IP = (d * QD) * dot + m * QM + SUM * QS + QK, dot = sum a_i * p_i +// where E and K fold the query's own rounding error into per-query constants. +// +// Layout: [int8 codes (dim)] [zero padding up to a multiple of 8 bytes] [8-byte constant slots] +// The slots are QD, QM, QS, QK and Y_MEAN_IP as double, then P as int64. +struct sq8_a8_query { + using value_type = int8_t; + + enum ConstIndex : size_t { + QD = 0, // s + QM = 1, // s * P + QS = 2, // c + E / dim + QK = 3, // K + Y_MEAN_IP = 4, // for a mean-centered index, else 0; not read by the kernels + P = 5, // sum of the codes (int64), for kernels that offset the stored codes + CONSTS_COUNT = 6 + }; + + static constexpr size_t consts_offset(size_t dim) { return (dim + 7) & ~size_t(7); } + + static constexpr size_t blob_bytes_count(size_t dim) { + return consts_offset(dim) + CONSTS_COUNT * sizeof(double); + } +}; + +static_assert(sizeof(double) == sizeof(int64_t), "A8 query constants use 8-byte slots"); + +} // namespace vecsim_types diff --git a/tests/unit/test_spaces.cpp b/tests/unit/test_spaces.cpp index 69d6c84d9..154aa5951 100644 --- a/tests/unit/test_spaces.cpp +++ b/tests/unit/test_spaces.cpp @@ -28,6 +28,8 @@ #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" #include "VecSim/spaces/quantized_kernels.h" +#include "VecSim/spaces/sq8_a8_query_encoder.h" +#include "VecSim/types/sq8_a8.h" #include "VecSim/spaces/functions/AVX512F.h" #include "VecSim/spaces/functions/AVX.h" #include "VecSim/spaces/functions/SSE.h" @@ -5574,3 +5576,481 @@ TEST_F(SpacesTest, SQ8_SQ8_L2_self_distance_is_near_zero) { << "dispatched kernel, dim " << dim; } } + +/* ======================== Tests SQ8 with an A8 query ========================= */ + +namespace { + +using sq8_a8 = vecsim_types::sq8_a8_query; + +double A8Const(const std::vector &query, size_t dim, size_t index) { + return load_unaligned(query.data() + sq8_a8::consts_offset(dim) + + index * sizeof(double)); +} + +int64_t A8CodeSum(const std::vector &query, size_t dim) { + return load_unaligned(query.data() + sq8_a8::consts_offset(dim) + + sq8_a8::P * sizeof(double)); +} + +// The production scalar encoder, into a blob whose padding starts non-zero. +template +std::vector A8Encode(const std::vector &y) { + std::vector blob(sq8_a8::blob_bytes_count(y.size()), 0xAB); + spaces::quantized::SQ8_A8_EncodeQuery(y.data(), y.size(), blob.data()); + return blob; +} + +std::vector A8RandomVector(size_t dim, int seed, float min_val = -1.0f, float max_val = 1.0f, + bool normalize = false) { + std::vector v(dim); + test_utils::populate_float_vec(v.data(), dim, seed, min_val, max_val); + if (normalize) { + spaces::GetNormalizeFunc()(v.data(), dim); + } + return v; +} + +// SQ8 storage from floats through the tests' quantizer. The blob has room for the L2 metadata; +// the IP kernels read min, delta and SUM only. +std::vector A8Stored(const std::vector &x) { + std::vector blob(x.size() + + sq8::storage_metadata_count() * sizeof(float)); + test_utils::quantize_float_vec_to_sq8_with_metadata(x.data(), x.size(), blob.data()); + return blob; +} + +// SQ8 storage with the given codes; SUM is derived from the integer code sum in double, as +// QuantPreprocessor does. +std::vector A8StoredFromCodes(const std::vector &codes, float min_val, + float delta) { + const size_t dim = codes.size(); + std::vector blob(dim + sq8::storage_metadata_count() * sizeof(float)); + std::memcpy(blob.data(), codes.data(), dim); + uint64_t code_sum = 0; + for (uint8_t a : codes) { + code_sum += a; + } + const float sum = static_cast(static_cast(dim) * min_val + + static_cast(delta) * code_sum); + auto *params = blob.data() + dim; + std::memcpy(params + sq8::MIN_VAL * sizeof(float), &min_val, sizeof(float)); + std::memcpy(params + sq8::DELTA * sizeof(float), &delta, sizeof(float)); + std::memcpy(params + sq8::SUM * sizeof(float), &sum, sizeof(float)); + return blob; +} + +// The A8 grid and codes by definition: s = (max y - min y) / 255 (1 for a constant query), +// c = min y + 128 * s and p_i = clamp(floor((y_i - c) / s + 0.5), -128, 127), all in double. +struct A8ReferenceCodes { + double s, c; + std::vector p; +}; + +A8ReferenceCodes A8ReferenceEncode(const std::vector &y) { + double lo = y[0], hi = y[0]; + for (float v : y) { + lo = v < lo ? v : lo; + hi = v > hi ? v : hi; + } + A8ReferenceCodes ref; + ref.s = hi > lo ? (hi - lo) / 255.0 : 1.0; + ref.c = lo + 128.0 * ref.s; + for (float v : y) { + const double t = std::floor((static_cast(v) - ref.c) / ref.s + 0.5); + ref.p.push_back(static_cast(t < -128.0 ? -128.0 : t > 127.0 ? 127.0 : t)); + } + return ref; +} + +// The A8 inner product estimate by its definition, in long double with two-pass sums: +// IP = d * s * dot + m * s * P + SUM * (c + E / dim) + K, K = s * sum (p_i - P / dim) * e_i +// with e_i = (y_i - c) - s * p_i. It shares no code with the encoder's one-pass constants or the +// kernels' double epilogue. true_ip is the inner product of the stored reconstruction with the +// float query. The estimate leaves -sum (w_i - mean(w)) * e_i of it, w_i = xh_i - yh_i, plus the +// FP32 rounding of SUM times (c + E / dim); bound is the worst case of both. +struct A8Reference { + long double ip, scale, true_ip, bound, slack; +}; + +A8Reference A8ReferenceIP(const uint8_t *stored, const std::vector &y) { + const size_t dim = y.size(); + const A8ReferenceCodes q = A8ReferenceEncode(y); + const auto *params = stored + dim; + const long double m = load_unaligned(params + sq8::MIN_VAL * sizeof(float)); + const long double d = load_unaligned(params + sq8::DELTA * sizeof(float)); + const long double sum = load_unaligned(params + sq8::SUM * sizeof(float)); + const long double s = q.s, c = q.c, D = static_cast(dim); + + int64_t dot = 0, P = 0, A = 0; + long double E = 0; + std::vector e(dim); + for (size_t i = 0; i < dim; i++) { + dot += static_cast(stored[i]) * q.p[i]; + P += q.p[i]; + A += stored[i]; + e[i] = (static_cast(y[i]) - c) - s * q.p[i]; + E += e[i]; + } + const long double p_mean = static_cast(P) / D; + long double K = 0; + for (size_t i = 0; i < dim; i++) { + K += (q.p[i] - p_mean) * e[i]; + } + K *= s; + + A8Reference ref; + const long double qs = c + E / D; + ref.ip = d * s * dot + m * s * P + sum * qs + K; + ref.scale = + std::fabs(d * s * dot) + std::fabs(m * s * P) + std::fabs(sum * qs) + std::fabs(K) + 1.0L; + + long double true_ip = 0, true_ip_terms = 0, w_mean = 0, e_max = 0, w_dev = 0; + std::vector w(dim); + for (size_t i = 0; i < dim; i++) { + const long double xh = m + d * stored[i]; + true_ip += xh * y[i]; + true_ip_terms += std::fabs(xh * y[i]); + w[i] = xh - (c + s * q.p[i]); + w_mean += w[i]; + e_max = std::fabs(e[i]) > e_max ? std::fabs(e[i]) : e_max; + } + w_mean /= D; + for (size_t i = 0; i < dim; i++) { + w_dev += std::fabs(w[i] - w_mean); + } + ref.true_ip = true_ip; + ref.bound = e_max * w_dev + std::fabs(qs) * std::fabs(sum - (D * m + d * A)); + // Rounding of the long double sums themselves. + ref.slack = 1e-15L * (ref.scale + true_ip_terms); + return ref; +} + +// A kernel's float distance against the reference: one ulp of the float result for the final +// rounding, plus 1e-12 of the term scale for the double arithmetic (the study's matching rule). +double A8Tolerance(const A8Reference &ref) { + const float magnitude = std::fabs(static_cast(1.0L - ref.ip)); + const float ulp = std::nextafter(magnitude, std::numeric_limits::infinity()) - magnitude; + return ulp + 1e-12 * static_cast(ref.scale); +} + +void ExpectA8MatchesReference(const std::vector &stored, const std::vector &y, + const char *what) { + const size_t dim = y.size(); + const std::vector query = A8Encode(y); + const A8Reference ref = A8ReferenceIP(stored.data(), y); + EXPECT_NEAR(SQ8_A8_InnerProduct(stored.data(), query.data(), dim), + static_cast(1.0L - ref.ip), A8Tolerance(ref)) + << what << ", dim " << dim; + EXPECT_LE(std::fabs(ref.ip - ref.true_ip), ref.bound + ref.slack) + << "estimate outside its error bound: " << what << ", dim " << dim; +} + +// Checks one encoding against the definition: codes, zeroed padding and the exact constants +// bit for bit, the summed constants within 1e-12 of their scale. +void ExpectA8EncodingMatchesDefinition(const std::vector &y, const char *what) { + const size_t dim = y.size(); + const std::vector blob = A8Encode(y); + const A8ReferenceCodes ref = A8ReferenceEncode(y); + + int64_t P = 0; + for (size_t i = 0; i < dim; i++) { + ASSERT_EQ(static_cast(static_cast(blob[i])), ref.p[i]) + << what << ", dim " << dim << " at " << i; + P += ref.p[i]; + } + for (size_t i = dim; i < sq8_a8::consts_offset(dim); i++) { + ASSERT_EQ(blob[i], 0) << "padding byte " << i << ", " << what << ", dim " << dim; + } + EXPECT_EQ(A8CodeSum(blob, dim), P) << what << ", dim " << dim; + EXPECT_EQ(A8Const(blob, dim, sq8_a8::QD), ref.s) << what << ", dim " << dim; + EXPECT_EQ(A8Const(blob, dim, sq8_a8::QM), ref.s * static_cast(P)) + << what << ", dim " << dim; + EXPECT_EQ(A8Const(blob, dim, sq8_a8::Y_MEAN_IP), 0.0) << what << ", dim " << dim; + + const long double s = ref.s, c = ref.c, D = static_cast(dim); + long double E = 0; + std::vector e(dim); + for (size_t i = 0; i < dim; i++) { + e[i] = (static_cast(y[i]) - c) - s * ref.p[i]; + E += e[i]; + } + long double K = 0; + for (size_t i = 0; i < dim; i++) { + K += (ref.p[i] - static_cast(P) / D) * e[i]; + } + K *= s; + EXPECT_NEAR(A8Const(blob, dim, sq8_a8::QS), static_cast(c + E / D), + 1e-12 * static_cast(std::fabs(c) + s)) + << what << ", dim " << dim; + // |K| is at most 128 * s^2 * dim. + EXPECT_NEAR(A8Const(blob, dim, sq8_a8::QK), static_cast(K), + 1e-12 * static_cast(128.0L * s * s * D)) + << what << ", dim " << dim; +} + +} // namespace + +TEST_F(SpacesTest, SQ8_A8_QueryBlobLayout) { + EXPECT_EQ(sq8_a8::consts_offset(1), 8u); + EXPECT_EQ(sq8_a8::consts_offset(8), 8u); + EXPECT_EQ(sq8_a8::consts_offset(9), 16u); + EXPECT_EQ(sq8_a8::consts_offset(768), 768u); + EXPECT_EQ(sq8_a8::consts_offset(769), 776u); + EXPECT_EQ(sq8_a8::blob_bytes_count(769), 776u + 6 * sizeof(double)); +} + +TEST_F(SpacesTest, SQ8_A8_EncoderMatchesDefinition) { + for (const size_t dim : {1UL, 3UL, 8UL, 9UL, 63UL, 768UL, 769UL, 1536UL}) { + for (int seed : {1, 2}) { + ExpectA8EncodingMatchesDefinition(A8RandomVector(dim, seed, -1.0f, 1.0f, true), + "normalized"); + ExpectA8EncodingMatchesDefinition(A8RandomVector(dim, seed, 1000.0f, 1001.0f), + "offset"); + ExpectA8EncodingMatchesDefinition(A8RandomVector(dim, seed, -1e-3f, 1e-3f), "small"); + } + } +} + +TEST_F(SpacesTest, SQ8_A8_EncoderFixedCases) { + constexpr size_t dim = 769; + + // A constant query has no range: s = 1, every code is -128 and nothing is rounded, so the + // constants reconstruct it exactly. + for (const float value : {0.0f, 0.25f, -3.5f}) { + const std::vector blob = A8Encode(std::vector(dim, value)); + for (size_t i = 0; i < dim; i++) { + ASSERT_EQ(static_cast(static_cast(blob[i])), -128) + << "value " << value << " at " << i; + } + EXPECT_EQ(A8CodeSum(blob, dim), -128 * static_cast(dim)); + EXPECT_EQ(A8Const(blob, dim, sq8_a8::QD), 1.0); + EXPECT_EQ(A8Const(blob, dim, sq8_a8::QM), -128.0 * dim); + EXPECT_EQ(A8Const(blob, dim, sq8_a8::QS), value + 128.0); + EXPECT_EQ(A8Const(blob, dim, sq8_a8::QK), 0.0); + EXPECT_EQ(A8Const(blob, dim, sq8_a8::Y_MEAN_IP), 0.0); + } + + // A range that overflows FP32 is still finite in double, so the codes reach both ends of the + // grid and the constants stay finite. + std::vector wide(dim); + for (size_t i = 0; i < dim; i++) { + wide[i] = i % 2 ? std::numeric_limits::max() : -std::numeric_limits::max(); + } + const std::vector wide_blob = A8Encode(wide); + for (size_t i = 0; i < dim; i++) { + ASSERT_EQ(static_cast(static_cast(wide_blob[i])), i % 2 ? 127 : -128) + << "at " << i; + } + for (size_t index : {sq8_a8::QD, sq8_a8::QM, sq8_a8::QS, sq8_a8::QK}) { + EXPECT_TRUE(std::isfinite(A8Const(wide_blob, dim, index))) << "constant " << index; + } + + // A NaN coordinate is left out of the range and gets the defined code -128, so the other codes + // are those of the same query with that coordinate at its minimum. + std::vector with_nan = A8RandomVector(dim, 7); + with_nan[5] = std::numeric_limits::quiet_NaN(); + std::vector with_min = with_nan; + float min_val = with_nan[0]; + for (size_t i = 0; i < dim; i++) { + min_val = i != 5 && with_nan[i] < min_val ? with_nan[i] : min_val; + } + with_min[5] = min_val; + const std::vector nan_blob = A8Encode(with_nan); + const std::vector min_blob = A8Encode(with_min); + EXPECT_EQ(static_cast(static_cast(nan_blob[5])), -128); + EXPECT_EQ(std::memcmp(nan_blob.data(), min_blob.data(), dim), 0); + + // An FP16 query encodes exactly as its FP32 widening. + for (const size_t fp16_dim : {1UL, 768UL, 1001UL}) { + std::vector y16(fp16_dim); + test_utils::populate_float16_vec(y16.data(), fp16_dim, 11); + std::vector y32(fp16_dim); + for (size_t i = 0; i < fp16_dim; i++) { + y32[i] = vecsim_types::FP16_to_FP32(y16[i]); + } + EXPECT_EQ(A8Encode(y16), A8Encode(y32)) << "dim " << fp16_dim; + } +} + +TEST_F(SpacesTest, SQ8_A8_ScalarMatchesExactReference) { + for (const size_t dim : {1UL, 2UL, 7UL, 64UL, 100UL, 767UL, 768UL, 1000UL, 1536UL}) { + for (int seed : {1, 2}) { + ExpectA8MatchesReference(A8Stored(A8RandomVector(dim, seed, -1.0f, 1.0f, true)), + A8RandomVector(dim, seed + 50, -1.0f, 1.0f, true), + "normalized"); + } + ExpectA8MatchesReference(A8Stored(A8RandomVector(dim, 3, 1000.0f, 1001.0f)), + A8RandomVector(dim, 4, 1000.0f, 1001.0f), "offset"); + const std::vector stored = A8Stored(A8RandomVector(dim, 5)); + ExpectA8MatchesReference(stored, std::vector(dim, 0.25f), "constant query"); + ExpectA8MatchesReference(stored, std::vector(dim, 0.0f), "zero query"); + std::vector outlier = A8RandomVector(dim, 6); + outlier[dim / 2] = 100.0f; + ExpectA8MatchesReference(stored, outlier, "outlier query"); + ExpectA8MatchesReference(A8StoredFromCodes(std::vector(dim, 0), -0.5f, 1.0f / 255), + A8RandomVector(dim, 8), "all-0 codes"); + ExpectA8MatchesReference( + A8StoredFromCodes(std::vector(dim, 255), -0.5f, 1.0f / 255), + A8RandomVector(dim, 9), "all-255 codes"); + } +} + +// The chooser is the only switch for the A8 query path, so its gates are pinned here. +TEST_F(SpacesTest, SQ8_A8_ChooserGates) { + // The bound must sit exactly where a signed 32-bit total of 255 * 128 products runs out. + constexpr uint64_t max_term = 255ULL * 128ULL; + constexpr uint64_t int32_max = std::numeric_limits::max(); + ASSERT_EQ(SQ8_A8_MAX_EXACT_SIMD_DIM, 65793u); + ASSERT_LE(max_term * SQ8_A8_MAX_EXACT_SIMD_DIM, int32_max); + ASSERT_GT(max_term * (SQ8_A8_MAX_EXACT_SIMD_DIM + 1), int32_max); + ASSERT_EQ(SQ8_A8_MIN_DIM, 768u); + + const auto cpu = getCpuOptimizationFeatures(); + for (const size_t dim : + {size_t{1}, SQ8_A8_MIN_DIM - 1, SQ8_A8_MAX_EXACT_SIMD_DIM + 1, size_t{100000}}) { + EXPECT_EQ(IP_SQ8_A8_GetDistFunc(dim, nullptr, &cpu), nullptr) << "dim " << dim; + } +#if defined(CPU_FEATURES_ARCH_X86_64) && defined(OPT_AVX512_F_BW_VL_VNNI) + // Every feature reported, so the gates are checked whatever this CPU has. The kernels are only + // compared here, not called. + auto all_on = cpu; + std::memset(&all_on, 0xFF, sizeof(all_on)); + for (const size_t dim : {SQ8_A8_MIN_DIM, size_t{1024}, SQ8_A8_MAX_EXACT_SIMD_DIM}) { + unsigned char alignment = 0; + const auto expected = Choose_SQ8_A8_IP_implementation_AVX512F_BW_VL_VNNI(dim); + EXPECT_EQ(IP_SQ8_A8_GetDistFunc(dim, &alignment, &all_on), expected) << "dim " << dim; + EXPECT_EQ(alignment, 0) << "dim " << dim; + } + EXPECT_EQ(IP_SQ8_A8_GetDistFunc(SQ8_A8_MIN_DIM - 1, nullptr, &all_on), nullptr); + EXPECT_EQ(IP_SQ8_A8_GetDistFunc(SQ8_A8_MAX_EXACT_SIMD_DIM + 1, nullptr, &all_on), nullptr); + // Any one of the four features missing turns the path off. + for (int missing = 0; missing < 4; missing++) { + auto features = all_on; + switch (missing) { + case 0: + features.avx512f = 0; + break; + case 1: + features.avx512bw = 0; + break; + case 2: + features.avx512vl = 0; + break; + default: + features.avx512vnni = 0; + break; + } + EXPECT_EQ(IP_SQ8_A8_GetDistFunc(SQ8_A8_MIN_DIM, nullptr, &features), nullptr) + << "missing feature " << missing; + } +#else + // This build has no A8 kernels, so the path is off at every dimension. + EXPECT_EQ(IP_SQ8_A8_GetDistFunc(SQ8_A8_MIN_DIM, nullptr, &cpu), nullptr); +#endif +} + +#if defined(CPU_FEATURES_ARCH_X86_64) && defined(OPT_AVX512_F_BW_VL_VNNI) +namespace { + +// For these blobs the distance is 1 - (dot - expected): min 0, delta 1 and SUM 0 on the stored +// side, QD 1, QM 0, QS 0 and QK -expected on the query side, every term an integer below 2^53. +// So it is exactly 1 iff the kernel's integer dot equals expected. +float A8DotProbe(dist_func_t func, const std::vector &codes, + const std::vector &query_codes, int64_t expected) { + const size_t dim = codes.size(); + std::vector stored = A8StoredFromCodes(codes, 0.0f, 1.0f); + const float zero_sum = 0.0f; + std::memcpy(stored.data() + dim + sq8::SUM * sizeof(float), &zero_sum, sizeof(float)); + std::vector query(sq8_a8::blob_bytes_count(dim), 0); + std::memcpy(query.data(), query_codes.data(), dim); + const double consts[] = {1.0, 0.0, 0.0, -static_cast(expected)}; + std::memcpy(query.data() + sq8_a8::consts_offset(dim), consts, sizeof(consts)); + return func(stored.data(), query.data(), dim); +} + +} // namespace + +class SQ8_A8_AVX512VNNITest : public testing::TestWithParam {}; + +TEST_P(SQ8_A8_AVX512VNNITest, MatchesScalarBitForBit) { + auto optimization = getCpuOptimizationFeatures(); + if (!optimization.avx512f || !optimization.avx512bw || !optimization.avx512vl || + !optimization.avx512vnni) { + GTEST_SKIP() << "AVX512 F/BW/VL/VNNI is unavailable"; + } + const size_t dim = GetParam(); + const dist_func_t simd = Choose_SQ8_A8_IP_implementation_AVX512F_BW_VL_VNNI(dim); + const bool gate_on = dim >= SQ8_A8_MIN_DIM && dim <= SQ8_A8_MAX_EXACT_SIMD_DIM; + unsigned char alignment = 0; + EXPECT_EQ(IP_SQ8_A8_GetDistFunc(dim, &alignment, &optimization), gate_on ? simd : nullptr); + EXPECT_EQ(alignment, 0); + + // The integer dot is exact, for random codes and for both extremes of each operand. + std::vector codes(dim); + std::vector query_codes(dim); + const auto expect_exact_dot = [&](const char *what) { + int64_t expected = 0; + for (size_t i = 0; i < dim; i++) { + expected += static_cast(codes[i]) * query_codes[i]; + } + EXPECT_EQ(A8DotProbe(SQ8_A8_InnerProduct, codes, query_codes, expected), 1.0f) + << "scalar, " << what << ", dim " << dim; + EXPECT_EQ(A8DotProbe(simd, codes, query_codes, expected), 1.0f) + << "AVX512, " << what << ", dim " << dim; + }; + for (int seed : {1, 2}) { + test_utils::populate_uint8_vec(codes.data(), dim, seed); + test_utils::populate_int8_vec(query_codes.data(), dim, seed + 100); + expect_exact_dot("random codes"); + } + for (const uint8_t a : {uint8_t{0}, uint8_t{255}}) { + for (const int8_t p : {int8_t{-128}, int8_t{127}}) { + codes.assign(dim, a); + query_codes.assign(dim, p); + expect_exact_dot("extreme codes"); + } + } + + // Distances of encoded queries are bit-identical to the scalar reference, also from blobs + // that start at an odd address (stored) or 8 bytes past an allocation (query). + const auto expect_bit_identical = [&](const std::vector &stored, + const std::vector &query, const char *what) { + const float scalar = SQ8_A8_InnerProduct(stored.data(), query.data(), dim); + const float vectorized = simd(stored.data(), query.data(), dim); + EXPECT_EQ(std::memcmp(&scalar, &vectorized, sizeof(float)), 0) + << what << ", dim " << dim << ": scalar " << scalar << ", AVX512 " << vectorized; + std::vector stored_buf(stored.size() + 1); + std::vector query_buf(query.size() + 8); + std::memcpy(stored_buf.data() + 1, stored.data(), stored.size()); + std::memcpy(query_buf.data() + 8, query.data(), query.size()); + const float shifted = simd(stored_buf.data() + 1, query_buf.data() + 8, dim); + EXPECT_EQ(std::memcmp(&scalar, &shifted, sizeof(float)), 0) + << what << ", shifted blobs, dim " << dim; + }; + for (int seed : {1, 2}) { + expect_bit_identical(A8Stored(A8RandomVector(dim, seed, -1.0f, 1.0f, true)), + A8Encode(A8RandomVector(dim, seed + 50, -1.0f, 1.0f, true)), + "normalized"); + } + expect_bit_identical(A8Stored(A8RandomVector(dim, 3, 1000.0f, 1001.0f)), + A8Encode(A8RandomVector(dim, 4, 1000.0f, 1001.0f)), "offset"); + expect_bit_identical(A8StoredFromCodes(std::vector(dim, 0), -0.5f, 1.0f / 255), + A8Encode(A8RandomVector(dim, 5)), "all-0 codes"); + expect_bit_identical(A8StoredFromCodes(std::vector(dim, 255), -0.5f, 1.0f / 255), + A8Encode(A8RandomVector(dim, 6)), "all-255 codes"); +} + +// Every residual dim % 64 inside the gate. +INSTANTIATE_TEST_SUITE_P(SQ8_A8_EveryResidual, SQ8_A8_AVX512VNNITest, + testing::Range(size_t{768}, size_t{832})); +// Below the gate (the kernel itself handles any dim), each count of leftover 64-byte chunks after +// the 256-byte loop, and the largest exact dim. +INSTANTIATE_TEST_SUITE_P(SQ8_A8_Dims, SQ8_A8_AVX512VNNITest, + testing::Values(size_t{1}, size_t{15}, size_t{63}, size_t{64}, size_t{65}, + size_t{127}, size_t{128}, size_t{255}, size_t{256}, + size_t{257}, size_t{512}, size_t{767}, size_t{832}, + size_t{896}, size_t{960}, size_t{1000}, size_t{1024}, + size_t{1088}, size_t{1152}, size_t{1216}, size_t{1536}, + spaces::SQ8_A8_MAX_EXACT_SIMD_DIM)); +#endif From 4c2a5de8d2bb8c2da8626f1a361a03fac099bbc6 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Thu, 12 Nov 2026 12:05:55 +0200 Subject: [PATCH 2/3] MOD-19169: Add the AVX-512 A8 query encoder and mean centering The A8 query encoder now runs vectorized and handles mean-centered indexes, so the factory can quantize SQ8 IP queries once per query at a cost close to today's query preparation. - sq8_a8_query_encoder.h: SQ8_A8_EncodeQuery takes an optional mean. With one, it encodes y - mean (the same FP32 subtraction the stored vectors get) and writes of the raw query to the Y_MEAN_IP slot. The three-argument form is unchanged. - IP_AVX512F_SQ8_A8_Encoder.h: the AVX-512F encoder for FP32 and FP16 input (vcvtph2ps). Each lane does the scalar encoder's double operations in the same order, and the range uses the same min / max operand order, so the codes equal the scalar encoder's bit for bit, including NaN (code -128), infinities and constant queries. Only the summed constants may differ in the last bits. - SQ8_A8_GetQueryEncoder(arch_opt) picks it when the CPU has AVX-512F, and the scalar encoder otherwise. - test_spaces.cpp: AVX-512F against scalar for 14 dims and 17 kinds of query (ties at exact and inexact steps, NaN first, in a block and in the tail, infinities, +-FLT_MAX, subnormals), with and without a mean, FP32 and FP16; every finite FP16 value; the chooser; and the centered encoding against the encoding of y - mean. Nothing calls the encoder yet. Co-Authored-By: Claude Opus 5.5 --- .../spaces/IP/IP_AVX512F_SQ8_A8_Encoder.h | 163 ++++++++++++ src/VecSim/spaces/IP_space.cpp | 22 ++ src/VecSim/spaces/IP_space.h | 4 + src/VecSim/spaces/functions/AVX512F.cpp | 9 + src/VecSim/spaces/functions/AVX512F.h | 5 + src/VecSim/spaces/spaces.h | 6 + src/VecSim/spaces/sq8_a8_query_encoder.h | 49 +++- tests/unit/test_spaces.cpp | 237 ++++++++++++++++++ 8 files changed, 488 insertions(+), 7 deletions(-) create mode 100644 src/VecSim/spaces/IP/IP_AVX512F_SQ8_A8_Encoder.h diff --git a/src/VecSim/spaces/IP/IP_AVX512F_SQ8_A8_Encoder.h b/src/VecSim/spaces/IP/IP_AVX512F_SQ8_A8_Encoder.h new file mode 100644 index 000000000..9d401966a --- /dev/null +++ b/src/VecSim/spaces/IP/IP_AVX512F_SQ8_A8_Encoder.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once +#include "VecSim/spaces/space_includes.h" +#include "VecSim/spaces/sq8_a8_query_encoder.h" +#include "VecSim/types/float16.h" + +#include + +/* + * Vectorized A8 query encoder (vecsim_types::sq8_a8_query), AVX-512F only. + * + * Pass 1 finds the float range 16 lanes at a time. Pass 2 converts 16 floats per iteration to two + * halves of 8 doubles, and each lane performs the scalar encoder's double operations in the same + * order (v = y - c, v / s + 0.5, floor, max with -128, then min with 127), so every code equals + * SQ8_A8_EncodeElement's. The min / max operand order matches the scalar comparisons too: a NaN + * element is skipped by the range unless it is the first one, and its code is -128. The last + * dim % 16 elements use the scalar step. The sums E and PE are added in a different order than + * the scalar encoder's, so the constants QS and QK may differ in their last bits. + * + * With a mean (mean-centered index), each element is centered in FP32 first, the same float + * subtraction the scalar encoder does, and is accumulated in double. + */ + +namespace spaces { +namespace a8_avx512f { + +// 16 input elements widened to FP32 (FP16 through vcvtph2ps, exact like FP16_to_FP32), centered +// when WithMean. +template +static inline __m512 LoadValues16(const InputType *query, const float *mean, size_t i, + __m512 &raw) { + if constexpr (std::is_same_v) { + raw = _mm512_cvtph_ps(_mm256_loadu_si256(reinterpret_cast(query + i))); + } else { + raw = _mm512_loadu_ps(query + i); + } + if constexpr (WithMean) { + return _mm512_sub_ps(raw, _mm512_loadu_ps(mean + i)); + } else { + return raw; + } +} + +static inline __m512d LowHalf(__m512 x) { return _mm512_cvtps_pd(_mm512_castps512_ps256(x)); } +static inline __m512d HighHalf(__m512 x) { + return _mm512_cvtps_pd(_mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(x), 1))); +} + +struct Sums8 { + __m512d P = _mm512_setzero_pd(); + __m512d E = _mm512_setzero_pd(); + __m512d PE = _mm512_setzero_pd(); + __m512d mean_ip = _mm512_setzero_pd(); +}; + +// Codes of 8 elements as int32, accumulating P, E and PE. The quotient is a true division: a +// multiply by 1 / s needs a division fallback near integers to keep the codes exact, and is not +// faster on AVX-512 hardware. +static inline __m256i EncodeHalf(__m512d y, __m512d offset, __m512d step, Sums8 &sums) { + const __m512d v = _mm512_sub_pd(y, offset); + const __m512d t = _mm512_add_pd(_mm512_div_pd(v, step), _mm512_set1_pd(0.5)); + __m512d code = _mm512_roundscale_pd(t, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); + // max(code, -128) returns -128 for NaN, like the scalar clamp. + code = _mm512_max_pd(code, _mm512_set1_pd(-128.0)); + code = _mm512_min_pd(code, _mm512_set1_pd(127.0)); + const __m512d err = _mm512_sub_pd(v, _mm512_mul_pd(step, code)); + sums.P = _mm512_add_pd(sums.P, code); + sums.E = _mm512_add_pd(sums.E, err); + sums.PE = _mm512_fmadd_pd(code, err, sums.PE); + return _mm512_cvtpd_epi32(code); // exact: code is an integer in [-128, 127] +} + +template +static void EncodeQuery(const InputType *query, const float *mean, size_t dim, void *blob) { + assert(dim > 0); + const auto value = [=](size_t i) -> float { + const float y = quantized::SQ8_A8_InputValue(query, i); + if constexpr (WithMean) { + return y - mean[i]; + } else { + return y; + } + }; + + // Pass 1: range. Every lane starts from element 0, as the scalar loop does. + float min_val = value(0); + float max_val = min_val; + size_t i = 0; + __m512 raw; + { + __m512 lo = _mm512_set1_ps(min_val); + __m512 hi = lo; + for (; i + 16 <= dim; i += 16) { + const __m512 x = LoadValues16(query, mean, i, raw); + lo = _mm512_min_ps(x, lo); // x < lo ? x : lo + hi = _mm512_max_ps(x, hi); // x > hi ? x : hi + } + min_val = _mm512_reduce_min_ps(lo); + max_val = _mm512_reduce_max_ps(hi); + } + for (; i < dim; i++) { + const float y = value(i); + min_val = y < min_val ? y : min_val; + max_val = y > max_val ? y : max_val; + } + const quantized::SQ8_A8_QueryGrid grid = quantized::SQ8_A8_QueryGridFromRange(min_val, max_val); + + // Pass 2: codes and sums. + const __m512d offset = _mm512_set1_pd(grid.offset); + const __m512d step = _mm512_set1_pd(grid.step); + auto *codes = static_cast(blob); + Sums8 low_sums, high_sums; + i = 0; + for (; i + 16 <= dim; i += 16) { + const __m512 x = LoadValues16(query, mean, i, raw); + const __m256i low = EncodeHalf(LowHalf(x), offset, step, low_sums); + const __m256i high = EncodeHalf(HighHalf(x), offset, step, high_sums); + const __m512i codes32 = _mm512_inserti64x4(_mm512_castsi256_si512(low), high, 1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(codes + i), _mm512_cvtepi32_epi8(codes32)); + if constexpr (WithMean) { + const __m512 m = _mm512_loadu_ps(mean + i); + low_sums.mean_ip = _mm512_fmadd_pd(LowHalf(m), LowHalf(raw), low_sums.mean_ip); + high_sums.mean_ip = _mm512_fmadd_pd(HighHalf(m), HighHalf(raw), high_sums.mean_ip); + } + } + quantized::SQ8_A8_QuerySums sums; + sums.P = static_cast(_mm512_reduce_add_pd(_mm512_add_pd(low_sums.P, high_sums.P))); + sums.E = _mm512_reduce_add_pd(_mm512_add_pd(low_sums.E, high_sums.E)); + sums.PE = _mm512_reduce_add_pd(_mm512_add_pd(low_sums.PE, high_sums.PE)); + [[maybe_unused]] double y_mean_ip = + _mm512_reduce_add_pd(_mm512_add_pd(low_sums.mean_ip, high_sums.mean_ip)); + for (size_t tail = i; tail < dim; tail++) { + codes[tail] = quantized::SQ8_A8_EncodeElement(value(tail), grid, sums); + } + quantized::SQ8_A8_FinishQuery(grid, sums, dim, blob); + + if constexpr (WithMean) { + for (; i < dim; i++) { + y_mean_ip += static_cast(mean[i]) * quantized::SQ8_A8_InputValue(query, i); + } + quantized::SQ8_A8_SetQueryMeanIP(y_mean_ip, dim, blob); + } +} + +} // namespace a8_avx512f + +template +void SQ8_A8_EncodeQuery_AVX512F(const InputType *query, const float *mean, size_t dim, void *blob) { + if (mean) { + a8_avx512f::EncodeQuery(query, mean, dim, blob); + } else { + a8_avx512f::EncodeQuery(query, nullptr, dim, blob); + } +} + +} // namespace spaces diff --git a/src/VecSim/spaces/IP_space.cpp b/src/VecSim/spaces/IP_space.cpp index cd86e5919..07eb92214 100644 --- a/src/VecSim/spaces/IP_space.cpp +++ b/src/VecSim/spaces/IP_space.cpp @@ -9,6 +9,7 @@ #include "VecSim/spaces/space_includes.h" #include "VecSim/spaces/IP_space.h" #include "VecSim/spaces/IP/IP.h" +#include "VecSim/spaces/sq8_a8_query_encoder.h" #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/spaces/functions/AVX512F.h" @@ -556,6 +557,27 @@ dist_func_t IP_SQ8_A8_GetDistFunc(size_t dim, unsigned char *alignment, return nullptr; } +template +sq8_a8_encode_func_t SQ8_A8_GetQueryEncoder(const void *arch_opt) { + [[maybe_unused]] auto features = getCpuOptimizationFeatures(arch_opt); + +#ifdef CPU_FEATURES_ARCH_X86_64 +#ifdef OPT_AVX512F + if (features.avx512f) { + if constexpr (std::is_same_v) { + return Choose_SQ8_A8_FP16_QueryEncoder_AVX512F(); + } else { + return Choose_SQ8_A8_FP32_QueryEncoder_AVX512F(); + } + } +#endif +#endif // __x86_64__ + return quantized::SQ8_A8_EncodeQuery; +} + +template sq8_a8_encode_func_t SQ8_A8_GetQueryEncoder(const void *arch_opt); +template sq8_a8_encode_func_t SQ8_A8_GetQueryEncoder(const void *arch_opt); + dist_func_t IP_FP32_GetDistFunc(size_t dim, unsigned char *alignment, const void *arch_opt) { unsigned char dummy_alignment; if (alignment == nullptr) { diff --git a/src/VecSim/spaces/IP_space.h b/src/VecSim/spaces/IP_space.h index 3c6f0d8a2..0caf4ab3c 100644 --- a/src/VecSim/spaces/IP_space.h +++ b/src/VecSim/spaces/IP_space.h @@ -55,4 +55,8 @@ dist_func_t Cosine_SQ8_SQ8_GetDistFunc(size_t dim, unsigned char *alignme // alignment, so the hint is left unchanged. dist_func_t IP_SQ8_A8_GetDistFunc(size_t dim, unsigned char *alignment = nullptr, const void *arch_opt = nullptr); +// The A8 query encoder for float or float16 input: vectorized where the CPU has AVX-512F, the +// scalar encoder otherwise. Both write the same codes. +template +sq8_a8_encode_func_t SQ8_A8_GetQueryEncoder(const void *arch_opt = nullptr); } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX512F.cpp b/src/VecSim/spaces/functions/AVX512F.cpp index 0b68d505e..91934b510 100644 --- a/src/VecSim/spaces/functions/AVX512F.cpp +++ b/src/VecSim/spaces/functions/AVX512F.cpp @@ -17,6 +17,7 @@ #include "VecSim/spaces/IP/IP_AVX512F_FP32.h" #include "VecSim/spaces/IP/IP_AVX512F_FP64.h" #include "VecSim/spaces/IP/IP_AVX512F_SQ8_FP16.h" +#include "VecSim/spaces/IP/IP_AVX512F_SQ8_A8_Encoder.h" namespace spaces { @@ -77,6 +78,14 @@ dist_func_t Choose_SQ8_FP16_L2_implementation_AVX512F(size_t dim) { return ret_dist_func; } +// The A8 query encoders need only AVX-512F; the A8 kernels that read their output need VNNI. +sq8_a8_encode_func_t Choose_SQ8_A8_FP32_QueryEncoder_AVX512F() { + return SQ8_A8_EncodeQuery_AVX512F; +} +sq8_a8_encode_func_t Choose_SQ8_A8_FP16_QueryEncoder_AVX512F() { + return SQ8_A8_EncodeQuery_AVX512F; +} + #include "implementation_chooser_cleanup.h" } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX512F.h b/src/VecSim/spaces/functions/AVX512F.h index 8d600f961..1223879c5 100644 --- a/src/VecSim/spaces/functions/AVX512F.h +++ b/src/VecSim/spaces/functions/AVX512F.h @@ -9,6 +9,7 @@ #pragma once #include "VecSim/spaces/spaces.h" +#include "VecSim/types/float16.h" namespace spaces { @@ -25,4 +26,8 @@ dist_func_t Choose_SQ8_FP16_IP_implementation_AVX512F(size_t dim); dist_func_t Choose_SQ8_FP16_Cosine_implementation_AVX512F(size_t dim); dist_func_t Choose_SQ8_FP16_L2_implementation_AVX512F(size_t dim); +// A8 query encoders (vecsim_types::sq8_a8_query) for float and float16 input. +sq8_a8_encode_func_t Choose_SQ8_A8_FP32_QueryEncoder_AVX512F(); +sq8_a8_encode_func_t Choose_SQ8_A8_FP16_QueryEncoder_AVX512F(); + } // namespace spaces diff --git a/src/VecSim/spaces/spaces.h b/src/VecSim/spaces/spaces.h index c00529123..2c15d9113 100644 --- a/src/VecSim/spaces/spaces.h +++ b/src/VecSim/spaces/spaces.h @@ -49,6 +49,12 @@ using normalizeVector_f = void (*)(void *input_vector, const size_t dim); template normalizeVector_f GetNormalizeFunc(); +// Encodes a query into an A8 query blob (vecsim_types::sq8_a8_query); mean is nullptr or the mean +// of a mean-centered index. See spaces/sq8_a8_query_encoder.h. +template +using sq8_a8_encode_func_t = void (*)(const InputType *query, const float *mean, size_t dim, + void *blob); + static int inline is_little_endian() { unsigned int x = 1; return *(char *)&x; diff --git a/src/VecSim/spaces/sq8_a8_query_encoder.h b/src/VecSim/spaces/sq8_a8_query_encoder.h index 34f31f81c..f7aed9ec2 100644 --- a/src/VecSim/spaces/sq8_a8_query_encoder.h +++ b/src/VecSim/spaces/sq8_a8_query_encoder.h @@ -85,21 +85,42 @@ static inline void SQ8_A8_FinishQuery(const SQ8_A8_QueryGrid &grid, const SQ8_A8 std::memcpy(consts_out + a8::P * sizeof(double), &sums.P, sizeof(int64_t)); } +// Writes of the uncentered query to the Y_MEAN_IP slot. Each product of two floats is +// exact in double, so only the summation order can differ between encoders. +static inline void SQ8_A8_SetQueryMeanIP(double y_mean_ip, size_t dim, void *blob) { + using a8 = vecsim_types::sq8_a8_query; + std::memcpy(static_cast(blob) + a8::consts_offset(dim) + + a8::Y_MEAN_IP * sizeof(double), + &y_mean_ip, sizeof(y_mean_ip)); +} + +// Widens one input element to FP32. +template +static inline float SQ8_A8_InputValue(const InputType *query, size_t i) { + if constexpr (std::is_same_v) { + return vecsim_types::FP16_to_FP32(query[i]); + } else { + return query[i]; + } +} + // Writes the full A8 query blob (vecsim_types::sq8_a8_query::blob_bytes_count(dim) bytes) for a // float or float16 query. FP16 values are widened to FP32 first, so an FP16 query encodes exactly // as its FP32 widening. +// +// mean is nullptr, or the mean of a mean-centered index (dim FP32 values). Then the query is +// centered in FP32 as y - mean, exactly as the stored vectors are, and of the uncentered +// query is written to Y_MEAN_IP. template -static inline void SQ8_A8_EncodeQuery(const InputType *query, size_t dim, void *blob) { +static inline void SQ8_A8_EncodeQuery(const InputType *query, const float *mean, size_t dim, + void *blob) { static_assert(std::is_same_v || std::is_same_v, "The A8 query encoder takes float or float16 input"); assert(dim > 0); - const auto value = [query](size_t i) -> float { - if constexpr (std::is_same_v) { - return vecsim_types::FP16_to_FP32(query[i]); - } else { - return query[i]; - } + const auto value = [query, mean](size_t i) -> float { + const float y = SQ8_A8_InputValue(query, i); + return mean ? y - mean[i] : y; }; float min_val = value(0); @@ -117,6 +138,20 @@ static inline void SQ8_A8_EncodeQuery(const InputType *query, size_t dim, void * codes[i] = SQ8_A8_EncodeElement(value(i), grid, sums); } SQ8_A8_FinishQuery(grid, sums, dim, blob); + + if (mean) { + double y_mean_ip = 0; + for (size_t i = 0; i < dim; i++) { + y_mean_ip += static_cast(mean[i]) * SQ8_A8_InputValue(query, i); + } + SQ8_A8_SetQueryMeanIP(y_mean_ip, dim, blob); + } +} + +// The query of an index without a mean. +template +static inline void SQ8_A8_EncodeQuery(const InputType *query, size_t dim, void *blob) { + SQ8_A8_EncodeQuery(query, nullptr, dim, blob); } } // namespace quantized diff --git a/tests/unit/test_spaces.cpp b/tests/unit/test_spaces.cpp index 154aa5951..b2c668e7b 100644 --- a/tests/unit/test_spaces.cpp +++ b/tests/unit/test_spaces.cpp @@ -6054,3 +6054,240 @@ INSTANTIATE_TEST_SUITE_P(SQ8_A8_Dims, SQ8_A8_AVX512VNNITest, size_t{1088}, size_t{1152}, size_t{1216}, size_t{1536}, spaces::SQ8_A8_MAX_EXACT_SIMD_DIM)); #endif + +#if defined(CPU_FEATURES_ARCH_X86_64) && defined(OPT_AVX512F) +namespace { + +template +std::vector A8CastInput(const std::vector &y) { + if constexpr (std::is_same_v) { + std::vector out(y.size()); + for (size_t i = 0; i < y.size(); i++) { + out[i] = vecsim_types::FP32_to_FP16(y[i]); + } + return out; + } else { + return y; + } +} + +// A constant may be NaN when the query holds NaN or an infinity; then both encoders must agree. +void ExpectA8ConstNear(double scalar, double simd, double tolerance, const std::string &what) { + if (std::isnan(scalar) || std::isnan(simd)) { + EXPECT_TRUE(std::isnan(scalar) && std::isnan(simd)) + << what << ": scalar " << scalar << ", AVX512F " << simd; + } else if (std::isinf(scalar) || std::isinf(simd)) { + EXPECT_EQ(scalar, simd) << what; + } else { + EXPECT_NEAR(simd, scalar, tolerance) << what; + } +} + +// The AVX-512F encoder against the scalar one on one query: the codes, the zeroed padding, P, QD +// and QM bit for bit; QS, QK and Y_MEAN_IP are sums in another order, so within 1e-12 of their +// scale (the scalar encoder's tolerance against the definition). +template +void ExpectA8EncodersMatch(sq8_a8_encode_func_t simd_encode, + const std::vector &y, const float *mean, + const std::string &what) { + const size_t dim = y.size(); + std::vector scalar(sq8_a8::blob_bytes_count(dim), 0xAB); + std::vector simd(sq8_a8::blob_bytes_count(dim), 0xCD); + spaces::quantized::SQ8_A8_EncodeQuery(y.data(), mean, dim, scalar.data()); + simd_encode(y.data(), mean, dim, simd.data()); + + for (size_t i = 0; i < sq8_a8::consts_offset(dim); i++) { + ASSERT_EQ(simd[i], scalar[i]) << what << ", dim " << dim << ", byte " << i; + } + for (size_t i = dim; i < sq8_a8::consts_offset(dim); i++) { + ASSERT_EQ(simd[i], 0) << what << ", dim " << dim << ", padding byte " << i; + } + EXPECT_EQ(A8CodeSum(simd, dim), A8CodeSum(scalar, dim)) << what << ", dim " << dim; + for (size_t index : {sq8_a8::QD, sq8_a8::QM}) { + const double a = A8Const(scalar, dim, index), b = A8Const(simd, dim, index); + EXPECT_TRUE(std::memcmp(&a, &b, sizeof(double)) == 0 || (std::isnan(a) && std::isnan(b))) + << what << ", dim " << dim << ", constant " << index << ": " << a << " vs " << b; + } + const double s = A8Const(scalar, dim, sq8_a8::QD); + const double qs = A8Const(scalar, dim, sq8_a8::QS); + ExpectA8ConstNear(qs, A8Const(simd, dim, sq8_a8::QS), 1e-12 * (std::fabs(qs) + s), + what + ", QS, dim " + std::to_string(dim)); + // |K| is at most 128 * s^2 * dim. + ExpectA8ConstNear(A8Const(scalar, dim, sq8_a8::QK), A8Const(simd, dim, sq8_a8::QK), + 1e-12 * 128.0 * s * s * static_cast(dim), + what + ", QK, dim " + std::to_string(dim)); + double mean_ip_scale = 0; + for (size_t i = 0; mean && i < dim; i++) { + mean_ip_scale += std::fabs(static_cast(mean[i]) * + spaces::quantized::SQ8_A8_InputValue(y.data(), i)); + } + ExpectA8ConstNear(A8Const(scalar, dim, sq8_a8::Y_MEAN_IP), + A8Const(simd, dim, sq8_a8::Y_MEAN_IP), 1e-12 * mean_ip_scale, + what + ", Y_MEAN_IP, dim " + std::to_string(dim)); + if (!mean) { + EXPECT_EQ(A8Const(simd, dim, sq8_a8::Y_MEAN_IP), 0.0) << what << ", dim " << dim; + } +} + +// Queries that reach every branch of the code computation: ordinary data, constant and zero +// queries, exact rounding ties, saturated ends, NaN (first element, inside a vector block, in the +// tail), infinities and subnormals. +std::vector>> A8EncoderCases(size_t dim) { + std::vector>> cases; + cases.emplace_back("normalized", A8RandomVector(dim, 1, -1.0f, 1.0f, true)); + cases.emplace_back("offset", A8RandomVector(dim, 2, 1000.0f, 1001.0f)); + cases.emplace_back("small", A8RandomVector(dim, 3, -1e-3f, 1e-3f)); + cases.emplace_back("constant", std::vector(dim, 0.25f)); + cases.emplace_back("zero", std::vector(dim, 0.0f)); + // With min 0 and max 255 the grid is s = 1, c = 128, so y = k + 0.5 lands exactly on a + // rounding tie (v / s + 0.5 is an integer). + std::vector ties(dim); + for (size_t i = 0; i < dim; i++) { + ties[i] = static_cast(i % 255) + 0.5f; + } + ties[0] = 0.0f; + if (dim > 1) { + ties[dim - 1] = 255.0f; + } + cases.emplace_back("rounding ties", ties); + // The same with an inexact step (s = 0.3): the quotients land within rounding of an integer, + // so any change to the rounding of the code computation shows. + std::vector inexact_ties(dim); + for (size_t i = 0; i < dim; i++) { + inexact_ties[i] = static_cast((static_cast(i % 255) + 0.5) * 0.3); + } + inexact_ties[0] = 0.0f; + if (dim > 1) { + inexact_ties[dim - 1] = 76.5f; + } + cases.emplace_back("rounding ties, inexact step", inexact_ties); + std::vector wide(dim); + for (size_t i = 0; i < dim; i++) { + wide[i] = i % 2 ? std::numeric_limits::max() : -std::numeric_limits::max(); + } + cases.emplace_back("+-FLT_MAX", wide); + const float nan = std::numeric_limits::quiet_NaN(); + for (size_t at : {size_t{0}, size_t{20}, dim - 1}) { + if (at < dim) { + std::vector with_nan = A8RandomVector(dim, 4); + with_nan[at] = nan; + cases.emplace_back("NaN at " + std::to_string(at), with_nan); + } + } + std::vector with_inf = A8RandomVector(dim, 5); + with_inf[dim / 2] = std::numeric_limits::infinity(); + cases.emplace_back("+inf", with_inf); + std::vector with_neg_inf = A8RandomVector(dim, 6); + with_neg_inf[dim / 3] = -std::numeric_limits::infinity(); + cases.emplace_back("-inf", with_neg_inf); + std::vector outlier = A8RandomVector(dim, 7); + outlier[dim / 2] = 100.0f; + cases.emplace_back("outlier", outlier); + cases.emplace_back("subnormal", A8RandomVector(dim, 8, -1e-39f, 1e-39f)); + return cases; +} + +template +void ExpectA8EncodersMatchOnAllCases(sq8_a8_encode_func_t simd_encode) { + for (const size_t dim : {1UL, 7UL, 15UL, 16UL, 17UL, 31UL, 32UL, 33UL, 767UL, 768UL, 769UL, + 1000UL, 1024UL, 1536UL}) { + const std::vector mean = A8RandomVector(dim, 9, -0.5f, 0.5f); + for (const auto &[name, values] : A8EncoderCases(dim)) { + const std::vector y = A8CastInput(values); + ExpectA8EncodersMatch(simd_encode, y, nullptr, name); + ExpectA8EncodersMatch(simd_encode, y, mean.data(), name + ", centered"); + } + } +} + +} // namespace + +TEST_F(SpacesTest, SQ8_A8_AVX512FEncoderMatchesScalar) { + const auto cpu = getCpuOptimizationFeatures(); + if (!cpu.avx512f) { + GTEST_SKIP() << "AVX512F is unavailable"; + } + ExpectA8EncodersMatchOnAllCases(Choose_SQ8_A8_FP32_QueryEncoder_AVX512F()); + ExpectA8EncodersMatchOnAllCases(Choose_SQ8_A8_FP16_QueryEncoder_AVX512F()); +} + +// vcvtph2ps against FP16_to_FP32 for every finite FP16 value: each query is a run of consecutive +// encodings, so its range is narrow and each conversion moves its code. +TEST_F(SpacesTest, SQ8_A8_AVX512FEncoderMatchesScalarOnEveryFP16Value) { + const auto cpu = getCpuOptimizationFeatures(); + if (!cpu.avx512f) { + GTEST_SKIP() << "AVX512F is unavailable"; + } + const auto simd_encode = Choose_SQ8_A8_FP16_QueryEncoder_AVX512F(); + constexpr size_t dim = 1031; + std::vector y; + for (uint32_t bits = 0; bits <= 0xFFFF; bits++) { + const float16 value{static_cast(bits)}; + if ((bits & 0x7C00u) == 0x7C00u) { + continue; // infinity or NaN + } + y.push_back(value); + if (y.size() == dim) { + ExpectA8EncodersMatch(simd_encode, y, nullptr, + "FP16 run ending at " + std::to_string(bits)); + y.clear(); + } + } + if (!y.empty()) { + ExpectA8EncodersMatch(simd_encode, y, nullptr, "last FP16 run"); + } +} + +TEST_F(SpacesTest, SQ8_A8_QueryEncoderChooser) { + auto features = getCpuOptimizationFeatures(); + std::memset(&features, 0xFF, sizeof(features)); + EXPECT_EQ(SQ8_A8_GetQueryEncoder(&features), Choose_SQ8_A8_FP32_QueryEncoder_AVX512F()); + EXPECT_EQ(SQ8_A8_GetQueryEncoder(&features), + Choose_SQ8_A8_FP16_QueryEncoder_AVX512F()); + features.avx512f = 0; + // The scalar encoder; it has internal linkage, so it is identified by what it writes. + const auto fp32_fallback = SQ8_A8_GetQueryEncoder(&features); + const auto fp16_fallback = SQ8_A8_GetQueryEncoder(&features); + ASSERT_NE(fp32_fallback, nullptr); + ASSERT_NE(fp16_fallback, nullptr); + EXPECT_NE(fp32_fallback, Choose_SQ8_A8_FP32_QueryEncoder_AVX512F()); + EXPECT_NE(fp16_fallback, Choose_SQ8_A8_FP16_QueryEncoder_AVX512F()); + const std::vector y = A8RandomVector(769, 10); + const std::vector mean = A8RandomVector(769, 11); + std::vector expected(sq8_a8::blob_bytes_count(y.size())); + std::vector got(expected.size()); + spaces::quantized::SQ8_A8_EncodeQuery(y.data(), mean.data(), y.size(), expected.data()); + fp32_fallback(y.data(), mean.data(), y.size(), got.data()); + EXPECT_EQ(got, expected); + const std::vector y16 = A8CastInput(y); + spaces::quantized::SQ8_A8_EncodeQuery(y16.data(), mean.data(), y16.size(), expected.data()); + fp16_fallback(y16.data(), mean.data(), y16.size(), got.data()); + EXPECT_EQ(got, expected); +} +#endif + +// The centered encoding is the encoding of y - mean (in FP32), plus of the raw query. +TEST_F(SpacesTest, SQ8_A8_EncoderCentersOnMean) { + for (const size_t dim : {1UL, 768UL, 1001UL}) { + const std::vector y = A8RandomVector(dim, 12); + const std::vector mean = A8RandomVector(dim, 13, -0.5f, 0.5f); + std::vector centered(dim); + long double mean_ip = 0; + for (size_t i = 0; i < dim; i++) { + centered[i] = y[i] - mean[i]; + mean_ip += static_cast(mean[i]) * y[i]; + } + std::vector blob(sq8_a8::blob_bytes_count(dim), 0xAB); + spaces::quantized::SQ8_A8_EncodeQuery(y.data(), mean.data(), dim, blob.data()); + const std::vector expected = A8Encode(centered); + const size_t y_mean_ip_offset = sq8_a8::consts_offset(dim) + sq8_a8::Y_MEAN_IP * 8; + EXPECT_EQ(std::memcmp(blob.data(), expected.data(), y_mean_ip_offset), 0) << "dim " << dim; + EXPECT_EQ(std::memcmp(blob.data() + y_mean_ip_offset + 8, + expected.data() + y_mean_ip_offset + 8, 8), + 0) + << "dim " << dim; + EXPECT_NEAR(A8Const(blob, dim, sq8_a8::Y_MEAN_IP), static_cast(mean_ip), + 1e-12 * static_cast(dim)) + << "dim " << dim; + } +} From 0eb35e0c9b8dbd4e5b09862105fbfcf2791f7ee3 Mon Sep 17 00:00:00 2001 From: Dor Forer Date: Thu, 12 Nov 2026 12:05:55 +0200 Subject: [PATCH 3/3] MOD-19169: Use A8 queries for SQ8 IP HNSW indexes above the gate SQ8 IP and Cosine HNSW indexes now quantize FP32 and FP16 queries once to 8-bit A8 codes where IP_SQ8_A8_GetDistFunc has a kernel (AVX-512 F/BW/VL/VNNI, 768 <= dim <= 65,793), for searches and for graph inserts, with and without the learned mean. Elsewhere nothing changes: FP32 and FP16 queries stay asymmetric, and L2 and SQ4 are untouched. - components_factory.h: CreateQuantizedIndexComponents tries the A8 chooser first (SQ8 IP only, under if constexpr) and builds the A8 classes when it returns a kernel; otherwise it runs the existing asymmetric path unchanged. The shared assembly moved into AssembleQuantizedIndexComponents. An optional arch_opt (tests only) reaches the chooser. - preprocessors.h: SQ8A8QueryPreprocessor derives from QuantPreprocessor, so applySQ8Mean's cast still holds. Storage blobs are the base's; the query is the A8 blob (64-byte aligned), encoded with the mean as it is when the query arrives, so a mean installed later applies. The encoder is chosen once in a function-local static. - calculator.h: SQ8A8QueryDistanceCalculatorWithNorm derives from DistanceCalculatorWithNorm and applies base - x_mean_ip - y_mean_ip + mean_sum_squares to the A8 kernel, in double; the stored-to-stored path is the base's. Without a mean the A8 kernel is the plain query function. - The new classes add no data members; static_asserts in EstimateInitialSize keep the estimates valid. - Tests: the factory's choice above and below the gate and with the CPU forced off; blob layout, padding, alignment and ownership; distances against the scalar kernel and against a bound derived from the two quantization steps (SQ8A8DistanceTolerance); the deferred mean through applySQ8Mean; and the public API at dims 768, 777 (not a multiple of 64, so the tails run) and 1024 (top-K, range, batch iterator, multi-value, GetDistanceFrom_Unsafe) for standalone indexes, and at 768 and 1024 for tiered indexes with a late mean. Co-Authored-By: Claude Opus 5.5 --- .../components/components_factory.h | 86 ++-- src/VecSim/index_factories/hnsw_factory.cpp | 7 + src/VecSim/spaces/computer/calculator.h | 49 +- src/VecSim/spaces/computer/preprocessors.h | 35 ++ src/VecSim/types/sq8_a8.h | 4 + tests/unit/test_components.cpp | 432 ++++++++++++++++++ tests/unit/test_hnsw_sq8.cpp | 408 +++++++++++++++++ tests/unit/unit_test_utils.h | 51 +++ 8 files changed, 1045 insertions(+), 27 deletions(-) diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index 39fc2486a..32a416340 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -9,6 +9,8 @@ #pragma once #include "VecSim/spaces/spaces.h" +#include "VecSim/spaces/IP_space.h" +#include "VecSim/types/sq8_a8.h" #include "VecSim/vec_sim_common.h" #include "VecSim/vec_sim_index.h" #include "VecSim/index_factories/components/preprocessors_factory.h" @@ -40,22 +42,14 @@ template return alignment; } -template +// Builds the quantization preprocessor, its container and the distance calculator of a quantized +// index. query_func compares a stored blob with a query blob, sym_func two stored blobs. +template IndexComponents -CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, - const float *mean_ptr) { - using Preprocessor = QuantPreprocessor; - using QueryType = typename Preprocessor::QueryType; - unsigned char storage_alignment = 0, asym_storage_alignment = 0; - - // Graph construction compares two stored quantized blobs; search compares a stored blob with a - // QueryType query. Both dispatchers report alignment for the stored operand. - auto sym_func = spaces::GetDistFunc(Metric, dim, &storage_alignment); - auto asym_func = - spaces::GetDistFunc(Metric, dim, &asym_storage_alignment); - storage_alignment = spaces::combineAlignments(storage_alignment, asym_storage_alignment); - const unsigned char query_alignment = GetQueryAlignment(Metric, dim); - +AssembleQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, + const float *mean_ptr, spaces::dist_func_t query_func, + spaces::dist_func_t sym_func, unsigned char query_alignment, + unsigned char storage_alignment) { PreprocessorInterface *pp = nullptr; IndexCalculatorInterface *calc = nullptr; @@ -69,11 +63,10 @@ CreateQuantizedIndexComponents(const std::shared_ptr &allocator } pp = new (allocator) Preprocessor(allocator, dim, mean_vec); - calc = new (allocator) DistanceCalculatorWithNorm( - allocator, asym_func, sym_func, mean_sum_squares); + calc = new (allocator) Calculator(allocator, query_func, sym_func, mean_sum_squares); } else { pp = new (allocator) Preprocessor(allocator, dim); - calc = new (allocator) DistanceCalculatorCommon(allocator, sym_func, asym_func); + calc = new (allocator) DistanceCalculatorCommon(allocator, sym_func, query_func); } auto *container = new (allocator) @@ -84,33 +77,74 @@ CreateQuantizedIndexComponents(const std::shared_ptr &allocator return {calc, container}; } +// SQ8 IP queries take the A8 path where IP_SQ8_A8_GetDistFunc has a kernel for this dim and CPU: +// FP32 and FP16 queries are quantized once to int8 codes (vecsim_types::sq8_a8_query). Everywhere +// else, and for every other storage type and metric, queries stay asymmetric. arch_opt overrides +// the CPU features for the A8 choice (tests only). +template +IndexComponents +CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, + const float *mean_ptr, const void *arch_opt = nullptr) { + if constexpr (std::is_same_v && Metric == VecSimMetric_IP) { + unsigned char a8_storage_alignment = 0; + if (auto a8_func = spaces::IP_SQ8_A8_GetDistFunc(dim, &a8_storage_alignment, arch_opt)) { + unsigned char storage_alignment = 0; + auto sym_func = + spaces::GetDistFunc(Metric, dim, &storage_alignment); + return AssembleQuantizedIndexComponents< + DataType, WithMean, SQ8A8QueryPreprocessor, + SQ8A8QueryDistanceCalculatorWithNorm>( + allocator, dim, mean_ptr, a8_func, sym_func, + vecsim_types::sq8_a8_query::blob_alignment, + spaces::combineAlignments(storage_alignment, a8_storage_alignment)); + } + } + + using Preprocessor = QuantPreprocessor; + using QueryType = typename Preprocessor::QueryType; + unsigned char storage_alignment = 0, asym_storage_alignment = 0; + + // Graph construction compares two stored quantized blobs; search compares a stored blob with a + // QueryType query. Both dispatchers report alignment for the stored operand. + auto sym_func = spaces::GetDistFunc(Metric, dim, &storage_alignment); + auto asym_func = + spaces::GetDistFunc(Metric, dim, &asym_storage_alignment); + storage_alignment = spaces::combineAlignments(storage_alignment, asym_storage_alignment); + const unsigned char query_alignment = GetQueryAlignment(Metric, dim); + + return AssembleQuantizedIndexComponents< + DataType, WithMean, Preprocessor, + DistanceCalculatorWithNorm>( + allocator, dim, mean_ptr, asym_func, sym_func, query_alignment, storage_alignment); +} + // Run-time overload: selects WithMean from whether mean_ptr is set. template IndexComponents CreateQuantizedIndexComponents(const std::shared_ptr &allocator, size_t dim, - const float *mean_ptr) { + const float *mean_ptr, const void *arch_opt = nullptr) { if (mean_ptr) { - return CreateQuantizedIndexComponents(allocator, dim, - mean_ptr); + return CreateQuantizedIndexComponents( + allocator, dim, mean_ptr, arch_opt); } return CreateQuantizedIndexComponents(allocator, dim, - mean_ptr); + mean_ptr, arch_opt); } template IndexComponents CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, - const float *mean_ptr) { + const float *mean_ptr, const void *arch_opt = nullptr) { return CreateQuantizedIndexComponents( - allocator, dim, mean_ptr); + allocator, dim, mean_ptr, arch_opt); } template IndexComponents CreateSQ8IndexComponents(const std::shared_ptr &allocator, size_t dim, - const float *mean_ptr) { + const float *mean_ptr, const void *arch_opt = nullptr) { return CreateQuantizedIndexComponents(allocator, dim, - mean_ptr); + mean_ptr, arch_opt); } template diff --git a/src/VecSim/index_factories/hnsw_factory.cpp b/src/VecSim/index_factories/hnsw_factory.cpp index 2d03fc233..786b29a8b 100644 --- a/src/VecSim/index_factories/hnsw_factory.cpp +++ b/src/VecSim/index_factories/hnsw_factory.cpp @@ -216,6 +216,13 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi sizeof(QuantPreprocessor)); static_assert(sizeof(QuantPreprocessor) == sizeof(QuantPreprocessor)); + // Nor does the query path: the A8 classes add no data members. + static_assert(sizeof(SQ8A8QueryPreprocessor) == + sizeof(QuantPreprocessor)); + static_assert(sizeof(SQ8A8QueryPreprocessor) == + sizeof(QuantPreprocessor)); + static_assert(sizeof(SQ8A8QueryDistanceCalculatorWithNorm) == + sizeof(DistanceCalculatorWithNorm)); if (with_mean) { est += allocations_overhead + sizeof(DistanceCalculatorWithNorm); diff --git a/src/VecSim/spaces/computer/calculator.h b/src/VecSim/spaces/computer/calculator.h index 230042ab2..385099abb 100644 --- a/src/VecSim/spaces/computer/calculator.h +++ b/src/VecSim/spaces/computer/calculator.h @@ -13,6 +13,7 @@ #include "VecSim/memory/vecsim_base.h" #include "VecSim/spaces/spaces.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq8_a8.h" #include "VecSim/utils/alignment.h" enum class DistanceMode { @@ -156,7 +157,7 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP, "DistanceCalculatorWithNorm only supports L2 and IP metrics"); -private: +protected: struct WithNormDistanceContext { spaces::dist_func_t stored_func; spaces::dist_func_t query_func; @@ -166,6 +167,7 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { // HNSW may install the accumulated mean before storing the first vector. WithNormDistanceContext context_; +private: static DistType calcStoredWithContext(const void *opaque_context, const void *v1, const void *v2, size_t dim) { const auto *context = static_cast(opaque_context); @@ -250,3 +252,48 @@ class DistanceCalculatorWithNorm : public IndexCalculatorInterface { return DistanceDispatch::stateful(&context_, calcQueryWithContext); } }; + +// SQ8 storage with an A8 query (vecsim_types::sq8_a8_query) in a mean-centered IP index. The +// query is encoded centered, y' = y - mean, like the stored vectors, so the query path applies the +// stored-to-stored correction, with y_mean_ip = read from the query's constants. The +// stored-to-stored path is the base class's. +template +class SQ8A8QueryDistanceCalculatorWithNorm + : public DistanceCalculatorWithNorm { + using Base = DistanceCalculatorWithNorm; + using Context = typename Base::WithNormDistanceContext; + static_assert(Metric == VecSimMetric_IP, "A8 queries serve SQ8 inner product only"); + + static DistType calcA8QueryWithContext(const void *opaque_context, const void *candidate, + const void *query, size_t dim) { + using sq8 = vecsim_types::sq8; + using a8 = vecsim_types::sq8_a8_query; + const auto *context = static_cast(opaque_context); + // base = 1 - IP(x', y'). As for two stored blobs: + // 1 - IP(x, y) = base - x_mean_ip - y_mean_ip + mean_sum_squares + const DistType base = context->query_func(candidate, query, dim); + const float x_mean_ip = load_unaligned(static_cast(candidate) + + sq8::data_bytes_count(dim) + + sq8::mean_ip_index() * sizeof(float)); + const double y_mean_ip = + load_unaligned(static_cast(query) + a8::consts_offset(dim) + + a8::Y_MEAN_IP * sizeof(double)); + return static_cast(static_cast(base) - x_mean_ip - y_mean_ip + + context->mean_sum_squares); + } + +public: + using Base::Base; + + DistType calcDistanceForQuery(const void *candidate, const void *query, + size_t dim) const override { + return calcA8QueryWithContext(&this->context_, candidate, query, dim); + } + + DistanceDispatch getDistanceDispatch(DistanceMode mode) const override { + if (mode == DistanceMode::StoredToStored) { + return Base::getDistanceDispatch(mode); + } + return DistanceDispatch::stateful(&this->context_, calcA8QueryWithContext); + } +}; diff --git a/src/VecSim/spaces/computer/preprocessors.h b/src/VecSim/spaces/computer/preprocessors.h index 1dbcd2aef..889caa318 100644 --- a/src/VecSim/spaces/computer/preprocessors.h +++ b/src/VecSim/spaces/computer/preprocessors.h @@ -22,9 +22,11 @@ #include "VecSim/memory/vecsim_base.h" #include "VecSim/spaces/spaces.h" +#include "VecSim/spaces/IP_space.h" #include "VecSim/memory/memory_utils.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq8_a8.h" #include "VecSim/types/sq4.h" #include "VecSim/utils/vecsim_stl.h" @@ -667,11 +669,44 @@ class QuantPreprocessor : public PreprocessorInterface { } } +protected: // Mean vector for WithNorm=true; zero-size placeholder otherwise (no runtime overhead). [[no_unique_address]] std::conditional_t, std::monostate> mean; const size_t dim; + +private: const size_t storage_bytes_count; const size_t query_bytes_count; }; + +// SQ8 storage with an A8 query (vecsim_types::sq8_a8_query): the query is quantized once to int8 +// codes on its own grid, so each distance is one integer dot product. Storage blobs are the base +// class's, and preprocess() returns them with a separate A8 query blob. With WithNorm the query is +// centered like the stored vectors, reading the mean when it is encoded, so a mean installed later +// with setMean applies to later queries. +template +class SQ8A8QueryPreprocessor : public QuantPreprocessor { + using Base = QuantPreprocessor; + static_assert(Metric == VecSimMetric_IP, "A8 queries serve SQ8 inner product only"); + +public: + using Base::Base; + + void preprocessQuery(const void *original_blob, void *&blob, size_t &query_blob_size, + unsigned char alignment) const override { + assert(!blob && "query_blob must be nullptr"); + // Chosen once from the CPU features; the choice does not depend on the index. + static const auto encode = spaces::SQ8_A8_GetQueryEncoder(); + + const size_t bytes = vecsim_types::sq8_a8_query::blob_bytes_count(this->dim); + blob = this->allocator->allocate_aligned(bytes, alignment); + const float *mean = nullptr; + if constexpr (WithNorm) { + mean = this->mean.data(); + } + encode(static_cast(original_blob), mean, this->dim, blob); + query_blob_size = bytes; + } +}; diff --git a/src/VecSim/types/sq8_a8.h b/src/VecSim/types/sq8_a8.h index 58bfd30a1..330830862 100644 --- a/src/VecSim/types/sq8_a8.h +++ b/src/VecSim/types/sq8_a8.h @@ -43,6 +43,10 @@ struct sq8_a8_query { static constexpr size_t blob_bytes_count(size_t dim) { return consts_offset(dim) + CONSTS_COUNT * sizeof(double); } + + // Allocation alignment of a query blob. The kernels need none (unaligned and masked loads, + // constants read with memcpy); 64 keeps the codes on whole cache lines. + static constexpr unsigned char blob_alignment = 64; }; static_assert(sizeof(double) == sizeof(int64_t), "A8 query constants use 8-byte slots"); diff --git a/tests/unit/test_components.cpp b/tests/unit/test_components.cpp index 6744756b3..b60ac3a87 100644 --- a/tests/unit/test_components.cpp +++ b/tests/unit/test_components.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "gtest/gtest.h" #include "VecSim/vec_sim.h" @@ -22,6 +23,10 @@ #include "VecSim/index_factories/components/components_factory.h" #include "VecSim/spaces/IP_space.h" #include "VecSim/spaces/L2_space.h" +#include "VecSim/spaces/IP/IP.h" +#include "VecSim/spaces/sq8_a8_query_encoder.h" +#include "VecSim/quantization/sq8_mean.h" +#include "VecSim/types/sq8_a8.h" #include "unit_test_utils.h" #include "tests_utils.h" @@ -2846,3 +2851,430 @@ TEST(SQ8TrainingStateTest, AccumulatesInFP64) { EXPECT_DOUBLE_EQ(state.getRunningSum()[0], 1.0); EXPECT_FLOAT_EQ(state.calculateMean()[0], 1.0f); } + +/* ======================== SQ8 IP with A8 queries ========================= */ + +namespace { + +using sq8_a8 = vecsim_types::sq8_a8_query; + +// Whether this build and CPU have the A8 query path at all. +bool HostHasA8Queries() { return spaces::IP_SQ8_A8_GetDistFunc(spaces::SQ8_A8_MIN_DIM) != nullptr; } + +// CPU features with everything off, so the factory takes the paths of a CPU without AVX-512. +auto CpuFeaturesOff() { + auto features = spaces::getCpuOptimizationFeatures(); + std::memset(&features, 0, sizeof(features)); + return features; +} + +template +PreprocessorInterface *SQ8FactoryPreprocessor(const IndexComponents &components) { + return static_cast *>(components.preprocessors) + ->getPreprocessors()[0]; +} + +template +DataType SQ8A8FromFloat(float value) { + if constexpr (std::is_same_v) { + return vecsim_types::FP32_to_FP16(value); + } else { + return value; + } +} + +template +float SQ8A8ToFloat(DataType value) { + if constexpr (std::is_same_v) { + return vecsim_types::FP16_to_FP32(value); + } else { + return value; + } +} + +template +std::vector SQ8A8RandomVector(size_t dim, int seed, float min_val, float max_val) { + std::vector values(dim); + test_utils::populate_float_vec(values.data(), dim, seed, min_val, max_val); + std::vector out(dim); + for (size_t i = 0; i < dim; i++) { + out[i] = SQ8A8FromFloat(values[i]); + } + return out; +} + +template +std::vector SQ8A8Widen(const std::vector &v) { + std::vector out(v.size()); + for (size_t i = 0; i < v.size(); i++) { + out[i] = SQ8A8ToFloat(v[i]); + } + return out; +} + +double SQ8A8Const(const void *query, size_t dim, size_t index) { + return load_unaligned(static_cast(query) + sq8_a8::consts_offset(dim) + + index * sizeof(double)); +} + +// The components the factory builds for an SQ8 IP index: the A8 classes where expect_a8, +// otherwise the asymmetric query for FP32 and FP16 alike, exactly as below the gate. +template +void verifySQ8IPFactoryPath(size_t dim, const void *arch_opt, bool expect_a8) { + SCOPED_TRACE(::testing::Message() + << "dim=" << dim << " mean=" << WithMean + << " fp16=" << (std::is_same_v) << " forced_off=" + << (arch_opt != nullptr)); + auto allocator = VecSimAllocator::newVecsimAllocator(); + std::vector mean(dim, 0.125f); + auto components = CreateSQ8IndexComponents( + allocator, dim, WithMean ? mean.data() : nullptr, arch_opt); + PreprocessorInterface &pp = *SQ8FactoryPreprocessor(components); + IndexCalculatorInterface &calc = *components.indexCalculator; + + using A8Preprocessor = SQ8A8QueryPreprocessor; + EXPECT_EQ(dynamic_cast(&pp) != nullptr, expect_a8); + if constexpr (WithMean) { + // applySQ8Mean's casts hold on every path. + EXPECT_NE((dynamic_cast *>(&pp)), + nullptr); + EXPECT_NE( + (dynamic_cast *>(&calc)), + nullptr); + } else { + EXPECT_EQ(typeid(calc), typeid(DistanceCalculatorCommon)); + } + EXPECT_TRUE(calc.getDistanceDispatch(DistanceMode::StoredToQuery).isValid()); + EXPECT_TRUE(calc.getDistanceDispatch(DistanceMode::StoredToStored).isValid()); + + const std::vector y = SQ8A8RandomVector(dim, 3, -1.0f, 1.0f); + { + auto query = components.preprocessors->preprocessQuery(y.data(), dim * sizeof(DataType)); + auto storage = + components.preprocessors->preprocessForStorage(y.data(), dim * sizeof(DataType)); + if (expect_a8) { + EXPECT_EQ(typeid(pp), typeid(A8Preprocessor)); + if constexpr (WithMean) { + EXPECT_EQ( + typeid(calc), + typeid(SQ8A8QueryDistanceCalculatorWithNorm)); + } + EXPECT_EQ(components.preprocessors->getQueryAlignment(), sq8_a8::blob_alignment); + } else { + EXPECT_EQ(typeid(pp), typeid(QuantPreprocessor)); + if constexpr (WithMean) { + EXPECT_EQ(typeid(calc), + typeid(DistanceCalculatorWithNorm)); + } + // The query is the raw input vector followed by its FP32 metadata. + EXPECT_EQ(std::memcmp(query.get(), y.data(), dim * sizeof(DataType)), 0); + } + } + delete components.preprocessors; + delete components.indexCalculator; +} + +template +void verifySQ8IPFactoryPaths() { + const bool host_a8 = HostHasA8Queries(); + const auto off = CpuFeaturesOff(); + for (const size_t dim : {spaces::SQ8_A8_MIN_DIM - 1, spaces::SQ8_A8_MIN_DIM, size_t{1024}}) { + const bool a8 = host_a8 && dim >= spaces::SQ8_A8_MIN_DIM; + verifySQ8IPFactoryPath(dim, nullptr, a8); + verifySQ8IPFactoryPath(dim, nullptr, a8); + verifySQ8IPFactoryPath(dim, &off, false); + verifySQ8IPFactoryPath(dim, &off, false); + } +} + +} // namespace + +TEST(SQ8A8FactoryTest, ChoosesQueryPathByGate) { +#if defined(CPU_FEATURES_ARCH_X86_64) && defined(OPT_AVX512_F_BW_VL_VNNI) + const auto cpu = spaces::getCpuOptimizationFeatures(); + EXPECT_EQ(HostHasA8Queries(), + cpu.avx512f && cpu.avx512bw && cpu.avx512vl && cpu.avx512vnni ? true : false); +#endif + verifySQ8IPFactoryPaths(); + verifySQ8IPFactoryPaths(); +} + +// L2 and SQ4 never take the A8 path. +TEST(SQ8A8FactoryTest, L2AndSQ4KeepAsymmetricQueriesAboveTheGate) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + constexpr size_t dim = 1024; + std::vector mean(dim, 0.125f); + const float *const mean_ptrs[] = {nullptr, mean.data()}; + for (const float *mean_ptr : mean_ptrs) { + auto l2 = CreateSQ8IndexComponents(allocator, dim, mean_ptr); + auto l2_fp16 = CreateSQ8IndexComponents( + allocator, dim, mean_ptr); + auto sq4 = CreateQuantizedIndexComponents( + allocator, dim, mean_ptr); + const PreprocessorInterface &l2_pp = *SQ8FactoryPreprocessor(l2); + const PreprocessorInterface &l2_fp16_pp = *SQ8FactoryPreprocessor(l2_fp16); + const PreprocessorInterface &sq4_pp = *SQ8FactoryPreprocessor(sq4); + if (mean_ptr) { + EXPECT_EQ(typeid(l2_pp), typeid(QuantPreprocessor)); + EXPECT_EQ(typeid(l2_fp16_pp), + typeid(QuantPreprocessor)); + EXPECT_EQ(typeid(sq4_pp), + typeid(QuantPreprocessor)); + } else { + EXPECT_EQ(typeid(l2_pp), typeid(QuantPreprocessor)); + EXPECT_EQ(typeid(l2_fp16_pp), + typeid(QuantPreprocessor)); + EXPECT_EQ(typeid(sq4_pp), + typeid(QuantPreprocessor)); + } + delete l2.preprocessors; + delete l2.indexCalculator; + delete l2_fp16.preprocessors; + delete l2_fp16.indexCalculator; + delete sq4.preprocessors; + delete sq4.indexCalculator; + } +} + +namespace { + +// The A8 query blob through the container: the encoder's codes, zeroed padding and constants, +// 64-byte alignment, and preprocess() returning the storage blob and a separate A8 blob, both +// released with the processed blobs. +template +void verifySQ8A8QueryBlob(size_t dim) { + SCOPED_TRACE(::testing::Message() << "dim=" << dim << " mean=" << WithMean << " fp16=" + << (std::is_same_v)); + auto allocator = VecSimAllocator::newVecsimAllocator(); + const std::vector mean = SQ8A8Widen(SQ8A8RandomVector(dim, 5, -0.1f, 0.1f)); + const float *mean_ptr = WithMean ? mean.data() : nullptr; + auto components = CreateSQ8IndexComponents(allocator, dim, mean_ptr); + const std::vector y = SQ8A8RandomVector(dim, 6, -1.0f, 1.0f); + const std::vector original_y = y; + const size_t query_bytes = sq8_a8::blob_bytes_count(dim); + const size_t storage_bytes = sq8::storage_bytes_count(dim); + + // The scalar encoder is the reference; the production encoder may be the AVX-512 one. + std::vector expected(query_bytes); + spaces::quantized::SQ8_A8_EncodeQuery(y.data(), mean_ptr, dim, expected.data()); + + const uint64_t baseline = allocator->getAllocationSize(); + { + auto query = components.preprocessors->preprocessQuery(y.data(), dim * sizeof(DataType)); + auto storage = + components.preprocessors->preprocessForStorage(y.data(), dim * sizeof(DataType)); + auto paired = components.preprocessors->preprocess(y.data(), dim * sizeof(DataType)); + ASSERT_NE(query.get(), nullptr); + EXPECT_EQ(reinterpret_cast(query.get()) % sq8_a8::blob_alignment, 0u); + const auto *bytes = static_cast(query.get()); + // Codes and the zeroed padding. + EXPECT_EQ(std::memcmp(bytes, expected.data(), sq8_a8::consts_offset(dim)), 0); + for (size_t i = dim; i < sq8_a8::consts_offset(dim); i++) { + EXPECT_EQ(bytes[i], 0) << "padding byte " << i; + } + for (size_t index : {sq8_a8::QD, sq8_a8::QM, sq8_a8::P}) { + EXPECT_EQ( + std::memcmp(bytes + sq8_a8::consts_offset(dim) + index * sizeof(double), + expected.data() + sq8_a8::consts_offset(dim) + index * sizeof(double), + sizeof(double)), + 0) + << "constant " << index; + } + const double s = SQ8A8Const(expected.data(), dim, sq8_a8::QD); + EXPECT_NEAR(SQ8A8Const(bytes, dim, sq8_a8::QS), + SQ8A8Const(expected.data(), dim, sq8_a8::QS), + 1e-12 * (std::fabs(SQ8A8Const(expected.data(), dim, sq8_a8::QS)) + s)); + EXPECT_NEAR(SQ8A8Const(bytes, dim, sq8_a8::QK), + SQ8A8Const(expected.data(), dim, sq8_a8::QK), + 1e-12 * 128.0 * s * s * static_cast(dim)); + if constexpr (WithMean) { + double y_mean_ip = 0; + for (size_t i = 0; i < dim; i++) { + y_mean_ip += static_cast(mean[i]) * SQ8A8ToFloat(y[i]); + } + EXPECT_NEAR(SQ8A8Const(bytes, dim, sq8_a8::Y_MEAN_IP), y_mean_ip, + 1e-12 * static_cast(dim)); + } else { + EXPECT_EQ(SQ8A8Const(bytes, dim, sq8_a8::Y_MEAN_IP), 0.0); + } + + // preprocess() returns two blobs: the storage blob and an identical A8 query blob. + ASSERT_NE(paired.getStorageBlob(), nullptr); + ASSERT_NE(paired.getQueryBlob(), nullptr); + EXPECT_NE(paired.getStorageBlob(), paired.getQueryBlob()); + EXPECT_EQ(std::memcmp(paired.getStorageBlob(), storage.get(), storage_bytes), 0); + EXPECT_EQ(std::memcmp(paired.getQueryBlob(), query.get(), query_bytes), 0); + EXPECT_EQ(reinterpret_cast(paired.getQueryBlob()) % sq8_a8::blob_alignment, 0u); + if (const auto alignment = components.preprocessors->getStorageAlignment()) { + EXPECT_EQ(reinterpret_cast(paired.getStorageBlob()) % alignment, 0u); + } + EXPECT_GT(allocator->getAllocationSize(), baseline); + } + // Both blobs of preprocess() (and the single ones) were released. + EXPECT_EQ(allocator->getAllocationSize(), baseline); + EXPECT_EQ(std::memcmp(y.data(), original_y.data(), dim * sizeof(DataType)), 0); + delete components.preprocessors; + delete components.indexCalculator; +} + +} // namespace + +TEST(SQ8A8FactoryTest, QueryBlobLayoutAlignmentAndOwnership) { + if (!HostHasA8Queries()) { + GTEST_SKIP() << "No A8 query kernel for this CPU"; + } + for (const size_t dim : {size_t{768}, size_t{1001}}) { + verifySQ8A8QueryBlob(dim); + verifySQ8A8QueryBlob(dim); + verifySQ8A8QueryBlob(dim); + verifySQ8A8QueryBlob(dim); + } +} + +namespace { + +// Query distances are the A8 kernel plus, with a mean, the stored-to-stored correction; they stay +// within the bound derived from the two quantization steps of the true distance. Stored-to-stored +// distances are those of the asymmetric path. +template +void verifySQ8A8Distances(size_t dim) { + SCOPED_TRACE(::testing::Message() << "dim=" << dim << " mean=" << WithMean << " fp16=" + << (std::is_same_v)); + auto allocator = VecSimAllocator::newVecsimAllocator(); + const std::vector mean = SQ8A8Widen(SQ8A8RandomVector(dim, 7, -0.1f, 0.1f)); + const float *mean_ptr = WithMean ? mean.data() : nullptr; + const auto off = CpuFeaturesOff(); + auto a8 = CreateSQ8IndexComponents(allocator, dim, mean_ptr); + auto fallback = + CreateSQ8IndexComponents(allocator, dim, mean_ptr, &off); + const auto query_dispatch = + a8.indexCalculator->getDistanceDispatch(DistanceMode::StoredToQuery); + const auto stored_dispatch = + a8.indexCalculator->getDistanceDispatch(DistanceMode::StoredToStored); + ASSERT_TRUE(query_dispatch.isValid()); + ASSERT_TRUE(stored_dispatch.isValid()); + float mean_sum_squares = 0.0f; + for (float v : mean) { + mean_sum_squares += v * v; + } + + const std::vector y = SQ8A8RandomVector(dim, 8, -1.0f, 1.0f); + const std::vector y_f = SQ8A8Widen(y); + const size_t storage_bytes = sq8::storage_bytes_count(dim); + auto query = a8.preprocessors->preprocessQuery(y.data(), dim * sizeof(DataType)); + auto y_storage = a8.preprocessors->preprocessForStorage(y.data(), dim * sizeof(DataType)); + for (int seed : {9, 10, 11}) { + const std::vector x = SQ8A8RandomVector(dim, seed, -1.0f, 1.0f); + const std::vector x_f = SQ8A8Widen(x); + auto stored = a8.preprocessors->preprocessForStorage(x.data(), dim * sizeof(DataType)); + auto fallback_stored = + fallback.preprocessors->preprocessForStorage(x.data(), dim * sizeof(DataType)); + ASSERT_EQ(std::memcmp(stored.get(), fallback_stored.get(), storage_bytes), 0); + + const float got = a8.indexCalculator->calcDistanceForQuery(stored.get(), query.get(), dim); + // The A8 kernel is bit-identical to the scalar reference. + const float kernel = SQ8_A8_InnerProduct(stored.get(), query.get(), dim); + if constexpr (WithMean) { + const float x_mean_ip = load_unaligned( + static_cast(stored.get()) + sq8::data_bytes_count(dim) + + sq8::mean_ip_index() * sizeof(float)); + const double y_mean_ip = SQ8A8Const(query.get(), dim, sq8_a8::Y_MEAN_IP); + EXPECT_EQ(got, static_cast(static_cast(kernel) - x_mean_ip - y_mean_ip + + mean_sum_squares)); + } else { + EXPECT_EQ(got, kernel); + } + EXPECT_EQ(query_dispatch(stored.get(), query.get(), dim), got); + + double ip = 0; + for (size_t i = 0; i < dim; i++) { + ip += static_cast(x_f[i]) * y_f[i]; + } + const double tolerance = SQ8A8DistanceTolerance(x_f.data(), y_f.data(), mean_ptr, dim); + EXPECT_NEAR(got, 1.0 - ip, tolerance) << "seed " << seed; + + // Stored-to-stored is unchanged from the asymmetric path. + const float sym = a8.indexCalculator->calcDistance(stored.get(), y_storage.get(), dim); + EXPECT_EQ(sym, fallback.indexCalculator->calcDistance(fallback_stored.get(), + y_storage.get(), dim)); + EXPECT_EQ(stored_dispatch(stored.get(), y_storage.get(), dim), sym); + } + query.reset(); + y_storage.reset(); + for (auto *components : {&a8, &fallback}) { + delete components->preprocessors; + delete components->indexCalculator; + } +} + +} // namespace + +TEST(SQ8A8FactoryTest, DistancesAndDispatch) { + if (!HostHasA8Queries()) { + GTEST_SKIP() << "No A8 query kernel for this CPU"; + } + for (const size_t dim : {size_t{768}, size_t{1024}}) { + verifySQ8A8Distances(dim); + verifySQ8A8Distances(dim); + verifySQ8A8Distances(dim); + verifySQ8A8Distances(dim); + } +} + +// A tiered index creates its components with a zero mean and installs the learned one later +// through applySQ8Mean. Queries encoded after that, including through a dispatch cached before +// it, match components built with the mean from the start. +TEST(SQ8A8FactoryTest, DeferredMeanTakesEffect) { + if (!HostHasA8Queries()) { + GTEST_SKIP() << "No A8 query kernel for this CPU"; + } + const auto run = [](auto tag) { + using DataType = decltype(tag); + constexpr size_t dim = 768; + SCOPED_TRACE(::testing::Message() + << "fp16=" << (std::is_same_v)); + auto allocator = VecSimAllocator::newVecsimAllocator(); + const std::vector zero(dim, 0.0f); + const std::vector mean = SQ8A8Widen(SQ8A8RandomVector(dim, 12, -0.2f, 0.2f)); + auto deferred = + CreateSQ8IndexComponents(allocator, dim, zero.data()); + auto direct = + CreateSQ8IndexComponents(allocator, dim, mean.data()); + const auto cached = + deferred.indexCalculator->getDistanceDispatch(DistanceMode::StoredToQuery); + ASSERT_NE((dynamic_cast *>( + SQ8FactoryPreprocessor(deferred))), + nullptr); + applySQ8Mean(*deferred.preprocessors, *deferred.indexCalculator, + mean); + const std::vector x = SQ8A8RandomVector(dim, 13, -1.0f, 1.0f); + const std::vector y = SQ8A8RandomVector(dim, 14, -1.0f, 1.0f); + { + auto x_deferred = + deferred.preprocessors->preprocessForStorage(x.data(), dim * sizeof(DataType)); + auto y_deferred = + deferred.preprocessors->preprocessQuery(y.data(), dim * sizeof(DataType)); + auto x_direct = + direct.preprocessors->preprocessForStorage(x.data(), dim * sizeof(DataType)); + auto y_direct = direct.preprocessors->preprocessQuery(y.data(), dim * sizeof(DataType)); + EXPECT_EQ(std::memcmp(x_deferred.get(), x_direct.get(), + sq8::storage_bytes_count(dim)), + 0); + EXPECT_EQ(std::memcmp(y_deferred.get(), y_direct.get(), sq8_a8::blob_bytes_count(dim)), + 0); + EXPECT_NE(SQ8A8Const(y_deferred.get(), dim, sq8_a8::Y_MEAN_IP), 0.0); + const float expected = + direct.indexCalculator->calcDistanceForQuery(x_direct.get(), y_direct.get(), dim); + EXPECT_EQ(cached(x_deferred.get(), y_deferred.get(), dim), expected); + EXPECT_EQ(deferred.indexCalculator->calcDistanceForQuery(x_deferred.get(), + y_deferred.get(), dim), + expected); + } + for (auto *components : {&deferred, &direct}) { + delete components->preprocessors; + delete components->indexCalculator; + } + }; + run(float{}); + run(vecsim_types::float16{}); +} diff --git a/tests/unit/test_hnsw_sq8.cpp b/tests/unit/test_hnsw_sq8.cpp index 8708c1f1c..de9091770 100644 --- a/tests/unit/test_hnsw_sq8.cpp +++ b/tests/unit/test_hnsw_sq8.cpp @@ -11,6 +11,7 @@ #include "VecSim/algorithms/hnsw/hnsw_single.h" #include "VecSim/algorithms/hnsw/hnsw_multi.h" #include "VecSim/index_factories/hnsw_factory.h" +#include "VecSim/index_factories/components/components_factory.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" #include "VecSim/vec_sim.h" @@ -26,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -2064,3 +2066,409 @@ TEST(SQ8TieredHNSWTest, MeanCenteredFP16L2BufferedAndMigratedValues) { verify_queries(); } } + +/* ======================== SQ8 IP with A8 queries ========================= */ + +namespace { + +bool HostHasA8Queries() { return spaces::IP_SQ8_A8_GetDistFunc(spaces::SQ8_A8_MIN_DIM) != nullptr; } + +template +DataType A8FromFloat(float value) { + if constexpr (std::is_same_v) { + return vecsim_types::FP32_to_FP16(value); + } else { + return value; + } +} + +template +float A8ToFloat(DataType value) { + if constexpr (std::is_same_v) { + return vecsim_types::FP16_to_FP32(value); + } else { + return value; + } +} + +template +constexpr VecSimType A8IndexType() { + return std::is_same_v ? VecSimType_FLOAT16 + : VecSimType_FLOAT32; +} + +// Vectors whose distances to the query are far apart compared with the A8 error: vector k is +// t_k * q + sqrt(1 - t_k^2) * r_k + o, with q a random unit query, r_k a random unit vector +// orthogonal to q, t_k evenly spaced over [-1, 1] and the common offset o = 0.3 q + 0.5 w (w a +// random unit vector orthogonal to q), so IP(x_k, q) = t_k + 0.3. The offset gives a learned mean +// a large , so a wrong mean correction shows. The values are stored in the index data +// type, and the references use them widened back to FP32. +template +struct A8SearchData { + std::vector query; + std::vector> vectors; +}; + +template +A8SearchData MakeA8SearchData(size_t dim, size_t count, unsigned seed) { + std::mt19937 gen(seed); + std::uniform_real_distribution dist(-1.0, 1.0); + const auto random_unit = [&](const std::vector *orthogonal_to) { + std::vector v(dim); + for (auto &value : v) { + value = dist(gen); + } + if (orthogonal_to) { + double dot = 0; + for (size_t i = 0; i < dim; i++) { + dot += v[i] * (*orthogonal_to)[i]; + } + for (size_t i = 0; i < dim; i++) { + v[i] -= dot * (*orthogonal_to)[i]; + } + } + double norm = 0; + for (double value : v) { + norm += value * value; + } + for (auto &value : v) { + value /= std::sqrt(norm); + } + return v; + }; + const std::vector q = random_unit(nullptr); + A8SearchData data; + for (double value : q) { + data.query.push_back(A8FromFloat(static_cast(value))); + } + const std::vector w = random_unit(&q); + for (size_t k = 0; k < count; k++) { + const double t = -1.0 + 2.0 * static_cast(k) / static_cast(count - 1); + const std::vector r = random_unit(&q); + std::vector x(dim); + for (size_t i = 0; i < dim; i++) { + const double offset = 0.3 * q[i] + 0.5 * w[i]; + x[i] = A8FromFloat( + static_cast(t * q[i] + std::sqrt(1 - t * t) * r[i] + offset)); + } + data.vectors.push_back(std::move(x)); + } + return data; +} + +template +std::vector A8Widen(const std::vector &v) { + std::vector out(v.size()); + for (size_t i = 0; i < v.size(); i++) { + out[i] = A8ToFloat(v[i]); + } + return out; +} + +// What a Cosine index stores: the vector normalized, rounded to the data type. +template +std::vector A8Normalized(const std::vector &v) { + double norm = 0; + for (DataType value : v) { + norm += static_cast(A8ToFloat(value)) * A8ToFloat(value); + } + std::vector out(v.size()); + for (size_t i = 0; i < v.size(); i++) { + out[i] = + A8ToFloat(A8FromFloat(static_cast(A8ToFloat(v[i]) / std::sqrt(norm)))); + } + return out; +} + +struct A8Expected { + double distance; + double tolerance; +}; + +// The true distance of a stored vector to the query, with the A8 bound (unit_test_utils.h). +// extra_relative covers what the bound does not: the data-type rounding of Cosine's normalization. +A8Expected ExpectedA8Distance(const std::vector &x, const std::vector &y, + const float *mean, double extra_relative) { + double ip = 0, ip_abs = 0; + for (size_t i = 0; i < x.size(); i++) { + ip += static_cast(x[i]) * y[i]; + ip_abs += std::fabs(static_cast(x[i]) * y[i]); + } + return {1.0 - ip, + SQ8A8DistanceTolerance(x.data(), y.data(), mean, x.size()) + extra_relative * ip_abs}; +} + +// Top-K, GetDistanceFrom_Unsafe, range query and batch iterator against the references. A +// multi-value label scores its closest vector. The spacing of the data keeps the ranking +// unambiguous at these tolerances, which is checked first. +void VerifyA8Searches(VecSimIndex *index, const void *query, const std::vector &labels, + const std::vector &expected) { + std::map best; + for (size_t k = 0; k < labels.size(); k++) { + auto it = best.find(labels[k]); + if (it == best.end() || expected[k].distance < it->second.distance) { + best[labels[k]] = expected[k]; + } + } + std::vector order; + for (const auto &[label, value] : best) { + order.push_back(label); + } + std::sort(order.begin(), order.end(), + [&](size_t a, size_t b) { return best[a].distance < best[b].distance; }); + for (size_t r = 0; r + 1 < order.size(); r++) { + const A8Expected &a = best[order[r]], &b = best[order[r + 1]]; + ASSERT_GT(b.distance - a.distance, a.tolerance + b.tolerance) << "rank " << r; + } + + const auto verify = [&](size_t id, double score, size_t rank) { + ASSERT_LT(rank, order.size()); + EXPECT_EQ(id, order[rank]) << "rank " << rank; + EXPECT_NEAR(score, best[id].distance, best[id].tolerance) << "label " << id; + }; + const size_t n = order.size(); + runTopKSearchTest(index, query, n, verify); + for (const auto &[label, value] : best) { + EXPECT_NEAR(VecSimIndex_GetDistanceFrom_Unsafe(index, label, query), value.distance, + value.tolerance) + << "label " << label; + } + // The closest three labels; the radius splits the gap to the fourth in proportion to the two + // tolerances, so it clears both. + const A8Expected &in = best[order[2]], &out = best[order[3]]; + const double radius = + in.distance + (out.distance - in.distance) * in.tolerance / (in.tolerance + out.tolerance); + runRangeQueryTest(index, query, radius, verify, 3, BY_SCORE); + VecSimBatchIterator *iterator = VecSimBatchIterator_New(index, query, nullptr); + ASSERT_NE(iterator, nullptr); + runBatchIteratorSearchTest(iterator, n, verify); + VecSimBatchIterator_Free(iterator); +} + +// The query distance of each vector as components built by the factory compute it: the A8 path on +// this CPU, or with arch_opt the path of a CPU without AVX-512. +template +std::vector A8ComponentDistances(const A8SearchData &data, const float *mean, + const void *arch_opt) { + const size_t dim = data.query.size(); + auto allocator = VecSimAllocator::newVecsimAllocator(); + auto components = + CreateSQ8IndexComponents(allocator, dim, mean, arch_opt); + std::vector distances; + { + auto query = + components.preprocessors->preprocessQuery(data.query.data(), dim * sizeof(DataType)); + for (const auto &x : data.vectors) { + auto stored = + components.preprocessors->preprocessForStorage(x.data(), dim * sizeof(DataType)); + distances.push_back( + components.indexCalculator->calcDistanceForQuery(stored.get(), query.get(), dim)); + } + } + delete components.preprocessors; + delete components.indexCalculator; + return distances; +} + +// The index scores with the A8 path: GetDistanceFrom_Unsafe equals the A8 components' distance +// bit for bit (a multi-value label takes its closest vector), and differs from the path of a CPU +// without AVX-512 for some label. +template +void ExpectIndexUsesA8Queries(VecSimIndex *index, const A8SearchData &data, + const std::vector &labels, const float *mean) { + auto features_off = spaces::getCpuOptimizationFeatures(); + std::memset(&features_off, 0, sizeof(features_off)); + const std::vector a8 = A8ComponentDistances(data, mean, nullptr); + const std::vector fallback = A8ComponentDistances(data, mean, &features_off); + std::map> best; + for (size_t k = 0; k < labels.size(); k++) { + auto [it, inserted] = best.try_emplace(labels[k], a8[k], fallback[k]); + if (!inserted) { + it->second.first = std::min(it->second.first, a8[k]); + it->second.second = std::min(it->second.second, fallback[k]); + } + } + bool differs = false; + for (const auto &[label, distances] : best) { + EXPECT_EQ(VecSimIndex_GetDistanceFrom_Unsafe(index, label, data.query.data()), + distances.first) + << "label " << label; + differs = differs || distances.first != distances.second; + } + EXPECT_TRUE(differs) << "the A8 and fallback paths gave identical distances"; +} + +constexpr size_t kA8VectorCount = 9; + +template +void RunA8StandaloneTest(size_t dim, bool with_mean, bool multi) { + SCOPED_TRACE(::testing::Message() + << "dim=" << dim << " fp16=" + << (std::is_same_v) << " mean=" << with_mean + << " multi=" << multi); + const auto data = MakeA8SearchData(dim, kA8VectorCount, static_cast(dim)); + std::vector mean(dim); + std::mt19937 gen(7); + std::uniform_real_distribution mean_dist(-0.05f, 0.05f); + for (auto &value : mean) { + value = mean_dist(gen); + } + HNSWParams params = {.type = A8IndexType(), + .dim = dim, + .metric = VecSimMetric_IP, + .multi = multi, + .initialCapacity = kA8VectorCount, + .M = 16, + .efConstruction = 100, + .efRuntime = kA8VectorCount, + .quantType = VecSimQuant_SQ8, + .quantParams = with_mean ? mean.data() : nullptr}; + VecSimParams vecsim_params = CreateParams(params); + std::unique_ptr index(VecSimIndex_New(&vecsim_params), + VecSimIndex_Free); + ASSERT_NE(index, nullptr); + + std::vector labels; + std::vector expected; + const std::vector query = A8Widen(data.query); + for (size_t k = 0; k < kA8VectorCount; k++) { + labels.push_back(multi ? k / 2 : k); + ASSERT_EQ(VecSimIndex_AddVector(index.get(), data.vectors[k].data(), labels.back()), 1); + expected.push_back(ExpectedA8Distance(A8Widen(data.vectors[k]), query, + with_mean ? mean.data() : nullptr, 0.0)); + } + const auto original_query = data.query; + ExpectIndexUsesA8Queries(index.get(), data, labels, with_mean ? mean.data() : nullptr); + VerifyA8Searches(index.get(), data.query.data(), labels, expected); + EXPECT_EQ(std::memcmp(data.query.data(), original_query.data(), dim * sizeof(DataType)), 0); +} + +// A tiered index learns its mean from the first vectors, which wait in the flat buffer; the +// backend is built with a zero mean and gets the learned one before any vector migrates. Queries +// are checked with every vector buffered, with some migrated and with all migrated. +template +void RunA8TieredLateMeanTest(size_t dim, VecSimMetric metric, bool multi) { + SCOPED_TRACE(::testing::Message() + << "dim=" << dim << " fp16=" + << (std::is_same_v) << " metric=" << metric + << " multi=" << multi); + const auto data = + MakeA8SearchData(dim, kA8VectorCount, static_cast(dim) + 1); + tieredIndexMock mock_thread_pool; + HNSWParams hnsw_params = {.type = A8IndexType(), + .dim = dim, + .metric = metric, + .multi = multi, + .M = 16, + .efConstruction = 100, + .efRuntime = kA8VectorCount, + .quantType = VecSimQuant_SQ8}; + VecSimParams primary_params = CreateParams(hnsw_params); + TieredIndexParams tiered_params = { + .jobQueue = &mock_thread_pool.jobQ, + .jobQueueCtx = mock_thread_pool.ctx, + .submitCb = tieredIndexMock::submit_callback, + .flatBufferLimit = kA8VectorCount, + .primaryIndexParams = &primary_params, + .specificParams = {TieredHNSWParams{.QuantNormalizationSetSize = kA8VectorCount}}}; + VecSimParams params = CreateParams(tiered_params); + auto *index = VecSimIndex_New(¶ms); + if (!index) { + mock_thread_pool.reset_ctx(); + FAIL() << "the tiered SQ8 index must construct"; + } + mock_thread_pool.ctx->index_strong_ref.reset(index); + const auto tier_sizes = [index] { + const auto info = VecSimIndex_DebugInfo(index); + return std::pair{info.tieredInfo.frontendCommonInfo.indexSize, + info.tieredInfo.backendCommonInfo.indexSize}; + }; + + // What the index stores (normalized for Cosine) and the mean it learns from it. + std::vector> stored; + std::vector sum(dim, 0.0); + for (const auto &x : data.vectors) { + stored.push_back(metric == VecSimMetric_Cosine ? A8Normalized(x) : A8Widen(x)); + for (size_t i = 0; i < dim; i++) { + sum[i] += stored.back()[i]; + } + } + std::vector mean(dim); + for (size_t i = 0; i < dim; i++) { + mean[i] = static_cast(sum[i] / kA8VectorCount); + } + const std::vector query = + metric == VecSimMetric_Cosine ? A8Normalized(data.query) : A8Widen(data.query); + // Cosine normalizes in the data type, which the references only reproduce to about one unit + // in the last place of each operand: FP16 values may differ by one ulp (2^-10 relative) in the + // vector and in the query, FP32 ones by the rounding of the FP32 norm. + const double normalization_error = metric != VecSimMetric_Cosine ? 0.0 + : std::is_same_v ? 0x1p-9 + : 1e-4; + std::vector labels; + std::vector expected; + for (size_t k = 0; k < kA8VectorCount; k++) { + labels.push_back(multi ? k / 2 : k); + ASSERT_EQ(VecSimIndex_AddVector(index, data.vectors[k].data(), labels.back()), 1); + expected.push_back(ExpectedA8Distance(stored[k], query, mean.data(), normalization_error)); + } + // Training finished with the last vector, and every insertion job waits. + ASSERT_EQ(mock_thread_pool.jobQ.size(), kA8VectorCount); + { + SCOPED_TRACE("all buffered"); + VerifyA8Searches(index, data.query.data(), labels, expected); + } + for (size_t job = 0; job < kA8VectorCount / 2; job++) { + mock_thread_pool.thread_iteration(); + } + ASSERT_EQ(tier_sizes().second, kA8VectorCount / 2); + { + SCOPED_TRACE("partly migrated"); + VerifyA8Searches(index, data.query.data(), labels, expected); + } + while (!mock_thread_pool.jobQ.empty()) { + mock_thread_pool.thread_iteration(); + } + ASSERT_EQ(tier_sizes(), (std::pair{0, kA8VectorCount})); + { + SCOPED_TRACE("all migrated"); + VerifyA8Searches(index, data.query.data(), labels, expected); + } + // The backend scores with the A8 path and the learned mean. Only for IP: with Cosine the + // references cannot reproduce the index's normalized values bit for bit. + if (metric == VecSimMetric_IP) { + ExpectIndexUsesA8Queries(index, data, labels, mean.data()); + } +} + +} // namespace + +// The A8 query path (dim >= SQ8_A8_MIN_DIM on a CPU with AVX-512 VNNI) through the public API. +// 777 is not a multiple of 64, so the kernel and encoder tails run too. +TEST(HNSWSQ8A8QueryTest, PublicAPIsAboveTheGate) { + if (!HostHasA8Queries()) { + GTEST_SKIP() << "No A8 query kernel for this CPU"; + } + for (const size_t dim : {size_t{768}, size_t{777}, size_t{1024}}) { + for (bool with_mean : {false, true}) { + for (bool multi : {false, true}) { + RunA8StandaloneTest(dim, with_mean, multi); + RunA8StandaloneTest(dim, with_mean, multi); + } + } + } +} + +TEST(SQ8TieredHNSWTest, A8QueriesWithLateMeanBufferedAndMigrated) { + if (!HostHasA8Queries()) { + GTEST_SKIP() << "No A8 query kernel for this CPU"; + } + for (const size_t dim : {size_t{768}, size_t{1024}}) { + for (auto metric : {VecSimMetric_IP, VecSimMetric_Cosine}) { + for (bool multi : {false, true}) { + RunA8TieredLateMeanTest(dim, metric, multi); + RunA8TieredLateMeanTest(dim, metric, multi); + } + } + } +} diff --git a/tests/unit/unit_test_utils.h b/tests/unit/unit_test_utils.h index b6e3538f3..915de07df 100644 --- a/tests/unit/unit_test_utils.h +++ b/tests/unit/unit_test_utils.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include "VecSim/vec_sim.h" #include "VecSim/algorithms/hnsw/hnsw_tiered.h" @@ -265,6 +267,55 @@ inline void ComputeSQ8Quantization(const float *original_blob, size_t dim, uint8 std::memcpy(output + dim, metadata, sizeof(metadata)); } +/** + * Error bound of an SQ8 IP distance whose query is an A8 query (vecsim_types::sq8_a8_query), + * derived from the two quantization steps rather than picked. + * + * x is the stored vector and y the query (FP32 values of the index's input), mean the index mean + * or nullptr; the index quantizes both centered, x' = x - mean and y' = y - mean. With the stored + * step dx = range(x') / 255 and the query step s = range(y') / 255, and u = x' - y': + * stored rounding: |IP(x', y') - IP(xh', y')| <= dx / 2 * sum |y'_i| + * query rounding: |IP(xh', y') - IP_A8| <= s / 2 * sum (|u_i - mean(u)| + dx + s) + * The A8 constants cancel the query's rounding except for its error e_i (|e_i| <= s / 2) times the + * deviation of xh'_i - yh'_i from its mean, which is at most |u_i - mean(u)| + dx + s. The rest is + * the FP32 rounding of the stored metadata, of x_mean_ip and of the final distance. + */ +inline double SQ8A8DistanceTolerance(const float *x, const float *y, const float *mean, + size_t dim) { + std::vector xc(dim), yc(dim); + for (size_t i = 0; i < dim; i++) { + const double m = mean ? mean[i] : 0.0; + xc[i] = x[i] - m; + yc[i] = y[i] - m; + } + const auto [x_lo, x_hi] = std::minmax_element(xc.begin(), xc.end()); + const auto [y_lo, y_hi] = std::minmax_element(yc.begin(), yc.end()); + const double dx = (*x_hi - *x_lo) / 255.0; + const double s = (*y_hi - *y_lo) / 255.0; + double y_abs = 0, x_abs = 0, xy_abs = 0, mean_terms = 0, u_mean = 0; + for (size_t i = 0; i < dim; i++) { + const double m = mean ? mean[i] : 0.0; + y_abs += std::fabs(yc[i]); + x_abs += std::fabs(xc[i]); + xy_abs += std::fabs(static_cast(x[i]) * y[i]); + mean_terms += std::fabs(x[i] * m) + std::fabs(y[i] * m) + m * m; + u_mean += xc[i] - yc[i]; + } + u_mean /= static_cast(dim); + double u_dev = 0; + for (size_t i = 0; i < dim; i++) { + u_dev += std::fabs(xc[i] - yc[i] - u_mean); + } + const double stored = dx / 2 * y_abs; + const double query = s / 2 * (u_dev + static_cast(dim) * (dx + s)); + const double y_max = std::max(std::fabs(*y_lo), std::fabs(*y_hi)); + // FP32 epsilon per summed term, generously: the stored SUM (times the query offset, at most + // max |y'|), the FP32 sums behind x_mean_ip and mean_sum_squares, and the result. + const double rounding = + 1e-5 + 1e-6 * (x_abs * y_max + xy_abs) + 2e-7 * static_cast(dim) * mean_terms; + return 1.01 * (stored + query) + rounding; +} + // TODO: Move all test_utils to this namespace namespace test_utils { size_t CalcVectorDataSize(VecSimIndex *index, VecSimType data_type);