From 3ec8d5871f1249bd207f5d69e081cfe63727e832 Mon Sep 17 00:00:00 2001 From: Alex Dimitrov Date: Thu, 12 Nov 2026 11:46:05 +0200 Subject: [PATCH 1/8] Add FP64 SQ8 quantization components and scalar distances --- .../components/components_factory.h | 22 ++ src/VecSim/spaces/CMakeLists.txt | 1 + .../spaces/computer/sq8_fp64_preprocessor.h | 127 ++++++++++++ src/VecSim/spaces/spaces.cpp | 36 ++++ src/VecSim/spaces/sq8_fp64.cpp | 63 ++++++ src/VecSim/spaces/sq8_fp64.h | 18 ++ src/VecSim/types/sq8_fp64.h | 40 ++++ tests/unit/CMakeLists.txt | 4 +- tests/unit/test_sq8_fp64_components.cpp | 129 ++++++++++++ tests/unit/test_sq8_fp64_distances.cpp | 189 ++++++++++++++++++ 10 files changed, 627 insertions(+), 2 deletions(-) create mode 100644 src/VecSim/spaces/computer/sq8_fp64_preprocessor.h create mode 100644 src/VecSim/spaces/sq8_fp64.cpp create mode 100644 src/VecSim/spaces/sq8_fp64.h create mode 100644 src/VecSim/types/sq8_fp64.h create mode 100644 tests/unit/test_sq8_fp64_components.cpp create mode 100644 tests/unit/test_sq8_fp64_distances.cpp diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index 39fc2486a..a0a8ae4c1 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -13,6 +13,7 @@ #include "VecSim/vec_sim_index.h" #include "VecSim/index_factories/components/preprocessors_factory.h" #include "VecSim/spaces/computer/calculator.h" +#include "VecSim/spaces/computer/sq8_fp64_preprocessor.h" template IndexComponents @@ -113,6 +114,27 @@ CreateSQ8IndexComponents(const std::shared_ptr &allocator, size mean_ptr); } +// Cosine storage and query inputs must already be normalized by the caller. +template +IndexComponents +CreateSQ8FP64IndexComponents(const std::shared_ptr &allocator, size_t dim) { + unsigned char storage_alignment = 0; + const auto symmetric = spaces::GetDistFunc( + Metric, dim, &storage_alignment); + unsigned char asymmetric_alignment = 0; + const auto asymmetric = spaces::GetDistFunc( + Metric, dim, &asymmetric_alignment); + storage_alignment = spaces::combineAlignments(storage_alignment, asymmetric_alignment); + auto *calculator = new (allocator) + DistanceCalculatorCommon(allocator, symmetric, asymmetric); + auto *container = new (allocator) + MultiPreprocessorsContainer(allocator, 0, storage_alignment); + auto *preprocessor = new (allocator) SQ8FP64Preprocessor(allocator, dim); + [[maybe_unused]] const int result = container->addPreprocessor(preprocessor); + assert(result != -1); + return {calculator, container}; +} + template size_t EstimateComponentsMemory(VecSimMetric metric, bool is_normalized) { size_t allocations_overhead = VecSimAllocator::getAllocationOverheadSize(); diff --git a/src/VecSim/spaces/CMakeLists.txt b/src/VecSim/spaces/CMakeLists.txt index eb8a15fdd..bfae49369 100644 --- a/src/VecSim/spaces/CMakeLists.txt +++ b/src/VecSim/spaces/CMakeLists.txt @@ -1,6 +1,7 @@ # Build non optimized code in a single project without architecture optimization flag. project(VectorSimilaritySpaces_no_optimization) add_library(VectorSimilaritySpaces_no_optimization + sq8_fp64.cpp L2/L2.cpp IP/IP.cpp ) diff --git a/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h b/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h new file mode 100644 index 000000000..82466832f --- /dev/null +++ b/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include "VecSim/spaces/computer/preprocessors.h" +#include "VecSim/types/sq8_fp64.h" + +// Cosine inputs must already be normalized by the caller. +template +class SQ8FP64Preprocessor : public PreprocessorInterface { + static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP || + Metric == VecSimMetric_Cosine); + +public: + SQ8FP64Preprocessor(std::shared_ptr allocator, size_t dim); + void preprocess(const void *original, void *&storage, void *&query, size_t &storage_size, + size_t &query_size, unsigned char storage_alignment, + unsigned char query_alignment) const override; + void preprocessForStorage(const void *original, void *&storage, size_t &storage_size, + unsigned char alignment) const override; + void preprocessQuery(const void *original, void *&query, size_t &query_size, + unsigned char alignment) const override; + void preprocessStorageInPlace(void *original, size_t input_size) const override; + +private: + static uint8_t to_code(double scaled); + void quantize(const double *input, uint8_t *output) const; + const size_t dim; +}; + +template +SQ8FP64Preprocessor::SQ8FP64Preprocessor(std::shared_ptr allocator, + size_t dim) + : PreprocessorInterface(allocator), dim(dim) { + assert(dim > 0); +} + +template +uint8_t SQ8FP64Preprocessor::to_code(double scaled) { + if (!(scaled > 0.0)) { + return 0; + } + + if (scaled >= vecsim_types::sq8_fp64::max_code) { + return vecsim_types::sq8_fp64::max_code; + } + + return static_cast(scaled + 0.5); +} + +template +void SQ8FP64Preprocessor::quantize(const double *input, uint8_t *output) const { + const auto [min_it, max_it] = std::minmax_element(input, input + dim); + const double minimum = *min_it; + const double range = *max_it - minimum; + const double delta = range == 0.0 ? 1.0 : range / vecsim_types::sq8_fp64::max_code; + double sum = 0.0; + double squares = 0.0; + for (size_t i = 0; i < dim; ++i) { + const uint8_t code = to_code((input[i] - minimum) / delta); + output[i] = code; + const double value = minimum + delta * code; + sum += value; + if constexpr (Metric == VecSimMetric_L2) { + squares += value * value; + } + } + const double metadata[] = {minimum, delta, sum, squares}; + std::memcpy(output + dim, metadata, + vecsim_types::sq8_fp64::storage_metadata_count() * sizeof(double)); +} + +template +void SQ8FP64Preprocessor::preprocess(const void *original, void *&storage, void *&query, + size_t &storage_size, size_t &query_size, + unsigned char storage_alignment, + unsigned char query_alignment) const { + assert(!storage && !query); + assert(&storage_size != &query_size); + preprocessForStorage(original, storage, storage_size, storage_alignment); + preprocessQuery(original, query, query_size, query_alignment); +} + +template +void SQ8FP64Preprocessor::preprocessForStorage(const void *original, void *&storage, + size_t &storage_size, + unsigned char alignment) const { + assert(!storage); + storage_size = vecsim_types::sq8_fp64::storage_bytes_count(dim); + storage = this->allocator->allocate_aligned(storage_size, alignment); + quantize(static_cast(original), static_cast(storage)); +} + +template +void SQ8FP64Preprocessor::preprocessQuery(const void *original, void *&query, + size_t &query_size, + unsigned char alignment) const { + assert(!query); + const auto *input = static_cast(original); + double sum = 0.0; + double squares = 0.0; + for (size_t i = 0; i < dim; ++i) { + sum += input[i]; + if constexpr (Metric == VecSimMetric_L2) { + squares += input[i] * input[i]; + } + } + query_size = vecsim_types::sq8_fp64::query_bytes_count(dim); + query = this->allocator->allocate_aligned(query_size, alignment); + std::memcpy(query, original, dim * sizeof(double)); + const double metadata[] = {sum, squares}; + std::memcpy(static_cast(query) + dim * sizeof(double), metadata, + vecsim_types::sq8_fp64::query_metadata_count() * sizeof(double)); +} + +template +void SQ8FP64Preprocessor::preprocessStorageInPlace(void *original, + size_t input_size) const { + assert(input_size >= vecsim_types::sq8_fp64::storage_bytes_count(dim)); + quantize(static_cast(original), static_cast(original)); +} diff --git a/src/VecSim/spaces/spaces.cpp b/src/VecSim/spaces/spaces.cpp index 1e62c0b5c..e5484ab38 100644 --- a/src/VecSim/spaces/spaces.cpp +++ b/src/VecSim/spaces/spaces.cpp @@ -9,6 +9,8 @@ #include "VecSim/types/bfloat16.h" #include "VecSim/types/float16.h" #include "VecSim/types/sq8.h" +#include "VecSim/types/sq8_fp64.h" +#include "VecSim/spaces/sq8_fp64.h" #include "VecSim/types/sq4.h" #include "VecSim/spaces/IP/IP.h" #include "VecSim/spaces/L2/L2.h" @@ -104,6 +106,40 @@ dist_func_t GetDistFunc(VecSimMetric metric, size_t dim, throw std::invalid_argument("Invalid metric"); } +template <> +dist_func_t GetDistFunc(VecSimMetric metric, + size_t dim, + unsigned char *alignment) { + if (alignment) { + *alignment = 0; + } + switch (metric) { + case VecSimMetric_Cosine: + case VecSimMetric_IP: + return IP_SQ8FP64; + case VecSimMetric_L2: + return L2_SQ8FP64; + } + throw std::invalid_argument("Invalid metric"); +} + +template <> +dist_func_t GetDistFunc(VecSimMetric metric, + size_t dim, + unsigned char *alignment) { + if (alignment) { + *alignment = 0; + } + switch (metric) { + case VecSimMetric_Cosine: + case VecSimMetric_IP: + return IP_SQ8FP64_FP64; + case VecSimMetric_L2: + return L2_SQ8FP64_FP64; + } + throw std::invalid_argument("Invalid metric"); +} + template <> dist_func_t GetDistFunc(VecSimMetric metric, size_t dim, unsigned char *alignment) { diff --git a/src/VecSim/spaces/sq8_fp64.cpp b/src/VecSim/spaces/sq8_fp64.cpp new file mode 100644 index 000000000..4c9096f59 --- /dev/null +++ b/src/VecSim/spaces/sq8_fp64.cpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#include "VecSim/spaces/sq8_fp64.h" +#include "VecSim/types/sq8_fp64.h" +#include "VecSim/utils/alignment.h" + +namespace spaces { +namespace { +double metadata(const void *blob, size_t dim, size_t index) { + return load_unaligned(static_cast(blob) + dim + + index * sizeof(double)); +} + +template +double distance(const void *left, const void *right, size_t dim) { + const auto *a = static_cast(left); + const auto *b = static_cast(right); + const double min_a = metadata(left, dim, vecsim_types::sq8_fp64::MIN_VAL); + const double delta_a = metadata(left, dim, vecsim_types::sq8_fp64::DELTA); + double min_b = 0.0; + double delta_b = 0.0; + if constexpr (!Asymmetric) { + min_b = metadata(right, dim, vecsim_types::sq8_fp64::MIN_VAL); + delta_b = metadata(right, dim, vecsim_types::sq8_fp64::DELTA); + } + double result = 0.0; + for (size_t i = 0; i < dim; ++i) { + if constexpr (L2) { + const double residual = + Asymmetric + ? delta_a * a[i] + (min_a - load_unaligned(b + i * sizeof(double))) + : (min_a - min_b) + (delta_a - delta_b) * a[i] + + delta_b * (int(a[i]) - int(b[i])); + result += residual * residual; + } else { + const double y = Asymmetric ? load_unaligned(b + i * sizeof(double)) + : min_b + delta_b * b[i]; + result += (min_a + delta_a * a[i]) * y; + } + } + return L2 ? result : 1.0 - result; +} +} // namespace + +double L2_SQ8FP64(const void *left, const void *right, size_t dim) { + return distance(left, right, dim); +} +double L2_SQ8FP64_FP64(const void *left, const void *right, size_t dim) { + return distance(left, right, dim); +} +double IP_SQ8FP64(const void *left, const void *right, size_t dim) { + return distance(left, right, dim); +} +double IP_SQ8FP64_FP64(const void *left, const void *right, size_t dim) { + return distance(left, right, dim); +} +} // namespace spaces diff --git a/src/VecSim/spaces/sq8_fp64.h b/src/VecSim/spaces/sq8_fp64.h new file mode 100644 index 000000000..769c7b63e --- /dev/null +++ b/src/VecSim/spaces/sq8_fp64.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include + +namespace spaces { +double L2_SQ8FP64(const void *left, const void *right, size_t dim); +double L2_SQ8FP64_FP64(const void *left, const void *right, size_t dim); +double IP_SQ8FP64(const void *left, const void *right, size_t dim); +double IP_SQ8FP64_FP64(const void *left, const void *right, size_t dim); +} // namespace spaces diff --git a/src/VecSim/types/sq8_fp64.h b/src/VecSim/types/sq8_fp64.h new file mode 100644 index 000000000..9cdb3aed1 --- /dev/null +++ b/src/VecSim/types/sq8_fp64.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include +#include "VecSim/types/quantized_metadata.h" + +namespace vecsim_types { + +// SQ8 bytes followed by double reconstruction metadata; distinct from the FP32 SQ8 layout. +struct sq8_fp64 : quantized_metadata { + using value_type = uint8_t; + static constexpr uint8_t max_code = std::numeric_limits::max(); + static constexpr size_t data_bytes_count(size_t dim); + template + static constexpr size_t storage_bytes_count(size_t dim); + template + static constexpr size_t query_bytes_count(size_t dim); +}; + +constexpr size_t sq8_fp64::data_bytes_count(size_t dim) { return dim; } + +template +constexpr size_t sq8_fp64::storage_bytes_count(size_t dim) { + return dim + storage_metadata_count() * sizeof(double); +} + +template +constexpr size_t sq8_fp64::query_bytes_count(size_t dim) { + return (dim + query_metadata_count()) * sizeof(double); +} + +} // namespace vecsim_types diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index fd8e18721..db364d876 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -39,10 +39,10 @@ add_executable(test_hnsw_sq4 ../utils/test_main_with_timeout.cpp ../utils/mock_t add_executable(test_hnsw_sq8 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_hnsw_sq8.cpp unit_test_utils.cpp) add_executable(test_bruteforce ../utils/test_main_with_timeout.cpp test_bruteforce.cpp test_bruteforce_multi.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) add_executable(test_allocator ../utils/test_main_with_timeout.cpp test_allocator.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) -add_executable(test_spaces ../utils/test_main_with_timeout.cpp test_spaces.cpp) +add_executable(test_spaces ../utils/test_main_with_timeout.cpp test_spaces.cpp test_sq8_fp64_distances.cpp) add_executable(test_types ../utils/test_main_with_timeout.cpp test_types.cpp) add_executable(test_common ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_common.cpp unit_test_utils.cpp) -add_executable(test_components ../utils/test_main_with_timeout.cpp test_components.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) +add_executable(test_components ../utils/test_main_with_timeout.cpp test_components.cpp test_sq8_fp64_components.cpp ../utils/mock_thread_pool.cpp unit_test_utils.cpp) add_executable(test_bf16 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_bf16.cpp unit_test_utils.cpp) add_executable(test_fp16 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_fp16.cpp unit_test_utils.cpp) add_executable(test_int8 ../utils/test_main_with_timeout.cpp ../utils/mock_thread_pool.cpp test_int8.cpp unit_test_utils.cpp) diff --git a/tests/unit/test_sq8_fp64_components.cpp b/tests/unit/test_sq8_fp64_components.cpp new file mode 100644 index 000000000..9de1bf4e3 --- /dev/null +++ b/tests/unit/test_sq8_fp64_components.cpp @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#include +#include +#include +#include "gtest/gtest.h" +#include "VecSim/index_factories/components/components_factory.h" + +namespace { +using vecsim_types::sq8_fp64; + +double readMetadata(const void *blob, size_t offset, size_t index) { + double result; + std::memcpy(&result, static_cast(blob) + offset + index * sizeof(double), + sizeof(result)); + return result; +} + +template +void checkLayout(size_t dim, double constant) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + SQ8FP64Preprocessor preprocessor(allocator, dim); + std::vector input(dim, constant); + const auto original = input; + void *storage = nullptr; + void *query = nullptr; + size_t storage_size = input.size() * sizeof(double); + size_t query_size = storage_size; + preprocessor.preprocess(input.data(), storage, query, storage_size, query_size, 32, 64); + EXPECT_EQ(storage_size, sq8_fp64::storage_bytes_count(dim)); + EXPECT_EQ(query_size, sq8_fp64::query_bytes_count(dim)); + EXPECT_EQ(reinterpret_cast(storage) % 32, 0); + EXPECT_EQ(reinterpret_cast(query) % 64, 0); + EXPECT_EQ(std::memcmp(input.data(), query, dim * sizeof(double)), 0); + EXPECT_EQ(input, original); + EXPECT_DOUBLE_EQ(readMetadata(storage, dim, sq8_fp64::MIN_VAL), constant); + EXPECT_DOUBLE_EQ(readMetadata(storage, dim, sq8_fp64::DELTA), 1.0); + EXPECT_DOUBLE_EQ(readMetadata(storage, dim, sq8_fp64::SUM), dim * constant); + EXPECT_DOUBLE_EQ(readMetadata(query, dim * sizeof(double), sq8_fp64::SUM_QUERY), + dim * constant); + if constexpr (Metric == VecSimMetric_L2) { + EXPECT_DOUBLE_EQ(readMetadata(storage, dim, sq8_fp64::SUM_SQUARES), + dim * constant * constant); + EXPECT_DOUBLE_EQ(readMetadata(query, dim * sizeof(double), sq8_fp64::SUM_SQUARES_QUERY), + dim * constant * constant); + } + for (size_t i = 0; i < dim; ++i) { + EXPECT_EQ(static_cast(storage)[i], 0); + } + std::vector inplace((std::max(storage_size, dim * sizeof(double)) + 7) / 8); + std::memcpy(inplace.data(), input.data(), dim * sizeof(double)); + preprocessor.preprocessStorageInPlace(inplace.data(), inplace.size() * sizeof(double)); + EXPECT_EQ(std::memcmp(inplace.data(), storage, storage_size), 0); + allocator->free_allocation(storage); + allocator->free_allocation(query); +} + +TEST(SQ8FP64Components, LayoutAlignmentConstantsAndInPlace) { + for (size_t dim : {1, 3, 4, 5, 17}) { + checkLayout(dim, 0.0); + checkLayout(dim, 1.25); + checkLayout(dim, -2.0); + checkLayout(dim, 1.0 / std::sqrt(double(dim))); + } +} + +TEST(SQ8FP64Components, DoublePrecisionRangeAndMetadata) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + SQ8FP64Preprocessor preprocessor(allocator, 3); + for (double scale : {1.0, 1e100, 1e-300}) { + const double input[] = {scale, 1.25 * scale, 2.0 * scale}; + void *storage = nullptr; + size_t size = sizeof(input); + preprocessor.preprocessForStorage(input, storage, size, 0); + const auto *codes = static_cast(storage); + EXPECT_EQ(codes[0], 0); + EXPECT_EQ(codes[1], 64); + EXPECT_EQ(codes[2], 255); + const double delta = readMetadata(storage, 3, sq8_fp64::DELTA); + EXPECT_GT(delta, 0.0); + EXPECT_TRUE(std::isfinite(delta)); + const double middle = scale + delta * 64; + EXPECT_DOUBLE_EQ(readMetadata(storage, 3, sq8_fp64::SUM), scale + middle + 2 * scale); + EXPECT_DOUBLE_EQ(readMetadata(storage, 3, sq8_fp64::SUM_SQUARES), + scale * scale + middle * middle + 4 * scale * scale); + allocator->free_allocation(storage); + } +} + +TEST(SQ8FP64Components, DefinedConversionOutsideRepresentableDeltaDomain) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + SQ8FP64Preprocessor preprocessor(allocator, 2); + const double tiny = std::numeric_limits::denorm_min(); + for (const auto &input : {std::vector{-1e308, 1e308}, std::vector{0.0, tiny}}) { + void *storage = nullptr; + size_t size = input.size() * sizeof(double); + preprocessor.preprocessForStorage(input.data(), storage, size, 0); + const auto *codes = static_cast(storage); + EXPECT_EQ(codes[0], 0); + EXPECT_EQ(codes[1], input[0] == 0.0 ? 255 : 0); + allocator->free_allocation(storage); + } +} + +TEST(SQ8FP64Components, NearbyLargeValuesKeepDoublePrecision) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto components = CreateSQ8FP64IndexComponents(allocator, 2); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + const double input[] = {1e8, 1e8 + 1.0}; + const double query[] = {1e8, 1e8}; + const auto stored = preprocessors->preprocessForStorage(input, sizeof(input)); + const auto processed = preprocessors->preprocessQuery(query, sizeof(query)); + EXPECT_EQ(static_cast(stored.get())[0], 0); + EXPECT_EQ(static_cast(stored.get())[1], 255); + EXPECT_NEAR(calculator->calcDistanceForQuery(stored.get(), processed.get(), 2), 1.0, 1e-14); + const double close[] = {1.0, 1.0 + std::ldexp(1.0, -40)}; + const auto precise = preprocessors->preprocessForStorage(close, sizeof(close)); + EXPECT_DOUBLE_EQ(readMetadata(precise.get(), 2, sq8_fp64::DELTA), std::ldexp(1.0, -40) / 255); + EXPECT_NEAR(calculator->calcDistance(precise.get(), precise.get(), 2), 0.0, 1e-40); +} +} // namespace diff --git a/tests/unit/test_sq8_fp64_distances.cpp b/tests/unit/test_sq8_fp64_distances.cpp new file mode 100644 index 000000000..d29f962ee --- /dev/null +++ b/tests/unit/test_sq8_fp64_distances.cpp @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#include +#include +#include +#include +#include +#include +#include +#include "gtest/gtest.h" +#include "VecSim/index_factories/components/components_factory.h" + +namespace { +using vecsim_types::sq8_fp64; + +std::vector reconstruct(const void *blob, size_t dim, double &delta) { + const auto *bytes = static_cast(blob); + double minimum; + std::memcpy(&minimum, bytes + dim, sizeof(double)); + std::memcpy(&delta, bytes + dim + sizeof(double), sizeof(double)); + std::vector values(dim); + for (size_t i = 0; i < dim; ++i) { + values[i] = minimum + delta * bytes[i]; + } + return values; +} + +template +double reference(const std::vector &a, const std::vector &b) { + double result = 0.0; + for (size_t i = 0; i < a.size(); ++i) { + if constexpr (Metric == VecSimMetric_L2) { + const double residual = a[i] - b[i]; + result += residual * residual; + } else { + result += a[i] * b[i]; + } + } + return Metric == VecSimMetric_L2 ? result : 1.0 - result; +} + +void normalize(std::vector &values) { + const double norm = + std::sqrt(std::inner_product(values.begin(), values.end(), values.begin(), 0.0)); + for (auto &value : values) { + value /= norm; + } +} + +void checkBound(VecSimMetric metric, double actual, double expected, double delta_a, double delta_b, + const std::vector &a, const std::vector &b) { + double norm1_a = 0.0; + double norm1_b = 0.0; + double scale = 1.0; + for (size_t i = 0; i < a.size(); ++i) { + norm1_a += std::abs(a[i]); + norm1_b += std::abs(b[i]); + scale = std::max({scale, std::abs(a[i]), std::abs(b[i])}); + } + const double slack = 64 * std::numeric_limits::epsilon() * (a.size() + 1) * scale; + if (metric == VecSimMetric_L2) { + EXPECT_GE(actual, 0.0); + EXPECT_LE(std::abs(std::sqrt(actual) - std::sqrt(expected)), + (delta_a + delta_b) * std::sqrt(double(a.size())) / 2 + slack); + } else { + EXPECT_LE(std::abs(actual - expected), delta_a * norm1_b / 2 + delta_b * norm1_a / 2 + + a.size() * delta_a * delta_b / 4 + + slack * scale); + } +} + +template +void checkDistances(size_t dim, bool nearly_parallel) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto components = CreateSQ8FP64IndexComponents(allocator, dim); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + std::vector a(dim); + std::vector b(dim); + for (size_t i = 0; i < dim; ++i) { + a[i] = 0.31 * (int(i % 7) - 3) + 0.11; + b[i] = nearly_parallel ? a[i] + 1e-7 * i : 0.23 * (int(i % 5) - 2) - 0.07; + } + if constexpr (Metric == VecSimMetric_Cosine) { + normalize(a); + normalize(b); + } + const size_t input_size = dim * sizeof(double); + const auto left = preprocessors->preprocessForStorage(a.data(), input_size); + const auto right = preprocessors->preprocessForStorage(b.data(), input_size); + const auto query = preprocessors->preprocessQuery(b.data(), input_size); + const auto combined = preprocessors->preprocess(b.data(), input_size); + EXPECT_EQ(std::memcmp(right.get(), combined.getStorageBlob(), + sq8_fp64::storage_bytes_count(dim)), + 0); + EXPECT_EQ( + std::memcmp(query.get(), combined.getQueryBlob(), sq8_fp64::query_bytes_count(dim)), + 0); + double delta_a; + double delta_b; + const auto ra = reconstruct(left.get(), dim, delta_a); + const auto rb = reconstruct(right.get(), dim, delta_b); + const double symmetric = calculator->calcDistance(left.get(), right.get(), dim); + const double asymmetric = calculator->calcDistanceForQuery(left.get(), query.get(), dim); + EXPECT_NEAR(symmetric, reference(ra, rb), 1e-13 * (dim + 1)); + EXPECT_NEAR(asymmetric, reference(ra, b), 1e-13 * (dim + 1)); + checkBound(Metric, symmetric, reference(a, b), delta_a, delta_b, a, b); + checkBound(Metric, asymmetric, reference(a, b), delta_a, 0, a, b); + EXPECT_DOUBLE_EQ( + calculator->getDistanceDispatch(DistanceMode::StoredToStored)(left.get(), right.get(), dim), + symmetric); + EXPECT_DOUBLE_EQ( + calculator->getDistanceDispatch(DistanceMode::StoredToQuery)(left.get(), query.get(), dim), + asymmetric); + const auto self_query = preprocessors->preprocessQuery(a.data(), input_size); + checkBound(Metric, calculator->calcDistanceForQuery(left.get(), self_query.get(), dim), + reference(a, a), delta_a, 0, a, a); +} + +TEST(SQ8FP64Distances, MetricsReconstructionAndAnalyticalBounds) { + for (size_t dim : {1, 3, 4, 5, 17}) { + checkDistances(dim, false); + checkDistances(dim, false); + checkDistances(dim, false); + checkDistances(dim, true); + } +} + +TEST(SQ8FP64Distances, OriginalDoubleTopTwoRanking) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto components = CreateSQ8FP64IndexComponents(allocator, 3); + std::unique_ptr> calculator(components.indexCalculator); + std::unique_ptr preprocessors(components.preprocessors); + const std::vector query(3, 1e8 + 3); + const auto processed = preprocessors->preprocessQuery(query.data(), 3 * sizeof(double)); + const std::array offsets = {0, 2, 5, 9}; + const std::array expected = {27, 3, 12, 108}; + std::array original_order = {0, 1, 2, 3}; + auto quantized_order = original_order; + std::array scores; + for (size_t i = 0; i < offsets.size(); ++i) { + const std::vector input(3, 1e8 + offsets[i]); + const auto stored = preprocessors->preprocessForStorage(input.data(), 3 * sizeof(double)); + scores[i] = calculator->calcDistanceForQuery(stored.get(), processed.get(), 3); + EXPECT_DOUBLE_EQ(reference(input, query), expected[i]); + EXPECT_DOUBLE_EQ(scores[i], expected[i]); + } + std::sort(original_order.begin(), original_order.end(), + [&](size_t a, size_t b) { return expected[a] < expected[b]; }); + std::sort(quantized_order.begin(), quantized_order.end(), + [&](size_t a, size_t b) { return scores[a] < scores[b]; }); + EXPECT_EQ(original_order, quantized_order); + EXPECT_EQ(quantized_order[0], 1); + EXPECT_EQ(quantized_order[1], 2); +} + +TEST(SQ8FP64Distances, DispatchAlignmentUnalignedBlobsAndInvalidMetric) { + auto allocator = VecSimAllocator::newVecsimAllocator(); + SQ8FP64Preprocessor preprocessor(allocator, 3); + const double input[] = {1e8, 1e8 + 1, 1e8 + 2}; + void *storage = nullptr; + size_t size = sizeof(input); + preprocessor.preprocessForStorage(input, storage, size, 0); + std::vector unaligned(size + 1); + std::memcpy(unaligned.data() + 1, storage, size); + unsigned char alignment = 99; + const auto symmetric = spaces::GetDistFunc(VecSimMetric_L2, 3, &alignment); + EXPECT_EQ(alignment, 0); + EXPECT_DOUBLE_EQ(symmetric(unaligned.data() + 1, storage, 3), 0.0); + const auto asymmetric = + spaces::GetDistFunc(VecSimMetric_L2, 3, nullptr); + std::vector unaligned_query(sizeof(input) + 1); + std::memcpy(unaligned_query.data() + 1, input, sizeof(input)); + EXPECT_NEAR(asymmetric(unaligned.data() + 1, unaligned_query.data() + 1, 3), + std::pow(1.0 / 255, 2), 1e-15); + EXPECT_THROW((spaces::GetDistFunc(VecSimMetric(99), 3, nullptr)), + std::invalid_argument); + EXPECT_THROW((spaces::GetDistFunc(VecSimMetric(99), 3, nullptr)), + std::invalid_argument); + allocator->free_allocation(storage); +} +} // namespace From 74916ca1ddb8e37c843ea7f54edadbb039d55250 Mon Sep 17 00:00:00 2001 From: Alex Dimitrov Date: Thu, 12 Nov 2026 12:00:42 +0200 Subject: [PATCH 2/8] Tighten FP64 SQ8 accuracy tests and const bindings --- .../components/components_factory.h | 6 +-- .../spaces/computer/sq8_fp64_preprocessor.h | 2 +- src/VecSim/spaces/sq8_fp64.cpp | 4 +- tests/unit/test_sq8_fp64_components.cpp | 38 ++++++++++++++----- tests/unit/test_sq8_fp64_distances.cpp | 28 ++++++++------ 5 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index a0a8ae4c1..63aac197e 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -125,11 +125,11 @@ CreateSQ8FP64IndexComponents(const std::shared_ptr &allocator, const auto asymmetric = spaces::GetDistFunc( Metric, dim, &asymmetric_alignment); storage_alignment = spaces::combineAlignments(storage_alignment, asymmetric_alignment); - auto *calculator = new (allocator) + auto *const calculator = new (allocator) DistanceCalculatorCommon(allocator, symmetric, asymmetric); - auto *container = new (allocator) + auto *const container = new (allocator) MultiPreprocessorsContainer(allocator, 0, storage_alignment); - auto *preprocessor = new (allocator) SQ8FP64Preprocessor(allocator, dim); + auto *const preprocessor = new (allocator) SQ8FP64Preprocessor(allocator, dim); [[maybe_unused]] const int result = container->addPreprocessor(preprocessor); assert(result != -1); return {calculator, container}; diff --git a/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h b/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h index 82466832f..19d5b7a28 100644 --- a/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h +++ b/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h @@ -102,7 +102,7 @@ void SQ8FP64Preprocessor::preprocessQuery(const void *original, void *&q size_t &query_size, unsigned char alignment) const { assert(!query); - const auto *input = static_cast(original); + const auto *const input = static_cast(original); double sum = 0.0; double squares = 0.0; for (size_t i = 0; i < dim; ++i) { diff --git a/src/VecSim/spaces/sq8_fp64.cpp b/src/VecSim/spaces/sq8_fp64.cpp index 4c9096f59..7010a60f6 100644 --- a/src/VecSim/spaces/sq8_fp64.cpp +++ b/src/VecSim/spaces/sq8_fp64.cpp @@ -19,8 +19,8 @@ double metadata(const void *blob, size_t dim, size_t index) { template double distance(const void *left, const void *right, size_t dim) { - const auto *a = static_cast(left); - const auto *b = static_cast(right); + const auto *const a = static_cast(left); + const auto *const b = static_cast(right); const double min_a = metadata(left, dim, vecsim_types::sq8_fp64::MIN_VAL); const double delta_a = metadata(left, dim, vecsim_types::sq8_fp64::DELTA); double min_b = 0.0; diff --git a/tests/unit/test_sq8_fp64_components.cpp b/tests/unit/test_sq8_fp64_components.cpp index 9de1bf4e3..92d4a4420 100644 --- a/tests/unit/test_sq8_fp64_components.cpp +++ b/tests/unit/test_sq8_fp64_components.cpp @@ -25,9 +25,9 @@ double readMetadata(const void *blob, size_t offset, size_t index) { template void checkLayout(size_t dim, double constant) { - auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto allocator = VecSimAllocator::newVecsimAllocator(); SQ8FP64Preprocessor preprocessor(allocator, dim); - std::vector input(dim, constant); + const std::vector input(dim, constant); const auto original = input; void *storage = nullptr; void *query = nullptr; @@ -72,14 +72,14 @@ TEST(SQ8FP64Components, LayoutAlignmentConstantsAndInPlace) { } TEST(SQ8FP64Components, DoublePrecisionRangeAndMetadata) { - auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto allocator = VecSimAllocator::newVecsimAllocator(); SQ8FP64Preprocessor preprocessor(allocator, 3); for (double scale : {1.0, 1e100, 1e-300}) { const double input[] = {scale, 1.25 * scale, 2.0 * scale}; void *storage = nullptr; size_t size = sizeof(input); preprocessor.preprocessForStorage(input, storage, size, 0); - const auto *codes = static_cast(storage); + const auto *const codes = static_cast(storage); EXPECT_EQ(codes[0], 0); EXPECT_EQ(codes[1], 64); EXPECT_EQ(codes[2], 255); @@ -95,25 +95,45 @@ TEST(SQ8FP64Components, DoublePrecisionRangeAndMetadata) { } TEST(SQ8FP64Components, DefinedConversionOutsideRepresentableDeltaDomain) { - auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto allocator = VecSimAllocator::newVecsimAllocator(); SQ8FP64Preprocessor preprocessor(allocator, 2); const double tiny = std::numeric_limits::denorm_min(); for (const auto &input : {std::vector{-1e308, 1e308}, std::vector{0.0, tiny}}) { void *storage = nullptr; size_t size = input.size() * sizeof(double); preprocessor.preprocessForStorage(input.data(), storage, size, 0); - const auto *codes = static_cast(storage); + const auto *const codes = static_cast(storage); EXPECT_EQ(codes[0], 0); EXPECT_EQ(codes[1], input[0] == 0.0 ? 255 : 0); allocator->free_allocation(storage); } } +template +void checkZeroDistance(double expected) { + const auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto components = CreateSQ8FP64IndexComponents(allocator, 3); + const std::unique_ptr> calculator(components.indexCalculator); + const std::unique_ptr preprocessors(components.preprocessors); + const double input[] = {0.0, 0.0, 0.0}; + const auto blobs = preprocessors->preprocess(input, sizeof(input)); + EXPECT_DOUBLE_EQ(calculator->calcDistance(blobs.getStorageBlob(), blobs.getStorageBlob(), 3), + expected); + EXPECT_DOUBLE_EQ( + calculator->calcDistanceForQuery(blobs.getStorageBlob(), blobs.getQueryBlob(), 3), + expected); +} + +TEST(SQ8FP64Components, ZeroDistances) { + checkZeroDistance(0.0); + checkZeroDistance(1.0); +} + TEST(SQ8FP64Components, NearbyLargeValuesKeepDoublePrecision) { - auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto allocator = VecSimAllocator::newVecsimAllocator(); const auto components = CreateSQ8FP64IndexComponents(allocator, 2); - std::unique_ptr> calculator(components.indexCalculator); - std::unique_ptr preprocessors(components.preprocessors); + const std::unique_ptr> calculator(components.indexCalculator); + const std::unique_ptr preprocessors(components.preprocessors); const double input[] = {1e8, 1e8 + 1.0}; const double query[] = {1e8, 1e8}; const auto stored = preprocessors->preprocessForStorage(input, sizeof(input)); diff --git a/tests/unit/test_sq8_fp64_distances.cpp b/tests/unit/test_sq8_fp64_distances.cpp index d29f962ee..11432fc20 100644 --- a/tests/unit/test_sq8_fp64_distances.cpp +++ b/tests/unit/test_sq8_fp64_distances.cpp @@ -21,7 +21,7 @@ namespace { using vecsim_types::sq8_fp64; std::vector reconstruct(const void *blob, size_t dim, double &delta) { - const auto *bytes = static_cast(blob); + const auto *const bytes = static_cast(blob); double minimum; std::memcpy(&minimum, bytes + dim, sizeof(double)); std::memcpy(&delta, bytes + dim + sizeof(double), sizeof(double)); @@ -64,24 +64,30 @@ void checkBound(VecSimMetric metric, double actual, double expected, double delt norm1_b += std::abs(b[i]); scale = std::max({scale, std::abs(a[i]), std::abs(b[i])}); } + const double error_a = + std::all_of(a.begin(), a.end(), [&](double value) { return value == a.front(); }) ? 0.0 + : delta_a; + const double error_b = + std::all_of(b.begin(), b.end(), [&](double value) { return value == b.front(); }) ? 0.0 + : delta_b; const double slack = 64 * std::numeric_limits::epsilon() * (a.size() + 1) * scale; if (metric == VecSimMetric_L2) { EXPECT_GE(actual, 0.0); EXPECT_LE(std::abs(std::sqrt(actual) - std::sqrt(expected)), - (delta_a + delta_b) * std::sqrt(double(a.size())) / 2 + slack); + (error_a + error_b) * std::sqrt(double(a.size())) / 2 + slack); } else { - EXPECT_LE(std::abs(actual - expected), delta_a * norm1_b / 2 + delta_b * norm1_a / 2 + - a.size() * delta_a * delta_b / 4 + + EXPECT_LE(std::abs(actual - expected), error_a * norm1_b / 2 + error_b * norm1_a / 2 + + a.size() * error_a * error_b / 4 + slack * scale); } } template void checkDistances(size_t dim, bool nearly_parallel) { - auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto allocator = VecSimAllocator::newVecsimAllocator(); const auto components = CreateSQ8FP64IndexComponents(allocator, dim); - std::unique_ptr> calculator(components.indexCalculator); - std::unique_ptr preprocessors(components.preprocessors); + const std::unique_ptr> calculator(components.indexCalculator); + const std::unique_ptr preprocessors(components.preprocessors); std::vector a(dim); std::vector b(dim); for (size_t i = 0; i < dim; ++i) { @@ -134,10 +140,10 @@ TEST(SQ8FP64Distances, MetricsReconstructionAndAnalyticalBounds) { } TEST(SQ8FP64Distances, OriginalDoubleTopTwoRanking) { - auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto allocator = VecSimAllocator::newVecsimAllocator(); const auto components = CreateSQ8FP64IndexComponents(allocator, 3); - std::unique_ptr> calculator(components.indexCalculator); - std::unique_ptr preprocessors(components.preprocessors); + const std::unique_ptr> calculator(components.indexCalculator); + const std::unique_ptr preprocessors(components.preprocessors); const std::vector query(3, 1e8 + 3); const auto processed = preprocessors->preprocessQuery(query.data(), 3 * sizeof(double)); const std::array offsets = {0, 2, 5, 9}; @@ -162,7 +168,7 @@ TEST(SQ8FP64Distances, OriginalDoubleTopTwoRanking) { } TEST(SQ8FP64Distances, DispatchAlignmentUnalignedBlobsAndInvalidMetric) { - auto allocator = VecSimAllocator::newVecsimAllocator(); + const auto allocator = VecSimAllocator::newVecsimAllocator(); SQ8FP64Preprocessor preprocessor(allocator, 3); const double input[] = {1e8, 1e8 + 1, 1e8 + 2}; void *storage = nullptr; From c91659c885a230b3f5de49579575fe587dcad7ab Mon Sep 17 00:00:00 2001 From: Alex Dimitrov Date: Thu, 12 Nov 2026 12:02:39 +0200 Subject: [PATCH 3/8] Format FP64 SQ8 component and dispatch additions --- .../index_factories/components/components_factory.h | 12 ++++++------ src/VecSim/spaces/spaces.cpp | 9 ++++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/VecSim/index_factories/components/components_factory.h b/src/VecSim/index_factories/components/components_factory.h index 63aac197e..21e8a2c2d 100644 --- a/src/VecSim/index_factories/components/components_factory.h +++ b/src/VecSim/index_factories/components/components_factory.h @@ -119,16 +119,16 @@ template IndexComponents CreateSQ8FP64IndexComponents(const std::shared_ptr &allocator, size_t dim) { unsigned char storage_alignment = 0; - const auto symmetric = spaces::GetDistFunc( - Metric, dim, &storage_alignment); + const auto symmetric = + spaces::GetDistFunc(Metric, dim, &storage_alignment); unsigned char asymmetric_alignment = 0; const auto asymmetric = spaces::GetDistFunc( Metric, dim, &asymmetric_alignment); storage_alignment = spaces::combineAlignments(storage_alignment, asymmetric_alignment); - auto *const calculator = new (allocator) - DistanceCalculatorCommon(allocator, symmetric, asymmetric); - auto *const container = new (allocator) - MultiPreprocessorsContainer(allocator, 0, storage_alignment); + auto *const calculator = + new (allocator) DistanceCalculatorCommon(allocator, symmetric, asymmetric); + auto *const container = + new (allocator) MultiPreprocessorsContainer(allocator, 0, storage_alignment); auto *const preprocessor = new (allocator) SQ8FP64Preprocessor(allocator, dim); [[maybe_unused]] const int result = container->addPreprocessor(preprocessor); assert(result != -1); diff --git a/src/VecSim/spaces/spaces.cpp b/src/VecSim/spaces/spaces.cpp index e5484ab38..8c6b3f37b 100644 --- a/src/VecSim/spaces/spaces.cpp +++ b/src/VecSim/spaces/spaces.cpp @@ -107,9 +107,8 @@ dist_func_t GetDistFunc(VecSimMetric metric, size_t dim, } template <> -dist_func_t GetDistFunc(VecSimMetric metric, - size_t dim, - unsigned char *alignment) { +dist_func_t GetDistFunc(VecSimMetric metric, size_t dim, + unsigned char *alignment) { if (alignment) { *alignment = 0; } @@ -125,8 +124,8 @@ dist_func_t GetDistFunc(VecSimMetric met template <> dist_func_t GetDistFunc(VecSimMetric metric, - size_t dim, - unsigned char *alignment) { + size_t dim, + unsigned char *alignment) { if (alignment) { *alignment = 0; } From df6419d50d638515107e5e03b3bbe9ea1ebc82f9 Mon Sep 17 00:00:00 2001 From: Alex Dimitrov Date: Thu, 12 Nov 2026 14:39:27 +0200 Subject: [PATCH 4/8] Polish FP64 SQ8 component coverage --- src/VecSim/spaces/sq8_fp64.cpp | 8 ++------ tests/unit/test_sq8_fp64_components.cpp | 9 +++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/VecSim/spaces/sq8_fp64.cpp b/src/VecSim/spaces/sq8_fp64.cpp index 7010a60f6..02f001cdc 100644 --- a/src/VecSim/spaces/sq8_fp64.cpp +++ b/src/VecSim/spaces/sq8_fp64.cpp @@ -23,12 +23,8 @@ double distance(const void *left, const void *right, size_t dim) { const auto *const b = static_cast(right); const double min_a = metadata(left, dim, vecsim_types::sq8_fp64::MIN_VAL); const double delta_a = metadata(left, dim, vecsim_types::sq8_fp64::DELTA); - double min_b = 0.0; - double delta_b = 0.0; - if constexpr (!Asymmetric) { - min_b = metadata(right, dim, vecsim_types::sq8_fp64::MIN_VAL); - delta_b = metadata(right, dim, vecsim_types::sq8_fp64::DELTA); - } + const double min_b = Asymmetric ? 0.0 : metadata(right, dim, vecsim_types::sq8_fp64::MIN_VAL); + const double delta_b = Asymmetric ? 0.0 : metadata(right, dim, vecsim_types::sq8_fp64::DELTA); double result = 0.0; for (size_t i = 0; i < dim; ++i) { if constexpr (L2) { diff --git a/tests/unit/test_sq8_fp64_components.cpp b/tests/unit/test_sq8_fp64_components.cpp index 92d4a4420..4a16d2347 100644 --- a/tests/unit/test_sq8_fp64_components.cpp +++ b/tests/unit/test_sq8_fp64_components.cpp @@ -7,8 +7,12 @@ * GNU Affero General Public License v3 (AGPLv3). */ +#include +#include +#include #include #include +#include #include #include "gtest/gtest.h" #include "VecSim/index_factories/components/components_factory.h" @@ -79,6 +83,11 @@ TEST(SQ8FP64Components, DoublePrecisionRangeAndMetadata) { void *storage = nullptr; size_t size = sizeof(input); preprocessor.preprocessForStorage(input, storage, size, 0); + std::vector inplace((std::max(size, sizeof(input)) + sizeof(double) - 1) / + sizeof(double)); + std::memcpy(inplace.data(), input, sizeof(input)); + preprocessor.preprocessStorageInPlace(inplace.data(), inplace.size() * sizeof(double)); + EXPECT_EQ(std::memcmp(inplace.data(), storage, size), 0); const auto *const codes = static_cast(storage); EXPECT_EQ(codes[0], 0); EXPECT_EQ(codes[1], 64); From 87cbcc58cc716f341f530571b1ed690c6478a15f Mon Sep 17 00:00:00 2001 From: Alex Dimitrov Date: Thu, 12 Nov 2026 16:24:54 +0200 Subject: [PATCH 5/8] Use SQ8 FP64 data size helper for offsets --- src/VecSim/spaces/computer/sq8_fp64_preprocessor.h | 2 +- src/VecSim/spaces/sq8_fp64.cpp | 3 ++- src/VecSim/types/sq8_fp64.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h b/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h index 19d5b7a28..c431c4f82 100644 --- a/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h +++ b/src/VecSim/spaces/computer/sq8_fp64_preprocessor.h @@ -72,7 +72,7 @@ void SQ8FP64Preprocessor::quantize(const double *input, uint8_t *output) } } const double metadata[] = {minimum, delta, sum, squares}; - std::memcpy(output + dim, metadata, + std::memcpy(output + vecsim_types::sq8_fp64::data_bytes_count(dim), metadata, vecsim_types::sq8_fp64::storage_metadata_count() * sizeof(double)); } diff --git a/src/VecSim/spaces/sq8_fp64.cpp b/src/VecSim/spaces/sq8_fp64.cpp index 02f001cdc..d66e0eeb2 100644 --- a/src/VecSim/spaces/sq8_fp64.cpp +++ b/src/VecSim/spaces/sq8_fp64.cpp @@ -13,7 +13,8 @@ namespace spaces { namespace { double metadata(const void *blob, size_t dim, size_t index) { - return load_unaligned(static_cast(blob) + dim + + return load_unaligned(static_cast(blob) + + vecsim_types::sq8_fp64::data_bytes_count(dim) + index * sizeof(double)); } diff --git a/src/VecSim/types/sq8_fp64.h b/src/VecSim/types/sq8_fp64.h index 9cdb3aed1..4b10127d4 100644 --- a/src/VecSim/types/sq8_fp64.h +++ b/src/VecSim/types/sq8_fp64.h @@ -29,7 +29,7 @@ constexpr size_t sq8_fp64::data_bytes_count(size_t dim) { return dim; } template constexpr size_t sq8_fp64::storage_bytes_count(size_t dim) { - return dim + storage_metadata_count() * sizeof(double); + return data_bytes_count(dim) + storage_metadata_count() * sizeof(double); } template From e277cc78ce3c777c10e9fc2e4150d74729344223 Mon Sep 17 00:00:00 2001 From: Alex Dimitrov Date: Thu, 12 Nov 2026 17:01:40 +0200 Subject: [PATCH 6/8] Share stored-query SQ8 L2 across float precisions --- src/VecSim/spaces/quantized_kernels.h | 41 +++++++++++++--------- src/VecSim/spaces/sq8_fp64.cpp | 46 +++++++++++++++---------- src/VecSim/types/sq8.h | 1 + src/VecSim/types/sq8_fp64.h | 4 +++ tests/unit/test_sq8_fp64_components.cpp | 17 +++++++++ 5 files changed, 74 insertions(+), 35 deletions(-) diff --git a/src/VecSim/spaces/quantized_kernels.h b/src/VecSim/spaces/quantized_kernels.h index 245a8fa47..7fa13530f 100644 --- a/src/VecSim/spaces/quantized_kernels.h +++ b/src/VecSim/spaces/quantized_kernels.h @@ -18,11 +18,12 @@ // // Scalar kernels for scalar quantized storage, generic over a StorageType that provides // get_code(data, i), data_bytes_count(dim) and the MIN_VAL / DELTA / SUM / SUM_SQUARES metadata -// indices (stored after the codes) and SUM_QUERY (stored after the FP32 query values). -// It also provides the code dot hooks code_dot(a, b, dim) and code_dot_fp32(codes, y, dim). +// indices (stored after the codes) and SUM_QUERY (stored after the query values). +// StoredQueryL2 also requires metadata_type, matching the query value type. +// The float IP helpers use code_dot(a, b, dim) and code_dot_fp32(codes, y, dim). // // Stored layout: [codes (data_bytes_count(dim) bytes)] [min_val] [delta] [sum] [sum_squares (L2)] -// FP32 query layout: [float values (dim)] [y_sum] [y_sum_squares (L2 only)] +// Query layout: [values (dim)] [y_sum] [y_sum_squares (L2 only)] // SQ4 L2 uses only the query body; its two trailing FP32 slots are reserved and zero-initialized. // // The operand order in these bodies is load-bearing and relies on FP addition not being @@ -58,7 +59,7 @@ static inline float FP32_InnerProduct_Impl(const void *stored, const void *query } /* - * Asymmetric stored-FP32 L2 squared distance, accumulated directly per component: + * Asymmetric stored-query L2 squared distance, accumulated directly per component: * ||x - y||^2 = sum((dequant(x_i) - y_i)^2), where dequant(x_i) = min_val + delta * q_i * * Accumulating residuals keeps each component's rounding at the scale of |x_i - y_i|. Expanding @@ -66,29 +67,37 @@ static inline float FP32_InnerProduct_Impl(const void *stored, const void *query * distance when x and y share a large common offset relative to their spread. */ template -static inline float FP32_L2Sqr(const void *stored, const void *query, size_t dim) { - // Storage metadata follows a byte payload and is not necessarily float-aligned. - const auto *pVect1 = static_cast(stored); - const auto *pVect2 = static_cast(query); - - const auto *params1 = pVect1 + StorageType::data_bytes_count(dim); - const float min_val = load_unaligned(params1 + StorageType::MIN_VAL * sizeof(float)); - const float delta = load_unaligned(params1 + StorageType::DELTA * sizeof(float)); - - float res = 0; +static inline typename StorageType::metadata_type StoredQueryL2(const void *stored, + const void *query, size_t dim) { + using Real = typename StorageType::metadata_type; + // Storage metadata follows a byte payload and is not necessarily aligned. + const auto *const pVect1 = static_cast(stored); + const auto *const pVect2 = static_cast(query); + + const auto *const params1 = pVect1 + StorageType::data_bytes_count(dim); + const Real min_val = load_unaligned(params1 + StorageType::MIN_VAL * sizeof(Real)); + const Real delta = load_unaligned(params1 + StorageType::DELTA * sizeof(Real)); + + Real res = 0; for (size_t i = 0; i < dim; i++) { // Order matters. When the stored range sits far from zero, min_val and y_i are within a // factor of two of each other, so min_val - y_i is exact (Sterbenz) and rounding happens // only at the scale of the residual. Forming (min_val + delta*q_i) first would round at // the scale of min_val and wipe out small residuals. When min_val and y_i are not close, // the residual is large and either order is fine. - float diff = - delta * static_cast(StorageType::get_code(pVect1, i)) + (min_val - pVect2[i]); + const Real query_value = load_unaligned(pVect2 + i * sizeof(Real)); + const Real diff = + delta * static_cast(StorageType::get_code(pVect1, i)) + (min_val - query_value); res += diff * diff; } return res; } +template +static inline float FP32_L2Sqr(const void *stored, const void *query, size_t dim) { + return StoredQueryL2(stored, query, dim); +} + /* * Symmetric stored-stored inner product using precomputed sums: * IP = min1*sum2 + min2*sum1 - dim*min1*min2 + delta1*delta2*sum(q1_i*q2_i) diff --git a/src/VecSim/spaces/sq8_fp64.cpp b/src/VecSim/spaces/sq8_fp64.cpp index d66e0eeb2..c08213d7e 100644 --- a/src/VecSim/spaces/sq8_fp64.cpp +++ b/src/VecSim/spaces/sq8_fp64.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "VecSim/spaces/sq8_fp64.h" +#include "VecSim/spaces/quantized_kernels.h" #include "VecSim/types/sq8_fp64.h" #include "VecSim/utils/alignment.h" @@ -18,8 +19,24 @@ double metadata(const void *blob, size_t dim, size_t index) { index * sizeof(double)); } -template -double distance(const void *left, const void *right, size_t dim) { +double storedL2(const void *left, const void *right, size_t dim) { + const auto *const a = static_cast(left); + const auto *const b = static_cast(right); + const double min_a = metadata(left, dim, vecsim_types::sq8_fp64::MIN_VAL); + const double delta_a = metadata(left, dim, vecsim_types::sq8_fp64::DELTA); + const double min_b = metadata(right, dim, vecsim_types::sq8_fp64::MIN_VAL); + const double delta_b = metadata(right, dim, vecsim_types::sq8_fp64::DELTA); + double result = 0.0; + for (size_t i = 0; i < dim; ++i) { + const double residual = + (min_a - min_b) + (delta_a - delta_b) * a[i] + delta_b * (int(a[i]) - int(b[i])); + result += residual * residual; + } + return result; +} + +template +double innerProduct(const void *left, const void *right, size_t dim) { const auto *const a = static_cast(left); const auto *const b = static_cast(right); const double min_a = metadata(left, dim, vecsim_types::sq8_fp64::MIN_VAL); @@ -28,33 +45,24 @@ double distance(const void *left, const void *right, size_t dim) { const double delta_b = Asymmetric ? 0.0 : metadata(right, dim, vecsim_types::sq8_fp64::DELTA); double result = 0.0; for (size_t i = 0; i < dim; ++i) { - if constexpr (L2) { - const double residual = - Asymmetric - ? delta_a * a[i] + (min_a - load_unaligned(b + i * sizeof(double))) - : (min_a - min_b) + (delta_a - delta_b) * a[i] + - delta_b * (int(a[i]) - int(b[i])); - result += residual * residual; - } else { - const double y = Asymmetric ? load_unaligned(b + i * sizeof(double)) - : min_b + delta_b * b[i]; - result += (min_a + delta_a * a[i]) * y; - } + const double y = + Asymmetric ? load_unaligned(b + i * sizeof(double)) : min_b + delta_b * b[i]; + result += (min_a + delta_a * a[i]) * y; } - return L2 ? result : 1.0 - result; + return 1.0 - result; } } // namespace double L2_SQ8FP64(const void *left, const void *right, size_t dim) { - return distance(left, right, dim); + return storedL2(left, right, dim); } double L2_SQ8FP64_FP64(const void *left, const void *right, size_t dim) { - return distance(left, right, dim); + return quantized::StoredQueryL2(left, right, dim); } double IP_SQ8FP64(const void *left, const void *right, size_t dim) { - return distance(left, right, dim); + return innerProduct(left, right, dim); } double IP_SQ8FP64_FP64(const void *left, const void *right, size_t dim) { - return distance(left, right, dim); + return innerProduct(left, right, dim); } } // namespace spaces diff --git a/src/VecSim/types/sq8.h b/src/VecSim/types/sq8.h index 55adccf74..2aaf10472 100644 --- a/src/VecSim/types/sq8.h +++ b/src/VecSim/types/sq8.h @@ -18,6 +18,7 @@ namespace vecsim_types { // Represents a scalar-quantized 8-bit blob with reconstruction metadata struct sq8 : quantized_metadata { using value_type = uint8_t; + using metadata_type = float; static constexpr uint8_t max_code = std::numeric_limits::max(); static constexpr size_t data_bytes_count(size_t dim) { return dim; } diff --git a/src/VecSim/types/sq8_fp64.h b/src/VecSim/types/sq8_fp64.h index 4b10127d4..d8fc09f4b 100644 --- a/src/VecSim/types/sq8_fp64.h +++ b/src/VecSim/types/sq8_fp64.h @@ -17,8 +17,10 @@ namespace vecsim_types { // SQ8 bytes followed by double reconstruction metadata; distinct from the FP32 SQ8 layout. struct sq8_fp64 : quantized_metadata { using value_type = uint8_t; + using metadata_type = double; static constexpr uint8_t max_code = std::numeric_limits::max(); static constexpr size_t data_bytes_count(size_t dim); + static uint8_t get_code(const uint8_t *data, size_t index); template static constexpr size_t storage_bytes_count(size_t dim); template @@ -27,6 +29,8 @@ struct sq8_fp64 : quantized_metadata { constexpr size_t sq8_fp64::data_bytes_count(size_t dim) { return dim; } +inline uint8_t sq8_fp64::get_code(const uint8_t *data, size_t index) { return data[index]; } + template constexpr size_t sq8_fp64::storage_bytes_count(size_t dim) { return data_bytes_count(dim) + storage_metadata_count() * sizeof(double); diff --git a/tests/unit/test_sq8_fp64_components.cpp b/tests/unit/test_sq8_fp64_components.cpp index 4a16d2347..9fbef4e91 100644 --- a/tests/unit/test_sq8_fp64_components.cpp +++ b/tests/unit/test_sq8_fp64_components.cpp @@ -16,6 +16,7 @@ #include #include "gtest/gtest.h" #include "VecSim/index_factories/components/components_factory.h" +#include "VecSim/spaces/sq8_fp64.h" namespace { using vecsim_types::sq8_fp64; @@ -156,3 +157,19 @@ TEST(SQ8FP64Components, NearbyLargeValuesKeepDoublePrecision) { EXPECT_NEAR(calculator->calcDistance(precise.get(), precise.get(), 2), 0.0, 1e-40); } } // namespace + +TEST(SQ8FP64Components, AsymmetricL2UnalignedQuery) { + constexpr size_t dim = 3; + std::vector stored(sq8_fp64::storage_bytes_count(dim)); + stored[0] = 2; + stored[1] = 5; + stored[2] = 9; + const double minimum = 100000000.0; + const double delta = 0.5; + std::memcpy(stored.data() + dim, &minimum, sizeof(minimum)); + std::memcpy(stored.data() + dim + sizeof(double), &delta, sizeof(delta)); + const double values[] = {100000000.5, 100000003.5, 100000002.0}; + alignas(double) uint8_t query[sizeof(values) + 1]; + std::memcpy(query + 1, values, sizeof(values)); + EXPECT_DOUBLE_EQ(spaces::L2_SQ8FP64_FP64(stored.data(), query + 1, dim), 7.5); +} From 5999e573c0e2d0150c0c171df1dbec3ec9935e4b Mon Sep 17 00:00:00 2001 From: Alex Dimitrov Date: Thu, 12 Nov 2026 17:28:56 +0200 Subject: [PATCH 7/8] Guard FP32 SQ8 L2 wrapper precision --- src/VecSim/spaces/quantized_kernels.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/VecSim/spaces/quantized_kernels.h b/src/VecSim/spaces/quantized_kernels.h index 7fa13530f..f4b38625d 100644 --- a/src/VecSim/spaces/quantized_kernels.h +++ b/src/VecSim/spaces/quantized_kernels.h @@ -10,6 +10,7 @@ #include #include +#include #include "VecSim/utils/alignment.h" #include "VecSim/spaces/quantized_distance_helpers.h" @@ -80,11 +81,7 @@ static inline typename StorageType::metadata_type StoredQueryL2(const void *stor Real res = 0; for (size_t i = 0; i < dim; i++) { - // Order matters. When the stored range sits far from zero, min_val and y_i are within a - // factor of two of each other, so min_val - y_i is exact (Sterbenz) and rounding happens - // only at the scale of the residual. Forming (min_val + delta*q_i) first would round at - // the scale of min_val and wipe out small residuals. When min_val and y_i are not close, - // the residual is large and either order is fine. + // Subtract min and query first to preserve small residuals with a large common offset. const Real query_value = load_unaligned(pVect2 + i * sizeof(Real)); const Real diff = delta * static_cast(StorageType::get_code(pVect1, i)) + (min_val - query_value); @@ -95,6 +92,7 @@ static inline typename StorageType::metadata_type StoredQueryL2(const void *stor template static inline float FP32_L2Sqr(const void *stored, const void *query, size_t dim) { + static_assert(std::is_same_v); return StoredQueryL2(stored, query, dim); } From 50fb62c3b21a096f15702b558c508fbe09a5028c Mon Sep 17 00:00:00 2001 From: Alex Dimitrov Date: Thu, 12 Nov 2026 17:42:57 +0200 Subject: [PATCH 8/8] Test FP64 SQ8 small residual precision --- tests/unit/test_sq8_fp64_components.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/unit/test_sq8_fp64_components.cpp b/tests/unit/test_sq8_fp64_components.cpp index 9fbef4e91..a66626d70 100644 --- a/tests/unit/test_sq8_fp64_components.cpp +++ b/tests/unit/test_sq8_fp64_components.cpp @@ -173,3 +173,17 @@ TEST(SQ8FP64Components, AsymmetricL2UnalignedQuery) { std::memcpy(query + 1, values, sizeof(values)); EXPECT_DOUBLE_EQ(spaces::L2_SQ8FP64_FP64(stored.data(), query + 1, dim), 7.5); } + +TEST(SQ8FP64Components, AsymmetricL2PreservesSmallResiduals) { + constexpr size_t dim = 3; + std::vector stored(sq8_fp64::storage_bytes_count(dim)); + stored[0] = 1; + stored[1] = 2; + stored[2] = 3; + const double minimum = 1e15; + const double delta = 0x1p-10; + std::memcpy(stored.data() + dim, &minimum, sizeof(minimum)); + std::memcpy(stored.data() + dim + sizeof(double), &delta, sizeof(delta)); + const double query[] = {minimum + 0.125, minimum + 0.25, minimum + 0.375}; + EXPECT_EQ(spaces::L2_SQ8FP64_FP64(stored.data(), query, dim), 112903.0 / 524288.0); +}