Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions src/VecSim/index_factories/components/components_factory.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "VecSim/vec_sim_index.h"
#include "VecSim/index_factories/components/preprocessors_factory.h"
#include "VecSim/spaces/computer/calculator.h"
#include "VecSim/spaces/computer/sq8_fp64_preprocessor.h"

template <typename DataType, typename DistType>
IndexComponents<DataType, DistType>
Expand Down Expand Up @@ -113,6 +114,27 @@ CreateSQ8IndexComponents(const std::shared_ptr<VecSimAllocator> &allocator, size
mean_ptr);
}

// Cosine storage and query inputs must already be normalized by the caller.
template <VecSimMetric Metric>
IndexComponents<double, double>
CreateSQ8FP64IndexComponents(const std::shared_ptr<VecSimAllocator> &allocator, size_t dim) {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we extend the existing quantized component factory and preprocessor to support FP64 instead of adding parallel implementations? Most of the setup and preprocessing flow is shared. Precision-specific arithmetic can remain separate where necessary.

unsigned char storage_alignment = 0;
const auto symmetric =
spaces::GetDistFunc<vecsim_types::sq8_fp64, double>(Metric, dim, &storage_alignment);
unsigned char asymmetric_alignment = 0;
const auto asymmetric = spaces::GetDistFunc<vecsim_types::sq8_fp64, double, double>(
Metric, dim, &asymmetric_alignment);
storage_alignment = spaces::combineAlignments(storage_alignment, asymmetric_alignment);
auto *const calculator =
new (allocator) DistanceCalculatorCommon<double>(allocator, symmetric, asymmetric);
auto *const container =
new (allocator) MultiPreprocessorsContainer<double, 1>(allocator, 0, storage_alignment);
auto *const preprocessor = new (allocator) SQ8FP64Preprocessor<Metric>(allocator, dim);
[[maybe_unused]] const int result = container->addPreprocessor(preprocessor);
assert(result != -1);
return {calculator, container};
}

template <typename DataType, typename DistType>
size_t EstimateComponentsMemory(VecSimMetric metric, bool is_normalized) {
size_t allocations_overhead = VecSimAllocator::getAllocationOverheadSize();
Expand Down
1 change: 1 addition & 0 deletions src/VecSim/spaces/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Build non optimized code in a single project without architecture optimization flag.
project(VectorSimilaritySpaces_no_optimization)
add_library(VectorSimilaritySpaces_no_optimization
sq8_fp64.cpp
L2/L2.cpp
IP/IP.cpp
)
Expand Down
127 changes: 127 additions & 0 deletions src/VecSim/spaces/computer/sq8_fp64_preprocessor.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
* Copyright (c) 2006-Present, Redis Ltd.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
* GNU Affero General Public License v3 (AGPLv3).
*/
#pragma once

#include "VecSim/spaces/computer/preprocessors.h"
#include "VecSim/types/sq8_fp64.h"

// Cosine inputs must already be normalized by the caller.
template <VecSimMetric Metric>
class SQ8FP64Preprocessor : public PreprocessorInterface {
static_assert(Metric == VecSimMetric_L2 || Metric == VecSimMetric_IP ||
Metric == VecSimMetric_Cosine);

public:
SQ8FP64Preprocessor(std::shared_ptr<VecSimAllocator> allocator, size_t dim);

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering if we extend QuantPreprocessor to support FP64 instead of introducing a parallel implementation? Most of the preprocessing flow is identical. The arithmetic and metadata types could follow the storage type, with direct division retained for FP64 where reciprocal overflow matters. Is there a concrete constraint that makes sharing the implementation impractical?

void preprocess(const void *original, void *&storage, void *&query, size_t &storage_size,
size_t &query_size, unsigned char storage_alignment,
unsigned char query_alignment) const override;
void preprocessForStorage(const void *original, void *&storage, size_t &storage_size,
unsigned char alignment) const override;
void preprocessQuery(const void *original, void *&query, size_t &query_size,
unsigned char alignment) const override;
void preprocessStorageInPlace(void *original, size_t input_size) const override;

private:
static uint8_t to_code(double scaled);
void quantize(const double *input, uint8_t *output) const;
const size_t dim;
};

template <VecSimMetric Metric>
SQ8FP64Preprocessor<Metric>::SQ8FP64Preprocessor(std::shared_ptr<VecSimAllocator> allocator,
size_t dim)
: PreprocessorInterface(allocator), dim(dim) {
assert(dim > 0);
}

template <VecSimMetric Metric>
uint8_t SQ8FP64Preprocessor<Metric>::to_code(double scaled) {
if (!(scaled > 0.0)) {
return 0;
}

if (scaled >= vecsim_types::sq8_fp64::max_code) {
return vecsim_types::sq8_fp64::max_code;
}

return static_cast<uint8_t>(scaled + 0.5);
}

template <VecSimMetric Metric>
void SQ8FP64Preprocessor<Metric>::quantize(const double *input, uint8_t *output) const {
const auto [min_it, max_it] = std::minmax_element(input, input + dim);
const double minimum = *min_it;
const double range = *max_it - minimum;
const double delta = range == 0.0 ? 1.0 : range / vecsim_types::sq8_fp64::max_code;
double sum = 0.0;
double squares = 0.0;
for (size_t i = 0; i < dim; ++i) {
const uint8_t code = to_code((input[i] - minimum) / delta);
output[i] = code;
const double value = minimum + delta * code;
sum += value;
if constexpr (Metric == VecSimMetric_L2) {
squares += value * value;
}
}
const double metadata[] = {minimum, delta, sum, squares};

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The FP64 IP and L2 kernels only consume the codes, minimum, and delta. Do we need the stored/query sums and sums of squares yet? Unless another consumer requires them, could we avoid computing and storing unused metadata?

std::memcpy(output + vecsim_types::sq8_fp64::data_bytes_count(dim), metadata,
vecsim_types::sq8_fp64::storage_metadata_count<Metric>() * sizeof(double));
}

template <VecSimMetric Metric>
void SQ8FP64Preprocessor<Metric>::preprocess(const void *original, void *&storage, void *&query,
size_t &storage_size, size_t &query_size,
unsigned char storage_alignment,
unsigned char query_alignment) const {
assert(!storage && !query);
assert(&storage_size != &query_size);
preprocessForStorage(original, storage, storage_size, storage_alignment);
preprocessQuery(original, query, query_size, query_alignment);
}

template <VecSimMetric Metric>
void SQ8FP64Preprocessor<Metric>::preprocessForStorage(const void *original, void *&storage,
size_t &storage_size,
unsigned char alignment) const {
assert(!storage);
storage_size = vecsim_types::sq8_fp64::storage_bytes_count<Metric>(dim);
storage = this->allocator->allocate_aligned(storage_size, alignment);
quantize(static_cast<const double *>(original), static_cast<uint8_t *>(storage));
}

template <VecSimMetric Metric>
void SQ8FP64Preprocessor<Metric>::preprocessQuery(const void *original, void *&query,
size_t &query_size,
unsigned char alignment) const {
assert(!query);
const auto *const input = static_cast<const double *>(original);
double sum = 0.0;
double squares = 0.0;
for (size_t i = 0; i < dim; ++i) {
sum += input[i];
if constexpr (Metric == VecSimMetric_L2) {
squares += input[i] * input[i];
}
}
query_size = vecsim_types::sq8_fp64::query_bytes_count<Metric>(dim);
query = this->allocator->allocate_aligned(query_size, alignment);
std::memcpy(query, original, dim * sizeof(double));
const double metadata[] = {sum, squares};
std::memcpy(static_cast<uint8_t *>(query) + dim * sizeof(double), metadata,
vecsim_types::sq8_fp64::query_metadata_count<Metric>() * sizeof(double));
}

template <VecSimMetric Metric>
void SQ8FP64Preprocessor<Metric>::preprocessStorageInPlace(void *original,
size_t input_size) const {
assert(input_size >= vecsim_types::sq8_fp64::storage_bytes_count<Metric>(dim));
quantize(static_cast<const double *>(original), static_cast<uint8_t *>(original));
}
49 changes: 28 additions & 21 deletions src/VecSim/spaces/quantized_kernels.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

#include <cstddef>
#include <cstdint>
#include <type_traits>
#include "VecSim/utils/alignment.h"
#include "VecSim/spaces/quantized_distance_helpers.h"

Expand All @@ -18,11 +19,12 @@
//
// Scalar kernels for scalar quantized storage, generic over a StorageType that provides
// get_code(data, i), data_bytes_count(dim) and the MIN_VAL / DELTA / SUM / SUM_SQUARES metadata
// indices (stored after the codes) and SUM_QUERY (stored after the FP32 query values).
// It also provides the code dot hooks code_dot(a, b, dim) and code_dot_fp32(codes, y, dim).
// indices (stored after the codes) and SUM_QUERY (stored after the query values).
// StoredQueryL2 also requires metadata_type, matching the query value type.
// The float IP helpers use code_dot(a, b, dim) and code_dot_fp32(codes, y, dim).
//
// Stored layout: [codes (data_bytes_count(dim) bytes)] [min_val] [delta] [sum] [sum_squares (L2)]
// FP32 query layout: [float values (dim)] [y_sum] [y_sum_squares (L2 only)]
// Query layout: [values (dim)] [y_sum] [y_sum_squares (L2 only)]
// SQ4 L2 uses only the query body; its two trailing FP32 slots are reserved and zero-initialized.
//
// The operand order in these bodies is load-bearing and relies on FP addition not being
Expand Down Expand Up @@ -58,37 +60,42 @@ static inline float FP32_InnerProduct_Impl(const void *stored, const void *query
}

/*
* Asymmetric stored-FP32 L2 squared distance, accumulated directly per component:
* Asymmetric stored-query L2 squared distance, accumulated directly per component:
* ||x - y||^2 = sum((dequant(x_i) - y_i)^2), where dequant(x_i) = min_val + delta * q_i
*
* Accumulating residuals keeps each component's rounding at the scale of |x_i - y_i|. Expanding
* to ||x||^2 + ||y||^2 - 2*IP(x, y) would round at the scale of the norms, which loses the
* distance when x and y share a large common offset relative to their spread.
*/
template <typename StorageType>
static inline float FP32_L2Sqr(const void *stored, const void *query, size_t dim) {
// Storage metadata follows a byte payload and is not necessarily float-aligned.
const auto *pVect1 = static_cast<const uint8_t *>(stored);
const auto *pVect2 = static_cast<const float *>(query);

const auto *params1 = pVect1 + StorageType::data_bytes_count(dim);
const float min_val = load_unaligned<float>(params1 + StorageType::MIN_VAL * sizeof(float));
const float delta = load_unaligned<float>(params1 + StorageType::DELTA * sizeof(float));

float res = 0;
static inline typename StorageType::metadata_type StoredQueryL2(const void *stored,
const void *query, size_t dim) {
using Real = typename StorageType::metadata_type;
// Storage metadata follows a byte payload and is not necessarily aligned.
const auto *const pVect1 = static_cast<const uint8_t *>(stored);
const auto *const pVect2 = static_cast<const uint8_t *>(query);

const auto *const params1 = pVect1 + StorageType::data_bytes_count(dim);
const Real min_val = load_unaligned<Real>(params1 + StorageType::MIN_VAL * sizeof(Real));
const Real delta = load_unaligned<Real>(params1 + StorageType::DELTA * sizeof(Real));

Real res = 0;
for (size_t i = 0; i < dim; i++) {
// Order matters. When the stored range sits far from zero, min_val and y_i are within a
// factor of two of each other, so min_val - y_i is exact (Sterbenz) and rounding happens
// only at the scale of the residual. Forming (min_val + delta*q_i) first would round at
// the scale of min_val and wipe out small residuals. When min_val and y_i are not close,
// the residual is large and either order is fine.
float diff =
delta * static_cast<float>(StorageType::get_code(pVect1, i)) + (min_val - pVect2[i]);
// Subtract min and query first to preserve small residuals with a large common offset.
const Real query_value = load_unaligned<Real>(pVect2 + i * sizeof(Real));
const Real diff =
delta * static_cast<Real>(StorageType::get_code(pVect1, i)) + (min_val - query_value);
res += diff * diff;
}
return res;
}

template <typename StorageType>
static inline float FP32_L2Sqr(const void *stored, const void *query, size_t dim) {
static_assert(std::is_same_v<typename StorageType::metadata_type, float>);
return StoredQueryL2<StorageType>(stored, query, dim);
}

/*
* Symmetric stored-stored inner product using precomputed sums:
* IP = min1*sum2 + min2*sum1 - dim*min1*min2 + delta1*delta2*sum(q1_i*q2_i)
Expand Down
35 changes: 35 additions & 0 deletions src/VecSim/spaces/spaces.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
#include "VecSim/types/bfloat16.h"
#include "VecSim/types/float16.h"
#include "VecSim/types/sq8.h"
#include "VecSim/types/sq8_fp64.h"
#include "VecSim/spaces/sq8_fp64.h"
#include "VecSim/types/sq4.h"
#include "VecSim/spaces/IP/IP.h"
#include "VecSim/spaces/L2/L2.h"
Expand Down Expand Up @@ -104,6 +106,39 @@ dist_func_t<float> GetDistFunc<uint8_t, float>(VecSimMetric metric, size_t dim,
throw std::invalid_argument("Invalid metric");
}

template <>
dist_func_t<double> GetDistFunc<vecsim_types::sq8_fp64, double>(VecSimMetric metric, size_t dim,
unsigned char *alignment) {
if (alignment) {
*alignment = 0;
}
switch (metric) {
case VecSimMetric_Cosine:
case VecSimMetric_IP:
return IP_SQ8FP64;
case VecSimMetric_L2:
return L2_SQ8FP64;
}
throw std::invalid_argument("Invalid metric");
}

template <>
dist_func_t<double> GetDistFunc<vecsim_types::sq8_fp64, double, double>(VecSimMetric metric,
size_t dim,
unsigned char *alignment) {
if (alignment) {
*alignment = 0;
}
switch (metric) {
case VecSimMetric_Cosine:
case VecSimMetric_IP:
return IP_SQ8FP64_FP64;
case VecSimMetric_L2:
return L2_SQ8FP64_FP64;
}
throw std::invalid_argument("Invalid metric");
}

template <>
dist_func_t<float> GetDistFunc<vecsim_types::sq8, float>(VecSimMetric metric, size_t dim,
unsigned char *alignment) {
Expand Down
68 changes: 68 additions & 0 deletions src/VecSim/spaces/sq8_fp64.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* Copyright (c) 2006-Present, Redis Ltd.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
* GNU Affero General Public License v3 (AGPLv3).
*/
#include "VecSim/spaces/sq8_fp64.h"
#include "VecSim/spaces/quantized_kernels.h"
#include "VecSim/types/sq8_fp64.h"
#include "VecSim/utils/alignment.h"

namespace spaces {
namespace {
double metadata(const void *blob, size_t dim, size_t index) {
return load_unaligned<double>(static_cast<const uint8_t *>(blob) +
vecsim_types::sq8_fp64::data_bytes_count(dim) +
index * sizeof(double));
}

double storedL2(const void *left, const void *right, size_t dim) {
const auto *const a = static_cast<const uint8_t *>(left);
const auto *const b = static_cast<const uint8_t *>(right);
const double min_a = metadata(left, dim, vecsim_types::sq8_fp64::MIN_VAL);
const double delta_a = metadata(left, dim, vecsim_types::sq8_fp64::DELTA);
const double min_b = metadata(right, dim, vecsim_types::sq8_fp64::MIN_VAL);
const double delta_b = metadata(right, dim, vecsim_types::sq8_fp64::DELTA);
double result = 0.0;
for (size_t i = 0; i < dim; ++i) {
const double residual =
(min_a - min_b) + (delta_a - delta_b) * a[i] + delta_b * (int(a[i]) - int(b[i]));
result += residual * residual;
}
return result;
}

template <bool Asymmetric>
double innerProduct(const void *left, const void *right, size_t dim) {
const auto *const a = static_cast<const uint8_t *>(left);
const auto *const b = static_cast<const uint8_t *>(right);
const double min_a = metadata(left, dim, vecsim_types::sq8_fp64::MIN_VAL);
const double delta_a = metadata(left, dim, vecsim_types::sq8_fp64::DELTA);
const double min_b = Asymmetric ? 0.0 : metadata(right, dim, vecsim_types::sq8_fp64::MIN_VAL);
const double delta_b = Asymmetric ? 0.0 : metadata(right, dim, vecsim_types::sq8_fp64::DELTA);
double result = 0.0;
for (size_t i = 0; i < dim; ++i) {
const double y =
Asymmetric ? load_unaligned<double>(b + i * sizeof(double)) : min_b + delta_b * b[i];
result += (min_a + delta_a * a[i]) * y;
}
return 1.0 - result;
}
} // namespace

double L2_SQ8FP64(const void *left, const void *right, size_t dim) {
return storedL2(left, right, dim);
}
double L2_SQ8FP64_FP64(const void *left, const void *right, size_t dim) {
return quantized::StoredQueryL2<vecsim_types::sq8_fp64>(left, right, dim);
}
double IP_SQ8FP64(const void *left, const void *right, size_t dim) {
return innerProduct<false>(left, right, dim);
}
double IP_SQ8FP64_FP64(const void *left, const void *right, size_t dim) {
return innerProduct<true>(left, right, dim);
}
} // namespace spaces
18 changes: 18 additions & 0 deletions src/VecSim/spaces/sq8_fp64.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* Copyright (c) 2006-Present, Redis Ltd.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
* GNU Affero General Public License v3 (AGPLv3).
*/
#pragma once

#include <cstddef>

namespace spaces {
double L2_SQ8FP64(const void *left, const void *right, size_t dim);
double L2_SQ8FP64_FP64(const void *left, const void *right, size_t dim);
double IP_SQ8FP64(const void *left, const void *right, size_t dim);
double IP_SQ8FP64_FP64(const void *left, const void *right, size_t dim);
} // namespace spaces
Loading
Loading