Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 60 additions & 26 deletions src/VecSim/index_factories/components/components_factory.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
#pragma once

#include "VecSim/spaces/spaces.h"
#include "VecSim/spaces/IP_space.h"
#include "VecSim/types/sq8_a8.h"
#include "VecSim/vec_sim_common.h"
#include "VecSim/vec_sim_index.h"
#include "VecSim/index_factories/components/preprocessors_factory.h"
Expand Down Expand Up @@ -40,22 +42,14 @@ template <typename DataType>
return alignment;
}

template <typename DataType, VecSimMetric Metric, bool WithMean, typename StorageType>
// Builds the quantization preprocessor, its container and the distance calculator of a quantized
// index. query_func compares a stored blob with a query blob, sym_func two stored blobs.
template <typename DataType, bool WithMean, typename Preprocessor, typename Calculator>
IndexComponents<DataType, float>
CreateQuantizedIndexComponents(const std::shared_ptr<VecSimAllocator> &allocator, size_t dim,
const float *mean_ptr) {
using Preprocessor = QuantPreprocessor<DataType, Metric, WithMean, StorageType>;
using QueryType = typename Preprocessor::QueryType;
unsigned char storage_alignment = 0, asym_storage_alignment = 0;

// Graph construction compares two stored quantized blobs; search compares a stored blob with a
// QueryType query. Both dispatchers report alignment for the stored operand.
auto sym_func = spaces::GetDistFunc<StorageType, float>(Metric, dim, &storage_alignment);
auto asym_func =
spaces::GetDistFunc<StorageType, float, QueryType>(Metric, dim, &asym_storage_alignment);
storage_alignment = spaces::combineAlignments(storage_alignment, asym_storage_alignment);
const unsigned char query_alignment = GetQueryAlignment<QueryType>(Metric, dim);

AssembleQuantizedIndexComponents(const std::shared_ptr<VecSimAllocator> &allocator, size_t dim,
const float *mean_ptr, spaces::dist_func_t<float> query_func,
spaces::dist_func_t<float> sym_func, unsigned char query_alignment,
unsigned char storage_alignment) {
PreprocessorInterface *pp = nullptr;
IndexCalculatorInterface<float> *calc = nullptr;

Expand All @@ -69,11 +63,10 @@ CreateQuantizedIndexComponents(const std::shared_ptr<VecSimAllocator> &allocator
}

pp = new (allocator) Preprocessor(allocator, dim, mean_vec);
calc = new (allocator) DistanceCalculatorWithNorm<DataType, float, Metric, StorageType>(
allocator, asym_func, sym_func, mean_sum_squares);
calc = new (allocator) Calculator(allocator, query_func, sym_func, mean_sum_squares);
} else {
pp = new (allocator) Preprocessor(allocator, dim);
calc = new (allocator) DistanceCalculatorCommon<float>(allocator, sym_func, asym_func);
calc = new (allocator) DistanceCalculatorCommon<float>(allocator, sym_func, query_func);
}

auto *container = new (allocator)
Expand All @@ -84,33 +77,74 @@ CreateQuantizedIndexComponents(const std::shared_ptr<VecSimAllocator> &allocator
return {calc, container};
}

// SQ8 IP queries take the A8 path where IP_SQ8_A8_GetDistFunc has a kernel for this dim and CPU:
// FP32 and FP16 queries are quantized once to int8 codes (vecsim_types::sq8_a8_query). Everywhere
// else, and for every other storage type and metric, queries stay asymmetric. arch_opt overrides
// the CPU features for the A8 choice (tests only).
template <typename DataType, VecSimMetric Metric, bool WithMean, typename StorageType>
IndexComponents<DataType, float>
CreateQuantizedIndexComponents(const std::shared_ptr<VecSimAllocator> &allocator, size_t dim,
const float *mean_ptr, const void *arch_opt = nullptr) {
if constexpr (std::is_same_v<StorageType, vecsim_types::sq8> && Metric == VecSimMetric_IP) {
unsigned char a8_storage_alignment = 0;
if (auto a8_func = spaces::IP_SQ8_A8_GetDistFunc(dim, &a8_storage_alignment, arch_opt)) {
unsigned char storage_alignment = 0;
auto sym_func =
spaces::GetDistFunc<StorageType, float>(Metric, dim, &storage_alignment);
return AssembleQuantizedIndexComponents<
DataType, WithMean, SQ8A8QueryPreprocessor<DataType, Metric, WithMean>,
SQ8A8QueryDistanceCalculatorWithNorm<DataType, float, Metric>>(
allocator, dim, mean_ptr, a8_func, sym_func,
vecsim_types::sq8_a8_query::blob_alignment,
spaces::combineAlignments(storage_alignment, a8_storage_alignment));
}
}

using Preprocessor = QuantPreprocessor<DataType, Metric, WithMean, StorageType>;
using QueryType = typename Preprocessor::QueryType;
unsigned char storage_alignment = 0, asym_storage_alignment = 0;

// Graph construction compares two stored quantized blobs; search compares a stored blob with a
// QueryType query. Both dispatchers report alignment for the stored operand.
auto sym_func = spaces::GetDistFunc<StorageType, float>(Metric, dim, &storage_alignment);
auto asym_func =
spaces::GetDistFunc<StorageType, float, QueryType>(Metric, dim, &asym_storage_alignment);
storage_alignment = spaces::combineAlignments(storage_alignment, asym_storage_alignment);
const unsigned char query_alignment = GetQueryAlignment<QueryType>(Metric, dim);

return AssembleQuantizedIndexComponents<
DataType, WithMean, Preprocessor,
DistanceCalculatorWithNorm<DataType, float, Metric, StorageType>>(
allocator, dim, mean_ptr, asym_func, sym_func, query_alignment, storage_alignment);
}

// Run-time overload: selects WithMean from whether mean_ptr is set.
template <typename DataType, VecSimMetric Metric, typename StorageType>
IndexComponents<DataType, float>
CreateQuantizedIndexComponents(const std::shared_ptr<VecSimAllocator> &allocator, size_t dim,
const float *mean_ptr) {
const float *mean_ptr, const void *arch_opt = nullptr) {
if (mean_ptr) {
return CreateQuantizedIndexComponents<DataType, Metric, true, StorageType>(allocator, dim,
mean_ptr);
return CreateQuantizedIndexComponents<DataType, Metric, true, StorageType>(
allocator, dim, mean_ptr, arch_opt);
}
return CreateQuantizedIndexComponents<DataType, Metric, false, StorageType>(allocator, dim,
mean_ptr);
mean_ptr, arch_opt);
}

template <typename DataType, VecSimMetric Metric, bool WithMean>
IndexComponents<DataType, float>
CreateSQ8IndexComponents(const std::shared_ptr<VecSimAllocator> &allocator, size_t dim,
const float *mean_ptr) {
const float *mean_ptr, const void *arch_opt = nullptr) {
return CreateQuantizedIndexComponents<DataType, Metric, WithMean, vecsim_types::sq8>(
allocator, dim, mean_ptr);
allocator, dim, mean_ptr, arch_opt);
}

template <typename DataType, VecSimMetric Metric>
IndexComponents<DataType, float>
CreateSQ8IndexComponents(const std::shared_ptr<VecSimAllocator> &allocator, size_t dim,
const float *mean_ptr) {
const float *mean_ptr, const void *arch_opt = nullptr) {
return CreateQuantizedIndexComponents<DataType, Metric, vecsim_types::sq8>(allocator, dim,
mean_ptr);
mean_ptr, arch_opt);
}

template <typename DataType, typename DistType>
Expand Down
7 changes: 7 additions & 0 deletions src/VecSim/index_factories/hnsw_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,13 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi
sizeof(QuantPreprocessor<float, VecSimMetric_L2, true>));
static_assert(sizeof(QuantPreprocessor<float, VecSimMetric_L2, false, sq4>) ==
sizeof(QuantPreprocessor<float, VecSimMetric_L2>));
// Nor does the query path: the A8 classes add no data members.
static_assert(sizeof(SQ8A8QueryPreprocessor<float, VecSimMetric_IP, true>) ==
sizeof(QuantPreprocessor<float, VecSimMetric_L2, true>));
static_assert(sizeof(SQ8A8QueryPreprocessor<float, VecSimMetric_IP>) ==
sizeof(QuantPreprocessor<float, VecSimMetric_L2>));
static_assert(sizeof(SQ8A8QueryDistanceCalculatorWithNorm<float, float, VecSimMetric_IP>) ==
sizeof(DistanceCalculatorWithNorm<float, float, VecSimMetric_L2>));
if (with_mean) {
est += allocations_overhead +
sizeof(DistanceCalculatorWithNorm<float, float, VecSimMetric_L2>);
Expand Down
4 changes: 4 additions & 0 deletions src/VecSim/spaces/IP/IP.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ float SQ8_FP32_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension
return SQ8_FP32_InnerProduct(pVect1v, pVect2v, dimension);
}

float SQ8_A8_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) {
return spaces::quantized::SQ8_A8_IPDistance(pVect1v, pVect2v, dimension);
}

/*
* Optimized asymmetric SQ8-FP16 inner product using algebraic identity:
* IP(x, y) = Σ(x_i * y_i)
Expand Down
6 changes: 6 additions & 0 deletions src/VecSim/spaces/IP/IP.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@ float SQ8_SQ8_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dime
// Vector layout: [uint8_t values (dim)] [min_val (float)] [delta (float)] [sum (float)]
float SQ8_SQ8_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension);

// SQ8 storage vs an A8 query, the scalar reference of the A8 kernels. It is not selected by the
// dispatcher; IP_SQ8_A8_GetDistFunc returns only SIMD kernels.
// pVect1 is storage (SQ8): [uint8_t values (dim)] [min_val] [delta] [sum]
// pVect2 is the A8 query (vecsim_types::sq8_a8_query): [int8 codes (dim)] [pad] [constants]
float SQ8_A8_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension);

float FP32_InnerProduct(const void *pVect1, const void *pVect2, size_t dimension);

double FP64_InnerProduct(const void *pVect1, const void *pVect2, size_t dimension);
Expand Down
96 changes: 96 additions & 0 deletions src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_A8.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/*
* Copyright (c) 2006-Present, Redis Ltd.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
* GNU Affero General Public License v3 (AGPLv3).
*/
#pragma once
#include "VecSim/spaces/space_includes.h"
#include "VecSim/spaces/quantized_distance_helpers.h"
#include <immintrin.h>

namespace {

/*
* SQ8 storage vs an A8 query (vecsim_types::sq8_a8_query). The stored uint8 codes and the int8
* query codes are exactly the unsigned-by-signed operands of vpdpbusd, which adds four products
* into each int32 lane without saturation. The rest of the distance is the shared double
* epilogue, so this kernel returns the scalar reference's float for the same code dot.
*
* Four independent accumulators take 256 bytes per iteration, up to three remaining 64-byte
* chunks go to their own accumulators, and one masked load of each operand reads the last
* `residual` bytes, so nothing past the codes is read.
*
* Exactness: |a_i * p_i| <= 255 * 128, and every lane and every partial sum of the reduction adds
* at most dim products, so all of them are exact in int32 while 32,640 * dim fits INT32_MAX
* (spaces::SQ8_A8_MAX_EXACT_SIMD_DIM).
*/

// A distinct zero for each accumulator. When the accumulators start from one shared zero register,
// GCC 13 copies each tied vpdpbusd accumulator twice per loop iteration; the empty asm makes every
// initial zero its own value, which removes the copies. It changes no arithmetic.
static inline __m512i SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI() {
__m512i zero = _mm512_setzero_si512();
__asm__ volatile("" : "+v"(zero));
return zero;
}

template <unsigned char residual> // 0..63
static inline int32_t SQ8_A8_CodeDot_AVX512F_BW_VL_VNNI(const uint8_t *codes,
const int8_t *query_codes, size_t dim) {
__m512i acc0 = SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI();
__m512i acc1 = SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI();
__m512i acc2 = SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI();
__m512i acc3 = SQ8_A8_ZeroAccumulator_AVX512F_BW_VL_VNNI();
const size_t complete = dim - residual;
size_t i = 0;
for (; i + 256 <= complete; i += 256) {
acc0 = _mm512_dpbusd_epi32(acc0, _mm512_loadu_si512(codes + i),
_mm512_loadu_si512(query_codes + i));
acc1 = _mm512_dpbusd_epi32(acc1, _mm512_loadu_si512(codes + i + 64),
_mm512_loadu_si512(query_codes + i + 64));
acc2 = _mm512_dpbusd_epi32(acc2, _mm512_loadu_si512(codes + i + 128),
_mm512_loadu_si512(query_codes + i + 128));
acc3 = _mm512_dpbusd_epi32(acc3, _mm512_loadu_si512(codes + i + 192),
_mm512_loadu_si512(query_codes + i + 192));
}
if (i < complete) {
acc0 = _mm512_dpbusd_epi32(acc0, _mm512_loadu_si512(codes + i),
_mm512_loadu_si512(query_codes + i));
i += 64;
if (i < complete) {
acc1 = _mm512_dpbusd_epi32(acc1, _mm512_loadu_si512(codes + i),
_mm512_loadu_si512(query_codes + i));
i += 64;
if (i < complete) {
acc2 = _mm512_dpbusd_epi32(acc2, _mm512_loadu_si512(codes + i),
_mm512_loadu_si512(query_codes + i));
i += 64;
}
}
}
__m512i acc = _mm512_add_epi32(_mm512_add_epi32(acc0, acc1), _mm512_add_epi32(acc2, acc3));
// The tail goes into the summed accumulator: accumulating it into one of the loop's
// accumulators makes GCC 13 copy that accumulator twice per loop iteration.
if constexpr (residual > 0) {
constexpr __mmask64 mask = (uint64_t(1) << residual) - 1;
acc = _mm512_dpbusd_epi32(acc, _mm512_maskz_loadu_epi8(mask, codes + i),
_mm512_maskz_loadu_epi8(mask, query_codes + i));
}
return _mm512_reduce_add_epi32(acc);
}

// Returns the distance 1 - IP; Cosine on normalized vectors uses the same kernel.
template <unsigned char residual> // 0..63
float SQ8_A8_InnerProductSIMD64_AVX512F_BW_VL_VNNI(const void *pVect1v, const void *pVect2v,
size_t dimension) {
const auto *stored = static_cast<const uint8_t *>(pVect1v);
const auto *query = static_cast<const uint8_t *>(pVect2v);
const int32_t code_dot = SQ8_A8_CodeDot_AVX512F_BW_VL_VNNI<residual>(
stored, reinterpret_cast<const int8_t *>(query), dimension);
return spaces::quantized::SQ8_A8_InnerProduct_FromCodeDot(stored, query, dimension, code_dot);
}

} // namespace
Loading
Loading