Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 39 additions & 5 deletions src/VecSim/index_factories/hnsw_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@
#include "VecSim/algorithms/hnsw/hnsw.h"
#include "VecSim/types/bfloat16.h"
#include "VecSim/types/float16.h"
#include "VecSim/types/sq4.h"

#include <type_traits>

using bfloat16 = vecsim_types::bfloat16;
using float16 = vecsim_types::float16;
using sq8 = vecsim_types::sq8;
using sq4 = vecsim_types::sq4;

namespace HNSWFactory {

Expand Down Expand Up @@ -56,10 +58,13 @@ template <VecSimMetric Metric, typename StorageType = sq8>
VecSimType type,
VecSimMetric resolved_metric) {
// Reject unknown quantizers instead of silently creating an unquantized index.
if (quant_type != VecSimQuant_SQ8) {
if (quant_type != VecSimQuant_SQ8 && quant_type != VecSimQuant_SQ4) {
return false;
}
// SQ8 kernels accept only FLOAT32 and FLOAT16 input.
if (quant_type == VecSimQuant_SQ4 && type != VecSimType_FLOAT32) {
return false;
}
// SQ8 additionally accepts FLOAT16 input.
if (type != VecSimType_FLOAT32 && type != VecSimType_FLOAT16) {
return false;
}
Expand Down Expand Up @@ -106,6 +111,15 @@ VecSimIndex *NewIndex(const VecSimParams *params, bool is_normalized) {
return NULL;
}

if (hnswParams->quantType == VecSimQuant_SQ4) {
if (metric == VecSimMetric_L2) {
return NewIndex_Quantized<float, VecSimMetric_L2, sq4>(
hnswParams, abstractInitParams, mean_ptr);
}
return NewIndex_Quantized<float, VecSimMetric_IP, sq4>(hnswParams, abstractInitParams,
mean_ptr);
}

if (hnswParams->type == VecSimType_FLOAT32) {
if (metric == VecSimMetric_L2) {
return NewIndex_Quantized<float, VecSimMetric_L2>(hnswParams, abstractInitParams,
Expand Down Expand Up @@ -196,7 +210,12 @@ size_t EstimateInitialSize(const HNSWParams *params, bool is_normalized, bool wi
ResolveQuantizedMetric(params->metric, is_normalized))) {
throw std::invalid_argument("Unsupported quantization params for HNSW index");
}
// Template arguments do not affect these component sizes.
// Template arguments do not affect these component sizes. The storage type does not
// either; the asserts keep that true, so one estimate serves SQ8 and SQ4.
static_assert(sizeof(QuantPreprocessor<float, VecSimMetric_L2, true, sq4>) ==
sizeof(QuantPreprocessor<float, VecSimMetric_L2, true>));
static_assert(sizeof(QuantPreprocessor<float, VecSimMetric_L2, false, sq4>) ==
sizeof(QuantPreprocessor<float, VecSimMetric_L2>));
if (with_mean) {
est += allocations_overhead +
sizeof(DistanceCalculatorWithNorm<float, float, VecSimMetric_L2>);
Expand Down Expand Up @@ -245,7 +264,12 @@ size_t EstimateElementSize(const HNSWParams *params, bool with_mean) {

size_t stored_data_size;
// Preserve the element estimator's existing contract: return a size and validate in NewIndex.
if (params->quantType == VecSimQuant_SQ8) {
if (params->quantType == VecSimQuant_SQ4) {
stored_data_size =
params->metric == VecSimMetric_L2
? GetQuantizedStoredDataSize<VecSimMetric_L2, sq4>(params->dim, with_mean)
: GetQuantizedStoredDataSize<VecSimMetric_IP, sq4>(params->dim, with_mean);
} else if (params->quantType == VecSimQuant_SQ8) {
if (params->metric == VecSimMetric_L2) {
stored_data_size = GetQuantizedStoredDataSize<VecSimMetric_L2>(params->dim, with_mean);
} else {
Expand Down Expand Up @@ -336,7 +360,8 @@ static bool InitializeParams(std::ifstream &source_params, HNSWParams &params,
// VecSimQuantType, so it must never be loaded into the enum.
std::underlying_type_t<VecSimQuantType> raw_quant_type;
Serializer::readBinaryPOD(source_params, raw_quant_type);
if (raw_quant_type != VecSimQuant_NONE && raw_quant_type != VecSimQuant_SQ8) {
if (raw_quant_type != VecSimQuant_NONE && raw_quant_type != VecSimQuant_SQ8 &&
raw_quant_type != VecSimQuant_SQ4) {
return false;
}
params.quantType = static_cast<VecSimQuantType>(raw_quant_type);
Expand Down Expand Up @@ -392,6 +417,15 @@ VecSimIndex *NewIndex(const std::string &location, bool is_normalized) {
return NULL;
}

if (params.quantType == VecSimQuant_SQ4) {
if (metric == VecSimMetric_L2) {
return LoadIndex_Quantized<float, VecSimMetric_L2, sq4>(
input, params, abstractInitParams, version, mean_ptr);
}
return LoadIndex_Quantized<float, VecSimMetric_IP, sq4>(
input, params, abstractInitParams, version, mean_ptr);
}

if (params.type == VecSimType_FLOAT32) {
if (metric == VecSimMetric_L2) {
return LoadIndex_Quantized<float, VecSimMetric_L2>(
Expand Down
3 changes: 3 additions & 0 deletions src/VecSim/index_factories/tiered_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) {
inline size_t EstimateInitialSize(const TieredIndexParams *params) {
const auto &hnsw_params = params->primaryIndexParams->algoParams.hnswParams;

if (hnsw_params.quantType == VecSimQuant_SQ4) {
throw std::invalid_argument("Tiered HNSW SQ4 is not supported");
}
const bool requires_training = RequiresQuantizationTraining(params);
const bool with_mean = requires_training || hnsw_params.quantParams != nullptr;
size_t est = HNSWFactory::EstimateInitialSize(&hnsw_params, true, with_mean);
Expand Down
10 changes: 10 additions & 0 deletions src/VecSim/spaces/IP/IP.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,21 @@
#include "VecSim/types/bfloat16.h"
#include "VecSim/types/float16.h"
#include "VecSim/types/sq8.h"
#include "VecSim/types/sq4.h"
#include "VecSim/utils/alignment.h"

using bfloat16 = vecsim_types::bfloat16;
using float16 = vecsim_types::float16;
using sq8 = vecsim_types::sq8;
using sq4 = vecsim_types::sq4;

float SQ4_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) {
return 1.0f - spaces::quantized::FP32_InnerProduct_Impl<sq4>(pVect1v, pVect2v, dimension);
}

float SQ4_SQ4_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) {
return 1.0f - spaces::quantized::Stored_InnerProduct_Impl<sq4>(pVect1v, pVect2v, dimension);
}

/*
* Optimized asymmetric SQ8-FP32 inner product using algebraic identity:
Expand Down
4 changes: 4 additions & 0 deletions src/VecSim/spaces/IP/IP.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@

#include <cstdlib>

// Packed SQ4 storage, compared with FP32 queries or another SQ4 vector.
float SQ4_FP32_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension);
float SQ4_SQ4_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension);

// SQ8-FP32: Common inner product implementation that returns the raw inner product value
// (not distance). Used by SQ8_FP32_InnerProduct, SQ8_FP32_Cosine, and SQ8_FP32_L2Sqr.
// pVect1 is storage (SQ8): [uint8_t values (dim)] [min_val] [delta] [x_sum] [x_sum_squares (L2
Expand Down
18 changes: 18 additions & 0 deletions src/VecSim/spaces/IP_space.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,24 @@ using bfloat16 = vecsim_types::bfloat16;
using float16 = vecsim_types::float16;

namespace spaces {
// SQ4-FP32: packed storage and unaligned metadata, so no alignment requirement.
dist_func_t<float> IP_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment,
const void *arch_opt) {
if (alignment) {
*alignment = 0;
}
return SQ4_FP32_InnerProduct;
}

// SQ4-SQ4: packed storage and unaligned metadata, so no alignment requirement.
dist_func_t<float> IP_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment,
const void *arch_opt) {
if (alignment) {
*alignment = 0;
}
return SQ4_SQ4_InnerProduct;
}

// SQ8-FP32: asymmetric distance between SQ8 storage and FP32 query
dist_func_t<float> IP_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment,
const void *arch_opt) {
Expand Down
6 changes: 6 additions & 0 deletions src/VecSim/spaces/IP_space.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@
#include "VecSim/spaces/spaces.h"

namespace spaces {
// Packed SQ4 storage has no alignment requirement, including its metadata.
dist_func_t<float> IP_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr,
const void *arch_opt = nullptr);
dist_func_t<float> IP_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment = nullptr,
const void *arch_opt = nullptr);

// SQ8-FP32: asymmetric distance between FP32 query and SQ8 storage
dist_func_t<float> IP_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr,
const void *arch_opt = nullptr);
Expand Down
36 changes: 36 additions & 0 deletions src/VecSim/spaces/L2/L2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,48 @@
#include "VecSim/types/bfloat16.h"
#include "VecSim/types/float16.h"
#include "VecSim/types/sq8.h"
#include "VecSim/types/sq4.h"
#include "VecSim/utils/alignment.h"
#include <iostream>

using bfloat16 = vecsim_types::bfloat16;
using float16 = vecsim_types::float16;
using sq8 = vecsim_types::sq8;
using sq4 = vecsim_types::sq4;

float SQ4_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) {
const auto *codes = static_cast<const uint8_t *>(pVect1v);
const auto *query = static_cast<const float *>(pVect2v);
const auto *params = codes + sq4::data_bytes_count(dimension);
const float min_val = load_unaligned<float>(params + sq4::MIN_VAL * sizeof(float));
const float delta = load_unaligned<float>(params + sq4::DELTA * sizeof(float));

float result = 0.0f;
const size_t full_bytes = dimension / 2;
for (size_t j = 0; j < full_bytes; ++j) {
// Load once and use both nibbles. Keep min - query before adding delta * code so
// rounding stays at the residual's scale, and accumulate in coordinate order.
const uint8_t byte = codes[j];
const float even = delta * float(byte & sq4::max_code) + (min_val - query[2 * j]);
const float odd = delta * float(byte >> sq4::bits_per_code) + (min_val - query[2 * j + 1]);
result += even * even;
result += odd * odd;
}
if (dimension % 2) {
// Only the low nibble is a coordinate; never read a query value for the padding.
const float last =
delta * float(codes[full_bytes] & sq4::max_code) + (min_val - query[dimension - 1]);
result += last * last;
}
return result;
}

// SQ4-SQ4 L2 squared distance with the same identity as SQ8-SQ8:
// ||x - y||^2 = ||x||^2 + ||y||^2 - 2*IP(x, y)
// from the stored SUM_SQUARES metadata and the stored-to-stored inner product.
float SQ4_SQ4_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension) {
return spaces::quantized::Stored_L2Sqr<sq4>(pVect1v, pVect2v, dimension);
}

/*
* Asymmetric SQ8-FP32 L2 squared distance, accumulated directly per component:
Expand Down
4 changes: 4 additions & 0 deletions src/VecSim/spaces/L2/L2.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@

#include <cstdlib>

// Packed SQ4 storage, compared with FP32 queries or another SQ4 vector.
float SQ4_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension);
float SQ4_SQ4_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension);

// SQ8-FP32: pVect1v vector of type uint8 (SQ8) and pVect2v vector of type fp32
float SQ8_FP32_L2Sqr(const void *pVect1v, const void *pVect2v, size_t dimension);

Expand Down
18 changes: 18 additions & 0 deletions src/VecSim/spaces/L2_space.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,24 @@ using float16 = vecsim_types::float16;

namespace spaces {

// SQ4-FP32: packed storage and unaligned metadata, so no alignment requirement.
dist_func_t<float> L2_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment,
const void *arch_opt) {
if (alignment) {
*alignment = 0;
}
return SQ4_FP32_L2Sqr;
}

// SQ4-SQ4: packed storage and unaligned metadata, so no alignment requirement.
dist_func_t<float> L2_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment,
const void *arch_opt) {
if (alignment) {
*alignment = 0;
}
return SQ4_SQ4_L2Sqr;
}

// SQ8-FP32: asymmetric L2 distance between SQ8 storage and FP32 query
dist_func_t<float> L2_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment,
const void *arch_opt) {
Expand Down
6 changes: 6 additions & 0 deletions src/VecSim/spaces/L2_space.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@ dist_func_t<float> L2_INT8_GetDistFunc(size_t dim, unsigned char *alignment = nu
const void *arch_opt = nullptr);
dist_func_t<float> L2_UINT8_GetDistFunc(size_t dim, unsigned char *alignment = nullptr,
const void *arch_opt = nullptr);
// SQ4-FP32: packed storage has no alignment requirement, including its metadata.
dist_func_t<float> L2_SQ4_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr,
const void *arch_opt = nullptr);
// SQ4-SQ4: packed storage has no alignment requirement, including its metadata.
dist_func_t<float> L2_SQ4_SQ4_GetDistFunc(size_t dim, unsigned char *alignment = nullptr,
const void *arch_opt = nullptr);
// SQ8-FP32: asymmetric L2 distance between FP32 query and SQ8 storage
dist_func_t<float> L2_SQ8_FP32_GetDistFunc(size_t dim, unsigned char *alignment = nullptr,
const void *arch_opt = nullptr);
Expand Down
38 changes: 25 additions & 13 deletions src/VecSim/spaces/computer/preprocessors.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include "VecSim/memory/memory_utils.h"
#include "VecSim/types/float16.h"
#include "VecSim/types/sq8.h"
#include "VecSim/types/sq4.h"
#include "VecSim/utils/vecsim_stl.h"

class PreprocessorInterface : public VecsimBaseObject {
Expand Down Expand Up @@ -143,8 +144,8 @@ class CosinePreprocessor : public PreprocessorInterface {

/*
* QuantPreprocessor is a preprocessor that quantizes storage vectors from DataType to a
* lower precision representation defined by StorageType (SQ8, the current storage type,
* stores one uint8_t code per coordinate).
* lower precision representation (SQ8 stores one uint8_t code per coordinate, SQ4 packs
* two 4-bit codes per byte).
* Query vectors remain as DataType for asymmetric distance computation.
*
* The quantized storage blob contains the quantized values along with metadata (min value,
Expand All @@ -153,7 +154,7 @@ class CosinePreprocessor : public PreprocessorInterface {
* and then scaling the values to fit in the storage type's code range.
*
* Storage layout:
* | quantized_values[dim] | min_val | delta | x_sum | (x_sum_squares for L2 only) |
* | quantized_values (packed for SQ4) | min_val | delta | x_sum | (L2 sum_squares) |
* where, writing x_r[i] = min_val + delta * a[i] for the value a blob actually represents:
* x_sum = Σx_r[i]: sum of the reconstructed values,
* x_sum_squares = Σx_r[i]²: sum of their squares.
Expand All @@ -178,11 +179,13 @@ class CosinePreprocessor : public PreprocessorInterface {
* store metric-specific FP32 values from the query body to accelerate asymmetric distance
* computation:
* - For IP/Cosine: y_sum = Σy_i (sum of query values)
* - For L2: y_sum = Σy_i (sum of query values), y_sum_squares = Σy_i² (sum of squared query values)
* - For SQ8 L2: y_sum = Σy_i, y_sum_squares = Σy_i²
* - For SQ4 L2: two zero-initialized reserved FP32 slots; direct residual kernels need no sums.
*
* Query blob layout:
* - For IP/Cosine: | query_values[dim] | y_sum |
* - For L2: | query_values[dim] | y_sum | y_sum_squares |
* SQ4 L2 keeps the same layout, with both trailing slots reserved and zero-initialized.
*
* Query metadata is always FP32. The query blob size is:
* - For IP/Cosine: dim * sizeof(DataType) + 1 * sizeof(float)
Expand All @@ -201,6 +204,7 @@ class CosinePreprocessor : public PreprocessorInterface {
* where y_sum = Σy_i is precomputed and stored in the query blob.
*
* For L2:
* The identity below explains the metadata; direct residual query kernels do not consume it.
* ||x_r - y||² = Σx_r[i]² - 2*Σ(x_r[i] * y_i) + Σy_i²
* = x_sum_squares - 2 * IP(x_r, y) + y_sum_squares
* where:
Expand Down Expand Up @@ -389,7 +393,7 @@ class QuantPreprocessor : public PreprocessorInterface {
}

// Metadata uses MetadataType. Use memcpy because the metadata offset (after the
// quantized body) is not guaranteed to be sizeof(MetadataType)-aligned.
// quantized body, packed for SQ4) is not guaranteed to be sizeof(MetadataType)-aligned.
void *meta_dst = quantized + StorageType::data_bytes_count(this->dim);
MetadataType buf[5] = {min_val, delta, sum};
size_t n = 3;
Expand Down Expand Up @@ -512,9 +516,9 @@ class QuantPreprocessor : public PreprocessorInterface {
/**
* Preprocesses the original blob into separate storage and query blobs.
*
* Storage vectors are quantized to StorageType scalar codes (SQ8: one uint8_t per
* coordinate), with metadata (min, delta, sum, and sum_squares for L2) appended for
* distance reconstruction.
* Storage vectors are quantized to scalar codes (SQ8: one uint8_t per coordinate, SQ4: two
* 4-bit codes packed per byte), with metadata (min, delta, sum, and
* sum_squares for L2) appended for distance reconstruction.
*
* Query vectors use QueryType for asymmetric distance computation, with a precomputed
* sum (for IP/Cosine) or sum of squares (for L2) appended for efficient distance calculation.
Expand Down Expand Up @@ -575,12 +579,13 @@ class QuantPreprocessor : public PreprocessorInterface {
* the original query values.
*
* - For IP/Cosine: y_sum = Σy_i (sum of query values)
* - For L2: y_sum = Σy_i (sum of query values), y_sum_squares = Σy_i² (sum of squared query
* values)
* - For SQ8 L2: y_sum = Σy_i, y_sum_squares = Σy_i²
* - For SQ4 L2: two zero-initialized reserved FP32 slots; no query reductions are needed.
*
* Query blob layout:
* - For IP/Cosine: | query_values[dim] | y_sum |
* - For L2: | query_values[dim] | y_sum | y_sum_squares |
* SQ4 L2 reserves and zero-initializes both trailing slots instead of computing sums.
*
* Query blob size:
* - For IP/Cosine: dim * sizeof(QueryType) + 1 * sizeof(float)
Expand All @@ -606,7 +611,14 @@ class QuantPreprocessor : public PreprocessorInterface {
// Compute and write FP32 query metadata after the query body. The metadata offset is
// body_bytes, which is not guaranteed to be 4-byte aligned for FP16 query bodies.
void *metadata_dst = static_cast<uint8_t *>(blob) + body_bytes;
assign_query_metadata(query_values, input, metadata_dst);
if constexpr (std::is_same_v<StorageType, vecsim_types::sq4> && Metric == VecSimMetric_L2) {
// SQ4 L2 computes residuals directly, so query sums are unused. Keep initialized
// padding for SIMD tails that load a final query pair when dim is odd.
const MetadataType padding[2] = {};
memcpy(metadata_dst, padding, sizeof(padding));
} else {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why? and why not in assign_query_metadata?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SQ8’s FP32-query L2 path also does not use these sums. Both kernels accumulate residual squares directly: we found precision loss from cancellation when expanding L2 into squared norms and a dot product. SQ8 preprocessing still computes the unused sums on this path, whereas SQ4 skips them.

The trailing slots currently remain as initialized padding, including for odd SVE query-pair tails in #1103. We opened MOD-21066 to benchmark and test skipping the unused calculations and removing the trailing fields, including any required tail-load changes, before deciding whether to remove them entirely.

assign_query_metadata(query_values, input, metadata_dst);
}

query_blob_size = this->query_bytes_count;
}
Expand Down Expand Up @@ -638,8 +650,8 @@ class QuantPreprocessor : public PreprocessorInterface {
float value = first_input_value - mean[0];
float min_val = value;
float max_val = value;
// x_mean_ip only feeds the IP correction term (see DistanceCalculatorWithNorm); the
// normalized L2 correction cancels the mean terms exactly, so skip the extra work.
// Use the original input for the IP correction: a correction from reconstructed
// values would also multiply quantization error by the mean. L2 needs no correction.
if constexpr (Metric == VecSimMetric_IP)
x_mean_ip = first_input_value * mean[0];

Expand Down
Loading
Loading