Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions bindings/c/docs/C_API_Design.md
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,8 @@ The main search structure providing vector similarity search operations.
**Dynamic Index Parameters:**

The `_ex` variants of the dynamic functions (`svs_index_build_dynamic_ex`,
`svs_index_load_dynamic_ex`, `svs_index_convert_dynamic_ex`,
`svs_index_load_dynamic_ex`, `svs_index_load_stream_dynamic_ex`,
`svs_index_convert_dynamic_ex`,
`svs_index_builder_estimate_memory_dynamic_ex`,
`svs_index_builder_estimate_search_memory_dynamic_ex`) take a versioned
`const svs_dynamic_index_params_t*` instead of a bare `blocksize_bytes` argument.
Expand Down Expand Up @@ -654,7 +655,7 @@ for full signatures, parameters, and Doxygen documentation.
| **Search params** | `svs_search_params_create_vamana`, `svs_search_params_free` |
| **Builder** | `svs_index_builder_create`, `svs_index_builder_set_{storage,threadpool,threadpool_custom,allocator,allocator_custom}`, `svs_index_builder_free` |
| **Memory estimation** | `svs_index_builder_estimate_memory`, `svs_index_builder_estimate_memory_dynamic[_ex]`, `svs_index_builder_estimate_search_memory`, `svs_index_builder_estimate_search_memory_dynamic[_ex]`, `svs_index_builder_get_default_blocksize_bytes` |
| **Index lifecycle** | `svs_index_build`, `svs_index_build_dynamic[_ex]`, `svs_index_convert`, `svs_index_convert_dynamic[_ex]`, `svs_index_load`, `svs_index_load_dynamic[_ex]`, `svs_index_load_stream`, `svs_index_load_stream_dynamic`, `svs_index_save`, `svs_index_save_stream`, `svs_index_free` |
| **Index lifecycle** | `svs_index_build`, `svs_index_build_dynamic[_ex]`, `svs_index_convert`, `svs_index_convert_dynamic[_ex]`, `svs_index_load`, `svs_index_load_dynamic[_ex]`, `svs_index_load_stream`, `svs_index_load_stream_dynamic[_ex]`, `svs_index_save`, `svs_index_save_stream`, `svs_index_free` |
| **Dynamic ops** | `svs_index_dynamic_{add_points,delete_points,has_id,consolidate,compact,get_sync_kind}` |
| **Introspection** | `svs_index_get_num_threads` / `set_num_threads`, `svs_index_get_distance`, `svs_index_reconstruct`, `svs_index_get_memory_usage`, `svs_index_get_memory_breakdown` |
| **Search** | `svs_index_search_topk` (+ deprecated `svs_index_search`), `svs_search_results_free` |
Expand Down
21 changes: 21 additions & 0 deletions bindings/c/include/svs/c/svs_c.h
Original file line number Diff line number Diff line change
Expand Up @@ -1411,6 +1411,27 @@ SVS_API svs_index_h svs_index_load_stream_dynamic(
svs_error_h out_err /*=NULL*/
);

/// @brief Load a dynamic index from a caller-supplied stream with explicit parameters
/// @param builder The index builder handle (used for configuration)
/// @param stream The stream interface to read the index from
/// @param params Pointer to the dynamic index parameters structure, or NULL for defaults
/// @param out_err An optional error handle to capture errors
/// @return A handle to the loaded dynamic index
/// @remarks The block parameters only affect the in-memory layout, not the on-disk
/// format. The sync kind is not persisted with the index and must be passed on every load.
/// @remarks The operations table is copied, but @p stream->self is retained as-is. It only
/// needs to remain valid until this function returns.
/// @remarks Accepts both the native stream encoding and a packed directory archive. The
/// native encoding carries no total length, so a load may read up to 64 KiB past the end
/// of the index; frame the payload if embedding it in a larger stream.
/// @error SVS_ERROR_INVALID_ARGUMENT if @p params is invalid (see svs_dynamic_index_params)
SVS_API svs_index_h svs_index_load_stream_dynamic_ex(
svs_index_builder_h builder,
svs_stream_i stream,
const svs_dynamic_index_params_t* params /*=NULL*/,
svs_error_h out_err /*=NULL*/
);

/// @brief Convert an index using new builder configuration
/// @param builder The index builder handle (used for configuration)
/// @param src_index The source index handle to convert from
Expand Down
60 changes: 59 additions & 1 deletion bindings/c/src/dispatcher_concurrent_vamana.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
#include <svs/orchestrators/concurrent_dynamic_vamana.h>

#include <filesystem>
#include <istream>
#include <memory>
#include <span>
#include <utility>
#include <variant>
Expand Down Expand Up @@ -196,9 +198,38 @@ svs::DynamicVamana load_concurrent_vamana_index(
);
}

template <typename DataLoader, typename Distance>
svs::DynamicVamana load_stream_concurrent_vamana_index(
const svs::index::vamana::VamanaBuildParameters& SVS_UNUSED(build_params),
std::unique_ptr<std::istream> stream,
DataLoader SVS_UNUSED(loader),
Distance D,
svs::threads::ThreadPoolHandle pool,
const AllocatorBuilder& allocator_builder,
const svs::data::BlockingParameters& block_params,
svs::logging::logger_ptr logger
) {
using allocator_type = typename DataLoader::allocator_type;
using value_type = typename allocator_type::value_type;
using data_type = typename DataLoader::data_type;
auto allocator = allocator_type{block_params, allocator_builder.build<value_type>()};
auto graph_allocator = ConcurrentGraphAllocator{
block_params, allocator_builder.build_for_graph<uint32_t>()};

return svs::ConcurrentDynamicVamana::assemble<float>(
*stream,
svs::make_stream_loader<ConcurrentGraph>(graph_allocator),
svs::make_stream_loader<data_type>(allocator),
std::move(D),
std::move(pool),
std::move(logger)
);
}

using ConcurrentVamanaSource = std::variant<
std::pair<svs::data::ConstSimpleDataView<float>, std::span<const size_t>>,
std::filesystem::path>;
std::filesystem::path,
std::unique_ptr<std::istream>>;

using BuildConcurrentIndexDispatcher = svs::lib::Dispatcher<
svs::DynamicVamana,
Expand All @@ -221,10 +252,15 @@ const BuildConcurrentIndexDispatcher& build_concurrent_vamana_index_dispatcher()
d.register_target(&load_concurrent_vamana_index<DataLoader, Distance>);
};

auto load_stream_closure = [&d]<typename DataLoader, typename Distance>() {
d.register_target(&load_stream_concurrent_vamana_index<DataLoader, Distance>);
};

constexpr auto kind = ConcurrentVamanaFlavor::block_kind;

for_all_specializations<kind>(build_closure);
for_all_specializations<kind>(load_closure);
for_all_specializations<kind>(load_stream_closure);
return d;
}();
return dispatcher;
Expand Down Expand Up @@ -293,6 +329,28 @@ svs::DynamicVamana dispatch_concurrent_vamana_index_load(
);
}

svs::DynamicVamana dispatch_concurrent_vamana_index_load_stream(
const svs::index::vamana::VamanaBuildParameters& build_params,
std::unique_ptr<std::istream> stream,
const Storage* storage,
svs::DistanceType distance_type,
svs::threads::ThreadPoolHandle pool,
const AllocatorBuilder& allocator_builder,
const svs::data::BlockingParameters& block_params,
svs::logging::logger_ptr logger
) {
return build_concurrent_vamana_index_dispatcher().invoke(
build_params,
ConcurrentVamanaSource{std::move(stream)},
storage,
distance_type,
std::move(pool),
allocator_builder,
block_params,
std::move(logger)
);
}

svs::DynamicVamana dispatch_concurrent_vamana_index_copy(
const svs::index::vamana::VamanaBuildParameters& build_params,
const svs::DynamicVamana& src_index,
Expand Down
11 changes: 11 additions & 0 deletions bindings/c/src/dispatcher_concurrent_vamana.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,17 @@ svs::DynamicVamana dispatch_concurrent_vamana_index_load(
svs::logging::logger_ptr logger
);

svs::DynamicVamana dispatch_concurrent_vamana_index_load_stream(
const svs::index::vamana::VamanaBuildParameters& build_params,
std::unique_ptr<std::istream> stream,
const Storage* storage,
svs::DistanceType distance_type,
svs::threads::ThreadPoolHandle pool,
const AllocatorBuilder& allocator_builder,
const svs::data::BlockingParameters& block_params,
svs::logging::logger_ptr logger
);

// Copies between a concurrent and a regular or concurrent dynamic index;
// at least one of `src_concurrent` and `dst_concurrent` must be true.
svs::DynamicVamana dispatch_concurrent_vamana_index_copy(
Expand Down
40 changes: 23 additions & 17 deletions bindings/c/src/index_builder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -346,23 +346,29 @@ std::shared_ptr<DynamicIndex> IndexBuilder::load_stream_dynamic(
) {
if (algorithm->type == SVS_ALGORITHM_TYPE_VAMANA) {
auto vamana_algorithm = static_cast<AlgorithmVamana*>(algorithm.get());

auto index = std::make_shared<DynamicIndexVamana>(
*this,
dispatch_dynamic_vamana_index_load_stream(
vamana_algorithm->build_parameters(),
std::move(stream),
storage.get(),
to_distance_type(distance_metric),
pool_builder.build(sync_kind),
allocator_builder,
block_params,
get_logger()
),
sync_kind
);

return index;
const auto& build_params = vamana_algorithm->build_parameters();
auto index = uses_concurrent_index(sync_kind)
? dispatch_concurrent_vamana_index_load_stream(
build_params,
std::move(stream),
storage.get(),
to_distance_type(distance_metric),
pool_builder.build(sync_kind),
allocator_builder,
block_params,
get_logger()
)
: dispatch_dynamic_vamana_index_load_stream(
build_params,
std::move(stream),
storage.get(),
to_distance_type(distance_metric),
pool_builder.build(sync_kind),
allocator_builder,
block_params,
get_logger()
);
return make_dynamic_index(*this, std::move(index), sync_kind);
}
return nullptr;
}
Expand Down
34 changes: 34 additions & 0 deletions bindings/c/src/svs_c.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1245,6 +1245,40 @@ extern "C" svs_index_h svs_index_load_stream_dynamic(
);
}

extern "C" svs_index_h svs_index_load_stream_dynamic_ex(
svs_index_builder_h builder,
svs_stream_i stream,
const svs_dynamic_index_params_t* params,
svs_error_h out_err
) {
using namespace svs::c_runtime;
return wrap_exceptions(
[&]() {
EXPECT_ARG_NOT_NULL(builder);
EXPECT_ARG_NOT_NULL(stream);
NOT_IMPLEMENTED_IF(
(builder->impl->algorithm->type != SVS_ALGORITHM_TYPE_VAMANA),
"Only Vamana algorithm is currently supported for dynamic index loading"
);
StreamBuf::validate(stream, /*need_write=*/false);
auto dynamic_params = read_dynamic_index_params(params);
auto index = builder->impl->load_stream_dynamic(
std::make_unique<InputStream>(*stream->ops, stream->self),
dynamic_params.block_params,
dynamic_params.sync_kind
Comment on lines +1265 to +1268

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added: saves to a stream, loads with svs_index_load_stream_dynamic_ex and checks the FINE_GRAIN sync kind and size, and that params = NULL gives NONE.

);
if (index == nullptr) {
SET_ERROR(out_err, SVS_ERROR_RUNTIME, "Dynamic index load failed");
return svs_index_h{nullptr};
}
auto result = new svs_index;
result->impl = index;
return result;
},
out_err
);
}

extern "C" svs_index_h
svs_index_convert(svs_index_builder_h builder, svs_index_h src_index, svs_error_h out_err) {
using namespace svs::c_runtime;
Expand Down
45 changes: 45 additions & 0 deletions bindings/c/tests/c_api_stream.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -676,6 +676,51 @@ CATCH_TEST_CASE("C API Stream Save and Load", "[c_api][index][stream]") {
svs_index_free(index);
}

CATCH_SECTION("Dynamic stream load with sync kind") {
std::vector<size_t> ids(NUM_VECTORS);
std::iota(ids.begin(), ids.end(), size_t{0});
const size_t BLOCK_SIZE = 1024 * 1024;
svs_index_h index = svs_index_build_dynamic(
builder, data.data(), ids.data(), NUM_VECTORS, BLOCK_SIZE, error
);
CATCH_REQUIRE(index != nullptr);

MemoryStream stream;
svs_stream_interface_ops write_ops =
SVS_INIT_STREAM_OPS(nullptr, memory_stream_write);
svs_stream_interface out_stream = SVS_MAKE_INTERFACE(&stream, write_ops);
CATCH_REQUIRE(svs_index_save_stream(index, &out_stream, error));
CATCH_REQUIRE(svs_error_ok(error));

svs_stream_interface_ops read_ops =
SVS_INIT_STREAM_OPS(memory_stream_read, nullptr);
svs_stream_interface in_stream = SVS_MAKE_INTERFACE(&stream, read_ops);
svs_dynamic_index_params_t params = SVS_INIT_DYNAMIC_INDEX_PARAMS();
params.blocksize_bytes = BLOCK_SIZE;
params.sync_kind = SVS_SYNC_KIND_FINE_GRAIN;
svs_index_h loaded =
svs_index_load_stream_dynamic_ex(builder, &in_stream, &params, error);
CATCH_REQUIRE(loaded != nullptr);

svs_sync_kind_t kind = SVS_SYNC_KIND_NONE;
CATCH_REQUIRE(svs_index_dynamic_get_sync_kind(loaded, &kind, error));
CATCH_REQUIRE(kind == SVS_SYNC_KIND_FINE_GRAIN);
size_t size = 0;
CATCH_REQUIRE(svs_index_get_size(loaded, &size, error));
CATCH_REQUIRE(size == NUM_VECTORS);
svs_index_free(loaded);

// NULL params selects the defaults, i.e. no synchronization.
stream.pos = 0;
loaded = svs_index_load_stream_dynamic_ex(builder, &in_stream, nullptr, error);
CATCH_REQUIRE(loaded != nullptr);
kind = SVS_SYNC_KIND_FINE_GRAIN;
CATCH_REQUIRE(svs_index_dynamic_get_sync_kind(loaded, &kind, error));
CATCH_REQUIRE(kind == SVS_SYNC_KIND_NONE);
svs_index_free(loaded);
svs_index_free(index);
}

CATCH_SECTION("Dynamic Stream Load Uses Custom Allocator For Graph") {
// Assert same number of bytes are allocated during original construction and after
// a streaming I/O loop
Expand Down
Loading