diff --git a/.install/ubuntu_26.04.sh b/.install/ubuntu_26.04.sh new file mode 100755 index 000000000..3967cdaf2 --- /dev/null +++ b/.install/ubuntu_26.04.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e +export DEBIAN_FRONTEND=noninteractive +MODE=$1 # whether to install using sudo or not + +$MODE apt update -qq + +# apt.llvm.org has no clang-18 repo for 26.04, but the Ubuntu archive ships clang-format-18. +# check-format.sh calls the unversioned clang-format, so point it at version 18. +$MODE apt install -yqq clang-format-18 +$MODE ln -sf /usr/bin/clang-format-18 /usr/local/bin/clang-format +clang-format --version + +$MODE apt install -yqq git wget build-essential lcov openssl libssl-dev \ + python3 python3-venv python3-dev unzip rsync curl +source install_cmake.sh $MODE diff --git a/cmake/x86_64InstructionFlags.cmake b/cmake/x86_64InstructionFlags.cmake index dadd550a8..f19ef7662 100644 --- a/cmake/x86_64InstructionFlags.cmake +++ b/cmake/x86_64InstructionFlags.cmake @@ -21,15 +21,35 @@ CHECK_CXX_COMPILER_FLAG(-msse4.1 CXX_SSE4) CHECK_CXX_COMPILER_FLAG(-msse3 CXX_SSE3) CHECK_CXX_COMPILER_FLAG(-msse CXX_SSE) -# Turn off AVX512BF16 on Ubuntu 18.04 as it is not supported by its binutils assembler version. +# Check binutils version for AVX512 instruction support. +# Even if the compiler supports certain flags, the assembler (binutils) may not. +# - AVX512-BF16 requires binutils >= 2.34 +# - AVX512-FP16 requires binutils >= 2.38 if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") - execute_process(COMMAND lsb_release -rs - OUTPUT_VARIABLE UBUNTU_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - - if("${UBUNTU_VERSION}" STREQUAL "18.04") - message(STATUS "Compiling on Ubuntu 18.04, turning off CXX_AVX512BF16 flag.") - set(CXX_AVX512BF16 FALSE) + # Get binutils/assembler version + execute_process(COMMAND as --version + OUTPUT_VARIABLE AS_VERSION_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) + # Extract version number (e.g., "2.34" from "GNU assembler (GNU Binutils for Ubuntu) 2.34") + string(REGEX MATCH "[0-9]+\\.[0-9]+" BINUTILS_VERSION "${AS_VERSION_OUTPUT}") + + if(BINUTILS_VERSION) + message(STATUS "Detected binutils version: ${BINUTILS_VERSION}") + + # AVX512-BF16 requires binutils >= 2.34 + if(BINUTILS_VERSION VERSION_LESS "2.34") + message(STATUS "binutils ${BINUTILS_VERSION} < 2.34, turning off CXX_AVX512BF16 flag.") + set(CXX_AVX512BF16 FALSE) + endif() + + # AVX512-FP16 requires binutils >= 2.38 + if(BINUTILS_VERSION VERSION_LESS "2.38") + message(STATUS "binutils ${BINUTILS_VERSION} < 2.38, turning off CXX_AVX512FP16 flag.") + set(CXX_AVX512FP16 FALSE) + endif() + else() + message(WARNING "Could not detect binutils version, AVX512 features may fail to assemble") endif() endif() diff --git a/src/VecSim/memory/vecsim_malloc.h b/src/VecSim/memory/vecsim_malloc.h index 178d22d3a..2962c69a4 100644 --- a/src/VecSim/memory/vecsim_malloc.h +++ b/src/VecSim/memory/vecsim_malloc.h @@ -113,6 +113,10 @@ inline void vecsim_free(void *p) { VecSimAllocator::memFunctions.freeFunction(p) template struct VecsimSTLAllocator { using value_type = T; + // Tiered indexes use separate allocators for frontend/backend/management layers. + // Swapping containers across these layers is safe (same underlying alloc functions), + // so we must tell std::vector::swap to swap the allocator along with the data. + using propagate_on_container_swap = std::true_type; private: VecsimSTLAllocator() {} @@ -148,3 +152,9 @@ template bool operator!=(const VecsimSTLAllocator &a, const VecsimSTLAllocator &b) { return a.vecsim_allocator != b.vecsim_allocator; } + +// Guard against regressions of the allocator swap-propagation trait. Tiered indexes swap +// std::vectors whose allocators reference different VecSimAllocator instances, which is UB +// unless the allocator is swapped along with the buffer (see hnsw_tiered.h getNextResults). +static_assert(std::allocator_traits>::propagate_on_container_swap::value, + "VecsimSTLAllocator must propagate on container swap."); diff --git a/src/VecSim/spaces/CMakeLists.txt b/src/VecSim/spaces/CMakeLists.txt index d88750e91..fe354ded5 100644 --- a/src/VecSim/spaces/CMakeLists.txt +++ b/src/VecSim/spaces/CMakeLists.txt @@ -13,7 +13,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall") set(OPTIMIZATIONS "") -if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") +if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") # Check that the compiler supports instructions flag. # from gcc14+ -mavx512bw is implicitly enabled when -mavx512vbmi2 is requested include(${root}/cmake/x86_64InstructionFlags.cmake) @@ -93,7 +93,7 @@ if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") endif() endif() -if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)|(ARM64)|(armv.*)") +if (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)|(ARM64)|(armv.*)") include(${root}/cmake/aarch64InstructionFlags.cmake) # Create different optimization implementations for ARM architecture diff --git a/tests/benchmark/spaces_benchmarks/bm_spaces_int8.cpp b/tests/benchmark/spaces_benchmarks/bm_spaces_int8.cpp index a4b63ee73..ec3c4e84f 100644 --- a/tests/benchmark/spaces_benchmarks/bm_spaces_int8.cpp +++ b/tests/benchmark/spaces_benchmarks/bm_spaces_int8.cpp @@ -49,8 +49,8 @@ INITIALIZE_BENCHMARKS_SET_Cosine(BM_VecSimSpaces_Integers_INT8, INT8, SVE2, 32, #endif #ifdef OPT_SVE bool sve_supported = opt.sve; // Check for SVE support -INITIALIZE_BENCHMARKS_SET_L2_IP(BM_VecSimSpaces_Integers_INT8, INT8, SVE, 32, sve2_supported); -INITIALIZE_BENCHMARKS_SET_Cosine(BM_VecSimSpaces_Integers_INT8, INT8, SVE, 32, sve2_supported); +INITIALIZE_BENCHMARKS_SET_L2_IP(BM_VecSimSpaces_Integers_INT8, INT8, SVE, 32, sve_supported); +INITIALIZE_BENCHMARKS_SET_Cosine(BM_VecSimSpaces_Integers_INT8, INT8, SVE, 32, sve_supported); #endif #ifdef OPT_NEON_DOTPROD bool neon_dotprod_supported = opt.asimddp; diff --git a/tests/benchmark/spaces_benchmarks/bm_spaces_uint8.cpp b/tests/benchmark/spaces_benchmarks/bm_spaces_uint8.cpp index 0d8bc35cc..602fff719 100644 --- a/tests/benchmark/spaces_benchmarks/bm_spaces_uint8.cpp +++ b/tests/benchmark/spaces_benchmarks/bm_spaces_uint8.cpp @@ -49,8 +49,8 @@ INITIALIZE_BENCHMARKS_SET_Cosine(BM_VecSimSpaces_Integers_UINT8, UINT8, SVE2, 32 #endif #ifdef OPT_SVE bool sve_supported = opt.sve; // Check for SVE support -INITIALIZE_BENCHMARKS_SET_L2_IP(BM_VecSimSpaces_Integers_UINT8, UINT8, SVE, 32, sve2_supported); -INITIALIZE_BENCHMARKS_SET_Cosine(BM_VecSimSpaces_Integers_UINT8, UINT8, SVE, 32, sve2_supported); +INITIALIZE_BENCHMARKS_SET_L2_IP(BM_VecSimSpaces_Integers_UINT8, UINT8, SVE, 32, sve_supported); +INITIALIZE_BENCHMARKS_SET_Cosine(BM_VecSimSpaces_Integers_UINT8, UINT8, SVE, 32, sve_supported); #endif #ifdef OPT_NEON_DOTPROD // NEON_DOTPROD functions