From b59de180184173eec05b08da9a81512e532a2a56 Mon Sep 17 00:00:00 2001 From: maboroshi Date: Fri, 9 Oct 2026 13:18:54 +0000 Subject: [PATCH 1/4] add neurostore example --- examples/neurostore/model_compression.cpp | 104 ++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 examples/neurostore/model_compression.cpp diff --git a/examples/neurostore/model_compression.cpp b/examples/neurostore/model_compression.cpp new file mode 100644 index 000000000..80fbabd82 --- /dev/null +++ b/examples/neurostore/model_compression.cpp @@ -0,0 +1,104 @@ + +/** + * @file model_compression.cpp + * Demonstrate model compression using NeurStore + */ + +// make sure you have local neurostore runtime enviroment +#include "neurstore/neurstore.h" + +#include +#include +#include + +namespace fs = std::filesystem; +using namespace std::chrono; + +std::vector read_file(const std::string &path) { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) { + throw std::runtime_error("Failed to open file: " + path); + } + std::streamsize size = file.tellg(); + if (size <= 0) { + throw std::runtime_error("File is empty or unreadable: " + path); + } + file.seekg(0, std::ios::beg); + std::vector buffer(size); + if (!file.read(reinterpret_cast(buffer.data()), size)) { + throw std::runtime_error("Failed to read file contents: " + path); + } + return buffer; +} + +int main(int argc, char **argv) { + if (argc != 4) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + const std::string model_folder = argv[1]; + const double tolerance = std::stod(argv[2]); + const int parallelism = std::stoi(argv[3]); + + omp_set_num_threads(parallelism); + + std::vector model_names; + + for (const auto& entry : fs::directory_iterator(model_folder)) { + if (entry.is_regular_file()) { + const std::string filename = entry.path().filename().string(); + if (entry.path().extension() == ".onnx") { + model_names.push_back(filename.substr(0, filename.size() - 5)); + } + } + } + + std::string outputFolder = "./neurstore_compression_example"; + fs::create_directories(outputFolder); + + const NeurStore neurstore(outputFolder, std::make_shared(outputFolder)); + auto start = high_resolution_clock::now(); + bool success = false; + try { + success = neurstore.saveModels(model_names, tolerance, model_folder); + } catch (const std::exception &e) { + std::cerr << "Exception during compression: " << e.what() << std::endl; + return 1; + } + if (!success) { + std::cerr << "Compression failed." << std::endl; + return 1; + } + const auto end = high_resolution_clock::now(); + const double elapsed = duration_cast>(end - start).count(); + + double compressed_size = 0.0; // MB + for (const auto &file : std::filesystem::recursive_directory_iterator(outputFolder)) { + if (file.is_regular_file()) { + compressed_size += static_cast(file.file_size()) / (1024 * 1024); + } + } + + double original_size = 0.0; // MB + for (const auto &name: model_names) { + std::string path = fs::path(model_folder) / (name + ".onnx"); + try { + auto raw = read_file(path); + original_size += static_cast(raw.size()) / (1024 * 1024); + } catch (...) { + std::cerr << "Failed to read: " << path << std::endl; + } + } + + std::cout << "================ Compression Summary ================" << std::endl; + std::cout << "Compressing models from folder: " << model_folder << std::endl; + std::cout << "Tolerance: " << tolerance << ", Parallelism: " << parallelism << std::endl; + std::cout << "-----------------------------------------------------" << std::endl; + std::cout << "Total models compressed: " << model_names.size() << std::endl; + std::cout << "Total time: " << elapsed << " sec" << std::endl; + std::cout << "Total original size: " << original_size << " MB" << std::endl; + std::cout << "Total compressed size: " << compressed_size << " MB" << std::endl; + std::cout << "Compression ratio: " << (original_size / compressed_size) << std::endl; + std::cout << "=====================================================" << std::endl; + return 0; +} From bc5af9609affcfe0b4779b5bd49ee717e0d8d0ec Mon Sep 17 00:00:00 2001 From: maboroshi Date: Mon, 26 Oct 2026 16:03:37 +0800 Subject: [PATCH 2/4] add neurostore example --- ...ompression_and_decompression_from_list.cpp | 105 ++++++++++++ examples/neurostore/model_save_load.cpp | 149 ++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 examples/neurostore/model_compression_and_decompression_from_list.cpp create mode 100644 examples/neurostore/model_save_load.cpp diff --git a/examples/neurostore/model_compression_and_decompression_from_list.cpp b/examples/neurostore/model_compression_and_decompression_from_list.cpp new file mode 100644 index 000000000..b7b340a4a --- /dev/null +++ b/examples/neurostore/model_compression_and_decompression_from_list.cpp @@ -0,0 +1,105 @@ +#include "neurstore/neurstore.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +std::vector readModelList(const std::string &path) { + std::ifstream file(path); + if (!file.is_open()) + throw std::runtime_error("Failed to open model list file: " + path); + + std::vector modelPaths; + std::string line; + while (std::getline(file, line)) + if (!line.empty()) modelPaths.emplace_back(std::move(line)); + return modelPaths; +} + +std::vector readFile(const std::string &path) { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file.is_open()) + throw std::runtime_error("Failed to open file: " + path); + + std::streamsize size = file.tellg(); + if (size <= 0) + throw std::runtime_error("Empty or invalid file: " + path); + + std::vector buffer(static_cast(size)); + file.seekg(0, std::ios::beg); + if (!file.read(buffer.data(), size)) + throw std::runtime_error("Failed to read file: " + path); + return buffer; +} + +int main(int argc, char **argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] + << " \n"; + return 1; + } + + const std::string modelListPath = argv[1]; + std::string storeDir = argv[2]; + const double tolerance = std::stod(argv[3]); + const int parallelism = std::stoi(argv[4]); + const std::string loadModelDir = argv[5]; + + omp_set_num_threads(parallelism); + fs::create_directories(storeDir); + fs::create_directories(loadModelDir); + + const auto modelPaths = readModelList(modelListPath); + auto cacheManager = std::make_shared(storeDir); + NeurStore neurstore(storeDir, cacheManager); + + try { + if (!neurstore.saveModelsFromPaths(modelPaths, tolerance)) { + std::cerr << "Failed to save models to NeurStore.\n"; + return 1; + } + cacheManager->clearCache(); + } catch (const std::exception &e) { + std::cerr << "Compression exception: " << e.what() << '\n'; + return 1; + } + + for (const auto &modelPath: modelPaths) { + fs::path p(modelPath); + const std::string modelName = p.stem().string(); + + try { + auto modelProto = neurstore.loadModel(modelName); + std::string buffer; + modelProto->SerializeToString(&buffer); + + fs::path outPath = fs::path(loadModelDir) / (modelName + ".onnx"); + std::ofstream fout(outPath, std::ios::binary); + fout.write(buffer.data(), static_cast(buffer.size())); + std::cout << "[INFO] Loaded & dumped model: " << modelName << " -> " + << outPath << '\n'; + } catch (const std::exception &e) { + std::cerr << "[WARN] Failed to load/dump model " << modelName + << ": " << e.what() << '\n'; + } + } + + double totalOriginalSize = 0.0, totalCompressedSize = 0.0; + for (const auto &mp: modelPaths) + totalOriginalSize += static_cast(readFile(mp).size()) / (1024 * 1024); + + for (const auto &entry: fs::directory_iterator(storeDir)) + if (entry.is_regular_file()) + totalCompressedSize += static_cast(entry.file_size()) / (1024 * 1024); + + std::cout << "\n================ Overall Summary (NeurStore) ================\n"; + std::cout << "Total models processed : " << modelPaths.size() << '\n'; + std::cout << "Compression ratio : " << (totalOriginalSize / totalCompressedSize) << '\n'; + std::cout << "Total original size : " << totalOriginalSize << " MB\n"; + std::cout << "Total compressed size : " << totalCompressedSize << " MB\n"; + std::cout << "=============================================================\n"; + return 0; +} diff --git a/examples/neurostore/model_save_load.cpp b/examples/neurostore/model_save_load.cpp new file mode 100644 index 000000000..48f0f3549 --- /dev/null +++ b/examples/neurostore/model_save_load.cpp @@ -0,0 +1,149 @@ +/** + * @file model_save_load.cpp + * Demonstrate model save/load via libpq + */ + +#include +#include +#include +#include +#include +#include + + +void set_parallelism(PGconn *conn, const int parallelism) { + const int32_t parallelism_be = htonl(parallelism); + const char *paramValues[1] = {reinterpret_cast(¶llelism_be)}; + constexpr int paramLengths[1] = {sizeof(int32_t)}; + constexpr int paramFormats[1] = {1}; + constexpr Oid paramTypes[1] = {23}; // OID for integer + + PGresult *res = PQexecParams( + conn, + "SELECT ns_set_parallelism($1)", + 1, + paramTypes, + paramValues, + paramLengths, + paramFormats, + 0 + ); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + std::cerr << "Failed to set parallelism: " << PQresultErrorMessage(res) << std::endl; + } else { + std::cout << "Set neurstore parallelism to " << parallelism << std::endl; + std::cout << "--------------------------------" << std::endl; + } + PQclear(res); +} + +void save_models_from_folder(PGconn *conn, const std::string &folder_path, const float tolerance) { + const char *paramValues[2]; + int paramLengths[2]; + int paramFormats[2]; + Oid paramTypes[2] = {25, 700}; // OID for text and float4 + + paramValues[0] = folder_path.c_str(); + paramValues[1] = reinterpret_cast(&tolerance); + + paramLengths[0] = folder_path.size(); + paramLengths[1] = sizeof(float); + + paramFormats[0] = 0; + paramFormats[1] = 1; + + PGresult *res = PQexecParams( + conn, + "SELECT ns_save_models_from_folder($1, $2)", + 2, + paramTypes, + paramValues, + paramLengths, + paramFormats, + 0 + ); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + std::cerr << "Failed to save models from folder: " << PQresultErrorMessage(res) << std::endl; + } else { + std::cout << "Saved all models from: " << folder_path << std::endl; + std::cout << "--------------------------------" << std::endl; + } + PQclear(res); +} + +void load_model(PGconn* conn, const int model_id, const std::string& udf, const bool return_serialized = false) { + const int32_t model_id_be = htonl(model_id); + const char* return_serialized_model = return_serialized ? "t" : "f"; + const char* paramValues[2] = { reinterpret_cast(&model_id_be), return_serialized_model }; + constexpr int paramLengths[2] = { sizeof(int32_t), 1 }; + constexpr int paramFormats[2] = { 1, 0 }; + constexpr Oid paramTypes[2] = { 23, 16 }; + + const std::string query = "SELECT " + udf + "($1, $2)"; + + PGresult* res = PQexecParams( + conn, + query.c_str(), + 2, + paramTypes, + paramValues, + paramLengths, + paramFormats, + 1 + ); + + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) == 0) { + std::cerr << "Failed to load model with " << udf << ": " << PQresultErrorMessage(res) << std::endl; + PQclear(res); + return; + } + + if (return_serialized) { + const char* data = PQgetvalue(res, 0, 0); + const int size = PQgetlength(res, 0, 0); + onnx::ModelProto model; + if (!model.ParseFromArray(data, size)) { + std::cerr << "Failed to parse model data." << std::endl; + PQclear(res); + return; + } + } + + std::cout << "Loaded model (id = " << model_id << ") using " << udf << std::endl; + std::cout << "--------------------------------" << std::endl; + PQclear(res); +} + +int main(int argc, char* argv[]) { + if (argc != 4) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + const std::string conninfo = argv[1]; + const float tolerance = std::stof(argv[2]); + const std::string model_folder = argv[3]; + + const std::string abs_model_folder = std::filesystem::absolute(model_folder).string(); + + PGconn *conn = PQconnectdb(conninfo.c_str()); + if (PQstatus(conn) != CONNECTION_OK) { + std::cerr << "Connection failed: " << PQerrorMessage(conn) << std::endl; + return 1; + } + std::cout << "PostgreSQL connection established" << std::endl; + std::cout << "conninfo: " << conninfo << std::endl; + std::cout << "compression tolerance: " << tolerance << std::endl; + std::cout << "--------------------------------" << std::endl; + + set_parallelism(conn, 1); + save_models_from_folder(conn, abs_model_folder, tolerance); + + constexpr int model_id = 1; + load_model(conn, model_id, "ns_load_model"); + load_model(conn, model_id, "ns_load_model_as_uint8_delta", true); + + PQfinish(conn); + return 0; +} From ff030d2a8c5adb0e6696f00dd6d3a3bafa25dc02 Mon Sep 17 00:00:00 2001 From: maboroshi Date: Fri, 30 Oct 2026 13:35:09 +0800 Subject: [PATCH 3/4] add neurostore example --- CMakeLists.txt | 10 +- cmake/Dependencies.cmake | 1 + examples/neurostore/model_compression.cpp | 104 ------------ ...ompression_and_decompression_from_list.cpp | 105 ------------ examples/neurostore/model_save_load.cpp | 149 ------------------ java/CMakeLists.txt | 2 +- python/CMakeLists.txt | 22 ++- python/singa/model.py | 2 +- python/singa/sonnx.py | 22 +-- python/singa/tensor.py | 8 +- src/CMakeLists.txt | 4 +- test/python/test_tensor.py | 2 +- 12 files changed, 45 insertions(+), 386 deletions(-) delete mode 100644 examples/neurostore/model_compression.cpp delete mode 100644 examples/neurostore/model_compression_and_decompression_from_list.cpp delete mode 100644 examples/neurostore/model_save_load.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cc025fe16..d87024b00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(singa) +IF(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + SET(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) +ENDIF() + LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Thirdparty) #include(GetGitRevisionDescription) @@ -47,10 +51,10 @@ message(STATUS "SINGA Version ${SINGA_VERSION}") # Flags IF(UNIX) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g -O2 -fPIC -Wall -pthread") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -Wall -pthread") ENDIF() IF(APPLE) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -g -O2 -fPIC -Wall -pthread") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -fPIC -Wall -pthread") SET(CMAKE_MACOSX_RPATH 1) ENDIF() @@ -61,7 +65,7 @@ IF(WIN32) #MESSAGE(STATUS "Using msvc compiler ${CMAKE_CXX_FLAGS}") ENDIF() ENDIF() -IF(CMAKE_BUILD_TYPE=Debug) +IF(CMAKE_BUILD_TYPE STREQUAL "Debug") SET(NVCC_FLAG "${NVCC_FLAG} -g -G ") ENDIF() #message(STATUS "${CMAKE_CXX_FLAGS}") diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 60c432729..a9b5264ed 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -52,6 +52,7 @@ IF(USE_MODULES) #ENDIF() ELSE() FIND_PACKAGE( Protobuf 3.0 REQUIRED ) + INCLUDE_DIRECTORIES(${Protobuf_INCLUDE_DIRS}) #MESSAGE(STATUS "proto libs " ${PROTOBUF_LIBRARY}) LIST(APPEND SINGA_LINKER_LIBS ${PROTOBUF_LIBRARY}) #IF(USE_CBLAS) diff --git a/examples/neurostore/model_compression.cpp b/examples/neurostore/model_compression.cpp deleted file mode 100644 index 80fbabd82..000000000 --- a/examples/neurostore/model_compression.cpp +++ /dev/null @@ -1,104 +0,0 @@ - -/** - * @file model_compression.cpp - * Demonstrate model compression using NeurStore - */ - -// make sure you have local neurostore runtime enviroment -#include "neurstore/neurstore.h" - -#include -#include -#include - -namespace fs = std::filesystem; -using namespace std::chrono; - -std::vector read_file(const std::string &path) { - std::ifstream file(path, std::ios::binary | std::ios::ate); - if (!file) { - throw std::runtime_error("Failed to open file: " + path); - } - std::streamsize size = file.tellg(); - if (size <= 0) { - throw std::runtime_error("File is empty or unreadable: " + path); - } - file.seekg(0, std::ios::beg); - std::vector buffer(size); - if (!file.read(reinterpret_cast(buffer.data()), size)) { - throw std::runtime_error("Failed to read file contents: " + path); - } - return buffer; -} - -int main(int argc, char **argv) { - if (argc != 4) { - std::cerr << "Usage: " << argv[0] << " " << std::endl; - return 1; - } - const std::string model_folder = argv[1]; - const double tolerance = std::stod(argv[2]); - const int parallelism = std::stoi(argv[3]); - - omp_set_num_threads(parallelism); - - std::vector model_names; - - for (const auto& entry : fs::directory_iterator(model_folder)) { - if (entry.is_regular_file()) { - const std::string filename = entry.path().filename().string(); - if (entry.path().extension() == ".onnx") { - model_names.push_back(filename.substr(0, filename.size() - 5)); - } - } - } - - std::string outputFolder = "./neurstore_compression_example"; - fs::create_directories(outputFolder); - - const NeurStore neurstore(outputFolder, std::make_shared(outputFolder)); - auto start = high_resolution_clock::now(); - bool success = false; - try { - success = neurstore.saveModels(model_names, tolerance, model_folder); - } catch (const std::exception &e) { - std::cerr << "Exception during compression: " << e.what() << std::endl; - return 1; - } - if (!success) { - std::cerr << "Compression failed." << std::endl; - return 1; - } - const auto end = high_resolution_clock::now(); - const double elapsed = duration_cast>(end - start).count(); - - double compressed_size = 0.0; // MB - for (const auto &file : std::filesystem::recursive_directory_iterator(outputFolder)) { - if (file.is_regular_file()) { - compressed_size += static_cast(file.file_size()) / (1024 * 1024); - } - } - - double original_size = 0.0; // MB - for (const auto &name: model_names) { - std::string path = fs::path(model_folder) / (name + ".onnx"); - try { - auto raw = read_file(path); - original_size += static_cast(raw.size()) / (1024 * 1024); - } catch (...) { - std::cerr << "Failed to read: " << path << std::endl; - } - } - - std::cout << "================ Compression Summary ================" << std::endl; - std::cout << "Compressing models from folder: " << model_folder << std::endl; - std::cout << "Tolerance: " << tolerance << ", Parallelism: " << parallelism << std::endl; - std::cout << "-----------------------------------------------------" << std::endl; - std::cout << "Total models compressed: " << model_names.size() << std::endl; - std::cout << "Total time: " << elapsed << " sec" << std::endl; - std::cout << "Total original size: " << original_size << " MB" << std::endl; - std::cout << "Total compressed size: " << compressed_size << " MB" << std::endl; - std::cout << "Compression ratio: " << (original_size / compressed_size) << std::endl; - std::cout << "=====================================================" << std::endl; - return 0; -} diff --git a/examples/neurostore/model_compression_and_decompression_from_list.cpp b/examples/neurostore/model_compression_and_decompression_from_list.cpp deleted file mode 100644 index b7b340a4a..000000000 --- a/examples/neurostore/model_compression_and_decompression_from_list.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include "neurstore/neurstore.h" - -#include -#include -#include -#include - -namespace fs = std::filesystem; - -std::vector readModelList(const std::string &path) { - std::ifstream file(path); - if (!file.is_open()) - throw std::runtime_error("Failed to open model list file: " + path); - - std::vector modelPaths; - std::string line; - while (std::getline(file, line)) - if (!line.empty()) modelPaths.emplace_back(std::move(line)); - return modelPaths; -} - -std::vector readFile(const std::string &path) { - std::ifstream file(path, std::ios::binary | std::ios::ate); - if (!file.is_open()) - throw std::runtime_error("Failed to open file: " + path); - - std::streamsize size = file.tellg(); - if (size <= 0) - throw std::runtime_error("Empty or invalid file: " + path); - - std::vector buffer(static_cast(size)); - file.seekg(0, std::ios::beg); - if (!file.read(buffer.data(), size)) - throw std::runtime_error("Failed to read file: " + path); - return buffer; -} - -int main(int argc, char **argv) { - if (argc != 6) { - std::cerr << "Usage: " << argv[0] - << " \n"; - return 1; - } - - const std::string modelListPath = argv[1]; - std::string storeDir = argv[2]; - const double tolerance = std::stod(argv[3]); - const int parallelism = std::stoi(argv[4]); - const std::string loadModelDir = argv[5]; - - omp_set_num_threads(parallelism); - fs::create_directories(storeDir); - fs::create_directories(loadModelDir); - - const auto modelPaths = readModelList(modelListPath); - auto cacheManager = std::make_shared(storeDir); - NeurStore neurstore(storeDir, cacheManager); - - try { - if (!neurstore.saveModelsFromPaths(modelPaths, tolerance)) { - std::cerr << "Failed to save models to NeurStore.\n"; - return 1; - } - cacheManager->clearCache(); - } catch (const std::exception &e) { - std::cerr << "Compression exception: " << e.what() << '\n'; - return 1; - } - - for (const auto &modelPath: modelPaths) { - fs::path p(modelPath); - const std::string modelName = p.stem().string(); - - try { - auto modelProto = neurstore.loadModel(modelName); - std::string buffer; - modelProto->SerializeToString(&buffer); - - fs::path outPath = fs::path(loadModelDir) / (modelName + ".onnx"); - std::ofstream fout(outPath, std::ios::binary); - fout.write(buffer.data(), static_cast(buffer.size())); - std::cout << "[INFO] Loaded & dumped model: " << modelName << " -> " - << outPath << '\n'; - } catch (const std::exception &e) { - std::cerr << "[WARN] Failed to load/dump model " << modelName - << ": " << e.what() << '\n'; - } - } - - double totalOriginalSize = 0.0, totalCompressedSize = 0.0; - for (const auto &mp: modelPaths) - totalOriginalSize += static_cast(readFile(mp).size()) / (1024 * 1024); - - for (const auto &entry: fs::directory_iterator(storeDir)) - if (entry.is_regular_file()) - totalCompressedSize += static_cast(entry.file_size()) / (1024 * 1024); - - std::cout << "\n================ Overall Summary (NeurStore) ================\n"; - std::cout << "Total models processed : " << modelPaths.size() << '\n'; - std::cout << "Compression ratio : " << (totalOriginalSize / totalCompressedSize) << '\n'; - std::cout << "Total original size : " << totalOriginalSize << " MB\n"; - std::cout << "Total compressed size : " << totalCompressedSize << " MB\n"; - std::cout << "=============================================================\n"; - return 0; -} diff --git a/examples/neurostore/model_save_load.cpp b/examples/neurostore/model_save_load.cpp deleted file mode 100644 index 48f0f3549..000000000 --- a/examples/neurostore/model_save_load.cpp +++ /dev/null @@ -1,149 +0,0 @@ -/** - * @file model_save_load.cpp - * Demonstrate model save/load via libpq - */ - -#include -#include -#include -#include -#include -#include - - -void set_parallelism(PGconn *conn, const int parallelism) { - const int32_t parallelism_be = htonl(parallelism); - const char *paramValues[1] = {reinterpret_cast(¶llelism_be)}; - constexpr int paramLengths[1] = {sizeof(int32_t)}; - constexpr int paramFormats[1] = {1}; - constexpr Oid paramTypes[1] = {23}; // OID for integer - - PGresult *res = PQexecParams( - conn, - "SELECT ns_set_parallelism($1)", - 1, - paramTypes, - paramValues, - paramLengths, - paramFormats, - 0 - ); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) { - std::cerr << "Failed to set parallelism: " << PQresultErrorMessage(res) << std::endl; - } else { - std::cout << "Set neurstore parallelism to " << parallelism << std::endl; - std::cout << "--------------------------------" << std::endl; - } - PQclear(res); -} - -void save_models_from_folder(PGconn *conn, const std::string &folder_path, const float tolerance) { - const char *paramValues[2]; - int paramLengths[2]; - int paramFormats[2]; - Oid paramTypes[2] = {25, 700}; // OID for text and float4 - - paramValues[0] = folder_path.c_str(); - paramValues[1] = reinterpret_cast(&tolerance); - - paramLengths[0] = folder_path.size(); - paramLengths[1] = sizeof(float); - - paramFormats[0] = 0; - paramFormats[1] = 1; - - PGresult *res = PQexecParams( - conn, - "SELECT ns_save_models_from_folder($1, $2)", - 2, - paramTypes, - paramValues, - paramLengths, - paramFormats, - 0 - ); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) { - std::cerr << "Failed to save models from folder: " << PQresultErrorMessage(res) << std::endl; - } else { - std::cout << "Saved all models from: " << folder_path << std::endl; - std::cout << "--------------------------------" << std::endl; - } - PQclear(res); -} - -void load_model(PGconn* conn, const int model_id, const std::string& udf, const bool return_serialized = false) { - const int32_t model_id_be = htonl(model_id); - const char* return_serialized_model = return_serialized ? "t" : "f"; - const char* paramValues[2] = { reinterpret_cast(&model_id_be), return_serialized_model }; - constexpr int paramLengths[2] = { sizeof(int32_t), 1 }; - constexpr int paramFormats[2] = { 1, 0 }; - constexpr Oid paramTypes[2] = { 23, 16 }; - - const std::string query = "SELECT " + udf + "($1, $2)"; - - PGresult* res = PQexecParams( - conn, - query.c_str(), - 2, - paramTypes, - paramValues, - paramLengths, - paramFormats, - 1 - ); - - if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) == 0) { - std::cerr << "Failed to load model with " << udf << ": " << PQresultErrorMessage(res) << std::endl; - PQclear(res); - return; - } - - if (return_serialized) { - const char* data = PQgetvalue(res, 0, 0); - const int size = PQgetlength(res, 0, 0); - onnx::ModelProto model; - if (!model.ParseFromArray(data, size)) { - std::cerr << "Failed to parse model data." << std::endl; - PQclear(res); - return; - } - } - - std::cout << "Loaded model (id = " << model_id << ") using " << udf << std::endl; - std::cout << "--------------------------------" << std::endl; - PQclear(res); -} - -int main(int argc, char* argv[]) { - if (argc != 4) { - std::cerr << "Usage: " << argv[0] << " " << std::endl; - return 1; - } - const std::string conninfo = argv[1]; - const float tolerance = std::stof(argv[2]); - const std::string model_folder = argv[3]; - - const std::string abs_model_folder = std::filesystem::absolute(model_folder).string(); - - PGconn *conn = PQconnectdb(conninfo.c_str()); - if (PQstatus(conn) != CONNECTION_OK) { - std::cerr << "Connection failed: " << PQerrorMessage(conn) << std::endl; - return 1; - } - std::cout << "PostgreSQL connection established" << std::endl; - std::cout << "conninfo: " << conninfo << std::endl; - std::cout << "compression tolerance: " << tolerance << std::endl; - std::cout << "--------------------------------" << std::endl; - - set_parallelism(conn, 1); - save_models_from_folder(conn, abs_model_folder, tolerance); - - constexpr int model_id = 1; - load_model(conn, model_id, "ns_load_model"); - load_model(conn, model_id, "ns_load_model_as_uint8_delta", true); - - PQfinish(conn); - return 0; -} diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 7f4419832..50f437062 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -104,7 +104,7 @@ PROTOBUF_GENERATE_JAVA(proto_javas ${proto_files}) # MESSAGE(STATUS "proto javas: ${proto_javas}") execute_process( - COMMAND swig -c++ -java -I${CMAKE_SOURCE_DIR}/include + COMMAND ${SWIG_EXECUTABLE} -c++ -java -I${CMAKE_BINARY_DIR}/src/api -I${CMAKE_SOURCE_DIR}/include -I${JAVA_INCLUDE_PATH} -I${JAVA_INCLUDE_PATH2} -package org.apache.singa.swig -outdir ${CMAKE_BINARY_DIR}/java/src/main/java/org/apache/singa/swig diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 22a7c7d0f..c71703ba2 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -94,17 +94,22 @@ file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/python/singa/proto) file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/python/rafiki) file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/src/api) -IF(USE_PYTHON3) - SET(SWIG_PYTHON3 "-py3") +IF(USE_PYTHON3 AND SWIG_VERSION VERSION_LESS "4.1") + SET(SWIG_PYTHON3 "-py3") ELSE() - SET(SWIG_PYTHON3 "") + SET(SWIG_PYTHON3 "") ENDIF() execute_process( - COMMAND swig -c++ -python ${SWIG_PYTHON3} -I${CMAKE_SOURCE_DIR}/include + COMMAND ${SWIG_EXECUTABLE} -c++ -python ${SWIG_PYTHON3} + -I${CMAKE_BINARY_DIR}/src/api -I${CMAKE_SOURCE_DIR}/include -outdir ${CMAKE_BINARY_DIR}/python/singa -o ${CMAKE_BINARY_DIR}/src/api/singa_wrap.cxx - ${CMAKE_SOURCE_DIR}/src/api/singa.i) + ${CMAKE_SOURCE_DIR}/src/api/singa.i + RESULT_VARIABLE SWIG_RESULT) +IF(NOT SWIG_RESULT EQUAL 0) + MESSAGE(FATAL_ERROR "SINGA SWIG generation failed (${SWIG_RESULT})") +ENDIF() set(python_srcs "${CMAKE_BINARY_DIR}/src/api/singa_wrap.cxx") @@ -115,7 +120,12 @@ create_symlinks(${python_source_files}) execute_process( COMMAND ${PYTHON_EXECUTABLE} -c "from __future__ import print_function; import numpy; print(numpy.get_include())" - OUTPUT_VARIABLE NUMPY_INCLUDE_DIR) + OUTPUT_VARIABLE NUMPY_INCLUDE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE NUMPY_RESULT) +IF(NOT NUMPY_RESULT EQUAL 0) + MESSAGE(FATAL_ERROR "NumPy is unavailable in ${PYTHON_EXECUTABLE}") +ENDIF() #message(status "numpy path ${NUMPY_INCLUDE_DIR}") diff --git a/python/singa/model.py b/python/singa/model.py index 5f1ed2c21..ebfd36817 100644 --- a/python/singa/model.py +++ b/python/singa/model.py @@ -27,7 +27,7 @@ import zipfile import numpy as np from functools import wraps -from collections import Iterable +from collections.abc import Iterable from singa import tensor from singa import autograd diff --git a/python/singa/sonnx.py b/python/singa/sonnx.py index 625a30c29..528e45a81 100755 --- a/python/singa/sonnx.py +++ b/python/singa/sonnx.py @@ -23,9 +23,10 @@ import onnx import onnx.utils +import onnxoptimizer as optimizer from onnx.backend.base import Backend, BackendRep from onnx import (checker, helper, numpy_helper, GraphProto, NodeProto, - TensorProto, OperatorSetIdProto, optimizer, mapping, + TensorProto, OperatorSetIdProto, mapping, shape_inference) import warnings @@ -38,6 +39,7 @@ from . import singa_wrap as singa import collections +from collections.abc import Iterable OrderedDict = collections.OrderedDict namedtuple = collections.namedtuple @@ -57,7 +59,7 @@ np.dtype('complex128'): None, np.dtype('uint32'): None, np.dtype('uint64'): None, - np.dtype(np.object): None + np.dtype(object): None } @@ -900,7 +902,7 @@ def singa_op_to_onnx_node(cls, op, op_t): else: translator = cls._common_singa_tensor_to_onnx_node nodes = translator(op, op_t) - if not isinstance(nodes, collections.Iterable): + if not isinstance(nodes, Iterable): nodes = [nodes] nodes = [node for node in nodes if node is not None] return nodes @@ -1826,7 +1828,7 @@ def _run_node(cls, operator, inputs): list, the output """ outputs = operator(*inputs) - if not isinstance(outputs, collections.Iterable): + if not isinstance(outputs, Iterable): outputs = [outputs] return outputs @@ -1919,10 +1921,10 @@ def prepare(cls, model, device='CPU', **kwargs): """ super(SingaBackend, cls).prepare(model, device, **kwargs) # optimize and infer the shape of the model - try: - model = onnx.utils.polish_model(model) - except IndexError as err: - model = shape_inference.infer_shapes(model) + checker.check_model(model) + model = shape_inference.infer_shapes(model) + model = optimizer.optimize(model) + checker.check_model(model) # check the opset version and ir version # SINGA supports opset version(11), ir version(1.6.0 -> 6) @@ -1939,8 +1941,8 @@ def prepare(cls, model, device='CPU', **kwargs): if model.ir_version > cls._ir_version: warnings.warn( - "The imported ir verion {} is larger than the supported version {}." - .format(cls._ir_version, imp.version)) + "The imported IR version {} is larger than the supported version {}." + .format(model.ir_version, cls._ir_version)) graph = model.graph params, inputs, outputs, layers = cls._onnx_model_to_singa_ops( diff --git a/python/singa/tensor.py b/python/singa/tensor.py index 963ad1a00..3d50a07e6 100755 --- a/python/singa/tensor.py +++ b/python/singa/tensor.py @@ -372,7 +372,7 @@ def copy_from_numpy(self, np_array, offset=0): self.data.CopyFloatDataFromHostPtr(np_array) elif dt == np.float16: self.data.CopyHalfFloatDataFromHostPtr(np_array) - elif dt == np.int or dt == np.int32: + elif dt == int or dt == np.int32: self.data.CopyIntDataFromHostPtr(np_array) else: raise NotImplementedError('Not implemented yet for ', dt) @@ -886,10 +886,10 @@ def from_numpy(np_array, dev=None): ''' assert type(np_array) is np.ndarray, 'Must input numpy array' # convert to float32 array - if np_array.dtype == np.float64 or np_array.dtype == np.float: + if np_array.dtype == np.float64 or np_array.dtype == float: np_array = np_array.astype(np.float32) - if np_array.dtype == np.int64 or np_array.dtype == np.int: + if np_array.dtype == np.int64 or np_array.dtype == int: np_array = np_array.astype(np.int32) if np_array.dtype == np.float32: @@ -1791,7 +1791,7 @@ def copy_from_numpy(data, np_array): data.CopyFloatDataFromHostPtr(np_array) elif dt == np.float16: data.CopyHalfFloatDataFromHostPtr(np_array) - elif dt == np.int or dt == np.int32: + elif dt == int or dt == np.int32: data.CopyIntDataFromHostPtr(np_array) else: raise NotImplementedError('Not implemented yet for ', dt) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5f302992b..e14f5cf2c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -111,7 +111,7 @@ IF(CODE_COVERAGE) ENDIF(CODE_COVERAGE) #pass configure infor to swig -FILE(REMOVE "${CMAKE_CURRENT_SOURCE_DIR}/api/config.i") -CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/api/config.i.in" "${CMAKE_CURRENT_SOURCE_DIR}/api/config.i") +FILE(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/src/api") +CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/api/config.i.in" "${CMAKE_BINARY_DIR}/src/api/config.i") diff --git a/test/python/test_tensor.py b/test/python/test_tensor.py index f7044f1d2..dc7568e12 100644 --- a/test/python/test_tensor.py +++ b/test/python/test_tensor.py @@ -208,7 +208,7 @@ def test_tensor_inplace_api(self): self.assertTrue(y is x) def test_numpy_convert(self): - a = np.asarray([[1, 0, 0], [0, 1, 0]], dtype=np.int) + a = np.asarray([[1, 0, 0], [0, 1, 0]], dtype=int) t = tensor.from_numpy(a) b = tensor.to_numpy(t) self.assertEqual(np.sum(a - b), 0) From 1f2ffecb9dabb725dc63c392d4397dfe366c5b09 Mon Sep 17 00:00:00 2001 From: maboroshi Date: Fri, 30 Oct 2026 13:37:43 +0800 Subject: [PATCH 4/4] add neurostore example --- examples/neurostore/prepare.py | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 examples/neurostore/prepare.py diff --git a/examples/neurostore/prepare.py b/examples/neurostore/prepare.py new file mode 100644 index 000000000..e3c037a75 --- /dev/null +++ b/examples/neurostore/prepare.py @@ -0,0 +1,67 @@ +""" +prepare.py + +This script downloads models from Hugging Face and exports them to ONNX format. +In our example, we use three DistilBERT models: +1. google-bert/bert-base-uncased +2. lorenzozan/ME2-BERT +3. prithivMLmods/Spam-Bert-Uncased + +To download and export other models, modify: 1). the `models` list in the main function. 2). the +dummy_inputs and torch.onnx.export function parameters. +""" + +import os +import torch +from transformers import AutoModel + +DUMMY_SEQ_LENGTH = 32 + + +def export_model_to_onnx(model_name: str, save_dir: str): + """ + Downloads and exports a Hugging Face model to ONNX format. + """ + print(f"Processing {model_name}...") + try: + model = AutoModel.from_pretrained(model_name) + model.eval() + + dummy_inputs = { + "input_ids": torch.randint(0, 30522, (1, DUMMY_SEQ_LENGTH)), + "attention_mask": torch.ones((1, DUMMY_SEQ_LENGTH), dtype=torch.long) + } + + safe_model_name = model_name.replace("/", "_") + export_path = os.path.join(save_dir, f"{safe_model_name}.onnx") + + torch.onnx.export( + model, + (dummy_inputs["input_ids"], dummy_inputs["attention_mask"]), + export_path, + input_names=["input_ids", "attention_mask"], + output_names=["last_hidden_state"], + dynamic_axes={ + "input_ids": {0: "batch_size", 1: "sequence_length"}, + "attention_mask": {0: "batch_size", 1: "sequence_length"}, + "last_hidden_state": {0: "batch_size", 1: "sequence_length"} + }, + opset_version=14, + ) + print(f"Exported ONNX to {export_path}") + except Exception as e: + print(f"Failed to export {model_name}: {e}") + + +if __name__ == "__main__": + model_dir = "./models" + os.makedirs(model_dir, exist_ok=True) + + models = [ + "distilbert/distilbert-base-uncased", + "winegarj/distilbert-base-uncased-finetuned-sst2", + "eliasalbouzidi/distilbert-nsfw-text-classifier" + ] + + for model_name in models: + export_model_to_onnx(model_name, model_dir)