From 7745304c3ca5f951fc2c1278edc0e16f5de3fa2c Mon Sep 17 00:00:00 2001 From: 076923 <076923@naver.com> Date: Fri, 6 Nov 2026 17:52:35 +0900 Subject: [PATCH] Release 0.12.0: finite store timeouts, MCP row and column limits with MemoryError reporting, evidence_scope and needs_full_audit, MCP security tests, verification doc in five areas --- .agents/skills/vecscope/SKILL.md | 4 + CHANGELOG.md | 21 +++- README.md | 13 ++- docs/ko/README.md | 12 ++- docs/ko/verification.md | 130 +++++++++++++++++++------ docs/verification.md | 140 ++++++++++++++++++++------ server.json | 4 +- src/vecscope/__init__.py | 2 +- src/vecscope/adapters.py | 64 +++++++----- src/vecscope/evidence.py | 9 ++ src/vecscope/mcp_server.py | 122 +++++++++++++++++++---- tests/golden/anisotropic.json | 2 + tests/golden/collapsed.json | 2 + tests/golden/healthy.json | 2 + tests/test_adapters.py | 4 +- tests/test_mcp_security.py | 162 +++++++++++++++++++++++++++++++ 16 files changed, 584 insertions(+), 109 deletions(-) create mode 100644 tests/test_mcp_security.py diff --git a/.agents/skills/vecscope/SKILL.md b/.agents/skills/vecscope/SKILL.md index a42c48a..264898c 100644 --- a/.agents/skills/vecscope/SKILL.md +++ b/.agents/skills/vecscope/SKILL.md @@ -110,6 +110,10 @@ Inputs: one vector per row in `.npy/.npz/.parquet/.jsonl/.json/.csv`. - The response lists `inconclusive_findings` (hubness, antihubs and connectivity verdicts other than `fail`, outlier verdicts other than `ok`): never report those as healthy; say they need `--sample 0`. +- Check `evidence_scope` first: `sampled` means every number is an estimate from a subset, + and `needs_full_audit` names the findings (inconclusive or unstable) whose verdict cannot + stand for the corpus. Report those as unsettled and re-run with `--sample 0` (MCP: + `sample=0`) before acting on them or passing the JSON on. - anisotropy, effective_rank, rogue_dimensions, norms, metric_mismatch and distance_concentration are stable estimates. - `value_range` is the spread between repeated samples, not an interval that contains the diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dfdc18..b8983ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,20 @@ All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses [Semantic Versioning](https://semver.org/). -## [Unreleased] +## [0.12.0] - 2026-11-06 ### Added +- `evidence_scope` (`full` or `sampled`) and `needs_full_audit` (the inconclusive and unstable + findings of a sampled audit) at the top level of the evidence JSON, and the MCP summary line + starts with the scope, so a sampled estimate is never read as a full audit. +- MCP server options `--max-rows` and `--max-dim` (`VECSCOPE_MAX_ROWS`, `VECSCOPE_MAX_DIM`) + refuse a matrix over the limit whatever its byte size; a store stops being read at + `--max-rows`. A `MemoryError` during a call is returned to the agent as a tool error that + names the ways out. +- `tests/test_mcp_security.py` pins the server's guards: paths outside the allowed + directories, the read, row and column limits, network stores without opt-in, secrets kept + out of errors, the timeout option, the `MemoryError` path and the scope of a sampled response. - Documentation site at , built with Zensical from the README, `docs/` and the Korean translations, and deployed by `.github/workflows/docs.yml`. - `CODE_OF_CONDUCT.md` (Contributor Covenant 2.1), `CITATION.cff`, a logo and social preview @@ -16,6 +26,9 @@ All notable changes to this project are documented here. The format follows ### Changed +- `docs/verification.md` is organized into five areas (algorithm exactness, thresholds and + retrieval quality, sampled audits, ANN recall and index sweeps, MCP security and operations), + each ending with what it does not cover. - Workflows run with no default token permissions, pin actions to commit SHAs, and do not keep the checkout credentials. CI also builds the package, checks it with `twine`, and lints the workflows with zizmor. The release workflow checks that the built version matches the tag. @@ -23,6 +36,12 @@ All notable changes to this project are documented here. The format follows - The Korean README and the agent cases page link to repository files by URL, so the links work on the documentation site as well as on GitHub. +### Fixed + +- The `timeout=` option of a network store rejects `inf`, `nan`, non-numeric and missing + values and caps the wait at 3600 s; before, `timeout=inf` passed the positive check and + disabled the timeout. + ## [0.11.0] - 2026-11-01 ### Changed diff --git a/README.md b/README.md index 6f74edd..9a8fa56 100644 --- a/README.md +++ b/README.md @@ -497,7 +497,8 @@ Options go in the query string: - `pk=` (Milvus primary key) - `api_key_env=NAME` (the key is read from that environment variable; keys in the URI itself are not used) -- `timeout=` seconds per request to a network store (default 60) +- `timeout=` seconds per request to a network store (default 60; a finite positive number, + at most 3600) Reading is read-only, except that a Milvus collection that is not loaded is loaded for the read and released again. @@ -646,8 +647,14 @@ One request cannot exhaust the server's memory: - A file larger than `--max-read-mb` (default 4096; `VECSCOPE_MAX_READ_MB`; 0 for no limit) is refused before it is read. - A store stops being read once its vectors pass the same size. +- `--max-rows` and `--max-dim` (`VECSCOPE_MAX_ROWS`, `VECSCOPE_MAX_DIM`; default 0 for no + limit) refuse a matrix with more rows or columns than that, whatever its byte size; the + queries file is checked the same way, and a store stops being read at `--max-rows`. - Files are parsed straight from the checked handle, never copied into memory first. On a 500,000 x 384 `.npy` file (0.72 GiB) an MCP read peaks at 0.79 GiB, as the CLI does. +- A `MemoryError` while reading or auditing is returned to the agent as a tool error that + names the ways out (a sample, a smaller input, the limits above) instead of ending the + call silently. Calls in parallel, or with expensive settings, are bounded too: @@ -970,6 +977,10 @@ about the sample. Which findings need every row: A sampled audit therefore: +- sets `evidence_scope` to `sampled` (`full` when every row was audited) and lists in + `needs_full_audit` the findings whose verdict cannot stand for the corpus: the inconclusive + ones and, with `--sample-repeats`, the unstable ones. Anything consuming the JSON can gate + on these two keys without reading the per-finding caveats - lists `inconclusive_findings` (the verdicts the table marks inconclusive) - flags them `INCONCLUSIVE (sampled)` in the text output and names them in the gate line - takes the collapse verdict from `details.estimated_full_share` (see below) diff --git a/docs/ko/README.md b/docs/ko/README.md index ecc79ca..09909db 100644 --- a/docs/ko/README.md +++ b/docs/ko/README.md @@ -486,7 +486,8 @@ vecscope audit "weaviate://localhost:8080/Docs?grpc_port=50051" - `https=1` - `pk=`(Milvus 기본 키) - `api_key_env=NAME`(키는 그 환경 변수에서 읽고, URI에 직접 적은 키는 쓰지 않음) -- `timeout=`(네트워크 저장소 요청 하나의 제한 시간, 초 단위, 기본값 60) +- `timeout=`(네트워크 저장소 요청 하나의 제한 시간, 초 단위, 기본값 60. 유한한 양수여야 하고 최대 + 3600) 읽기만 하며, 로드되지 않은 Milvus 컬렉션만 읽는 동안 로드했다가 다시 해제합니다. @@ -629,8 +630,13 @@ args = ["--allow-dir", "/data/embeddings"] - `--max-read-mb`(기본값 4096, `VECSCOPE_MAX_READ_MB`, 0이면 제한 없음)보다 큰 파일은 읽기 전에 거부합니다. - 저장소도 벡터가 같은 크기를 넘으면 읽기를 멈춥니다. +- `--max-rows`와 `--max-dim`(`VECSCOPE_MAX_ROWS`, `VECSCOPE_MAX_DIM`, 기본값 0이면 제한 없음)은 + 바이트 크기와 상관없이 행이나 열이 그보다 많은 행렬을 거부합니다. 질의 파일도 같은 기준으로 + 확인하고, 저장소는 `--max-rows`에서 읽기를 멈춥니다. - 파일은 확인한 핸들에서 바로 파싱하고, 먼저 메모리에 통째로 복사하지 않습니다. 500,000 x 384 `.npy` 파일(0.72 GiB)을 MCP로 읽을 때 최대 메모리는 CLI와 같은 0.79 GiB입니다. +- 읽기나 감사 중에 `MemoryError`가 나면 호출을 조용히 끝내지 않고, 빠져나갈 방법(표본, 더 작은 + 입력, 위의 상한)을 적은 도구 오류로 에이전트에게 돌려줍니다. 병렬 호출과 비용이 큰 설정에도 한도가 있습니다. @@ -950,6 +956,10 @@ zipf)을 받고 실행 환경 정보를 JSON에 함께 기록합니다. 그래서 표본 감사는 다음과 같이 처리합니다. +- `evidence_scope`를 `sampled`로 둡니다(모든 행을 감사했으면 `full`). 코퍼스 전체를 대신할 수 + 없는 판정, 즉 불확정(inconclusive) 항목과 `--sample-repeats`를 썼을 때의 불안정 항목을 + `needs_full_audit`에 나열합니다. JSON을 받는 쪽은 항목별 주의 문구를 읽지 않아도 이 두 키로 + 게이트를 걸 수 있습니다. - 표에서 inconclusive로 표시한 판정들이 `inconclusive_findings` 목록으로 들어갑니다. - 그 판정들은 텍스트 출력에서는 `INCONCLUSIVE (sampled)`로, gate 줄에서는 이름으로 표시됩니다. diff --git a/docs/ko/verification.md b/docs/ko/verification.md index f3d96bc..f411d6f 100644 --- a/docs/ko/verification.md +++ b/docs/ko/verification.md @@ -2,18 +2,20 @@ # 검증 결과 -vecscope의 주장을 어떻게 확인했고 측정 결과가 무엇을 보여 주는지 정리합니다. 대상은 다음과 -같습니다. +vecscope의 주장을 어떻게 확인했고 측정 결과가 무엇을 보여 주는지 다섯 영역으로 나누어 +정리합니다. -- float64 기준 정확성 -- ANN recall 채점기 -- 표본 감사 -- 라벨이 있는 데이터에서 검색 품질과 임곗값의 관계 -- 질의 항목 -- 계획(plan)의 조치를 적용했을 때 검색 품질이 어떻게 움직이는지 -- 에이전트가 MCP로 계획을 따라가는 과정 +1. [알고리즘 정확성](#1-알고리즘-정확성): k-NN, 증분 감사, 공개한 표를 float64와 전체 재계산에 + 대조한 결과 +2. [임곗값과 검색 품질](#2-임곗값과-검색-품질): 라벨이 있는 데이터에서 모든 진단 항목과 + nDCG@10의 관계, 질의 항목, 계획(plan)의 조치를 적용했을 때 검색 품질의 변화 +3. [표본 감사](#3-표본-감사): 표본이 대신할 수 있는 것과 없는 것 +4. [ANN recall과 인덱스 스윕](#4-ann-recall과-인덱스-스윕): recall 채점기와 스윕 +5. [MCP 보안과 운영](#5-mcp-보안과-운영): 서버의 보호 장치와 에이전트가 MCP로 계획을 따라가는 + 과정 -임곗값과 계획 규칙은 이 결과를 바탕으로 정했습니다. +임곗값과 계획 규칙은 이 결과를 바탕으로 정했습니다. 각 영역의 끝에 검증하지 않은 것을 적었고, +[한계](#한계)에 모아 두었습니다. - 실행 환경: AMD EPYC 7R13 8 vCPU, RAM 26 GB, NVIDIA L4. Python 3.12.3, numpy 2.2.6, faiss 1.15.1, sentence-transformers 3.4.1, torch 2.6.0 @@ -25,23 +27,30 @@ vecscope의 주장을 어떻게 확인했고 측정 결과가 무엇을 보여 한국어 4개: ko-strategyqa, MTEB의 XPQA kor-kor·AutoRAG·MultiLongDoc ko) - 벡터 저장소: faiss 1.15.1, Qdrant client 1.20(로컬 모드), Chroma 1.5, Milvus Lite 3.2, Weaviate client 4.24 +- 테스트: `pytest -q`와 `pytest -m slow`가 통과합니다. 벡터 저장소 테스트는 실제 로컬 + Qdrant·Chroma·Milvus Lite 인스턴스와 faiss 인덱스 파일로 실행합니다. -## 결과 요약 +## 1. 알고리즘 정확성 -| 영역 | 확인한 사실 | +| 주장 | 확인한 사실 | |---|---| -| 테스트 | `pytest -q`와 `pytest -m slow`가 통과합니다. 벡터 저장소 테스트는 실제 로컬 Qdrant·Chroma·Milvus Lite 인스턴스와 faiss 인덱스 파일로 실행합니다. | | 정확한 k-NN (`exact_knn`, `query_knn`, faiss flat CPU·GPU) | 100k × 768 분포 6종과 실제 임베딩 76개 (모델, 데이터세트) 조합에서 float64와 비교한 순위 오류가 0개입니다. float32 거리 오차는 최대 5.9e-7입니다. | -| ANN recall 채점 | 동점 허용 오차는 `1e-6 + 2e-6 × 거리`로, float32 오차의 약 3배입니다. 100k × 768 인덱스 설정 24개에서 엄격한 float64 채점기와 1e-4 이내(질의 1,000개 중 최대 1개)로 같고, 완전한 복제는 여전히 서로 바꿔도 되는 이웃으로 셉니다. 실제 질의에서는 3.1e-4 이내(질의 323개 중 1개)입니다. | -| 표본 감사의 값 | 분포형 항목은 전체 값과 2.3% 이내로 같고, hubness와 antihubs는 모든 경우 전체 값보다 낮게 나옵니다(하한). collapse의 `estimated_full_share`는 전체 비율의 1.015~1.026배입니다. | -| 표본 감사의 판정 | 전체 코퍼스보다 낮게 나온 표본 판정은 모두 `inconclusive_findings`에 들어 있습니다(clustered·duplicates의 hubness·antihubs, zipf의 antihubs·connectivity). | -| `sample_repeats` | `value_range`는 표본 사이의 흩어짐입니다. 전체 코퍼스 값이 그 안에 없는 경우가 잦으므로 구간 추정이 아닙니다. | -| 임곗값 | 공개 nDCG@10을 0.007 이내로 재현한 파이프라인 18개에서 `anisotropy`, `query_gap`, `query_hubness` 판정이 모두 `ok`입니다. 한국어 데이터에 쓴 영어 전용 모델 11개는 모두 `hubness`나 `query_hubness`를 올립니다. | | README "실제 임베딩" 표 | 각 데이터의 앞 4만 행에서 값과 판정 88개가 모두 재현됩니다. | | 증분 감사 (`--knn-cache`) | 100k × 768에 5천 행이나 2만 행을 덧붙였을 때 판정은 전체 감사와 같고 값은 상대 7e-5 이내입니다. 15초와 47초가 걸려 전체 감사(94초, 123초)보다 빠릅니다. 1,000행을 제자리에서 다시 임베딩하면 10,625행을 다시 계산했고 판정이 같으며 값은 4e-5 이내, 19초로 전체 감사 95초보다 빠릅니다. 1만 행이면 갱신이 83초로 전체 감사 89초와 비슷해서, 다시 계산할 행이 절반을 넘으면 캐시를 새로 만듭니다. 이 기준으로 다시 돌리자 1만 행 변경에서 캐시를 새로 만들었고 87초로 전체 감사(87초)와 같았습니다. | -| sweep | 추천 설정이 seed 3개에서 같습니다. nprobe를 고정한 IVF recall은 데이터의 군집이 nlist와 어떻게 겹치는지에 따라 규모가 커질 때 오르기도 내리기도 합니다. | +| 결정성 | 같은 입력으로 두 번 실행하면 JSON이 완전히 같습니다(고정 seed, 정확한 이웃). `tests/golden/`의 파일이 예제 행렬의 결과를 고정합니다. | -## 진단값과 nDCG@10의 관계 +검증하지 않은 것: 2만 행이 넘는 실제 코퍼스의 정확성은 합성 데이터로 확인했고, 실제 데이터 +검사는 그보다 작은 76개 조합에서 했습니다. + +## 2. 임곗값과 검색 품질 + +| 주장 | 확인한 사실 | +|---|---| +| 임곗값 | 공개 nDCG@10을 0.007 이내로 재현한 파이프라인 18개에서 `anisotropy`, `query_gap`, `query_hubness` 판정이 모두 `ok`입니다. 한국어 데이터에 쓴 영어 전용 모델 11개는 모두 `hubness`나 `query_hubness`를 올립니다. | +| 계획이 다루는 항목은 검색 품질과 함께 움직임 | 중복 제거, 지배 좌표의 z-score, ef_search 상향이 각각 nDCG@10을 회복시킵니다([개입 실험](#개입-실험)). | +| 계획이 요구하지 않은 변경은 공짜가 아님 | 건강한 모델을 중심화하면 `anisotropy`는 사라지지만 nDCG@10이 최대 0.034 떨어집니다. | + +### 진단값과 nDCG@10의 관계 데이터세트마다 모델 19종에 대해 각 항목의 값과 nDCG@10 사이의 Spearman 상관을 구했습니다. 음수면 검색이 나빠질수록 값이 커진다는 뜻입니다. @@ -86,7 +95,7 @@ ArguAna의 nDCG@10은 BEIR과 같이 질의 자신의 문서(코퍼스에 들어 전용 모델)은 여러 항목을 동시에 올립니다. 이런 파이프라인을 가르는 것은 경고의 개수이지 `fail` 하나가 아닙니다. -## 한국어 데이터와 문서 길이 +### 한국어 데이터와 문서 길이 MTEB의 한국어 데이터세트 3개를 더 측정했습니다. 어느 것도 임곗값을 정하는 데 쓰지 않았고, 주로 문서 길이가 다릅니다. 모든 모델에서 문서를 512토큰으로 잘랐습니다. @@ -121,7 +130,7 @@ Qwen3-Embedding-0.6B입니다. - "검색에서 서로 바꿔 써도 되는 문서가 많다"로 읽어야 하고, 모든 모델의 낮은 nDCG@10이 이를 뒷받침합니다. 계획은 `query_gap`을 중복과 같은 단계에 넣으므로 질의 탓으로 돌리지 않습니다. -## 질의 항목 +### 질의 항목 `query_gap`과 `query_shift`는 SciFact, NFCorpus, ko-strategyqa에서 모델 19종의 임베딩 캐시로 후보 세 가지를 측정해 골랐습니다. 모든 (모델, 데이터세트) 조합을 자기 질의와 주입한 결함 세 @@ -166,7 +175,7 @@ q_nn은 질의에서 코퍼스로의 최근접 거리입니다. 임곗값은 재 인코딩하므로 둘 사이에 비대칭이 없습니다. 이 경우는 `query_hubness`가 11개 중 10개를 fail(11~30), 나머지 하나를 warn(8.3)으로 잡습니다. -## 개입 실험 +### 개입 실험 계획의 단계를 적용하면 검색 품질이 움직일까요? @@ -197,12 +206,64 @@ q_nn은 질의에서 코퍼스로의 최근접 거리입니다. 임곗값은 재 `vecscope sweep`(README "인덱스 스윕")으로 확인합니다. - 건강한 모델을 중심화해도 나아지지 않습니다. `anisotropy`는 사라졌지만 nDCG@10이 최대 0.034 떨어졌습니다. 계획은 `anisotropy`가 `ok`가 아닐 때만 이 조치를 제안하고, 재측정 안내에서 - 라벨이 있는 질의로 검색 품질을 확인하라고 적습니다. + 라벨이 있는 질의로 검색 품질을 확인하라고 적습니다. 계획의 단계는 지금 코퍼스에서 확인해야 + 할 가설이지 진단이 아니므로, 조치를 적용한 에이전트는 반드시 다시 감사하고 검색 품질을 + 비교한 뒤에 채택해야 합니다. - 지배 좌표는 노름도 흩뜨리고 코사인과 내적의 순위도 갈라 놓습니다. 정규화로는 그 좌표가 사라지지 않으므로, 계획은 이 항목들을 z-score 조치와 함께 `dominant_dimension` 단계에 넣습니다. -## 에이전트 연동 +검증하지 않은 것: 임곗값과 질의 항목은 위 데이터에 맞춰 정했고 홀드아웃 분할이 없으며, 여기 +있는 데이터세트 9개 밖에서 모델·언어·문서 길이·코퍼스 크기별 오탐과 미탐을 추적하지 +않았습니다. 아주 짧은 정형 문서나 좁은 도메인 코퍼스에서는 일부 기본값(`collapse`, +`effective_rank`, `query_gap`)이 좋은 모델도 잡으므로, 그런 데이터에서는 게이트로 쓰기 전에 +보정하십시오. 운영 질의 로그와 비공개 코퍼스는 측정하지 않았습니다. + +## 3. 표본 감사 + +| 주장 | 확인한 사실 | +|---|---| +| 값 | 분포형 항목은 전체 값과 2.3% 이내로 같고, hubness와 antihubs는 모든 경우 전체 값보다 낮게 나옵니다(하한). collapse의 `estimated_full_share`는 전체 비율의 1.015~1.026배입니다. | +| 판정 | 전체 코퍼스보다 낮게 나온 표본 판정은 모두 `inconclusive_findings`에 들어 있습니다(clustered·duplicates의 hubness·antihubs, zipf의 antihubs·connectivity). | +| `sample_repeats` | `value_range`는 표본 사이의 흩어짐입니다. 전체 코퍼스 값이 그 안에 없는 경우가 잦으므로 구간 추정이 아닙니다. | +| 출력의 범위 표시 | 표본 감사는 `evidence_scope`를 `"sampled"`로 두고, 불확정(inconclusive) 항목과 불안정 항목을 `needs_full_audit`에 나열합니다. MCP 요약 줄은 범위로 시작하므로 표본 추정치가 전체 감사로 읽히지 않습니다(`tests/test_mcp_security.py`). | + +검증하지 않은 것: 표본 감사는 균등 무작위 행을 씁니다. 군집별 층화 표본은 검토했지만 넣지 +않았습니다. 섬은 max(k + 1, 행의 0.5%)보다 작고 떨어져 있어야 하므로 층을 어떻게 나눠도 +표본은 작은 섬을 놓치고, 층화는 분포형 항목을 편향시킵니다. 대신 완전 복제는 전체 행에서 +셉니다. + +## 4. ANN recall과 인덱스 스윕 + +| 주장 | 확인한 사실 | +|---|---| +| ANN recall 채점 | 동점 허용 오차는 `1e-6 + 2e-6 × 거리`로, float32 오차의 약 3배입니다. 100k × 768 인덱스 설정 24개에서 엄격한 float64 채점기와 1e-4 이내(질의 1,000개 중 최대 1개)로 같고, 완전한 복제는 여전히 서로 바꿔도 되는 이웃으로 셉니다. 실제 질의에서는 3.1e-4 이내(질의 323개 중 1개)입니다. | +| `ann_recall`은 ranked 검색 품질을 따라감 | HNSW M=16, ef_search 4, 8, 16, 64에서 `ann_recall` 중앙값 0.883, 0.958, 0.984, 0.998이고 정확 검색 대비 ranked nDCG@10은 0.83, 0.92, 0.96, 0.995입니다. 84회 전체에서 Spearman 0.94입니다([개입 실험](#개입-실험)). | +| sweep | 추천 설정이 seed 3개에서 같습니다. nprobe를 고정한 IVF recall은 데이터의 군집이 nlist와 어떻게 겹치는지에 따라 규모가 커질 때 오르기도 내리기도 합니다. | + +검증하지 않은 것: 5천 행 이하 코퍼스의 검색 지연 시간은 측정 잡음보다 작으므로, 지연 시간 +쪽은 100k 행에서 재는 `vecscope sweep`(README "인덱스 스윕")으로 확인합니다. HNSW와 IVF 외의 +인덱스 종류는 스윕하지 않았습니다. + +## 5. MCP 보안과 운영 + +서버의 보호 장치는 `tests/test_mcp_security.py`가 작은 행렬로 고정합니다. + +- 허용 디렉터리 밖의 코퍼스·질의·비교 파일은 거부하고, 디렉터리나 없는 파일은 사용자 오류로 + 보고합니다. +- `--max-read-mb`는 파일을 파싱하기 전에 거부하고, `0`이면 제한을 풀며, 숫자가 아닌 값은 + 오류로 보고합니다. `--max-rows`와 `--max-dim`은 바이트 크기와 상관없이 상한을 넘는 + 행렬(질의 파일 포함)을 거부하고, 명령줄 옵션이 이 값을 설정합니다. +- 네트워크 저장소는 `--allow-network` 없이는 거부하고, 오류 메시지에 비밀번호·API 키·URI의 + 다른 비밀값이 실리지 않습니다. `timeout=` 옵션은 유한한 양수 초여야 하고 최대 3,600입니다. +- 읽기나 감사 중의 `MemoryError`는 호출을 조용히 끝내지 않고, 빠져나갈 방법(표본, 더 작은 + 입력, 상한 옵션)을 적은 도구 오류로 에이전트에게 전달됩니다. +- 표본 감사 응답은 `evidence_scope`, `needs_full_audit`, 요약 줄의 범위 표시를 담습니다. + +벡터 저장소 읽기는 실제 로컬 Qdrant·Chroma·Milvus Lite 인스턴스와 faiss 인덱스 파일로 +확인했고(`tests/test_adapters.py`), 바이트 상한에서 읽기가 멈추는 것도 포함합니다. + +### 에이전트 연동 - `tests/test_agent_workflow.py`는 MCP 서버를 stdio로 구동합니다. 정규화되지 않은 벡터에 공통 오프셋을 더한 코퍼스를 라벨 질의와 함께 감사합니다. @@ -239,6 +300,14 @@ q_nn은 질의에서 코퍼스로의 최근접 거리입니다. 임곗값은 재 - 스윕이 nlist를 스스로 정해(132) 둘 다 손으로 다시 재야 했고, 이 때문에 `sweep --nlist`를 추가했습니다. +검증하지 않은 것: Weaviate 어댑터는 옵트인 테스트(`VECSCOPE_TEST_WEAVIATE=1`)에서 내장 서버로만 +확인했습니다. 원격 Qdrant·Milvus·Weaviate 서버는 테스트하지 않았습니다. `timeout=` 옵션과 +클라이언트 오류의 비밀값 제거는 응답하지 않는 Qdrant 포트와 대역 클라이언트로 확인했고, 느린 +서버로는 확인하지 않았습니다. 행·차원 상한과 `MemoryError` 경로는 상한을 낮게 둔 작은 행렬로 +확인했고, 실제로 메모리를 고갈시켜 보지는 않았습니다. 많은 호출을 받으며 오래 실행되는 서버의 +메모리는 측정하지 않았습니다. 에이전트 실행은 모델 하나, 과제 세 개입니다. 도구와 계획을 쓸 수 +있다는 것을 보여 줄 뿐, 모든 에이전트가 그대로 따른다는 뜻은 아닙니다. + ## 한계 - 임곗값과 질의 항목은 위 데이터에 맞춰 정했고 홀드아웃 분할은 없습니다. 운영 질의 로그와 @@ -247,21 +316,22 @@ q_nn은 질의에서 코퍼스로의 최근접 거리입니다. 임곗값은 재 일부 기본값(`collapse`, `effective_rank`, `query_gap`)이 좋은 모델도 잡으므로, 그런 데이터에서는 게이트로 쓰기 전에 보정하십시오. 2만 행이 넘는 코퍼스의 정확성은 합성 데이터로 확인했고, 실제 데이터 정확성 검사는 그보다 작은 76개 조합에서 했습니다. -- 표본 감사는 균등 무작위 행을 씁니다. 군집별 층화 표본은 검토했지만 넣지 않았습니다. 섬은 - max(k + 1, 행의 0.5%)보다 작고 떨어져 있어야 하므로 층을 어떻게 나눠도 표본은 작은 섬을 놓치고, - 층화는 분포형 항목을 편향시킵니다. 대신 완전 복제는 전체 행에서 셉니다. +- 표본 감사는 균등 무작위 행을 씁니다(층화 표본을 넣지 않은 이유는 [표본 감사](#3-표본-감사) + 참고). - Weaviate 어댑터는 옵트인 테스트(`VECSCOPE_TEST_WEAVIATE=1`)에서 내장 서버로만 확인했습니다. 원격 Qdrant·Milvus·Weaviate 서버는 테스트하지 않았습니다. `timeout=` 옵션과 클라이언트 오류의 비밀값 제거는 응답하지 않는 Qdrant 포트와 대역 클라이언트로 확인했고, 느린 서버로는 확인하지 않았습니다. +- MCP 보호 장치는 작은 입력에 상한을 낮게 두고 확인했고, 실제로 메모리를 고갈시키거나 오래 + 실행되는 서버를 측정하지는 않았습니다. - 에이전트 실행은 모델 하나, 과제 세 개입니다. 도구와 계획을 쓸 수 있다는 것을 보여 줄 뿐, 모든 에이전트가 그대로 따른다는 뜻은 아닙니다. ## 재현 방법 - `pytest -q`는 검증 테스트(`tests/test_new_features.py`, `tests/test_scale_exactness.py`, - `tests/test_agent_workflow.py`)를 실행합니다. `pytest -m slow`를 주면 100k × 384 정확성 - 검사도 실행합니다. + `tests/test_agent_workflow.py`, `tests/test_mcp_security.py`)를 실행합니다. `pytest -m slow`를 + 주면 100k × 384 정확성 검사도 실행합니다. - `python examples/benchmark_scale.py --rows 10000 50000 100000 --dim 768 --json scale.json`과 `python examples/benchmark_sweep.py --rows 100000 --dim 768 --corpus clustered zipf --json sweep.json`으로 표를 다시 잴 수 있습니다. 실행 환경 정보가 JSON에 함께 기록됩니다. diff --git a/docs/verification.md b/docs/verification.md index 3b59f2e..ae15f8e 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -2,17 +2,22 @@ # Verification -This page records how vecscope's claims were checked and what the measurements show: - -- exactness against float64 -- the ANN recall scorer -- sampled audits -- the thresholds against retrieval quality on labeled data -- the query findings -- what happens to retrieval when a fix from the plan is applied -- an agent following the plan over MCP - -The thresholds and the plan rules are set from these results. +This page records how vecscope's claims were checked and what the measurements show, in five +areas: + +1. [Algorithm exactness](#1-algorithm-exactness): the k-NN, the incremental audit and the + published tables against float64 and against a full recomputation +2. [Thresholds and retrieval quality](#2-thresholds-and-retrieval-quality): every finding + against nDCG@10 on labeled data, the query findings, and what retrieval does when a plan + step is applied +3. [Sampled audits](#3-sampled-audits): what a sample can and cannot stand for +4. [ANN recall and index sweeps](#4-ann-recall-and-index-sweeps): the recall scorer and the + sweep +5. [MCP security and operations](#5-mcp-security-and-operations): the server's guards and + agents following the plan over MCP + +The thresholds and the plan rules are set from these results. Each area ends with what it +does not cover; the [limitations](#limitations) collect them. - Environment: AMD EPYC 7R13, 8 vCPU, 26 GB RAM, NVIDIA L4; Python 3.12.3, numpy 2.2.6, faiss 1.15.1, sentence-transformers 3.4.1, torch 2.6.0 @@ -25,23 +30,30 @@ The thresholds and the plan rules are set from these results. and MultiLongDoc ko) - Vector stores: faiss 1.15.1, Qdrant client 1.20 (local mode), Chroma 1.5, Milvus Lite 3.2, Weaviate client 4.24 +- Tests: `pytest -q` and `pytest -m slow` pass; the vector store tests run against real + local Qdrant, Chroma and Milvus Lite instances and a faiss index file -## Results +## 1. Algorithm exactness -| area | result | +| claim | result | |---|---| -| tests | `pytest -q` and `pytest -m slow` pass; the vector store tests run against real local Qdrant, Chroma and Milvus Lite instances and a faiss index file | | exact k-NN (`exact_knn`, `query_knn`, faiss flat CPU and GPU) | 0 rank errors against float64 on six shapes at 100k x 768 and on 76 (model, dataset) pairs of real embeddings; float32 distance error at most 5.9e-7 | -| ANN recall scoring | tie slack is `1e-6 + 2e-6 x distance`, about three times the float32 error. On 24 index settings at 100k x 768 the score is within 1e-4 of a strict float64 scorer (at most one query in 1,000) and exact copies still count as interchangeable; on real queries within 3.1e-4 (one query in 323) | -| sampled audit, values | distribution findings within 2.3% of the full value; hubness and antihubs a lower bound everywhere; the collapse `estimated_full_share` is 1.015 to 1.026 x the full share | -| sampled audit, verdicts | every sampled verdict below the full-corpus one is listed in `inconclusive_findings` (clustered and duplicates hubness and antihubs, zipf antihubs and connectivity) | -| `sample_repeats` | `value_range` is the spread between samples; it often does not contain the full-corpus value, so it is not an interval estimate | -| thresholds | on the 18 pipelines that reproduce published nDCG@10 within 0.007, no `anisotropy`, `query_gap` or `query_hubness` verdict is worse than `ok`; all 11 English-only models on Korean data raise `hubness` or `query_hubness` | | README "Real embeddings" table | all 88 values and verdicts reproduced on the first 40k rows of each set | | incremental audit (`--knn-cache`) | on 100k x 768 with 5k or 20k appended rows, verdicts identical to a full audit and values within 7e-5 relative; 15 s and 47 s against 94 s and 123 s for the full audit. With 1,000 rows re-embedded in place, 10,625 rows recomputed, same verdicts, values within 4e-5, 19 s against 95 s; with 10,000 the update took 83 s against 89 s, which is why the cache rebuilds once more than half the rows would be recomputed; re-run with that rule, 10,000 changed rows rebuilt it in 87 s against 87 s | -| sweep | recommendations identical across three seeds; IVF recall at a fixed nprobe can rise or fall with corpus size depending on how the data's clusters line up with nlist | +| determinism | two runs on the same input produce identical JSON (fixed seeds, exact neighbors); the golden files under `tests/golden/` pin the example matrices | + +Not covered: exactness on real corpora over 20,000 rows was checked on synthetic data; the +real-data check covers the 76 smaller pairs. -## Finding values against nDCG@10 +## 2. Thresholds and retrieval quality + +| claim | result | +|---|---| +| thresholds | on the 18 pipelines that reproduce published nDCG@10 within 0.007, no `anisotropy`, `query_gap` or `query_hubness` verdict is worse than `ok`; all 11 English-only models on Korean data raise `hubness` or `query_hubness` | +| a finding the plan acts on tracks retrieval | deduplicating, z-scoring a dominant coordinate and raising ef_search each recover nDCG@10 ([interventions](#interventions)) | +| a change the plan does not ask for is not free | centering a healthy model removed `anisotropy` and cost up to 0.034 nDCG@10 | + +### Finding values against nDCG@10 Spearman correlation between each finding's value and nDCG@10 across the 19 models, per dataset. A negative value means the finding rises as retrieval gets worse. @@ -89,7 +101,7 @@ What this supports: an English-only model on Korean) raises several findings at once. The count of warnings separates such pipelines; a single `fail` does not. -## Korean data and document length +### Korean data and document length Three more Korean sets from MTEB, none used to set a threshold, differ mainly in document length. Documents were truncated to 512 tokens for every model. @@ -126,7 +138,7 @@ gte-multilingual-base and Qwen3-Embedding-0.6B. every model confirms. The plan puts `query_gap` with the duplicates, so the queries are not blamed for it. -## Query findings +### Query findings `query_gap` and `query_shift` were chosen from three candidates measured on the cached embeddings of the 19 models on SciFact, NFCorpus and ko-strategyqa. Every (model, dataset) @@ -171,7 +183,7 @@ Three of the fault criteria turned out to describe something other than a query- and documents equally badly, so nothing is asymmetric between them; `query_hubness` fails 10 of the 11 (11 to 30) and warns on the last (8.3). -## Interventions +### Interventions Does retrieval quality move when a plan step is applied? @@ -207,12 +219,67 @@ What this supports: "Index sweep"), not here. - Centering a healthy model does not help: it removed `anisotropy` and cost up to 0.034 nDCG@10. The plan only proposes it when `anisotropy` is not `ok`, and its re-measure line - says to check retrieval on labeled queries. + says to check retrieval on labeled queries. A plan step is a hypothesis to test on the + corpus at hand, not a diagnosis; an agent that applies one must re-audit and compare + retrieval before adopting it. - A dominant coordinate also spreads the norms and splits cosine from inner product. Normalizing does not remove the coordinate, so the plan puts those findings in the `dominant_dimension` step with the z-score change. -## Agent integration +Not covered: the thresholds and the query findings were fitted on the data above, with no +held-out split, and false positives and misses were not tracked per model, language, +document length or corpus size beyond the nine datasets here. On very short templated +documents and on narrow-domain corpora some defaults (`collapse`, `effective_rank`, +`query_gap`) flag good models; calibrate on such data before gating. Production query logs +and private corpora were not measured. + +## 3. Sampled audits + +| claim | result | +|---|---| +| values | distribution findings within 2.3% of the full value; hubness and antihubs a lower bound everywhere; the collapse `estimated_full_share` is 1.015 to 1.026 x the full share | +| verdicts | every sampled verdict below the full-corpus one is listed in `inconclusive_findings` (clustered and duplicates hubness and antihubs, zipf antihubs and connectivity) | +| `sample_repeats` | `value_range` is the spread between samples; it often does not contain the full-corpus value, so it is not an interval estimate | +| scope in the output | a sampled audit sets `evidence_scope: "sampled"` and lists in `needs_full_audit` the inconclusive and unstable findings; the MCP summary line starts with the scope, so a sampled estimate is never read as a full audit (`tests/test_mcp_security.py`) | + +Not covered: sampled audits use uniform random rows. Stratified (per-cluster) sampling was +considered and not adopted: an island must be smaller than max(k + 1, 0.5% of the rows) and +isolated, so a sample misses small islands whatever the strata, while stratifying would bias +the distribution findings. Exact copies are counted over every row instead. + +## 4. ANN recall and index sweeps + +| claim | result | +|---|---| +| ANN recall scoring | tie slack is `1e-6 + 2e-6 x distance`, about three times the float32 error. On 24 index settings at 100k x 768 the score is within 1e-4 of a strict float64 scorer (at most one query in 1,000) and exact copies still count as interchangeable; on real queries within 3.1e-4 (one query in 323) | +| `ann_recall` tracks ranked retrieval | HNSW M=16 at ef_search 4, 8, 16, 64: `ann_recall` median 0.883, 0.958, 0.984, 0.998 against ranked nDCG@10 at 0.83, 0.92, 0.96, 0.995 of exact; Spearman 0.94 over 84 runs ([interventions](#interventions)) | +| sweep | recommendations identical across three seeds; IVF recall at a fixed nprobe can rise or fall with corpus size depending on how the data's clusters line up with nlist | + +Not covered: search latency on corpora under 5,000 rows is below the measurement noise; the +latency side of the trade-off is measured by `vecscope sweep` on 100k rows (README "Index +sweep"). Index types other than HNSW and IVF were not swept. + +## 5. MCP security and operations + +The server's guards are pinned by `tests/test_mcp_security.py` on small matrices: + +- a corpus, queries or comparison file outside the allowed directories is refused, and a + directory or a missing file is reported as a user error +- `--max-read-mb` refuses a file before it is parsed, `0` lifts the limit, and a value that + is not a number is reported; `--max-rows` and `--max-dim` refuse a matrix (or a queries + file) over the limit whatever its byte size, and the command-line options set them +- a network store is refused without `--allow-network`, and the error never carries the + password, the API key or the other URI secrets; the `timeout=` option must be a finite + positive number of seconds and at most 3,600 +- a `MemoryError` while reading or auditing reaches the agent as a tool error that names the + ways out (a sample, a smaller input, the limits) instead of ending the call silently +- a sampled response states `evidence_scope`, `needs_full_audit` and the scope in its + summary line + +Vector store reads were tested against real local Qdrant, Chroma and Milvus Lite instances +and a faiss index file (`tests/test_adapters.py`), including the byte limit stopping a read. + +### Agent integration - `tests/test_agent_workflow.py` drives the MCP server over stdio: a corpus with unnormalized vectors and a shared offset, audited with labeled queries. @@ -251,6 +318,15 @@ What this supports: - Both had to re-measure by hand because the sweep chose its own nlist (132), which led to `sweep --nlist`. +Not covered: the Weaviate adapter was tested only against an embedded server in opt-in tests +(`VECSCOPE_TEST_WEAVIATE=1`); remote Qdrant, Milvus and Weaviate servers were not tested. +The `timeout=` option and the scrubbing of client errors were tested against an unreachable +Qdrant port and a stubbed client, not against a slow server. The row and column limits and +the `MemoryError` path were tested on small matrices with the limits set low, not by +exhausting a machine. The memory of a long-running server under many calls was not measured. +The agent runs are one model on three tasks; they show the tools and the plan are usable, +not that every agent follows them. + ## Limitations - The thresholds and the query findings were fitted on the data above; there is no held-out @@ -260,22 +336,22 @@ What this supports: `query_gap`) flag good models; calibrate on such data before gating. Exactness on corpora over 20,000 rows was checked on synthetic data; the real-data exactness check covers the 76 smaller pairs. -- Sampled audits use uniform random rows. Stratified (per-cluster) sampling was considered and - not adopted: an island must be smaller than max(k + 1, 0.5% of the rows) and isolated, so a - sample misses small islands whatever the strata, while stratifying would bias the - distribution findings. Exact copies are counted over every row instead. +- Sampled audits use uniform random rows (see [sampled audits](#3-sampled-audits) for why + stratified sampling was not adopted). - The Weaviate adapter was tested only against an embedded server in opt-in tests (`VECSCOPE_TEST_WEAVIATE=1`); remote Qdrant, Milvus and Weaviate servers were not tested. The `timeout=` option and the scrubbing of client errors were tested against an unreachable Qdrant port and a stubbed client, not against a slow server. +- The MCP guards were tested with low limits on small inputs, not by exhausting a machine, and + a long-running server was not measured. - The agent runs are one model on three tasks; they show the tools and the plan are usable, not that every agent follows them. ## Reproducing - `pytest -q` runs the verification tests (`tests/test_new_features.py`, - `tests/test_scale_exactness.py`, `tests/test_agent_workflow.py`); `pytest -m slow` adds the - 100k x 384 exactness checks. + `tests/test_scale_exactness.py`, `tests/test_agent_workflow.py`, + `tests/test_mcp_security.py`); `pytest -m slow` adds the 100k x 384 exactness checks. - `python examples/benchmark_scale.py --rows 10000 50000 100000 --dim 768 --json scale.json` and `python examples/benchmark_sweep.py --rows 100000 --dim 768 --corpus clustered zipf --json sweep.json` re-measure the tables and record the machine in the JSON. diff --git a/server.json b/server.json index f2374e9..47ae14f 100644 --- a/server.json +++ b/server.json @@ -7,13 +7,13 @@ "url": "https://github.com/076923/vecscope", "source": "github" }, - "version": "0.11.0", + "version": "0.12.0", "packages": [ { "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "vecscope", - "version": "0.11.0", + "version": "0.12.0", "runtimeHint": "uvx", "transport": { "type": "stdio" diff --git a/src/vecscope/__init__.py b/src/vecscope/__init__.py index 804a524..2c8dbe0 100644 --- a/src/vecscope/__init__.py +++ b/src/vecscope/__init__.py @@ -3,5 +3,5 @@ from vecscope.compare import CompareResult, compare from vecscope.evidence import Evidence, Finding, audit -__version__ = "0.11.0" +__version__ = "0.12.0" __all__ = ["CompareResult", "Evidence", "Finding", "__version__", "audit", "compare"] diff --git a/src/vecscope/adapters.py b/src/vecscope/adapters.py index 231d739..1f7728c 100644 --- a/src/vecscope/adapters.py +++ b/src/vecscope/adapters.py @@ -26,6 +26,7 @@ from __future__ import annotations +import math import os from dataclasses import dataclass, field from pathlib import Path @@ -45,6 +46,7 @@ ) FAISS_SUFFIXES = (".faiss", ".index") _BATCH = 1024 +MAX_TIMEOUT = 3600.0 # seconds; a request to a store may not wait longer than this DEFAULT_TIMEOUT = 60.0 # query options that are recorded as given; every other value is redacted _SAFE_OPTS = ("collection", "vector", "https", "pk", "grpc_port", "api_key_env", "timeout") @@ -116,22 +118,29 @@ def _api_key(o: dict) -> str | None: def _timeout(o: dict) -> float: + """``timeout=`` seconds from the URI options: a finite positive number of seconds, at most + ``MAX_TIMEOUT``, so a store that never answers cannot hold a request open forever.""" + raw = o.get("timeout", DEFAULT_TIMEOUT) try: - t = float(o.get("timeout", DEFAULT_TIMEOUT)) - except ValueError: - raise ValueError(f"timeout={o['timeout']!r} is not a number of seconds") from None - if not t > 0: - raise ValueError(f"timeout must be positive, got {t}") + t = float(raw) + except (TypeError, ValueError): + raise ValueError(f"timeout={raw!r} is not a number of seconds") from None + if not math.isfinite(t) or t <= 0: + raise ValueError(f"timeout must be a finite positive number of seconds, got {raw!r}") + if t > MAX_TIMEOUT: + raise ValueError(f"timeout={raw!r} is over the {MAX_TIMEOUT:.0f} s most a request may wait") return t class _Rows: """Ids and float32 vectors as they are read, refusing more than ``max_bytes`` of vectors - (a client's list of Python floats is several times larger than the array it becomes).""" + or more than ``max_rows`` rows. Each vector is converted to a float32 array as it + arrives, so the client's list of Python floats (several times larger than the array it + becomes) lives only as long as one batch.""" - def __init__(self, max_bytes: int | None, what: str): + def __init__(self, max_bytes: int | None, what: str, max_rows: int | None = None): self.ids, self.vectors, self.nbytes = [], [], 0 - self.max_bytes, self.what = max_bytes, what + self.max_bytes, self.max_rows, self.what = max_bytes, max_rows, what def add(self, i, v) -> None: v = np.asarray(v, dtype=np.float32).ravel() @@ -142,6 +151,11 @@ def add(self, i, v) -> None: "vectors, the most this reader may load; audit a smaller collection " "or raise the limit" ) + if self.max_rows is not None and len(self.ids) >= self.max_rows: + raise ValueError( + f"{self.what} holds more than {self.max_rows} rows, the most this reader " + "may load; audit a smaller collection or raise the limit" + ) self.ids.append(i) self.vectors.append(v) @@ -272,7 +286,7 @@ def search(q, k): ) -def _qdrant(uri: str, max_bytes: int | None = None) -> Source: +def _qdrant(uri: str, max_bytes: int | None = None, max_rows: int | None = None) -> Source: try: from qdrant_client import QdrantClient, models except ImportError as e: @@ -294,7 +308,7 @@ def _qdrant(uri: str, max_bytes: int | None = None) -> Source: if not collection: raise ValueError(f"{_redact(uri)}: name the collection") name = o.get("vector") - rows, offset = _Rows(max_bytes, f"qdrant collection {collection}"), None + rows, offset = _Rows(max_bytes, f"qdrant collection {collection}", max_rows), None while True: points, offset = client.scroll( collection, limit=_BATCH, offset=offset, with_vectors=True, with_payload=False @@ -318,7 +332,7 @@ def search(q, k): ) -def _chroma(uri: str, max_bytes: int | None = None) -> Source: +def _chroma(uri: str, max_bytes: int | None = None, max_rows: int | None = None) -> Source: try: import chromadb except ImportError as e: @@ -345,7 +359,7 @@ def _chroma(uri: str, max_bytes: int | None = None) -> Source: if not collection: raise ValueError(f"{_redact(uri)}: name the collection") col = client.get_collection(collection) - rows, offset = _Rows(max_bytes, f"chroma collection {collection}"), 0 + rows, offset = _Rows(max_bytes, f"chroma collection {collection}", max_rows), 0 while True: got = col.get(include=["embeddings"], limit=_BATCH, offset=offset) if not got["ids"]: @@ -364,7 +378,7 @@ def search(q, k): ) -def _milvus(uri: str, max_bytes: int | None = None) -> Source: +def _milvus(uri: str, max_bytes: int | None = None, max_rows: int | None = None) -> Source: try: from pymilvus import MilvusClient except ImportError as e: @@ -399,7 +413,7 @@ def _milvus(uri: str, max_bytes: int | None = None) -> Source: loaded_here = "notload" in state.replace("_", "") if loaded_here: client.load_collection(collection) - rows = _Rows(max_bytes, f"milvus collection {collection}") + rows = _Rows(max_bytes, f"milvus collection {collection}", max_rows) it = client.query_iterator(collection, batch_size=_BATCH, output_fields=[pk, name]) try: while True: @@ -442,7 +456,7 @@ def release(): return src -def _weaviate(uri: str, max_bytes: int | None = None) -> Source: +def _weaviate(uri: str, max_bytes: int | None = None, max_rows: int | None = None) -> Source: try: import weaviate from weaviate.classes.init import AdditionalConfig, Auth, Timeout @@ -470,7 +484,7 @@ def _weaviate(uri: str, max_bytes: int | None = None) -> Source: raise ValueError(f"{_redact(uri)}: name the collection") col = client.collections.get(collection) name = o.get("vector") - rows = _Rows(max_bytes, f"weaviate collection {collection}") + rows = _Rows(max_bytes, f"weaviate collection {collection}", max_rows) try: for obj in col.iterator(include_vector=True): rows.add(str(obj.uuid), _pick_vector(obj.vector, name, "weaviate")) @@ -495,23 +509,29 @@ def search(q, k): ) -def load_source(uri: str, data: bytes | None = None, max_bytes: int | None = None) -> Source: +def load_source( + uri: str, + data: bytes | None = None, + max_bytes: int | None = None, + max_rows: int | None = None, +) -> Source: """Vectors (ascending id order), their ids, and a search function from a vector store. ``data`` is a faiss index file's content when the caller has already read it; - ``max_bytes`` caps the float32 vectors read.""" + ``max_bytes`` caps the float32 vectors read and ``max_rows`` the rows read from a store + (a faiss index is checked by its caller, which knows its size up front).""" from vecscope.incremental import fingerprint scheme = scheme_of(uri) if scheme == "faiss" or Path(str(uri)).suffix.lower() in FAISS_SUFFIXES and not scheme: read, args = _faiss, (str(uri), data, max_bytes) elif scheme.startswith("qdrant"): - read, args = _qdrant, (uri, max_bytes) + read, args = _qdrant, (uri, max_bytes, max_rows) elif scheme.startswith("chroma"): - read, args = _chroma, (uri, max_bytes) + read, args = _chroma, (uri, max_bytes, max_rows) elif scheme.startswith("milvus"): - read, args = _milvus, (uri, max_bytes) + read, args = _milvus, (uri, max_bytes, max_rows) elif scheme == "weaviate": - read, args = _weaviate, (uri, max_bytes) + read, args = _weaviate, (uri, max_bytes, max_rows) else: raise ValueError( f"unsupported source {_redact(uri)!r}; known schemes: {', '.join(SCHEMES)}" diff --git a/src/vecscope/evidence.py b/src/vecscope/evidence.py index d0936a5..d831af0 100644 --- a/src/vecscope/evidence.py +++ b/src/vecscope/evidence.py @@ -108,6 +108,11 @@ class Evidence: unstable_findings: list[str] = field(default_factory=list) near_dup_cos: float = 0.99 inconclusive_findings: list[str] = field(default_factory=list) + # "full" when every row was audited, "sampled" when the findings describe a seeded + # subset of the rows; the verdicts in needs_full_audit cannot stand for the whole corpus + # (the inconclusive and the unstable ones) and need a run with sample=0 + evidence_scope: str = "full" + needs_full_audit: list[str] = field(default_factory=list) # ordered steps for the non-ok findings (vecscope.plan): shared cause, first change plan: list[dict] = field(default_factory=list) # quality on labeled queries (vecscope.retrieval); empty without qrels @@ -1256,6 +1261,10 @@ def audit( unstable_findings=unstable, near_dup_cos=near_dup_cos, inconclusive_findings=inconclusive, + evidence_scope="sampled" if sampled else "full", + needs_full_audit=sorted( + set(inconclusive) | set(unstable), key=inconclusive.__add__(unstable).index + ), plan=build_plan([asdict(f) for f in findings], retrieval or None), retrieval=retrieval, corpus_fingerprint=fingerprint(x_full), diff --git a/src/vecscope/mcp_server.py b/src/vecscope/mcp_server.py index 5f62208..6ad4525 100644 --- a/src/vecscope/mcp_server.py +++ b/src/vecscope/mcp_server.py @@ -21,7 +21,10 @@ A file larger than ``--max-read-mb`` (default 4096, ``VECSCOPE_MAX_READ_MB``, 0 = no limit) is refused before it is read, and a store stops being read once its vectors pass that size, so one request cannot exhaust the server's memory. Files are parsed straight from the -verified handle, never copied into memory first. +verified handle, never copied into memory first. ``--max-rows`` and ``--max-dim`` +(``VECSCOPE_MAX_ROWS``, ``VECSCOPE_MAX_DIM``, default 0 = no limit) refuse a matrix with more +rows or columns than that, whatever its byte size, and a ``MemoryError`` while reading or +auditing is reported to the agent as such instead of killing the call silently. The SDK runs every tool call in its own thread. Audits, sweeps and compares take one of ``--max-concurrent`` slots (default 1, ``VECSCOPE_MAX_CONCURRENT``) and further calls wait, @@ -61,12 +64,14 @@ from vecscope.evidence import ACTIONS, THRESHOLDS, audit, diff_evidence, same_corpus from vecscope.io import load_neighbors, load_vectors -_USER_ERRORS = (ValueError, TypeError, FileNotFoundError, ImportError, OSError) +_USER_ERRORS = (ValueError, TypeError, FileNotFoundError, ImportError, OSError, MemoryError) ALLOWED_DIRS_ENV = "VECSCOPE_ALLOWED_DIRS" ALLOW_NETWORK_ENV = "VECSCOPE_ALLOW_NETWORK" ALLOW_ANY_ENV = "VECSCOPE_ALLOW_ANY" MAX_READ_ENV = "VECSCOPE_MAX_READ_MB" DEFAULT_MAX_READ_MB = 4096 +MAX_ROWS_ENV = "VECSCOPE_MAX_ROWS" +MAX_DIM_ENV = "VECSCOPE_MAX_DIM" MAX_CONCURRENT_ENV = "VECSCOPE_MAX_CONCURRENT" # the most a call may ask for of a setting that multiplies its cost; the CLI has no caps LIMITS = { @@ -177,6 +182,13 @@ def _call(tool: str, **paths): def _user_error(e: Exception) -> ToolError: """Return a ToolError whose message reaches the agent (plain exceptions are masked).""" + if isinstance(e, MemoryError): + return ToolError( + "MemoryError: the server ran out of memory on this call; audit a sample " + "(sample=, the default is 100000 rows), a smaller file or collection, or start " + "the server on a machine with more memory. Its limits are --max-read-mb, " + "--max-rows and --max-dim" + ) return ToolError(f"{type(e).__name__}: {e}") @@ -203,6 +215,43 @@ def max_read_bytes() -> int | None: return None if mb <= 0 else int(mb * 2**20) +def _count_limit(env: str) -> int | None: + raw = os.environ.get(env, "") + if not raw.strip(): + return None + try: + n = int(raw) + except ValueError: + raise ValueError(f"{env}={raw!r} is not a whole number") from None + return None if n <= 0 else n + + +def max_rows() -> int | None: + """The most rows one matrix may have, from VECSCOPE_MAX_ROWS (unset or 0 = no limit).""" + return _count_limit(MAX_ROWS_ENV) + + +def max_dim() -> int | None: + """The most columns one matrix may have, from VECSCOPE_MAX_DIM (unset or 0 = no limit).""" + return _count_limit(MAX_DIM_ENV) + + +def check_shape(x, what: str = "path"): + """Refuse a matrix over --max-rows or --max-dim; returns it unchanged otherwise.""" + rows, dim = max_rows(), max_dim() + if rows is not None and x.shape[0] > rows: + raise ValueError( + f"{what} has {x.shape[0]} rows, over the {rows} this server audits " + f"(--max-rows / {MAX_ROWS_ENV}); audit a smaller file or raise the limit" + ) + if dim is not None and x.shape[1] > dim: + raise ValueError( + f"{what} has {x.shape[1]} columns, over the {dim} this server audits " + f"(--max-dim / {MAX_DIM_ENV}); audit a smaller file or raise the limit" + ) + return x + + @contextlib.contextmanager def open_checked(path: str, what: str = "path"): """A binary handle on ``path`` that is verified to be a regular file inside the allowed @@ -271,7 +320,7 @@ def _json_file(path: str, what: str) -> dict: def _vectors(path: str, what: str = "path"): with open_checked(path, what) as fh: - return load_vectors(path, fh) + return check_shape(load_vectors(path, fh), what) def _neighbors(path: str | None, what: str): @@ -288,19 +337,26 @@ def _open_source(path: str): scheme = scheme_of(path) if scheme in ("", "faiss"): # one index file: read through a verified handle - return load_source( + src = load_source( path, read_checked(local_path(path) if scheme else path), max_bytes=max_read_bytes() ) - if scheme.endswith("+local"): - # a local database directory: its client opens the files itself, so only the - # directory is checked - check_path(local_path(path)) - elif os.environ.get(ALLOW_NETWORK_ENV) != "1": - raise PermissionError( - f"{scheme} sources reach the network; start the server with " - f"--allow-network (or {ALLOW_NETWORK_ENV}=1) to read them" - ) - return load_source(path, max_bytes=max_read_bytes()) + else: + if scheme.endswith("+local"): + # a local database directory: its client opens the files itself, so only the + # directory is checked + check_path(local_path(path)) + elif os.environ.get(ALLOW_NETWORK_ENV) != "1": + raise PermissionError( + f"{scheme} sources reach the network; start the server with " + f"--allow-network (or {ALLOW_NETWORK_ENV}=1) to read them" + ) + src = load_source(path, max_bytes=max_read_bytes(), max_rows=max_rows()) + try: + check_shape(src.vectors, "path") + except ValueError: + src.close() + raise + return src def check_path(path: str | None, what: str = "path") -> str | None: @@ -377,7 +433,9 @@ def _summary(ev: dict) -> str: bad = [f for f in ev["findings"] if f["verdict"] != "ok"] head = f"verdict={ev['verdict']} n={ev['n']} dim={ev['dim']} k={ev['k']}" if ev.get("sampled"): - head += f" (sampled from {ev['n_total']} rows)" + head += f" scope=sampled ({ev['n']} of {ev['n_total']} rows: estimates, not a full audit)" + else: + head += " scope=full" if ev.get("unstable_findings"): head += f"; verdict unstable across sample repeats: {', '.join(ev['unstable_findings'])}" if ev.get("inconclusive_findings"): @@ -385,6 +443,8 @@ def _summary(ev: dict) -> str: f"; inconclusive on this sample (not evidence of health): " f"{', '.join(ev['inconclusive_findings'])}" ) + if ev.get("needs_full_audit"): + head += f"; needs a full audit (sample=0) to settle: {', '.join(ev['needs_full_audit'])}" if not bad: head += "; all findings ok" else: @@ -481,8 +541,12 @@ def audit_embeddings( `verbose=False` drops details, causes, actions and explanations of findings that are ok and the thresholds table; it saves most when most findings are ok. - The response starts with `summary` (one line: non-ok findings, retrieval, the first plan - step and the baseline comparison), `verdict` and `audit_id`. Each finding carries value, + The response starts with `summary` (one line: the evidence scope, non-ok findings, + retrieval, the first plan step and the baseline comparison), `verdict` and `audit_id`. + `evidence_scope` is "full" when every row was audited and "sampled" otherwise; a + sampled response lists in `needs_full_audit` the findings whose verdict cannot stand for + the corpus (the inconclusive and the unstable ones). Report those as unsettled, never as + healthy, and re-run with sample=0 to settle them. Each finding carries value, healthy_range, warn_at, fail_at, explanation, details, likely_causes, suggested_action and verify_by. Treat actions as hypotheses: apply the smallest change, re-audit, and compare retrieval on real queries before adopting. @@ -759,6 +823,22 @@ def main(argv: list[str] | None = None) -> None: help="refuse files larger than this and stop reading a store past it " f"(default {DEFAULT_MAX_READ_MB}; 0 = no limit); same as {MAX_READ_ENV}", ) + p.add_argument( + "--max-rows", + type=int, + default=None, + metavar="N", + help="refuse a matrix with more rows than this, whatever its byte size " + f"(default 0 = no limit); same as {MAX_ROWS_ENV}", + ) + p.add_argument( + "--max-dim", + type=int, + default=None, + metavar="N", + help="refuse a matrix with more columns than this (default 0 = no limit); " + f"same as {MAX_DIM_ENV}", + ) p.add_argument( "--max-concurrent", type=int, @@ -780,6 +860,14 @@ def main(argv: list[str] | None = None) -> None: if args.max_read_mb < 0: p.error("--max-read-mb must be 0 (no limit) or positive") os.environ[MAX_READ_ENV] = str(args.max_read_mb) + for flag, env, value in ( + ("--max-rows", MAX_ROWS_ENV, args.max_rows), + ("--max-dim", MAX_DIM_ENV, args.max_dim), + ): + if value is not None: + if value < 0: + p.error(f"{flag} must be 0 (no limit) or positive") + os.environ[env] = str(value) if args.allow_network: os.environ[ALLOW_NETWORK_ENV] = "1" if args.allow_any: diff --git a/tests/golden/anisotropic.json b/tests/golden/anisotropic.json index e54ec94..b61f119 100644 --- a/tests/golden/anisotropic.json +++ b/tests/golden/anisotropic.json @@ -406,6 +406,8 @@ "unstable_findings": [], "near_dup_cos": 0.99, "inconclusive_findings": [], + "evidence_scope": "full", + "needs_full_audit": [], "plan": [ { "order": 1, diff --git a/tests/golden/collapsed.json b/tests/golden/collapsed.json index 6e52944..31cf8e1 100644 --- a/tests/golden/collapsed.json +++ b/tests/golden/collapsed.json @@ -461,6 +461,8 @@ "unstable_findings": [], "near_dup_cos": 0.99, "inconclusive_findings": [], + "evidence_scope": "full", + "needs_full_audit": [], "plan": [ { "order": 1, diff --git a/tests/golden/healthy.json b/tests/golden/healthy.json index e3d2157..94be013 100644 --- a/tests/golden/healthy.json +++ b/tests/golden/healthy.json @@ -407,6 +407,8 @@ "unstable_findings": [], "near_dup_cos": 0.99, "inconclusive_findings": [], + "evidence_scope": "full", + "needs_full_audit": [], "plan": [], "retrieval": {}, "source": {}, diff --git a/tests/test_adapters.py b/tests/test_adapters.py index c8491ed..cc1d059 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -174,7 +174,7 @@ def test_client_errors_are_scrubbed_of_secrets(monkeypatch): monkeypatch.setenv("VS_KEY", "s3cret-key") - def leaky(uri, max_bytes=None): + def leaky(uri, max_bytes=None, max_rows=None): raise RuntimeError("401 for api-key s3cret-key, token tok-1234, user alice:pw1234") monkeypatch.setattr(adapters, "_qdrant", leaky) @@ -190,7 +190,7 @@ def test_timeout_option_and_an_unreachable_store(): from vecscope.adapters import _timeout assert _timeout({}) == 60.0 and _timeout({"timeout": "2.5"}) == 2.5 - for bad in ("x", "0", "-1"): + for bad in ("x", "0", "-1", "inf", "nan", None): with pytest.raises(ValueError, match="timeout"): _timeout({"timeout": bad}) pytest.importorskip("qdrant_client") diff --git a/tests/test_mcp_security.py b/tests/test_mcp_security.py new file mode 100644 index 0000000..ef2464b --- /dev/null +++ b/tests/test_mcp_security.py @@ -0,0 +1,162 @@ +"""The MCP server's guards: paths outside the allowed directories, the read, row and column +limits, network stores without opt-in, secrets in errors, the store timeout option, and a +MemoryError surfacing as a tool error with advice. All on small matrices.""" + +from __future__ import annotations + +import json + +import numpy as np +import pytest + +pytest.importorskip("mcp") + +from vecscope import mcp_server + + +@pytest.fixture +def sandbox(tmp_path, monkeypatch): + """An allowed directory with a small corpus inside and a copy outside, every limit off.""" + inside, outside = tmp_path / "in", tmp_path / "out" + inside.mkdir() + outside.mkdir() + x = np.random.default_rng(0).standard_normal((300, 16)).astype(np.float32) + x /= np.linalg.norm(x, axis=1, keepdims=True) + np.save(inside / "x.npy", x) + np.save(outside / "x.npy", x) + monkeypatch.setenv(mcp_server.ALLOWED_DIRS_ENV, str(inside)) + for env in ( + mcp_server.ALLOW_ANY_ENV, + mcp_server.ALLOW_NETWORK_ENV, + mcp_server.MAX_READ_ENV, + mcp_server.MAX_ROWS_ENV, + mcp_server.MAX_DIM_ENV, + ): + monkeypatch.delenv(env, raising=False) + return inside, outside + + +def test_paths_outside_the_allowed_directories_are_refused(sandbox): + inside, outside = sandbox + with pytest.raises(Exception, match="outside the directories"): + mcp_server.audit_embeddings(str(outside / "x.npy")) + with pytest.raises(Exception, match="outside the directories"): + mcp_server.audit_embeddings(str(inside / "x.npy"), queries_path=str(outside / "x.npy")) + with pytest.raises(Exception, match="outside the directories"): + mcp_server.compare_embeddings(str(inside / "x.npy"), str(outside / "x.npy")) + res = mcp_server.audit_embeddings(str(inside / "x.npy"), k=5, verbose=False) + assert res["verdict"] in ("ok", "warn", "fail") and res["evidence_scope"] == "full" + + +def test_a_directory_and_a_missing_file_are_user_errors(sandbox): + inside, _ = sandbox + with pytest.raises(Exception, match="cannot be read as a file|not a regular file"): + mcp_server.audit_embeddings(str(inside)) + with pytest.raises(Exception, match="not found"): + mcp_server.audit_embeddings(str(inside / "missing.npy")) + + +def test_read_limit_refuses_a_file_before_parsing_it(sandbox, monkeypatch): + inside, _ = sandbox + size_mb = (inside / "x.npy").stat().st_size / 2**20 + monkeypatch.setenv(mcp_server.MAX_READ_ENV, str(size_mb / 2)) + with pytest.raises(Exception, match="over the .* MiB this server reads"): + mcp_server.audit_embeddings(str(inside / "x.npy")) + monkeypatch.setenv(mcp_server.MAX_READ_ENV, "0") # 0 lifts the limit + assert mcp_server.audit_embeddings(str(inside / "x.npy"), k=5, verbose=False)["n"] == 300 + monkeypatch.setenv(mcp_server.MAX_READ_ENV, "lots") + with pytest.raises(Exception, match="not a number of MiB"): + mcp_server.audit_embeddings(str(inside / "x.npy")) + + +def test_row_and_column_limits(sandbox, monkeypatch): + inside, _ = sandbox + monkeypatch.setenv(mcp_server.MAX_ROWS_ENV, "299") + with pytest.raises(Exception, match="300 rows, over the 299"): + mcp_server.audit_embeddings(str(inside / "x.npy")) + monkeypatch.setenv(mcp_server.MAX_ROWS_ENV, "300") + monkeypatch.setenv(mcp_server.MAX_DIM_ENV, "15") + with pytest.raises(Exception, match="16 columns, over the 15"): + mcp_server.audit_embeddings(str(inside / "x.npy")) + monkeypatch.setenv(mcp_server.MAX_DIM_ENV, "0") + assert mcp_server.audit_embeddings(str(inside / "x.npy"), k=5, verbose=False)["dim"] == 16 + monkeypatch.setenv(mcp_server.MAX_ROWS_ENV, "many") + with pytest.raises(Exception, match="not a whole number"): + mcp_server.audit_embeddings(str(inside / "x.npy")) + # the queries file is checked by the same limits + monkeypatch.setenv(mcp_server.MAX_ROWS_ENV, "0") + monkeypatch.setenv(mcp_server.MAX_DIM_ENV, "16") + np.save(inside / "q32.npy", np.ones((4, 32), dtype=np.float32)) + with pytest.raises(Exception, match="queries_path has 32 columns"): + mcp_server.audit_embeddings(str(inside / "x.npy"), queries_path=str(inside / "q32.npy")) + + +def test_the_server_options_set_the_limits(monkeypatch, tmp_path): + for env in (mcp_server.MAX_ROWS_ENV, mcp_server.MAX_DIM_ENV, mcp_server.MAX_READ_ENV): + monkeypatch.setenv(env, "") # main() writes os.environ; this restores it afterwards + monkeypatch.setattr(mcp_server.mcp, "run", lambda *a, **k: None) + monkeypatch.chdir(tmp_path) + mcp_server.main(["--max-rows", "1000", "--max-dim", "2048", "--max-read-mb", "64"]) + assert mcp_server.max_rows() == 1000 + assert mcp_server.max_dim() == 2048 + assert mcp_server.max_read_bytes() == 64 * 2**20 + with pytest.raises(SystemExit): + mcp_server.main(["--max-rows", "-1"]) + + +def test_memory_error_reaches_the_agent_with_advice(sandbox, monkeypatch): + inside, _ = sandbox + + def exhausted(*a, **k): + raise MemoryError + + monkeypatch.setattr(mcp_server, "audit", exhausted) + with pytest.raises(Exception, match="MemoryError: .*sample=.*--max-rows") as err: + mcp_server.audit_embeddings(str(inside / "x.npy")) + assert "ran out of memory" in str(err.value) + + +def test_network_stores_need_opt_in_and_errors_carry_no_secrets(sandbox, monkeypatch): + with pytest.raises(Exception, match="allow-network") as err: + mcp_server.audit_embeddings("qdrant://alice:pw1234@host:6333/docs?api_key_env=VS_KEY") + assert "pw1234" not in str(err.value) + pytest.importorskip("qdrant_client") + monkeypatch.setenv(mcp_server.ALLOW_NETWORK_ENV, "1") + monkeypatch.setenv("VS_KEY", "s3cret-key") + with pytest.raises(Exception) as err: + mcp_server.audit_embeddings( + "qdrant://alice:pw1234@127.0.0.1:1/docs?api_key_env=VS_KEY&timeout=1" + ) + msg = str(err.value) + assert "127.0.0.1:1/docs" in msg + for secret in ("pw1234", "s3cret-key"): + assert secret not in msg + + +def test_store_timeout_must_be_a_finite_positive_bounded_number(): + from vecscope.adapters import MAX_TIMEOUT, _timeout + + assert _timeout({}) == 60.0 + assert _timeout({"timeout": "2.5"}) == 2.5 + assert _timeout({"timeout": str(MAX_TIMEOUT)}) == MAX_TIMEOUT + for bad in ("inf", "-inf", "nan", "0", "-1", "x", "", None, [1]): + with pytest.raises(ValueError, match="timeout"): + _timeout({"timeout": bad}) + with pytest.raises(ValueError, match="most a request may wait"): + _timeout({"timeout": "1e9"}) + + +def test_sampled_response_states_its_scope(sandbox): + inside, _ = sandbox + res = mcp_server.audit_embeddings(str(inside / "x.npy"), k=5, sample=100, verbose=False) + assert res["evidence_scope"] == "sampled" and res["sampled"] is True + assert res["summary"].startswith(f"verdict={res['verdict']} n=100 dim=16 k=5 scope=sampled") + assert set(res["needs_full_audit"]) == set(res["inconclusive_findings"]) | set( + res["unstable_findings"] + ) + if res["needs_full_audit"]: + assert "needs a full audit (sample=0)" in res["summary"] + full = mcp_server.audit_embeddings(str(inside / "x.npy"), k=5, sample=0, verbose=False) + assert full["evidence_scope"] == "full" and full["needs_full_audit"] == [] + assert "scope=full" in full["summary"] + json.dumps(res) # the response stays plain JSON