From 89171a53ea24526d7a72c4b2e7583b078d6014a7 Mon Sep 17 00:00:00 2001 From: manuc66 Date: Mon, 16 Nov 2026 07:04:51 +0100 Subject: [PATCH 1/6] Add BM25 over stock PostgreSQL and a tsquery match mode pg_search is the faster route to BM25 on PostgreSQL, but it needs a preloaded library and none of RDS, Cloud SQL or Azure Database for PostgreSQL lets you install one, so the only BM25 backend was unavailable on managed deployments. PostgresBm25SearchEngine ranks the same tsvector term frequencies with Bm25Scorer, over the lexical engine's table, DDL and analysis. N, the average document length and each term's document frequency come from one ts_stat snapshot, invalidated by writes and rebuildable with RefreshStatistics(); term frequencies and the length come from the candidate row's own tsvector. PostgreSQL selects the best CandidatePool matches by ts_rank_cd (default 1,000, 0 for the exact ranking up to the engine's 100,000-row ceiling) and the engine re-ranks them, because scoring every match would make a query proportional to the corpus. PostgresIndexOptions.MatchMode separates the other half of ts_rank_cd's gap: AllTerms is websearch_to_tsquery's AND (the previous behaviour), AnyTerm ORs the query's lexemes. The gate and the rank expression now come from one PostgresMatchExpression, and the documents upsert from one PostgresDocumentWriter, so the two text engines cannot drift. Measured by the harness on the same corpora, queries and qrels (nDCG@10, harness defaults): all any pool 1k pool 0 in-mem ParadeDB NFCorpus 0.1864 0.1881 0.2999 0.3072 0.308 0.3084 SciFact 0.0188 0.0988 0.5636 0.6665 0.662 0.6619 At pool 0 the portable BM25 reproduces the in-memory row to within the analysis. The default pool is the whole 0.103-point SciFact difference, which is why --backend-pool exists and why the row prints the pool it ran with. The evaluation page had attributed ts_rank_cd's NFCorpus gap to its matching rule; OR-ing the terms moves NFCorpus 0.1864 -> 0.1881, so that gap is the ranker (coverage has no idf). The rows behind both statements are now published, and the backend guide carries a managed-PostgreSQL matrix: contrib and pgvector are available, pg_search is not. --- CHANGELOG.md | 38 ++ README.md | 6 +- .../PostgresBackendBenchmark.cs | 120 ++++++- bench/LexiSharp.Eval/Program.cs | 85 ++++- bench/LexiSharp.Eval/README.md | 31 +- docs/api.md | 5 +- docs/backends.md | 144 +++++++- docs/evaluation.md | 38 +- docs/index.md | 9 +- docs/querying.md | 4 +- .../PostgresBm25IndexView.cs | 118 ++++++ src/LexiSharp.Postgres/PostgresBm25Options.cs | 131 +++++++ .../PostgresBm25SearchEngine.cs | 338 ++++++++++++++++++ .../PostgresBm25Statistics.cs | 109 ++++++ .../PostgresDocumentStore.cs | 112 ++++++ .../PostgresIndexOptions.cs | 48 ++- .../PostgresMatchExpression.cs | 108 ++++++ src/LexiSharp.Postgres/PostgresSchema.cs | 6 +- .../PostgresTextSearchEngine.cs | 131 +------ src/LexiSharp.Postgres/PostgresTsQueryText.cs | 67 ++++ src/LexiSharp.Postgres/PostgresTsvText.cs | 176 +++++++++ ...ostgresBm25SearchEngineConformanceTests.cs | 36 ++ .../PostgresBm25SearchEngineTests.cs | 301 ++++++++++++++++ .../LexiSharp.Tests/PostgresMatchModeTests.cs | 157 ++++++++ tests/LexiSharp.Tests/PostgresTsvTextTests.cs | 102 ++++++ tests/LexiSharp.Tests/PublicApiBaseline.txt | 38 ++ 26 files changed, 2276 insertions(+), 182 deletions(-) create mode 100644 src/LexiSharp.Postgres/PostgresBm25IndexView.cs create mode 100644 src/LexiSharp.Postgres/PostgresBm25Options.cs create mode 100644 src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs create mode 100644 src/LexiSharp.Postgres/PostgresBm25Statistics.cs create mode 100644 src/LexiSharp.Postgres/PostgresDocumentStore.cs create mode 100644 src/LexiSharp.Postgres/PostgresMatchExpression.cs create mode 100644 src/LexiSharp.Postgres/PostgresTsQueryText.cs create mode 100644 src/LexiSharp.Postgres/PostgresTsvText.cs create mode 100644 tests/LexiSharp.Tests/Conformance/PostgresBm25SearchEngineConformanceTests.cs create mode 100644 tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs create mode 100644 tests/LexiSharp.Tests/PostgresMatchModeTests.cs create mode 100644 tests/LexiSharp.Tests/PostgresTsvTextTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f944d4c..cd2edc78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,38 @@ ## [Unreleased] +### Added + +- **`PostgresBm25SearchEngine`** (`LexiSharp.Postgres`): Okapi BM25 over **stock PostgreSQL**. The + `tsvector` column and its GIN index do the matching and `Bm25Scorer` does the ranking over the + term frequencies the vector already stores, so the only server-side requirement is + `CREATE EXTENSION unaccent`. That is what makes a real BM25 available on RDS, Cloud SQL and Azure + Database for PostgreSQL, where ParadeDB's `pg_search` cannot be installed. `N`, the average + length and each term's document frequency come from one cached `ts_stat` snapshot (invalidated by + writes, rebuildable with `RefreshStatistics()`); PostgreSQL selects the best `CandidatePool` + matches by `ts_rank_cd` (default 1,000, `0` for the exact ranking) and the engine re-ranks them, + because scoring every match would make a query proportional to the corpus. Measured at pool 0: + **0.3072 nDCG@10 on NFCorpus and 0.6665 on SciFact**, against 0.308 / 0.662 for the in-memory + scorer and 0.3084 / 0.6619 for ParadeDB. At the default pool the same rows read 0.2999 / 0.5636 — + the difference is recall, and it is stated rather than hidden. + +- **`PostgresIndexOptions.MatchMode`** (`PostgresMatchMode.AllTerms | AnyTerm`): how a multi-term + query becomes a `tsquery` on the lexical engine. `AllTerms` is the previous behaviour + (`websearch_to_tsquery`, AND); `AnyTerm` ORs the query's lexemes, takes them from the column's own + configuration, and keeps quoted phrases as hard gates. On long queries the AND rule was what buried + the corpus: SciFact moves 0.0188 → 0.0988 nDCG@10 (5.3×), while the ranker is still the rest — + 0.0988 is far from BM25's 0.662 — and NFCorpus barely moves (0.1864 → 0.1881). + +- **`--backend bm25`, `--backend-match all|any` and `--backend-pool `** in + `bench/LexiSharp.Eval`, so the portable BM25 row and the match rule are measured by the same + harness, corpus and qrels as every other row. + +### Changed + +- The shared documents table's upsert and row reader are now one implementation + (`PostgresDocumentWriter`/`PostgresDocumentReader`), used by the lexical and BM25 engines, and the + `tsquery` a search is matched and ranked with is built once (`PostgresMatchExpression`). A query + that means one thing at the gate cannot mean another at the rank. + ### Fixed - **The evaluation harness measured a rounded run in the engine's tie-break, not in the run reader's.** @@ -17,6 +50,11 @@ the metrics are measured at (`MetricDepth.Cutoff`) instead of `--top-k`, which had printed `nDCG@1000` over an nDCG@10 value. +- **The evaluation page attributed `ts_rank_cd`'s NFCorpus gap to its matching rule.** It is not: + OR-ing the terms moves NFCorpus 0.1864 → 0.1881, so the gap there is the ranker (coverage has no + idf). On SciFact the rule is worth 5.3× and the ranker is still the rest. The rows behind both + statements are now published. + ## [1.0.0] — 2026-11-10 The first stable release. The public API is now versioned by semantic versioning: a breaking diff --git a/README.md b/README.md index 448a50f0..32d24a42 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ service. | Package | What it adds | |---|---| | `LexiSharp` | Core: index, scorers, fusion, reranking, metrics, traces — no dependencies | -| `LexiSharp.Postgres` | tsvector · pgvector · sparsevec · pg_trgm · pg_search (Tantivy) BM25 | +| `LexiSharp.Postgres` | tsvector · BM25 without an extension · pgvector · sparsevec · pg_trgm · pg_search (Tantivy) BM25 | | `LexiSharp.Onnx` | Local ONNX embeddings and cross-encoder reranking (no model shipped) | | `LexiSharp.MessagePack` | Binary persistence for the in-memory index | | `LexiSharp.AspNetCore` | `GET /search` minimal-API endpoint over `LexiSharpIndex` | @@ -119,6 +119,10 @@ linear classifier): [text analysis →](docs/text-analysis.md). reproducible by any code path. - Not every engine/scorer/reranker/merger combination is tested; treat unusual pairings as supported but unproven until tested on your data. +- The stock-PostgreSQL BM25 re-ranks a bounded candidate pool (1,000 by default): the pool bounds + recall, and on SciFact it costs 0.10 nDCG@10 against the exact ranking. `CandidatePool = 0` + removes the bound. The ParadeDB engine needs `pg_search`, which no major managed service lets you + install; the `tsvector`, BM25 and `pg_trgm` engines do not. - SQL backends are measured on ranking for the engines that rank on their own; sparse and fuzzy engines are covered by integration tests only. diff --git a/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs b/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs index a40eb816..4577d1f1 100644 --- a/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs +++ b/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs @@ -15,6 +15,13 @@ internal enum BackendKind /// tsvector + ts_rank_cd over stock PostgreSQL: coverage-based, not BM25. Lexical, + /// + /// tsvector term frequencies + over stock + /// PostgreSQL: a real BM25 with no extension, unlike , and with no + /// pg_search install, unlike . + /// + Bm25, + /// pg_search + pdb.score: Okapi BM25 computed by Tantivy. ParadeDb, @@ -59,6 +66,12 @@ internal enum BackendKind /// normalisation to go with them. Without a weighted title, a document's body counts as /// much as its title and the row measures nothing new. /// +/// +/// How a multi-term query is matched on the lexical and BM25 rows, or null for each +/// engine's own default: AND for the lexical row (what it has always run) and OR for BM25, whose +/// ranking needs a candidate set to rank. The rule is stated in the row's name either way, because +/// it moves the number further than most ranking changes. +/// /// /// The pg_trgm similarity thresholds a fuzzy row is measured at, one row each. The threshold /// is the % operator's match rule (set_limit), not a ranking parameter: a document @@ -80,7 +93,9 @@ internal sealed record BackendLane( IReadOnlyList? EfSearchValues = null, IReadOnlyDictionary? FieldWeights = null, IReadOnlyList? SimilarityThresholds = null, - string? SparseVectorsDirectory = null) + string? SparseVectorsDirectory = null, + PostgresMatchMode? MatchMode = null, + int? CandidatePool = null) { /// The ef_search values in force; pgvector's own default when none was given. public IReadOnlyList EfValues => EfSearchValues is { Count: > 0 } values ? values : [40]; @@ -98,19 +113,22 @@ public static BackendLane Parse( IReadOnlyList? efSearchValues = null, IReadOnlyDictionary? fieldWeights = null, IReadOnlyList? similarityThresholds = null, - string? sparseVectorsDirectory = null) + string? sparseVectorsDirectory = null, + PostgresMatchMode? matchMode = null, + int? candidatePool = null) { ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); BackendKind parsed = kind switch { "lexical" => BackendKind.Lexical, + "bm25" => BackendKind.Bm25, "paradedb" => BackendKind.ParadeDb, "vector" => BackendKind.Vector, "fuzzy" => BackendKind.Fuzzy, "sparse" => BackendKind.Sparse, _ => throw new ArgumentException( - $"--backend expects 'lexical', 'paradedb', 'vector', 'fuzzy' or 'sparse', got '{kind}'."), + $"--backend expects 'lexical', 'bm25', 'paradedb', 'vector', 'fuzzy' or 'sparse', got '{kind}'."), }; if (parsed == BackendKind.Sparse) @@ -154,7 +172,25 @@ public static BackendLane Parse( { throw new ArgumentException( "--backend-bm25 sets k1/b, and ts_rank_cd has neither: the lexical row ranks by coverage " - + "and takes no parameter. Drop the flag, or use --backend paradedb."); + + "and takes no parameter. Drop the flag, or use --backend bm25 (stock PostgreSQL, no " + + "extension) or --backend paradedb."); + } + + if (parsed != BackendKind.Bm25 && candidatePool is not null) + { + throw new ArgumentException( + "--backend-pool bounds the matches the BM25 engine re-ranks, and no other backend has a " + + "pool: the lexical row ranks in SQL, the vector and sparse rows rank by distance after " + + "an index scan, and ParadeDB and fuzzy rank their whole match set. Drop the flag, or " + + "use --backend bm25."); + } + + if (parsed is not (BackendKind.Lexical or BackendKind.Bm25) && matchMode is not null) + { + throw new ArgumentException( + "--backend-match selects the tsquery rule the lexical and BM25 engines match with, and the " + + "vector, sparse, ParadeDB and fuzzy rows rank by an embedding, a dot product, pdb.score or " + + "trigram similarity. Drop the flag, or use --backend lexical or --backend bm25."); } if (parsed == BackendKind.ParadeDb && textSearchConfig != "simple") @@ -228,7 +264,9 @@ public static BackendLane Parse( return new BackendLane( parsed, connectionString, textSearchConfig, bm25, efSearchValues, fieldWeights, similarityThresholds, - sparseVectorsDirectory is null ? null : Path.GetFullPath(sparseVectorsDirectory)); + sparseVectorsDirectory is null ? null : Path.GetFullPath(sparseVectorsDirectory), + matchMode, + candidatePool); } /// @@ -251,6 +289,10 @@ public string TableFor(string dataset) /// Names the engine and its analysis, because a nDCG without them is not a figure. public string Description => Kind switch { + BackendKind.Bm25 => + $"PostgreSQL BM25 (Bm25Scorer over tsvector term frequencies, text search config " + + $"'{TextSearchConfig}' + unaccent, {MatchText}, {Bm25OperatingPoint}, candidate pool " + + $"{EffectivePool:N0}" + FieldWeightsText + ")", BackendKind.ParadeDb => "ParadeDB BM25 (pdb.score, Tantivy)" + Bm25Text + " — index over content only, title concatenated into it", @@ -264,10 +306,38 @@ public string TableFor(string dataset) BackendKind.Sparse => "pgvector sparsevec (inner product) + HNSW (m = 16, ef_construction = 64), pgvector's " + "default hnsw.ef_search 40, and the same table after DROP INDEX", - _ => $"PostgreSQL lexical (ts_rank_cd, text search config '{TextSearchConfig}' + unaccent" - + FieldWeightsText + ")", + _ => $"PostgreSQL lexical (ts_rank_cd, text search config '{TextSearchConfig}' + unaccent, " + + $"{MatchText}" + FieldWeightsText + ")", }; + /// The pool the BM25 row ran with: the flag when given, the engine's default otherwise. + public int EffectivePool => CandidatePool ?? PostgresBm25Options.DefaultCandidatePool; + + /// + /// The rule the row actually ran under: the flag when given, otherwise each engine's own + /// default — AND for the lexical row (unchanged since it was measured) and OR for BM25, whose + /// ranking needs a candidate set to rank. + /// + public PostgresMatchMode ResolvedMatchMode => + MatchMode ?? (Kind == BackendKind.Bm25 ? PostgresMatchMode.AnyTerm : PostgresMatchMode.AllTerms); + + /// The match rule in words, because two nDCGs under one name are two different rows. + private string MatchText => ResolvedMatchMode == PostgresMatchMode.AnyTerm + ? "any term" + : "all terms"; + + /// The pool in words: a count when bounded, the ceiling when the bound was removed. + private string PoolText => EffectivePool == 0 + ? $"every match up to the engine's {PostgresBm25Options.MaximumCandidatePool:N0}-row ceiling (pool unbounded)" + : $"the top {EffectivePool:N0} matches by ts_rank_cd"; + + /// The k1/b the BM25 engine ran at: the flag when given, the engine's defaults otherwise. + private string Bm25OperatingPoint => Bm25 is { } parameters + ? string.Create(CultureInfo.InvariantCulture, $"k1={parameters.K1:0.##}, b={parameters.B:0.##}") + : string.Create( + CultureInfo.InvariantCulture, + $"k1={PostgresBm25Options.DefaultK1:0.##}, b={PostgresBm25Options.DefaultB:0.##}"); + private static string FormatThreshold(double threshold) => threshold.ToString("0.####", CultureInfo.InvariantCulture); @@ -351,10 +421,24 @@ private static string FormatThreshold(double threshold) => + "the lexical and vector rows refuse — which is the query set the in-memory sparse row " + "is scored on.", ], + BackendKind.Bm25 => + [ + "The ranking is Bm25Scorer over the term frequencies the tsv column stores: N, the average " + + "length and each term's document frequency come from one ts_stat snapshot of the table, so " + + "the scores are on the in-memory scorer's scale — comparable with the in-memory BM25 row, and " + + "not with ts_rank_cd's or pdb.score's.", + $"The database selects {PoolText} and the engine re-ranks them. A bounded pool bounds " + + "recall — a document outside it cannot be returned — and does not change the ordering " + + "inside it.", + AnalysisCaveat, + ], _ => [ - "ts_rank_cd is a coverage-based rank, not BM25, and it takes no k1/b. Its ORDER BY has no " - + "tie-break, so the order among equal scores is the executor's and can differ between runs.", + "ts_rank_cd is a coverage-based rank, not BM25, and it takes no k1/b. Equal scores are broken " + + "by the document id, ordinally, in the SQL and in the C# side alike.", + "The match rule is in the row's name: 'all terms' is websearch_to_tsquery's AND, 'any term' is " + + "one tsquery OR-ing the query's lexemes. It changes which documents are candidates, not how " + + "they are ranked, and it moves this row further than any ranking change would.", FieldWeights is { Count: > 0 } ? "The weighting is a tsvector class per field, so what it changes is the ratio between " + "classes (A = 1.0 against D = 0.1 in ts_rank_cd's default mapping). It has no per-field " @@ -453,6 +537,23 @@ public static BackendMeasurement Measure( return Run(parade, parade.DropSchema); } + if (lane.Kind == BackendKind.Bm25) + { + using var bm25 = new PostgresBm25SearchEngine( + lane.ConnectionString, + new PostgresBm25Options + { + Table = table, + TextSearchConfig = lane.TextSearchConfig, + MatchMode = lane.ResolvedMatchMode, + K1 = lane.Bm25?.K1 ?? PostgresBm25Options.DefaultK1, + B = lane.Bm25?.B ?? PostgresBm25Options.DefaultB, + CandidatePool = lane.EffectivePool, + }); + + return Run(bm25, bm25.DropSchema); + } + using var lexical = new PostgresTextSearchEngine( lane.ConnectionString, new PostgresIndexOptions @@ -460,6 +561,7 @@ public static BackendMeasurement Measure( Table = table, TextSearchConfig = lane.TextSearchConfig, FieldWeights = lane.FieldWeights, + MatchMode = lane.ResolvedMatchMode, }); return Run(lexical, lexical.DropSchema); diff --git a/bench/LexiSharp.Eval/Program.cs b/bench/LexiSharp.Eval/Program.cs index 61787f44..66a2bf5a 100644 --- a/bench/LexiSharp.Eval/Program.cs +++ b/bench/LexiSharp.Eval/Program.cs @@ -118,6 +118,15 @@ public static async Task Main(string[] args) // with the file the run actually read. string? sparseVectorsDir = null; + // The lexical and BM25 rows' tsquery rule: "all" (the historical AND) or "any" (a + // disjunction of the query's lexemes). Parsed with the lane below, so a typo fails on a + // message rather than an unhandled exception. + string? backendMatch = null; + + // The BM25 row's re-rank pool. A flag because it is the one knob that trades recall for + // cost, and a measurement of what the pool costs is what a reader needs to set it. + string? backendPool = null; + // The lexical row's tsv weighting, as "title=A[,summary=B]" — parsed after the pair check // below, so a malformed spec fails on a message rather than an unhandled exception. string? backendFieldWeights = null; @@ -341,6 +350,12 @@ public static async Task Main(string[] args) case "--backend-bm25" when i + 1 < args.Length: backendBm25 = ParseBm25Parameters(args[++i]); break; + case "--backend-match" when i + 1 < args.Length: + backendMatch = args[++i]; + break; + case "--backend-pool" when i + 1 < args.Length: + backendPool = args[++i]; + break; case "--backend-ef" when i + 1 < args.Length: backendEfValues.AddRange(ParsePositiveInts(args[++i], "--backend-ef")); break; @@ -550,6 +565,14 @@ public static async Task Main(string[] args) return 2; } + if (backendMatch is not null && backendKind is null) + { + Console.Error.WriteLine( + "--backend-match selects the tsquery rule the lexical and BM25 engines match with, so it " + + "needs --backend lexical or --backend bm25 with --backend-connection."); + return 2; + } + BackendLane? backendLane = null; if (backendKind is not null && backendConnection is not null) @@ -564,7 +587,9 @@ public static async Task Main(string[] args) backendEfValues, backendFieldWeights is null ? null : ParseFieldWeights(backendFieldWeights), backendThresholds, - sparseVectorsDir); + sparseVectorsDir, + backendMatch is null ? null : ParseMatchMode(backendMatch), + backendPool is null ? null : ParseNonNegative(backendPool, "--backend-pool")); } catch (ArgumentException error) { @@ -1829,14 +1854,27 @@ private static Bm25Parameters ParseBm25Parameters(string value) return new Bm25Parameters(k1, b); } + /// + /// --backend-match: the tsquery rule the lexical and BM25 rows match with. Spelled + /// all and any because those are the two English words websearch_to_tsquery + /// and a disjunction of lexemes are — ands/ors would name an operator, not a rule. + /// + private static PostgresMatchMode ParseMatchMode(string value) => + value.ToLowerInvariant() switch + { + "all" => PostgresMatchMode.AllTerms, + "any" => PostgresMatchMode.AnyTerm, + _ => throw new ArgumentException( + $"--backend-match expects 'all' or 'any', got '{value}'."), + }; + /// /// Parses title=A,summary=B into the class map the lexical backend indexes with. The /// class names are PostgreSQL's own four; a value that is not one is refused by name rather /// than quantized to the nearest, because a silent approximation here would be a number /// measured under a weighting nobody asked for. /// - private static IReadOnlyDictionary ParseFieldWeights(string value) - { + private static IReadOnlyDictionary ParseFieldWeights(string value) { var weights = new Dictionary(StringComparer.Ordinal); foreach (string pair in value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) @@ -1937,9 +1975,9 @@ private static void PrintHelp() [--rerank-cache ] [--diagnose-doc ] [--diagnose-query ] [--diagnose-depth ] [--analysis-diff ] - [--backend --backend-connection ] - [--backend-bm25 ] [--backend-ef ] [--backend-threshold ] - [--backend-config ] + [--backend --backend-connection ] + [--backend-bm25 ] [--backend-match ] [--backend-ef ] + [--backend-threshold ] [--backend-config ] [--sparse ] --data Base directory for datasets (default: /data). @@ -2127,7 +2165,9 @@ forward passes and hours of CPU. The three reranked configurations The target database needs the vector extension. --backend Add a retrieval-quality row measured through a SQL backend on the same corpus, queries and qrels: 'lexical' (tsvector + ts_rank_cd over stock - PostgreSQL), 'paradedb' (pg_search + pdb.score, Okapi BM25 computed + PostgreSQL), 'bm25' (BM25 over the same tsvector term frequencies, + computed by this library, over stock PostgreSQL — no extension beyond + unaccent), 'paradedb' (pg_search + pdb.score, Okapi BM25 computed by Tantivy; needs the paradedb image), 'vector' (pgvector HNSW over the cached dense embeddings, needs --dense), 'fuzzy' (pg_trgm similarity, one row per similarity threshold) or 'sparse' (pgvector @@ -2140,19 +2180,34 @@ table is one index with many scorers. Needs --backend-connection. lexisharp_eval_ and drops it afterwards, so it never reads or empties a table an application uses. --backend-config - PostgreSQL text search configuration for the lexical backend (default: - simple). This is the row's analysis: 'english' applies PostgreSQL's own - Snowball stemmer and stop list, which is close to — and not identical - with — the harness's --analyzer english. Not read by ParadeDB, the - vector, sparse or fuzzy engines. + PostgreSQL text search configuration for the lexical and bm25 backends + (default: simple). This is the row's analysis: 'english' applies + PostgreSQL's own Snowball stemmer and stop list, which is close to — + and not identical with — the harness's --analyzer english. Not read by + ParadeDB, the vector, sparse or fuzzy engines. --backend-bm25 k1 and b the ParadeDB backend is indexed with, e.g. 0.9,0.4 — the parameters the published baselines were produced at. pg_search takes them per field at index creation, so the row is built at these values rather than re-weighted afterwards. Omitted, Tantivy's defaults - (k1=1.2, b=0.75) apply. Refused with --backend lexical, whose - ts_rank_cd has no such parameter, and with the vector, sparse and - fuzzy backends, which have no k1/b. + (k1=1.2, b=0.75) apply. For --backend bm25 the same flag sets the + engine's parameters, whose own defaults are 1.5,0.75. Refused with + --backend lexical, whose ts_rank_cd has no such parameter, and with + the vector, sparse and fuzzy backends, which have no k1/b. + --backend-pool + How many matches the bm25 backend re-ranks, default 1000. The database + selects them by ts_rank_cd and the engine scores them with BM25, so the + pool bounds recall: a document outside it cannot be returned at any + page size. It is widened to the requested page and capped at 100,000. + 0 removes the bound up to that ceiling — the exact ranking, at a cost + that grows with the match set. Refused with the other backends. + --backend-match + How the lexical and bm25 backends turn a multi-term query into a + tsquery: 'all' (default) requires every term, websearch_to_tsquery's + rule; 'any' ORs the query's lexemes, so a document holding one of them + is a candidate. The rule decides most of ts_rank_cd's distance from + BM25 on long queries. Refused with the other backends, which do not + match by tsquery. --backend-ef pgvector hnsw.ef_search values the vector backend is measured at, comma-separated (e.g. 10,40,160). One row per value, over one index: diff --git a/bench/LexiSharp.Eval/README.md b/bench/LexiSharp.Eval/README.md index d197bae0..738981de 100644 --- a/bench/LexiSharp.Eval/README.md +++ b/bench/LexiSharp.Eval/README.md @@ -185,6 +185,7 @@ same qrels as the table above, through the same metric code at the same depth: | `--backend` | engine | ranking | needs | |---|---|---|---| | `lexical` | `PostgresTextSearchEngine` | `ts_rank_cd` | stock PostgreSQL (the lane installs `unaccent`) | +| `bm25` | `PostgresBm25SearchEngine` | `Bm25Scorer` over the `tsvector` term frequencies | stock PostgreSQL (the lane installs `unaccent`) | | `paradedb` | `ParadeDBTextSearchEngine` | `pdb.score`, Tantivy's BM25 | `paradedb/paradedb:pg16` (`pg_search`) | | `vector` | `PostgresVectorSearchEngine` | cosine over pgvector HNSW | `--dense` (it ranks the cached e5 vectors) | | `fuzzy` | `PostgresFuzzySearchEngine` | `similarity(content, query)` over `pg_trgm` | stock PostgreSQL (the lane installs `pg_trgm`) | @@ -213,6 +214,22 @@ Out of the box, each engine at its own defaults: | SciFact | 0.662 | 0.6619 | 0.0188 | not measured | | ArguAna | 0.320 | not measured | not measured | not measured | +`--backend-match all|any` picks how a multi-term query becomes a `tsquery`: `all` (the lexical row's +historical behaviour) requires every term, `any` ORs the query's lexemes. `--backend-pool ` +bounds how many matches the BM25 row re-ranks (default 1,000; `0` = exact, up to the engine's +100,000-row ceiling). The two knobs, and the rows they produce (harness defaults, exponential +gains): + +| corpus | `ts_rank_cd` all | `ts_rank_cd` any | BM25 pool 1,000 | BM25 pool 0 | in-memory BM25 | ParadeDB | +|---|---:|---:|---:|---:|---:|---:| +| NFCorpus | 0.1864 | 0.1881 | 0.2999 | 0.3072 | 0.308 | 0.3084 | +| SciFact | 0.0188 | 0.0988 | 0.5636 | **0.6665** | 0.662 | 0.6619 | + +The matching rule is the SciFact story (5.3×) and almost nothing on NFCorpus, whose queries average +3.4 terms; the ranker is the rest. `--backend bm25 --backend-pool 0` reproduces BM25 to within the +analysis (PostgreSQL's `simple` config, not the harness tokenizer), which is what makes the pool's +0.103-point cost on SciFact visible rather than attributed to the scoring. + What each row is, and is not: - **ParadeDB is a BM25 and reads where this library's BM25 reads.** At matched parameters the two @@ -221,11 +238,15 @@ What each row is, and is not: because the `pg_search` index covers `content` alone, and the matcher is a disjunction (`|||`) rather than this library's scored term loop. Not a reproduction, and not bit-identical. - **`ts_rank_cd` is not BM25** — it is coverage-based and takes no k1/b, so `--backend-bm25` is - refused with it rather than silently ignored. Most of its distance from the BM25 column is the - *matching* rule rather than the ranking: `websearch_to_tsquery` ANDs every term, so its idea of a - match is a document containing all of them. NFCorpus's test queries average 3.4 words and read - 0.1864; SciFact's average 13.1 and read 0.0188. PostgreSQL's own English analysis - (`--backend-config english`) moves NFCorpus 0.1864 → 0.2043, not closer. + refused with it rather than silently ignored. Its distance has two causes, separated by the table + above: the AND *matching* rule buries long queries (SciFact 0.0188 → 0.0988 under `any`), and the + *ranking* has no idf (NFCorpus barely moves under `any`, 0.1864 → 0.1881, because its queries + average 3.4 terms). PostgreSQL's own English analysis (`--backend-config english`) moves NFCorpus + 0.1864 → 0.2043, not closer. +- **`--backend bm25` is the same corpus with a real BM25 and no extension.** At pool 0 it reads + 0.3072 / 0.6665 against 0.308 / 0.662 in memory and 0.3084 / 0.6619 for ParadeDB; at the default + pool, 0.2999 / 0.5636. The gap between pool 0 and pool 1,000 is recall, not scoring, and it is + the reason the flag exists. - **ArguAna has no row.** Both text engines refuse query text their matcher would read as a prefix operator, and 2 of the 1,406 test queries carry a `*` attached to a term. The lane prints that refusal and its count instead of a metric over the other 1,404, which would be a different query diff --git a/docs/api.md b/docs/api.md index f7871856..3d9626ca 100644 --- a/docs/api.md +++ b/docs/api.md @@ -503,11 +503,14 @@ expression, and calling it a `struct` would be as wrong as calling a `record` a | Type | Kind | Contracts | What it is | | --- | --- | --- | --- | +| `PostgresBm25Options` | record | — | Naming, analysis and ranking options for the stock-PostgreSQL BM25 engine. | +| `PostgresBm25SearchEngine` | class | `IDisposable`, `IQuerySyntaxSupport`, `ITextSearchEngine` | Okapi BM25 on stock PostgreSQL: the tsvector column and GIN index do the matching, and Bm25Scorer does the ranking over the term frequencies the vector already… | | `PostgresFieldWeight` | enum | — | The four weight classes a tsvector lexeme can carry, which setweight writes and ts_rank_cd reads back through its weights array. | | `PostgresFuzzyOptions` | record | — | Naming and behavior options for the PostgresFuzzySearchEngine. | | `PostgresFuzzySearchEngine` | class | `IDisposable`, `IQuerySyntaxSupport`, `ITextSearchEngine` | PostgreSQL-backed ITextSearchEngine for approximate, typo-tolerant matching on top of the pg_trgm extension (trigram similarity) and fuzzystrmatch (levenshtein… | | `PostgresIndexOptions` | record | — | Naming and behavior options for the PostgreSQL-backed index. | -| `PostgresSchema` | class | — | Shared DDL for the documents table. Used by the lexical (tsvector), vector (pgvector ANN) and ParadeDB (pg_search BM25) engines so they can all point at the… | +| `PostgresMatchMode` | enum | — | How the lexical engine turns a multi-term query into a match. | +| `PostgresSchema` | class | — | Shared DDL for the documents table. Used by the lexical (tsvector), BM25 (PostgresBm25SearchEngine), vector (pgvector ANN) and ParadeDB (pg_search BM25) engines… | | `PostgresSparseOptions` | record | — | Naming, vocabulary and behavior options for the pgvector sparsevec index. The table layout is shared with the lexical engine so both engines can serve a single… | | `PostgresSparseSearchEngine` | class | `IDisposable`, `IQuerySyntaxSupport`, `ITextSearchEngine` | PostgreSQL-backed learned-sparse search engine (pgvector sparsevec + HNSW): stores each document's learned sparse vector (SPLADE-style term weights) in the… | | `PostgresTextSearchEngine` | class | `IDisposable`, `IQuerySyntaxSupport`, `ITextSearchEngine` | PostgreSQL-backed ITextSearchEngine: lexical full-text search over a tsvector column with a GIN index, queried with websearch_to_tsquery and ranked with… | diff --git a/docs/backends.md b/docs/backends.md index dbade9e1..3e964796 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -2,8 +2,9 @@ title: PostgreSQL and ParadeDB backends nav_order: 9 description: >- - Full-text search on tsvector, pgvector nearest-neighbour search, sparse - retrieval, pg_trgm fuzzy search, and true BM25 on ParadeDB pg_search. + Full-text search on tsvector, BM25 without an extension, pgvector + nearest-neighbour search, sparse retrieval, pg_trgm fuzzy search, and true BM25 on ParadeDB + pg_search. --- # PostgreSQL and ParadeDB backends @@ -17,27 +18,30 @@ than a rewrite. | `LexiSharp` | records, interfaces, in-memory index, scorers, tokenizer, all decorators | **nothing** | | `LexiSharp.MessagePack` | binary persistence for the in-memory index and the sparse engine | MessagePack | | `LexiSharp.AspNetCore` | a `GET /search` minimal-API endpoint | ASP.NET Core | -| `LexiSharp.Postgres` | lexical, vector, sparse, fuzzy and BM25 engines over PostgreSQL | Npgsql | +| `LexiSharp.Postgres` | lexical, BM25, vector, sparse and fuzzy engines over PostgreSQL | Npgsql | -`LexiSharp.Postgres` ships five engines over **one shared documents table**, picked at +`LexiSharp.Postgres` ships six engines over **one shared documents table**, picked at instantiation: - **Lexical** — `tsvector` + GIN + `unaccent`, ranked with `ts_rank_cd`. +- **BM25 (stock PostgreSQL)** — `PostgresBm25SearchEngine`: BM25 over the term frequencies the + `tsvector` column already stores, computed by `Bm25Scorer`. No extension beyond `unaccent`, so + it runs where `pg_search` cannot be installed. - **Vector** — ANN over `pgvector` (HNSW or IVFFlat), fed by your own `IEmbeddingProvider`. - **Sparse** — learned-sparse ANN over `pgvector sparsevec` (HNSW only), fed by your own `ISparseEmbeddingProvider`. - **Fuzzy** — approximate typo-tolerant matching over `pg_trgm`, optionally refined with `fuzzystrmatch`. -- **BM25** — `ParadeDBTextSearchEngine`, true Okapi BM25 computed by Tantivy through - `pg_search` rather than `ts_rank_cd`. **The extension is AGPL-3 licensed** — fine for - SaaS and internal use, worth a review if you distribute the stack. +- **BM25 (ParadeDB)** — `ParadeDBTextSearchEngine`, true Okapi BM25 computed by Tantivy through + `pg_search`. **The extension is AGPL-3 licensed** — fine for SaaS and internal use, worth a + review if you distribute the stack. **What is verified, and what is not.** The integration tests run against a live PostgreSQL in CI (`pgvector/pgvector:pg16` and `paradedb/paradedb:pg16`) and self-skip without `POSTGRES_TEST_CONNECTION`, so what they cover is executed on every build: schema installation, the query paths, and exact cosine behaviour for the vector engine. Retrieval -*quality* is measured for the five engines that rank on their own: the evaluation harness's -`--backend lexical|paradedb|vector|fuzzy|sparse` adds a nDCG row for the same BEIR corpus, +*quality* is measured for the six engines that rank on their own: the evaluation harness's +`--backend lexical|bm25|paradedb|vector|fuzzy|sparse` adds a nDCG row for the same BEIR corpus, queries and qrels as the in-memory table, and [the evaluation page](evaluation.md#the-sql-backends-on-the-same-corpora) carries the figures and what each one is comparable with. `--backend sparse` indexes the same precomputed SPLADE vectors as the in-memory @@ -152,8 +156,8 @@ dotnet run --project bench/LexiSharp.Eval -c Release -- --dataset nfcorpus --no- | SciFact (300 queries) | 0.0188 | 0.0188 | 0.0188 | The direction is consistent with the title carrying a little signal on NFCorpus and none on a -corpus whose queries are long enough that `websearch_to_tsquery`'s AND gate, not the ranking, -decides the match set. A difference this small on one host is a measurement of this configuration, +corpus whose queries are long enough for the AND gate to leave too little to rank for a weighting +to show through. A difference this small on one host is a measurement of this configuration, not a claim that field weighting does not work: the classes are coarse (four buckets, fixed values), and the engine has no per-field length to normalise against. @@ -219,6 +223,31 @@ live instance (e.g. `Host=localhost;Port=5432;Username=postgres;Password=postgre The vector and sparse tests additionally require the `vector` extension: use the `pgvector/pgvector:pg16` image (lexical tests only need stock PostgreSQL). +**Match rule (`MatchMode`).** A multi-term query becomes a `tsquery` in one of two ways: + +| mode | expression | a document must contain | +|---|---|---| +| `AllTerms` (default) | `websearch_to_tsquery` | every term | +| `AnyTerm` | one `tsquery` OR-ing the query's lexemes | any term | + +`AnyTerm` takes its lexemes from `to_tsvector`, so they come from the column's own +configuration; it is a bag of words (the websearch operators `or`, `-term` and a trailing `*` are +not honoured) and quoted phrases are still hard gates. It changes which documents are candidates, +not how they are ranked — still `ts_rank_cd`. + +On this repository's BEIR runs the rule is worth more than the ranking difference on long queries, +and close to nothing on short ones: + +| corpus | `AllTerms` | `AnyTerm` | +|---|---:|---:| +| NFCorpus (queries average 3.4 terms) | 0.1864 | 0.1881 | +| SciFact (13.1 terms) | 0.0188 | 0.0988 | + +SciFact is where the AND rule buries the corpus: requiring all thirteen terms of an abstract leaves +almost nothing to rank. NFCorpus barely moves, because three terms usually do co-occur — its +distance from BM25 is the ranker, not the rule. `AnyTerm` still lands far from BM25's 0.662 on +SciFact, which is what `PostgresBm25SearchEngine` (below) is for. + **Sparse (`PostgresSparseSearchEngine`)** — learned-sparse ANN over `pgvector sparsevec` (HNSW only — IVFFlat is unavailable for `sparsevec`), fed by your own sparse model: @@ -296,6 +325,53 @@ are in-sample: each threshold was scored on the qrels it is reported against, so upper bound and not a tuned configuration. `Nearest` mode, the Levenshtein and phonetic refinements, and `%>`/`word_similarity` are not measured. +## BM25 over stock PostgreSQL (`PostgresBm25SearchEngine`) + +True Okapi BM25 with **no extension beyond `unaccent`**. The `tsvector` column and its GIN index do +the matching — the same table, the same DDL and the same `tsv` expression as the lexical engine — +and `Bm25Scorer` does the ranking over the term frequencies the vector already stores. + +```csharp +using LexiSharp.Postgres; + +ITextSearchEngine engine = new PostgresBm25SearchEngine(connectionString); +engine.Add(new SearchDocument("1", "the quick brown fox jumps over the lazy dog")); +engine.Search("quick fox"); // Bm25Scorer scores, not ts_rank_cd's +``` + +It exists because `pg_search` (next section) is the faster route to BM25 and the one with the +mature Tantivy index, but it needs a preloaded library, and no major managed service lets you +install one. This engine needs `CREATE EXTENSION unaccent` and nothing else, so on RDS, Cloud SQL +and Azure Database for PostgreSQL the only server-side step is allowing that one extension. + +**Where the numbers come from.** `N`, the average document length and every query term's document +frequency are read once through `ts_stat` and cached; the term frequencies and the length come from +the candidate row's own `tsvector`. The query passes through the same text search configuration as +the column, so the scorer sums over the terms the index matched. Any write through the engine +invalidates the snapshot and the next search rebuilds it; a write by another engine on the shared +table (or a bulk load) does not, and `RefreshStatistics()` is how that caller says so. Scores are +on `Bm25Scorer`'s scale, unlike `ts_rank_cd`'s and `pdb.score`'s, so they are comparable with the +in-memory BM25 row. + +**The candidate pool.** PostgreSQL selects the best `CandidatePool` matches by `ts_rank_cd` +(default 1,000; `0` removes the bound up to the engine's 100,000-row ceiling) and only those are +re-ranked. The pool bounds recall — a document outside it cannot be returned — and does not change +the ordering inside it. On the same BEIR corpora: + +| corpus | pool 1,000 | pool 0 (exact) | in-memory BM25 | ParadeDB | +|---|---:|---:|---:|---:| +| NFCorpus | 0.2999 | 0.3072 | 0.308 | 0.3084 | +| SciFact | 0.5636 | 0.6665 | 0.662 | 0.6619 | + +At the default pool the row is within 0.008 of the in-memory BM25 on NFCorpus and 0.10 short on +SciFact: on a corpus where a thirteen-term query matches half of it, the pool's 1,000 candidates are +the whole gap. Raise the pool, or set it to `0`, when the match sets are that large — the cost is +one `tsvector` per match transferred and parsed per query. + +**What it is not.** The analysis is PostgreSQL's, not this library's tokenizer, so it matches the +in-memory rows at most and no published figure. Weights (`FieldWeights`) are a `ts_rank_cd` +concept and do not reach this ranking. + ## ParadeDB BM25 backend (`ParadeDBTextSearchEngine`) Okapi BM25 ranking computed by Tantivy inside PostgreSQL through the `pg_search` @@ -342,11 +418,41 @@ the in-memory engines return under the default `TieBreak.DocumentId`. The statem page differently after a plan change. `TieBreak.InsertionOrder` has no counterpart on a SQL table and is ignored there: the id order is the answer. -BM25 scores are PostgreSQL-native, so they are **not** numerically comparable to -`Bm25Scorer`/`TfIdfScorer` — use the hybrid engine's `ReciprocalRankFusionMerger` (or -re-rank) for a single cross-engine ordering. Note the extension is **AGPL-3 licensed**: fine -for SaaS/internal use, but review it if you distribute the stack. - -By default the tests are skipped unless `POSTGRES_TEST_CONNECTION` points at a live instance -**with `pg_search` available** (the `paradedb/paradedb:pg16` image ships it, preloaded) — -they self-skip when the extension is absent. +ParadeDB's scores are Tantivy's and PostgreSQL-native, so they are **not** numerically comparable +to `Bm25Scorer`/`TfIdfScorer` — use the hybrid engine's `ReciprocalRankFusionMerger` (or re-rank) +for a single cross-engine ordering. The stock-PostgreSQL BM25 engine above is the opposite: its +scores are `Bm25Scorer`'s. Note the extension is **AGPL-3 licensed**: fine for SaaS/internal use, +but review it if you distribute the stack. + +## Managed PostgreSQL + +What each engine needs server-side, and whether the large managed services provide it: + +| requirement | RDS / Aurora | Cloud SQL | Azure Database for PostgreSQL | +|---|---|---|---| +| `unaccent`, `pg_trgm`, `fuzzystrmatch` | yes | yes | yes | +| `pgvector` (vector engine) | yes | yes | yes | +| `pgvector` ≥ 0.7.0 (sparse engine, `sparsevec` + HNSW) | yes on current majors | version-gated | yes on current majors; older servers ship 0.5.1, which has no `sparsevec` | +| `pg_search` (ParadeDB engine) | **no** | **no** | **no** | + +Everything else the engines need is stock PostgreSQL: the lexical and BM25 engines run anywhere, +and the fuzzy engine needs only contrib. + +Three things bite in practice: + +- **`CREATE EXTENSION` is privileged.** RDS wants `rds_superuser`, Cloud SQL `cloudsqlsuperuser`, + Azure `azure_pg_admin`. A deployment role without it fails at `EnsureSchema`, not at first query. +- **Azure and Cloud SQL gate extension installation by allow-list.** On Azure the extension must + also be listed in the `azure.extensions` server parameter before `CREATE EXTENSION` succeeds. + Set it in the deployment steps, not in application code. +- **`pg_search` is not installable on any of the three.** It is a preloaded library plus an index + access method, and the managed services ship a curated extension list. The portable BM25 is + `PostgresBm25SearchEngine`; if you want ParadeDB itself, run it beside the managed database (it + documents logical replication from RDS, Cloud SQL, Azure, Supabase and Neon into a ParadeDB + cluster) or use a managed ParadeDB offering. + +By default the tests are skipped unless `POSTGRES_TEST_CONNECTION` points at a live instance. What +each group needs is what its engine needs: the lexical and BM25 tests run on stock PostgreSQL with +contrib (`unaccent`), the vector and sparse tests need `pgvector` (`pgvector/pgvector:pg16`), and +the ParadeDB tests need `pg_search` (`paradedb/paradedb:pg16`, which ships it preloaded) — they +self-skip when the extension is absent. diff --git a/docs/evaluation.md b/docs/evaluation.md index 22c0bad1..5605ae76 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -448,7 +448,9 @@ dotnet run --project bench/LexiSharp.Eval -c Release -- --dataset nfcorpus --no- ``` `--backend lexical` measures `PostgresTextSearchEngine` (`tsvector` + `ts_rank_cd`); -`--backend paradedb` measures `ParadeDBTextSearchEngine` (`pg_search` + `pdb.score`, Tantivy's BM25); +`--backend bm25` measures `PostgresBm25SearchEngine` (BM25 over the same `tsvector` term +frequencies, no extension); `--backend paradedb` measures `ParadeDBTextSearchEngine` (`pg_search` + +`pdb.score`, Tantivy's BM25); `--backend vector` measures `PostgresVectorSearchEngine` over the cached dense embeddings, with one row per `ef_search`; `--backend fuzzy` measures `PostgresFuzzySearchEngine`, with one row per `pg_trgm` similarity threshold; `--backend sparse` measures `PostgresSparseSearchEngine` over the @@ -475,6 +477,14 @@ Out of the box, each engine at its own defaults (`k1=1.5, b=0.75` here, `1.2, 0. | SciFact | 0.662 | 0.6619 | 0.0188 | not measured | | ArguAna | 0.320 | not measured | not measured | not measured | +The two knobs that moved `ts_rank_cd` and then closed the gap, measured on the same corpora and the +same qrels (harness defaults, exponential gains, 10 retrieved at 10): + +| corpus | `ts_rank_cd` all terms | `ts_rank_cd` any term | BM25 pool 1,000 | BM25 pool 0 | in-memory BM25 | ParadeDB | +|---|---:|---:|---:|---:|---:|---:| +| NFCorpus | 0.1864 | 0.1881 | 0.2999 | 0.3072 | 0.308 | 0.3084 | +| SciFact | 0.0188 | 0.0988 | 0.5636 | **0.6665** | 0.662 | 0.6619 | + - **ParadeDB is a BM25 and reads where this library's BM25 reads.** At matched parameters the two differ by 0.0005 on NFCorpus and 0.0083 on SciFact, which is analysis rather than ranking: the index is built on `pdb.simple` with ASCII folding, the title is concatenated into the indexed @@ -482,10 +492,13 @@ Out of the box, each engine at its own defaults (`k1=1.5, b=0.75` here, `1.2, 0. (`|||`) rather than this library's scored term loop. It is not a reproduction, and it is not bit-identical. - **`ts_rank_cd` is not BM25** — it is coverage-based and takes no k1/b, so `--backend-bm25` is - refused with it rather than silently ignored. Most of its distance from the BM25 column is the - matching rule rather than the ranking: `websearch_to_tsquery` ANDs every term, so a match is a - document containing all of them. NFCorpus's test queries average 3.4 words and read 0.1864; - SciFact's average 13.1 and read 0.0188. PostgreSQL's own English analysis + refused with it rather than silently ignored. Its distance from BM25 has two causes, and the + rows above separate them. The *matching* rule is `websearch_to_tsquery`'s AND; `--backend-match + any` ORs the query's lexemes instead, which is worth 0.0188 → 0.0988 on SciFact (a 5.3× lift, + because a thirteen-term query matched with AND leaves almost nothing to rank) and 0.1864 → + 0.1881 on NFCorpus (whose queries average 3.4 terms, so the relevant documents usually do + contain all of them). What remains after the rule is the *ranker*: coverage has no idf, so a + common word counts as much as a rare one. PostgreSQL's own English analysis (`--backend-config english`) moves NFCorpus 0.1864 → 0.2043, not closer. Weighting the title with `--backend-field-weights title=A` leaves both rows essentially unchanged — NFCorpus 0.1869, SciFact 0.0188 — and [the backend guide](backends.md#postgresql-backend-lexisharpostgres) carries @@ -499,10 +512,17 @@ Out of the box, each engine at its own defaults (`k1=1.5, b=0.75` here, `1.2, 0. parameters, and the pinned job runs without a PostgreSQL service, so a pin there could only ever skip — a gate that cannot fail. They are reproducible from the command above, and nothing catches a drift in them. -- **Scores are PostgreSQL-native and not numerically comparable** with `Bm25Scorer`; the metric - columns are. The analysis is PostgreSQL's own, not this library's tokenizer, which is why these - rows are not comparable with the published figures or with the aligned rows either — the - parameter-matched table above compares two analyses, not two implementations of one. +- **`--backend bm25` closes the ranker gap with no extension.** It scores the same `tsvector` term + frequencies with `Bm25Scorer`, and at an unbounded pool reads 0.3072 / 0.6665 against the + in-memory row's 0.308 / 0.662 and ParadeDB's 0.3084 / 0.6619 — so what is left is analysis, not + arithmetic. The default pool of 1,000 reads 0.2999 / 0.5636: on SciFact, where a thirteen-term + query matches half the corpus, the pool is the whole 0.10 difference. That is the trade + `--backend-pool` states, and the reason the row prints the pool it ran with. +- **`ts_rank_cd`'s and ParadeDB's scores are PostgreSQL-native and not numerically comparable** + with `Bm25Scorer`; the stock-PostgreSQL BM25 row's are `Bm25Scorer`'s and are. The metric + columns compare either way. The analysis is PostgreSQL's own, not this library's tokenizer, which + is why these rows are not comparable with the published figures or with the aligned rows either — + the parameter-matched table above compares two analyses, not two implementations of one. The vector engine is measured the same way, over the harness's cached embeddings, with one row per `ef_search` — `--backend vector --backend-ef 10,40,160`, which needs `--dense`: diff --git a/docs/index.md b/docs/index.md index 71d5c6ed..cf274afb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -115,8 +115,11 @@ twice. Every figure below is reproducible with a command on the - **The SQL backends, on the same corpora.** The PostgreSQL text, BM25 and vector engines are scored over the same queries and qrels as the in-memory table: at matched k1=0.9/b=0.4, ParadeDB's Tantivy BM25 reads **0.3055** and **0.6633** on NFCorpus and SciFact against this library's - **0.305** and **0.655**, `ts_rank_cd` reads **0.1864** and **0.0188** — mostly its matcher, which - ANDs every term — and pgvector at `ef_search` 10/40/160 reads **0.2882 / 0.2960 / 0.3013** on + **0.305** and **0.655**, `ts_rank_cd` reads **0.1864** and **0.0188**: its matcher ANDs every + term, and OR-ing them (`--backend-match any`) recovers **0.0988** on SciFact but only **0.1881** + on NFCorpus, so the rest of that gap is the ranker. The stock-PostgreSQL BM25 engine closes it + with no extension: **0.3072** and **0.6665** at an unbounded candidate pool, **0.2999** and + **0.5636** at its default 1,000. pgvector at `ef_search` 10/40/160 reads **0.2882 / 0.2960 / 0.3013** on NFCorpus against the exact in-memory dense row's **0.304**, and **0.5998 / 0.6528 / 0.6550** on SciFact against **0.655**: that gap is the index's cost in nDCG. ArguAna produces no row from any of the three: the text engines reject 2 of its 1,406 test queries, and the vector engine rejects @@ -155,7 +158,7 @@ twice. Every figure below is reproducible with a command on the | [Rerankers and hybrid search](pipelines.md) | rerankers (MMR, cascade, cross-encoder, MaxSim), hybrid federation, cost-based and intent-based routing | | [Dense, sparse and query expansion](embeddings.md) | dense, learned-sparse and corpus-derived expansion, all behind consumer-supplied seams | | [Classification and text analysis](text-analysis.md) | lexical similarity, keyword extraction, Naive Bayes, trees/forest/linear, centroids, nearest neighbours and blends | -| [PostgreSQL and ParadeDB backends](backends.md) | PostgreSQL lexical / vector / sparse / fuzzy, ParadeDB BM25, what the integration tests cover | +| [PostgreSQL and ParadeDB backends](backends.md) | PostgreSQL lexical / BM25 / vector / sparse / fuzzy, ParadeDB BM25, managed-service support, what the integration tests cover | | [Search traces and telemetry](observability.md) | `SearchTrace`, production telemetry, source agreement, calibrated score confidence | | [Retrieval quality and IR metrics](evaluation.md) | the BEIR numbers in full, IR metrics, the benchmark CLI, per-query diffs, parameter tuning | | [Allocation and latency benchmarks](benchmarks.md) | allocations and timings, with the machine they came from and the command to reproduce them | diff --git a/docs/querying.md b/docs/querying.md index 469dfc36..085f867c 100644 --- a/docs/querying.md +++ b/docs/querying.md @@ -174,7 +174,7 @@ raw text is then that backend's own grammar to read. | Engine | Query syntax | |---|---| | `RankedTextSearchEngine` | phrases, prefix/fuzzy, required and prohibited terms; field scopes when its scorer and index declare them (see [Which engines honor field scopes](#which-engines-honor-field-scopes)) | -| `PostgresTextSearchEngine`, `ParadeDBTextSearchEngine` | phrases | +| `PostgresTextSearchEngine`, `PostgresBm25SearchEngine`, `ParadeDBTextSearchEngine` | phrases | | `PostgresFuzzySearchEngine`, `PostgresVectorSearchEngine`, `PostgresSparseSearchEngine`, `InMemoryVectorSearchEngine`, `SparseTextSearchEngine` | plain queries only | The SQL backends are declared unsupported rather than approximated: the lexical engine forwards @@ -256,7 +256,7 @@ statistics but not per-field positions answers a scoped single term and throws |---|---| | `RankedTextSearchEngine` + `Bm25FScorer` over an `IFieldStatisticsIndex` | yes | | `RankedTextSearchEngine` + any other scorer, or over an index without field statistics | refused | -| `PostgresTextSearchEngine`, `ParadeDBTextSearchEngine` | refused when `KnownFields` is set; otherwise the prefix is text | +| `PostgresTextSearchEngine`, `PostgresBm25SearchEngine`, `ParadeDBTextSearchEngine` | refused when `KnownFields` is set; otherwise the prefix is text | | `PostgresFuzzySearchEngine`, `PostgresVectorSearchEngine`, `PostgresSparseSearchEngine`, `InMemoryVectorSearchEngine`, `SparseTextSearchEngine` | refused when `KnownFields` is set; otherwise the prefix is text | The SQL backends have no field model in their table (the text is one `tsvector`/`pg_search` diff --git a/src/LexiSharp.Postgres/PostgresBm25IndexView.cs b/src/LexiSharp.Postgres/PostgresBm25IndexView.cs new file mode 100644 index 00000000..72e2b563 --- /dev/null +++ b/src/LexiSharp.Postgres/PostgresBm25IndexView.cs @@ -0,0 +1,118 @@ +using LexiSharp.Core; + +namespace LexiSharp.Postgres; + +/// One re-ranked match: its document, its term frequencies and its length. +internal sealed record PostgresBm25Candidate( + SearchDocument Document, + IReadOnlyDictionary TermFrequencies, + int Length); + +/// +/// The candidate set of one search, presented as an so +/// can score it. +/// +/// +/// +/// DocumentFrequency, N and the average length answer from the corpus snapshot; +/// TermFrequency and DocumentLength answer from the candidate row's own +/// tsvector. That split is what makes the score a corpus statistic rather than a statistic +/// of the page: computing idf over the candidates would give a rare term the frequency of +/// "however many of the top 1,000 happened to contain it". +/// +/// +/// Positions are not kept. Nothing on the BM25 path reads them, and a caller that asks gets +/// rather than an empty list that would read as "the term +/// occurs nowhere". +/// +/// +internal sealed class PostgresBm25IndexView : IReadOnlyTextIndex +{ + private readonly PostgresBm25Statistics _statistics; + private readonly IReadOnlyList _candidates; + private readonly Dictionary _byId; + + public PostgresBm25IndexView(PostgresBm25Statistics statistics, IReadOnlyList candidates) + { + ArgumentNullException.ThrowIfNull(statistics); + ArgumentNullException.ThrowIfNull(candidates); + + _statistics = statistics; + _candidates = candidates; + _byId = candidates.ToDictionary(candidate => candidate.Document.Id, StringComparer.Ordinal); + } + + /// + public IReadOnlyCollection Documents => + _candidates.Select(candidate => candidate.Document).ToList(); + + /// + public int Count => _candidates.Count; + + /// + public double AverageDocumentLength => _statistics.AverageDocumentLength; + + /// + public int StatisticDocumentCount => _statistics.DocumentCount; + + /// + public int VocabularySize => _statistics.VocabularySize; + + /// + public long CorpusTokenCount => _statistics.CorpusTokenCount; + + /// + public bool Contains(string documentId) => + documentId is not null && _byId.ContainsKey(documentId); + + /// + public IReadOnlyList GetTerms(string documentId) => + Candidate(documentId).TermFrequencies.Keys.OrderBy(term => term, StringComparer.Ordinal).ToList(); + + /// + /// Always: frequencies are kept, positions are not. + public IReadOnlyList GetTermPositions(string documentId, string term) => + throw new NotSupportedException( + "The BM25 pool keeps per-term frequencies, not positions: a phrase is enforced by the " + + "tsquery that selected the candidates, not re-checked while scoring."); + + /// + public int DocumentFrequency(string term) => + term is null ? 0 : _statistics.DocumentFrequency(term); + + /// + public int CorpusFrequency(string term) => + term is null ? 0 : _statistics.CorpusFrequency(term); + + /// + public int TermFrequency(string documentId, string term) => + term is not null && Candidate(documentId).TermFrequencies.TryGetValue(term, out int frequency) + ? frequency + : 0; + + /// + public int DocumentLength(string documentId) => Candidate(documentId).Length; + + /// + public bool TryGetDocument(string documentId, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out SearchDocument? document) + { + if (documentId is not null && _byId.TryGetValue(documentId, out PostgresBm25Candidate? candidate)) + { + document = candidate.Document; + return true; + } + + document = null; + return false; + } + + private PostgresBm25Candidate Candidate(string documentId) + { + ArgumentNullException.ThrowIfNull(documentId); + + return _byId.TryGetValue(documentId, out PostgresBm25Candidate? candidate) + ? candidate + : throw new ArgumentException( + $"Document '{documentId}' is not in this search's candidate pool.", nameof(documentId)); + } +} diff --git a/src/LexiSharp.Postgres/PostgresBm25Options.cs b/src/LexiSharp.Postgres/PostgresBm25Options.cs new file mode 100644 index 00000000..93ec60f3 --- /dev/null +++ b/src/LexiSharp.Postgres/PostgresBm25Options.cs @@ -0,0 +1,131 @@ +namespace LexiSharp.Postgres; + +/// +/// Naming, analysis and ranking options for the stock-PostgreSQL BM25 engine. +/// +/// +/// +/// The table layout is the lexical engine's: reads the same +/// documents table, the same tsv column and the same GIN index, so one corpus serves +/// ts_rank_cd, BM25, vector and fuzzy search without a second copy. +/// +/// +/// What makes this BM25 and not ts_rank_cd. The matching rule is still PostgreSQL's — +/// the same tsquery, the same GIN scan — but the score is +/// Σ idf(t)·tf(t,d)·(k1+1)/(tf(t,d)+k1(1−b+b·|d|/avgdl)) with +/// idf(t) = ln(1+(N−df(t)+0.5)/(df(t)+0.5)), computed by +/// over the term frequencies the tsvector already stores. +/// It takes k1 and b, which ts_rank_cd does not have, and it returns scores on +/// the in-memory scorer's scale rather than PostgreSQL's. +/// +/// +public sealed record PostgresBm25Options +{ + /// Schema that hosts the documents table (default: public). + public string Schema { get; init; } = "public"; + + /// Table that stores documents (default: lexisharp_documents). + public string Table { get; init; } = "lexisharp_documents"; + + /// + /// The PostgreSQL text search configuration that analysis, matching and scoring all read + /// (default: simple). The tsv column, the query lexemes and the term frequencies + /// come from this one setting, which is why the three cannot disagree. + /// + public string TextSearchConfig { get; init; } = "simple"; + + /// + /// When true (default), the constructor installs unaccent, the documents table and its + /// GIN index — the lexical engine's schema, unchanged. + /// + public bool AutoCreateSchema { get; init; } = true; + + /// + /// Weight class per entry, applied when the + /// tsv column is written (default: null). + /// + /// + /// setweight classes are a ts_rank_cd concept: this engine reads term frequencies + /// out of the vector and scores them with BM25, which has no place for a class. Setting weights + /// here only matters if another engine reads the same rows — pass the same value to every + /// engine writing the table so the tsv column is written one way. + /// + public IReadOnlyDictionary? FieldWeights { get; init; } + + /// Class given to when weighting is on. + public PostgresFieldWeight DefaultFieldWeight { get; init; } = PostgresFieldWeight.D; + + /// Term-frequency saturation (default 1.5, the usual Okapi value). + public const double DefaultK1 = 1.5; + + /// Length normalisation in [0, 1] (default 0.75); 0 disables it. + public const double DefaultB = 0.75; + + /// How many matches are re-ranked with BM25 when is unset. + public const int DefaultCandidatePool = 1000; + + /// + /// The most candidates one query may ask for, shared with every SQL engine's per-query fetch + /// ceiling. = 0 means this, not an unbounded scan. + /// + public const int MaximumCandidatePool = PostgresDocumentExclusion.MaxRowsPerQuery; + + /// Term-frequency saturation (default 1.5, the usual Okapi value). + public double K1 { get; init; } = DefaultK1; + + /// Length normalisation in [0, 1] (default 0.75); 0 disables it. + public double B { get; init; } = DefaultB; + + /// + /// How a multi-term query is matched: any term (, the + /// default) or every term (). + /// + /// + /// AnyTerm is the default because BM25 ranks a candidate set: an AND rule on a long query + /// leaves almost nothing to rank (0.0188 nDCG@10 on SciFact in this repository's own run), and + /// the rare-term idf that makes BM25 useful never gets to see the documents that hold only part + /// of the query. stays available for a corpus where a + /// result must contain every word. + /// + public PostgresMatchMode MatchMode { get; init; } = PostgresMatchMode.AnyTerm; + + /// + /// How many matches are re-ranked with BM25 (default 1000). + /// + /// + /// + /// The database chooses candidates with ts_rank_cd and the engine scores only the top + /// of them: transferring and parsing the whole match set would make + /// a search proportional to the corpus. The pool is therefore a recall bound — a document + /// outside it cannot be returned, however well it would have scored — and it is widened + /// automatically when the requested page is larger. + /// + /// + /// 0 removes the bound, up to the engine's own ceiling of 100,000 rows per query. That is + /// the exact ranking and a cost that grows with the match set; use it to check what the pool + /// costs on a corpus, not in a request path. + /// + /// + public int CandidatePool { get; init; } = DefaultCandidatePool; + + internal bool IsValid => + AsIndexOptions().IsValid + && !double.IsNaN(K1) && !double.IsInfinity(K1) && K1 >= 0 + && !double.IsNaN(B) && !double.IsInfinity(B) && B is >= 0 and <= 1 + && CandidatePool >= 0; + + /// Fully qualified table name, e.g. public.lexisharp_documents. + public string QualifiedTableName => AsIndexOptions().QualifiedTableName; + + internal PostgresIndexOptions AsIndexOptions() => + new() + { + Schema = Schema, + Table = Table, + TextSearchConfig = TextSearchConfig, + AutoCreateSchema = AutoCreateSchema, + FieldWeights = FieldWeights, + DefaultFieldWeight = DefaultFieldWeight, + MatchMode = MatchMode, + }; +} diff --git a/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs b/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs new file mode 100644 index 00000000..57284b69 --- /dev/null +++ b/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs @@ -0,0 +1,338 @@ +using LexiSharp.Core; +using LexiSharp.Ranking; +using Npgsql; + +namespace LexiSharp.Postgres; + +/// +/// Okapi BM25 on stock PostgreSQL: the tsvector column and GIN index do the matching, and +/// does the ranking over the term frequencies the vector already holds. +/// No extension beyond unaccent, so it runs on RDS, Cloud SQL and Azure Database for +/// PostgreSQL — where the ParadeDB pg_search extension cannot be installed — with only the +/// usual privileged role needed to create that one extension. +/// +/// +/// +/// Where each number comes from. The document count, the average length and every term's +/// document frequency are read once by through +/// ts_stat and cached until a write invalidates them; the per-document term frequencies and +/// length come from the candidate row's own tsvector. The query passes through the same +/// text search configuration as the column, so the terms the scorer sums over are the terms the +/// index matched — not a second tokenizer's opinion of them. +/// +/// +/// What the pool costs. PostgreSQL selects the +/// best matches by ts_rank_cd and the engine re-ranks that set; scoring the whole match set +/// would make a query proportional to the corpus and mean shipping every tsvector to the +/// client. A document outside the pool cannot be returned. The pool is a recall bound, not a +/// ranking approximation: inside it the ordering is exact BM25. +/// +/// +/// Scores. Unlike the ts_rank_cd engine, these are scores on +/// the in-memory scorer's scale, with the same conventions — so a document's score is comparable +/// with the in-memory BM25 row, and 's decomposition applies to the +/// same formula. What is not identical is the analysis: PostgreSQL's configuration, not this +/// library's tokenizer, produced the terms on both sides. +/// +/// +/// Statistics lag. The cached snapshot is invalidated by , +/// , and , so the first search after a +/// write pays one ts_stat walk of the table. An application that writes and searches in a +/// tight loop should batch its writes through , or accept that cost. +/// +/// +public sealed class PostgresBm25SearchEngine : ITextSearchEngine, IDisposable, IQuerySyntaxSupport +{ + /// + public QueryFeatures SupportedQueryFeatures => QueryFeatures.Phrases; + + private readonly NpgsqlDataSource _dataSource; + private readonly PostgresBm25Options _options; + private readonly PostgresIndexOptions _indexOptions; + private readonly Bm25Scorer _scorer; + private PostgresBm25Statistics? _statistics; + private bool _schemaReady; + + /// A PostgreSQL connection string (Npgsql format). + /// Naming, analysis and ranking options (default: ). + /// + /// Shortcut for ; overrides the option when set. + /// + public PostgresBm25SearchEngine( + string connectionString, + PostgresBm25Options? options = null, + bool? autoCreateSchema = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); + + _options = options ?? new PostgresBm25Options(); + + if (!_options.IsValid) + throw new ArgumentException( + "Schema, table and text search config must match [A-Za-z0-9_], a field weight must name " + + "a non-empty field with a defined class, k1 must be a finite non-negative number, b a " + + "finite number in [0, 1], and CandidatePool a non-negative count.", + nameof(options)); + + if (autoCreateSchema is not null) + _options = _options with { AutoCreateSchema = autoCreateSchema.Value }; + + _indexOptions = _options.AsIndexOptions(); + _scorer = new Bm25Scorer(_options.K1, _options.B); + + _dataSource = NpgsqlDataSource.Create(connectionString); + + if (_options.AutoCreateSchema) + EnsureSchema(); + } + + /// + /// Installs the unaccent extension, the shared documents table and its GIN index. + /// Idempotent, and the same schema the lexical engine creates. + /// + public void EnsureSchema() + { + if (_schemaReady) + return; + + using var connection = _dataSource.OpenConnection(); + PostgresSchema.CreateDocumentTable(connection, _indexOptions); + _schemaReady = true; + } + + /// Async variant of . + public async Task EnsureSchemaAsync(CancellationToken cancellationToken = default) + { + if (_schemaReady) + return; + + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await PostgresSchema.CreateDocumentTableAsync(connection, _indexOptions, cancellationToken).ConfigureAwait(false); + _schemaReady = true; + } + + /// For tests: the physical table name in use. + internal string TestTableName => _options.Table; + + /// Drops the documents table and its indexes. Useful for tests and clean teardowns. + public void DropSchema() + { + using var connection = _dataSource.OpenConnection(); + + using var command = connection.CreateCommand(); + command.CommandText = $"DROP TABLE IF EXISTS {_options.QualifiedTableName}"; // NOSONAR:S2077 (identifier only, validated + quoted) + command.ExecuteNonQuery(); + } + + /// + /// Rebuilds the corpus statistics now, instead of waiting for the next write to invalidate them. + /// + /// + /// The snapshot is exact for the corpus at the moment it was read and unchanged until a write + /// through this engine invalidates it. A writer that changes the table by other means — another + /// engine on the same table, a bulk load, a restore — invalidates nothing, and this is how that + /// caller says so. Returns the number of terms in the snapshot. + /// + public int RefreshStatistics() + { + using var connection = _dataSource.OpenConnection(); + _statistics = PostgresBm25Statistics.Load(connection, _indexOptions); + return _statistics.VocabularySize; + } + + /// + public void Index(IEnumerable documents) + { + ArgumentNullException.ThrowIfNull(documents); + + using var connection = _dataSource.OpenConnection(); + + using (var command = connection.CreateCommand()) + using (var transaction = connection.BeginTransaction()) + { + command.CommandText = $"TRUNCATE {_options.QualifiedTableName}"; // NOSONAR:S2077 (identifier only, validated + quoted) + command.Transaction = transaction; + command.ExecuteNonQuery(); + + foreach (var document in documents) + PostgresDocumentWriter.Insert(connection, transaction, _indexOptions, document); + + transaction.Commit(); + } + + _statistics = null; + } + + /// + public void Add(SearchDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + using var connection = _dataSource.OpenConnection(); + PostgresDocumentWriter.Insert(connection, null, _indexOptions, document); + + _statistics = null; + } + + /// + public bool Remove(string documentId) + { + ArgumentNullException.ThrowIfNull(documentId); + + using var connection = _dataSource.OpenConnection(); + + using var command = connection.CreateCommand(); + command.CommandText = $"DELETE FROM {_options.QualifiedTableName} WHERE id = @id"; // NOSONAR:S2077 (identifiers only; id is parameterized) + command.Parameters.AddWithValue("id", documentId); + bool removed = command.ExecuteNonQuery() > 0; + + if (removed) + _statistics = null; + + return removed; + } + + /// + public void Clear() + { + using var connection = _dataSource.OpenConnection(); + + using var command = connection.CreateCommand(); + command.CommandText = $"TRUNCATE {_options.QualifiedTableName}"; // NOSONAR:S2077 (identifier only, validated + quoted) + command.ExecuteNonQuery(); + + _statistics = null; + } + + /// + public IReadOnlyList Search( + string query, SearchOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + cancellationToken.ThrowIfCancellationRequested(); + + options ??= SearchOptions.Default; + + if (options.IsEmpty || string.IsNullOrWhiteSpace(query)) + return Array.Empty(); + + // ParseQuerySyntax: false is the caller saying "literal text, do not consult the query + // parser" — and EnsureSupported is a parser call. + if (options.ParseQuerySyntax) + QuerySyntax.EnsureSupported(query, SupportedQueryFeatures, nameof(PostgresBm25SearchEngine), options.KnownFields); + + using var connection = _dataSource.OpenConnection(); + + // The same analysis on both sides: PostgreSQL's, via the column's configuration. + string[] queryTerms = ReadQueryLexemes(connection, query); + + if (queryTerms.Length == 0) + return Array.Empty(); + + // Benign race: two concurrent searches may build the snapshot twice and both publish a + // complete one. A reference assignment is atomic, so no reader can see a half-built object. + var statistics = _statistics ??= PostgresBm25Statistics.Load(connection, _indexOptions); + var match = PostgresMatchExpression.Build(_indexOptions, query); + + using var command = connection.CreateCommand(); + + var filters = PostgresMetadataFilterSql.Build(options.Filters); + filters.Apply(command); + match.Bind(command); + + string candidatesSql = $""" + SELECT id, content, category, fields, text_fields, tsv::text + FROM {_options.QualifiedTableName} + WHERE ({match.Gate}){filters.Fragment} + ORDER BY {match.Rank} DESC, {PostgresSchema.OrdinalIdTieBreak} + LIMIT @pool; + """; + + // Identifiers/config only are interpolated (validated [A-Za-z0-9_] and quoted); query text is parameterized. + command.CommandText = candidatesSql; // NOSONAR:S2077 + command.Parameters.AddWithValue("pool", PoolSize(_options, options)); + + var candidates = new List(); + + using (var cancelling = PostgresCancellation.Register(command, cancellationToken)) + using (var reader = command.ExecuteReader()) + { + while (reader.Read()) + { + SearchDocument document = PostgresDocumentReader.Read(reader); + var (termFrequencies, length) = + PostgresTsvText.Parse(reader.IsDBNull(5) ? null : reader.GetString(5)); + + candidates.Add(new PostgresBm25Candidate(document, termFrequencies, length)); + } + } + + if (candidates.Count == 0) + return Array.Empty(); + + var view = new PostgresBm25IndexView(statistics, candidates); + var results = new List(candidates.Count); + + foreach (PostgresBm25Candidate candidate in candidates) + { + double score = _scorer.Score(candidate.Document.Id, queryTerms, view); + + if (double.IsNaN(score) || double.IsInfinity(score) || score <= 0) + continue; + + if (score < options.MinimumScore) + continue; + + if (!PostgresDocumentExclusion.Passes(options, candidate.Document.Id)) + continue; + + results.Add(new SearchResult(candidate.Document.Id, score, candidate.Document)); + } + + // Score descending, then the ordinal id: the same tie-break the SQL engines write into + // their ORDER BY, applied here because the ordering is computed in this process. + results.Sort(static (left, right) => + { + int byScore = right.Score.CompareTo(left.Score); + return byScore != 0 ? byScore : string.CompareOrdinal(left.DocumentId, right.DocumentId); + }); + + return results.Skip(options.Offset).Take(options.Limit).ToList(); + } + + /// Releases the underlying Npgsql data source. + public void Dispose() => _dataSource.Dispose(); + + /// + /// How many matches the database hands over: the configured pool, widened to at least the + /// requested page, and capped by the engine's per-query ceiling. + /// + /// + /// The widening is what keeps pagination honest: a page beyond the pool would otherwise come + /// back empty even though more matches exist. The pool decides which candidates are scored, and + /// it can never be smaller than the window the caller asked for. + /// + internal static int PoolSize(PostgresBm25Options options, SearchOptions searchOptions) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(searchOptions); + + long wanted = (long)searchOptions.Window + (searchOptions.ExcludedDocumentIds?.Count ?? 0); + + if (options.CandidatePool == 0) + return PostgresBm25Options.MaximumCandidatePool; + + return (int)Math.Min( + Math.Max(options.CandidatePool, wanted), + PostgresBm25Options.MaximumCandidatePool); + } + + private string[] ReadQueryLexemes(NpgsqlConnection connection, string query) + { + using var command = connection.CreateCommand(); + command.CommandText = PostgresTsQueryText.QueryLexemes(_indexOptions.TextSearchConfig); // NOSONAR:S2077 (validated config only) + command.Parameters.AddWithValue("query", query); + + return command.ExecuteScalar() as string[] ?? Array.Empty(); + } +} diff --git a/src/LexiSharp.Postgres/PostgresBm25Statistics.cs b/src/LexiSharp.Postgres/PostgresBm25Statistics.cs new file mode 100644 index 00000000..2609a19b --- /dev/null +++ b/src/LexiSharp.Postgres/PostgresBm25Statistics.cs @@ -0,0 +1,109 @@ +using Npgsql; + +namespace LexiSharp.Postgres; + +/// +/// The corpus statistics a needs, read from the table once. +/// +/// +/// +/// Why a snapshot. BM25 needs three corpus-wide numbers: the document count N, the +/// average document length, and each query term's document frequency. Stock PostgreSQL exposes +/// them through ts_stat, which walks the table — cheap once, unaffordable per query. So one +/// call builds this object and the engine keeps it until a write invalidates it; the numbers are +/// then exact for the corpus as it stood at that moment, and stale until the next rebuild. That is +/// the same trade Lucene makes with per-segment statistics, and it is stated rather than hidden. +/// +/// +/// Why N is not count(*). A document with no indexed term contributes nothing +/// to any score and cannot be a result; counting it would deflate every idf. Under +/// --backend lexical this repository already measured what a corpus with one empty document +/// costs (1 of 8,674 on ArguAna); the count here is the one +/// documents, so the two BM25 implementations agree. +/// +/// +internal sealed class PostgresBm25Statistics +{ + private readonly Dictionary _terms; + + private PostgresBm25Statistics(int documentCount, long corpusTokenCount, Dictionary terms) + { + DocumentCount = documentCount; + CorpusTokenCount = corpusTokenCount; + _terms = terms; + } + + /// Documents holding at least one indexed term: the N of the idf. + public int DocumentCount { get; } + + /// Total indexed tokens across the corpus (the sum of every term's frequency). + public long CorpusTokenCount { get; } + + /// Mean indexed tokens per counted document. + public double AverageDocumentLength => + DocumentCount == 0 ? 0 : (double)CorpusTokenCount / DocumentCount; + + /// Terms the snapshot knows. + public int VocabularySize => _terms.Count; + + /// Number of documents containing (0 when unknown). + public int DocumentFrequency(string term) => + _terms.TryGetValue(term, out TermStatistics statistics) ? statistics.DocumentFrequency : 0; + + /// Total occurrences of (0 when unknown). + public int CorpusFrequency(string term) => + _terms.TryGetValue(term, out TermStatistics statistics) ? statistics.CorpusFrequency : 0; + + /// + /// Reads the statistics of ' table. + /// + /// + /// Two statements: ts_stat answers per term, and the count of non-empty vectors answers + /// N (a scalar subquery would put both in one round trip, but ts_stat cannot be + /// correlated, so the two are separate reads of the same connection). + /// + public static PostgresBm25Statistics Load(NpgsqlConnection connection, PostgresIndexOptions options) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentNullException.ThrowIfNull(options); + + int documentCount; + long corpusTokenCount = 0; + + using (var count = connection.CreateCommand()) + { + // Identifiers only are interpolated (validated [A-Za-z0-9_]+ and quoted). + count.CommandText = + $"SELECT count(*) FROM {options.QualifiedTableName} WHERE tsv IS NOT NULL AND tsv::text <> ''"; // NOSONAR:S2077 + + documentCount = Convert.ToInt32(count.ExecuteScalar()); + } + + var terms = new Dictionary(StringComparer.Ordinal); + + using (var command = connection.CreateCommand()) + { + // The inner query is a literal ts_stat evaluates; the table name is a validated, + // quoted identifier and carries no quote character of its own. + command.CommandText = + $"SELECT word, ndoc, nentry FROM ts_stat('SELECT tsv FROM {options.QualifiedTableName}')"; // NOSONAR:S2077 + + using var reader = command.ExecuteReader(); + + while (reader.Read()) + { + string term = reader.GetString(0); + int ndoc = reader.GetInt32(1); + int nentry = reader.GetInt32(2); + + terms[term] = new TermStatistics(ndoc, nentry); + corpusTokenCount += nentry; + } + } + + return new PostgresBm25Statistics(documentCount, corpusTokenCount, terms); + } + + /// One term's two counts, kept together so they cannot be read apart. + private readonly record struct TermStatistics(int DocumentFrequency, int CorpusFrequency); +} diff --git a/src/LexiSharp.Postgres/PostgresDocumentStore.cs b/src/LexiSharp.Postgres/PostgresDocumentStore.cs new file mode 100644 index 00000000..4023d4ad --- /dev/null +++ b/src/LexiSharp.Postgres/PostgresDocumentStore.cs @@ -0,0 +1,112 @@ +using LexiSharp.Core; +using Npgsql; +using NpgsqlTypes; + +namespace LexiSharp.Postgres; + +/// +/// The upsert the two tsvector text engines issue into the shared documents table. +/// +/// +/// +/// Extracted for the same reason was: a second engine now +/// writes the same table, and the two must write the same row. The tsv column in particular +/// is derived here from , from the same options, so a +/// document written by the lexical engine and the same document written by the BM25 engine score +/// through one analysis. +/// +/// +/// Identifiers only are interpolated (validated [A-Za-z0-9_]+ and quoted); every value is a +/// parameter. +/// +/// +internal static class PostgresDocumentWriter +{ + /// Upserts one document, deriving tsv from its text and text fields. + public static void Insert( + NpgsqlConnection connection, + NpgsqlTransaction? transaction, + PostgresIndexOptions options, + SearchDocument document) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(document); + + using var command = connection.CreateCommand(); + + string tsvExpr = PostgresSchema.TsvExpression(options, "@content", "@text_fields"); + string tsvExprUpsert = PostgresSchema.TsvExpression(options, "EXCLUDED.content", "EXCLUDED.text_fields"); + + string insertSql = $""" + INSERT INTO {options.QualifiedTableName} (id, content, category, fields, text_fields, tsv) + VALUES (@id, @content, @category, @fields, @text_fields, {tsvExpr}) + ON CONFLICT (id) DO UPDATE + SET content = EXCLUDED.content, + category = EXCLUDED.category, + fields = EXCLUDED.fields, + text_fields = EXCLUDED.text_fields, + tsv = {tsvExprUpsert}; + """; + + command.CommandText = insertSql; // NOSONAR:S2077 + + if (transaction is not null) + command.Transaction = transaction; + + command.Parameters.AddWithValue("id", document.Id); + command.Parameters.AddWithValue("content", document.Text); + command.Parameters.AddWithValue("category", (object?)document.Category ?? DBNull.Value); + + var fieldsParameter = command.Parameters.AddWithValue( + "fields", + document.Fields is null ? DBNull.Value : System.Text.Json.JsonSerializer.Serialize(document.Fields, PostgresJsonContext.Default.IReadOnlyDictionaryStringString)); + fieldsParameter.NpgsqlDbType = NpgsqlDbType.Jsonb; + + NpgsqlParameter textFieldsParameter; + if (document.TextFields is not null) + { + textFieldsParameter = command.Parameters.AddWithValue( + "text_fields", + System.Text.Json.JsonSerializer.Serialize(document.TextFields, PostgresJsonContext.Default.IReadOnlyDictionaryStringString)); + textFieldsParameter.NpgsqlDbType = NpgsqlDbType.Jsonb; + } + else + { + textFieldsParameter = command.Parameters.AddWithValue("text_fields", DBNull.Value); + textFieldsParameter.NpgsqlDbType = NpgsqlDbType.Jsonb; + } + + command.ExecuteNonQuery(); + } +} + +/// +/// Reads the five document columns the engines select, in the order they select them. +/// +internal static class PostgresDocumentReader +{ + /// + /// Reads one row whose first five columns are id, content, category, fields, text_fields. + /// + public static SearchDocument Read(NpgsqlDataReader reader) + { + ArgumentNullException.ThrowIfNull(reader); + + string id = reader.GetString(0); + string content = reader.GetString(1); + string? category = reader.IsDBNull(2) ? null : reader.GetString(2); + var fields = reader.IsDBNull(3) ? null : Deserialize(reader.GetString(3)); + var textFields = reader.IsDBNull(4) ? null : Deserialize(reader.GetString(4)); + + return new SearchDocument(id, content, fields, category, textFields); + } + + private static IReadOnlyDictionary? Deserialize(string json) + { + if (string.IsNullOrEmpty(json)) + return null; + + return System.Text.Json.JsonSerializer.Deserialize>(json, PostgresJsonContext.Default.DictionaryStringString); + } +} diff --git a/src/LexiSharp.Postgres/PostgresIndexOptions.cs b/src/LexiSharp.Postgres/PostgresIndexOptions.cs index b21e6b51..58be4d6b 100644 --- a/src/LexiSharp.Postgres/PostgresIndexOptions.cs +++ b/src/LexiSharp.Postgres/PostgresIndexOptions.cs @@ -25,6 +25,41 @@ public enum PostgresFieldWeight D, } +/// +/// How the lexical engine turns a multi-term query into a match. +/// +public enum PostgresMatchMode +{ + /// + /// Every term must appear (websearch_to_tsquery). The default, and the rule a search box + /// that promises "all words" implies. + /// + /// + /// Long queries pay for it: SciFact's test queries average 13.1 terms, and requiring all of + /// them on a ~500-word abstract leaves almost nothing to rank. On this repository's BEIR run + /// the rule reads 0.0188 nDCG@10 there. + /// + AllTerms, + + /// + /// Any term may appear: the query's lexemes are OR-ed into one tsquery. + /// + /// + /// + /// The lexemes come from to_tsvector over the configured text search configuration, so + /// they are the same strings the tsv column holds. It is a bag of words: the websearch + /// operators (or, -term, a trailing *) are not honoured. Quoted phrases + /// still are, because the engines split them out before they build the expression. + /// + /// + /// This changes which documents are candidates, not how they are ranked: the ordering is still + /// ts_rank_cd's coverage score, which is not BM25. Use + /// when the ranking itself has to be BM25. + /// + /// + AnyTerm, +} + /// /// Naming and behavior options for the PostgreSQL-backed index. /// @@ -88,8 +123,19 @@ public sealed record PostgresIndexOptions /// public PostgresFieldWeight DefaultFieldWeight { get; init; } = PostgresFieldWeight.D; + /// + /// How a multi-term query is matched: every term (, the + /// default) or any term (). + /// + /// + /// The mode decides the candidate set, not the ranking: results are ordered by + /// ts_rank_cd either way. Row storage does not change, so the same table serves both — + /// only the query's tsquery moves. + /// + public PostgresMatchMode MatchMode { get; init; } = PostgresMatchMode.AllTerms; + internal bool IsValid => SafeIdentifier.IsValid(Schema) && SafeIdentifier.IsValid(Table) && SafeIdentifier.IsValid(TextSearchConfig) - && IsValidWeighting; + && IsValidWeighting && Enum.IsDefined(MatchMode); /// Field names reach SQL as quoted literals, so only empty or NUL-bearing keys are refused. internal bool IsValidWeighting => IsWeightingValid(FieldWeights, DefaultFieldWeight); diff --git a/src/LexiSharp.Postgres/PostgresMatchExpression.cs b/src/LexiSharp.Postgres/PostgresMatchExpression.cs new file mode 100644 index 00000000..08867fe9 --- /dev/null +++ b/src/LexiSharp.Postgres/PostgresMatchExpression.cs @@ -0,0 +1,108 @@ +using LexiSharp.Core; +using Npgsql; + +namespace LexiSharp.Postgres; + +/// +/// The tsquery a search is matched and ranked with, built once for both stock-PostgreSQL +/// text engines. +/// +/// +/// +/// Mixed queries honour the same contract as the in-memory engine: quoted segments are a hard +/// corpus gate (each phrase must appear, AND-ed) while the free terms only contribute to scoring. +/// Gating on the whole query would wrongly require the free terms too. A query without quotes is +/// matched by . +/// +/// +/// It lives here rather than in one engine because two of them issue it now — +/// scores with ts_rank_cd, and +/// uses the same expression to choose the candidate pool it +/// then re-ranks. One copy is what keeps a query that means one thing at the gate from meaning +/// another at the rank. +/// +/// +internal sealed class PostgresMatchExpression +{ + private readonly IReadOnlyList> _parameters; + + private PostgresMatchExpression( + string gate, string rank, IReadOnlyList> parameters) + { + Gate = gate; + Rank = rank; + _parameters = parameters; + } + + /// The tsquery a row must match, as an SQL predicate on tsv. + public string Gate { get; } + + /// The expression ts_rank_cd is called with, weights included. + public string Rank { get; } + + /// Builds the expression for one query. + /// Naming, analysis and match-mode options. + /// The raw query text. + public static PostgresMatchExpression Build(PostgresIndexOptions options, string query) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(query); + + var segments = QueryParser.SplitRaw(query); + var parameters = new List>(segments.Phrases.Count + 2); + + string gate; + string rank; + + if (segments.Phrases.Count == 0) + { + string match = PostgresTsQueryText.For(options.MatchMode, options.TextSearchConfig, "@query"); + + gate = $"tsv @@ {match}"; + rank = match; + parameters.Add(new("query", query)); + } + else + { + var gates = new List(segments.Phrases.Count); + var ranks = new List(segments.Phrases.Count + 1); + + if (!string.IsNullOrWhiteSpace(segments.FreeText)) + { + ranks.Add(PostgresTsQueryText.For(options.MatchMode, options.TextSearchConfig, "@freeText")); + parameters.Add(new("freeText", segments.FreeText)); + } + + for (int i = 0; i < segments.Phrases.Count; i++) + { + gates.Add($"tsv @@ phraseto_tsquery('{options.TextSearchConfig}', unaccent(@phrase{i}))"); + ranks.Add($"phraseto_tsquery('{options.TextSearchConfig}', unaccent(@phrase{i}))"); + parameters.Add(new($"phrase{i}", segments.Phrases[i])); + } + + gate = string.Join(" AND ", gates); + + // OR the free terms with each phrase so a phrase-only match still scores above zero + // (the ts_rank_cd of the full AND query is 0 when a free term is missing). + rank = string.Join(" || ", ranks); + } + + // The weights array is the class-to-value mapping the stored letters are read through, so + // the query side of a weighted index is stated here rather than left to ts_rank_cd's + // default. Without FieldWeights the call is the one the engine has always issued. + string rankSql = PostgresSchema.RankWeightsArray(options) is { } weights + ? $"ts_rank_cd({weights}, tsv, {rank})" + : $"ts_rank_cd(tsv, {rank})"; + + return new PostgresMatchExpression(gate, rankSql, parameters); + } + + /// Adds the query's parameters to a command, under the names the expressions use. + public void Bind(NpgsqlCommand command) + { + ArgumentNullException.ThrowIfNull(command); + + foreach (var parameter in _parameters) + command.Parameters.AddWithValue(parameter.Key, parameter.Value); + } +} diff --git a/src/LexiSharp.Postgres/PostgresSchema.cs b/src/LexiSharp.Postgres/PostgresSchema.cs index 456af75a..5a9726ea 100644 --- a/src/LexiSharp.Postgres/PostgresSchema.cs +++ b/src/LexiSharp.Postgres/PostgresSchema.cs @@ -5,9 +5,9 @@ namespace LexiSharp.Postgres; /// -/// Shared DDL for the documents table. Used by the lexical (tsvector), vector -/// (pgvector ANN) and ParadeDB (pg_search BM25) engines so they can all point -/// at the same table. +/// Shared DDL for the documents table. Used by the lexical (tsvector), BM25 +/// (PostgresBm25SearchEngine), vector (pgvector ANN) and ParadeDB +/// (pg_search BM25) engines so they can all point at the same table. /// public static class PostgresSchema { diff --git a/src/LexiSharp.Postgres/PostgresTextSearchEngine.cs b/src/LexiSharp.Postgres/PostgresTextSearchEngine.cs index 394c31c3..cdc65c0e 100644 --- a/src/LexiSharp.Postgres/PostgresTextSearchEngine.cs +++ b/src/LexiSharp.Postgres/PostgresTextSearchEngine.cs @@ -1,6 +1,5 @@ using LexiSharp.Core; using Npgsql; -using NpgsqlTypes; namespace LexiSharp.Postgres; @@ -36,7 +35,15 @@ namespace LexiSharp.Postgres; /// /// Ranking uses the database's native ts_rank_cd, so scores are not numerically /// comparable to LexiSharp's BM25/TF-IDF. When identical scoring matters, wrap this engine in -/// the hybrid package's merger strategy (re-rank on the union) or recompute scores in C#. +/// the hybrid package's merger strategy (re-rank on the union), recompute scores in C#, or use +/// , which ranks the same rows with BM25 and no extension. +/// +/// +/// decides how a multi-term query becomes a +/// tsquery: (the default, and what this engine has +/// always issued) requires every term, ORs the query's +/// lexemes. The mode changes the candidate set, not the ranking — ts_rank_cd is coverage +/// based either way. /// /// /// Embeddings/ANN (pgvector) are intentionally not created here: add a @@ -212,54 +219,14 @@ public IReadOnlyList Search(string query, SearchOptions? options = // Mixed queries honor the same contract as the stock engine: quoted segments are a hard // corpus gate (each phrase must appear, AND-ed) while the free terms only contribute to - // scoring. Gating on the whole query (websearch_to_tsquery) would wrongly require the - // free terms too. A query without quotes keeps its exact previous behavior. - var segments = QueryParser.SplitRaw(query); - - string gateCondition; - string rankExpression; - - if (segments.Phrases.Count == 0) - { - gateCondition = $"tsv @@ websearch_to_tsquery('{_options.TextSearchConfig}', unaccent(@query))"; - rankExpression = $"websearch_to_tsquery('{_options.TextSearchConfig}', unaccent(@query))"; - command.Parameters.AddWithValue("query", query); - } - else - { - var gates = new List(segments.Phrases.Count); - var ranks = new List(segments.Phrases.Count + 1); - - if (!string.IsNullOrWhiteSpace(segments.FreeText)) - { - ranks.Add($"websearch_to_tsquery('{_options.TextSearchConfig}', unaccent(@freeText))"); - command.Parameters.AddWithValue("freeText", segments.FreeText); - } - - for (int i = 0; i < segments.Phrases.Count; i++) - { - gates.Add($"tsv @@ phraseto_tsquery('{_options.TextSearchConfig}', unaccent(@phrase{i}))"); - ranks.Add($"phraseto_tsquery('{_options.TextSearchConfig}', unaccent(@phrase{i}))"); - command.Parameters.AddWithValue($"phrase{i}", segments.Phrases[i]); - } - - gateCondition = string.Join(" AND ", gates); - // OR the free terms with each phrase so a phrase-only match still scores above zero - // (the ts_rank_cd of the full AND query is 0 when a free term is missing). - rankExpression = string.Join(" || ", ranks); - } - - // The weights array is the class-to-value mapping the stored letters are read through, so - // the query side of a weighted index is stated here rather than left to ts_rank_cd's - // default. Without FieldWeights the call is the one the engine has always issued. - string rank = PostgresSchema.RankWeightsArray(_options) is { } weights - ? $"ts_rank_cd({weights}, tsv, {rankExpression})" - : $"ts_rank_cd(tsv, {rankExpression})"; + // scoring. A query without quotes is matched by PostgresIndexOptions.MatchMode. + var match = PostgresMatchExpression.Build(_options, query); + match.Bind(command); string searchSql = $""" - SELECT id, content, category, fields, text_fields, {rank} AS score + SELECT id, content, category, fields, text_fields, {match.Rank} AS score FROM {_options.QualifiedTableName} - WHERE ({gateCondition}){filters.Fragment} + WHERE ({match.Gate}){filters.Fragment} ORDER BY score DESC, {PostgresSchema.OrdinalIdTieBreak} LIMIT @limit; """; @@ -305,72 +272,8 @@ public IReadOnlyList Search(string query, SearchOptions? options = /// Releases the underlying Npgsql data source. public void Dispose() => _dataSource.Dispose(); - private void Insert(NpgsqlConnection connection, NpgsqlTransaction? transaction, SearchDocument document) - { - using var command = connection.CreateCommand(); - - string tsvExpr = PostgresSchema.TsvExpression(_options, "@content", "@text_fields"); - string tsvExprUpsert = PostgresSchema.TsvExpression(_options, "EXCLUDED.content", "EXCLUDED.text_fields"); - - string insertSql = $""" - INSERT INTO {_options.QualifiedTableName} (id, content, category, fields, text_fields, tsv) - VALUES (@id, @content, @category, @fields, @text_fields, {tsvExpr}) - ON CONFLICT (id) DO UPDATE - SET content = EXCLUDED.content, - category = EXCLUDED.category, - fields = EXCLUDED.fields, - text_fields = EXCLUDED.text_fields, - tsv = {tsvExprUpsert}; - """; - - // Identifiers only are interpolated (validated [A-Za-z0-9_]+ and quoted); values are parameters. - command.CommandText = insertSql; // NOSONAR:S2077 - - if (transaction is not null) - command.Transaction = transaction; - - command.Parameters.AddWithValue("id", document.Id); - command.Parameters.AddWithValue("content", document.Text); - command.Parameters.AddWithValue("category", (object?)document.Category ?? DBNull.Value); + private void Insert(NpgsqlConnection connection, NpgsqlTransaction? transaction, SearchDocument document) => + PostgresDocumentWriter.Insert(connection, transaction, _options, document); - var fieldsParameter = command.Parameters.AddWithValue( - "fields", - document.Fields is null ? DBNull.Value : System.Text.Json.JsonSerializer.Serialize(document.Fields, PostgresJsonContext.Default.IReadOnlyDictionaryStringString)); - fieldsParameter.NpgsqlDbType = NpgsqlDbType.Jsonb; - - NpgsqlParameter textFieldsParameter; - if (document.TextFields is not null) - { - textFieldsParameter = command.Parameters.AddWithValue( - "text_fields", - System.Text.Json.JsonSerializer.Serialize(document.TextFields, PostgresJsonContext.Default.IReadOnlyDictionaryStringString)); - textFieldsParameter.NpgsqlDbType = NpgsqlDbType.Jsonb; - } - else - { - textFieldsParameter = command.Parameters.AddWithValue("text_fields", DBNull.Value); - textFieldsParameter.NpgsqlDbType = NpgsqlDbType.Jsonb; - } - - command.ExecuteNonQuery(); - } - - private static SearchDocument ReadDocument(NpgsqlDataReader reader) - { - string id = reader.GetString(0); - string content = reader.GetString(1); - string? category = reader.IsDBNull(2) ? null : reader.GetString(2); - var fields = reader.IsDBNull(3) ? null : DeserializeFields(reader.GetString(3)); - var textFields = reader.IsDBNull(4) ? null : DeserializeFields(reader.GetString(4)); - - return new SearchDocument(id, content, fields, category, textFields); - } - - private static IReadOnlyDictionary? DeserializeFields(string? json) - { - if (string.IsNullOrEmpty(json)) - return null; - - return System.Text.Json.JsonSerializer.Deserialize>(json, PostgresJsonContext.Default.DictionaryStringString); - } + private static SearchDocument ReadDocument(NpgsqlDataReader reader) => PostgresDocumentReader.Read(reader); } \ No newline at end of file diff --git a/src/LexiSharp.Postgres/PostgresTsQueryText.cs b/src/LexiSharp.Postgres/PostgresTsQueryText.cs new file mode 100644 index 00000000..f588d04b --- /dev/null +++ b/src/LexiSharp.Postgres/PostgresTsQueryText.cs @@ -0,0 +1,67 @@ +namespace LexiSharp.Postgres; + +/// +/// The two ways a raw query becomes a tsquery for the stock-PostgreSQL engines. +/// +/// +/// +/// websearch_to_tsquery is the safe, documented entry point, and it ANDs every term: +/// a document matches only if it contains all of them. That is the right rule for a search box +/// promising "all words", and the wrong one for a corpus where queries carry a dozen terms — +/// SciFact's average 13.1 terms and, at that rule, almost nothing matches. On this repository's +/// own BEIR run the difference is not a nuance: 0.0188 nDCG@10 for the AND rule against 0.662 for +/// BM25 on SciFact. +/// +/// +/// ORs the lexemes instead. The lexemes are taken from the tsvector of +/// the query, so they come from the same configuration as the indexed column, and each is emitted +/// through quote_literal because a lexeme is not necessarily a bare word: the simple +/// configuration keeps a-b as one lexeme, and an unquoted a-b in a tsquery +/// literal would be parsed as a phrase. An empty result (a query of stop words only) stays +/// NULL, so tsv @@ NULL matches nothing rather than raising. +/// +/// +/// AnyTerm is a bag of words: the websearch operators (or, -term, a trailing +/// *) are not read as operators, because the query goes through to_tsvector. Phrases +/// are unaffected — the engines split them out before they build a match expression. +/// +/// +internal static class PostgresTsQueryText +{ + /// + /// websearch_to_tsquery over the parameter: every term must appear (AND). + /// + /// Validated as [A-Za-z0-9_] by the options. + /// An SQL parameter reference such as @query. + public static string AllTerms(string textSearchConfig, string parameter) => + $"websearch_to_tsquery('{textSearchConfig}', unaccent({parameter}))"; + + /// + /// A disjunction of the query's lexemes: any term may appear (OR). + /// + /// Validated as [A-Za-z0-9_] by the options. + /// An SQL parameter reference such as @query. + public static string AnyTerm(string textSearchConfig, string parameter) => $""" + (SELECT string_agg(quote_literal(lexeme), ' | ') + FROM unnest(tsvector_to_array(to_tsvector('{textSearchConfig}', unaccent({parameter})))) AS lexeme)::tsquery + """; + + /// The expression a selects. + public static string For(PostgresMatchMode mode, string textSearchConfig, string parameter) => + mode == PostgresMatchMode.AnyTerm + ? AnyTerm(textSearchConfig, parameter) + : AllTerms(textSearchConfig, parameter); + + /// + /// The query's lexemes, as the indexed column's configuration produced them. + /// + /// + /// The C# side needs the same terms the tsvector column holds — a BM25 computed over + /// LexiSharp's tokenizer against term frequencies taken from PostgreSQL's analysis would sum + /// over terms that never meet. This is the query-side counterpart of + /// , and it is a separate statement because the term + /// list is a value of the search, not of each returned row. + /// + public static string QueryLexemes(string textSearchConfig) => + $"SELECT coalesce(tsvector_to_array(to_tsvector('{textSearchConfig}', unaccent(@query))), ARRAY[]::text[])"; +} diff --git a/src/LexiSharp.Postgres/PostgresTsvText.cs b/src/LexiSharp.Postgres/PostgresTsvText.cs new file mode 100644 index 00000000..2cf236e5 --- /dev/null +++ b/src/LexiSharp.Postgres/PostgresTsvText.cs @@ -0,0 +1,176 @@ +using System.Text; + +namespace LexiSharp.Postgres; + +/// +/// Reads the text form of a tsvector into per-lexeme frequencies and a document length. +/// +/// +/// +/// This exists because PostgreSQL exposes no set-returning function for "the term frequencies of +/// one vector": tsvector_to_array drops positions and therefore duplicates (it returns +/// {cat,fat,rat} for 'fat:2,4 cat:3 rat:5A'), and ts_stat only answers for a +/// whole table. The text form keeps the positions, and counting them is the frequency. +/// +/// +/// The grammar is the output of tsvector_out: lexemes are single-quoted, an embedded quote +/// is doubled, positions follow a colon as a comma-separated list, and a position may carry a +/// weight letter (A-D). A lexeme with no position list (a vector written by +/// strip) counts as one occurrence: the vector says the term is present, and no more. +/// +/// +/// The length is the number of positions, which is the number of indexed tokens — the quantity +/// ts_stat's nentry sums to over the table, so the per-document length and the corpus +/// average are one measurement rather than two definitions. +/// +/// +internal static class PostgresTsvText +{ + /// Parses a tsvector text value. + /// The value, as tsv::text produced it; empty for an empty vector. + /// Per-lexeme frequencies and the document length in indexed tokens. + public static (Dictionary TermFrequencies, int Length) Parse(string? text) + { + var terms = new Dictionary(StringComparer.Ordinal); + int length = 0; + + if (string.IsNullOrEmpty(text)) + return (terms, length); + + int i = 0; + + while (i < text.Length) + { + while (i < text.Length && char.IsWhiteSpace(text[i])) + i++; + + if (i >= text.Length) + break; + + string lexeme; + + if (text[i] == '\'') + { + i++; + var builder = new StringBuilder(); + + while (i < text.Length) + { + if (text[i] == '\'') + { + // A doubled quote is the escape; a single one closes the lexeme. + if (i + 1 < text.Length && text[i + 1] == '\'') + { + builder.Append('\''); + i += 2; + continue; + } + + i++; + break; + } + + builder.Append(text[i]); + i++; + } + + lexeme = builder.ToString(); + } + else + { + // Defensive: every released version quotes lexemes, but an unquoted one would + // otherwise be swallowed into the next token's positions. + int start = i; + + while (i < text.Length && text[i] != ':' && !char.IsWhiteSpace(text[i])) + i++; + + lexeme = text[start..i]; + } + + if (lexeme.Length == 0) + continue; + + int count = 1; + + if (i < text.Length && text[i] == ':') + { + i++; + count = 0; + bool digits = false; + + while (i < text.Length && !char.IsWhiteSpace(text[i])) + { + char c = text[i]; + + if (c is >= '0' and <= '9') + { + digits = true; + i++; + continue; + } + + if (c == ',') + { + if (digits) + count++; + + digits = false; + i++; + continue; + } + + // A weight letter (or anything else) qualifies the position it follows; it + // neither starts a new position nor ends the one it belongs to. + i++; + } + + if (digits) + count++; + + // A malformed list still says the term is present. + count = Math.Max(count, 1); + } + + terms[lexeme] = terms.GetValueOrDefault(lexeme) + count; + length += count; + } + + return (terms, length); + } + + /// + /// Formats a term→frequency map as a tsvector literal, for tests that need a row without + /// a server. Positions are sequential and carry no weights, which is all the BM25 mode reads. + /// + internal static string Format(IReadOnlyDictionary termFrequencies) + { + ArgumentNullException.ThrowIfNull(termFrequencies); + + var builder = new StringBuilder(); + int position = 0; + + foreach (var entry in termFrequencies.OrderBy(entry => entry.Key, StringComparer.Ordinal)) + { + if (builder.Length > 0) + builder.Append(' '); + + builder.Append('\'').Append(entry.Key.Replace("'", "''", StringComparison.Ordinal)).Append('\''); + + if (entry.Value > 0) + { + builder.Append(':'); + + for (int i = 0; i < entry.Value; i++) + { + if (i > 0) + builder.Append(','); + + builder.Append(++position); + } + } + } + + return builder.ToString(); + } +} diff --git a/tests/LexiSharp.Tests/Conformance/PostgresBm25SearchEngineConformanceTests.cs b/tests/LexiSharp.Tests/Conformance/PostgresBm25SearchEngineConformanceTests.cs new file mode 100644 index 00000000..aacab815 --- /dev/null +++ b/tests/LexiSharp.Tests/Conformance/PostgresBm25SearchEngineConformanceTests.cs @@ -0,0 +1,36 @@ +using LexiSharp.Core; +using LexiSharp.Postgres; + +namespace LexiSharp.Tests.Conformance; + +public class PostgresBm25SearchEngineConformanceTests : SearchEngineConformanceTests +{ + private static string? ConnectionString => + Environment.GetEnvironmentVariable("POSTGRES_TEST_CONNECTION"); + + protected override bool IsUnavailable => ConnectionString is null; + + protected override string UnavailableReason => "POSTGRES_TEST_CONNECTION not set."; + + protected override ITextSearchEngine CreateEngine() + { + var suffix = Guid.NewGuid().ToString("N")[..8]; + return new PostgresBm25SearchEngine( + ConnectionString!, + new PostgresBm25Options { Table = $"lexisharp_bm25_conformance_{suffix}" }); + } + + protected override void DestroyEngine(ITextSearchEngine engine) + { + var postgres = (PostgresBm25SearchEngine)engine; + + try + { + postgres.DropSchema(); + } + finally + { + postgres.Dispose(); + } + } +} diff --git a/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs b/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs new file mode 100644 index 00000000..923f7e12 --- /dev/null +++ b/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs @@ -0,0 +1,301 @@ +using LexiSharp.Core; +using LexiSharp.Indexing; +using LexiSharp.Postgres; +using LexiSharp.Ranking; +using Xunit; + +namespace LexiSharp.Tests; + +/// +/// The stock-PostgreSQL BM25 engine: portability (no extension beyond unaccent), the score +/// itself, and the two things it does differently from every other SQL backend — a cached corpus +/// snapshot and a bounded candidate pool. +/// +public class PostgresBm25SearchEngineTests +{ + private static string? ConnectionString => + Environment.GetEnvironmentVariable("POSTGRES_TEST_CONNECTION"); + + private static string Table(string suffix) => $"lexisharp_bm25_{suffix}_{Guid.NewGuid():N}"[..44]; + + private static PostgresBm25SearchEngine NewEngine(string suffix, Action? configure = null) + { + var options = new PostgresBm25Options { Table = Table(suffix) }; + configure?.Invoke(options); + return new PostgresBm25SearchEngine(ConnectionString!, options); + } + + private static void Run(string suffix, Action body, Action? configure = null) + { + using var engine = NewEngine(suffix, configure); + + try + { + body(engine); + } + finally + { + engine.DropSchema(); + } + } + + /// + /// Plain lowercase ASCII, so PostgreSQL's simple analysis and LexiSharp's tokenizer + /// produce the same terms — which is what makes the two scorers comparable at all. + /// + private static readonly SearchDocument[] Corpus = + [ + new("d1", "quick brown fox"), + new("d2", "quick quick brown brown brown"), + new("d3", "lazy dog sleeps"), + new("d4", "quick thinking saves the day"), + new("d5", "unrelated content here"), + ]; + + [SkippableFact] + public void Search_ScoresWhatTheInMemoryBm25Scores() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("parity", engine => + { + engine.Index(Corpus); + + var index = new InMemoryTextIndex(); + index.Index(Corpus); + + var scorer = new Bm25Scorer(1.5, 0.75); + var terms = index.Tokenizer.Tokenize("quick brown"); + + var results = engine.Search("quick brown", new SearchOptions(Limit: 10)); + + Assert.Equal( + results.Select(result => result.DocumentId), + results.OrderByDescending(result => result.Score) + .ThenBy(result => result.DocumentId, StringComparer.Ordinal) + .Select(result => result.DocumentId)); + + foreach (SearchResult result in results) + { + double expected = scorer.Score(result.DocumentId, terms, index); + + Assert.InRange(result.Score, expected - 1e-9, expected + 1e-9); + } + + // d3 and d5 share no term with the query: BM25 scores them 0, so they are absent. + Assert.Equal(3, results.Count); + }); + } + + [SkippableFact] + public void Search_IsAccentInsensitive() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("accents", engine => + { + engine.Add(new SearchDocument("cafe", "un café à Paris")); + + Assert.Equal("cafe", engine.Search("CAFE").Single().DocumentId); + }); + } + + [SkippableFact] + public void Search_EmptyQueryOrEmptyTableReturnsNothing() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("empty", engine => + { + Assert.Empty(engine.Search("quick")); + + engine.Index(Corpus); + + Assert.Empty(engine.Search(" ")); + Assert.Empty(engine.Search("quantum")); + }); + } + + [SkippableFact] + public void Search_HonoursAQuotedPhraseAsAGate() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("phrase", engine => + { + engine.Index(Corpus); + + Assert.Equal( + "d1", + engine.Search("\"brown fox\"").Single().DocumentId); + + Assert.Empty(engine.Search("\"fox brown\"")); + }); + } + + [SkippableFact] + public void Search_HonoursMetadataFilters() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("filters", engine => + { + engine.Index( + [ + new("article", "machine learning systems", + new Dictionary { ["kind"] = "article" }), + new("note", "machine learning notes", + new Dictionary { ["kind"] = "note" }), + ]); + + var results = engine.Search("machine learning", new SearchOptions( + Limit: 10, + Filters: [new MetadataFilter("kind", MetadataFilterOperator.Equal, "note")])); + + Assert.Equal("note", Assert.Single(results).DocumentId); + }); + } + + [SkippableFact] + public void Search_ExcludesTheDocumentAndStillFillsThePage() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("excluded", engine => + { + engine.Index( + new[] { "d1", "d2", "d3", "d4" } + .Select(id => new SearchDocument(id, "shared common term"))); + + var all = engine.Search("shared common term", new SearchOptions(Limit: 10)); + Assert.Equal(4, all.Count); + + var after = engine.Search("shared common term", new SearchOptions( + Limit: 10, + ExcludedDocumentIds: new HashSet(StringComparer.Ordinal) { all[0].DocumentId })); + + Assert.DoesNotContain(after, result => result.DocumentId == all[0].DocumentId); + Assert.Equal(3, after.Count); + }); + } + + [SkippableFact] + public void Add_InvalidatesTheStatisticsSnapshot() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("invalidate", engine => + { + engine.Index(Corpus); + + double before = engine.Search("quick").Single(result => result.DocumentId == "d1").Score; + + engine.Add(new SearchDocument("d6", "quick")); + + var after = engine.Search("quick", new SearchOptions(Limit: 10)); + + // The new document is a candidate immediately (the SQL reads the table), and the idf + // moved because the snapshot was rebuilt: N grew from 4 to 5. + Assert.Contains(after, result => result.DocumentId == "d6"); + Assert.NotEqual(before, after.Single(result => result.DocumentId == "d1").Score); + }); + } + + [SkippableFact] + public void Remove_InvalidatesTheStatisticsSnapshot() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("remove", engine => + { + engine.Index(Corpus); + + Assert.True(engine.Remove("d2")); + Assert.DoesNotContain(engine.Search("quick", new SearchOptions(Limit: 10)), + result => result.DocumentId == "d2"); + + Assert.False(engine.Remove("d2")); + }); + } + + [SkippableFact] + public void RefreshStatistics_PicksUpAWriteIssuedByAnotherEngine() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + string table = Table("refresh"); + + using var bm25 = new PostgresBm25SearchEngine( + ConnectionString!, new PostgresBm25Options { Table = table }); + + using var lexical = new PostgresTextSearchEngine( + ConnectionString!, new PostgresIndexOptions { Table = table }); + + try + { + bm25.Index([new SearchDocument("d1", "alpha")]); + Assert.Equal(1, bm25.RefreshStatistics()); + + // A write through the lexical engine on the shared table invalidates nothing here: the + // snapshot is this engine's, not the table's. + lexical.Add(new SearchDocument("d2", "beta")); + + Assert.Equal(2, bm25.RefreshStatistics()); + + // Candidates were never stale — that is the live SQL — so the document is found either + // way; what the refresh restores is its idf. + Assert.Equal("d2", bm25.Search("beta").Single().DocumentId); + } + finally + { + bm25.DropSchema(); + } + } + + [SkippableFact] + public void Index_ReplacesTheCorpus() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("reindex", engine => + { + engine.Index(Corpus); + engine.Index([new SearchDocument("only", "quick brown fox")]); + + var results = engine.Search("quick brown", new SearchOptions(Limit: 10)); + + Assert.Equal("only", Assert.Single(results).DocumentId); + }); + } + + [Theory] + [InlineData(1000, 10, 0, 1000)] + [InlineData(1000, 10, 2000, 2010)] + [InlineData(5, 10, 0, 10)] + [InlineData(0, 10, 0, 100_000)] + [InlineData(1000, 10, int.MaxValue, 100_000)] + public void PoolSize_WidensToThePageAndCapsAtTheCeiling( + int candidatePool, int limit, int offset, int expected) + { + var options = new PostgresBm25Options { CandidatePool = candidatePool }; + + int pool = PostgresBm25SearchEngine.PoolSize(options, new SearchOptions(Limit: limit, Offset: offset)); + + Assert.Equal(expected, pool); + } + + [Fact] + public void Constructor_RefusesInvalidRankingParameters() + { + const string connection = "Host=localhost;Database=unused"; + + Assert.Throws(() => new PostgresBm25SearchEngine( + connection, new PostgresBm25Options { K1 = -1 }, autoCreateSchema: false)); + + Assert.Throws(() => new PostgresBm25SearchEngine( + connection, new PostgresBm25Options { B = 1.5 }, autoCreateSchema: false)); + + Assert.Throws(() => new PostgresBm25SearchEngine( + connection, new PostgresBm25Options { CandidatePool = -1 }, autoCreateSchema: false)); + } +} diff --git a/tests/LexiSharp.Tests/PostgresMatchModeTests.cs b/tests/LexiSharp.Tests/PostgresMatchModeTests.cs new file mode 100644 index 00000000..d84ecbf4 --- /dev/null +++ b/tests/LexiSharp.Tests/PostgresMatchModeTests.cs @@ -0,0 +1,157 @@ +using LexiSharp.Core; +using LexiSharp.Postgres; +using Xunit; + +namespace LexiSharp.Tests; + +/// +/// , against a real server. The default (every term) +/// is what the engine always did; changes which documents +/// are candidates, and these tests are what keeps it from changing anything else. +/// +public class PostgresMatchModeTests +{ + private static string? ConnectionString => + Environment.GetEnvironmentVariable("POSTGRES_TEST_CONNECTION"); + + private static string Table(string suffix) => $"lexisharp_match_{suffix}_{Guid.NewGuid():N}"[..44]; + + private static PostgresTextSearchEngine NewEngine(PostgresMatchMode mode, string suffix) + { + return new PostgresTextSearchEngine( + ConnectionString!, + new PostgresIndexOptions { Table = Table(suffix), MatchMode = mode }); + } + + private static void Run(PostgresMatchMode mode, string suffix, Action body) + { + using var engine = NewEngine(mode, suffix); + + try + { + body(engine); + } + finally + { + engine.DropSchema(); + } + } + + private static readonly SearchDocument[] Corpus = + [ + new("d1", "neural machine translation"), + new("d2", "pasta recipe with tomato"), + ]; + + [SkippableFact] + public void AllTerms_RequiresEveryTerm() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run(PostgresMatchMode.AllTerms, "all", engine => + { + engine.Index(Corpus); + + Assert.Empty(engine.Search("neural tomato")); + Assert.Single(engine.Search("neural translation")); + }); + } + + [SkippableFact] + public void AnyTerm_MatchesADocumentHoldingOneOfTheTerms() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run(PostgresMatchMode.AnyTerm, "any", engine => + { + engine.Index(Corpus); + + var results = engine.Search("neural tomato", new SearchOptions(Limit: 10)); + + Assert.Equal(["d1", "d2"], results.Select(result => result.DocumentId).Order().ToArray()); + }); + } + + [SkippableFact] + public void AnyTerm_StillHonoursAQuotedPhraseAsAGate() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run(PostgresMatchMode.AnyTerm, "phrase", engine => + { + engine.Index(Corpus); + + // The phrase is a hard gate in both modes; the free term beside it only scores, so a + // document holding only the free term ("tomato") does not become a result. + var mixed = engine.Search("\"machine translation\" tomato", new SearchOptions(Limit: 10)); + + Assert.Equal("d1", Assert.Single(mixed).DocumentId); + Assert.Empty(engine.Search("\"machine learning\" tomato")); + Assert.Equal("d1", engine.Search("\"machine translation\"").Single().DocumentId); + }); + } + + [SkippableFact] + public void AnyTerm_IsAccentInsensitiveAndCaseInsensitive() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run(PostgresMatchMode.AnyTerm, "accents", engine => + { + engine.Add(new SearchDocument("cafe", "un café à Paris")); + + Assert.Equal("cafe", engine.Search("CAFE").Single().DocumentId); + }); + } + + [SkippableFact] + public void AnyTerm_OnAQueryOfStopWordsReturnsNothing() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + using var engine = new PostgresTextSearchEngine( + ConnectionString!, + new PostgresIndexOptions + { + Table = Table("stop"), + MatchMode = PostgresMatchMode.AnyTerm, + TextSearchConfig = "english", + }); + + try + { + engine.Add(new SearchDocument("d1", "the quick brown fox")); + + // All stop words: the tsquery is NULL rather than an error, and nothing matches. + Assert.Empty(engine.Search("the of and")); + } + finally + { + engine.DropSchema(); + } + } + + [SkippableFact] + public void AnyTerm_KeepsTheRankingMonotoneInTermOverlap() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run(PostgresMatchMode.AnyTerm, "rank", engine => + { + engine.Index( + [ + new("one", "neural related"), + new("two", "neural translation"), + new("three", "unrelated words entirely"), + ]); + + var results = engine.Search("neural translation", new SearchOptions(Limit: 3)); + + // Same length, so coverage decides: the document matching both terms outranks the one + // matching a single term, and the one matching neither is absent. + Assert.Equal("two", results[0].DocumentId); + Assert.Equal("one", results[1].DocumentId); + Assert.DoesNotContain(results, result => result.DocumentId == "three"); + }); + } +} diff --git a/tests/LexiSharp.Tests/PostgresTsvTextTests.cs b/tests/LexiSharp.Tests/PostgresTsvTextTests.cs new file mode 100644 index 00000000..fc69ba9b --- /dev/null +++ b/tests/LexiSharp.Tests/PostgresTsvTextTests.cs @@ -0,0 +1,102 @@ +using LexiSharp.Postgres; +using Xunit; + +namespace LexiSharp.Tests; + +/// +/// The tsvector text reader the BM25 engine scores from. No server: the input is the exact +/// string tsvector_out produces, and the expected numbers are what BM25's tf and +/// |d| have to be for that vector. +/// +public class PostgresTsvTextTests +{ + [Fact] + public void Parse_CountsEveryPositionAsOneOccurrence() + { + var (terms, length) = PostgresTsvText.Parse("'fat':2,4 'cat':3 'rat':5A"); + + Assert.Equal(2, terms["fat"]); + Assert.Equal(1, terms["cat"]); + Assert.Equal(1, terms["rat"]); + Assert.Equal(4, length); + } + + [Fact] + public void Parse_TreatsWeightLettersAsPartOfThePosition() + { + var (terms, length) = PostgresTsvText.Parse("'a':1A,2B 'b':3C 'c':4D"); + + Assert.Equal(2, terms["a"]); + Assert.Equal(1, terms["b"]); + Assert.Equal(1, terms["c"]); + Assert.Equal(4, length); + } + + [Fact] + public void Parse_ReadsALexemeWithNoPositionsAsOneOccurrence() + { + // A vector written with strip(): presence without positions. One is the only count the + // vector can support, and reading it as zero would make the term disappear from the score. + var (terms, length) = PostgresTsvText.Parse("'quick' 'brown'"); + + Assert.Equal(1, terms["quick"]); + Assert.Equal(1, terms["brown"]); + Assert.Equal(2, length); + } + + [Fact] + public void Parse_UnescapesADoubledQuoteInsideALexeme() + { + var (terms, _) = PostgresTsvText.Parse("'o''brien':1 's':2"); + + Assert.Equal(1, terms["o'brien"]); + Assert.Equal(1, terms["s"]); + } + + [Fact] + public void Parse_ReadsALexemeContainingASpace() + { + // A custom dictionary can emit a multi-word lexeme, and the positions still count. + var (terms, length) = PostgresTsvText.Parse("'new york':1,7 'city':2"); + + Assert.Equal(2, terms["new york"]); + Assert.Equal(1, terms["city"]); + Assert.Equal(3, length); + } + + [Fact] + public void Parse_ReturnsNothingForAnEmptyVector() + { + var (terms, length) = PostgresTsvText.Parse(string.Empty); + + Assert.Empty(terms); + Assert.Equal(0, length); + } + + [Fact] + public void Parse_UsesTheOrdinalComparer() + { + // 'A' and 'a' are different lexemes to tsvector and must stay different to the scorer. + var (terms, _) = PostgresTsvText.Parse("'A':1 'a':2"); + + Assert.Equal(2, terms.Count); + Assert.Equal(1, terms["A"]); + Assert.Equal(1, terms["a"]); + } + + [Fact] + public void Format_RoundTripsThroughParse() + { + var frequencies = new Dictionary(StringComparer.Ordinal) + { + ["quick"] = 3, + ["brown"] = 1, + ["o'brien"] = 2, + }; + + var (terms, length) = PostgresTsvText.Parse(PostgresTsvText.Format(frequencies)); + + Assert.Equal(frequencies, terms); + Assert.Equal(6, length); + } +} diff --git a/tests/LexiSharp.Tests/PublicApiBaseline.txt b/tests/LexiSharp.Tests/PublicApiBaseline.txt index e4674cec..dc2e4465 100644 --- a/tests/LexiSharp.Tests/PublicApiBaseline.txt +++ b/tests/LexiSharp.Tests/PublicApiBaseline.txt @@ -216,6 +216,8 @@ C|LexiSharp.Onnx.SentencePieceOnnxTokenizerOptions|Void .ctor() C|LexiSharp.Onnx.SentencePieceOnnxTokenizer|Void .ctor(System.String, LexiSharp.Onnx.SentencePieceOnnxTokenizerOptions) C|LexiSharp.ParadeDB.ParadeDBOptions|Void .ctor() C|LexiSharp.ParadeDB.ParadeDBTextSearchEngine|Void .ctor(System.String, LexiSharp.ParadeDB.ParadeDBOptions, System.Nullable`1[System.Boolean]) +C|LexiSharp.Postgres.PostgresBm25Options|Void .ctor() +C|LexiSharp.Postgres.PostgresBm25SearchEngine|Void .ctor(System.String, LexiSharp.Postgres.PostgresBm25Options, System.Nullable`1[System.Boolean]) C|LexiSharp.Postgres.PostgresFuzzyOptions|Void .ctor() C|LexiSharp.Postgres.PostgresFuzzySearchEngine|Void .ctor(System.String, LexiSharp.Postgres.PostgresFuzzyOptions, System.Nullable`1[System.Boolean]) C|LexiSharp.Postgres.PostgresIndexOptions|Void .ctor() @@ -397,10 +399,16 @@ F|LexiSharp.Onnx.OnnxPooling|Mean|LexiSharp.Onnx.OnnxPooling|static|1 F|LexiSharp.Onnx.OnnxScoreTransform|None|LexiSharp.Onnx.OnnxScoreTransform|static|0 F|LexiSharp.Onnx.OnnxScoreTransform|Sigmoid|LexiSharp.Onnx.OnnxScoreTransform|static|1 F|LexiSharp.Onnx.OnnxScoreTransform|Softmax|LexiSharp.Onnx.OnnxScoreTransform|static|2 +F|LexiSharp.Postgres.PostgresBm25Options|DefaultB|System.Double|static|0.75 +F|LexiSharp.Postgres.PostgresBm25Options|DefaultCandidatePool|System.Int32|static|1000 +F|LexiSharp.Postgres.PostgresBm25Options|DefaultK1|System.Double|static|1.5 +F|LexiSharp.Postgres.PostgresBm25Options|MaximumCandidatePool|System.Int32|static|100000 F|LexiSharp.Postgres.PostgresFieldWeight|A|LexiSharp.Postgres.PostgresFieldWeight|static|0 F|LexiSharp.Postgres.PostgresFieldWeight|B|LexiSharp.Postgres.PostgresFieldWeight|static|1 F|LexiSharp.Postgres.PostgresFieldWeight|C|LexiSharp.Postgres.PostgresFieldWeight|static|2 F|LexiSharp.Postgres.PostgresFieldWeight|D|LexiSharp.Postgres.PostgresFieldWeight|static|3 +F|LexiSharp.Postgres.PostgresMatchMode|AllTerms|LexiSharp.Postgres.PostgresMatchMode|static|0 +F|LexiSharp.Postgres.PostgresMatchMode|AnyTerm|LexiSharp.Postgres.PostgresMatchMode|static|1 F|LexiSharp.Postgres.SparseDistance|Cosine|LexiSharp.Postgres.SparseDistance|static|1 F|LexiSharp.Postgres.SparseDistance|InnerProduct|LexiSharp.Postgres.SparseDistance|static|0 F|LexiSharp.Postgres.SparseDistance|L1|LexiSharp.Postgres.SparseDistance|static|3 @@ -1625,6 +1633,20 @@ M|LexiSharp.ParadeDB.ParadeDBTextSearchEngine|Void Index(System.Collections.Gene M|LexiSharp.PathSafety|Boolean IsSafe(System.String) M|LexiSharp.PathSafety|Void ThrowIfUnsafe(System.String, System.String) M|LexiSharp.Postgres.IColumnAwareEmbeddingProvider|System.Threading.Tasks.Task`1[System.ReadOnlyMemory`1[System.Single]] GetTextEmbeddingAsync(System.String, LexiSharp.Core.EmbeddingUse, System.String, System.Threading.CancellationToken) +M|LexiSharp.Postgres.PostgresBm25Options|Boolean Equals(LexiSharp.Postgres.PostgresBm25Options) +M|LexiSharp.Postgres.PostgresBm25Options|Boolean Equals(System.Object) +M|LexiSharp.Postgres.PostgresBm25Options|Int32 GetHashCode() +M|LexiSharp.Postgres.PostgresBm25Options|System.String ToString() +M|LexiSharp.Postgres.PostgresBm25SearchEngine|Boolean Remove(System.String) +M|LexiSharp.Postgres.PostgresBm25SearchEngine|Int32 RefreshStatistics() +M|LexiSharp.Postgres.PostgresBm25SearchEngine|System.Collections.Generic.IReadOnlyList`1[LexiSharp.Core.SearchResult] Search(System.String, LexiSharp.Core.SearchOptions, System.Threading.CancellationToken) +M|LexiSharp.Postgres.PostgresBm25SearchEngine|System.Threading.Tasks.Task EnsureSchemaAsync(System.Threading.CancellationToken) +M|LexiSharp.Postgres.PostgresBm25SearchEngine|Void Add(LexiSharp.Core.SearchDocument) +M|LexiSharp.Postgres.PostgresBm25SearchEngine|Void Clear() +M|LexiSharp.Postgres.PostgresBm25SearchEngine|Void Dispose() +M|LexiSharp.Postgres.PostgresBm25SearchEngine|Void DropSchema() +M|LexiSharp.Postgres.PostgresBm25SearchEngine|Void EnsureSchema() +M|LexiSharp.Postgres.PostgresBm25SearchEngine|Void Index(System.Collections.Generic.IEnumerable`1[LexiSharp.Core.SearchDocument]) M|LexiSharp.Postgres.PostgresFuzzyOptions|Boolean Equals(LexiSharp.Postgres.PostgresFuzzyOptions) M|LexiSharp.Postgres.PostgresFuzzyOptions|Boolean Equals(System.Object) M|LexiSharp.Postgres.PostgresFuzzyOptions|Int32 GetHashCode() @@ -2652,6 +2674,18 @@ P|LexiSharp.ParadeDB.ParadeDBOptions|QualifiedTableName|System.String|get P|LexiSharp.ParadeDB.ParadeDBOptions|Schema|System.String|getset P|LexiSharp.ParadeDB.ParadeDBOptions|Table|System.String|getset P|LexiSharp.ParadeDB.ParadeDBTextSearchEngine|SupportedQueryFeatures|LexiSharp.Core.QueryFeatures|get +P|LexiSharp.Postgres.PostgresBm25Options|AutoCreateSchema|System.Boolean|getset +P|LexiSharp.Postgres.PostgresBm25Options|B|System.Double|getset +P|LexiSharp.Postgres.PostgresBm25Options|CandidatePool|System.Int32|getset +P|LexiSharp.Postgres.PostgresBm25Options|DefaultFieldWeight|LexiSharp.Postgres.PostgresFieldWeight|getset +P|LexiSharp.Postgres.PostgresBm25Options|FieldWeights|System.Collections.Generic.IReadOnlyDictionary|getset +P|LexiSharp.Postgres.PostgresBm25Options|K1|System.Double|getset +P|LexiSharp.Postgres.PostgresBm25Options|MatchMode|LexiSharp.Postgres.PostgresMatchMode|getset +P|LexiSharp.Postgres.PostgresBm25Options|QualifiedTableName|System.String|get +P|LexiSharp.Postgres.PostgresBm25Options|Schema|System.String|getset +P|LexiSharp.Postgres.PostgresBm25Options|Table|System.String|getset +P|LexiSharp.Postgres.PostgresBm25Options|TextSearchConfig|System.String|getset +P|LexiSharp.Postgres.PostgresBm25SearchEngine|SupportedQueryFeatures|LexiSharp.Core.QueryFeatures|get P|LexiSharp.Postgres.PostgresFuzzyOptions|AutoCreateSchema|System.Boolean|getset P|LexiSharp.Postgres.PostgresFuzzyOptions|ContentField|System.String|getset P|LexiSharp.Postgres.PostgresFuzzyOptions|IncludePhonetic|System.Boolean|getset @@ -2667,6 +2701,7 @@ P|LexiSharp.Postgres.PostgresFuzzySearchEngine|SupportedQueryFeatures|LexiSharp. P|LexiSharp.Postgres.PostgresIndexOptions|AutoCreateSchema|System.Boolean|getset P|LexiSharp.Postgres.PostgresIndexOptions|DefaultFieldWeight|LexiSharp.Postgres.PostgresFieldWeight|getset P|LexiSharp.Postgres.PostgresIndexOptions|FieldWeights|System.Collections.Generic.IReadOnlyDictionary|getset +P|LexiSharp.Postgres.PostgresIndexOptions|MatchMode|LexiSharp.Postgres.PostgresMatchMode|getset P|LexiSharp.Postgres.PostgresIndexOptions|QualifiedTableName|System.String|get P|LexiSharp.Postgres.PostgresIndexOptions|Schema|System.String|getset P|LexiSharp.Postgres.PostgresIndexOptions|Table|System.String|getset @@ -3137,10 +3172,13 @@ T|LexiSharp.ParadeDB.ParadeDBOptions|class||System.IEquatable +T|LexiSharp.Postgres.PostgresBm25SearchEngine|class||LexiSharp.Core.IQuerySyntaxSupport,LexiSharp.Core.ITextSearchEngine,System.IDisposable T|LexiSharp.Postgres.PostgresFieldWeight|enum||System.IComparable,System.IConvertible,System.IFormattable,System.ISpanFormattable T|LexiSharp.Postgres.PostgresFuzzyOptions|class||System.IEquatable T|LexiSharp.Postgres.PostgresFuzzySearchEngine|class||LexiSharp.Core.IQuerySyntaxSupport,LexiSharp.Core.ITextSearchEngine,System.IDisposable T|LexiSharp.Postgres.PostgresIndexOptions|class||System.IEquatable +T|LexiSharp.Postgres.PostgresMatchMode|enum||System.IComparable,System.IConvertible,System.IFormattable,System.ISpanFormattable T|LexiSharp.Postgres.PostgresSchema|class|| T|LexiSharp.Postgres.PostgresSparseOptions|class||System.IEquatable T|LexiSharp.Postgres.PostgresSparseSearchEngine|class||LexiSharp.Core.IQuerySyntaxSupport,LexiSharp.Core.ITextSearchEngine,System.IDisposable From c739d4cccef520f11c81683ad319b9981adc0210 Mon Sep 17 00:00:00 2001 From: manuc66 Date: Mon, 16 Nov 2026 08:25:04 +0100 Subject: [PATCH 2/6] Measure the stock-PostgreSQL engines and read BM25 candidates in two phases The quality harness reports no latency on purpose, so the portable BM25's cost was a claim -- "bounded by the pool" -- without a number. A `postgres` lane measures it: index build, ts_stat build and p50/p95/p99/mean per engine and per pool size, interleaved per query so a pool size does not inherit the previous row's cache state. Measured on NFCorpus (3,633 documents) and SciFact (5,183), one process, one thread, a pooled data source, a page of 10: corpus engine stats p50 p95 mean NFCorpus ts_rank_cd all - 0.6 1.3 0.7 NFCorpus ts_rank_cd any - 4.1 74.6 21.3 NFCorpus bm25 pool 1,000 220ms 16.0 107.3 37.3 NFCorpus bm25 pool 0 201ms 15.8 177.1 58.1 SciFact ts_rank_cd all - 0.7 0.8 0.7 SciFact ts_rank_cd any - 170.1 347.2 177.6 SciFact bm25 pool 1,000 252ms 203.3 385.2 210.7 SciFact bm25 pool 0 246ms 299.7 490.0 301.0 The candidate selection dominates, not the BM25 arithmetic: on SciFact the lexical any-term row alone is 170 ms, because a thirteen-term query matches almost the whole corpus and ts_rank_cd is computed for every match before the pool cut. That is the price of the recall the rule buys, and it is published rather than absorbed. The pool is mostly a quality knob: 500 candidates cost 0.005 nDCG@10 on NFCorpus and 0.077 on SciFact and barely move the median, so 1,000 stays the default. Two changes make the BM25 rows cheaper. The candidate query ranks ids and carries one tsvector per candidate; the page's documents are fetched afterwards by id, so a thousand candidates no longer drag a thousand contents and jsonb rows through a sort PostgreSQL answered by spilling to disk. The per-candidate reader keeps only the query's terms and counts every position for the length without allocating the skipped lexemes. On NFCorpus, pool 1,000: 26.9 -> 16.0 ms at the median and 63.7 -> 37.3 ms at the mean. `--keep` leaves the benchmark table in place for EXPLAIN. --- CHANGELOG.md | 14 + README.md | 6 +- bench/LexiSharp.Benchmarks/PostgresLane.cs | 373 ++++++++++++++++++ bench/LexiSharp.Benchmarks/Program.cs | 7 + docs/backends.md | 37 ++ docs/benchmarks.md | 61 +++ .../PostgresBm25IndexView.cs | 26 +- .../PostgresBm25SearchEngine.cs | 103 ++++- src/LexiSharp.Postgres/PostgresTsvText.cs | 153 +++++-- tests/LexiSharp.Tests/PostgresTsvTextTests.cs | 23 ++ 10 files changed, 734 insertions(+), 69 deletions(-) create mode 100644 bench/LexiSharp.Benchmarks/PostgresLane.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index cd2edc78..181aa4d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,13 @@ `bench/LexiSharp.Eval`, so the portable BM25 row and the match rule are measured by the same harness, corpus and qrels as every other row. +- **`postgres` lane in `bench/LexiSharp.Benchmarks`**: index build, statistics build and search + latency (p50/p95/p99/mean) of the stock-PostgreSQL text engines on the BEIR corpora, per match + rule and per candidate-pool size, interleaved so one row does not inherit another's cache state. + It is the measurement the quality harness deliberately does not take. On NFCorpus the pool-1,000 + BM25 row reads 16.0 ms p50 / 37.3 ms mean; on SciFact 203.3 / 210.7 ms, of which the OR + candidate selection alone is 170.1 / 177.6 ms. `--keep` leaves the table for `EXPLAIN`. + ### Changed - The shared documents table's upsert and row reader are now one implementation @@ -33,6 +40,13 @@ `tsquery` a search is matched and ranked with is built once (`PostgresMatchExpression`). A query that means one thing at the gate cannot mean another at the rank. +- **The BM25 candidate read is two-phase.** The candidate query ranks ids and returns one + `tsvector` per candidate; the page's documents are fetched afterwards, by id. Carrying content, + category and jsonb through the sort made PostgreSQL spill it to disk. The per-candidate + `tsvector` reader also keeps only the query's terms now, while still counting every position for + the document length. On NFCorpus the pool-1,000 row moved from 26.9 ms to 16.0 ms at the median + and from 63.7 ms to 37.3 ms at the mean. + ### Fixed - **The evaluation harness measured a rounded run in the engine's tie-break, not in the run reader's.** diff --git a/README.md b/README.md index 32d24a42..7474d994 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,10 @@ linear classifier): [text analysis →](docs/text-analysis.md). supported but unproven until tested on your data. - The stock-PostgreSQL BM25 re-ranks a bounded candidate pool (1,000 by default): the pool bounds recall, and on SciFact it costs 0.10 nDCG@10 against the exact ranking. `CandidatePool = 0` - removes the bound. The ParadeDB engine needs `pg_search`, which no major managed service lets you - install; the `tsvector`, BM25 and `pg_trgm` engines do not. + removes the bound. It is also the slow path where `pg_search` cannot run: selecting the pool is + the cost, and on a 5,000-document corpus with 13-term queries a search reads ~200 ms p50. The + ParadeDB engine needs `pg_search`, which no major managed service lets you install; the + `tsvector`, BM25 and `pg_trgm` engines do not. - SQL backends are measured on ranking for the engines that rank on their own; sparse and fuzzy engines are covered by integration tests only. diff --git a/bench/LexiSharp.Benchmarks/PostgresLane.cs b/bench/LexiSharp.Benchmarks/PostgresLane.cs new file mode 100644 index 00000000..e8d07460 --- /dev/null +++ b/bench/LexiSharp.Benchmarks/PostgresLane.cs @@ -0,0 +1,373 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using LexiSharp.Core; +using LexiSharp.Eval; +using LexiSharp.Postgres; + +namespace LexiSharp.Benchmarks; + +/// +/// What a search costs on the stock-PostgreSQL text engines: index build, corpus-statistics build, +/// and the query latency distribution — ts_rank_cd against the portable BM25 at several +/// candidate-pool sizes. +/// +/// +/// +/// The evaluation harness deliberately reports no latency: its number would be dominated by +/// whatever connection shape its lane happens to have, and a metric table that mixes quality with a +/// deployment detail invites the wrong comparison. This lane is the other question, asked on +/// purpose. It measures one process, one thread, one query at a time, against a server on the same +/// host, with the engine's own pooled NpgsqlDataSource warmed — the shape a request path +/// has, not a connection handshake per call. +/// +/// +/// What the pool costs, in milliseconds. The portable BM25 selects its candidates with +/// ts_rank_cd and re-ranks the top CandidatePool; the pool is a recall bound, and the +/// quality page states what it costs in nDCG. Here is the other side of that choice: the same +/// corpus, the same queries, at pool 1,000 / 5,000 / unbounded. A pool that looks cheap in nDCG +/// and expensive here (or the reverse) is why both measurements exist. +/// +/// +/// The statistics build is not a search. ts_stat walks the table, so the first search +/// after a write pays for the whole corpus. It is measured by calling +/// explicitly, printed in its own column, +/// and excluded from the query distribution — a caller that writes between searches pays it, and +/// should see the number rather than infer it from a p99. +/// +/// +internal static class PostgresLane +{ + /// Where the BEIR corpora live, relative to the repository root and to any directory above the assembly. + private const string DefaultDataDirectory = "bench/LexiSharp.Eval/data"; + + private static readonly string[] DefaultDatasets = ["nfcorpus", "scifact"]; + + private static long _sink; + + public static void Run(string[] args) + { + Options options = Parse(args); + + PrintHeader(options); + + var dropper = new List<(string Table, string Connection)>(); + + try + { + foreach (string dataset in options.Datasets) + Measure(dataset, options, dropper); + } + finally + { + if (options.Keep) + { + foreach (var (table, _) in dropper) + Console.WriteLine($"kept {table} (not dropped)"); + } + else + { + foreach (var (table, connection) in dropper) + { + using var engine = new PostgresTextSearchEngine( + connection, + new PostgresIndexOptions { Table = table }, + autoCreateSchema: false); + + engine.DropSchema(); + } + } + } + } + + private static void Measure(string datasetName, Options options, List<(string, string)> dropper) + { + BeirDataset dataset = BeirDataset.Resolve(datasetName); + + BeirCorpus corpus = BeirLoader.LoadOrDownloadAsync(options.DataDirectory, dataset) + .GetAwaiter() + .GetResult(); + + IReadOnlyList documents = Evaluation.BuildDocuments(corpus); + + string[] queries = Evaluation + .BuildQueries(corpus, limit: null, excludeQueryDocument: false) + .Select(query => query.Query.Text) + .ToArray(); + + string table = $"lexisharp_bench_pg_{dataset.Name}_{Guid.NewGuid().ToString("N")[..8]}"; + + dropper.Add((table, options.Connection)); + + var indexOptions = new PostgresIndexOptions { Table = table }; + + // Indexed once, by one engine, and read by every row: the table and the analysis are the + // same for all of them, so rebuilding per row would measure the rebuild. + var indexing = Stopwatch.StartNew(); + + using (var writer = new PostgresTextSearchEngine(options.Connection, indexOptions)) + { + writer.Index(documents); + + // A read that touches the index, so the timing is not the first plan the server compiles. + _sink += writer.Search(queries[0], new SearchOptions(Limit: options.Limit)).Count; + } + + indexing.Stop(); + + Console.WriteLine( + string.Create( + CultureInfo.InvariantCulture, + $"== {dataset.Name} — {documents.Count:N0} documents, {queries.Length:N0} queries, " + + $"Limit {options.Limit}, {options.Passes} pass(es)")); + Console.WriteLine( + $"index {indexing.Elapsed.TotalMilliseconds:N0} ms (one table, shared by every row)"); + Console.WriteLine(); + Console.WriteLine(" engine stats ms p50 p95 p99 mean q/s"); + + var rows = Rows(options, table).ToList(); + var engines = new List(rows.Count); + var statistics = new List(rows.Count); + var samples = new List[rows.Count]; + + try + { + for (int i = 0; i < rows.Count; i++) + { + engines.Add(rows[i].Engine()); + statistics.Add(rows[i].Statistics()); + samples[i] = new List(queries.Length * options.Passes); + } + + // Warm-up: every engine's connection pool, the server's plan cache and the JIT. Nothing + // here is reported, and the passes below start from a process that has already answered + // these queries. + foreach (string query in queries.Take(Math.Min(queries.Length, 50))) + { + for (int i = 0; i < engines.Count; i++) + _sink += engines[i].Search(query, new SearchOptions(Limit: options.Limit)).Count; + } + + // Interleaved rather than row after row: a row measured alone pays for whatever the + // previous row warmed (buffer cache, pool, tiered JIT) or left behind (a gen2 GC), and + // a pool-size sweep is exactly where that reads as a ranking. + for (int pass = 0; pass < options.Passes; pass++) + { + foreach (string query in queries) + { + for (int i = 0; i < engines.Count; i++) + { + long start = Stopwatch.GetTimestamp(); + _sink += engines[i].Search(query, new SearchOptions(Limit: options.Limit)).Count; + samples[i].Add(Stopwatch.GetElapsedTime(start).TotalMilliseconds); + } + } + } + + for (int i = 0; i < rows.Count; i++) + { + samples[i].Sort(); + + double mean = samples[i].Average(); + double qps = mean > 0 ? 1_000 / mean : 0; + + Console.WriteLine( + string.Create( + CultureInfo.InvariantCulture, + $" {rows[i].Name,-25} {Format(statistics[i]),8} {Percentile(samples[i], 50.0),8:0.00} " + + $"{Percentile(samples[i], 95.0),8:0.00} {Percentile(samples[i], 99.0),8:0.00} " + + $"{mean,8:0.00} {qps,6:N0}")); + } + } + finally + { + foreach (ITextSearchEngine engine in engines) + (engine as IDisposable)?.Dispose(); + } + + Console.WriteLine(); + } + + /// + /// The rows measured on one table, in the order a reader decides with them: the engine that was + /// there before, the mode that changes its candidate set, then BM25 at increasing pool sizes. + /// + private static IEnumerable Rows(Options options, string table) => + [ + new( + "ts_rank_cd (all terms)", + () => new PostgresTextSearchEngine( + options.Connection, + new PostgresIndexOptions { Table = table, MatchMode = PostgresMatchMode.AllTerms }, + autoCreateSchema: false), + () => null), + new( + "ts_rank_cd (any term)", + () => new PostgresTextSearchEngine( + options.Connection, + new PostgresIndexOptions { Table = table, MatchMode = PostgresMatchMode.AnyTerm }, + autoCreateSchema: false), + () => null), + new( + "bm25 pool 1,000", + () => new PostgresBm25SearchEngine( + options.Connection, + new PostgresBm25Options { Table = table, CandidatePool = 1000 }, + autoCreateSchema: false), + () => Statistics(() => new PostgresBm25SearchEngine( + options.Connection, + new PostgresBm25Options { Table = table, CandidatePool = 1000 }, + autoCreateSchema: false))), + new( + "bm25 pool 5,000", + () => new PostgresBm25SearchEngine( + options.Connection, + new PostgresBm25Options { Table = table, CandidatePool = 5000 }, + autoCreateSchema: false), + () => Statistics(() => new PostgresBm25SearchEngine( + options.Connection, + new PostgresBm25Options { Table = table, CandidatePool = 5000 }, + autoCreateSchema: false))), + new( + "bm25 pool unbounded", + () => new PostgresBm25SearchEngine( + options.Connection, + new PostgresBm25Options { Table = table, CandidatePool = 0 }, + autoCreateSchema: false), + () => Statistics(() => new PostgresBm25SearchEngine( + options.Connection, + new PostgresBm25Options { Table = table, CandidatePool = 0 }, + autoCreateSchema: false))), + ]; + + /// + /// One statistics build, on its own engine: the snapshot belongs to the instance, so timing it + /// on the instance the row then searches with would start that engine from a warm snapshot and + /// hide exactly the cost this column exists for. + /// + private static double Statistics(Func create) + { + using PostgresBm25SearchEngine engine = create(); + var timer = Stopwatch.StartNew(); + _sink += engine.RefreshStatistics(); + timer.Stop(); + return timer.Elapsed.TotalMilliseconds; + } + + private static string Format(double? value) => + value is { } milliseconds ? milliseconds.ToString("N0", CultureInfo.InvariantCulture) : "—"; + + /// The percentile of a sorted sample, nearest-rank: no interpolation and no outlier removal. + private static double Percentile(List sorted, double percentile) + { + if (sorted.Count == 0) + return 0; + + int rank = (int)Math.Ceiling(percentile / 100.0 * sorted.Count) - 1; + return sorted[Math.Clamp(rank, 0, sorted.Count - 1)]; + } + + private static void PrintHeader(Options options) + { + Console.WriteLine("LexiSharp PostgreSQL lane — index, statistics and query latency on stock PostgreSQL"); + Console.WriteLine($"host {RuntimeInformation.OSDescription} / {RuntimeInformation.ProcessArchitecture}"); + Console.WriteLine($"cpu {Environment.ProcessorCount} logical processors"); + Console.WriteLine($"runtime {RuntimeInformation.FrameworkDescription}"); + Console.WriteLine($"server {options.Connection}"); + Console.WriteLine( + "shape one process, one thread, one query at a time, the engine's pooled data source warmed; " + + "a search is a page of 10"); + Console.WriteLine($"datasets {string.Join(", ", options.Datasets)}"); + Console.WriteLine(); + } + + private static Options Parse(string[] args) + { + var datasets = new List(); + string? connection = null; + string? data = null; + int limit = 10; + int passes = 3; + bool keep = false; + + for (int i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--dataset" when i + 1 < args.Length: + datasets.AddRange(args[++i].Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + break; + case "--connection" when i + 1 < args.Length: + connection = args[++i]; + break; + case "--data" when i + 1 < args.Length: + data = args[++i]; + break; + case "--limit" when i + 1 < args.Length: + limit = int.Parse(args[++i], CultureInfo.InvariantCulture); + break; + case "--passes" when i + 1 < args.Length: + passes = int.Parse(args[++i], CultureInfo.InvariantCulture); + break; + case "--keep": + keep = true; + break; + default: + throw new ArgumentException( + $"Unrecognised postgres argument '{args[i]}'. Usage: postgres --connection " + + "[--dataset nfcorpus,scifact] [--data ] [--limit 10] [--passes 3] [--keep]."); + } + } + + if (string.IsNullOrWhiteSpace(connection)) + throw new ArgumentException("postgres needs --connection ."); + + if (limit <= 0 || passes <= 0) + throw new ArgumentException("--limit and --passes must be positive."); + + if (datasets.Count == 0) + datasets.AddRange(DefaultDatasets); + + foreach (string dataset in datasets) + _ = BeirDataset.Resolve(dataset); + + return new Options([.. datasets], connection, ResolveDataDirectory(data), limit, passes, keep); + } + + /// + /// The directory holding the BEIR corpora. Defaulted against the working directory first and then + /// against the tree above the assembly, so the documented command runs from the repository root + /// and from a `bin/Release` directory alike. + /// + private static string ResolveDataDirectory(string? given) + { + if (given is not null) + return given; + + foreach (string candidate in Candidates()) + { + if (Directory.Exists(candidate)) + return candidate; + } + + return DefaultDataDirectory; + + static IEnumerable Candidates() + { + yield return DefaultDataDirectory; + + for (DirectoryInfo? directory = new(AppContext.BaseDirectory); directory is not null; directory = directory.Parent) + yield return Path.Combine(directory.FullName, DefaultDataDirectory); + } + } + + private sealed record Options( + IReadOnlyList Datasets, + string Connection, + string DataDirectory, + int Limit, + int Passes, + bool Keep); + + private sealed record Row(string Name, Func Engine, Func Statistics); +} diff --git a/bench/LexiSharp.Benchmarks/Program.cs b/bench/LexiSharp.Benchmarks/Program.cs index 916ff56e..fbc6a094 100644 --- a/bench/LexiSharp.Benchmarks/Program.cs +++ b/bench/LexiSharp.Benchmarks/Program.cs @@ -56,6 +56,13 @@ public static void Main(string[] args) return; } + // Index, corpus statistics and query latency of the stock-PostgreSQL text engines. See PostgresLane. + if (args.Length > 0 && args[0] == "postgres") + { + PostgresLane.Run(args[1..]); + return; + } + var switcher = new BenchmarkSwitcher(new[] { typeof(TokenizerBenchmarks), diff --git a/docs/backends.md b/docs/backends.md index 3e964796..af25d8e7 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -368,6 +368,43 @@ SciFact: on a corpus where a thirteen-term query matches half of it, the pool's the whole gap. Raise the pool, or set it to `0`, when the match sets are that large — the cost is one `tsvector` per match transferred and parsed per query. +**What it costs, in milliseconds.** The evaluation harness reports quality and not time on purpose; +the [PostgreSQL lane](benchmarks.md#postgresql-lane-postgres-what-a-search-costs-on-stock-postgresql) +measures the other half — one process, one thread, a pooled data source, a page of 10, on a local +server: + +| corpus (documents) | engine | stats build | p50 | p95 | mean | +|---|---|---:|---:|---:|---:| +| NFCorpus (3,633) | `ts_rank_cd`, all terms | — | 0.6 ms | 1.3 ms | 0.7 ms | +| NFCorpus | `ts_rank_cd`, any term | — | 4.1 ms | 74.6 ms | 21.3 ms | +| NFCorpus | BM25, pool 1,000 | 220 ms | 16.0 ms | 107.3 ms | 37.3 ms | +| NFCorpus | BM25, pool 0 | 201 ms | 15.8 ms | 177.1 ms | 58.1 ms | +| SciFact (5,183) | `ts_rank_cd`, all terms | — | 0.7 ms | 0.8 ms | 0.7 ms | +| SciFact | `ts_rank_cd`, any term | — | 170.1 ms | 347.2 ms | 177.6 ms | +| SciFact | BM25, pool 1,000 | 252 ms | 203.3 ms | 385.2 ms | 210.7 ms | +| SciFact | BM25, pool 0 | 246 ms | 299.7 ms | 490.0 ms | 301.0 ms | + +- **The candidate selection is the cost, not the BM25.** On SciFact the `any term` row alone is + 170 ms: a thirteen-term query matches nearly the whole corpus, and `ts_rank_cd` is evaluated over + every match before the pool cut. The BM25 row adds about 30 ms on top of that. On NFCorpus the + selection is 4 ms at the median with a long tail, because the match set is smaller. +- **On a small corpus with long queries, OR matching is a full scan.** That is the recall it buys, + and it is the lexical engine's cost as much as this one's. +- **The pool is a quality knob more than a time one.** Halving it to 500 costs 0.005 nDCG@10 on + NFCorpus and 0.077 on SciFact and barely moves the median; doubling it costs 0.10 in SciFact's + tail. 1,000 is the default for that reason. +- **This is not a speed replacement for `pg_search`.** Tantivy answers from its own inverted index + instead of ranking the match set in PostgreSQL, which is why the extension exists. What this + engine buys is portability; the table is its price. +- **The statistics build is the first search after a write**, not a search: `ts_stat` walks the + table, and the column is measured on its own so it does not hide inside a p99. + +The read is two-phase for this reason: the candidate query ranks ids and carries one `tsvector` per +candidate, and the page's documents are fetched afterwards, so a thousand candidates do not drag a +thousand contents and jsonb rows through the sort. The per-candidate reader keeps the query's terms +and counts the rest without allocating their strings. On NFCorpus those two changes took the +pool-1,000 row from 27 ms to 16 ms at the median and from 64 ms to 37 ms at the mean. + **What it is not.** The analysis is PostgreSQL's, not this library's tokenizer, so it matches the in-memory rows at most and no published figure. Weights (`FieldWeights`) are a `ts_rank_cd` concept and do not reach this ranking. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 450314ce..c85cb4be 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1612,6 +1612,64 @@ whose fetch buffer became per thread on 2026-11-02 — but it does **not** verif returned; the deterministic test for that is `SegmentReaderConcurrencyTests`, which drives the copy step through a barrier rather than hoping for a schedule. A wrong answer would not fail this lane. +## PostgreSQL lane (`postgres`): what a search costs on stock PostgreSQL + +The quality table in [evaluation](evaluation.md#the-sql-backends-on-the-same-corpora) says which +backend ranks better; this lane says what a search costs on the one that has to run where +`pg_search` cannot be installed. Same two BEIR corpora, same test queries, one process, one thread, +a page of 10, the engine's own pooled `NpgsqlDataSource` warmed, and a PostgreSQL 16 server in a +container on the same host. + +```bash +dotnet run --project bench/LexiSharp.Benchmarks -c Release -- postgres \ + --dataset nfcorpus,scifact \ + --connection "Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=lexisharp" +``` + +Measured 2026-11-16 on an 8-core Linux host, .NET 10.0.14, two passes over every query, p50/p95/p99 +by nearest rank. The index is built once per corpus and shared by every row; the rows are +interleaved per query, so a pool size does not inherit the previous row's cache state or its GC. + +| corpus (documents) | engine | index | stats build | p50 | p95 | p99 | mean | q/s | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| NFCorpus (3,633) | `ts_rank_cd`, all terms | 2.9 s | — | 0.6 ms | 1.3 ms | 1.7 ms | 0.7 ms | 1,459 | +| NFCorpus | `ts_rank_cd`, any term | | — | 4.1 ms | 74.6 ms | 102.3 ms | 21.3 ms | 47 | +| NFCorpus | BM25, pool 1,000 | | 220 ms | 16.0 ms | 107.3 ms | 131.1 ms | 37.3 ms | 27 | +| NFCorpus | BM25, pool 5,000 | | 205 ms | 15.7 ms | 176.9 ms | 202.2 ms | 57.7 ms | 17 | +| NFCorpus | BM25, pool 0 | | 201 ms | 15.8 ms | 177.1 ms | 209.0 ms | 58.1 ms | 17 | +| SciFact (5,183) | `ts_rank_cd`, all terms | 3.5 s | — | 0.7 ms | 0.8 ms | 1.3 ms | 0.7 ms | 1,433 | +| SciFact | `ts_rank_cd`, any term | | — | 170.1 ms | 347.2 ms | 425.0 ms | 177.6 ms | 6 | +| SciFact | BM25, pool 1,000 | | 252 ms | 203.3 ms | 385.2 ms | 466.7 ms | 210.7 ms | 5 | +| SciFact | BM25, pool 5,000 | | 248 ms | 300.2 ms | 483.4 ms | 569.6 ms | 301.2 ms | 3 | +| SciFact | BM25, pool 0 | | 246 ms | 299.7 ms | 490.0 ms | 570.4 ms | 301.0 ms | 3 | + +What the table says, and what it does not: + +- **The candidate selection dominates, not the BM25 arithmetic.** On SciFact the lexical + `any term` row is already 170 ms: a thirteen-term query matches almost the whole corpus, and + `ts_rank_cd` is computed for every match before the pool cut. The BM25 row adds about 30 ms. + On NFCorpus the same selection is 4 ms at the median, and the BM25 row is 16 ms. +- **OR matching on a small corpus with long queries is a scan.** The all-terms rows are two to + three orders of magnitude faster because the AND match set is tiny and the GIN index answers it + directly. That is the recall the any-term rule buys, priced. +- **The pool is mostly a quality knob.** 500 candidates cost 0.005 nDCG@10 on NFCorpus and 0.077 + on SciFact ([evaluation](evaluation.md)) and barely move the median; 5,000 and 0 differ from + each other only in the tail. The default of 1,000 is chosen from both tables. +- **`stats build` is not a search.** It is the `ts_stat` walk a `PostgresBm25SearchEngine` pays on + its first search after a write, measured by calling `RefreshStatistics()` explicitly. A write and + a search in a tight loop pays ~200-250 ms per corpus of this size. +- **It is not a speed comparison with `pg_search`.** Tantivy answers from its own inverted index + instead of ranking the match set inside PostgreSQL, which is why the extension exists. This lane + measures the portable path; the extension is the one to measure when it can be installed. + +Two reads make the BM25 rows what they are: the candidate query ranks ids and carries one +`tsvector` per candidate, and the page's documents are fetched afterwards; the per-candidate +`tsvector` reader keeps only the query's terms and counts every position for the length without +allocating the skipped strings. Before those two changes, NFCorpus's pool-1,000 row read 26.9 ms +p50 / 63.7 ms mean (and the first, uncontended run read 45 ms / 112 ms). `--keep` leaves the table +in place for `EXPLAIN`; the plan that mattered was a sort spilling to disk because the candidate +rows carried content and jsonb. + ## Tokenizer (`TokenizerBenchmarks`) Tokenizing the same text under different normalizations. The `Ratio` column is only @@ -1778,6 +1836,9 @@ dotnet run --project bench/LexiSharp.Benchmarks -c Release -- cursor 100000 --mi dotnet run --project bench/LexiSharp.Benchmarks -c Release -- store 100000 # posting entries a block bound can skip, per query and corpus size (see the pruning lane above): dotnet run --project bench/LexiSharp.Benchmarks -c Release -- wand 100000 +# stock-PostgreSQL index, statistics and query latency, per engine and pool size (see the PostgreSQL lane above): +dotnet run --project bench/LexiSharp.Benchmarks -c Release -- postgres --dataset nfcorpus,scifact \ + --connection "Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=lexisharp" # one shared index at 1/2/4/8 callers, raw and synchronized (see the concurrency lane above): dotnet run --project bench/LexiSharp.Benchmarks -c Release -- concurrency --runs 5 # the same callers against a committed store, array and mapped (see the concurrency lane above): diff --git a/src/LexiSharp.Postgres/PostgresBm25IndexView.cs b/src/LexiSharp.Postgres/PostgresBm25IndexView.cs index 72e2b563..e7d7be9a 100644 --- a/src/LexiSharp.Postgres/PostgresBm25IndexView.cs +++ b/src/LexiSharp.Postgres/PostgresBm25IndexView.cs @@ -2,9 +2,14 @@ namespace LexiSharp.Postgres; -/// One re-ranked match: its document, its term frequencies and its length. +/// One re-ranked match: its id, its term frequencies and its length. +/// +/// The document itself is not carried: the candidate query fetches only id and the +/// tsvector for the whole pool, and the page's documents are fetched after scoring. A +/// thousand rows of content and jsonb are transferred to be thrown away otherwise. +/// internal sealed record PostgresBm25Candidate( - SearchDocument Document, + string DocumentId, IReadOnlyDictionary TermFrequencies, int Length); @@ -23,7 +28,8 @@ internal sealed record PostgresBm25Candidate( /// /// Positions are not kept. Nothing on the BM25 path reads them, and a caller that asks gets /// rather than an empty list that would read as "the term -/// occurs nowhere". +/// occurs nowhere". The same goes for the documents: this view scores a candidate pool, and the +/// page's documents are fetched once the pool has been ranked. /// /// internal sealed class PostgresBm25IndexView : IReadOnlyTextIndex @@ -39,12 +45,15 @@ public PostgresBm25IndexView(PostgresBm25Statistics statistics, IReadOnlyList candidate.Document.Id, StringComparer.Ordinal); + _byId = candidates.ToDictionary(candidate => candidate.DocumentId, StringComparer.Ordinal); } /// + /// Always: the pool carries ids, not documents. public IReadOnlyCollection Documents => - _candidates.Select(candidate => candidate.Document).ToList(); + throw new NotSupportedException( + "The BM25 candidate pool carries ids and term frequencies. The page's documents are " + + "fetched after the pool is scored."); /// public int Count => _candidates.Count; @@ -94,14 +103,9 @@ public int TermFrequency(string documentId, string term) => public int DocumentLength(string documentId) => Candidate(documentId).Length; /// + /// Always false: this view is a scoring view, and holds no documents. public bool TryGetDocument(string documentId, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out SearchDocument? document) { - if (documentId is not null && _byId.TryGetValue(documentId, out PostgresBm25Candidate? candidate)) - { - document = candidate.Document; - return true; - } - document = null; return false; } diff --git a/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs b/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs index 57284b69..1a7b3433 100644 --- a/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs +++ b/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs @@ -229,6 +229,10 @@ public IReadOnlyList Search( if (queryTerms.Length == 0) return Array.Empty(); + // The query's terms, as the set the per-candidate reader keeps: the length needs every + // position, the frequencies only need these. + var keptTerms = new HashSet(queryTerms, StringComparer.Ordinal); + // Benign race: two concurrent searches may build the snapshot twice and both publish a // complete one. A reference assignment is atomic, so no reader can see a half-built object. var statistics = _statistics ??= PostgresBm25Statistics.Load(connection, _indexOptions); @@ -240,12 +244,20 @@ public IReadOnlyList Search( filters.Apply(command); match.Bind(command); + // Two shapes in one statement: the inner query ranks the match set and keeps the top + // @pool, carrying only (id, score) so the sort stays in memory; the outer join then reads + // one tsv per candidate. Selecting content and jsonb here as well would put ~1 KB of + // payload per row into the sort, which PostgreSQL answered by spilling it to disk. string candidatesSql = $""" - SELECT id, content, category, fields, text_fields, tsv::text - FROM {_options.QualifiedTableName} - WHERE ({match.Gate}){filters.Fragment} - ORDER BY {match.Rank} DESC, {PostgresSchema.OrdinalIdTieBreak} - LIMIT @pool; + SELECT t.id, t.tsv::text + FROM {_options.QualifiedTableName} t + JOIN ( + SELECT id, {match.Rank} AS score + FROM {_options.QualifiedTableName} + WHERE ({match.Gate}){filters.Fragment} + ORDER BY score DESC, {PostgresSchema.OrdinalIdTieBreak} + LIMIT @pool + ) s ON t.id = s.id; """; // Identifiers/config only are interpolated (validated [A-Za-z0-9_] and quoted); query text is parameterized. @@ -259,11 +271,10 @@ public IReadOnlyList Search( { while (reader.Read()) { - SearchDocument document = PostgresDocumentReader.Read(reader); var (termFrequencies, length) = - PostgresTsvText.Parse(reader.IsDBNull(5) ? null : reader.GetString(5)); + PostgresTsvText.Parse(reader.IsDBNull(1) ? null : reader.GetString(1), keptTerms); - candidates.Add(new PostgresBm25Candidate(document, termFrequencies, length)); + candidates.Add(new PostgresBm25Candidate(reader.GetString(0), termFrequencies, length)); } } @@ -271,11 +282,11 @@ public IReadOnlyList Search( return Array.Empty(); var view = new PostgresBm25IndexView(statistics, candidates); - var results = new List(candidates.Count); + var scored = new List<(string DocumentId, double Score)>(candidates.Count); foreach (PostgresBm25Candidate candidate in candidates) { - double score = _scorer.Score(candidate.Document.Id, queryTerms, view); + double score = _scorer.Score(candidate.DocumentId, queryTerms, view); if (double.IsNaN(score) || double.IsInfinity(score) || score <= 0) continue; @@ -283,21 +294,85 @@ public IReadOnlyList Search( if (score < options.MinimumScore) continue; - if (!PostgresDocumentExclusion.Passes(options, candidate.Document.Id)) + if (!PostgresDocumentExclusion.Passes(options, candidate.DocumentId)) continue; - results.Add(new SearchResult(candidate.Document.Id, score, candidate.Document)); + scored.Add((candidate.DocumentId, score)); } // Score descending, then the ordinal id: the same tie-break the SQL engines write into // their ORDER BY, applied here because the ordering is computed in this process. - results.Sort(static (left, right) => + scored.Sort(static (left, right) => { int byScore = right.Score.CompareTo(left.Score); return byScore != 0 ? byScore : string.CompareOrdinal(left.DocumentId, right.DocumentId); }); - return results.Skip(options.Offset).Take(options.Limit).ToList(); + List<(string DocumentId, double Score)> page = scored + .Skip(options.Offset) + .Take(options.Limit) + .ToList(); + + if (page.Count == 0) + return Array.Empty(); + + Dictionary documents = ReadDocuments(connection, page, cancellationToken); + + // A document deleted between the two reads is skipped rather than returned without its + // content. The window is not refilled: the alternative is a third query for a row that + // another writer is deleting, and the page is a read of a corpus that moved. + var results = new List(page.Count); + + foreach (var (documentId, score) in page) + { + if (documents.TryGetValue(documentId, out SearchDocument? document)) + results.Add(new SearchResult(documentId, score, document)); + } + + return results; + } + + /// + /// Reads the documents of the page, by id — the second half of the two-phase read. + /// + /// + /// The candidate query fetched ids and term frequencies for the whole pool; only the page's + /// rows need their content, category and fields. One statement with = ANY(@ids) rather + /// than one per id: it is the same round trip for a page of ten. + /// + private Dictionary ReadDocuments( + NpgsqlConnection connection, + IReadOnlyList<(string DocumentId, double Score)> page, + CancellationToken cancellationToken) + { + var ids = new string[page.Count]; + + for (int i = 0; i < page.Count; i++) + ids[i] = page[i].DocumentId; + + using var command = connection.CreateCommand(); + + command.CommandText = $""" + SELECT id, content, category, fields, text_fields + FROM {_options.QualifiedTableName} + WHERE id = ANY(@ids); + """; // NOSONAR:S2077 (identifier only, validated + quoted) + + command.Parameters.AddWithValue("ids", ids); + + var documents = new Dictionary(page.Count, StringComparer.Ordinal); + + using (var cancelling = PostgresCancellation.Register(command, cancellationToken)) + using (var reader = command.ExecuteReader()) + { + while (reader.Read()) + { + SearchDocument document = PostgresDocumentReader.Read(reader); + documents[document.Id] = document; + } + } + + return documents; } /// Releases the underlying Npgsql data source. diff --git a/src/LexiSharp.Postgres/PostgresTsvText.cs b/src/LexiSharp.Postgres/PostgresTsvText.cs index 2cf236e5..054f8e01 100644 --- a/src/LexiSharp.Postgres/PostgresTsvText.cs +++ b/src/LexiSharp.Postgres/PostgresTsvText.cs @@ -23,13 +23,32 @@ namespace LexiSharp.Postgres; /// ts_stat's nentry sums to over the table, so the per-document length and the corpus /// average are one measurement rather than two definitions. /// +/// +/// The keep set. A search needs the frequency of the query's terms and nothing +/// else, but the length still needs every position. Passing the query's terms lets the reader skip +/// the of every other lexeme: a candidate's tsvector holds a few +/// hundred lexemes and a query carries a handful, so without it the search allocates hundreds of +/// strings per candidate to throw nearly all of them away. The lookup is span-based +/// (), and the canonical string the set +/// already holds is what the frequency map is keyed by. +/// /// internal static class PostgresTsvText { - /// Parses a tsvector text value. + /// Parses a tsvector text value, keeping every lexeme. /// The value, as tsv::text produced it; empty for an empty vector. /// Per-lexeme frequencies and the document length in indexed tokens. - public static (Dictionary TermFrequencies, int Length) Parse(string? text) + public static (Dictionary TermFrequencies, int Length) Parse(string? text) => + Parse(text, keep: null); + + /// Parses a tsvector text value, keeping only the lexemes in . + /// The value, as tsv::text produced it; empty for an empty vector. + /// + /// The terms whose frequencies are wanted, or null for all of them. The length is exact + /// either way: skipping a lexeme skips its string, not its positions. + /// + /// Per-lexeme frequencies and the document length in indexed tokens. + public static (Dictionary TermFrequencies, int Length) Parse(string? text, HashSet? keep) { var terms = new Dictionary(StringComparer.Ordinal); int length = 0; @@ -37,6 +56,12 @@ public static (Dictionary TermFrequencies, int Length) Parse(string if (string.IsNullOrEmpty(text)) return (terms, length); + HashSet.AlternateLookup> lookup = default; + bool filtered = keep is not null; + + if (filtered) + lookup = keep!.GetAlternateLookup>(); + int i = 0; while (i < text.Length) @@ -47,12 +72,14 @@ public static (Dictionary TermFrequencies, int Length) Parse(string if (i >= text.Length) break; - string lexeme; + int start; + int end; + bool escaped = false; if (text[i] == '\'') { i++; - var builder = new StringBuilder(); + start = i; while (i < text.Length) { @@ -61,82 +88,124 @@ public static (Dictionary TermFrequencies, int Length) Parse(string // A doubled quote is the escape; a single one closes the lexeme. if (i + 1 < text.Length && text[i + 1] == '\'') { - builder.Append('\''); + escaped = true; i += 2; continue; } - i++; break; } - builder.Append(text[i]); i++; } - lexeme = builder.ToString(); + end = i; + + if (i < text.Length) + i++; } else { // Defensive: every released version quotes lexemes, but an unquoted one would // otherwise be swallowed into the next token's positions. - int start = i; + start = i; while (i < text.Length && text[i] != ':' && !char.IsWhiteSpace(text[i])) i++; - lexeme = text[start..i]; + end = i; } - if (lexeme.Length == 0) + if (end <= start) continue; - int count = 1; + int count = ReadCount(text, ref i); + length += count; - if (i < text.Length && text[i] == ':') + if (!filtered) { - i++; - count = 0; - bool digits = false; + string lexeme = escaped ? Unescape(text, start, end) : text[start..end]; + terms[lexeme] = terms.GetValueOrDefault(lexeme) + count; + continue; + } - while (i < text.Length && !char.IsWhiteSpace(text[i])) - { - char c = text[i]; + if (!escaped) + { + if (lookup.TryGetValue(text.AsSpan(start, end - start), out string? kept)) + terms[kept] = terms.GetValueOrDefault(kept) + count; - if (c is >= '0' and <= '9') - { - digits = true; - i++; - continue; - } + continue; + } - if (c == ',') - { - if (digits) - count++; + string unescaped = Unescape(text, start, end); - digits = false; - i++; - continue; - } + if (keep!.Contains(unescaped)) + terms[unescaped] = terms.GetValueOrDefault(unescaped) + count; + } - // A weight letter (or anything else) qualifies the position it follows; it - // neither starts a new position nor ends the one it belongs to. - i++; - } + return (terms, length); + } + + /// + /// Reads the position list at and returns how many positions it holds, + /// advancing past it. A lexeme with no list counts as one occurrence. + /// + private static int ReadCount(string text, ref int i) + { + if (i >= text.Length || text[i] != ':') + return 1; + + i++; + int count = 0; + bool digits = false; + + while (i < text.Length && !char.IsWhiteSpace(text[i])) + { + char c = text[i]; + + if (c is >= '0' and <= '9') + { + digits = true; + i++; + continue; + } + if (c == ',') + { if (digits) count++; - // A malformed list still says the term is present. - count = Math.Max(count, 1); + digits = false; + i++; + continue; } - terms[lexeme] = terms.GetValueOrDefault(lexeme) + count; - length += count; + // A weight letter (or anything else) qualifies the position it follows; it neither + // starts a new position nor ends the one it belongs to. + i++; } - return (terms, length); + if (digits) + count++; + + // A malformed list still says the term is present. + return Math.Max(count, 1); + } + + /// An escaped lexeme as a string, with each doubled quote collapsed to one. + private static string Unescape(string text, int start, int end) + { + var builder = new StringBuilder(end - start); + + for (int i = start; i < end; i++) + { + builder.Append(text[i]); + + if (text[i] == '\'' && i + 1 < end && text[i + 1] == '\'') + i++; + } + + return builder.ToString(); } /// diff --git a/tests/LexiSharp.Tests/PostgresTsvTextTests.cs b/tests/LexiSharp.Tests/PostgresTsvTextTests.cs index fc69ba9b..a12e77cf 100644 --- a/tests/LexiSharp.Tests/PostgresTsvTextTests.cs +++ b/tests/LexiSharp.Tests/PostgresTsvTextTests.cs @@ -84,6 +84,29 @@ public void Parse_UsesTheOrdinalComparer() Assert.Equal(1, terms["a"]); } + [Fact] + public void Parse_WithKeepReadsOnlyTheWantedFrequenciesAndTheExactLength() + { + var (terms, length) = PostgresTsvText.Parse( + "'a':1,2 'b':3 'o''brien':4", new HashSet(["a", "o'brien"], StringComparer.Ordinal)); + + Assert.Equal(2, terms["a"]); + Assert.Equal(1, terms["o'brien"]); + Assert.False(terms.ContainsKey("b")); + + // The length is every position, including the lexeme whose string was skipped. + Assert.Equal(4, length); + } + + [Fact] + public void Parse_WithAnEmptyKeepStillCountsTheLength() + { + var (terms, length) = PostgresTsvText.Parse("'a':1,2 'b':3", []); + + Assert.Empty(terms); + Assert.Equal(3, length); + } + [Fact] public void Format_RoundTripsThroughParse() { From 9a42ad78ac38923400b497c3bffae8f410cc3831 Mon Sep 17 00:00:00 2001 From: manuc66 Date: Mon, 16 Nov 2026 22:11:20 +0100 Subject: [PATCH 3/6] Generate BM25 candidates from the query's rare terms A GIN index has no score order, so the portable BM25 had to rank every match before the pool cut. On SciFact that is 265 ms p50, and worse: the pool then truncated the ranking by ts_rank_cd coverage, reading 0.5636 nDCG@10 against 0.6665 for the exact ranking. The scan was the cost, and the pool was the wrong selector. The gate generates candidates from the query's rarest terms instead: terms are taken by ascending document frequency while their summed document frequency stays inside RareTermGateBudget (1,000 rows by default), and only those matches are ranked. The whole query still scores them. This is what BM25's idf says to do -- a term held by ten documents carries ~6 nats, one held by half the corpus ~0.4 -- and it is a recall bound: a document holding none of the gated terms cannot be returned. It falls back to the whole match set when even the rarest term exceeds the budget, and a quoted phrase bypasses it, because the phrase is the query's contract. Measured on the same corpora, queries and qrels (nDCG@10, pool 1,000): corpus no gate gate 250 gate 500 gate 1,000 exact NFCorpus 0.2999 0.3036 0.3053 0.3072 0.3072 SciFact 0.5636 0.6643 0.6665 0.6665 0.6665 and latencies (p50 / mean, one thread, pooled data source): corpus no gate gate 250 gate 500 gate 1,000 NFCorpus 25.7/50.0 ms 5.0/7.0 ms 7.0/10.5 ms 10.0/17.8 ms SciFact 265.0/284.4 ms 15.7/17.8 ms 31.4/34.3 ms 58.8/63.9 ms At the default budget the gate reads the exact ranking's nDCG on both corpora, 4-9x faster than the ungated scan. The claim is bounded by the evidence: both corpora are scientific and entity-rich, which is the gate's best case, and a rare term that is a false friend can gate out relevant documents. `--backend-gate off --backend-gate-budget ` measures that on another corpus. Tests pin the selection (budget, fallback, deterministic order) and the behaviour (a document matching only common terms is excluded when the gate is on and returned when it is off). --- CHANGELOG.md | 32 +++-- README.md | 9 +- bench/LexiSharp.Benchmarks/PostgresLane.cs | 28 +++-- .../PostgresBackendBenchmark.cs | 42 ++++++- bench/LexiSharp.Eval/Program.cs | 36 +++++- bench/LexiSharp.Eval/README.md | 35 +++--- docs/backends.md | 104 +++++++++------- docs/benchmarks.md | 51 ++++---- docs/evaluation.md | 18 +-- docs/index.md | 4 +- src/LexiSharp.Postgres/PostgresBm25Options.cs | 48 +++++++- .../PostgresBm25SearchEngine.cs | 87 ++++++++++++- .../PostgresMatchExpression.cs | 12 +- .../PostgresBm25SearchEngineTests.cs | 115 +++++++++++++++++- tests/LexiSharp.Tests/PublicApiBaseline.txt | 3 + 15 files changed, 493 insertions(+), 131 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 181aa4d7..e9db7cb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,10 @@ `CREATE EXTENSION unaccent`. That is what makes a real BM25 available on RDS, Cloud SQL and Azure Database for PostgreSQL, where ParadeDB's `pg_search` cannot be installed. `N`, the average length and each term's document frequency come from one cached `ts_stat` snapshot (invalidated by - writes, rebuildable with `RefreshStatistics()`); PostgreSQL selects the best `CandidatePool` - matches by `ts_rank_cd` (default 1,000, `0` for the exact ranking) and the engine re-ranks them, - because scoring every match would make a query proportional to the corpus. Measured at pool 0: - **0.3072 nDCG@10 on NFCorpus and 0.6665 on SciFact**, against 0.308 / 0.662 for the in-memory - scorer and 0.3084 / 0.6619 for ParadeDB. At the default pool the same rows read 0.2999 / 0.5636 — - the difference is recall, and it is stated rather than hidden. + writes, rebuildable with `RefreshStatistics()`), while the term frequencies and the length come + from the candidate row's own vector. Measured with the candidate gate below: **0.3072 nDCG@10 on + NFCorpus and 0.6665 on SciFact**, against 0.308 / 0.662 for the in-memory scorer and 0.3084 / + 0.6619 for ParadeDB. - **`PostgresIndexOptions.MatchMode`** (`PostgresMatchMode.AllTerms | AnyTerm`): how a multi-term query becomes a `tsquery` on the lexical engine. `AllTerms` is the previous behaviour @@ -22,16 +20,28 @@ the corpus: SciFact moves 0.0188 → 0.0988 nDCG@10 (5.3×), while the ranker is still the rest — 0.0988 is far from BM25's 0.662 — and NFCorpus barely moves (0.1864 → 0.1881). -- **`--backend bm25`, `--backend-match all|any` and `--backend-pool `** in +- **A candidate gate on the stock-PostgreSQL BM25** (`RareTermGate`, on by default). A GIN index + cannot return rows in score order, so without it a query whose common terms + match most of the corpus pays a full scan. The gate generates candidates from the query's rarest + terms — ascending document frequency while their summed frequency stays inside + `RareTermGateBudget` (default 1,000 rows) — and scores them with the whole query. It is a recall + bound, it falls back to the whole match set when even the rarest term exceeds the budget, and a + quoted phrase bypasses it. Measured at the default budget it reads the exact ranking's nDCG on + both BEIR corpora (0.3072 / 0.6665) at 10.0 / 58.8 ms p50, against 0.2999 / 0.5636 at 25.7 / + 265.0 ms with the gate off. + +- **`--backend bm25`, `--backend-match all|any`, `--backend-gate rare|off`, `--backend-gate-budget` + and `--backend-pool `** in `bench/LexiSharp.Eval`, so the portable BM25 row and the match rule are measured by the same harness, corpus and qrels as every other row. - **`postgres` lane in `bench/LexiSharp.Benchmarks`**: index build, statistics build and search latency (p50/p95/p99/mean) of the stock-PostgreSQL text engines on the BEIR corpora, per match - rule and per candidate-pool size, interleaved so one row does not inherit another's cache state. - It is the measurement the quality harness deliberately does not take. On NFCorpus the pool-1,000 - BM25 row reads 16.0 ms p50 / 37.3 ms mean; on SciFact 203.3 / 210.7 ms, of which the OR - candidate selection alone is 170.1 / 177.6 ms. `--keep` leaves the table for `EXPLAIN`. + rule, candidate-gate budget and pool size, interleaved so one row does not inherit another's + cache state. It is the measurement the quality harness deliberately does not take. At the default + gate the BM25 row reads 10.0 ms p50 / 17.8 ms mean on NFCorpus and 58.8 / 63.9 ms on SciFact, + against 25.7 / 50.0 and 265.0 / 284.4 ms with the gate off. `--keep` leaves the table for + `EXPLAIN`. ### Changed diff --git a/README.md b/README.md index 7474d994..13052011 100644 --- a/README.md +++ b/README.md @@ -121,10 +121,11 @@ linear classifier): [text analysis →](docs/text-analysis.md). supported but unproven until tested on your data. - The stock-PostgreSQL BM25 re-ranks a bounded candidate pool (1,000 by default): the pool bounds recall, and on SciFact it costs 0.10 nDCG@10 against the exact ranking. `CandidatePool = 0` - removes the bound. It is also the slow path where `pg_search` cannot run: selecting the pool is - the cost, and on a 5,000-document corpus with 13-term queries a search reads ~200 ms p50. The - ParadeDB engine needs `pg_search`, which no major managed service lets you install; the - `tsvector`, BM25 and `pg_trgm` engines do not. + removes the bound. The rare-term gate is on by default and is what makes the path usable: it + reads ~59 ms p50 on a 5,000-document corpus with 13-term queries, against ~265 ms with the gate + off. It is a recall bound — a document holding none of the query's rare terms cannot be returned + — measured on two scientific corpora. The ParadeDB engine needs `pg_search`, which no major + managed service lets you install; the `tsvector`, BM25 and `pg_trgm` engines do not. - SQL backends are measured on ranking for the engines that rank on their own; sparse and fuzzy engines are covered by integration tests only. diff --git a/bench/LexiSharp.Benchmarks/PostgresLane.cs b/bench/LexiSharp.Benchmarks/PostgresLane.cs index e8d07460..2c3d8ad1 100644 --- a/bench/LexiSharp.Benchmarks/PostgresLane.cs +++ b/bench/LexiSharp.Benchmarks/PostgresLane.cs @@ -209,34 +209,44 @@ private static IEnumerable Rows(Options options, string table) => autoCreateSchema: false), () => null), new( - "bm25 pool 1,000", + "bm25 gate 250", () => new PostgresBm25SearchEngine( options.Connection, - new PostgresBm25Options { Table = table, CandidatePool = 1000 }, + new PostgresBm25Options { Table = table, CandidatePool = 1000, RareTermGateBudget = 250 }, autoCreateSchema: false), () => Statistics(() => new PostgresBm25SearchEngine( options.Connection, - new PostgresBm25Options { Table = table, CandidatePool = 1000 }, + new PostgresBm25Options { Table = table, CandidatePool = 1000, RareTermGateBudget = 250 }, autoCreateSchema: false))), new( - "bm25 pool 5,000", + "bm25 gate 500", () => new PostgresBm25SearchEngine( options.Connection, - new PostgresBm25Options { Table = table, CandidatePool = 5000 }, + new PostgresBm25Options { Table = table, CandidatePool = 1000, RareTermGateBudget = 500 }, autoCreateSchema: false), () => Statistics(() => new PostgresBm25SearchEngine( options.Connection, - new PostgresBm25Options { Table = table, CandidatePool = 5000 }, + new PostgresBm25Options { Table = table, CandidatePool = 1000, RareTermGateBudget = 500 }, autoCreateSchema: false))), new( - "bm25 pool unbounded", + "bm25 gate 1,000", () => new PostgresBm25SearchEngine( options.Connection, - new PostgresBm25Options { Table = table, CandidatePool = 0 }, + new PostgresBm25Options { Table = table, CandidatePool = 1000, RareTermGateBudget = 1000 }, autoCreateSchema: false), () => Statistics(() => new PostgresBm25SearchEngine( options.Connection, - new PostgresBm25Options { Table = table, CandidatePool = 0 }, + new PostgresBm25Options { Table = table, CandidatePool = 1000, RareTermGateBudget = 1000 }, + autoCreateSchema: false))), + new( + "bm25 no gate", + () => new PostgresBm25SearchEngine( + options.Connection, + new PostgresBm25Options { Table = table, CandidatePool = 1000, RareTermGate = false }, + autoCreateSchema: false), + () => Statistics(() => new PostgresBm25SearchEngine( + options.Connection, + new PostgresBm25Options { Table = table, CandidatePool = 1000, RareTermGate = false }, autoCreateSchema: false))), ]; diff --git a/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs b/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs index 4577d1f1..8847e629 100644 --- a/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs +++ b/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs @@ -95,7 +95,9 @@ internal sealed record BackendLane( IReadOnlyList? SimilarityThresholds = null, string? SparseVectorsDirectory = null, PostgresMatchMode? MatchMode = null, - int? CandidatePool = null) + int? CandidatePool = null, + bool? RareTermGate = null, + int? RareTermGateBudget = null) { /// The ef_search values in force; pgvector's own default when none was given. public IReadOnlyList EfValues => EfSearchValues is { Count: > 0 } values ? values : [40]; @@ -115,7 +117,9 @@ public static BackendLane Parse( IReadOnlyList? similarityThresholds = null, string? sparseVectorsDirectory = null, PostgresMatchMode? matchMode = null, - int? candidatePool = null) + int? candidatePool = null, + bool? rareTermGate = null, + int? rareTermGateBudget = null) { ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); @@ -185,6 +189,14 @@ public static BackendLane Parse( + "use --backend bm25."); } + if (parsed != BackendKind.Bm25 && (rareTermGate is not null || rareTermGateBudget is not null)) + { + throw new ArgumentException( + "--backend-gate and --backend-gate-df configure the BM25 engine's candidate gate, which " + + "selects candidates from the query's rare terms; the other backends have no such gate. " + + "Drop the flags, or use --backend bm25."); + } + if (parsed is not (BackendKind.Lexical or BackendKind.Bm25) && matchMode is not null) { throw new ArgumentException( @@ -266,7 +278,9 @@ public static BackendLane Parse( similarityThresholds, sparseVectorsDirectory is null ? null : Path.GetFullPath(sparseVectorsDirectory), matchMode, - candidatePool); + candidatePool, + rareTermGate, + rareTermGateBudget); } /// @@ -292,7 +306,7 @@ public string TableFor(string dataset) BackendKind.Bm25 => $"PostgreSQL BM25 (Bm25Scorer over tsvector term frequencies, text search config " + $"'{TextSearchConfig}' + unaccent, {MatchText}, {Bm25OperatingPoint}, candidate pool " - + $"{EffectivePool:N0}" + FieldWeightsText + ")", + + $"{EffectivePool:N0}, {GateText}" + FieldWeightsText + ")", BackendKind.ParadeDb => "ParadeDB BM25 (pdb.score, Tantivy)" + Bm25Text + " — index over content only, title concatenated into it", @@ -326,6 +340,18 @@ public string TableFor(string dataset) ? "any term" : "all terms"; + /// The candidate gate in words: the rare-term one, or the whole query. + private string GateText => ResolvedRareTermGate + ? $"rare-term gate, budget {ResolvedGateBudget:N0} rows" + : "no candidate gate"; + + /// The gate the row ran with: the flag when given, the engine's default otherwise. + public bool ResolvedRareTermGate => RareTermGate ?? true; + + /// The gate's document-frequency threshold: the flag when given, the engine's default otherwise. + public int ResolvedGateBudget => + RareTermGateBudget ?? PostgresBm25Options.DefaultRareTermGateBudget; + /// The pool in words: a count when bounded, the ceiling when the bound was removed. private string PoolText => EffectivePool == 0 ? $"every match up to the engine's {PostgresBm25Options.MaximumCandidatePool:N0}-row ceiling (pool unbounded)" @@ -430,6 +456,12 @@ private static string FormatThreshold(double threshold) => $"The database selects {PoolText} and the engine re-ranks them. A bounded pool bounds " + "recall — a document outside it cannot be returned — and does not change the ordering " + "inside it.", + GateText is "no candidate gate" + ? "Every match of the query is a candidate: the database ranks the whole match set, which " + + "is what the row's latency includes." + : "Candidates are the matches of the query's terms whose document frequency is at most " + + $"{ResolvedGateBudget:N0}, not of the whole query: a document holding none of " + + "them cannot be returned. The full query still scores the candidates.", AnalysisCaveat, ], _ => @@ -549,6 +581,8 @@ public static BackendMeasurement Measure( K1 = lane.Bm25?.K1 ?? PostgresBm25Options.DefaultK1, B = lane.Bm25?.B ?? PostgresBm25Options.DefaultB, CandidatePool = lane.EffectivePool, + RareTermGate = lane.ResolvedRareTermGate, + RareTermGateBudget = lane.ResolvedGateBudget, }); return Run(bm25, bm25.DropSchema); diff --git a/bench/LexiSharp.Eval/Program.cs b/bench/LexiSharp.Eval/Program.cs index 66a2bf5a..3a951b08 100644 --- a/bench/LexiSharp.Eval/Program.cs +++ b/bench/LexiSharp.Eval/Program.cs @@ -127,6 +127,12 @@ public static async Task Main(string[] args) // cost, and a measurement of what the pool costs is what a reader needs to set it. string? backendPool = null; + // The BM25 row's candidate gate: rare terms or the whole query, and the document-frequency + // ceiling a term must stay under. Two flags because the second only means anything with + // the first. + string? backendGate = null; + string? backendGateBudget = null; + // The lexical row's tsv weighting, as "title=A[,summary=B]" — parsed after the pair check // below, so a malformed spec fails on a message rather than an unhandled exception. string? backendFieldWeights = null; @@ -356,6 +362,12 @@ public static async Task Main(string[] args) case "--backend-pool" when i + 1 < args.Length: backendPool = args[++i]; break; + case "--backend-gate" when i + 1 < args.Length: + backendGate = args[++i]; + break; + case "--backend-gate-budget" when i + 1 < args.Length: + backendGateBudget = args[++i]; + break; case "--backend-ef" when i + 1 < args.Length: backendEfValues.AddRange(ParsePositiveInts(args[++i], "--backend-ef")); break; @@ -589,7 +601,9 @@ public static async Task Main(string[] args) backendThresholds, sparseVectorsDir, backendMatch is null ? null : ParseMatchMode(backendMatch), - backendPool is null ? null : ParseNonNegative(backendPool, "--backend-pool")); + backendPool is null ? null : ParseNonNegative(backendPool, "--backend-pool"), + backendGate is null ? null : ParseGate(backendGate), + backendGateBudget is null ? null : ParsePositive(backendGateBudget, "--backend-gate-budget")); } catch (ArgumentException error) { @@ -1859,6 +1873,15 @@ private static Bm25Parameters ParseBm25Parameters(string value) /// all and any because those are the two English words websearch_to_tsquery /// and a disjunction of lexemes are — ands/ors would name an operator, not a rule. /// + /// --backend-gate: the BM25 candidate gate, spelled by what it selects. + private static bool ParseGate(string value) => + value.ToLowerInvariant() switch + { + "rare" => true, + "off" or "none" => false, + _ => throw new ArgumentException($"--backend-gate expects 'rare' or 'off', got '{value}'."), + }; + private static PostgresMatchMode ParseMatchMode(string value) => value.ToLowerInvariant() switch { @@ -2194,6 +2217,17 @@ parameters the published baselines were produced at. pg_search takes engine's parameters, whose own defaults are 1.5,0.75. Refused with --backend lexical, whose ts_rank_cd has no such parameter, and with the vector, sparse and fuzzy backends, which have no k1/b. + --backend-gate + How the bm25 backend generates candidates: 'rare' (default) gates on + the query's rarest terms until --backend-gate-budget rows are + read, then scores those with the whole query; 'off' scores every + match of the query. A rare-term gate is a recall bound: + a document holding none of the rare terms is not returned. Refused + with the other backends, which have no such gate. + --backend-gate-budget + How many candidate rows --backend-gate rare may read, by ascending + term document frequency (default 1000). If even the rarest term + exceeds it, the row falls back to the whole match set. --backend-pool How many matches the bm25 backend re-ranks, default 1000. The database selects them by ts_rank_cd and the engine scores them with BM25, so the diff --git a/bench/LexiSharp.Eval/README.md b/bench/LexiSharp.Eval/README.md index 738981de..823e4d55 100644 --- a/bench/LexiSharp.Eval/README.md +++ b/bench/LexiSharp.Eval/README.md @@ -215,20 +215,23 @@ Out of the box, each engine at its own defaults: | ArguAna | 0.320 | not measured | not measured | not measured | `--backend-match all|any` picks how a multi-term query becomes a `tsquery`: `all` (the lexical row's -historical behaviour) requires every term, `any` ORs the query's lexemes. `--backend-pool ` -bounds how many matches the BM25 row re-ranks (default 1,000; `0` = exact, up to the engine's -100,000-row ceiling). The two knobs, and the rows they produce (harness defaults, exponential -gains): +historical behaviour) requires every term, `any` ORs the query's lexemes. `--backend-gate rare|off` +turns the BM25 row's candidate gate on or off, and `--backend-gate-budget ` bounds how many rows +it may read (default 1,000). `--backend-pool ` bounds how many of the gate's candidates are +re-ranked (default 1,000; `0` = the engine's ceiling). The knobs, and the rows they produce (harness +defaults, exponential gains): -| corpus | `ts_rank_cd` all | `ts_rank_cd` any | BM25 pool 1,000 | BM25 pool 0 | in-memory BM25 | ParadeDB | -|---|---:|---:|---:|---:|---:|---:| -| NFCorpus | 0.1864 | 0.1881 | 0.2999 | 0.3072 | 0.308 | 0.3084 | -| SciFact | 0.0188 | 0.0988 | 0.5636 | **0.6665** | 0.662 | 0.6619 | +| corpus | `ts_rank_cd` all | `ts_rank_cd` any | BM25 gate 1,000 | BM25 no gate | BM25 pool 0 | in-memory BM25 | ParadeDB | +|---|---:|---:|---:|---:|---:|---:|---:| +| NFCorpus | 0.1864 | 0.1881 | **0.3072** | 0.2999 | 0.3072 | 0.308 | 0.3084 | +| SciFact | 0.0188 | 0.0988 | **0.6665** | 0.5636 | 0.6665 | 0.662 | 0.6619 | The matching rule is the SciFact story (5.3×) and almost nothing on NFCorpus, whose queries average -3.4 terms; the ranker is the rest. `--backend bm25 --backend-pool 0` reproduces BM25 to within the -analysis (PostgreSQL's `simple` config, not the harness tokenizer), which is what makes the pool's -0.103-point cost on SciFact visible rather than attributed to the scoring. +3.4 terms; the ranker is the rest. The gate is what makes the BM25 row recover the exact ranking +(the `pool 0` column) while reading ~1,000 candidate rows instead of the whole match set; the +no-gate column is the pool-only path, where `ts_rank_cd` coverage chooses the candidates and SciFact +loses 0.10. `--backend-gate off --backend-pool 0` reproduces untruncated BM25 to within the analysis +(PostgreSQL's `simple` config, not the harness tokenizer). What each row is, and is not: @@ -243,10 +246,12 @@ What each row is, and is not: *ranking* has no idf (NFCorpus barely moves under `any`, 0.1864 → 0.1881, because its queries average 3.4 terms). PostgreSQL's own English analysis (`--backend-config english`) moves NFCorpus 0.1864 → 0.2043, not closer. -- **`--backend bm25` is the same corpus with a real BM25 and no extension.** At pool 0 it reads - 0.3072 / 0.6665 against 0.308 / 0.662 in memory and 0.3084 / 0.6619 for ParadeDB; at the default - pool, 0.2999 / 0.5636. The gap between pool 0 and pool 1,000 is recall, not scoring, and it is - the reason the flag exists. +- **`--backend bm25` is the same corpus with a real BM25 and no extension.** At the default gate it + reads 0.3072 / 0.6665 against 0.308 / 0.662 in memory and 0.3084 / 0.6619 for ParadeDB; with the + gate off, 0.2999 / 0.5636, because the candidate set is then chosen by coverage and truncated. + Both columns are recall, not scoring: the score is the same `Bm25Scorer` in every one of them. + The gate is a bound — a document holding none of the query's gated terms cannot be returned — and + `--backend-gate off` is how to measure that on another corpus. - **ArguAna has no row.** Both text engines refuse query text their matcher would read as a prefix operator, and 2 of the 1,406 test queries carry a `*` attached to a term. The lane prints that refusal and its count instead of a metric over the other 1,404, which would be a different query diff --git a/docs/backends.md b/docs/backends.md index af25d8e7..41a80061 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -353,57 +353,67 @@ table (or a bulk load) does not, and `RefreshStatistics()` is how that caller sa on `Bm25Scorer`'s scale, unlike `ts_rank_cd`'s and `pdb.score`'s, so they are comparable with the in-memory BM25 row. -**The candidate pool.** PostgreSQL selects the best `CandidatePool` matches by `ts_rank_cd` -(default 1,000; `0` removes the bound up to the engine's 100,000-row ceiling) and only those are -re-ranked. The pool bounds recall — a document outside it cannot be returned — and does not change -the ordering inside it. On the same BEIR corpora: - -| corpus | pool 1,000 | pool 0 (exact) | in-memory BM25 | ParadeDB | -|---|---:|---:|---:|---:| -| NFCorpus | 0.2999 | 0.3072 | 0.308 | 0.3084 | -| SciFact | 0.5636 | 0.6665 | 0.662 | 0.6619 | - -At the default pool the row is within 0.008 of the in-memory BM25 on NFCorpus and 0.10 short on -SciFact: on a corpus where a thirteen-term query matches half of it, the pool's 1,000 candidates are -the whole gap. Raise the pool, or set it to `0`, when the match sets are that large — the cost is -one `tsvector` per match transferred and parsed per query. - -**What it costs, in milliseconds.** The evaluation harness reports quality and not time on purpose; -the [PostgreSQL lane](benchmarks.md#postgresql-lane-postgres-what-a-search-costs-on-stock-postgresql) +**The candidate gate.** A GIN index cannot return rows in score order, so without a gate the only +option is to score every match before the pool cut — and a query whose common terms match most of +the corpus then costs a full scan. `RareTermGate` (on by default) generates candidates from the +query's **rarest terms** instead: terms are taken by ascending document frequency while their summed +document frequency stays inside `RareTermGateBudget` (default 1,000 rows), and only those matches are +ranked. The full query still scores them. This is what BM25's own idf says to do — a term held by ten +documents carries about 6 nats, one held by half the corpus about 0.4 — and it is a **recall bound**: +a document holding none of the gated terms cannot be returned by any score. + +Two fallbacks keep it honest. When even the rarest term exceeds the budget, the thread falls back to +the whole match set, so a corpus of common terms keeps the previous behaviour. And a quoted phrase +bypasses the gate entirely: the phrase is the query's contract, and a disjunction of terms must not +stand in for it. `CandidatePool` still bounds how many of the gate's candidates are read (default +1,000); with the gate on it rarely binds. + +**What it buys, in quality.** nDCG@10 over the same corpora, queries and qrels: + +| corpus | no gate | gate, 250 rows | gate, 500 rows | gate, 1,000 rows | exact (no gate, pool 0) | +|---|---:|---:|---:|---:|---:| +| NFCorpus | 0.2999 | 0.3036 | 0.3053 | **0.3072** | 0.3072 | +| SciFact | 0.5636 | 0.6643 | 0.6665 | **0.6665** | 0.6665 | + +At the default budget the gate reads the exact ranking's nDCG on both corpora. The ungated row does +not because its 1,000 candidates are chosen by `ts_rank_cd` coverage, which is not what BM25 ranks +by; on SciFact that truncation is the whole 0.10 gap. The gate chooses by idf instead, and its +candidate set is small enough not to truncate. Lowering the budget trades a little quality on +NFCorpus for speed; raising it changes nothing here. + +**What it costs, in time.** The evaluation harness reports quality and not time on purpose; the +[PostgreSQL lane](benchmarks.md#postgresql-lane-postgres-what-a-search-costs-on-stock-postgresql) measures the other half — one process, one thread, a pooled data source, a page of 10, on a local -server: - -| corpus (documents) | engine | stats build | p50 | p95 | mean | -|---|---|---:|---:|---:|---:| -| NFCorpus (3,633) | `ts_rank_cd`, all terms | — | 0.6 ms | 1.3 ms | 0.7 ms | -| NFCorpus | `ts_rank_cd`, any term | — | 4.1 ms | 74.6 ms | 21.3 ms | -| NFCorpus | BM25, pool 1,000 | 220 ms | 16.0 ms | 107.3 ms | 37.3 ms | -| NFCorpus | BM25, pool 0 | 201 ms | 15.8 ms | 177.1 ms | 58.1 ms | -| SciFact (5,183) | `ts_rank_cd`, all terms | — | 0.7 ms | 0.8 ms | 0.7 ms | -| SciFact | `ts_rank_cd`, any term | — | 170.1 ms | 347.2 ms | 177.6 ms | -| SciFact | BM25, pool 1,000 | 252 ms | 203.3 ms | 385.2 ms | 210.7 ms | -| SciFact | BM25, pool 0 | 246 ms | 299.7 ms | 490.0 ms | 301.0 ms | - -- **The candidate selection is the cost, not the BM25.** On SciFact the `any term` row alone is - 170 ms: a thirteen-term query matches nearly the whole corpus, and `ts_rank_cd` is evaluated over - every match before the pool cut. The BM25 row adds about 30 ms on top of that. On NFCorpus the - selection is 4 ms at the median with a long tail, because the match set is smaller. -- **On a small corpus with long queries, OR matching is a full scan.** That is the recall it buys, - and it is the lexical engine's cost as much as this one's. -- **The pool is a quality knob more than a time one.** Halving it to 500 costs 0.005 nDCG@10 on - NFCorpus and 0.077 on SciFact and barely moves the median; doubling it costs 0.10 in SciFact's - tail. 1,000 is the default for that reason. -- **This is not a speed replacement for `pg_search`.** Tantivy answers from its own inverted index - instead of ranking the match set in PostgreSQL, which is why the extension exists. What this - engine buys is portability; the table is its price. +server. p50 / mean: + +| corpus (documents) | `ts_rank_cd`, any term | BM25, no gate | gate, 250 | gate, 500 | gate, 1,000 | +|---|---:|---:|---:|---:|---:| +| NFCorpus (3,633) | 6.6 / 28.3 ms | 25.7 / 50.0 ms | 5.0 / 7.0 ms | 7.0 / 10.5 ms | **10.0 / 17.8 ms** | +| SciFact (5,183) | 219 / 240 ms | 265 / 284 ms | 15.7 / 17.8 ms | 31.4 / 34.3 ms | **58.8 / 63.9 ms** | + +- **The scan was the cost, and the gate removes it.** On SciFact the ungated row is 265 ms because a + thirteen-term query matches nearly the whole corpus and `ts_rank_cd` is evaluated over every + match; the gated row is 59 ms, and 16 ms at a 250-row budget. +- **The pool is no longer the interesting knob.** With the gate on, `pool 0` and `pool 1,000` read + the same (SciFact 50.1 vs 49.7 ms in the lane); the pool matters on the ungated path, where it is + the thing that truncates. +- **It is still not a speed replacement for `pg_search`.** Tantivy answers from its own inverted + index instead of ranking a candidate set inside PostgreSQL, which is why the extension exists. + What this engine buys is portability; the table is its price. - **The statistics build is the first search after a write**, not a search: `ts_stat` walks the table, and the column is measured on its own so it does not hide inside a p99. -The read is two-phase for this reason: the candidate query ranks ids and carries one `tsvector` per -candidate, and the page's documents are fetched afterwards, so a thousand candidates do not drag a -thousand contents and jsonb rows through the sort. The per-candidate reader keeps the query's terms -and counts the rest without allocating their strings. On NFCorpus those two changes took the -pool-1,000 row from 27 ms to 16 ms at the median and from 64 ms to 37 ms at the mean. +Two facts about the evidence, because they bound the claim. Both corpora are scientific and their +queries are rich in specific terms — SciFact's rarest query term has a median document frequency of +1 — which is the case the gate is built for. And the failure mode is real: a rare term that is a +false friend (a name variant, an entity absent from the relevant documents) can gate them out. The +harness flag `--backend-gate off` is how to measure that on your own corpus. + +The read is two-phase: the candidate query ranks ids and carries one `tsvector` per candidate, and +the page's documents are fetched afterwards, so a thousand candidates do not drag a thousand contents +and jsonb rows through the sort. The per-candidate reader keeps the query's terms and counts the rest +without allocating their strings. On NFCorpus those two changes took the ungated pool-1,000 row from +27 ms to 16 ms at the median and from 64 ms to 37 ms at the mean. **What it is not.** The analysis is PostgreSQL's, not this library's tokenizer, so it matches the in-memory rows at most and no published figure. Weights (`FieldWeights`) are a `ts_rank_cd` diff --git a/docs/benchmarks.md b/docs/benchmarks.md index c85cb4be..1e316f71 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1628,39 +1628,46 @@ dotnet run --project bench/LexiSharp.Benchmarks -c Release -- postgres \ Measured 2026-11-16 on an 8-core Linux host, .NET 10.0.14, two passes over every query, p50/p95/p99 by nearest rank. The index is built once per corpus and shared by every row; the rows are -interleaved per query, so a pool size does not inherit the previous row's cache state or its GC. +interleaved per query, so a gate budget does not inherit the previous row's cache state or its GC. | corpus (documents) | engine | index | stats build | p50 | p95 | p99 | mean | q/s | |---|---|---:|---:|---:|---:|---:|---:|---:| -| NFCorpus (3,633) | `ts_rank_cd`, all terms | 2.9 s | — | 0.6 ms | 1.3 ms | 1.7 ms | 0.7 ms | 1,459 | -| NFCorpus | `ts_rank_cd`, any term | | — | 4.1 ms | 74.6 ms | 102.3 ms | 21.3 ms | 47 | -| NFCorpus | BM25, pool 1,000 | | 220 ms | 16.0 ms | 107.3 ms | 131.1 ms | 37.3 ms | 27 | -| NFCorpus | BM25, pool 5,000 | | 205 ms | 15.7 ms | 176.9 ms | 202.2 ms | 57.7 ms | 17 | -| NFCorpus | BM25, pool 0 | | 201 ms | 15.8 ms | 177.1 ms | 209.0 ms | 58.1 ms | 17 | -| SciFact (5,183) | `ts_rank_cd`, all terms | 3.5 s | — | 0.7 ms | 0.8 ms | 1.3 ms | 0.7 ms | 1,433 | -| SciFact | `ts_rank_cd`, any term | | — | 170.1 ms | 347.2 ms | 425.0 ms | 177.6 ms | 6 | -| SciFact | BM25, pool 1,000 | | 252 ms | 203.3 ms | 385.2 ms | 466.7 ms | 210.7 ms | 5 | -| SciFact | BM25, pool 5,000 | | 248 ms | 300.2 ms | 483.4 ms | 569.6 ms | 301.2 ms | 3 | -| SciFact | BM25, pool 0 | | 246 ms | 299.7 ms | 490.0 ms | 570.4 ms | 301.0 ms | 3 | +| NFCorpus (3,633) | `ts_rank_cd`, all terms | 3.3 s | — | 0.9 ms | 1.8 ms | 3.4 ms | 1.0 ms | 982 | +| NFCorpus | `ts_rank_cd`, any term | | — | 6.6 ms | 100.9 ms | 136.7 ms | 28.3 ms | 35 | +| NFCorpus | BM25, gate 250 | | 268 ms | 5.0 ms | 17.6 ms | 25.3 ms | 7.0 ms | 143 | +| NFCorpus | BM25, gate 500 | | 224 ms | 7.0 ms | 28.4 ms | 38.9 ms | 10.5 ms | 95 | +| NFCorpus | BM25, gate 1,000 (default) | | 210 ms | 10.0 ms | 55.2 ms | 66.0 ms | 17.8 ms | 56 | +| NFCorpus | BM25, no gate | | 227 ms | 25.7 ms | 148.1 ms | 177.3 ms | 50.0 ms | 20 | +| SciFact (5,183) | `ts_rank_cd`, all terms | 6.5 s | — | 0.8 ms | 2.6 ms | 3.8 ms | 1.1 ms | 936 | +| SciFact | `ts_rank_cd`, any term | | — | 219.3 ms | 498.4 ms | 649.9 ms | 239.5 ms | 4 | +| SciFact | BM25, gate 250 | | 450 ms | 15.7 ms | 36.6 ms | 51.5 ms | 17.8 ms | 56 | +| SciFact | BM25, gate 500 | | 384 ms | 31.4 ms | 68.7 ms | 88.1 ms | 34.3 ms | 29 | +| SciFact | BM25, gate 1,000 (default) | | 387 ms | 58.8 ms | 119.8 ms | 153.0 ms | 63.9 ms | 16 | +| SciFact | BM25, no gate | | 434 ms | 265.0 ms | 531.8 ms | 734.3 ms | 284.4 ms | 4 | What the table says, and what it does not: - **The candidate selection dominates, not the BM25 arithmetic.** On SciFact the lexical - `any term` row is already 170 ms: a thirteen-term query matches almost the whole corpus, and - `ts_rank_cd` is computed for every match before the pool cut. The BM25 row adds about 30 ms. - On NFCorpus the same selection is 4 ms at the median, and the BM25 row is 16 ms. -- **OR matching on a small corpus with long queries is a scan.** The all-terms rows are two to - three orders of magnitude faster because the AND match set is tiny and the GIN index answers it - directly. That is the recall the any-term rule buys, priced. -- **The pool is mostly a quality knob.** 500 candidates cost 0.005 nDCG@10 on NFCorpus and 0.077 - on SciFact ([evaluation](evaluation.md)) and barely move the median; 5,000 and 0 differ from - each other only in the tail. The default of 1,000 is chosen from both tables. + `any term` row alone is 219 ms: a thirteen-term query matches almost the whole corpus, and + `ts_rank_cd` is computed for every match. The BM25 row without a gate is 265 ms. +- **The rare-term gate removes that scan.** It generates candidates from the query's rarest terms, + by ascending document frequency while their summed document frequency stays inside the budget + (1,000 rows by default), and scores them with the whole query. On SciFact that is 265 ms → 59 ms + at the default budget and 16 ms at 250; on NFCorpus 25.7 → 10.0 ms. It also recovers the exact + ranking's nDCG on both corpora, which the ungated pool does not + ([backends](backends.md#bm25-over-stock-postgresql-postgresbm25searchengine)). +- **The pool is mostly a fallback knob now.** With the gate on, the pool rarely binds; the gate's + budget is the cost control that matters. With the gate off, the pool is what truncates, and on + SciFact that truncation is 0.10 nDCG@10. - **`stats build` is not a search.** It is the `ts_stat` walk a `PostgresBm25SearchEngine` pays on its first search after a write, measured by calling `RefreshStatistics()` explicitly. A write and - a search in a tight loop pays ~200-250 ms per corpus of this size. + a search in a tight loop pays ~200-450 ms per corpus of this size. - **It is not a speed comparison with `pg_search`.** Tantivy answers from its own inverted index - instead of ranking the match set inside PostgreSQL, which is why the extension exists. This lane + instead of ranking a candidate set inside PostgreSQL, which is why the extension exists. This lane measures the portable path; the extension is the one to measure when it can be installed. +- **The gate is a recall bound.** Both corpora are scientific and entity-rich, which is its best + case; a rare term that is a false friend can gate out relevant documents. `--backend-gate off` + measures that on another corpus. Two reads make the BM25 rows what they are: the candidate query ranks ids and carries one `tsvector` per candidate, and the page's documents are fetched afterwards; the per-candidate diff --git a/docs/evaluation.md b/docs/evaluation.md index 5605ae76..0bf4a01e 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -480,10 +480,10 @@ Out of the box, each engine at its own defaults (`k1=1.5, b=0.75` here, `1.2, 0. The two knobs that moved `ts_rank_cd` and then closed the gap, measured on the same corpora and the same qrels (harness defaults, exponential gains, 10 retrieved at 10): -| corpus | `ts_rank_cd` all terms | `ts_rank_cd` any term | BM25 pool 1,000 | BM25 pool 0 | in-memory BM25 | ParadeDB | -|---|---:|---:|---:|---:|---:|---:| -| NFCorpus | 0.1864 | 0.1881 | 0.2999 | 0.3072 | 0.308 | 0.3084 | -| SciFact | 0.0188 | 0.0988 | 0.5636 | **0.6665** | 0.662 | 0.6619 | +| corpus | `ts_rank_cd` all terms | `ts_rank_cd` any term | BM25, gate 1,000 | BM25, no gate | BM25 pool 0 | in-memory BM25 | ParadeDB | +|---|---:|---:|---:|---:|---:|---:|---:| +| NFCorpus | 0.1864 | 0.1881 | **0.3072** | 0.2999 | 0.3072 | 0.308 | 0.3084 | +| SciFact | 0.0188 | 0.0988 | **0.6665** | 0.5636 | 0.6665 | 0.662 | 0.6619 | - **ParadeDB is a BM25 and reads where this library's BM25 reads.** At matched parameters the two differ by 0.0005 on NFCorpus and 0.0083 on SciFact, which is analysis rather than ranking: the @@ -513,11 +513,13 @@ same qrels (harness defaults, exponential gains, 10 retrieved at 10): skip — a gate that cannot fail. They are reproducible from the command above, and nothing catches a drift in them. - **`--backend bm25` closes the ranker gap with no extension.** It scores the same `tsvector` term - frequencies with `Bm25Scorer`, and at an unbounded pool reads 0.3072 / 0.6665 against the + frequencies with `Bm25Scorer`, and at the default rare-term gate reads 0.3072 / 0.6665 against the in-memory row's 0.308 / 0.662 and ParadeDB's 0.3084 / 0.6619 — so what is left is analysis, not - arithmetic. The default pool of 1,000 reads 0.2999 / 0.5636: on SciFact, where a thirteen-term - query matches half the corpus, the pool is the whole 0.10 difference. That is the trade - `--backend-pool` states, and the reason the row prints the pool it ran with. + arithmetic. The gate is a candidate-recall bound, not a scoring change: it generates candidates + from the query's rarest terms and scores them with the whole query. With the gate off, the pool of + 1,000 reads 0.2999 / 0.5636, because the candidates are chosen by `ts_rank_cd` coverage rather + than by idf — on SciFact that truncation is the whole 0.10 difference. The numbers behind the + choice, and what it costs in milliseconds, are on the [backend guide](backends.md#bm25-over-stock-postgresql-postgresbm25searchengine). - **`ts_rank_cd`'s and ParadeDB's scores are PostgreSQL-native and not numerically comparable** with `Bm25Scorer`; the stock-PostgreSQL BM25 row's are `Bm25Scorer`'s and are. The metric columns compare either way. The analysis is PostgreSQL's own, not this library's tokenizer, which diff --git a/docs/index.md b/docs/index.md index cf274afb..58bdf764 100644 --- a/docs/index.md +++ b/docs/index.md @@ -118,8 +118,8 @@ twice. Every figure below is reproducible with a command on the **0.305** and **0.655**, `ts_rank_cd` reads **0.1864** and **0.0188**: its matcher ANDs every term, and OR-ing them (`--backend-match any`) recovers **0.0988** on SciFact but only **0.1881** on NFCorpus, so the rest of that gap is the ranker. The stock-PostgreSQL BM25 engine closes it - with no extension: **0.3072** and **0.6665** at an unbounded candidate pool, **0.2999** and - **0.5636** at its default 1,000. pgvector at `ef_search` 10/40/160 reads **0.2882 / 0.2960 / 0.3013** on + with no extension: **0.3072** and **0.6665** at its default candidate gate, **0.2999** and + **0.5636** with the gate off. pgvector at `ef_search` 10/40/160 reads **0.2882 / 0.2960 / 0.3013** on NFCorpus against the exact in-memory dense row's **0.304**, and **0.5998 / 0.6528 / 0.6550** on SciFact against **0.655**: that gap is the index's cost in nDCG. ArguAna produces no row from any of the three: the text engines reject 2 of its 1,406 test queries, and the vector engine rejects diff --git a/src/LexiSharp.Postgres/PostgresBm25Options.cs b/src/LexiSharp.Postgres/PostgresBm25Options.cs index 93ec60f3..0397174a 100644 --- a/src/LexiSharp.Postgres/PostgresBm25Options.cs +++ b/src/LexiSharp.Postgres/PostgresBm25Options.cs @@ -64,6 +64,9 @@ public sealed record PostgresBm25Options /// How many matches are re-ranked with BM25 when is unset. public const int DefaultCandidatePool = 1000; + /// How many candidate rows the rare-term gate may read when unset. + public const int DefaultRareTermGateBudget = 1000; + /// /// The most candidates one query may ask for, shared with every SQL engine's per-query fetch /// ceiling. = 0 means this, not an unbounded scan. @@ -89,6 +92,48 @@ public sealed record PostgresBm25Options /// public PostgresMatchMode MatchMode { get; init; } = PostgresMatchMode.AnyTerm; + /// + /// Whether candidate selection is generated from the query's rare terms (default true). + /// + /// + /// + /// A GIN index cannot return rows in score order, so the alternative is to score every match + /// before the pool cut — and on a corpus where a query's common terms match most of it, that is + /// the whole cost of a search. BM25's own idf says the rare terms are the discriminative ones: + /// a term held by ten documents carries ~6 nats of idf where one held by half the corpus + /// carries ~0.4. The gate is therefore the disjunction of the rarest of the query's terms, + /// taken by ascending document frequency until rows would be + /// read, and only those matches are ranked. The full query still scores them. + /// + /// + /// This is a recall bound, like : a document holding none of + /// the gated terms cannot be returned, however much of the common part of the query it matches. + /// It falls back to the whole query when even the rarest term exceeds the budget, so a corpus + /// whose terms are all common keeps the previous behaviour. Measured on BEIR, see the backend + /// guide. + /// + /// + public bool RareTermGate { get; init; } = true; + + /// + /// How many candidate rows the gate may read (default 1,000). Only read when + /// is on. + /// + /// + /// + /// The gate is the union of the selected terms' postings, and the sum of their document + /// frequencies is an upper bound on its size — so this is a ceiling on rows read, not a + /// frequency threshold whose meaning would change with the corpus. Terms are added by + /// ascending document frequency while the running sum stays inside the budget; if the rarest + /// term alone exceeds it, the query falls back to the whole match set. + /// + /// + /// Raise it for recall, lower it for speed. It is a cost bound first: the query's rare terms + /// fit inside it on the corpora this was measured on, which is what makes the gate cheap. + /// + /// + public int RareTermGateBudget { get; init; } = DefaultRareTermGateBudget; + /// /// How many matches are re-ranked with BM25 (default 1000). /// @@ -112,7 +157,8 @@ public sealed record PostgresBm25Options AsIndexOptions().IsValid && !double.IsNaN(K1) && !double.IsInfinity(K1) && K1 >= 0 && !double.IsNaN(B) && !double.IsInfinity(B) && B is >= 0 and <= 1 - && CandidatePool >= 0; + && CandidatePool >= 0 + && RareTermGateBudget >= 1; /// Fully qualified table name, e.g. public.lexisharp_documents. public string QualifiedTableName => AsIndexOptions().QualifiedTableName; diff --git a/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs b/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs index 1a7b3433..3d614080 100644 --- a/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs +++ b/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs @@ -21,6 +21,16 @@ namespace LexiSharp.Postgres; /// index matched — not a second tokenizer's opinion of them. /// /// +/// The candidate gate. A GIN index returns posting lists, not rows in score order, so the +/// only way to cut the cost of a query whose common terms match most of the corpus is to rank a +/// smaller set. generates candidates from the +/// query's terms whose document frequency is at most +/// — the terms BM25's idf says +/// are the discriminative ones — and scores those with the full query. It is a recall bound: +/// a document holding none of the rare terms is not returned. See the backend guide for what it +/// measured. +/// +/// /// What the pool costs. PostgreSQL selects the /// best matches by ts_rank_cd and the engine re-ranks that set; scoring the whole match set /// would make a query proportional to the corpus and mean shipping every tsvector to the @@ -244,6 +254,22 @@ public IReadOnlyList Search( filters.Apply(command); match.Bind(command); + // The candidate gate: the query's rare terms when it has any, the whole query otherwise. + // A GIN index cannot rank, so this is what keeps the scan proportional to the + // discriminative part of the query instead of to every document its common part matches. + string gate = match.Gate; + + // Only for a free-text query. A mixed query's gate is the phrase conjunction, and the free + // terms beside it are scoring-only by contract: substituting the rare-term disjunction for + // the phrases would let a document without the phrase in. + if (_options.RareTermGate + && !match.HasPhrases + && GateTerms(queryTerms, statistics.DocumentFrequency, _options.RareTermGateBudget) is { Length: > 0 } gateTerms) + { + gate = "tsv @@ (SELECT string_agg(quote_literal(term), ' | ') FROM unnest(@gateTerms::text[]) AS term)::tsquery"; + command.Parameters.AddWithValue("gateTerms", gateTerms); + } + // Two shapes in one statement: the inner query ranks the match set and keeps the top // @pool, carrying only (id, score) so the sort stays in memory; the outer join then reads // one tsv per candidate. Selecting content and jsonb here as well would put ~1 KB of @@ -254,7 +280,7 @@ public IReadOnlyList Search( JOIN ( SELECT id, {match.Rank} AS score FROM {_options.QualifiedTableName} - WHERE ({match.Gate}){filters.Fragment} + WHERE ({gate}){filters.Fragment} ORDER BY score DESC, {PostgresSchema.OrdinalIdTieBreak} LIMIT @pool ) s ON t.id = s.id; @@ -402,6 +428,65 @@ internal static int PoolSize(PostgresBm25Options options, SearchOptions searchOp PostgresBm25Options.MaximumCandidatePool); } + /// + /// The query's terms the candidate gate matches on: the rarest ones, by ascending document + /// frequency, while their summed document frequency stays inside . + /// + /// + /// + /// The sum of the selected terms' document frequencies is an upper bound on the gate's match + /// set, so the budget is a ceiling on rows read rather than a frequency threshold whose meaning + /// would change with the corpus. Terms are added while they fit; when even the rarest one does + /// not, the caller falls back to the whole query. + /// + /// + /// Absent terms (df = 0) are skipped: gating on one would match nothing, and it cannot + /// contribute to a score either. The order is frequency then ordinal, so the selection does not + /// depend on the order a dictionary happened to produce. + /// + /// + internal static string[] GateTerms( + IReadOnlyList terms, + Func documentFrequency, + int budget) + { + ArgumentNullException.ThrowIfNull(terms); + ArgumentNullException.ThrowIfNull(documentFrequency); + + if (budget <= 0) + return []; + + var candidates = new List<(string Term, int DocumentFrequency)>(terms.Count); + + foreach (string term in terms) + { + int frequency = documentFrequency(term); + + if (frequency > 0) + candidates.Add((term, frequency)); + } + + candidates.Sort(static (left, right) => + { + int byFrequency = left.DocumentFrequency.CompareTo(right.DocumentFrequency); + return byFrequency != 0 ? byFrequency : string.CompareOrdinal(left.Term, right.Term); + }); + + var selected = new List(candidates.Count); + long bound = 0; + + foreach (var candidate in candidates) + { + if (bound + candidate.DocumentFrequency > budget) + break; + + selected.Add(candidate.Term); + bound += candidate.DocumentFrequency; + } + + return [.. selected]; + } + private string[] ReadQueryLexemes(NpgsqlConnection connection, string query) { using var command = connection.CreateCommand(); diff --git a/src/LexiSharp.Postgres/PostgresMatchExpression.cs b/src/LexiSharp.Postgres/PostgresMatchExpression.cs index 08867fe9..56388007 100644 --- a/src/LexiSharp.Postgres/PostgresMatchExpression.cs +++ b/src/LexiSharp.Postgres/PostgresMatchExpression.cs @@ -27,11 +27,12 @@ internal sealed class PostgresMatchExpression private readonly IReadOnlyList> _parameters; private PostgresMatchExpression( - string gate, string rank, IReadOnlyList> parameters) + string gate, string rank, IReadOnlyList> parameters, bool hasPhrases) { Gate = gate; Rank = rank; _parameters = parameters; + HasPhrases = hasPhrases; } /// The tsquery a row must match, as an SQL predicate on tsv. @@ -40,6 +41,13 @@ private PostgresMatchExpression( /// The expression ts_rank_cd is called with, weights included. public string Rank { get; } + /// + /// Whether the gate is made of quoted phrases. A gate that is false matches the query's + /// terms by the match mode; one that is true is a conjunction of phrase predicates, and + /// nothing may be substituted for it — the phrase is the query's contract. + /// + public bool HasPhrases { get; } + /// Builds the expression for one query. /// Naming, analysis and match-mode options. /// The raw query text. @@ -94,7 +102,7 @@ public static PostgresMatchExpression Build(PostgresIndexOptions options, string ? $"ts_rank_cd({weights}, tsv, {rank})" : $"ts_rank_cd(tsv, {rank})"; - return new PostgresMatchExpression(gate, rankSql, parameters); + return new PostgresMatchExpression(gate, rankSql, parameters, segments.Phrases.Count > 0); } /// Adds the query's parameters to a command, under the names the expressions use. diff --git a/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs b/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs index 923f7e12..8b0577d3 100644 --- a/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs +++ b/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs @@ -18,14 +18,19 @@ public class PostgresBm25SearchEngineTests private static string Table(string suffix) => $"lexisharp_bm25_{suffix}_{Guid.NewGuid():N}"[..44]; - private static PostgresBm25SearchEngine NewEngine(string suffix, Action? configure = null) + private static PostgresBm25SearchEngine NewEngine( + string suffix, Func? configure = null) { var options = new PostgresBm25Options { Table = Table(suffix) }; - configure?.Invoke(options); - return new PostgresBm25SearchEngine(ConnectionString!, options); + + return new PostgresBm25SearchEngine( + ConnectionString!, configure is null ? options : configure(options)); } - private static void Run(string suffix, Action body, Action? configure = null) + private static void Run( + string suffix, + Action body, + Func? configure = null) { using var engine = NewEngine(suffix, configure); @@ -284,6 +289,108 @@ public void PoolSize_WidensToThePageAndCapsAtTheCeiling( Assert.Equal(expected, pool); } + [Fact] + public void GateTerms_TakesTheRarestTermsThatFitTheBudget() + { + var frequencies = new Dictionary(StringComparer.Ordinal) + { + ["common"] = 900, + ["rare"] = 5, + ["rarer"] = 1, + ["absent"] = 0, + }; + + string[] terms = PostgresBm25SearchEngine.GateTerms( + ["common", "rare", "rarer", "absent"], term => frequencies[term], budget: 10); + + // 1 + 5 <= 10, and adding 900 would not. + Assert.Equal(["rarer", "rare"], terms); + } + + [Fact] + public void GateTerms_FallsBackWhenTheRarestTermExceedsTheBudget() + { + var frequencies = new Dictionary { ["a"] = 20, ["b"] = 30 }; + + string[] terms = PostgresBm25SearchEngine.GateTerms( + ["a", "b"], term => frequencies[term], budget: 10); + + // Empty is the caller's signal to gate on the whole query: a gate it cannot afford is not + // a smaller gate, it is a different query. + Assert.Empty(terms); + } + + [Fact] + public void GateTerms_IsDeterministicOnEqualFrequencies() + { + var frequencies = new Dictionary { ["b"] = 5, ["a"] = 5 }; + + string[] terms = PostgresBm25SearchEngine.GateTerms( + ["b", "a"], term => frequencies[term], budget: 100); + + Assert.Equal(["a", "b"], terms); + } + + [SkippableFact] + public void RareTermGate_ExcludesADocumentThatMatchesOnlyCommonTerms() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + // df(alpha) = 1, df(beta) = 2. With a budget of 2, only alpha fits: the gate is alpha, and + // d2 — which matches beta but not alpha — is not a candidate. With the gate off, it is. + Run("gate", engine => + { + engine.Index( + [ + new("d1", "alpha beta"), + new("d2", "beta gamma"), + new("d3", "gamma delta"), + ]); + + Assert.Equal("d1", Assert.Single(engine.Search("alpha beta")).DocumentId); + }, options => options with { RareTermGateBudget = 2 }); + + Run("nogate", engine => + { + engine.Index( + [ + new("d1", "alpha beta"), + new("d2", "beta gamma"), + new("d3", "gamma delta"), + ]); + + Assert.Equal( + ["d1", "d2"], + engine.Search("alpha beta", new SearchOptions(Limit: 10)) + .Select(result => result.DocumentId) + .Order(StringComparer.Ordinal)); + }, options => options with { RareTermGateBudget = 2, RareTermGate = false }); + } + + [SkippableFact] + public void RareTermGate_FallsBackToTheWholeQueryWhenNothingFitsTheBudget() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + // df(beta) = 2 and df(gamma) = 2, both above a budget of 1: the gate is empty, so the + // engine ranks the whole match set and every matching document comes back. + Run("gatefallback", engine => + { + engine.Index( + [ + new("d1", "alpha beta"), + new("d2", "beta gamma"), + new("d3", "gamma delta"), + ]); + + Assert.Equal( + ["d1", "d2", "d3"], + engine.Search("beta gamma", new SearchOptions(Limit: 10)) + .Select(result => result.DocumentId) + .Order(StringComparer.Ordinal)); + }, options => options with { RareTermGateBudget = 1 }); + } + [Fact] public void Constructor_RefusesInvalidRankingParameters() { diff --git a/tests/LexiSharp.Tests/PublicApiBaseline.txt b/tests/LexiSharp.Tests/PublicApiBaseline.txt index dc2e4465..ddaad345 100644 --- a/tests/LexiSharp.Tests/PublicApiBaseline.txt +++ b/tests/LexiSharp.Tests/PublicApiBaseline.txt @@ -402,6 +402,7 @@ F|LexiSharp.Onnx.OnnxScoreTransform|Softmax|LexiSharp.Onnx.OnnxScoreTransform|st F|LexiSharp.Postgres.PostgresBm25Options|DefaultB|System.Double|static|0.75 F|LexiSharp.Postgres.PostgresBm25Options|DefaultCandidatePool|System.Int32|static|1000 F|LexiSharp.Postgres.PostgresBm25Options|DefaultK1|System.Double|static|1.5 +F|LexiSharp.Postgres.PostgresBm25Options|DefaultRareTermGateBudget|System.Int32|static|1000 F|LexiSharp.Postgres.PostgresBm25Options|MaximumCandidatePool|System.Int32|static|100000 F|LexiSharp.Postgres.PostgresFieldWeight|A|LexiSharp.Postgres.PostgresFieldWeight|static|0 F|LexiSharp.Postgres.PostgresFieldWeight|B|LexiSharp.Postgres.PostgresFieldWeight|static|1 @@ -2682,6 +2683,8 @@ P|LexiSharp.Postgres.PostgresBm25Options|FieldWeights|System.Collections.Generic P|LexiSharp.Postgres.PostgresBm25Options|K1|System.Double|getset P|LexiSharp.Postgres.PostgresBm25Options|MatchMode|LexiSharp.Postgres.PostgresMatchMode|getset P|LexiSharp.Postgres.PostgresBm25Options|QualifiedTableName|System.String|get +P|LexiSharp.Postgres.PostgresBm25Options|RareTermGateBudget|System.Int32|getset +P|LexiSharp.Postgres.PostgresBm25Options|RareTermGate|System.Boolean|getset P|LexiSharp.Postgres.PostgresBm25Options|Schema|System.String|getset P|LexiSharp.Postgres.PostgresBm25Options|Table|System.String|getset P|LexiSharp.Postgres.PostgresBm25Options|TextSearchConfig|System.String|getset From 01ae1c6019839d2467dbb06f18d6b96dd1bf2395 Mon Sep 17 00:00:00 2001 From: manuc66 Date: Mon, 16 Nov 2026 23:18:45 +0100 Subject: [PATCH 4/6] Split the three methods CodeFactor flags as complex No behaviour change: the same statements, in named pieces. - `BackendLane.Parse` (55) was one method holding every backend's rule. Each group is now a `Refuse...` method that returns early for the backends it does not concern, so the parse itself reads as a list of the rules applied. This also fixes a stale message: `--backend-gate-df` is `--backend-gate-budget`. - `PostgresBm25SearchEngine.Search` (22) is now the sequence of its steps: build the candidate command, read the candidates, score them, read the page, assemble the results. - `PostgresTsvText.Parse` (26) keeps the loop and hands each lexeme to a reader and a recorder, which is also where the `keep` decision now lives. --- .../PostgresBackendBenchmark.cs | 258 +++++++++++------- .../PostgresBm25SearchEngine.cs | 98 +++++-- src/LexiSharp.Postgres/PostgresTsvText.cs | 142 ++++++---- 3 files changed, 317 insertions(+), 181 deletions(-) diff --git a/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs b/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs index 8847e629..54128e7f 100644 --- a/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs +++ b/bench/LexiSharp.Eval/PostgresBackendBenchmark.cs @@ -123,55 +123,90 @@ public static BackendLane Parse( { ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); - BackendKind parsed = kind switch - { - "lexical" => BackendKind.Lexical, - "bm25" => BackendKind.Bm25, - "paradedb" => BackendKind.ParadeDb, - "vector" => BackendKind.Vector, - "fuzzy" => BackendKind.Fuzzy, - "sparse" => BackendKind.Sparse, - _ => throw new ArgumentException( - $"--backend expects 'lexical', 'bm25', 'paradedb', 'vector', 'fuzzy' or 'sparse', got '{kind}'."), - }; + BackendKind parsed = ParseKind(kind); + + RefuseSparseOptions(parsed, sparseVectorsDirectory, bm25, textSearchConfig, efSearchValues); + RefuseLexicalOptions(parsed, bm25); + RefuseParadeDbOptions(parsed, textSearchConfig); + RefuseVectorOptions(parsed, bm25, textSearchConfig); + RefuseFuzzyOptions(parsed, bm25, textSearchConfig); + RefuseBm25OnlyOptions(parsed, candidatePool, rareTermGate, rareTermGateBudget); + RefuseMatchMode(parsed, matchMode); + RefuseEfSearch(parsed, efSearchValues); + RefuseThresholds(parsed, similarityThresholds); + RefuseFieldWeights(parsed, fieldWeights); + + return new BackendLane( + parsed, connectionString, textSearchConfig, bm25, efSearchValues, fieldWeights, + similarityThresholds, + sparseVectorsDirectory is null ? null : Path.GetFullPath(sparseVectorsDirectory), + matchMode, + candidatePool, + rareTermGate, + rareTermGateBudget); + } + + private static BackendKind ParseKind(string kind) => kind switch + { + "lexical" => BackendKind.Lexical, + "bm25" => BackendKind.Bm25, + "paradedb" => BackendKind.ParadeDb, + "vector" => BackendKind.Vector, + "fuzzy" => BackendKind.Fuzzy, + "sparse" => BackendKind.Sparse, + _ => throw new ArgumentException( + $"--backend expects 'lexical', 'bm25', 'paradedb', 'vector', 'fuzzy' or 'sparse', got '{kind}'."), + }; - if (parsed == BackendKind.Sparse) + /// A sparse row needs its vectors directory and reads none of the text options. + private static void RefuseSparseOptions( + BackendKind parsed, + string? sparseVectorsDirectory, + Bm25Parameters? bm25, + string textSearchConfig, + IReadOnlyList? efSearchValues) + { + if (parsed != BackendKind.Sparse) + return; + + if (string.IsNullOrWhiteSpace(sparseVectorsDirectory)) { - if (string.IsNullOrWhiteSpace(sparseVectorsDirectory)) - { - throw new ArgumentException( - "--backend sparse measures PostgresSparseSearchEngine over precomputed learned-sparse " - + "vectors, and the engine has no model of its own: give them with --sparse , the " - + "same directory the in-memory sparse row reads. --dense is not a substitute — those " - + "are the dense lane's embeddings, not sparse term weights."); - } + throw new ArgumentException( + "--backend sparse measures PostgresSparseSearchEngine over precomputed learned-sparse " + + "vectors, and the engine has no model of its own: give them with --sparse , the " + + "same directory the in-memory sparse row reads. --dense is not a substitute — those " + + "are the dense lane's embeddings, not sparse term weights."); + } - if (bm25 is not null) - { - throw new ArgumentException( - "--backend-bm25 sets k1/b, and the sparse row ranks by the inner product of two " - + "learned sparse vectors: no BM25 parameter reaches it. Drop the flag."); - } + if (bm25 is not null) + { + throw new ArgumentException( + "--backend-bm25 sets k1/b, and the sparse row ranks by the inner product of two " + + "learned sparse vectors: no BM25 parameter reaches it. Drop the flag."); + } - if (textSearchConfig != "simple") - { - throw new ArgumentException( - $"--backend-config '{textSearchConfig}' is PostgreSQL's text search configuration, " - + "which the sparse engine never reads: the shared table's tsv column is written but " - + "the sparse ranking is a dot product over term coordinates. Drop the flag, or use " - + "--backend lexical."); - } + if (textSearchConfig != "simple") + { + throw new ArgumentException( + $"--backend-config '{textSearchConfig}' is PostgreSQL's text search configuration, " + + "which the sparse engine never reads: the shared table's tsv column is written but " + + "the sparse ranking is a dot product over term coordinates. Drop the flag, or use " + + "--backend lexical."); + } - if (efSearchValues is { Count: > 0 }) - { - throw new ArgumentException( - "--backend-ef sets pgvector's hnsw.ef_search, and PostgresSparseSearchEngine has no " - + "option that reaches it: the sparse row is measured at pgvector's default 40. Set " - + "it on the server (a database-level setting applies, because the engine never " - + "overrides it) and say so in the row beside the number, or drop the flag."); - } + if (efSearchValues is { Count: > 0 }) + { + throw new ArgumentException( + "--backend-ef sets pgvector's hnsw.ef_search, and PostgresSparseSearchEngine has no " + + "option that reaches it: the sparse row is measured at pgvector's default 40. Set " + + "it on the server (a database-level setting applies, because the engine never " + + "overrides it) and say so in the row beside the number, or drop the flag."); } + } + /// ts_rank_cd takes no k1/b. + private static void RefuseLexicalOptions(BackendKind parsed, Bm25Parameters? bm25) + { if (parsed == BackendKind.Lexical && bm25 is not null) { throw new ArgumentException( @@ -179,83 +214,118 @@ public static BackendLane Parse( + "and takes no parameter. Drop the flag, or use --backend bm25 (stock PostgreSQL, no " + "extension) or --backend paradedb."); } + } - if (parsed != BackendKind.Bm25 && candidatePool is not null) + /// ParadeDB builds its own index and does not read the text search configuration. + private static void RefuseParadeDbOptions(BackendKind parsed, string textSearchConfig) + { + if (parsed == BackendKind.ParadeDb && textSearchConfig != "simple") { throw new ArgumentException( - "--backend-pool bounds the matches the BM25 engine re-ranks, and no other backend has a " - + "pool: the lexical row ranks in SQL, the vector and sparse rows rank by distance after " - + "an index scan, and ParadeDB and fuzzy rank their whole match set. Drop the flag, or " - + "use --backend bm25."); + $"--backend-config '{textSearchConfig}' selects PostgreSQL's text search configuration, " + + "which the lexical engine reads and ParadeDB does not. Drop the flag, or use " + + "--backend lexical."); } + } - if (parsed != BackendKind.Bm25 && (rareTermGate is not null || rareTermGateBudget is not null)) + /// An embedding index has no BM25 parameter and no text search configuration. + private static void RefuseVectorOptions(BackendKind parsed, Bm25Parameters? bm25, string textSearchConfig) + { + if (parsed != BackendKind.Vector) + return; + + if (bm25 is not null) { throw new ArgumentException( - "--backend-gate and --backend-gate-df configure the BM25 engine's candidate gate, which " - + "selects candidates from the query's rare terms; the other backends have no such gate. " - + "Drop the flags, or use --backend bm25."); + "--backend-bm25 sets k1/b, which no BM25 parameter reaches in a vector index: the " + + "ranking there is cosine distance over embeddings. Drop the flag."); } - if (parsed is not (BackendKind.Lexical or BackendKind.Bm25) && matchMode is not null) + if (textSearchConfig != "simple") { throw new ArgumentException( - "--backend-match selects the tsquery rule the lexical and BM25 engines match with, and the " - + "vector, sparse, ParadeDB and fuzzy rows rank by an embedding, a dot product, pdb.score or " - + "trigram similarity. Drop the flag, or use --backend lexical or --backend bm25."); + $"--backend-config '{textSearchConfig}' is PostgreSQL's text search configuration, " + + "which the vector engine never reads: its documents are ranked by embedding. Drop " + + "the flag, or use --backend lexical."); } + } - if (parsed == BackendKind.ParadeDb && textSearchConfig != "simple") + /// A trigram index has no BM25 parameter and no text search configuration. + private static void RefuseFuzzyOptions(BackendKind parsed, Bm25Parameters? bm25, string textSearchConfig) + { + if (parsed != BackendKind.Fuzzy) + return; + + if (bm25 is not null) { throw new ArgumentException( - $"--backend-config '{textSearchConfig}' selects PostgreSQL's text search configuration, " - + "which the lexical engine reads and ParadeDB does not. Drop the flag, or use " - + "--backend lexical."); + "--backend-bm25 sets k1/b, which no BM25 parameter reaches in a trigram index: the " + + "ranking there is similarity over character trigrams. Drop the flag."); } - if (parsed == BackendKind.Vector) + if (textSearchConfig != "simple") { - if (bm25 is not null) - { - throw new ArgumentException( - "--backend-bm25 sets k1/b, which no BM25 parameter reaches in a vector index: the " - + "ranking there is cosine distance over embeddings. Drop the flag."); - } + throw new ArgumentException( + $"--backend-config '{textSearchConfig}' is PostgreSQL's text search configuration, " + + "which the fuzzy engine never reads: its documents are matched by trigram " + + "similarity. Drop the flag, or use --backend lexical."); + } + } - if (textSearchConfig != "simple") - { - throw new ArgumentException( - $"--backend-config '{textSearchConfig}' is PostgreSQL's text search configuration, " - + "which the vector engine never reads: its documents are ranked by embedding. Drop " - + "the flag, or use --backend lexical."); - } + /// The pool and the gate exist only on the stock-PostgreSQL BM25 engine. + private static void RefuseBm25OnlyOptions( + BackendKind parsed, + int? candidatePool, + bool? rareTermGate, + int? rareTermGateBudget) + { + if (parsed == BackendKind.Bm25) + return; + + if (candidatePool is not null) + { + throw new ArgumentException( + "--backend-pool bounds the matches the BM25 engine re-ranks, and no other backend has a " + + "pool: the lexical row ranks in SQL, the vector and sparse rows rank by distance after " + + "an index scan, and ParadeDB and fuzzy rank their whole match set. Drop the flag, or " + + "use --backend bm25."); } - if (parsed == BackendKind.Fuzzy) + if (rareTermGate is not null || rareTermGateBudget is not null) { - if (bm25 is not null) - { - throw new ArgumentException( - "--backend-bm25 sets k1/b, which no BM25 parameter reaches in a trigram index: the " - + "ranking there is similarity over character trigrams. Drop the flag."); - } + throw new ArgumentException( + "--backend-gate and --backend-gate-budget configure the BM25 engine's candidate gate, " + + "which selects candidates from the query's rare terms; the other backends have no " + + "such gate. Drop the flags, or use --backend bm25."); + } + } - if (textSearchConfig != "simple") - { - throw new ArgumentException( - $"--backend-config '{textSearchConfig}' is PostgreSQL's text search configuration, " - + "which the fuzzy engine never reads: its documents are matched by trigram " - + "similarity. Drop the flag, or use --backend lexical."); - } + /// The tsquery match rule is the lexical and BM25 engines' alone. + private static void RefuseMatchMode(BackendKind parsed, PostgresMatchMode? matchMode) + { + if (parsed is not (BackendKind.Lexical or BackendKind.Bm25) && matchMode is not null) + { + throw new ArgumentException( + "--backend-match selects the tsquery rule the lexical and BM25 engines match with, and the " + + "vector, sparse, ParadeDB and fuzzy rows rank by an embedding, a dot product, pdb.score or " + + "trigram similarity. Drop the flag, or use --backend lexical or --backend bm25."); } + } + /// hnsw.ef_search is the vector engine's. + private static void RefuseEfSearch(BackendKind parsed, IReadOnlyList? efSearchValues) + { if (parsed != BackendKind.Vector && efSearchValues is { Count: > 0 }) { throw new ArgumentException( "--backend-ef sets pgvector's hnsw.ef_search, which only the vector backend reads. " + "Drop the flag, or use --backend vector."); } + } + /// pg_trgm's threshold is the fuzzy engine's. + private static void RefuseThresholds(BackendKind parsed, IReadOnlyList? similarityThresholds) + { if (parsed != BackendKind.Fuzzy && similarityThresholds is { Count: > 0 }) { throw new ArgumentException( @@ -263,7 +333,12 @@ public static BackendLane Parse( + "reads: the other engines match by term or by embedding and have no threshold. Drop " + "the flag, or use --backend fuzzy."); } + } + /// setweight classes are read by ts_rank_cd, so they are the lexical row's. + private static void RefuseFieldWeights( + BackendKind parsed, IReadOnlyDictionary? fieldWeights) + { if (parsed != BackendKind.Lexical && fieldWeights is { Count: > 0 }) { throw new ArgumentException( @@ -272,15 +347,6 @@ public static BackendLane Parse( + "embedding, by a sparse dot product, by pdb.score or by trigram similarity. Drop the " + "flag, or use --backend lexical."); } - - return new BackendLane( - parsed, connectionString, textSearchConfig, bm25, efSearchValues, fieldWeights, - similarityThresholds, - sparseVectorsDirectory is null ? null : Path.GetFullPath(sparseVectorsDirectory), - matchMode, - candidatePool, - rareTermGate, - rareTermGateBudget); } /// diff --git a/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs b/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs index 3d614080..cc130062 100644 --- a/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs +++ b/src/LexiSharp.Postgres/PostgresBm25SearchEngine.cs @@ -248,7 +248,48 @@ public IReadOnlyList Search( var statistics = _statistics ??= PostgresBm25Statistics.Load(connection, _indexOptions); var match = PostgresMatchExpression.Build(_indexOptions, query); - using var command = connection.CreateCommand(); + using var command = CreateCandidatesCommand(connection, match, statistics, queryTerms, options); + + List candidates = ReadCandidates(command, keptTerms, cancellationToken); + + if (candidates.Count == 0) + return Array.Empty(); + + List<(string DocumentId, double Score)> scored = + ScoreCandidates(candidates, queryTerms, statistics, options); + + List<(string DocumentId, double Score)> page = scored + .Skip(options.Offset) + .Take(options.Limit) + .ToList(); + + if (page.Count == 0) + return Array.Empty(); + + Dictionary documents = ReadDocuments(connection, page, cancellationToken); + + return BuildResults(page, documents); + } + + /// + /// Builds the candidate query: the match expression, the metadata filters, the rare-term gate + /// and the pool, in one statement. + /// + /// + /// Two shapes in one statement: the inner query ranks the match set and keeps the top + /// @pool, carrying only (id, score) so the sort stays in memory; the outer join + /// then reads one tsv per candidate. Selecting content and jsonb here as well would put + /// about a kilobyte of payload per row into the sort, which PostgreSQL answered by spilling it + /// to disk. + /// + private NpgsqlCommand CreateCandidatesCommand( + NpgsqlConnection connection, + PostgresMatchExpression match, + PostgresBm25Statistics statistics, + string[] queryTerms, + SearchOptions options) + { + var command = connection.CreateCommand(); var filters = PostgresMetadataFilterSql.Build(options.Filters); filters.Apply(command); @@ -257,11 +298,12 @@ public IReadOnlyList Search( // The candidate gate: the query's rare terms when it has any, the whole query otherwise. // A GIN index cannot rank, so this is what keeps the scan proportional to the // discriminative part of the query instead of to every document its common part matches. - string gate = match.Gate; - + // // Only for a free-text query. A mixed query's gate is the phrase conjunction, and the free // terms beside it are scoring-only by contract: substituting the rare-term disjunction for // the phrases would let a document without the phrase in. + string gate = match.Gate; + if (_options.RareTermGate && !match.HasPhrases && GateTerms(queryTerms, statistics.DocumentFrequency, _options.RareTermGateBudget) is { Length: > 0 } gateTerms) @@ -270,11 +312,7 @@ public IReadOnlyList Search( command.Parameters.AddWithValue("gateTerms", gateTerms); } - // Two shapes in one statement: the inner query ranks the match set and keeps the top - // @pool, carrying only (id, score) so the sort stays in memory; the outer join then reads - // one tsv per candidate. Selecting content and jsonb here as well would put ~1 KB of - // payload per row into the sort, which PostgreSQL answered by spilling it to disk. - string candidatesSql = $""" + command.CommandText = $""" SELECT t.id, t.tsv::text FROM {_options.QualifiedTableName} t JOIN ( @@ -284,12 +322,17 @@ public IReadOnlyList Search( ORDER BY score DESC, {PostgresSchema.OrdinalIdTieBreak} LIMIT @pool ) s ON t.id = s.id; - """; + """; // NOSONAR:S2077 (identifiers validated and quoted; query text parameterized) - // Identifiers/config only are interpolated (validated [A-Za-z0-9_] and quoted); query text is parameterized. - command.CommandText = candidatesSql; // NOSONAR:S2077 command.Parameters.AddWithValue("pool", PoolSize(_options, options)); + return command; + } + + /// Reads the candidate rows: an id and the term frequencies the gate and the scorer need. + private static List ReadCandidates( + NpgsqlCommand command, HashSet keptTerms, CancellationToken cancellationToken) + { var candidates = new List(); using (var cancelling = PostgresCancellation.Register(command, cancellationToken)) @@ -304,9 +347,16 @@ LIMIT @pool } } - if (candidates.Count == 0) - return Array.Empty(); + return candidates; + } + /// Scores the candidates with BM25 and returns the survivors, best first. + private List<(string DocumentId, double Score)> ScoreCandidates( + List candidates, + string[] queryTerms, + PostgresBm25Statistics statistics, + SearchOptions options) + { var view = new PostgresBm25IndexView(statistics, candidates); var scored = new List<(string DocumentId, double Score)>(candidates.Count); @@ -334,19 +384,17 @@ LIMIT @pool return byScore != 0 ? byScore : string.CompareOrdinal(left.DocumentId, right.DocumentId); }); - List<(string DocumentId, double Score)> page = scored - .Skip(options.Offset) - .Take(options.Limit) - .ToList(); - - if (page.Count == 0) - return Array.Empty(); - - Dictionary documents = ReadDocuments(connection, page, cancellationToken); + return scored; + } - // A document deleted between the two reads is skipped rather than returned without its - // content. The window is not refilled: the alternative is a third query for a row that - // another writer is deleting, and the page is a read of a corpus that moved. + /// + /// Pairs the page's scores with the documents read for it, skipping a row another writer + /// deleted between the two reads. + /// + private static List BuildResults( + List<(string DocumentId, double Score)> page, + Dictionary documents) + { var results = new List(page.Count); foreach (var (documentId, score) in page) diff --git a/src/LexiSharp.Postgres/PostgresTsvText.cs b/src/LexiSharp.Postgres/PostgresTsvText.cs index 054f8e01..087e51a1 100644 --- a/src/LexiSharp.Postgres/PostgresTsvText.cs +++ b/src/LexiSharp.Postgres/PostgresTsvText.cs @@ -57,93 +57,115 @@ public static (Dictionary TermFrequencies, int Length) Parse(string return (terms, length); HashSet.AlternateLookup> lookup = default; - bool filtered = keep is not null; - if (filtered) - lookup = keep!.GetAlternateLookup>(); + if (keep is not null) + lookup = keep.GetAlternateLookup>(); int i = 0; - while (i < text.Length) + while (TryReadLexeme(text, ref i, out int start, out int end, out bool escaped)) { - while (i < text.Length && char.IsWhiteSpace(text[i])) - i++; + if (end <= start) + continue; - if (i >= text.Length) - break; + int count = ReadCount(text, ref i); + length += count; - int start; - int end; - bool escaped = false; + Record(terms, text, start, end, escaped, count, keep, lookup); + } - if (text[i] == '\'') - { - i++; - start = i; + return (terms, length); + } - while (i < text.Length) + /// + /// Reads one lexeme at , skipping leading whitespace and advancing past the + /// closing quote. Returns false at the end of the value. + /// + private static bool TryReadLexeme( + string text, ref int i, out int start, out int end, out bool escaped) + { + start = 0; + end = 0; + escaped = false; + + while (i < text.Length && char.IsWhiteSpace(text[i])) + i++; + + if (i >= text.Length) + return false; + + if (text[i] == '\'') + { + i++; + start = i; + + while (i < text.Length) + { + if (text[i] == '\'') { - if (text[i] == '\'') + // A doubled quote is the escape; a single one closes the lexeme. + if (i + 1 < text.Length && text[i + 1] == '\'') { - // A doubled quote is the escape; a single one closes the lexeme. - if (i + 1 < text.Length && text[i + 1] == '\'') - { - escaped = true; - i += 2; - continue; - } - - break; + escaped = true; + i += 2; + continue; } - i++; + break; } - end = i; - - if (i < text.Length) - i++; + i++; } - else - { - // Defensive: every released version quotes lexemes, but an unquoted one would - // otherwise be swallowed into the next token's positions. - start = i; - while (i < text.Length && text[i] != ':' && !char.IsWhiteSpace(text[i])) - i++; + end = i; - end = i; - } + if (i < text.Length) + i++; - if (end <= start) - continue; + return true; + } - int count = ReadCount(text, ref i); - length += count; + // Defensive: every released version quotes lexemes, but an unquoted one would otherwise be + // swallowed into the next token's positions. + start = i; - if (!filtered) - { - string lexeme = escaped ? Unescape(text, start, end) : text[start..end]; - terms[lexeme] = terms.GetValueOrDefault(lexeme) + count; - continue; - } + while (i < text.Length && text[i] != ':' && !char.IsWhiteSpace(text[i])) + i++; - if (!escaped) - { - if (lookup.TryGetValue(text.AsSpan(start, end - start), out string? kept)) - terms[kept] = terms.GetValueOrDefault(kept) + count; + end = i; + return true; + } - continue; - } + /// Adds one lexeme's occurrences to the frequency map, honoring the keep set. + private static void Record( + Dictionary terms, + string text, + int start, + int end, + bool escaped, + int count, + HashSet? keep, + HashSet.AlternateLookup> lookup) + { + if (keep is null) + { + string lexeme = escaped ? Unescape(text, start, end) : text[start..end]; + terms[lexeme] = terms.GetValueOrDefault(lexeme) + count; + return; + } - string unescaped = Unescape(text, start, end); + if (!escaped) + { + if (lookup.TryGetValue(text.AsSpan(start, end - start), out string? kept)) + terms[kept] = terms.GetValueOrDefault(kept) + count; - if (keep!.Contains(unescaped)) - terms[unescaped] = terms.GetValueOrDefault(unescaped) + count; + return; } - return (terms, length); + string unescaped = Unescape(text, start, end); + + if (keep.Contains(unescaped)) + terms[unescaped] = terms.GetValueOrDefault(unescaped) + count; } /// From 19d646787a6ac70e6f6064ff9101e5c9d2b01781 Mon Sep 17 00:00:00 2001 From: manuc66 Date: Mon, 16 Nov 2026 23:23:50 +0100 Subject: [PATCH 5/6] Drop the index-name comparison from the BM25 notes The BM25 guide and the benchmark lane both said why the other backend is faster by naming its index. The comparison stands without the name: that backend answers from its own inverted index instead of ranking a candidate set inside PostgreSQL. --- docs/backends.md | 9 ++++----- docs/benchmarks.md | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/backends.md b/docs/backends.md index 41a80061..2b344ba2 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -339,9 +339,8 @@ engine.Add(new SearchDocument("1", "the quick brown fox jumps over the lazy dog" engine.Search("quick fox"); // Bm25Scorer scores, not ts_rank_cd's ``` -It exists because `pg_search` (next section) is the faster route to BM25 and the one with the -mature Tantivy index, but it needs a preloaded library, and no major managed service lets you -install one. This engine needs `CREATE EXTENSION unaccent` and nothing else, so on RDS, Cloud SQL +It exists because `pg_search` (next section) is the faster route to BM25, but it needs a preloaded +library, and no major managed service lets you install one. This engine needs `CREATE EXTENSION unaccent` and nothing else, so on RDS, Cloud SQL and Azure Database for PostgreSQL the only server-side step is allowing that one extension. **Where the numbers come from.** `N`, the average document length and every query term's document @@ -397,8 +396,8 @@ server. p50 / mean: - **The pool is no longer the interesting knob.** With the gate on, `pool 0` and `pool 1,000` read the same (SciFact 50.1 vs 49.7 ms in the lane); the pool matters on the ungated path, where it is the thing that truncates. -- **It is still not a speed replacement for `pg_search`.** Tantivy answers from its own inverted - index instead of ranking a candidate set inside PostgreSQL, which is why the extension exists. +- **It is still not a speed replacement for `pg_search`.** That engine answers from its own + inverted index instead of ranking a candidate set inside PostgreSQL, which is why it exists. What this engine buys is portability; the table is its price. - **The statistics build is the first search after a write**, not a search: `ts_stat` walks the table, and the column is measured on its own so it does not hide inside a p99. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 1e316f71..bb035669 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1662,8 +1662,8 @@ What the table says, and what it does not: - **`stats build` is not a search.** It is the `ts_stat` walk a `PostgresBm25SearchEngine` pays on its first search after a write, measured by calling `RefreshStatistics()` explicitly. A write and a search in a tight loop pays ~200-450 ms per corpus of this size. -- **It is not a speed comparison with `pg_search`.** Tantivy answers from its own inverted index - instead of ranking a candidate set inside PostgreSQL, which is why the extension exists. This lane +- **It is not a speed comparison with `pg_search`.** That engine answers from its own inverted + index instead of ranking a candidate set inside PostgreSQL, which is why it exists. This lane measures the portable path; the extension is the one to measure when it can be installed. - **The gate is a recall bound.** Both corpora are scientific and entity-rich, which is its best case; a rare term that is a false friend can gate out relevant documents. `--backend-gate off` From 7492995161ec11d50bc987234fc497de70fa5de4 Mon Sep 17 00:00:00 2001 From: manuc66 Date: Mon, 16 Nov 2026 23:34:27 +0100 Subject: [PATCH 6/6] Cover the BM25 code the scorer does not call, and fix an empty-lexeme hang The coverage report on the new files was mostly measuring the scorer's path: the pool's other IReadOnlyTextIndex members, the engine's schema and option paths, and the parser's defensive branches were never reached. Writing the tests found a real bug. - An empty quoted lexeme (`'':1`) left the ':' unread, so the outer loop saw it forever: `Parse` did not terminate. The position list is now consumed even for a lexeme that is skipped, and a test pins the case. - A statistics factory gives the pool's view something to be built from without a server, and the view's members are pinned: the snapshot answers, the candidate answers, the absent document throws, and Documents / GetTermPositions throw rather than returning an empty answer. - The engine gains tests for the second EnsureSchema call, EnsureSchemaAsync, autoCreateSchema: false, a query of stop words (no lexeme at all), the minimum score, Clear, and ParseQuerySyntax: false. - The parser gains the unquoted lexeme, the unterminated quote, an escaped lexeme under keep, a malformed position list, and a trailing doubled quote. Line coverage of the five new files under the Postgres/Conformance filter is now 100% except one defensive line in the row reader's json guard. --- .../PostgresBm25Statistics.cs | 26 ++++ src/LexiSharp.Postgres/PostgresTsvText.cs | 5 + .../PostgresBm25IndexViewTests.cs | 125 +++++++++++++++ .../PostgresBm25SearchEngineTests.cs | 142 ++++++++++++++++++ tests/LexiSharp.Tests/PostgresTsvTextTests.cs | 60 ++++++++ 5 files changed, 358 insertions(+) create mode 100644 tests/LexiSharp.Tests/PostgresBm25IndexViewTests.cs diff --git a/src/LexiSharp.Postgres/PostgresBm25Statistics.cs b/src/LexiSharp.Postgres/PostgresBm25Statistics.cs index 2609a19b..bb309b5e 100644 --- a/src/LexiSharp.Postgres/PostgresBm25Statistics.cs +++ b/src/LexiSharp.Postgres/PostgresBm25Statistics.cs @@ -104,6 +104,32 @@ public static PostgresBm25Statistics Load(NpgsqlConnection connection, PostgresI return new PostgresBm25Statistics(documentCount, corpusTokenCount, terms); } + /// + /// Builds a snapshot from counts a caller already has. + /// + /// + /// The seam a test uses, and the shape a caller that reads the two counts elsewhere would + /// want. could go through it, but it already has the counts in the form it + /// reads them and copying them twice would be work for no reader. + /// + internal static PostgresBm25Statistics From( + int documentCount, + IReadOnlyDictionary terms) + { + ArgumentNullException.ThrowIfNull(terms); + + var copy = new Dictionary(terms.Count, StringComparer.Ordinal); + long corpusTokenCount = 0; + + foreach (var (term, counts) in terms) + { + copy[term] = new TermStatistics(counts.DocumentFrequency, counts.CorpusFrequency); + corpusTokenCount += counts.CorpusFrequency; + } + + return new PostgresBm25Statistics(documentCount, corpusTokenCount, copy); + } + /// One term's two counts, kept together so they cannot be read apart. private readonly record struct TermStatistics(int DocumentFrequency, int CorpusFrequency); } diff --git a/src/LexiSharp.Postgres/PostgresTsvText.cs b/src/LexiSharp.Postgres/PostgresTsvText.cs index 087e51a1..f1d78499 100644 --- a/src/LexiSharp.Postgres/PostgresTsvText.cs +++ b/src/LexiSharp.Postgres/PostgresTsvText.cs @@ -66,7 +66,12 @@ public static (Dictionary TermFrequencies, int Length) Parse(string while (TryReadLexeme(text, ref i, out int start, out int end, out bool escaped)) { if (end <= start) + { + // Still consume the position list: an empty lexeme is not a token, but leaving its + // ":1" unread would make the outer loop see ':' forever. + _ = ReadCount(text, ref i); continue; + } int count = ReadCount(text, ref i); length += count; diff --git a/tests/LexiSharp.Tests/PostgresBm25IndexViewTests.cs b/tests/LexiSharp.Tests/PostgresBm25IndexViewTests.cs new file mode 100644 index 00000000..a7596589 --- /dev/null +++ b/tests/LexiSharp.Tests/PostgresBm25IndexViewTests.cs @@ -0,0 +1,125 @@ +using LexiSharp.Core; +using LexiSharp.Postgres; +using Xunit; + +namespace LexiSharp.Tests; + +/// +/// The candidate pool as an : the members +/// reads, and the ones it does not, which are pinned here so a future caller finds the stated +/// answer rather than an accidental one. +/// +public class PostgresBm25IndexViewTests +{ + private static PostgresBm25Statistics Statistics() => + PostgresBm25Statistics.From( + 10, + new Dictionary(StringComparer.Ordinal) + { + ["quick"] = (3, 5), + ["brown"] = (2, 2), + }); + + private static PostgresBm25IndexView View() => + new(Statistics(), + [ + new PostgresBm25Candidate( + "d1", new Dictionary(StringComparer.Ordinal) { ["quick"] = 2 }, 4), + new PostgresBm25Candidate( + "d2", new Dictionary(StringComparer.Ordinal) { ["brown"] = 1 }, 2), + ]); + + [Fact] + public void Count_IsTheCandidateCount() => Assert.Equal(2, View().Count); + + [Fact] + public void CorpusStatistics_ComeFromTheSnapshot() + { + PostgresBm25IndexView view = View(); + + Assert.Equal(10, view.StatisticDocumentCount); + Assert.Equal(2, view.VocabularySize); + Assert.Equal(7, view.CorpusTokenCount); + Assert.Equal(0.7, view.AverageDocumentLength, precision: 10); + } + + [Fact] + public void DocumentFrequency_ComesFromTheSnapshot() + { + PostgresBm25IndexView view = View(); + + Assert.Equal(3, view.DocumentFrequency("quick")); + Assert.Equal(0, view.DocumentFrequency("absent")); + Assert.Equal(0, view.DocumentFrequency(null!)); + } + + [Fact] + public void CorpusFrequency_ComesFromTheSnapshot() + { + PostgresBm25IndexView view = View(); + + Assert.Equal(5, view.CorpusFrequency("quick")); + Assert.Equal(0, view.CorpusFrequency("absent")); + Assert.Equal(0, view.CorpusFrequency(null!)); + } + + [Fact] + public void PerDocumentAnswers_ComeFromTheCandidate() + { + PostgresBm25IndexView view = View(); + + Assert.Equal(2, view.TermFrequency("d1", "quick")); + Assert.Equal(0, view.TermFrequency("d1", "brown")); + Assert.Equal(0, view.TermFrequency("d1", null!)); + Assert.Equal(4, view.DocumentLength("d1")); + Assert.True(view.Contains("d1")); + Assert.False(view.Contains("absent")); + Assert.False(view.Contains(null!)); + } + + [Fact] + public void GetTerms_IsOrdinalSorted() + { + var view = new PostgresBm25IndexView( + Statistics(), + [ + new PostgresBm25Candidate( + "d1", + new Dictionary(StringComparer.Ordinal) { ["b"] = 1, ["a"] = 1, ["C"] = 1 }, + 3), + ]); + + Assert.Equal(["C", "a", "b"], view.GetTerms("d1")); + } + + [Fact] + public void UnknownDocument_Throws() + { + PostgresBm25IndexView view = View(); + + Assert.Throws(() => view.TermFrequency("absent", "quick")); + Assert.Throws(() => view.DocumentLength("absent")); + Assert.Throws(() => view.GetTerms("absent")); + } + + [Fact] + public void ThePoolKeepsNoDocumentsAndNoPositions() + { + PostgresBm25IndexView view = View(); + + Assert.Throws(() => view.Documents); + Assert.Throws(() => view.GetTermPositions("d1", "quick")); + Assert.False(view.TryGetDocument("d1", out _)); + } + + [Fact] + public void AnEmptySnapshot_HasNoAverageLength() + { + PostgresBm25Statistics statistics = PostgresBm25Statistics.From( + 0, new Dictionary()); + + Assert.Equal(0, statistics.AverageDocumentLength); + Assert.Equal(0, statistics.VocabularySize); + Assert.Equal(0, statistics.CorpusTokenCount); + } +} diff --git a/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs b/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs index 8b0577d3..b62e4aa7 100644 --- a/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs +++ b/tests/LexiSharp.Tests/PostgresBm25SearchEngineTests.cs @@ -2,6 +2,7 @@ using LexiSharp.Indexing; using LexiSharp.Postgres; using LexiSharp.Ranking; +using Npgsql; using Xunit; namespace LexiSharp.Tests; @@ -44,6 +45,19 @@ private static void Run( } } + /// Whether the physical table exists, asked of the server rather than inferred. + private static bool TableExists(string table) + { + using var connection = new NpgsqlConnection(ConnectionString!); + connection.Open(); + + using var command = connection.CreateCommand(); + command.CommandText = "SELECT to_regclass(@name) IS NOT NULL"; + command.Parameters.AddWithValue("name", "public." + table); + + return (bool)command.ExecuteScalar()!; + } + /// /// Plain lowercase ASCII, so PostgreSQL's simple analysis and LexiSharp's tokenizer /// produce the same terms — which is what makes the two scorers comparable at all. @@ -391,6 +405,134 @@ public void RareTermGate_FallsBackToTheWholeQueryWhenNothingFitsTheBudget() }, options => options with { RareTermGateBudget = 1 }); } + [Fact] + public void GateTerms_WithAZeroBudgetSelectsNothing() + { + Assert.Empty(PostgresBm25SearchEngine.GateTerms(["a"], _ => 1, budget: 0)); + } + + [SkippableFact] + public void EnsureSchema_IsIdempotentAndNamesItsTable() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + using var engine = NewEngine("schema"); + + try + { + // The constructor already installed it; this is the early return. + engine.EnsureSchema(); + + Assert.True(TableExists(engine.TestTableName)); + } + finally + { + engine.DropSchema(); + } + } + + [SkippableFact] + public async Task EnsureSchemaAsync_CreatesTheTableWhenTheConstructorDidNot() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + using var engine = new PostgresBm25SearchEngine( + ConnectionString!, + new PostgresBm25Options { Table = Table("async") }, + autoCreateSchema: false); + + try + { + Assert.False(TableExists(engine.TestTableName)); + + await engine.EnsureSchemaAsync(); + + // The second call is the early return. + await engine.EnsureSchemaAsync(); + + engine.Add(new SearchDocument("d1", "quick brown fox")); + + Assert.Equal("d1", engine.Search("quick").Single().DocumentId); + } + finally + { + engine.DropSchema(); + } + } + + [SkippableFact] + public void Constructor_AutoCreateSchemaFalseLeavesTheTableAlone() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + using var engine = new PostgresBm25SearchEngine( + ConnectionString!, + new PostgresBm25Options { Table = Table("noschema") }, + autoCreateSchema: false); + + Assert.False(TableExists(engine.TestTableName)); + } + + [SkippableFact] + public void Search_OnAQueryOfStopWordsReturnsNothing() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("stopwords", engine => + { + engine.Add(new SearchDocument("d1", "the quick brown fox")); + + // Every term is a stop word under the English configuration, so the query has no + // lexeme at all and there is nothing to gate or score. + Assert.Empty(engine.Search("the of and")); + }, options => options with { TextSearchConfig = "english" }); + } + + [SkippableFact] + public void Search_HonoursMinimumScore() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("minscore", engine => + { + engine.Index(Corpus); + + Assert.NotEmpty(engine.Search("quick", new SearchOptions(Limit: 10))); + Assert.Empty(engine.Search("quick", new SearchOptions(Limit: 10, MinimumScore: double.MaxValue))); + }); + } + + [SkippableFact] + public void Search_WithQuerySyntaxOffTreatsTheTextAsAValue() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("syntaxoff", engine => + { + engine.Index(Corpus); + + // ParseQuerySyntax: false is the caller saying "literal text": the parser is not + // consulted, and the phrase is still matched by the engine's own split. + Assert.NotEmpty(engine.Search("\"brown fox\"", new SearchOptions(Limit: 10, ParseQuerySyntax: false))); + }); + } + + [SkippableFact] + public void Clear_EmptiesTheCorpusAndTheNextSearchRebuildsTheSnapshot() + { + Skip.If(ConnectionString is null, "POSTGRES_TEST_CONNECTION not set."); + + Run("clear", engine => + { + engine.Index(Corpus); + Assert.NotEmpty(engine.Search("quick")); + + engine.Clear(); + + Assert.Empty(engine.Search("quick")); + }); + } + [Fact] public void Constructor_RefusesInvalidRankingParameters() { diff --git a/tests/LexiSharp.Tests/PostgresTsvTextTests.cs b/tests/LexiSharp.Tests/PostgresTsvTextTests.cs index a12e77cf..41eb2a98 100644 --- a/tests/LexiSharp.Tests/PostgresTsvTextTests.cs +++ b/tests/LexiSharp.Tests/PostgresTsvTextTests.cs @@ -107,6 +107,66 @@ public void Parse_WithAnEmptyKeepStillCountsTheLength() Assert.Equal(3, length); } + [Fact] + public void Parse_ReadsAnUnquotedLexeme() + { + // Defensive path: no released version writes one, but swallowing it into the next token + // would mis-attribute its positions. + var (terms, length) = PostgresTsvText.Parse("abc:1,2 'd':3"); + + Assert.Equal(2, terms["abc"]); + Assert.Equal(1, terms["d"]); + Assert.Equal(3, length); + } + + [Fact] + public void Parse_SkipsAnEmptyLexemeWithoutLooping() + { + // An empty quoted lexeme is not a token, but its position list still has to be consumed: + // leaving the ':' unread made the outer loop spin forever before this was pinned. + var (terms, length) = PostgresTsvText.Parse("'':1 'a':2"); + + Assert.Equal(1, terms["a"]); + Assert.Equal(1, length); + } + + [Fact] + public void Parse_ReadsAnUnterminatedLexemeToTheEnd() + { + var (terms, length) = PostgresTsvText.Parse("'abc"); + + Assert.Equal(1, terms["abc"]); + Assert.Equal(1, length); + } + + [Fact] + public void Parse_WithKeepUnescapesAnEscapedLexeme() + { + var (terms, _) = PostgresTsvText.Parse( + "'o''brien':1 'other':2", new HashSet(["o'brien"], StringComparer.Ordinal)); + + Assert.Equal(1, terms["o'brien"]); + Assert.False(terms.ContainsKey("other")); + } + + [Fact] + public void Parse_TreatsAMalformedPositionListAsOneOccurrence() + { + var (terms, length) = PostgresTsvText.Parse("'a':x 'b':2"); + + Assert.Equal(1, terms["a"]); + Assert.Equal(1, terms["b"]); + Assert.Equal(2, length); + } + + [Fact] + public void Parse_UnescapesATrailingDoubledQuote() + { + var (terms, _) = PostgresTsvText.Parse("'a''':1"); + + Assert.Equal(1, terms["a'"]); + } + [Fact] public void Format_RoundTripsThroughParse() {