From 8b86a27900d005c159e42ef16baad7d15fb717de Mon Sep 17 00:00:00 2001 From: manuc66 Date: Tue, 17 Nov 2026 00:22:16 +0100 Subject: [PATCH] Add FiQA-2018 as the candidate gate's third corpus The gate was measured on NFCorpus and SciFact, both scientific and rich in specific terms -- its best case. FiQA-2018 is the other nature: 57,638 documents, 648 test queries, financial Q&A with short queries. At pool 1,000 the gate reads 0.2061 nDCG@10 against 0.1077 with it off; Anserini's published BM25 is 0.2361 with its own analysis. The direction is SciFact's, so the result is the candidate set and not a scientific-corpus accident. It is selectable with --dataset fiqa and deliberately out of --dataset all: the pinned tables were built on the three corpora, and a fourth row that is not pinned must not silently join them. A test pins that choice. Anserini indexes 57,600 of the file's 57,638 documents because 38 have no text; the fingerprint's 3% tolerance reports no difference, and the entry says where the number comes from. --- CHANGELOG.md | 7 +++++++ bench/LexiSharp.Eval/BeirCorpus.cs | 18 +++++++++++++++++- bench/LexiSharp.Eval/Program.cs | 2 +- bench/LexiSharp.Eval/README.md | 11 +++++++++-- docs/backends.md | 13 ++++++++++--- docs/evaluation.md | 5 +++++ docs/index.md | 3 ++- .../LexiSharp.Tests/EvaluationHarnessTests.cs | 12 ++++++++++++ 8 files changed, 63 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de80459..cca1c0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,13 @@ against 25.7 / 50.0 and 265.0 / 284.4 ms with the gate off. `--keep` leaves the table for `EXPLAIN`. +- **FiQA-2018 as a fourth selectable corpus** (`--dataset fiqa`; 57,638 documents, 648 test + queries). It is not part of `--dataset all` or the pinned comparison: it is the candidate gate's + evidence on a corpus of another nature (financial Q&A, short queries). At pool 1,000 the gate + reads **0.2061** nDCG@10 against **0.1077** without it; Anserini's published BM25 is 0.2361 with + its own analysis. Same cause as SciFact: the ungated candidates are chosen by coverage and + truncated, the gated ones by idf. + - **`SearchTrace` on the SQL backends, and a `TraceStage.Gate`.** The PostgreSQL engines now record the `Score` stage for the page they return — `Postgres lexical`, `Postgres BM25`, `pgvector`, `sparsevec`, `pg_trgm`, `ParadeDB BM25` — where they previously ignored `SearchOptions.Trace` diff --git a/bench/LexiSharp.Eval/BeirCorpus.cs b/bench/LexiSharp.Eval/BeirCorpus.cs index 768162e..d5eddf1 100644 --- a/bench/LexiSharp.Eval/BeirCorpus.cs +++ b/bench/LexiSharp.Eval/BeirCorpus.cs @@ -49,7 +49,8 @@ internal sealed record BeirDataset( long? PublishedTotalTerms = null, int? Documents = null, int? NonEmptyDocuments = null, - string? PublishedSource = null) + string? PublishedSource = null, + bool InAll = true) { public static readonly IReadOnlyList All = [ @@ -86,6 +87,21 @@ internal sealed record BeirDataset( Documents: 8_674, NonEmptyDocuments: 8_673, PublishedSource: "https://github.com/castorini/anserini/blob/master/src/main/resources/reproduce/from-document-collection/configs/beir-v1.0.0-arguana.flat.yaml"), + // Not part of --dataset all: the three above are the pinned comparison, and this one is the + // candidate gate's evidence on a corpus of another nature (financial Q&A, short queries). + // The 38 documents with no text are why Anserini indexes 57,600 of the file's 57,638. + new( + "fiqa", + "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/fiqa.zip", + "17918ed23cd04fb15047f73e6c3bd9d9", + Bm25Ndcg10Ref: 0.236, + Graded: false, + PublishedNdcg10Ref: 0.2361, + PublishedTotalTerms: 5_288_635, + Documents: 57_600, + NonEmptyDocuments: 57_600, + PublishedSource: "https://github.com/castorini/anserini/blob/master/src/main/resources/reproduce/from-document-collection/configs/beir-v1.0.0-fiqa.flat.yaml", + InAll: false), ]; public static BeirDataset Resolve(string name) diff --git a/bench/LexiSharp.Eval/Program.cs b/bench/LexiSharp.Eval/Program.cs index 3a951b0..fc28862 100644 --- a/bench/LexiSharp.Eval/Program.cs +++ b/bench/LexiSharp.Eval/Program.cs @@ -691,7 +691,7 @@ public static async Task Main(string[] args) (ITokenizer tokenizer, string analysisDescription, TokenizerOptions tokenizerOptions) = BuildAnalysis(analyzer, stem); IReadOnlyList datasets = datasetArg == "all" - ? BeirDataset.All + ? BeirDataset.All.Where(dataset => dataset.InAll).ToList() : [BeirDataset.Resolve(datasetArg)]; IReranker? reranker = null; diff --git a/bench/LexiSharp.Eval/README.md b/bench/LexiSharp.Eval/README.md index 823e4d5..7de7785 100644 --- a/bench/LexiSharp.Eval/README.md +++ b/bench/LexiSharp.Eval/README.md @@ -46,6 +46,7 @@ BEIR's own code is Apache-2.0; that license covers the repository, not the data | SciFact | CC BY 4.0 | **ODC-By 1.0** (abstracts from Semantic Scholar S2ORC) | [`allenai/scifact` LICENSE.md](https://github.com/allenai/scifact/blob/master/LICENSE.md) | | NFCorpus | **unconfirmed** | **unconfirmed** | dataset homepage states no data license | | ArguAna | **unconfirmed** | **unconfirmed** | dataset homepage unreachable (HTTP 403) at time of writing | +| FiQA-2018 | **unconfirmed** | **unconfirmed** | re-packaged by BEIR; no license stated in this repository | Two things worth stating rather than glossing over: @@ -70,7 +71,8 @@ benchmark CLI is hand-built and owned by this project rather than sliced out of dotnet run --project bench/LexiSharp.Eval -c Release # NFCorpus, full dotnet run --project bench/LexiSharp.Eval -c Release -- --dataset scifact # SciFact dotnet run --project bench/LexiSharp.Eval -c Release -- --dataset arguana --no-tuned # ArguAna, skip oracle -dotnet run --project bench/LexiSharp.Eval -c Release -- --dataset all --no-tuned # all datasets +dotnet run --project bench/LexiSharp.Eval -c Release -- --dataset fiqa # FiQA-2018 (not in --dataset all) +dotnet run --project bench/LexiSharp.Eval -c Release -- --dataset all --no-tuned # the three pinned datasets dotnet run --project bench/LexiSharp.Eval -c Release -- --dense # add dense retrieval (e5-small) dotnet run --project bench/LexiSharp.Eval -c Release -- --rerank # add cross-encoder reranking (MiniLM) dotnet run --project bench/LexiSharp.Eval -c Release -- --rerank-top 50 # rerank depth (default 100) @@ -227,7 +229,12 @@ defaults, exponential gains): | SciFact | 0.0188 | 0.0988 | **0.6665** | 0.5636 | 0.6665 | 0.662 | 0.6619 | The matching rule is the SciFact story (5.3×) and almost nothing on NFCorpus, whose queries average -3.4 terms; the ranker is the rest. The gate is what makes the BM25 row recover the exact ranking +3.4 terms; the ranker is the rest. + +`--dataset fiqa` adds a corpus of another nature (57,638 documents, 648 test queries, financial Q&A +with short queries) without joining `--dataset all` or the pinned table. On it the gate reads +**0.2061** nDCG@10 against **0.1077** with the gate off at pool 1,000; Anserini's published BM25 is +0.2361 with its own analysis. The direction is SciFact's, not a scientific-corpus accident. The gate is what makes the BM25 row recover the exact ranking (the `pool 0` column) while reading ~1,000 candidate rows instead of the whole match set; the no-gate column is the pool-only path, where `ts_rank_cd` coverage chooses the candidates and SciFact loses 0.10. `--backend-gate off --backend-pool 0` reproduces untruncated BM25 to within the analysis diff --git a/docs/backends.md b/docs/backends.md index b98ef7d..023ab85 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -382,6 +382,12 @@ by; on SciFact that truncation is the whole 0.10 gap. The gate chooses by idf in candidate set is small enough not to truncate. Lowering the budget trades a little quality on NFCorpus for speed; raising it changes nothing here. +**A corpus of another nature.** FiQA-2018 (`--dataset fiqa`; 57,638 documents, 648 test queries, +financial Q&A with short queries) is not part of the pinned three. At the same settings the gate +reads **0.2061** nDCG@10 and the ungated pool **0.1077**, against Anserini's published **0.2361** +for its own analysis. The pattern is SciFact's rather than a scientific-corpus accident: the +ungated pool truncates a coverage-ranked candidate set, and the gate's idf-selected one does not. + **What it costs, in time.** The evaluation harness reports quality and not time on purpose; the [PostgreSQL lane](benchmarks.md#postgresql-lane-postgres-what-a-search-costs-on-stock-postgresql) measures the other half — one process, one thread, a pooled data source, a page of 10, on a local @@ -404,9 +410,10 @@ server. p50 / mean: - **The statistics build is the first search after a write**, not a search: `ts_stat` walks the table, and the column is measured on its own so it does not hide inside a p99. -Two facts about the evidence, because they bound the claim. Both corpora are scientific and their -queries are rich in specific terms — SciFact's rarest query term has a median document frequency of -1 — which is the case the gate is built for. And the failure mode is real: a rare term that is a +Two facts about the evidence, because they bound the claim. Two of the three corpora measured +(NFCorpus, SciFact) are scientific and rich in specific terms — SciFact's rarest query term has a +median document frequency of 1 — which is the gate's best case; the third, FiQA, is financial with +short queries and shows the same direction. And the failure mode is real: a rare term that is a false friend (a name variant, an entity absent from the relevant documents) can gate them out. The harness flag `--backend-gate off` is how to measure that on your own corpus. diff --git a/docs/evaluation.md b/docs/evaluation.md index 0bf4a01..864cc19 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -520,6 +520,11 @@ same qrels (harness defaults, exponential gains, 10 retrieved at 10): 1,000 reads 0.2999 / 0.5636, because the candidates are chosen by `ts_rank_cd` coverage rather than by idf — on SciFact that truncation is the whole 0.10 difference. The numbers behind the choice, and what it costs in milliseconds, are on the [backend guide](backends.md#bm25-over-stock-postgresql-postgresbm25searchengine). +- **The gate on a corpus of another nature.** FiQA-2018 is a selectable dataset (`--dataset fiqa`) + but not part of `--dataset all` or the pinned comparison. On its 57,638 documents and 648 test + queries, the stock-PostgreSQL BM25 row reads **0.2061** nDCG@10 with the gate and **0.1077** + without it at pool 1,000; Anserini's published BM25 is 0.2361 with its own analysis. Same cause as + SciFact: the ungated candidates are chosen by coverage and truncated, the gated ones by idf. - **`ts_rank_cd`'s and ParadeDB's scores are PostgreSQL-native and not numerically comparable** with `Bm25Scorer`; the stock-PostgreSQL BM25 row's are `Bm25Scorer`'s and are. The metric columns compare either way. The analysis is PostgreSQL's own, not this library's tokenizer, which diff --git a/docs/index.md b/docs/index.md index 58bdf76..0782d6d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -119,7 +119,8 @@ twice. Every figure below is reproducible with a command on the term, and OR-ing them (`--backend-match any`) recovers **0.0988** on SciFact but only **0.1881** on NFCorpus, so the rest of that gap is the ranker. The stock-PostgreSQL BM25 engine closes it with no extension: **0.3072** and **0.6665** at its default candidate gate, **0.2999** and - **0.5636** with the gate off. pgvector at `ef_search` 10/40/160 reads **0.2882 / 0.2960 / 0.3013** on + **0.5636** with the gate off; on FiQA, the corpus of another nature, **0.2061** against + **0.1077**. pgvector at `ef_search` 10/40/160 reads **0.2882 / 0.2960 / 0.3013** on NFCorpus against the exact in-memory dense row's **0.304**, and **0.5998 / 0.6528 / 0.6550** on SciFact against **0.655**: that gap is the index's cost in nDCG. ArguAna produces no row from any of the three: the text engines reject 2 of its 1,406 test queries, and the vector engine rejects diff --git a/tests/LexiSharp.Tests/EvaluationHarnessTests.cs b/tests/LexiSharp.Tests/EvaluationHarnessTests.cs index 24e4220..7bac3c7 100644 --- a/tests/LexiSharp.Tests/EvaluationHarnessTests.cs +++ b/tests/LexiSharp.Tests/EvaluationHarnessTests.cs @@ -17,6 +17,18 @@ namespace LexiSharp.Tests; /// public class EvaluationHarnessTests { + [Fact] + public void FiqaIsSelectableButNotPartOfAll() + { + // It is the candidate gate's evidence on a corpus of another nature, not a fourth row in + // the pinned comparison: --dataset all must keep meaning the three the tables were built on. + BeirDataset fiqa = BeirDataset.Resolve("fiqa"); + + Assert.False(fiqa.InAll); + Assert.Equal(3, BeirDataset.All.Count(dataset => dataset.InAll)); + Assert.DoesNotContain("fiqa", BeirDataset.All.Where(dataset => dataset.InAll).Select(dataset => dataset.Name)); + } + [Fact] public void TheBm25FTunedRowIsOptInAndSaysWhetherWeightingHelped() {