diff --git a/CHANGELOG.md b/CHANGELOG.md index 60620ed..0f944d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## [Unreleased] + +### Fixed + +- **The evaluation harness measured a rounded run in the engine's tie-break, not in the run reader's.** + The four-decimal rounding reproduces what the reference writes down, and a written run is read back + by `trec_eval`, which ignores the rank column and breaks an exact score tie by document number + **descending** (`comp_sim_docno` in `form_res_rels.c`). The harness kept the engine's order + (document id ascending), so on ArguAna it reported **0.4005** where `trec_eval` reads **0.3970** + from the same file: 5,699 exact duplicate scores, 183 queries with a tie in the top 10, 84 queries + whose nDCG differs. Applying the reader's tie-break to a rounded run makes the two per-query equal, + 1,406 of 1,406, and the pin `arguana/uax29-aligned` is re-recorded from 0.4005 to **0.3970** as a + measured value, not a copied one. The replay also retrieves at the reference run's depth (1,000) + rather than 10: a document at rank 11 can enter the top ten by tying the tenth score, and that + boundary alone is worth 0.0004 in the aggregate, across four queries. NFCorpus and SciFact are + unchanged; no configuration had a tie inside their top ten. The table header now names the depth + the metrics are measured at (`MetricDepth.Cutoff`) instead of `--top-k`, which had printed + `nDCG@1000` over an nDCG@10 value. + ## [1.0.0] — 2026-11-10 The first stable release. The public API is now versioned by semantic versioning: a breaking diff --git a/README.md b/README.md index 3a2adbe..448a50f 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ aligned to the published figures: |---|---:|---:| | NFCorpus | 0.3218 | 0.3218 | | SciFact | 0.6789 | 0.6789 | -| ArguAna | 0.4005 | 0.3970 | +| ArguAna | 0.3970 | 0.3970 | Point estimates; the harness also reports paired-bootstrap 95 % CIs. Under the library's own default analysis and parameters the same scorer reads 0.308 / 0.662 / 0.320 — the difference is @@ -113,8 +113,10 @@ linear classifier): [text analysis →](docs/text-analysis.md). - **net10.0**, **1.0.0**: the public API is stable within a major version. Breaking changes require a major release, additions are minor and fixes are patches. Pin a major version and read the [changelog](CHANGELOG.md). -- ArguAna reads 0.4005 against 0.3970 published (0.0035 short; cause not established). An earlier - ArguAna figure was withdrawn and is not reproducible by any code path. +- ArguAna reads 0.3970 against 0.3970 published, equal to four decimals on a run file read by + `trec_eval`; the 0.4005 an earlier version reported was the same ranking under the engine's own + tie-break inside an exact score tie. An earlier ArguAna figure was withdrawn and is not + reproducible by any code path. - Not every engine/scorer/reranker/merger combination is tested; treat unusual pairings as supported but unproven until tested on your data. - SQL backends are measured on ranking for the engines that rank on their own; sparse and fuzzy diff --git a/bench/LexiSharp.Eval/Evaluation.cs b/bench/LexiSharp.Eval/Evaluation.cs index 34837d6..c7d94b7 100644 --- a/bench/LexiSharp.Eval/Evaluation.cs +++ b/bench/LexiSharp.Eval/Evaluation.cs @@ -718,20 +718,27 @@ void Score(int i) { EvaluatedQuery evaluated = queries[i]; - string[] retrieved = engine.Search( - evaluated.Query.Text, - // The rounding is a reproduction of what the reference writes down, not a scoring - // choice, so it is off unless the run asks for it: on it, the metrics measure a - // different ranking function. - new SearchOptions( - topK, - ExcludedDocumentIds: evaluated.Excluded, - ParseQuerySyntax: parseQuerySyntax) - .WithScoreRounding(scoreRounding - ? ScoreRounding.FourDecimals - : ScoreRounding.None)) - .Select(result => result.DocumentId) - .ToArray(); + IReadOnlyList hits = engine.Search( + evaluated.Query.Text, + // The rounding is a reproduction of what the reference writes down, not a scoring + // choice, so it is off unless the run asks for it: on it, the metrics measure a + // different ranking function. + new SearchOptions( + topK, + ExcludedDocumentIds: evaluated.Excluded, + ParseQuerySyntax: parseQuerySyntax) + .WithScoreRounding(scoreRounding + ? ScoreRounding.FourDecimals + : ScoreRounding.None)); + + // Rounding merges scores that were distinct into a tie, and a run file is read back by a + // tool whose tie-break is part of the published number. Measuring the engine's order + // instead would score a ranking no run file contains: on ArguAna that is 0.4005 against + // the 0.3970 a reader computes from the same file. + if (scoreRounding) + hits = RunFileOrder(hits); + + string[] retrieved = hits.Select(result => result.DocumentId).ToArray(); string[] relevant = evaluated.Graded.Keys.ToArray(); @@ -801,6 +808,33 @@ void Score(int i) .ToList()); } + /// + /// Rewrites a page into the order a run file's reader reconstructs: score descending, then + /// document id descending. + /// + /// + /// + /// This is trec_eval's comp_sim_docno, which is what turns a run file into a + /// ranking: the file's rank column is ignored, and the scores are sorted with + /// strcmp(p2->docno, p1->docno) on an exact tie. The order matters because + /// deliberately rounds scores onto the same + /// ten-thousandth, so a tie the engine had already broken becomes an exact tie in the file. The + /// walk-down step cannot always separate it either: at an ArguAna score of 118 the float spacing + /// is larger than the one-millionth it subtracts, so the scores stay bit-identical and the reader's + /// tie-break decides. + /// + /// + /// Only the reproduction path calls this. Without rounding, an exact tie is the engine's own and + /// its configured is the ranking being measured; with rounding, the run file + /// is the object being measured, and the engine order is a ranking that file no longer carries. + /// + /// + internal static SearchResult[] RunFileOrder(IReadOnlyList results) => + results + .OrderByDescending(result => result.Score) + .ThenByDescending(result => result.DocumentId, StringComparer.Ordinal) + .ToArray(); + private static string RunTuned( ITextIndex index, IReadOnlyList fitQueries, diff --git a/bench/LexiSharp.Eval/Program.cs b/bench/LexiSharp.Eval/Program.cs index 28a19ca..61787f4 100644 --- a/bench/LexiSharp.Eval/Program.cs +++ b/bench/LexiSharp.Eval/Program.cs @@ -1149,7 +1149,7 @@ void CloseRerankCache() referenceBm25 ?? new Bm25Parameters(0.9, 0.4), parseQuerySyntax); } - PrintTable(results, topK, jobs); + PrintTable(results, jobs); if (bootstrap) RetrievalBootstrap.Print(results, bootstrapBaseline, bootstrapResamples, bootstrapCorrection); @@ -1597,9 +1597,14 @@ private static IEnumerable ParseStrengths(string value) $"--expand-ranking expects 'count', 'pmi' or 'both', got '{value}'."), }; - private static void PrintTable(IReadOnlyList results, int topK, int jobs) + private static void PrintTable(IReadOnlyList results, int jobs) { - var header = new[] { "Config", $"nDCG@{topK}", $"MAP@{topK}", $"MRR@{topK}", $"R@{topK}", "Queries", "Wall" }; + // The header names the depth the metrics are measured at, not the depth the run retrieved. + // --top-k 1000 still scores at rank 10 (MetricDepth.Cutoff), and printing "nDCG@1000" over an + // nDCG@10 value is a label a reader has no way to catch: the parity command retrieves 1,000 + // documents to compare scores one by one and measures ten. + int depth = MetricDepth.Cutoff; + var header = new[] { "Config", $"nDCG@{depth}", $"MAP@{depth}", $"MRR@{depth}", $"R@{depth}", "Queries", "Wall" }; var widths = new[] { 30, 9, 9, 9, 9, 8, 8 }; Console.WriteLine(); diff --git a/bench/LexiSharp.Eval/README.md b/bench/LexiSharp.Eval/README.md index d79a0fb..d197bae 100644 --- a/bench/LexiSharp.Eval/README.md +++ b/bench/LexiSharp.Eval/README.md @@ -952,22 +952,24 @@ document count, and a metric cutoff. | + reference analysis, at k1=0.9/b=0.4 | 0.240 | | + reference analysis, + query-frequency counting, at k1=0.9/b=0.4 | 0.301 | | + reference analysis, + query-frequency counting, at k1=3.0/b=0.75 | 0.367 | -| + the parity options, at k1=0.9/b=0.4 | 0.4005 | +| + the parity options, at k1=0.9/b=0.4 | 0.3970 | | published reference, at k1=0.9/b=0.4 | 0.3970 | Read the third and fourth rows together: the setting is worth +0.061 at the reference's own parameters and +0.033 at k1=3.0. Those are settings the library exposes. The fifth row adds the four options a bit-for-bit comparison needs — the document count that carries the field rather than every indexed document, the reference's scale and precision, its four-decimal rounding, and the query's own -document removed — all off by default. It reads **0.4005** against the published **0.3970**, 0.0035 -short, and **that gap is not closed by any configuration tried**. - -A parity row in `reference/pinned.json` used to carry 0.3970 as this library's own figure, with a -claim that all 15,466 returned scores matched the reference's on the raw bits. Both are withdrawn: -the parity section is displayed by `--verify-reference` but never replayed, and the figure was the -published one rather than what the documented configuration measures. The reproduction command in -`docs/evaluation.md` now produces 0.4005, and that value is a pinned regression, so the gate replays -it. +document removed — all off by default. It reads **0.3970** against the published **0.3970**: the run +file is read with `trec_eval`'s tie-break (score descending, then document number descending), which +is the convention the published figure was produced under. Scored in the engine's own order the same +ranking reads 0.4005, which is the order inside an exact score tie, not a different result. + +A parity row in `reference/pinned.json` used to carry 0.3970 as this library's own figure without +replaying it, with a claim that all 15,466 returned scores matched the reference's on the raw bits. +That row was withdrawn: the parity section is displayed by `--verify-reference` but never replayed, +and the figure was the published one rather than a measurement. The value is now in a pinned +regression (`arguana/uax29-aligned`) that `--verify-reference` replays, and the run-file comparison is +per query. An earlier figure published for this corpus was withdrawn; it is not reproducible by any code path in this repository. See the [changelog](https://github.com/manuc66/LexiSharp/blob/main/CHANGELOG.md). The diff --git a/bench/LexiSharp.Eval/ReferenceVerifier.cs b/bench/LexiSharp.Eval/ReferenceVerifier.cs index d39c990..093e43a 100644 --- a/bench/LexiSharp.Eval/ReferenceVerifier.cs +++ b/bench/LexiSharp.Eval/ReferenceVerifier.cs @@ -24,6 +24,13 @@ namespace LexiSharp.Eval; /// internal static class ReferenceVerifier { + /// + /// The depth the reference's run files are written at, and therefore the depth a rounded pin has + /// to retrieve before it can be read back: the reader's tie-break can promote the document at + /// rank 11 into the measured top ten when it ties the tenth score. + /// + private const int ReferenceRunDepth = 1000; + public static async Task RunAsync( PinnedReference pinned, string dataBaseDir, bool write, CancellationToken cancellationToken) { @@ -184,14 +191,27 @@ private static double Measure( // on BEIR queries a straight quote is ordinary prose and a query language collapses it // (146 of ArguAna's 1,406 test queries carry one). The field exists so the pin says // which of the two it was recorded under. - var retrieved = engine + // A rounded pin is a run file, and a run file's reader can move a document from beyond + // rank 10 into the top 10 when it ties the tenth score. The published ArguAna run was + // written at depth 1000; retrieving only 10 leaves four queries reading 0.3974 instead of + // the reader's 0.3970, because the document that decides those four is the eleventh. + int depth = pin.ScoreRounding ? ReferenceRunDepth : MetricDepth.Cutoff; + + IReadOnlyList hits = engine .Search(query.Text, new SearchOptions( - 10, + depth, ExcludedDocumentIds: excluded, ParseQuerySyntax: pin.QuerySyntax) - .WithScoreRounding(pin.ScoreRounding ? ScoreRounding.FourDecimals : ScoreRounding.None)) - .Select(result => result.DocumentId) - .ToArray(); + .WithScoreRounding(pin.ScoreRounding ? ScoreRounding.FourDecimals : ScoreRounding.None)); + + // A rounded pin is replaying a run file, and a run file is ranked by its reader: score + // descending, then document id descending. Measuring the engine's tie-break against a + // published number computed by trec_eval is what made ArguAna read 0.4005 where the same + // ranking read 0.3970 from the file. + if (pin.ScoreRounding) + hits = Evaluation.RunFileOrder(hits); + + var retrieved = hits.Select(result => result.DocumentId).ToArray(); total += RetrievalMetrics.NdcgAtK(retrieved, corpus.TestRelevance[query.Id], 10, gain); } diff --git a/bench/LexiSharp.Eval/reference/pinned.json b/bench/LexiSharp.Eval/reference/pinned.json index 96e9111..83c48c1 100644 --- a/bench/LexiSharp.Eval/reference/pinned.json +++ b/bench/LexiSharp.Eval/reference/pinned.json @@ -212,13 +212,13 @@ }, "excludeQueryDocument": true, "queries": 1406, - "expectedNdcg": 0.4005, + "expectedNdcg": 0.397, "tolerance": 0.0005, "referenceIndexStatistics": true, "scoreRounding": true, "omitSaturationConstant": true, "singlePrecision": true, - "note": "The alignment row for ArguAna, under the reference's conventions: uax29, k1=0.9/b=0.4, linear gains, the repeated-term rule, the query document excluded, and the four arithmetic options. It reads 0.4005 against the published 0.3970 - a gap of 0.0035 that is not reproduced by the configuration the parity row named. The 0.3970 that row carried was not what this configuration measures; it is withdrawn, and this pin is what the documented command produces.", + "note": "The alignment row for ArguAna, under the reference's conventions: uax29, k1=0.9/b=0.4, linear gains, the repeated-term rule, the query document excluded, and the four arithmetic options. It reads 0.3970 against the published 0.3970. The 0.4005 this row carried measured the same ranking under the engine's own tie-break (document id ascending): score rounding puts scores on a ten-thousandth grid, and a published figure is read from a run file by trec_eval, which ignores the rank column and breaks an exact score tie by document number descending (comp_sim_docno). The metric path now applies the reader's tie-break whenever scoreRounding is on, which is the only convention under which the two numbers are the same measurement.", "queryTermFrequency": true }, { @@ -377,7 +377,7 @@ { "corpus": "arguana", "lexisharpConfig": "arguana/uax29-aligned at k1=0.9, b=0.4, reference conventions on", - "lexisharpNdcg": 0.4005, + "lexisharpNdcg": 0.397, "referenceNdcg": 0.397, "referenceParameters": { "k1": 0.9, @@ -385,10 +385,10 @@ }, "alignedParameters": true, "reproducibilityStatus": "reproduced", - "lexisharpNdcgMeasured": 0.4005, - "reproducibilityNote": "Measured today with the same third-party tool on a run written by this harness, and independently by this harness. trec_eval 9.0.8 reads ndcg_cut_10 0.3970 and recall_100 0.9324 on it, against 0.3970 and 0.9324 published; this harness reads 0.3970 on the same run. The two metrics are computed differently and agree to four decimals. With the reference's own scale, precision and post-processing - see --reference-score-rounding, --omit-saturation-constant, --single-precision-bm25 and --reference-index-statistics, all off by default - 15,466 of 15,466 compared (query, document) scores over all 1,406 queries match on the raw bits, so the ranking is the reference's ranking and not a lookalike. A residual 0.0004 between this harness and trec_eval on the rounded run is confined to 102 of 1,406 queries, every one containing a tie in the returned scores. A tie in the returned scores is the plausible mechanism, because it is the only property those 102 queries share, and it is not established: neither the order written in the run file, nor document id, nor a reversal of the tied group reproduces trec_eval's reading, and 93 of 183 fit a reversal. Which order produced the published figure is not established either, and is not claimed. The row is left at 0.397, which is what it now measures: re-recording a baseline is a reviewed act, and a value that was measured to land on the recorded one needs no edit.", + "lexisharpNdcgMeasured": 0.397, + "reproducibilityNote": "Measured today with the same third-party tool on a run written by this harness, and independently by this harness. trec_eval 9.0.8 reads ndcg_cut_10 0.3970 and recall_100 0.9324 on it, against 0.3970 and 0.9324 published; this harness reads 0.3970 on the same run. The 0.4005 recorded here before was the same ranking measured under the engine's own tie-break: the score rounding deliberately puts scores on a ten-thousandth grid, and trec_eval ignores the rank column, sorting the file by score descending and breaking an exact tie by document number descending (comp_sim_docno in form_res_rels.c). The walk-down that is meant to separate a tied group cannot always do it either - at an ArguAna score of 118 the float spacing is larger than the millionth it subtracts - so the run file leaves exact ties and the reader decides them. Applying that reader's tie-break to a rounded run makes the harness and trec_eval agree; measured on the first 120 queries of the same run, file order reads 0.4418 and the reader's order reads 0.4391, which is exactly what trec_eval prints. The two metrics are computed differently and agree to four decimals. With the reference's own scale, precision and post-processing - see --reference-score-rounding, --omit-saturation-constant, --single-precision-bm25 and --reference-index-statistics, all off by default - the index is the reference's index; the bit-level score comparison that used to be recorded here is not re-asserted, because the reference's per-query run file is not published and equal scores leave the tie order to the reader.", "variants": "uax29 + reference index statistics + parity options -> 0.3970. The 0.401 and 0.403 recorded earlier on this row were measured with the metrics taken at the run's retrieval depth rather than at rank 10: a parity run retrieves one document deeper than it measures, so --top-k 11 produced nDCG@11 under a heading reading nDCG@10. That 0.006 was the eleventh document. Fixed in Evaluation.cs as MetricDepth, and the aggregate is unaffected by it at the published depth.", - "note": "Like for like: 0.4005 at the reference's own k1=0.9, b=0.4, under its analyzer (uax29), its linear gains, its repeated-term rule, the query document excluded, its index statistics, score rounding, omitted saturation constant and single precision. Against 0.3970 published, a gap of 0.0035. The 0.3970 this row carried was not what the documented configuration measures - it is the published figure, and the parity section is displayed but never replayed, so nothing caught the difference. The bit-level score claim on this corpus is withdrawn with it." + "note": "Like for like: 0.3970 at the reference's own k1=0.9, b=0.4, under its analyzer (uax29), its linear gains, its repeated-term rule, the query document excluded, its index statistics, score rounding, omitted saturation constant and single precision. The 0.4005 this row used to carry was the same ranking under the engine's own tie-break (document id ascending); the published figure is read from a run file, whose reader breaks an exact score tie by document number descending, and the rounding deliberately creates those ties. The metric path now applies the reader's tie-break for a rounded run, and the two numbers are equal to four decimals. The bit-level score claim on this corpus stays withdrawn: it belonged to the row that carried the published figure, and equal scores do not by themselves settle an aggregate that depends on the reader's tie order." } ], "referenceSource": { diff --git a/docs/evaluation.md b/docs/evaluation.md index 3aa463c..22c0bad 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -36,16 +36,17 @@ comparison. nDCG@10: | ArguAna | defaults | 0.320 | 0.315 | +1.6 % | | NFCorpus | aligned: uax29 analysis, k1=0.9, b=0.4, reference conventions | 0.3218 | 0.3218 | 0.0000 | | SciFact | aligned: same conventions | 0.6789 | 0.6789 | 0.0000 | -| ArguAna | aligned | 0.4005 | 0.3970 at k1=0.9, b=0.4 | −0.0035 | +| ArguAna | aligned | 0.3970 | 0.3970 at k1=0.9, b=0.4 | 0.0000 | The **defaults** rows answer *what the library does out of the box*: no stemming, stop words kept. The **aligned** rows answer *is this comparable to the published figure* — and with the reference's own analyzer, its parameters, its gain convention, its repeated-term rule, its index statistics, its score rounding, its omitted saturation constant and its single precision, the two -agree to the fourth decimal on NFCorpus and SciFact. On ArguAna this library reads **0.4005** against -the published **0.3970** — 0.0035 short under the same conventions — and that gap is not reproduced by -any configuration tried. The gap between the two row groups is configuration, not ranking; the -ArguAna gap is not yet explained. +agree to the fourth decimal on all three corpora. The gap between the two row groups is configuration, +not ranking. On ArguAna the aligned row reads **0.3970** when the metric is computed the way the +published figure was: on a run file, whose reader breaks an exact score tie by document number +descending. Computing it in the engine's own tie-break reads 0.4005 on the same ranking — the same +index, the same scores, the same documents — and is not a different result. ### Paired bootstrap over queries @@ -59,23 +60,25 @@ the family of configurations in the table. dotnet run --project bench/LexiSharp.Eval -c Release -- --dataset nfcorpus --no-tuned \ --bootstrap --analyzer uax29 --ndcg-gain linear --reference-bm25 0.9,0.4 \ --query-term-frequency --reference-index-statistics --reference-score-rounding \ - --omit-saturation-constant --single-precision-bm25 + --omit-saturation-constant --single-precision-bm25 --top-k 1000 ``` -The aligned rows, with their intervals: +The parity rows retrieve at the reference run's depth (1,000) even though they measure at 10: a +document at rank 11 can tie the tenth score and be promoted into the measured top ten by the reader's +tie-break. The aligned rows, with their intervals: | Corpus | nDCG@10 [95 % CI] | Δ vs BM25 1.5/0.75 [95 % CI] | p | p (Holm) | |---|---:|---:|---:|---:| | NFCorpus | 0.3218 [0.2869, 0.3564] | −0.0042 [−0.0083, 0.0000] | 0.048 | 0.240 | | SciFact | 0.6789 [0.6334, 0.7237] | −0.0087 [−0.0202, +0.0032] | 0.151 | 1.000 | -| ArguAna | 0.4005 [0.3820, 0.4194] | −0.0879 [−0.0967, −0.0794] | 0.0001 | 0.001 | +| ArguAna | 0.3970 [0.3787, 0.4155] | −0.0875 [−0.0963, −0.0791] | 0.0001 | 0.0014 | The published figures (0.325, 0.665, 0.397) all fall **inside** those intervals. Equality to four decimals is a property of the point estimate, not evidence that the two systems are statistically equivalent. The NFCorpus difference is the one the Holm correction removes: across 14 configurations, a raw p=0.048 is not a family-wise result. The ArguAna difference is not removed: -under the same exclusion convention, BM25 1.5/0.75 reads **0.4884** [0.4697, 0.5077], 0.0879 above -the aligned 0.9/0.4 row (paired difference [0.0794, 0.0967], p<0.0001, Holm p=0.001). The aligned +under the same exclusion convention, BM25 1.5/0.75 reads **0.4845** [0.4661, 0.5035], 0.0875 above +the aligned 0.9/0.4 row (paired difference [0.0791, 0.0963], p<0.0001, Holm p=0.0014). The aligned row is a reproduction target, not the best operating point. The bootstrap covers the table `Evaluation.Run` produces: the lexical scorers, their variants, the @@ -142,9 +145,11 @@ the reference's published term count on NFCorpus and SciFact. Adding the four ar precision) brings both figures to the published ones at four decimals: this library reads **0.32175501** and **0.67890943** unrounded, against a reference published as 0.3218 and 0.6789. The published figures are rounded to four decimals, so the score comparison cannot see beyond that; -what can be seen, and is exact, is the index — see below. Bit-level score parity — the only claim -that would settle the ranking — exists on ArguAna, where 15,466 of 15,466 compared scores match, and -is not established on these two. Both fingerprints and both scores are pinned +what can be seen, and is exact, is the index — see below. Bit-level score parity was measured on +ArguAna, where 15,466 of 15,466 compared scores match, and is not established on these two. It is not +the claim that settles an aggregate, though: equal scores still leave the order inside a tie to the +run reader, which is the 0.0035 the ArguAna section below resolves. Both fingerprints and both scores +are pinned (`nfcorpus/uax29`, `scifact/uax29`, `nfcorpus/uax29-aligned`, `scifact/uax29-aligned`), so `--verify-reference` replays them. @@ -161,25 +166,37 @@ zero tolerance. Term for term, not only in total: diffing this library's token m Lucene 10.5.2's over these files leaves **zero** tokens on either side for NFCorpus and SciFact, and ArguAna contains none of the characters the two rules touch. The indexes are the same index. -**ArguAna: the metric is reproduced, and so are the raw scores.** The two are verified by different -means and the distinction is the point. +**ArguAna: the metric is reproduced, per query.** The aggregate and the scores are verified by +different means, and the distinction is the point: the scores were already the same, and the aggregate +was not, because equal scores leave the order inside a tie to the file's reader. The aggregate is verified without this repository's own metric code being involved. `trec_eval` 9.0.8, run on a `trec_run` this harness writes at that configuration, reads nDCG@10 **0.3970** and recall@100 -**0.9324** — the two published figures, to four decimals. This harness reads 0.3970 on the same run. -The metrics are computed differently and agree; where they do not, the difference is 0.0004, confined -to the 102 of 1,406 queries whose returned scores contain a tie. A tie is the plausible mechanism, -being the only property those 102 queries share, and it is not established: neither the order written -in the run file, nor document id, nor a reversal of the tied group reproduces `trec_eval`'s reading. -Which order produced the published figure is not established either. +**0.9324** — the two published figures, to four decimals. This harness reads 0.3970 on the same run, +and the agreement is per query, not only in aggregate: once the two read the file the same way, all +1,406 query values agree to four decimals. + +The rule that makes them agree is not a tolerance. `trec_eval` ignores the rank column, sorts the file +by score descending, and breaks an exact score tie by document number **descending** +(`comp_sim_docno` in `form_res_rels.c`). The four-decimal rounding this harness applies to reproduce a +written run deliberately puts scores on the same ten-thousandth, and the walk-down that is meant to +separate a tied group cannot always do it: at an ArguAna score of 118, the float spacing is coarser +than the millionth it subtracts. The parity run therefore contains **5,699** exact duplicate scores; +**183** queries have a tie inside the top 10; **84** change nDCG when the reader's order replaces the +engine's. Read in the engine's order the same file averages **0.4005**, read by `trec_eval`'s rule it +averages **0.3970**. A document at rank 11 can even enter the top 10 when it ties the tenth score, +which is why the parity run retrieves 1,000 deep and not 10. The harness now applies the reader's +tie-break to a rounded run, so the two numbers are the same measurement. One thing is worth knowing about reading a metric from this harness, because it changed a number that looked like a ranking result. The harness measures at **rank 10**, not at the depth the run retrieved, and the two being the same value is a coincidence of the default: a bit-for-bit -reproduction run retrieves one document deeper than it measures, so measuring at the retrieval depth -reported nDCG@11 under a heading reading nDCG@10. Retrieval depth and evaluation depth are different -things, and the cutoff is named (`MetricDepth`) and independent of `--top-k`. The table below -holds the full set of figures the published config produces, at the cutoff: +reproduction run retrieves at the reference run's depth, so measuring at the retrieval depth reported +nDCG@11 under a heading reading nDCG@10. Retrieval depth and evaluation depth are different things, +and the cutoff is named (`MetricDepth`) and independent of `--top-k`. For a rounded run the retrieval +depth is not arbitrary either: the reader's tie-break can promote the document at rank 11 into the +measured top ten, so the parity run retrieves 1,000 documents and measures ten. The table below holds +the full set of figures the published config produces, at the cutoff: | | LexiSharp | reference | Δ | |---|---:|---:|---:| @@ -190,9 +207,11 @@ holds the full set of figures the published config produces, at the cutoff: | MAP@100 | 0.3280 | 0.3280 | 0.0000 | | reciprocal rank | 0.3282 | 0.3282 | 0.0000 | -**What is reproduced is the scores, and on the raw bits.** A metric sees an order, and an order can -agree while every number behind it differs — which is exactly what was happening. Against a run produced -by the reference implementation's own searcher over its own index, for all 1,406 test queries: +**Equal scores are necessary, and not sufficient.** A metric sees an order, and an order can agree +while every number behind it differs — that is what the three arithmetic options above fixed. The +converse is also true, and it is what the 0.0035 was: equal scores leave the order inside a tie to +whoever reads the file, and the reader's rule is part of the published number. Measured against a run +produced by the reference implementation's own searcher over its own index, for all 1,406 test queries: | | paired scores identical on the raw bits | |---|---:| @@ -201,7 +220,11 @@ by the reference implementation's own searcher over its own index, for all 1,406 | … and the reference's scale, precision and rounding | **14,168 / 14,168 — 100.00 %** | At the depth the comparison runs, one document deeper than the metric is measured at, the count is -**15,466 / 15,466** over all 1,406 queries. +**15,466 / 15,466** over all 1,406 queries. That comparison was recorded earlier and is not +re-asserted as this row's parity claim: the reference's per-query run file is not published here, and +bit-identical scores do not settle an aggregate that depends on the reader's order inside a tie. The +reproducible claim is the aggregate under that order, **0.3970**, which both `trec_eval` and this +harness now measure. The three differences that got there, each behind an option whose default is unchanged: @@ -323,14 +346,16 @@ two settings could not have produced different rankings. The engine hands the sc list, the setting is observable through `Search`, and `QueryTermWeightingTests` fails if it ever stops being. -**ArguAna is 0.0035 short, and that is the measured state.** Under the reference's analyzer, +**ArguAna is 0.3970, and the 0.0035 was the reader's tie-break.** Under the reference's analyzer, parameters, gain convention, repeated-term rule, exclusion and arithmetic, this library reads -**0.4005** where the reference publishes **0.3970**. A parity row used to carry 0.3970 as this -library's own figure; it was the published number, not a measurement — the parity section is -displayed by `--verify-reference` but never replayed, so nothing caught the difference. That row is -withdrawn, the value is now a pinned regression (`arguana/uax29-aligned`), and the raw-score claim -that went with it is withdrawn too. The cause is not established: the index is term for term the -reference's, so what remains is the scorer's arithmetic, and no configuration tried here closes it. +**0.3970** where the reference publishes **0.3970**. The 0.4005 this page used to report was the same +ranking scored in the engine's order, with an exact score tie broken by document id ascending instead +of the run reader's descending rule. The index is term for term the reference's and the scores are the +same; there is no scorer-arithmetic gap left to close. The row is a pinned regression +(`arguana/uax29-aligned`) at 0.3970, and the run-file comparison is per query: 1,406 of 1,406 values +agree with `trec_eval` once the same tie-break is applied. The earlier note that the parity row +carried the published figure as a measurement is still true of that row; it is not true of this pin, +which is replayed by `--verify-reference`. The reference is also **newer than the table it is first compared against**: a current per-corpus regression for this exact index puts ArguAna at 0.3970, not the 0.315 of the 2021 paper. diff --git a/docs/index.md b/docs/index.md index 1308b0c..71d5c6e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -96,7 +96,10 @@ twice. Every figure below is reproducible with a command on the plain BM25 scorer reaches nDCG@10 **0.3218** on NFCorpus and **0.6789** on SciFact, against **0.3218 / 0.6789** — equal to the fourth decimal, with an index that matches the reference's term for term. On ArguAna, at the reference's own k1=0.9/b=0.4 and under the same conventions, it - reaches **0.4005** against **0.3970**, 0.0035 short. Under the library's own defaults the same scorer reads + reaches **0.3970** against **0.3970** — equal to the fourth decimal on a run file read by + `trec_eval`, whose tie-break the harness now applies to a rounded run. The same ranking scored in + the engine's own tie-break reads 0.4005; that is a different order inside an exact tie, not a + different result. Under the library's own defaults the same scorer reads 0.308 / 0.662 / 0.320; that difference is the configuration, not the ranking. The corpora are md5-verified on download, the harness ships no licence over them, and every number here is pinned and re-checked by the `Pinned reference` workflow. diff --git a/docs/ranking.md b/docs/ranking.md index 37d53e5..9eca726 100644 --- a/docs/ranking.md +++ b/docs/ranking.md @@ -97,10 +97,13 @@ rather than a constructor parameter for the same reason: naming the call is the something other than searching is being asked for. With `saturationConstant: false`, single precision, reference statistics and the rounding, this -library reads **0.4005** on ArguAna against the published **0.3970**, 0.0035 short. The bit-level -raw-score comparison this paragraph used to report for all 15,466 (query, document) pairs is -withdrawn: the figure it belonged to is not what the documented configuration measures, and no -configuration tried reproduces it. See [Evaluation](evaluation.md). +library reads **0.3970** on ArguAna against the published **0.3970**, equal to four decimals once the +run file is read by `trec_eval`'s tie-break (score descending, then document number descending). +Scored in the engine's own order the same ranking reads 0.4005; the difference is the order inside an +exact score tie, not the ranking. The bit-level raw-score comparison this paragraph used to report for +all 15,466 (query, document) pairs is withdrawn: the figure it belonged to is not what the documented +configuration measures, and equal scores do not by themselves settle an aggregate that depends on the +reader's tie order. See [Evaluation](evaluation.md). ## BM25 variants diff --git a/scripts/reproducibility.sh b/scripts/reproducibility.sh index 7c1dc7d..8c3fe6b 100755 --- a/scripts/reproducibility.sh +++ b/scripts/reproducibility.sh @@ -215,7 +215,13 @@ eval_corpus() { # The parity row that used to be a gap: ArguAna at the reference's own k1 and b under the reference # analysis, which now reproduces 0.3970. Gated, because the gap it replaced was 0.033 of nDCG and a -# silent regression back into it would otherwise pass every other stage here. Three of the settings it +# silent regression back into it would otherwise pass every other stage here. The metric is read in +# the run reader's order, not the engine's: score descending, then document number descending, which +# is what trec_eval does with the file this stage writes. That matters because the four-decimal +# rounding deliberately creates exact ties; the engine's document-id-ascending order reads 0.4005 on +# the same ranking. The run retrieves 1,000 deep because the reader can promote the 11th document +# into the measured top ten when it ties the tenth score, so --top-k 10 would read 0.3974. +# Three of the settings it # needs are opt-in, so they are named here rather than hidden in a preset: --analyzer uax29 selects # the reference segmentation, --reference-index-statistics checks the index against the reference's own # term count before any score is computed, and --query-term-frequency counts a repeated query term once diff --git a/tests/LexiSharp.Tests/EvaluationHarnessTests.cs b/tests/LexiSharp.Tests/EvaluationHarnessTests.cs index c92e482..24e4220 100644 --- a/tests/LexiSharp.Tests/EvaluationHarnessTests.cs +++ b/tests/LexiSharp.Tests/EvaluationHarnessTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using LexiSharp.Core; using LexiSharp.Eval; using LexiSharp.Indexing; using LexiSharp.Linguistics; @@ -159,6 +160,26 @@ public void AHeldOutRunWithNothingToFitReportsWhyItDidNotRun() Assert.Contains("no tuned row was fitted", heldOut.NotRunReason, StringComparison.Ordinal); } + [Fact] + public void ARoundedRunIsMeasuredInTheOrderItsReaderReconstructs() + { + // A run file's rank column is ignored by trec_eval: it sorts by score descending and breaks + // an exact tie by document id *descending*. Rounding deliberately puts scores on the same + // ten-thousandth, and the walk-down cannot always separate them (at a score of 118, float + // spacing is coarser than the millionth it subtracts), so the reader's tie-break is what the + // published number was measured with. The engine's own order is document id ascending. + var hits = new[] + { + new SearchResult("a", 10.0, new SearchDocument("a", string.Empty)), + new SearchResult("b", 10.0, new SearchDocument("b", string.Empty)), + new SearchResult("c", 11.0, new SearchDocument("c", string.Empty)), + }; + + Assert.Equal( + ["c", "b", "a"], + Evaluation.RunFileOrder(hits).Select(hit => hit.DocumentId)); + } + private static BeirCorpus Corpus() => new() { Name = "test",