From 509daaca6ab468e399b9a5b994971a1e04293c7d Mon Sep 17 00:00:00 2001 From: manuc66 Date: Tue, 17 Nov 2026 19:44:44 +0100 Subject: [PATCH 1/8] Emit emoji as terms under the Unicode word boundaries The reference tokenizer starts an emoji token on every code point with the Extended_Pictographic property, and FiQA's index holds 34 of them that this library dropped. The rule is the property, not the OtherSymbol category: the same corpus holds 143 other So characters (degree sign, black square, numero) that the reference drops, so a category rule would add terms on the other side. The tables are pinned to Unicode 12.1 because the reference grammar declares that version; later revisions reassign code points it already classifies. FiQA: 32 documents differing -> 8; missing 50 -> 16, extra 11 unchanged; total terms 5288596 -> 5288630. --- src/LexiSharp/Linguistics/Tokenizer.cs | 65 ++++++++++---- src/LexiSharp/Linguistics/WordBreakClasses.cs | 82 +++++++++++++++++ tests/LexiSharp.Tests/EmojiTokenTests.cs | 89 +++++++++++++++++++ .../LexiSharp.Tests/WordSegmentationTests.cs | 8 +- 4 files changed, 226 insertions(+), 18 deletions(-) create mode 100644 src/LexiSharp/Linguistics/WordBreakClasses.cs create mode 100644 tests/LexiSharp.Tests/EmojiTokenTests.cs diff --git a/src/LexiSharp/Linguistics/Tokenizer.cs b/src/LexiSharp/Linguistics/Tokenizer.cs index 2c7a7cf0..5b456c50 100644 --- a/src/LexiSharp/Linguistics/Tokenizer.cs +++ b/src/LexiSharp/Linguistics/Tokenizer.cs @@ -348,24 +348,26 @@ private void ScanJoining(ReadOnlySpan text, List? terms, List= 128 && _symbolTerms.Contains(text[position])) { - if (_options.KeepSingleCharTerms) - { - string symbol = NormalizeToken(text.Slice(position, 1)); - - if (_stopWords is null || !_stopWords.Contains(symbol)) - { - if (_options.Stemmer is not null) - symbol = _options.Stemmer.Stem(symbol); - - terms?.Add(symbol); - spans?.Add(new TokenSpan(symbol, position, 1)); - } - } - + EmitStandalone(text, position, 1, terms, spans); position++; continue; } + // Under the measured boundaries an emoji is a term of its own, listed as a symbol or + // not: the reference's grammar starts an emoji token on every `Extended_Pictographic` + // code point and on nothing else. That distinction is why this asks the emoji property + // rather than the character category — `✔`, `♦` and `😂` are terms in FiQA's index, + // while `°`, `■` and `№` are dropped by both implementations. + if (_conditionalJoins + && text[position] >= 128 + && Rune.DecodeFromUtf16(text[position..], out Rune standalone, out int standaloneLength) == OperationStatus.Done + && WordBreakClasses.IsEmoji(standalone)) + { + EmitStandalone(text, position, standaloneLength, terms, spans); + position += standaloneLength; + continue; + } + bool beginsWord = StartsWordAt(text, position); // An unconditional joiner can open a token when a word character follows it: measured @@ -460,6 +462,38 @@ private void ScanJoining(ReadOnlySpan text, List? terms, List + /// Emits a one-code-point term that the reference tokenizes on its own — a configured symbol, + /// an emoji, an ideograph — through the same normalization, stop-word and stemming pipeline as + /// a word. + /// + /// + /// still decides whether it survives, because + /// that is the option a caller uses to ask for no one-character terms at all; the uax29 analysis + /// sets it, so every symbol, emoji and ideograph reaches the index. + /// + private void EmitStandalone( + ReadOnlySpan text, + int start, + int length, + List? terms, + List? spans) + { + if (!_options.KeepSingleCharTerms) + return; + + string term = NormalizeToken(text.Slice(start, length)); + + if (_stopWords is not null && _stopWords.Contains(term)) + return; + + if (_options.Stemmer is not null) + term = _options.Stemmer.Stem(term); + + terms?.Add(term); + spans?.Add(new TokenSpan(term, start, length)); + } + /// /// Whether ends in the two-character possessive 's or ’s. /// @@ -537,8 +571,7 @@ or UnicodeCategory.SpacingCombiningMark /// Whether a word starts at . A joiner never starts one. private static bool StartsWordAt(ReadOnlySpan text, int position) - { - char c = text[position]; + { char c = text[position]; if (c < 128) return AsciiWordChars.Contains(c); diff --git a/src/LexiSharp/Linguistics/WordBreakClasses.cs b/src/LexiSharp/Linguistics/WordBreakClasses.cs new file mode 100644 index 00000000..15132714 --- /dev/null +++ b/src/LexiSharp/Linguistics/WordBreakClasses.cs @@ -0,0 +1,82 @@ +using System.Text; + +namespace LexiSharp.Linguistics; + +/// +/// The Unicode character classes the reference word tokenizer tests directly, which the base class +/// library does not expose: the emoji property and the ideographic script. +/// +/// +/// +/// The tables are pinned to Unicode 12.1 on purpose. The tokenizer this library reproduces — Lucene +/// 10.5.2's StandardTokenizer — is generated from a grammar declared as %unicode 12.1, +/// so its answers are the 12.1 answers even where a later Unicode version reassigns a code point. +/// Two of those reassignments are visible here: U+0600..U+0605 are Format in 12.1 and become +/// Prepend later, and the Extended_Pictographic property was laid out over reserved +/// ranges that later shrank. Using the newer tables would part company with the reference on +/// characters assigned after 12.1 — rare in a corpus, but not impossible in a URL or a name. +/// +/// +/// This is deliberately not a general-purpose Unicode classification API: it exists so that +/// can emit the same tokens as that grammar. +/// The English and identifier analyses never call it. +/// +/// +internal static class WordBreakClasses +{ + /// + /// Whether carries the Extended_Pictographic property — the set + /// the reference grammar starts an emoji token on. Not every OtherSymbol is emoji: + /// © and ™ are, while ° and ■ are not, and the reference drops the + /// latter where it emits the former. + /// + public static bool IsEmoji(Rune rune) + { + if (rune.Value < 0xA9) + return false; + + return InRanges(rune.Value, EmojiRanges); + } + + /// + /// The Extended_Pictographic code point ranges of Unicode 12.1, as adjacent + /// [start, end] pairs sorted ascending. + /// + private static readonly int[] EmojiRanges = + [ + 0xA9, 0xA9, 0xAE, 0xAE, 0x203C, 0x203C, 0x2049, 0x2049, 0x2122, 0x2122, 0x2139, 0x2139, 0x2194, 0x2199, 0x21A9, 0x21AA, + 0x231A, 0x231B, 0x2328, 0x2328, 0x2388, 0x2388, 0x23CF, 0x23CF, 0x23E9, 0x23F3, 0x23F8, 0x23FA, 0x24C2, 0x24C2, 0x25AA, 0x25AB, + 0x25B6, 0x25B6, 0x25C0, 0x25C0, 0x25FB, 0x25FE, 0x2600, 0x2605, 0x2607, 0x2612, 0x2614, 0x2685, 0x2690, 0x2705, 0x2708, 0x2712, + 0x2714, 0x2714, 0x2716, 0x2716, 0x271D, 0x271D, 0x2721, 0x2721, 0x2728, 0x2728, 0x2733, 0x2734, 0x2744, 0x2744, 0x2747, 0x2747, + 0x274C, 0x274C, 0x274E, 0x274E, 0x2753, 0x2755, 0x2757, 0x2757, 0x2763, 0x2767, 0x2795, 0x2797, 0x27A1, 0x27A1, 0x27B0, 0x27B0, + 0x27BF, 0x27BF, 0x2934, 0x2935, 0x2B05, 0x2B07, 0x2B1B, 0x2B1C, 0x2B50, 0x2B50, 0x2B55, 0x2B55, 0x3030, 0x3030, 0x303D, 0x303D, + 0x3297, 0x3297, 0x3299, 0x3299, 0x1F000, 0x1F0FF, 0x1F10D, 0x1F10F, 0x1F12F, 0x1F12F, 0x1F16C, 0x1F171, 0x1F17E, 0x1F17F, 0x1F18E, 0x1F18E, + 0x1F191, 0x1F19A, 0x1F1AD, 0x1F1E5, 0x1F201, 0x1F20F, 0x1F21A, 0x1F21A, 0x1F22F, 0x1F22F, 0x1F232, 0x1F23A, 0x1F23C, 0x1F23F, 0x1F249, 0x1F3FA, + 0x1F400, 0x1F53D, 0x1F546, 0x1F64F, 0x1F680, 0x1F6FF, 0x1F774, 0x1F77F, 0x1F7D5, 0x1F7FF, 0x1F80C, 0x1F80F, 0x1F848, 0x1F84F, 0x1F85A, 0x1F85F, + 0x1F888, 0x1F88F, 0x1F8AE, 0x1F8FF, 0x1F90C, 0x1F93A, 0x1F93C, 0x1F945, 0x1F947, 0x1FFFD, + ]; + + /// + /// Whether falls inside one of the [start, end] pairs in + /// . + /// + private static bool InRanges(int value, ReadOnlySpan ranges) + { + int low = 0; + int high = (ranges.Length / 2) - 1; + + while (low <= high) + { + int middle = (low + high) / 2; + + if (value < ranges[middle * 2]) + high = middle - 1; + else if (value > ranges[(middle * 2) + 1]) + low = middle + 1; + else + return true; + } + + return false; + } +} diff --git a/tests/LexiSharp.Tests/EmojiTokenTests.cs b/tests/LexiSharp.Tests/EmojiTokenTests.cs new file mode 100644 index 00000000..e2c3c769 --- /dev/null +++ b/tests/LexiSharp.Tests/EmojiTokenTests.cs @@ -0,0 +1,89 @@ +using LexiSharp.Linguistics; +using Xunit; + +namespace LexiSharp.Tests; + +/// +/// Under an emoji code point is a term of its +/// own, and a symbol that is not emoji is not. +/// +/// +/// The rule is the Extended_Pictographic property, not the OtherSymbol category, and +/// the difference is not academic: FiQA's index holds ✔, ♦, ☏ and the emoji, +/// while it holds no °, ■ or № at all. A rule written on the category emits +/// 143 terms the reference does not have. +/// +/// The code point is emitted whole, so a supplementary-plane emoji is one term rather than two +/// surrogate halves. Multi-code-point emoji sequences — a ZWJ family, a flag pair — are not +/// reproduced; see the remark on WordBreakClasses and the plan's residual note. +/// +/// +public class EmojiTokenTests +{ + private static Tokenizer Segmenter() => new(new TokenizerOptions + { + KeepSingleCharTerms = true, + FoldDiacritics = false, + WordSegmentation = WordSegmentation.UnicodeWordBoundaries, + }); + + [Theory] + [InlineData("✔")] + [InlineData("♦")] + [InlineData("☏")] + [InlineData("©")] + [InlineData("®")] + [InlineData("™")] + [InlineData("↔")] + [InlineData("🤔")] + [InlineData("😂")] + [InlineData("👻")] + public void AnEmojiCodePointIsATerm(string symbol) + { + Assert.Equal([symbol], Segmenter().Tokenize(symbol)); + } + + [Fact] + public void ASupplementaryEmojiIsOneTermNotTwoSurrogateHalves() + { + IReadOnlyList terms = Segmenter().Tokenize("👨‍👩‍👧‍👦 🤔"); + + // The ZWJ family is four emoji code points plus three joiners and becomes four terms; the + // point of the assertion is that the surrogate pairs are not split. + Assert.Equal(["👨", "👩", "👧", "👦", "🤔"], terms); + } + + [Fact] + public void AnEmojiDoesNotExtendTheWordBesideIt() + { + Assert.Equal(["aspirin", "®"], Segmenter().Tokenize("Aspirin®")); + Assert.Equal(["a", "🤔", "b"], Segmenter().Tokenize("a🤔b")); + Assert.Equal(["2024", "©"], Segmenter().Tokenize("2024©")); + } + + [Fact] + public void RepeatedEmojiAreOneTermEach() + { + Assert.Equal(["🤔", "🤔", "🤔"], Segmenter().Tokenize("🤔🤔🤔")); + Assert.Equal(["✔", "✔"], Segmenter().Tokenize("✔ ✔")); + } + + /// + /// The other half of the rule: the category is not enough. These characters are + /// OtherSymbol and the reference drops them. + /// + [Theory] + [InlineData("°")] + [InlineData("¦")] + [InlineData("●")] + [InlineData("■")] + [InlineData("▲")] + [InlineData("№")] + [InlineData("℅")] + [InlineData("│")] + public void ANonEmojiSymbolIsNotATerm(string symbol) + { + Assert.Empty(Segmenter().Tokenize(symbol)); + Assert.Equal(["a", "b"], Segmenter().Tokenize("a" + symbol + "b")); + } +} diff --git a/tests/LexiSharp.Tests/WordSegmentationTests.cs b/tests/LexiSharp.Tests/WordSegmentationTests.cs index c9f92af5..179ccccc 100644 --- a/tests/LexiSharp.Tests/WordSegmentationTests.cs +++ b/tests/LexiSharp.Tests/WordSegmentationTests.cs @@ -206,8 +206,12 @@ public void AConfiguredSymbolIsItsOwnTokenAndDoesNotExtendTheWord() } [Fact] - public void WithoutTheOptionASymbolIsStillDropped() + public void AnEmojiSymbolIsATermWithoutTheOptionAndANonEmojiSymbolIsNot() { - Assert.Equal(["aspirin"], Segmenter().Tokenize("Aspirin®")); + // `®` carries the emoji property, so the mode emits it on its own; `°` does not, and the + // reference drops it. SymbolTerms is then redundant for the four characters the uax29 + // analysis used to list — it remains for configurations that want an explicit set. + Assert.Equal(["aspirin", "®"], Segmenter().Tokenize("Aspirin®")); + Assert.Equal(["20", "c"], Segmenter().Tokenize("20°c")); } } From 310b8d71d474ada61a147cbea1b537a6fb5e527f Mon Sep 17 00:00:00 2001 From: manuc66 Date: Tue, 17 Nov 2026 19:46:09 +0100 Subject: [PATCH 2/8] Emit each Han ideograph as its own term under the word boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reference grammar's IDEOGRAPHIC rule emits a single CJK ideograph per token, so a run of them is not a word. FiQA holds nine ideographs; joining 理财 and 雙元貨幣產品 lost eight terms and invented two. The rule is Script=Han minus its word letters, not a block range: U+3005 and U+303B are Word_Break=ALetter and the reference keeps them inside an alphanumeric run, which this library's letter scan already did. Katakana and Hangul runs are untouched. FiQA: 8 documents differing -> 5; missing 16 -> 6, extra 11 -> 8; total terms 5288630 -> 5288637. --- src/LexiSharp/Linguistics/Tokenizer.cs | 23 +++++- src/LexiSharp/Linguistics/WordBreakClasses.cs | 28 +++++++ .../LexiSharp.Tests/IdeographicTokenTests.cs | 77 +++++++++++++++++++ 3 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 tests/LexiSharp.Tests/IdeographicTokenTests.cs diff --git a/src/LexiSharp/Linguistics/Tokenizer.cs b/src/LexiSharp/Linguistics/Tokenizer.cs index 5b456c50..68fc6997 100644 --- a/src/LexiSharp/Linguistics/Tokenizer.cs +++ b/src/LexiSharp/Linguistics/Tokenizer.cs @@ -358,10 +358,13 @@ private void ScanJoining(ReadOnlySpan text, List? terms, List= 128 && Rune.DecodeFromUtf16(text[position..], out Rune standalone, out int standaloneLength) == OperationStatus.Done - && WordBreakClasses.IsEmoji(standalone)) + && (WordBreakClasses.IsEmoji(standalone) || WordBreakClasses.IsIdeographic(standalone))) { EmitStandalone(text, position, standaloneLength, terms, spans); position += standaloneLength; @@ -410,6 +413,11 @@ private void ScanJoining(ReadOnlySpan text, List? terms, List text, int position) && IsCombiningMark(rune); } + /// + /// Whether an ideograph starts at — a term of its own under the + /// measured boundaries rather than part of the run beside it. + /// + private static bool IsIdeographicAt(ReadOnlySpan text, int position) + { + if (text[position] < 0x2E80) + return false; + + return Rune.DecodeFromUtf16(text[position..], out Rune rune, out _) == OperationStatus.Done + && WordBreakClasses.IsIdeographic(rune); + } + /// /// Whether is one of the three Unicode mark categories. /// diff --git a/src/LexiSharp/Linguistics/WordBreakClasses.cs b/src/LexiSharp/Linguistics/WordBreakClasses.cs index 15132714..78d81a97 100644 --- a/src/LexiSharp/Linguistics/WordBreakClasses.cs +++ b/src/LexiSharp/Linguistics/WordBreakClasses.cs @@ -56,6 +56,34 @@ public static bool IsEmoji(Rune rune) 0x1F888, 0x1F88F, 0x1F8AE, 0x1F8FF, 0x1F90C, 0x1F93A, 0x1F93C, 0x1F945, 0x1F947, 0x1FFFD, ]; + /// + /// Whether is a CJK ideograph: a character with the Han script + /// that the reference grammar tokenizes one code point at a time. + /// + /// + /// The exceptions are the script's word letters: U+3005 and U+303B, the ideographic iteration + /// marks, are Script=Han but Word_Break=ALetter, so the reference folds them into + /// an alphanumeric run instead of emitting them alone. U+3007, the ideographic number zero, + /// stays here because it is Word_Break=Other. + /// + public static bool IsIdeographic(Rune rune) + { + if (rune.Value < 0x2E80) + return false; + + return InRanges(rune.Value, IdeographicRanges); + } + + /// + /// The Script=Han code points of Unicode 12.1 that are not word letters, as adjacent + /// [start, end] pairs sorted ascending. + /// + private static readonly int[] IdeographicRanges = + [ + 0x2E80, 0x2E99, 0x2E9B, 0x2EF3, 0x2F00, 0x2FD5, 0x3007, 0x3007, 0x3021, 0x3029, 0x3038, 0x303A, 0x3400, 0x4DB5, 0x4E00, 0x9FEF, + 0xF900, 0xFA6D, 0xFA70, 0xFAD9, 0x20000, 0x2A6D6, 0x2A700, 0x2B734, 0x2B740, 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2F800, 0x2FA1D, + ]; + /// /// Whether falls inside one of the [start, end] pairs in /// . diff --git a/tests/LexiSharp.Tests/IdeographicTokenTests.cs b/tests/LexiSharp.Tests/IdeographicTokenTests.cs new file mode 100644 index 00000000..16cf747c --- /dev/null +++ b/tests/LexiSharp.Tests/IdeographicTokenTests.cs @@ -0,0 +1,77 @@ +using LexiSharp.Linguistics; +using Xunit; + +namespace LexiSharp.Tests; + +/// +/// Under a Han ideograph is a term of its own: +/// 理财 is two terms, not one. +/// +/// +/// The reference tokenizer's IDEOGRAPHIC rule emits a single CJK ideograph per token, so a +/// run of them is not a word. FiQA holds nine ideographs, and joining them lost six terms and +/// invented a seventh. +/// +/// The rule is the script with its word letters removed, not a range: U+3005 and U+303B (the +/// ideographic iteration marks) are Script=Han and Word_Break=ALetter, so the +/// reference keeps them inside an alphanumeric run — a々b is one token — and this library's +/// letter scan already did the same. Only the ideographs that are not word letters split. +/// +/// +public class IdeographicTokenTests +{ + private static Tokenizer Segmenter() => new(new TokenizerOptions + { + KeepSingleCharTerms = true, + FoldDiacritics = false, + WordSegmentation = WordSegmentation.UnicodeWordBoundaries, + }); + + [Fact] + public void EachIdeographIsATerm() + { + Assert.Equal(["理", "财"], Segmenter().Tokenize("理财")); + Assert.Equal(["雙", "元", "貨", "幣", "產", "品"], Segmenter().Tokenize("雙元貨幣產品")); + } + + [Fact] + public void AnIdeographEndsTheRunBesideIt() + { + Assert.Equal(["6", "尺"], Segmenter().Tokenize("6尺")); + Assert.Equal(["abc", "尺", "def"], Segmenter().Tokenize("abc尺def")); + Assert.Equal(["尺", "abc"], Segmenter().Tokenize("尺abc")); + } + + [Fact] + public void RepeatedIdeographsAreOneTermEach() + { + Assert.Equal(["尺", "尺"], Segmenter().Tokenize("尺尺")); + } + + /// + /// The exceptions that keep this from being a script test: iteration marks are word letters in + /// the reference's word-break data, and an alphanumeric run keeps them. + /// + [Fact] + public void TheIdeographicIterationMarksStayInTheWord() + { + Assert.Equal(["a々b"], Segmenter().Tokenize("a々b")); + Assert.Equal(["a〻b"], Segmenter().Tokenize("a〻b")); + } + + [Fact] + public void TheIdeographicNumberZeroIsATerm() + { + Assert.Equal(["〇"], Segmenter().Tokenize("〇")); + } + + /// + /// The control: the rule is about ideographs only. A Katakana run was one token before this + /// change and has to stay one. + /// + [Fact] + public void KatakanaIsStillOneRun() + { + Assert.Equal(["カタカナ"], Segmenter().Tokenize("カタカナ")); + } +} From 30323d1de026feb16a923bf0eb759dcdb89c953f Mon Sep 17 00:00:00 2001 From: manuc66 Date: Tue, 17 Nov 2026 19:47:09 +0100 Subject: [PATCH 3/8] Read a word's class past its combining marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The separator table joins a full stop between two letters, and it asks for the class of the characters on each side. A word ending in a combining mark put the mark on that side, and a mark is neither a letter nor a digit, so tɕɛ̝.bʌl split into two terms where the reference holds one. The class now skips the marks that modify the base character, which is what they are: tɕɛ̝ has the class of ɛ. FiQA: 5 documents differing -> 3; missing 6 -> 4, extra 8 -> 4; total terms 5288637 -> 5288635. --- src/LexiSharp/Linguistics/Tokenizer.cs | 18 ++++++++++++++++++ tests/LexiSharp.Tests/CombiningMarkTests.cs | 15 +++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/LexiSharp/Linguistics/Tokenizer.cs b/src/LexiSharp/Linguistics/Tokenizer.cs index 68fc6997..27d78bbd 100644 --- a/src/LexiSharp/Linguistics/Tokenizer.cs +++ b/src/LexiSharp/Linguistics/Tokenizer.cs @@ -722,6 +722,24 @@ private static bool JoinsUnderCondition(ReadOnlySpan text, int position) private static CharClass ClassAt(ReadOnlySpan text, int index) { + // A combining mark takes its class from the character it modifies. The stop in `tɕɛ̝.bʌl` + // sits between two letters, but the character immediately before it is U+031D, a mark; the + // reference joins the full stop on both sides being letters, so the class has to be read + // past the mark rather than off it. + while (index >= 0 && text[index] >= 128) + { + if (Rune.DecodeLastFromUtf16(text[..(index + 1)], out Rune mark, out int markLength) != OperationStatus.Done + || !IsCombiningMark(mark)) + { + break; + } + + index -= markLength; + } + + if (index < 0) + return CharClass.Other; + char c = text[index]; if (c < 128) diff --git a/tests/LexiSharp.Tests/CombiningMarkTests.cs b/tests/LexiSharp.Tests/CombiningMarkTests.cs index d8031b9b..4e97df1f 100644 --- a/tests/LexiSharp.Tests/CombiningMarkTests.cs +++ b/tests/LexiSharp.Tests/CombiningMarkTests.cs @@ -87,4 +87,19 @@ public void PrecomposedLettersAreUnaffected() Assert.Equal(["क़"], tokenizer.Tokenize("क़")); Assert.Equal(["ẋ"], tokenizer.Tokenize("ẋ")); } + + /// + /// A mark does not change the class of the word that ends with it, so a separator that joins two + /// letters still joins when a mark sits between the last letter and the separator. FiQA's + /// tɕɛ̝.bʌl is one term in the reference; reading the class off U+031D — a mark, neither a + /// letter nor a digit — split it into two and was the whole of that document's difference. + /// + [Fact] + public void AMarkBeforeTheSeparatorDoesNotChangeTheWordClass() + { + Assert.Equal(["tɕɛ̝.bʌl"], Tokenizing().Tokenize("tɕɛ̝.bʌl")); + Assert.Equal(["ab" + Dot + ".cd"], Tokenizing().Tokenize("ab" + Dot + ".cd")); + Assert.Equal(["ab" + Dot + ":cd"], Tokenizing().Tokenize("ab" + Dot + ":cd")); + Assert.Equal(["12" + Dot + ",34"], Tokenizing().Tokenize("12" + Dot + ",34")); + } } \ No newline at end of file From 1c46d17877d8fdabe7e27f314aadb48021861ef3 Mon Sep 17 00:00:00 2001 From: manuc66 Date: Tue, 17 Nov 2026 19:48:10 +0100 Subject: [PATCH 4/8] Keep format characters inside the word under the word boundaries The reference's word-break data classes a byte-order mark, a left-to-right mark and a soft hyphen as Format, which a word continues through; the term keeps the character. FiQA holds 1990 and bank, both split here before this rule. U+200B is the exception that makes this a word-break rule rather than a category test: it is Format by category and Other by word-break class, and the reference splits on it. A format character still cannot open a word, and the separator class lookup now reads past one, as it reads past a combining mark. FiQA: 3 documents differing -> 1; missing 4 -> 2, extra 4 -> 1; total terms 5288635 -> 5288634. --- src/LexiSharp/Linguistics/Tokenizer.cs | 26 +++++-- src/LexiSharp/Linguistics/WordBreakClasses.cs | 31 ++++++++ tests/LexiSharp.Tests/FormatCharacterTests.cs | 77 +++++++++++++++++++ 3 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 tests/LexiSharp.Tests/FormatCharacterTests.cs diff --git a/src/LexiSharp/Linguistics/Tokenizer.cs b/src/LexiSharp/Linguistics/Tokenizer.cs index 27d78bbd..e1a9c7f1 100644 --- a/src/LexiSharp/Linguistics/Tokenizer.cs +++ b/src/LexiSharp/Linguistics/Tokenizer.cs @@ -542,8 +542,8 @@ private static bool LeadsWordAt(ReadOnlySpan text, int position) } /// - /// Whether the character at continues a word: a combining mark does, a - /// mark cannot start one. + /// Whether the character at continues a word: a combining mark or a + /// format character does, and neither can start one. /// /// /// Measured, and the asymmetry is the point: a◌̇b is one token and a◌̇ is one token, @@ -554,14 +554,25 @@ private static bool LeadsWordAt(ReadOnlySpan text, int position) /// two. On the BEIR corpora in this repository the effect is tiny but real: ArguAna holds no combining /// mark at all, SciFact none either, NFCorpus four characters out of 5,779,318. /// + /// + /// A format character is the same kind of thing for the reference's word-break data: it cannot + /// open a word and does not close one. FiQA holds two terms that depend on it — a byte-order + /// mark inside 1990 and a left-to-right mark after bank — where this library + /// split the word at the format character and lost the term. + /// /// - private static bool ContinuesWordAt(ReadOnlySpan text, int position) + private bool ContinuesWordAt(ReadOnlySpan text, int position) { if (text[position] < 128) return false; - return Rune.DecodeFromUtf16(text[position..], out Rune rune, out _) == OperationStatus.Done - && IsCombiningMark(rune); + if (Rune.DecodeFromUtf16(text[position..], out Rune rune, out _) != OperationStatus.Done) + return false; + + // Format characters extend a word only under the measured boundaries, where the reference's + // word-break data says so. The identifiers analysis shares this scan and keeps its behavior: + // a format character still ends the identifier there. + return IsCombiningMark(rune) || (_conditionalJoins && WordBreakClasses.IsFormatExtend(rune)); } /// @@ -725,11 +736,12 @@ private static CharClass ClassAt(ReadOnlySpan text, int index) // A combining mark takes its class from the character it modifies. The stop in `tɕɛ̝.bʌl` // sits between two letters, but the character immediately before it is U+031D, a mark; the // reference joins the full stop on both sides being letters, so the class has to be read - // past the mark rather than off it. + // past the mark rather than off it. A format character is skipped for the same reason: it + // belongs to the word it sits in and has no class of its own. while (index >= 0 && text[index] >= 128) { if (Rune.DecodeLastFromUtf16(text[..(index + 1)], out Rune mark, out int markLength) != OperationStatus.Done - || !IsCombiningMark(mark)) + || (!IsCombiningMark(mark) && !WordBreakClasses.IsFormatExtend(mark))) { break; } diff --git a/src/LexiSharp/Linguistics/WordBreakClasses.cs b/src/LexiSharp/Linguistics/WordBreakClasses.cs index 78d81a97..bb789afb 100644 --- a/src/LexiSharp/Linguistics/WordBreakClasses.cs +++ b/src/LexiSharp/Linguistics/WordBreakClasses.cs @@ -84,6 +84,37 @@ public static bool IsIdeographic(Rune rune) 0xF900, 0xFA6D, 0xFA70, 0xFAD9, 0x20000, 0x2A6D6, 0x2A700, 0x2B734, 0x2B740, 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2F800, 0x2FA1D, ]; + /// + /// Whether is a format character the reference's word-break data lets a + /// word continue through: a soft hyphen, a byte-order mark, a left-to-right mark, and the other + /// Word_Break=Format code points. + /// + /// + /// A format character cannot open a word, but it does not end one: a byte-order mark between + /// the digits of 1990, or a left-to-right mark after bank, stays inside the term + /// in the reference index. U+200B is deliberately absent — it is Word_Break=Other, and + /// the reference splits a word around it, which this library already did. + /// + public static bool IsFormatExtend(Rune rune) + { + if (rune.Value < 0xAD) + return false; + + return InRanges(rune.Value, FormatRanges); + } + + /// + /// The Word_Break=Format and Word_Break=ZWJ code points of Unicode 12.1 that are + /// format characters, plus U+200C, whose word-break class is Extend, as adjacent + /// [start, end] pairs sorted ascending. + /// + private static readonly int[] FormatRanges = + [ + 0xAD, 0xAD, 0x600, 0x605, 0x61C, 0x61C, 0x6DD, 0x6DD, 0x70F, 0x70F, 0x8E2, 0x8E2, 0x180E, 0x180E, 0x200C, 0x200F, + 0x202A, 0x202E, 0x2060, 0x2064, 0x2066, 0x206F, 0xFEFF, 0xFEFF, 0xFFF9, 0xFFFB, 0x110BD, 0x110BD, 0x110CD, 0x110CD, 0x13430, 0x13438, + 0x1BCA0, 0x1BCA3, 0x1D173, 0x1D17A, 0xE0001, 0xE0001, 0xE0020, 0xE007F, + ]; + /// /// Whether falls inside one of the [start, end] pairs in /// . diff --git a/tests/LexiSharp.Tests/FormatCharacterTests.cs b/tests/LexiSharp.Tests/FormatCharacterTests.cs new file mode 100644 index 00000000..96e229d2 --- /dev/null +++ b/tests/LexiSharp.Tests/FormatCharacterTests.cs @@ -0,0 +1,77 @@ +using LexiSharp.Linguistics; +using Xunit; + +namespace LexiSharp.Tests; + +/// +/// Under a format character does not end the +/// word it sits in, and does not open one of its own. +/// +/// +/// The reference's word-break data classes a byte-order mark, a left-to-right mark and a soft hyphen +/// as Format, which a word continues through. FiQA holds 199 + U+FEFF + 0 and +/// bank + U+200E, both one term in the reference and two here before this rule. +/// +/// The exception is U+200B, the zero-width space: its word-break class is Other, and the +/// reference splits a word around it. Treating every Format character the same would merge +/// 145 of FiQA's word boundaries. +/// +/// +public class FormatCharacterTests +{ + private const string Bom = "\uFEFF"; + private const string LeftToRightMark = "\u200E"; + private const string ZeroWidthSpace = "\u200B"; + private const string SoftHyphen = "\u00AD"; + + private static Tokenizer Segmenter() => new(new TokenizerOptions + { + KeepSingleCharTerms = true, + FoldDiacritics = false, + WordSegmentation = WordSegmentation.UnicodeWordBoundaries, + }); + + [Fact] + public void AFormatCharacterInsideAWordStaysInTheTerm() + { + Assert.Equal(["199" + Bom + "0"], Segmenter().Tokenize("199" + Bom + "0")); + Assert.Equal(["bank" + LeftToRightMark], Segmenter().Tokenize("bank" + LeftToRightMark)); + Assert.Equal(["x" + LeftToRightMark + "y"], Segmenter().Tokenize("x" + LeftToRightMark + "y")); + } + + [Fact] + public void AFormatCharacterCannotOpenAWord() + { + Assert.Equal(["ab"], Segmenter().Tokenize(Bom + "ab")); + Assert.Equal(["ab"], Segmenter().Tokenize(LeftToRightMark + "ab")); + Assert.Empty(Segmenter().Tokenize(Bom)); + } + + [Fact] + public void AFormatCharacterDoesNotChangeTheWordClassForASeparator() + { + Assert.Equal(["bank" + LeftToRightMark + ".foo"], Segmenter().Tokenize("bank" + LeftToRightMark + ".foo")); + } + + /// + /// The zero-width space is Format by category and Other by word-break class, and + /// the reference splits on it. This is the pin that keeps the rule from being a category test. + /// + [Fact] + public void AZeroWidthSpaceStillSplits() + { + Assert.Equal(["a", "b"], Segmenter().Tokenize("a" + ZeroWidthSpace + "b")); + Assert.Equal(["a", "b", "c"], Segmenter().Tokenize("a" + ZeroWidthSpace + "b" + ZeroWidthSpace + "c")); + } + + /// + /// The soft hyphen was already a joiner before this rule, and it still is; this checks that the + /// format-character rule did not change what the soft hyphen does at the end of a word. + /// + [Fact] + public void ASoftHyphenStillJoinsAndStaysAtTheEnd() + { + Assert.Equal(["co" + SoftHyphen + "operate"], Segmenter().Tokenize("co" + SoftHyphen + "operate")); + Assert.Equal(["bank" + SoftHyphen], Segmenter().Tokenize("bank" + SoftHyphen)); + } +} From 52e9e305f19c092a9db38c39f67708080a66d4f6 Mon Sep 17 00:00:00 2001 From: manuc66 Date: Tue, 17 Nov 2026 19:51:23 +0100 Subject: [PATCH 5/8] Chop a token at the reference scanner's 255-unit buffer The reference tokenizer emits a token when its 255-unit input buffer fills and continues the rest as a new token, so FiQA's 468-character blob is indexed as one term of 255 and one of 213. The limit is the scanner's buffer width, so it belongs to the mode that reproduces that scanner: the identifiers analysis runs through the same joining scan and keeps a long identifier whole, pinned by the new scope tests alongside emoji, ideographs and format characters. A cut that would land inside a surrogate pair moves back one unit, matching the scanner's hold-back of a trailing high surrogate. FiQA: 1 document differing -> 0; missing 2 -> 0, extra 1 -> 0; total terms 5288634 -> 5288635, identical to the Lucene 10.5.2 dump. --- src/LexiSharp/Linguistics/Tokenizer.cs | 58 ++++++++++--- src/LexiSharp/Linguistics/WordBreakClasses.cs | 3 +- tests/LexiSharp.Tests/AnalysisScopeTests.cs | 81 +++++++++++++++++++ tests/LexiSharp.Tests/LongTokenTests.cs | 81 +++++++++++++++++++ 4 files changed, 209 insertions(+), 14 deletions(-) create mode 100644 tests/LexiSharp.Tests/AnalysisScopeTests.cs create mode 100644 tests/LexiSharp.Tests/LongTokenTests.cs diff --git a/src/LexiSharp/Linguistics/Tokenizer.cs b/src/LexiSharp/Linguistics/Tokenizer.cs index e1a9c7f1..c7ba7e36 100644 --- a/src/LexiSharp/Linguistics/Tokenizer.cs +++ b/src/LexiSharp/Linguistics/Tokenizer.cs @@ -451,23 +451,47 @@ private void ScanJoining(ReadOnlySpan text, List? terms, List + /// Emits the word occupying [start, end), chopped the way the reference's scanner chops a + /// token longer than its input buffer. + /// + /// + /// The reference tokenizer fills a buffer of 255 UTF-16 units and emits what it has when a token + /// reaches the end of it, then continues scanning the rest as a new token: a 468-character token + /// is indexed as one of 255 and one of 213, both terms. That is a property of the scanner, not a + /// length filter — nothing is dropped — and it is reproduced only for + /// , where the scanner being reproduced is + /// what the mode means. The identifiers analysis also runs through this scan and must keep + /// emitting a long identifier whole. + /// + /// A boundary that would land inside a surrogate pair moves back one unit, because the reference + /// scanner holds a trailing high surrogate over to the next refill rather than splitting it. + /// + /// + private void EmitWordRun( + ReadOnlySpan text, + int start, + int end, + List? terms, + List? spans) + { + var none = new NormalizedTermsBuilder(default, enabled: false); - if (_stopWords is not null && _stopWords.Contains(term)) - continue; + do + { + int chunkEnd = _conditionalJoins ? Math.Min(start + MaxTokenLength, end) : end; - if (_options.Stemmer is not null) - term = _options.Stemmer.Stem(term); + if (chunkEnd < end && chunkEnd - start > 1 && char.IsHighSurrogate(text[chunkEnd - 1])) + chunkEnd--; - terms?.Add(term); - spans?.Add(new TokenSpan(term, start, position - start)); + EmitWord(text, start, chunkEnd, terms, spans, ref none); + start = chunkEnd; } + while (start < end); } /// @@ -603,7 +627,8 @@ or UnicodeCategory.SpacingCombiningMark /// Whether a word starts at . A joiner never starts one. private static bool StartsWordAt(ReadOnlySpan text, int position) - { char c = text[position]; + { + char c = text[position]; if (c < 128) return AsciiWordChars.Contains(c); @@ -993,4 +1018,11 @@ private static string NormalizeSlow(string term) /// implementation's disagree on, measured over 505 positions whose lowercase differs. /// private const char TurkishDottedI = 'İ'; + + /// + /// The longest token the reference tokenizer emits before it continues with the rest as another + /// token. Its scanner's buffer is 255 UTF-16 units, so a longer token is chopped at that width — + /// not dropped, as the claim about Lucene's maxTokenLength is sometimes read. + /// + private const int MaxTokenLength = 255; } diff --git a/src/LexiSharp/Linguistics/WordBreakClasses.cs b/src/LexiSharp/Linguistics/WordBreakClasses.cs index bb789afb..5da76842 100644 --- a/src/LexiSharp/Linguistics/WordBreakClasses.cs +++ b/src/LexiSharp/Linguistics/WordBreakClasses.cs @@ -4,7 +4,8 @@ namespace LexiSharp.Linguistics; /// /// The Unicode character classes the reference word tokenizer tests directly, which the base class -/// library does not expose: the emoji property and the ideographic script. +/// library does not expose: the emoji property, the ideographic script and the word-continuing +/// format class. /// /// /// diff --git a/tests/LexiSharp.Tests/AnalysisScopeTests.cs b/tests/LexiSharp.Tests/AnalysisScopeTests.cs new file mode 100644 index 00000000..c925bc79 --- /dev/null +++ b/tests/LexiSharp.Tests/AnalysisScopeTests.cs @@ -0,0 +1,81 @@ +using LexiSharp.Linguistics; +using Xunit; + +namespace LexiSharp.Tests; + +/// +/// The four parity rules added for — emoji and +/// ideographs as terms, combining and format characters inside a word, and the 255-unit chop — do +/// not reach the other analyses. +/// +/// +/// This is the regression pin for the scope of the change: default, porter, +/// english and identifiers are the analyses every published figure and every caller +/// outside the uax29 comparison uses, and each keeps the behavior it had. The joining path is +/// shared with the identifiers analysis, which is why the chop is gated on the mode rather than on +/// the scan. +/// +public class AnalysisScopeTests +{ + public static TheoryData Analyses => new() { "default", "porter", "english", "identifiers" }; + + private static Tokenizer Build(string analyzer) => analyzer switch + { + "default" => new Tokenizer(TokenizerOptions.Default), + "porter" => new Tokenizer(new TokenizerOptions + { + Stemmer = new PorterStemmer(), + RemoveStopWords = true, + }), + "english" => new Tokenizer(new TokenizerOptions + { + Stemmer = new PorterStemmer(), + RemoveStopWords = true, + StopWords = StopWords.EnglishFunction, + KeepSingleCharTerms = true, + }), + "identifiers" => new Tokenizer(new TokenizerOptions + { + WordJoiners = "-_.:/", + KeepSingleCharTerms = true, + FoldDiacritics = false, + }), + _ => throw new ArgumentOutOfRangeException(nameof(analyzer)), + }; + + [Theory] + [MemberData(nameof(Analyses))] + public void AnEmojiIsNotATermOutsideTheMode(string analyzer) + { + Tokenizer tokenizer = Build(analyzer); + + Assert.Equal(["ab", "cd"], tokenizer.Tokenize("ab🤔cd")); + Assert.Empty(tokenizer.Tokenize("✔")); + } + + [Theory] + [MemberData(nameof(Analyses))] + public void AnIdeographicRunStaysOneWordOutsideTheMode(string analyzer) + { + Assert.Equal(["理财"], Build(analyzer).Tokenize("理财")); + } + + [Theory] + [MemberData(nameof(Analyses))] + public void AFormatCharacterStillSplitsOutsideTheMode(string analyzer) + { + // The ASCII number survives every analysis; the `0` after the byte-order mark is kept by + // `english` and `identifiers`, which keep single-character terms, and dropped by the two + // that do not. + string[] expected = analyzer is "english" or "identifiers" ? ["199", "0"] : ["199"]; + + Assert.Equal(expected, Build(analyzer).Tokenize("199\uFEFF0")); + } + + [Theory] + [MemberData(nameof(Analyses))] + public void ALongTokenIsNotChoppedOutsideTheMode(string analyzer) + { + Assert.Equal([new string('x', 468)], Build(analyzer).Tokenize(new string('x', 468))); + } +} diff --git a/tests/LexiSharp.Tests/LongTokenTests.cs b/tests/LexiSharp.Tests/LongTokenTests.cs new file mode 100644 index 00000000..b55eb230 --- /dev/null +++ b/tests/LexiSharp.Tests/LongTokenTests.cs @@ -0,0 +1,81 @@ +using LexiSharp.Linguistics; +using Xunit; + +namespace LexiSharp.Tests; + +/// +/// Under a token longer than the reference +/// scanner's 255-unit buffer is emitted in consecutive chunks, not dropped and not left whole. +/// +/// +/// FiQA holds one document whose token is 468 characters; the reference index holds it as one term +/// of 255 and one of 213. The limit is the scanner's buffer width, so it is a property of the mode +/// that reproduces that scanner and not of whitespace options: the other analyses keep a long +/// identifier whole, and pins that. +/// +public class LongTokenTests +{ + private static Tokenizer Segmenter(bool keepSingleCharTerms = true) => new(new TokenizerOptions + { + KeepSingleCharTerms = keepSingleCharTerms, + FoldDiacritics = false, + WordSegmentation = WordSegmentation.UnicodeWordBoundaries, + }); + + [Fact] + public void ATokenOf468CharactersIsChoppedAt255() + { + IReadOnlyList terms = Segmenter().Tokenize(new string('x', 468)); + + Assert.Equal(2, terms.Count); + Assert.Equal(new string('x', 255), terms[0]); + Assert.Equal(new string('x', 213), terms[1]); + } + + [Fact] + public void ATokenOfExactly255CharactersIsOneTerm() + { + Assert.Equal([new string('z', 255)], Segmenter().Tokenize(new string('z', 255))); + } + + [Fact] + public void ATokenOf256CharactersIs255And1() + { + Assert.Equal([new string('y', 255), "y"], Segmenter().Tokenize(new string('y', 256))); + } + + [Fact] + public void ATokenOf510CharactersIsTwoChunksOf255() + { + Assert.Equal( + [new string('w', 255), new string('w', 255)], + Segmenter().Tokenize(new string('w', 510))); + } + + /// + /// The chunk boundary never lands inside a surrogate pair. A 258-unit word whose 255th unit is + /// the high half of a Deseret letter is chopped at 254, so the pair moves to the second chunk + /// whole — which is what the reference scanner does when it holds a trailing high surrogate over. + /// + [Fact] + public void AChunkBoundaryDoesNotSplitASurrogatePair() + { + string word = new string('a', 254) + "\U00010400" + "bb"; + + IReadOnlyList terms = Segmenter().Tokenize(word); + + Assert.Equal(2, terms.Count); + Assert.Equal(new string('a', 254), terms[0]); + Assert.Equal("\U00010428bb", terms[1]); + } + + /// + /// Each chunk is a token in its own right, so the single-character rule applies to the tail: a + /// 256-character token keeps a one-character tail only when single-character terms are kept. + /// + [Fact] + public void TheTailChunkObeysTheSingleCharacterRule() + { + Assert.Equal([new string('y', 255)], Segmenter(keepSingleCharTerms: false).Tokenize(new string('y', 256))); + } +} From d6abd6c85ed0c1f4defa4ce8f7aa02e397645995 Mon Sep 17 00:00:00 2001 From: manuc66 Date: Tue, 17 Nov 2026 19:56:56 +0100 Subject: [PATCH 6/8] Include the narrow no-break space in the word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit U+202F is Word_Break=ExtendNumLet in the reference's Unicode 12.1 data, not a separator: it joins on both sides, and a leading one belongs to the token that follows, so FiQA's section sign + U+202F + 1950.6 is one term where this library had 1950.6 alone. The residual was invisible to the plan's diff, which splits on any Unicode whitespace and splits U+202F away. Comparing on ASCII spaces only — the dumps join terms with one — finds it, and the two files are now byte-identical. Only U+202F is handled: it is the only ExtendNumLet member the corpora exercise, and the rest are left until a document needs them. FiQA: 1 document differing -> 0, missing 1 -> 0, extra 1 -> 0; total terms unchanged at 5288635. --- src/LexiSharp/Linguistics/Tokenizer.cs | 36 ++++++++++++++++--- tests/LexiSharp.Tests/AnalysisScopeTests.cs | 7 ++++ .../LexiSharp.Tests/WordSegmentationTests.cs | 25 +++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/src/LexiSharp/Linguistics/Tokenizer.cs b/src/LexiSharp/Linguistics/Tokenizer.cs index c7ba7e36..610aaf91 100644 --- a/src/LexiSharp/Linguistics/Tokenizer.cs +++ b/src/LexiSharp/Linguistics/Tokenizer.cs @@ -390,6 +390,17 @@ private void ScanJoining(ReadOnlySpan text, List? terms, List text, List? terms, List text, int position) && WordBreakClasses.IsIdeographic(rune); } + /// + /// Whether the character at is U+202F, the narrow no-break space, + /// which the reference's word-break data classes as ExtendNumLet instead of a separator. + /// + /// + /// ExtendNumLet is a small class — the underscore, U+202F, four connector punctuation + /// characters and their presentation forms — and only U+202F is handled here, because it is the + /// only member a corpus in this repository has shown to matter: FiQA's section sign, U+202F and + /// 1950.6 form one term in the reference, the space and all. The rest can join, split or + /// be dropped without a measured consequence yet, and adding them speculatively would change + /// documents no corpus here exercises. Only the measured boundaries consult this; the flat and + /// identifier analyses still treat the character as a separator. + /// + private bool IsExtendNumLetAt(ReadOnlySpan text, int position) => + _conditionalJoins && text[position] == '\u202F'; + /// /// Whether is one of the three Unicode mark categories. /// diff --git a/tests/LexiSharp.Tests/AnalysisScopeTests.cs b/tests/LexiSharp.Tests/AnalysisScopeTests.cs index c925bc79..3436ebd4 100644 --- a/tests/LexiSharp.Tests/AnalysisScopeTests.cs +++ b/tests/LexiSharp.Tests/AnalysisScopeTests.cs @@ -72,6 +72,13 @@ public void AFormatCharacterStillSplitsOutsideTheMode(string analyzer) Assert.Equal(expected, Build(analyzer).Tokenize("199\uFEFF0")); } + [Theory] + [MemberData(nameof(Analyses))] + public void ANarrowNoBreakSpaceStillSplitsOutsideTheMode(string analyzer) + { + Assert.Equal(["ab", "cd"], Build(analyzer).Tokenize("ab\u202Fcd")); + } + [Theory] [MemberData(nameof(Analyses))] public void ALongTokenIsNotChoppedOutsideTheMode(string analyzer) diff --git a/tests/LexiSharp.Tests/WordSegmentationTests.cs b/tests/LexiSharp.Tests/WordSegmentationTests.cs index 179ccccc..a771486c 100644 --- a/tests/LexiSharp.Tests/WordSegmentationTests.cs +++ b/tests/LexiSharp.Tests/WordSegmentationTests.cs @@ -165,6 +165,31 @@ public void FlatIsTheDefault() Assert.Equal(WordSegmentation.Flat, TokenizerOptions.Default.WordSegmentation); Assert.Equal(["1", "2", "3"], tokenizer.Tokenize("1,2,3")); } + + /// + /// U+202F, the narrow no-break space, is Word_Break=ExtendNumLet in the reference's + /// Unicode data — a connector, not a separator. It joins the word on either side, and a leading + /// one belongs to the token: FiQA's section sign + U+202F + 1950.6 is one term in the + /// reference index, where this library had 1950.6 alone. + /// + [Fact] + public void TheNarrowNoBreakSpaceIsPartOfTheWord() + { + // Folding would decompose U+202F to a plain space before the term is emitted, which is not + // what the uax29 analysis does, so the configuration states that. + var tokenizer = new Tokenizer(new TokenizerOptions + { + KeepSingleCharTerms = true, + FoldDiacritics = false, + WordSegmentation = WordSegmentation.UnicodeWordBoundaries, + }); + + Assert.Equal(["\u202F1950.6"], tokenizer.Tokenize("\u202F1950.6")); + Assert.Equal(["a\u202Fb"], tokenizer.Tokenize("a\u202Fb")); + Assert.Equal(["12\u202Fab"], tokenizer.Tokenize("12\u202Fab")); + Assert.Equal(["a\u202F"], tokenizer.Tokenize("a\u202F")); + Assert.Empty(tokenizer.Tokenize("\u202F")); + } } /// /// The two rules that closed the last term gap against the reference index: the middle dot joins two From 083f56e8e23e7393184edbb118c55c9c1dae796b Mon Sep 17 00:00:00 2001 From: manuc66 Date: Tue, 17 Nov 2026 20:10:20 +0100 Subject: [PATCH 7/8] Document the uax29/Lucene index parity FiQA's index is now the reference's: 5,288,635 terms and a per-document stream byte-identical to Lucene 10.5.2's EnglishAnalyzer. The evaluation and backend pages say so, the changelog carries the rules and the one knowingly unimplemented case (multi-code-point emoji sequences, absent from FiQA). --- CHANGELOG.md | 11 +++++++++++ docs/backends.md | 3 ++- docs/evaluation.md | 7 ++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cca1c0cc..25b8459d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,17 @@ against 25.7 / 50.0 and 265.0 / 284.4 ms with the gate off. `--keep` leaves the table for `EXPLAIN`. +- **`uax29` now reproduces Lucene 10.5.2's `StandardTokenizer` term for term.** Under + `WordSegmentation.UnicodeWordBoundaries`, the tokenizer emits Extended_Pictographic symbols + (Unicode 12.1, the revision the reference grammar declares), one term per Han ideograph, keeps + combining marks and format characters inside the word, chops a token at the reference's 255-unit + buffer, and treats U+202F as a word extender. The other analyzers are unchanged, pinned by + `AnalysisScopeTests`. FiQA-2018's index now holds the reference's **5,288,635** terms and its + per-document term stream is **byte-identical** to Lucene 10.5.2's `EnglishAnalyzer`; the + NFCorpus/SciFact/ArguAna fingerprints and aligned scores are unchanged. Multi-code-point emoji + sequences (ZWJ, flags, keycaps, skin tones) are the one knowingly unimplemented rule — FiQA + contains none, and a test pins the divergence. + - **FiQA-2018 as a fourth selectable corpus** (`--dataset fiqa`; 57,638 documents, 648 test queries). It is not part of `--dataset all` or the pinned comparison: it is the candidate gate's evidence on a corpus of another nature (financial Q&A, short queries). At pool 1,000 the gate diff --git a/docs/backends.md b/docs/backends.md index 2669d85f..782b51bf 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -389,7 +389,8 @@ financial Q&A with short queries) is not part of the pinned three. Under this ta analysis at k1=0.9/b=0.4. Under the reference's own conventions — `--analyzer uax29`, linear gains, k1=0.9/b=0.4, query-term frequency, reference index statistics, score rounding, omitted saturation constant, single precision, depth 1,000 — the in-memory row reads **0.236** against 0.2361 and the -paper's 0.236: the figure is reproduced. This engine cannot use `uax29`; with its closest +paper's 0.236: the figure is reproduced, and the index is the reference's: 5,288,635 terms, with a +per-document term stream byte-identical to Lucene 10.5.2's `EnglishAnalyzer`. This engine cannot use `uax29`; with its closest configuration (`english`, k1=0.9/b=0.4) it reads **0.2291**. The gap was decomposed: a PostgreSQL configuration with the reference's 33-word stop list and Snowball reads 0.2196, so the stop list is not the cause (the longer stock list helps) and the remainder is the stemmer and the parser — not diff --git a/docs/evaluation.md b/docs/evaluation.md index 8e5add05..c3bc10ad 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -529,7 +529,12 @@ same qrels (harness defaults, exponential gains, 10 retrieved at 10): gains, k1=0.9/b=0.4, query-term frequency, reference index statistics, score rounding, omitted saturation constant, single precision, depth 1,000 — the in-memory BM25 reads **0.236** against the published **0.2361** (the paper's 0.236): the figure is reproduced at the precision it was - published at. The PostgreSQL row cannot use `uax29`; with its closest configuration (`english`, + published at. The index is the reference's, not merely a close one: `--fingerprint` reports + 5,288,635 terms, and the per-document term stream is byte-identical to Lucene 10.5.2's + `EnglishAnalyzer` over the corpus (Extended_Pictographic symbols, one term per Han ideograph, + combining marks, format characters, the 255-unit token chop, U+202F). Multi-code-point emoji + sequences are the one knowingly unimplemented rule — FiQA contains none, and a test pins the + divergence. The PostgreSQL row cannot use `uax29`; with its closest configuration (`english`, k1=0.9/b=0.4) it reads **0.2291**, the same ~0.007 analysis gap the other corpora show, not a gate cost. Where the gap sits was measured rather than assumed: a PostgreSQL configuration carrying the reference's 33-word stop list and its Snowball stemmer reads **0.2196**, so the stop list is not From 10777a8265b4e774035580f80c6185392c29155f Mon Sep 17 00:00:00 2001 From: manuc66 Date: Tue, 17 Nov 2026 20:16:27 +0100 Subject: [PATCH 8/8] Split ScanJoining along the rules it now holds The emoji, ideograph, format-character and ExtendNumLet rules had grown the scan to 129 lines and a complexity CodeFactor flags. Each step is now a named method -- standalone tokens, whether a word begins, leading joiners, the word run -- with the comments moved to the rule they explain. No behaviour change: the FiQA term dump is still byte-identical to Lucene 10.5.2's. --- src/LexiSharp/Linguistics/Tokenizer.cs | 227 ++++++++++++++----------- 1 file changed, 126 insertions(+), 101 deletions(-) diff --git a/src/LexiSharp/Linguistics/Tokenizer.cs b/src/LexiSharp/Linguistics/Tokenizer.cs index 610aaf91..ea0edd15 100644 --- a/src/LexiSharp/Linguistics/Tokenizer.cs +++ b/src/LexiSharp/Linguistics/Tokenizer.cs @@ -343,128 +343,153 @@ private void ScanJoining(ReadOnlySpan text, List? terms, List= 128 && _symbolTerms.Contains(text[position])) - { - EmitStandalone(text, position, 1, terms, spans); - position++; + if (EmitStandaloneAt(text, ref position, terms, spans)) continue; - } - // Under the measured boundaries an emoji is a term of its own, listed as a symbol or - // not: the reference's grammar starts an emoji token on every `Extended_Pictographic` - // code point and on nothing else. That distinction is why this asks the emoji property - // rather than the character category — `✔`, `♦` and `😂` are terms in FiQA's index, - // while `°`, `■` and `№` are dropped by both implementations. - // - // An ideograph is the same kind of term: `理财` is `理` and `财`, and `6尺` is `6` and - // `尺`. Both are checked before the word logic because neither may open a run. - if (_conditionalJoins - && text[position] >= 128 - && Rune.DecodeFromUtf16(text[position..], out Rune standalone, out int standaloneLength) == OperationStatus.Done - && (WordBreakClasses.IsEmoji(standalone) || WordBreakClasses.IsIdeographic(standalone))) + if (!BeginsWordAt(text, position)) { - EmitStandalone(text, position, standaloneLength, terms, spans); - position += standaloneLength; + position++; continue; } - bool beginsWord = StartsWordAt(text, position); - - // An unconditional joiner can open a token when a word character follows it: measured - // `_630888` indexes whole and `_alpha` as `_alpha`, so the underscore is part of the word - // here — while `-alpha` is just `alpha`, so this is the underscore's rule and not a rule - // about any separator. Tested here rather than inside the run scan, because the run scan - // only asks whether a word continues past a character it is already inside. - // - // A run of joiners on its own is not a token: `_`, `__` and `___` alone all produce nothing. - // So the joiners have to lead to a word character, not merely to another joiner, and - // `ContinuesWord` is what finds the word character at the far end of the run. - if (!beginsWord) - { - beginsWord = text[position] < 128 - && RuleFor(text[position]) == JoinRule.Always - && position + 1 < text.Length - && LeadsWordAt(text, position + 1); - } + int start = position; + position = ConsumeLeadingJoiners(text, position + 1); + position = ScanWordRun(text, position); - // U+202F joins the same way from the other side of the ASCII boundary: the reference's - // word-break data calls it ExtendNumLet, so a leading one belongs to the token that - // follows it. FiQA's section sign + U+202F + `1950.6` is one term, the space included. - if (!beginsWord - && IsExtendNumLetAt(text, position) - && position + 1 < text.Length - && LeadsWordAt(text, position + 1)) - { - beginsWord = true; - } + EmitWordRun(text, start, position, terms, spans); + } + } - if (!beginsWord) - { - position++; - continue; - } + /// + /// Emits the character at as a term of its own when it is one, and + /// advances past it. + /// + /// + /// A configured symbol is a term of its own and does not extend the word beside it: + /// Aspirin® is aspirin and ®, not aspirin®. Under the measured + /// boundaries an emoji or an ideograph is the same kind of term: the reference's grammar starts + /// an emoji token on every Extended_Pictographic code point and on nothing else, and + /// emits each Han ideograph alone (理财 is 理 and 财). Both are checked + /// before the word logic because neither may open a run. + /// + private bool EmitStandaloneAt( + ReadOnlySpan text, ref int position, List? terms, List? spans) + { + if (_symbolTerms is not null && text[position] >= 128 && _symbolTerms.Contains(text[position])) + { + EmitStandalone(text, position, 1, terms, spans); + position++; + return true; + } - int start = position; + if (_conditionalJoins + && text[position] >= 128 + && Rune.DecodeFromUtf16(text[position..], out Rune standalone, out int standaloneLength) == OperationStatus.Done + && (WordBreakClasses.IsEmoji(standalone) || WordBreakClasses.IsIdeographic(standalone))) + { + EmitStandalone(text, position, standaloneLength, terms, spans); + position += standaloneLength; + return true; + } + + return false; + } + + /// Whether a token starts at . + /// + /// An unconditional joiner can open a token when a word character follows it: measured + /// _630888 indexes whole and _alpha as _alpha, so the underscore is part of + /// the word — while -alpha is just alpha. A run of joiners on its own is not a + /// token, so the joiners have to lead to a word character, not merely to another joiner. + /// U+202F joins the same way from the other side of the ASCII boundary: the reference's + /// word-break data calls it ExtendNumLet, so a leading one belongs to the token that + /// follows it (FiQA's section sign + U+202F + 1950.6 is one term). + /// + private bool BeginsWordAt(ReadOnlySpan text, int position) + { + if (StartsWordAt(text, position)) + return true; + + if (text[position] < 128 + && RuleFor(text[position]) == JoinRule.Always + && position + 1 < text.Length + && LeadsWordAt(text, position + 1)) + { + return true; + } + + return IsExtendNumLetAt(text, position) + && position + 1 < text.Length + && LeadsWordAt(text, position + 1); + } + + /// + /// Consumes the unconditional joiners that open a token and returns the position after them. + /// + /// + /// A leading joiner belongs to the token even though nothing precedes it for the "word + /// character on both sides" test to succeed on: _630888 indexes whole, _alpha as + /// _alpha, __alpha as __alpha — while -alpha is just alpha. + /// The whole run is consumed here because only asks whether a word + /// continues past a character it is already inside. + /// + private int ConsumeLeadingJoiners(ReadOnlySpan text, int position) + { + while (position < text.Length + && RuleFor(text[position]) == JoinRule.Always + && position + 1 < text.Length + && ContinuesWord(text, position + 1)) + { position++; + } - // An unconditional joiner that opens the token belongs to it, even though nothing precedes - // it for the "word character on both sides" test to succeed on. Measured: `_630888` indexes - // whole, `_alpha` as `_alpha` and `__alpha` as `__alpha` — while `-alpha` is just `alpha`, - // so this is the underscore's rule and not a rule about any separator. Every leading joiner - // is consumed here, including a run of them, because the run scan below only asks whether a - // word continues past a character it is already inside. - while (position < text.Length && RuleFor(text[position]) == JoinRule.Always - && position + 1 < text.Length && ContinuesWord(text, position + 1)) - { - position++; - } + return position; + } - while (position < text.Length) - { - // An ideograph ends the run: the reference emits it as a token of its own, so - // `abc尺def` is three terms and not one. - if (_conditionalJoins && IsIdeographicAt(text, position)) - break; + /// + /// Walks the word that starts at and returns where it ends. + /// + /// + /// ASCII runs are skipped in bulk: everything up to the first non-letter or non-digit is inside + /// the word. A non-ASCII word character, a combining mark or format character that continues + /// one, or U+202F advances by its rune length, since a surrogate pair is two chars and the + /// boundary has to land between runes. An ideograph always ends the run: the reference emits it + /// alone, so abc尺def is three terms and not one. + /// + private int ScanWordRun(ReadOnlySpan text, int position) + { + while (position < text.Length) + { + if (_conditionalJoins && IsIdeographicAt(text, position)) + break; - if (text[position] < 128) - { - // Bulk-skip the rest of an ASCII word run: everything up to the first character - // that is not an ASCII letter or digit is inside the word. A negative result means - // every remaining character matches, so the run reaches the end of the text — the - // case a token at the very end of the input would otherwise fall into. - int relativeEnd = text[position..].IndexOfAnyExcept(AsciiWordChars); - - if (relativeEnd != 0) - { - position += relativeEnd < 0 ? text.Length - position : relativeEnd; - continue; - } - } - else if (StartsWordAt(text, position) || ContinuesWordAt(text, position) || IsExtendNumLetAt(text, position)) - { - // A non-ASCII word character, a combining mark or format character that - // continues one, or U+202F. Decode to learn how many UTF-16 units it spans, - // since a surrogate pair is two chars and the boundary has to land between - // runes. - Rune.DecodeFromUtf16(text[position..], out _, out int runeLength); - position += runeLength; - continue; - } + if (text[position] < 128) + { + int relativeEnd = text[position..].IndexOfAnyExcept(AsciiWordChars); - if (JoinsToNext(text, position)) + if (relativeEnd != 0) { - position++; + position += relativeEnd < 0 ? text.Length - position : relativeEnd; continue; } + } + else if (StartsWordAt(text, position) || ContinuesWordAt(text, position) || IsExtendNumLetAt(text, position)) + { + Rune.DecodeFromUtf16(text[position..], out _, out int runeLength); + position += runeLength; + continue; + } - break; + if (JoinsToNext(text, position)) + { + position++; + continue; } - EmitWordRun(text, start, position, terms, spans); + break; } + + return position; } ///