diff --git a/.github/workflows/references.yml b/.github/workflows/references.yml index c368862b87..5e3e85104d 100644 --- a/.github/workflows/references.yml +++ b/.github/workflows/references.yml @@ -296,3 +296,64 @@ jobs: python-version: ${{ matrix.python }} - name: Benchmark latency run: python references/table/latency.py tablecenternet --it 5 --size 512 + + train-formula-recognition: + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python: ["3.11"] + steps: + - uses: actions/checkout@v7 + - name: Setup environment + uses: ./.github/actions/setup-env + with: + python-version: ${{ matrix.python }} + extra-requirements: references/requirements.txt + - name: Download and extract toy set + run: | + wget https://github.com/mindee/doctr/releases/download/v1.1.0/toy_formula_set-8f54029b.zip + unzip toy_formula_set-8f54029b.zip -d formula_set + - name: Train for a short epoch + run: python references/formula/train.py formulanet --train_path ./formula_set --val_path ./formula_set -b 2 --epochs 1 --max_decode_length 64 + + evaluate-formula-recognition: + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python: ["3.11"] + steps: + - uses: actions/checkout@v7 + - name: Setup environment + uses: ./.github/actions/setup-env + with: + python-version: ${{ matrix.python }} + extra-requirements: references/requirements.txt + - name: Download and extract toy set + run: | + wget https://github.com/mindee/doctr/releases/download/v1.1.0/toy_formula_set-8f54029b.zip + unzip toy_formula_set-8f54029b.zip -d formula_set + - name: Evaluate formula recognition + run: python references/formula/evaluate.py formulanet ./formula_set -b 2 --max_decode_length 64 + + latency-formula-recognition: + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python: ["3.11"] + steps: + - uses: actions/checkout@v7 + - name: Setup environment + uses: ./.github/actions/setup-env + with: + python-version: ${{ matrix.python }} + - name: Benchmark latency + run: python references/formula/latency.py formulanet --it 5 --tokens 32 diff --git a/README.md b/README.md index ee52fdb638..a13131a1e3 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,10 @@ Credits where it's due: this repository is implementing, among others, architect - TableCenterNet: [A one-stage network for table structure recognition](https://arxiv.org/abs/2504.17522). +### Formula Recognition + +- FormulaNet (HGNetV2 encoder + transformer decoder): [PP-FormulaNet: Bridging Accuracy and Efficiency in Advanced Formula Recognition](https://arxiv.org/abs/2503.18382). + ## More goodies ### Documentation diff --git a/api/app/routes/ocr.py b/api/app/routes/ocr.py index 9a053c3ffc..c6cbf360d8 100644 --- a/api/app/routes/ocr.py +++ b/api/app/routes/ocr.py @@ -6,7 +6,18 @@ from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status -from app.schemas import LayoutElementOut, OCRBlock, OCRIn, OCRLine, OCROut, OCRPage, OCRWord, TableCellOut, TableOut +from app.schemas import ( + FormulaOut, + LayoutElementOut, + OCRBlock, + OCRIn, + OCRLine, + OCROut, + OCRPage, + OCRWord, + TableCellOut, + TableOut, +) from app.utils import get_documents, resolve_geometry from app.vision import init_predictor @@ -63,6 +74,15 @@ async def perform_ocr(request: OCRIn = Depends(), files: list[UploadFile] = [Fil ) for table in page.tables ], + formulas=[ + FormulaOut( + value=formula.value, + geometry=resolve_geometry(formula.geometry), + confidence=round(formula.confidence, 2), + objectness_score=(None if formula.objectness_score is None else round(formula.objectness_score, 2)), + ) + for formula in page.formulas + ], items=[ OCRPage( blocks=[ diff --git a/api/app/schemas.py b/api/app/schemas.py index c269b51ef9..7c46447a7a 100644 --- a/api/app/schemas.py +++ b/api/app/schemas.py @@ -32,6 +32,7 @@ class OCRIn(KIEIn, BaseModel): resolve_blocks: bool = Field(default=False, examples=[False]) paragraph_break: float = Field(default=0.0035, examples=[0.0035]) detect_tables: bool = Field(default=False, examples=[False]) + detect_formulas: bool = Field(default=False, examples=[False]) class RecognitionIn(BaseModel): @@ -173,6 +174,13 @@ class TableOut(BaseModel): ) +class FormulaOut(BaseModel): + value: str = Field(..., examples=["\\frac { a } { b }"]) + geometry: list[float] = Field(..., examples=[[0.0, 0.0, 0.0, 0.0]]) + confidence: float = Field(..., examples=[0.99]) + objectness_score: float | None = Field(..., examples=[0.99]) + + class OCROut(BaseModel): name: str = Field(..., examples=["example.jpg"]) orientation: dict[str, float | None] = Field(..., examples=[{"value": 0.0, "confidence": 0.99}]) @@ -207,6 +215,19 @@ class OCROut(BaseModel): ] ], ) + formulas: list[FormulaOut] = Field( + default=[], + examples=[ + [ + { + "value": "\\frac { a } { b }", + "geometry": [0.0, 0.0, 0.0, 0.0], + "confidence": 0.99, + "objectness_score": 0.99, + } + ] + ], + ) items: list[OCRPage] = Field( ..., examples=[ diff --git a/api/tests/routes/test_ocr.py b/api/tests/routes/test_ocr.py index df34d831b3..e7c509b251 100644 --- a/api/tests/routes/test_ocr.py +++ b/api/tests/routes/test_ocr.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from doctr.io.elements import Document, Page, Table, TableCell +from doctr.io.elements import Document, Formula, Page, Table, TableCell def common_test(json_response, expected_response): @@ -144,6 +144,39 @@ async def test_ocr_table_cells(test_app_asyncio, mock_detection_image, monkeypat assert [cell["objectness_score"] for cell in out_cells] == [0.81, None] +@pytest.mark.asyncio +async def test_ocr_formulas(test_app_asyncio, mock_detection_image, monkeypatch): + # The recognized formulas are returned with their LaTeX source, and `detect_formulas` reaches the predictor + formulas = [ + Formula("\\frac { a } { b }", 0.9876, ((0.1, 0.1), (0.5, 0.2)), objectness_score=0.8123), + Formula("x ^ { 2 }", 0.5, ((0.1, 0.3), (0.5, 0.4))), + ] + doc = Document([Page(np.zeros((100, 100, 3), dtype=np.uint8), [], 0, (100, 100), formulas=formulas)]) + requests = [] + + def fake_init_predictor(request): + requests.append(request) + return lambda pages: doc + + monkeypatch.setattr("app.routes.ocr.init_predictor", fake_init_predictor) + files = [("files", ("test.jpg", mock_detection_image, "image/jpeg"))] + response = await test_app_asyncio.post( + "/ocr", params={"detect_formulas": True}, files=files, headers={"accept": "application/json"} + ) + assert response.status_code == 200 + assert requests[-1].detect_formulas is True + assert response.json()[0]["formulas"] == [ + {"value": "\\frac { a } { b }", "geometry": [0.1, 0.1, 0.5, 0.2], "confidence": 0.99, "objectness_score": 0.81}, + {"value": "x ^ { 2 }", "geometry": [0.1, 0.3, 0.5, 0.4], "confidence": 0.5, "objectness_score": None}, + ] + + # No formulas by default + doc = Document([Page(np.zeros((100, 100, 3), dtype=np.uint8), [], 0, (100, 100))]) + response = await test_app_asyncio.post("/ocr", files=files, headers={"accept": "application/json"}) + assert requests[-1].detect_formulas is False + assert response.json()[0]["formulas"] == [] + + @pytest.mark.asyncio async def test_ocr_invalid_file(test_app_asyncio, mock_txt_file): headers = { diff --git a/demo/app.py b/demo/app.py index aa502ccd9b..c31e5255f2 100644 --- a/demo/app.py +++ b/demo/app.py @@ -73,6 +73,8 @@ def main(det_archs, reco_archs, layout_archs): layout_arch = st.sidebar.selectbox("Layout detection model", layout_archs, disabled=not detect_layout) # Table detection (relies on the layout model to locate tables) detect_tables = st.sidebar.checkbox("Detect tables", value=False) + # Formula recognition (relies on the layout model to locate formulas) + detect_formulas = st.sidebar.checkbox("Detect formulas", value=False) st.sidebar.write("\n") # Binarization threshold bin_thresh = st.sidebar.slider("Binarization threshold", min_value=0.1, max_value=0.9, value=0.3, step=0.1) @@ -101,6 +103,7 @@ def main(det_archs, reco_archs, layout_archs): detect_layout=detect_layout, layout_arch=layout_arch, detect_tables=detect_tables, + detect_formulas=detect_formulas, ) with st.spinner("Analyzing..."): @@ -133,6 +136,14 @@ def main(det_archs, reco_archs, layout_archs): st.markdown(f"**Table {idx + 1}** ({table.num_rows} x {table.num_cols})") st.dataframe(pd.DataFrame(table.to_grid())) + # Display recognized formulas (if any): rendered, and their LaTeX source + if out.pages[0].formulas: + st.markdown("\nRecognized formulas:") + for idx, formula in enumerate(out.pages[0].formulas): + st.markdown(f"**Formula {idx + 1}** (confidence {formula.confidence:.2f})") + st.latex(formula.value) + st.code(formula.value, language="latex") + # Display JSON st.markdown("\nHere are your analysis results in JSON format:") st.json(page_export, expanded=False) diff --git a/demo/backend/pytorch.py b/demo/backend/pytorch.py index 980b1137a0..8a4ebad316 100644 --- a/demo/backend/pytorch.py +++ b/demo/backend/pytorch.py @@ -51,6 +51,7 @@ def load_predictor( detect_layout: bool, layout_arch: str, detect_tables: bool, + detect_formulas: bool = False, ) -> OCRPredictor: """Load a predictor from doctr.models @@ -68,6 +69,8 @@ def load_predictor( detect_layout: whether to run a layout detection model and attach the regions to each page layout_arch: layout architecture to use when detect_layout is True detect_tables: whether to detect tables (via the layout model), structure them and attach them to each page + detect_formulas: whether to read the formulas (located via the layout model) as LaTeX and attach them to each + page Returns: instance of OCRPredictor @@ -85,6 +88,7 @@ def load_predictor( detect_layout=detect_layout, layout_arch=layout_arch, detect_tables=detect_tables, + detect_formulas=detect_formulas, ).to(device) predictor.det_predictor.model.postprocessor.bin_thresh = bin_thresh predictor.det_predictor.model.postprocessor.box_thresh = box_thresh diff --git a/docs/source/api/datasets.md b/docs/source/api/datasets.md index f7b0c9e827..962143377b 100644 --- a/docs/source/api/datasets.md +++ b/docs/source/api/datasets.md @@ -72,6 +72,13 @@ for a guide. :type: dict[str, str] The supported vocabularies, by name. See :doc:`../models/vocabs` for the complete list. + +.. data:: LATEX_TOKENS + :type: list[str] + + The 1,324 LaTeX tokens of formula recognition: the printable ASCII characters, the control symbols, the commands and + environments supported by KaTeX, and the other commands and characters found in public formula datasets (UniMER-1M, + HME100K, UniMER-Test). See :doc:`../models/formula`. ``` ## Utilities @@ -79,6 +86,10 @@ for a guide. ```{eval-rst} .. autofunction:: translate +.. autofunction:: tokenize_latex + +.. autofunction:: normalize_latex + .. autofunction:: encode_string .. autofunction:: decode_sequence diff --git a/docs/source/api/io.md b/docs/source/api/io.md index 3b17f59507..b77fce0591 100644 --- a/docs/source/api/io.md +++ b/docs/source/api/io.md @@ -93,7 +93,7 @@ The elements returned by the predictors, from the largest to the smallest. See .. autoclass:: Artefact ``` -### Layout and tables +### Layout, tables and formulas ```{eval-rst} .. autoclass:: LayoutElement @@ -105,6 +105,8 @@ The elements returned by the predictors, from the largest to the smallest. See .. autoclass:: TableCell :members: row_span, col_span + +.. autoclass:: Formula ``` ### Key information extraction diff --git a/docs/source/api/models.md b/docs/source/api/models.md index ef43708046..915035ec5c 100644 --- a/docs/source/api/models.md +++ b/docs/source/api/models.md @@ -94,6 +94,17 @@ directly. See the [model zoo](../models/index.md) for benchmarks. .. autofunction:: doctr.models.table_structure.tablecenternet ``` +## Formula recognition + +```{eval-rst} +.. autofunction:: doctr.models.formula.formula_predictor + +.. autoclass:: doctr.models.formula.predictor.FormulaPredictor + :members: predict_regions + +.. autofunction:: doctr.models.formula.formulanet +``` + ## Classification Orientation classifiers, and the backbones used by the detection and recognition models. @@ -146,6 +157,8 @@ Orientation classifiers, and the backbones used by the detection and recognition .. autofunction:: doctr.models.classification.vit_det_m .. autofunction:: doctr.models.classification.starnet_s3 + +.. autofunction:: doctr.models.classification.hgnetv2_b4 ``` ## Hugging Face Hub diff --git a/docs/source/api/transforms.md b/docs/source/api/transforms.md index 67d5aa1db8..b22fab06d6 100644 --- a/docs/source/api/transforms.md +++ b/docs/source/api/transforms.md @@ -28,6 +28,8 @@ img_transforms = Compose([ .. autoclass:: RandomResize +.. autoclass:: RandomPad + .. autoclass:: GaussianNoise .. autoclass:: GaussianBlur diff --git a/docs/source/api/utils.md b/docs/source/api/utils.md index 6f43421576..7e537e0a7b 100644 --- a/docs/source/api/utils.md +++ b/docs/source/api/utils.md @@ -50,4 +50,7 @@ Metrics to evaluate models on each task, used by the evaluation scripts of the r .. autoclass:: TableCellMetric :members: update, summary + +.. autoclass:: FormulaMetric + :members: update, summary ``` diff --git a/docs/source/conf.py b/docs/source/conf.py index 3c3064cbbb..f20a96b8d8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -206,7 +206,7 @@ # Vocabularies listed on `models/vocabs.md`, grouped by script. Any vocabulary missing from these groups is listed # under "Other scripts", so that the page always reflects `doctr.datasets.VOCABS`. VOCAB_GROUPS = { - "General purpose": ["latin", "english", "multilingual", "latex"], + "General purpose": ["latin", "english", "multilingual"], "Latin script": [ "albanian", "afrikaans", diff --git a/docs/source/deployment/configuration.md b/docs/source/deployment/configuration.md index 7e93aa2a6c..bbf6226eee 100644 --- a/docs/source/deployment/configuration.md +++ b/docs/source/deployment/configuration.md @@ -30,6 +30,7 @@ from doctr.models import ocr_predictor ocr_predictor(pretrained=True) # detection + recognition ocr_predictor(pretrained=True, assume_straight_pages=False) # + orientation classifiers ocr_predictor(pretrained=True, detect_tables=True) # + layout and table models +ocr_predictor(pretrained=True, detect_formulas=True) # + layout and formula models ``` Copy the cache directory (`~/.cache/doctr` or `$DOCTR_CACHE_DIR`) to the same location on the offline machine. diff --git a/docs/source/deployment/performance.md b/docs/source/deployment/performance.md index 826a56cac8..d2662776a4 100644 --- a/docs/source/deployment/performance.md +++ b/docs/source/deployment/performance.md @@ -76,7 +76,8 @@ Pass all the pages of a document (or several documents) in **one call** so that the more expensive rotated-box post-processing. - With rotated documents, `disable_page_orientation=True` and `disable_crop_orientation=True` skip the orientation classifiers when you know the text is horizontal (see [](../user_guide/rotated_documents.md)). -- Only enable `detect_layout`, `detect_tables`, `detect_orientation` and `detect_language` when you use them. +- Only enable `detect_layout`, `detect_tables`, `detect_formulas`, `detect_orientation` and `detect_language` when you + use them. The formula model decodes its LaTeX token by token: its latency grows with the length of the formulas. - Use lighter architectures, see the [model zoo](../models/index.md). ## Compile the models diff --git a/docs/source/models/formula.md b/docs/source/models/formula.md new file mode 100644 index 0000000000..d5cbf185aa --- /dev/null +++ b/docs/source/models/formula.md @@ -0,0 +1,46 @@ +# Formula recognition + +Formula recognition models read the image of a mathematical formula and predict its LaTeX source. In the OCR +pipeline, they read the `Formula` regions found by the [layout model](layout.md), which are exported as display math +(e.g. `$$...$$` in Markdown). + +## Architectures + +| Architecture | Paper | Pretrained weights | +| --- | --- | --- | +| {func}`~doctr.models.formula.formulanet` {bdg-secondary}`default` | [PP-FormulaNet: Bridging Accuracy and Efficiency in Advanced Formula Recognition](https://arxiv.org/abs/2503.18382) | {bdg-success}`available` | + +`formulanet` combines a {func}`~doctr.models.classification.hgnetv2_b4` encoder with a small transformer decoder, +which predicts LaTeX tokens from {data}`~doctr.datasets.LATEX_TOKENS`, e.g. `\frac { a } { b ^ { 2 } }`. The +pretrained model is trained on the printed and handwritten formulas of +[UniMER-1M](https://huggingface.co/datasets/wanderkid/UniMER_Dataset). + +## Benchmark + +| Architecture | Input shape | # params | Exact match | Edit similarity | BLEU | +| --- | --- | ---: | ---: | ---: | ---: | +| `formulanet` | (256, 768, 3) | 26.4 M | 44.40 | 91.30 | 0.840 | + +The metrics are produced by `references/formula/evaluate.py` with {class}`~doctr.utils.metrics.FormulaMetric` on +[UniMER-Test](https://huggingface.co/datasets/wanderkid/UniMER_Dataset) (23,757 formulas), on the LaTeX tokens: +**exact match** rate, **edit similarity** (one minus the relative token edit distance) and corpus-level **BLEU-4**. + +:::{note} +Equation numbers are predicted with `\tag{...}` or as text after the formula. A few formulas may still use TeX commands that KaTeX does not render +(e.g. `\boldmath`): they render in LaTeX, but not in a KaTeX-based Markdown or HTML viewer. +::: + +## Usage + +```python +from doctr.models import formula_predictor, ocr_predictor + +# In the OCR pipeline +predictor = ocr_predictor(pretrained=True, detect_formulas=True) + +# On its own +formulas = formula_predictor("formulanet", pretrained=True) +``` + +See [](../user_guide/layout_and_tables.md#formula-recognition) to work with the recognized formulas, and the +[reference scripts](https://github.com/mindee/doctr/tree/main/references/formula#readme) to train on your own data. diff --git a/docs/source/models/index.md b/docs/source/models/index.md index 8599be215e..b5dac8800d 100644 --- a/docs/source/models/index.md +++ b/docs/source/models/index.md @@ -11,6 +11,7 @@ This section lists them with their benchmarks, so you can pick the right trade-o | [Text recognition](recognition.md) | CRNN, SAR, MASTER, ViTSTR, PARSeq, VIPTR | `crnn_vgg16_bn` | `ocr_predictor(reco_arch=...)` | | [Layout analysis](layout.md) | LW-DETR | `lw_detr_s` | `ocr_predictor(detect_layout=True, layout_arch=...)` | | [Table structure](table_structure.md) | TableCenterNet | `tablecenternet` | `ocr_predictor(detect_tables=True)` | +| [Formula recognition](formula.md) | FormulaNet | `formulanet` | `ocr_predictor(detect_formulas=True)` | | [Orientation](#orientation-classifiers) | MobileNetV3 | `mobilenet_v3_small_*_orientation` | `ocr_predictor(assume_straight_pages=False)` | [End-to-end benchmarks](end_to_end.md) compare detection + recognition pairs, including commercial OCR services. @@ -66,6 +67,7 @@ detection recognition layout table_structure +formula end_to_end vocabs ``` diff --git a/docs/source/training/custom_models.md b/docs/source/training/custom_models.md index 392b34546f..caa96d37ba 100644 --- a/docs/source/training/custom_models.md +++ b/docs/source/training/custom_models.md @@ -113,6 +113,28 @@ predictor = ocr_predictor(pretrained=True, detect_tables=True) predictor.table_predictor = table_predictor(table_model) # replace the default table model ``` +## Formula recognition models + +A model trained with your own vocabulary must be built with the same one: + +```python +import json + +from doctr.models import formula_predictor, formulanet, ocr_predictor + +vocab = json.load(open("path/to/vocab.json")) # cf. references/formula/build_vocab.py +formula_model = formulanet(pretrained=False, vocab=vocab) +formula_model.from_pretrained("path/to/formula.pt") + +predictor = ocr_predictor(pretrained=True, detect_formulas=True) +predictor.formula_predictor = formula_predictor(formula_model) # replace the default formula model +``` + +:::{note} +Formula recognition (`detect_formulas=True`) reads the regions labelled `Formula`: keep this class name if your layout +model should feed the formula model. +::: + ## Orientation classifiers The orientation classifiers are used with `assume_straight_pages=False`, `straighten_pages=True` or diff --git a/docs/source/training/datasets.md b/docs/source/training/datasets.md index 46e58130bc..765d5497e9 100644 --- a/docs/source/training/datasets.md +++ b/docs/source/training/datasets.md @@ -98,6 +98,7 @@ Load annotated data stored in the [training format](train.md) of each task: | --- | --- | --- | | {class}`~doctr.datasets.DetectionDataset` | Text detection | `images/` + `labels.json` with polygons (optionally per class) | | {class}`~doctr.datasets.RecognitionDataset` | Text recognition | word images + `labels.json` mapping each image to its text | +| {class}`~doctr.datasets.RecognitionDataset` | Formula recognition | formula images + `labels.json` mapping each image to its LaTeX source | | {class}`~doctr.datasets.OCRDataset` | End-to-end OCR | `images/` + `labels.json` with boxes and transcriptions | | {class}`~doctr.datasets.LayoutDataset` | Layout analysis | `images/` + `labels.json` with polygons and classes | | {class}`~doctr.datasets.TableStructureDataset` | Table structure | `images/` + `labels.json` with cells and their logical coordinates | diff --git a/docs/source/training/sharing_models.md b/docs/source/training/sharing_models.md index 62450d06fe..85b17491f9 100644 --- a/docs/source/training/sharing_models.md +++ b/docs/source/training/sharing_models.md @@ -32,7 +32,7 @@ model = recognition.crnn_mobilenet_v3_large(pretrained=True) push_to_hf_hub( model, model_name="doctr-crnn-mobilenet-v3-large-french-v1", - task="recognition", # classification, detection, recognition, layout or table_structure + task="recognition", # classification, detection, recognition, layout, table_structure or formula arch="crnn_mobilenet_v3_large", ) ``` @@ -53,6 +53,7 @@ python references/recognition/train.py crnn_mobilenet_v3_large --name doctr-crnn | Recognition | `doctr--` | | Layout | `doctr-` | | Table structure | `doctr-` | +| Formula recognition | `doctr-` | ## Community models diff --git a/docs/source/training/train.md b/docs/source/training/train.md index fa0b22657e..1650be0af5 100644 --- a/docs/source/training/train.md +++ b/docs/source/training/train.md @@ -118,6 +118,22 @@ Cell polygons are ordered top-left, top-right, bottom-right, bottom-left. Logica inclusive: a cell spanning one row and one column has equal start and end indices. ::: +:::{tab-item} Formula +`images/` contains images of **formulas** and `labels.json` their LaTeX source, without math delimiters: + +```{code-block} javascript +:caption: labels.json + +{ + "img_1.png": "\\frac{a}{b^{2}}", + "img_2.png": "\\sum_{i=1}^{n} x_{i}" +} +``` + +The labels are split into LaTeX tokens with {func}`~doctr.datasets.tokenize_latex`, raw or already separated by +spaces. +::: + :::{tab-item} Orientation No labels: a folder of **upright** images. The script rotates them to generate the training targets. @@ -175,6 +191,17 @@ python references/table/train.py tablecenternet \ [Full guide](https://github.com/mindee/doctr/tree/main/references/table#readme), including a toy dataset to try the pipeline end to end. +### Formula recognition + +```bash +python references/formula/train.py formulanet \ + --train_path path/to/train_set --val_path path/to/val_set --epochs 5 +``` + +The samples with a token out of the vocabulary are skipped: `build_vocab.py` extends the vocabulary with the tokens of +your labels (`--vocab vocab.json`). [Full guide](https://github.com/mindee/doctr/tree/main/references/formula#readme), +including a toy dataset to try the pipeline end to end. + ### Orientation classifiers ```bash diff --git a/docs/source/user_guide/cli.md b/docs/source/user_guide/cli.md index 24fdd434a9..9ecb7a466c 100644 --- a/docs/source/user_guide/cli.md +++ b/docs/source/user_guide/cli.md @@ -95,6 +95,7 @@ Run `doctr-cli --help` for the full list with default values. | `--detect_language` | off | Estimate the language of each page | | `--detect_layout` | off | Attach the layout regions to each page | | `--detect_tables` | off | Recognize tables (enables the layout model) | +| `--detect_formulas` | off | Read formulas as LaTeX (enables the layout model) | | `--ignore_regions CLASS [CLASS ...]` | none | Layout classes to mask before OCR, e.g. `Picture Table` | | `--export_as_straight_boxes` | off | Export straight boxes with `--no-assume_straight_pages` | | `--preserve_original_coords` | off | With `--straighten_pages`, map the boxes back to the original pages | @@ -131,7 +132,7 @@ Run `doctr-cli --help` for the full list with default values. ## Output files - **JSON** contains the full structure: pages (dimensions, orientation, language, layout), blocks, lines and words - with their confidence and geometry, and the tables with `--detect_tables`. + with their confidence and geometry, the tables with `--detect_tables` and the formulas with `--detect_formulas`. - **Text-like formats** (`txt`, `md`, `adoc`, `html`) are written to a single file. - **hOCR** describes one page per file: `--output doc.xml` on a three-page document writes `doc_page_1.xml`, `doc_page_2.xml` and `doc_page_3.xml`. diff --git a/docs/source/user_guide/index.md b/docs/source/user_guide/index.md index 8c09b7a4b7..83fc40dcfd 100644 --- a/docs/source/user_guide/index.md +++ b/docs/source/user_guide/index.md @@ -46,7 +46,7 @@ Text, JSON, Markdown, AsciiDoc, HTML and hOCR, in reading order. :link: layout_and_tables :link-type: doc -Detect titles, figures and tables, and extract tables as row/column grids. +Detect titles, figures and tables, extract tables as row/column grids and read formulas as LaTeX. ::: :::{grid-item-card} {octicon}`sync;1.1em` Rotated documents diff --git a/docs/source/user_guide/layout_and_tables.md b/docs/source/user_guide/layout_and_tables.md index c00201bcfc..3d7d2d801b 100644 --- a/docs/source/user_guide/layout_and_tables.md +++ b/docs/source/user_guide/layout_and_tables.md @@ -1,7 +1,7 @@ # Layout analysis & tables Text alone loses the structure of a document. docTR can also tell *what* each region of a page is (a title, a -paragraph, a table, a figure...) and rebuild tables as rows and columns. +paragraph, a table, a figure...), rebuild tables as rows and columns and read formulas as LaTeX. ## Layout analysis @@ -61,7 +61,8 @@ predictor = ocr_predictor(pretrained=True, detect_layout=True, ignore_regions=[" ``` :::{important} -`ignore_regions` relies on the layout model: enable it with `detect_layout=True` (or `detect_tables=True`), +`ignore_regions` relies on the layout model: enable it with `detect_layout=True` (or `detect_tables=True` / +`detect_formulas=True`), otherwise the regions cannot be located and nothing is masked. The CLI enables it automatically. ::: @@ -139,9 +140,44 @@ for cell in table.cells: | `confidence` | Mean recognition confidence of the words of the cell | | `objectness_score` | Table model score of the cell (`None` for a cell made from words) | +## Formula recognition + +With `detect_formulas=True`, each region the layout model labels as a `Formula` (a display formula, on its own lines) +is masked out of the text detection, cropped and read by a formula recognition model (`formulanet`), which predicts its +LaTeX source. The formulas are attached to every page as {class}`~doctr.io.Formula` objects: + +```python +from doctr.io import DocumentFile +from doctr.models import ocr_predictor + +predictor = ocr_predictor(pretrained=True, detect_formulas=True) +result = predictor(DocumentFile.from_pdf("attention_is_all_you_need.pdf")) # arXiv:1706.03762 + +formula = result.pages[4].formulas[0] +print(formula.value) +# \mathrm { F F N } ( x ) = \operatorname* { m a x } ( 0 , x W _ { 1 } + b _ { 1 } ) W _ { 2 } + b _ { 2 } \tag { $ 2 $ } +print(formula.confidence) # 0.98 +``` + +The formulas are read in place with the text, and exported as display math: + +| Export | Rendering | +| --- | --- | +| `render()` | the LaTeX source | +| Markdown | `$$`, the LaTeX source and `$$` on their own lines | +| HTML | `
$$...$$
`, rendered by MathJax or KaTeX auto-render | +| AsciiDoc | a `[latexmath]` passthrough block | +| hOCR | an `ocr_math` element holding the LaTeX source | +| JSON | the `formulas` key of each page: `value`, `confidence`, `geometry`, `objectness_score` | + +:::{important} +The formula regions are **not** read by the text recognition model, so a formula is not returned twice. The LaTeX +source is never escaped as Markdown or AsciiDoc text, and it is HTML-escaped in HTML exports. +::: + ## Standalone predictors -Both models can also run on their own, without text recognition: +The models can also run on their own, without text recognition: ::::{tab-set} @@ -186,6 +222,31 @@ tables = predictor.predict_regions(pages, regions, word_boxes) # one (N, 4) arr tables[0][0]["cells"][0]["words"] # indices of the words of the cell in `word_boxes[0]`, in reading order ``` ::: + +:::{tab-item} Formula recognition +Pass images of formulas, e.g. crops of the `Formula` regions, margins included: + +```python +from doctr.models import formula_predictor + +predictor = formula_predictor("formulanet", pretrained=True) +out = predictor([formula_image]) + +# The LaTeX tokens separated by spaces, and the confidence +out[0] # ('J ( u , y ) = \\int _ { 0 } ^ { 1 } \\frac { | u | ^ { 2 } } { 2 } \\, d x + \\Psi ( y )', 0.98) +``` + +With the layout regions of whole pages, the formula regions are cropped and read: + +```python +from doctr.models import layout_predictor + +regions = layout_predictor("lw_detr_s", pretrained=True)(pages) +formulas = predictor.predict_regions(pages, regions) + +formulas[0][0] # {'geometry': ..., 'value': '\\frac { a } { b ^ { 2 } }', 'confidence': 0.97, 'score': 0.91} +``` +::: :::: See [](standalone_predictors.md) for all the task-specific predictors. diff --git a/docs/source/user_guide/standalone_predictors.md b/docs/source/user_guide/standalone_predictors.md index 8d7c1d50c9..84d2746178 100644 --- a/docs/source/user_guide/standalone_predictors.md +++ b/docs/source/user_guide/standalone_predictors.md @@ -10,6 +10,7 @@ reading pre-cropped words, classifying orientations... | {func}`~doctr.models.recognition.recognition_predictor` | Word crops | `(text, confidence)` | | {func}`~doctr.models.layout.layout_predictor` | Pages | `{"class_names", "boxes", "scores"}` | | {func}`~doctr.models.table_structure.table_predictor` | Table images | `{"cells", "num_rows", "num_cols"}` | +| {func}`~doctr.models.formula.formula_predictor` | Formula images | `(latex, confidence)` | | {func}`~doctr.models.classification.page_orientation_predictor` | Pages | Orientation class, angle and probability | | {func}`~doctr.models.classification.crop_orientation_predictor` | Word crops | Orientation class, angle and probability | @@ -58,7 +59,7 @@ lines or pages, use the OCR predictor, which detects the words first. Very wide crops are split into overlapping parts that are read separately and merged. `batch_size` (default `128`) sets how many crops are processed at once. -## Layout analysis & table structure +## Layout analysis, table structure & formula recognition See [](layout_and_tables.md#standalone-predictors). @@ -106,6 +107,7 @@ words = predictor(DocumentFile.from_images("path/to/word_crops/", lazy=True)) | --------------------------------------------------------------- | -------------------- | | Detection, layout, table structure, page orientation | `32` images | | Recognition, crop orientation (much smaller crops) | `1024` crops | +| Formula recognition | `128` crops | `chunk_size` is rounded up to a multiple of `batch_size`, so that the model processes the same batches as for a list, and the predictions do not change (the recognition predictor splits very wide crops within each chunk, which can only diff --git a/doctr/cli/main.py b/doctr/cli/main.py index 97da9466ed..e9ce4b7ee0 100644 --- a/doctr/cli/main.py +++ b/doctr/cli/main.py @@ -285,7 +285,7 @@ def _build_predictor(args: argparse.Namespace) -> Any: "keep_reading_order": args.keep_reading_order, } - model = ocr_predictor(detect_tables=args.detect_tables, **kwargs) + model = ocr_predictor(detect_tables=args.detect_tables, detect_formulas=args.detect_formulas, **kwargs) _set_thresholds(model, args) return _to_device(model, args) @@ -548,6 +548,11 @@ def _parse_args(argv=None): action="store_true", help="regroup the words of the detected tables into structured tables (enables the layout model)", ) + parser.add_argument( + "--detect_formulas", + action="store_true", + help="read the detected formulas as LaTeX, exported as display math (enables the layout model)", + ) parser.add_argument( "--ignore_regions", type=str, diff --git a/doctr/datasets/utils.py b/doctr/datasets/utils.py index 4346e75562..ea595848f5 100644 --- a/doctr/datasets/utils.py +++ b/doctr/datasets/utils.py @@ -3,11 +3,12 @@ # This program is licensed under the Apache License 2.0. # See LICENSE or go to for full license details. +import re import string import unicodedata from collections.abc import Sequence from collections.abc import Sequence as SequenceType -from functools import lru_cache, partial +from functools import lru_cache from pathlib import Path from typing import Any, TypeVar @@ -21,6 +22,8 @@ __all__ = [ "translate", + "tokenize_latex", + "normalize_latex", "encode_string", "decode_sequence", "encode_sequences", @@ -66,45 +69,103 @@ def translate( return translated +# A LaTeX token: an environment delimiter (e.g. \begin{array}), a command (e.g. \frac, \operatorname*), a control +# symbol (e.g. \{, \, or \\) or any other non-whitespace character +_LATEX_TOKEN = re.compile(r"\\(?:begin|end)\s*\{[a-zA-Z]+\*?\}|\\[a-zA-Z]+\*?|\\.|\S", re.DOTALL) + + +def tokenize_latex(latex: str) -> list[str]: + r"""Split a LaTeX string into tokens: commands, environment delimiters, control symbols and single characters + + >>> from doctr.datasets import tokenize_latex + >>> tokenize_latex(r"\frac{a}{b^2}") + ['\\frac', '{', 'a', '}', '{', 'b', '^', '2', '}'] + + The whitespaces are not part of the tokens (a control space `\ ` is tokenized as `\space`, which renders the same): + tokenizing a string and joining the tokens with spaces gives a normalized string which tokenizes into the same + tokens. + + Args: + latex: the LaTeX string to tokenize + + Returns: + the list of tokens + """ + tokens = [] + for tok in _LATEX_TOKEN.findall(latex): + if tok.startswith(("\\begin", "\\end")): + # Drop the whitespaces between an environment delimiter and its name (e.g. "\begin {array}") + tok = re.sub(r"\s+", "", tok) + elif len(tok) == 2 and tok[1].isspace(): + tok = "\\space" + tokens.append(tok) + return tokens + + +def normalize_latex(latex: str) -> str: + r"""Normalize a LaTeX string into space-separated tokens (cf. :func:`tokenize_latex`) + + >>> from doctr.datasets import normalize_latex + >>> normalize_latex(r"\frac{a}{b^2}") + '\\frac { a } { b ^ 2 }' + + Args: + latex: the LaTeX string to normalize + + Returns: + the normalized string, rendering the same as the input + """ + return " ".join(tokenize_latex(latex)) + + @lru_cache(maxsize=32) -def _vocab_lookup_table(vocab: str) -> dict[str, int]: - """Cached char -> index table for a vocab""" - return {char: idx for idx, char in reversed(list(enumerate(vocab)))} +def _vocab_lookup_table(vocab: str | tuple[str, ...]) -> dict[str, int]: + """Cached token -> index table for a vocab (the first occurrence of a duplicated token wins)""" + return {token: idx for idx, token in reversed(list(enumerate(vocab)))} + + +def _split_tokens(input_string: str, vocab: str | Sequence[str]) -> list[str]: + """Split a string into the units of a vocab: characters for a string vocab, whitespace-separated tokens for a + list of tokens""" + return list(input_string) if isinstance(vocab, str) else input_string.split() def encode_string( input_string: str, - vocab: str, + vocab: str | Sequence[str], ) -> list[int]: """Given a predefined mapping, encode the string to a sequence of numbers Args: input_string: string to encode - vocab: vocabulary (string), the encoding is given by the indexing of the character sequence + vocab: vocabulary, the encoding is given by the indexing of its units: a string of characters, or a list of + tokens (e.g. LaTeX commands), in which case the input string is split on whitespaces Returns: A list encoding the input_string """ - table = _vocab_lookup_table(vocab) + table = _vocab_lookup_table(vocab if isinstance(vocab, str) else tuple(vocab)) + tokens = _split_tokens(input_string, vocab) try: - return [table[char] for char in input_string] + return [table[token] for token in tokens] except KeyError as e: - missing_chars = [char for char in input_string if char not in table] + missing = [token for token in tokens if token not in table] raise ValueError( - f"Some characters cannot be found in 'vocab': {set(missing_chars)}.\n" + f"Some characters cannot be found in 'vocab': {set(missing)}.\n" f"Please check the input string `{input_string}` and the vocabulary `{vocab}`" ) from e def decode_sequence( input_seq: np.ndarray | SequenceType[int], - mapping: str, + mapping: str | Sequence[str], ) -> str: """Given a predefined mapping, decode the sequence of numbers to a string Args: input_seq: array to decode - mapping: vocabulary (string), the encoding is given by the indexing of the character sequence + mapping: vocabulary, the encoding is given by the indexing of its units: a string of characters, or a list of + tokens, in which case the decoded tokens are joined with spaces Returns: A string, decoded from input_seq @@ -117,12 +178,12 @@ def decode_sequence( if len(input_seq) > 0 and (min(input_seq) < 0 or max(input_seq) >= len(mapping)): raise AssertionError("Input must be an array of int, with values between 0 and the mapping size") - return "".join(map(mapping.__getitem__, input_seq)) + return ("" if isinstance(mapping, str) else " ").join(map(mapping.__getitem__, input_seq)) def encode_sequences( sequences: list[str], - vocab: str, + vocab: str | Sequence[str], target_size: int | None = None, eos: int = -1, sos: int | None = None, @@ -133,7 +194,8 @@ def encode_sequences( Args: sequences: the list of character sequences of size N - vocab: the ordered vocab to use for encoding + vocab: the ordered vocab to use for encoding: a string of characters, or a list of tokens (cf. + :func:`encode_string`) target_size: maximum length of the encoded data eos: encoding of End Of String sos: optional encoding of Start Of String @@ -146,9 +208,10 @@ def encode_sequences( if 0 <= eos < len(vocab): raise ValueError("argument 'eos' needs to be outside of vocab possible indices") + encoded_seqs = [encode_string(seq, vocab) for seq in sequences] if not isinstance(target_size, int) or dynamic_seq_length: - # Maximum string length + EOS - max_length = max((len(w) for w in sequences), default=0) + 1 + # Maximum sequence length (in vocab units) + EOS + max_length = max((len(seq) for seq in encoded_seqs), default=0) + 1 if isinstance(sos, int): max_length += 1 if isinstance(pad, int): @@ -166,7 +229,7 @@ def encode_sequences( encoded_data: np.ndarray = np.full([len(sequences), target_size], default_symbol, dtype=np.int32) # Encode the strings - for idx, seq in enumerate(map(partial(encode_string, vocab=vocab), sequences)): + for idx, seq in enumerate(encoded_seqs): if isinstance(pad, int): # add eos at the end of the sequence seq.append(eos) encoded_data[idx, : min(len(seq), target_size)] = seq[: min(len(seq), target_size)] diff --git a/doctr/datasets/vocabs.py b/doctr/datasets/vocabs.py index 0211b0ae8d..1562695125 100644 --- a/doctr/datasets/vocabs.py +++ b/doctr/datasets/vocabs.py @@ -6,11 +6,9 @@ import re import string -__all__ = ["VOCABS"] +__all__ = ["VOCABS", "LATEX_TOKENS"] _BASE_VOCABS = { - # LaTeX - "latex": "".join(sorted(set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\\{}^_=+-<>()[]/.,| "))), # Latin "digits": string.digits, "ascii_letters": string.ascii_letters, @@ -1171,3 +1169,142 @@ + VOCABS["greek"] ) ) + + +# LaTeX tokens of formula recognition: the printable ASCII characters, the control symbols, the environments and +# commands supported by KaTeX (https://katex.org/docs/supported), which renders the predictions, and the other +# TeX / LaTeX commands and characters found in the labels of public formula datasets (UniMER-1M, HME100K, UniMER-Test) +# NOTE: a backslash followed by a whitespace is tokenized as \space (cf. `doctr.datasets.tokenize_latex`) +_LATEX_CONTROL_SYMBOLS = ( + "\\\\ \\{ \\} \\| \\, \\: \\; \\! \\# \\$ \\% \\& \\_ \\' \\` \\^ \\~ \\= \\. \\\" \\> \\< \\/" +).split() +_LATEX_ENVIRONMENTS = ( + "array darray matrix matrix* pmatrix pmatrix* bmatrix bmatrix* Bmatrix Bmatrix* vmatrix vmatrix* Vmatrix " + "Vmatrix* smallmatrix subarray cases dcases rcases drcases aligned alignedat gathered split equation equation* " + "align align* alignat alignat* gather gather* CD" +).split() +_LATEX_COMMANDS = ( + # Greek and Hebrew letters + "alpha beta gamma delta epsilon varepsilon zeta eta theta vartheta iota kappa varkappa lambda mu nu xi omicron pi " + "varpi rho varrho sigma varsigma tau upsilon phi varphi chi psi omega digamma " + "Alpha Beta Gamma Delta Epsilon Zeta Eta Theta Iota Kappa Lambda Mu Nu Xi Omicron Pi Rho Sigma Tau Upsilon Phi " + "Chi Psi Omega varGamma varDelta varTheta varLambda varXi varPi varSigma varUpsilon varPhi varPsi varOmega " + "aleph beth gimel daleth " + # Letter-like symbols + "ell hbar hslash imath jmath wp Re Im partial nabla infty eth Bbbk complement Finv Game mho " + # Fractions, binomials and roots + "frac dfrac tfrac cfrac genfrac over atop above choose binom dbinom tbinom brace brack sqrt " + # Big operators + "sum prod coprod int iint iiint oint oiint oiiint intop smallint bigcup bigcap bigvee bigwedge bigoplus " + "bigotimes bigodot biguplus bigsqcup " + # Named operators + "lim liminf limsup varliminf varlimsup injlim projlim varinjlim varprojlim max min sup inf det gcd deg dim exp " + "hom ker lg ln log Pr arg argmax argmin sin cos tan cot sec csc arcsin arccos arctan arctg arcctg sinh cosh tanh " + "coth sh ch th tg ctg cotg cosec cth mod bmod pmod pod " + # Binary operators + "pm mp times div cdot ast star circ bullet oplus ominus otimes oslash odot cap cup uplus sqcap sqcup vee wedge " + "lor land setminus smallsetminus wr amalg dagger ddagger dotplus divideontimes ltimes rtimes leftthreetimes " + "rightthreetimes curlywedge curlyvee barwedge doublebarwedge boxplus boxminus boxtimes boxdot circledast " + "circledcirc circleddash centerdot intercal gtrdot lessdot Cap Cup doublecap doublecup bigcirc diamond " + "triangleleft triangleright bigtriangleup bigtriangledown lhd rhd unlhd unrhd veebar And " + # Relations + "leq le geq ge neq ne equiv approx approxeq sim simeq cong propto varpropto prec succ preceq succeq ll gg lll " + "ggg subset supset subseteq supseteq subsetneq supsetneq subseteqq supseteqq subsetneqq supsetneqq varsubsetneq " + "varsupsetneq nsubseteq nsupseteq Subset Supset sqsubset sqsupset sqsubseteq sqsupseteq in ni notin owns mid " + "nmid parallel nparallel shortmid shortparallel nshortmid nshortparallel perp models vdash dashv vDash Vdash " + "Vvdash nvdash nvDash nVdash nVDash asymp bowtie Join smile frown doteq doteqdot Doteq fallingdotseq " + "risingdotseq coloneqq eqqcolon colonequals coloneq eqcolon Coloneqq leqq geqq leqslant geqslant lesssim gtrsim " + "lessapprox gtrapprox lessgtr gtrless lesseqgtr gtreqless lesseqqgtr gtreqqless lneq gneq lneqq gneqq lnsim " + "gnsim lnapprox gnapprox nless ngtr nleq ngeq nleqq ngeqq nleqslant ngeqslant nprec nsucc npreceq nsucceq " + "precsim succsim precapprox succapprox precneqq succneqq precnsim succnsim precnapprox succnapprox preccurlyeq " + "succcurlyeq curlyeqprec curlyeqsucc trianglelefteq trianglerighteq ntriangleleft ntriangleright " + "ntrianglelefteq ntrianglerighteq vartriangle vartriangleleft vartriangleright triangleq between pitchfork " + "backsim backsimeq eqsim thicksim thickapprox nsim ncong eqcirc circeq bumpeq Bumpeq multimap therefore because " + # Arrows + "leftarrow rightarrow to gets uparrow downarrow updownarrow leftrightarrow Leftarrow Rightarrow Uparrow " + "Downarrow Updownarrow Leftrightarrow longleftarrow longrightarrow longleftrightarrow Longleftarrow " + "Longrightarrow Longleftrightarrow iff implies impliedby mapsto longmapsto hookleftarrow hookrightarrow " + "leftharpoonup leftharpoondown rightharpoonup rightharpoondown rightleftharpoons leftrightharpoons nearrow " + "searrow swarrow nwarrow nleftarrow nrightarrow nLeftarrow nRightarrow nleftrightarrow nLeftrightarrow " + "leftleftarrows rightrightarrows leftrightarrows rightleftarrows Lleftarrow Rrightarrow twoheadleftarrow " + "twoheadrightarrow leftarrowtail rightarrowtail looparrowleft looparrowright curvearrowleft curvearrowright " + "circlearrowleft circlearrowright Lsh Rsh upuparrows downdownarrows upharpoonleft upharpoonright downharpoonleft " + "downharpoonright restriction leftrightsquigarrow rightsquigarrow dashleftarrow dashrightarrow " + "xrightarrow xleftarrow xleftrightarrow xRightarrow xLeftarrow xLeftrightarrow xmapsto xhookleftarrow " + "xhookrightarrow xtwoheadrightarrow xtwoheadleftarrow xrightharpoonup xrightharpoondown xleftharpoonup " + "xleftharpoondown xrightleftharpoons xleftrightharpoons xlongequal xtofrom " + # Delimiters and their sizing + "left right middle big Big bigg Bigg bigl bigr Bigl Bigr biggl biggr Biggl Biggr bigm Bigm biggm Biggm langle " + "rangle lceil rceil lfloor rfloor lvert rvert lVert rVert vert Vert lbrace rbrace lbrack rbrack lgroup rgroup " + "lmoustache rmoustache ulcorner urcorner llcorner lrcorner backslash " + # Accents and annotations + "hat widehat check widecheck tilde widetilde utilde bar overline underline underbar vec overrightarrow " + "overleftarrow overleftrightarrow underrightarrow underleftarrow underleftrightarrow overlinesegment " + "underlinesegment Overrightarrow dot ddot dddot ddddot acute grave breve mathring overbrace underbrace " + "overgroup undergroup overset underset stackrel boxed cancel bcancel xcancel sout not neg lnot " + # Fonts and classes + "mathrm mathbf mathit mathsf mathtt mathcal mathscr mathfrak mathbb mathnormal boldsymbol bm bold pmb Bbb text " + "textrm textbf textit textsf texttt textnormal textup textmd emph rm bf it sf tt cal frak operatorname " + "operatorname* operatornamewithlimits mathop mathbin mathrel mathopen mathclose mathpunct mathinner mathord " + # Styles and sizes + "tiny scriptsize footnotesize small normalsize large Large LARGE huge Huge displaystyle textstyle scriptstyle " + "scriptscriptstyle limits nolimits displaylimits " + # Spacing and layout + "quad qquad enspace thinspace medspace thickspace negthinspace negmedspace negthickspace hspace hskip mskip kern " + "mkern mathstrut phantom hphantom vphantom space nobreakspace llap rlap clap mathllap mathrlap mathclap smash " + "raisebox mbox hbox fbox rule substack sideset cr newline hline hdashline tag notag nonumber color textcolor " + "colorbox fcolorbox " + # Other symbols + "dots ldots cdots vdots ddots dotsc dotsb dotsm dotsi dotso cdotp ldotp hdots prime backprime forall exists " + "nexists emptyset varnothing angle measuredangle sphericalangle top bot triangle triangledown square " + "blacksquare lozenge blacklozenge blacktriangle blacktriangledown blacktriangleleft blacktriangleright bigstar " + "diagup diagdown clubsuit diamondsuit heartsuit spadesuit flat natural sharp checkmark maltese circledR " + "circledS surd degree yen pounds dag ddag S P copyright textdegree colon vcentcolon" +).split() + +# Found in the formula datasets, beyond KaTeX: control symbols (a lone backslash ends some labels), TeX / LaTeX commands +# (text-mode accents, grouping, spacing and font switches, `\up*` upright Greek letters...) and Unicode characters +# (Greek, Cyrillic and accented letters, math symbols, CJK punctuation and the Chinese characters of HME100K) +_DATASET_CONTROL_SYMBOLS = ["\\", "\\(", "\\)", "\\*", "\\-"] +_DATASET_COMMANDS = ( + "AA AE Bar Breve Check DOTSB Ddot Dot H Hat L Lt MakeUppercase O SS Tilde Vec a aa ae aftergroup aligned " + "allowbreak allowdisplaybreaks angles arraycolsep arraystretch astrosun atopwithdelims b backepsilon " + "baselineskip begingroup bfseries bgroup bigskip bigtimes boldmath boxbslash break c centering cite d def depth " + "detokenize diameter dj do downuparrows dprime edef egroup else em empty endaligned endgroup enskip ensuremath " + "eqno eqslantgtr eqslantless everymath expandafter fcmp fi fill fint fontfamily footnote footnotemark framebox " + "gdef global hexagon hfil hfill hss i idotsint if ifmmode iiiint ij indent index interleave isvec itGamma " + "itOmega itSigma j joinrel k l label lat leavevmode lefteqn leftmoon leftrightarrowtriangle less linespread " + "llangle lowercase makebox mathbfcal mathbfit mathbfscr mathchoice mathellipsis mathgroup mathpalette " + "mathparagraph mathrsf mathsection mathsfit medskip mit ng noindent nolinebreak normalbaselineskip normalcolor " + "normalfont nsimeq null nulldelimiterspace o of offinterlineskip ointctrclockwise ointop overbar overbracket " + "overrightharpoon overwithdelims pentagon protect pullback qopname r raggedleft raisetag ref relax rightmoon " + "romannumeral root rq rrangle s sb sc scshape selectfont shuffle skew sl slash smallskip smt sp special ss " + "string strut sumint sun textbackslash textbar textcent textcircled textcopyright textemdash texteuro " + "textgreater textlangle textless textmu textperiodcentered textperthousand textquoteright textrangle textrecipe " + "textregistered textsc textsl textsubscript textsuperscript texttimes textunderscore the theboldgroup " + "theequation thesubequation thinmuskip time tmspace today tplus u unboldmath underbracket upDelta upOmega " + "upSigma upalpha upbeta upchi updelta updownarrows upepsilon upeta upgamma upkappa uplambda upmu upnu upomega " + "uppercase upphi uppi uppsi uprho upshape upsigma uptau uptheta upvarepsilon upvarphi upxi upzeta v verb vline " + "vrule" +).split() +_DATASET_CHARACTERS = ( + # Latin-1, accented Latin and Greek / Cyrillic letters + "¡£¥¨®¯°±²´µ·¸ÅÆ×àáäåéðöćčΓΔΩαβδεηθικλμνοπστυφχψωόДНагдежинч" + # Punctuation, math symbols and letter-like symbols (U+2061 is the invisible function application) + "ḊḌ‐–—’“†…⁄\u2061ℓ↦∈∑−∘√∞∫≅≤⊗⊘⋅◁⟨⟩𝓝𝕍𝕎" + # CJK punctuation and characters (HME100K, UniMER-Test), U+FFFD is the replacement character of broken labels + "、。万丙个为乙人代以倍元入公分即原可周和因大天妈将小岁年式得或所把数方时明是最本次求法满然甲的种积第答组能自行袋解该负足面页项,\ufffd" +) + +# NOTE: the tokens found in the datasets come last, so that the indices of the other tokens do not depend on them +LATEX_TOKENS: list[str] = list( + dict.fromkeys([ + *(char for char in string.printable if not char.isspace() and char != "\\"), + *_LATEX_CONTROL_SYMBOLS, + *(f"\\{boundary}{{{env}}}" for env in _LATEX_ENVIRONMENTS for boundary in ("begin", "end")), + *(f"\\{command}" for command in _LATEX_COMMANDS), + *_DATASET_CONTROL_SYMBOLS, + *(f"\\{command}" for command in _DATASET_COMMANDS), + *_DATASET_CHARACTERS, + ]) +) diff --git a/doctr/io/elements.py b/doctr/io/elements.py index f1da19db1b..bf55589c42 100644 --- a/doctr/io/elements.py +++ b/doctr/io/elements.py @@ -35,6 +35,7 @@ "LayoutElement", "TableCell", "Table", + "Formula", ] @@ -365,6 +366,50 @@ def from_dict(cls, save_dict: dict[str, Any], **kwargs): return cls(**kwargs) +class Formula(Element): + r"""Implements a mathematical formula recognized on a page as LaTeX + + The formula regions are located by the layout model and kept out of the regular text recognition, so the + formula is not duplicated in the `blocks` of the page. + + Args: + value: the LaTeX source of the formula (tokens separated by spaces, e.g. `\frac { a } { b }`), without + math delimiters + confidence: the confidence of the formula recognition + geometry: bounding box of the formula in format ((xmin, ymin), (xmax, ymax)) or a (4, 2) polygon, with + coordinates relative to the page's size + objectness_score: the score of the formula region predicted by the layout model, None if unknown + """ + + _exported_keys: list[str] = ["geometry", "value", "confidence", "objectness_score"] + _children_names: list[str] = [] + + def __init__( + self, + value: str, + confidence: float, + geometry: BoundingBox | np.ndarray, + objectness_score: float | None = None, + ) -> None: + super().__init__() + self.value = value + self.confidence = confidence + self.geometry = geometry + self.objectness_score = objectness_score + + def render(self) -> str: + """Renders the LaTeX source of the formula""" + return self.value + + def extra_repr(self) -> str: + return f"value='{self.value}', confidence={self.confidence:.2}" + + @classmethod + def from_dict(cls, save_dict: dict[str, Any], **kwargs): + kwargs = {k: save_dict[k] for k in cls._exported_keys if k != "objectness_score"} + return cls(**kwargs, objectness_score=save_dict.get("objectness_score")) + + class Line(Element): """Implements a line element as a collection of words @@ -497,13 +542,15 @@ class Page(PageExportsMixin, Element): language: a dictionary with the language value and confidence of the prediction layout: optional list of layout regions detected on the page tables: optional list of tables recognized on the page. Words assigned to a table are removed from `blocks`. + formulas: optional list of formulas recognized on the page. Formula regions are not part of `blocks`. """ _exported_keys: list[str] = ["page_idx", "dimensions", "orientation", "language"] - _children_names: list[str] = ["blocks", "layout", "tables"] + _children_names: list[str] = ["blocks", "layout", "tables", "formulas"] blocks: list[Block] = [] layout: list[LayoutElement] = [] tables: list[Table] = [] + formulas: list[Formula] = [] page = _page_image_property @@ -517,11 +564,13 @@ def __init__( language: dict[str, Any] | None = None, layout: list[LayoutElement] | None = None, tables: list[Table] | None = None, + formulas: list[Formula] | None = None, ) -> None: super().__init__( blocks=blocks, layout=layout if layout is not None else [], tables=tables if tables is not None else [], + formulas=formulas if formulas is not None else [], ) self.page = page self.page_idx = page_idx @@ -573,6 +622,7 @@ def from_dict(cls, save_dict: dict[str, Any], page: np.ndarray | None = None, ** "blocks": [Block.from_dict(block_dict) for block_dict in save_dict["blocks"]], "layout": [LayoutElement.from_dict(region_dict) for region_dict in save_dict.get("layout", [])], "tables": [Table.from_dict(table_dict) for table_dict in save_dict.get("tables", [])], + "formulas": [Formula.from_dict(formula_dict) for formula_dict in save_dict.get("formulas", [])], }) # The page image is not part of the export: pass it back explicitly to restore a fully usable page return cls(page=_empty_page_image(page), **_kwargs) diff --git a/doctr/io/exporters.py b/doctr/io/exporters.py index c626c41409..52593f24b0 100644 --- a/doctr/io/exporters.py +++ b/doctr/io/exporters.py @@ -17,7 +17,7 @@ from doctr.utils.common_types import BoundingBox if TYPE_CHECKING: # pragma: no cover - from doctr.io.elements import Block, KIEPage, LayoutElement, Line, Page, Table + from doctr.io.elements import Block, Formula, KIEPage, LayoutElement, Line, Page, Table __all__ = [ "AsciiDocExporter", @@ -99,6 +99,13 @@ def _merge_overlapping_regions(elt_regions: list[int], region_geoms: list[Any], return [groups[reg] if reg >= 0 else reg for reg in elt_regions] +def _is_atomic(element: Any) -> bool: + """Whether a reading-order element is an item of its own (a table, a formula or a figure), not a text line""" + from doctr.io.elements import Formula, LayoutElement, Table + + return isinstance(element, (Table, Formula, LayoutElement)) + + def _xyxy(geometry: Any) -> tuple[float, float, float, float]: """Return the enclosing box (xmin, ymin, xmax, ymax) of a box or a polygon""" pts = np.asarray(geometry, dtype=np.float64).reshape(-1, 2) @@ -122,13 +129,15 @@ def _join_caption_lines(lines: list[str]) -> str: def _reading_order_signature(page: "Page", direction: str) -> tuple[Any, ...]: """Return a cheap fingerprint of a page, used to invalidate the reading-order cache - It covers the direction and the identity of the blocks (with their line count), tables and layout regions. + It covers the direction and the identity of the blocks (with their line count), tables, formulas and layout + regions. In-place edits of the words of a line are not detected: drop `_reading_order_cache` after such edits. """ return ( direction, tuple((id(block), len(block.lines)) for block in page.blocks), tuple(id(table) for table in getattr(page, "tables", ()) or ()), + tuple(id(formula) for formula in getattr(page, "formulas", ()) or ()), tuple(id(region) for region in getattr(page, "layout", ()) or ()), ) @@ -144,7 +153,7 @@ def _store_reading_order(page: "Page", signature: tuple[Any, ...], result: tuple def page_reading_order( page: "Page", direction: str = "auto", include_figures: bool = False ) -> tuple[list[Any], list[str | None], str]: - """Linearize the content of a page (blocks, tables and optionally figures) in reading order + """Linearize the content of a page (blocks, tables, formulas and optionally figures) in reading order The result is cached on the page, so a page exported to several formats is ordered once. The figures always take part in the ordering, `include_figures` only controls whether they are returned. @@ -171,8 +180,8 @@ def page_reading_order( direction = ReadingOrderPredictor(direction=direction).resolve_direction( [word.value for word in words], language=language ) - # Lines, tables and figures (as floats) are ordered together - elements: list[Any] = [*lines, *page.tables, *picture_regions(page)] + # Lines, tables and figures (as floats) and formulas (in the text flow) are ordered together + elements: list[Any] = [*lines, *page.tables, *getattr(page, "formulas", []), *picture_regions(page)] items: list[Any] = [] labels: list[str | None] = [] if len(elements) > 0: @@ -203,12 +212,13 @@ def page_reading_order( def _element_labels( page: "Page", elements: list[Any], elt_geoms: list[Any], elt_regions: list[int], region_geoms: list[Any] ) -> list[str | None]: - """Return the layout label of each element: the label of its layout region (None for none), 'Table' for a table + """Return the layout label of each element: the label of its layout region (None for none), 'Table' for a table, + 'Formula' for a formula A figure takes its own type, unless it lies in page furniture (e.g. a logo in the page header): it is then read with the furniture, along with its text. """ - from doctr.io.elements import LayoutElement, Table + from doctr.io.elements import Formula, LayoutElement, Table from doctr.models.reading_order import layout_label_role from doctr.models.reading_order.base import _FURNITURE_ROLES @@ -228,6 +238,8 @@ def _element_labels( for idx, elt in enumerate(elements): if isinstance(elt, Table): labels[idx] = "Table" + elif isinstance(elt, Formula): + labels[idx] = "Formula" elif not isinstance(elt, LayoutElement) and elt_regions[idx] in furniture_of_figure: labels[idx] = furniture_of_figure[elt_regions[idx]] return labels @@ -275,11 +287,11 @@ def _segments_to_items( page: "Page", elements: list[Any], segments: list[list[int]], elt_labels: list[str | None], elt_regions: list[int] ) -> tuple[list[Any], list[str | None]]: """Turn reading-ordered segments of elements into page items: one block per paragraph, one block per list item - (a list item wrapped over several lines is one block), tables and figures as they are + (a list item wrapped over several lines is one block), tables, formulas and figures as they are The artefacts of a block follow its first line; those of blocks without lines go to the last block. """ - from doctr.io.elements import LayoutElement, Table + from doctr.io.elements import Formula, Table from doctr.models.reading_order import normalize_layout_label artefacts = _ArtefactClaims(page.blocks) @@ -288,9 +300,9 @@ def _segments_to_items( list_region: int | None = None # region of the list item being built for run in (run for segment in segments for run in _float_runs(elements, segment)): first, label = elements[run[0]], elt_labels[run[0]] - if isinstance(first, (Table, LayoutElement)): + if _is_atomic(first): items.append(first) - labels.append("Table" if isinstance(first, Table) else label) + labels.append("Table" if isinstance(first, Table) else "Formula" if isinstance(first, Formula) else label) list_region = None elif normalize_layout_label(label) in _LIST_LABELS: # One bullet per list-item region: consecutive lines of the same region are a single bullet @@ -343,13 +355,11 @@ def attach_leftovers(self, items: list[Any]) -> None: def _float_runs(elements: list[Any], segment: list[int]) -> list[list[int]]: - """Split the tables and figures out of a segment (a figure in page furniture can share one)""" - from doctr.io.elements import LayoutElement, Table - + """Split the tables, formulas and figures out of a segment (a figure in page furniture can share one)""" runs: list[list[int]] = [] for idx in segment: - is_float = isinstance(elements[idx], (Table, LayoutElement)) - if runs and not is_float and not isinstance(elements[runs[-1][-1]], (Table, LayoutElement)): + is_atomic = _is_atomic(elements[idx]) + if runs and not is_atomic and not _is_atomic(elements[runs[-1][-1]]): runs[-1].append(idx) else: runs.append([idx]) @@ -601,6 +611,10 @@ def render_table(self, table: "Table", escape: bool = True) -> str: """Render a recognized table in the target format""" raise NotImplementedError + def render_formula(self, formula: "Formula", escape: bool = True) -> str: + """Render a recognized formula in the target format (its LaTeX source is never escaped as text)""" + return formula.value + def render_figure(self, source: str | None, caption: str | None = None, escape: bool = True) -> str: """Render a figure in the target format @@ -728,7 +742,7 @@ def export_page( Returns: the exported page as a string """ - from doctr.io.elements import LayoutElement, Table + from doctr.io.elements import Formula, LayoutElement, Table from doctr.models.reading_order import layout_label_role, normalize_layout_label from doctr.models.reading_order.base import _FURNITURE_ROLES @@ -756,9 +770,13 @@ def _flush_list() -> None: continue if index in absorbed_captions or index in figure_text: continue - if isinstance(item, Table): + if isinstance(item, (Table, Formula)): _flush_list() - rendered = self.render_table(item, escape=escape) + rendered = ( + self.render_table(item, escape=escape) + if isinstance(item, Table) + else self.render_formula(item, escape=escape) + ) if rendered: parts.append(rendered) continue @@ -881,6 +899,10 @@ def _cell(value: str) -> str: separator = "| " + " | ".join("---" for _ in grid[0]) + " |" return "\n".join([rows[0], separator, *rows[1:]]) + def render_formula(self, formula: "Formula", escape: bool = True) -> str: + """Render a formula as display math: `$$`, the LaTeX source and `$$` on their own lines""" + return f"$$\n{formula.value}\n$$" if formula.value else "" + def render_figure(self, source: str | None, caption: str | None = None, escape: bool = True) -> str: """Render a figure as an image, with its caption as alt text and as a visible italic line below""" if source is None: @@ -936,6 +958,10 @@ def _row(row: list[str]) -> str: return "\n".join(["|===", _row(grid[0]), "", *[_row(row) for row in grid[1:]], "|==="]) + def render_formula(self, formula: "Formula", escape: bool = True) -> str: + """Render a formula as a `latexmath` passthrough block""" + return f"[latexmath]\n++++\n{formula.value}\n++++" if formula.value else "" + def render_figure(self, source: str | None, caption: str | None = None, escape: bool = True) -> str: """Render a figure as a block image macro, titled with its caption""" if source is None: @@ -959,7 +985,8 @@ def class_header(self, class_name: str, escape: bool = True) -> str: class HTMLExporter(_PageTextExporter): """Export OCR results to semantic HTML, with the content sorted in reading order. - Headings map to `

`/`

`, list items to `
  • `, recognized tables to `` and + Headings map to `

    `/`

    `, list items to `
    • `, recognized tables to `

    `, recognized formulas to + `
    ` holding `$$...$$` display math (rendered by MathJax or KaTeX auto-render) and paragraphs to `

    ` (with `
    ` between the visual lines of a paragraph). The output is a fragment, not a full document: it carries no doctype, `` or charset declaration. @@ -1002,6 +1029,12 @@ def render_figure(self, source: str | None, caption: str | None = None, escape: figcaption = f"\n

    {self.escape_text(caption) if escape else caption}
    " if caption else "" return f'
    {alt}{figcaption}
    ' + def render_formula(self, formula: "Formula", escape: bool = True) -> str: + """Render a formula as `$$...$$` display math in a `
    `""" + if not formula.value: + return "" + return f'
    $${self.escape_text(formula.value) if escape else formula.value}$$
    ' + def render_table(self, table: "Table", escape: bool = True) -> str: """Render a table as an HTML table (first row used as header)""" grid = table.to_grid() @@ -1085,7 +1118,7 @@ class XMLExporter: >>> xml_bytes, xml_tree = XMLExporter().export_page(page) """ - ocr_capabilities: ClassVar[str] = "ocr_page ocr_carea ocr_par ocr_line ocrx_word ocr_photo" + ocr_capabilities: ClassVar[str] = "ocr_page ocr_carea ocr_par ocr_line ocrx_word ocr_photo ocr_math" def _new_document(self, file_title: str, language: str) -> tuple[ETElement, ETElement]: """Create the hOCR root element with its , returning the root and its element.""" @@ -1163,6 +1196,39 @@ def _add_table( cell_span.text = cell.value return table_count + 1 + def _add_formula(self, page_div: ETElement, formula: "Formula", width: int, height: int, formula_count: int) -> int: + """Add a formula as an hOCR text area holding an `ocr_math` element, with its LaTeX source as text + + Args: + page_div: the `ocr_page` element + formula: the formula + width: page width in pixels + height: page height in pixels + formula_count: 1-based index of the formula on the page + + Returns: + the index of the next formula + """ + xmin, ymin, xmax, ymax = _xyxy(formula.geometry) + bbox = _hocr_bbox(((xmin, ymin), (xmax, ymax)), width, height) + area = SubElement( + page_div, "div", attrib={"class": "ocr_carea", "id": f"formula_area_{formula_count}", "title": bbox} + ) + paragraph = SubElement( + area, "p", attrib={"class": "ocr_par", "id": f"formula_par_{formula_count}", "title": bbox} + ) + math = SubElement( + paragraph, + "span", + attrib={ + "class": "ocr_math", + "id": f"formula_{formula_count}", + "title": f"{bbox}; x_wconf {int(round(formula.confidence * 100))}", + }, + ) + math.text = formula.value + return formula_count + 1 + def _add_figure( self, page_div: ETElement, region: "LayoutElement", width: int, height: int, figure_count: int ) -> int: @@ -1200,26 +1266,27 @@ def export_page( ) -> tuple[bytes, ET.ElementTree]: """Export a page as hOCR XML, with its content sorted in reading order. - Figures are exported as `ocr_photo` areas. + Figures are exported as `ocr_photo` areas, formulas as `ocr_math` elements holding their LaTeX source. Args: page: the page to export file_title: the title of the XML file direction: reading direction, one of 'auto', 'ltr', 'rtl', 'ttb-rtl' or 'ttb-ltr' reading_order: whether the content should be linearized in reading order. Pass False to serialize - `page.blocks` then `page.tables` in their raw order. + `page.blocks`, `page.tables` then `page.formulas` in their raw order. dpi: the page resolution in dots per inch, used to estimate font sizes (`x_size`, `x_fsize`) Returns: a tuple of the XML byte string, and its ElementTree """ - from doctr.io.elements import LayoutElement, Table + from doctr.io.elements import Formula, LayoutElement, Table block_count: int = 1 line_count: int = 1 word_count: int = 1 table_count: int = 1 figure_count: int = 1 + formula_count: int = 1 height, width = page.dimensions page_hocr, body = self._new_document(file_title, _resolve_hocr_language(page.language)) page_div = SubElement( @@ -1235,7 +1302,7 @@ def export_page( if reading_order: items, _, direction = page_reading_order(page, direction, include_figures=True) else: - items = [*page.blocks, *page.tables, *picture_regions(page)] + items = [*page.blocks, *page.tables, *page.formulas, *picture_regions(page)] # iterate over the blocks / lines / words and create the XML elements line by line with the attributes for item in items: if isinstance(item, Table): @@ -1244,6 +1311,9 @@ def export_page( if isinstance(item, LayoutElement): figure_count = self._add_figure(page_div, item, width, height, figure_count) continue + if isinstance(item, Formula): + formula_count = self._add_formula(page_div, item, width, height, formula_count) + continue block = item if len(block.geometry) != 2: raise TypeError("XML export is only available for straight bounding boxes for now.") @@ -1408,6 +1478,7 @@ class PageExportsMixin: language: dict[str, Any] layout: list[Any] tables: list["Table"] + formulas: list["Formula"] def render(self, block_break: str = "\n\n", direction: str = "auto", include_furniture: bool = True) -> str: """Renders the full text of the page, with its content sorted in reading order. @@ -1438,7 +1509,7 @@ def export(self, reading_order: bool = True) -> dict[str, Any]: export_dict = Element.export(cast("Element", self)) if reading_order: - # Tables and layout regions have their own export keys + # Tables, formulas and layout regions have their own export keys blocks = [item for item in page_reading_order(cast("Page", self))[0] if isinstance(item, Block)] if blocks: # an empty linearization (no line on the page) leaves the stored blocks untouched export_dict["blocks"] = [block.export() for block in blocks] @@ -1469,15 +1540,15 @@ def export_as_xml( def items_in_reading_order( self, direction: str = "auto", include_figures: bool = False - ) -> list["Block | Table | LayoutElement"]: - """Return the content of the page (blocks, tables and optionally figures) sorted in reading order. + ) -> list["Block | Table | Formula | LayoutElement"]: + """Return the content of the page (blocks, tables, formulas and optionally figures) sorted in reading order. Args: direction: reading direction, one of 'auto', 'ltr', 'rtl', 'ttb-rtl' or 'ttb-ltr' include_figures: whether to return the figure regions too Returns: - list of blocks, tables and, if requested, figures in reading order + list of blocks, tables, formulas and, if requested, figures in reading order """ return page_reading_order(cast("Page", self), direction, include_figures=include_figures)[0] diff --git a/doctr/models/__init__.py b/doctr/models/__init__.py index 893998d536..a229085dc2 100644 --- a/doctr/models/__init__.py +++ b/doctr/models/__init__.py @@ -3,5 +3,6 @@ from .recognition import * from .layout import * from .table_structure import * +from .formula import * from .zoo import * from .factory import * diff --git a/doctr/models/builder.py b/doctr/models/builder.py index 946cae4e18..8176c437e8 100644 --- a/doctr/models/builder.py +++ b/doctr/models/builder.py @@ -15,6 +15,7 @@ from doctr.io.elements import ( Block, Document, + Formula, KIEDocument, KIEPage, LayoutElement, @@ -514,6 +515,35 @@ def _build_tables( ) return tables, consumed + def _build_formulas(self, page_formulas: list[dict[str, Any]] | None, straight: bool) -> list[Formula]: + """Build the formulas of a page from their recognition + + Args: + page_formulas: the formulas recognized on the page `{"geometry", "value", "confidence", "score"}` + (cf. `FormulaPredictor.predict_regions`), with a straight box (4,) or a polygon (4, 2) in relative + coordinates, or None + straight: whether to export the geometries as straight boxes (else as polygons) + + Returns: + the list of `Formula` objects of the page, the empty predictions left out + """ + formulas: list[Formula] = [] + for raw in page_formulas or []: + if not raw.get("value"): + continue + geom = np.asarray(raw["geometry"], dtype=np.float64) + if geom.ndim == 1: # (xmin, ymin, xmax, ymax) + geom = np.array([[geom[0], geom[1]], [geom[2], geom[1]], [geom[2], geom[3]], [geom[0], geom[3]]]) + formulas.append( + Formula( + value=raw["value"], + confidence=float(raw["confidence"]), + geometry=self._export_geometry(geom, straight or self.export_as_straight_boxes), + objectness_score=None if raw.get("score") is None else float(raw["score"]), + ) + ) + return formulas + @staticmethod def _export_geometry(poly: np.ndarray, straight: bool) -> Any: """Export a (4, 2) polygon as a straight 2-point box or as a 4-point polygon, with Python floats""" @@ -648,6 +678,7 @@ def __call__( languages: list[dict[str, Any]] | None = None, regions: list[dict[str, Any] | None] | None = None, tables: Sequence[dict[str, Any] | list[dict[str, Any]] | None] | None = None, + formulas: Sequence[list[dict[str, Any]] | None] | None = None, ) -> Document: """Re-arrange detected words into structured blocks @@ -670,6 +701,8 @@ def __call__( page: a single grid `{"cells": [...], "num_rows": int, "num_cols": int}` or a list of such grids (one per table region detected by the layout model). Words assigned to any table are removed from the `blocks` output of that page. + formulas: optional, list of N elements, where each element is the list of formulas recognized on a page + `{"geometry", "value", "confidence", "score"}` (cf. `FormulaPredictor.predict_regions`) Returns: document object @@ -684,6 +717,7 @@ def __call__( _languages = languages if isinstance(languages, list) else [None] * len(boxes) _regions = regions if isinstance(regions, list) else [None] * len(boxes) _tables = tables if isinstance(tables, list) else [None] * len(boxes) + _formulas = formulas if isinstance(formulas, list) else [None] * len(boxes) if self.export_as_straight_boxes and len(boxes) > 0: # If boxes are already straight OK, else fit a bounding rect if boxes[0].ndim == 3: @@ -703,6 +737,7 @@ def __call__( language, page_regions, page_table, + page_formulas, ) in zip( # noqa: E501 pages, range(len(boxes)), @@ -715,6 +750,7 @@ def __call__( _languages, _regions, _tables, + _formulas, ): # Build the page tables and flag the words that belong to a table page_tables, consumed = self._build_tables(page_boxes, word_preds, page_table, shape) @@ -744,6 +780,7 @@ def __call__( language, page_layout, page_tables, + self._build_formulas(page_formulas, straight=page_boxes.ndim != 3), ) if self.keep_reading_order: self._apply_reading_order(_page) @@ -780,6 +817,7 @@ def __call__( # type: ignore[override] languages: list[dict[str, Any]] | None = None, regions: list[dict[str, Any] | None] | None = None, tables: Sequence[dict[str, Any] | list[dict[str, Any]] | None] | None = None, + formulas: Sequence[list[dict[str, Any]] | None] | None = None, ) -> KIEDocument: """Re-arrange detected words into structured predictions @@ -802,6 +840,8 @@ def __call__( # type: ignore[override] page: a single grid `{"cells": [...], "num_rows": int, "num_cols": int}` or a list of such grids (one per table region detected by the layout model). Words assigned to any table are removed from the `blocks` output of that page. Unused for KIE documents, as tables are not supported in KIE. + formulas: optional, list of N elements, where each element is the list of formulas recognized on a page. + Unused for KIE documents, as formulas are not supported in KIE. Returns: document object diff --git a/doctr/models/classification/__init__.py b/doctr/models/classification/__init__.py index b1e03e86e6..0ca5b7535a 100644 --- a/doctr/models/classification/__init__.py +++ b/doctr/models/classification/__init__.py @@ -7,4 +7,5 @@ from .starnet import * from .vip import * from .vit_det import * +from .hgnetv2 import * from .zoo import * diff --git a/doctr/models/classification/hgnetv2/__init__.py b/doctr/models/classification/hgnetv2/__init__.py new file mode 100644 index 0000000000..e3c861310c --- /dev/null +++ b/doctr/models/classification/hgnetv2/__init__.py @@ -0,0 +1 @@ +from .pytorch import * diff --git a/doctr/models/classification/hgnetv2/pytorch.py b/doctr/models/classification/hgnetv2/pytorch.py new file mode 100644 index 0000000000..32464c8176 --- /dev/null +++ b/doctr/models/classification/hgnetv2/pytorch.py @@ -0,0 +1,352 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +# Credits: architecture of PP-HGNetV2 from PaddleClas / PaddleOCR (https://github.com/PaddlePaddle/PaddleOCR, +# Apache 2.0), as also used by D-FINE (https://github.com/Peterande/D-FINE, Apache 2.0). + +from copy import deepcopy +from typing import Any + +import torch +from torch import nn +from torch.nn import functional as F + +from doctr.datasets import VOCABS + +from ...modules.layers import DropPath +from ...utils import load_pretrained_params + +__all__ = ["HGNetV2", "hgnetv2_b4"] + + +default_cfgs: dict[str, dict[str, Any]] = { + "hgnetv2_b4": { + "mean": (0.694, 0.695, 0.693), + "std": (0.299, 0.296, 0.301), + "input_shape": (3, 32, 32), + "classes": list(VOCABS["french"]), + "url": "https://github.com/mindee/doctr/releases/download/v1.1.0/hgnetv2_b4-0f90b8d0.pt", + }, +} + + +class ConvBNAct(nn.Sequential): + """A convolution (without bias) followed by batch-norm and an optional ReLU + + Args: + in_channels: number of input channels + out_channels: number of output channels + kernel_size: size of the convolution kernel + stride: stride of the convolution + groups: number of groups of the convolution + use_act: whether to add the ReLU activation + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int = 3, + stride: int = 1, + groups: int = 1, + use_act: bool = True, + ) -> None: + layers: list[nn.Module] = [ + nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=(kernel_size - 1) // 2, + groups=groups, + bias=False, + ), + nn.BatchNorm2d(out_channels), + ] + if use_act: + layers.append(nn.ReLU(inplace=True)) + super().__init__(*layers) + + +class LightConvBNAct(nn.Sequential): + """A point-wise convolution followed by a depth-wise convolution + + Args: + in_channels: number of input channels + out_channels: number of output channels + kernel_size: size of the depth-wise convolution kernel + """ + + def __init__(self, in_channels: int, out_channels: int, kernel_size: int) -> None: + super().__init__( + ConvBNAct(in_channels, out_channels, kernel_size=1, use_act=False), + ConvBNAct(out_channels, out_channels, kernel_size=kernel_size, groups=out_channels), + ) + + +class StemBlock(nn.Module): + """Stem of HGNetV2 (stride 4): a strided convolution, then a two-branch (convolutions | max-pooling) block + + Args: + in_channels: number of input channels + mid_channels: number of intermediate channels + out_channels: number of output channels + """ + + def __init__(self, in_channels: int, mid_channels: int, out_channels: int) -> None: + super().__init__() + self.stem1 = ConvBNAct(in_channels, mid_channels, kernel_size=3, stride=2) + # 2x2 kernels, padded on the bottom-right to keep the resolution + self.stem2a = nn.Sequential( + nn.Conv2d(mid_channels, mid_channels // 2, 2, bias=False), nn.BatchNorm2d(mid_channels // 2), nn.ReLU(True) + ) + self.stem2b = nn.Sequential( + nn.Conv2d(mid_channels // 2, mid_channels, 2, bias=False), nn.BatchNorm2d(mid_channels), nn.ReLU(True) + ) + self.pool = nn.MaxPool2d(kernel_size=2, stride=1, ceil_mode=True) + self.stem3 = ConvBNAct(mid_channels * 2, mid_channels, kernel_size=3, stride=2) + self.stem4 = ConvBNAct(mid_channels, out_channels, kernel_size=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = F.pad(self.stem1(x), (0, 1, 0, 1)) + x2 = self.stem2b(F.pad(self.stem2a(x), (0, 1, 0, 1))) + x1 = self.pool(x) + return self.stem4(self.stem3(torch.cat([x1, x2], dim=1))) + + +class HGBlock(nn.Module): + """HGNetV2 block: a chain of convolutions whose outputs are all concatenated (with the input) and aggregated by + point-wise convolutions, with an optional residual connection + + Args: + in_channels: number of input channels + mid_channels: number of channels of each convolution of the chain + out_channels: number of output channels + layer_num: number of convolutions in the chain + kernel_size: size of the convolution kernels + residual: whether to add a residual connection + light_block: whether to use light (point-wise + depth-wise) convolutions + drop_path: stochastic depth rate of the residual branch + """ + + def __init__( + self, + in_channels: int, + mid_channels: int, + out_channels: int, + layer_num: int, + kernel_size: int = 3, + residual: bool = False, + light_block: bool = False, + drop_path: float = 0.0, + ) -> None: + super().__init__() + self.residual = residual + self.layers = nn.ModuleList([ + (LightConvBNAct if light_block else ConvBNAct)( + in_channels if idx == 0 else mid_channels, mid_channels, kernel_size=kernel_size + ) + for idx in range(layer_num) + ]) + total_channels = in_channels + layer_num * mid_channels + self.aggregation = nn.Sequential( + ConvBNAct(total_channels, out_channels // 2, kernel_size=1), + ConvBNAct(out_channels // 2, out_channels, kernel_size=1), + ) + self.drop_path = DropPath(drop_path) if drop_path > 0 else nn.Identity() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + identity = x + outputs = [x] + for layer in self.layers: + x = layer(x) + outputs.append(x) + x = self.aggregation(torch.cat(outputs, dim=1)) + return self.drop_path(x) + identity if self.residual else x + + +class HGStage(nn.Sequential): + """HGNetV2 stage: an optional depth-wise strided convolution, then a sequence of HGBlocks + + Args: + in_channels: number of input channels + mid_channels: number of channels of the convolutions of the blocks + out_channels: number of output channels + num_blocks: number of blocks + downsample: whether to downsample the input by 2 first + light_block: whether the blocks use light convolutions + kernel_size: size of the convolution kernels of the blocks + layer_num: number of convolutions per block + drop_path: stochastic depth rate of each block + """ + + def __init__( + self, + in_channels: int, + mid_channels: int, + out_channels: int, + num_blocks: int, + downsample: bool, + light_block: bool, + kernel_size: int, + layer_num: int, + drop_path: list[float], + ) -> None: + super().__init__( + ConvBNAct(in_channels, in_channels, kernel_size=3, stride=2, groups=in_channels, use_act=False) + if downsample + else nn.Identity(), + *[ + HGBlock( + in_channels if idx == 0 else out_channels, + mid_channels, + out_channels, + layer_num, + kernel_size=kernel_size, + residual=idx > 0, + light_block=light_block, + drop_path=drop_path[idx], + ) + for idx in range(num_blocks) + ], + ) + + +class HGNetV2(nn.Sequential): + """Implements the HGNetV2 architecture from PaddlePaddle's `PP-HGNetV2 + `_. + + The model is a stem (stride 4) followed by 4 stages (strides 4, 8, 16 and 32). + + Args: + stem_channels: the input, intermediate and output channels of the stem + stages: the configuration of each stage `(in_channels, mid_channels, out_channels, num_blocks, downsample, + light_block, kernel_size, layer_num)` + drop_path_rate: maximum stochastic depth rate (linearly scaled across blocks) + num_classes: number of output classes of the classification head + include_top: whether to add the classification head + cfg: the configuration dict of the model + """ + + def __init__( + self, + stem_channels: tuple[int, int, int], + stages: list[tuple[int, int, int, int, bool, bool, int, int]], + drop_path_rate: float = 0.0, + num_classes: int = 1000, + include_top: bool = True, + cfg: dict[str, Any] | None = None, + ) -> None: + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(stage[3] for stage in stages))] + _layers: list[nn.Module] = [StemBlock(*stem_channels)] + cur = 0 + for in_ch, mid_ch, out_ch, num_blocks, downsample, light_block, kernel_size, layer_num in stages: + _layers.append( + HGStage( + in_ch, + mid_ch, + out_ch, + num_blocks, + downsample, + light_block, + kernel_size, + layer_num, + dpr[cur : cur + num_blocks], + ) + ) + cur += num_blocks + + if include_top: + _layers.append( + nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Flatten(1), + nn.Linear(stages[-1][2], num_classes), + ) + ) + + super().__init__(*_layers) + # Output channels of the stem and of each stage + self.channels = [stem_channels[2], *(stage[2] for stage in stages)] + self.cfg = cfg + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") + elif isinstance(m, nn.BatchNorm2d): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.trunc_normal_(m.weight, std=0.02) + nn.init.zeros_(m.bias) + + def from_pretrained(self, path_or_url: str, **kwargs: Any) -> None: + """Load pretrained parameters onto the model + + Args: + path_or_url: the path or URL to the model parameters (checkpoint) + **kwargs: additional arguments to be passed to `doctr.models.utils.load_pretrained_params` + """ + load_pretrained_params(self, path_or_url, **kwargs) + + +def _hgnetv2( + arch: str, + pretrained: bool, + ignore_keys: list[str] | None = None, + **kwargs: Any, +) -> HGNetV2: + kwargs["num_classes"] = kwargs.get("num_classes", len(default_cfgs[arch]["classes"])) + kwargs["classes"] = kwargs.get("classes", default_cfgs[arch]["classes"]) + + _cfg = deepcopy(default_cfgs[arch]) + _cfg["num_classes"] = kwargs["num_classes"] + _cfg["classes"] = kwargs["classes"] + kwargs.pop("classes") + + # Build the model + model = HGNetV2(**kwargs) + # Load pretrained parameters + if pretrained: + # The number of classes is not the same as the number of classes in the pretrained model => + # remove the last layer weights + _ignore_keys = ignore_keys if kwargs["num_classes"] != len(default_cfgs[arch]["classes"]) else None + model.from_pretrained(default_cfgs[arch]["url"], ignore_keys=_ignore_keys) + + model.cfg = _cfg + + return model + + +def hgnetv2_b4(pretrained: bool = False, **kwargs: Any) -> HGNetV2: + """HGNetV2-B4 from PaddlePaddle's `PP-HGNetV2 + `_. + + >>> import torch + >>> from doctr.models import hgnetv2_b4 + >>> model = hgnetv2_b4(pretrained=False) + >>> input_tensor = torch.rand((1, 3, 32, 32), dtype=torch.float32) + >>> out = model(input_tensor) + + Args: + pretrained: boolean, True if model is pretrained + **kwargs: keyword arguments of the HGNetV2 architecture + + Returns: + A HGNetV2-B4 model + """ + return _hgnetv2( + "hgnetv2_b4", + pretrained, + stem_channels=(3, 32, 48), + stages=[ + (48, 48, 128, 1, False, False, 3, 6), + (128, 96, 512, 1, True, False, 3, 6), + (512, 192, 1024, 3, True, True, 5, 6), + (1024, 384, 2048, 1, True, True, 5, 6), + ], + ignore_keys=["5.2.weight", "5.2.bias"], + **kwargs, + ) diff --git a/doctr/models/classification/zoo.py b/doctr/models/classification/zoo.py index e6e7919a10..ae680803f0 100644 --- a/doctr/models/classification/zoo.py +++ b/doctr/models/classification/zoo.py @@ -35,6 +35,7 @@ "vit_det_s", "vit_det_m", "starnet_s3", + "hgnetv2_b4", ] ORIENTATION_ARCHS: list[str] = ["mobilenet_v3_small_crop_orientation", "mobilenet_v3_small_page_orientation"] diff --git a/doctr/models/factory/hub.py b/doctr/models/factory/hub.py index f039ceae63..8850d8f3af 100644 --- a/doctr/models/factory/hub.py +++ b/doctr/models/factory/hub.py @@ -33,6 +33,7 @@ "recognition": models.recognition.zoo.ARCHS, "layout": models.layout.zoo.ARCHS, "table_structure": models.table_structure.zoo.ARCHS, + "formula": models.formula.zoo.ARCHS, } @@ -99,8 +100,8 @@ def push_to_hf_hub(model: Any, model_name: str, task: str, **kwargs) -> None: # if run_config is None and arch is None: raise ValueError("run_config or arch must be specified") - if task not in ["classification", "detection", "recognition", "layout", "table_structure"]: - raise ValueError("task must be one of classification, detection, recognition, layout, table_structure") + if task not in ["classification", "detection", "recognition", "layout", "table_structure", "formula"]: + raise ValueError("task must be one of classification, detection, recognition, layout, table_structure, formula") # default readme readme = f"""--- @@ -223,6 +224,8 @@ def from_hub(repo_id: str, **kwargs: Any): model = models.layout.__dict__[arch](pretrained=False, class_names=cfg["class_names"]) elif task == "table_structure": model = models.table_structure.__dict__[arch](pretrained=False) + elif task == "formula": + model = models.formula.__dict__[arch](pretrained=False, input_shape=cfg["input_shape"], vocab=cfg["vocab"]) # update model cfg model.cfg = cfg diff --git a/doctr/models/formula/__init__.py b/doctr/models/formula/__init__.py new file mode 100644 index 0000000000..537f46382c --- /dev/null +++ b/doctr/models/formula/__init__.py @@ -0,0 +1,2 @@ +from .formulanet import * +from .zoo import * diff --git a/doctr/models/formula/formulanet/__init__.py b/doctr/models/formula/formulanet/__init__.py new file mode 100644 index 0000000000..b2a33fe1a3 --- /dev/null +++ b/doctr/models/formula/formulanet/__init__.py @@ -0,0 +1,2 @@ +from .base import * +from .pytorch import * diff --git a/doctr/models/formula/formulanet/base.py b/doctr/models/formula/formulanet/base.py new file mode 100644 index 0000000000..52e7343510 --- /dev/null +++ b/doctr/models/formula/formulanet/base.py @@ -0,0 +1,70 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +from collections.abc import Sequence + +import numpy as np + +from doctr.datasets import encode_sequences, normalize_latex +from doctr.utils.repr import NestedObject + +from ..._utils import ConfidenceAggregation, _confidence_aggregation_repr, _resolve_confidence_aggregation + +__all__ = ["_FormulaNet", "_FormulaNetPostProcessor"] + + +class _FormulaNet: + vocab: list[str] + max_length: int + + def build_target(self, gts: list[str]) -> tuple[np.ndarray, list[int]]: + r"""Encode LaTeX labels into token sequences `[SOS, tokens..., EOS, PAD...]` + + The labels are tokenized first (cf. :func:`doctr.datasets.normalize_latex`), so they can be given either raw + (e.g. `\frac{a}{b}`) or normalized (e.g. `\frac { a } { b }`). + + Args: + gts: list of ground-truth LaTeX labels + + Returns: + the encoded sequences, of shape (N, L), and the number of tokens of each label + """ + normalized = [normalize_latex(gt) for gt in gts] + encoded = encode_sequences( + sequences=normalized, + vocab=self.vocab, + target_size=self.max_length + 2, # SOS + tokens + EOS + eos=len(self.vocab), + sos=len(self.vocab) + 1, + pad=len(self.vocab) + 2, + dynamic_seq_length=True, + ) + seq_len = [len(label.split()) for label in normalized] + return encoded, seq_len + + +class _FormulaNetPostProcessor(NestedObject): + """Turns the token scores of a formula recognition model into LaTeX strings + + Args: + vocab: the list of LaTeX tokens of the model + confidence_aggregation: aggregation method of the token probabilities into the formula confidence: + "mean", "min", "max", "median", "geometric_mean", "harmonic_mean" or a callable + """ + + def __init__( + self, + vocab: Sequence[str], + confidence_aggregation: ConfidenceAggregation = "mean", + ) -> None: + _resolve_confidence_aggregation(confidence_aggregation) + self.vocab = list(vocab) + self.confidence_aggregation = confidence_aggregation + + def extra_repr(self) -> str: + return ( + f"vocab_size={len(self.vocab)}, " + f"confidence_aggregation={_confidence_aggregation_repr(self.confidence_aggregation)}" + ) diff --git a/doctr/models/formula/formulanet/pytorch.py b/doctr/models/formula/formulanet/pytorch.py new file mode 100644 index 0000000000..be08dcb6cd --- /dev/null +++ b/doctr/models/formula/formulanet/pytorch.py @@ -0,0 +1,429 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +import math +from collections.abc import Callable, Sequence +from copy import deepcopy +from typing import Any, cast + +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F +from torchvision.models._utils import IntermediateLayerGetter + +from doctr.datasets import LATEX_TOKENS + +from ..._utils import ConfidenceAggregation, aggregate_confidence +from ...classification import hgnetv2_b4 +from ...utils import _bf16_to_float32, load_pretrained_params +from .base import _FormulaNet, _FormulaNetPostProcessor + +__all__ = ["FormulaNet", "formulanet"] + +default_cfgs: dict[str, dict[str, Any]] = { + "formulanet": { + "mean": (0.7931, 0.7931, 0.7931), + "std": (0.1738, 0.1738, 0.1738), + "input_shape": (3, 256, 768), + "vocab": LATEX_TOKENS, + "url": "https://github.com/mindee/doctr/releases/download/v1.1.0/formulanet-66790059.pt", + }, +} + + +def _sincos_2d(height: int, width: int, dim: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + """Fixed 2D sinusoidal positional encoding of a feature map, half of the channels for each axis + + Returns: + the encoding, of shape (height * width, dim) + """ + quarter = dim // 4 + omega = 1.0 / (10000 ** (torch.arange(quarter, device=device, dtype=torch.float32) / quarter)) + ys = torch.arange(height, device=device, dtype=torch.float32)[:, None] * omega[None] # (H, D/4) + xs = torch.arange(width, device=device, dtype=torch.float32)[:, None] * omega[None] # (W, D/4) + pos_y = torch.cat([ys.sin(), ys.cos()], dim=1)[:, None].expand(height, width, 2 * quarter) + pos_x = torch.cat([xs.sin(), xs.cos()], dim=1)[None].expand(height, width, 2 * quarter) + pos = torch.cat([pos_y, pos_x], dim=-1).reshape(height * width, 4 * quarter) + return F.pad(pos, (0, dim - 4 * quarter)).to(dtype) + + +class MultiHeadAttention(nn.Module): + """Multi-head attention with separate key / value projections, to cache them during decoding + + Args: + d_model: dimension of the model + num_heads: number of attention heads + dropout: dropout rate of the attention weights + """ + + def __init__(self, d_model: int, num_heads: int, dropout: float = 0.0) -> None: + super().__init__() + if d_model % num_heads != 0: + raise ValueError(f"d_model ({d_model}) must be divisible by num_heads ({num_heads})") + self.num_heads = num_heads + self.head_dim = d_model // num_heads + self.dropout = dropout + self.q_proj = nn.Linear(d_model, d_model) + self.k_proj = nn.Linear(d_model, d_model) + self.v_proj = nn.Linear(d_model, d_model) + self.out_proj = nn.Linear(d_model, d_model) + + def _split_heads(self, x: torch.Tensor) -> torch.Tensor: + # (N, L, D) -> (N, H, L, D / H) + return x.reshape(x.shape[0], x.shape[1], self.num_heads, self.head_dim).transpose(1, 2) + + def project_kv(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Project a sequence into the keys and values of the attention, of shape (N, H, L, D / H)""" + return self._split_heads(self.k_proj(x)), self._split_heads(self.v_proj(x)) + + def forward( + self, query: torch.Tensor, key_value: tuple[torch.Tensor, torch.Tensor], is_causal: bool = False + ) -> torch.Tensor: + q = self._split_heads(self.q_proj(query)) + out = F.scaled_dot_product_attention( + q, *key_value, dropout_p=self.dropout if self.training else 0.0, is_causal=is_causal + ) + return self.out_proj(out.transpose(1, 2).reshape(query.shape[0], query.shape[1], -1)) + + +class FormulaDecoderLayer(nn.Module): + """Pre-norm transformer decoder layer (causal self-attention, cross-attention on the image features, MLP) + + Args: + d_model: dimension of the model + num_heads: number of attention heads + ffn_dim: hidden dimension of the feed-forward network + dropout: dropout rate + """ + + def __init__(self, d_model: int, num_heads: int, ffn_dim: int, dropout: float = 0.1) -> None: + super().__init__() + self.self_attn = MultiHeadAttention(d_model, num_heads, dropout) + self.cross_attn = MultiHeadAttention(d_model, num_heads, dropout) + self.ffn = nn.Sequential( + nn.Linear(d_model, ffn_dim), nn.GELU(), nn.Dropout(dropout), nn.Linear(ffn_dim, d_model) + ) + self.self_attn_norm = nn.LayerNorm(d_model) + self.cross_attn_norm = nn.LayerNorm(d_model) + self.ffn_norm = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + def forward( + self, + x: torch.Tensor, + memory_kv: tuple[torch.Tensor, torch.Tensor], + cache: tuple[torch.Tensor, torch.Tensor] | None = None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]: + """Decode a sequence (causally), or the next tokens of a sequence given the cached keys / values + + Args: + x: the embedded tokens, of shape (N, L, D) + memory_kv: the keys and values of the image features (cf. `MultiHeadAttention.project_kv`) + cache: the keys and values of the previous tokens, None for the start of the sequence + + Returns: + the decoded tokens, and the keys and values of the sequence decoded so far + """ + h = self.self_attn_norm(x) + k, v = self.self_attn.project_kv(h) + if cache is not None: + k, v = torch.cat([cache[0], k], dim=2), torch.cat([cache[1], v], dim=2) + # With a cache, the new tokens attend to all the previous ones (decoding one token at a time) + x = x + self.dropout(self.self_attn(h, (k, v), is_causal=cache is None)) + x = x + self.dropout(self.cross_attn(self.cross_attn_norm(x), memory_kv)) + x = x + self.dropout(self.ffn(self.ffn_norm(x))) + return x, (k, v) + + +class FormulaNet(_FormulaNet, nn.Module): + """Implements a formula recognition model: a convolutional encoder (HGNetV2) whose last feature map is flattened + into a sequence, and a small autoregressive transformer decoder (MBart-style) predicting LaTeX tokens, as in + `"PP-FormulaNet" `_. + + Args: + feature_extractor: the backbone serving as feature extractor + vocab: the list of LaTeX tokens + d_model: dimension of the decoder + num_layers: number of decoder layers + num_heads: number of attention heads of the decoder + ffn_dim: hidden dimension of the feed-forward networks of the decoder + max_length: maximum number of tokens of a formula: size of the positional embeddings and maximum length of the + training labels + max_decode_length: maximum number of tokens generated by the greedy decoding (at most `max_length`). The + decoding stops once every formula of a batch is complete: this caps the time spent on a formula the model + fails to end (e.g. a repetition loop), which holds up its whole batch + dropout_prob: dropout probability of the decoder + feat_channels: number of channels of the feature map of the backbone + input_shape: input shape of the image + exportable: onnx exportable returns only logits + cfg: dictionary containing information about the model + confidence_aggregation: aggregation method of the token probabilities into the formula confidence + """ + + def __init__( + self, + feature_extractor: nn.Module, + vocab: Sequence[str], + d_model: int = 384, + num_layers: int = 4, + num_heads: int = 16, + ffn_dim: int = 1536, + max_length: int = 4096, + max_decode_length: int = 2048, + dropout_prob: float = 0.1, + feat_channels: int = 2048, + input_shape: tuple[int, int, int] = (3, 256, 768), + exportable: bool = False, + cfg: dict[str, Any] | None = None, + confidence_aggregation: ConfidenceAggregation = "mean", + ) -> None: + super().__init__() + self.vocab = list(vocab) + self.vocab_size = len(self.vocab) + self.max_length = max_length + self.max_decode_length = max_decode_length + self.d_model = d_model + self.exportable = exportable + self.cfg = cfg + # Special tokens, after the vocab + self.eos, self.sos, self.pad = self.vocab_size, self.vocab_size + 1, self.vocab_size + 2 + + self.feat_extractor = feature_extractor + self.feat_proj = nn.Linear(feat_channels, d_model) + self.feat_norm = nn.LayerNorm(d_model) + + self.embed = nn.Embedding(self.vocab_size + 3, d_model, padding_idx=self.pad) # +3 for EOS, SOS, PAD + self.pos_embed = nn.Embedding(max_length + 1, d_model) # SOS + tokens + self.embed_norm = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout_prob) + self.layers = nn.ModuleList([ + FormulaDecoderLayer(d_model, num_heads, ffn_dim, dropout_prob) for _ in range(num_layers) + ]) + self.norm = nn.LayerNorm(d_model) + self.head = nn.Linear(d_model, self.vocab_size + 1) # +1 for EOS + + self.postprocessor = FormulaNetPostProcessor(vocab=self.vocab, confidence_aggregation=confidence_aggregation) + + for n, m in self.named_modules(): + # Don't override the initialization of the backbone + if n.startswith("feat_extractor."): + continue + if isinstance(m, nn.Linear): + nn.init.trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Embedding): + nn.init.trunc_normal_(m.weight, std=0.02) + if m.padding_idx is not None: + m.weight.data[m.padding_idx].zero_() + + def from_pretrained(self, path_or_url: str, **kwargs: Any) -> None: + """Load pretrained parameters onto the model + + Args: + path_or_url: the path or URL to the model parameters (checkpoint) + **kwargs: additional arguments to be passed to `doctr.models.utils.load_pretrained_params` + """ + load_pretrained_params(self, path_or_url, **kwargs) + + def _decoder_layers(self) -> list[FormulaDecoderLayer]: + return cast(list[FormulaDecoderLayer], list(self.layers)) + + def encode(self, x: torch.Tensor) -> torch.Tensor: + """Encode images into a sequence of features, of shape (N, H / 32 * W / 32, d_model)""" + features = self.feat_extractor(x)["features"] # (N, C, H, W) + _, _, height, width = features.shape + memory = self.feat_proj(features.flatten(2).transpose(1, 2)) + memory = memory + _sincos_2d(height, width, self.d_model, memory.device, memory.dtype) + return self.feat_norm(memory) + + def _embed_tokens(self, tokens: torch.Tensor, start: int = 0) -> torch.Tensor: + positions = torch.arange(start, start + tokens.shape[1], device=tokens.device) + x = math.sqrt(self.d_model) * self.embed(tokens) + self.pos_embed(positions)[None] + return self.dropout(self.embed_norm(x)) + + def decode(self, tokens: torch.Tensor, memory: torch.Tensor) -> torch.Tensor: + """Decode full token sequences in parallel (teacher forcing) + + Args: + tokens: the input tokens, starting with SOS, of shape (N, L) + memory: the encoded images + + Returns: + the logits of the next token at each position, of shape (N, L, vocab_size + 1) + """ + x = self._embed_tokens(tokens) + for layer in self._decoder_layers(): + x, _ = layer(x, layer.cross_attn.project_kv(memory)) + return self.head(self.norm(x)) + + # The greedy decoding is not differentiable (argmax), and tracking the gradients of the growing key / value caches + # would take a memory quadratic in the number of tokens: the training relies on `decode` (teacher forcing) + @torch.no_grad() + def decode_greedy(self, memory: torch.Tensor, max_len: int | None = None) -> torch.Tensor: + """Generate token sequences greedily, one token at a time, caching the keys / values of the decoded tokens + + Args: + memory: the encoded images + max_len: maximum number of tokens to generate (`max_decode_length` by default), at most `max_length` + + Returns: + the logits of the generated tokens, of shape (N, L, vocab_size + 1) + """ + steps = min(self.max_decode_length if max_len is None else max_len, self.max_length) + 1 # tokens + EOS + memory_kv = [layer.cross_attn.project_kv(memory) for layer in self._decoder_layers()] + caches: list[tuple[torch.Tensor, torch.Tensor] | None] = [None] * len(self.layers) + token = torch.full((memory.shape[0], 1), self.sos, dtype=torch.long, device=memory.device) + done = torch.zeros(memory.shape[0], dtype=torch.bool, device=memory.device) + logits = [] + for step in range(steps): + x = self._embed_tokens(token, start=step) + for idx, layer in enumerate(self.layers): + x, caches[idx] = layer(x, memory_kv[idx], caches[idx]) + step_logits = self.head(self.norm(x)) + logits.append(step_logits) + token = step_logits.argmax(-1) + # Stop once every sequence has produced EOS + # NOTE: `break` isn't correctly translated to ONNX so the export always runs every step + if not self.exportable: + done |= token[:, 0] == self.eos + if bool(done.all()): + break + return torch.cat(logits, dim=1) + + def forward( + self, + x: torch.Tensor, + target: list[str] | None = None, + return_model_output: bool = False, + return_preds: bool = False, + ) -> dict[str, Any]: + memory = self.encode(x) + + if self.training and target is None: + raise ValueError("Need to provide labels during training") + + loss = None + if target is not None: + # Teacher forcing: predict each token from the previous ones + _gt, _seq_len = self.build_target(target) + gt = torch.from_numpy(_gt).to(dtype=torch.long, device=x.device) + gt = gt[:, : max(_seq_len, default=0) + 2] # SOS + longest sequence + EOS + logits = self.decode(gt[:, :-1], memory) + loss = F.cross_entropy(logits.flatten(end_dim=1), gt[:, 1:].flatten(), ignore_index=self.pad) + + if target is None or return_preds: + logits = self.decode_greedy(memory) + + logits = _bf16_to_float32(logits) + + out: dict[str, Any] = {} + if self.exportable: + out["logits"] = logits + return out + + if return_model_output: + out["out_map"] = logits + + if target is None or return_preds: + # Disable for torch.compile compatibility + @torch.compiler.disable + def _postprocess(logits: torch.Tensor) -> list[tuple[str, float]]: + return self.postprocessor(logits) + + out["preds"] = _postprocess(logits) + + if loss is not None: + out["loss"] = loss + + return out + + +class FormulaNetPostProcessor(_FormulaNetPostProcessor): + """Post processor for FormulaNet architecture + + Args: + vocab: the list of LaTeX tokens of the model + confidence_aggregation: aggregation method of the token probabilities into the formula confidence + """ + + def __call__(self, logits: torch.Tensor) -> list[tuple[str, float]]: + out_idxs = logits.argmax(-1).detach().cpu().numpy() + probs = torch.softmax(logits, -1).max(dim=-1)[0].detach().float().cpu().numpy() + eos = len(self.vocab) + + results: list[tuple[str, float]] = [] + for idxs, token_probs in zip(out_idxs, probs): + ends = np.flatnonzero(idxs == eos) + length = int(ends[0]) if ends.size > 0 else len(idxs) + latex = " ".join(self.vocab[idx] for idx in idxs[:length]) + results.append((latex, aggregate_confidence(token_probs[:length], self.confidence_aggregation))) + return results + + +def _formulanet( + arch: str, + pretrained: bool, + backbone_fn: Callable[..., nn.Module], + layer: str, + pretrained_backbone: bool = True, + ignore_keys: list[str] | None = None, + **kwargs: Any, +) -> FormulaNet: + pretrained_backbone = pretrained_backbone and not pretrained + + # Patch the config + _cfg = deepcopy(default_cfgs[arch]) + _cfg["vocab"] = list(kwargs.get("vocab", _cfg["vocab"])) + _cfg["input_shape"] = kwargs.get("input_shape", _cfg["input_shape"]) + + kwargs["vocab"] = _cfg["vocab"] + kwargs["input_shape"] = _cfg["input_shape"] + + # Feature extractor: the classification model (head included, to load its pretrained weights as they are), of which + # only the layers up to `layer` are kept + backbone = backbone_fn(pretrained_backbone) + feat_extractor = IntermediateLayerGetter(backbone, {layer: "features"}) + + # Build the model + feat_channels = cast(list[int], backbone.channels)[-1] + model = FormulaNet(feat_extractor, feat_channels=feat_channels, cfg=_cfg, **kwargs) + # Load pretrained parameters + if pretrained: + # The number of tokens is not the same as the number of tokens of the pretrained model => + # remove the embedding and the last layer weights + _ignore_keys = ignore_keys if _cfg["vocab"] != default_cfgs[arch]["vocab"] else None + model.from_pretrained(default_cfgs[arch]["url"], ignore_keys=_ignore_keys) + + return model + + +def formulanet(pretrained: bool = False, **kwargs: Any) -> FormulaNet: + """FormulaNet: a HGNetV2-B4 encoder with a transformer decoder predicting LaTeX tokens, following + `"PP-FormulaNet: Bridging Accuracy and Efficiency in Advanced Formula Recognition" + `_. + + >>> import torch + >>> from doctr.models import formulanet + >>> model = formulanet(pretrained=False) + >>> input_tensor = torch.rand((1, 3, 256, 768)) + >>> out = model(input_tensor) + + Args: + pretrained: If True, returns a model pre-trained on a formula recognition dataset + **kwargs: keyword arguments of the FormulaNet architecture + + Returns: + formula recognition architecture + """ + return _formulanet( + "formulanet", + pretrained, + hgnetv2_b4, + "4", + ignore_keys=["embed.weight", "head.weight", "head.bias"], + **kwargs, + ) diff --git a/doctr/models/formula/predictor/__init__.py b/doctr/models/formula/predictor/__init__.py new file mode 100644 index 0000000000..e3c861310c --- /dev/null +++ b/doctr/models/formula/predictor/__init__.py @@ -0,0 +1 @@ +from .pytorch import * diff --git a/doctr/models/formula/predictor/pytorch.py b/doctr/models/formula/predictor/pytorch.py new file mode 100644 index 0000000000..a31b122a31 --- /dev/null +++ b/doctr/models/formula/predictor/pytorch.py @@ -0,0 +1,132 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +from collections.abc import Iterable, Sequence +from typing import Any + +import numpy as np +import torch +from torch import nn + +from doctr.models._utils import predict_by_chunks +from doctr.models.preprocessor import PreProcessor +from doctr.models.utils import set_device_and_dtype +from doctr.utils.geometry import extract_crops, extract_rcrops + +__all__ = ["FormulaPredictor"] + + +def _crop_region(page: np.ndarray, region: np.ndarray) -> np.ndarray | None: + """Crop a page-relative region, a straight box (4,) or a polygon (4, 2), None for a degenerate region""" + region = np.asarray(region, dtype=np.float32) + if region.ndim == 1: + crop = extract_crops(page, np.clip(region, 0, 1)[None])[0] + else: + crop = extract_rcrops(page, np.clip(region, 0, 1)[None], assume_horizontal=False)[0] + return crop if crop.shape[0] >= 2 and crop.shape[1] >= 2 else None + + +class FormulaPredictor(nn.Module): + """Implements an object able to recognize mathematical formulas as LaTeX strings + + Each formula image is resized (keeping its aspect ratio), padded and normalized. The margins around a formula + (e.g. of a layout region) are kept: the model is trained to read formulas with margins. + + Args: + pre_processor: transform inputs for easier batched model inference + model: core formula recognition architecture + chunk_size: the maximum number of formula images decoded and processed at once (128 by default). Longer + inputs, lazy page sources (cf. `DocumentFile.from_images(..., lazy=True)`) and iterators are processed + chunk by chunk, with a memory footprint that does not grow with the number of images. It is rounded + up to a multiple of the batch size, so that the predictions are the same as in a single pass. + """ + + def __init__( + self, + pre_processor: PreProcessor, + model: nn.Module, + chunk_size: int = 128, + ) -> None: + super().__init__() + self.pre_processor = pre_processor + self.model = model.eval() + if chunk_size < 1: + raise ValueError("`chunk_size` must be a positive integer") + self.chunk_size = chunk_size + + @torch.inference_mode() + def forward( + self, + crops: Sequence[np.ndarray] | Iterable[np.ndarray], + **kwargs: Any, + ) -> list[tuple[str, float]]: + """Recognize the formula of each image + + Args: + crops: a list of formula images (numpy arrays of shape H x W x C), a lazy page source + (cf. `DocumentFile.from_images(..., lazy=True)`) or any iterable of images + **kwargs: keyword arguments forwarded to the model + + Returns: + the LaTeX string (tokens separated by spaces) and the confidence of each image + """ + return predict_by_chunks( + lambda chunk: self._predict(chunk, **kwargs), crops, self.chunk_size, self.pre_processor.batch_size + ) + + @torch.inference_mode() + def _predict(self, crops: Sequence[np.ndarray], **kwargs: Any) -> list[tuple[str, float]]: + if len(crops) == 0: + return [] + # Dimension check + if any(crop.ndim != 3 for crop in crops): + raise ValueError("incorrect input shape: all crops are expected to be multi-channel 2D images.") + empty = next((crop for crop in crops if not all(s > 0 for s in crop.shape)), None) + if empty is not None: + raise ValueError(f"incorrect input shape: crops are expected to be non-empty, got shape {empty.shape}.") + + processed_batches = self.pre_processor(list(crops)) + _params = next(self.model.parameters()) + self.model, processed_batches = set_device_and_dtype( # type: ignore[assignment] + self.model, processed_batches, _params.device, _params.dtype + ) + raw = [self.model(batch, return_preds=True, **kwargs)["preds"] for batch in processed_batches] + return [pred for batch in raw for pred in batch] + + @torch.inference_mode() + def predict_regions( + self, + pages: list[np.ndarray], + regions: list[dict[str, Any] | None], + class_name: str = "Formula", + **kwargs: Any, + ) -> list[list[dict[str, Any]]]: + """Recognize the formulas a layout model located on pages + + Args: + pages: the page images + regions: the per-page layout predictions `{"class_names", "boxes", "scores"}` (None for none) + class_name: the layout class of the formula regions + **kwargs: keyword arguments forwarded to the model + + Returns: + a per-page list of formulas `{"geometry", "value", "confidence", "score"}`, with the geometry of the + layout region (page-relative) and its layout score + """ + crops: list[np.ndarray] = [] + meta: list[tuple[int, np.ndarray, float | None]] = [] + for p_idx, (page, region) in enumerate(zip(pages, regions)): + region = region or {} + scores = region.get("scores") + for r_idx, (cls_name, box) in enumerate(zip(region.get("class_names", []), region.get("boxes", []))): + crop = _crop_region(page, box) if cls_name == class_name else None + if crop is not None: + crops.append(crop) + meta.append((p_idx, np.asarray(box), None if scores is None else float(scores[r_idx]))) + + formulas: list[list[dict[str, Any]]] = [[] for _ in pages] + for (p_idx, box, score), (latex, confidence) in zip(meta, self(crops, **kwargs) if crops else []): + formulas[p_idx].append({"geometry": box, "value": latex, "confidence": confidence, "score": score}) + return formulas diff --git a/doctr/models/formula/zoo.py b/doctr/models/formula/zoo.py new file mode 100644 index 0000000000..be555e469b --- /dev/null +++ b/doctr/models/formula/zoo.py @@ -0,0 +1,102 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +from typing import Any + +from doctr.models.utils import _CompiledModule + +from .. import formula +from .._utils import ConfidenceAggregation, _resolve_confidence_aggregation +from ..preprocessor import PreProcessor +from .predictor import FormulaPredictor + +__all__ = ["formula_predictor"] + +ARCHS: list[str] = ["formulanet"] + + +def _predictor( + arch: Any, + pretrained: bool, + confidence_aggregation: ConfidenceAggregation | None = None, + max_decode_length: int | None = None, + **kwargs: Any, +) -> FormulaPredictor: + if confidence_aggregation is not None: + _resolve_confidence_aggregation(confidence_aggregation) + + if isinstance(arch, str): + if arch not in ARCHS: + raise ValueError(f"unknown architecture '{arch}'") + _model = formula.__dict__[arch]( + pretrained=pretrained, pretrained_backbone=kwargs.get("pretrained_backbone", True) + ) + else: + # Adding the type for torch compiled models to the allowed architectures + allowed_archs = [formula.FormulaNet, _CompiledModule] + if not isinstance(arch, tuple(allowed_archs)): + raise ValueError(f"unknown architecture: {type(arch)}") + _model = arch + + if confidence_aggregation is not None: + _model.postprocessor.confidence_aggregation = confidence_aggregation + if max_decode_length is not None: + _model.max_decode_length = max_decode_length + + kwargs.pop("pretrained_backbone", None) + kwargs["mean"] = kwargs.get("mean", _model.cfg["mean"]) + kwargs["std"] = kwargs.get("std", _model.cfg["std"]) + kwargs["batch_size"] = kwargs.get("batch_size", 16) + chunk_size = kwargs.pop("chunk_size", 128) + # Same resizing as during training: aspect ratio preserved, padded at the bottom-right + kwargs.setdefault("symmetric_pad", False) + predictor = FormulaPredictor( + PreProcessor(_model.cfg["input_shape"][1:], preserve_aspect_ratio=True, **kwargs), + _model, + chunk_size=chunk_size, + ) + return predictor + + +def formula_predictor( + arch: Any = "formulanet", + pretrained: bool = False, + batch_size: int = 16, + confidence_aggregation: ConfidenceAggregation | None = None, + max_decode_length: int | None = None, + **kwargs: Any, +) -> FormulaPredictor: + """Formula recognition architecture: turns images of mathematical formulas into LaTeX strings. + + >>> import numpy as np + >>> from doctr.models import formula_predictor + >>> model = formula_predictor(arch='formulanet', pretrained=True) + >>> input_crop = (255 * np.random.rand(64, 256, 3)).astype(np.uint8) + >>> out = model([input_crop]) + + Args: + arch: name of the architecture or model itself to use (e.g. 'formulanet') + pretrained: If True, returns a model pre-trained on a formula recognition dataset + batch_size: number of samples the model processes in parallel + confidence_aggregation: aggregation method of the token probabilities into the formula confidence: + "mean", "min", "max", "median", "geometric_mean", "harmonic_mean" or a callable. If None, the one of + the model is kept ("mean" by default) + max_decode_length: maximum number of tokens generated per formula. If None, the one of the model is kept + (2048 by default). A model instance passed as `arch` is modified. + chunk_size: maximum number of formula images decoded and processed at once (128 by default): longer inputs and + lazy page sources (cf. `DocumentFile.from_images(..., lazy=True)`) are processed chunk by chunk + **kwargs: optional keyword arguments passed to the architecture + + Returns: + Formula recognition predictor + """ + return _predictor( + arch=arch, + pretrained=pretrained, + batch_size=batch_size, + confidence_aggregation=confidence_aggregation, + max_decode_length=max_decode_length, + **kwargs, + ) diff --git a/doctr/models/predictor/base.py b/doctr/models/predictor/base.py index 9696fc87f9..5692b63bea 100644 --- a/doctr/models/predictor/base.py +++ b/doctr/models/predictor/base.py @@ -244,13 +244,22 @@ def _get_orientations( ] return general_pages_orientations, origin_page_orientations + def _masked_classes(self) -> set[str]: + """The layout classes masked out before text detection: the ignored regions, and the formula regions when + they are read by a formula model""" + classes = set(self.ignore_regions or ()) + if getattr(self, "formula_predictor", None) is not None: + classes.add(getattr(self, "formula_class_name", "Formula")) + return classes + def _mask_regions( self, pages: list[np.ndarray], regions: list[dict[str, Any]] | None, fill_value: int = 0, ) -> list[np.ndarray]: - if regions is None or not self.ignore_regions: + masked_classes = self._masked_classes() + if regions is None or not masked_classes: return pages out_pages: list[np.ndarray] = [] @@ -259,7 +268,7 @@ def _mask_regions( out_pages.append(page) continue boxes = np.asarray(region["boxes"], dtype=np.float32) - keep = np.array([name in self.ignore_regions for name in region["class_names"]], dtype=bool) + keep = np.array([name in masked_classes for name in region["class_names"]], dtype=bool) out_pages.append(mask_boxes(page, boxes[keep], fill_value=fill_value) if keep.any() else page) return out_pages @@ -440,6 +449,8 @@ def _remap_to_original_coords( self._remap_geometry(table, m_inv, sw, sh, oh, ow) for cell in table.cells: self._remap_geometry(cell, m_inv, sw, sh, oh, ow) + for formula in page.formulas: + self._remap_geometry(formula, m_inv, sw, sh, oh, ow) if orig_pages is not None: out.pages[pidx].page = orig_pages[pidx] diff --git a/doctr/models/predictor/pytorch.py b/doctr/models/predictor/pytorch.py index 027f28c183..5abf54ac95 100644 --- a/doctr/models/predictor/pytorch.py +++ b/doctr/models/predictor/pytorch.py @@ -13,6 +13,7 @@ from doctr.io.elements import Document from doctr.models._utils import get_language from doctr.models.detection.predictor import DetectionPredictor +from doctr.models.formula.predictor import FormulaPredictor from doctr.models.layout.predictor import LayoutPredictor from doctr.models.recognition.predictor import RecognitionPredictor from doctr.models.table_structure.predictor import TablePredictor @@ -44,6 +45,8 @@ class OCRPredictor(nn.Module, _OCRPredictor): table_predictor: optional table structure recognition module. Requires `layout_predictor`: table regions are located by the layout model, cropped, and passed to this module. Words falling inside a detected table are regrouped into a structured table and removed from the regular text output. + formula_predictor: optional formula recognition module. Requires `layout_predictor`: formula regions are + located by the layout model, masked out of the text detection, cropped, and read as LaTeX by this module. page_chunk_size: the maximum number of pages processed at once (32 by default) for lazy page sources (cf. `DocumentFile.from_pdf(..., lazy=True)`), iterators and :meth:`stream`, with a memory footprint that does not grow with the number of pages. Lists and tuples of pages are processed in a single pass. @@ -63,6 +66,7 @@ def __init__( preserve_original_coords: bool = False, layout_predictor: LayoutPredictor | None = None, table_predictor: TablePredictor | None = None, + formula_predictor: FormulaPredictor | None = None, **kwargs: Any, ) -> None: nn.Module.__init__(self) @@ -89,6 +93,14 @@ def __init__( "`table_predictor` requires a `layout_predictor`: tables are located with the layout model, " "cropped, and then passed to the table model." ) + self.formula_predictor = formula_predictor.eval() if formula_predictor is not None else None + # Layout class label whose regions are read by the formula model (and masked out of the text detection) + self.formula_class_name = "Formula" + if self.formula_predictor is not None and self.layout_predictor is None: + raise ValueError( + "`formula_predictor` requires a `layout_predictor`: formulas are located with the layout model, " + "cropped, and then passed to the formula model." + ) @torch.inference_mode() def forward( @@ -214,6 +226,12 @@ def _predict_pages( if self.table_predictor is not None and regions is not None else None ) + # Recognize formulas: read the formula regions of the layout model as LaTeX + formulas = ( + self.formula_predictor.predict_regions(pages, regions, self.formula_class_name, **kwargs) + if self.formula_predictor is not None and regions is not None + else None + ) out = self.doc_builder( pages, @@ -226,6 +244,7 @@ def _predict_pages( languages_dict, regions, tables, + formulas, ) if self.straighten_pages and self.preserve_original_coords: diff --git a/doctr/models/zoo.py b/doctr/models/zoo.py index 85cad31243..d6ae1569c5 100644 --- a/doctr/models/zoo.py +++ b/doctr/models/zoo.py @@ -7,6 +7,7 @@ from ._utils import ConfidenceAggregation, _resolve_confidence_aggregation from .detection.zoo import detection_predictor +from .formula.zoo import formula_predictor from .kie_predictor import KIEPredictor from .layout.zoo import layout_predictor from .predictor import OCRPredictor @@ -33,6 +34,7 @@ def _predictor( layout_arch: Any = "lw_detr_s", ignore_regions: list[str] | None = None, detect_tables: bool = False, + detect_formulas: bool = False, confidence_aggregation: ConfidenceAggregation | None = None, **kwargs, ) -> OCRPredictor: @@ -60,7 +62,7 @@ def _predictor( confidence_aggregation=confidence_aggregation, ) - # Layout - required for table detection, so build it whenever layout or tables are requested + # Layout - required for table and formula recognition, so build it whenever one of them is requested layout_pred = ( layout_predictor( layout_arch, @@ -70,7 +72,7 @@ def _predictor( symmetric_pad=symmetric_pad, batch_size=det_bs, ) - if (detect_layout or detect_tables) + if (detect_layout or detect_tables or detect_formulas) else None ) @@ -88,6 +90,17 @@ def _predictor( else None ) + # Formula recognition - optional, applied on the cropped formula regions found by the layout model + formula_pred = ( + formula_predictor( + "formulanet", + pretrained=pretrained, + pretrained_backbone=pretrained_backbone, + ) + if detect_formulas + else None + ) + return OCRPredictor( det_predictor, reco_predictor, @@ -99,6 +112,7 @@ def _predictor( detect_language=detect_language, layout_predictor=layout_pred, table_predictor=table_pred, + formula_predictor=formula_pred, ignore_regions=ignore_regions, **kwargs, ) @@ -120,6 +134,7 @@ def ocr_predictor( layout_arch: Any = "lw_detr_s", ignore_regions: list[str] | None = None, detect_tables: bool = False, + detect_formulas: bool = False, confidence_aggregation: ConfidenceAggregation | None = None, **kwargs: Any, ) -> OCRPredictor: @@ -164,6 +179,10 @@ def ocr_predictor( structure model. Words falling inside a detected table are regrouped into a structured table (accessible via `page.tables`) and removed from the regular text output. This enables the layout model and slightly deteriorates the overall latency. + detect_formulas: if True, formula regions found by the layout model are masked out of the text detection, + cropped and read as LaTeX by a formula recognition model (accessible via `page.formulas`, and exported as + display math, e.g. `$$...$$` in Markdown). This enables the layout model and deteriorates the overall + latency. confidence_aggregation: aggregation method of the character probabilities of the recognition model into the word confidence: "mean", "min", "max", "median", "geometric_mean", "harmonic_mean" or a Python callable taking the 1D array of probabilities and returning a float. If None (default), the method of the @@ -190,6 +209,7 @@ def ocr_predictor( detect_layout=detect_layout, layout_arch=layout_arch, detect_tables=detect_tables, + detect_formulas=detect_formulas, ignore_regions=ignore_regions, confidence_aggregation=confidence_aggregation, **kwargs, diff --git a/doctr/transforms/modules/pytorch.py b/doctr/transforms/modules/pytorch.py index 3aa103a97d..39f649ef20 100644 --- a/doctr/transforms/modules/pytorch.py +++ b/doctr/transforms/modules/pytorch.py @@ -24,6 +24,7 @@ "RandomHorizontalFlip", "RandomShadow", "RandomResize", + "RandomPad", "GaussianBlur", "RandomGlare", "RandomLighting", @@ -455,6 +456,77 @@ def extra_repr(self) -> str: return f"scale_range={self.scale_range}, preserve_aspect_ratio={self.preserve_aspect_ratio}, symmetric_pad={self.symmetric_pad}, p={self.p}" # noqa: E501 +class RandomPad(torch.nn.Module): + """Randomly pad the input image with its background color and align the corresponding targets + + Each side gets an independent padding, filled with the background color of the image (the median color of its + border pixels: white for a rendered document, the paper color for a photo). Used for instance to teach a model to + read the crops of layout regions with their margins. + + >>> import torch + >>> from doctr.transforms import RandomPad + >>> from doctr.utils import Sample + >>> transfo = RandomPad(max_ratio=0.5, p=0.5) + >>> out = transfo(Sample(image=torch.rand((3, 64, 256)))) + + Args: + max_ratio: maximum padding of each side, relative to the image height + p: probability to apply the transformation + """ + + def __init__(self, max_ratio: float = 0.5, p: float = 0.5) -> None: + super().__init__() + if max_ratio < 0: + raise ValueError("`max_ratio` must be non-negative") + self.max_ratio = max_ratio + self.p = p + + @staticmethod + def _pad_target(target: np.ndarray, offset: tuple[int, int], size: tuple[int, int], new_size: tuple[int, int]): + """Map relative boxes (N, 4) or polygons (N, 4, 2) of the image to the padded image""" + if not ((target.ndim == 2 and target.shape[1] == 4) or (target.ndim == 3 and target.shape[1:] == (4, 2))): + raise ValueError(f"expected relative boxes (N, 4) or polygons (N, 4, 2), got shape {target.shape}") + (left, top), (h, w), (new_h, new_w) = offset, size, new_size + out = target.astype(np.float32, copy=True) + xs, ys = (out[..., [0, 2]], out[..., [1, 3]]) if out.ndim == 2 else (out[..., 0], out[..., 1]) + new_xs, new_ys = (left + xs * w) / new_w, (top + ys * h) / new_h + if out.ndim == 2: + out[..., [0, 2]], out[..., [1, 3]] = new_xs, new_ys + else: + out[..., 0], out[..., 1] = new_xs, new_ys + return out + + def forward(self, sample: Sample) -> Sample: + if torch.rand(1) >= self.p: + return sample + img = sample.image # (C, H, W) + _, h, w = img.shape + top, bottom, left, right = (torch.rand(4) * self.max_ratio * h).long().tolist() + new_h, new_w = top + h + bottom, left + w + right + # Background color: the median color of the border pixels + border = torch.cat([img[:, 0], img[:, -1], img[:, :, 0], img[:, :, -1]], dim=1) + background = border.median(dim=1).values + out = background[:, None, None].repeat(1, new_h, new_w) + out[:, top : top + h, left : left + w] = img + + target = sample.target + if isinstance(target, np.ndarray): + target = self._pad_target(target, (left, top), (h, w), (new_h, new_w)) + elif isinstance(target, dict): + target = { + key: self._pad_target(value, (left, top), (h, w), (new_h, new_w)) + if isinstance(value, np.ndarray) + else value + for key, value in target.items() + } + # The padded areas are not valid image content + mask = None if sample.mask is None else pad(sample.mask, (left, right, top, bottom), value=False) + return sample.replace(image=out, target=target, mask=mask) + + def extra_repr(self) -> str: + return f"max_ratio={self.max_ratio}, p={self.p}" + + def _apply_float_photometric(sample: Sample, fn) -> Sample: """Run a photometric function expecting a float image in [0, 1] on a float or uint8 sample.""" if sample.image.dtype == torch.uint8: diff --git a/doctr/utils/metrics.py b/doctr/utils/metrics.py index 5f4ea29cc0..7ac32e22dc 100644 --- a/doctr/utils/metrics.py +++ b/doctr/utils/metrics.py @@ -4,8 +4,11 @@ # See LICENSE or go to for full license details. +from collections import Counter + import numpy as np from anyascii import anyascii +from rapidfuzz.distance import Levenshtein from scipy.optimize import linear_sum_assignment from shapely import STRtree, area, convex_hull, intersection, is_valid, polygons @@ -19,6 +22,7 @@ "DetectionMetric", "ObjectDetectionMetric", "TableCellMetric", + "FormulaMetric", ] @@ -417,6 +421,102 @@ def reset(self) -> None: self.struct_matches = 0 +class FormulaMetric: + r"""Implements formula recognition metrics, computed on LaTeX tokens (cf. :func:`doctr.datasets.tokenize_latex`), + so they do not depend on the whitespaces of the LaTeX sources. + + It reports the **exact match** rate (the share of formulas whose token sequences are identical), the **edit + similarity** (the mean over formulas of :math:`1 - \frac{d(X, Y)}{\max(|X|, |Y|)}` with :math:`d` the + token-level Levenshtein distance) and the corpus-level **BLEU-4** score (geometric mean of the 1- to 4-gram + precisions with a brevity penalty, without smoothing). + + >>> from doctr.utils import FormulaMetric + >>> metric = FormulaMetric() + >>> metric.update([r"\frac{a}{b}"], [r"\frac { a } { c }"]) + >>> metric.summary() + """ + + max_order: int = 4 + # Scalar accumulators, which can be summed over several processes + counters: tuple[str, ...] = ( + "exact", + "edit_similarity", + "total", + "pred_len", + "gt_len", + *(f"ngram_matches_{n}" for n in range(1, max_order + 1)), + *(f"ngram_totals_{n}" for n in range(1, max_order + 1)), + ) + + def __init__(self) -> None: + self.reset() + + def update(self, gt: list[str], pred: list[str]) -> None: + """Update the state of the metric with new predictions + + Args: + gt: list of ground-truth LaTeX strings + pred: list of predicted LaTeX strings + """ + from doctr.datasets import tokenize_latex + + if len(gt) != len(pred): + raise AssertionError("prediction size does not match with ground-truth labels size") + + for gt_latex, pred_latex in zip(gt, pred): + gt_tokens, pred_tokens = tokenize_latex(gt_latex), tokenize_latex(pred_latex) + self.exact += int(gt_tokens == pred_tokens) + length = max(len(gt_tokens), len(pred_tokens)) + self.edit_similarity += 1 - Levenshtein.distance(gt_tokens, pred_tokens) / length if length > 0 else 1.0 + self.pred_len += len(pred_tokens) + self.gt_len += len(gt_tokens) + for n in range(1, self.max_order + 1): + gt_ngrams = Counter(tuple(gt_tokens[i : i + n]) for i in range(len(gt_tokens) - n + 1)) + pred_ngrams = Counter(tuple(pred_tokens[i : i + n]) for i in range(len(pred_tokens) - n + 1)) + matches = sum((gt_ngrams & pred_ngrams).values()) + setattr(self, f"ngram_matches_{n}", getattr(self, f"ngram_matches_{n}") + matches) + setattr( + self, f"ngram_totals_{n}", getattr(self, f"ngram_totals_{n}") + max(len(pred_tokens) - n + 1, 0) + ) + self.total += len(gt) + + def summary(self) -> dict[str, float]: + """Computes the aggregated metrics + + Returns: + a dictionary with the `exact_match` rate, the mean `edit_similarity` and the corpus `bleu` score + """ + if self.total == 0: + raise AssertionError("you need to update the metric before getting the summary") + + precisions = [ + getattr(self, f"ngram_matches_{n}") / getattr(self, f"ngram_totals_{n}") + if getattr(self, f"ngram_totals_{n}") > 0 + else 0.0 + for n in range(1, self.max_order + 1) + ] + if min(precisions) > 0 and self.pred_len > 0: + brevity_penalty = 1.0 if self.pred_len > self.gt_len else float(np.exp(1 - self.gt_len / self.pred_len)) + bleu = brevity_penalty * float(np.exp(np.mean(np.log(precisions)))) + else: + bleu = 0.0 + return dict( + exact_match=self.exact / self.total, + edit_similarity=self.edit_similarity / self.total, + bleu=bleu, + ) + + def reset(self) -> None: + self.exact = 0 + self.edit_similarity = 0.0 + self.total = 0 + self.pred_len = 0 + self.gt_len = 0 + for n in range(1, self.max_order + 1): + setattr(self, f"ngram_matches_{n}", 0) + setattr(self, f"ngram_totals_{n}", 0) + + class OCRMetric: r"""Implements an end-to-end OCR metric. diff --git a/references/formula/README.md b/references/formula/README.md new file mode 100644 index 0000000000..e8de8c865e --- /dev/null +++ b/references/formula/README.md @@ -0,0 +1,184 @@ +# Formula recognition + +The sample scripts in this folder let you train, evaluate and benchmark formula recognition models with docTR. +A formula recognition model reads the image of a mathematical formula and predicts its LaTeX source, token by token. + +## Setup + +First, you need to install `doctr` (with pip, for instance) + +```shell +pip install -e . --upgrade +pip install -r references/requirements.txt +``` + +## Usage + +You can start your training in PyTorch: + +```shell +python references/formula/train.py formulanet --train_path path/to/your/train_set --val_path path/to/your/val_set --epochs 5 +``` + +To try the pipeline end-to-end on a small toy dataset (32 formulas of UniMER-1M and UniMER-Test): + +```shell +wget https://github.com/mindee/doctr/releases/download/v1.1.0/toy_formula_set-8f54029b.zip +unzip toy_formula_set-8f54029b.zip -d formula_set +python references/formula/train.py formulanet --train_path ./formula_set --val_path ./formula_set -b 2 --epochs 1 +``` + +`--input_size H W` sets the input size of the model (`256 768` by default): each formula is resized to fit it while +keeping its aspect ratio, and padded. The training adds random margins around half of the formulas +(`doctr.transforms.RandomPad`: each side up to half the image height, in the background color of the image), so that the +model reads the crops of layout regions as they are: the formulas are not trimmed, neither at training nor at inference. + +`--max_length` sets the maximum number of tokens of a formula (`4096` by default, enough for the longest UniMER-1M +label). + +Long formulas are memory-hungry: a batch is padded to its longest formula. `--max-tokens` bounds the number of padded +target tokens per forward pass (`32768` by default, about 8.5 GB of GPU memory with `--amp-dtype bfloat16` and the +default batch size): a batch above it is split into chunks of formulas of similar length, whose gradients are +accumulated, so the update is the same as for the whole batch. Lower it if you run out of memory. + +The validation decodes each formula greedily until its end token, or up to `--max_decode_length` tokens (`2048` by +default, enough for 99.99% of the UniMER-Test labels). The decoding of a batch lasts until its longest formula ends, so +early in a training, before the model learns to stop, a validation batch of 32 formulas can take about 3.5 s on an +RTX 4080. + +### Multi-GPU support + +We now use the built-in [`torchrun`](https://pytorch.org/docs/stable/elastic/run.html) launcher to spawn your DDP workers. `torchrun` will set all the necessary environment variables (`LOCAL_RANK`, `RANK`, etc.) for you. Arguments are the same than the ones from single GPU, except: + +- `--backend`: you can specify another `backend` for `DistributedDataParallel` if the default one is not available on +your operating system. Fastest one is `nccl` according to [PyTorch Documentation](https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html). + +#### Key `torchrun` parameters + +- `--nproc_per_node=` + Spawn `` processes on the local machine (typically equal to the number of GPUs you want to use). +- `--nnodes=` + (Optional) Total number of nodes in your job. Default is 1. +- `--rdzv_backend`, `--rdzv_endpoint`, `--rdzv_id` + (Optional) Rendezvous settings for multi-node jobs. See the [torchrun docs](https://pytorch.org/docs/stable/elastic/run.html) for details. + +#### GPU selection + +By default all visible GPUs will be used. To limit which GPUs participate, set the `CUDA_VISIBLE_DEVICES` environment variable **before** running `torchrun`. For example, to use only CUDA devices 0 and 2: + +```shell +CUDA_VISIBLE_DEVICES=0,2 \ +torchrun --nproc_per_node=2 references/formula/train.py \ + formulanet \ + --train_path path/to/train \ + --val_path path/to/val \ + --epochs 5 \ + --backend nccl +``` + +## Device and mixed precision + +Every training and evaluation script accepts `--device`: a CUDA index (`0`), `cuda:N`, `mps` (Apple Silicon GPU) or `cpu`. Without it the script picks CUDA if available, then MPS, then CPU. In distributed mode (`torchrun`) the argument is ignored and each process uses its own GPU. + +`--amp` enables automatic mixed precision and is only supported on CUDA. `--amp-dtype bfloat16` (Ampere or newer GPUs) uses bfloat16 instead of float16: it has the range of float32, so it needs no loss scaling and avoids the overflows float16 can produce in some losses. + +```shell +# NVIDIA GPU with bfloat16 mixed precision +python references/formula/train.py formulanet --train_path path/to/train --val_path path/to/val --epochs 5 --device 0 --amp --amp-dtype bfloat16 +``` + +## Checkpoints + +Each run writes its metadata once, as `.json` next to the checkpoints it saves. It holds what is needed to rebuild the model for inference and to reproduce the run: the architecture and its task settings (`vocab`, `input_shape`, `max_length`, `max_decode_length`), the dataset hashes, the docTR / PyTorch versions, the git revision and the full list of arguments. + +## Evaluation + +You can evaluate a model (the pretrained one by default, or your own checkpoint with `--resume`) on a dataset: + +```shell +python references/formula/evaluate.py formulanet path/to/your/dataset --resume path/to/your/checkpoint.pt +``` + +The script reports, with `doctr.utils.metrics.FormulaMetric`, on the LaTeX tokens (so the whitespaces of the labels do not matter): + +- the **exact match** rate: the share of formulas whose tokens are all correct; +- the **edit similarity**: one minus the token-level edit distance, relative to the longest of the two sequences, averaged over the formulas; +- the corpus-level **BLEU-4** score. + +To compare with published results, evaluate each subset of [UniMER-Test](https://huggingface.co/datasets/wanderkid/UniMER_Dataset) +separately (`spe`, `cpe`, `sce`, `hwe`). + +## Latency benchmark + +You can measure the inference latency of an architecture: + +```shell +python references/formula/latency.py formulanet --it 100 --size 256 768 --tokens 128 --gpu +``` + +The decoding time grows with the length of the formulas: `--tokens` sets the number of tokens decoded per formula. + +## Data format + +You need to provide both `train_path` and `val_path` arguments to start training (`evaluate.py` takes a single `dataset_path`). +Each path must lead to a folder with 1 subfolder and 1 file (`train_path` can also lead to a folder of such folders, which are merged): + +```shell +├── images +│ ├── sample_img_01.png +│ ├── sample_img_02.png +│ ├── sample_img_03.png +│ └── ... +└── labels.json +``` + +`labels.json` maps each image file name to the LaTeX source of its formula, without math delimiters (`$`, `$$`, `\[`): + +```shell +{ + "sample_img_01.png": "\\frac{a}{b^{2}}", + "sample_img_02.png": "\\sum _ { i = 1 } ^ { n } x _ { i }", + ... +} +``` + +The labels are split into tokens with `doctr.datasets.tokenize_latex`: commands (`\frac`, `\operatorname*`), environment +delimiters (`\begin{array}`), control symbols (`\{`, `\,`, `\\`) and single characters. They can be given raw or +already separated by spaces. The model predicts the tokens separated by spaces (e.g. `\frac { a } { b ^ { 2 } }`), which +renders the same in KaTeX or MathJax. + +The samples holding a token out of the vocabulary, or more tokens than `--max_length`, are skipped (the scripts report +how many). For the best results, normalize the labels with the [KaTeX](https://katex.org/) parser before training +(e.g. matrices as `\begin{array}`), so that a formula has a single spelling. + +### Vocabulary + +The default vocabulary, `doctr.datasets.LATEX_TOKENS`, holds the printable ASCII characters, the commands and +environments supported by KaTeX, and every other token of the UniMER-1M, HME100K and UniMER-Test labels (TeX commands +such as `\ensuremath` or `\textsc`, Greek, Cyrillic and CJK characters...): it covers all of them, no `--vocab` is needed +to train on these datasets. To recognize other symbols, extend it with the tokens of your labels that occur often +enough, and train with it: + +```shell +python references/formula/build_vocab.py path/to/train_set --output vocab.json --min-count 10 +python references/formula/train.py formulanet --train_path path/to/train_set --val_path path/to/val_set --vocab vocab.json +``` + +Pass the same `--vocab` to `evaluate.py`, and the vocabulary to the model to use it (`formulanet(vocab=...)`). + +## Slack Logging with tqdm + +To enable Slack logging using `tqdm`, you need to set the following environment variables: + +- `TQDM_SLACK_TOKEN`: the Slack Bot Token +- `TQDM_SLACK_CHANNEL`: you can retrieve it using `Right Click on Channel > Copy > Copy link`. You should get something like `https://xxxxxx.slack.com/archives/yyyyyyyy`. Keep only the `yyyyyyyy` part. + +You can follow this page on [how to create a Slack App](https://api.slack.com/quickstart). + +## Advanced options + +Feel free to inspect the multiple script option to customize your training to your own needs! + +```python +python references/formula/train.py --help +``` diff --git a/references/formula/build_vocab.py b/references/formula/build_vocab.py new file mode 100644 index 0000000000..6cd332eb97 --- /dev/null +++ b/references/formula/build_vocab.py @@ -0,0 +1,52 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +"""Build a LaTeX token vocabulary for formula recognition: the default docTR tokens, extended with the tokens of +training labels which occur often enough (e.g. CJK characters in the formulas of Chinese documents)""" + +import argparse +import json +from collections import Counter +from pathlib import Path + +from doctr.datasets import LATEX_TOKENS, tokenize_latex + + +def main(args): + counts: Counter[str] = Counter() + for path in args.datasets: + labels_path = Path(path).joinpath("labels.json") if Path(path).is_dir() else Path(path) + with open(labels_path, encoding="utf-8") as f: + labels = json.load(f) + for label in labels.values(): + counts.update(tokenize_latex(label)) + + base = [] if args.from_scratch else list(LATEX_TOKENS) + known = set(base) + extra = [token for token, count in counts.most_common() if count >= args.min_count and token not in known] + vocab = base + extra + with open(args.output, "w", encoding="utf-8") as f: + json.dump(vocab, f, ensure_ascii=False, indent=0) + + rare = sum(1 for token, count in counts.items() if count < args.min_count and token not in known) + print(f"{len(vocab)} tokens written to {args.output} ({len(extra)} added, {rare} rare tokens left out)") + if extra: + print(f"Added tokens: {extra[:50]}{' ...' if len(extra) > 50 else ''}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Build a LaTeX token vocabulary from formula datasets", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("datasets", type=str, nargs="+", help="dataset folders (with labels.json) or labels files") + parser.add_argument("--output", type=str, default="vocab.json", help="path of the JSON vocab to write") + parser.add_argument("--min-count", type=int, default=10, help="minimum number of occurrences of an added token") + parser.add_argument( + "--from-scratch", + action="store_true", + help="only keep the tokens of the datasets, instead of extending the default docTR tokens", + ) + main(parser.parse_args()) diff --git a/references/formula/ddp_utils.py b/references/formula/ddp_utils.py new file mode 100644 index 0000000000..3172a2fb6e --- /dev/null +++ b/references/formula/ddp_utils.py @@ -0,0 +1,122 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +from collections.abc import Iterable, Iterator, Sequence, Sized +from typing import Any + +import torch +import torch.distributed as dist +from torch.utils.data import Sampler + +__all__ = [ + "ShardSampler", + "barrier_download", + "ddp_device", + "is_distributed", + "is_main_rank", + "reduce_sum", + "sync_val_metric", +] + + +def is_distributed() -> bool: + """Whether a process group with more than one rank is active""" + return dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1 + + +def is_main_rank() -> bool: + """Whether this process is rank 0 (always True when not distributed)""" + return not (dist.is_available() and dist.is_initialized()) or dist.get_rank() == 0 + + +def ddp_device() -> torch.device: + """Device to put collective payloads on""" + if torch.cuda.is_available(): + return torch.device("cuda", torch.cuda.current_device()) + return torch.device("cpu") + + +def reduce_sum(values: Sequence[float]) -> list[float]: + """Sum a short list of scalars across all ranks. No-op when not distributed""" + if not is_distributed(): + return [float(v) for v in values] + tensor = torch.tensor([float(v) for v in values], dtype=torch.float64, device=ddp_device()) + dist.all_reduce(tensor, op=dist.ReduceOp.SUM) + return tensor.tolist() + + +def _gather_lists(value: list[Any]) -> list[Any]: + """Concatenate a per-rank list into the full list, in rank order""" + gathered: list[list[Any] | None] = [None] * dist.get_world_size() + dist.all_gather_object(gathered, value) + return [item for part in gathered if part is not None for item in part] + + +def sync_val_metric( + val_metric: Any, + counters: Iterable[str] = (), + buffers: Iterable[str] = (), +) -> None: + """Merge per-rank validation state so `summary()` reflects the whole validation set + + Args: + val_metric: the metric instance to update in place + counters: names of scalar accumulators to sum (e.g. `num_gts`, `matches`) + buffers: names of list accumulators to concatenate (e.g. `_gts`, `_preds`) + """ + if not is_distributed(): + return + + counter_names = [name for name in counters if hasattr(val_metric, name)] + if counter_names: + totals = reduce_sum([float(getattr(val_metric, name)) for name in counter_names]) + for name, total in zip(counter_names, totals): + # Preserve int/float so downstream divisions behave as before + setattr(val_metric, name, type(getattr(val_metric, name))(total)) + + for name in buffers: + if hasattr(val_metric, name): + setattr(val_metric, name, _gather_lists(getattr(val_metric, name))) + + +def barrier_download(rank: int, distributed: bool): + """Context manager that lets rank 0 populate the docTR cache before the others read it""" + + def _barrier(): + if torch.cuda.is_available() and dist.get_backend() == "nccl": + dist.barrier(device_ids=[torch.cuda.current_device()]) + else: + dist.barrier() + + class _BarrierDownload: + def __enter__(self): + if distributed and rank != 0: + _barrier() # wait for rank 0 to finish downloading + return self + + def __exit__(self, *exc_info): + if distributed and rank == 0: + _barrier() # release the other ranks + return False + + return _BarrierDownload() + + +class ShardSampler(Sampler[int]): + """Split a dataset across ranks without padding, so no sample is evaluated twice""" + + def __init__(self, dataset: Sized, rank: int | None = None, num_replicas: int | None = None): + if num_replicas is None: + num_replicas = dist.get_world_size() if is_distributed() else 1 + if rank is None: + rank = dist.get_rank() if is_distributed() else 0 + # Strided split: rank 0 takes 0, N, 2N - Every index appears on exactly one rank. + self.indices = list(range(rank, len(dataset), num_replicas)) + + def __iter__(self) -> Iterator[int]: + return iter(self.indices) + + def __len__(self) -> int: + return len(self.indices) diff --git a/references/formula/evaluate.py b/references/formula/evaluate.py new file mode 100644 index 0000000000..1b3b81cab0 --- /dev/null +++ b/references/formula/evaluate.py @@ -0,0 +1,195 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +import multiprocessing as mp +import os +import time +from pathlib import Path + +import torch +from torch.utils.data import DataLoader, SequentialSampler +from torchvision.transforms.v2 import Normalize + +if os.getenv("TQDM_SLACK_TOKEN") and os.getenv("TQDM_SLACK_CHANNEL"): + from tqdm.contrib.slack import tqdm +else: + from tqdm.auto import tqdm + +from doctr import transforms as T +from doctr.datasets import RecognitionDataset +from doctr.models import formula +from doctr.utils.metrics import FormulaMetric +from utils import amp_dtype, filter_samples, load_vocab, model_device, resolve_device + +AMP_DTYPE = torch.float16 # set from --amp-dtype in main() + + +def _autocast(): + return torch.amp.autocast("cuda", dtype=AMP_DTYPE) + + +@torch.inference_mode() +def evaluate(model, val_loader, batch_transforms, val_metric, amp=False): + # Model in eval mode + model.eval() + # Reset val metric + val_metric.reset() + # Validation loop + val_loss, sample_cnt = 0.0, 0 + pbar = tqdm(val_loader) + for images, targets in pbar: + images = images.to(model_device(model), non_blocking=True) + images = batch_transforms(images) + if amp: + with _autocast(): + out = model(images, targets, return_preds=True) + else: + out = model(images, targets, return_preds=True) + # Compute metric + val_metric.update(targets, [latex for latex, _ in out["preds"]]) + + val_loss += out["loss"].item() * images.shape[0] + sample_cnt += images.shape[0] + + val_loss /= sample_cnt + return val_loss, val_metric.summary() + + +def main(args): + global AMP_DTYPE + AMP_DTYPE = amp_dtype(args.amp_dtype) + slack_token = os.getenv("TQDM_SLACK_TOKEN") + slack_channel = os.getenv("TQDM_SLACK_CHANNEL") + + pbar = tqdm(disable=False if slack_token and slack_channel else True) + if slack_token and slack_channel: + # Monkey patch tqdm write method to send messages directly to Slack + pbar.write = lambda msg: pbar.sio.client.chat_postMessage(channel=slack_channel, text=msg) + pbar.write(str(args)) + device = resolve_device(args.device) + if args.amp and device.type != "cuda": + raise ValueError("--amp (automatic mixed precision) is only supported on CUDA devices") + + if device.type == "cuda": + torch.backends.cudnn.benchmark = True + + if not isinstance(args.workers, int): + args.workers = min(16, mp.cpu_count()) + + vocab = load_vocab(args.vocab) + + # Load doctr model + model = formula.__dict__[args.arch]( + pretrained=True if args.resume is None else False, + input_shape=(3, *args.input_size), + vocab=vocab, + max_length=args.max_length, + max_decode_length=args.max_decode_length, + ).eval() + + # Resume weights + if isinstance(args.resume, str): + pbar.write(f"Resuming {args.resume}") + model.from_pretrained(args.resume) + + st = time.time() + dataset_path = Path(args.dataset_path) + ds = RecognitionDataset( + img_folder=str(dataset_path.joinpath("images")), + labels_path=str(dataset_path.joinpath("labels.json")), + img_transforms=T.Resize(args.input_size, preserve_aspect_ratio=True), + ) + unknown, too_long = filter_samples(ds, vocab, args.max_length) + if unknown or too_long: + pbar.write(f"{unknown} samples with unknown tokens and {too_long} too long samples skipped") + + test_loader = DataLoader( + ds, + batch_size=args.batch_size, + drop_last=False, + num_workers=args.workers, + sampler=SequentialSampler(ds), + pin_memory=device.type == "cuda", + collate_fn=ds.collate_fn, + ) + pbar.write(f"Test set loaded in {time.time() - st:.4}s ({len(ds)} samples in {len(test_loader)} batches)") + + batch_transforms = Normalize(mean=model.cfg["mean"], std=model.cfg["std"]) + + # Metrics + val_metric = FormulaMetric() + + # GPU + if device.type == "cpu": + pbar.write("No accessible GPU, target device set to CPU.") + if device.type == "cuda": + torch.cuda.set_device(device) + model = model.to(device) + + pbar.write("Running evaluation") + val_loss, result = evaluate(model, test_loader, batch_transforms, val_metric, amp=args.amp) + pbar.write( + f"Validation loss: {val_loss:.6} (Exact match: {result['exact_match']:.2%} | " + f"Edit similarity: {result['edit_similarity']:.2%} | BLEU: {result['bleu']:.4})" + ) + + +def parse_args(): + import argparse + + parser = argparse.ArgumentParser( + description="docTR evaluation script for formula recognition (PyTorch)", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument("arch", type=str, help="formula recognition model to evaluate") + parser.add_argument("dataset_path", type=str, help="path to the evaluation data folder (images/ + labels.json)") + parser.add_argument( + "--vocab", + type=str, + default=None, + help="path to a JSON list of LaTeX tokens (cf. build_vocab.py), the default LaTeX tokens of docTR if omitted", + ) + parser.add_argument( + "--device", + default=None, + type=str, + help="Device for single-process runs: a CUDA index (e.g. 0), 'cuda:N', 'mps' (Apple Silicon) or 'cpu'. " + "Default: CUDA if available, else MPS, else CPU.", + ) + parser.add_argument("-b", "--batch_size", type=int, default=16, help="batch size for evaluation") + parser.add_argument( + "--input_size", + type=int, + nargs=2, + default=(256, 768), + metavar=("H", "W"), + help="input size (H, W) of the model", + ) + parser.add_argument("--max_length", type=int, default=4096, help="maximum number of tokens of a formula") + parser.add_argument( + "--max_decode_length", + type=int, + default=2048, + help="maximum number of tokens generated per formula by the greedy decoding of the predictions", + ) + parser.add_argument("-j", "--workers", type=int, default=None, help="number of workers used for dataloading") + parser.add_argument("--resume", type=str, default=None, help="Checkpoint to resume") + parser.add_argument("--amp", dest="amp", help="Use Automatic Mixed Precision", action="store_true") + parser.add_argument( + "--amp-dtype", + choices=["float16", "bfloat16"], + default="float16", + help="autocast dtype for --amp; bfloat16 (Ampere+ GPUs) avoids float16 overflows, no loss scaling", + ) + args = parser.parse_args() + args.input_size = tuple(args.input_size) + + return args + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/references/formula/latency.py b/references/formula/latency.py new file mode 100644 index 0000000000..21855c091d --- /dev/null +++ b/references/formula/latency.py @@ -0,0 +1,78 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +"""Formula recognition latency benchmark""" + +import argparse +import time + +import numpy as np +import torch + +from doctr.models import formula + + +@torch.inference_mode() +def main(args): + device = torch.device("cuda:0" if args.gpu else "cpu") + + model = ( + formula + .__dict__[args.arch]( + pretrained=args.pretrained, + pretrained_backbone=False, + input_shape=(3, *args.size), + ) + .eval() + .to(device=device) + ) + # The decoding time grows with the number of tokens: decode a fixed number of them (an untrained model rarely + # stops on its own, a trained one stops at the end of each formula) + model.max_decode_length = args.tokens + + # Input + img_tensor = torch.rand((args.batch_size, 3, *args.size)).to(device=device) + + # Warmup + for _ in range(10): + _ = model(img_tensor) + + timings = [] + + # Evaluation runs + for _ in range(args.it): + start_ts = time.perf_counter() + _ = model(img_tensor) + if args.gpu: + torch.cuda.synchronize() + timings.append(time.perf_counter() - start_ts) + + _timings = np.array(timings) + print( + f"{args.arch} ({args.it} runs on {tuple(args.size)} inputs in batches of {args.batch_size}, " + f"up to {args.tokens} tokens)" + ) + print(f"mean {1000 * _timings.mean():.2f}ms, std {1000 * _timings.std():.2f}ms") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="docTR latency benchmark for formula recognition (PyTorch)", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("arch", type=str, help="Architecture to use") + parser.add_argument("--batch-size", "-b", type=int, default=16, help="The batch_size") + parser.add_argument( + "--size", type=int, nargs=2, default=(256, 768), metavar=("H", "W"), help="The image input size (H, W)" + ) + parser.add_argument("--tokens", type=int, default=128, help="Maximum number of tokens to decode") + parser.add_argument("--gpu", dest="gpu", help="Should the benchmark be performed on GPU", action="store_true") + parser.add_argument("--it", type=int, default=100, help="Number of iterations to run") + parser.add_argument( + "--pretrained", dest="pretrained", help="Use pre-trained models from the modelzoo", action="store_true" + ) + args = parser.parse_args() + + main(args) diff --git a/references/formula/train.py b/references/formula/train.py new file mode 100644 index 0000000000..2ba7569dd7 --- /dev/null +++ b/references/formula/train.py @@ -0,0 +1,787 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +import datetime +import hashlib +import logging +import multiprocessing +import os +import time +from contextlib import nullcontext +from pathlib import Path + +import numpy as np +import torch +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.optim.lr_scheduler import ( + CosineAnnealingLR, + LinearLR, + MultiplicativeLR, + OneCycleLR, + PolynomialLR, + SequentialLR, +) +from torch.utils.data import DataLoader, RandomSampler, SequentialSampler +from torch.utils.data.distributed import DistributedSampler +from torchvision.transforms.v2 import ( + Compose, + Normalize, + RandomPerspective, + RandomPhotometricDistort, +) + +if os.getenv("TQDM_SLACK_TOKEN") and os.getenv("TQDM_SLACK_CHANNEL"): + from tqdm.contrib.slack import tqdm +else: + from tqdm.auto import tqdm + +from ddp_utils import ShardSampler, barrier_download, is_main_rank, reduce_sum, sync_val_metric + +from doctr import transforms as T +from doctr.datasets import RecognitionDataset, tokenize_latex +from doctr.models import formula, login_to_hub, push_to_hf_hub +from doctr.utils.metrics import FormulaMetric +from utils import ( + EarlyStopper, + amp_dtype, + filter_samples, + load_vocab, + model_device, + plot_recorder, + plot_samples, + resolve_device, + run_metadata, + save_checkpoint, + save_run_metadata, +) + +AMP_DTYPE = torch.float16 # set from --amp-dtype in main() + + +def _autocast(): + return torch.amp.autocast("cuda", dtype=AMP_DTYPE) + + +def _scaler(): + # bfloat16 has the range of float32: no loss scaling needed (GradScaler only makes sense for float16) + return torch.amp.GradScaler("cuda", enabled=AMP_DTYPE == torch.float16) + + +def split_batch(images: torch.Tensor, targets: list[str], max_tokens: int) -> list[tuple]: + """Split a batch into chunks of at most `max_tokens` padded target tokens, to bound the memory of long formulas + + The samples are sorted by length, so that the long ones do not pad the short ones. A batch which fits is returned as + a single chunk. + + Returns: + a list of (images, targets, share of the target tokens of the batch) chunks + """ + lengths = [len(tokenize_latex(target)) + 1 for target in targets] # tokens + EOS + if len(targets) * max(lengths) <= max_tokens: + return [(images, targets, 1.0)] + total = sum(lengths) + chunks: list[list[int]] = [[]] + for idx in sorted(range(len(targets)), key=lengths.__getitem__): + # The samples come by increasing length: the current one is the longest of its chunk + if chunks[-1] and (len(chunks[-1]) + 1) * lengths[idx] > max_tokens: + chunks.append([]) + chunks[-1].append(idx) + return [ + (images[chunk], [targets[idx] for idx in chunk], sum(lengths[idx] for idx in chunk) / total) for chunk in chunks + ] + + +def train_step(model, images, targets, optimizer, scaler=None, max_tokens: int = 32768) -> torch.Tensor: + """Forward, backward & update on a batch, accumulating the gradients of its chunks (cf. `split_batch`) + + The loss of each chunk is weighted by its share of the target tokens, so the update is the one of the whole batch. + + Returns: + the loss of the batch + """ + optimizer.zero_grad() + chunks = split_batch(images, targets, max_tokens) + batch_loss = torch.zeros((), device=images.device) + for idx, (chunk_images, chunk_targets, weight) in enumerate(chunks): + # With DDP, synchronize the gradients on the last chunk only + sync = idx == len(chunks) - 1 or not hasattr(model, "no_sync") + with nullcontext() if sync else model.no_sync(): + if scaler is not None: + with _autocast(): + loss = model(chunk_images, chunk_targets)["loss"] * weight + scaler.scale(loss).backward() + else: + loss = model(chunk_images, chunk_targets)["loss"] * weight + loss.backward() + batch_loss += loss.detach() + if scaler is not None: + # Gradient clipping + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), 5) + # Update the params + scaler.step(optimizer) + scaler.update() + else: + torch.nn.utils.clip_grad_norm_(model.parameters(), 5) + optimizer.step() + return batch_loss + + +def record_lr( + model: torch.nn.Module, + train_loader: DataLoader, + batch_transforms, + optimizer, + start_lr: float = 1e-7, + end_lr: float = 1, + num_it: int = 100, + amp: bool = False, + max_tokens: int = 32768, +): + """Gridsearch the optimal learning rate for the training. + Adapted from https://github.com/frgfm/Holocron/blob/master/holocron/trainer/core.py + """ + if num_it > len(train_loader): + raise ValueError("the value of `num_it` needs to be lower than the number of available batches") + + model = model.train() + # Update param groups & LR + optimizer.defaults["lr"] = start_lr + for pgroup in optimizer.param_groups: + pgroup["lr"] = start_lr + + gamma = (end_lr / start_lr) ** (1 / (num_it - 1)) + scheduler = MultiplicativeLR(optimizer, lambda step: gamma) + + lr_recorder = [start_lr * gamma**idx for idx in range(num_it)] + loss_recorder = [] + + scaler = _scaler() if amp else None + + for batch_idx, (images, targets) in enumerate(train_loader): + images = images.to(model_device(model), non_blocking=True) + + images = batch_transforms(images) + + # Forward, Backward & update + train_loss = train_step(model, images, targets, optimizer, scaler, max_tokens) + # Update LR + scheduler.step() + + # Record + if not torch.isfinite(train_loss): + if batch_idx == 0: + raise ValueError("loss value is NaN or inf.") + else: + break + loss_recorder.append(train_loss.item()) + # Stop after the number of iterations + if batch_idx + 1 == num_it: + break + + return lr_recorder[: len(loss_recorder)], loss_recorder + + +def fit_one_epoch( + model, + device, + train_loader, + batch_transforms, + optimizer, + scheduler, + amp=False, + log=None, + rank=0, + max_tokens: int = 32768, +): + scaler = _scaler() if amp else None + + model.train() + # Iterate over the batches of the dataset + epoch_train_loss, batch_cnt = 0, 0 + pbar = tqdm(train_loader, dynamic_ncols=True, disable=(rank != 0)) + for images, targets in pbar: + images = images.to(device, non_blocking=True) + images = batch_transforms(images) + + train_loss = train_step(model, images, targets, optimizer, scaler, max_tokens) + + scheduler.step() + last_lr = scheduler.get_last_lr()[0] + + pbar.set_description(f"Training loss: {train_loss.item():.6} | LR: {last_lr:.6}") + if log: + log(train_loss=train_loss.item(), lr=last_lr) + + epoch_train_loss += train_loss.item() + batch_cnt += 1 + + epoch_train_loss /= batch_cnt + return epoch_train_loss, last_lr + + +@torch.no_grad() +def evaluate(model, device, val_loader, batch_transforms, val_metric, amp=False, log=None): + # Evaluate the underlying module: DDP's forward hooks add nothing under no_grad and + # would otherwise tie every rank to an identical number of batches. + model = model.module if hasattr(model, "module") else model + # Model in eval mode + model.eval() + # Reset val metric + val_metric.reset() + # Validation loop + # Weight by samples, not batches, so the result is independent of the sharding + val_loss, sample_cnt = 0, 0 + pbar = tqdm(val_loader, dynamic_ncols=True, disable=not is_main_rank()) + for images, targets in pbar: + images = images.to(device) + images = batch_transforms(images) + # The loss is computed with teacher forcing, the predictions are decoded greedily + if amp: + with _autocast(): + out = model(images, targets, return_preds=True) + else: + out = model(images, targets, return_preds=True) + # Compute metric + preds = [latex for latex, _ in out["preds"]] + val_metric.update(targets, preds) + + pbar.set_description(f"Validation loss: {out['loss'].item():.6}") + if log: + log(val_loss=out["loss"].item()) + + val_loss += out["loss"].item() * images.shape[0] + sample_cnt += images.shape[0] + + val_loss, sample_cnt = reduce_sum([val_loss, float(sample_cnt)]) + sync_val_metric(val_metric, counters=val_metric.counters) + val_loss /= sample_cnt + return val_loss, val_metric.summary() + + +def _load_dataset(path: Path, img_transforms) -> RecognitionDataset: + """Load a formula dataset (`images/` + `labels.json`), or the union of the datasets of the subfolders of `path`""" + parts = [path] if path.joinpath("labels.json").is_file() else sorted(p for p in path.iterdir() if p.is_dir()) + dataset = RecognitionDataset( + str(parts[0].joinpath("images")), str(parts[0].joinpath("labels.json")), img_transforms=img_transforms + ) + for subfolder in parts[1:]: + dataset.merge_dataset( + RecognitionDataset(str(subfolder.joinpath("images")), str(subfolder.joinpath("labels.json"))) + ) + return dataset + + +def _labels_hash(path: Path) -> str: + parts = [path] if path.joinpath("labels.json").is_file() else sorted(p for p in path.iterdir() if p.is_dir()) + digest = hashlib.sha256() + for part in parts: + digest.update(part.joinpath("labels.json").read_bytes()) + return digest.hexdigest() + + +def main(args): + global AMP_DTYPE + AMP_DTYPE = amp_dtype(args.amp_dtype) + # Detect distributed setup + # variable is set by torchrun + world_size = int(os.environ.get("WORLD_SIZE", 1)) + distributed = world_size > 1 + + # GPU setup + if distributed: + rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", rank) + torch.cuda.set_device(device) + dist.init_process_group(backend=args.backend, device_id=device) + else: + # single process + rank = 0 + device = resolve_device(args.device) + if device.type == "cpu": + logging.warning("No accessible GPU, target device set to CPU.") # noqa: LOG015 + if args.amp and device.type != "cuda": + raise ValueError("--amp (automatic mixed precision) is only supported on CUDA devices") + use_cuda = device.type == "cuda" + + slack_token = os.getenv("TQDM_SLACK_TOKEN") + slack_channel = os.getenv("TQDM_SLACK_CHANNEL") + + pbar = tqdm(disable=False if (slack_token and slack_channel) and (rank == 0) else True) + if slack_token and slack_channel: + # Monkey patch tqdm write method to send messages directly to Slack + pbar.write = lambda msg: pbar.sio.client.chat_postMessage(channel=slack_channel, text=msg) + pbar.write(str(args)) + + if rank == 0 and args.push_to_hub: + login_to_hub() + + if not isinstance(args.workers, int): + args.workers = min(16, multiprocessing.cpu_count()) + + if use_cuda: + torch.backends.cudnn.benchmark = True + + vocab = load_vocab(args.vocab) + input_shape = (3, *args.input_size) + + # Load doctr model + with barrier_download(rank, distributed): + model = formula.__dict__[args.arch]( + pretrained=args.pretrained, + vocab=vocab, + input_shape=input_shape, + max_length=args.max_length, + max_decode_length=args.max_decode_length, + ) + batch_transforms = Normalize(mean=model.cfg["mean"], std=model.cfg["std"]) + + # Load val data + st = time.time() + val_path = Path(args.val_path) + val_hash = _labels_hash(val_path) + val_set = _load_dataset( + val_path, + img_transforms=T.Resize(args.input_size, preserve_aspect_ratio=True), + ) + unknown, too_long = filter_samples(val_set, vocab, args.max_length) + if rank == 0 and (unknown or too_long): + pbar.write(f"Validation set: {unknown} samples with unknown tokens and {too_long} too long samples skipped") + + val_loader = DataLoader( + val_set, + batch_size=args.batch_size, + drop_last=False, + num_workers=args.workers, + # Shard without padding so no sample is validated twice (see ShardSampler) + sampler=ShardSampler(val_set, rank=rank) if distributed else SequentialSampler(val_set), + pin_memory=use_cuda, + collate_fn=val_set.collate_fn, + ) + if rank == 0: + batch_info = f"{len(val_loader)} batches/rank" if distributed else f"{len(val_loader)} batches" + pbar.write(f"Validation set loaded in {time.time() - st:.4}s ({len(val_set)} samples in {batch_info})") + + # Resume weights + if isinstance(args.resume, str): + pbar.write(f"Resuming {args.resume}") + model.from_pretrained(args.resume) + + # Backbone freezing + if args.freeze_backbone: + for p in model.feat_extractor.parameters(): + p.requires_grad = False + + if use_cuda: + torch.cuda.set_device(device) + model = model.to(device) + + if distributed: + # construct DDP model + model = DDP(model, device_ids=[rank]) + + # Metrics + val_metric = FormulaMetric() + + if args.test_only: + if rank == 0: + pbar.write("Running evaluation") + val_loss, result = evaluate(model, device, val_loader, batch_transforms, val_metric, amp=args.amp) + if rank == 0: + pbar.write( + f"Validation loss: {val_loss:.6} (Exact match: {result['exact_match']:.2%} | " + f"Edit similarity: {result['edit_similarity']:.2%} | BLEU: {result['bleu']:.4})" + ) + if distributed: + dist.destroy_process_group() + return + + # Load train data + st = time.time() + train_path = Path(args.train_path) + train_hash = _labels_hash(train_path) + train_set = _load_dataset( + train_path, + img_transforms=Compose([ + # The crops of layout regions come with margins: learn to read them as they are + T.RandomPad(max_ratio=0.5, p=0.5), + T.Resize(args.input_size, preserve_aspect_ratio=True), + # Augmentations + T.ImageTorchvisionTransform(RandomPhotometricDistort(p=0.1)), + T.RandomApply(T.GaussianNoise(mean=0, std=0.05), 0.1), + T.RandomApply(T.GaussianBlur(sigma=(0.5, 1.0)), 0.2), + T.ImageTorchvisionTransform(RandomPerspective(distortion_scale=0.1, p=0.2)), + ]), + ) + unknown, too_long = filter_samples(train_set, vocab, args.max_length) + if rank == 0 and (unknown or too_long): + pbar.write(f"Training set: {unknown} samples with unknown tokens and {too_long} too long samples skipped") + + if distributed: + sampler = DistributedSampler(train_set, rank=rank, shuffle=True, drop_last=True) + else: + sampler = RandomSampler(train_set) + + train_loader = DataLoader( + train_set, + batch_size=args.batch_size, + drop_last=True, + num_workers=args.workers, + sampler=sampler, + pin_memory=use_cuda, + collate_fn=train_set.collate_fn, + ) + if rank == 0: + pbar.write( + f"Train set loaded in {time.time() - st:.4}s ({len(train_set)} samples in {len(train_loader)} batches)" + ) + + if args.show_samples: + if rank == 0: + x, target = next(iter(train_loader)) + plot_samples(x, target) + # Rank 0 arrives late here, so rendezvous before every rank leaves + if distributed: + dist.barrier() + dist.destroy_process_group() + return + + # Optimizer + if args.optim == "adam": + optimizer = torch.optim.Adam( + [p for p in model.parameters() if p.requires_grad], + args.lr, + betas=(0.95, 0.999), + eps=1e-6, + weight_decay=args.weight_decay, + ) + elif args.optim == "adamw": + optimizer = torch.optim.AdamW( + [p for p in model.parameters() if p.requires_grad], + args.lr, + betas=(0.9, 0.999), + eps=1e-6, + weight_decay=args.weight_decay or 1e-4, + ) + + # LR finder + if args.find_lr: + if rank == 0: + # Unwrap DDP: rank 0 runs alone here, gradient sync would deadlock + lrs, losses = record_lr( + model.module if hasattr(model, "module") else model, + train_loader, + batch_transforms, + optimizer, + amp=args.amp, + max_tokens=args.max_tokens, + ) + plot_recorder(lrs, losses) + # Rank 0 arrives late here, so rendezvous before every rank leaves + if distributed: + dist.barrier() + dist.destroy_process_group() + return + + # Scheduler, after a linear warmup + total_steps = args.epochs * len(train_loader) + warmup_steps = min(args.warmup_steps, total_steps // 2) + main_steps = total_steps - warmup_steps + if args.sched == "cosine": + scheduler = CosineAnnealingLR(optimizer, main_steps, eta_min=args.lr / 25e4) + elif args.sched == "onecycle": + scheduler = OneCycleLR(optimizer, args.lr, main_steps) + elif args.sched == "poly": + scheduler = PolynomialLR(optimizer, main_steps) + if warmup_steps > 0: + scheduler = SequentialLR( + optimizer, + [LinearLR(optimizer, start_factor=0.01, total_iters=warmup_steps), scheduler], + milestones=[warmup_steps], + ) + + # Training monitoring + current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + exp_name = f"{args.arch}_{current_time}" if args.name is None else args.name + + if rank == 0: + config = { + "learning_rate": args.lr, + "epochs": args.epochs, + "weight_decay": args.weight_decay, + "batch_size": args.batch_size, + "architecture": args.arch, + "input_size": args.input_size, + "max_length": args.max_length, + "max_decode_length": args.max_decode_length, + "max_tokens": args.max_tokens, + "optimizer": args.optim, + "framework": "pytorch", + "scheduler": args.sched, + "warmup_steps": warmup_steps, + "vocab": args.vocab, + "train_hash": train_hash, + "val_hash": val_hash, + "pretrained": args.pretrained, + "amp": args.amp, + } + # the run metadata describes the whole run: written once, next to the checkpoint `.pt` + save_run_metadata( + args.output_dir, + exp_name, + { + **config, + **run_metadata( + args, + task="formula", + vocab=vocab, + input_shape=input_shape, + max_length=args.max_length, + ), + }, + ) + + global global_step + global_step = 0 # Shared global step counter + # W&B + if rank == 0 and args.wb: + import wandb + + run = wandb.init( + name=exp_name, + project="formula-recognition", + config=config, + ) + + def wandb_log_at_step(train_loss=None, val_loss=None, lr=None): + wandb.log({ + **({"train_loss_step": train_loss} if train_loss is not None else {}), + **({"val_loss_step": val_loss} if val_loss is not None else {}), + **({"step_lr": lr} if lr is not None else {}), + }) + + # ClearML + if rank == 0 and args.clearml: + from clearml import Logger, Task + + task = Task.init(project_name="docTR/formula-recognition", task_name=exp_name, reuse_last_task_id=False) + task.upload_artifact("config", config) + + def clearml_log_at_step(train_loss=None, val_loss=None, lr=None): + logger = Logger.current_logger() + if train_loss is not None: + logger.report_scalar( + title="Training Step Loss", + series="train_loss_step", + iteration=global_step, + value=train_loss, + ) + if val_loss is not None: + logger.report_scalar( + title="Validation Step Loss", + series="val_loss_step", + iteration=global_step, + value=val_loss, + ) + if lr is not None: + logger.report_scalar( + title="Step Learning Rate", + series="step_lr", + iteration=global_step, + value=lr, + ) + + def log_at_step(train_loss=None, val_loss=None, lr=None): + global global_step + if args.wb and rank == 0: + wandb_log_at_step(train_loss, val_loss, lr) + if args.clearml and rank == 0: + clearml_log_at_step(train_loss, val_loss, lr) + global_step += 1 # Increment the shared global step counter + + # Create loss queue + min_loss = np.inf + if args.early_stop: + early_stopper = EarlyStopper(patience=args.early_stop_epochs, min_delta=args.early_stop_delta) + # Training loop + for epoch in range(args.epochs): + if distributed: + sampler.set_epoch(epoch) + train_loss, actual_lr = fit_one_epoch( + model, + device, + train_loader, + batch_transforms, + optimizer, + scheduler, + amp=args.amp, + log=log_at_step, + rank=rank, + max_tokens=args.max_tokens, + ) + + # Validation loop at the end of each epoch + val_loss, result = evaluate( + model, device, val_loader, batch_transforms, val_metric, amp=args.amp, log=log_at_step + ) + + if rank == 0: + pbar.write(f"Epoch {epoch + 1}/{args.epochs} - Training loss: {train_loss:.6} | LR: {actual_lr:.6}") + params = model.module if hasattr(model, "module") else model + if val_loss < min_loss: + pbar.write(f"Validation loss decreased {min_loss:.6} --> {val_loss:.6}: saving state...") + save_checkpoint(params, args.output_dir, exp_name) + min_loss = val_loss + if args.save_interval_epoch: + save_checkpoint(params, args.output_dir, f"{exp_name}_epoch{epoch + 1}") + pbar.write( + f"Epoch {epoch + 1}/{args.epochs} - Validation loss: {val_loss:.6} " + f"(Exact match: {result['exact_match']:.2%} | Edit similarity: {result['edit_similarity']:.2%} | " + f"BLEU: {result['bleu']:.4})" + ) + # W&B + if args.wb: + wandb.log({ + "train_loss": train_loss, + "val_loss": val_loss, + "learning_rate": actual_lr, + **result, + }) + + # ClearML + if args.clearml: + from clearml import Logger + + logger = Logger.current_logger() + logger.report_scalar(title="Training Loss", series="train_loss", value=train_loss, iteration=epoch) + logger.report_scalar(title="Validation Loss", series="val_loss", value=val_loss, iteration=epoch) + logger.report_scalar(title="Learning Rate", series="lr", value=actual_lr, iteration=epoch) + for name, value in result.items(): + logger.report_scalar(title=name, series=name, value=value, iteration=epoch) + + if args.early_stop and early_stopper.early_stop(val_loss): + if rank == 0: + pbar.write("Training halted early due to reaching patience limit.") + break + + if rank == 0: + if args.wb: + run.finish() + + if args.push_to_hub: + push_to_hf_hub(model, exp_name, task="formula", run_config=args) + + if distributed: + dist.destroy_process_group() + + +def parse_args(): + import argparse + + parser = argparse.ArgumentParser( + description="DocTR training script for formula recognition (PyTorch)", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # DDP related args + parser.add_argument("--backend", default="nccl", type=str, help="Backend to use for torch.distributed") + + parser.add_argument("arch", type=str, help="formula recognition model to train") + parser.add_argument("--output_dir", type=str, default=".", help="path to save checkpoints and final model") + parser.add_argument( + "--train_path", + type=str, + required=True, + help="path to the train data folder (images/ + labels.json), or to a folder of such folders", + ) + parser.add_argument("--val_path", type=str, required=True, help="path to the validation data folder") + parser.add_argument("--name", type=str, default=None, help="Name of your training experiment") + parser.add_argument("--epochs", type=int, default=10, help="number of epochs to train the model on") + parser.add_argument("-b", "--batch_size", type=int, default=32, help="batch size for training") + parser.add_argument( + "--save-interval-epoch", dest="save_interval_epoch", action="store_true", help="Save model every epoch" + ) + parser.add_argument( + "--input_size", + type=int, + nargs=2, + default=(256, 768), + metavar=("H", "W"), + help="input size (H, W) of the model: the formulas are resized to it, keeping their aspect ratio", + ) + parser.add_argument("--max_length", type=int, default=4096, help="maximum number of tokens of a formula") + parser.add_argument( + "--max_decode_length", + type=int, + default=2048, + help="maximum number of tokens generated per formula by the greedy decoding of the predictions", + ) + parser.add_argument( + "--max-tokens", + type=int, + default=32768, + help="maximum number of padded target tokens per forward pass: longer batches are split into chunks whose " + "gradients are accumulated, to bound the memory of long formulas", + ) + parser.add_argument( + "--vocab", + type=str, + default=None, + help="path to a JSON list of LaTeX tokens (cf. build_vocab.py), the default LaTeX tokens of docTR if omitted", + ) + parser.add_argument( + "--device", + default=None, + type=str, + help="Device for single-process runs: a CUDA index (e.g. 0), 'cuda:N', 'mps' (Apple Silicon) or 'cpu'. " + "Default: CUDA if available, else MPS, else CPU.", + ) + + parser.add_argument("--lr", type=float, default=0.001, help="learning rate for the optimizer (Adam or AdamW)") + parser.add_argument("--wd", "--weight-decay", default=0.01, type=float, help="weight decay", dest="weight_decay") + parser.add_argument("--warmup-steps", type=int, default=1000, help="number of linear warmup steps of the LR") + parser.add_argument("-j", "--workers", type=int, default=None, help="number of workers used for dataloading") + parser.add_argument("--resume", type=str, default=None, help="Path to your checkpoint") + parser.add_argument("--test-only", dest="test_only", action="store_true", help="Run the validation loop") + parser.add_argument( + "--freeze-backbone", dest="freeze_backbone", action="store_true", help="freeze model backbone for fine-tuning" + ) + parser.add_argument( + "--show-samples", dest="show_samples", action="store_true", help="Display unormalized training samples" + ) + parser.add_argument("--wb", dest="wb", action="store_true", help="Log to Weights & Biases") + parser.add_argument("--clearml", dest="clearml", action="store_true", help="Log to ClearML") + parser.add_argument("--push-to-hub", dest="push_to_hub", action="store_true", help="Push to Huggingface Hub") + parser.add_argument( + "--pretrained", + dest="pretrained", + action="store_true", + help="Load pretrained parameters before starting the training", + ) + parser.add_argument("--optim", type=str, default="adamw", choices=["adam", "adamw"], help="optimizer to use") + parser.add_argument( + "--sched", type=str, default="cosine", choices=["cosine", "onecycle", "poly"], help="scheduler to use" + ) + parser.add_argument("--amp", dest="amp", help="Use Automatic Mixed Precision", action="store_true") + parser.add_argument( + "--amp-dtype", + choices=["float16", "bfloat16"], + default="float16", + help="autocast dtype for --amp; bfloat16 (Ampere+ GPUs) avoids float16 overflows, no loss scaling", + ) + parser.add_argument("--find-lr", action="store_true", help="Gridsearch the optimal LR") + parser.add_argument("--early-stop", action="store_true", help="Enable early stopping") + parser.add_argument("--early-stop-epochs", type=int, default=5, help="Patience for early stopping") + parser.add_argument("--early-stop-delta", type=float, default=0.01, help="Minimum Delta for early stopping") + args = parser.parse_args() + args.input_size = tuple(args.input_size) + + return args + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/references/formula/utils.py b/references/formula/utils.py new file mode 100644 index 0000000000..fad60d6a38 --- /dev/null +++ b/references/formula/utils.py @@ -0,0 +1,243 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +import json +import math +import subprocess +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import torch + +import doctr +from doctr.datasets import LATEX_TOKENS, tokenize_latex + + +def plot_samples(images, targets): + # Unnormalize image + num_samples = min(len(images), 12) + num_cols = min(len(images), 4) + num_rows = int(math.ceil(num_samples / num_cols)) + _, axes = plt.subplots(num_rows, num_cols, figsize=(20, 5)) + for idx in range(num_samples): + img = (255 * images[idx].numpy()).round().clip(0, 255).astype(np.uint8) + if img.shape[0] == 3 and img.shape[2] != 3: + img = img.transpose(1, 2, 0) + + row_idx = idx // num_cols + col_idx = idx % num_cols + ax = axes[row_idx] if num_rows > 1 else axes + ax = ax[col_idx] if num_cols > 1 else ax + + ax.imshow(img) + # Long formulas would overflow the subplot + title = targets[idx] if len(targets[idx]) <= 60 else f"{targets[idx][:57]}..." + ax.set_title(title, fontsize=8) + # Disable axis + for ax in axes.ravel(): + ax.axis("off") + + plt.show() + + +def plot_recorder(lr_recorder, loss_recorder, beta: float = 0.95, **kwargs) -> None: + """Display the results of the LR grid search. + Adapted from https://github.com/frgfm/Holocron/blob/master/holocron/trainer/core.py. + + Args: + lr_recorder: list of LR values + loss_recorder: list of loss values + beta (float, optional): smoothing factor + **kwargs: keyword arguments from `matplotlib.pyplot.show`. + """ + if len(lr_recorder) != len(loss_recorder) or len(lr_recorder) == 0: + raise AssertionError("Both `lr_recorder` and `loss_recorder` should have the same length") + + # Exp moving average of loss + smoothed_losses = [] + avg_loss = 0.0 + for idx, loss in enumerate(loss_recorder): + avg_loss = beta * avg_loss + (1 - beta) * loss + smoothed_losses.append(avg_loss / (1 - beta ** (idx + 1))) + + # Properly rescale Y-axis + data_slice = slice( + min(len(loss_recorder) // 10, 10), + -min(len(loss_recorder) // 20, 5) if len(loss_recorder) >= 20 else len(loss_recorder), + ) + vals = np.array(smoothed_losses[data_slice]) + min_idx = vals.argmin() + max_val = vals.max() if min_idx is None else vals[: min_idx + 1].max() # type: ignore[misc] + delta = max_val - vals[min_idx] + + plt.plot(lr_recorder[data_slice], smoothed_losses[data_slice]) + plt.xscale("log") + plt.xlabel("Learning Rate") + plt.ylabel("Training loss") + plt.ylim(vals[min_idx] - 0.1 * delta, max_val + 0.2 * delta) + plt.grid(True, linestyle="--", axis="x") + plt.show(**kwargs) + + +class EarlyStopper: + def __init__(self, patience: int = 5, min_delta: float = 0.01): + self.patience = patience + self.min_delta = min_delta + self.counter = 0 + self.min_validation_loss = float("inf") + + def early_stop(self, validation_loss: float) -> bool: + if validation_loss < self.min_validation_loss: + self.min_validation_loss = validation_loss + self.counter = 0 + elif validation_loss > (self.min_validation_loss + self.min_delta): + self.counter += 1 + if self.counter >= self.patience: + return True + return False + + +def resolve_device(device: str | int | None) -> torch.device: + """Turn a CLI device spec (None, "cpu", "mps", "cuda", "cuda:1", 0) into a torch device. + + With `None`, pick CUDA if available, then MPS, then CPU. + + Args: + device: the device specification + + Returns: + the resolved torch device + """ + if device is None or device == "": + if torch.cuda.is_available(): + return torch.device("cuda", 0) + if torch.backends.mps.is_available(): + return torch.device("mps") + return torch.device("cpu") + if isinstance(device, int) or (isinstance(device, str) and device.isdigit()): + index = int(device) + if not torch.cuda.is_available(): + raise AssertionError("PyTorch cannot access your GPU. Please investigate!") + if index >= torch.cuda.device_count(): + raise ValueError("Invalid device index") + return torch.device("cuda", index) + dev = torch.device(device) + if dev.type == "cuda": + if not torch.cuda.is_available(): + raise AssertionError("PyTorch cannot access your GPU. Please investigate!") + if dev.index is None: + # "cuda" without an index: pin it to the current device so that `torch.cuda.set_device` works + dev = torch.device("cuda", torch.cuda.current_device()) + elif dev.index >= torch.cuda.device_count(): + raise ValueError("Invalid device index") + if dev.type == "mps" and not torch.backends.mps.is_available(): + raise AssertionError("MPS backend is not available on this machine.") + return dev + + +def model_device(model: torch.nn.Module) -> torch.device: + """Device holding the parameters of a model (works through DDP wrappers).""" + return next(model.parameters()).device + + +def amp_dtype(name: str) -> torch.dtype: + """Autocast dtype for `--amp-dtype`.""" + return torch.bfloat16 if name == "bfloat16" else torch.float16 + + +def save_checkpoint(model: torch.nn.Module, output_dir: str, name: str) -> Path: + """Save the model weights as `/.pt`. + + Args: + model: the model to save (unwrapped from DDP if needed) + output_dir: destination folder, created if missing + name: file stem + + Returns: + the path of the weights file + """ + out_dir = Path(output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + weights = out_dir / f"{name}.pt" + torch.save(model.state_dict(), weights) + return weights + + +def save_run_metadata(output_dir: str, name: str, metadata: dict) -> Path: + """Write the run metadata as `/.json`. + + The metadata (architecture, task settings such as class names or vocab, dataset hashes, versions, arguments) + is what is needed to rebuild the model for inference without remembering how it was trained. It describes the + run, not one of its checkpoints, so it is written once per run and shared by every checkpoint the run saves. + + Args: + output_dir: destination folder, created if missing + name: file stem, the experiment name + metadata: JSON-serializable run description + + Returns: + the path of the metadata file + """ + out_dir = Path(output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / f"{name}.json" + with open(path, "w", encoding="utf-8") as f: + json.dump(metadata, f, indent=1, default=str) + return path + + +def run_metadata(args, **task_specific) -> dict: + """Common run description written next to every checkpoint: versions, git revision and the full arguments.""" + try: + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL, text=True, cwd=Path(__file__).parent + ).strip() + except Exception: + revision = None + return { + "framework": "pytorch", + "doctr_version": doctr.__version__, + "torch_version": torch.__version__, + "git_revision": revision, + "args": dict(vars(args)), + **task_specific, + } + + +def load_vocab(path: str | None) -> list[str]: + """Load a list of LaTeX tokens from a JSON file (cf. `build_vocab.py`), the default `LATEX_TOKENS` without path""" + if path is None: + return list(LATEX_TOKENS) + with open(path, encoding="utf-8") as f: + vocab = json.load(f) + if not isinstance(vocab, list) or not all(isinstance(token, str) for token in vocab): + raise ValueError(f"{path} must hold a JSON list of tokens") + return vocab + + +def filter_samples(dataset, vocab: list[str], max_length: int) -> tuple[int, int]: + """Drop the samples a model cannot learn, in place: labels with a token out of the vocab or too long + + Args: + dataset: a dataset whose `data` is a list of (image, LaTeX label) + vocab: the tokens of the model + max_length: the maximum number of tokens of the model + + Returns: + the number of samples dropped for an unknown token, and for their length + """ + tokens = set(vocab) + kept, unknown, too_long = [], 0, 0 + for img, label in dataset.data: + label_tokens = tokenize_latex(label) + if not set(label_tokens) <= tokens: + unknown += 1 + elif len(label_tokens) > max_length: + too_long += 1 + else: + kept.append((img, label)) + dataset.data = kept + return unknown, too_long diff --git a/tests/common/test_cli.py b/tests/common/test_cli.py index 22eac78204..0ddbd6136b 100644 --- a/tests/common/test_cli.py +++ b/tests/common/test_cli.py @@ -101,6 +101,7 @@ def test_parse_args_defaults(): assert args.detect_language is False assert args.detect_layout is False assert args.detect_tables is False + assert args.detect_formulas is False assert args.ignore_regions is None assert args.export_as_straight_boxes is False assert args.preserve_original_coords is False @@ -184,6 +185,7 @@ def test_parse_args_advanced_options(): "3", "--detect_layout", "--detect_tables", + "--detect_formulas", "--ignore_regions", "Picture", "Table", @@ -208,6 +210,7 @@ def test_parse_args_advanced_options(): assert args.pdf_scale == 3 assert args.detect_layout is True assert args.detect_tables is True + assert args.detect_formulas is True assert args.ignore_regions == ["Picture", "Table"] assert args.export_as_straight_boxes is True assert args.preserve_original_coords is True @@ -433,6 +436,7 @@ def test_build_predictor_forwards_every_option(recorded_predictor): "--detect_language", "--detect_layout", "--detect_tables", + "--detect_formulas", "--export_as_straight_boxes", "--preserve_original_coords", "--disable_page_orientation", @@ -461,6 +465,7 @@ def test_build_predictor_forwards_every_option(recorded_predictor): "layout_arch": "lw_detr_s", "ignore_regions": None, "detect_tables": True, + "detect_formulas": True, "det_bs": 4, "reco_bs": 64, "confidence_aggregation": "geometric_mean", diff --git a/tests/common/test_datasets_utils.py b/tests/common/test_datasets_utils.py index a40bdcec4c..036be10265 100644 --- a/tests/common/test_datasets_utils.py +++ b/tests/common/test_datasets_utils.py @@ -170,3 +170,43 @@ def test_pre_transform_multiclass(): # a class outside class_names is an error, not a silently dropped box with pytest.raises(ValueError, match="unknown class"): utils.pre_transform_multiclass(img, (boxes, classes), class_names=["a"]) + + +@pytest.mark.parametrize( + "latex, tokens", + [ + [r"\frac{a}{b^2}", ["\\frac", "{", "a", "}", "{", "b", "^", "2", "}"]], + [ + r"\begin {array}{cc} a & b \\ c \end{array}", + ["\\begin{array}", "{", "c", "c", "}", "a", "&", "b", "\\\\", "c", "\\end{array}"], + ], + [ + r"\operatorname*{max}_x \, \{x\}", + ["\\operatorname*", "{", "m", "a", "x", "}", "_", "x", "\\,", "\\{", "x", "\\}"], + ], + [r"a\ b", ["a", "\\space", "b"]], + [" ", []], + ], +) +def test_tokenize_latex(latex, tokens): + assert utils.tokenize_latex(latex) == tokens + normalized = utils.normalize_latex(latex) + assert normalized == " ".join(tokens) + # Normalization is idempotent + assert utils.normalize_latex(normalized) == normalized + + +def test_token_vocab_encoding(): + vocab = ["a", "b", "\\frac", "{", "}"] + assert utils.encode_string("\\frac { a } b", vocab) == [2, 3, 0, 4, 1] + assert utils.decode_sequence([2, 3, 0, 4, 1], vocab) == "\\frac { a } b" + with pytest.raises(ValueError): + utils.encode_string("\\alpha", vocab) + with pytest.raises(AssertionError): + utils.decode_sequence([5], vocab) + + # The sequence lengths are counted in tokens + encoded = utils.encode_sequences(["\\frac { a } { b }", "a"], vocab, eos=5, sos=6, pad=7) + assert encoded.tolist() == [[6, 2, 3, 0, 4, 3, 1, 4, 5, 7], [6, 0, 5, 7, 7, 7, 7, 7, 7, 7]] + encoded = utils.encode_sequences(["a b", "a"], vocab, target_size=10, eos=5, dynamic_seq_length=True) + assert encoded.tolist() == [[0, 1, 5], [0, 5, 5]] diff --git a/tests/common/test_datasets_vocabs.py b/tests/common/test_datasets_vocabs.py index cd84bf7ac7..04ce19a2bd 100644 --- a/tests/common/test_datasets_vocabs.py +++ b/tests/common/test_datasets_vocabs.py @@ -1,6 +1,6 @@ from collections import Counter -from doctr.datasets import VOCABS +from doctr.datasets import LATEX_TOKENS, VOCABS, tokenize_latex def test_vocabs_duplicates(): @@ -9,3 +9,18 @@ def test_vocabs_duplicates(): duplicates = [char for char, count in Counter(vocab).items() if count > 1] assert not duplicates, f"Duplicate characters in {key} vocab: {duplicates}" + + +def test_latex_tokens(): + assert isinstance(LATEX_TOKENS, list) and all(isinstance(token, str) for token in LATEX_TOKENS) + # No duplicate, and every token is a single token for the LaTeX tokenizer + assert len(LATEX_TOKENS) == len(set(LATEX_TOKENS)) + assert all(tokenize_latex(token) == [token] for token in LATEX_TOKENS) + assert all(not any(char.isspace() for char in token) for token in LATEX_TOKENS) + for token in ("a", "{", "^", "\\frac", "\\alpha", "\\operatorname*", "\\begin{pmatrix}", "\\end{array}", "\\\\"): + assert token in LATEX_TOKENS + # Tokens of the public formula datasets, beyond KaTeX + for token in ("\\ensuremath", "\\textsc", "\\upmu", "\\-", "、", "θ", "数"): + assert token in LATEX_TOKENS + # The KaTeX tokens come first: their indices do not depend on the tokens found in the datasets + assert LATEX_TOKENS.index("\\vcentcolon") < LATEX_TOKENS.index("\\ensuremath") diff --git a/tests/common/test_io_elements.py b/tests/common/test_io_elements.py index 337afa202c..e71ad934dc 100644 --- a/tests/common/test_io_elements.py +++ b/tests/common/test_io_elements.py @@ -529,6 +529,7 @@ def test_page(): "language": language, "layout": [r.export() for r in layout], "tables": [], + "formulas": [], } assert json.dumps(page.export()) and json.dumps(page.export(reading_order=False)) @@ -809,3 +810,23 @@ def test_kie_page_and_kie_document_from_dict(): restored_plain = elements.Document.from_dict(plain.export()) assert all(isinstance(page, elements.Page) for page in restored_plain.pages) assert restored_plain.export() == plain.export() + + +def test_formula(): + geometry = ((0.2, 0.3), (0.6, 0.4)) + formula = elements.Formula(r"x ^ { 2 }", 0.75, geometry, objectness_score=0.9) + assert formula.value == r"x ^ { 2 }" and formula.confidence == 0.75 and formula.geometry == geometry + assert formula.render() == r"x ^ { 2 }" + assert repr(formula) == "Formula(value='x ^ { 2 }', confidence=0.75)" + exported = formula.export() + assert exported == {"geometry": geometry, "value": "x ^ { 2 }", "confidence": 0.75, "objectness_score": 0.9} + assert elements.Formula.from_dict(exported).export() == exported + # Exports without objectness score + reloaded = elements.Formula.from_dict({k: v for k, v in exported.items() if k != "objectness_score"}) + assert reloaded.objectness_score is None + + page = elements.Page(np.zeros((10, 10, 3), dtype=np.uint8), [], 0, (10, 10), formulas=[formula]) + assert page.formulas == [formula] + assert elements.Page.from_dict(page.export()).formulas[0].export() == exported + # Exports of earlier versions have no formulas + assert elements.Page.from_dict({k: v for k, v in page.export().items() if k != "formulas"}).formulas == [] diff --git a/tests/common/test_io_exporters.py b/tests/common/test_io_exporters.py index 95a27116d7..d43b8440e4 100644 --- a/tests/common/test_io_exporters.py +++ b/tests/common/test_io_exporters.py @@ -1304,3 +1304,44 @@ def test_page_export_as_xml_figures(picture): assert "ocr_photo" in xml.split('name="ocr-capabilities" content="')[1].split('"')[0] # The raw order also carries the figures assert 'class="ocr_photo"' in page.export_as_xml(reading_order=False)[0].decode() + + +def test_formula_exports(): + # A display formula between two paragraphs is read in the text flow, and rendered as display math + lines = [_line_at(f"intro line {i} of the body", 0.1, 0.05 + 0.04 * i, 0.9, 0.08 + 0.04 * i) for i in range(3)] + lines += [_line_at(f"outro line {i} of the body", 0.1, 0.40 + 0.04 * i, 0.9, 0.43 + 0.04 * i) for i in range(3)] + formula = elements.Formula(r"\frac { a } { b } < c", 0.8, ((0.3, 0.22), (0.7, 0.32)), 0.9) + layout = [ + elements.LayoutElement("Text", 0.9, ((0.08, 0.04), (0.92, 0.17))), + elements.LayoutElement("Formula", 0.9, ((0.3, 0.22), (0.7, 0.32))), + elements.LayoutElement("Text", 0.9, ((0.08, 0.39), (0.92, 0.52))), + ] + page = _page(lines, layout=layout, formulas=[formula]) + items, labels, _ = page_reading_order(page) + assert [type(item).__name__ for item in items] == ["Block", "Formula", "Block"] + assert labels[1] == "Formula" + assert page.items_in_reading_order()[1] is formula + + # The LaTeX source is not escaped as Markdown / AsciiDoc text, but HTML-escaped in HTML + assert "\n\n$$\n\\frac { a } { b } < c\n$$\n\n" in page.export_as_markdown() + assert "\n\n[latexmath]\n++++\n\\frac { a } { b } < c\n++++\n\n" in page.export_as_asciidoc() + assert '\n
    $$\\frac { a } { b } < c$$
    \n' in page.export_as_html() + assert "\n\n\\frac { a } { b } < c\n\n" in page.render() + xml = page.export_as_xml()[0].decode() + assert '' in xml + assert "ocr_math" in ET.fromstring(page.export_as_xml()[0]).find(".//{*}meta[@name='ocr-capabilities']").get( + "content" + ) + # Raw order: blocks, then tables, then formulas + assert '= 0.9, latex + assert 0 < confidence <= 1 + + +def test_ocrpredictor_formulas(mock_pdf, mock_vocab, monkeypatch): + det_predictor = models.detection_predictor( + detection.db_mobilenet_v3_large(pretrained=False, pretrained_backbone=False, assume_straight_pages=True), + batch_size=2, + ) + reco_predictor = models.recognition_predictor( + recognition.crnn_vgg16_bn(pretrained=False, pretrained_backbone=False, vocab=mock_vocab), batch_size=32 + ) + layout_pred = layout_predictor("lw_detr_s", pretrained=False) + formula_pred = formula_predictor("formulanet", pretrained=True) + + # A formula predictor requires a layout predictor (formulas are located with the layout model) + with pytest.raises(ValueError): + OCRPredictor(det_predictor, reco_predictor, formula_predictor=formula_pred) + + doc = DocumentFile.from_pdf(mock_pdf) + page_h, page_w = doc[0].shape[:2] + + # A layout model locating one formula in the middle of each page + def fake_layout(pages, **kwargs): + return [ + { + "class_names": ["Text", "Formula"], + "boxes": np.array([[0.1, 0.1, 0.9, 0.2], [0.3, 0.4, 0.7, 0.5]], dtype=np.float32), + "scores": np.array([0.9, 0.8], dtype=np.float32), + } + for _ in pages + ] + + monkeypatch.setattr(layout_pred, "forward", fake_layout) + masked_pages = [] + det_forward = det_predictor.forward + + def spy_detection(pages, **kwargs): + masked_pages.extend(pages) + return det_forward(pages, **kwargs) + + monkeypatch.setattr(det_predictor, "forward", spy_detection) + monkeypatch.setattr(formula_pred, "forward", lambda crops, **kwargs: [("x ^ 2", 0.75) for _ in crops]) + + predictor = OCRPredictor( + det_predictor, reco_predictor, layout_predictor=layout_pred, formula_predictor=formula_pred + ) + assert isinstance(predictor.formula_predictor, FormulaPredictor) + out = predictor(doc) + assert isinstance(out, Document) + # The formula regions are masked out of the text detection + region = masked_pages[0][int(0.4 * page_h) : int(0.5 * page_h), int(0.3 * page_w) : int(0.7 * page_w)] + assert np.all(region == 0) + for page in out.pages: + assert len(page.formulas) == 1 and isinstance(page.formulas[0], Formula) + formula_elt = page.formulas[0] + assert formula_elt.value == "x ^ 2" and formula_elt.confidence == 0.75 + assert formula_elt.objectness_score == pytest.approx(0.8) + np.testing.assert_allclose(formula_elt.geometry, ((0.3, 0.4), (0.7, 0.5)), atol=1e-6) + assert page.export()["formulas"] == [formula_elt.export()] + assert "$$\nx ^ 2\n$$" in page.export_as_markdown() + assert formula_elt in page.items_in_reading_order() + + # Without a formula predictor, the formula regions are left to the text detection + masked_pages.clear() + predictor = OCRPredictor(det_predictor, reco_predictor, layout_predictor=layout_pred) + out = predictor(doc) + assert all(page.formulas == [] for page in out.pages) + assert not np.all(masked_pages[0] == 0) + + +def test_ocrpredictor_formulas_factory(): + # The factory exposes a single `detect_formulas` flag, which also enables the layout model + predictor = models.ocr_predictor( + "db_mobilenet_v3_large", "crnn_vgg16_bn", pretrained=False, pretrained_backbone=False, detect_formulas=True + ) + assert isinstance(predictor.formula_predictor, FormulaPredictor) + assert predictor.layout_predictor is not None + assert predictor._masked_classes() == {"Formula"} + + # No formulas by default + predictor = models.ocr_predictor("db_mobilenet_v3_large", "crnn_vgg16_bn", pretrained=False) + assert predictor.formula_predictor is None + assert predictor._masked_classes() == set() + + +def test_formula_save_config_for_hub(): + model = formula.formulanet(pretrained=True).eval() + with tempfile.TemporaryDirectory() as tmp_dir: + _save_model_and_config_for_hf_hub(model, arch="formulanet", task="formula", save_dir=tmp_dir) + assert os.path.exists(os.path.join(tmp_dir, "pytorch_model.bin")) + with open(os.path.join(tmp_dir, "config.json")) as f: + config = json.load(f) + assert config["arch"] == "formulanet" and config["task"] == "formula" + assert config["vocab"] == LATEX_TOKENS + # The config is enough to rebuild the model and reload the weights + rebuilt = formula.formulanet( + pretrained=False, + pretrained_backbone=False, + input_shape=tuple(config["input_shape"]), + vocab=config["vocab"], + ) + rebuilt.from_pretrained(os.path.join(tmp_dir, "pytorch_model.bin")) + for (name, param), rebuilt_param in zip(model.state_dict().items(), rebuilt.state_dict().values()): + assert torch.equal(param, rebuilt_param), name + + +def test_formulanet_max_decode_length(): + model = formula.formulanet(pretrained=True, max_decode_length=8) + assert model.max_length == 4096 and model.max_decode_length == 8 + assert model.pos_embed.num_embeddings == 4097 # the size of the model follows `max_length` + x = torch.rand((2, 3, 256, 768)) + # Training labels longer than `max_decode_length` are learned entirely + label = " ".join(["a"] * 30) + out = model.train()(x, [label, "b"], return_model_output=True) + assert out["out_map"].shape[1] == 31 + # The decoding generates at most `max_decode_length` tokens (+ EOS), even when the model never ends a formula + model.exportable = True + with torch.no_grad(): + assert model.eval()(x)["logits"].shape[1] == 9 + # An explicit length overrides it + assert model.decode_greedy(model.encode(x), max_len=12).shape[1] == 13 + # The default cap, and its override in the predictor + assert formula.formulanet(pretrained=True).max_decode_length == 2048 + predictor = formula_predictor(model, max_decode_length=4) + assert predictor.model.max_decode_length == 4 diff --git a/tests/pytorch/test_transforms_pt.py b/tests/pytorch/test_transforms_pt.py index 0ef89d3c5a..d0336c3fdb 100644 --- a/tests/pytorch/test_transforms_pt.py +++ b/tests/pytorch/test_transforms_pt.py @@ -18,6 +18,7 @@ RandomGlare, RandomHorizontalFlip, RandomLighting, + RandomPad, RandomPerspective, RandomResize, RandomRotate, @@ -799,6 +800,57 @@ def test_random_resize_small_image(input_shape, preserve_aspect_ratio): assert data.target.shape == target.shape +def test_random_pad(): + transfo = RandomPad(max_ratio=0.5, p=1.0) + assert repr(transfo) == "RandomPad(max_ratio=0.5, p=1.0)" + with pytest.raises(ValueError): + RandomPad(max_ratio=-0.1) + + # A white image with a dark rectangle: the padding is white, the content is untouched + img = torch.ones((3, 40, 100)) + img[:, 10:30, 20:80] = 0.0 + boxes = np.array([[0.2, 0.25, 0.8, 0.75]], dtype=np.float32) + polys = np.array([[[0.2, 0.25], [0.8, 0.25], [0.8, 0.75], [0.2, 0.75]]], dtype=np.float32) + mask = torch.ones((40, 100), dtype=torch.bool) + torch.manual_seed(0) + for target in (boxes, polys, {"words": boxes, "label": "abc"}): + out = transfo(Sample(image=img, target=target, mask=mask)) + _, new_h, new_w = out.image.shape + assert new_h >= 40 and new_w >= 100 and new_h <= 40 + 2 * 20 and new_w <= 100 + 2 * 20 + assert out.image.dtype == img.dtype + # The content is the dark rectangle on white, wherever it was moved + rows, cols = torch.nonzero(out.image[0] < 0.5, as_tuple=True) + top, left = int(rows.min()), int(cols.min()) + assert (int(rows.max()) - top + 1, int(cols.max()) - left + 1) == (20, 60) + assert torch.equal(out.image[:, top - 10 : top + 30, left - 20 : left + 80], img) + assert torch.all(out.image[:, :, : left - 20] == 1) and torch.all(out.image[:, : top - 10] == 1) + # The targets follow the content + box = (out.target["words"] if isinstance(out.target, dict) else out.target).reshape(-1, 2) + np.testing.assert_allclose(box.min(axis=0) * [new_w, new_h], [left, top], atol=1e-3) + np.testing.assert_allclose(box.max(axis=0) * [new_w, new_h], [left + 60, top + 20], atol=1e-3) + if isinstance(target, dict): + assert out.target["label"] == "abc" + # The padded areas are not valid content + assert out.mask.shape == (new_h, new_w) and int(out.mask.sum()) == 40 * 100 + + # The padding takes the background color, for float and uint8 images + gray = torch.full((3, 20, 50), 0.6) + out = RandomPad(max_ratio=1.0, p=1.0)(Sample(image=gray)) + assert torch.allclose(out.image, torch.tensor(0.6)) + gray_uint8 = torch.full((3, 20, 50), 153, dtype=torch.uint8) + out = RandomPad(max_ratio=1.0, p=1.0)(Sample(image=gray_uint8)) + assert out.image.dtype == torch.uint8 and torch.all(out.image == 153) + + # Non-geometric targets (e.g. a LaTeX label) are kept, invalid geometries are rejected + assert RandomPad(p=1.0)(Sample(image=img, target="x ^ { 2 }")).target == "x ^ { 2 }" + with pytest.raises(ValueError): + RandomPad(p=1.0)(Sample(image=img, target=np.zeros((2, 2), dtype=np.float32))) + + # p=0: unchanged + sample = Sample(image=img, target=boxes) + assert RandomPad(p=0.0)(sample) is sample + + # ---------------------------------------------------------------------------- # End-to-end tests for SampleCompose with geometric and photometric transforms # ----------------------------------------------------------------------------