Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/source/api/models.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ directly. See the [model zoo](../models/index.md) for benchmarks.
```{eval-rst}
.. autofunction:: doctr.models.table_structure.table_predictor

.. autoclass:: doctr.models.table_structure.predictor.TablePredictor
:members: predict_regions

.. autofunction:: doctr.models.table_structure.tablecenternet
```

Expand Down
2 changes: 1 addition & 1 deletion docs/source/models/table_structure.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ Table structure models parse the image of a table into a machine-understandable

| Architecture | Input shape | # params | Recall | Precision | F1 | Structure acc. | sec/it (batch 1) |
| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |
| `tablecenternet` | (1024, 1024, 3) | 7.1 M | 93.82 | 96.96 | 95.36 | 76.84 | 0.5 |
| `tablecenternet` | (1024, 1024, 3) | 7.1 M | 96.41 | 96.50 | 96.46 | 78.85 | 0.5 |

The metrics are produced by `references/table/evaluate.py` with
{class}`~doctr.utils.metrics.TableCellMetric`: cell detection **recall**, **precision** and **F1** (cells matched
Expand Down
20 changes: 17 additions & 3 deletions docs/source/user_guide/layout_and_tables.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,10 @@ otherwise the regions cannot be located and nothing is masked. The CLI enables i
## Table structure recognition

With `detect_tables=True`, each region the layout model labels as a `Table` is cropped and passed to a table
structure model (`tablecenternet`), which predicts its cells and the rows and columns they span. The words falling
inside a cell are regrouped into a {class}`~doctr.io.Table`:
structure model (`tablecenternet`), which predicts its cells and the rows and columns they span. Every word inside the
table region is regrouped into a {class}`~doctr.io.Table`: the words outside of the predicted cells (e.g. a missed
header row or a row of totals) complete the grid, and the rows and columns follow the text lines of the cells (the
rows and columns without any text are removed):

```python
from doctr.io import DocumentFile
Expand Down Expand Up @@ -135,7 +137,7 @@ for cell in table.cells:
| `row_span`, `col_span` | Number of rows / columns spanned |
| `geometry` | Position of the cell, relative to the page |
| `confidence` | Mean recognition confidence of the words of the cell |
| `objectness_score` | Score of the cell predicted by the table model |
| `objectness_score` | Table model score of the cell (`None` for a cell made from words) |

## Standalone predictors

Expand Down Expand Up @@ -171,6 +173,18 @@ out[0]["cells"][0]
# {'geometry': [[x1, y1], ..., [x4, y4]], # relative to the table image
# 'score': 0.42, 'row_start': 0, 'row_end': 0, 'col_start': 0, 'col_end': 0}
```

With the layout regions of whole pages and their words (e.g. from your own OCR), the table regions are cropped and
the words assigned to the cells:

```python
from doctr.models import layout_predictor

regions = layout_predictor("lw_detr_s", pretrained=True)(pages)
tables = predictor.predict_regions(pages, regions, word_boxes) # one (N, 4) array of relative boxes per page

tables[0][0]["cells"][0]["words"] # indices of the words of the cell in `word_boxes[0]`, in reading order
```
:::
::::

Expand Down
260 changes: 33 additions & 227 deletions doctr/models/builder.py

Large diffs are not rendered by default.

94 changes: 1 addition & 93 deletions doctr/models/predictor/pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from collections.abc import Iterable, Sequence
from typing import Any

import cv2
import numpy as np
import torch
from torch import nn
Expand Down Expand Up @@ -211,7 +210,7 @@ def _predict_pages(

# Recognize table structure: locate tables with the layout model, crop each one and run the table model
tables = (
self._tables_from_regions(pages, regions, **kwargs)
self.table_predictor.predict_regions(pages, regions, boxes, self.table_class_name, **kwargs)
if self.table_predictor is not None and regions is not None
else None
)
Expand All @@ -232,94 +231,3 @@ def _predict_pages(
if self.straighten_pages and self.preserve_original_coords:
out = self._remap_to_original_coords(out, _orig_shapes, _straight_shapes, m_invs, _orig_pages)
return out

def _tables_from_regions(
self,
pages: list[np.ndarray],
regions: list[dict[str, Any]],
**kwargs: Any,
) -> list[list[dict[str, Any]]]:
"""Crop the table regions found by the layout model and run the table model on each crop.

Each `Table` region is rectified to an upright crop with a perspective transform (so straight boxes and
rotated polygons are both handled), the table model is applied per crop, and the predicted cell polygons
are mapped back to page-relative coordinates through the inverse transform. A page therefore yields one
structured table per detected `Table` region.

Args:
pages: the (possibly straightened) page images
regions: the per-page layout predictions `{"class_names", "boxes", "scores"}`
**kwargs: keyword arguments forwarded to the table predictor

Returns:
a per-page list of table grids `{"cells": [...], "num_rows": int, "num_cols": int}` in page-relative
coordinates
"""
crops: list[np.ndarray] = []
# (page index, inverse transform, crop width, crop height, page width, page height)
crop_meta: list[tuple[int, np.ndarray, int, int, int, int]] = []
for p_idx, (page, region) in enumerate(zip(pages, regions)):
if region is None:
continue
h, w = page.shape[:2]
for cls_name, box in zip(region["class_names"], region["boxes"]):
if cls_name != self.table_class_name:
continue
pts = np.asarray(box, dtype=np.float32).reshape(-1, 2)
if pts.shape[0] == 2: # straight box (x_min, y_min, x_max, y_max) -> corners
(bx0, by0), (bx1, by1) = pts
src = np.array([[bx0, by0], [bx1, by0], [bx1, by1], [bx0, by1]], dtype=np.float32)
else: # rotated 4-point polygon, already ordered (top-left, top-right, bottom-right, bottom-left)
src = pts.copy()
# Relative -> absolute pixel corners
src[:, 0] *= w
src[:, 1] *= h
# Upright crop size from the region side lengths
crop_w = int(round(max(np.linalg.norm(src[1] - src[0]), np.linalg.norm(src[2] - src[3]))))
crop_h = int(round(max(np.linalg.norm(src[3] - src[0]), np.linalg.norm(src[2] - src[1]))))
if crop_w < 2 or crop_h < 2:
continue
# Map the region onto an inset rectangle so the warp samples `pad` extra pixels of page
# context on each side. Crop-relative [0, 1] still maps exactly to [0, out_w] / [0, out_h].
pad = max(20, round(0.01 * max(crop_w, crop_h)))
out_w, out_h = crop_w + 2 * pad, crop_h + 2 * pad
dst = np.array(
[[pad, pad], [pad + crop_w, pad], [pad + crop_w, pad + crop_h], [pad, pad + crop_h]],
dtype=np.float32,
)
transform = cv2.getPerspectiveTransform(src, dst)
inverse = cv2.getPerspectiveTransform(dst, src)
crops.append(cv2.warpPerspective(page, transform, (out_w, out_h), borderMode=cv2.BORDER_REPLICATE))
crop_meta.append((p_idx, inverse, out_w, out_h, w, h))

tables_per_page: list[list[dict[str, Any]]] = [[] for _ in pages]
if len(crops) == 0:
return tables_per_page

grids = self.table_predictor(crops, **kwargs) # type: ignore[misc]
for (p_idx, inverse, crop_w, crop_h, w, h), grid in zip(crop_meta, grids):
remapped_cells: list[dict[str, Any]] = []
for cell in grid["cells"]:
# Cell polygon is crop-relative -> crop pixels -> page pixels (inverse transform) -> page-relative
raw = np.asarray(cell["geometry"], dtype=np.float32)
if raw.ndim == 1 and raw.size == 4: # straight box (xmin, ymin, xmax, ymax)
x0, y0, x1, y1 = raw
poly = np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32)
else:
poly = raw.reshape(-1, 2)
poly[:, 0] *= crop_w
poly[:, 1] *= crop_h
poly = cv2.perspectiveTransform(poly[None, :, :], inverse)[0]
poly[:, 0] /= w
poly[:, 1] /= h
new_cell = dict(cell)
new_cell["geometry"] = poly.tolist()
remapped_cells.append(new_cell)
if not remapped_cells:
continue
tables_per_page[p_idx].append({
"cells": remapped_cells,
"num_rows": grid["num_rows"],
"num_cols": grid["num_cols"],
})
return tables_per_page
70 changes: 60 additions & 10 deletions doctr/models/reading_order/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ def _key_value_rows(

def _column_gutter(
boxes: np.ndarray, voters: np.ndarray, y_overlap: np.ndarray, y_overlap_threshold: float
) -> np.ndarray | None:
) -> float | None:
"""Detect a multi-column page: a vertical line separating the voters into two groups crossed by few of them

Args:
Expand All @@ -421,30 +421,66 @@ def _column_gutter(
y_overlap_threshold: the vertical overlap above which two boxes lie on the same row

Returns:
the mask of the boxes crossing the gutter for a multi-column page, None for a single-column page
the position of the gutter for a multi-column page, None for a single-column page
"""
if np.count_nonzero(voters) < 3:
voters = np.ones(boxes.shape[0], dtype=bool)
x0, x1 = boxes[:, 0], boxes[:, 2]
vx0, vx1, num_voters = x0[voters], x1[voters], int(np.count_nonzero(voters))
if num_voters < 3:
return None
vy = (boxes[voters, 1] + boxes[voters, 3]) / 2
# Measured on the voters only, so that a wide figure does not shift the search window
left_edge = float(vx0.min())
span = float(vx1.max()) - left_edge or 1.0
tolerance = max(1, int(0.05 * num_voters))
centers = (vx0 + vx1) / 2
lo, hi = left_edge + 0.25 * span, left_edge + 0.75 * span
for split in np.unique(vx1[(vx1 >= lo) & (vx1 <= hi)]):
crossing = int(np.count_nonzero(np.minimum(vx1 - split, split - vx0) > 0.02 * span))
across = np.minimum(vx1 - split, split - vx0) > 0.02 * span
on_left, on_right = ~across & (centers <= split), ~across & (centers > split)
if not on_left.any() or not on_right.any():
continue
# Elements above or below the columns (e.g. a title and its authors) do not count as crossing
columns_y = vy[on_left | on_right]
crossing = int(np.count_nonzero(across & (vy > columns_y.min()) & (vy < columns_y.max())))
left = int(np.count_nonzero(centers <= split))
if crossing <= tolerance and left >= 0.25 * num_voters and num_voters - left >= 0.25 * num_voters:
if _key_value_rows(boxes, voters, float(split), y_overlap, y_overlap_threshold, span):
beside = voters.copy()
beside[np.flatnonzero(voters)[across]] = False
if _key_value_rows(boxes, beside, float(split), y_overlap, y_overlap_threshold, span):
return None # rows of keys and values (e.g. a receipt): read row after row
return np.minimum(x1 - split, split - x0) > 0.02 * span
return float(split)
return None


def _detached_above(y0: np.ndarray, y1: np.ndarray, column: np.ndarray, top: float) -> np.ndarray:
"""The elements of a column separated by a wide gap from both the rest of the column and the other columns below
them (e.g. a right-aligned title or address block above the columns of a page)

Args:
y0: the top of each box
y1: the bottom of each box
column: the indices of the elements of the column
top: the position the other columns start at

Returns:
the indices of the detached elements
"""
column = column[np.argsort(y0[column], kind="stable")]
level = np.flatnonzero(y1[column] > top)
if level.size == 0 or level[0] == 0:
return column[:0]
first = int(level[0])
bottoms = np.maximum.accumulate(y1[column])
gaps = y0[column[1:]] - bottoms[:-1]
# Wide: more than 3 times the usual gap of the column, and than the height of an element
limit = max(3 * float(np.median(gaps[first:])) if gaps[first:].size else 0.0, float(np.median(y1 - y0)))
while first > 0 and gaps[first - 1] <= limit:
first -= 1
return column[:first] if first > 0 and top - bottoms[first - 1] > limit else column[:0]


def _topological_order(
boxes: np.ndarray, x_overlap_threshold: float, y_overlap_threshold: float, voters: np.ndarray | None = None
) -> list[int]:
Expand Down Expand Up @@ -494,15 +530,27 @@ def _topological_order(
np.fill_diagonal(edges, False)

# A multi-column page favors the continuation of the current column (cf. `_continuation`)
spanning = (x1 - x0) > 0.5 * (float(x1.max() - x0.min()) or 1.0)
crossing = _column_gutter(
span = float(x1.max() - x0.min()) or 1.0
spanning = (x1 - x0) > 0.5 * span
gutter = _column_gutter(
boxes, np.ones(num_boxes, dtype=bool) if voters is None else voters, y_overlap, y_overlap_threshold
)
multi_column = crossing is not None
multi_column = gutter is not None
crossing = np.zeros(num_boxes, dtype=bool)
# 1 for the columns after the gutter, read after the ones before it even when they start higher (only when both
# sides hold several elements)
after_gutter = np.zeros(num_boxes, dtype=int)
if gutter is not None:
crossing = np.minimum(x1 - gutter, gutter - x0) > 0.02 * span
after = x0 >= gutter - 0.02 * span
before = ~after & ~crossing
if np.count_nonzero(after) >= 2 and np.count_nonzero(before) >= 2:
after_gutter = after.astype(int)
after_gutter[_detached_above(y0, y1, np.flatnonzero(after), float(y0[before].min()))] = 0
# Column components: elements connected through horizontal overlap, except those which must not merge the columns
# (page-wide or crossing the gutter). This keeps the traversal inside a column even when the direct top-to-bottom
# continuation is momentarily broken by a fragmented or misaligned OCR line.
bridging = spanning | crossing if multi_column else spanning
bridging = spanning | crossing
col_edges = np.triu(x_linked, 1) # each pair once
col_edges &= ~bridging[:, None]
col_edges &= ~bridging[None, :]
Expand All @@ -529,9 +577,11 @@ def _continuation(last: int, ready: np.ndarray) -> np.ndarray:
if ready.size == 0: # cycle safety net (can only happen with degenerate overlapping geometries)
candidates = np.flatnonzero(~emitted)
else:
# The columns before the gutter first
ready = ready[after_gutter[ready] == after_gutter[ready].min()]
candidates = _continuation(order[-1], ready) if order else ready
# Topmost first, then leftmost (in canonical space)
next_idx = int(candidates[np.lexsort((x0[candidates], y0[candidates]))[0]])
next_idx = int(candidates[np.lexsort((x0[candidates], y0[candidates], after_gutter[candidates]))[0]])
order.append(next_idx)
emitted[next_idx] = True
in_degree = in_degree - edges[next_idx]
Expand Down
Loading
Loading