diff --git a/docs/source/api/models.md b/docs/source/api/models.md index 091f0efff6..ef43708046 100644 --- a/docs/source/api/models.md +++ b/docs/source/api/models.md @@ -88,6 +88,9 @@ directly. See the [model zoo](../models/index.md) for benchmarks. ```{eval-rst} .. autofunction:: doctr.models.table_structure.table_predictor +.. autoclass:: doctr.models.table_structure.predictor.TablePredictor + :members: predict_regions + .. autofunction:: doctr.models.table_structure.tablecenternet ``` diff --git a/docs/source/models/table_structure.md b/docs/source/models/table_structure.md index e51a493103..0f1579c0ee 100644 --- a/docs/source/models/table_structure.md +++ b/docs/source/models/table_structure.md @@ -13,7 +13,7 @@ Table structure models parse the image of a table into a machine-understandable | Architecture | Input shape | # params | Recall | Precision | F1 | Structure acc. | sec/it (batch 1) | | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| `tablecenternet` | (1024, 1024, 3) | 7.1 M | 93.82 | 96.96 | 95.36 | 76.84 | 0.5 | +| `tablecenternet` | (1024, 1024, 3) | 7.1 M | 96.41 | 96.50 | 96.46 | 78.85 | 0.5 | The metrics are produced by `references/table/evaluate.py` with {class}`~doctr.utils.metrics.TableCellMetric`: cell detection **recall**, **precision** and **F1** (cells matched diff --git a/docs/source/user_guide/layout_and_tables.md b/docs/source/user_guide/layout_and_tables.md index 5ad7ece8ff..c00201bcfc 100644 --- a/docs/source/user_guide/layout_and_tables.md +++ b/docs/source/user_guide/layout_and_tables.md @@ -68,8 +68,10 @@ otherwise the regions cannot be located and nothing is masked. The CLI enables i ## Table structure recognition With `detect_tables=True`, each region the layout model labels as a `Table` is cropped and passed to a table -structure model (`tablecenternet`), which predicts its cells and the rows and columns they span. The words falling -inside a cell are regrouped into a {class}`~doctr.io.Table`: +structure model (`tablecenternet`), which predicts its cells and the rows and columns they span. Every word inside the +table region is regrouped into a {class}`~doctr.io.Table`: the words outside of the predicted cells (e.g. a missed +header row or a row of totals) complete the grid, and the rows and columns follow the text lines of the cells (the +rows and columns without any text are removed): ```python from doctr.io import DocumentFile @@ -135,7 +137,7 @@ for cell in table.cells: | `row_span`, `col_span` | Number of rows / columns spanned | | `geometry` | Position of the cell, relative to the page | | `confidence` | Mean recognition confidence of the words of the cell | -| `objectness_score` | Score of the cell predicted by the table model | +| `objectness_score` | Table model score of the cell (`None` for a cell made from words) | ## Standalone predictors @@ -171,6 +173,18 @@ out[0]["cells"][0] # {'geometry': [[x1, y1], ..., [x4, y4]], # relative to the table image # 'score': 0.42, 'row_start': 0, 'row_end': 0, 'col_start': 0, 'col_end': 0} ``` + +With the layout regions of whole pages and their words (e.g. from your own OCR), the table regions are cropped and +the words assigned to the cells: + +```python +from doctr.models import layout_predictor + +regions = layout_predictor("lw_detr_s", pretrained=True)(pages) +tables = predictor.predict_regions(pages, regions, word_boxes) # one (N, 4) array of relative boxes per page + +tables[0][0]["cells"][0]["words"] # indices of the words of the cell in `word_boxes[0]`, in reading order +``` ::: :::: diff --git a/doctr/models/builder.py b/doctr/models/builder.py index eb74d37c82..946cae4e18 100644 --- a/doctr/models/builder.py +++ b/doctr/models/builder.py @@ -9,7 +9,6 @@ from collections.abc import Sequence from typing import Any -import cv2 import numpy as np from scipy.cluster.hierarchy import fclusterdata @@ -26,9 +25,9 @@ TableCell, Word, ) +from doctr.models.table_structure.grid import assemble_page_table from doctr.utils.geometry import ( estimate_page_angle, - order_points, resolve_enclosing_bbox, resolve_enclosing_rbbox, rotate_boxes, @@ -450,260 +449,67 @@ def _build_layout_elements(regions: dict[str, Any] | None) -> list[LayoutElement elements.append(LayoutElement(layout_type=str(cname), confidence=float(score), geometry=geometry)) return elements - @staticmethod - def _word_centers(boxes: np.ndarray) -> np.ndarray: - """Return the (x, y) center of each word box. - - Args: - boxes: word boxes of shape (N, 4) (straight: x1, y1, x2, y2) or (N, 4, 2) (rotated polygon) - - Returns: - array of shape (N, 2) with the relative center coordinates of each box - """ - if boxes.ndim == 3: # rotated polygons (N, 4, 2) - return boxes.mean(axis=1) - return np.stack([(boxes[:, 0] + boxes[:, 2]) / 2, (boxes[:, 1] + boxes[:, 3]) / 2], axis=1) - - @staticmethod - def _as_cell_polygon(geometry: Any) -> np.ndarray: - """Normalize a cell geometry to a (4, 2) polygon. - - Straight-page table predictions store cells as flat (xmin, ymin, xmax, ymax) boxes, while rotated-page - predictions store (4, 2) corner polygons: both are mapped to a (4, 2) polygon (TL, TR, BR, BL). - - Args: - geometry: the raw cell geometry - - Returns: - array of shape (4, 2) with the polygon vertices - """ - arr = np.asarray(geometry, dtype=np.float32) - if arr.ndim == 1: # straight box (xmin, ymin, xmax, ymax) - x0, y0, x1, y1 = arr - return np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) - return order_points(arr.reshape(-1, 2)) - - @staticmethod - def _points_in_polygons(points: np.ndarray, polys: np.ndarray) -> np.ndarray: - """Vectorized ray casting: test every point against every polygon at once. - - Args: - points: array of shape (N, 2) with the (x, y) coordinates of the points - polys: array of shape (C, V, 2) with the polygon vertices - - Returns: - boolean array of shape (N, C), True where point n lies inside polygon c - """ - if points.shape[0] == 0 or polys.shape[0] == 0: - return np.zeros((points.shape[0], polys.shape[0]), dtype=bool) - px = points[:, 0].astype(np.float64)[:, None, None] - py = points[:, 1].astype(np.float64)[:, None, None] - vi = polys.astype(np.float64) # (C, V, 2) - vj = np.roll(vi, 1, axis=1) # previous vertex, with wrap-around - xi, yi = vi[None, ..., 0], vi[None, ..., 1] - xj, yj = vj[None, ..., 0], vj[None, ..., 1] - crossing = ((yi > py) != (yj > py)) & (px < (xj - xi) * (py - yi) / (yj - yi + 1e-12) + xi) - # A point is inside when a ray crosses the polygon boundary an odd number of times - return (crossing.sum(axis=-1) % 2).astype(bool) - - @staticmethod - def _order_cell_words(w_idcs: list[int], centers: np.ndarray, heights: np.ndarray) -> list[int]: - """Order the words of a table cell in reading order: rows top to bottom, words left to right. - - Args: - w_idcs: indices of the cell's words - centers: word centers of shape (N, 2), de-skewed for rotated pages - heights: per-word heights of shape (N,) (rotation-invariant) - - Returns: - the cell's word indices in reading order - """ - idcs = sorted(w_idcs, key=lambda i: float(centers[i][1])) - med_height = float(np.median([heights[i] for i in idcs])) - if not np.isfinite(med_height) or med_height <= 0: - med_height = 1.0 - rows: list[list[int]] = [[idcs[0]]] - y_sum = float(centers[idcs[0]][1]) - for idx in idcs[1:]: - if float(centers[idx][1]) - y_sum / len(rows[-1]) < med_height / 2: - rows[-1].append(idx) - y_sum += float(centers[idx][1]) - else: - rows.append([idx]) - y_sum = float(centers[idx][1]) - return [idx for row in rows for idx in sorted(row, key=lambda i: float(centers[i][0]))] - - @staticmethod - def _localize_logic(cells: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int, int]: - """Re-index a table's logical coordinates to a local 0-based grid. - - The table model returns logical (row/column) coordinates that may carry a constant offset; shifting them - so that the smallest row/column start is 0 makes the grid directly usable by :meth:`Table.to_grid`. - - Args: - cells: the cells of a single table - - Returns: - a tuple `(cells, num_rows, num_cols)` with the re-indexed cells and the table dimensions - """ - min_row = min(int(c["row_start"]) for c in cells) - min_col = min(int(c["col_start"]) for c in cells) - norm: list[dict[str, Any]] = [] - max_row = max_col = 0 - for c in cells: - nc = dict(c) - nc["row_start"] = int(c["row_start"]) - min_row - nc["row_end"] = int(c["row_end"]) - min_row - nc["col_start"] = int(c["col_start"]) - min_col - nc["col_end"] = int(c["col_end"]) - min_col - max_row, max_col = max(max_row, nc["row_end"]), max(max_col, nc["col_end"]) - norm.append(nc) - return norm, max_row + 1, max_col + 1 - - @classmethod - def _table_grids( - cls, page_table: dict[str, Any] | list[dict[str, Any]] | None - ) -> list[tuple[list[dict[str, Any]], int, int]]: - """Normalize the table structure prediction(s) of a page to non-empty grids with local logical coordinates - - Args: - page_table: a single grid, a list of grids or None (cf. `_build_tables`) - - Returns: - the cells, number of rows and number of columns of each non-empty grid - """ - raw_tables = [] if page_table is None else [page_table] if isinstance(page_table, dict) else list(page_table) - return [cls._localize_logic(raw["cells"]) for raw in raw_tables if raw and len(raw.get("cells", [])) > 0] - - @classmethod - def _assign_cell_words(cls, centers: np.ndarray, polys: np.ndarray, consumed: np.ndarray) -> list[list[int]]: - """Assign each word not yet consumed to at most one cell of a table, and flag it as consumed - - A word goes to the first cell containing its center. A word landing just outside every cell but inside the - table goes to the nearest cell, within half a median cell diagonal, so table text is not dropped into the body. - - Args: - centers: the (N, 2) word centers - polys: the (C, 4, 2) cell polygons, vertices ordered TL, TR, BR, BL - consumed: the (N,) mask of the words already assigned to a table, updated in place - - Returns: - the word indices of each cell - """ - cell_words: list[list[int]] = [[] for _ in polys] - free = np.flatnonzero(~consumed) - if free.size == 0: - return cell_words - inside = cls._points_in_polygons(centers[free], polys) # (F, C) - assigned = inside.any(axis=1) - for w_idx, c_idx in zip(free[assigned], inside[assigned].argmax(axis=1)): - cell_words[c_idx].append(int(w_idx)) - consumed[w_idx] = True - leftover = free[~assigned] - if leftover.size > 0: - cell_centers = polys.mean(axis=1) # (C, 2) - max_dist = 0.5 * float(np.median(np.linalg.norm(polys[:, 2] - polys[:, 0], axis=1))) # TL->BR diagonal - outline = cv2.boxPoints(cv2.minAreaRect(polys.reshape(-1, 2).astype(np.float32))).reshape(1, 4, 2) - in_table = cls._points_in_polygons(centers[leftover], outline)[:, 0] - for w_idx in leftover[in_table]: - dists = np.linalg.norm(cell_centers - centers[w_idx], axis=1) - nearest = int(dists.argmin()) - if dists[nearest] <= max_dist: - cell_words[nearest].append(int(w_idx)) - consumed[w_idx] = True - return cell_words - - @staticmethod - def _deskew_centers(centers: np.ndarray, polys: np.ndarray) -> np.ndarray: - """Rotate word centers by the angle of a table, to order the words along its own axes - - Args: - centers: the (N, 2) word centers - polys: the (C, 4, 2) cell polygons, vertices ordered TL, TR, BR, BL - - Returns: - the (N, 2) de-skewed centers - """ - top_edges = polys[:, 1] - polys[:, 0] # TR - TL - angle = float(np.median(np.arctan2(top_edges[:, 1], top_edges[:, 0]))) - cos_a, sin_a = np.cos(-angle), np.sin(-angle) - pivot = centers.mean(axis=0) - shifted = centers - pivot - return np.stack( - [ - pivot[0] + shifted[:, 0] * cos_a - shifted[:, 1] * sin_a, - pivot[1] + shifted[:, 0] * sin_a + shifted[:, 1] * cos_a, - ], - axis=1, - ) - def _build_tables( self, boxes: np.ndarray, word_preds: list[tuple[str, float]], page_table: dict[str, Any] | list[dict[str, Any]] | None, + page_shape: tuple[int, int], ) -> tuple[list[Table], np.ndarray]: - """Assign detected words to table cells and build the page tables. + """Build the tables of a page from their structure and the words of the page. Args: boxes: word boxes of the page, of shape (N, 4) or (N, 4, 2), in relative coordinates word_preds: list of (text, confidence) for each of the N words page_table: the table structure prediction(s) for the page. Either a single grid `{"cells": [{"geometry", "score", "row_start", "row_end", "col_start", "col_end"}], "num_rows", - "num_cols"}` (cell geometries in page-relative coordinates), a list of such grids, or None + "num_cols"}` (cell geometries in page-relative coordinates), a list of such grids, or None. The cells + may list the indices of their words (`"words"`, cf. `TablePredictor.predict_regions`), otherwise the + words are assigned to them + page_shape: the (height, width) of the page Returns: a tuple with the list of `Table` objects of the page (one per provided table) and a boolean mask of shape (N,) that is True for words assigned to a table (to be removed from `blocks`) """ consumed = np.zeros(boxes.shape[0], dtype=bool) - grids = self._table_grids(page_table) - if len(grids) == 0: - return [], consumed - # The geometries follow the page's word geometry: straight 2-point boxes for (N, 4) word boxes, 4-point polygons - # for (N, 4, 2) ones. Word heights are rotation-invariant (left edge length for polygons). + raw_tables = [] if page_table is None else [page_table] if isinstance(page_table, dict) else list(page_table) + # Geometries follow the word boxes: 2-point boxes for (N, 4), 4-point polygons for (N, 4, 2) straight = boxes.ndim != 3 - centers = self._word_centers(boxes) if boxes.shape[0] > 0 else np.empty((0, 2)) - if boxes.shape[0] == 0: - heights = np.empty(0) - else: - heights = boxes[:, 3] - boxes[:, 1] if straight else np.linalg.norm(boxes[:, 3] - boxes[:, 0], axis=1) - tables: list[Table] = [] - for cells, num_rows, num_cols in grids: - polys = np.stack([self._as_cell_polygon(cell["geometry"]) for cell in cells]) # (C, 4, 2) - order_centers = centers if straight or centers.shape[0] == 0 else self._deskew_centers(centers, polys) + for raw in raw_tables: + if not raw or len(raw.get("cells", [])) == 0: + continue + cells, num_rows, num_cols = assemble_page_table(raw["cells"], boxes, page_shape, consumed) table_cells = [] - for cell, poly, w_idcs in zip(cells, polys, self._assign_cell_words(centers, polys, consumed)): - if len(w_idcs) > 0: - # Reading order inside the cell: rows top to bottom, words left to right (table axes) - ordered = self._order_cell_words(w_idcs, order_centers, heights) - value = " ".join(word_preds[i][0] for i in ordered) - confidence = float(np.mean([word_preds[i][1] for i in ordered])) - else: - value, confidence = "", float(cell["score"]) + for cell in cells: + words = [word_preds[idx] for idx in cell["words"]] table_cells.append( TableCell( - value=value, - confidence=confidence, - geometry=self._export_geometry(poly, straight), - row_start=int(cell["row_start"]), - row_end=int(cell["row_end"]), - col_start=int(cell["col_start"]), - col_end=int(cell["col_end"]), - objectness_score=float(cell["score"]), + value=" ".join(value for value, _ in words), + # An empty cell falls back to the score of the table model + confidence=float(np.mean([conf for _, conf in words])) if words else float(cell["score"] or 0), + geometry=self._export_geometry(cell["geometry"], straight), + row_start=cell["row_start"], + row_end=cell["row_end"], + col_start=cell["col_start"], + col_end=cell["col_end"], + objectness_score=cell["score"], ) ) - # Enclosing geometry of the whole table - (xmin, ymin), (xmax, ymax) = polys.reshape(-1, 2).min(axis=0), polys.reshape(-1, 2).max(axis=0) - corners = np.asarray([[xmin, ymin], [xmax, ymin], [xmax, ymax], [xmin, ymax]]) + pts = np.concatenate([cell["geometry"] for cell in cells]) + (xmin, ymin), (xmax, ymax) = pts.min(axis=0), pts.max(axis=0) + # The cells made from words have no score + scores = [cell["score"] for cell in cells if cell["score"] is not None] tables.append( Table( cells=table_cells, num_rows=num_rows, num_cols=num_cols, - geometry=self._export_geometry(corners, straight), - confidence=float(np.mean([float(cell["score"]) for cell in cells])), + geometry=self._export_geometry( + np.array([[xmin, ymin], [xmax, ymin], [xmax, ymax], [xmin, ymax]]), straight + ), + confidence=float(np.mean(scores)) if scores else 0.0, ) ) return tables, consumed @@ -911,7 +717,7 @@ def __call__( _tables, ): # Build the page tables and flag the words that belong to a table - page_tables, consumed = self._build_tables(page_boxes, word_preds, page_table) + page_tables, consumed = self._build_tables(page_boxes, word_preds, page_table, shape) if consumed.any(): # Remove the words assigned to a table from the regular blocks output keep = ~consumed diff --git a/doctr/models/predictor/pytorch.py b/doctr/models/predictor/pytorch.py index ada7b96c21..027f28c183 100644 --- a/doctr/models/predictor/pytorch.py +++ b/doctr/models/predictor/pytorch.py @@ -6,7 +6,6 @@ from collections.abc import Iterable, Sequence from typing import Any -import cv2 import numpy as np import torch from torch import nn @@ -211,7 +210,7 @@ def _predict_pages( # Recognize table structure: locate tables with the layout model, crop each one and run the table model tables = ( - self._tables_from_regions(pages, regions, **kwargs) + self.table_predictor.predict_regions(pages, regions, boxes, self.table_class_name, **kwargs) if self.table_predictor is not None and regions is not None else None ) @@ -232,94 +231,3 @@ def _predict_pages( if self.straighten_pages and self.preserve_original_coords: out = self._remap_to_original_coords(out, _orig_shapes, _straight_shapes, m_invs, _orig_pages) return out - - def _tables_from_regions( - self, - pages: list[np.ndarray], - regions: list[dict[str, Any]], - **kwargs: Any, - ) -> list[list[dict[str, Any]]]: - """Crop the table regions found by the layout model and run the table model on each crop. - - Each `Table` region is rectified to an upright crop with a perspective transform (so straight boxes and - rotated polygons are both handled), the table model is applied per crop, and the predicted cell polygons - are mapped back to page-relative coordinates through the inverse transform. A page therefore yields one - structured table per detected `Table` region. - - Args: - pages: the (possibly straightened) page images - regions: the per-page layout predictions `{"class_names", "boxes", "scores"}` - **kwargs: keyword arguments forwarded to the table predictor - - Returns: - a per-page list of table grids `{"cells": [...], "num_rows": int, "num_cols": int}` in page-relative - coordinates - """ - crops: list[np.ndarray] = [] - # (page index, inverse transform, crop width, crop height, page width, page height) - crop_meta: list[tuple[int, np.ndarray, int, int, int, int]] = [] - for p_idx, (page, region) in enumerate(zip(pages, regions)): - if region is None: - continue - h, w = page.shape[:2] - for cls_name, box in zip(region["class_names"], region["boxes"]): - if cls_name != self.table_class_name: - continue - pts = np.asarray(box, dtype=np.float32).reshape(-1, 2) - if pts.shape[0] == 2: # straight box (x_min, y_min, x_max, y_max) -> corners - (bx0, by0), (bx1, by1) = pts - src = np.array([[bx0, by0], [bx1, by0], [bx1, by1], [bx0, by1]], dtype=np.float32) - else: # rotated 4-point polygon, already ordered (top-left, top-right, bottom-right, bottom-left) - src = pts.copy() - # Relative -> absolute pixel corners - src[:, 0] *= w - src[:, 1] *= h - # Upright crop size from the region side lengths - crop_w = int(round(max(np.linalg.norm(src[1] - src[0]), np.linalg.norm(src[2] - src[3])))) - crop_h = int(round(max(np.linalg.norm(src[3] - src[0]), np.linalg.norm(src[2] - src[1])))) - if crop_w < 2 or crop_h < 2: - continue - # Map the region onto an inset rectangle so the warp samples `pad` extra pixels of page - # context on each side. Crop-relative [0, 1] still maps exactly to [0, out_w] / [0, out_h]. - pad = max(20, round(0.01 * max(crop_w, crop_h))) - out_w, out_h = crop_w + 2 * pad, crop_h + 2 * pad - dst = np.array( - [[pad, pad], [pad + crop_w, pad], [pad + crop_w, pad + crop_h], [pad, pad + crop_h]], - dtype=np.float32, - ) - transform = cv2.getPerspectiveTransform(src, dst) - inverse = cv2.getPerspectiveTransform(dst, src) - crops.append(cv2.warpPerspective(page, transform, (out_w, out_h), borderMode=cv2.BORDER_REPLICATE)) - crop_meta.append((p_idx, inverse, out_w, out_h, w, h)) - - tables_per_page: list[list[dict[str, Any]]] = [[] for _ in pages] - if len(crops) == 0: - return tables_per_page - - grids = self.table_predictor(crops, **kwargs) # type: ignore[misc] - for (p_idx, inverse, crop_w, crop_h, w, h), grid in zip(crop_meta, grids): - remapped_cells: list[dict[str, Any]] = [] - for cell in grid["cells"]: - # Cell polygon is crop-relative -> crop pixels -> page pixels (inverse transform) -> page-relative - raw = np.asarray(cell["geometry"], dtype=np.float32) - if raw.ndim == 1 and raw.size == 4: # straight box (xmin, ymin, xmax, ymax) - x0, y0, x1, y1 = raw - poly = np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) - else: - poly = raw.reshape(-1, 2) - poly[:, 0] *= crop_w - poly[:, 1] *= crop_h - poly = cv2.perspectiveTransform(poly[None, :, :], inverse)[0] - poly[:, 0] /= w - poly[:, 1] /= h - new_cell = dict(cell) - new_cell["geometry"] = poly.tolist() - remapped_cells.append(new_cell) - if not remapped_cells: - continue - tables_per_page[p_idx].append({ - "cells": remapped_cells, - "num_rows": grid["num_rows"], - "num_cols": grid["num_cols"], - }) - return tables_per_page diff --git a/doctr/models/reading_order/base.py b/doctr/models/reading_order/base.py index ff1265ece9..975a2f0217 100644 --- a/doctr/models/reading_order/base.py +++ b/doctr/models/reading_order/base.py @@ -411,7 +411,7 @@ def _key_value_rows( def _column_gutter( boxes: np.ndarray, voters: np.ndarray, y_overlap: np.ndarray, y_overlap_threshold: float -) -> np.ndarray | None: +) -> float | None: """Detect a multi-column page: a vertical line separating the voters into two groups crossed by few of them Args: @@ -421,7 +421,7 @@ def _column_gutter( y_overlap_threshold: the vertical overlap above which two boxes lie on the same row Returns: - the mask of the boxes crossing the gutter for a multi-column page, None for a single-column page + the position of the gutter for a multi-column page, None for a single-column page """ if np.count_nonzero(voters) < 3: voters = np.ones(boxes.shape[0], dtype=bool) @@ -429,6 +429,7 @@ def _column_gutter( vx0, vx1, num_voters = x0[voters], x1[voters], int(np.count_nonzero(voters)) if num_voters < 3: return None + vy = (boxes[voters, 1] + boxes[voters, 3]) / 2 # Measured on the voters only, so that a wide figure does not shift the search window left_edge = float(vx0.min()) span = float(vx1.max()) - left_edge or 1.0 @@ -436,15 +437,50 @@ def _column_gutter( centers = (vx0 + vx1) / 2 lo, hi = left_edge + 0.25 * span, left_edge + 0.75 * span for split in np.unique(vx1[(vx1 >= lo) & (vx1 <= hi)]): - crossing = int(np.count_nonzero(np.minimum(vx1 - split, split - vx0) > 0.02 * span)) + across = np.minimum(vx1 - split, split - vx0) > 0.02 * span + on_left, on_right = ~across & (centers <= split), ~across & (centers > split) + if not on_left.any() or not on_right.any(): + continue + # Elements above or below the columns (e.g. a title and its authors) do not count as crossing + columns_y = vy[on_left | on_right] + crossing = int(np.count_nonzero(across & (vy > columns_y.min()) & (vy < columns_y.max()))) left = int(np.count_nonzero(centers <= split)) if crossing <= tolerance and left >= 0.25 * num_voters and num_voters - left >= 0.25 * num_voters: - if _key_value_rows(boxes, voters, float(split), y_overlap, y_overlap_threshold, span): + beside = voters.copy() + beside[np.flatnonzero(voters)[across]] = False + if _key_value_rows(boxes, beside, float(split), y_overlap, y_overlap_threshold, span): return None # rows of keys and values (e.g. a receipt): read row after row - return np.minimum(x1 - split, split - x0) > 0.02 * span + return float(split) return None +def _detached_above(y0: np.ndarray, y1: np.ndarray, column: np.ndarray, top: float) -> np.ndarray: + """The elements of a column separated by a wide gap from both the rest of the column and the other columns below + them (e.g. a right-aligned title or address block above the columns of a page) + + Args: + y0: the top of each box + y1: the bottom of each box + column: the indices of the elements of the column + top: the position the other columns start at + + Returns: + the indices of the detached elements + """ + column = column[np.argsort(y0[column], kind="stable")] + level = np.flatnonzero(y1[column] > top) + if level.size == 0 or level[0] == 0: + return column[:0] + first = int(level[0]) + bottoms = np.maximum.accumulate(y1[column]) + gaps = y0[column[1:]] - bottoms[:-1] + # Wide: more than 3 times the usual gap of the column, and than the height of an element + limit = max(3 * float(np.median(gaps[first:])) if gaps[first:].size else 0.0, float(np.median(y1 - y0))) + while first > 0 and gaps[first - 1] <= limit: + first -= 1 + return column[:first] if first > 0 and top - bottoms[first - 1] > limit else column[:0] + + def _topological_order( boxes: np.ndarray, x_overlap_threshold: float, y_overlap_threshold: float, voters: np.ndarray | None = None ) -> list[int]: @@ -494,15 +530,27 @@ def _topological_order( np.fill_diagonal(edges, False) # A multi-column page favors the continuation of the current column (cf. `_continuation`) - spanning = (x1 - x0) > 0.5 * (float(x1.max() - x0.min()) or 1.0) - crossing = _column_gutter( + span = float(x1.max() - x0.min()) or 1.0 + spanning = (x1 - x0) > 0.5 * span + gutter = _column_gutter( boxes, np.ones(num_boxes, dtype=bool) if voters is None else voters, y_overlap, y_overlap_threshold ) - multi_column = crossing is not None + multi_column = gutter is not None + crossing = np.zeros(num_boxes, dtype=bool) + # 1 for the columns after the gutter, read after the ones before it even when they start higher (only when both + # sides hold several elements) + after_gutter = np.zeros(num_boxes, dtype=int) + if gutter is not None: + crossing = np.minimum(x1 - gutter, gutter - x0) > 0.02 * span + after = x0 >= gutter - 0.02 * span + before = ~after & ~crossing + if np.count_nonzero(after) >= 2 and np.count_nonzero(before) >= 2: + after_gutter = after.astype(int) + after_gutter[_detached_above(y0, y1, np.flatnonzero(after), float(y0[before].min()))] = 0 # Column components: elements connected through horizontal overlap, except those which must not merge the columns # (page-wide or crossing the gutter). This keeps the traversal inside a column even when the direct top-to-bottom # continuation is momentarily broken by a fragmented or misaligned OCR line. - bridging = spanning | crossing if multi_column else spanning + bridging = spanning | crossing col_edges = np.triu(x_linked, 1) # each pair once col_edges &= ~bridging[:, None] col_edges &= ~bridging[None, :] @@ -529,9 +577,11 @@ def _continuation(last: int, ready: np.ndarray) -> np.ndarray: if ready.size == 0: # cycle safety net (can only happen with degenerate overlapping geometries) candidates = np.flatnonzero(~emitted) else: + # The columns before the gutter first + ready = ready[after_gutter[ready] == after_gutter[ready].min()] candidates = _continuation(order[-1], ready) if order else ready # Topmost first, then leftmost (in canonical space) - next_idx = int(candidates[np.lexsort((x0[candidates], y0[candidates]))[0]]) + next_idx = int(candidates[np.lexsort((x0[candidates], y0[candidates], after_gutter[candidates]))[0]]) order.append(next_idx) emitted[next_idx] = True in_degree = in_degree - edges[next_idx] diff --git a/doctr/models/table_structure/grid.py b/doctr/models/table_structure/grid.py new file mode 100644 index 0000000000..f9985445ff --- /dev/null +++ b/doctr/models/table_structure/grid.py @@ -0,0 +1,430 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +"""Assemble the cells of a table structure model and the words of the table into a consistent grid. + +Boxes are `(x0, y0, x1, y1)` in the upright table frame, logical coordinates are `(row_start, row_end, col_start, +col_end)` (0-indexed, inclusive). +""" + +from typing import Any + +import numpy as np + +__all__ = ["fill_table_grid", "assemble_page_table"] + + +def as_polygons(geometries: Any) -> np.ndarray: + """Convert straight boxes or (TL, TR, BR, BL) polygons to (N, 4, 2) float32 polygons""" + flat = np.asarray(geometries, dtype=np.float32) + flat = flat.reshape(len(flat), -1) if flat.size else np.zeros((0, 4), dtype=np.float32) + if flat.shape[1] == 4: + x0, y0, x1, y1 = flat.T + return np.stack([np.stack(pt, axis=-1) for pt in ((x0, y0), (x1, y0), (x1, y1), (x0, y1))], axis=1) + return flat.reshape(-1, 4, 2) + + +def _grid_lines(start: np.ndarray, end: np.ndarray, lo: np.ndarray, hi: np.ndarray, size: int) -> np.ndarray: + """Positions of the `size + 1` grid lines of an axis: the median of the cell edges on each line, interpolated + where no edge lies""" + edges: list[list[float]] = [[] for _ in range(size + 1)] + for s, e, low, high in zip(start, end, lo, hi): + edges[int(s)].append(float(low)) + edges[int(e) + 1].append(float(high)) + known = [k for k, values in enumerate(edges) if values] + return np.maximum.accumulate(np.interp(np.arange(size + 1), known, [float(np.median(edges[k])) for k in known])) + + +def _group(values: np.ndarray, tol: float) -> np.ndarray: + """Group index of each 1D position, a group gathering the positions closer than `tol` to its mean""" + groups = np.zeros(values.size, dtype=np.int64) + group = 0 + members: list[int] = [] + for j in np.argsort(values, kind="stable"): + if members and values[j] - np.mean(values[members]) >= tol: + group, members = group + 1, [] + groups[j] = group + members.append(j) + return groups + + +def _axis_slots(lo: np.ndarray, hi: np.ndarray, lines: np.ndarray, line_tol: float | None) -> np.ndarray: + """Index of each word along an axis. The words beyond the grid get new indices, grouped by text line (rows, + `line_tol`) or by overlapping extent (columns, `line_tol=None`)""" + size, center = lines.size - 1, (lo + hi) / 2 + idx = np.clip(np.searchsorted(lines, center, side="right") - 1, 0, size - 1) + for before, mask in ((True, center < lines[0]), (False, center > lines[-1])): + members = np.flatnonzero(mask) + if members.size == 0: + continue + if line_tol is not None: + groups = _group(center[members], line_tol) + else: # chain the overlapping extents + groups, group, reach = np.zeros(members.size, dtype=np.int64), -1, -np.inf + for j in np.argsort(lo[members], kind="stable"): + if lo[members[j]] > reach: + group += 1 + groups[j], reach = group, max(reach, hi[members[j]]) + idx[members] = groups - (groups.max() + 1) if before else size + groups + return idx + + +class _Grid: + """The cells of a table: boxes, logical coordinates, word indices and source cell (-1 if made from words)""" + + def __init__(self, boxes: np.ndarray, logic: np.ndarray, words: np.ndarray): + self.boxes = [np.asarray(b, dtype=np.float64) for b in boxes] + self.logic = [list(map(int, lc)) for lc in logic] + self.cell_words: list[list[int]] = [[] for _ in self.boxes] + self.origin = list(range(len(self.boxes))) + self.words = words + + def add(self, box: np.ndarray, logic: list[int], words: list[int], origin: int = -1) -> None: + self.boxes.append(np.asarray(box, dtype=np.float64)) + self.logic.append(logic) + self.cell_words.append(words) + self.origin.append(origin) + + def keep(self, mask: np.ndarray) -> None: + for name in ("boxes", "logic", "cell_words", "origin"): + setattr(self, name, [v for v, k in zip(getattr(self, name), mask) if k]) + + def localize(self) -> None: + """Shift the logical coordinates so that the first row and column are 0""" + logic = np.asarray(self.logic) + logic -= [logic[:, 0].min()] * 2 + [logic[:, 2].min()] * 2 + self.logic = logic.tolist() + + def text_box(self, c_idx: int) -> np.ndarray: + boxes = self.words[self.cell_words[c_idx]] + return np.concatenate([boxes[:, :2].min(axis=0), boxes[:, 2:].max(axis=0)]) + + def lines(self, axis: int) -> np.ndarray: + """The grid lines of the rows (axis 0) or columns (axis 1)""" + logic, boxes = np.asarray(self.logic), np.asarray(self.boxes) + lo, hi = (1, 3) if axis == 0 else (0, 2) + size = int(logic[:, 2 * axis + 1].max()) + 1 + return _grid_lines(logic[:, 2 * axis], logic[:, 2 * axis + 1], boxes[:, lo], boxes[:, hi], size) + + +def _complete(grid: _Grid, leftover: np.ndarray, line_height: float) -> None: + """Place the words lying in no cell: in the cell covering their slot, or in a new cell (a missed cell, or a new + row / column beyond the grid)""" + words = grid.words[leftover] + row_lines, col_lines = grid.lines(0), grid.lines(1) + rows = _axis_slots(words[:, 1], words[:, 3], row_lines, line_height / 2) + cols = _axis_slots(words[:, 0], words[:, 2], col_lines, None) + occupied: dict[tuple[int, int], int] = {} + for c_idx, (rs, re, cs, ce) in enumerate(grid.logic): + for r in range(rs, re + 1): + for c in range(cs, ce + 1): + occupied.setdefault((r, c), c_idx) + + new_cells: dict[tuple[int, int, int, int], list[int]] = {} + in_grid = (rows >= 0) & (rows < row_lines.size - 1) + for j in np.flatnonzero(in_grid): + slot = (int(rows[j]), int(cols[j])) + if slot in occupied: + grid.cell_words[occupied[slot]].append(int(leftover[j])) + else: + new_cells.setdefault((slot[0], slot[0], slot[1], slot[1]), []).append(int(j)) + for r in np.unique(rows[~in_grid]): + # New row: a word starts a new cell in another column, unless it continues a sentence + members = np.flatnonzero(rows == r) + groups: list[list[int]] = [] + for j in members[np.argsort(words[members, 0], kind="stable")]: + prev = groups[-1][-1] if groups else None + if prev is None or (cols[j] != cols[prev] and words[j, 0] - words[prev, 2] >= line_height / 2): + groups.append([]) + groups[-1].append(int(j)) + spans: list[tuple[int, int, list[int]]] = [] + for group in groups: + lo, hi = int(cols[group].min()), int(cols[group].max()) + while spans and lo <= spans[-1][1]: # overlapping words: a column belongs to a single cell + prev_lo, prev_hi, prev_group = spans.pop() + lo, hi, group = min(lo, prev_lo), max(hi, prev_hi), prev_group + group + spans.append((lo, hi, group)) + for lo, hi, group in spans: + new_cells[(int(r), int(r), lo, hi)] = group + + def band(lines: np.ndarray, start: int, end: int) -> tuple[float, float]: + # Unbounded beyond the grid + size = lines.size - 1 + return (lines[start] if 0 <= start < size else np.inf, lines[end + 1] if 0 <= end < size else -np.inf) + + for (rs, re, cs, ce), positions in new_cells.items(): + text = words[positions] + (x0, x1), (y0, y1) = band(col_lines, cs, ce), band(row_lines, rs, re) + box = [min(x0, text[:, 0].min()), min(y0, text[:, 1].min())] + box += [max(x1, text[:, 2].max()), max(y1, text[:, 3].max())] + grid.add(np.array(box), [rs, re, cs, ce], [int(leftover[j]) for j in positions]) + grid.localize() + + +def _merge_and_prune(grid: _Grid) -> None: + """Merge the adjacent rows / columns the model split (complementary text on the same text line / column), then + remove the rows / columns in which no single-span cell holds text""" + logic = np.asarray(grid.logic) + filled = np.array([len(w) > 0 for w in grid.cell_words]) + if not filled.any(): + return + text = np.array([grid.text_box(i) if filled[i] else np.full(4, np.nan) for i in range(len(grid.boxes))]) + boxes = np.asarray(grid.boxes) + alive = np.ones(len(boxes), dtype=bool) + + def mergeable(axis: int, k: int) -> bool: + start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] + o_start, o_end = logic[:, 2 - 2 * axis], logic[:, 3 - 2 * axis] + in_k, in_next = alive & (start <= k) & (end >= k), alive & (start <= k + 1) & (end >= k + 1) + a, b = np.flatnonzero(in_k & ~in_next & filled), np.flatnonzero(in_next & ~in_k & filled) + if a.size == 0 or b.size == 0: + return False + if np.any((o_start[a, None] <= o_end[None, b]) & (o_start[None, b] <= o_end[a, None])): + return False # both hold text in a same column / row + lo, hi = (1, 3) if axis == 0 else (0, 2) + a_lo, a_hi, b_lo, b_hi = text[a, lo].min(), text[a, hi].max(), text[b, lo].min(), text[b, hi].max() + return bool(min(a_hi, b_hi) - max(a_lo, b_lo) >= 0.5 * min(a_hi - a_lo, b_hi - b_lo)) + + def shift(axis: int, k: int, merge: bool) -> None: + # Merge the index k + 1 into k, or remove the index k + start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] + end[alive & ((end > k) if merge else (end >= k))] -= 1 + start[alive & (start > k)] -= 1 + + def absorb_overlaps() -> None: + # A cell absorbs the empty cells now sharing a slot with it + for i in np.flatnonzero(alive & filled).tolist() + np.flatnonzero(alive & ~filled).tolist(): + if not alive[i]: + continue + others = np.flatnonzero(alive & ~filled) + others = others[others != i] + hits = others[ + (logic[others, 0] <= logic[i, 1]) + & (logic[i, 0] <= logic[others, 1]) + & (logic[others, 2] <= logic[i, 3]) + & (logic[i, 2] <= logic[others, 3]) + ] + if hits.size: + boxes[i, :2] = np.minimum(boxes[i, :2], boxes[hits, :2].min(axis=0)) + boxes[i, 2:] = np.maximum(boxes[i, 2:], boxes[hits, 2:].max(axis=0)) + logic[i, ::2] = np.minimum(logic[i, ::2], logic[hits, ::2].min(axis=0)) + logic[i, 1::2] = np.maximum(logic[i, 1::2], logic[hits, 1::2].max(axis=0)) + alive[hits] = False + + changed = True + while changed: + changed = False + for axis in (0, 1): + for k in range(int(logic[alive, 2 * axis + 1].max())): + if mergeable(axis, k): + shift(axis, k, merge=True) + absorb_overlaps() + changed = True + break + for axis in (0, 1): + start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] + for k in range(int(end[alive].max()), -1, -1): + if not np.any(alive & filled & (start == k) & (end == k)): + alive[alive & (start == k) & (end == k)] = False + shift(axis, k, merge=False) + grid.boxes, grid.logic = list(boxes), logic.tolist() + grid.keep(alive) + grid.localize() + + +def _split_rows(grid: _Grid, line_height: float) -> None: + """Split the rows the model merged along their text lines: a row most of whose filled cells hold several lines + (a single cell wrapping its text does not split its row), and a spanning cell whose text lines each lie level + with a different row""" + centers = (grid.words[:, 1] + grid.words[:, 3]) / 2 + + def split(c_idx: int, line_of: np.ndarray, spans: list[tuple[int, int]]) -> None: + # One cell per text line, cut midway between the lines (the text may lie beyond the box of its cell) + words, (x0, y0, x1, y1) = np.asarray(grid.cell_words[c_idx]), grid.boxes[c_idx] + tops = [grid.words[words[line_of == k], 1].min() for k in range(len(spans))] + bottoms = [grid.words[words[line_of == k], 3].max() for k in range(len(spans))] + cuts = [min(y0, tops[0]), *[(b + t) / 2 for b, t in zip(bottoms[:-1], tops[1:])], max(y1, bottoms[-1])] + _, _, cs, ce = grid.logic[c_idx] + for k, (rs, re) in enumerate(spans): + box, logic, members = ( + np.array([x0, cuts[k], x1, cuts[k + 1]]), + [rs, re, cs, ce], + words[line_of == k].tolist(), + ) + if k == 0: + grid.boxes[c_idx], grid.logic[c_idx], grid.cell_words[c_idx] = box, logic, members + else: + grid.add(box, logic, members, grid.origin[c_idx]) + + # From the bottom up, so that the rows still to process do not move + for r in range(max(lc[1] for lc in grid.logic), -1, -1): + in_row = [i for i, lc in enumerate(grid.logic) if lc[0] == lc[1] == r] + filled = [i for i in in_row if grid.cell_words[i]] + words = [w for i in filled for w in grid.cell_words[i]] + if len(filled) < 2: + continue + line_of = dict(zip(words, _group(centers[words], line_height / 2).tolist())) + n_lines = max(line_of.values()) + 1 + multi = [i for i in filled if len({line_of[w] for w in grid.cell_words[i]}) > 1] + if len(multi) < 2 or 2 * len(multi) < len(filled): + continue + for lc in grid.logic: # the rows below move down, the cells spanning the row grow + if not lc[0] == lc[1] == r: + lc[0] += n_lines - 1 if lc[0] > r else 0 + lc[1] += n_lines - 1 if lc[1] >= r else 0 + for i in in_row: + if not grid.cell_words[i]: + grid.logic[i][1] = r + n_lines - 1 + continue + lines = np.array([line_of[w] for w in grid.cell_words[i]]) + present = np.unique(lines) + split(i, np.searchsorted(present, lines), [(r + k, r + k) for k in present]) + + def level_with_row(c_idx: int, row: int, lo: float, hi: float) -> bool: + # Whether a text line (lo to hi) is level with the text of another cell of the row + return any( + j != c_idx + and grid.logic[j][0] == row == grid.logic[j][1] + and any(min(hi, y1) - max(lo, y0) > 0.5 * (hi - lo) for _, y0, _, y1 in grid.words[grid.cell_words[j]]) + for j in range(len(grid.boxes)) + ) + + row_lines = grid.lines(0) + for i in range(len(grid.boxes)): + rs, re = grid.logic[i][:2] + if re == rs or len(grid.cell_words[i]) < 2: + continue + idcs = np.asarray(grid.cell_words[i]) + cell_lines = _group(centers[idcs], line_height / 2) + rows: list[int] = [] # row of each text line + for k in range(int(cell_lines.max()) + 1): + lo, hi = grid.words[idcs[cell_lines == k], 1].min(), grid.words[idcs[cell_lines == k], 3].max() + row = int(np.clip(np.searchsorted(row_lines, (lo + hi) / 2, side="right") - 1, rs, re)) + if row in rows or not level_with_row(i, row, lo, hi): + rows = [] + break + rows.append(row) + if len(rows) > 1: + # Each text line spans until the row of the next one + starts, ends = [rs, *rows[1:]], [*(row - 1 for row in rows[1:]), re] + split(i, cell_lines, list(zip(starts, ends))) + + +def _reading_order(words: np.ndarray, idcs: list[int], line_height: float) -> list[int]: + """Order the words of a cell: text lines top to bottom, words left to right""" + lines = _group((words[idcs, 1] + words[idcs, 3]) / 2, line_height / 2) + return [idcs[j] for j in np.lexsort((words[idcs, 0], lines))] + + +def fill_table_grid( + cells: np.ndarray, logic: np.ndarray, words: np.ndarray +) -> tuple[np.ndarray, np.ndarray, list[list[int]], list[int]]: + """Make the grid of a table consistent with its text, and assign every word of the table to a cell. + + The words outside of every cell complete the grid, the rows / columns split by the model are merged, the ones + without text are removed, and the rows merged by the model are split along their text lines. + + Args: + cells: the (C, 4) cell boxes predicted by the table model + logic: the (C, 4) logical coordinates of the cells + words: the (W, 4) boxes of the words of the table, in the same frame + + Returns: + the cell boxes, their local logical coordinates, the indices of their words in reading order, and the index of + the predicted cell each one comes from (-1 for a cell made from words) + """ + grid = _Grid(cells, logic, np.asarray(words, dtype=np.float64).reshape(-1, 4)) + if len(grid.boxes) == 0: + return np.zeros((0, 4)), np.zeros((0, 4), dtype=np.int64), [], [] + grid.localize() + w = grid.words + if w.shape[0] > 0: + centers = (w[:, :2] + w[:, 2:]) / 2 + boxes = np.asarray(grid.boxes) + inside = ( + (boxes[None, :, 0] <= centers[:, None, 0]) + & (centers[:, None, 0] <= boxes[None, :, 2]) + & (boxes[None, :, 1] <= centers[:, None, 1]) + & (centers[:, None, 1] <= boxes[None, :, 3]) + ) + assigned = inside.any(axis=1) + for w_idx, c_idx in zip(np.flatnonzero(assigned), inside[assigned].argmax(axis=1)): + grid.cell_words[c_idx].append(int(w_idx)) + line_height = float(np.median(w[:, 3] - w[:, 1])) + line_height = line_height if line_height > 0 else 1.0 + if not assigned.all(): + _complete(grid, np.flatnonzero(~assigned), line_height) + _merge_and_prune(grid) + _split_rows(grid, line_height) + grid.localize() + grid.cell_words = [_reading_order(w, idcs, line_height) if idcs else [] for idcs in grid.cell_words] + return np.asarray(grid.boxes), np.asarray(grid.logic, dtype=np.int64), grid.cell_words, grid.origin + + +def _rotate(points: np.ndarray, angle: float, pivot: np.ndarray) -> np.ndarray: + """Rotate (..., 2) points by `angle` radians around `pivot`""" + cos_a, sin_a = np.cos(angle), np.sin(angle) + x, y = points[..., 0] - pivot[0], points[..., 1] - pivot[1] + return np.stack([pivot[0] + x * cos_a - y * sin_a, pivot[1] + x * sin_a + y * cos_a], axis=-1) + + +def assemble_page_table( + cells: list[dict[str, Any]], word_boxes: np.ndarray, page_shape: tuple[int, int], taken: np.ndarray +) -> tuple[list[dict[str, Any]], int, int]: + """Assemble a table given in page coordinates with the words of the page. + + The cells listing their words (`"words"`) keep them, otherwise the free words inside the outline of the cells are + assigned with `fill_table_grid`. + + Args: + cells: the cells `{"geometry", "score", "row_start", "row_end", "col_start", "col_end"}` of the table, with + page-relative geometries (straight boxes or polygons) + word_boxes: the (N, 4) or (N, 4, 2) page-relative word boxes + page_shape: the (height, width) of the page + taken: the (N,) mask of the words already assigned to a table, updated in place + + Returns: + the cells (local logical coordinates, (4, 2) page-relative polygon, `"words"` in reading order), the number of + rows and the number of columns + """ + scale = np.array([page_shape[1], page_shape[0]], dtype=np.float64) + polys = as_polygons([cell["geometry"] for cell in cells]) * scale + logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells], dtype=np.int64) + if all("words" in cell for cell in cells): + cell_words = [[int(i) for i in cell["words"] if not taken[i]] for cell in cells] + origin = list(range(len(cells))) + logic -= [logic[:, 0].min()] * 2 + [logic[:, 2].min()] * 2 + else: + # Upright frame of the table rows + top_edges = polys[:, 1] - polys[:, 0] + angle, pivot = float(np.median(np.arctan2(top_edges[:, 1], top_edges[:, 0]))), polys.reshape(-1, 2).mean(0) + upright = _rotate(polys, -angle, pivot) + boxes = np.concatenate([upright.min(axis=1), upright.max(axis=1)], axis=1) + free, word_coords = np.flatnonzero(~taken), np.zeros((0, 4)) + if free.size: + words = _rotate(as_polygons(word_boxes[free]) * scale, -angle, pivot) + centers = words.mean(axis=1) + inside = np.all((centers >= boxes[:, :2].min(axis=0)) & (centers <= boxes[:, 2:].max(axis=0)), axis=1) + free, words = free[inside], words[inside] + word_coords = np.concatenate([words.min(axis=1), words.max(axis=1)], axis=1) + boxes, logic, local_words, origin = fill_table_grid(boxes, logic, word_coords) + cell_words = [[int(free[j]) for j in idcs] for idcs in local_words] + corners = boxes[:, [[0, 1], [2, 1], [2, 3], [0, 3]]] + polys = _rotate(corners, angle, pivot) + for idcs in cell_words: + taken[idcs] = True + out = [] + for poly, (rs, re, cs, ce), idcs, source in zip(polys / scale, logic, cell_words, origin): + score = None if source < 0 else cells[source]["score"] + out.append({ + "geometry": poly, + "score": None if score is None else float(score), + "row_start": int(rs), + "row_end": int(re), + "col_start": int(cs), + "col_end": int(ce), + "words": idcs, + }) + return out, int(logic[:, 1].max()) + 1, int(logic[:, 3].max()) + 1 diff --git a/doctr/models/table_structure/predictor/pytorch.py b/doctr/models/table_structure/predictor/pytorch.py index 26a54b14a3..fd58efc756 100644 --- a/doctr/models/table_structure/predictor/pytorch.py +++ b/doctr/models/table_structure/predictor/pytorch.py @@ -6,6 +6,7 @@ from collections.abc import Iterable, Sequence from typing import Any +import cv2 import numpy as np import torch from torch import nn @@ -15,9 +16,31 @@ from doctr.models.preprocessor import PreProcessor from doctr.models.utils import set_device_and_dtype +from ..grid import as_polygons, fill_table_grid +from ..tablecenternet.base import _logic_from_geometry + __all__ = ["TablePredictor"] +def _crop_region(page: np.ndarray, region: Any) -> tuple[np.ndarray, np.ndarray, int] | None: + """Rectify a page-relative table region to an upright crop with a margin of page context + + Returns: + the crop, the page-to-crop perspective transform and the margin in pixels, or None for a degenerate region + """ + h, w = page.shape[:2] + src = as_polygons([region])[0] * np.array([w, h], dtype=np.float32) + crop_w = int(round(max(np.linalg.norm(src[1] - src[0]), np.linalg.norm(src[2] - src[3])))) + crop_h = int(round(max(np.linalg.norm(src[3] - src[0]), np.linalg.norm(src[2] - src[1])))) + if crop_w < 2 or crop_h < 2: + return None + pad = max(20, round(0.01 * max(crop_w, crop_h))) + dst = np.array([[pad, pad], [pad + crop_w, pad], [pad + crop_w, pad + crop_h], [pad, pad + crop_h]], np.float32) + transform = cv2.getPerspectiveTransform(src, dst) + crop = cv2.warpPerspective(page, transform, (crop_w + 2 * pad, crop_h + 2 * pad), borderMode=cv2.BORDER_REPLICATE) + return crop, transform, pad + + class TablePredictor(nn.Module): """Implements an object able to recognize the cell structure of tables in a document. @@ -106,3 +129,103 @@ def _predict(self, pages: list[np.ndarray], **kwargs: Any) -> list[dict[str, Any # logical coordinates are 0-indexed, so the table size is the largest index + 1 (0 if no cells) results.append({"cells": cells, "num_rows": max_row + 1, "num_cols": max_col + 1}) return results + + @torch.inference_mode() + def predict_regions( + self, + pages: list[np.ndarray], + regions: list[dict[str, Any] | None], + word_boxes: list[np.ndarray] | None = None, + class_name: str = "Table", + **kwargs: Any, + ) -> list[list[dict[str, Any]]]: + """Recognize the tables a layout model located on pages, and assign the words of the pages to their cells + + Each table region (straight box or polygon) is rectified to an upright crop, and the words lying in it are + assigned to the cells and complete the grid (cf. :func:`~doctr.models.table_structure.grid.fill_table_grid`). + + Args: + pages: the page images + regions: the per-page layout predictions `{"class_names", "boxes", "scores"}` (None for none) + word_boxes: the per-page word boxes, of shape (N, 4) or (N, 4, 2) in relative coordinates + class_name: the layout class of the table regions + **kwargs: keyword arguments forwarded to the model + + Returns: + a per-page list of tables `{"cells": [...], "num_rows": int, "num_cols": int}` in page-relative + coordinates, each cell listing the page indices of its words in reading order (`"words"`) + """ + crops, meta = [], [] + for p_idx, (page, region) in enumerate(zip(pages, regions)): + region = region or {} + for cls_name, box in zip(region.get("class_names", []), region.get("boxes", [])): + cropped = _crop_region(page, box) if cls_name == class_name else None + if cropped is not None: + crops.append(cropped[0]) + meta.append((p_idx, cropped[1], cropped[2])) + tables: list[list[dict[str, Any]]] = [[] for _ in pages] + for (p_idx, transform, pad), crop, grid in zip(meta, crops, self(crops, **kwargs) if crops else []): + words = None if word_boxes is None else word_boxes[p_idx] + table = self._region_table(grid, crop.shape[:2], pages[p_idx].shape[:2], transform, pad, words) + if table is not None: + tables[p_idx].append(table) + return tables + + def _region_table( + self, + grid: dict[str, Any], + crop_shape: tuple[int, int], + page_shape: tuple[int, int], + transform: np.ndarray, + pad: int, + word_boxes: np.ndarray | None, + ) -> dict[str, Any] | None: + """Assemble the table of a region crop with the words of the page, in page-relative coordinates""" + (crop_h, crop_w), (h, w) = crop_shape, page_shape + if len(grid["cells"]) == 0: + return None + + def in_region(points: np.ndarray) -> np.ndarray: + return np.flatnonzero(np.all((points >= pad) & (points <= [crop_w - pad, crop_h - pad]), axis=1)) + + # Cells in crop pixels; the ones centered in the context margin belong to the neighbouring content + polys = as_polygons([cell["geometry"] for cell in grid["cells"]]) * [crop_w, crop_h] + keep = in_region(polys.mean(axis=1)) + if keep.size == 0: + return None + polys, cells = polys[keep], [grid["cells"][idx] for idx in keep] + logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells]) + geometric_logic = getattr(getattr(self.model, "postprocessor", None), "geometric_logic", False) + if geometric_logic: + # Index the rows / columns from the cell edges (TableCenterNet logic is ordered columns first) + by_col = _logic_from_geometry(polys, logic[:, [2, 3, 0, 1]]) + logic = by_col[:, [2, 3, 0, 1]] + # Words of the region, in crop pixels + word_idcs, words = np.zeros(0, dtype=np.int64), np.zeros((0, 4)) + if word_boxes is not None and len(word_boxes) > 0: + pts = as_polygons(word_boxes) * np.array([w, h], dtype=np.float32) + pts = cv2.perspectiveTransform(pts.reshape(1, -1, 2), transform).reshape(-1, 4, 2) + word_idcs = in_region(pts.mean(axis=1)) + words = np.concatenate([pts[word_idcs].min(axis=1), pts[word_idcs].max(axis=1)], axis=1) + boxes, cell_logic, cell_words, origin = fill_table_grid( + np.concatenate([polys.min(axis=1), polys.max(axis=1)], axis=1), logic, words + ) + # Crop pixels -> page-relative coordinates + corners = boxes[:, [[0, 1], [2, 1], [2, 3], [0, 3]]].reshape(1, -1, 2).astype(np.float32) + page_polys = cv2.perspectiveTransform(corners, np.linalg.inv(transform)).reshape(-1, 4, 2) / [w, h] + return { + "cells": [ + { + "geometry": poly.tolist(), + "score": None if source < 0 else float(cells[source]["score"]), # None for a cell made from words + "row_start": int(rs), + "row_end": int(re), + "col_start": int(cs), + "col_end": int(ce), + "words": [int(word_idcs[idx]) for idx in w_idcs], + } + for poly, (rs, re, cs, ce), w_idcs, source in zip(page_polys, cell_logic, cell_words, origin) + ], + "num_rows": int(cell_logic[:, 1].max()) + 1, + "num_cols": int(cell_logic[:, 3].max()) + 1, + } diff --git a/doctr/models/table_structure/tablecenternet/base.py b/doctr/models/table_structure/tablecenternet/base.py index b82b19c9e6..4725e8009e 100644 --- a/doctr/models/table_structure/tablecenternet/base.py +++ b/doctr/models/table_structure/tablecenternet/base.py @@ -147,79 +147,33 @@ def window(lo: np.ndarray | float, hi: np.ndarray | float): return np.sort(np.asarray(lines)) -def _vote_line_indices( - n_lines: int, start: np.ndarray, end: np.ndarray, model_logic: np.ndarray -) -> tuple[np.ndarray, np.ndarray]: - """Logical index of each grid line, voted by the model's logical coordinates of the cells lying on it. +def _index_lines(lines: np.ndarray, start: np.ndarray, end: np.ndarray, cell_size: float) -> np.ndarray: + """Logical index of each grid line of an axis, from the cells lying between the lines. + + Every cell spans at least one index, and the lines no cell separates share an index. A gap covered by no cell + counts as many indices as cells of the typical size fit in it. Args: - n_lines: number of grid lines + lines: the sorted positions of the grid lines, shape (L,) start: index of the line each cell starts on, shape (N,) - end: index of the line each cell ends on, shape (N,) - model_logic: the model's (start, end) logical coordinates of the cells along this axis, shape (N, 2) + end: index of the line each cell ends on (after its start line), shape (N,) + cell_size: the typical size of a cell along the axis Returns: - the (non-decreasing) logical index of each line, shape (n_lines,), and whether a strict majority of the cells - on the line voted for it, shape (n_lines,) + the (non-decreasing) logical index of each line, shape (L,) """ - votes: list[list[int]] = [[] for _ in range(n_lines)] - starts_of: list[list[int]] = [[] for _ in range(n_lines)] # start lines of the cells ending on each line - for s, e, (m_start, m_end) in zip(start, end, model_logic): - votes[s].append(int(m_start)) # a cell starts on its line at its start index - votes[e].append(int(m_end) + 1) # and ends on the line following its last index - starts_of[e].append(int(s)) - values = np.zeros(n_lines, dtype=np.int64) - majority = np.zeros(n_lines, dtype=bool) - prev = -1 - for k, v in enumerate(votes): - value = max(prev, 0) - if v: - uniq, counts = np.unique(v, return_counts=True) - value = int(uniq[counts.argmax()]) # most voted (the smallest one on ties) - # A split vote is not a drift of the model: some annotations index a same line differently per row - majority[k] = counts.max() > len(v) / 2 - if majority[k]: - # Lines are ordered: their indices can not decrease. Several lines may share an index (the edges of a - # row / column do not always line up), so they are not forced to strictly increase - value = max(value, prev) - # But a cell spans at least one index: the line moves on when at least half of the cells ending on it - # would span none - need = [values[s] + 1 for s in starts_of[k] if majority[s] and values[s] + 1 > value] - if need and 2 * len(need) >= len(starts_of[k]): - value = max(need) - values[k] = prev = value - else: - values[k] = value - return values, majority - - -def _connected_tables(left: np.ndarray, right: np.ndarray, top: np.ndarray, bottom: np.ndarray) -> np.ndarray: - """Label the groups of touching cells (deskewed extents), i.e. the separate tables of an image.""" - n = left.size - # Neighbouring cells of a table share their edges, up to a fraction of the smallest cell sizes - margin = 0.25 * float(min(np.percentile(right - left, 10), np.percentile(bottom - top, 10))) - touch = ( - (left[:, None] <= right[None] + margin) - & (left[None] <= right[:, None] + margin) - & (top[:, None] <= bottom[None] + margin) - & (top[None] <= bottom[:, None] + margin) - ) - labels = -np.ones(n, dtype=np.int64) - for seed in range(n): - if labels[seed] >= 0: - continue - labels[seed] = seed - stack = [seed] - while stack: - i = stack.pop() - for j in np.flatnonzero(touch[i] & (labels < 0)): - labels[j] = seed - stack.append(int(j)) - return labels + values = np.zeros(lines.size, dtype=np.int64) + for k in range(1, lines.size): + value = values[k - 1] + if not np.any((start <= k - 1) & (end >= k)): # no cell covers the gap between the 2 lines + value += int(round((lines[k] - lines[k - 1]) / max(cell_size, 1e-9))) + ending = start[end == k] + values[k] = max(value, int(values[ending].max()) + 1) if ending.size else value + return values def _align_table_logic(edges: tuple[np.ndarray, ...], logic: np.ndarray) -> np.ndarray: - """Align the (N, 4) logical coordinates of the cells of a single table on their deskewed edges.""" + """Align the (N, 4) logical coordinates of the cells of a table on their deskewed edges.""" left, right, top, bottom = edges logic = logic.copy() # The table may be rotated by any multiple of 90 degrees (or the corners ordered by the image axes): match each @@ -238,7 +192,7 @@ def _align_table_logic(edges: tuple[np.ndarray, ...], logic: np.ndarray) -> np.n if corr[g, a] < 0: # the logical indices grow against the geometric axis start_edge, end_edge = -end_edge, -start_edge # Tolerance on the alignment of the edges, from the narrowest cells along this axis: merging the 2 lines of a - # narrow row / column would lose it, while splitting a line only makes lines sharing the same index + # narrow row / column would lose it, while the lines split from a line that no cell separates share an index tol = max(0.4 * float(np.percentile(end_edge - start_edge, 10)), 1e-6) lines = _cluster_lines(np.concatenate([start_edge, end_edge]), tol) if lines.size < 2: # degenerate (flat) cells: nothing to align @@ -248,25 +202,20 @@ def _align_table_logic(edges: tuple[np.ndarray, ...], logic: np.ndarray) -> np.n end = np.abs(end_edge[:, None] - lines[None]).argmin(axis=1) end = np.clip(end, 1, lines.size - 1) start = np.minimum(start, end - 1) - values, majority = _vote_line_indices(lines.size, start, end, logic[:, offset : offset + 2]) - model_start, model_end = logic[:, offset], logic[:, offset + 1] - span = np.maximum(model_end - model_start, 0) - line_start, line_end = values[start], values[end] - 1 - new_start = np.where(majority[start], line_start, np.where(majority[end], line_end - span, model_start)) - new_end = np.where(majority[end], line_end, np.where(majority[start], line_start + span, model_end)) - logic[:, offset] = new_start - logic[:, offset + 1] = np.maximum(new_end, new_start) + values = _index_lines(lines, start, end, float(np.median(end_edge - start_edge))) + # Keep the absolute position the model predicts for most of the cells + values += max(int(round(float(np.median(logic[:, offset] - values[start])))), 0) + logic[:, offset] = values[start] + logic[:, offset + 1] = values[end] - 1 return logic def _logic_from_geometry(polys: np.ndarray, model_logic: np.ndarray) -> np.ndarray: - """Make the logical coordinates `[start_col, end_col, start_row, end_row]` (0-indexed) of (N, 4, 2) cells - consistent with their geometry. + """Make the logical coordinates `[start_col, end_col, start_row, end_row]` (0-indexed) of the (N, 4, 2) cells of a + table consistent with their geometry. - The cells are split into tables (groups of touching cells), and in each table the cell edges are grouped into - the row and column lines of the grid: each line takes the logical index the model predicts for most of the cells - on it. The cells of a visual row / column share their indices, even when the model drifts on some of them, while - the absolute indices of the model are kept (e.g. a row with no detected cell still counts). + The cell edges are grouped into the row and column lines of the grid, indexed from the geometry (cf. + `_index_lines`). The model coordinates only give the direction of the axes and the absolute position of the table. Args: polys: the cells, of shape (N, 4, 2) @@ -275,13 +224,10 @@ def _logic_from_geometry(polys: np.ndarray, model_logic: np.ndarray) -> np.ndarr Returns: the logical coordinates, of shape (N, 4) """ - edges = _cell_edges(_deskew_cells(polys)) - logic = np.asarray(model_logic, dtype=np.int64).copy() - labels = _connected_tables(*edges) - for label in np.unique(labels): - idx = np.flatnonzero(labels == label) - logic[idx] = _align_table_logic(tuple(e[idx] for e in edges), logic[idx]) - return logic + logic = np.asarray(model_logic, dtype=np.int64) + if logic.shape[0] == 0: + return logic.copy() + return _align_table_logic(_cell_edges(_deskew_cells(polys)), logic) def _lookup_logic(lc_map: np.ndarray, x: float, y: float) -> np.ndarray: @@ -305,9 +251,8 @@ class TableCenterNetPostProcessor: corner_thresh: minimum score for a corner to be used during relocation not_relocate: if True, skip the corner-relocation step (faster, less accurate) assume_straight_pages: whether the pages are assumed to be straight (i.e., no rotation) - geometric_logic: if True, remove the duplicate (overlapping) cells and make the logical coordinates - consistent with the cell geometry: the cells whose edges lie on the same grid line share the logical - index most of them are predicted with (the logical-coordinate head drifts on borderless tables) + geometric_logic: if True, remove the duplicate (overlapping) cells. `TablePredictor.predict_regions` then + indexes the rows and columns from the cell edges instead of the logical coordinates of the model """ def __init__( @@ -411,9 +356,9 @@ def __call__(self, decoded: dict[str, np.ndarray]) -> list[dict[str, np.ndarray] scores = cs[keep].astype(np.float32) logic = (logic[keep] - 1).astype(np.int32) # start_col, end_col, start_row, end_row (0-indexed) if self.geometric_logic and polys.shape[0] > 0: + # The logical coordinates stay the model's: `TablePredictor._region_table` indexes them from geometry kept = _suppress_overlapping_cells(polys, scores) - polys, scores = polys[kept], scores[kept] - logic = _logic_from_geometry(polys, logic[kept]).astype(np.int32) + polys, scores, logic = polys[kept], scores[kept], logic[kept] if self.assume_straight_pages: cells = np.concatenate([polys.min(axis=1), polys.max(axis=1)], axis=1).astype(np.float32) else: diff --git a/doctr/models/table_structure/tablecenternet/pytorch.py b/doctr/models/table_structure/tablecenternet/pytorch.py index 0a6e74a4b2..858a7ed6f8 100644 --- a/doctr/models/table_structure/tablecenternet/pytorch.py +++ b/doctr/models/table_structure/tablecenternet/pytorch.py @@ -26,7 +26,7 @@ "input_shape": (3, 1024, 1024), "mean": (0.798, 0.785, 0.772), "std": (0.264, 0.2749, 0.287), - "url": "https://github.com/mindee/doctr/releases/download/v1.1.0/tablecenternet-fdc82796.pt", + "url": "https://github.com/mindee/doctr/releases/download/v1.1.0/tablecenternet-b0d6f603.pt", }, } @@ -149,7 +149,7 @@ class TableCenterNet(nn.Module, _TableCenterNet): center_k: maximum number of cell centers corner_k: maximum number of corners not_relocate: if True, skip the corner-relocation step - geometric_logic: if True, remove the duplicate cells and align the logical coordinates on the cell geometry + geometric_logic: if True, remove the duplicate cells (cf. `TableCenterNetPostProcessor`) assume_straight_pages: if True, the predictor will fit straight boxes to the cells exportable: onnx exportable returns only the raw head maps cfg: the configuration dict of the model diff --git a/tests/common/test_models_builder.py b/tests/common/test_models_builder.py index 252960d3a7..1d15bf01f6 100644 --- a/tests/common/test_models_builder.py +++ b/tests/common/test_models_builder.py @@ -465,29 +465,6 @@ def test_resolve_lines(input_boxes, shape, lines): assert doc_builder._resolve_lines(np.asarray(input_boxes), shape) == lines -def test_table_geometry_helpers(): - polys = np.array( - [ - [[0.1, 0.1], [0.4, 0.1], [0.4, 0.3], [0.1, 0.3]], # axis-aligned quad - [[0.5, 0.5], [0.8, 0.6], [0.7, 0.9], [0.45, 0.8]], # rotated quad - ], - dtype=np.float32, - ) - points = np.array([[0.2, 0.2], [0.6, 0.7], [0.95, 0.95], [0.05, 0.05]], dtype=np.float32) - inside = builder.DocumentBuilder._points_in_polygons(points, polys) - assert inside.tolist() == [[True, False], [False, True], [False, False], [False, False]] - # empty inputs yield empty masks instead of raising - assert builder.DocumentBuilder._points_in_polygons(np.zeros((0, 2)), polys).shape == (0, 2) - assert builder.DocumentBuilder._points_in_polygons(points, np.zeros((0, 4, 2))).shape == (4, 0) - # flat straight box (xmin, ymin, xmax, ymax) -> (4, 2) polygon - poly = builder.DocumentBuilder._as_cell_polygon([0.1, 0.2, 0.5, 0.6]) - assert poly.shape == (4, 2) - assert np.allclose(poly, [[0.1, 0.2], [0.5, 0.2], [0.5, 0.6], [0.1, 0.6]]) - # (4, 2) polygons pass through unchanged - quad = np.array([[0.1, 0.2], [0.5, 0.25], [0.45, 0.6], [0.05, 0.55]], dtype=np.float32) - assert np.allclose(builder.DocumentBuilder._as_cell_polygon(quad), quad) - - def _table_cell(x0, y0, x1, y1, rs, re, cs, ce, score=0.9): return { "geometry": [[x0, y0], [x1, y0], [x1, y1], [x0, y1]], @@ -573,17 +550,18 @@ def test_documentbuilder_tables(): _table_cell(0.10, 0.10, 0.25, 0.20, 0, 0, 0, 0, score=0.8), _table_cell(0.28, 0.10, 0.45, 0.20, 0, 0, 1, 1, score=0.6), _table_cell(0.10, 0.22, 0.25, 0.32, 1, 1, 0, 0, score=0.4), # no word inside this cell + _table_cell(0.28, 0.22, 0.45, 0.32, 1, 1, 1, 1, score=0.5), ], "num_rows": 2, "num_cols": 2, } - page = doc_builder(*_table_words("Name", "Age", confidences=[0.95, 0.85]), tables=[table]).pages[0] + page = doc_builder(*_table_words("Name", "Age", "30", confidences=[0.95, 0.85, 0.75]), tables=[table]).pages[0] cells = page.tables[0].cells - assert [cell.value for cell in cells] == ["Name", "Age", ""] - assert [cell.objectness_score for cell in cells] == [0.8, 0.6, 0.4] - assert [cell.confidence for cell in cells] == pytest.approx([0.95, 0.85, 0.4]) - assert page.tables[0].confidence == pytest.approx(0.6) - assert [cell["objectness_score"] for cell in page.export()["tables"][0]["cells"]] == [0.8, 0.6, 0.4] + assert [cell.value for cell in cells] == ["Name", "Age", "", "30"] + assert [cell.objectness_score for cell in cells] == [0.8, 0.6, 0.4, 0.5] + assert [cell.confidence for cell in cells] == pytest.approx([0.95, 0.85, 0.4, 0.75]) + assert page.tables[0].confidence == pytest.approx(0.575) + assert [cell["objectness_score"] for cell in page.export()["tables"][0]["cells"]] == [0.8, 0.6, 0.4, 0.5] # Multi-word cells are read line by line, left to right words = ["world", "hello", "second", "line", "solo"] @@ -656,6 +634,24 @@ def test_documentbuilder_tables(): json.dumps(page.export()) +def test_documentbuilder_tables_assigned_words(): + # The OCR predictor assigns the words of a table to its cells (in reading order): the builder uses them as is + cells = [ + {**_table_cell(0.10, 0.10, 0.25, 0.20, 0, 0, 0, 0), "words": [1, 0]}, + {**_table_cell(0.28, 0.10, 0.45, 0.20, 0, 0, 1, 1), "words": []}, + # a cell made from the words of the table: not scored by the table model + {**_table_cell(0.10, 0.22, 0.45, 0.32, 1, 1, 0, 1, score=None), "words": [2]}, + ] + words = ["world", "hello", "total", "caption"] + boxes = [[0.15, 0.12, 0.2, 0.14], [0.11, 0.12, 0.14, 0.14], [0.6, 0.6, 0.7, 0.7], [0.3, 0.9, 0.4, 0.95]] + page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[{"cells": cells}]]).pages[0] + assert page.tables[0].to_grid() == [["hello world", ""], ["total", ""]] + assert [cell.objectness_score for cell in page.tables[0].cells] == [0.9, 0.9, None] + assert page.tables[0].confidence == pytest.approx(0.9) + assert _words(page) == ["caption"] + json.dumps(page.export()) + + @pytest.mark.parametrize( "words, boxes, grids, tables", [ diff --git a/tests/common/test_models_reading_order.py b/tests/common/test_models_reading_order.py index bc458397a8..fab1312b47 100644 --- a/tests/common/test_models_reading_order.py +++ b/tests/common/test_models_reading_order.py @@ -346,6 +346,52 @@ def test_layout_labels(label, normalized, role): list(range(7)), id="header-over-columns", ), + pytest.param( + # A centered title and its authors above 2 columns (e.g. the first page of a paper): more lines crossing + # the gutter than its tolerance, but none level with the columns + _column(0.15, 0.85, 0.02, 0.03, 2, 0.02) + + _column(0.35, 0.65, 0.09, 0.025, 4, 0.015) + + _column(0.05, 0.45, 0.2, 0.02, 40, 0.015) + + _column(0.55, 0.95, 0.2, 0.02, 40, 0.015), + {}, + list(range(86)), + id="title-and-authors-over-columns", + ), + pytest.param( + # The right column starts higher than the left one (e.g. beside a table whose caption is read with it): + # the left column is read first + _column(0.05, 0.45, 0.2, 0.03, 10, 0.02) + _column(0.55, 0.95, 0.15, 0.03, 10, 0.02), + {}, + list(range(20)), + id="right-column-starting-higher", + ), + pytest.param( + # A right-aligned title far above both columns is not the start of the right column + [_box(0.6, 0.03, 0.95, 0.06)] + + _column(0.05, 0.45, 0.2, 0.05, 6, 0.035) + + _column(0.55, 0.95, 0.2, 0.05, 6, 0.035), + {}, + list(range(13)), + id="right-aligned-title-over-columns", + ), + pytest.param( + # The same title above a right column starting higher: title, left column, right column + [_box(0.6, 0.03, 0.95, 0.06)] + + _column(0.05, 0.45, 0.3, 0.05, 6, 0.035) + + _column(0.55, 0.95, 0.2, 0.05, 8, 0.035), + {}, + list(range(15)), + id="right-aligned-title-over-higher-right-column", + ), + pytest.param( + # A heading right above the right column belongs to it + [_box(0.55, 0.14, 0.95, 0.18)] + + _column(0.05, 0.45, 0.2, 0.05, 6, 0.035) + + _column(0.55, 0.95, 0.2, 0.05, 6, 0.035), + {}, + [*range(1, 7), 0, *range(7, 13)], + id="heading-of-the-right-column", + ), pytest.param( SPLIT_CAPTION, {"labels": SPLIT_CAPTION_LABELS}, diff --git a/tests/common/test_models_table_structure.py b/tests/common/test_models_table_structure.py index 602097cbf4..5f3510f435 100644 --- a/tests/common/test_models_table_structure.py +++ b/tests/common/test_models_table_structure.py @@ -5,9 +5,9 @@ from doctr.models.table_structure.tablecenternet.base import ( _build_table_target, _cluster_lines, + _index_lines, _logic_from_geometry, _suppress_overlapping_cells, - _vote_line_indices, ) @@ -105,47 +105,46 @@ def test_tablecenternet_logic_from_geometry(angle): ) -def test_tablecenternet_logic_from_geometry_tables(): +def test_tablecenternet_logic_from_geometry_model_drift(): target = _grid_target(3, 3, use_polygons=True) polys, gt = target["cells"], target["logic"] drifted = gt.copy() drifted[2] += [0, 0, 1, 1] # last cell of row 0 one row too low - # 2 tables in an image: each one keeps its own indices (both start at 0) - top, bottom = polys * [1, 0.4], polys * [1, 0.4] + [0, 0.55] - cells = np.concatenate([top, bottom]) - np.testing.assert_array_equal(_logic_from_geometry(cells, np.concatenate([drifted, gt])), np.concatenate([gt, gt])) - # An upside-down table (rows growing upwards): the axes follow the model coordinates flipped = polys.copy() flipped[..., 1] = 1 - flipped[..., 1] np.testing.assert_array_equal(_logic_from_geometry(flipped, drifted), gt) - # The cells on a line do not agree on its index (2 against 2): the model coordinates are kept + # The model counts one more column in the 1st row than in the 2nd one, on a regular grid: the cells of a + # visual column share their index two_rows = _grid_target(2, 3, use_polygons=True) split = two_rows["logic"].copy() - split[:3, :2] += [[0, 1], [1, 1], [1, 1]] # the 1st row counts one more column than the 2nd one - np.testing.assert_array_equal(_logic_from_geometry(two_rows["cells"], split), split) - - -def test_tablecenternet_vote_line_indices(): - def vote(cells): - start, end, logic = (np.array(v) for v in zip(*cells)) - return _vote_line_indices(3, start, end, logic) - - # 3 cells between 3 lines: the 2 cells after the 1st line drift to the index of the 1st cell. The vote of line 1 - # is 0 but its cell would span no index: the line moves on (and so does the next one) - values, majority = vote([(0, 1, (0, 0)), (1, 2, (0, 0)), (1, 2, (0, 0))]) - np.testing.assert_array_equal(values, [0, 1, 2]) - assert majority.all() - # A minority of cells spanning no index (4 cells of a split header row, against 6 rows-high cells) does not move - # the line: the other ones would be shifted - cells = [(0, 1, (0, 0))] * 4 + [(1, 2, (1, 1))] * 4 + [(0, 2, (0, 0))] * 6 - values, _ = vote(cells) - np.testing.assert_array_equal(values, [0, 1, 1]) - # A split vote does not decide the index of a line - values, majority = vote([(0, 1, (0, 0)), (0, 1, (0, 1)), (1, 2, (1, 1)), (1, 2, (2, 2))]) - np.testing.assert_array_equal(majority, [True, False, False]) + split[:3, :2] += [[0, 1], [1, 1], [1, 1]] + np.testing.assert_array_equal(_logic_from_geometry(two_rows["cells"], split), two_rows["logic"]) + + # The model drifts by one row on half of the columns (e.g. a borderless table): the rows follow the geometry + grid = _grid_target(3, 4, use_polygons=True) + drifted = grid["logic"].copy() + drifted[drifted[:, 0] >= 2] += [0, 0, 1, 1] + np.testing.assert_array_equal(_logic_from_geometry(grid["cells"], drifted), grid["logic"]) + + +def test_tablecenternet_index_lines(): + def index(lines, cells, size=10.0): + start, end = (np.array(v) for v in zip(*cells)) + return _index_lines(np.asarray(lines, dtype=float), start, end, size).tolist() + + # A regular grid: one index per line + assert index([0, 10, 20, 30], [(0, 1), (1, 2), (2, 3)]) == [0, 1, 2, 3] + # A stray line (a shorter cell) separates no cell from the line after it: both lines share an index + assert index([0, 7, 10, 20], [(0, 1), (0, 2), (2, 3), (2, 3)]) == [0, 1, 1, 2] + # A cell spanning 2 rows of the other cells + assert index([0, 10, 20], [(0, 2), (0, 1), (1, 2)]) == [0, 1, 2] + # A gap of 2 cell sizes covered by no cell counts as 2 empty indices, a narrow one (between cells that do not + # touch) as none + assert index([0, 10, 30, 40], [(0, 1), (2, 3)]) == [0, 1, 3, 4] + assert index([0, 10, 12, 22], [(0, 1), (2, 3)]) == [0, 1, 1, 2] def test_tablecenternet_cluster_lines(): diff --git a/tests/pytorch/test_models_table_grid.py b/tests/pytorch/test_models_table_grid.py new file mode 100644 index 0000000000..38efa188a1 --- /dev/null +++ b/tests/pytorch/test_models_table_grid.py @@ -0,0 +1,186 @@ +import numpy as np + +from doctr.models.table_structure.grid import ( + _axis_slots, + _grid_lines, + _group, + as_polygons, + assemble_page_table, + fill_table_grid, +) + + +def _grid(xs, ys): + """Boxes and logical coordinates (row_start, row_end, col_start, col_end) of a regular grid""" + boxes, logic = [], [] + for r in range(len(ys) - 1): + for c in range(len(xs) - 1): + boxes.append([xs[c], ys[r], xs[c + 1], ys[r + 1]]) + logic.append([r, r, c, c]) + return np.array(boxes, dtype=float), np.array(logic) + + +def _word(cx, cy, w=40, h=20): + return [cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2] + + +def _table(cells, logic, words): + """The text grid of a table, from named words""" + names, boxes = list(words), np.array(list(words.values()), dtype=float).reshape(-1, 4) + boxes, logic, cell_words, origin = fill_table_grid(np.asarray(cells, dtype=float), np.asarray(logic), boxes) + grid = [["" for _ in range(logic[:, 3].max() + 1)] for _ in range(logic[:, 1].max() + 1)] + for (rs, _, cs, _), idcs in zip(logic, cell_words): + grid[rs][cs] = " ".join(filter(None, [grid[rs][cs], *(names[i] for i in idcs)])) + return grid, logic, origin + + +def test_table_grid_helpers(): + # Straight boxes (flat or 2-point) and polygons, as (N, 4, 2) polygons + expected = [[[0.1, 0.2], [0.5, 0.2], [0.5, 0.6], [0.1, 0.6]]] + np.testing.assert_allclose(as_polygons([[0.1, 0.2, 0.5, 0.6]]), expected) + np.testing.assert_allclose(as_polygons([((0.1, 0.2), (0.5, 0.6))]), expected) + np.testing.assert_allclose(as_polygons(expected), expected) + assert as_polygons(np.zeros((0, 4))).shape == (0, 4, 2) + # Grid lines at the median of the cell edges, interpolated when no edge lies on them + lines = _grid_lines(np.array([0, 2]), np.array([0, 2]), np.array([0.0, 20.0]), np.array([10.0, 30.0]), 3) + np.testing.assert_allclose(lines, [0, 10, 20, 30]) + # Positions grouped by distance to the mean of their group + np.testing.assert_array_equal(_group(np.array([0.0, 30.0, 4.0, 33.0]), 10), [0, 1, 0, 1]) + # Words beyond the grid make new indices, grouped by text line (rows) or overlapping extent (columns) + lines = np.array([100.0, 200.0, 300.0]) + lo, hi = np.array([150.0, 40.0, 45.0, 10.0, 310.0]), np.array([160.0, 60.0, 65.0, 30.0, 330.0]) + np.testing.assert_array_equal(_axis_slots(lo, hi, lines, 15), [0, -1, -1, -2, 2]) + np.testing.assert_array_equal(_axis_slots(lo, hi, lines, None), [0, -1, -1, -2, 2]) + + +def test_fill_table_grid_complete(): + # A 2 x 2 grid whose last cell is missed by the model, inside a table holding more text around it + cells, logic = _grid([200, 400, 600], [300, 400, 500]) + words = { + "A": _word(300, 350), + "B": _word(500, 350), + "C": _word(300, 450), + "missed": _word(500, 450), # in the slot of the missed cell + "Col1": _word(300, 250), # a header row above the grid + "Col2": _word(500, 250), + "Total": _word(250, 580, w=60), # a sentence below the grid, across both columns + "amount": _word(322.5, 580, w=75), + "due": _word(405, 580, w=80), + "key": _word(150, 350), # a column on the left of the grid + } + grid, _, origin = _table(cells[:3], logic[:3], words) + assert grid == [["", "Col1", "Col2"], ["key", "A", "B"], ["", "C", "missed"], ["", "Total amount due", ""]] + # The cells made from the words have no source cell + assert sorted(origin) == [-1] * 5 + [0, 1, 2] + + # Overlapping words below the grid (e.g. duplicate detections) do not make cells sharing a column + words = {"A": _word(300, 350), "B": _word(500, 350)} + words |= {"a": [210, 440, 250, 460], "b": [300, 440, 520, 460], "c": [320, 440, 360, 460]} + grid, out_logic, origin = _table(cells[:2], logic[:2], words) + assert grid == [["A", "B"], ["a b c", ""]] + assert out_logic.tolist() == [[0, 0, 0, 0], [0, 0, 1, 1], [1, 1, 0, 1]] and origin == [0, 1, -1] + + # Without words: the predicted grid, with local logical coordinates + boxes, out_logic, cell_words, origin = fill_table_grid(cells, logic + 3, np.zeros((0, 4))) + np.testing.assert_array_equal(out_logic, logic) + assert cell_words == [[]] * 4 and origin == [0, 1, 2, 3] + + +def test_fill_table_grid_merge_and_prune(): + # The model splits a row in 2 (the edges of its cells do not line up) and predicts an empty row + cells = [ + [100, 100, 300, 200], + [300, 100, 500, 200], + [100, 200, 300, 300], + [300, 200, 500, 240], # empty, a row of its own + [300, 240, 500, 340], + [100, 340, 300, 440], # an empty row + [300, 340, 500, 440], + [100, 440, 300, 540], + [300, 440, 500, 540], + ] + logic = [[0, 0, 0, 0], [0, 0, 1, 1], [1, 1, 0, 0], [1, 1, 1, 1], [2, 2, 1, 1]] + logic += [[3, 3, 0, 0], [3, 3, 1, 1], [4, 4, 0, 0], [4, 4, 1, 1]] + words = {"h1": _word(200, 150), "h2": _word(400, 150), "a": _word(200, 250), "b": _word(400, 255)} + words |= {"c": _word(200, 490), "d": _word(400, 490)} + # The text of the split row lies on the same line: the 2 rows are merged, and the empty row is removed + grid, _, _ = _table(cells, logic, words) + assert grid == [["h1", "h2"], ["a", "b"], ["c", "d"]] + # Text on different lines is not merged + words["b"] = _word(400, 310) + grid, _, _ = _table(cells, logic, words) + assert grid == [["h1", "h2"], ["a", ""], ["", "b"], ["c", "d"]] + + +def test_fill_table_grid_split_rows(): + # The model merges the 3 text lines of a dense table into a single row + cells, logic = _grid([100, 300, 500], [100, 160, 340]) + words = {"Name": _word(200, 130), "Value": _word(400, 130)} + for line, cy in enumerate((190, 250, 310)): + words |= {f"key{line}": _word(200, cy), f"{line}.00": _word(400, cy)} + grid, _, _ = _table(cells, logic, words) + assert grid == [["Name", "Value"], ["key0", "0.00"], ["key1", "1.00"], ["key2", "2.00"]] + + # A single cell wrapping its text (e.g. a long header) does not split its row + words = {"Unit": _word(200, 120), "price": _word(200, 140), "Qty": _word(400, 130)} + words |= {"10": _word(200, 250), "2": _word(400, 250)} + grid, _, _ = _table(cells, logic, words) + assert grid == [["Unit price", "Qty"], ["10", "2"]] + + # A cell shorter than its row holds a word of its slot lying below it: the split cells enclose their text + cells, logic = _grid([100, 300, 500, 700], [100, 160, 340]) + cells[4, 3] = 200 + words = { + f"{name}{line}": _word(cx, cy) for line, cy in enumerate((190, 310)) for name, cx in zip("abc", (200, 400, 600)) + } + boxes, out_logic, cell_words, _ = fill_table_grid(cells, logic, np.array(list(words.values()), dtype=float)) + assert out_logic[:, 1].max() == 1 and all(len(idcs) == 1 for idcs in cell_words) + assert np.all(boxes[:, 3] > boxes[:, 1]) + + # A cell spanning 2 rows whose text lines lie in each of them, aligned with the text of the row + cells, logic = [[100, 100, 300, 200], [100, 200, 300, 300], [300, 100, 500, 300]], [[0, 0, 0, 0], [1, 1, 0, 0]] + logic.append([0, 1, 1, 1]) + words = {"Acquired": _word(200, 150), "Taken": _word(200, 250), "51": _word(400, 150), "9": _word(400, 250)} + grid, _, origin = _table(cells, logic, words) + assert grid == [["Acquired", "51"], ["Taken", "9"]] + assert sorted(origin) == [0, 1, 2, 2] # both parts keep the score of the spanning cell + # A label spanning 2 rows on a single line is kept as is + words = {"Acquired": _word(200, 150), "Taken": _word(200, 250), "label": _word(400, 200)} + _, out_logic, _ = _table(cells, logic, words) + np.testing.assert_array_equal(out_logic, logic) + + +def _rotated(box, angle, pivot=(500.0, 500.0)): + """The corners (TL, TR, BR, BL) of a box rotated by `angle` degrees, in a 1000 x 1000 page (relative)""" + rad = np.deg2rad(angle) + rot = np.array([[np.cos(rad), -np.sin(rad)], [np.sin(rad), np.cos(rad)]]) + return (((as_polygons([box])[0] - pivot) @ rot.T + pivot) / 1000).tolist() + + +def test_assemble_page_table(): + # A 2 x 2 table rotated by 20 degrees on a page, whose words are assigned in the frame of its rows + cells, logic = _grid([300, 500, 700], [400, 500, 600]) + page_cells = [ + {"geometry": _rotated(box, 20), "score": 0.9, "row_start": rs, "row_end": re, "col_start": cs, "col_end": ce} + for box, (rs, re, cs, ce) in zip(cells, logic + 1) # offset logical coordinates + ] + words = [_rotated(_word(*center), 20) for center in ((400, 450), (600, 450), (400, 550), (600, 550), (900, 900))] + taken = np.zeros(len(words), dtype=bool) + out, num_rows, num_cols = assemble_page_table(page_cells, np.array(words), (1000, 1000), taken) + assert (num_rows, num_cols) == (2, 2) + assert [(c["row_start"], c["col_start"], c["words"]) for c in out] == [ + (0, 0, [0]), + (0, 1, [1]), + (1, 0, [2]), + (1, 1, [3]), + ] + # The cell geometries stay in the page frame, the word outside of the table is left free + np.testing.assert_allclose(out[0]["geometry"], page_cells[0]["geometry"], atol=1e-5) + assert taken.tolist() == [True, True, True, True, False] + + # Cells listing their words keep them, except the ones already taken by another table + listed = [{**cell, "words": [idx]} for idx, cell in enumerate(page_cells)] + taken = np.array([True, False, False, False, False]) + out, _, _ = assemble_page_table(listed, np.array(words), (1000, 1000), taken) + assert [c["words"] for c in out] == [[], [1], [2], [3]] + assert taken.tolist() == [True, True, True, True, False] diff --git a/tests/pytorch/test_models_table_structure_pt.py b/tests/pytorch/test_models_table_structure_pt.py index a12e9dd07b..f2cc9f6ac6 100644 --- a/tests/pytorch/test_models_table_structure_pt.py +++ b/tests/pytorch/test_models_table_structure_pt.py @@ -186,3 +186,52 @@ def test_torch_compiled_models(arch_name, mock_table_image): assert np.allclose(cell["geometry"], compiled_cell["geometry"], atol=1e-4) assert np.isclose(cell["score"], compiled_cell["score"], atol=1e-4) assert all(cell[k] == compiled_cell[k] for k in ("row_start", "row_end", "col_start", "col_end")) + + +def test_table_predictor_regions(monkeypatch): + predictor = table_structure.zoo.table_predictor("tablecenternet", pretrained=False) + crops = [] + + def fake_forward(images, **kwargs): + # A fixed grid (crop-relative cells) for every crop: 2 cells, and 1 of the context margin, above the region + crops.extend(images) + h, w = images[0].shape[:2] + pad = 20 # the context margin of a region smaller than 2000 px + + def cell(x0, y0, x1, y1, r, c): + box = [(pad + x0) / w, (pad + y0) / h, (pad + x1) / w, (pad + y1) / h] + return {"geometry": box, "score": 0.9, "row_start": r, "row_end": r, "col_start": c, "col_end": c} + + cells = [cell(0, 0, 100, 30, 1, 0), cell(100, 0, 200, 30, 1, 1), cell(0, -18, 200, -4, 0, 0)] + return [{"cells": cells, "num_rows": 2, "num_cols": 2} for _ in images] + + monkeypatch.setattr(predictor, "forward", fake_forward) + page = np.full((400, 600, 3), 255, dtype=np.uint8) + x0, y0, x1 = 0.1, 0.25, 0.1 + 200 / 600 # a 200 x 50 px table region, its cells cover its top 30 px + regions = [{"class_names": ["Text", "Table"], "boxes": np.array([[0, 0, 0.1, 0.1], [x0, y0, x1, 0.375]])}, None] + # Words in each cell, one in the region below the cells, and one outside of the region + words = np.array([ + [0.12, 0.27, 0.15, 0.29], + [0.30, 0.27, 0.33, 0.29], + [0.12, 0.345, 0.15, 0.365], + [0.12, 0.80, 0.15, 0.82], + ]) + tables = predictor.predict_regions([page, page], regions, [words, words[:0]]) + # Only the table region is cropped, with a context margin + assert len(crops) == 1 and crops[0].shape[:2] == (90, 240) + assert len(tables) == 2 and len(tables[0]) == 1 and tables[1] == [] + table = tables[0][0] + # The cell of the context margin is dropped, the word below the cells makes a new row + assert (table["num_rows"], table["num_cols"]) == (2, 2) + cells = sorted(table["cells"], key=lambda c: (c["row_start"], c["col_start"])) + assert [(c["row_start"], c["col_start"], c["words"], c["score"]) for c in cells] == [ + (0, 0, [0], 0.9), + (0, 1, [1], 0.9), + (1, 0, [2], None), + ] + # The cells are mapped back to page-relative coordinates + expected = [[x0, y0], [x0 + 100 / 600, y0], [x0 + 100 / 600, 0.325], [x0, 0.325]] + np.testing.assert_allclose(cells[0]["geometry"], expected, atol=1e-5) + # Without words, the cells are empty + tables = predictor.predict_regions([page], regions[:1]) + assert all(cell["words"] == [] for cell in tables[0][0]["cells"])