From 234e8d7a63f2cc28cf8ac621067d53c2d93b6217 Mon Sep 17 00:00:00 2001 From: felix Date: Mon, 9 Nov 2026 15:21:01 +0100 Subject: [PATCH 1/8] Update --- doctr/models/builder.py | 590 ++++++++++++++++-- doctr/models/predictor/pytorch.py | 72 ++- .../table_structure/tablecenternet/base.py | 89 ++- .../table_structure/tablecenternet/pytorch.py | 2 +- tests/common/test_models_builder.py | 141 ++++- tests/common/test_models_table_structure.py | 52 +- tests/pytorch/test_models_zoo_pt.py | 49 ++ 7 files changed, 823 insertions(+), 172 deletions(-) diff --git a/doctr/models/builder.py b/doctr/models/builder.py index eb74d37c82..80fa893ec0 100644 --- a/doctr/models/builder.py +++ b/doctr/models/builder.py @@ -450,20 +450,6 @@ def _build_layout_elements(regions: dict[str, Any] | None) -> list[LayoutElement elements.append(LayoutElement(layout_type=str(cname), confidence=float(score), geometry=geometry)) return elements - @staticmethod - def _word_centers(boxes: np.ndarray) -> np.ndarray: - """Return the (x, y) center of each word box. - - Args: - boxes: word boxes of shape (N, 4) (straight: x1, y1, x2, y2) or (N, 4, 2) (rotated polygon) - - Returns: - array of shape (N, 2) with the relative center coordinates of each box - """ - if boxes.ndim == 3: # rotated polygons (N, 4, 2) - return boxes.mean(axis=1) - return np.stack([(boxes[:, 0] + boxes[:, 2]) / 2, (boxes[:, 1] + boxes[:, 3]) / 2], axis=1) - @staticmethod def _as_cell_polygon(geometry: Any) -> np.ndarray: """Normalize a cell geometry to a (4, 2) polygon. @@ -478,8 +464,8 @@ def _as_cell_polygon(geometry: Any) -> np.ndarray: array of shape (4, 2) with the polygon vertices """ arr = np.asarray(geometry, dtype=np.float32) - if arr.ndim == 1: # straight box (xmin, ymin, xmax, ymax) - x0, y0, x1, y1 = arr + if arr.size == 4: # straight box (xmin, ymin, xmax, ymax) or ((xmin, ymin), (xmax, ymax)) + x0, y0, x1, y1 = arr.ravel() return np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) return order_points(arr.reshape(-1, 2)) @@ -563,28 +549,36 @@ def _localize_logic(cells: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], @classmethod def _table_grids( cls, page_table: dict[str, Any] | list[dict[str, Any]] | None - ) -> list[tuple[list[dict[str, Any]], int, int]]: + ) -> list[tuple[list[dict[str, Any]], int, int, np.ndarray | None]]: """Normalize the table structure prediction(s) of a page to non-empty grids with local logical coordinates Args: page_table: a single grid, a list of grids or None (cf. `_build_tables`) Returns: - the cells, number of rows and number of columns of each non-empty grid + the cells, number of rows, number of columns and (4, 2) region polygon (None if unknown) of each + non-empty grid """ raw_tables = [] if page_table is None else [page_table] if isinstance(page_table, dict) else list(page_table) - return [cls._localize_logic(raw["cells"]) for raw in raw_tables if raw and len(raw.get("cells", [])) > 0] + grids = [] + for raw in raw_tables: + if not raw or len(raw.get("cells", [])) == 0: + continue + region = raw.get("geometry") + grids.append(( + *cls._localize_logic(raw["cells"]), + None if region is None else cls._as_cell_polygon(region), + )) + return grids @classmethod def _assign_cell_words(cls, centers: np.ndarray, polys: np.ndarray, consumed: np.ndarray) -> list[list[int]]: - """Assign each word not yet consumed to at most one cell of a table, and flag it as consumed - - A word goes to the first cell containing its center. A word landing just outside every cell but inside the - table goes to the nearest cell, within half a median cell diagonal, so table text is not dropped into the body. + """Assign each word not yet consumed whose center lies in a cell to the first such cell, and flag it as + consumed Args: centers: the (N, 2) word centers - polys: the (C, 4, 2) cell polygons, vertices ordered TL, TR, BR, BL + polys: the (C, 4, 2) cell polygons consumed: the (N,) mask of the words already assigned to a table, updated in place Returns: @@ -599,58 +593,494 @@ def _assign_cell_words(cls, centers: np.ndarray, polys: np.ndarray, consumed: np for w_idx, c_idx in zip(free[assigned], inside[assigned].argmax(axis=1)): cell_words[c_idx].append(int(w_idx)) consumed[w_idx] = True - leftover = free[~assigned] - if leftover.size > 0: - cell_centers = polys.mean(axis=1) # (C, 2) - max_dist = 0.5 * float(np.median(np.linalg.norm(polys[:, 2] - polys[:, 0], axis=1))) # TL->BR diagonal - outline = cv2.boxPoints(cv2.minAreaRect(polys.reshape(-1, 2).astype(np.float32))).reshape(1, 4, 2) - in_table = cls._points_in_polygons(centers[leftover], outline)[:, 0] - for w_idx in leftover[in_table]: - dists = np.linalg.norm(cell_centers - centers[w_idx], axis=1) - nearest = int(dists.argmin()) - if dists[nearest] <= max_dist: - cell_words[nearest].append(int(w_idx)) - consumed[w_idx] = True return cell_words @staticmethod - def _deskew_centers(centers: np.ndarray, polys: np.ndarray) -> np.ndarray: - """Rotate word centers by the angle of a table, to order the words along its own axes + def _rotate_points(points: np.ndarray, angle: float, pivot: np.ndarray) -> np.ndarray: + """Rotate (..., 2) points by `angle` radians around `pivot`""" + cos_a, sin_a = np.cos(angle), np.sin(angle) + shifted = points - pivot + return np.stack( + [ + pivot[0] + shifted[..., 0] * cos_a - shifted[..., 1] * sin_a, + pivot[1] + shifted[..., 0] * sin_a + shifted[..., 1] * cos_a, + ], + axis=-1, + ) + + @staticmethod + def _grid_lines(start: np.ndarray, end: np.ndarray, lo: np.ndarray, hi: np.ndarray, size: int) -> np.ndarray: + """Positions of the `size + 1` grid lines of an axis, from the cell edges lying on each of them + + Line `k` is where index `k` starts and index `k - 1` ends: it lies at the median of the corresponding cell + edges, the lines on which no cell edge lies are interpolated between their neighbours. Args: - centers: the (N, 2) word centers - polys: the (C, 4, 2) cell polygons, vertices ordered TL, TR, BR, BL + start: the first index of each cell along the axis, shape (C,) + end: the last index of each cell along the axis, shape (C,) + lo: the lower edge of each cell along the axis (in the table frame), shape (C,) + hi: the upper edge of each cell along the axis (in the table frame), shape (C,) + size: the number of indices (rows or columns) of the axis Returns: - the (N, 2) de-skewed centers + the non-decreasing positions of the grid lines, shape (size + 1,) """ - top_edges = polys[:, 1] - polys[:, 0] # TR - TL - angle = float(np.median(np.arctan2(top_edges[:, 1], top_edges[:, 0]))) - cos_a, sin_a = np.cos(-angle), np.sin(-angle) - pivot = centers.mean(axis=0) - shifted = centers - pivot - return np.stack( - [ - pivot[0] + shifted[:, 0] * cos_a - shifted[:, 1] * sin_a, - pivot[1] + shifted[:, 0] * sin_a + shifted[:, 1] * cos_a, - ], - axis=1, + edges: list[list[float]] = [[] for _ in range(size + 1)] + for s, e, low, high in zip(start, end, lo, hi): + edges[int(s)].append(float(low)) + edges[int(e) + 1].append(float(high)) + known = [k for k, values in enumerate(edges) if values] + lines = np.interp(np.arange(size + 1), known, [float(np.median(edges[k])) for k in known]) + return np.maximum.accumulate(lines) + + @staticmethod + def _axis_slots( + center: np.ndarray, lo: np.ndarray, hi: np.ndarray, lines: np.ndarray, line_tol: float | None = None + ) -> tuple[np.ndarray, dict[int, tuple[float, float]]]: + """Index of each word along an axis of a table grid, adding new indices for the words beyond the grid + + A word whose center lies between two grid lines takes their index. The words before the first line (resp. + after the last one) are grouped into new indices before (resp. after) the grid: by text line (centers closer + than `line_tol`) for rows, by overlapping extents for columns (`line_tol` is None). + + Args: + center: the word centers along the axis (in the table frame), shape (W,) + lo: the lower word edges along the axis, shape (W,) + hi: the upper word edges along the axis, shape (W,) + lines: the positions of the grid lines of the axis, shape (size + 1,) + line_tol: the maximum distance between the centers of the words of a same new row + + Returns: + the (possibly negative or beyond the grid) index of each word, and the (lo, hi) extent of every index + """ + size = lines.size - 1 + idx = np.clip(np.searchsorted(lines, center, side="right") - 1, 0, size - 1) + bands = {k: (float(lines[k]), float(lines[k + 1])) for k in range(size)} + for before, mask in ((True, center < lines[0]), (False, center > lines[-1])): + members = np.flatnonzero(mask) + if members.size == 0: + continue + groups: list[list[int]] = [] + if line_tol is not None: + for j in members[np.argsort(center[members], kind="stable")]: + if groups and center[j] - np.mean(center[groups[-1]]) < line_tol: + groups[-1].append(int(j)) + else: + groups.append([int(j)]) + else: + reach = -np.inf + for j in members[np.argsort(lo[members], kind="stable")]: + if groups and lo[j] <= reach: + groups[-1].append(int(j)) + else: + groups.append([int(j)]) + reach = -np.inf + reach = max(reach, float(hi[j])) + new_ids = range(-len(groups), 0) if before else range(size, size + len(groups)) + for g_id, group in zip(new_ids, groups): + idx[group] = g_id + bands[g_id] = (float(lo[group].min()), float(hi[group].max())) + return idx, bands + + @classmethod + def _complete_grid( + cls, + cells: list[dict[str, Any]], + polys: np.ndarray, + cell_words: list[list[int]], + leftover: np.ndarray, + word_polys: np.ndarray, + angle: float, + ) -> tuple[list[dict[str, Any]], np.ndarray, list[list[int]]]: + """Place the words of a table that lie in no cell into its grid, so that no table text is left out + + The row and column lines of the grid are located from the cell edges. A word between two lines of each axis + joins the cell covering that grid slot, or a new cell when no cell covers it (e.g. a missed cell). The words + beyond the grid (e.g. a missed header row or a row of totals) make new rows / columns: in a new row, the + words closer than half a text line, like the words of a sentence, share a single (possibly spanning) cell. + + Args: + cells: the cells of the table, with local logical coordinates + polys: the (C, 4, 2) cell polygons (absolute coordinates) + cell_words: the word indices of each cell + leftover: the indices of the words of the table that lie in no cell + word_polys: the (N, 4, 2) word polygons (absolute coordinates) + angle: the angle of the table rows (radians) + + Returns: + the cells, their polygons and the word indices of each cell, new cells included (the logical coordinates + of the new rows / columns before the grid are negative) + """ + pivot = polys.reshape(-1, 2).mean(axis=0) + # Table frame: the rows and columns are horizontal and vertical + cell_pts = cls._rotate_points(polys, -angle, pivot) + word_pts = cls._rotate_points(word_polys[leftover], -angle, pivot) + w_x0, w_x1 = word_pts[..., 0].min(axis=1), word_pts[..., 0].max(axis=1) + w_y0, w_y1 = word_pts[..., 1].min(axis=1), word_pts[..., 1].max(axis=1) + logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells], dtype=int) + num_rows, num_cols = int(logic[:, 1].max()) + 1, int(logic[:, 3].max()) + 1 + row_lines = cls._grid_lines( + logic[:, 0], logic[:, 1], cell_pts[..., 1].min(axis=1), cell_pts[..., 1].max(axis=1), num_rows ) + col_lines = cls._grid_lines( + logic[:, 2], logic[:, 3], cell_pts[..., 0].min(axis=1), cell_pts[..., 0].max(axis=1), num_cols + ) + med_h = float(np.median(w_y1 - w_y0)) + med_h = med_h if np.isfinite(med_h) and med_h > 0 else 1.0 + col_idx, col_bands = cls._axis_slots((w_x0 + w_x1) / 2, w_x0, w_x1, col_lines) + row_idx, row_bands = cls._axis_slots((w_y0 + w_y1) / 2, w_y0, w_y1, row_lines, line_tol=med_h / 2) + + # The cell covering each slot of the grid + occupied: dict[tuple[int, int], int] = {} + for c_idx, (rs, re, cs, ce) in enumerate(logic): + for r in range(rs, re + 1): + for c in range(cs, ce + 1): + occupied.setdefault((r, c), c_idx) + + # New cells: (row_start, row_end, col_start, col_end) -> positions in `leftover` + new_cells: dict[tuple[int, int, int, int], list[int]] = {} + in_grid = (row_idx >= 0) & (row_idx < num_rows) + for j in np.flatnonzero(in_grid): + slot = (int(row_idx[j]), int(col_idx[j])) + if slot in occupied: + cell_words[occupied[slot]].append(int(leftover[j])) + else: + new_cells.setdefault((slot[0], slot[0], slot[1], slot[1]), []).append(int(j)) + # New rows: the words of a same column, or closer than half a text line, share a cell + for r in np.unique(row_idx[~in_grid]): + members = np.flatnonzero(row_idx == r) + members = members[np.argsort(w_x0[members], kind="stable")] + group = [int(members[0])] + for prev, j in zip(members[:-1], members[1:]): + if col_idx[j] == col_idx[group[-1]] or w_x0[j] - w_x1[prev] < med_h / 2: + group.append(int(j)) + continue + new_cells[(int(r), int(r), int(col_idx[group].min()), int(col_idx[group].max()))] = group + group = [int(j)] + new_cells[(int(r), int(r), int(col_idx[group].min()), int(col_idx[group].max()))] = group + + cells, new_polys = list(cells), [] + for (rs, re, cs, ce), positions in new_cells.items(): + x0 = min(col_bands[cs][0], float(w_x0[positions].min())) + x1 = max(col_bands[ce][1], float(w_x1[positions].max())) + y0 = min(row_bands[rs][0], float(w_y0[positions].min())) + y1 = max(row_bands[re][1], float(w_y1[positions].max())) + rect = np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float64) + new_polys.append(cls._rotate_points(rect, angle, pivot)) + # A cell added from the words: the table structure model did not score it + cells.append({"score": None, "row_start": rs, "row_end": re, "col_start": cs, "col_end": ce}) + cell_words.append([int(leftover[j]) for j in positions]) + if new_polys: + polys = np.concatenate([polys, np.stack(new_polys).astype(polys.dtype)]) + return cells, polys, cell_words + + @classmethod + def _simplify_grid( + cls, + cells: list[dict[str, Any]], + polys: np.ndarray, + cell_words: list[list[int]], + word_polys: np.ndarray, + angle: float, + ) -> tuple[list[dict[str, Any]], np.ndarray, list[list[int]]]: + """Merge the rows / columns of a table split by the structure model, and remove the ones without any text + + Two adjacent rows (resp. columns) are merged when their text is complementary (no column, resp. row, has text + in both) and lies on the same text line (resp. column), i.e. their text extents mostly overlap: the table + structure model sometimes splits a row whose cell edges do not line up. A row (resp. column) without text of its + own (no cell holds text in this row only) is removed: the grid only keeps the rows and columns of the table's + content, and the cells spanning the removed ones span one less. + + Args: + cells: the cells of the table, with local logical coordinates + polys: the (C, 4, 2) cell polygons (absolute coordinates) + cell_words: the word indices of each cell + word_polys: the (N, 4, 2) word polygons (absolute coordinates) + angle: the angle of the table rows (radians) + + Returns: + the remaining cells, their polygons and the word indices of each cell + """ + filled = np.array([len(w) > 0 for w in cell_words]) + if not filled.any(): + return cells, polys, cell_words + pivot = polys.reshape(-1, 2).mean(axis=0) + frame = cls._rotate_points(polys, -angle, pivot) + rects = np.concatenate([frame.min(axis=1), frame.max(axis=1)], axis=1) # (C, 4) x0, y0, x1, y1 + resized = np.zeros(len(cells), dtype=bool) + # Text extent of each cell in the table frame: x0, y0, x1, y1 + content = np.full((len(cells), 4), np.nan) + for c_idx in np.flatnonzero(filled): + pts = cls._rotate_points(word_polys[cell_words[c_idx]], -angle, pivot).reshape(-1, 2) + content[c_idx] = [*pts.min(axis=0), *pts.max(axis=0)] + # row_start, row_end, col_start, col_end + logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells], dtype=int) + alive = np.ones(len(cells), dtype=bool) + + def mergeable(axis: int, k: int) -> bool: + start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] + o_start, o_end = logic[:, 2 - 2 * axis], logic[:, 3 - 2 * axis] + in_k, in_next = alive & (start <= k) & (end >= k), alive & (start <= k + 1) & (end >= k + 1) + only_k, only_next = np.flatnonzero(in_k & ~in_next & filled), np.flatnonzero(in_next & ~in_k & filled) + if only_k.size == 0 or only_next.size == 0: + return False + # Complementary: no index of the other axis holds text in both + if np.any( + (o_start[only_k, None] <= o_end[None, only_next]) & (o_start[None, only_next] <= o_end[only_k, None]) + ): + return False + # On the same line: the text extents along the axis mostly overlap + lo, hi = (1, 3) if axis == 0 else (0, 2) + a_lo, a_hi = content[only_k, lo].min(), content[only_k, hi].max() + b_lo, b_hi = content[only_next, lo].min(), content[only_next, hi].max() + overlap = min(a_hi, b_hi) - max(a_lo, b_lo) + return bool(overlap >= 0.5 * min(a_hi - a_lo, b_hi - b_lo)) + + def shift(axis: int, k: int, merge: bool) -> None: + # Merge the index k + 1 into the index k, or remove the index k (its cells are removed beforehand) + start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] + end[alive & ((end > k) if merge else (end >= k))] -= 1 + start[alive & (start > k)] -= 1 + + def resolve_overlaps() -> None: + # The cells merged onto a same slot: the empty ones are absorbed by the cells holding text + idx = np.flatnonzero(alive) + for pos, i in enumerate(idx): + if not alive[i]: + continue + others = idx[pos + 1 :] + others = others[alive[others]] + hits = others[ + (logic[others, 0] <= logic[i, 1]) + & (logic[i, 0] <= logic[others, 1]) + & (logic[others, 2] <= logic[i, 3]) + & (logic[i, 2] <= logic[others, 3]) + ] + for j in hits: + keep, drop = (j, i) if filled[j] and not filled[i] else (i, j) + if filled[drop]: # both hold text: not merged + continue + rects[keep, :2] = np.minimum(rects[keep, :2], rects[drop, :2]) + rects[keep, 2:] = np.maximum(rects[keep, 2:], rects[drop, 2:]) + logic[keep, ::2] = np.minimum(logic[keep, ::2], logic[drop, ::2]) + logic[keep, 1::2] = np.maximum(logic[keep, 1::2], logic[drop, 1::2]) + resized[keep] = True + alive[drop] = False + if drop == i: + break + + changed = True + while changed: + changed = False + for axis in (0, 1): + size = int(logic[alive, 2 * axis + 1].max()) + 1 + for k in range(size - 1): + if mergeable(axis, k): + shift(axis, k, merge=True) + resolve_overlaps() + changed = True + break + # Remove the rows / columns without text of their own: no text, or only the text of cells spanning other rows + # / columns too (which then span one less) + for axis in (0, 1): + start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] + for k in range(int(end[alive].max()), -1, -1): + if np.any(alive & filled & (start == k) & (end == k)): + continue + alive[alive & (start == k) & (end == k)] = False + shift(axis, k, merge=False) + + keep = np.flatnonzero(alive) + new_polys = polys.copy() + for c_idx in np.flatnonzero(resized & alive): + x0, y0, x1, y1 = rects[c_idx] + rect = np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float64) + new_polys[c_idx] = cls._rotate_points(rect, angle, pivot) + new_cells = [] + for c_idx in keep: + rs, re, cs, ce = (int(v) for v in logic[c_idx]) + new_cells.append({**cells[c_idx], "row_start": rs, "row_end": re, "col_start": cs, "col_end": ce}) + return new_cells, new_polys[keep], [cell_words[c_idx] for c_idx in keep] + + @staticmethod + def _text_lines(centers: np.ndarray, tol: float) -> np.ndarray: + """Group 1D word centers (along the rows' normal) into text lines: the line index of each word, from the top""" + order = np.argsort(centers, kind="stable") + line_ids = np.zeros(centers.size, dtype=np.int64) + line, members = 0, [int(order[0])] if centers.size else [] + for j in order[1:]: + if centers[j] - np.mean(centers[members]) >= tol: + line, members = line + 1, [] + members.append(int(j)) + line_ids[j] = line + return line_ids + + @classmethod + def _split_rows( + cls, + cells: list[dict[str, Any]], + polys: np.ndarray, + cell_words: list[list[int]], + word_polys: np.ndarray, + angle: float, + ) -> tuple[list[dict[str, Any]], np.ndarray, list[list[int]]]: + """Split the rows merged by the table structure model, from the text lines of their cells + + The model may merge several rows of a dense table (e.g. without rulings) into a single one: a row is split + into its text lines when most of its cells holding text (at least 2) hold several of them, so that a single + cell wrapping its text (e.g. a long header) does not split its row. Likewise, a cell spanning several rows + is split when each of its text lines lies in a different row, aligned with the text of that row. + + Args: + cells: the cells of the table, with local logical coordinates + polys: the (C, 4, 2) cell polygons (absolute coordinates) + cell_words: the word indices of each cell + word_polys: the (N, 4, 2) word polygons (absolute coordinates) + angle: the angle of the table rows (radians) + + Returns: + the cells, their polygons and the word indices of each cell + """ + if not any(cell_words): + return cells, polys, cell_words + pivot = polys.reshape(-1, 2).mean(axis=0) + frame = cls._rotate_points(polys, -angle, pivot) + rects = [np.concatenate([f.min(axis=0), f.max(axis=0)]) for f in frame] # x0, y0, x1, y1 + all_words = np.unique(np.concatenate([w for w in cell_words if w])).astype(int) + w_pts = cls._rotate_points(word_polys[all_words], -angle, pivot) + w_y0 = dict(zip(all_words.tolist(), w_pts[..., 1].min(axis=1))) + w_y1 = dict(zip(all_words.tolist(), w_pts[..., 1].max(axis=1))) + heights = w_pts[..., 1].max(axis=1) - w_pts[..., 1].min(axis=1) + tol = 0.5 * float(np.median(heights)) if heights.size else 0.0 + if not tol > 0: + return cells, polys, cell_words + cells = [dict(c) for c in cells] + cell_words = [list(w) for w in cell_words] + resized: set[int] = set() + + def center(w: int) -> float: + return (w_y0[w] + w_y1[w]) / 2 + + def split_cell(c_idx: int, line_ids: np.ndarray, rows: list[tuple[int, int]]) -> None: + """Split a cell into one cell per text line, spanning the given rows""" + words = np.asarray(cell_words[c_idx]) + x0, y0, x1, y1 = rects[c_idx] + n_lines = int(line_ids.max()) + 1 + # The sub-cells share the cell's extent, split midway between its text lines + tops = [min(w_y0[w] for w in words[line_ids == line]) for line in range(n_lines)] + bottoms = [max(w_y1[w] for w in words[line_ids == line]) for line in range(n_lines)] + bounds = [y0] + [(bottoms[line] + tops[line + 1]) / 2 for line in range(n_lines - 1)] + [y1] + for line, (rs, re) in enumerate(rows): + new = {**cells[c_idx], "row_start": rs, "row_end": re} + rect = np.array([x0, bounds[line], x1, bounds[line + 1]]) + if line == 0: + cells[c_idx], rects[c_idx], cell_words[c_idx] = new, rect, words[line_ids == 0].tolist() + resized.add(c_idx) + else: + cells.append(new) + rects.append(rect) + cell_words.append(words[line_ids == line].tolist()) + resized.add(len(cells) - 1) + + # 1. Rows merged by the model, from the bottom up so that the rows still to process do not move + num_rows = max(c["row_end"] for c in cells) + 1 + for r in range(num_rows - 1, -1, -1): + in_row = [i for i, c in enumerate(cells) if c["row_start"] == r and c["row_end"] == r] + filled = [i for i in in_row if cell_words[i]] + if len(filled) < 2: + continue + words = [w for i in filled for w in cell_words[i]] + line_of = dict(zip(words, cls._text_lines(np.array([center(w) for w in words]), tol).tolist())) + n_lines = max(line_of.values()) + 1 + multi = [i for i in filled if len({line_of[w] for w in cell_words[i]}) > 1] + if n_lines < 2 or len(multi) < 2 or 2 * len(multi) < len(filled): + continue + # The rows below move down, the cells spanning the row span its new rows too + for c in cells: + if c["row_start"] == r and c["row_end"] == r: + continue + if c["row_start"] > r: + c["row_start"] += n_lines - 1 + if c["row_end"] >= r: + c["row_end"] += n_lines - 1 + for i in in_row: + if not cell_words[i]: + cells[i]["row_end"] = r + n_lines - 1 + continue + cell_lines = np.array([line_of[w] for w in cell_words[i]]) + present = np.unique(cell_lines) + # Local line indices of the cell -> rows of the table + split_cell(i, np.searchsorted(present, cell_lines), [(r + line, r + line) for line in present]) + + # 2. Cells spanning several rows whose text lines lie in different rows + logic = np.array([[c["row_start"], c["row_end"]] for c in cells], dtype=int) + if logic[:, 1].max() > 0: + row_y0 = np.array([r[1] for r in rects]) + row_y1 = np.array([r[3] for r in rects]) + lines = cls._grid_lines(logic[:, 0], logic[:, 1], row_y0, row_y1, int(logic[:, 1].max()) + 1) + for i in range(len(cells)): + rs, re = cells[i]["row_start"], cells[i]["row_end"] + if re == rs or len(cell_words[i]) < 2: + continue + cell_lines = cls._text_lines(np.array([center(w) for w in cell_words[i]]), tol) + n_lines = int(cell_lines.max()) + 1 + if n_lines < 2: + continue + line_rows = [] + for line in range(n_lines): + members = [w for w, lid in zip(cell_words[i], cell_lines) if lid == line] + lo, hi = min(w_y0[w] for w in members), max(w_y1[w] for w in members) + row = int(np.clip(np.searchsorted(lines, (lo + hi) / 2, side="right") - 1, rs, re)) + # The text of the other cells of the row lies on the same line + aligned = any( + cells[j]["row_start"] == row == cells[j]["row_end"] + and any(min(hi, w_y1[w]) - max(lo, w_y0[w]) > 0.5 * (hi - lo) for w in cell_words[j]) + for j in range(len(cells)) + if j != i and cell_words[j] + ) + if not aligned: + break + line_rows.append(row) + if len(line_rows) < n_lines or len(set(line_rows)) < n_lines: + continue + spans = [ + (rs if line == 0 else line_rows[line], re if line == n_lines - 1 else line_rows[line + 1] - 1) + for line in range(n_lines) + ] + split_cell(i, cell_lines, spans) + + if not resized: + return cells, polys, cell_words + new_polys = [ + cls._rotate_points( + np.array([[r[0], r[1]], [r[2], r[1]], [r[2], r[3]], [r[0], r[3]]], dtype=np.float64), angle, pivot + ) + if idx in resized + else polys[idx] + for idx, r in enumerate(rects) + ] + return cells, np.stack(new_polys).astype(polys.dtype), cell_words def _build_tables( self, boxes: np.ndarray, word_preds: list[tuple[str, float]], page_table: dict[str, Any] | list[dict[str, Any]] | None, + page_shape: tuple[int, int] | None = None, ) -> tuple[list[Table], np.ndarray]: """Assign detected words to table cells and build the page tables. + Every word whose center lies in a table (its layout region when provided, the cells otherwise) is part of + the table: the words outside the predicted cells complete the grid (cf. `_complete_grid`). + Args: boxes: word boxes of the page, of shape (N, 4) or (N, 4, 2), in relative coordinates word_preds: list of (text, confidence) for each of the N words page_table: the table structure prediction(s) for the page. Either a single grid `{"cells": [{"geometry", "score", "row_start", "row_end", "col_start", "col_end"}], "num_rows", - "num_cols"}` (cell geometries in page-relative coordinates), a list of such grids, or None + "num_cols", "geometry"}` (cell geometries in page-relative coordinates, the optional "geometry" being + the (4, 2) polygon of the table region), a list of such grids, or None + page_shape: the (height, width) of the page, to process the geometry in an isotropic space Returns: a tuple with the list of `Table` objects of the page (one per provided table) and a boolean mask of @@ -661,27 +1091,47 @@ def _build_tables( if len(grids) == 0: return [], consumed # The geometries follow the page's word geometry: straight 2-point boxes for (N, 4) word boxes, 4-point polygons - # for (N, 4, 2) ones. Word heights are rotation-invariant (left edge length for polygons). + # for (N, 4, 2) ones. straight = boxes.ndim != 3 - centers = self._word_centers(boxes) if boxes.shape[0] > 0 else np.empty((0, 2)) - if boxes.shape[0] == 0: - heights = np.empty(0) - else: - heights = boxes[:, 3] - boxes[:, 1] if straight else np.linalg.norm(boxes[:, 3] - boxes[:, 0], axis=1) + scale = np.ones(2) if page_shape is None else np.array([page_shape[1], page_shape[0]], dtype=np.float64) + if boxes.ndim == 3: # (N, 4, 2) polygons, ordered TL, TR, BR, BL + word_polys = boxes.astype(np.float64) * scale + else: # (N, 4) straight boxes (xmin, ymin, xmax, ymax) + x0, y0, x1, y1 = boxes.reshape(-1, 4).T.astype(np.float64) + word_polys = np.stack([np.stack(pt, axis=-1) for pt in ((x0, y0), (x1, y0), (x1, y1), (x0, y1))], 1) * scale + centers = word_polys.mean(axis=1) + heights = np.linalg.norm(word_polys[:, 3] - word_polys[:, 0], axis=1) # rotation-invariant (left edge) tables: list[Table] = [] - for cells, num_rows, num_cols in grids: - polys = np.stack([self._as_cell_polygon(cell["geometry"]) for cell in cells]) # (C, 4, 2) - order_centers = centers if straight or centers.shape[0] == 0 else self._deskew_centers(centers, polys) + for cells, num_rows, num_cols, region in grids: + polys = np.stack([self._as_cell_polygon(cell["geometry"]) for cell in cells]) * scale # (C, 4, 2) + cell_words = self._assign_cell_words(centers, polys, consumed) + top_edges = polys[:, 1] - polys[:, 0] # TR - TL + angle = 0.0 if straight else float(np.median(np.arctan2(top_edges[:, 1], top_edges[:, 0]))) + # The words of the table that lie in no cell: in its layout region, or in the cells' outline + free = np.flatnonzero(~consumed) + outline = cv2.boxPoints(cv2.minAreaRect(polys.reshape(-1, 2).astype(np.float32))).reshape(1, 4, 2) + in_table = self._points_in_polygons(centers[free], outline)[:, 0] + if region is not None: + in_table |= self._points_in_polygons(centers[free], (region * scale)[None])[:, 0] + leftover = free[in_table] + if leftover.size > 0: + cells, polys, cell_words = self._complete_grid(cells, polys, cell_words, leftover, word_polys, angle) + consumed[leftover] = True + cells, num_rows, num_cols = self._localize_logic(cells) + cells, polys, cell_words = self._simplify_grid(cells, polys, cell_words, word_polys, angle) + cells, polys, cell_words = self._split_rows(cells, polys, cell_words, word_polys, angle) + cells, num_rows, num_cols = self._localize_logic(cells) + # Reading order inside the cells: rows top to bottom, words left to right (table axes) + order_centers = self._rotate_points(centers, -angle, centers.mean(axis=0)) if centers.size else centers table_cells = [] - for cell, poly, w_idcs in zip(cells, polys, self._assign_cell_words(centers, polys, consumed)): + for cell, poly, w_idcs in zip(cells, polys / scale, cell_words): if len(w_idcs) > 0: - # Reading order inside the cell: rows top to bottom, words left to right (table axes) ordered = self._order_cell_words(w_idcs, order_centers, heights) value = " ".join(word_preds[i][0] for i in ordered) confidence = float(np.mean([word_preds[i][1] for i in ordered])) else: - value, confidence = "", float(cell["score"]) + value, confidence = "", float(cell["score"] or 0.0) table_cells.append( TableCell( value=value, @@ -691,19 +1141,21 @@ def _build_tables( row_end=int(cell["row_end"]), col_start=int(cell["col_start"]), col_end=int(cell["col_end"]), - objectness_score=float(cell["score"]), + objectness_score=None if cell["score"] is None else float(cell["score"]), ) ) # Enclosing geometry of the whole table - (xmin, ymin), (xmax, ymax) = polys.reshape(-1, 2).min(axis=0), polys.reshape(-1, 2).max(axis=0) + pts = (polys / scale).reshape(-1, 2) + (xmin, ymin), (xmax, ymax) = pts.min(axis=0), pts.max(axis=0) corners = np.asarray([[xmin, ymin], [xmax, ymin], [xmax, ymax], [xmin, ymax]]) + scores = [float(cell["score"]) for cell in cells if cell["score"] is not None] tables.append( Table( cells=table_cells, num_rows=num_rows, num_cols=num_cols, geometry=self._export_geometry(corners, straight), - confidence=float(np.mean([float(cell["score"]) for cell in cells])), + confidence=float(np.mean(scores)) if scores else 0.0, ) ) return tables, consumed @@ -911,7 +1363,7 @@ def __call__( _tables, ): # Build the page tables and flag the words that belong to a table - page_tables, consumed = self._build_tables(page_boxes, word_preds, page_table) + page_tables, consumed = self._build_tables(page_boxes, word_preds, page_table, shape) if consumed.any(): # Remove the words assigned to a table from the regular blocks output keep = ~consumed diff --git a/doctr/models/predictor/pytorch.py b/doctr/models/predictor/pytorch.py index ada7b96c21..b91efc0026 100644 --- a/doctr/models/predictor/pytorch.py +++ b/doctr/models/predictor/pytorch.py @@ -17,6 +17,7 @@ from doctr.models.layout.predictor import LayoutPredictor from doctr.models.recognition.predictor import RecognitionPredictor from doctr.models.table_structure.predictor import TablePredictor +from doctr.models.table_structure.tablecenternet.base import _logic_from_geometry from doctr.utils.geometry import detach_scores from .base import _OCRPredictor @@ -233,6 +234,15 @@ def _predict_pages( out = self._remap_to_original_coords(out, _orig_shapes, _straight_shapes, m_invs, _orig_pages) return out + @staticmethod + def _cell_polygon(geometry: Any) -> np.ndarray: + """Normalize a cell geometry, a straight box (xmin, ymin, xmax, ymax) or a polygon, to a (4, 2) polygon""" + raw = np.asarray(geometry, dtype=np.float32) + if raw.size == 4: # straight box (xmin, ymin, xmax, ymax) + x0, y0, x1, y1 = raw.ravel() + return np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) + return raw.reshape(-1, 2).copy() + def _tables_from_regions( self, pages: list[np.ndarray], @@ -244,7 +254,8 @@ def _tables_from_regions( Each `Table` region is rectified to an upright crop with a perspective transform (so straight boxes and rotated polygons are both handled), the table model is applied per crop, and the predicted cell polygons are mapped back to page-relative coordinates through the inverse transform. A page therefore yields one - structured table per detected `Table` region. + structured table per detected `Table` region. The crop includes a margin of page context around the region: + the cells predicted in this margin belong to the neighbouring content and are dropped. Args: pages: the (possibly straightened) page images @@ -252,12 +263,13 @@ def _tables_from_regions( **kwargs: keyword arguments forwarded to the table predictor Returns: - a per-page list of table grids `{"cells": [...], "num_rows": int, "num_cols": int}` in page-relative - coordinates + a per-page list of table grids `{"cells": [...], "num_rows": int, "num_cols": int, "geometry": region}` + in page-relative coordinates, `geometry` being the (4, 2) polygon of the table region """ crops: list[np.ndarray] = [] - # (page index, inverse transform, crop width, crop height, page width, page height) - crop_meta: list[tuple[int, np.ndarray, int, int, int, int]] = [] + # (page index, inverse transform, crop width, crop height, context margin, page width, page height, + # page-relative region) + crop_meta: list[tuple[int, np.ndarray, int, int, int, int, int, np.ndarray]] = [] for p_idx, (page, region) in enumerate(zip(pages, regions)): if region is None: continue @@ -290,30 +302,52 @@ def _tables_from_regions( transform = cv2.getPerspectiveTransform(src, dst) inverse = cv2.getPerspectiveTransform(dst, src) crops.append(cv2.warpPerspective(page, transform, (out_w, out_h), borderMode=cv2.BORDER_REPLICATE)) - crop_meta.append((p_idx, inverse, out_w, out_h, w, h)) + crop_meta.append((p_idx, inverse, out_w, out_h, pad, w, h, src / np.array([w, h], dtype=np.float32))) tables_per_page: list[list[dict[str, Any]]] = [[] for _ in pages] if len(crops) == 0: return tables_per_page grids = self.table_predictor(crops, **kwargs) # type: ignore[misc] - for (p_idx, inverse, crop_w, crop_h, w, h), grid in zip(crop_meta, grids): + # Whether the structure model makes the logical coordinates consistent with the cell geometry + postprocessor = getattr(getattr(self.table_predictor, "model", None), "postprocessor", None) + realign = getattr(postprocessor, "geometric_logic", False) + for (p_idx, inverse, crop_w, crop_h, pad, w, h, region_poly), grid in zip(crop_meta, grids): + if len(grid["cells"]) == 0: + continue + # Cell polygons: crop-relative -> crop pixels + polys = np.stack([self._cell_polygon(cell["geometry"]) for cell in grid["cells"]]) + polys *= np.array([crop_w, crop_h], dtype=np.float32) + # The context margin around the region shows the neighbouring content (e.g. the cells of an adjacent + # table): the cells centered there are not part of this table + centers = polys.mean(axis=1) + keep = np.flatnonzero( + (centers[:, 0] >= pad) + & (centers[:, 0] <= crop_w - pad) + & (centers[:, 1] >= pad) + & (centers[:, 1] <= crop_h - pad) + ) + if keep.size == 0: + continue + cells = [grid["cells"][idx] for idx in keep] + logic = np.array([[c["col_start"], c["col_end"], c["row_start"], c["row_end"]] for c in cells]) + if realign and keep.size < polys.shape[0]: + # The dropped cells made grid lines of their own: index the rows / columns of the remaining ones again + logic = _logic_from_geometry(polys[keep], logic, split_tables=False) remapped_cells: list[dict[str, Any]] = [] - for cell in grid["cells"]: - # Cell polygon is crop-relative -> crop pixels -> page pixels (inverse transform) -> page-relative - raw = np.asarray(cell["geometry"], dtype=np.float32) - if raw.ndim == 1 and raw.size == 4: # straight box (xmin, ymin, xmax, ymax) - x0, y0, x1, y1 = raw - poly = np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) - else: - poly = raw.reshape(-1, 2) - poly[:, 0] *= crop_w - poly[:, 1] *= crop_h + for cell, poly, (col_start, col_end, row_start, row_end) in zip(cells, polys[keep], logic): + # crop pixels -> page pixels (inverse transform) -> page-relative poly = cv2.perspectiveTransform(poly[None, :, :], inverse)[0] poly[:, 0] /= w poly[:, 1] /= h new_cell = dict(cell) - new_cell["geometry"] = poly.tolist() + new_cell.update( + geometry=poly.tolist(), + row_start=int(row_start), + row_end=int(row_end), + col_start=int(col_start), + col_end=int(col_end), + ) remapped_cells.append(new_cell) if not remapped_cells: continue @@ -321,5 +355,7 @@ def _tables_from_regions( "cells": remapped_cells, "num_rows": grid["num_rows"], "num_cols": grid["num_cols"], + # The layout region of the table (TL, TR, BR, BL): every word inside it belongs to the table + "geometry": region_poly.tolist(), }) return tables_per_page diff --git a/doctr/models/table_structure/tablecenternet/base.py b/doctr/models/table_structure/tablecenternet/base.py index b82b19c9e6..bea453b9c5 100644 --- a/doctr/models/table_structure/tablecenternet/base.py +++ b/doctr/models/table_structure/tablecenternet/base.py @@ -147,50 +147,31 @@ def window(lo: np.ndarray | float, hi: np.ndarray | float): return np.sort(np.asarray(lines)) -def _vote_line_indices( - n_lines: int, start: np.ndarray, end: np.ndarray, model_logic: np.ndarray -) -> tuple[np.ndarray, np.ndarray]: - """Logical index of each grid line, voted by the model's logical coordinates of the cells lying on it. +def _index_lines(lines: np.ndarray, start: np.ndarray, end: np.ndarray, cell_size: float) -> np.ndarray: + """Logical index of each grid line of an axis, from the cells lying between the lines. + + Every cell spans at least one index (the line it ends on comes after the line it starts on), and the lines are + otherwise given the smallest possible indices: the lines no cell separates (e.g. the edges of a row that do not + line up) share an index. A gap covered by no cell (e.g. a row with no detected cell) counts as many empty indices + as cells of the typical size fit in it. Args: - n_lines: number of grid lines + lines: the sorted positions of the grid lines, shape (L,) start: index of the line each cell starts on, shape (N,) - end: index of the line each cell ends on, shape (N,) - model_logic: the model's (start, end) logical coordinates of the cells along this axis, shape (N, 2) + end: index of the line each cell ends on (after its start line), shape (N,) + cell_size: the typical size of a cell along the axis Returns: - the (non-decreasing) logical index of each line, shape (n_lines,), and whether a strict majority of the cells - on the line voted for it, shape (n_lines,) + the (non-decreasing) logical index of each line, shape (L,) """ - votes: list[list[int]] = [[] for _ in range(n_lines)] - starts_of: list[list[int]] = [[] for _ in range(n_lines)] # start lines of the cells ending on each line - for s, e, (m_start, m_end) in zip(start, end, model_logic): - votes[s].append(int(m_start)) # a cell starts on its line at its start index - votes[e].append(int(m_end) + 1) # and ends on the line following its last index - starts_of[e].append(int(s)) - values = np.zeros(n_lines, dtype=np.int64) - majority = np.zeros(n_lines, dtype=bool) - prev = -1 - for k, v in enumerate(votes): - value = max(prev, 0) - if v: - uniq, counts = np.unique(v, return_counts=True) - value = int(uniq[counts.argmax()]) # most voted (the smallest one on ties) - # A split vote is not a drift of the model: some annotations index a same line differently per row - majority[k] = counts.max() > len(v) / 2 - if majority[k]: - # Lines are ordered: their indices can not decrease. Several lines may share an index (the edges of a - # row / column do not always line up), so they are not forced to strictly increase - value = max(value, prev) - # But a cell spans at least one index: the line moves on when at least half of the cells ending on it - # would span none - need = [values[s] + 1 for s in starts_of[k] if majority[s] and values[s] + 1 > value] - if need and 2 * len(need) >= len(starts_of[k]): - value = max(need) - values[k] = prev = value - else: - values[k] = value - return values, majority + values = np.zeros(lines.size, dtype=np.int64) + for k in range(1, lines.size): + value = values[k - 1] + if not np.any((start <= k - 1) & (end >= k)): # no cell covers the gap between the 2 lines + value += int(round((lines[k] - lines[k - 1]) / max(cell_size, 1e-9))) + ending = start[end == k] + values[k] = max(value, int(values[ending].max()) + 1) if ending.size else value + return values def _connected_tables(left: np.ndarray, right: np.ndarray, top: np.ndarray, bottom: np.ndarray) -> np.ndarray: @@ -238,7 +219,7 @@ def _align_table_logic(edges: tuple[np.ndarray, ...], logic: np.ndarray) -> np.n if corr[g, a] < 0: # the logical indices grow against the geometric axis start_edge, end_edge = -end_edge, -start_edge # Tolerance on the alignment of the edges, from the narrowest cells along this axis: merging the 2 lines of a - # narrow row / column would lose it, while splitting a line only makes lines sharing the same index + # narrow row / column would lose it, while the lines split from a line that no cell separates share an index tol = max(0.4 * float(np.percentile(end_edge - start_edge, 10)), 1e-6) lines = _cluster_lines(np.concatenate([start_edge, end_edge]), tol) if lines.size < 2: # degenerate (flat) cells: nothing to align @@ -248,36 +229,36 @@ def _align_table_logic(edges: tuple[np.ndarray, ...], logic: np.ndarray) -> np.n end = np.abs(end_edge[:, None] - lines[None]).argmin(axis=1) end = np.clip(end, 1, lines.size - 1) start = np.minimum(start, end - 1) - values, majority = _vote_line_indices(lines.size, start, end, logic[:, offset : offset + 2]) - model_start, model_end = logic[:, offset], logic[:, offset + 1] - span = np.maximum(model_end - model_start, 0) - line_start, line_end = values[start], values[end] - 1 - new_start = np.where(majority[start], line_start, np.where(majority[end], line_end - span, model_start)) - new_end = np.where(majority[end], line_end, np.where(majority[start], line_start + span, model_end)) - logic[:, offset] = new_start - logic[:, offset + 1] = np.maximum(new_end, new_start) + values = _index_lines(lines, start, end, float(np.median(end_edge - start_edge))) + # The indices follow the geometry, from the absolute position the model predicts for most of the cells (e.g. + # the cells below a row with no detected cell, apart from the cells above it, keep their indices) + values += max(int(round(float(np.median(logic[:, offset] - values[start])))), 0) + logic[:, offset] = values[start] + logic[:, offset + 1] = values[end] - 1 return logic -def _logic_from_geometry(polys: np.ndarray, model_logic: np.ndarray) -> np.ndarray: +def _logic_from_geometry(polys: np.ndarray, model_logic: np.ndarray, split_tables: bool = True) -> np.ndarray: """Make the logical coordinates `[start_col, end_col, start_row, end_row]` (0-indexed) of (N, 4, 2) cells consistent with their geometry. The cells are split into tables (groups of touching cells), and in each table the cell edges are grouped into - the row and column lines of the grid: each line takes the logical index the model predicts for most of the cells - on it. The cells of a visual row / column share their indices, even when the model drifts on some of them, while - the absolute indices of the model are kept (e.g. a row with no detected cell still counts). + the row and column lines of the grid, indexed from the geometry (cf. `_index_lines`): the cells of a visual row / + column share their indices, even when the logical-coordinate head of the model drifts (e.g. on borderless + tables). The model coordinates give the direction of the axes and the absolute position of each table (e.g. the + cells below a row with no detected cell keep their indices). Args: polys: the cells, of shape (N, 4, 2) model_logic: the logical coordinates predicted by the model, of shape (N, 4) + split_tables: if False, the cells are known to belong to a single table and are aligned together Returns: the logical coordinates, of shape (N, 4) """ edges = _cell_edges(_deskew_cells(polys)) logic = np.asarray(model_logic, dtype=np.int64).copy() - labels = _connected_tables(*edges) + labels = _connected_tables(*edges) if split_tables else np.zeros(logic.shape[0], dtype=np.int64) for label in np.unique(labels): idx = np.flatnonzero(labels == label) logic[idx] = _align_table_logic(tuple(e[idx] for e in edges), logic[idx]) @@ -306,8 +287,8 @@ class TableCenterNetPostProcessor: not_relocate: if True, skip the corner-relocation step (faster, less accurate) assume_straight_pages: whether the pages are assumed to be straight (i.e., no rotation) geometric_logic: if True, remove the duplicate (overlapping) cells and make the logical coordinates - consistent with the cell geometry: the cells whose edges lie on the same grid line share the logical - index most of them are predicted with (the logical-coordinate head drifts on borderless tables) + consistent with the cell geometry: the rows and columns are indexed from the grid lines the cell edges + lie on (the logical-coordinate head drifts on borderless tables) """ def __init__( diff --git a/doctr/models/table_structure/tablecenternet/pytorch.py b/doctr/models/table_structure/tablecenternet/pytorch.py index 0a6e74a4b2..cbc6a45b78 100644 --- a/doctr/models/table_structure/tablecenternet/pytorch.py +++ b/doctr/models/table_structure/tablecenternet/pytorch.py @@ -26,7 +26,7 @@ "input_shape": (3, 1024, 1024), "mean": (0.798, 0.785, 0.772), "std": (0.264, 0.2749, 0.287), - "url": "https://github.com/mindee/doctr/releases/download/v1.1.0/tablecenternet-fdc82796.pt", + "url": "https://github.com/mindee/doctr/releases/download/v1.1.0/tablecenternet-0d349a85.pt", }, } diff --git a/tests/common/test_models_builder.py b/tests/common/test_models_builder.py index 252960d3a7..93d7f7d7ad 100644 --- a/tests/common/test_models_builder.py +++ b/tests/common/test_models_builder.py @@ -573,17 +573,18 @@ def test_documentbuilder_tables(): _table_cell(0.10, 0.10, 0.25, 0.20, 0, 0, 0, 0, score=0.8), _table_cell(0.28, 0.10, 0.45, 0.20, 0, 0, 1, 1, score=0.6), _table_cell(0.10, 0.22, 0.25, 0.32, 1, 1, 0, 0, score=0.4), # no word inside this cell + _table_cell(0.28, 0.22, 0.45, 0.32, 1, 1, 1, 1, score=0.5), ], "num_rows": 2, "num_cols": 2, } - page = doc_builder(*_table_words("Name", "Age", confidences=[0.95, 0.85]), tables=[table]).pages[0] + page = doc_builder(*_table_words("Name", "Age", "30", confidences=[0.95, 0.85, 0.75]), tables=[table]).pages[0] cells = page.tables[0].cells - assert [cell.value for cell in cells] == ["Name", "Age", ""] - assert [cell.objectness_score for cell in cells] == [0.8, 0.6, 0.4] - assert [cell.confidence for cell in cells] == pytest.approx([0.95, 0.85, 0.4]) - assert page.tables[0].confidence == pytest.approx(0.6) - assert [cell["objectness_score"] for cell in page.export()["tables"][0]["cells"]] == [0.8, 0.6, 0.4] + assert [cell.value for cell in cells] == ["Name", "Age", "", "30"] + assert [cell.objectness_score for cell in cells] == [0.8, 0.6, 0.4, 0.5] + assert [cell.confidence for cell in cells] == pytest.approx([0.95, 0.85, 0.4, 0.75]) + assert page.tables[0].confidence == pytest.approx(0.575) + assert [cell["objectness_score"] for cell in page.export()["tables"][0]["cells"]] == [0.8, 0.6, 0.4, 0.5] # Multi-word cells are read line by line, left to right words = ["world", "hello", "second", "line", "solo"] @@ -656,6 +657,134 @@ def test_documentbuilder_tables(): json.dumps(page.export()) +def _grid_cells(xs, ys): + """Cells of a regular grid from its column and row lines""" + return [ + _table_cell(xs[c], ys[r], xs[c + 1], ys[r + 1], r, r, c, c) + for r in range(len(ys) - 1) + for c in range(len(xs) - 1) + ] + + +def _word_box(cx, cy, w=0.04, h=0.02): + return [cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2] + + +@pytest.mark.parametrize("straight", [True, False]) +def test_documentbuilder_tables_complete_grid(straight): + # A 2 x 2 grid predicted by the table model, inside a larger table region found by the layout model + xs, ys = [0.20, 0.40, 0.60], [0.30, 0.40, 0.50] + cells = _grid_cells(xs, ys)[:3] # the cell of the last row and column is missed + region = [[0.10, 0.15], [0.70, 0.15], [0.70, 0.65], [0.10, 0.65]] + words = { + "A": (0.30, 0.35), + "B": (0.50, 0.35), + "C": (0.30, 0.45), + "missed": (0.50, 0.45), # in the slot of the missed cell + "Col1": (0.30, 0.25), # a header row above the grid + "Col2": (0.50, 0.25), + "Total": (0.25, 0.58), # a sentence below the grid, across both columns + "amount": (0.3225, 0.58), + "due": (0.405, 0.58), + "key": (0.15, 0.35), # a column on the left of the grid + "caption": (0.40, 0.80), # outside of the table region + } + widths = {"Total": 0.06, "amount": 0.075, "due": 0.08} # the words of the sentence are 5px apart + boxes = [_word_box(*center, w=widths.get(name, 0.04)) for name, center in words.items()] + if not straight: + boxes = [[[x0, y0], [x1, y0], [x1, y1], [x0, y1]] for x0, y0, x1, y1 in boxes] + table = {"cells": cells, "num_rows": 2, "num_cols": 2, "geometry": region} + page = builder.DocumentBuilder()(*_word_args(list(words), boxes), tables=[[table]]).pages[0] + # Every word of the table region ends up in the table, the caption stays in the blocks + assert _words(page) == ["caption"] + grid = page.tables[0].to_grid() + assert grid == [ + ["", "Col1", "Col2"], + ["key", "A", "B"], + ["", "C", "missed"], + ["", "Total amount due", ""], + ] + # The cells added from the words are not scored by the table model, and span the columns of their text + total = next(cell for cell in page.tables[0].cells if cell.value.startswith("Total")) + assert total.objectness_score is None and (total.col_start, total.col_end) == (1, 2) + assert page.tables[0].confidence == pytest.approx(0.9) + json.dumps(page.export()) + + # Without the table region, the words around the cells stay in the blocks (only the cell outline is known) + table = {"cells": cells, "num_rows": 2, "num_cols": 2} + page = builder.DocumentBuilder()(*_word_args(list(words), boxes), tables=[[table]]).pages[0] + assert page.tables[0].to_grid() == [["A", "B"], ["C", "missed"]] + assert sorted(_words(page)) == sorted(["Col1", "Col2", "Total", "amount", "due", "key", "caption"]) + + +def test_documentbuilder_tables_simplify_grid(): + # The model splits a row in 2 (its cells do not line up) and predicts an empty row: the text of the split row + # lies on the same line, so the 2 rows are merged, and the empty row is removed + cells = [ + _table_cell(0.10, 0.10, 0.30, 0.20, 0, 0, 0, 0), + _table_cell(0.30, 0.10, 0.50, 0.20, 0, 0, 1, 1), + _table_cell(0.10, 0.20, 0.30, 0.30, 1, 1, 0, 0), + _table_cell(0.30, 0.20, 0.50, 0.24, 1, 1, 1, 1), # empty, a row of its own + _table_cell(0.30, 0.24, 0.50, 0.34, 2, 2, 1, 1), + _table_cell(0.10, 0.34, 0.30, 0.44, 3, 3, 0, 0), # an empty row + _table_cell(0.30, 0.34, 0.50, 0.44, 3, 3, 1, 1), + _table_cell(0.10, 0.44, 0.30, 0.54, 4, 4, 0, 0), + _table_cell(0.30, 0.44, 0.50, 0.54, 4, 4, 1, 1), + ] + words = ["h1", "h2", "a", "b", "c", "d"] + boxes = [_word_box(0.2, 0.15), _word_box(0.4, 0.15), _word_box(0.2, 0.25), _word_box(0.4, 0.255)] + boxes += [_word_box(0.2, 0.49), _word_box(0.4, 0.49)] + table = {"cells": cells, "num_rows": 5, "num_cols": 2} + page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] + assert page.tables[0].to_grid() == [["h1", "h2"], ["a", "b"], ["c", "d"]] + assert (page.tables[0].num_rows, page.tables[0].num_cols) == (3, 2) + # Text on different lines is not merged + boxes[3] = _word_box(0.4, 0.31) + page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] + assert page.tables[0].to_grid() == [["h1", "h2"], ["a", ""], ["", "b"], ["c", "d"]] + + +def test_documentbuilder_tables_split_rows(): + # The model merges the 3 text lines of a dense table into a single row + cells = [ + _table_cell(0.10, 0.10, 0.30, 0.16, 0, 0, 0, 0), + _table_cell(0.30, 0.10, 0.50, 0.16, 0, 0, 1, 1), + _table_cell(0.10, 0.16, 0.30, 0.34, 1, 1, 0, 0), + _table_cell(0.30, 0.16, 0.50, 0.34, 1, 1, 1, 1), + ] + words, boxes = ["Name", "Value"], [_word_box(0.2, 0.13), _word_box(0.4, 0.13)] + for line, cy in enumerate((0.19, 0.25, 0.31)): + words += [f"key{line}", f"{line}.00"] + boxes += [_word_box(0.2, cy), _word_box(0.4, cy)] + table = {"cells": cells, "num_rows": 2, "num_cols": 2} + page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] + assert page.tables[0].to_grid() == [["Name", "Value"], ["key0", "0.00"], ["key1", "1.00"], ["key2", "2.00"]] + + # A single cell wrapping its text (e.g. a long header) does not split its row + words = ["Unit", "price", "Qty", "10", "2"] + boxes = [_word_box(0.2, 0.12), _word_box(0.2, 0.14), _word_box(0.4, 0.13)] + boxes += [_word_box(0.2, 0.25), _word_box(0.4, 0.25)] + page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] + assert page.tables[0].to_grid() == [["Unit price", "Qty"], ["10", "2"]] + + # A cell spanning 2 rows whose text lines lie in each of them, aligned with the text of the row + cells = [ + _table_cell(0.10, 0.10, 0.30, 0.20, 0, 0, 0, 0), + _table_cell(0.10, 0.20, 0.30, 0.30, 1, 1, 0, 0), + _table_cell(0.30, 0.10, 0.50, 0.30, 0, 1, 1, 1), + ] + words = ["Acquired", "Taken", "51", "9"] + boxes = [_word_box(0.2, 0.15), _word_box(0.2, 0.25), _word_box(0.4, 0.15), _word_box(0.4, 0.25)] + table = {"cells": cells, "num_rows": 2, "num_cols": 2} + page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] + assert page.tables[0].to_grid() == [["Acquired", "51"], ["Taken", "9"]] + # A label spanning 2 rows on a single line is kept as is + page = builder.DocumentBuilder()( + *_word_args(words[:2] + ["label"], boxes[:2] + [_word_box(0.4, 0.2)]), tables=[[table]] + ).pages[0] + assert [(c.value, c.row_span) for c in page.tables[0].cells] == [("Acquired", 1), ("Taken", 1), ("label", 2)] + + @pytest.mark.parametrize( "words, boxes, grids, tables", [ diff --git a/tests/common/test_models_table_structure.py b/tests/common/test_models_table_structure.py index 602097cbf4..9cc1194a9f 100644 --- a/tests/common/test_models_table_structure.py +++ b/tests/common/test_models_table_structure.py @@ -5,9 +5,9 @@ from doctr.models.table_structure.tablecenternet.base import ( _build_table_target, _cluster_lines, + _index_lines, _logic_from_geometry, _suppress_overlapping_cells, - _vote_line_indices, ) @@ -121,31 +121,35 @@ def test_tablecenternet_logic_from_geometry_tables(): flipped[..., 1] = 1 - flipped[..., 1] np.testing.assert_array_equal(_logic_from_geometry(flipped, drifted), gt) - # The cells on a line do not agree on its index (2 against 2): the model coordinates are kept + # The model counts one more column in the 1st row than in the 2nd one, on a regular grid: the cells of a + # visual column share their index two_rows = _grid_target(2, 3, use_polygons=True) split = two_rows["logic"].copy() - split[:3, :2] += [[0, 1], [1, 1], [1, 1]] # the 1st row counts one more column than the 2nd one - np.testing.assert_array_equal(_logic_from_geometry(two_rows["cells"], split), split) - - -def test_tablecenternet_vote_line_indices(): - def vote(cells): - start, end, logic = (np.array(v) for v in zip(*cells)) - return _vote_line_indices(3, start, end, logic) - - # 3 cells between 3 lines: the 2 cells after the 1st line drift to the index of the 1st cell. The vote of line 1 - # is 0 but its cell would span no index: the line moves on (and so does the next one) - values, majority = vote([(0, 1, (0, 0)), (1, 2, (0, 0)), (1, 2, (0, 0))]) - np.testing.assert_array_equal(values, [0, 1, 2]) - assert majority.all() - # A minority of cells spanning no index (4 cells of a split header row, against 6 rows-high cells) does not move - # the line: the other ones would be shifted - cells = [(0, 1, (0, 0))] * 4 + [(1, 2, (1, 1))] * 4 + [(0, 2, (0, 0))] * 6 - values, _ = vote(cells) - np.testing.assert_array_equal(values, [0, 1, 1]) - # A split vote does not decide the index of a line - values, majority = vote([(0, 1, (0, 0)), (0, 1, (0, 1)), (1, 2, (1, 1)), (1, 2, (2, 2))]) - np.testing.assert_array_equal(majority, [True, False, False]) + split[:3, :2] += [[0, 1], [1, 1], [1, 1]] + np.testing.assert_array_equal(_logic_from_geometry(two_rows["cells"], split), two_rows["logic"]) + + # The model drifts by one row on half of the columns (e.g. a borderless table): the rows follow the geometry + grid = _grid_target(3, 4, use_polygons=True) + drifted = grid["logic"].copy() + drifted[drifted[:, 0] >= 2] += [0, 0, 1, 1] + np.testing.assert_array_equal(_logic_from_geometry(grid["cells"], drifted), grid["logic"]) + + +def test_tablecenternet_index_lines(): + def index(lines, cells, size=10.0): + start, end = (np.array(v) for v in zip(*cells)) + return _index_lines(np.asarray(lines, dtype=float), start, end, size).tolist() + + # A regular grid: one index per line + assert index([0, 10, 20, 30], [(0, 1), (1, 2), (2, 3)]) == [0, 1, 2, 3] + # A stray line (a shorter cell) separates no cell from the line after it: both lines share an index + assert index([0, 7, 10, 20], [(0, 1), (0, 2), (2, 3), (2, 3)]) == [0, 1, 1, 2] + # A cell spanning 2 rows of the other cells + assert index([0, 10, 20], [(0, 2), (0, 1), (1, 2)]) == [0, 1, 2] + # A gap of 2 cell sizes covered by no cell counts as 2 empty indices, a narrow one (between cells that do not + # touch) as none + assert index([0, 10, 30, 40], [(0, 1), (2, 3)]) == [0, 1, 3, 4] + assert index([0, 10, 12, 22], [(0, 1), (2, 3)]) == [0, 1, 1, 2] def test_tablecenternet_cluster_lines(): diff --git a/tests/pytorch/test_models_zoo_pt.py b/tests/pytorch/test_models_zoo_pt.py index 8af57e0541..e17620d9b7 100644 --- a/tests/pytorch/test_models_zoo_pt.py +++ b/tests/pytorch/test_models_zoo_pt.py @@ -408,6 +408,55 @@ def test_ocrpredictor_tables(mock_pdf, mock_vocab): assert exported["tables"] == [t.export() for t in page.tables] +def test_ocrpredictor_tables_from_regions(): + class _FakeTablePredictor: + """Returns a fixed grid (crop-relative cells) for every crop, with a cell in the context margin""" + + model = None + + def __init__(self): + self.crops = [] + + def __call__(self, crops, **kwargs): + self.crops = crops + h, w = crops[0].shape[:2] + pad = 20 # the context margin of a region smaller than 2000 px + + def cell(x0, y0, x1, y1, r, c): + box = [(pad + x0) / w, (pad + y0) / h, (pad + x1) / w, (pad + y1) / h] + return {"geometry": box, "score": 0.9, "row_start": r, "row_end": r, "col_start": c, "col_end": c} + + # 2 cells of the table, and a cell of the neighbouring content, above the region + cells = [cell(0, 0, 100, 50, 1, 0), cell(100, 0, 200, 50, 1, 1), cell(0, -18, 200, -4, 0, 0)] + return [{"cells": cells, "num_rows": 2, "num_cols": 2}] + + fake = type("FakeOCRPredictor", (), {})() + fake.table_predictor = _FakeTablePredictor() + fake.table_class_name = "Table" + fake._cell_polygon = OCRPredictor._cell_polygon + page = np.full((400, 600, 3), 255, dtype=np.uint8) + region = { + "class_names": ["Text", "Table"], + "boxes": np.array([[0, 0, 0.1, 0.1], [0.1, 0.25, 0.1 + 200 / 600, 0.375]]), + } + tables = OCRPredictor._tables_from_regions(fake, [page], [region]) + # Only the table region is cropped, with a context margin + assert len(fake.table_predictor.crops) == 1 and fake.table_predictor.crops[0].shape[:2] == (90, 240) + assert len(tables) == 1 and len(tables[0]) == 1 + table = tables[0][0] + # The cell of the context margin is dropped, the others are mapped back to page-relative coordinates + assert len(table["cells"]) == 2 + np.testing.assert_allclose( + table["cells"][0]["geometry"], + [[0.1, 0.25], [0.1 + 100 / 600, 0.25], [0.1 + 100 / 600, 0.375], [0.1, 0.375]], + atol=1e-5, + ) + # The table region is attached to the grid + np.testing.assert_allclose( + table["geometry"], [[0.1, 0.25], [0.1 + 200 / 600, 0.25], [0.1 + 200 / 600, 0.375], [0.1, 0.375]], atol=1e-6 + ) + + def test_ocrpredictor_tables_factory(): # The factory exposes a single `detect_tables` flag, which also enables the layout model predictor = models.ocr_predictor("db_mobilenet_v3_large", "crnn_vgg16_bn", pretrained=False, detect_tables=True) From 93597d9a5686dab0f3eab09285645ee375c53bff Mon Sep 17 00:00:00 2001 From: felix Date: Tue, 10 Nov 2026 10:37:20 +0100 Subject: [PATCH 2/8] Update --- doctr/models/builder.py | 600 ++++--------------------- doctr/models/predictor/pytorch.py | 104 ++--- doctr/models/table_structure/grid.py | 348 ++++++++++++++ tests/common/test_models_builder.py | 149 +----- tests/common/test_models_table_grid.py | 120 +++++ tests/pytorch/test_models_zoo_pt.py | 36 +- 6 files changed, 643 insertions(+), 714 deletions(-) create mode 100644 doctr/models/table_structure/grid.py create mode 100644 tests/common/test_models_table_grid.py diff --git a/doctr/models/builder.py b/doctr/models/builder.py index 80fa893ec0..d9475a17d2 100644 --- a/doctr/models/builder.py +++ b/doctr/models/builder.py @@ -450,6 +450,20 @@ def _build_layout_elements(regions: dict[str, Any] | None) -> list[LayoutElement elements.append(LayoutElement(layout_type=str(cname), confidence=float(score), geometry=geometry)) return elements + @staticmethod + def _word_centers(boxes: np.ndarray) -> np.ndarray: + """Return the (x, y) center of each word box. + + Args: + boxes: word boxes of shape (N, 4) (straight: x1, y1, x2, y2) or (N, 4, 2) (rotated polygon) + + Returns: + array of shape (N, 2) with the relative center coordinates of each box + """ + if boxes.ndim == 3: # rotated polygons (N, 4, 2) + return boxes.mean(axis=1) + return np.stack([(boxes[:, 0] + boxes[:, 2]) / 2, (boxes[:, 1] + boxes[:, 3]) / 2], axis=1) + @staticmethod def _as_cell_polygon(geometry: Any) -> np.ndarray: """Normalize a cell geometry to a (4, 2) polygon. @@ -464,8 +478,8 @@ def _as_cell_polygon(geometry: Any) -> np.ndarray: array of shape (4, 2) with the polygon vertices """ arr = np.asarray(geometry, dtype=np.float32) - if arr.size == 4: # straight box (xmin, ymin, xmax, ymax) or ((xmin, ymin), (xmax, ymax)) - x0, y0, x1, y1 = arr.ravel() + if arr.ndim == 1: # straight box (xmin, ymin, xmax, ymax) + x0, y0, x1, y1 = arr return np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) return order_points(arr.reshape(-1, 2)) @@ -549,36 +563,28 @@ def _localize_logic(cells: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], @classmethod def _table_grids( cls, page_table: dict[str, Any] | list[dict[str, Any]] | None - ) -> list[tuple[list[dict[str, Any]], int, int, np.ndarray | None]]: + ) -> list[tuple[list[dict[str, Any]], int, int]]: """Normalize the table structure prediction(s) of a page to non-empty grids with local logical coordinates Args: page_table: a single grid, a list of grids or None (cf. `_build_tables`) Returns: - the cells, number of rows, number of columns and (4, 2) region polygon (None if unknown) of each - non-empty grid + the cells, number of rows and number of columns of each non-empty grid """ raw_tables = [] if page_table is None else [page_table] if isinstance(page_table, dict) else list(page_table) - grids = [] - for raw in raw_tables: - if not raw or len(raw.get("cells", [])) == 0: - continue - region = raw.get("geometry") - grids.append(( - *cls._localize_logic(raw["cells"]), - None if region is None else cls._as_cell_polygon(region), - )) - return grids + return [cls._localize_logic(raw["cells"]) for raw in raw_tables if raw and len(raw.get("cells", [])) > 0] @classmethod def _assign_cell_words(cls, centers: np.ndarray, polys: np.ndarray, consumed: np.ndarray) -> list[list[int]]: - """Assign each word not yet consumed whose center lies in a cell to the first such cell, and flag it as - consumed + """Assign each word not yet consumed to at most one cell of a table, and flag it as consumed + + A word goes to the first cell containing its center. A word landing just outside every cell but inside the + table goes to the nearest cell, within half a median cell diagonal, so table text is not dropped into the body. Args: centers: the (N, 2) word centers - polys: the (C, 4, 2) cell polygons + polys: the (C, 4, 2) cell polygons, vertices ordered TL, TR, BR, BL consumed: the (N,) mask of the words already assigned to a table, updated in place Returns: @@ -593,494 +599,60 @@ def _assign_cell_words(cls, centers: np.ndarray, polys: np.ndarray, consumed: np for w_idx, c_idx in zip(free[assigned], inside[assigned].argmax(axis=1)): cell_words[c_idx].append(int(w_idx)) consumed[w_idx] = True + leftover = free[~assigned] + if leftover.size > 0: + cell_centers = polys.mean(axis=1) # (C, 2) + max_dist = 0.5 * float(np.median(np.linalg.norm(polys[:, 2] - polys[:, 0], axis=1))) # TL->BR diagonal + outline = cv2.boxPoints(cv2.minAreaRect(polys.reshape(-1, 2).astype(np.float32))).reshape(1, 4, 2) + in_table = cls._points_in_polygons(centers[leftover], outline)[:, 0] + for w_idx in leftover[in_table]: + dists = np.linalg.norm(cell_centers - centers[w_idx], axis=1) + nearest = int(dists.argmin()) + if dists[nearest] <= max_dist: + cell_words[nearest].append(int(w_idx)) + consumed[w_idx] = True return cell_words @staticmethod - def _rotate_points(points: np.ndarray, angle: float, pivot: np.ndarray) -> np.ndarray: - """Rotate (..., 2) points by `angle` radians around `pivot`""" - cos_a, sin_a = np.cos(angle), np.sin(angle) - shifted = points - pivot - return np.stack( - [ - pivot[0] + shifted[..., 0] * cos_a - shifted[..., 1] * sin_a, - pivot[1] + shifted[..., 0] * sin_a + shifted[..., 1] * cos_a, - ], - axis=-1, - ) - - @staticmethod - def _grid_lines(start: np.ndarray, end: np.ndarray, lo: np.ndarray, hi: np.ndarray, size: int) -> np.ndarray: - """Positions of the `size + 1` grid lines of an axis, from the cell edges lying on each of them - - Line `k` is where index `k` starts and index `k - 1` ends: it lies at the median of the corresponding cell - edges, the lines on which no cell edge lies are interpolated between their neighbours. - - Args: - start: the first index of each cell along the axis, shape (C,) - end: the last index of each cell along the axis, shape (C,) - lo: the lower edge of each cell along the axis (in the table frame), shape (C,) - hi: the upper edge of each cell along the axis (in the table frame), shape (C,) - size: the number of indices (rows or columns) of the axis - - Returns: - the non-decreasing positions of the grid lines, shape (size + 1,) - """ - edges: list[list[float]] = [[] for _ in range(size + 1)] - for s, e, low, high in zip(start, end, lo, hi): - edges[int(s)].append(float(low)) - edges[int(e) + 1].append(float(high)) - known = [k for k, values in enumerate(edges) if values] - lines = np.interp(np.arange(size + 1), known, [float(np.median(edges[k])) for k in known]) - return np.maximum.accumulate(lines) - - @staticmethod - def _axis_slots( - center: np.ndarray, lo: np.ndarray, hi: np.ndarray, lines: np.ndarray, line_tol: float | None = None - ) -> tuple[np.ndarray, dict[int, tuple[float, float]]]: - """Index of each word along an axis of a table grid, adding new indices for the words beyond the grid - - A word whose center lies between two grid lines takes their index. The words before the first line (resp. - after the last one) are grouped into new indices before (resp. after) the grid: by text line (centers closer - than `line_tol`) for rows, by overlapping extents for columns (`line_tol` is None). - - Args: - center: the word centers along the axis (in the table frame), shape (W,) - lo: the lower word edges along the axis, shape (W,) - hi: the upper word edges along the axis, shape (W,) - lines: the positions of the grid lines of the axis, shape (size + 1,) - line_tol: the maximum distance between the centers of the words of a same new row - - Returns: - the (possibly negative or beyond the grid) index of each word, and the (lo, hi) extent of every index - """ - size = lines.size - 1 - idx = np.clip(np.searchsorted(lines, center, side="right") - 1, 0, size - 1) - bands = {k: (float(lines[k]), float(lines[k + 1])) for k in range(size)} - for before, mask in ((True, center < lines[0]), (False, center > lines[-1])): - members = np.flatnonzero(mask) - if members.size == 0: - continue - groups: list[list[int]] = [] - if line_tol is not None: - for j in members[np.argsort(center[members], kind="stable")]: - if groups and center[j] - np.mean(center[groups[-1]]) < line_tol: - groups[-1].append(int(j)) - else: - groups.append([int(j)]) - else: - reach = -np.inf - for j in members[np.argsort(lo[members], kind="stable")]: - if groups and lo[j] <= reach: - groups[-1].append(int(j)) - else: - groups.append([int(j)]) - reach = -np.inf - reach = max(reach, float(hi[j])) - new_ids = range(-len(groups), 0) if before else range(size, size + len(groups)) - for g_id, group in zip(new_ids, groups): - idx[group] = g_id - bands[g_id] = (float(lo[group].min()), float(hi[group].max())) - return idx, bands - - @classmethod - def _complete_grid( - cls, - cells: list[dict[str, Any]], - polys: np.ndarray, - cell_words: list[list[int]], - leftover: np.ndarray, - word_polys: np.ndarray, - angle: float, - ) -> tuple[list[dict[str, Any]], np.ndarray, list[list[int]]]: - """Place the words of a table that lie in no cell into its grid, so that no table text is left out - - The row and column lines of the grid are located from the cell edges. A word between two lines of each axis - joins the cell covering that grid slot, or a new cell when no cell covers it (e.g. a missed cell). The words - beyond the grid (e.g. a missed header row or a row of totals) make new rows / columns: in a new row, the - words closer than half a text line, like the words of a sentence, share a single (possibly spanning) cell. + def _deskew_centers(centers: np.ndarray, polys: np.ndarray) -> np.ndarray: + """Rotate word centers by the angle of a table, to order the words along its own axes Args: - cells: the cells of the table, with local logical coordinates - polys: the (C, 4, 2) cell polygons (absolute coordinates) - cell_words: the word indices of each cell - leftover: the indices of the words of the table that lie in no cell - word_polys: the (N, 4, 2) word polygons (absolute coordinates) - angle: the angle of the table rows (radians) + centers: the (N, 2) word centers + polys: the (C, 4, 2) cell polygons, vertices ordered TL, TR, BR, BL Returns: - the cells, their polygons and the word indices of each cell, new cells included (the logical coordinates - of the new rows / columns before the grid are negative) + the (N, 2) de-skewed centers """ - pivot = polys.reshape(-1, 2).mean(axis=0) - # Table frame: the rows and columns are horizontal and vertical - cell_pts = cls._rotate_points(polys, -angle, pivot) - word_pts = cls._rotate_points(word_polys[leftover], -angle, pivot) - w_x0, w_x1 = word_pts[..., 0].min(axis=1), word_pts[..., 0].max(axis=1) - w_y0, w_y1 = word_pts[..., 1].min(axis=1), word_pts[..., 1].max(axis=1) - logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells], dtype=int) - num_rows, num_cols = int(logic[:, 1].max()) + 1, int(logic[:, 3].max()) + 1 - row_lines = cls._grid_lines( - logic[:, 0], logic[:, 1], cell_pts[..., 1].min(axis=1), cell_pts[..., 1].max(axis=1), num_rows - ) - col_lines = cls._grid_lines( - logic[:, 2], logic[:, 3], cell_pts[..., 0].min(axis=1), cell_pts[..., 0].max(axis=1), num_cols + top_edges = polys[:, 1] - polys[:, 0] # TR - TL + angle = float(np.median(np.arctan2(top_edges[:, 1], top_edges[:, 0]))) + cos_a, sin_a = np.cos(-angle), np.sin(-angle) + pivot = centers.mean(axis=0) + shifted = centers - pivot + return np.stack( + [ + pivot[0] + shifted[:, 0] * cos_a - shifted[:, 1] * sin_a, + pivot[1] + shifted[:, 0] * sin_a + shifted[:, 1] * cos_a, + ], + axis=1, ) - med_h = float(np.median(w_y1 - w_y0)) - med_h = med_h if np.isfinite(med_h) and med_h > 0 else 1.0 - col_idx, col_bands = cls._axis_slots((w_x0 + w_x1) / 2, w_x0, w_x1, col_lines) - row_idx, row_bands = cls._axis_slots((w_y0 + w_y1) / 2, w_y0, w_y1, row_lines, line_tol=med_h / 2) - - # The cell covering each slot of the grid - occupied: dict[tuple[int, int], int] = {} - for c_idx, (rs, re, cs, ce) in enumerate(logic): - for r in range(rs, re + 1): - for c in range(cs, ce + 1): - occupied.setdefault((r, c), c_idx) - - # New cells: (row_start, row_end, col_start, col_end) -> positions in `leftover` - new_cells: dict[tuple[int, int, int, int], list[int]] = {} - in_grid = (row_idx >= 0) & (row_idx < num_rows) - for j in np.flatnonzero(in_grid): - slot = (int(row_idx[j]), int(col_idx[j])) - if slot in occupied: - cell_words[occupied[slot]].append(int(leftover[j])) - else: - new_cells.setdefault((slot[0], slot[0], slot[1], slot[1]), []).append(int(j)) - # New rows: the words of a same column, or closer than half a text line, share a cell - for r in np.unique(row_idx[~in_grid]): - members = np.flatnonzero(row_idx == r) - members = members[np.argsort(w_x0[members], kind="stable")] - group = [int(members[0])] - for prev, j in zip(members[:-1], members[1:]): - if col_idx[j] == col_idx[group[-1]] or w_x0[j] - w_x1[prev] < med_h / 2: - group.append(int(j)) - continue - new_cells[(int(r), int(r), int(col_idx[group].min()), int(col_idx[group].max()))] = group - group = [int(j)] - new_cells[(int(r), int(r), int(col_idx[group].min()), int(col_idx[group].max()))] = group - - cells, new_polys = list(cells), [] - for (rs, re, cs, ce), positions in new_cells.items(): - x0 = min(col_bands[cs][0], float(w_x0[positions].min())) - x1 = max(col_bands[ce][1], float(w_x1[positions].max())) - y0 = min(row_bands[rs][0], float(w_y0[positions].min())) - y1 = max(row_bands[re][1], float(w_y1[positions].max())) - rect = np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float64) - new_polys.append(cls._rotate_points(rect, angle, pivot)) - # A cell added from the words: the table structure model did not score it - cells.append({"score": None, "row_start": rs, "row_end": re, "col_start": cs, "col_end": ce}) - cell_words.append([int(leftover[j]) for j in positions]) - if new_polys: - polys = np.concatenate([polys, np.stack(new_polys).astype(polys.dtype)]) - return cells, polys, cell_words - - @classmethod - def _simplify_grid( - cls, - cells: list[dict[str, Any]], - polys: np.ndarray, - cell_words: list[list[int]], - word_polys: np.ndarray, - angle: float, - ) -> tuple[list[dict[str, Any]], np.ndarray, list[list[int]]]: - """Merge the rows / columns of a table split by the structure model, and remove the ones without any text - - Two adjacent rows (resp. columns) are merged when their text is complementary (no column, resp. row, has text - in both) and lies on the same text line (resp. column), i.e. their text extents mostly overlap: the table - structure model sometimes splits a row whose cell edges do not line up. A row (resp. column) without text of its - own (no cell holds text in this row only) is removed: the grid only keeps the rows and columns of the table's - content, and the cells spanning the removed ones span one less. - - Args: - cells: the cells of the table, with local logical coordinates - polys: the (C, 4, 2) cell polygons (absolute coordinates) - cell_words: the word indices of each cell - word_polys: the (N, 4, 2) word polygons (absolute coordinates) - angle: the angle of the table rows (radians) - - Returns: - the remaining cells, their polygons and the word indices of each cell - """ - filled = np.array([len(w) > 0 for w in cell_words]) - if not filled.any(): - return cells, polys, cell_words - pivot = polys.reshape(-1, 2).mean(axis=0) - frame = cls._rotate_points(polys, -angle, pivot) - rects = np.concatenate([frame.min(axis=1), frame.max(axis=1)], axis=1) # (C, 4) x0, y0, x1, y1 - resized = np.zeros(len(cells), dtype=bool) - # Text extent of each cell in the table frame: x0, y0, x1, y1 - content = np.full((len(cells), 4), np.nan) - for c_idx in np.flatnonzero(filled): - pts = cls._rotate_points(word_polys[cell_words[c_idx]], -angle, pivot).reshape(-1, 2) - content[c_idx] = [*pts.min(axis=0), *pts.max(axis=0)] - # row_start, row_end, col_start, col_end - logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells], dtype=int) - alive = np.ones(len(cells), dtype=bool) - - def mergeable(axis: int, k: int) -> bool: - start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] - o_start, o_end = logic[:, 2 - 2 * axis], logic[:, 3 - 2 * axis] - in_k, in_next = alive & (start <= k) & (end >= k), alive & (start <= k + 1) & (end >= k + 1) - only_k, only_next = np.flatnonzero(in_k & ~in_next & filled), np.flatnonzero(in_next & ~in_k & filled) - if only_k.size == 0 or only_next.size == 0: - return False - # Complementary: no index of the other axis holds text in both - if np.any( - (o_start[only_k, None] <= o_end[None, only_next]) & (o_start[None, only_next] <= o_end[only_k, None]) - ): - return False - # On the same line: the text extents along the axis mostly overlap - lo, hi = (1, 3) if axis == 0 else (0, 2) - a_lo, a_hi = content[only_k, lo].min(), content[only_k, hi].max() - b_lo, b_hi = content[only_next, lo].min(), content[only_next, hi].max() - overlap = min(a_hi, b_hi) - max(a_lo, b_lo) - return bool(overlap >= 0.5 * min(a_hi - a_lo, b_hi - b_lo)) - - def shift(axis: int, k: int, merge: bool) -> None: - # Merge the index k + 1 into the index k, or remove the index k (its cells are removed beforehand) - start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] - end[alive & ((end > k) if merge else (end >= k))] -= 1 - start[alive & (start > k)] -= 1 - - def resolve_overlaps() -> None: - # The cells merged onto a same slot: the empty ones are absorbed by the cells holding text - idx = np.flatnonzero(alive) - for pos, i in enumerate(idx): - if not alive[i]: - continue - others = idx[pos + 1 :] - others = others[alive[others]] - hits = others[ - (logic[others, 0] <= logic[i, 1]) - & (logic[i, 0] <= logic[others, 1]) - & (logic[others, 2] <= logic[i, 3]) - & (logic[i, 2] <= logic[others, 3]) - ] - for j in hits: - keep, drop = (j, i) if filled[j] and not filled[i] else (i, j) - if filled[drop]: # both hold text: not merged - continue - rects[keep, :2] = np.minimum(rects[keep, :2], rects[drop, :2]) - rects[keep, 2:] = np.maximum(rects[keep, 2:], rects[drop, 2:]) - logic[keep, ::2] = np.minimum(logic[keep, ::2], logic[drop, ::2]) - logic[keep, 1::2] = np.maximum(logic[keep, 1::2], logic[drop, 1::2]) - resized[keep] = True - alive[drop] = False - if drop == i: - break - - changed = True - while changed: - changed = False - for axis in (0, 1): - size = int(logic[alive, 2 * axis + 1].max()) + 1 - for k in range(size - 1): - if mergeable(axis, k): - shift(axis, k, merge=True) - resolve_overlaps() - changed = True - break - # Remove the rows / columns without text of their own: no text, or only the text of cells spanning other rows - # / columns too (which then span one less) - for axis in (0, 1): - start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] - for k in range(int(end[alive].max()), -1, -1): - if np.any(alive & filled & (start == k) & (end == k)): - continue - alive[alive & (start == k) & (end == k)] = False - shift(axis, k, merge=False) - - keep = np.flatnonzero(alive) - new_polys = polys.copy() - for c_idx in np.flatnonzero(resized & alive): - x0, y0, x1, y1 = rects[c_idx] - rect = np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float64) - new_polys[c_idx] = cls._rotate_points(rect, angle, pivot) - new_cells = [] - for c_idx in keep: - rs, re, cs, ce = (int(v) for v in logic[c_idx]) - new_cells.append({**cells[c_idx], "row_start": rs, "row_end": re, "col_start": cs, "col_end": ce}) - return new_cells, new_polys[keep], [cell_words[c_idx] for c_idx in keep] - - @staticmethod - def _text_lines(centers: np.ndarray, tol: float) -> np.ndarray: - """Group 1D word centers (along the rows' normal) into text lines: the line index of each word, from the top""" - order = np.argsort(centers, kind="stable") - line_ids = np.zeros(centers.size, dtype=np.int64) - line, members = 0, [int(order[0])] if centers.size else [] - for j in order[1:]: - if centers[j] - np.mean(centers[members]) >= tol: - line, members = line + 1, [] - members.append(int(j)) - line_ids[j] = line - return line_ids - - @classmethod - def _split_rows( - cls, - cells: list[dict[str, Any]], - polys: np.ndarray, - cell_words: list[list[int]], - word_polys: np.ndarray, - angle: float, - ) -> tuple[list[dict[str, Any]], np.ndarray, list[list[int]]]: - """Split the rows merged by the table structure model, from the text lines of their cells - - The model may merge several rows of a dense table (e.g. without rulings) into a single one: a row is split - into its text lines when most of its cells holding text (at least 2) hold several of them, so that a single - cell wrapping its text (e.g. a long header) does not split its row. Likewise, a cell spanning several rows - is split when each of its text lines lies in a different row, aligned with the text of that row. - - Args: - cells: the cells of the table, with local logical coordinates - polys: the (C, 4, 2) cell polygons (absolute coordinates) - cell_words: the word indices of each cell - word_polys: the (N, 4, 2) word polygons (absolute coordinates) - angle: the angle of the table rows (radians) - - Returns: - the cells, their polygons and the word indices of each cell - """ - if not any(cell_words): - return cells, polys, cell_words - pivot = polys.reshape(-1, 2).mean(axis=0) - frame = cls._rotate_points(polys, -angle, pivot) - rects = [np.concatenate([f.min(axis=0), f.max(axis=0)]) for f in frame] # x0, y0, x1, y1 - all_words = np.unique(np.concatenate([w for w in cell_words if w])).astype(int) - w_pts = cls._rotate_points(word_polys[all_words], -angle, pivot) - w_y0 = dict(zip(all_words.tolist(), w_pts[..., 1].min(axis=1))) - w_y1 = dict(zip(all_words.tolist(), w_pts[..., 1].max(axis=1))) - heights = w_pts[..., 1].max(axis=1) - w_pts[..., 1].min(axis=1) - tol = 0.5 * float(np.median(heights)) if heights.size else 0.0 - if not tol > 0: - return cells, polys, cell_words - cells = [dict(c) for c in cells] - cell_words = [list(w) for w in cell_words] - resized: set[int] = set() - - def center(w: int) -> float: - return (w_y0[w] + w_y1[w]) / 2 - - def split_cell(c_idx: int, line_ids: np.ndarray, rows: list[tuple[int, int]]) -> None: - """Split a cell into one cell per text line, spanning the given rows""" - words = np.asarray(cell_words[c_idx]) - x0, y0, x1, y1 = rects[c_idx] - n_lines = int(line_ids.max()) + 1 - # The sub-cells share the cell's extent, split midway between its text lines - tops = [min(w_y0[w] for w in words[line_ids == line]) for line in range(n_lines)] - bottoms = [max(w_y1[w] for w in words[line_ids == line]) for line in range(n_lines)] - bounds = [y0] + [(bottoms[line] + tops[line + 1]) / 2 for line in range(n_lines - 1)] + [y1] - for line, (rs, re) in enumerate(rows): - new = {**cells[c_idx], "row_start": rs, "row_end": re} - rect = np.array([x0, bounds[line], x1, bounds[line + 1]]) - if line == 0: - cells[c_idx], rects[c_idx], cell_words[c_idx] = new, rect, words[line_ids == 0].tolist() - resized.add(c_idx) - else: - cells.append(new) - rects.append(rect) - cell_words.append(words[line_ids == line].tolist()) - resized.add(len(cells) - 1) - - # 1. Rows merged by the model, from the bottom up so that the rows still to process do not move - num_rows = max(c["row_end"] for c in cells) + 1 - for r in range(num_rows - 1, -1, -1): - in_row = [i for i, c in enumerate(cells) if c["row_start"] == r and c["row_end"] == r] - filled = [i for i in in_row if cell_words[i]] - if len(filled) < 2: - continue - words = [w for i in filled for w in cell_words[i]] - line_of = dict(zip(words, cls._text_lines(np.array([center(w) for w in words]), tol).tolist())) - n_lines = max(line_of.values()) + 1 - multi = [i for i in filled if len({line_of[w] for w in cell_words[i]}) > 1] - if n_lines < 2 or len(multi) < 2 or 2 * len(multi) < len(filled): - continue - # The rows below move down, the cells spanning the row span its new rows too - for c in cells: - if c["row_start"] == r and c["row_end"] == r: - continue - if c["row_start"] > r: - c["row_start"] += n_lines - 1 - if c["row_end"] >= r: - c["row_end"] += n_lines - 1 - for i in in_row: - if not cell_words[i]: - cells[i]["row_end"] = r + n_lines - 1 - continue - cell_lines = np.array([line_of[w] for w in cell_words[i]]) - present = np.unique(cell_lines) - # Local line indices of the cell -> rows of the table - split_cell(i, np.searchsorted(present, cell_lines), [(r + line, r + line) for line in present]) - - # 2. Cells spanning several rows whose text lines lie in different rows - logic = np.array([[c["row_start"], c["row_end"]] for c in cells], dtype=int) - if logic[:, 1].max() > 0: - row_y0 = np.array([r[1] for r in rects]) - row_y1 = np.array([r[3] for r in rects]) - lines = cls._grid_lines(logic[:, 0], logic[:, 1], row_y0, row_y1, int(logic[:, 1].max()) + 1) - for i in range(len(cells)): - rs, re = cells[i]["row_start"], cells[i]["row_end"] - if re == rs or len(cell_words[i]) < 2: - continue - cell_lines = cls._text_lines(np.array([center(w) for w in cell_words[i]]), tol) - n_lines = int(cell_lines.max()) + 1 - if n_lines < 2: - continue - line_rows = [] - for line in range(n_lines): - members = [w for w, lid in zip(cell_words[i], cell_lines) if lid == line] - lo, hi = min(w_y0[w] for w in members), max(w_y1[w] for w in members) - row = int(np.clip(np.searchsorted(lines, (lo + hi) / 2, side="right") - 1, rs, re)) - # The text of the other cells of the row lies on the same line - aligned = any( - cells[j]["row_start"] == row == cells[j]["row_end"] - and any(min(hi, w_y1[w]) - max(lo, w_y0[w]) > 0.5 * (hi - lo) for w in cell_words[j]) - for j in range(len(cells)) - if j != i and cell_words[j] - ) - if not aligned: - break - line_rows.append(row) - if len(line_rows) < n_lines or len(set(line_rows)) < n_lines: - continue - spans = [ - (rs if line == 0 else line_rows[line], re if line == n_lines - 1 else line_rows[line + 1] - 1) - for line in range(n_lines) - ] - split_cell(i, cell_lines, spans) - - if not resized: - return cells, polys, cell_words - new_polys = [ - cls._rotate_points( - np.array([[r[0], r[1]], [r[2], r[1]], [r[2], r[3]], [r[0], r[3]]], dtype=np.float64), angle, pivot - ) - if idx in resized - else polys[idx] - for idx, r in enumerate(rects) - ] - return cells, np.stack(new_polys).astype(polys.dtype), cell_words def _build_tables( self, boxes: np.ndarray, word_preds: list[tuple[str, float]], page_table: dict[str, Any] | list[dict[str, Any]] | None, - page_shape: tuple[int, int] | None = None, ) -> tuple[list[Table], np.ndarray]: """Assign detected words to table cells and build the page tables. - Every word whose center lies in a table (its layout region when provided, the cells otherwise) is part of - the table: the words outside the predicted cells complete the grid (cf. `_complete_grid`). - Args: boxes: word boxes of the page, of shape (N, 4) or (N, 4, 2), in relative coordinates word_preds: list of (text, confidence) for each of the N words page_table: the table structure prediction(s) for the page. Either a single grid `{"cells": [{"geometry", "score", "row_start", "row_end", "col_start", "col_end"}], "num_rows", - "num_cols", "geometry"}` (cell geometries in page-relative coordinates, the optional "geometry" being - the (4, 2) polygon of the table region), a list of such grids, or None - page_shape: the (height, width) of the page, to process the geometry in an isotropic space + "num_cols"}` (cell geometries in page-relative coordinates), a list of such grids, or None. When the + cells list the indices of their words (`"words"`, in reading order, cf. the OCR predictor), they are + used as is, otherwise each word goes to the cell containing its center Returns: a tuple with the list of `Table` objects of the page (one per provided table) and a boolean mask of @@ -1091,43 +663,33 @@ def _build_tables( if len(grids) == 0: return [], consumed # The geometries follow the page's word geometry: straight 2-point boxes for (N, 4) word boxes, 4-point polygons - # for (N, 4, 2) ones. + # for (N, 4, 2) ones. Word heights are rotation-invariant (left edge length for polygons). straight = boxes.ndim != 3 - scale = np.ones(2) if page_shape is None else np.array([page_shape[1], page_shape[0]], dtype=np.float64) - if boxes.ndim == 3: # (N, 4, 2) polygons, ordered TL, TR, BR, BL - word_polys = boxes.astype(np.float64) * scale - else: # (N, 4) straight boxes (xmin, ymin, xmax, ymax) - x0, y0, x1, y1 = boxes.reshape(-1, 4).T.astype(np.float64) - word_polys = np.stack([np.stack(pt, axis=-1) for pt in ((x0, y0), (x1, y0), (x1, y1), (x0, y1))], 1) * scale - centers = word_polys.mean(axis=1) - heights = np.linalg.norm(word_polys[:, 3] - word_polys[:, 0], axis=1) # rotation-invariant (left edge) + centers = self._word_centers(boxes) if boxes.shape[0] > 0 else np.empty((0, 2)) + if boxes.shape[0] == 0: + heights = np.empty(0) + else: + heights = boxes[:, 3] - boxes[:, 1] if straight else np.linalg.norm(boxes[:, 3] - boxes[:, 0], axis=1) tables: list[Table] = [] - for cells, num_rows, num_cols, region in grids: - polys = np.stack([self._as_cell_polygon(cell["geometry"]) for cell in cells]) * scale # (C, 4, 2) - cell_words = self._assign_cell_words(centers, polys, consumed) - top_edges = polys[:, 1] - polys[:, 0] # TR - TL - angle = 0.0 if straight else float(np.median(np.arctan2(top_edges[:, 1], top_edges[:, 0]))) - # The words of the table that lie in no cell: in its layout region, or in the cells' outline - free = np.flatnonzero(~consumed) - outline = cv2.boxPoints(cv2.minAreaRect(polys.reshape(-1, 2).astype(np.float32))).reshape(1, 4, 2) - in_table = self._points_in_polygons(centers[free], outline)[:, 0] - if region is not None: - in_table |= self._points_in_polygons(centers[free], (region * scale)[None])[:, 0] - leftover = free[in_table] - if leftover.size > 0: - cells, polys, cell_words = self._complete_grid(cells, polys, cell_words, leftover, word_polys, angle) - consumed[leftover] = True - cells, num_rows, num_cols = self._localize_logic(cells) - cells, polys, cell_words = self._simplify_grid(cells, polys, cell_words, word_polys, angle) - cells, polys, cell_words = self._split_rows(cells, polys, cell_words, word_polys, angle) - cells, num_rows, num_cols = self._localize_logic(cells) - # Reading order inside the cells: rows top to bottom, words left to right (table axes) - order_centers = self._rotate_points(centers, -angle, centers.mean(axis=0)) if centers.size else centers + for cells, num_rows, num_cols in grids: + polys = np.stack([self._as_cell_polygon(cell["geometry"]) for cell in cells]) # (C, 4, 2) + if all("words" in cell for cell in cells): + # The OCR predictor assigned the words of the table to its cells, in reading order + cell_words = [] + for cell in cells: + cell_words.append([int(i) for i in cell["words"] if not consumed[i]]) + consumed[cell_words[-1]] = True + else: + # Reading order inside the cell: rows top to bottom, words left to right (table axes) + order_centers = centers if straight or centers.shape[0] == 0 else self._deskew_centers(centers, polys) + cell_words = [ + self._order_cell_words(w_idcs, order_centers, heights) if w_idcs else [] + for w_idcs in self._assign_cell_words(centers, polys, consumed) + ] table_cells = [] - for cell, poly, w_idcs in zip(cells, polys / scale, cell_words): - if len(w_idcs) > 0: - ordered = self._order_cell_words(w_idcs, order_centers, heights) + for cell, poly, ordered in zip(cells, polys, cell_words): + if len(ordered) > 0: value = " ".join(word_preds[i][0] for i in ordered) confidence = float(np.mean([word_preds[i][1] for i in ordered])) else: @@ -1145,9 +707,9 @@ def _build_tables( ) ) # Enclosing geometry of the whole table - pts = (polys / scale).reshape(-1, 2) - (xmin, ymin), (xmax, ymax) = pts.min(axis=0), pts.max(axis=0) + (xmin, ymin), (xmax, ymax) = polys.reshape(-1, 2).min(axis=0), polys.reshape(-1, 2).max(axis=0) corners = np.asarray([[xmin, ymin], [xmax, ymin], [xmax, ymax], [xmin, ymax]]) + # The cells made from the words of the table are not scored by the table model scores = [float(cell["score"]) for cell in cells if cell["score"] is not None] tables.append( Table( @@ -1363,7 +925,7 @@ def __call__( _tables, ): # Build the page tables and flag the words that belong to a table - page_tables, consumed = self._build_tables(page_boxes, word_preds, page_table, shape) + page_tables, consumed = self._build_tables(page_boxes, word_preds, page_table) if consumed.any(): # Remove the words assigned to a table from the regular blocks output keep = ~consumed diff --git a/doctr/models/predictor/pytorch.py b/doctr/models/predictor/pytorch.py index b91efc0026..f65deedbe7 100644 --- a/doctr/models/predictor/pytorch.py +++ b/doctr/models/predictor/pytorch.py @@ -16,6 +16,7 @@ from doctr.models.detection.predictor import DetectionPredictor from doctr.models.layout.predictor import LayoutPredictor from doctr.models.recognition.predictor import RecognitionPredictor +from doctr.models.table_structure.grid import fill_table_grid from doctr.models.table_structure.predictor import TablePredictor from doctr.models.table_structure.tablecenternet.base import _logic_from_geometry from doctr.utils.geometry import detach_scores @@ -212,7 +213,7 @@ def _predict_pages( # Recognize table structure: locate tables with the layout model, crop each one and run the table model tables = ( - self._tables_from_regions(pages, regions, **kwargs) + self._tables_from_regions(pages, regions, boxes, **kwargs) if self.table_predictor is not None and regions is not None else None ) @@ -247,29 +248,32 @@ def _tables_from_regions( self, pages: list[np.ndarray], regions: list[dict[str, Any]], + word_boxes: list[np.ndarray] | None = None, **kwargs: Any, ) -> list[list[dict[str, Any]]]: """Crop the table regions found by the layout model and run the table model on each crop. Each `Table` region is rectified to an upright crop with a perspective transform (so straight boxes and - rotated polygons are both handled), the table model is applied per crop, and the predicted cell polygons - are mapped back to page-relative coordinates through the inverse transform. A page therefore yields one - structured table per detected `Table` region. The crop includes a margin of page context around the region: - the cells predicted in this margin belong to the neighbouring content and are dropped. + rotated polygons are both handled), and the table model is applied per crop. The crop includes a margin of page + context around the region: the cells predicted in this margin belong to the neighbouring content and are + dropped. The words of the page lying in the region are mapped into the crop, where they are assigned to the + cells and complete the grid (cf. :func:`~doctr.models.table_structure.grid.fill_table_grid`). The cells are + then mapped back to page-relative coordinates through the inverse transform: a page yields one structured + table per detected `Table` region. Args: pages: the (possibly straightened) page images regions: the per-page layout predictions `{"class_names", "boxes", "scores"}` + word_boxes: the per-page word boxes, of shape (N, 4) or (N, 4, 2) in relative coordinates **kwargs: keyword arguments forwarded to the table predictor Returns: - a per-page list of table grids `{"cells": [...], "num_rows": int, "num_cols": int, "geometry": region}` - in page-relative coordinates, `geometry` being the (4, 2) polygon of the table region + a per-page list of table grids `{"cells": [...], "num_rows": int, "num_cols": int}` in page-relative + coordinates, each cell listing the indices of its words in the page (`"words"`) in reading order """ crops: list[np.ndarray] = [] - # (page index, inverse transform, crop width, crop height, context margin, page width, page height, - # page-relative region) - crop_meta: list[tuple[int, np.ndarray, int, int, int, int, int, np.ndarray]] = [] + # (page index, transform, inverse transform, crop width, crop height, context margin) + crop_meta: list[tuple[int, np.ndarray, np.ndarray, int, int, int]] = [] for p_idx, (page, region) in enumerate(zip(pages, regions)): if region is None: continue @@ -277,22 +281,13 @@ def _tables_from_regions( for cls_name, box in zip(region["class_names"], region["boxes"]): if cls_name != self.table_class_name: continue - pts = np.asarray(box, dtype=np.float32).reshape(-1, 2) - if pts.shape[0] == 2: # straight box (x_min, y_min, x_max, y_max) -> corners - (bx0, by0), (bx1, by1) = pts - src = np.array([[bx0, by0], [bx1, by0], [bx1, by1], [bx0, by1]], dtype=np.float32) - else: # rotated 4-point polygon, already ordered (top-left, top-right, bottom-right, bottom-left) - src = pts.copy() - # Relative -> absolute pixel corners - src[:, 0] *= w - src[:, 1] *= h + src = self._cell_polygon(box) * np.array([w, h], dtype=np.float32) # absolute pixel corners # Upright crop size from the region side lengths crop_w = int(round(max(np.linalg.norm(src[1] - src[0]), np.linalg.norm(src[2] - src[3])))) crop_h = int(round(max(np.linalg.norm(src[3] - src[0]), np.linalg.norm(src[2] - src[1])))) if crop_w < 2 or crop_h < 2: continue - # Map the region onto an inset rectangle so the warp samples `pad` extra pixels of page - # context on each side. Crop-relative [0, 1] still maps exactly to [0, out_w] / [0, out_h]. + # Map the region onto an inset rectangle so the warp samples `pad` extra pixels of page context pad = max(20, round(0.01 * max(crop_w, crop_h))) out_w, out_h = crop_w + 2 * pad, crop_h + 2 * pad dst = np.array( @@ -302,7 +297,7 @@ def _tables_from_regions( transform = cv2.getPerspectiveTransform(src, dst) inverse = cv2.getPerspectiveTransform(dst, src) crops.append(cv2.warpPerspective(page, transform, (out_w, out_h), borderMode=cv2.BORDER_REPLICATE)) - crop_meta.append((p_idx, inverse, out_w, out_h, pad, w, h, src / np.array([w, h], dtype=np.float32))) + crop_meta.append((p_idx, transform, inverse, out_w, out_h, pad)) tables_per_page: list[list[dict[str, Any]]] = [[] for _ in pages] if len(crops) == 0: @@ -312,21 +307,16 @@ def _tables_from_regions( # Whether the structure model makes the logical coordinates consistent with the cell geometry postprocessor = getattr(getattr(self.table_predictor, "model", None), "postprocessor", None) realign = getattr(postprocessor, "geometric_logic", False) - for (p_idx, inverse, crop_w, crop_h, pad, w, h, region_poly), grid in zip(crop_meta, grids): + for (p_idx, transform, inverse, crop_w, crop_h, pad), grid in zip(crop_meta, grids): if len(grid["cells"]) == 0: continue + h, w = pages[p_idx].shape[:2] # Cell polygons: crop-relative -> crop pixels polys = np.stack([self._cell_polygon(cell["geometry"]) for cell in grid["cells"]]) polys *= np.array([crop_w, crop_h], dtype=np.float32) - # The context margin around the region shows the neighbouring content (e.g. the cells of an adjacent - # table): the cells centered there are not part of this table + # The cells centered in the context margin are not part of the table centers = polys.mean(axis=1) - keep = np.flatnonzero( - (centers[:, 0] >= pad) - & (centers[:, 0] <= crop_w - pad) - & (centers[:, 1] >= pad) - & (centers[:, 1] <= crop_h - pad) - ) + keep = np.flatnonzero(np.all((centers >= pad) & (centers <= [crop_w - pad, crop_h - pad]), axis=1)) if keep.size == 0: continue cells = [grid["cells"][idx] for idx in keep] @@ -334,28 +324,42 @@ def _tables_from_regions( if realign and keep.size < polys.shape[0]: # The dropped cells made grid lines of their own: index the rows / columns of the remaining ones again logic = _logic_from_geometry(polys[keep], logic, split_tables=False) + # The words of the region, in the crop + word_idcs, words = np.zeros(0, dtype=np.int64), np.zeros((0, 4)) + if word_boxes is not None and len(word_boxes[p_idx]) > 0: + page_words = np.asarray(word_boxes[p_idx], dtype=np.float32) + if page_words.ndim == 2: # (N, 4) straight boxes -> (N, 4, 2) polygons + page_words = page_words[:, [[0, 1], [2, 1], [2, 3], [0, 3]]] + pts = cv2.perspectiveTransform((page_words * [w, h]).reshape(1, -1, 2), transform).reshape(-1, 4, 2) + word_centers = pts.mean(axis=1) + inside = (word_centers >= pad) & (word_centers <= [crop_w - pad, crop_h - pad]) + word_idcs = np.flatnonzero(np.all(inside, axis=1)) + words = np.concatenate([pts[word_idcs].min(axis=1), pts[word_idcs].max(axis=1)], axis=1) + boxes, cell_logic, cell_words, origin = fill_table_grid( + np.concatenate([polys[keep].min(axis=1), polys[keep].max(axis=1)], axis=1), + logic[:, [2, 3, 0, 1]], # (row_start, row_end, col_start, col_end) + words, + ) remapped_cells: list[dict[str, Any]] = [] - for cell, poly, (col_start, col_end, row_start, row_end) in zip(cells, polys[keep], logic): + for (x0, y0, x1, y1), (row_start, row_end, col_start, col_end), w_idcs, source in zip( + boxes, cell_logic, cell_words, origin + ): # crop pixels -> page pixels (inverse transform) -> page-relative - poly = cv2.perspectiveTransform(poly[None, :, :], inverse)[0] - poly[:, 0] /= w - poly[:, 1] /= h - new_cell = dict(cell) - new_cell.update( - geometry=poly.tolist(), - row_start=int(row_start), - row_end=int(row_end), - col_start=int(col_start), - col_end=int(col_end), - ) - remapped_cells.append(new_cell) - if not remapped_cells: - continue + corners = np.array([[[x0, y0], [x1, y0], [x1, y1], [x0, y1]]], dtype=np.float32) + poly = cv2.perspectiveTransform(corners, inverse)[0] / np.array([w, h], dtype=np.float32) + remapped_cells.append({ + "geometry": poly.tolist(), + # The cells made from the words are not scored by the table model + "score": None if source < 0 else float(cells[source]["score"]), + "row_start": int(row_start), + "row_end": int(row_end), + "col_start": int(col_start), + "col_end": int(col_end), + "words": [int(word_idcs[idx]) for idx in w_idcs], + }) tables_per_page[p_idx].append({ "cells": remapped_cells, - "num_rows": grid["num_rows"], - "num_cols": grid["num_cols"], - # The layout region of the table (TL, TR, BR, BL): every word inside it belongs to the table - "geometry": region_poly.tolist(), + "num_rows": int(cell_logic[:, 1].max()) + 1, + "num_cols": int(cell_logic[:, 3].max()) + 1, }) return tables_per_page diff --git a/doctr/models/table_structure/grid.py b/doctr/models/table_structure/grid.py new file mode 100644 index 0000000000..e78971bfea --- /dev/null +++ b/doctr/models/table_structure/grid.py @@ -0,0 +1,348 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +"""Assemble the cells predicted by a table structure model and the words of the table into a consistent grid. + +Everything here works in the frame of the (rectified, upright) table image, where the cells and the words are +axis-aligned boxes `(x0, y0, x1, y1)`, and the logical coordinates of the cells are `(row_start, row_end, col_start, +col_end)` (0-indexed, inclusive). +""" + +import numpy as np + +__all__ = ["fill_table_grid"] + + +def _grid_lines(start: np.ndarray, end: np.ndarray, lo: np.ndarray, hi: np.ndarray, size: int) -> np.ndarray: + """Positions of the `size + 1` lines of an axis: line `k`, where index `k` starts and index `k - 1` ends, lies at + the median of the cell edges on it (interpolated when no edge lies on it)""" + edges: list[list[float]] = [[] for _ in range(size + 1)] + for s, e, low, high in zip(start, end, lo, hi): + edges[int(s)].append(float(low)) + edges[int(e) + 1].append(float(high)) + known = [k for k, values in enumerate(edges) if values] + return np.maximum.accumulate(np.interp(np.arange(size + 1), known, [float(np.median(edges[k])) for k in known])) + + +def _group(values: np.ndarray, tol: float) -> np.ndarray: + """Group 1D positions closer than `tol` to the mean of their group: the group index of each value, in order""" + groups = np.zeros(values.size, dtype=np.int64) + group, members = 0, list[int]() + for j in np.argsort(values, kind="stable"): + if members and values[j] - np.mean(values[members]) >= tol: + group, members = group + 1, [] + groups[j] = group + members.append(j) + return groups + + +def _axis_slots(lo: np.ndarray, hi: np.ndarray, lines: np.ndarray, line_tol: float | None) -> np.ndarray: + """Index of each word along an axis of the grid: the index between the 2 lines around its center, or a new index + before / after the grid for the words beyond it, grouped by text line (rows, `line_tol`) or by overlapping + extent (columns, `line_tol` is None)""" + size, center = lines.size - 1, (lo + hi) / 2 + idx = np.clip(np.searchsorted(lines, center, side="right") - 1, 0, size - 1) + for before, mask in ((True, center < lines[0]), (False, center > lines[-1])): + members = np.flatnonzero(mask) + if members.size == 0: + continue + if line_tol is not None: + groups = _group(center[members], line_tol) + else: # chain the overlapping extents + order = members[np.argsort(lo[members], kind="stable")] + breaks = lo[order][1:] > np.maximum.accumulate(hi[order])[:-1] + groups = np.empty(members.size, dtype=np.int64) + groups[np.searchsorted(members, order)] = np.concatenate([[0], np.cumsum(breaks)]) + idx[members] = groups - (groups.max() + 1) if before else size + groups + return idx + + +class _Grid: + """The cells of a table: boxes, logical coordinates, word indices and source cell (-1 for a cell made from the + words)""" + + def __init__(self, boxes: np.ndarray, logic: np.ndarray, words: np.ndarray): + self.boxes = [np.asarray(b, dtype=np.float64) for b in boxes] + self.logic = [list(map(int, lc)) for lc in logic] + self.cell_words: list[list[int]] = [[] for _ in self.boxes] + self.origin = list(range(len(self.boxes))) + self.words = words + + def add(self, box: np.ndarray, logic: list[int], words: list[int], origin: int = -1) -> None: + self.boxes.append(np.asarray(box, dtype=np.float64)) + self.logic.append(logic) + self.cell_words.append(words) + self.origin.append(origin) + + def keep(self, mask: np.ndarray) -> None: + for name in ("boxes", "logic", "cell_words", "origin"): + setattr(self, name, [v for v, k in zip(getattr(self, name), mask) if k]) + + def localize(self) -> None: + """Shift the logical coordinates so that the first row and column are 0""" + logic = np.asarray(self.logic) + logic -= [logic[:, 0].min()] * 2 + [logic[:, 2].min()] * 2 + self.logic = logic.tolist() + + def text_box(self, c_idx: int) -> np.ndarray: + boxes = self.words[self.cell_words[c_idx]] + return np.concatenate([boxes[:, :2].min(axis=0), boxes[:, 2:].max(axis=0)]) + + def lines(self, axis: int) -> np.ndarray: + """The grid lines of the rows (axis 0) or columns (axis 1)""" + logic, boxes = np.asarray(self.logic), np.asarray(self.boxes) + lo, hi = (1, 3) if axis == 0 else (0, 2) + size = int(logic[:, 2 * axis + 1].max()) + 1 + return _grid_lines(logic[:, 2 * axis], logic[:, 2 * axis + 1], boxes[:, lo], boxes[:, hi], size) + + +def _complete(grid: _Grid, leftover: np.ndarray, line_height: float) -> None: + """Place the words lying in no cell into the grid: in the cell covering their slot, or in a new cell (a missed + cell, or a new row / column beyond the grid). In a new row, the words of a column or closer than half a text line + (e.g. a sentence) share a cell.""" + words = grid.words[leftover] + row_lines, col_lines = grid.lines(0), grid.lines(1) + rows = _axis_slots(words[:, 1], words[:, 3], row_lines, line_height / 2) + cols = _axis_slots(words[:, 0], words[:, 2], col_lines, None) + occupied: dict[tuple[int, int], int] = {} + for c_idx, (rs, re, cs, ce) in enumerate(grid.logic): + for r in range(rs, re + 1): + for c in range(cs, ce + 1): + occupied.setdefault((r, c), c_idx) + + new_cells: dict[tuple[int, int, int, int], list[int]] = {} + in_grid = (rows >= 0) & (rows < row_lines.size - 1) + for j in np.flatnonzero(in_grid): + slot = (int(rows[j]), int(cols[j])) + if slot in occupied: + grid.cell_words[occupied[slot]].append(int(leftover[j])) + else: + new_cells.setdefault((slot[0], slot[0], slot[1], slot[1]), []).append(int(j)) + for r in np.unique(rows[~in_grid]): + members = np.flatnonzero(rows == r) + members = members[np.argsort(words[members, 0], kind="stable")] + group = [int(members[0])] + for prev, j in zip(members[:-1], members[1:]): + if cols[j] != cols[group[-1]] and words[j, 0] - words[prev, 2] >= line_height / 2: + new_cells[(int(r), int(r), int(cols[group].min()), int(cols[group].max()))] = group + group = [] + group.append(int(j)) + new_cells[(int(r), int(r), int(cols[group].min()), int(cols[group].max()))] = group + + def band(lines: np.ndarray, start: int, end: int) -> tuple[float, float]: + # The extent of the grid lines from index start to end, unbounded beyond the grid + size = lines.size - 1 + return (lines[start] if 0 <= start < size else np.inf, lines[end + 1] if 0 <= end < size else -np.inf) + + for (rs, re, cs, ce), positions in new_cells.items(): + text = words[positions] + (x0, x1), (y0, y1) = band(col_lines, cs, ce), band(row_lines, rs, re) + box = [min(x0, text[:, 0].min()), min(y0, text[:, 1].min())] + box += [max(x1, text[:, 2].max()), max(y1, text[:, 3].max())] + grid.add(np.array(box), [rs, re, cs, ce], [int(leftover[j]) for j in positions]) + grid.localize() + + +def _merge_and_prune(grid: _Grid) -> None: + """Merge the adjacent rows (resp. columns) whose text is complementary (no column, resp. row, holds text in both) + and lies on the same text line (resp. column): the model splits a row when the edges of its cells do not line up. + Then remove the rows / columns without text of their own (no cell holds text in this row / column only): the cells + spanning them span one less.""" + logic = np.asarray(grid.logic) + filled = np.array([len(w) > 0 for w in grid.cell_words]) + if not filled.any(): + return + text = np.array([grid.text_box(i) if filled[i] else np.full(4, np.nan) for i in range(len(grid.boxes))]) + boxes = np.asarray(grid.boxes) + alive = np.ones(len(boxes), dtype=bool) + + def mergeable(axis: int, k: int) -> bool: + start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] + o_start, o_end = logic[:, 2 - 2 * axis], logic[:, 3 - 2 * axis] + in_k, in_next = alive & (start <= k) & (end >= k), alive & (start <= k + 1) & (end >= k + 1) + a, b = np.flatnonzero(in_k & ~in_next & filled), np.flatnonzero(in_next & ~in_k & filled) + if a.size == 0 or b.size == 0: + return False + if np.any((o_start[a, None] <= o_end[None, b]) & (o_start[None, b] <= o_end[a, None])): + return False # some column (resp. row) holds text in both + lo, hi = (1, 3) if axis == 0 else (0, 2) + a_lo, a_hi, b_lo, b_hi = text[a, lo].min(), text[a, hi].max(), text[b, lo].min(), text[b, hi].max() + return bool(min(a_hi, b_hi) - max(a_lo, b_lo) >= 0.5 * min(a_hi - a_lo, b_hi - b_lo)) + + def shift(axis: int, k: int, merge: bool) -> None: + # Merge the index k + 1 into k, or remove the index k + start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] + end[alive & ((end > k) if merge else (end >= k))] -= 1 + start[alive & (start > k)] -= 1 + + def absorb_overlaps() -> None: + # The empty cells now sharing a slot with another cell are absorbed by it + for i in np.flatnonzero(alive & filled).tolist() + np.flatnonzero(alive & ~filled).tolist(): + if not alive[i]: + continue + others = np.flatnonzero(alive & ~filled) + others = others[others != i] + hits = others[ + (logic[others, 0] <= logic[i, 1]) + & (logic[i, 0] <= logic[others, 1]) + & (logic[others, 2] <= logic[i, 3]) + & (logic[i, 2] <= logic[others, 3]) + ] + if hits.size: + boxes[i, :2] = np.minimum(boxes[i, :2], boxes[hits, :2].min(axis=0)) + boxes[i, 2:] = np.maximum(boxes[i, 2:], boxes[hits, 2:].max(axis=0)) + logic[i, ::2] = np.minimum(logic[i, ::2], logic[hits, ::2].min(axis=0)) + logic[i, 1::2] = np.maximum(logic[i, 1::2], logic[hits, 1::2].max(axis=0)) + alive[hits] = False + + changed = True + while changed: + changed = False + for axis in (0, 1): + for k in range(int(logic[alive, 2 * axis + 1].max())): + if mergeable(axis, k): + shift(axis, k, merge=True) + absorb_overlaps() + changed = True + break + for axis in (0, 1): + start, end = logic[:, 2 * axis], logic[:, 2 * axis + 1] + for k in range(int(end[alive].max()), -1, -1): + if not np.any(alive & filled & (start == k) & (end == k)): + alive[alive & (start == k) & (end == k)] = False + shift(axis, k, merge=False) + grid.boxes, grid.logic = list(boxes), logic.tolist() + grid.keep(alive) + grid.localize() + + +def _split_rows(grid: _Grid, line_height: float) -> None: + """Split the rows the model merged, from the text lines of their cells: a row whose cells holding text (at least + 2) mostly hold several text lines is split into its text lines (a single cell wrapping its text, e.g. a long + header, does not split its row), and a cell spanning several rows is split when each of its text lines lies in a + different row, aligned with the text of that row.""" + centers = (grid.words[:, 1] + grid.words[:, 3]) / 2 + + def split(c_idx: int, line_of: np.ndarray, spans: list[tuple[int, int]]) -> None: + # One cell per text line, the cell being cut midway between its text lines + words, (x0, y0, x1, y1) = np.asarray(grid.cell_words[c_idx]), grid.boxes[c_idx] + tops = [grid.words[words[line_of == k], 1].min() for k in range(len(spans))] + bottoms = [grid.words[words[line_of == k], 3].max() for k in range(len(spans))] + cuts = [y0, *[(b + t) / 2 for b, t in zip(bottoms[:-1], tops[1:])], y1] + _, _, cs, ce = grid.logic[c_idx] + for k, (rs, re) in enumerate(spans): + box, logic, members = ( + np.array([x0, cuts[k], x1, cuts[k + 1]]), + [rs, re, cs, ce], + words[line_of == k].tolist(), + ) + if k == 0: + grid.boxes[c_idx], grid.logic[c_idx], grid.cell_words[c_idx] = box, logic, members + else: + grid.add(box, logic, members, grid.origin[c_idx]) + + # Rows merged by the model, from the bottom up so that the rows still to process do not move + for r in range(max(lc[1] for lc in grid.logic), -1, -1): + in_row = [i for i, lc in enumerate(grid.logic) if lc[0] == lc[1] == r] + filled = [i for i in in_row if grid.cell_words[i]] + words = [w for i in filled for w in grid.cell_words[i]] + if len(filled) < 2: + continue + line_of = dict(zip(words, _group(centers[words], line_height / 2).tolist())) + n_lines = max(line_of.values()) + 1 + multi = [i for i in filled if len({line_of[w] for w in grid.cell_words[i]}) > 1] + if len(multi) < 2 or 2 * len(multi) < len(filled): + continue + for lc in grid.logic: # the rows below move down, the cells spanning the row span its new rows too + if not lc[0] == lc[1] == r: + lc[0] += n_lines - 1 if lc[0] > r else 0 + lc[1] += n_lines - 1 if lc[1] >= r else 0 + for i in in_row: + if not grid.cell_words[i]: + grid.logic[i][1] = r + n_lines - 1 + continue + lines = np.array([line_of[w] for w in grid.cell_words[i]]) + present = np.unique(lines) + split(i, np.searchsorted(present, lines), [(r + k, r + k) for k in present]) + + # Cells spanning several rows whose text lines lie in different rows + row_lines = grid.lines(0) + for i in range(len(grid.boxes)): + rs, re = grid.logic[i][:2] + if re == rs or len(grid.cell_words[i]) < 2: + continue + cell_lines = _group(centers[grid.cell_words[i]], line_height / 2) + rows = [] + for k in range(int(cell_lines.max()) + 1): + text = grid.words[np.asarray(grid.cell_words[i])[cell_lines == k]] + lo, hi = text[:, 1].min(), text[:, 3].max() + row = int(np.clip(np.searchsorted(row_lines, (lo + hi) / 2, side="right") - 1, rs, re)) + aligned = any( + j != i + and grid.logic[j][0] == row == grid.logic[j][1] + and any(min(hi, w[3]) - max(lo, w[1]) > 0.5 * (hi - lo) for w in grid.words[grid.cell_words[j]]) + for j in range(len(grid.boxes)) + ) + if not aligned or row in rows: + break + rows.append(row) + else: + if len(rows) > 1: + spans = [ + (rs if k == 0 else row, re if k == len(rows) - 1 else rows[k + 1] - 1) for k, row in enumerate(rows) + ] + split(i, cell_lines, spans) + + +def _reading_order(words: np.ndarray, idcs: list[int], line_height: float) -> list[int]: + """Order the words of a cell: text lines top to bottom, words left to right""" + lines = _group((words[idcs, 1] + words[idcs, 3]) / 2, line_height / 2) + return [idcs[j] for j in np.lexsort((words[idcs, 0], lines))] + + +def fill_table_grid( + cells: np.ndarray, logic: np.ndarray, words: np.ndarray +) -> tuple[np.ndarray, np.ndarray, list[list[int]], list[int]]: + """Make the grid of a table consistent with its text, and assign every word of the table to a cell. + + The words in a cell are assigned to it, the other ones complete the grid (missed cells, header row, totals...). + The rows / columns split by the model are merged, the ones without text are removed, and the rows merged by the + model are split along their text lines. + + Args: + cells: the (C, 4) cell boxes `(x0, y0, x1, y1)` predicted by the table model, in the frame of the table image + logic: the (C, 4) logical coordinates `(row_start, row_end, col_start, col_end)` of the cells + words: the (W, 4) boxes of the words of the table, in the same frame + + Returns: + the cell boxes, their local logical coordinates, the indices of their words in reading order, and the index of + the predicted cell each one comes from (-1 for the cells made from the words) + """ + grid = _Grid(cells, logic, np.asarray(words, dtype=np.float64).reshape(-1, 4)) + if len(grid.boxes) == 0: + return np.zeros((0, 4)), np.zeros((0, 4), dtype=np.int64), [], [] + grid.localize() + w = grid.words + if w.shape[0] > 0: + centers = (w[:, :2] + w[:, 2:]) / 2 + boxes = np.asarray(grid.boxes) + inside = ( + (boxes[None, :, 0] <= centers[:, None, 0]) + & (centers[:, None, 0] <= boxes[None, :, 2]) + & (boxes[None, :, 1] <= centers[:, None, 1]) + & (centers[:, None, 1] <= boxes[None, :, 3]) + ) + assigned = inside.any(axis=1) + for w_idx, c_idx in zip(np.flatnonzero(assigned), inside[assigned].argmax(axis=1)): + grid.cell_words[c_idx].append(int(w_idx)) + line_height = float(np.median(w[:, 3] - w[:, 1])) + line_height = line_height if line_height > 0 else 1.0 + if not assigned.all(): + _complete(grid, np.flatnonzero(~assigned), line_height) + _merge_and_prune(grid) + _split_rows(grid, line_height) + grid.localize() + grid.cell_words = [_reading_order(w, idcs, line_height) if idcs else [] for idcs in grid.cell_words] + return np.asarray(grid.boxes), np.asarray(grid.logic, dtype=np.int64), grid.cell_words, grid.origin diff --git a/tests/common/test_models_builder.py b/tests/common/test_models_builder.py index 93d7f7d7ad..94769947af 100644 --- a/tests/common/test_models_builder.py +++ b/tests/common/test_models_builder.py @@ -573,18 +573,17 @@ def test_documentbuilder_tables(): _table_cell(0.10, 0.10, 0.25, 0.20, 0, 0, 0, 0, score=0.8), _table_cell(0.28, 0.10, 0.45, 0.20, 0, 0, 1, 1, score=0.6), _table_cell(0.10, 0.22, 0.25, 0.32, 1, 1, 0, 0, score=0.4), # no word inside this cell - _table_cell(0.28, 0.22, 0.45, 0.32, 1, 1, 1, 1, score=0.5), ], "num_rows": 2, "num_cols": 2, } - page = doc_builder(*_table_words("Name", "Age", "30", confidences=[0.95, 0.85, 0.75]), tables=[table]).pages[0] + page = doc_builder(*_table_words("Name", "Age", confidences=[0.95, 0.85]), tables=[table]).pages[0] cells = page.tables[0].cells - assert [cell.value for cell in cells] == ["Name", "Age", "", "30"] - assert [cell.objectness_score for cell in cells] == [0.8, 0.6, 0.4, 0.5] - assert [cell.confidence for cell in cells] == pytest.approx([0.95, 0.85, 0.4, 0.75]) - assert page.tables[0].confidence == pytest.approx(0.575) - assert [cell["objectness_score"] for cell in page.export()["tables"][0]["cells"]] == [0.8, 0.6, 0.4, 0.5] + assert [cell.value for cell in cells] == ["Name", "Age", ""] + assert [cell.objectness_score for cell in cells] == [0.8, 0.6, 0.4] + assert [cell.confidence for cell in cells] == pytest.approx([0.95, 0.85, 0.4]) + assert page.tables[0].confidence == pytest.approx(0.6) + assert [cell["objectness_score"] for cell in page.export()["tables"][0]["cells"]] == [0.8, 0.6, 0.4] # Multi-word cells are read line by line, left to right words = ["world", "hello", "second", "line", "solo"] @@ -657,133 +656,23 @@ def test_documentbuilder_tables(): json.dumps(page.export()) -def _grid_cells(xs, ys): - """Cells of a regular grid from its column and row lines""" - return [ - _table_cell(xs[c], ys[r], xs[c + 1], ys[r + 1], r, r, c, c) - for r in range(len(ys) - 1) - for c in range(len(xs) - 1) - ] - - -def _word_box(cx, cy, w=0.04, h=0.02): - return [cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2] - - -@pytest.mark.parametrize("straight", [True, False]) -def test_documentbuilder_tables_complete_grid(straight): - # A 2 x 2 grid predicted by the table model, inside a larger table region found by the layout model - xs, ys = [0.20, 0.40, 0.60], [0.30, 0.40, 0.50] - cells = _grid_cells(xs, ys)[:3] # the cell of the last row and column is missed - region = [[0.10, 0.15], [0.70, 0.15], [0.70, 0.65], [0.10, 0.65]] - words = { - "A": (0.30, 0.35), - "B": (0.50, 0.35), - "C": (0.30, 0.45), - "missed": (0.50, 0.45), # in the slot of the missed cell - "Col1": (0.30, 0.25), # a header row above the grid - "Col2": (0.50, 0.25), - "Total": (0.25, 0.58), # a sentence below the grid, across both columns - "amount": (0.3225, 0.58), - "due": (0.405, 0.58), - "key": (0.15, 0.35), # a column on the left of the grid - "caption": (0.40, 0.80), # outside of the table region - } - widths = {"Total": 0.06, "amount": 0.075, "due": 0.08} # the words of the sentence are 5px apart - boxes = [_word_box(*center, w=widths.get(name, 0.04)) for name, center in words.items()] - if not straight: - boxes = [[[x0, y0], [x1, y0], [x1, y1], [x0, y1]] for x0, y0, x1, y1 in boxes] - table = {"cells": cells, "num_rows": 2, "num_cols": 2, "geometry": region} - page = builder.DocumentBuilder()(*_word_args(list(words), boxes), tables=[[table]]).pages[0] - # Every word of the table region ends up in the table, the caption stays in the blocks - assert _words(page) == ["caption"] - grid = page.tables[0].to_grid() - assert grid == [ - ["", "Col1", "Col2"], - ["key", "A", "B"], - ["", "C", "missed"], - ["", "Total amount due", ""], +def test_documentbuilder_tables_assigned_words(): + # The OCR predictor assigns the words of a table to its cells (in reading order): the builder uses them as is + cells = [ + {**_table_cell(0.10, 0.10, 0.25, 0.20, 0, 0, 0, 0), "words": [1, 0]}, + {**_table_cell(0.28, 0.10, 0.45, 0.20, 0, 0, 1, 1), "words": []}, + # a cell made from the words of the table: not scored by the table model + {**_table_cell(0.10, 0.22, 0.45, 0.32, 1, 1, 0, 1, score=None), "words": [2]}, ] - # The cells added from the words are not scored by the table model, and span the columns of their text - total = next(cell for cell in page.tables[0].cells if cell.value.startswith("Total")) - assert total.objectness_score is None and (total.col_start, total.col_end) == (1, 2) + words = ["world", "hello", "total", "caption"] + boxes = [[0.15, 0.12, 0.2, 0.14], [0.11, 0.12, 0.14, 0.14], [0.6, 0.6, 0.7, 0.7], [0.3, 0.9, 0.4, 0.95]] + page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[{"cells": cells}]]).pages[0] + assert page.tables[0].to_grid() == [["hello world", ""], ["total", ""]] + assert [cell.objectness_score for cell in page.tables[0].cells] == [0.9, 0.9, None] assert page.tables[0].confidence == pytest.approx(0.9) + assert _words(page) == ["caption"] json.dumps(page.export()) - # Without the table region, the words around the cells stay in the blocks (only the cell outline is known) - table = {"cells": cells, "num_rows": 2, "num_cols": 2} - page = builder.DocumentBuilder()(*_word_args(list(words), boxes), tables=[[table]]).pages[0] - assert page.tables[0].to_grid() == [["A", "B"], ["C", "missed"]] - assert sorted(_words(page)) == sorted(["Col1", "Col2", "Total", "amount", "due", "key", "caption"]) - - -def test_documentbuilder_tables_simplify_grid(): - # The model splits a row in 2 (its cells do not line up) and predicts an empty row: the text of the split row - # lies on the same line, so the 2 rows are merged, and the empty row is removed - cells = [ - _table_cell(0.10, 0.10, 0.30, 0.20, 0, 0, 0, 0), - _table_cell(0.30, 0.10, 0.50, 0.20, 0, 0, 1, 1), - _table_cell(0.10, 0.20, 0.30, 0.30, 1, 1, 0, 0), - _table_cell(0.30, 0.20, 0.50, 0.24, 1, 1, 1, 1), # empty, a row of its own - _table_cell(0.30, 0.24, 0.50, 0.34, 2, 2, 1, 1), - _table_cell(0.10, 0.34, 0.30, 0.44, 3, 3, 0, 0), # an empty row - _table_cell(0.30, 0.34, 0.50, 0.44, 3, 3, 1, 1), - _table_cell(0.10, 0.44, 0.30, 0.54, 4, 4, 0, 0), - _table_cell(0.30, 0.44, 0.50, 0.54, 4, 4, 1, 1), - ] - words = ["h1", "h2", "a", "b", "c", "d"] - boxes = [_word_box(0.2, 0.15), _word_box(0.4, 0.15), _word_box(0.2, 0.25), _word_box(0.4, 0.255)] - boxes += [_word_box(0.2, 0.49), _word_box(0.4, 0.49)] - table = {"cells": cells, "num_rows": 5, "num_cols": 2} - page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] - assert page.tables[0].to_grid() == [["h1", "h2"], ["a", "b"], ["c", "d"]] - assert (page.tables[0].num_rows, page.tables[0].num_cols) == (3, 2) - # Text on different lines is not merged - boxes[3] = _word_box(0.4, 0.31) - page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] - assert page.tables[0].to_grid() == [["h1", "h2"], ["a", ""], ["", "b"], ["c", "d"]] - - -def test_documentbuilder_tables_split_rows(): - # The model merges the 3 text lines of a dense table into a single row - cells = [ - _table_cell(0.10, 0.10, 0.30, 0.16, 0, 0, 0, 0), - _table_cell(0.30, 0.10, 0.50, 0.16, 0, 0, 1, 1), - _table_cell(0.10, 0.16, 0.30, 0.34, 1, 1, 0, 0), - _table_cell(0.30, 0.16, 0.50, 0.34, 1, 1, 1, 1), - ] - words, boxes = ["Name", "Value"], [_word_box(0.2, 0.13), _word_box(0.4, 0.13)] - for line, cy in enumerate((0.19, 0.25, 0.31)): - words += [f"key{line}", f"{line}.00"] - boxes += [_word_box(0.2, cy), _word_box(0.4, cy)] - table = {"cells": cells, "num_rows": 2, "num_cols": 2} - page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] - assert page.tables[0].to_grid() == [["Name", "Value"], ["key0", "0.00"], ["key1", "1.00"], ["key2", "2.00"]] - - # A single cell wrapping its text (e.g. a long header) does not split its row - words = ["Unit", "price", "Qty", "10", "2"] - boxes = [_word_box(0.2, 0.12), _word_box(0.2, 0.14), _word_box(0.4, 0.13)] - boxes += [_word_box(0.2, 0.25), _word_box(0.4, 0.25)] - page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] - assert page.tables[0].to_grid() == [["Unit price", "Qty"], ["10", "2"]] - - # A cell spanning 2 rows whose text lines lie in each of them, aligned with the text of the row - cells = [ - _table_cell(0.10, 0.10, 0.30, 0.20, 0, 0, 0, 0), - _table_cell(0.10, 0.20, 0.30, 0.30, 1, 1, 0, 0), - _table_cell(0.30, 0.10, 0.50, 0.30, 0, 1, 1, 1), - ] - words = ["Acquired", "Taken", "51", "9"] - boxes = [_word_box(0.2, 0.15), _word_box(0.2, 0.25), _word_box(0.4, 0.15), _word_box(0.4, 0.25)] - table = {"cells": cells, "num_rows": 2, "num_cols": 2} - page = builder.DocumentBuilder()(*_word_args(words, boxes), tables=[[table]]).pages[0] - assert page.tables[0].to_grid() == [["Acquired", "51"], ["Taken", "9"]] - # A label spanning 2 rows on a single line is kept as is - page = builder.DocumentBuilder()( - *_word_args(words[:2] + ["label"], boxes[:2] + [_word_box(0.4, 0.2)]), tables=[[table]] - ).pages[0] - assert [(c.value, c.row_span) for c in page.tables[0].cells] == [("Acquired", 1), ("Taken", 1), ("label", 2)] - @pytest.mark.parametrize( "words, boxes, grids, tables", diff --git a/tests/common/test_models_table_grid.py b/tests/common/test_models_table_grid.py new file mode 100644 index 0000000000..ba40040bf2 --- /dev/null +++ b/tests/common/test_models_table_grid.py @@ -0,0 +1,120 @@ +import numpy as np + +from doctr.models.table_structure.grid import _axis_slots, _grid_lines, _group, fill_table_grid + + +def _grid(xs, ys): + """Boxes and logical coordinates (row_start, row_end, col_start, col_end) of a regular grid""" + boxes, logic = [], [] + for r in range(len(ys) - 1): + for c in range(len(xs) - 1): + boxes.append([xs[c], ys[r], xs[c + 1], ys[r + 1]]) + logic.append([r, r, c, c]) + return np.array(boxes, dtype=float), np.array(logic) + + +def _word(cx, cy, w=40, h=20): + return [cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2] + + +def _table(cells, logic, words): + """The text grid of a table, from named words""" + names, boxes = list(words), np.array(list(words.values()), dtype=float).reshape(-1, 4) + boxes, logic, cell_words, origin = fill_table_grid(np.asarray(cells, dtype=float), np.asarray(logic), boxes) + grid = [["" for _ in range(logic[:, 3].max() + 1)] for _ in range(logic[:, 1].max() + 1)] + for (rs, _, cs, _), idcs in zip(logic, cell_words): + grid[rs][cs] = " ".join([grid[rs][cs]] * bool(grid[rs][cs]) + [names[i] for i in idcs]) + return grid, logic, origin + + +def test_table_grid_helpers(): + # Grid lines at the median of the cell edges, interpolated when no edge lies on them + lines = _grid_lines(np.array([0, 2]), np.array([0, 2]), np.array([0.0, 20.0]), np.array([10.0, 30.0]), 3) + np.testing.assert_allclose(lines, [0, 10, 20, 30]) + # Positions grouped by distance to the mean of their group + np.testing.assert_array_equal(_group(np.array([0.0, 30.0, 4.0, 33.0]), 10), [0, 1, 0, 1]) + # Words beyond the grid make new indices, grouped by text line (rows) or overlapping extent (columns) + lines = np.array([100.0, 200.0, 300.0]) + lo, hi = np.array([150.0, 40.0, 45.0, 10.0, 310.0]), np.array([160.0, 60.0, 65.0, 30.0, 330.0]) + np.testing.assert_array_equal(_axis_slots(lo, hi, lines, 15), [0, -1, -1, -2, 2]) + np.testing.assert_array_equal(_axis_slots(lo, hi, lines, None), [0, -1, -1, -2, 2]) + + +def test_fill_table_grid_complete(): + # A 2 x 2 grid whose last cell is missed by the model, inside a table holding more text around it + cells, logic = _grid([200, 400, 600], [300, 400, 500]) + words = { + "A": _word(300, 350), + "B": _word(500, 350), + "C": _word(300, 450), + "missed": _word(500, 450), # in the slot of the missed cell + "Col1": _word(300, 250), # a header row above the grid + "Col2": _word(500, 250), + "Total": _word(250, 580, w=60), # a sentence below the grid, across both columns + "amount": _word(322.5, 580, w=75), + "due": _word(405, 580, w=80), + "key": _word(150, 350), # a column on the left of the grid + } + grid, _, origin = _table(cells[:3], logic[:3], words) + assert grid == [["", "Col1", "Col2"], ["key", "A", "B"], ["", "C", "missed"], ["", "Total amount due", ""]] + # The cells made from the words have no source cell + assert sorted(origin) == [-1] * 5 + [0, 1, 2] + + # Without words: the predicted grid, with local logical coordinates + boxes, out_logic, cell_words, origin = fill_table_grid(cells, logic + 3, np.zeros((0, 4))) + np.testing.assert_array_equal(out_logic, logic) + assert cell_words == [[]] * 4 and origin == [0, 1, 2, 3] + + +def test_fill_table_grid_merge_and_prune(): + # The model splits a row in 2 (the edges of its cells do not line up) and predicts an empty row + cells = [ + [100, 100, 300, 200], + [300, 100, 500, 200], + [100, 200, 300, 300], + [300, 200, 500, 240], # empty, a row of its own + [300, 240, 500, 340], + [100, 340, 300, 440], # an empty row + [300, 340, 500, 440], + [100, 440, 300, 540], + [300, 440, 500, 540], + ] + logic = [[0, 0, 0, 0], [0, 0, 1, 1], [1, 1, 0, 0], [1, 1, 1, 1], [2, 2, 1, 1]] + logic += [[3, 3, 0, 0], [3, 3, 1, 1], [4, 4, 0, 0], [4, 4, 1, 1]] + words = {"h1": _word(200, 150), "h2": _word(400, 150), "a": _word(200, 250), "b": _word(400, 255)} + words |= {"c": _word(200, 490), "d": _word(400, 490)} + # The text of the split row lies on the same line: the 2 rows are merged, and the empty row is removed + grid, _, _ = _table(cells, logic, words) + assert grid == [["h1", "h2"], ["a", "b"], ["c", "d"]] + # Text on different lines is not merged + words["b"] = _word(400, 310) + grid, _, _ = _table(cells, logic, words) + assert grid == [["h1", "h2"], ["a", ""], ["", "b"], ["c", "d"]] + + +def test_fill_table_grid_split_rows(): + # The model merges the 3 text lines of a dense table into a single row + cells, logic = _grid([100, 300, 500], [100, 160, 340]) + words = {"Name": _word(200, 130), "Value": _word(400, 130)} + for line, cy in enumerate((190, 250, 310)): + words |= {f"key{line}": _word(200, cy), f"{line}.00": _word(400, cy)} + grid, _, _ = _table(cells, logic, words) + assert grid == [["Name", "Value"], ["key0", "0.00"], ["key1", "1.00"], ["key2", "2.00"]] + + # A single cell wrapping its text (e.g. a long header) does not split its row + words = {"Unit": _word(200, 120), "price": _word(200, 140), "Qty": _word(400, 130)} + words |= {"10": _word(200, 250), "2": _word(400, 250)} + grid, _, _ = _table(cells, logic, words) + assert grid == [["Unit price", "Qty"], ["10", "2"]] + + # A cell spanning 2 rows whose text lines lie in each of them, aligned with the text of the row + cells, logic = [[100, 100, 300, 200], [100, 200, 300, 300], [300, 100, 500, 300]], [[0, 0, 0, 0], [1, 1, 0, 0]] + logic.append([0, 1, 1, 1]) + words = {"Acquired": _word(200, 150), "Taken": _word(200, 250), "51": _word(400, 150), "9": _word(400, 250)} + grid, _, origin = _table(cells, logic, words) + assert grid == [["Acquired", "51"], ["Taken", "9"]] + assert sorted(origin) == [0, 1, 2, 2] # both parts keep the score of the spanning cell + # A label spanning 2 rows on a single line is kept as is + words = {"Acquired": _word(200, 150), "Taken": _word(200, 250), "label": _word(400, 200)} + _, out_logic, _ = _table(cells, logic, words) + np.testing.assert_array_equal(out_logic, logic) diff --git a/tests/pytorch/test_models_zoo_pt.py b/tests/pytorch/test_models_zoo_pt.py index e17620d9b7..ed9bf2fbac 100644 --- a/tests/pytorch/test_models_zoo_pt.py +++ b/tests/pytorch/test_models_zoo_pt.py @@ -427,7 +427,7 @@ def cell(x0, y0, x1, y1, r, c): return {"geometry": box, "score": 0.9, "row_start": r, "row_end": r, "col_start": c, "col_end": c} # 2 cells of the table, and a cell of the neighbouring content, above the region - cells = [cell(0, 0, 100, 50, 1, 0), cell(100, 0, 200, 50, 1, 1), cell(0, -18, 200, -4, 0, 0)] + cells = [cell(0, 0, 100, 30, 1, 0), cell(100, 0, 200, 30, 1, 1), cell(0, -18, 200, -4, 0, 0)] return [{"cells": cells, "num_rows": 2, "num_cols": 2}] fake = type("FakeOCRPredictor", (), {})() @@ -435,25 +435,31 @@ def cell(x0, y0, x1, y1, r, c): fake.table_class_name = "Table" fake._cell_polygon = OCRPredictor._cell_polygon page = np.full((400, 600, 3), 255, dtype=np.uint8) - region = { - "class_names": ["Text", "Table"], - "boxes": np.array([[0, 0, 0.1, 0.1], [0.1, 0.25, 0.1 + 200 / 600, 0.375]]), - } - tables = OCRPredictor._tables_from_regions(fake, [page], [region]) + x0, y0, x1, y1 = 0.1, 0.25, 0.1 + 200 / 600, 0.375 # the cells cover the top 30 px of its 50 px + region = {"class_names": ["Text", "Table"], "boxes": np.array([[0, 0, 0.1, 0.1], [x0, y0, x1, y1]])} + # Words in each cell, one in the region below the cells, and one outside of the region + words = np.array([ + [0.12, 0.27, 0.15, 0.29], + [0.30, 0.27, 0.33, 0.29], + [0.12, 0.345, 0.15, 0.365], + [0.12, 0.80, 0.15, 0.82], + ]) + tables = OCRPredictor._tables_from_regions(fake, [page], [region], [words]) # Only the table region is cropped, with a context margin assert len(fake.table_predictor.crops) == 1 and fake.table_predictor.crops[0].shape[:2] == (90, 240) assert len(tables) == 1 and len(tables[0]) == 1 table = tables[0][0] - # The cell of the context margin is dropped, the others are mapped back to page-relative coordinates - assert len(table["cells"]) == 2 - np.testing.assert_allclose( - table["cells"][0]["geometry"], - [[0.1, 0.25], [0.1 + 100 / 600, 0.25], [0.1 + 100 / 600, 0.375], [0.1, 0.375]], - atol=1e-5, - ) - # The table region is attached to the grid + # The cell of the context margin is dropped, the word below the cells makes a new row + assert (table["num_rows"], table["num_cols"]) == (2, 2) + cells = sorted(table["cells"], key=lambda c: (c["row_start"], c["col_start"])) + assert [(c["row_start"], c["col_start"], c["words"], c["score"]) for c in cells] == [ + (0, 0, [0], 0.9), + (0, 1, [1], 0.9), + (1, 0, [2], None), + ] + # The cells are mapped back to page-relative coordinates np.testing.assert_allclose( - table["geometry"], [[0.1, 0.25], [0.1 + 200 / 600, 0.25], [0.1 + 200 / 600, 0.375], [0.1, 0.375]], atol=1e-6 + cells[0]["geometry"], [[x0, y0], [x0 + 100 / 600, y0], [x0 + 100 / 600, 0.325], [x0, 0.325]], atol=1e-5 ) From d076fa3adb988ecb4e090a0ac5c79e9b8eb2b489 Mon Sep 17 00:00:00 2001 From: felix Date: Tue, 10 Nov 2026 11:49:28 +0100 Subject: [PATCH 3/8] Update --- doctr/models/builder.py | 270 +++--------------- doctr/models/predictor/pytorch.py | 134 +-------- doctr/models/reading_order/base.py | 41 ++- doctr/models/table_structure/grid.py | 86 +++++- .../table_structure/predictor/pytorch.py | 126 ++++++++ tests/common/test_models_builder.py | 36 +-- tests/common/test_models_reading_order.py | 19 ++ tests/common/test_models_table_grid.py | 7 +- .../pytorch/test_models_table_structure_pt.py | 49 ++++ tests/pytorch/test_models_zoo_pt.py | 55 ---- 10 files changed, 354 insertions(+), 469 deletions(-) diff --git a/doctr/models/builder.py b/doctr/models/builder.py index d9475a17d2..5b3b63a6ad 100644 --- a/doctr/models/builder.py +++ b/doctr/models/builder.py @@ -9,7 +9,6 @@ from collections.abc import Sequence from typing import Any -import cv2 import numpy as np from scipy.cluster.hierarchy import fclusterdata @@ -26,9 +25,9 @@ TableCell, Word, ) +from doctr.models.table_structure.grid import assemble_page_table from doctr.utils.geometry import ( estimate_page_angle, - order_points, resolve_enclosing_bbox, resolve_enclosing_rbbox, rotate_boxes, @@ -450,273 +449,68 @@ def _build_layout_elements(regions: dict[str, Any] | None) -> list[LayoutElement elements.append(LayoutElement(layout_type=str(cname), confidence=float(score), geometry=geometry)) return elements - @staticmethod - def _word_centers(boxes: np.ndarray) -> np.ndarray: - """Return the (x, y) center of each word box. - - Args: - boxes: word boxes of shape (N, 4) (straight: x1, y1, x2, y2) or (N, 4, 2) (rotated polygon) - - Returns: - array of shape (N, 2) with the relative center coordinates of each box - """ - if boxes.ndim == 3: # rotated polygons (N, 4, 2) - return boxes.mean(axis=1) - return np.stack([(boxes[:, 0] + boxes[:, 2]) / 2, (boxes[:, 1] + boxes[:, 3]) / 2], axis=1) - - @staticmethod - def _as_cell_polygon(geometry: Any) -> np.ndarray: - """Normalize a cell geometry to a (4, 2) polygon. - - Straight-page table predictions store cells as flat (xmin, ymin, xmax, ymax) boxes, while rotated-page - predictions store (4, 2) corner polygons: both are mapped to a (4, 2) polygon (TL, TR, BR, BL). - - Args: - geometry: the raw cell geometry - - Returns: - array of shape (4, 2) with the polygon vertices - """ - arr = np.asarray(geometry, dtype=np.float32) - if arr.ndim == 1: # straight box (xmin, ymin, xmax, ymax) - x0, y0, x1, y1 = arr - return np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) - return order_points(arr.reshape(-1, 2)) - - @staticmethod - def _points_in_polygons(points: np.ndarray, polys: np.ndarray) -> np.ndarray: - """Vectorized ray casting: test every point against every polygon at once. - - Args: - points: array of shape (N, 2) with the (x, y) coordinates of the points - polys: array of shape (C, V, 2) with the polygon vertices - - Returns: - boolean array of shape (N, C), True where point n lies inside polygon c - """ - if points.shape[0] == 0 or polys.shape[0] == 0: - return np.zeros((points.shape[0], polys.shape[0]), dtype=bool) - px = points[:, 0].astype(np.float64)[:, None, None] - py = points[:, 1].astype(np.float64)[:, None, None] - vi = polys.astype(np.float64) # (C, V, 2) - vj = np.roll(vi, 1, axis=1) # previous vertex, with wrap-around - xi, yi = vi[None, ..., 0], vi[None, ..., 1] - xj, yj = vj[None, ..., 0], vj[None, ..., 1] - crossing = ((yi > py) != (yj > py)) & (px < (xj - xi) * (py - yi) / (yj - yi + 1e-12) + xi) - # A point is inside when a ray crosses the polygon boundary an odd number of times - return (crossing.sum(axis=-1) % 2).astype(bool) - - @staticmethod - def _order_cell_words(w_idcs: list[int], centers: np.ndarray, heights: np.ndarray) -> list[int]: - """Order the words of a table cell in reading order: rows top to bottom, words left to right. - - Args: - w_idcs: indices of the cell's words - centers: word centers of shape (N, 2), de-skewed for rotated pages - heights: per-word heights of shape (N,) (rotation-invariant) - - Returns: - the cell's word indices in reading order - """ - idcs = sorted(w_idcs, key=lambda i: float(centers[i][1])) - med_height = float(np.median([heights[i] for i in idcs])) - if not np.isfinite(med_height) or med_height <= 0: - med_height = 1.0 - rows: list[list[int]] = [[idcs[0]]] - y_sum = float(centers[idcs[0]][1]) - for idx in idcs[1:]: - if float(centers[idx][1]) - y_sum / len(rows[-1]) < med_height / 2: - rows[-1].append(idx) - y_sum += float(centers[idx][1]) - else: - rows.append([idx]) - y_sum = float(centers[idx][1]) - return [idx for row in rows for idx in sorted(row, key=lambda i: float(centers[i][0]))] - - @staticmethod - def _localize_logic(cells: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int, int]: - """Re-index a table's logical coordinates to a local 0-based grid. - - The table model returns logical (row/column) coordinates that may carry a constant offset; shifting them - so that the smallest row/column start is 0 makes the grid directly usable by :meth:`Table.to_grid`. - - Args: - cells: the cells of a single table - - Returns: - a tuple `(cells, num_rows, num_cols)` with the re-indexed cells and the table dimensions - """ - min_row = min(int(c["row_start"]) for c in cells) - min_col = min(int(c["col_start"]) for c in cells) - norm: list[dict[str, Any]] = [] - max_row = max_col = 0 - for c in cells: - nc = dict(c) - nc["row_start"] = int(c["row_start"]) - min_row - nc["row_end"] = int(c["row_end"]) - min_row - nc["col_start"] = int(c["col_start"]) - min_col - nc["col_end"] = int(c["col_end"]) - min_col - max_row, max_col = max(max_row, nc["row_end"]), max(max_col, nc["col_end"]) - norm.append(nc) - return norm, max_row + 1, max_col + 1 - - @classmethod - def _table_grids( - cls, page_table: dict[str, Any] | list[dict[str, Any]] | None - ) -> list[tuple[list[dict[str, Any]], int, int]]: - """Normalize the table structure prediction(s) of a page to non-empty grids with local logical coordinates - - Args: - page_table: a single grid, a list of grids or None (cf. `_build_tables`) - - Returns: - the cells, number of rows and number of columns of each non-empty grid - """ - raw_tables = [] if page_table is None else [page_table] if isinstance(page_table, dict) else list(page_table) - return [cls._localize_logic(raw["cells"]) for raw in raw_tables if raw and len(raw.get("cells", [])) > 0] - - @classmethod - def _assign_cell_words(cls, centers: np.ndarray, polys: np.ndarray, consumed: np.ndarray) -> list[list[int]]: - """Assign each word not yet consumed to at most one cell of a table, and flag it as consumed - - A word goes to the first cell containing its center. A word landing just outside every cell but inside the - table goes to the nearest cell, within half a median cell diagonal, so table text is not dropped into the body. - - Args: - centers: the (N, 2) word centers - polys: the (C, 4, 2) cell polygons, vertices ordered TL, TR, BR, BL - consumed: the (N,) mask of the words already assigned to a table, updated in place - - Returns: - the word indices of each cell - """ - cell_words: list[list[int]] = [[] for _ in polys] - free = np.flatnonzero(~consumed) - if free.size == 0: - return cell_words - inside = cls._points_in_polygons(centers[free], polys) # (F, C) - assigned = inside.any(axis=1) - for w_idx, c_idx in zip(free[assigned], inside[assigned].argmax(axis=1)): - cell_words[c_idx].append(int(w_idx)) - consumed[w_idx] = True - leftover = free[~assigned] - if leftover.size > 0: - cell_centers = polys.mean(axis=1) # (C, 2) - max_dist = 0.5 * float(np.median(np.linalg.norm(polys[:, 2] - polys[:, 0], axis=1))) # TL->BR diagonal - outline = cv2.boxPoints(cv2.minAreaRect(polys.reshape(-1, 2).astype(np.float32))).reshape(1, 4, 2) - in_table = cls._points_in_polygons(centers[leftover], outline)[:, 0] - for w_idx in leftover[in_table]: - dists = np.linalg.norm(cell_centers - centers[w_idx], axis=1) - nearest = int(dists.argmin()) - if dists[nearest] <= max_dist: - cell_words[nearest].append(int(w_idx)) - consumed[w_idx] = True - return cell_words - - @staticmethod - def _deskew_centers(centers: np.ndarray, polys: np.ndarray) -> np.ndarray: - """Rotate word centers by the angle of a table, to order the words along its own axes - - Args: - centers: the (N, 2) word centers - polys: the (C, 4, 2) cell polygons, vertices ordered TL, TR, BR, BL - - Returns: - the (N, 2) de-skewed centers - """ - top_edges = polys[:, 1] - polys[:, 0] # TR - TL - angle = float(np.median(np.arctan2(top_edges[:, 1], top_edges[:, 0]))) - cos_a, sin_a = np.cos(-angle), np.sin(-angle) - pivot = centers.mean(axis=0) - shifted = centers - pivot - return np.stack( - [ - pivot[0] + shifted[:, 0] * cos_a - shifted[:, 1] * sin_a, - pivot[1] + shifted[:, 0] * sin_a + shifted[:, 1] * cos_a, - ], - axis=1, - ) - def _build_tables( self, boxes: np.ndarray, word_preds: list[tuple[str, float]], page_table: dict[str, Any] | list[dict[str, Any]] | None, + page_shape: tuple[int, int], ) -> tuple[list[Table], np.ndarray]: - """Assign detected words to table cells and build the page tables. + """Build the tables of a page from their structure and the words of the page. Args: boxes: word boxes of the page, of shape (N, 4) or (N, 4, 2), in relative coordinates word_preds: list of (text, confidence) for each of the N words page_table: the table structure prediction(s) for the page. Either a single grid `{"cells": [{"geometry", "score", "row_start", "row_end", "col_start", "col_end"}], "num_rows", - "num_cols"}` (cell geometries in page-relative coordinates), a list of such grids, or None. When the - cells list the indices of their words (`"words"`, in reading order, cf. the OCR predictor), they are - used as is, otherwise each word goes to the cell containing its center + "num_cols"}` (cell geometries in page-relative coordinates), a list of such grids, or None. The cells + may list the indices of their words (`"words"`, in reading order, cf. + :meth:`~doctr.models.table_structure.predictor.TablePredictor.predict_regions`), otherwise the words + are assigned to them (cf. :func:`~doctr.models.table_structure.grid.assemble_page_table`) + page_shape: the (height, width) of the page Returns: a tuple with the list of `Table` objects of the page (one per provided table) and a boolean mask of shape (N,) that is True for words assigned to a table (to be removed from `blocks`) """ consumed = np.zeros(boxes.shape[0], dtype=bool) - grids = self._table_grids(page_table) - if len(grids) == 0: - return [], consumed + raw_tables = [] if page_table is None else [page_table] if isinstance(page_table, dict) else list(page_table) # The geometries follow the page's word geometry: straight 2-point boxes for (N, 4) word boxes, 4-point polygons - # for (N, 4, 2) ones. Word heights are rotation-invariant (left edge length for polygons). + # for (N, 4, 2) ones straight = boxes.ndim != 3 - centers = self._word_centers(boxes) if boxes.shape[0] > 0 else np.empty((0, 2)) - if boxes.shape[0] == 0: - heights = np.empty(0) - else: - heights = boxes[:, 3] - boxes[:, 1] if straight else np.linalg.norm(boxes[:, 3] - boxes[:, 0], axis=1) - tables: list[Table] = [] - for cells, num_rows, num_cols in grids: - polys = np.stack([self._as_cell_polygon(cell["geometry"]) for cell in cells]) # (C, 4, 2) - if all("words" in cell for cell in cells): - # The OCR predictor assigned the words of the table to its cells, in reading order - cell_words = [] - for cell in cells: - cell_words.append([int(i) for i in cell["words"] if not consumed[i]]) - consumed[cell_words[-1]] = True - else: - # Reading order inside the cell: rows top to bottom, words left to right (table axes) - order_centers = centers if straight or centers.shape[0] == 0 else self._deskew_centers(centers, polys) - cell_words = [ - self._order_cell_words(w_idcs, order_centers, heights) if w_idcs else [] - for w_idcs in self._assign_cell_words(centers, polys, consumed) - ] + for raw in raw_tables: + if not raw or len(raw.get("cells", [])) == 0: + continue + cells, num_rows, num_cols = assemble_page_table(raw["cells"], boxes, page_shape, consumed) table_cells = [] - for cell, poly, ordered in zip(cells, polys, cell_words): - if len(ordered) > 0: - value = " ".join(word_preds[i][0] for i in ordered) - confidence = float(np.mean([word_preds[i][1] for i in ordered])) - else: - value, confidence = "", float(cell["score"] or 0.0) + for cell in cells: + words = [word_preds[idx] for idx in cell["words"]] table_cells.append( TableCell( - value=value, - confidence=confidence, - geometry=self._export_geometry(poly, straight), - row_start=int(cell["row_start"]), - row_end=int(cell["row_end"]), - col_start=int(cell["col_start"]), - col_end=int(cell["col_end"]), - objectness_score=None if cell["score"] is None else float(cell["score"]), + value=" ".join(value for value, _ in words), + # The recognition confidence of the words, the structure model's score for an empty cell + confidence=float(np.mean([conf for _, conf in words])) if words else float(cell["score"] or 0), + geometry=self._export_geometry(cell["geometry"], straight), + row_start=cell["row_start"], + row_end=cell["row_end"], + col_start=cell["col_start"], + col_end=cell["col_end"], + objectness_score=cell["score"], ) ) - # Enclosing geometry of the whole table - (xmin, ymin), (xmax, ymax) = polys.reshape(-1, 2).min(axis=0), polys.reshape(-1, 2).max(axis=0) - corners = np.asarray([[xmin, ymin], [xmax, ymin], [xmax, ymax], [xmin, ymax]]) + pts = np.concatenate([cell["geometry"] for cell in cells]) + (xmin, ymin), (xmax, ymax) = pts.min(axis=0), pts.max(axis=0) # The cells made from the words of the table are not scored by the table model - scores = [float(cell["score"]) for cell in cells if cell["score"] is not None] + scores = [cell["score"] for cell in cells if cell["score"] is not None] tables.append( Table( cells=table_cells, num_rows=num_rows, num_cols=num_cols, - geometry=self._export_geometry(corners, straight), + geometry=self._export_geometry( + np.array([[xmin, ymin], [xmax, ymin], [xmax, ymax], [xmin, ymax]]), straight + ), confidence=float(np.mean(scores)) if scores else 0.0, ) ) @@ -925,7 +719,7 @@ def __call__( _tables, ): # Build the page tables and flag the words that belong to a table - page_tables, consumed = self._build_tables(page_boxes, word_preds, page_table) + page_tables, consumed = self._build_tables(page_boxes, word_preds, page_table, shape) if consumed.any(): # Remove the words assigned to a table from the regular blocks output keep = ~consumed diff --git a/doctr/models/predictor/pytorch.py b/doctr/models/predictor/pytorch.py index f65deedbe7..027f28c183 100644 --- a/doctr/models/predictor/pytorch.py +++ b/doctr/models/predictor/pytorch.py @@ -6,7 +6,6 @@ from collections.abc import Iterable, Sequence from typing import Any -import cv2 import numpy as np import torch from torch import nn @@ -16,9 +15,7 @@ from doctr.models.detection.predictor import DetectionPredictor from doctr.models.layout.predictor import LayoutPredictor from doctr.models.recognition.predictor import RecognitionPredictor -from doctr.models.table_structure.grid import fill_table_grid from doctr.models.table_structure.predictor import TablePredictor -from doctr.models.table_structure.tablecenternet.base import _logic_from_geometry from doctr.utils.geometry import detach_scores from .base import _OCRPredictor @@ -213,7 +210,7 @@ def _predict_pages( # Recognize table structure: locate tables with the layout model, crop each one and run the table model tables = ( - self._tables_from_regions(pages, regions, boxes, **kwargs) + self.table_predictor.predict_regions(pages, regions, boxes, self.table_class_name, **kwargs) if self.table_predictor is not None and regions is not None else None ) @@ -234,132 +231,3 @@ def _predict_pages( if self.straighten_pages and self.preserve_original_coords: out = self._remap_to_original_coords(out, _orig_shapes, _straight_shapes, m_invs, _orig_pages) return out - - @staticmethod - def _cell_polygon(geometry: Any) -> np.ndarray: - """Normalize a cell geometry, a straight box (xmin, ymin, xmax, ymax) or a polygon, to a (4, 2) polygon""" - raw = np.asarray(geometry, dtype=np.float32) - if raw.size == 4: # straight box (xmin, ymin, xmax, ymax) - x0, y0, x1, y1 = raw.ravel() - return np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) - return raw.reshape(-1, 2).copy() - - def _tables_from_regions( - self, - pages: list[np.ndarray], - regions: list[dict[str, Any]], - word_boxes: list[np.ndarray] | None = None, - **kwargs: Any, - ) -> list[list[dict[str, Any]]]: - """Crop the table regions found by the layout model and run the table model on each crop. - - Each `Table` region is rectified to an upright crop with a perspective transform (so straight boxes and - rotated polygons are both handled), and the table model is applied per crop. The crop includes a margin of page - context around the region: the cells predicted in this margin belong to the neighbouring content and are - dropped. The words of the page lying in the region are mapped into the crop, where they are assigned to the - cells and complete the grid (cf. :func:`~doctr.models.table_structure.grid.fill_table_grid`). The cells are - then mapped back to page-relative coordinates through the inverse transform: a page yields one structured - table per detected `Table` region. - - Args: - pages: the (possibly straightened) page images - regions: the per-page layout predictions `{"class_names", "boxes", "scores"}` - word_boxes: the per-page word boxes, of shape (N, 4) or (N, 4, 2) in relative coordinates - **kwargs: keyword arguments forwarded to the table predictor - - Returns: - a per-page list of table grids `{"cells": [...], "num_rows": int, "num_cols": int}` in page-relative - coordinates, each cell listing the indices of its words in the page (`"words"`) in reading order - """ - crops: list[np.ndarray] = [] - # (page index, transform, inverse transform, crop width, crop height, context margin) - crop_meta: list[tuple[int, np.ndarray, np.ndarray, int, int, int]] = [] - for p_idx, (page, region) in enumerate(zip(pages, regions)): - if region is None: - continue - h, w = page.shape[:2] - for cls_name, box in zip(region["class_names"], region["boxes"]): - if cls_name != self.table_class_name: - continue - src = self._cell_polygon(box) * np.array([w, h], dtype=np.float32) # absolute pixel corners - # Upright crop size from the region side lengths - crop_w = int(round(max(np.linalg.norm(src[1] - src[0]), np.linalg.norm(src[2] - src[3])))) - crop_h = int(round(max(np.linalg.norm(src[3] - src[0]), np.linalg.norm(src[2] - src[1])))) - if crop_w < 2 or crop_h < 2: - continue - # Map the region onto an inset rectangle so the warp samples `pad` extra pixels of page context - pad = max(20, round(0.01 * max(crop_w, crop_h))) - out_w, out_h = crop_w + 2 * pad, crop_h + 2 * pad - dst = np.array( - [[pad, pad], [pad + crop_w, pad], [pad + crop_w, pad + crop_h], [pad, pad + crop_h]], - dtype=np.float32, - ) - transform = cv2.getPerspectiveTransform(src, dst) - inverse = cv2.getPerspectiveTransform(dst, src) - crops.append(cv2.warpPerspective(page, transform, (out_w, out_h), borderMode=cv2.BORDER_REPLICATE)) - crop_meta.append((p_idx, transform, inverse, out_w, out_h, pad)) - - tables_per_page: list[list[dict[str, Any]]] = [[] for _ in pages] - if len(crops) == 0: - return tables_per_page - - grids = self.table_predictor(crops, **kwargs) # type: ignore[misc] - # Whether the structure model makes the logical coordinates consistent with the cell geometry - postprocessor = getattr(getattr(self.table_predictor, "model", None), "postprocessor", None) - realign = getattr(postprocessor, "geometric_logic", False) - for (p_idx, transform, inverse, crop_w, crop_h, pad), grid in zip(crop_meta, grids): - if len(grid["cells"]) == 0: - continue - h, w = pages[p_idx].shape[:2] - # Cell polygons: crop-relative -> crop pixels - polys = np.stack([self._cell_polygon(cell["geometry"]) for cell in grid["cells"]]) - polys *= np.array([crop_w, crop_h], dtype=np.float32) - # The cells centered in the context margin are not part of the table - centers = polys.mean(axis=1) - keep = np.flatnonzero(np.all((centers >= pad) & (centers <= [crop_w - pad, crop_h - pad]), axis=1)) - if keep.size == 0: - continue - cells = [grid["cells"][idx] for idx in keep] - logic = np.array([[c["col_start"], c["col_end"], c["row_start"], c["row_end"]] for c in cells]) - if realign and keep.size < polys.shape[0]: - # The dropped cells made grid lines of their own: index the rows / columns of the remaining ones again - logic = _logic_from_geometry(polys[keep], logic, split_tables=False) - # The words of the region, in the crop - word_idcs, words = np.zeros(0, dtype=np.int64), np.zeros((0, 4)) - if word_boxes is not None and len(word_boxes[p_idx]) > 0: - page_words = np.asarray(word_boxes[p_idx], dtype=np.float32) - if page_words.ndim == 2: # (N, 4) straight boxes -> (N, 4, 2) polygons - page_words = page_words[:, [[0, 1], [2, 1], [2, 3], [0, 3]]] - pts = cv2.perspectiveTransform((page_words * [w, h]).reshape(1, -1, 2), transform).reshape(-1, 4, 2) - word_centers = pts.mean(axis=1) - inside = (word_centers >= pad) & (word_centers <= [crop_w - pad, crop_h - pad]) - word_idcs = np.flatnonzero(np.all(inside, axis=1)) - words = np.concatenate([pts[word_idcs].min(axis=1), pts[word_idcs].max(axis=1)], axis=1) - boxes, cell_logic, cell_words, origin = fill_table_grid( - np.concatenate([polys[keep].min(axis=1), polys[keep].max(axis=1)], axis=1), - logic[:, [2, 3, 0, 1]], # (row_start, row_end, col_start, col_end) - words, - ) - remapped_cells: list[dict[str, Any]] = [] - for (x0, y0, x1, y1), (row_start, row_end, col_start, col_end), w_idcs, source in zip( - boxes, cell_logic, cell_words, origin - ): - # crop pixels -> page pixels (inverse transform) -> page-relative - corners = np.array([[[x0, y0], [x1, y0], [x1, y1], [x0, y1]]], dtype=np.float32) - poly = cv2.perspectiveTransform(corners, inverse)[0] / np.array([w, h], dtype=np.float32) - remapped_cells.append({ - "geometry": poly.tolist(), - # The cells made from the words are not scored by the table model - "score": None if source < 0 else float(cells[source]["score"]), - "row_start": int(row_start), - "row_end": int(row_end), - "col_start": int(col_start), - "col_end": int(col_end), - "words": [int(word_idcs[idx]) for idx in w_idcs], - }) - tables_per_page[p_idx].append({ - "cells": remapped_cells, - "num_rows": int(cell_logic[:, 1].max()) + 1, - "num_cols": int(cell_logic[:, 3].max()) + 1, - }) - return tables_per_page diff --git a/doctr/models/reading_order/base.py b/doctr/models/reading_order/base.py index ff1265ece9..46207cf077 100644 --- a/doctr/models/reading_order/base.py +++ b/doctr/models/reading_order/base.py @@ -411,7 +411,7 @@ def _key_value_rows( def _column_gutter( boxes: np.ndarray, voters: np.ndarray, y_overlap: np.ndarray, y_overlap_threshold: float -) -> np.ndarray | None: +) -> float | None: """Detect a multi-column page: a vertical line separating the voters into two groups crossed by few of them Args: @@ -421,7 +421,7 @@ def _column_gutter( y_overlap_threshold: the vertical overlap above which two boxes lie on the same row Returns: - the mask of the boxes crossing the gutter for a multi-column page, None for a single-column page + the position of the gutter for a multi-column page, None for a single-column page """ if np.count_nonzero(voters) < 3: voters = np.ones(boxes.shape[0], dtype=bool) @@ -429,6 +429,7 @@ def _column_gutter( vx0, vx1, num_voters = x0[voters], x1[voters], int(np.count_nonzero(voters)) if num_voters < 3: return None + vy = (boxes[voters, 1] + boxes[voters, 3]) / 2 # Measured on the voters only, so that a wide figure does not shift the search window left_edge = float(vx0.min()) span = float(vx1.max()) - left_edge or 1.0 @@ -436,12 +437,20 @@ def _column_gutter( centers = (vx0 + vx1) / 2 lo, hi = left_edge + 0.25 * span, left_edge + 0.75 * span for split in np.unique(vx1[(vx1 >= lo) & (vx1 <= hi)]): - crossing = int(np.count_nonzero(np.minimum(vx1 - split, split - vx0) > 0.02 * span)) + across = np.minimum(vx1 - split, split - vx0) > 0.02 * span + on_left, on_right = ~across & (centers <= split), ~across & (centers > split) + if not on_left.any() or not on_right.any(): + continue + # The elements above or below the columns (e.g. a title and its authors) do not cross the gutter + sides = vy[on_left | on_right] + crossing = int(np.count_nonzero(across & (vy > sides.min()) & (vy < sides.max()))) left = int(np.count_nonzero(centers <= split)) if crossing <= tolerance and left >= 0.25 * num_voters and num_voters - left >= 0.25 * num_voters: - if _key_value_rows(boxes, voters, float(split), y_overlap, y_overlap_threshold, span): + sides = voters.copy() + sides[np.flatnonzero(voters)[across]] = False # the elements on either side of the gutter + if _key_value_rows(boxes, sides, float(split), y_overlap, y_overlap_threshold, span): return None # rows of keys and values (e.g. a receipt): read row after row - return np.minimum(x1 - split, split - x0) > 0.02 * span + return float(split) return None @@ -495,14 +504,26 @@ def _topological_order( # A multi-column page favors the continuation of the current column (cf. `_continuation`) spanning = (x1 - x0) > 0.5 * (float(x1.max() - x0.min()) or 1.0) - crossing = _column_gutter( + gutter = _column_gutter( boxes, np.ones(num_boxes, dtype=bool) if voters is None else voters, y_overlap, y_overlap_threshold ) - multi_column = crossing is not None + multi_column = gutter is not None + span = float(x1.max() - x0.min()) or 1.0 + crossing = np.zeros(num_boxes, dtype=bool) + if gutter is not None: + crossing = np.minimum(x1 - gutter, gutter - x0) > 0.02 * span + # On a multi-column page, the columns after the gutter are read once the ones before it are done, even when they + # start higher (e.g. beside a table whose caption is read with it): 1 for the elements after the gutter, else 0. + # Only for columns of several elements on both sides, not a lone element beside a column + after_gutter = np.zeros(num_boxes, dtype=int) + if gutter is not None: + after_gutter = (x0 >= gutter - 0.02 * span).astype(int) + if min(np.count_nonzero(after_gutter), np.count_nonzero(~crossing & (after_gutter == 0))) < 2: + after_gutter[:] = 0 # Column components: elements connected through horizontal overlap, except those which must not merge the columns # (page-wide or crossing the gutter). This keeps the traversal inside a column even when the direct top-to-bottom # continuation is momentarily broken by a fragmented or misaligned OCR line. - bridging = spanning | crossing if multi_column else spanning + bridging = spanning | crossing col_edges = np.triu(x_linked, 1) # each pair once col_edges &= ~bridging[:, None] col_edges &= ~bridging[None, :] @@ -530,8 +551,8 @@ def _continuation(last: int, ready: np.ndarray) -> np.ndarray: candidates = np.flatnonzero(~emitted) else: candidates = _continuation(order[-1], ready) if order else ready - # Topmost first, then leftmost (in canonical space) - next_idx = int(candidates[np.lexsort((x0[candidates], y0[candidates]))[0]]) + # Topmost first, then leftmost (in canonical space), the columns before the gutter first + next_idx = int(candidates[np.lexsort((x0[candidates], y0[candidates], after_gutter[candidates]))[0]]) order.append(next_idx) emitted[next_idx] = True in_degree = in_degree - edges[next_idx] diff --git a/doctr/models/table_structure/grid.py b/doctr/models/table_structure/grid.py index e78971bfea..eb6ad11c68 100644 --- a/doctr/models/table_structure/grid.py +++ b/doctr/models/table_structure/grid.py @@ -5,14 +5,26 @@ """Assemble the cells predicted by a table structure model and the words of the table into a consistent grid. -Everything here works in the frame of the (rectified, upright) table image, where the cells and the words are +`fill_table_grid` works in the frame of the (rectified, upright) table image, where the cells and the words are axis-aligned boxes `(x0, y0, x1, y1)`, and the logical coordinates of the cells are `(row_start, row_end, col_start, -col_end)` (0-indexed, inclusive). +col_end)` (0-indexed, inclusive). `assemble_page_table` applies it to a table given in page coordinates. """ +from typing import Any + import numpy as np -__all__ = ["fill_table_grid"] +__all__ = ["fill_table_grid", "assemble_page_table"] + + +def as_polygon(geometry: Any) -> np.ndarray: + """A straight box `(x0, y0, x1, y1)` / `((x0, y0), (x1, y1))`, or a polygon ordered (TL, TR, BR, BL) along its + reading direction, as a (4, 2) float32 polygon""" + raw = np.asarray(geometry, dtype=np.float32) + if raw.size == 4: + x0, y0, x1, y1 = raw.ravel() + return np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) + return raw.reshape(-1, 2).copy() def _grid_lines(start: np.ndarray, end: np.ndarray, lo: np.ndarray, hi: np.ndarray, size: int) -> np.ndarray: @@ -346,3 +358,71 @@ def fill_table_grid( grid.localize() grid.cell_words = [_reading_order(w, idcs, line_height) if idcs else [] for idcs in grid.cell_words] return np.asarray(grid.boxes), np.asarray(grid.logic, dtype=np.int64), grid.cell_words, grid.origin + + +def _rotate(points: np.ndarray, angle: float, pivot: np.ndarray) -> np.ndarray: + """Rotate (..., 2) points by `angle` radians around `pivot`""" + cos_a, sin_a = np.cos(angle), np.sin(angle) + x, y = points[..., 0] - pivot[0], points[..., 1] - pivot[1] + return np.stack([pivot[0] + x * cos_a - y * sin_a, pivot[1] + x * sin_a + y * cos_a], axis=-1) + + +def assemble_page_table( + cells: list[dict[str, Any]], word_boxes: np.ndarray, page_shape: tuple[int, int], taken: np.ndarray +) -> tuple[list[dict[str, Any]], int, int]: + """Assemble a table given in page coordinates with the words of the page. + + The cells listing their words (`"words"`, cf. `TablePredictor.predict_regions`) keep them. Otherwise, the words + lying in the outline of the cells are assigned to the grid (cf. `fill_table_grid`), in the frame of the table rows. + + Args: + cells: the cells `{"geometry", "score", "row_start", "row_end", "col_start", "col_end"}` of the table, with + page-relative geometries (straight boxes or polygons) + word_boxes: the (N, 4) or (N, 4, 2) page-relative word boxes + page_shape: the (height, width) of the page + taken: the (N,) mask of the words already assigned to a table, updated in place + + Returns: + the cells with local logical coordinates, a page-relative (4, 2) polygon geometry, and the indices of their + words in reading order (`"words"`), and the numbers of rows and columns of the table + """ + scale = np.array([page_shape[1], page_shape[0]], dtype=np.float64) + polys = np.stack([as_polygon(cell["geometry"]) for cell in cells]) * scale + logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells], dtype=np.int64) + if all("words" in cell for cell in cells): + cell_words = [[int(i) for i in cell["words"] if not taken[i]] for cell in cells] + origin = list(range(len(cells))) + logic -= [logic[:, 0].min()] * 2 + [logic[:, 2].min()] * 2 + else: + # The frame of the table rows + top_edges = polys[:, 1] - polys[:, 0] + angle, pivot = float(np.median(np.arctan2(top_edges[:, 1], top_edges[:, 0]))), polys.reshape(-1, 2).mean(0) + upright = _rotate(polys, -angle, pivot) + boxes = np.concatenate([upright.min(axis=1), upright.max(axis=1)], axis=1) + # The free words in the outline of the cells + free, word_coords = np.flatnonzero(~taken), np.zeros((0, 4)) + if free.size: + words = _rotate(np.stack([as_polygon(box) for box in word_boxes[free]]) * scale, -angle, pivot) + centers = words.mean(axis=1) + inside = np.all((centers >= boxes[:, :2].min(axis=0)) & (centers <= boxes[:, 2:].max(axis=0)), axis=1) + free, words = free[inside], words[inside] + word_coords = np.concatenate([words.min(axis=1), words.max(axis=1)], axis=1) + boxes, logic, local_words, origin = fill_table_grid(boxes, logic, word_coords) + cell_words = [[int(free[j]) for j in idcs] for idcs in local_words] + corners = boxes[:, [[0, 1], [2, 1], [2, 3], [0, 3]]] + polys = _rotate(corners, angle, pivot) + for idcs in cell_words: + taken[idcs] = True + out = [] + for poly, (rs, re, cs, ce), idcs, source in zip(polys / scale, logic, cell_words, origin): + score = None if source < 0 else cells[source]["score"] + out.append({ + "geometry": poly, + "score": None if score is None else float(score), + "row_start": int(rs), + "row_end": int(re), + "col_start": int(cs), + "col_end": int(ce), + "words": idcs, + }) + return out, int(logic[:, 1].max()) + 1, int(logic[:, 3].max()) + 1 diff --git a/doctr/models/table_structure/predictor/pytorch.py b/doctr/models/table_structure/predictor/pytorch.py index 26a54b14a3..8a76deeea9 100644 --- a/doctr/models/table_structure/predictor/pytorch.py +++ b/doctr/models/table_structure/predictor/pytorch.py @@ -6,6 +6,7 @@ from collections.abc import Iterable, Sequence from typing import Any +import cv2 import numpy as np import torch from torch import nn @@ -15,9 +16,32 @@ from doctr.models.preprocessor import PreProcessor from doctr.models.utils import set_device_and_dtype +from ..grid import as_polygon, fill_table_grid +from ..tablecenternet.base import _logic_from_geometry + __all__ = ["TablePredictor"] +def _crop_region(page: np.ndarray, region: Any) -> tuple[np.ndarray, np.ndarray, int] | None: + """Rectify a (page-relative) table region to an upright crop, with a margin of page context around it + + Returns: + the crop, the perspective transform from page pixels to crop pixels, and the margin (pixels), None for a + degenerate region + """ + h, w = page.shape[:2] + src = as_polygon(region) * np.array([w, h], dtype=np.float32) + crop_w = int(round(max(np.linalg.norm(src[1] - src[0]), np.linalg.norm(src[2] - src[3])))) + crop_h = int(round(max(np.linalg.norm(src[3] - src[0]), np.linalg.norm(src[2] - src[1])))) + if crop_w < 2 or crop_h < 2: + return None + pad = max(20, round(0.01 * max(crop_w, crop_h))) + dst = np.array([[pad, pad], [pad + crop_w, pad], [pad + crop_w, pad + crop_h], [pad, pad + crop_h]], np.float32) + transform = cv2.getPerspectiveTransform(src, dst) + crop = cv2.warpPerspective(page, transform, (crop_w + 2 * pad, crop_h + 2 * pad), borderMode=cv2.BORDER_REPLICATE) + return crop, transform, pad + + class TablePredictor(nn.Module): """Implements an object able to recognize the cell structure of tables in a document. @@ -106,3 +130,105 @@ def _predict(self, pages: list[np.ndarray], **kwargs: Any) -> list[dict[str, Any # logical coordinates are 0-indexed, so the table size is the largest index + 1 (0 if no cells) results.append({"cells": cells, "num_rows": max_row + 1, "num_cols": max_col + 1}) return results + + @torch.inference_mode() + def predict_regions( + self, + pages: list[np.ndarray], + regions: list[dict[str, Any] | None], + word_boxes: list[np.ndarray] | None = None, + class_name: str = "Table", + **kwargs: Any, + ) -> list[list[dict[str, Any]]]: + """Recognize the tables a layout model located on pages, and assign the words of the pages to their cells + + Each table region is rectified to an upright crop (so straight boxes and rotated polygons are both handled), + with a margin of page context: the cells centered in this margin belong to the neighbouring content and are + dropped. The words of the page lying in the region are assigned to the cells and complete the grid (cf. + :func:`~doctr.models.table_structure.grid.fill_table_grid`), then the cells are mapped back to the page. + + Args: + pages: the page images + regions: the per-page layout predictions `{"class_names", "boxes", "scores"}` (None for none) + word_boxes: the per-page word boxes, of shape (N, 4) or (N, 4, 2) in relative coordinates + class_name: the layout class of the table regions + **kwargs: keyword arguments forwarded to the model + + Returns: + a per-page list of tables `{"cells": [...], "num_rows": int, "num_cols": int}` in page-relative + coordinates, one per table region, each cell listing the indices of its words in the page (`"words"`) in + reading order + """ + crops, meta = [], [] + for p_idx, (page, region) in enumerate(zip(pages, regions)): + region = region or {} + for cls_name, box in zip(region.get("class_names", []), region.get("boxes", [])): + cropped = _crop_region(page, box) if cls_name == class_name else None + if cropped is not None: + crops.append(cropped[0]) + meta.append((p_idx, cropped[1], cropped[2])) + tables: list[list[dict[str, Any]]] = [[] for _ in pages] + for (p_idx, transform, pad), crop, grid in zip(meta, crops, self(crops, **kwargs) if crops else []): + words = None if word_boxes is None else word_boxes[p_idx] + table = self._region_table(grid, crop.shape[:2], pages[p_idx].shape[:2], transform, pad, words) + if table is not None: + tables[p_idx].append(table) + return tables + + def _region_table( + self, + grid: dict[str, Any], + crop_shape: tuple[int, int], + page_shape: tuple[int, int], + transform: np.ndarray, + pad: int, + word_boxes: np.ndarray | None, + ) -> dict[str, Any] | None: + """Assemble the table of a region crop with the words of the page, in page-relative coordinates""" + (crop_h, crop_w), (h, w) = crop_shape, page_shape + if len(grid["cells"]) == 0: + return None + + def in_region(points: np.ndarray) -> np.ndarray: + return np.flatnonzero(np.all((points >= pad) & (points <= [crop_w - pad, crop_h - pad]), axis=1)) + + # Cells in crop pixels, without the ones of the context margin + polys = np.stack([as_polygon(cell["geometry"]) for cell in grid["cells"]]) * [crop_w, crop_h] + keep = in_region(polys.mean(axis=1)) + if keep.size == 0: + return None + cells = [grid["cells"][idx] for idx in keep] + logic = np.array([[c["col_start"], c["col_end"], c["row_start"], c["row_end"]] for c in cells]) + if keep.size < polys.shape[0] and getattr(getattr(self.model, "postprocessor", None), "geometric_logic", 0): + # The dropped cells made grid lines of their own: index the rows / columns of the remaining ones again + logic = _logic_from_geometry(polys[keep], logic, split_tables=False) + # The words of the region, in crop pixels + word_idcs, words = np.zeros(0, dtype=np.int64), np.zeros((0, 4)) + if word_boxes is not None and len(word_boxes) > 0: + pts = np.stack([as_polygon(box) for box in word_boxes]) * np.array([w, h], dtype=np.float32) + pts = cv2.perspectiveTransform(pts.reshape(1, -1, 2), transform).reshape(-1, 4, 2) + word_idcs = in_region(pts.mean(axis=1)) + words = np.concatenate([pts[word_idcs].min(axis=1), pts[word_idcs].max(axis=1)], axis=1) + boxes, cell_logic, cell_words, origin = fill_table_grid( + np.concatenate([polys[keep].min(axis=1), polys[keep].max(axis=1)], axis=1), logic[:, [2, 3, 0, 1]], words + ) + # Crop pixels -> page-relative coordinates + corners = boxes[:, [[0, 1], [2, 1], [2, 3], [0, 3]]].reshape(1, -1, 2).astype(np.float32) + page_polys = cv2.perspectiveTransform(corners, np.linalg.inv(transform)).reshape(-1, 4, 2) / [w, h] + return { + "cells": [ + { + "geometry": poly.tolist(), + # The cells made from the words are not scored by the table model + "score": None if source < 0 else float(cells[source]["score"]), + "row_start": int(rs), + "row_end": int(re), + "col_start": int(cs), + "col_end": int(ce), + "words": [int(word_idcs[idx]) for idx in w_idcs], + } + for poly, (rs, re, cs, ce), w_idcs, source in zip(page_polys, cell_logic, cell_words, origin) + ], + "num_rows": int(cell_logic[:, 1].max()) + 1, + "num_cols": int(cell_logic[:, 3].max()) + 1, + } diff --git a/tests/common/test_models_builder.py b/tests/common/test_models_builder.py index 94769947af..1d15bf01f6 100644 --- a/tests/common/test_models_builder.py +++ b/tests/common/test_models_builder.py @@ -465,29 +465,6 @@ def test_resolve_lines(input_boxes, shape, lines): assert doc_builder._resolve_lines(np.asarray(input_boxes), shape) == lines -def test_table_geometry_helpers(): - polys = np.array( - [ - [[0.1, 0.1], [0.4, 0.1], [0.4, 0.3], [0.1, 0.3]], # axis-aligned quad - [[0.5, 0.5], [0.8, 0.6], [0.7, 0.9], [0.45, 0.8]], # rotated quad - ], - dtype=np.float32, - ) - points = np.array([[0.2, 0.2], [0.6, 0.7], [0.95, 0.95], [0.05, 0.05]], dtype=np.float32) - inside = builder.DocumentBuilder._points_in_polygons(points, polys) - assert inside.tolist() == [[True, False], [False, True], [False, False], [False, False]] - # empty inputs yield empty masks instead of raising - assert builder.DocumentBuilder._points_in_polygons(np.zeros((0, 2)), polys).shape == (0, 2) - assert builder.DocumentBuilder._points_in_polygons(points, np.zeros((0, 4, 2))).shape == (4, 0) - # flat straight box (xmin, ymin, xmax, ymax) -> (4, 2) polygon - poly = builder.DocumentBuilder._as_cell_polygon([0.1, 0.2, 0.5, 0.6]) - assert poly.shape == (4, 2) - assert np.allclose(poly, [[0.1, 0.2], [0.5, 0.2], [0.5, 0.6], [0.1, 0.6]]) - # (4, 2) polygons pass through unchanged - quad = np.array([[0.1, 0.2], [0.5, 0.25], [0.45, 0.6], [0.05, 0.55]], dtype=np.float32) - assert np.allclose(builder.DocumentBuilder._as_cell_polygon(quad), quad) - - def _table_cell(x0, y0, x1, y1, rs, re, cs, ce, score=0.9): return { "geometry": [[x0, y0], [x1, y0], [x1, y1], [x0, y1]], @@ -573,17 +550,18 @@ def test_documentbuilder_tables(): _table_cell(0.10, 0.10, 0.25, 0.20, 0, 0, 0, 0, score=0.8), _table_cell(0.28, 0.10, 0.45, 0.20, 0, 0, 1, 1, score=0.6), _table_cell(0.10, 0.22, 0.25, 0.32, 1, 1, 0, 0, score=0.4), # no word inside this cell + _table_cell(0.28, 0.22, 0.45, 0.32, 1, 1, 1, 1, score=0.5), ], "num_rows": 2, "num_cols": 2, } - page = doc_builder(*_table_words("Name", "Age", confidences=[0.95, 0.85]), tables=[table]).pages[0] + page = doc_builder(*_table_words("Name", "Age", "30", confidences=[0.95, 0.85, 0.75]), tables=[table]).pages[0] cells = page.tables[0].cells - assert [cell.value for cell in cells] == ["Name", "Age", ""] - assert [cell.objectness_score for cell in cells] == [0.8, 0.6, 0.4] - assert [cell.confidence for cell in cells] == pytest.approx([0.95, 0.85, 0.4]) - assert page.tables[0].confidence == pytest.approx(0.6) - assert [cell["objectness_score"] for cell in page.export()["tables"][0]["cells"]] == [0.8, 0.6, 0.4] + assert [cell.value for cell in cells] == ["Name", "Age", "", "30"] + assert [cell.objectness_score for cell in cells] == [0.8, 0.6, 0.4, 0.5] + assert [cell.confidence for cell in cells] == pytest.approx([0.95, 0.85, 0.4, 0.75]) + assert page.tables[0].confidence == pytest.approx(0.575) + assert [cell["objectness_score"] for cell in page.export()["tables"][0]["cells"]] == [0.8, 0.6, 0.4, 0.5] # Multi-word cells are read line by line, left to right words = ["world", "hello", "second", "line", "solo"] diff --git a/tests/common/test_models_reading_order.py b/tests/common/test_models_reading_order.py index bc458397a8..336e087c93 100644 --- a/tests/common/test_models_reading_order.py +++ b/tests/common/test_models_reading_order.py @@ -346,6 +346,25 @@ def test_layout_labels(label, normalized, role): list(range(7)), id="header-over-columns", ), + pytest.param( + # A centered title and its authors above 2 columns (e.g. the first page of a paper): more lines crossing + # the gutter than its tolerance, but none level with the columns + _column(0.15, 0.85, 0.02, 0.03, 2, 0.02) + + _column(0.35, 0.65, 0.09, 0.025, 4, 0.015) + + _column(0.05, 0.45, 0.2, 0.02, 40, 0.015) + + _column(0.55, 0.95, 0.2, 0.02, 40, 0.015), + {}, + list(range(86)), + id="title-and-authors-over-columns", + ), + pytest.param( + # The right column starts higher than the left one (e.g. beside a table whose caption is read with it): + # the left column is read first + _column(0.05, 0.45, 0.2, 0.03, 10, 0.02) + _column(0.55, 0.95, 0.15, 0.03, 10, 0.02), + {}, + list(range(20)), + id="right-column-starting-higher", + ), pytest.param( SPLIT_CAPTION, {"labels": SPLIT_CAPTION_LABELS}, diff --git a/tests/common/test_models_table_grid.py b/tests/common/test_models_table_grid.py index ba40040bf2..ee27b36721 100644 --- a/tests/common/test_models_table_grid.py +++ b/tests/common/test_models_table_grid.py @@ -1,6 +1,6 @@ import numpy as np -from doctr.models.table_structure.grid import _axis_slots, _grid_lines, _group, fill_table_grid +from doctr.models.table_structure.grid import _axis_slots, _grid_lines, _group, as_polygon, fill_table_grid def _grid(xs, ys): @@ -28,6 +28,11 @@ def _table(cells, logic, words): def test_table_grid_helpers(): + # Straight boxes (flat or 2-point) and polygons, as (4, 2) polygons + expected = [[0.1, 0.2], [0.5, 0.2], [0.5, 0.6], [0.1, 0.6]] + np.testing.assert_allclose(as_polygon([0.1, 0.2, 0.5, 0.6]), expected) + np.testing.assert_allclose(as_polygon(((0.1, 0.2), (0.5, 0.6))), expected) + np.testing.assert_allclose(as_polygon(expected), expected) # Grid lines at the median of the cell edges, interpolated when no edge lies on them lines = _grid_lines(np.array([0, 2]), np.array([0, 2]), np.array([0.0, 20.0]), np.array([10.0, 30.0]), 3) np.testing.assert_allclose(lines, [0, 10, 20, 30]) diff --git a/tests/pytorch/test_models_table_structure_pt.py b/tests/pytorch/test_models_table_structure_pt.py index a12e9dd07b..f2cc9f6ac6 100644 --- a/tests/pytorch/test_models_table_structure_pt.py +++ b/tests/pytorch/test_models_table_structure_pt.py @@ -186,3 +186,52 @@ def test_torch_compiled_models(arch_name, mock_table_image): assert np.allclose(cell["geometry"], compiled_cell["geometry"], atol=1e-4) assert np.isclose(cell["score"], compiled_cell["score"], atol=1e-4) assert all(cell[k] == compiled_cell[k] for k in ("row_start", "row_end", "col_start", "col_end")) + + +def test_table_predictor_regions(monkeypatch): + predictor = table_structure.zoo.table_predictor("tablecenternet", pretrained=False) + crops = [] + + def fake_forward(images, **kwargs): + # A fixed grid (crop-relative cells) for every crop: 2 cells, and 1 of the context margin, above the region + crops.extend(images) + h, w = images[0].shape[:2] + pad = 20 # the context margin of a region smaller than 2000 px + + def cell(x0, y0, x1, y1, r, c): + box = [(pad + x0) / w, (pad + y0) / h, (pad + x1) / w, (pad + y1) / h] + return {"geometry": box, "score": 0.9, "row_start": r, "row_end": r, "col_start": c, "col_end": c} + + cells = [cell(0, 0, 100, 30, 1, 0), cell(100, 0, 200, 30, 1, 1), cell(0, -18, 200, -4, 0, 0)] + return [{"cells": cells, "num_rows": 2, "num_cols": 2} for _ in images] + + monkeypatch.setattr(predictor, "forward", fake_forward) + page = np.full((400, 600, 3), 255, dtype=np.uint8) + x0, y0, x1 = 0.1, 0.25, 0.1 + 200 / 600 # a 200 x 50 px table region, its cells cover its top 30 px + regions = [{"class_names": ["Text", "Table"], "boxes": np.array([[0, 0, 0.1, 0.1], [x0, y0, x1, 0.375]])}, None] + # Words in each cell, one in the region below the cells, and one outside of the region + words = np.array([ + [0.12, 0.27, 0.15, 0.29], + [0.30, 0.27, 0.33, 0.29], + [0.12, 0.345, 0.15, 0.365], + [0.12, 0.80, 0.15, 0.82], + ]) + tables = predictor.predict_regions([page, page], regions, [words, words[:0]]) + # Only the table region is cropped, with a context margin + assert len(crops) == 1 and crops[0].shape[:2] == (90, 240) + assert len(tables) == 2 and len(tables[0]) == 1 and tables[1] == [] + table = tables[0][0] + # The cell of the context margin is dropped, the word below the cells makes a new row + assert (table["num_rows"], table["num_cols"]) == (2, 2) + cells = sorted(table["cells"], key=lambda c: (c["row_start"], c["col_start"])) + assert [(c["row_start"], c["col_start"], c["words"], c["score"]) for c in cells] == [ + (0, 0, [0], 0.9), + (0, 1, [1], 0.9), + (1, 0, [2], None), + ] + # The cells are mapped back to page-relative coordinates + expected = [[x0, y0], [x0 + 100 / 600, y0], [x0 + 100 / 600, 0.325], [x0, 0.325]] + np.testing.assert_allclose(cells[0]["geometry"], expected, atol=1e-5) + # Without words, the cells are empty + tables = predictor.predict_regions([page], regions[:1]) + assert all(cell["words"] == [] for cell in tables[0][0]["cells"]) diff --git a/tests/pytorch/test_models_zoo_pt.py b/tests/pytorch/test_models_zoo_pt.py index ed9bf2fbac..8af57e0541 100644 --- a/tests/pytorch/test_models_zoo_pt.py +++ b/tests/pytorch/test_models_zoo_pt.py @@ -408,61 +408,6 @@ def test_ocrpredictor_tables(mock_pdf, mock_vocab): assert exported["tables"] == [t.export() for t in page.tables] -def test_ocrpredictor_tables_from_regions(): - class _FakeTablePredictor: - """Returns a fixed grid (crop-relative cells) for every crop, with a cell in the context margin""" - - model = None - - def __init__(self): - self.crops = [] - - def __call__(self, crops, **kwargs): - self.crops = crops - h, w = crops[0].shape[:2] - pad = 20 # the context margin of a region smaller than 2000 px - - def cell(x0, y0, x1, y1, r, c): - box = [(pad + x0) / w, (pad + y0) / h, (pad + x1) / w, (pad + y1) / h] - return {"geometry": box, "score": 0.9, "row_start": r, "row_end": r, "col_start": c, "col_end": c} - - # 2 cells of the table, and a cell of the neighbouring content, above the region - cells = [cell(0, 0, 100, 30, 1, 0), cell(100, 0, 200, 30, 1, 1), cell(0, -18, 200, -4, 0, 0)] - return [{"cells": cells, "num_rows": 2, "num_cols": 2}] - - fake = type("FakeOCRPredictor", (), {})() - fake.table_predictor = _FakeTablePredictor() - fake.table_class_name = "Table" - fake._cell_polygon = OCRPredictor._cell_polygon - page = np.full((400, 600, 3), 255, dtype=np.uint8) - x0, y0, x1, y1 = 0.1, 0.25, 0.1 + 200 / 600, 0.375 # the cells cover the top 30 px of its 50 px - region = {"class_names": ["Text", "Table"], "boxes": np.array([[0, 0, 0.1, 0.1], [x0, y0, x1, y1]])} - # Words in each cell, one in the region below the cells, and one outside of the region - words = np.array([ - [0.12, 0.27, 0.15, 0.29], - [0.30, 0.27, 0.33, 0.29], - [0.12, 0.345, 0.15, 0.365], - [0.12, 0.80, 0.15, 0.82], - ]) - tables = OCRPredictor._tables_from_regions(fake, [page], [region], [words]) - # Only the table region is cropped, with a context margin - assert len(fake.table_predictor.crops) == 1 and fake.table_predictor.crops[0].shape[:2] == (90, 240) - assert len(tables) == 1 and len(tables[0]) == 1 - table = tables[0][0] - # The cell of the context margin is dropped, the word below the cells makes a new row - assert (table["num_rows"], table["num_cols"]) == (2, 2) - cells = sorted(table["cells"], key=lambda c: (c["row_start"], c["col_start"])) - assert [(c["row_start"], c["col_start"], c["words"], c["score"]) for c in cells] == [ - (0, 0, [0], 0.9), - (0, 1, [1], 0.9), - (1, 0, [2], None), - ] - # The cells are mapped back to page-relative coordinates - np.testing.assert_allclose( - cells[0]["geometry"], [[x0, y0], [x0 + 100 / 600, y0], [x0 + 100 / 600, 0.325], [x0, 0.325]], atol=1e-5 - ) - - def test_ocrpredictor_tables_factory(): # The factory exposes a single `detect_tables` flag, which also enables the layout model predictor = models.ocr_predictor("db_mobilenet_v3_large", "crnn_vgg16_bn", pretrained=False, detect_tables=True) From 1ec225d2f061869dd824ef92d7c7a8db8c9c8831 Mon Sep 17 00:00:00 2001 From: felix Date: Tue, 10 Nov 2026 12:08:16 +0100 Subject: [PATCH 4/8] Update --- docs/source/api/models.md | 3 + docs/source/user_guide/layout_and_tables.md | 19 +- doctr/models/reading_order/base.py | 28 ++- doctr/models/table_structure/grid.py | 88 ++++----- .../table_structure/predictor/pytorch.py | 21 ++- tests/pytorch/test_models_table_grid.py | 169 ++++++++++++++++++ 6 files changed, 260 insertions(+), 68 deletions(-) create mode 100644 tests/pytorch/test_models_table_grid.py diff --git a/docs/source/api/models.md b/docs/source/api/models.md index 091f0efff6..ef43708046 100644 --- a/docs/source/api/models.md +++ b/docs/source/api/models.md @@ -88,6 +88,9 @@ directly. See the [model zoo](../models/index.md) for benchmarks. ```{eval-rst} .. autofunction:: doctr.models.table_structure.table_predictor +.. autoclass:: doctr.models.table_structure.predictor.TablePredictor + :members: predict_regions + .. autofunction:: doctr.models.table_structure.tablecenternet ``` diff --git a/docs/source/user_guide/layout_and_tables.md b/docs/source/user_guide/layout_and_tables.md index 5ad7ece8ff..53d2fb6fe1 100644 --- a/docs/source/user_guide/layout_and_tables.md +++ b/docs/source/user_guide/layout_and_tables.md @@ -68,8 +68,9 @@ otherwise the regions cannot be located and nothing is masked. The CLI enables i ## Table structure recognition With `detect_tables=True`, each region the layout model labels as a `Table` is cropped and passed to a table -structure model (`tablecenternet`), which predicts its cells and the rows and columns they span. The words falling -inside a cell are regrouped into a {class}`~doctr.io.Table`: +structure model (`tablecenternet`), which predicts its cells and the rows and columns they span. Every word inside the +table region is regrouped into a {class}`~doctr.io.Table`: the words outside of the predicted cells (e.g. a missed +header row or a row of totals) complete the grid, and the rows and columns follow the text lines of the cells: ```python from doctr.io import DocumentFile @@ -135,7 +136,7 @@ for cell in table.cells: | `row_span`, `col_span` | Number of rows / columns spanned | | `geometry` | Position of the cell, relative to the page | | `confidence` | Mean recognition confidence of the words of the cell | -| `objectness_score` | Score of the cell predicted by the table model | +| `objectness_score` | Table model score of the cell (`None` for a cell made from words) | ## Standalone predictors @@ -171,6 +172,18 @@ out[0]["cells"][0] # {'geometry': [[x1, y1], ..., [x4, y4]], # relative to the table image # 'score': 0.42, 'row_start': 0, 'row_end': 0, 'col_start': 0, 'col_end': 0} ``` + +With the layout regions of whole pages and their words (e.g. from your own OCR), the table regions are cropped and +the words assigned to the cells: + +```python +from doctr.models import layout_predictor + +regions = layout_predictor("lw_detr_s", pretrained=True)(pages) +tables = predictor.predict_regions(pages, regions, word_boxes) # one (N, 4) array of relative boxes per page + +tables[0][0]["cells"][0]["words"] # indices of the words of the cell in `word_boxes[0]`, in reading order +``` ::: :::: diff --git a/doctr/models/reading_order/base.py b/doctr/models/reading_order/base.py index 46207cf077..ccf38ecc22 100644 --- a/doctr/models/reading_order/base.py +++ b/doctr/models/reading_order/base.py @@ -442,13 +442,13 @@ def _column_gutter( if not on_left.any() or not on_right.any(): continue # The elements above or below the columns (e.g. a title and its authors) do not cross the gutter - sides = vy[on_left | on_right] - crossing = int(np.count_nonzero(across & (vy > sides.min()) & (vy < sides.max()))) + columns_y = vy[on_left | on_right] + crossing = int(np.count_nonzero(across & (vy > columns_y.min()) & (vy < columns_y.max()))) left = int(np.count_nonzero(centers <= split)) if crossing <= tolerance and left >= 0.25 * num_voters and num_voters - left >= 0.25 * num_voters: - sides = voters.copy() - sides[np.flatnonzero(voters)[across]] = False # the elements on either side of the gutter - if _key_value_rows(boxes, sides, float(split), y_overlap, y_overlap_threshold, span): + beside = voters.copy() + beside[np.flatnonzero(voters)[across]] = False # the elements on either side of the gutter + if _key_value_rows(boxes, beside, float(split), y_overlap, y_overlap_threshold, span): return None # rows of keys and values (e.g. a receipt): read row after row return float(split) return None @@ -503,23 +503,21 @@ def _topological_order( np.fill_diagonal(edges, False) # A multi-column page favors the continuation of the current column (cf. `_continuation`) - spanning = (x1 - x0) > 0.5 * (float(x1.max() - x0.min()) or 1.0) + span = float(x1.max() - x0.min()) or 1.0 + spanning = (x1 - x0) > 0.5 * span gutter = _column_gutter( boxes, np.ones(num_boxes, dtype=bool) if voters is None else voters, y_overlap, y_overlap_threshold ) multi_column = gutter is not None - span = float(x1.max() - x0.min()) or 1.0 crossing = np.zeros(num_boxes, dtype=bool) - if gutter is not None: - crossing = np.minimum(x1 - gutter, gutter - x0) > 0.02 * span - # On a multi-column page, the columns after the gutter are read once the ones before it are done, even when they - # start higher (e.g. beside a table whose caption is read with it): 1 for the elements after the gutter, else 0. - # Only for columns of several elements on both sides, not a lone element beside a column + # 1 for the columns after the gutter: they are read once the ones before it are done, even when they start higher + # (e.g. beside a table whose caption is read with it). Only for columns of several elements, not a lone element after_gutter = np.zeros(num_boxes, dtype=int) if gutter is not None: - after_gutter = (x0 >= gutter - 0.02 * span).astype(int) - if min(np.count_nonzero(after_gutter), np.count_nonzero(~crossing & (after_gutter == 0))) < 2: - after_gutter[:] = 0 + crossing = np.minimum(x1 - gutter, gutter - x0) > 0.02 * span + after = x0 >= gutter - 0.02 * span + if np.count_nonzero(after) >= 2 and np.count_nonzero(~after & ~crossing) >= 2: + after_gutter = after.astype(int) # Column components: elements connected through horizontal overlap, except those which must not merge the columns # (page-wide or crossing the gutter). This keeps the traversal inside a column even when the direct top-to-bottom # continuation is momentarily broken by a fragmented or misaligned OCR line. diff --git a/doctr/models/table_structure/grid.py b/doctr/models/table_structure/grid.py index eb6ad11c68..3571a14c1b 100644 --- a/doctr/models/table_structure/grid.py +++ b/doctr/models/table_structure/grid.py @@ -17,14 +17,15 @@ __all__ = ["fill_table_grid", "assemble_page_table"] -def as_polygon(geometry: Any) -> np.ndarray: - """A straight box `(x0, y0, x1, y1)` / `((x0, y0), (x1, y1))`, or a polygon ordered (TL, TR, BR, BL) along its - reading direction, as a (4, 2) float32 polygon""" - raw = np.asarray(geometry, dtype=np.float32) - if raw.size == 4: - x0, y0, x1, y1 = raw.ravel() - return np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], dtype=np.float32) - return raw.reshape(-1, 2).copy() +def as_polygons(geometries: Any) -> np.ndarray: + """Straight boxes `(x0, y0, x1, y1)` / `((x0, y0), (x1, y1))`, or polygons ordered (TL, TR, BR, BL) along their + reading direction, as (N, 4, 2) float32 polygons""" + flat = np.asarray(geometries, dtype=np.float32) + flat = flat.reshape(len(flat), -1) if flat.size else np.zeros((0, 4), dtype=np.float32) + if flat.shape[1] == 4: + x0, y0, x1, y1 = flat.T + return np.stack([np.stack(pt, axis=-1) for pt in ((x0, y0), (x1, y0), (x1, y1), (x0, y1))], axis=1) + return flat.reshape(-1, 4, 2) def _grid_lines(start: np.ndarray, end: np.ndarray, lo: np.ndarray, hi: np.ndarray, size: int) -> np.ndarray: @@ -41,7 +42,8 @@ def _grid_lines(start: np.ndarray, end: np.ndarray, lo: np.ndarray, hi: np.ndarr def _group(values: np.ndarray, tol: float) -> np.ndarray: """Group 1D positions closer than `tol` to the mean of their group: the group index of each value, in order""" groups = np.zeros(values.size, dtype=np.int64) - group, members = 0, list[int]() + group = 0 + members: list[int] = [] # the values of the current group for j in np.argsort(values, kind="stable"): if members and values[j] - np.mean(values[members]) >= tol: group, members = group + 1, [] @@ -62,11 +64,12 @@ def _axis_slots(lo: np.ndarray, hi: np.ndarray, lines: np.ndarray, line_tol: flo continue if line_tol is not None: groups = _group(center[members], line_tol) - else: # chain the overlapping extents - order = members[np.argsort(lo[members], kind="stable")] - breaks = lo[order][1:] > np.maximum.accumulate(hi[order])[:-1] - groups = np.empty(members.size, dtype=np.int64) - groups[np.searchsorted(members, order)] = np.concatenate([[0], np.cumsum(breaks)]) + else: # chain the overlapping extents, from left to right + groups, group, reach = np.zeros(members.size, dtype=np.int64), -1, -np.inf + for j in np.argsort(lo[members], kind="stable"): + if lo[members[j]] > reach: # starts after the extents of the current group + group += 1 + groups[j], reach = group, max(reach, hi[members[j]]) idx[members] = groups - (groups.max() + 1) if before else size + groups return idx @@ -133,15 +136,16 @@ def _complete(grid: _Grid, leftover: np.ndarray, line_height: float) -> None: else: new_cells.setdefault((slot[0], slot[0], slot[1], slot[1]), []).append(int(j)) for r in np.unique(rows[~in_grid]): + # A new row, from left to right: a word starts a new cell in another column, unless it continues a sentence members = np.flatnonzero(rows == r) - members = members[np.argsort(words[members, 0], kind="stable")] - group = [int(members[0])] - for prev, j in zip(members[:-1], members[1:]): - if cols[j] != cols[group[-1]] and words[j, 0] - words[prev, 2] >= line_height / 2: - new_cells[(int(r), int(r), int(cols[group].min()), int(cols[group].max()))] = group - group = [] - group.append(int(j)) - new_cells[(int(r), int(r), int(cols[group].min()), int(cols[group].max()))] = group + groups: list[list[int]] = [] + for j in members[np.argsort(words[members, 0], kind="stable")]: + prev = groups[-1][-1] if groups else None + if prev is None or (cols[j] != cols[prev] and words[j, 0] - words[prev, 2] >= line_height / 2): + groups.append([]) + groups[-1].append(int(j)) + for group in groups: + new_cells.setdefault((int(r), int(r), int(cols[group].min()), int(cols[group].max())), []).extend(group) def band(lines: np.ndarray, start: int, end: int) -> tuple[float, float]: # The extent of the grid lines from index start to end, unbounded beyond the grid @@ -279,33 +283,35 @@ def split(c_idx: int, line_of: np.ndarray, spans: list[tuple[int, int]]) -> None present = np.unique(lines) split(i, np.searchsorted(present, lines), [(r + k, r + k) for k in present]) + def level_with_row(c_idx: int, row: int, lo: float, hi: float) -> bool: + # Whether a text line, from lo to hi, is level with the text of another cell of the row + return any( + j != c_idx + and grid.logic[j][0] == row == grid.logic[j][1] + and any(min(hi, y1) - max(lo, y0) > 0.5 * (hi - lo) for _, y0, _, y1 in grid.words[grid.cell_words[j]]) + for j in range(len(grid.boxes)) + ) + # Cells spanning several rows whose text lines lie in different rows row_lines = grid.lines(0) for i in range(len(grid.boxes)): rs, re = grid.logic[i][:2] if re == rs or len(grid.cell_words[i]) < 2: continue - cell_lines = _group(centers[grid.cell_words[i]], line_height / 2) - rows = [] + idcs = np.asarray(grid.cell_words[i]) + cell_lines = _group(centers[idcs], line_height / 2) + rows: list[int] = [] # the row of each text line for k in range(int(cell_lines.max()) + 1): - text = grid.words[np.asarray(grid.cell_words[i])[cell_lines == k]] - lo, hi = text[:, 1].min(), text[:, 3].max() + lo, hi = grid.words[idcs[cell_lines == k], 1].min(), grid.words[idcs[cell_lines == k], 3].max() row = int(np.clip(np.searchsorted(row_lines, (lo + hi) / 2, side="right") - 1, rs, re)) - aligned = any( - j != i - and grid.logic[j][0] == row == grid.logic[j][1] - and any(min(hi, w[3]) - max(lo, w[1]) > 0.5 * (hi - lo) for w in grid.words[grid.cell_words[j]]) - for j in range(len(grid.boxes)) - ) - if not aligned or row in rows: + if row in rows or not level_with_row(i, row, lo, hi): + rows = [] break rows.append(row) - else: - if len(rows) > 1: - spans = [ - (rs if k == 0 else row, re if k == len(rows) - 1 else rows[k + 1] - 1) for k, row in enumerate(rows) - ] - split(i, cell_lines, spans) + if len(rows) > 1: + # Each text line until the row of the next one, the cell keeping its first and last rows + starts, ends = [rs, *rows[1:]], [*(row - 1 for row in rows[1:]), re] + split(i, cell_lines, list(zip(starts, ends))) def _reading_order(words: np.ndarray, idcs: list[int], line_height: float) -> list[int]: @@ -387,7 +393,7 @@ def assemble_page_table( words in reading order (`"words"`), and the numbers of rows and columns of the table """ scale = np.array([page_shape[1], page_shape[0]], dtype=np.float64) - polys = np.stack([as_polygon(cell["geometry"]) for cell in cells]) * scale + polys = as_polygons([cell["geometry"] for cell in cells]) * scale logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells], dtype=np.int64) if all("words" in cell for cell in cells): cell_words = [[int(i) for i in cell["words"] if not taken[i]] for cell in cells] @@ -402,7 +408,7 @@ def assemble_page_table( # The free words in the outline of the cells free, word_coords = np.flatnonzero(~taken), np.zeros((0, 4)) if free.size: - words = _rotate(np.stack([as_polygon(box) for box in word_boxes[free]]) * scale, -angle, pivot) + words = _rotate(as_polygons(word_boxes[free]) * scale, -angle, pivot) centers = words.mean(axis=1) inside = np.all((centers >= boxes[:, :2].min(axis=0)) & (centers <= boxes[:, 2:].max(axis=0)), axis=1) free, words = free[inside], words[inside] diff --git a/doctr/models/table_structure/predictor/pytorch.py b/doctr/models/table_structure/predictor/pytorch.py index 8a76deeea9..ee8e2bc929 100644 --- a/doctr/models/table_structure/predictor/pytorch.py +++ b/doctr/models/table_structure/predictor/pytorch.py @@ -16,7 +16,7 @@ from doctr.models.preprocessor import PreProcessor from doctr.models.utils import set_device_and_dtype -from ..grid import as_polygon, fill_table_grid +from ..grid import as_polygons, fill_table_grid from ..tablecenternet.base import _logic_from_geometry __all__ = ["TablePredictor"] @@ -30,7 +30,7 @@ def _crop_region(page: np.ndarray, region: Any) -> tuple[np.ndarray, np.ndarray, degenerate region """ h, w = page.shape[:2] - src = as_polygon(region) * np.array([w, h], dtype=np.float32) + src = as_polygons([region])[0] * np.array([w, h], dtype=np.float32) crop_w = int(round(max(np.linalg.norm(src[1] - src[0]), np.linalg.norm(src[2] - src[3])))) crop_h = int(round(max(np.linalg.norm(src[3] - src[0]), np.linalg.norm(src[2] - src[1])))) if crop_w < 2 or crop_h < 2: @@ -193,24 +193,27 @@ def in_region(points: np.ndarray) -> np.ndarray: return np.flatnonzero(np.all((points >= pad) & (points <= [crop_w - pad, crop_h - pad]), axis=1)) # Cells in crop pixels, without the ones of the context margin - polys = np.stack([as_polygon(cell["geometry"]) for cell in grid["cells"]]) * [crop_w, crop_h] + polys = as_polygons([cell["geometry"] for cell in grid["cells"]]) * [crop_w, crop_h] keep = in_region(polys.mean(axis=1)) if keep.size == 0: return None - cells = [grid["cells"][idx] for idx in keep] - logic = np.array([[c["col_start"], c["col_end"], c["row_start"], c["row_end"]] for c in cells]) - if keep.size < polys.shape[0] and getattr(getattr(self.model, "postprocessor", None), "geometric_logic", 0): + polys, cells = polys[keep], [grid["cells"][idx] for idx in keep] + logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells]) + geometric_logic = getattr(getattr(self.model, "postprocessor", None), "geometric_logic", False) + if geometric_logic and keep.size < len(grid["cells"]): # The dropped cells made grid lines of their own: index the rows / columns of the remaining ones again - logic = _logic_from_geometry(polys[keep], logic, split_tables=False) + # (the logical coordinates of TableCenterNet are ordered (start_col, end_col, start_row, end_row)) + by_col = _logic_from_geometry(polys, logic[:, [2, 3, 0, 1]], split_tables=False) + logic = by_col[:, [2, 3, 0, 1]] # The words of the region, in crop pixels word_idcs, words = np.zeros(0, dtype=np.int64), np.zeros((0, 4)) if word_boxes is not None and len(word_boxes) > 0: - pts = np.stack([as_polygon(box) for box in word_boxes]) * np.array([w, h], dtype=np.float32) + pts = as_polygons(word_boxes) * np.array([w, h], dtype=np.float32) pts = cv2.perspectiveTransform(pts.reshape(1, -1, 2), transform).reshape(-1, 4, 2) word_idcs = in_region(pts.mean(axis=1)) words = np.concatenate([pts[word_idcs].min(axis=1), pts[word_idcs].max(axis=1)], axis=1) boxes, cell_logic, cell_words, origin = fill_table_grid( - np.concatenate([polys[keep].min(axis=1), polys[keep].max(axis=1)], axis=1), logic[:, [2, 3, 0, 1]], words + np.concatenate([polys.min(axis=1), polys.max(axis=1)], axis=1), logic, words ) # Crop pixels -> page-relative coordinates corners = boxes[:, [[0, 1], [2, 1], [2, 3], [0, 3]]].reshape(1, -1, 2).astype(np.float32) diff --git a/tests/pytorch/test_models_table_grid.py b/tests/pytorch/test_models_table_grid.py new file mode 100644 index 0000000000..9a606c7a97 --- /dev/null +++ b/tests/pytorch/test_models_table_grid.py @@ -0,0 +1,169 @@ +import numpy as np + +from doctr.models.table_structure.grid import ( + _axis_slots, + _grid_lines, + _group, + as_polygons, + assemble_page_table, + fill_table_grid, +) + + +def _grid(xs, ys): + """Boxes and logical coordinates (row_start, row_end, col_start, col_end) of a regular grid""" + boxes, logic = [], [] + for r in range(len(ys) - 1): + for c in range(len(xs) - 1): + boxes.append([xs[c], ys[r], xs[c + 1], ys[r + 1]]) + logic.append([r, r, c, c]) + return np.array(boxes, dtype=float), np.array(logic) + + +def _word(cx, cy, w=40, h=20): + return [cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2] + + +def _table(cells, logic, words): + """The text grid of a table, from named words""" + names, boxes = list(words), np.array(list(words.values()), dtype=float).reshape(-1, 4) + boxes, logic, cell_words, origin = fill_table_grid(np.asarray(cells, dtype=float), np.asarray(logic), boxes) + grid = [["" for _ in range(logic[:, 3].max() + 1)] for _ in range(logic[:, 1].max() + 1)] + for (rs, _, cs, _), idcs in zip(logic, cell_words): + grid[rs][cs] = " ".join(filter(None, [grid[rs][cs], *(names[i] for i in idcs)])) + return grid, logic, origin + + +def test_table_grid_helpers(): + # Straight boxes (flat or 2-point) and polygons, as (N, 4, 2) polygons + expected = [[[0.1, 0.2], [0.5, 0.2], [0.5, 0.6], [0.1, 0.6]]] + np.testing.assert_allclose(as_polygons([[0.1, 0.2, 0.5, 0.6]]), expected) + np.testing.assert_allclose(as_polygons([((0.1, 0.2), (0.5, 0.6))]), expected) + np.testing.assert_allclose(as_polygons(expected), expected) + assert as_polygons(np.zeros((0, 4))).shape == (0, 4, 2) + # Grid lines at the median of the cell edges, interpolated when no edge lies on them + lines = _grid_lines(np.array([0, 2]), np.array([0, 2]), np.array([0.0, 20.0]), np.array([10.0, 30.0]), 3) + np.testing.assert_allclose(lines, [0, 10, 20, 30]) + # Positions grouped by distance to the mean of their group + np.testing.assert_array_equal(_group(np.array([0.0, 30.0, 4.0, 33.0]), 10), [0, 1, 0, 1]) + # Words beyond the grid make new indices, grouped by text line (rows) or overlapping extent (columns) + lines = np.array([100.0, 200.0, 300.0]) + lo, hi = np.array([150.0, 40.0, 45.0, 10.0, 310.0]), np.array([160.0, 60.0, 65.0, 30.0, 330.0]) + np.testing.assert_array_equal(_axis_slots(lo, hi, lines, 15), [0, -1, -1, -2, 2]) + np.testing.assert_array_equal(_axis_slots(lo, hi, lines, None), [0, -1, -1, -2, 2]) + + +def test_fill_table_grid_complete(): + # A 2 x 2 grid whose last cell is missed by the model, inside a table holding more text around it + cells, logic = _grid([200, 400, 600], [300, 400, 500]) + words = { + "A": _word(300, 350), + "B": _word(500, 350), + "C": _word(300, 450), + "missed": _word(500, 450), # in the slot of the missed cell + "Col1": _word(300, 250), # a header row above the grid + "Col2": _word(500, 250), + "Total": _word(250, 580, w=60), # a sentence below the grid, across both columns + "amount": _word(322.5, 580, w=75), + "due": _word(405, 580, w=80), + "key": _word(150, 350), # a column on the left of the grid + } + grid, _, origin = _table(cells[:3], logic[:3], words) + assert grid == [["", "Col1", "Col2"], ["key", "A", "B"], ["", "C", "missed"], ["", "Total amount due", ""]] + # The cells made from the words have no source cell + assert sorted(origin) == [-1] * 5 + [0, 1, 2] + + # Without words: the predicted grid, with local logical coordinates + boxes, out_logic, cell_words, origin = fill_table_grid(cells, logic + 3, np.zeros((0, 4))) + np.testing.assert_array_equal(out_logic, logic) + assert cell_words == [[]] * 4 and origin == [0, 1, 2, 3] + + +def test_fill_table_grid_merge_and_prune(): + # The model splits a row in 2 (the edges of its cells do not line up) and predicts an empty row + cells = [ + [100, 100, 300, 200], + [300, 100, 500, 200], + [100, 200, 300, 300], + [300, 200, 500, 240], # empty, a row of its own + [300, 240, 500, 340], + [100, 340, 300, 440], # an empty row + [300, 340, 500, 440], + [100, 440, 300, 540], + [300, 440, 500, 540], + ] + logic = [[0, 0, 0, 0], [0, 0, 1, 1], [1, 1, 0, 0], [1, 1, 1, 1], [2, 2, 1, 1]] + logic += [[3, 3, 0, 0], [3, 3, 1, 1], [4, 4, 0, 0], [4, 4, 1, 1]] + words = {"h1": _word(200, 150), "h2": _word(400, 150), "a": _word(200, 250), "b": _word(400, 255)} + words |= {"c": _word(200, 490), "d": _word(400, 490)} + # The text of the split row lies on the same line: the 2 rows are merged, and the empty row is removed + grid, _, _ = _table(cells, logic, words) + assert grid == [["h1", "h2"], ["a", "b"], ["c", "d"]] + # Text on different lines is not merged + words["b"] = _word(400, 310) + grid, _, _ = _table(cells, logic, words) + assert grid == [["h1", "h2"], ["a", ""], ["", "b"], ["c", "d"]] + + +def test_fill_table_grid_split_rows(): + # The model merges the 3 text lines of a dense table into a single row + cells, logic = _grid([100, 300, 500], [100, 160, 340]) + words = {"Name": _word(200, 130), "Value": _word(400, 130)} + for line, cy in enumerate((190, 250, 310)): + words |= {f"key{line}": _word(200, cy), f"{line}.00": _word(400, cy)} + grid, _, _ = _table(cells, logic, words) + assert grid == [["Name", "Value"], ["key0", "0.00"], ["key1", "1.00"], ["key2", "2.00"]] + + # A single cell wrapping its text (e.g. a long header) does not split its row + words = {"Unit": _word(200, 120), "price": _word(200, 140), "Qty": _word(400, 130)} + words |= {"10": _word(200, 250), "2": _word(400, 250)} + grid, _, _ = _table(cells, logic, words) + assert grid == [["Unit price", "Qty"], ["10", "2"]] + + # A cell spanning 2 rows whose text lines lie in each of them, aligned with the text of the row + cells, logic = [[100, 100, 300, 200], [100, 200, 300, 300], [300, 100, 500, 300]], [[0, 0, 0, 0], [1, 1, 0, 0]] + logic.append([0, 1, 1, 1]) + words = {"Acquired": _word(200, 150), "Taken": _word(200, 250), "51": _word(400, 150), "9": _word(400, 250)} + grid, _, origin = _table(cells, logic, words) + assert grid == [["Acquired", "51"], ["Taken", "9"]] + assert sorted(origin) == [0, 1, 2, 2] # both parts keep the score of the spanning cell + # A label spanning 2 rows on a single line is kept as is + words = {"Acquired": _word(200, 150), "Taken": _word(200, 250), "label": _word(400, 200)} + _, out_logic, _ = _table(cells, logic, words) + np.testing.assert_array_equal(out_logic, logic) + + +def _rotated(box, angle, pivot=(500.0, 500.0)): + """The corners (TL, TR, BR, BL) of a box rotated by `angle` degrees, in a 1000 x 1000 page (relative)""" + rad = np.deg2rad(angle) + rot = np.array([[np.cos(rad), -np.sin(rad)], [np.sin(rad), np.cos(rad)]]) + return (((as_polygons([box])[0] - pivot) @ rot.T + pivot) / 1000).tolist() + + +def test_assemble_page_table(): + # A 2 x 2 table rotated by 20 degrees on a page, whose words are assigned in the frame of its rows + cells, logic = _grid([300, 500, 700], [400, 500, 600]) + page_cells = [ + {"geometry": _rotated(box, 20), "score": 0.9, "row_start": rs, "row_end": re, "col_start": cs, "col_end": ce} + for box, (rs, re, cs, ce) in zip(cells, logic + 1) # offset logical coordinates + ] + words = [_rotated(_word(*center), 20) for center in ((400, 450), (600, 450), (400, 550), (600, 550), (900, 900))] + taken = np.zeros(len(words), dtype=bool) + out, num_rows, num_cols = assemble_page_table(page_cells, np.array(words), (1000, 1000), taken) + assert (num_rows, num_cols) == (2, 2) + assert [(c["row_start"], c["col_start"], c["words"]) for c in out] == [ + (0, 0, [0]), + (0, 1, [1]), + (1, 0, [2]), + (1, 1, [3]), + ] + # The cell geometries stay in the page frame, the word outside of the table is left free + np.testing.assert_allclose(out[0]["geometry"], page_cells[0]["geometry"], atol=1e-5) + assert taken.tolist() == [True, True, True, True, False] + + # Cells listing their words keep them, except the ones already taken by another table + listed = [{**cell, "words": [idx]} for idx, cell in enumerate(page_cells)] + taken = np.array([True, False, False, False, False]) + out, _, _ = assemble_page_table(listed, np.array(words), (1000, 1000), taken) + assert [c["words"] for c in out] == [[], [1], [2], [3]] + assert taken.tolist() == [True, True, True, True, False] From 58aadc2d354d87b8c0845626337624691c6964f5 Mon Sep 17 00:00:00 2001 From: felix Date: Tue, 10 Nov 2026 12:09:20 +0100 Subject: [PATCH 5/8] Update --- docs/source/models/table_structure.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/models/table_structure.md b/docs/source/models/table_structure.md index e51a493103..3b6b3936fd 100644 --- a/docs/source/models/table_structure.md +++ b/docs/source/models/table_structure.md @@ -13,7 +13,7 @@ Table structure models parse the image of a table into a machine-understandable | Architecture | Input shape | # params | Recall | Precision | F1 | Structure acc. | sec/it (batch 1) | | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| `tablecenternet` | (1024, 1024, 3) | 7.1 M | 93.82 | 96.96 | 95.36 | 76.84 | 0.5 | +| `tablecenternet` | (1024, 1024, 3) | 7.1 M | 96.42 | 95.50 | 95.96 | 72.37 | 0.5 | The metrics are produced by `references/table/evaluate.py` with {class}`~doctr.utils.metrics.TableCellMetric`: cell detection **recall**, **precision** and **F1** (cells matched From 1d7cb6726abd520b94afec07f1115bd4778fa869 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 11 Nov 2026 09:31:21 +0100 Subject: [PATCH 6/8] Update --- docs/source/models/table_structure.md | 2 +- .../table_structure/predictor/pytorch.py | 2 +- .../table_structure/tablecenternet/base.py | 6 +- .../table_structure/tablecenternet/pytorch.py | 2 +- tests/common/test_models_table_grid.py | 125 ------------------ 5 files changed, 7 insertions(+), 130 deletions(-) delete mode 100644 tests/common/test_models_table_grid.py diff --git a/docs/source/models/table_structure.md b/docs/source/models/table_structure.md index 3b6b3936fd..0f1579c0ee 100644 --- a/docs/source/models/table_structure.md +++ b/docs/source/models/table_structure.md @@ -13,7 +13,7 @@ Table structure models parse the image of a table into a machine-understandable | Architecture | Input shape | # params | Recall | Precision | F1 | Structure acc. | sec/it (batch 1) | | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| `tablecenternet` | (1024, 1024, 3) | 7.1 M | 96.42 | 95.50 | 95.96 | 72.37 | 0.5 | +| `tablecenternet` | (1024, 1024, 3) | 7.1 M | 96.41 | 96.50 | 96.46 | 78.85 | 0.5 | The metrics are produced by `references/table/evaluate.py` with {class}`~doctr.utils.metrics.TableCellMetric`: cell detection **recall**, **precision** and **F1** (cells matched diff --git a/doctr/models/table_structure/predictor/pytorch.py b/doctr/models/table_structure/predictor/pytorch.py index ee8e2bc929..ee8cccf7b8 100644 --- a/doctr/models/table_structure/predictor/pytorch.py +++ b/doctr/models/table_structure/predictor/pytorch.py @@ -200,7 +200,7 @@ def in_region(points: np.ndarray) -> np.ndarray: polys, cells = polys[keep], [grid["cells"][idx] for idx in keep] logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells]) geometric_logic = getattr(getattr(self.model, "postprocessor", None), "geometric_logic", False) - if geometric_logic and keep.size < len(grid["cells"]): + if geometric_logic: # The dropped cells made grid lines of their own: index the rows / columns of the remaining ones again # (the logical coordinates of TableCenterNet are ordered (start_col, end_col, start_row, end_row)) by_col = _logic_from_geometry(polys, logic[:, [2, 3, 0, 1]], split_tables=False) diff --git a/doctr/models/table_structure/tablecenternet/base.py b/doctr/models/table_structure/tablecenternet/base.py index bea453b9c5..e155c33a21 100644 --- a/doctr/models/table_structure/tablecenternet/base.py +++ b/doctr/models/table_structure/tablecenternet/base.py @@ -392,9 +392,11 @@ def __call__(self, decoded: dict[str, np.ndarray]) -> list[dict[str, np.ndarray] scores = cs[keep].astype(np.float32) logic = (logic[keep] - 1).astype(np.int32) # start_col, end_col, start_row, end_row (0-indexed) if self.geometric_logic and polys.shape[0] > 0: + # The duplicate detections of a cell are dropped; the logical coordinates are the model's (the page + # pipeline, which assembles the words of a page on the grid lines of the cell edges, re-indexes them + # from the geometry itself, cf. `TablePredictor._region_table`) kept = _suppress_overlapping_cells(polys, scores) - polys, scores = polys[kept], scores[kept] - logic = _logic_from_geometry(polys, logic[kept]).astype(np.int32) + polys, scores, logic = polys[kept], scores[kept], logic[kept] if self.assume_straight_pages: cells = np.concatenate([polys.min(axis=1), polys.max(axis=1)], axis=1).astype(np.float32) else: diff --git a/doctr/models/table_structure/tablecenternet/pytorch.py b/doctr/models/table_structure/tablecenternet/pytorch.py index cbc6a45b78..190331907d 100644 --- a/doctr/models/table_structure/tablecenternet/pytorch.py +++ b/doctr/models/table_structure/tablecenternet/pytorch.py @@ -26,7 +26,7 @@ "input_shape": (3, 1024, 1024), "mean": (0.798, 0.785, 0.772), "std": (0.264, 0.2749, 0.287), - "url": "https://github.com/mindee/doctr/releases/download/v1.1.0/tablecenternet-0d349a85.pt", + "url": "https://github.com/mindee/doctr/releases/download/v1.1.0/tablecenternet-b0d6f603.pt", }, } diff --git a/tests/common/test_models_table_grid.py b/tests/common/test_models_table_grid.py deleted file mode 100644 index ee27b36721..0000000000 --- a/tests/common/test_models_table_grid.py +++ /dev/null @@ -1,125 +0,0 @@ -import numpy as np - -from doctr.models.table_structure.grid import _axis_slots, _grid_lines, _group, as_polygon, fill_table_grid - - -def _grid(xs, ys): - """Boxes and logical coordinates (row_start, row_end, col_start, col_end) of a regular grid""" - boxes, logic = [], [] - for r in range(len(ys) - 1): - for c in range(len(xs) - 1): - boxes.append([xs[c], ys[r], xs[c + 1], ys[r + 1]]) - logic.append([r, r, c, c]) - return np.array(boxes, dtype=float), np.array(logic) - - -def _word(cx, cy, w=40, h=20): - return [cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2] - - -def _table(cells, logic, words): - """The text grid of a table, from named words""" - names, boxes = list(words), np.array(list(words.values()), dtype=float).reshape(-1, 4) - boxes, logic, cell_words, origin = fill_table_grid(np.asarray(cells, dtype=float), np.asarray(logic), boxes) - grid = [["" for _ in range(logic[:, 3].max() + 1)] for _ in range(logic[:, 1].max() + 1)] - for (rs, _, cs, _), idcs in zip(logic, cell_words): - grid[rs][cs] = " ".join([grid[rs][cs]] * bool(grid[rs][cs]) + [names[i] for i in idcs]) - return grid, logic, origin - - -def test_table_grid_helpers(): - # Straight boxes (flat or 2-point) and polygons, as (4, 2) polygons - expected = [[0.1, 0.2], [0.5, 0.2], [0.5, 0.6], [0.1, 0.6]] - np.testing.assert_allclose(as_polygon([0.1, 0.2, 0.5, 0.6]), expected) - np.testing.assert_allclose(as_polygon(((0.1, 0.2), (0.5, 0.6))), expected) - np.testing.assert_allclose(as_polygon(expected), expected) - # Grid lines at the median of the cell edges, interpolated when no edge lies on them - lines = _grid_lines(np.array([0, 2]), np.array([0, 2]), np.array([0.0, 20.0]), np.array([10.0, 30.0]), 3) - np.testing.assert_allclose(lines, [0, 10, 20, 30]) - # Positions grouped by distance to the mean of their group - np.testing.assert_array_equal(_group(np.array([0.0, 30.0, 4.0, 33.0]), 10), [0, 1, 0, 1]) - # Words beyond the grid make new indices, grouped by text line (rows) or overlapping extent (columns) - lines = np.array([100.0, 200.0, 300.0]) - lo, hi = np.array([150.0, 40.0, 45.0, 10.0, 310.0]), np.array([160.0, 60.0, 65.0, 30.0, 330.0]) - np.testing.assert_array_equal(_axis_slots(lo, hi, lines, 15), [0, -1, -1, -2, 2]) - np.testing.assert_array_equal(_axis_slots(lo, hi, lines, None), [0, -1, -1, -2, 2]) - - -def test_fill_table_grid_complete(): - # A 2 x 2 grid whose last cell is missed by the model, inside a table holding more text around it - cells, logic = _grid([200, 400, 600], [300, 400, 500]) - words = { - "A": _word(300, 350), - "B": _word(500, 350), - "C": _word(300, 450), - "missed": _word(500, 450), # in the slot of the missed cell - "Col1": _word(300, 250), # a header row above the grid - "Col2": _word(500, 250), - "Total": _word(250, 580, w=60), # a sentence below the grid, across both columns - "amount": _word(322.5, 580, w=75), - "due": _word(405, 580, w=80), - "key": _word(150, 350), # a column on the left of the grid - } - grid, _, origin = _table(cells[:3], logic[:3], words) - assert grid == [["", "Col1", "Col2"], ["key", "A", "B"], ["", "C", "missed"], ["", "Total amount due", ""]] - # The cells made from the words have no source cell - assert sorted(origin) == [-1] * 5 + [0, 1, 2] - - # Without words: the predicted grid, with local logical coordinates - boxes, out_logic, cell_words, origin = fill_table_grid(cells, logic + 3, np.zeros((0, 4))) - np.testing.assert_array_equal(out_logic, logic) - assert cell_words == [[]] * 4 and origin == [0, 1, 2, 3] - - -def test_fill_table_grid_merge_and_prune(): - # The model splits a row in 2 (the edges of its cells do not line up) and predicts an empty row - cells = [ - [100, 100, 300, 200], - [300, 100, 500, 200], - [100, 200, 300, 300], - [300, 200, 500, 240], # empty, a row of its own - [300, 240, 500, 340], - [100, 340, 300, 440], # an empty row - [300, 340, 500, 440], - [100, 440, 300, 540], - [300, 440, 500, 540], - ] - logic = [[0, 0, 0, 0], [0, 0, 1, 1], [1, 1, 0, 0], [1, 1, 1, 1], [2, 2, 1, 1]] - logic += [[3, 3, 0, 0], [3, 3, 1, 1], [4, 4, 0, 0], [4, 4, 1, 1]] - words = {"h1": _word(200, 150), "h2": _word(400, 150), "a": _word(200, 250), "b": _word(400, 255)} - words |= {"c": _word(200, 490), "d": _word(400, 490)} - # The text of the split row lies on the same line: the 2 rows are merged, and the empty row is removed - grid, _, _ = _table(cells, logic, words) - assert grid == [["h1", "h2"], ["a", "b"], ["c", "d"]] - # Text on different lines is not merged - words["b"] = _word(400, 310) - grid, _, _ = _table(cells, logic, words) - assert grid == [["h1", "h2"], ["a", ""], ["", "b"], ["c", "d"]] - - -def test_fill_table_grid_split_rows(): - # The model merges the 3 text lines of a dense table into a single row - cells, logic = _grid([100, 300, 500], [100, 160, 340]) - words = {"Name": _word(200, 130), "Value": _word(400, 130)} - for line, cy in enumerate((190, 250, 310)): - words |= {f"key{line}": _word(200, cy), f"{line}.00": _word(400, cy)} - grid, _, _ = _table(cells, logic, words) - assert grid == [["Name", "Value"], ["key0", "0.00"], ["key1", "1.00"], ["key2", "2.00"]] - - # A single cell wrapping its text (e.g. a long header) does not split its row - words = {"Unit": _word(200, 120), "price": _word(200, 140), "Qty": _word(400, 130)} - words |= {"10": _word(200, 250), "2": _word(400, 250)} - grid, _, _ = _table(cells, logic, words) - assert grid == [["Unit price", "Qty"], ["10", "2"]] - - # A cell spanning 2 rows whose text lines lie in each of them, aligned with the text of the row - cells, logic = [[100, 100, 300, 200], [100, 200, 300, 300], [300, 100, 500, 300]], [[0, 0, 0, 0], [1, 1, 0, 0]] - logic.append([0, 1, 1, 1]) - words = {"Acquired": _word(200, 150), "Taken": _word(200, 250), "51": _word(400, 150), "9": _word(400, 250)} - grid, _, origin = _table(cells, logic, words) - assert grid == [["Acquired", "51"], ["Taken", "9"]] - assert sorted(origin) == [0, 1, 2, 2] # both parts keep the score of the spanning cell - # A label spanning 2 rows on a single line is kept as is - words = {"Acquired": _word(200, 150), "Taken": _word(200, 250), "label": _word(400, 200)} - _, out_logic, _ = _table(cells, logic, words) - np.testing.assert_array_equal(out_logic, logic) From 2f93f1b729a4348ceb1bc0f2b220596018998f02 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 11 Nov 2026 12:53:07 +0100 Subject: [PATCH 7/8] Apply fixes --- docs/source/user_guide/layout_and_tables.md | 3 +- doctr/models/builder.py | 12 +-- doctr/models/reading_order/base.py | 43 ++++++-- doctr/models/table_structure/grid.py | 102 +++++++++--------- .../table_structure/predictor/pytorch.py | 24 ++--- .../table_structure/tablecenternet/base.py | 24 ++--- .../table_structure/tablecenternet/pytorch.py | 2 +- tests/common/test_models_reading_order.py | 27 +++++ tests/pytorch/test_models_table_grid.py | 17 +++ 9 files changed, 155 insertions(+), 99 deletions(-) diff --git a/docs/source/user_guide/layout_and_tables.md b/docs/source/user_guide/layout_and_tables.md index 53d2fb6fe1..c00201bcfc 100644 --- a/docs/source/user_guide/layout_and_tables.md +++ b/docs/source/user_guide/layout_and_tables.md @@ -70,7 +70,8 @@ otherwise the regions cannot be located and nothing is masked. The CLI enables i With `detect_tables=True`, each region the layout model labels as a `Table` is cropped and passed to a table structure model (`tablecenternet`), which predicts its cells and the rows and columns they span. Every word inside the table region is regrouped into a {class}`~doctr.io.Table`: the words outside of the predicted cells (e.g. a missed -header row or a row of totals) complete the grid, and the rows and columns follow the text lines of the cells: +header row or a row of totals) complete the grid, and the rows and columns follow the text lines of the cells (the +rows and columns without any text are removed): ```python from doctr.io import DocumentFile diff --git a/doctr/models/builder.py b/doctr/models/builder.py index 5b3b63a6ad..946cae4e18 100644 --- a/doctr/models/builder.py +++ b/doctr/models/builder.py @@ -464,9 +464,8 @@ def _build_tables( page_table: the table structure prediction(s) for the page. Either a single grid `{"cells": [{"geometry", "score", "row_start", "row_end", "col_start", "col_end"}], "num_rows", "num_cols"}` (cell geometries in page-relative coordinates), a list of such grids, or None. The cells - may list the indices of their words (`"words"`, in reading order, cf. - :meth:`~doctr.models.table_structure.predictor.TablePredictor.predict_regions`), otherwise the words - are assigned to them (cf. :func:`~doctr.models.table_structure.grid.assemble_page_table`) + may list the indices of their words (`"words"`, cf. `TablePredictor.predict_regions`), otherwise the + words are assigned to them page_shape: the (height, width) of the page Returns: @@ -475,8 +474,7 @@ def _build_tables( """ consumed = np.zeros(boxes.shape[0], dtype=bool) raw_tables = [] if page_table is None else [page_table] if isinstance(page_table, dict) else list(page_table) - # The geometries follow the page's word geometry: straight 2-point boxes for (N, 4) word boxes, 4-point polygons - # for (N, 4, 2) ones + # Geometries follow the word boxes: 2-point boxes for (N, 4), 4-point polygons for (N, 4, 2) straight = boxes.ndim != 3 tables: list[Table] = [] for raw in raw_tables: @@ -489,7 +487,7 @@ def _build_tables( table_cells.append( TableCell( value=" ".join(value for value, _ in words), - # The recognition confidence of the words, the structure model's score for an empty cell + # An empty cell falls back to the score of the table model confidence=float(np.mean([conf for _, conf in words])) if words else float(cell["score"] or 0), geometry=self._export_geometry(cell["geometry"], straight), row_start=cell["row_start"], @@ -501,7 +499,7 @@ def _build_tables( ) pts = np.concatenate([cell["geometry"] for cell in cells]) (xmin, ymin), (xmax, ymax) = pts.min(axis=0), pts.max(axis=0) - # The cells made from the words of the table are not scored by the table model + # The cells made from words have no score scores = [cell["score"] for cell in cells if cell["score"] is not None] tables.append( Table( diff --git a/doctr/models/reading_order/base.py b/doctr/models/reading_order/base.py index ccf38ecc22..975a2f0217 100644 --- a/doctr/models/reading_order/base.py +++ b/doctr/models/reading_order/base.py @@ -441,19 +441,46 @@ def _column_gutter( on_left, on_right = ~across & (centers <= split), ~across & (centers > split) if not on_left.any() or not on_right.any(): continue - # The elements above or below the columns (e.g. a title and its authors) do not cross the gutter + # Elements above or below the columns (e.g. a title and its authors) do not count as crossing columns_y = vy[on_left | on_right] crossing = int(np.count_nonzero(across & (vy > columns_y.min()) & (vy < columns_y.max()))) left = int(np.count_nonzero(centers <= split)) if crossing <= tolerance and left >= 0.25 * num_voters and num_voters - left >= 0.25 * num_voters: beside = voters.copy() - beside[np.flatnonzero(voters)[across]] = False # the elements on either side of the gutter + beside[np.flatnonzero(voters)[across]] = False if _key_value_rows(boxes, beside, float(split), y_overlap, y_overlap_threshold, span): return None # rows of keys and values (e.g. a receipt): read row after row return float(split) return None +def _detached_above(y0: np.ndarray, y1: np.ndarray, column: np.ndarray, top: float) -> np.ndarray: + """The elements of a column separated by a wide gap from both the rest of the column and the other columns below + them (e.g. a right-aligned title or address block above the columns of a page) + + Args: + y0: the top of each box + y1: the bottom of each box + column: the indices of the elements of the column + top: the position the other columns start at + + Returns: + the indices of the detached elements + """ + column = column[np.argsort(y0[column], kind="stable")] + level = np.flatnonzero(y1[column] > top) + if level.size == 0 or level[0] == 0: + return column[:0] + first = int(level[0]) + bottoms = np.maximum.accumulate(y1[column]) + gaps = y0[column[1:]] - bottoms[:-1] + # Wide: more than 3 times the usual gap of the column, and than the height of an element + limit = max(3 * float(np.median(gaps[first:])) if gaps[first:].size else 0.0, float(np.median(y1 - y0))) + while first > 0 and gaps[first - 1] <= limit: + first -= 1 + return column[:first] if first > 0 and top - bottoms[first - 1] > limit else column[:0] + + def _topological_order( boxes: np.ndarray, x_overlap_threshold: float, y_overlap_threshold: float, voters: np.ndarray | None = None ) -> list[int]: @@ -510,14 +537,16 @@ def _topological_order( ) multi_column = gutter is not None crossing = np.zeros(num_boxes, dtype=bool) - # 1 for the columns after the gutter: they are read once the ones before it are done, even when they start higher - # (e.g. beside a table whose caption is read with it). Only for columns of several elements, not a lone element + # 1 for the columns after the gutter, read after the ones before it even when they start higher (only when both + # sides hold several elements) after_gutter = np.zeros(num_boxes, dtype=int) if gutter is not None: crossing = np.minimum(x1 - gutter, gutter - x0) > 0.02 * span after = x0 >= gutter - 0.02 * span - if np.count_nonzero(after) >= 2 and np.count_nonzero(~after & ~crossing) >= 2: + before = ~after & ~crossing + if np.count_nonzero(after) >= 2 and np.count_nonzero(before) >= 2: after_gutter = after.astype(int) + after_gutter[_detached_above(y0, y1, np.flatnonzero(after), float(y0[before].min()))] = 0 # Column components: elements connected through horizontal overlap, except those which must not merge the columns # (page-wide or crossing the gutter). This keeps the traversal inside a column even when the direct top-to-bottom # continuation is momentarily broken by a fragmented or misaligned OCR line. @@ -548,8 +577,10 @@ def _continuation(last: int, ready: np.ndarray) -> np.ndarray: if ready.size == 0: # cycle safety net (can only happen with degenerate overlapping geometries) candidates = np.flatnonzero(~emitted) else: + # The columns before the gutter first + ready = ready[after_gutter[ready] == after_gutter[ready].min()] candidates = _continuation(order[-1], ready) if order else ready - # Topmost first, then leftmost (in canonical space), the columns before the gutter first + # Topmost first, then leftmost (in canonical space) next_idx = int(candidates[np.lexsort((x0[candidates], y0[candidates], after_gutter[candidates]))[0]]) order.append(next_idx) emitted[next_idx] = True diff --git a/doctr/models/table_structure/grid.py b/doctr/models/table_structure/grid.py index 3571a14c1b..f9985445ff 100644 --- a/doctr/models/table_structure/grid.py +++ b/doctr/models/table_structure/grid.py @@ -3,11 +3,10 @@ # This program is licensed under the Apache License 2.0. # See LICENSE or go to for full license details. -"""Assemble the cells predicted by a table structure model and the words of the table into a consistent grid. +"""Assemble the cells of a table structure model and the words of the table into a consistent grid. -`fill_table_grid` works in the frame of the (rectified, upright) table image, where the cells and the words are -axis-aligned boxes `(x0, y0, x1, y1)`, and the logical coordinates of the cells are `(row_start, row_end, col_start, -col_end)` (0-indexed, inclusive). `assemble_page_table` applies it to a table given in page coordinates. +Boxes are `(x0, y0, x1, y1)` in the upright table frame, logical coordinates are `(row_start, row_end, col_start, +col_end)` (0-indexed, inclusive). """ from typing import Any @@ -18,8 +17,7 @@ def as_polygons(geometries: Any) -> np.ndarray: - """Straight boxes `(x0, y0, x1, y1)` / `((x0, y0), (x1, y1))`, or polygons ordered (TL, TR, BR, BL) along their - reading direction, as (N, 4, 2) float32 polygons""" + """Convert straight boxes or (TL, TR, BR, BL) polygons to (N, 4, 2) float32 polygons""" flat = np.asarray(geometries, dtype=np.float32) flat = flat.reshape(len(flat), -1) if flat.size else np.zeros((0, 4), dtype=np.float32) if flat.shape[1] == 4: @@ -29,8 +27,8 @@ def as_polygons(geometries: Any) -> np.ndarray: def _grid_lines(start: np.ndarray, end: np.ndarray, lo: np.ndarray, hi: np.ndarray, size: int) -> np.ndarray: - """Positions of the `size + 1` lines of an axis: line `k`, where index `k` starts and index `k - 1` ends, lies at - the median of the cell edges on it (interpolated when no edge lies on it)""" + """Positions of the `size + 1` grid lines of an axis: the median of the cell edges on each line, interpolated + where no edge lies""" edges: list[list[float]] = [[] for _ in range(size + 1)] for s, e, low, high in zip(start, end, lo, hi): edges[int(s)].append(float(low)) @@ -40,10 +38,10 @@ def _grid_lines(start: np.ndarray, end: np.ndarray, lo: np.ndarray, hi: np.ndarr def _group(values: np.ndarray, tol: float) -> np.ndarray: - """Group 1D positions closer than `tol` to the mean of their group: the group index of each value, in order""" + """Group index of each 1D position, a group gathering the positions closer than `tol` to its mean""" groups = np.zeros(values.size, dtype=np.int64) group = 0 - members: list[int] = [] # the values of the current group + members: list[int] = [] for j in np.argsort(values, kind="stable"): if members and values[j] - np.mean(values[members]) >= tol: group, members = group + 1, [] @@ -53,9 +51,8 @@ def _group(values: np.ndarray, tol: float) -> np.ndarray: def _axis_slots(lo: np.ndarray, hi: np.ndarray, lines: np.ndarray, line_tol: float | None) -> np.ndarray: - """Index of each word along an axis of the grid: the index between the 2 lines around its center, or a new index - before / after the grid for the words beyond it, grouped by text line (rows, `line_tol`) or by overlapping - extent (columns, `line_tol` is None)""" + """Index of each word along an axis. The words beyond the grid get new indices, grouped by text line (rows, + `line_tol`) or by overlapping extent (columns, `line_tol=None`)""" size, center = lines.size - 1, (lo + hi) / 2 idx = np.clip(np.searchsorted(lines, center, side="right") - 1, 0, size - 1) for before, mask in ((True, center < lines[0]), (False, center > lines[-1])): @@ -64,10 +61,10 @@ def _axis_slots(lo: np.ndarray, hi: np.ndarray, lines: np.ndarray, line_tol: flo continue if line_tol is not None: groups = _group(center[members], line_tol) - else: # chain the overlapping extents, from left to right + else: # chain the overlapping extents groups, group, reach = np.zeros(members.size, dtype=np.int64), -1, -np.inf for j in np.argsort(lo[members], kind="stable"): - if lo[members[j]] > reach: # starts after the extents of the current group + if lo[members[j]] > reach: group += 1 groups[j], reach = group, max(reach, hi[members[j]]) idx[members] = groups - (groups.max() + 1) if before else size + groups @@ -75,8 +72,7 @@ def _axis_slots(lo: np.ndarray, hi: np.ndarray, lines: np.ndarray, line_tol: flo class _Grid: - """The cells of a table: boxes, logical coordinates, word indices and source cell (-1 for a cell made from the - words)""" + """The cells of a table: boxes, logical coordinates, word indices and source cell (-1 if made from words)""" def __init__(self, boxes: np.ndarray, logic: np.ndarray, words: np.ndarray): self.boxes = [np.asarray(b, dtype=np.float64) for b in boxes] @@ -114,9 +110,8 @@ def lines(self, axis: int) -> np.ndarray: def _complete(grid: _Grid, leftover: np.ndarray, line_height: float) -> None: - """Place the words lying in no cell into the grid: in the cell covering their slot, or in a new cell (a missed - cell, or a new row / column beyond the grid). In a new row, the words of a column or closer than half a text line - (e.g. a sentence) share a cell.""" + """Place the words lying in no cell: in the cell covering their slot, or in a new cell (a missed cell, or a new + row / column beyond the grid)""" words = grid.words[leftover] row_lines, col_lines = grid.lines(0), grid.lines(1) rows = _axis_slots(words[:, 1], words[:, 3], row_lines, line_height / 2) @@ -136,7 +131,7 @@ def _complete(grid: _Grid, leftover: np.ndarray, line_height: float) -> None: else: new_cells.setdefault((slot[0], slot[0], slot[1], slot[1]), []).append(int(j)) for r in np.unique(rows[~in_grid]): - # A new row, from left to right: a word starts a new cell in another column, unless it continues a sentence + # New row: a word starts a new cell in another column, unless it continues a sentence members = np.flatnonzero(rows == r) groups: list[list[int]] = [] for j in members[np.argsort(words[members, 0], kind="stable")]: @@ -144,11 +139,18 @@ def _complete(grid: _Grid, leftover: np.ndarray, line_height: float) -> None: if prev is None or (cols[j] != cols[prev] and words[j, 0] - words[prev, 2] >= line_height / 2): groups.append([]) groups[-1].append(int(j)) + spans: list[tuple[int, int, list[int]]] = [] for group in groups: - new_cells.setdefault((int(r), int(r), int(cols[group].min()), int(cols[group].max())), []).extend(group) + lo, hi = int(cols[group].min()), int(cols[group].max()) + while spans and lo <= spans[-1][1]: # overlapping words: a column belongs to a single cell + prev_lo, prev_hi, prev_group = spans.pop() + lo, hi, group = min(lo, prev_lo), max(hi, prev_hi), prev_group + group + spans.append((lo, hi, group)) + for lo, hi, group in spans: + new_cells[(int(r), int(r), lo, hi)] = group def band(lines: np.ndarray, start: int, end: int) -> tuple[float, float]: - # The extent of the grid lines from index start to end, unbounded beyond the grid + # Unbounded beyond the grid size = lines.size - 1 return (lines[start] if 0 <= start < size else np.inf, lines[end + 1] if 0 <= end < size else -np.inf) @@ -162,10 +164,8 @@ def band(lines: np.ndarray, start: int, end: int) -> tuple[float, float]: def _merge_and_prune(grid: _Grid) -> None: - """Merge the adjacent rows (resp. columns) whose text is complementary (no column, resp. row, holds text in both) - and lies on the same text line (resp. column): the model splits a row when the edges of its cells do not line up. - Then remove the rows / columns without text of their own (no cell holds text in this row / column only): the cells - spanning them span one less.""" + """Merge the adjacent rows / columns the model split (complementary text on the same text line / column), then + remove the rows / columns in which no single-span cell holds text""" logic = np.asarray(grid.logic) filled = np.array([len(w) > 0 for w in grid.cell_words]) if not filled.any(): @@ -182,7 +182,7 @@ def mergeable(axis: int, k: int) -> bool: if a.size == 0 or b.size == 0: return False if np.any((o_start[a, None] <= o_end[None, b]) & (o_start[None, b] <= o_end[a, None])): - return False # some column (resp. row) holds text in both + return False # both hold text in a same column / row lo, hi = (1, 3) if axis == 0 else (0, 2) a_lo, a_hi, b_lo, b_hi = text[a, lo].min(), text[a, hi].max(), text[b, lo].min(), text[b, hi].max() return bool(min(a_hi, b_hi) - max(a_lo, b_lo) >= 0.5 * min(a_hi - a_lo, b_hi - b_lo)) @@ -194,7 +194,7 @@ def shift(axis: int, k: int, merge: bool) -> None: start[alive & (start > k)] -= 1 def absorb_overlaps() -> None: - # The empty cells now sharing a slot with another cell are absorbed by it + # A cell absorbs the empty cells now sharing a slot with it for i in np.flatnonzero(alive & filled).tolist() + np.flatnonzero(alive & ~filled).tolist(): if not alive[i]: continue @@ -235,18 +235,17 @@ def absorb_overlaps() -> None: def _split_rows(grid: _Grid, line_height: float) -> None: - """Split the rows the model merged, from the text lines of their cells: a row whose cells holding text (at least - 2) mostly hold several text lines is split into its text lines (a single cell wrapping its text, e.g. a long - header, does not split its row), and a cell spanning several rows is split when each of its text lines lies in a - different row, aligned with the text of that row.""" + """Split the rows the model merged along their text lines: a row most of whose filled cells hold several lines + (a single cell wrapping its text does not split its row), and a spanning cell whose text lines each lie level + with a different row""" centers = (grid.words[:, 1] + grid.words[:, 3]) / 2 def split(c_idx: int, line_of: np.ndarray, spans: list[tuple[int, int]]) -> None: - # One cell per text line, the cell being cut midway between its text lines + # One cell per text line, cut midway between the lines (the text may lie beyond the box of its cell) words, (x0, y0, x1, y1) = np.asarray(grid.cell_words[c_idx]), grid.boxes[c_idx] tops = [grid.words[words[line_of == k], 1].min() for k in range(len(spans))] bottoms = [grid.words[words[line_of == k], 3].max() for k in range(len(spans))] - cuts = [y0, *[(b + t) / 2 for b, t in zip(bottoms[:-1], tops[1:])], y1] + cuts = [min(y0, tops[0]), *[(b + t) / 2 for b, t in zip(bottoms[:-1], tops[1:])], max(y1, bottoms[-1])] _, _, cs, ce = grid.logic[c_idx] for k, (rs, re) in enumerate(spans): box, logic, members = ( @@ -259,7 +258,7 @@ def split(c_idx: int, line_of: np.ndarray, spans: list[tuple[int, int]]) -> None else: grid.add(box, logic, members, grid.origin[c_idx]) - # Rows merged by the model, from the bottom up so that the rows still to process do not move + # From the bottom up, so that the rows still to process do not move for r in range(max(lc[1] for lc in grid.logic), -1, -1): in_row = [i for i, lc in enumerate(grid.logic) if lc[0] == lc[1] == r] filled = [i for i in in_row if grid.cell_words[i]] @@ -271,7 +270,7 @@ def split(c_idx: int, line_of: np.ndarray, spans: list[tuple[int, int]]) -> None multi = [i for i in filled if len({line_of[w] for w in grid.cell_words[i]}) > 1] if len(multi) < 2 or 2 * len(multi) < len(filled): continue - for lc in grid.logic: # the rows below move down, the cells spanning the row span its new rows too + for lc in grid.logic: # the rows below move down, the cells spanning the row grow if not lc[0] == lc[1] == r: lc[0] += n_lines - 1 if lc[0] > r else 0 lc[1] += n_lines - 1 if lc[1] >= r else 0 @@ -284,7 +283,7 @@ def split(c_idx: int, line_of: np.ndarray, spans: list[tuple[int, int]]) -> None split(i, np.searchsorted(present, lines), [(r + k, r + k) for k in present]) def level_with_row(c_idx: int, row: int, lo: float, hi: float) -> bool: - # Whether a text line, from lo to hi, is level with the text of another cell of the row + # Whether a text line (lo to hi) is level with the text of another cell of the row return any( j != c_idx and grid.logic[j][0] == row == grid.logic[j][1] @@ -292,7 +291,6 @@ def level_with_row(c_idx: int, row: int, lo: float, hi: float) -> bool: for j in range(len(grid.boxes)) ) - # Cells spanning several rows whose text lines lie in different rows row_lines = grid.lines(0) for i in range(len(grid.boxes)): rs, re = grid.logic[i][:2] @@ -300,7 +298,7 @@ def level_with_row(c_idx: int, row: int, lo: float, hi: float) -> bool: continue idcs = np.asarray(grid.cell_words[i]) cell_lines = _group(centers[idcs], line_height / 2) - rows: list[int] = [] # the row of each text line + rows: list[int] = [] # row of each text line for k in range(int(cell_lines.max()) + 1): lo, hi = grid.words[idcs[cell_lines == k], 1].min(), grid.words[idcs[cell_lines == k], 3].max() row = int(np.clip(np.searchsorted(row_lines, (lo + hi) / 2, side="right") - 1, rs, re)) @@ -309,7 +307,7 @@ def level_with_row(c_idx: int, row: int, lo: float, hi: float) -> bool: break rows.append(row) if len(rows) > 1: - # Each text line until the row of the next one, the cell keeping its first and last rows + # Each text line spans until the row of the next one starts, ends = [rs, *rows[1:]], [*(row - 1 for row in rows[1:]), re] split(i, cell_lines, list(zip(starts, ends))) @@ -325,18 +323,17 @@ def fill_table_grid( ) -> tuple[np.ndarray, np.ndarray, list[list[int]], list[int]]: """Make the grid of a table consistent with its text, and assign every word of the table to a cell. - The words in a cell are assigned to it, the other ones complete the grid (missed cells, header row, totals...). - The rows / columns split by the model are merged, the ones without text are removed, and the rows merged by the - model are split along their text lines. + The words outside of every cell complete the grid, the rows / columns split by the model are merged, the ones + without text are removed, and the rows merged by the model are split along their text lines. Args: - cells: the (C, 4) cell boxes `(x0, y0, x1, y1)` predicted by the table model, in the frame of the table image - logic: the (C, 4) logical coordinates `(row_start, row_end, col_start, col_end)` of the cells + cells: the (C, 4) cell boxes predicted by the table model + logic: the (C, 4) logical coordinates of the cells words: the (W, 4) boxes of the words of the table, in the same frame Returns: the cell boxes, their local logical coordinates, the indices of their words in reading order, and the index of - the predicted cell each one comes from (-1 for the cells made from the words) + the predicted cell each one comes from (-1 for a cell made from words) """ grid = _Grid(cells, logic, np.asarray(words, dtype=np.float64).reshape(-1, 4)) if len(grid.boxes) == 0: @@ -378,8 +375,8 @@ def assemble_page_table( ) -> tuple[list[dict[str, Any]], int, int]: """Assemble a table given in page coordinates with the words of the page. - The cells listing their words (`"words"`, cf. `TablePredictor.predict_regions`) keep them. Otherwise, the words - lying in the outline of the cells are assigned to the grid (cf. `fill_table_grid`), in the frame of the table rows. + The cells listing their words (`"words"`) keep them, otherwise the free words inside the outline of the cells are + assigned with `fill_table_grid`. Args: cells: the cells `{"geometry", "score", "row_start", "row_end", "col_start", "col_end"}` of the table, with @@ -389,8 +386,8 @@ def assemble_page_table( taken: the (N,) mask of the words already assigned to a table, updated in place Returns: - the cells with local logical coordinates, a page-relative (4, 2) polygon geometry, and the indices of their - words in reading order (`"words"`), and the numbers of rows and columns of the table + the cells (local logical coordinates, (4, 2) page-relative polygon, `"words"` in reading order), the number of + rows and the number of columns """ scale = np.array([page_shape[1], page_shape[0]], dtype=np.float64) polys = as_polygons([cell["geometry"] for cell in cells]) * scale @@ -400,12 +397,11 @@ def assemble_page_table( origin = list(range(len(cells))) logic -= [logic[:, 0].min()] * 2 + [logic[:, 2].min()] * 2 else: - # The frame of the table rows + # Upright frame of the table rows top_edges = polys[:, 1] - polys[:, 0] angle, pivot = float(np.median(np.arctan2(top_edges[:, 1], top_edges[:, 0]))), polys.reshape(-1, 2).mean(0) upright = _rotate(polys, -angle, pivot) boxes = np.concatenate([upright.min(axis=1), upright.max(axis=1)], axis=1) - # The free words in the outline of the cells free, word_coords = np.flatnonzero(~taken), np.zeros((0, 4)) if free.size: words = _rotate(as_polygons(word_boxes[free]) * scale, -angle, pivot) diff --git a/doctr/models/table_structure/predictor/pytorch.py b/doctr/models/table_structure/predictor/pytorch.py index ee8cccf7b8..b414f73907 100644 --- a/doctr/models/table_structure/predictor/pytorch.py +++ b/doctr/models/table_structure/predictor/pytorch.py @@ -23,11 +23,10 @@ def _crop_region(page: np.ndarray, region: Any) -> tuple[np.ndarray, np.ndarray, int] | None: - """Rectify a (page-relative) table region to an upright crop, with a margin of page context around it + """Rectify a page-relative table region to an upright crop with a margin of page context Returns: - the crop, the perspective transform from page pixels to crop pixels, and the margin (pixels), None for a - degenerate region + the crop, the page-to-crop perspective transform and the margin in pixels, or None for a degenerate region """ h, w = page.shape[:2] src = as_polygons([region])[0] * np.array([w, h], dtype=np.float32) @@ -142,10 +141,8 @@ def predict_regions( ) -> list[list[dict[str, Any]]]: """Recognize the tables a layout model located on pages, and assign the words of the pages to their cells - Each table region is rectified to an upright crop (so straight boxes and rotated polygons are both handled), - with a margin of page context: the cells centered in this margin belong to the neighbouring content and are - dropped. The words of the page lying in the region are assigned to the cells and complete the grid (cf. - :func:`~doctr.models.table_structure.grid.fill_table_grid`), then the cells are mapped back to the page. + Each table region (straight box or polygon) is rectified to an upright crop, and the words lying in it are + assigned to the cells and complete the grid (cf. :func:`~doctr.models.table_structure.grid.fill_table_grid`). Args: pages: the page images @@ -156,8 +153,7 @@ def predict_regions( Returns: a per-page list of tables `{"cells": [...], "num_rows": int, "num_cols": int}` in page-relative - coordinates, one per table region, each cell listing the indices of its words in the page (`"words"`) in - reading order + coordinates, each cell listing the page indices of its words in reading order (`"words"`) """ crops, meta = [], [] for p_idx, (page, region) in enumerate(zip(pages, regions)): @@ -192,7 +188,7 @@ def _region_table( def in_region(points: np.ndarray) -> np.ndarray: return np.flatnonzero(np.all((points >= pad) & (points <= [crop_w - pad, crop_h - pad]), axis=1)) - # Cells in crop pixels, without the ones of the context margin + # Cells in crop pixels; the ones centered in the context margin belong to the neighbouring content polys = as_polygons([cell["geometry"] for cell in grid["cells"]]) * [crop_w, crop_h] keep = in_region(polys.mean(axis=1)) if keep.size == 0: @@ -201,11 +197,10 @@ def in_region(points: np.ndarray) -> np.ndarray: logic = np.array([[c["row_start"], c["row_end"], c["col_start"], c["col_end"]] for c in cells]) geometric_logic = getattr(getattr(self.model, "postprocessor", None), "geometric_logic", False) if geometric_logic: - # The dropped cells made grid lines of their own: index the rows / columns of the remaining ones again - # (the logical coordinates of TableCenterNet are ordered (start_col, end_col, start_row, end_row)) + # Index the rows / columns from the cell edges (TableCenterNet logic is ordered columns first) by_col = _logic_from_geometry(polys, logic[:, [2, 3, 0, 1]], split_tables=False) logic = by_col[:, [2, 3, 0, 1]] - # The words of the region, in crop pixels + # Words of the region, in crop pixels word_idcs, words = np.zeros(0, dtype=np.int64), np.zeros((0, 4)) if word_boxes is not None and len(word_boxes) > 0: pts = as_polygons(word_boxes) * np.array([w, h], dtype=np.float32) @@ -222,8 +217,7 @@ def in_region(points: np.ndarray) -> np.ndarray: "cells": [ { "geometry": poly.tolist(), - # The cells made from the words are not scored by the table model - "score": None if source < 0 else float(cells[source]["score"]), + "score": None if source < 0 else float(cells[source]["score"]), # None for a cell made from words "row_start": int(rs), "row_end": int(re), "col_start": int(cs), diff --git a/doctr/models/table_structure/tablecenternet/base.py b/doctr/models/table_structure/tablecenternet/base.py index e155c33a21..f8131befc5 100644 --- a/doctr/models/table_structure/tablecenternet/base.py +++ b/doctr/models/table_structure/tablecenternet/base.py @@ -150,10 +150,8 @@ def window(lo: np.ndarray | float, hi: np.ndarray | float): def _index_lines(lines: np.ndarray, start: np.ndarray, end: np.ndarray, cell_size: float) -> np.ndarray: """Logical index of each grid line of an axis, from the cells lying between the lines. - Every cell spans at least one index (the line it ends on comes after the line it starts on), and the lines are - otherwise given the smallest possible indices: the lines no cell separates (e.g. the edges of a row that do not - line up) share an index. A gap covered by no cell (e.g. a row with no detected cell) counts as many empty indices - as cells of the typical size fit in it. + Every cell spans at least one index, and the lines no cell separates share an index. A gap covered by no cell + counts as many indices as cells of the typical size fit in it. Args: lines: the sorted positions of the grid lines, shape (L,) @@ -230,8 +228,7 @@ def _align_table_logic(edges: tuple[np.ndarray, ...], logic: np.ndarray) -> np.n end = np.clip(end, 1, lines.size - 1) start = np.minimum(start, end - 1) values = _index_lines(lines, start, end, float(np.median(end_edge - start_edge))) - # The indices follow the geometry, from the absolute position the model predicts for most of the cells (e.g. - # the cells below a row with no detected cell, apart from the cells above it, keep their indices) + # Keep the absolute position the model predicts for most of the cells values += max(int(round(float(np.median(logic[:, offset] - values[start])))), 0) logic[:, offset] = values[start] logic[:, offset + 1] = values[end] - 1 @@ -243,10 +240,8 @@ def _logic_from_geometry(polys: np.ndarray, model_logic: np.ndarray, split_table consistent with their geometry. The cells are split into tables (groups of touching cells), and in each table the cell edges are grouped into - the row and column lines of the grid, indexed from the geometry (cf. `_index_lines`): the cells of a visual row / - column share their indices, even when the logical-coordinate head of the model drifts (e.g. on borderless - tables). The model coordinates give the direction of the axes and the absolute position of each table (e.g. the - cells below a row with no detected cell keep their indices). + the row and column lines of the grid, indexed from the geometry (cf. `_index_lines`). The model coordinates only + give the direction of the axes and the absolute position of each table. Args: polys: the cells, of shape (N, 4, 2) @@ -286,9 +281,8 @@ class TableCenterNetPostProcessor: corner_thresh: minimum score for a corner to be used during relocation not_relocate: if True, skip the corner-relocation step (faster, less accurate) assume_straight_pages: whether the pages are assumed to be straight (i.e., no rotation) - geometric_logic: if True, remove the duplicate (overlapping) cells and make the logical coordinates - consistent with the cell geometry: the rows and columns are indexed from the grid lines the cell edges - lie on (the logical-coordinate head drifts on borderless tables) + geometric_logic: if True, remove the duplicate (overlapping) cells. `TablePredictor.predict_regions` then + indexes the rows and columns from the cell edges instead of the logical coordinates of the model """ def __init__( @@ -392,9 +386,7 @@ def __call__(self, decoded: dict[str, np.ndarray]) -> list[dict[str, np.ndarray] scores = cs[keep].astype(np.float32) logic = (logic[keep] - 1).astype(np.int32) # start_col, end_col, start_row, end_row (0-indexed) if self.geometric_logic and polys.shape[0] > 0: - # The duplicate detections of a cell are dropped; the logical coordinates are the model's (the page - # pipeline, which assembles the words of a page on the grid lines of the cell edges, re-indexes them - # from the geometry itself, cf. `TablePredictor._region_table`) + # The logical coordinates stay the model's: `TablePredictor._region_table` indexes them from geometry kept = _suppress_overlapping_cells(polys, scores) polys, scores, logic = polys[kept], scores[kept], logic[kept] if self.assume_straight_pages: diff --git a/doctr/models/table_structure/tablecenternet/pytorch.py b/doctr/models/table_structure/tablecenternet/pytorch.py index 190331907d..858a7ed6f8 100644 --- a/doctr/models/table_structure/tablecenternet/pytorch.py +++ b/doctr/models/table_structure/tablecenternet/pytorch.py @@ -149,7 +149,7 @@ class TableCenterNet(nn.Module, _TableCenterNet): center_k: maximum number of cell centers corner_k: maximum number of corners not_relocate: if True, skip the corner-relocation step - geometric_logic: if True, remove the duplicate cells and align the logical coordinates on the cell geometry + geometric_logic: if True, remove the duplicate cells (cf. `TableCenterNetPostProcessor`) assume_straight_pages: if True, the predictor will fit straight boxes to the cells exportable: onnx exportable returns only the raw head maps cfg: the configuration dict of the model diff --git a/tests/common/test_models_reading_order.py b/tests/common/test_models_reading_order.py index 336e087c93..fab1312b47 100644 --- a/tests/common/test_models_reading_order.py +++ b/tests/common/test_models_reading_order.py @@ -365,6 +365,33 @@ def test_layout_labels(label, normalized, role): list(range(20)), id="right-column-starting-higher", ), + pytest.param( + # A right-aligned title far above both columns is not the start of the right column + [_box(0.6, 0.03, 0.95, 0.06)] + + _column(0.05, 0.45, 0.2, 0.05, 6, 0.035) + + _column(0.55, 0.95, 0.2, 0.05, 6, 0.035), + {}, + list(range(13)), + id="right-aligned-title-over-columns", + ), + pytest.param( + # The same title above a right column starting higher: title, left column, right column + [_box(0.6, 0.03, 0.95, 0.06)] + + _column(0.05, 0.45, 0.3, 0.05, 6, 0.035) + + _column(0.55, 0.95, 0.2, 0.05, 8, 0.035), + {}, + list(range(15)), + id="right-aligned-title-over-higher-right-column", + ), + pytest.param( + # A heading right above the right column belongs to it + [_box(0.55, 0.14, 0.95, 0.18)] + + _column(0.05, 0.45, 0.2, 0.05, 6, 0.035) + + _column(0.55, 0.95, 0.2, 0.05, 6, 0.035), + {}, + [*range(1, 7), 0, *range(7, 13)], + id="heading-of-the-right-column", + ), pytest.param( SPLIT_CAPTION, {"labels": SPLIT_CAPTION_LABELS}, diff --git a/tests/pytorch/test_models_table_grid.py b/tests/pytorch/test_models_table_grid.py index 9a606c7a97..38efa188a1 100644 --- a/tests/pytorch/test_models_table_grid.py +++ b/tests/pytorch/test_models_table_grid.py @@ -73,6 +73,13 @@ def test_fill_table_grid_complete(): # The cells made from the words have no source cell assert sorted(origin) == [-1] * 5 + [0, 1, 2] + # Overlapping words below the grid (e.g. duplicate detections) do not make cells sharing a column + words = {"A": _word(300, 350), "B": _word(500, 350)} + words |= {"a": [210, 440, 250, 460], "b": [300, 440, 520, 460], "c": [320, 440, 360, 460]} + grid, out_logic, origin = _table(cells[:2], logic[:2], words) + assert grid == [["A", "B"], ["a b c", ""]] + assert out_logic.tolist() == [[0, 0, 0, 0], [0, 0, 1, 1], [1, 1, 0, 1]] and origin == [0, 1, -1] + # Without words: the predicted grid, with local logical coordinates boxes, out_logic, cell_words, origin = fill_table_grid(cells, logic + 3, np.zeros((0, 4))) np.testing.assert_array_equal(out_logic, logic) @@ -120,6 +127,16 @@ def test_fill_table_grid_split_rows(): grid, _, _ = _table(cells, logic, words) assert grid == [["Unit price", "Qty"], ["10", "2"]] + # A cell shorter than its row holds a word of its slot lying below it: the split cells enclose their text + cells, logic = _grid([100, 300, 500, 700], [100, 160, 340]) + cells[4, 3] = 200 + words = { + f"{name}{line}": _word(cx, cy) for line, cy in enumerate((190, 310)) for name, cx in zip("abc", (200, 400, 600)) + } + boxes, out_logic, cell_words, _ = fill_table_grid(cells, logic, np.array(list(words.values()), dtype=float)) + assert out_logic[:, 1].max() == 1 and all(len(idcs) == 1 for idcs in cell_words) + assert np.all(boxes[:, 3] > boxes[:, 1]) + # A cell spanning 2 rows whose text lines lie in each of them, aligned with the text of the row cells, logic = [[100, 100, 300, 200], [100, 200, 300, 300], [300, 100, 500, 300]], [[0, 0, 0, 0], [1, 1, 0, 0]] logic.append([0, 1, 1, 1]) From 3ecdb047fdd09e2a9d8ef43127c8645692cabcf6 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 11 Nov 2026 13:18:54 +0100 Subject: [PATCH 8/8] Remove unused code --- .../table_structure/predictor/pytorch.py | 2 +- .../table_structure/tablecenternet/base.py | 50 ++++--------------- tests/common/test_models_table_structure.py | 7 +-- 3 files changed, 12 insertions(+), 47 deletions(-) diff --git a/doctr/models/table_structure/predictor/pytorch.py b/doctr/models/table_structure/predictor/pytorch.py index b414f73907..fd58efc756 100644 --- a/doctr/models/table_structure/predictor/pytorch.py +++ b/doctr/models/table_structure/predictor/pytorch.py @@ -198,7 +198,7 @@ def in_region(points: np.ndarray) -> np.ndarray: geometric_logic = getattr(getattr(self.model, "postprocessor", None), "geometric_logic", False) if geometric_logic: # Index the rows / columns from the cell edges (TableCenterNet logic is ordered columns first) - by_col = _logic_from_geometry(polys, logic[:, [2, 3, 0, 1]], split_tables=False) + by_col = _logic_from_geometry(polys, logic[:, [2, 3, 0, 1]]) logic = by_col[:, [2, 3, 0, 1]] # Words of the region, in crop pixels word_idcs, words = np.zeros(0, dtype=np.int64), np.zeros((0, 4)) diff --git a/doctr/models/table_structure/tablecenternet/base.py b/doctr/models/table_structure/tablecenternet/base.py index f8131befc5..4725e8009e 100644 --- a/doctr/models/table_structure/tablecenternet/base.py +++ b/doctr/models/table_structure/tablecenternet/base.py @@ -172,33 +172,8 @@ def _index_lines(lines: np.ndarray, start: np.ndarray, end: np.ndarray, cell_siz return values -def _connected_tables(left: np.ndarray, right: np.ndarray, top: np.ndarray, bottom: np.ndarray) -> np.ndarray: - """Label the groups of touching cells (deskewed extents), i.e. the separate tables of an image.""" - n = left.size - # Neighbouring cells of a table share their edges, up to a fraction of the smallest cell sizes - margin = 0.25 * float(min(np.percentile(right - left, 10), np.percentile(bottom - top, 10))) - touch = ( - (left[:, None] <= right[None] + margin) - & (left[None] <= right[:, None] + margin) - & (top[:, None] <= bottom[None] + margin) - & (top[None] <= bottom[:, None] + margin) - ) - labels = -np.ones(n, dtype=np.int64) - for seed in range(n): - if labels[seed] >= 0: - continue - labels[seed] = seed - stack = [seed] - while stack: - i = stack.pop() - for j in np.flatnonzero(touch[i] & (labels < 0)): - labels[j] = seed - stack.append(int(j)) - return labels - - def _align_table_logic(edges: tuple[np.ndarray, ...], logic: np.ndarray) -> np.ndarray: - """Align the (N, 4) logical coordinates of the cells of a single table on their deskewed edges.""" + """Align the (N, 4) logical coordinates of the cells of a table on their deskewed edges.""" left, right, top, bottom = edges logic = logic.copy() # The table may be rotated by any multiple of 90 degrees (or the corners ordered by the image axes): match each @@ -235,29 +210,24 @@ def _align_table_logic(edges: tuple[np.ndarray, ...], logic: np.ndarray) -> np.n return logic -def _logic_from_geometry(polys: np.ndarray, model_logic: np.ndarray, split_tables: bool = True) -> np.ndarray: - """Make the logical coordinates `[start_col, end_col, start_row, end_row]` (0-indexed) of (N, 4, 2) cells - consistent with their geometry. +def _logic_from_geometry(polys: np.ndarray, model_logic: np.ndarray) -> np.ndarray: + """Make the logical coordinates `[start_col, end_col, start_row, end_row]` (0-indexed) of the (N, 4, 2) cells of a + table consistent with their geometry. - The cells are split into tables (groups of touching cells), and in each table the cell edges are grouped into - the row and column lines of the grid, indexed from the geometry (cf. `_index_lines`). The model coordinates only - give the direction of the axes and the absolute position of each table. + The cell edges are grouped into the row and column lines of the grid, indexed from the geometry (cf. + `_index_lines`). The model coordinates only give the direction of the axes and the absolute position of the table. Args: polys: the cells, of shape (N, 4, 2) model_logic: the logical coordinates predicted by the model, of shape (N, 4) - split_tables: if False, the cells are known to belong to a single table and are aligned together Returns: the logical coordinates, of shape (N, 4) """ - edges = _cell_edges(_deskew_cells(polys)) - logic = np.asarray(model_logic, dtype=np.int64).copy() - labels = _connected_tables(*edges) if split_tables else np.zeros(logic.shape[0], dtype=np.int64) - for label in np.unique(labels): - idx = np.flatnonzero(labels == label) - logic[idx] = _align_table_logic(tuple(e[idx] for e in edges), logic[idx]) - return logic + logic = np.asarray(model_logic, dtype=np.int64) + if logic.shape[0] == 0: + return logic.copy() + return _align_table_logic(_cell_edges(_deskew_cells(polys)), logic) def _lookup_logic(lc_map: np.ndarray, x: float, y: float) -> np.ndarray: diff --git a/tests/common/test_models_table_structure.py b/tests/common/test_models_table_structure.py index 9cc1194a9f..5f3510f435 100644 --- a/tests/common/test_models_table_structure.py +++ b/tests/common/test_models_table_structure.py @@ -105,17 +105,12 @@ def test_tablecenternet_logic_from_geometry(angle): ) -def test_tablecenternet_logic_from_geometry_tables(): +def test_tablecenternet_logic_from_geometry_model_drift(): target = _grid_target(3, 3, use_polygons=True) polys, gt = target["cells"], target["logic"] drifted = gt.copy() drifted[2] += [0, 0, 1, 1] # last cell of row 0 one row too low - # 2 tables in an image: each one keeps its own indices (both start at 0) - top, bottom = polys * [1, 0.4], polys * [1, 0.4] + [0, 0.55] - cells = np.concatenate([top, bottom]) - np.testing.assert_array_equal(_logic_from_geometry(cells, np.concatenate([drifted, gt])), np.concatenate([gt, gt])) - # An upside-down table (rows growing upwards): the axes follow the model coordinates flipped = polys.copy() flipped[..., 1] = 1 - flipped[..., 1]