Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions docs/source/models/recognition.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ Word accuracy on FUNSD and CORD word crops (exact and partial match, see {class}
| `crnn_vgg16_bn` | (32, 128, 3) | 15.8 M | 88.21 | 88.95 | 95.47 | 95.91 | 0.6 |
| `crnn_mobilenet_v3_small` | (32, 128, 3) | 2.1 M | 87.25 | 87.99 | 93.91 | 94.34 | **0.05** |
| `crnn_mobilenet_v3_large` | (32, 128, 3) | 4.5 M | 87.38 | 88.09 | 94.46 | 94.92 | 0.08 |
| `master` | (32, 128, 3) | 58.7 M | **88.57** | **89.39** | **95.73** | **96.21** | 17.6 |
| `master` | (32, 128, 3) | 58.7 M | **88.57** | **89.39** | **95.73** | **96.21** | 2.7 |
| `sar_resnet31` | (32, 128, 3) | 55.4 M | 88.10 | 88.88 | 94.83 | 95.29 | 4.9 |
| `vitstr_small` | (32, 128, 3) | 21.4 M | 88.00 | 88.82 | 95.40 | 95.78 | 1.5 |
| `vitstr_base` | (32, 128, 3) | 85.2 M | 88.33 | 89.09 | 95.32 | 95.71 | 4.1 |
| `parseq` | (32, 128, 3) | 23.8 M | 88.53 | 89.24 | 95.56 | 95.91 | 2.2 |
| `vitstr_small` | (32, 128, 3) | 21.4 M | 88.00 | 88.82 | 95.40 | 95.78 | 1.4 |
| `vitstr_base` | (32, 128, 3) | 85.2 M | 88.33 | 89.09 | 95.32 | 95.71 | 3.6 |
| `parseq` | (32, 128, 3) | 23.8 M | 88.53 | 89.24 | 95.56 | 95.91 | 1.5 |
| `viptr_tiny` | (32, 128, 3) | 3.2 M | 86.03 | 86.71 | 93.08 | 93.47 | 0.08 |

## Usage
Expand Down
11 changes: 6 additions & 5 deletions doctr/datasets/ic03.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,9 @@ def __init__(
name, _resolution, rectangles = image

# File existence check
if not os.path.exists(os.path.join(tmp_root, name.text)):
raise FileNotFoundError(f"unable to locate {os.path.join(tmp_root, name.text)}")
img_name = name.text
if not img_name or not os.path.exists(os.path.join(tmp_root, img_name)):
raise FileNotFoundError(f"unable to locate {os.path.join(tmp_root, img_name or '')}")

if use_polygons:
# (x, y) coordinates of top left, top right, bottom right, bottom left corners
Expand Down Expand Up @@ -121,14 +122,14 @@ def __init__(
labels = [lab.text for rect in rectangles for lab in rect if lab.text]

if recognition_task:
crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, name.text), geoms=boxes)
crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, img_name), geoms=boxes)
for crop, label in zip(crops, labels):
if crop.shape[0] > 0 and crop.shape[1] > 0 and len(label) > 0 and " " not in label:
self.data.append((crop, label))
elif detection_task:
self.data.append((name.text, boxes))
self.data.append((img_name, boxes))
else:
self.data.append((name.text, dict(boxes=boxes, labels=labels)))
self.data.append((img_name, dict(boxes=boxes, labels=labels)))

self.root = tmp_root

Expand Down
13 changes: 7 additions & 6 deletions doctr/datasets/svt.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,9 @@ def __init__(
name, _, _, _resolution, rectangles = image

# File existence check
if not os.path.exists(os.path.join(tmp_root, name.text)):
raise FileNotFoundError(f"unable to locate {os.path.join(tmp_root, name.text)}")
img_name = name.text
if not img_name or not os.path.exists(os.path.join(tmp_root, img_name)):
raise FileNotFoundError(f"unable to locate {os.path.join(tmp_root, img_name or '')}")

if use_polygons:
# (x, y) coordinates of top left, top right, bottom right, bottom left corners
Expand Down Expand Up @@ -112,14 +113,14 @@ def __init__(
labels = [lab.text for rect in rectangles for lab in rect]

if recognition_task:
crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, name.text), geoms=boxes)
crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, img_name), geoms=boxes)
for crop, label in zip(crops, labels):
if crop.shape[0] > 0 and crop.shape[1] > 0 and len(label) > 0 and " " not in label:
if crop.shape[0] > 0 and crop.shape[1] > 0 and label and " " not in label:
self.data.append((crop, label))
elif detection_task:
self.data.append((name.text, boxes))
self.data.append((img_name, boxes))
else:
self.data.append((name.text, dict(boxes=boxes, labels=labels)))
self.data.append((img_name, dict(boxes=boxes, labels=labels)))

self.root = tmp_root

Expand Down
4 changes: 4 additions & 0 deletions doctr/io/image/pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ def tensor_from_numpy(npy_img: np.ndarray, dtype: torch.dtype = torch.float32) -
if dtype not in (torch.uint8, torch.float16, torch.float32):
raise ValueError("insupported value for dtype")

# torch.from_numpy requires a writable buffer without negative strides (e.g. a flipped or rotated view)
if not npy_img.flags.writeable or any(stride < 0 for stride in npy_img.strides):
npy_img = npy_img.copy()

if dtype == torch.float32:
img = to_tensor(npy_img)
else:
Expand Down
3 changes: 1 addition & 2 deletions doctr/models/detection/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import cv2
import numpy as np

from doctr.utils.multithreading import multithread_exec
from doctr.utils.repr import NestedObject

__all__ = ["DetectionPostProcessor"]
Expand Down Expand Up @@ -107,4 +106,4 @@ def __call__(
"""
if proba_map.ndim != 4:
raise AssertionError(f"arg `proba_map` is expected to be 4-dimensional, got {proba_map.ndim}.")
return list(multithread_exec(self._process_sample, proba_map))
return [self._process_sample(sample) for sample in proba_map]
2 changes: 1 addition & 1 deletion doctr/models/kie_predictor/pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ def _predict_pages(
det_pages = self._mask_regions(pages, regions)

# Forward again to get predictions on straight pages
loc_preds = self.det_predictor(det_pages, **kwargs)
loc_preds = self._detect_straightened(loc_preds, _orig_pages, pages, det_pages, **kwargs)

dict_loc_preds: dict[str, list[np.ndarray]] = invert_data_structure(loc_preds) # type: ignore[assignment]

Expand Down
115 changes: 89 additions & 26 deletions doctr/models/modules/transformer/pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@

import math
from collections.abc import Callable
from typing import Any
from typing import Any, cast

import torch
import torch.nn.functional as F
from torch import nn

__all__ = ["Decoder", "PositionalEncoding", "EncoderBlock", "MultiHeadAttention", "PositionwiseFeedForward"]
Expand All @@ -30,29 +31,37 @@
pe[:, 1::2] = torch.cos(position * div_term)
self.register_buffer("pe", pe.unsqueeze(0))

def forward(self, x: torch.Tensor) -> torch.Tensor:
def forward(self, x: torch.Tensor, start: int = 0) -> torch.Tensor:
"""Forward pass

Args:
x: embeddings (batch, max_len, d_model)
start: position of the first embedding in the sequence (e.g. for incremental decoding)

Returns:
positional embeddings (batch, max_len, d_model)
"""
x = x + self.pe[:, : x.size(1)] # type: ignore[index]
x = x + self.pe[:, start : start + x.size(1)] # type: ignore[index]
return self.dropout(x)


def scaled_dot_product_attention(
query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask: torch.Tensor | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
"""Scaled Dot-Product Attention"""
scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(query.size(-1))
if mask is not None:
# NOTE: to ensure the ONNX compatibility, masked_fill works only with int equal condition
scores = scores.masked_fill(mask == 0, float("-inf"))
p_attn = torch.softmax(scores, dim=-1)
return torch.matmul(p_attn, value), p_attn
) -> tuple[torch.Tensor, None]:
"""Scaled Dot-Product Attention, with the fused kernels of PyTorch (the attention weights are not materialized)

Check notice on line 51 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L51

First line should end with a period, question mark, or exclamation point (not ')') (D415)

Check notice on line 51 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L51

Missing blank line after last section ('Returns') (D413)

Check notice on line 51 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L51

Missing dashed underline after section ('Returns') (D407)

Check notice on line 51 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L51

Multi-line docstring summary should start at the second line (D213)

Check notice on line 51 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L51

Section name should end with a newline ('Returns', not 'Returns:') (D406)

Args:
query: queries of shape (*, L, E)
key: keys of shape (*, S, E)
value: values of shape (*, S, E)
mask: optional mask broadcastable to (*, L, S), where 0 marks the keys a query must not attend to

Returns:
the attention output of shape (*, L, E), and None in place of the attention weights
"""
# NOTE: the boolean mask is derived with an int comparison, to ensure the ONNX compatibility
attn_mask = None if mask is None else mask != 0
return F.scaled_dot_product_attention(query, key, value, attn_mask=attn_mask), None


class PositionwiseFeedForward(nn.Sequential):
Expand Down Expand Up @@ -83,23 +92,44 @@
self.linear_layers = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(3)])
self.output_linear = nn.Linear(d_model, d_model)

def forward(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask=None) -> torch.Tensor:
batch_size = query.size(0)
def _split_heads(self, x: torch.Tensor) -> torch.Tensor:
# (N, L, d_model) --> (N, num_heads, L, d_k)
return x.view(x.size(0), -1, self.num_heads, self.d_k).transpose(1, 2)

# linear projections of Q, K, V
query, key, value = [
linear(x).view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)
for linear, x in zip(self.linear_layers, (query, key, value))
]
def project_kv(self, key: torch.Tensor, value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Project the keys and values, e.g. once for the steps of an incremental decoding

Check notice on line 100 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L100

First line should end with a period, question mark, or exclamation point (not 'g') (D415)

Check notice on line 100 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L100

Missing blank line after last section ('Returns') (D413)

Check notice on line 100 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L100

Missing dashed underline after section ('Returns') (D407)

Check notice on line 100 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L100

Multi-line docstring summary should start at the second line (D213)

Check notice on line 100 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L100

Section name should end with a newline ('Returns', not 'Returns:') (D406)

# apply attention on all the projected vectors in batch
x, attn = scaled_dot_product_attention(query, key, value, mask=mask)
Args:
key: keys of shape (N, S, d_model)
value: values of shape (N, S, d_model)

# Concat attention heads
x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.num_heads * self.d_k)
Returns:
the projected keys and values, of shape (N, num_heads, S, d_k)
"""
return self._split_heads(self.linear_layers[1](key)), self._split_heads(self.linear_layers[2](value))

def attend(
self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask: torch.Tensor | None = None
) -> torch.Tensor:
"""Attention of the queries over keys and values projected with `project_kv`

Check notice on line 114 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L114

First line should end with a period, question mark, or exclamation point (not '`') (D415)

Check notice on line 114 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L114

Missing blank line after last section ('Returns') (D413)

Check notice on line 114 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L114

Missing dashed underline after section ('Returns') (D407)

Check notice on line 114 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L114

Multi-line docstring summary should start at the second line (D213)

Check notice on line 114 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L114

Section name should end with a newline ('Returns', not 'Returns:') (D406)

Args:
query: queries of shape (N, L, d_model)
key: projected keys of shape (N, num_heads, S, d_k)
value: projected values of shape (N, num_heads, S, d_k)
mask: optional mask broadcastable to (N, num_heads, L, S), where 0 marks the masked keys

Returns:
the attention output of shape (N, L, d_model)
"""
x, _ = scaled_dot_product_attention(self._split_heads(self.linear_layers[0](query)), key, value, mask=mask)
# Concat attention heads
x = x.transpose(1, 2).contiguous().view(query.size(0), -1, self.num_heads * self.d_k)
return self.output_linear(x)

def forward(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask=None) -> torch.Tensor:
return self.attend(query, *self.project_kv(key, value), mask=mask)


class EncoderBlock(nn.Module):
"""Transformer Encoder Block"""
Expand Down Expand Up @@ -184,16 +214,49 @@
memory: torch.Tensor,
source_mask: torch.Tensor | None = None,
target_mask: torch.Tensor | None = None,
cache: list[dict[str, tuple[torch.Tensor, torch.Tensor]]] | None = None,
) -> torch.Tensor:
"""Decode the target tokens

Check notice on line 219 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L219

First line should end with a period, question mark, or exclamation point (not 's') (D415)

Check notice on line 219 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L219

Missing blank line after last section ('Returns') (D413)

Check notice on line 219 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L219

Missing dashed underline after section ('Returns') (D407)

Check notice on line 219 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L219

Multi-line docstring summary should start at the second line (D213)

Check notice on line 219 in doctr/models/modules/transformer/pytorch.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/modules/transformer/pytorch.py#L219

Section name should end with a newline ('Returns', not 'Returns:') (D406)

Args:
tgt: target tokens of shape (N, L)
memory: encoded source of shape (N, S, d_model)
source_mask: optional mask of the source
target_mask: optional mask of the target
cache: for incremental decoding, an initially empty list that keeps the keys and values of every layer
between the calls: `tgt` then only holds the new tokens, which follow the ones of the previous calls
and attend to all of them (the source is projected once, at the first call)

Returns:
the decoded features of shape (N, L, d_model)
"""
start = 0 if not cache else cache[0]["self"][0].size(2)
tgt = self.embed(tgt) * math.sqrt(self.d_model)
pos_enc_tgt = self.positional_encoding(tgt)
output = pos_enc_tgt
output = self.positional_encoding(tgt, start)

for i in range(self.num_layers):
self_attention = cast(MultiHeadAttention, self.attention[i])
source_attention = cast(MultiHeadAttention, self.source_attention[i])
normed_output = self.layer_norm_input(output)
output = output + self.dropout(self.attention[i](normed_output, normed_output, normed_output, target_mask))
if cache is None:
attended = self_attention(normed_output, normed_output, normed_output, target_mask)
else:
if len(cache) <= i:
cache.append({"source": source_attention.project_kv(memory, memory)})
key, value = self_attention.project_kv(normed_output, normed_output)
if "self" in cache[i]:
key = torch.cat([cache[i]["self"][0], key], dim=2)
value = torch.cat([cache[i]["self"][1], value], dim=2)
cache[i]["self"] = (key, value)
attended = self_attention.attend(normed_output, key, value, target_mask)
output = output + self.dropout(attended)

normed_output = self.layer_norm_masked_attention(output)
output = output + self.dropout(self.source_attention[i](normed_output, memory, memory, source_mask))
if cache is None:
attended = source_attention(normed_output, memory, memory, source_mask)
else:
attended = source_attention.attend(normed_output, *cache[i]["source"], source_mask)
output = output + self.dropout(attended)
normed_output = self.layer_norm_attention(output)
output = output + self.dropout(self.position_feed_forward[i](normed_output))

Expand Down
35 changes: 35 additions & 0 deletions doctr/models/predictor/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from .._utils import estimate_orientation, mask_boxes, rectify_crops, rectify_loc_preds
from ..classification import crop_orientation_predictor, page_orientation_predictor
from ..classification.predictor import OrientationPredictor
from ..detection.predictor import DetectionPredictor

__all__ = ["_OCRPredictor"]

Expand Down Expand Up @@ -64,6 +65,8 @@
"""

crop_orientation_predictor: OrientationPredictor | None
# Set by the framework-specific predictors
det_predictor: DetectionPredictor
page_orientation_predictor: OrientationPredictor | None

def __init__(
Expand Down Expand Up @@ -306,6 +309,38 @@
out.append(straightened)
return out, m_invs

def _detect_straightened(
self,
loc_preds: list[Any],
orig_pages: list[np.ndarray],
pages: list[np.ndarray],
det_pages: list[np.ndarray],
**kwargs: Any,
) -> list[Any]:
"""Detect the text again on the straightened pages, except on the ones left unchanged by the straightening

Check notice on line 320 in doctr/models/predictor/base.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/predictor/base.py#L320

1 blank line required between summary line and description (found 0) (D205)

Check notice on line 320 in doctr/models/predictor/base.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/predictor/base.py#L320

First line should end with a period, question mark, or exclamation point (not 'g') (D415)

Check notice on line 320 in doctr/models/predictor/base.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/predictor/base.py#L320

Missing blank line after last section ('Returns') (D413)

Check notice on line 320 in doctr/models/predictor/base.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/predictor/base.py#L320

Missing dashed underline after section ('Returns') (D407)

Check notice on line 320 in doctr/models/predictor/base.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/predictor/base.py#L320

Multi-line docstring summary should start at the second line (D213)

Check notice on line 320 in doctr/models/predictor/base.py

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

doctr/models/predictor/base.py#L320

Section name should end with a newline ('Returns', not 'Returns:') (D406)
and the masking (e.g. straight pages): the detection made to estimate their orientation is reused

Args:
loc_preds: the detections on the original pages
orig_pages: the original pages
pages: the straightened pages
det_pages: the straightened pages passed to the detection model (masked layout regions)
**kwargs: keyword arguments of the detection predictor

Returns:
the detections on the straightened pages
"""
out = list(loc_preds)
changed = [
idx
for idx, (orig, page, det_page) in enumerate(zip(orig_pages, pages, det_pages))
if det_page is not page or orig.shape != page.shape or not np.array_equal(orig, page)
]
if changed:
for idx, pred in zip(changed, self.det_predictor([det_pages[idx] for idx in changed], **kwargs)):
out[idx] = pred
return out

@staticmethod
def _generate_crops(
pages: list[np.ndarray],
Expand Down
2 changes: 1 addition & 1 deletion doctr/models/predictor/pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ def _predict_pages(
det_pages = self._mask_regions(pages, regions)

# Forward again to get predictions on straight pages
loc_preds = self.det_predictor(det_pages, **kwargs)
loc_preds = self._detect_straightened(loc_preds, _orig_pages, pages, det_pages, **kwargs)

assert all(len(loc_pred) == 1 for loc_pred in loc_preds), (
"Detection Model in ocr_predictor should output only one class"
Expand Down
Loading
Loading