diff --git a/docs/source/models/recognition.md b/docs/source/models/recognition.md index 6d705fe593..e20f53cfe0 100644 --- a/docs/source/models/recognition.md +++ b/docs/source/models/recognition.md @@ -23,11 +23,11 @@ Word accuracy on FUNSD and CORD word crops (exact and partial match, see {class} | `crnn_vgg16_bn` | (32, 128, 3) | 15.8 M | 88.21 | 88.95 | 95.47 | 95.91 | 0.6 | | `crnn_mobilenet_v3_small` | (32, 128, 3) | 2.1 M | 87.25 | 87.99 | 93.91 | 94.34 | **0.05** | | `crnn_mobilenet_v3_large` | (32, 128, 3) | 4.5 M | 87.38 | 88.09 | 94.46 | 94.92 | 0.08 | -| `master` | (32, 128, 3) | 58.7 M | **88.57** | **89.39** | **95.73** | **96.21** | 17.6 | +| `master` | (32, 128, 3) | 58.7 M | **88.57** | **89.39** | **95.73** | **96.21** | 2.7 | | `sar_resnet31` | (32, 128, 3) | 55.4 M | 88.10 | 88.88 | 94.83 | 95.29 | 4.9 | -| `vitstr_small` | (32, 128, 3) | 21.4 M | 88.00 | 88.82 | 95.40 | 95.78 | 1.5 | -| `vitstr_base` | (32, 128, 3) | 85.2 M | 88.33 | 89.09 | 95.32 | 95.71 | 4.1 | -| `parseq` | (32, 128, 3) | 23.8 M | 88.53 | 89.24 | 95.56 | 95.91 | 2.2 | +| `vitstr_small` | (32, 128, 3) | 21.4 M | 88.00 | 88.82 | 95.40 | 95.78 | 1.4 | +| `vitstr_base` | (32, 128, 3) | 85.2 M | 88.33 | 89.09 | 95.32 | 95.71 | 3.6 | +| `parseq` | (32, 128, 3) | 23.8 M | 88.53 | 89.24 | 95.56 | 95.91 | 1.5 | | `viptr_tiny` | (32, 128, 3) | 3.2 M | 86.03 | 86.71 | 93.08 | 93.47 | 0.08 | ## Usage diff --git a/doctr/datasets/ic03.py b/doctr/datasets/ic03.py index 0216720322..cabd0af07a 100644 --- a/doctr/datasets/ic03.py +++ b/doctr/datasets/ic03.py @@ -85,8 +85,9 @@ def __init__( name, _resolution, rectangles = image # File existence check - if not os.path.exists(os.path.join(tmp_root, name.text)): - raise FileNotFoundError(f"unable to locate {os.path.join(tmp_root, name.text)}") + img_name = name.text + if not img_name or not os.path.exists(os.path.join(tmp_root, img_name)): + raise FileNotFoundError(f"unable to locate {os.path.join(tmp_root, img_name or '')}") if use_polygons: # (x, y) coordinates of top left, top right, bottom right, bottom left corners @@ -121,14 +122,14 @@ def __init__( labels = [lab.text for rect in rectangles for lab in rect if lab.text] if recognition_task: - crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, name.text), geoms=boxes) + crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, img_name), geoms=boxes) for crop, label in zip(crops, labels): if crop.shape[0] > 0 and crop.shape[1] > 0 and len(label) > 0 and " " not in label: self.data.append((crop, label)) elif detection_task: - self.data.append((name.text, boxes)) + self.data.append((img_name, boxes)) else: - self.data.append((name.text, dict(boxes=boxes, labels=labels))) + self.data.append((img_name, dict(boxes=boxes, labels=labels))) self.root = tmp_root diff --git a/doctr/datasets/svt.py b/doctr/datasets/svt.py index bcbac7769a..0ecc0fdfda 100644 --- a/doctr/datasets/svt.py +++ b/doctr/datasets/svt.py @@ -78,8 +78,9 @@ def __init__( name, _, _, _resolution, rectangles = image # File existence check - if not os.path.exists(os.path.join(tmp_root, name.text)): - raise FileNotFoundError(f"unable to locate {os.path.join(tmp_root, name.text)}") + img_name = name.text + if not img_name or not os.path.exists(os.path.join(tmp_root, img_name)): + raise FileNotFoundError(f"unable to locate {os.path.join(tmp_root, img_name or '')}") if use_polygons: # (x, y) coordinates of top left, top right, bottom right, bottom left corners @@ -112,14 +113,14 @@ def __init__( labels = [lab.text for rect in rectangles for lab in rect] if recognition_task: - crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, name.text), geoms=boxes) + crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, img_name), geoms=boxes) for crop, label in zip(crops, labels): - if crop.shape[0] > 0 and crop.shape[1] > 0 and len(label) > 0 and " " not in label: + if crop.shape[0] > 0 and crop.shape[1] > 0 and label and " " not in label: self.data.append((crop, label)) elif detection_task: - self.data.append((name.text, boxes)) + self.data.append((img_name, boxes)) else: - self.data.append((name.text, dict(boxes=boxes, labels=labels))) + self.data.append((img_name, dict(boxes=boxes, labels=labels))) self.root = tmp_root diff --git a/doctr/io/image/pytorch.py b/doctr/io/image/pytorch.py index 776051951a..53e913aa6e 100644 --- a/doctr/io/image/pytorch.py +++ b/doctr/io/image/pytorch.py @@ -83,6 +83,10 @@ def tensor_from_numpy(npy_img: np.ndarray, dtype: torch.dtype = torch.float32) - if dtype not in (torch.uint8, torch.float16, torch.float32): raise ValueError("insupported value for dtype") + # torch.from_numpy requires a writable buffer without negative strides (e.g. a flipped or rotated view) + if not npy_img.flags.writeable or any(stride < 0 for stride in npy_img.strides): + npy_img = npy_img.copy() + if dtype == torch.float32: img = to_tensor(npy_img) else: diff --git a/doctr/models/detection/core.py b/doctr/models/detection/core.py index 9ba5d2b786..7205cc8198 100644 --- a/doctr/models/detection/core.py +++ b/doctr/models/detection/core.py @@ -7,7 +7,6 @@ import cv2 import numpy as np -from doctr.utils.multithreading import multithread_exec from doctr.utils.repr import NestedObject __all__ = ["DetectionPostProcessor"] @@ -107,4 +106,4 @@ def __call__( """ if proba_map.ndim != 4: raise AssertionError(f"arg `proba_map` is expected to be 4-dimensional, got {proba_map.ndim}.") - return list(multithread_exec(self._process_sample, proba_map)) + return [self._process_sample(sample) for sample in proba_map] diff --git a/doctr/models/kie_predictor/pytorch.py b/doctr/models/kie_predictor/pytorch.py index 6ce3389683..643e1dc5fa 100644 --- a/doctr/models/kie_predictor/pytorch.py +++ b/doctr/models/kie_predictor/pytorch.py @@ -155,7 +155,7 @@ def _predict_pages( det_pages = self._mask_regions(pages, regions) # Forward again to get predictions on straight pages - loc_preds = self.det_predictor(det_pages, **kwargs) + loc_preds = self._detect_straightened(loc_preds, _orig_pages, pages, det_pages, **kwargs) dict_loc_preds: dict[str, list[np.ndarray]] = invert_data_structure(loc_preds) # type: ignore[assignment] diff --git a/doctr/models/modules/transformer/pytorch.py b/doctr/models/modules/transformer/pytorch.py index 7b9d890b47..c8467062c9 100644 --- a/doctr/models/modules/transformer/pytorch.py +++ b/doctr/models/modules/transformer/pytorch.py @@ -7,9 +7,10 @@ import math from collections.abc import Callable -from typing import Any +from typing import Any, cast import torch +import torch.nn.functional as F from torch import nn __all__ = ["Decoder", "PositionalEncoding", "EncoderBlock", "MultiHeadAttention", "PositionwiseFeedForward"] @@ -30,29 +31,37 @@ def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000) -> N pe[:, 1::2] = torch.cos(position * div_term) self.register_buffer("pe", pe.unsqueeze(0)) - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: torch.Tensor, start: int = 0) -> torch.Tensor: """Forward pass Args: x: embeddings (batch, max_len, d_model) + start: position of the first embedding in the sequence (e.g. for incremental decoding) Returns: positional embeddings (batch, max_len, d_model) """ - x = x + self.pe[:, : x.size(1)] # type: ignore[index] + x = x + self.pe[:, start : start + x.size(1)] # type: ignore[index] return self.dropout(x) def scaled_dot_product_attention( query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask: torch.Tensor | None = None -) -> tuple[torch.Tensor, torch.Tensor]: - """Scaled Dot-Product Attention""" - scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(query.size(-1)) - if mask is not None: - # NOTE: to ensure the ONNX compatibility, masked_fill works only with int equal condition - scores = scores.masked_fill(mask == 0, float("-inf")) - p_attn = torch.softmax(scores, dim=-1) - return torch.matmul(p_attn, value), p_attn +) -> tuple[torch.Tensor, None]: + """Scaled Dot-Product Attention, with the fused kernels of PyTorch (the attention weights are not materialized) + + Args: + query: queries of shape (*, L, E) + key: keys of shape (*, S, E) + value: values of shape (*, S, E) + mask: optional mask broadcastable to (*, L, S), where 0 marks the keys a query must not attend to + + Returns: + the attention output of shape (*, L, E), and None in place of the attention weights + """ + # NOTE: the boolean mask is derived with an int comparison, to ensure the ONNX compatibility + attn_mask = None if mask is None else mask != 0 + return F.scaled_dot_product_attention(query, key, value, attn_mask=attn_mask), None class PositionwiseFeedForward(nn.Sequential): @@ -83,23 +92,44 @@ def __init__(self, num_heads: int, d_model: int, dropout: float = 0.1) -> None: self.linear_layers = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(3)]) self.output_linear = nn.Linear(d_model, d_model) - def forward(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask=None) -> torch.Tensor: - batch_size = query.size(0) + def _split_heads(self, x: torch.Tensor) -> torch.Tensor: + # (N, L, d_model) --> (N, num_heads, L, d_k) + return x.view(x.size(0), -1, self.num_heads, self.d_k).transpose(1, 2) - # linear projections of Q, K, V - query, key, value = [ - linear(x).view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2) - for linear, x in zip(self.linear_layers, (query, key, value)) - ] + def project_kv(self, key: torch.Tensor, value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Project the keys and values, e.g. once for the steps of an incremental decoding - # apply attention on all the projected vectors in batch - x, attn = scaled_dot_product_attention(query, key, value, mask=mask) + Args: + key: keys of shape (N, S, d_model) + value: values of shape (N, S, d_model) - # Concat attention heads - x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.num_heads * self.d_k) + Returns: + the projected keys and values, of shape (N, num_heads, S, d_k) + """ + return self._split_heads(self.linear_layers[1](key)), self._split_heads(self.linear_layers[2](value)) + + def attend( + self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask: torch.Tensor | None = None + ) -> torch.Tensor: + """Attention of the queries over keys and values projected with `project_kv` + + Args: + query: queries of shape (N, L, d_model) + key: projected keys of shape (N, num_heads, S, d_k) + value: projected values of shape (N, num_heads, S, d_k) + mask: optional mask broadcastable to (N, num_heads, L, S), where 0 marks the masked keys + Returns: + the attention output of shape (N, L, d_model) + """ + x, _ = scaled_dot_product_attention(self._split_heads(self.linear_layers[0](query)), key, value, mask=mask) + # Concat attention heads + x = x.transpose(1, 2).contiguous().view(query.size(0), -1, self.num_heads * self.d_k) return self.output_linear(x) + def forward(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask=None) -> torch.Tensor: + return self.attend(query, *self.project_kv(key, value), mask=mask) + class EncoderBlock(nn.Module): """Transformer Encoder Block""" @@ -184,16 +214,49 @@ def forward( memory: torch.Tensor, source_mask: torch.Tensor | None = None, target_mask: torch.Tensor | None = None, + cache: list[dict[str, tuple[torch.Tensor, torch.Tensor]]] | None = None, ) -> torch.Tensor: + """Decode the target tokens + + Args: + tgt: target tokens of shape (N, L) + memory: encoded source of shape (N, S, d_model) + source_mask: optional mask of the source + target_mask: optional mask of the target + cache: for incremental decoding, an initially empty list that keeps the keys and values of every layer + between the calls: `tgt` then only holds the new tokens, which follow the ones of the previous calls + and attend to all of them (the source is projected once, at the first call) + + Returns: + the decoded features of shape (N, L, d_model) + """ + start = 0 if not cache else cache[0]["self"][0].size(2) tgt = self.embed(tgt) * math.sqrt(self.d_model) - pos_enc_tgt = self.positional_encoding(tgt) - output = pos_enc_tgt + output = self.positional_encoding(tgt, start) for i in range(self.num_layers): + self_attention = cast(MultiHeadAttention, self.attention[i]) + source_attention = cast(MultiHeadAttention, self.source_attention[i]) normed_output = self.layer_norm_input(output) - output = output + self.dropout(self.attention[i](normed_output, normed_output, normed_output, target_mask)) + if cache is None: + attended = self_attention(normed_output, normed_output, normed_output, target_mask) + else: + if len(cache) <= i: + cache.append({"source": source_attention.project_kv(memory, memory)}) + key, value = self_attention.project_kv(normed_output, normed_output) + if "self" in cache[i]: + key = torch.cat([cache[i]["self"][0], key], dim=2) + value = torch.cat([cache[i]["self"][1], value], dim=2) + cache[i]["self"] = (key, value) + attended = self_attention.attend(normed_output, key, value, target_mask) + output = output + self.dropout(attended) + normed_output = self.layer_norm_masked_attention(output) - output = output + self.dropout(self.source_attention[i](normed_output, memory, memory, source_mask)) + if cache is None: + attended = source_attention(normed_output, memory, memory, source_mask) + else: + attended = source_attention.attend(normed_output, *cache[i]["source"], source_mask) + output = output + self.dropout(attended) normed_output = self.layer_norm_attention(output) output = output + self.dropout(self.position_feed_forward[i](normed_output)) diff --git a/doctr/models/predictor/base.py b/doctr/models/predictor/base.py index 5692b63bea..bb4506713b 100644 --- a/doctr/models/predictor/base.py +++ b/doctr/models/predictor/base.py @@ -24,6 +24,7 @@ from .._utils import estimate_orientation, mask_boxes, rectify_crops, rectify_loc_preds from ..classification import crop_orientation_predictor, page_orientation_predictor from ..classification.predictor import OrientationPredictor +from ..detection.predictor import DetectionPredictor __all__ = ["_OCRPredictor"] @@ -64,6 +65,8 @@ class _OCRPredictor: """ crop_orientation_predictor: OrientationPredictor | None + # Set by the framework-specific predictors + det_predictor: DetectionPredictor page_orientation_predictor: OrientationPredictor | None def __init__( @@ -306,6 +309,38 @@ def _straighten_pages( out.append(straightened) return out, m_invs + def _detect_straightened( + self, + loc_preds: list[Any], + orig_pages: list[np.ndarray], + pages: list[np.ndarray], + det_pages: list[np.ndarray], + **kwargs: Any, + ) -> list[Any]: + """Detect the text again on the straightened pages, except on the ones left unchanged by the straightening + and the masking (e.g. straight pages): the detection made to estimate their orientation is reused + + Args: + loc_preds: the detections on the original pages + orig_pages: the original pages + pages: the straightened pages + det_pages: the straightened pages passed to the detection model (masked layout regions) + **kwargs: keyword arguments of the detection predictor + + Returns: + the detections on the straightened pages + """ + out = list(loc_preds) + changed = [ + idx + for idx, (orig, page, det_page) in enumerate(zip(orig_pages, pages, det_pages)) + if det_page is not page or orig.shape != page.shape or not np.array_equal(orig, page) + ] + if changed: + for idx, pred in zip(changed, self.det_predictor([det_pages[idx] for idx in changed], **kwargs)): + out[idx] = pred + return out + @staticmethod def _generate_crops( pages: list[np.ndarray], diff --git a/doctr/models/predictor/pytorch.py b/doctr/models/predictor/pytorch.py index 5abf54ac95..fb20967fea 100644 --- a/doctr/models/predictor/pytorch.py +++ b/doctr/models/predictor/pytorch.py @@ -177,7 +177,7 @@ def _predict_pages( det_pages = self._mask_regions(pages, regions) # Forward again to get predictions on straight pages - loc_preds = self.det_predictor(det_pages, **kwargs) + loc_preds = self._detect_straightened(loc_preds, _orig_pages, pages, det_pages, **kwargs) assert all(len(loc_pred) == 1 for loc_pred in loc_preds), ( "Detection Model in ocr_predictor should output only one class" diff --git a/doctr/models/preprocessor/pytorch.py b/doctr/models/preprocessor/pytorch.py index fcf5408285..0cfb7235a4 100644 --- a/doctr/models/preprocessor/pytorch.py +++ b/doctr/models/preprocessor/pytorch.py @@ -14,11 +14,19 @@ from doctr.transforms import Resize from doctr.utils import Sample -from doctr.utils.multithreading import multithread_exec __all__ = ["PreProcessor"] +def _torch_compatible(x: np.ndarray) -> np.ndarray: + """Make an array readable by `torch.from_numpy`, which requires a writable buffer without negative strides. + `np.ascontiguousarray` is not enough: it keeps a negative stride on an axis of size 1 (e.g. `np.rot90` of a crop + of 1 pixel in height, as rectified crops can be), which numpy still considers contiguous.""" + if not x.flags.writeable or any(stride < 0 for stride in x.strides): + return x.copy() + return np.ascontiguousarray(x) + + class PreProcessor(nn.Module): """Implements an abstract preprocessor object which performs casting, resizing, batching and normalization. @@ -98,9 +106,7 @@ def sample_transforms(self, x: np.ndarray) -> torch.Tensor | tuple[torch.Tensor, raise AssertionError("expected list of 3D Tensors") if x.dtype not in (np.uint8, np.float32, np.float16): raise TypeError("unsupported data type for numpy.ndarray") - if not x.flags.writeable: # torch.from_numpy requires a writable buffer - x = x.copy() - tensor = torch.from_numpy(np.ascontiguousarray(x)).permute(2, 0, 1) + tensor = torch.from_numpy(_torch_compatible(x)).permute(2, 0, 1) # Resizing if self.resize.return_padding_mask: sample = self.resize(Sample(image=tensor)) @@ -132,9 +138,7 @@ def __call__( raise AssertionError("expected 4D Tensor") if x.dtype not in (np.uint8, np.float32, np.float16): raise TypeError("unsupported data type for numpy.ndarray") - if not x.flags.writeable: # torch.from_numpy requires a writable buffer - x = x.copy() - tensor = torch.from_numpy(np.ascontiguousarray(x)).permute(0, 3, 1, 2) + tensor = torch.from_numpy(_torch_compatible(x)).permute(0, 3, 1, 2) # Resizing if tensor.shape[-2] != self.resize.size[0] or tensor.shape[-1] != self.resize.size[1]: @@ -156,7 +160,7 @@ def __call__( elif isinstance(x, list) and all(isinstance(sample, np.ndarray) for sample in x): # Sample transform (to tensor, resize) - samples = list(multithread_exec(self.sample_transforms, x)) + samples: Any = [self.sample_transforms(sample) for sample in x] # Batching if self.resize.return_padding_mask: img_batches, mask_batches = self.batch_inputs(samples) @@ -166,9 +170,7 @@ def __call__( raise TypeError(f"invalid input type: {type(x)}") # Batch transforms (normalize) + img_batches = [self.normalize(batch) for batch in img_batches] if self.resize.return_padding_mask: - img_batches = list(multithread_exec(self.normalize, img_batches)) return img_batches, mask_batches - - img_batches = list(multithread_exec(self.normalize, img_batches)) return img_batches diff --git a/doctr/models/recognition/master/pytorch.py b/doctr/models/recognition/master/pytorch.py index faf77d85ba..d2abf04c09 100644 --- a/doctr/models/recognition/master/pytorch.py +++ b/doctr/models/recognition/master/pytorch.py @@ -247,19 +247,24 @@ def decode(self, encoded: torch.Tensor) -> torch.Tensor: ys = torch.full((b, self.max_length), self.vocab_size + 2, dtype=torch.long, device=encoded.device) # pad ys[:, 0] = self.vocab_size + 1 # sos - # Final dimension include EOS/SOS/PAD - for i in range(self.max_length - 1): - source_mask, target_mask = self.make_source_and_target_mask(encoded, ys) - output = self.decoder(ys, encoded, source_mask, target_mask) - # update ys with the next token and ignore the first token (SOS) - ys[:, i + 1] = self.linear(output[:, i]).argmax(-1) - # Stop decoding if all sequences have reached the EOS token - # NOTE: `break` isn't correctly translated to Onnx so we don't break here if we want to export - if not self.exportable and (ys == self.vocab_size).any(dim=-1).all(): - break - - # Shape (N, max_length, vocab_size + 1) - return self.linear(output) + # Incremental decoding: each step only decodes the last token, the previous ones are cached + cache: list[dict[str, tuple[torch.Tensor, torch.Tensor]]] = [] + logits = [] + for i in range(self.max_length): + step_logits = self.linear(self.decoder(ys[:, i : i + 1], encoded, cache=cache)) + logits.append(step_logits) + if i + 1 < self.max_length: + # update ys with the next token + ys[:, i + 1] = step_logits[:, 0].argmax(-1) + # Stop decoding if all sequences have reached the EOS token + # NOTE: `break` isn't correctly translated to Onnx so we don't break here if we want to export + if not self.exportable and (ys == self.vocab_size).any(dim=-1).all(): + break + + out = torch.cat(logits, dim=1) + # Shape (N, max_length, vocab_size + 1): the positions after an early stop come after the EOS of every + # sequence, and are not decoded + return F.pad(out, (0, 0, 0, self.max_length - out.size(1))) class MASTERPostProcessor(_MASTERPostProcessor): diff --git a/doctr/models/recognition/parseq/pytorch.py b/doctr/models/recognition/parseq/pytorch.py index bf1763f0c7..ac4c5ed9c3 100644 --- a/doctr/models/recognition/parseq/pytorch.py +++ b/doctr/models/recognition/parseq/pytorch.py @@ -85,20 +85,43 @@ def __init__( self.cross_attention_dropout = nn.Dropout(dropout) self.feed_forward_dropout = nn.Dropout(dropout) + def project_content(self, content: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Normalize and project the content into the keys and values of the self-attention""" + content_norm = self.content_norm(content) + return self.attention.project_kv(content_norm, content_norm) + def forward( self, target, content, memory, target_mask: torch.Tensor | None = None, + content_kv: tuple[torch.Tensor, torch.Tensor] | None = None, + memory_kv: tuple[torch.Tensor, torch.Tensor] | None = None, ): + """Decode the target queries + + Args: + target: target queries of shape (N, L, d_model) + content: content of shape (N, S, d_model), unused if `content_kv` is given + memory: encoded image of shape (N, T, d_model), unused if `memory_kv` is given + target_mask: optional mask of the content for the target queries + content_kv: optional keys and values of the content, cf. `project_content` + memory_kv: optional keys and values of the memory, cf. `MultiHeadAttention.project_kv` + + Returns: + the decoded queries of shape (N, L, d_model) + """ query_norm = self.query_norm(target) - content_norm = self.content_norm(content) + if content_kv is None: + content_kv = self.project_content(content) target = target.clone() + self.attention_dropout( - self.attention(query_norm, content_norm, content_norm, mask=target_mask) + self.attention.attend(query_norm, *content_kv, mask=target_mask) ) + if memory_kv is None: + memory_kv = self.cross_attention.project_kv(memory, memory) target = target.clone() + self.cross_attention_dropout( - self.cross_attention(self.query_norm(target), memory, memory) + self.cross_attention.attend(self.query_norm(target), *memory_kv) ) target = target.clone() + self.feed_forward_dropout(self.position_feed_forward(self.feed_forward_norm(target))) return self.output_norm(target) @@ -284,14 +307,22 @@ def decode_autoregressive(self, features: torch.Tensor, max_len: int | None = No torch.tril(torch.ones((max_length, max_length), device=features.device), diagonal=0).to(dtype=torch.bool) ).int() + # Incremental decoding: project the image features once and the content one token at a time (cached) + memory_kv = self.decoder.cross_attention.project_kv(features, features) + content_kv: tuple[torch.Tensor, torch.Tensor] | None = None pos_logits = [] for i in range(max_length): + # Content of the last token: its embedding, with the position it is read at (none for SOS) + content = self.embed(ys[:, i : i + 1]) + if i > 0: + content = self.pos_queries[:, i - 1 : i] + content + key, value = self.decoder.project_content(self.dropout(content)) + if content_kv is not None: + key, value = torch.cat([content_kv[0], key], dim=2), torch.cat([content_kv[1], value], dim=2) + content_kv = (key, value) # Decode one token at a time without providing information about the future tokens - tgt_out = self.decode( - ys[:, : i + 1], - features, - query_mask[i : i + 1, : i + 1], - target_query=pos_queries[:, i : i + 1], + tgt_out = self.decoder( + self.dropout(pos_queries[:, i : i + 1]), None, None, content_kv=content_kv, memory_kv=memory_kv ) pos_prob = self.head(tgt_out) pos_logits.append(pos_prob) diff --git a/tests/common/test_models.py b/tests/common/test_models.py index 2d0507f891..bb6ea3973e 100644 --- a/tests/common/test_models.py +++ b/tests/common/test_models.py @@ -199,6 +199,28 @@ def test_aggregate_confidence_edge_cases(): aggregate_confidence([0.5], invalid) +def test_detect_straightened(): + # Only the pages changed by the straightening or the masking are detected again + calls = [] + + def det_predictor(pages, **kwargs): + calls.append(len(pages)) + return [f"redetected_{idx}" for idx in range(len(pages))] + + predictor = _OCRPredictor.__new__(_OCRPredictor) + predictor.det_predictor = det_predictor + orig = [np.zeros((4, 4, 3), dtype=np.uint8) for _ in range(4)] + straight = [orig[0].copy(), np.ones((4, 4, 3), dtype=np.uint8), np.zeros((4, 5, 3), dtype=np.uint8), orig[3].copy()] + masked = [straight[0], straight[1], straight[2], straight[3].copy()] # the last page is masked (a copy) + out = predictor._detect_straightened(["first_0", "first_1", "first_2", "first_3"], orig, straight, masked) + assert out == ["first_0", "redetected_0", "redetected_1", "redetected_2"] + assert calls == [3] + # Nothing changed: no detection at all + calls.clear() + assert predictor._detect_straightened(["a"], orig[:1], straight[:1], straight[:1]) == ["a"] + assert calls == [] + + def test_align_hook_output(): boxes = [ np.array([[0.1, 0.1, 0.2, 0.2], [0.5, 0.5, 0.6, 0.6], [0.8, 0.1, 0.9, 0.2]], dtype=np.float32), diff --git a/tests/common/test_models_builder.py b/tests/common/test_models_builder.py index 186d6b8c8a..f5f700b86b 100644 --- a/tests/common/test_models_builder.py +++ b/tests/common/test_models_builder.py @@ -531,6 +531,9 @@ def test_documentbuilder_tables(): assert page.tables == [] and page.export()["tables"] == [] assert sorted(_words(page)) == sorted(_TABLE_WORDS) + +def test_documentbuilder_tables_straight_cells(): + doc_builder = builder.DocumentBuilder() # Straight-mode table predictions store cells as flat (xmin, ymin, xmax, ymax) boxes # Regression test: this format used to crash the word-to-cell assignment with an IndexError straight = { @@ -544,6 +547,9 @@ def test_documentbuilder_tables(): assert len(page.tables) == 1 and page.tables[0].to_grid() == [["Name", "Age"]] assert _words(page) == ["caption"] + +def test_documentbuilder_tables_cell_scores(): + doc_builder = builder.DocumentBuilder() # Each cell keeps the score of the table structure model, next to the recognition confidence of its words table = { "cells": [ @@ -563,6 +569,9 @@ def test_documentbuilder_tables(): assert page.tables[0].confidence == pytest.approx(0.575) assert [cell["objectness_score"] for cell in page.export()["tables"][0]["cells"]] == [0.8, 0.6, 0.4, 0.5] + +def test_documentbuilder_tables_multi_word_cells(): + doc_builder = builder.DocumentBuilder() # Multi-word cells are read line by line, left to right words = ["world", "hello", "second", "line", "solo"] boxes = [ @@ -586,6 +595,9 @@ def test_documentbuilder_tables(): boxes = [[0.15, 0.15, 0.15, 0.15], [0.16, 0.15, 0.16, 0.15]] assert doc_builder(*_word_args(["a", "b"], boxes), tables=[[table]]).pages[0].tables[0].to_grid() == [["a b"]] + +def test_documentbuilder_tables_word_assignment(): + doc_builder = builder.DocumentBuilder() # A word in the gutter goes to its nearest cell, a word far away from the table stays in the regular blocks words = ["left", "right", "outside", "faraway"] boxes = [ @@ -604,6 +616,9 @@ def test_documentbuilder_tables(): assert grid[0][0] == "left" and "outside" in grid[0][1] assert _words(page) == ["faraway"] + +def test_documentbuilder_tables_rotated(): + doc_builder = builder.DocumentBuilder() # Rotated pages keep 4-point polygons for the table and its cells cell_boxes = [ (0.10, 0.10, 0.28, 0.20), diff --git a/tests/conftest.py b/tests/conftest.py index 0e33c61d6b..9506453b0e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import json +import os import shutil import tempfile from io import BytesIO @@ -8,6 +9,7 @@ import pytest import requests import scipy.io as sio +import torch from PIL import Image from doctr.datasets.generator.base import synthesize_text_img @@ -15,6 +17,24 @@ from doctr.utils import geometry +def pytest_configure(config): + # With pytest-xdist (`-n`), every worker would start one PyTorch / OpenCV thread per core: the workers then compete + # for the cores, and CPU-bound tests (e.g. autoregressive decoding) run 100x slower. Share the cores between them. + num_workers = int(os.environ.get("PYTEST_XDIST_WORKER_COUNT", 0)) + if num_workers > 1: + num_threads = max(1, (os.cpu_count() or 1) // num_workers) + torch.set_num_threads(num_threads) + cv2.setNumThreads(num_threads) + + +@pytest.fixture(autouse=True) +def _release_gpu_memory(): + yield + # With pytest-xdist, the workers share the GPU: give back the memory cached by a test, so that the others can use it + if int(os.environ.get("PYTEST_XDIST_WORKER_COUNT", 0)) > 1 and torch.cuda.is_available(): + torch.cuda.empty_cache() + + @pytest.fixture(scope="session") def mock_vocab(): return ( diff --git a/tests/pytorch/test_datasets_pt.py b/tests/pytorch/test_datasets_pt.py index fa80b2d952..de6b93d178 100644 --- a/tests/pytorch/test_datasets_pt.py +++ b/tests/pytorch/test_datasets_pt.py @@ -1,5 +1,6 @@ import json import os +from contextlib import contextmanager from shutil import move import numpy as np @@ -14,6 +15,16 @@ from doctr.transforms import Resize, SampleCompose +@contextmanager +def _moved_away(path): + # Remove a file of a session-wide fixture folder for the duration of a check, and restore it even if the check fails + move(path, f"{path}.tmp") + try: + yield + finally: + move(f"{path}.tmp", path) + + def _validate_dataset(ds, input_size, batch_size=2, class_indices=False, is_polygons=False): # Fetch one sample sample = ds[0] @@ -179,10 +190,8 @@ def test_detection_dataset(mock_image_folder, mock_detection_label): # File existence check img_name, _ = ds.data[0] - move(os.path.join(ds.root, img_name), os.path.join(ds.root, "tmp_file")) - with pytest.raises(FileNotFoundError): + with _moved_away(os.path.join(ds.root, img_name)), pytest.raises(FileNotFoundError): datasets.DetectionDataset(mock_image_folder, mock_detection_label) - move(os.path.join(ds.root, "tmp_file"), os.path.join(ds.root, img_name)) @pytest.mark.parametrize( @@ -236,74 +245,53 @@ def test_layout_dataset(mock_image_folder, mock_layout_label, use_polygons): assert set(target.keys()) == expected_classes assert all(isinstance(v, np.ndarray) for v in target.values()) - # File existence check - img_name, _ = ds.data[0] - move(os.path.join(ds.root, img_name), os.path.join(ds.root, "tmp_file")) - with pytest.raises(FileNotFoundError): - datasets.LayoutDataset( - img_folder=mock_image_folder, - label_path=mock_layout_label, - ) - move(os.path.join(ds.root, "tmp_file"), os.path.join(ds.root, img_name)) - with pytest.raises(FileNotFoundError): - datasets.LayoutDataset( - img_folder=mock_image_folder, - label_path="/tmp/does_not_exist.json", - ) +def _broken_layout_labels(label_path, tmp_path, first_label): + # Copy of the labels with the first entry replaced: the session-wide fixtures are never modified + with open(label_path) as f: + labels = json.load(f) + labels[next(iter(labels))] = first_label + path = tmp_path / "labels.json" + path.write_text(json.dumps(labels)) + return str(path) - with open(mock_layout_label) as f: - original_labels = json.load(f) - first_key = next(iter(original_labels)) - - test_cases = [ - ( - {"classes": ["Text"]}, - KeyError, - "missing 'polygons'", - ), - ( - {"polygons": [[[0, 0], [1, 0], [1, 1], [0, 1]]]}, - KeyError, - "missing 'classes'", - ), - ( - { - "polygons": [ - [[0, 0], [1, 0], [1, 1], [0, 1]], - [[0, 0], [1, 0], [1, 1], [0, 1]], - ], - "classes": ["Text"], - }, + +@pytest.mark.parametrize( + "first_label, exc_type, match", + [ + [{"classes": ["Text"]}, KeyError, "missing 'polygons'"], + [{"polygons": [[[0, 0], [1, 0], [1, 1], [0, 1]]]}, KeyError, "missing 'classes'"], + [ + {"polygons": [[[0, 0], [1, 0], [1, 1], [0, 1]], [[0, 0], [1, 0], [1, 1], [0, 1]]], "classes": ["Text"]}, ValueError, "number of polygons", - ), - ( - { - "polygons": [[[0, 0], [1, 0], [1, 1]]], # only 3 points - "classes": ["Text"], - }, + ], + # Only 3 points + [ + {"polygons": [[[0, 0], [1, 0], [1, 1]]], "classes": ["Text"]}, ValueError, "polygons are expected to have shape", - ), - ] - - for sample, exc_type, match in test_cases: - broken_labels = dict(original_labels) - broken_labels[first_key] = sample - - with open(mock_layout_label, "w") as f: - json.dump(broken_labels, f) + ], + ], +) +def test_layout_dataset_invalid_labels(mock_image_folder, mock_layout_label, tmp_path, first_label, exc_type, match): + label_path = _broken_layout_labels(mock_layout_label, tmp_path, first_label) + with pytest.raises(exc_type, match=match): + datasets.LayoutDataset(img_folder=mock_image_folder, label_path=label_path) - with pytest.raises(exc_type, match=match): - datasets.LayoutDataset( - img_folder=mock_image_folder, - label_path=mock_layout_label, - ) - # Restore original labels - with open(mock_layout_label, "w") as f: - json.dump(original_labels, f) +def test_layout_dataset_missing_files(mock_image_folder, mock_layout_label, tmp_path): + # A labeled image that is not in the folder + with open(mock_layout_label) as f: + labels = json.load(f) + labels["missing_image.jpeg"] = next(iter(labels.values())) + label_path = tmp_path / "labels.json" + label_path.write_text(json.dumps(labels)) + with pytest.raises(FileNotFoundError): + datasets.LayoutDataset(img_folder=mock_image_folder, label_path=str(label_path)) + # A missing label file + with pytest.raises(FileNotFoundError): + datasets.LayoutDataset(img_folder=mock_image_folder, label_path=str(tmp_path / "does_not_exist.json")) @pytest.mark.parametrize("use_polygons", [False, True]) @@ -344,10 +332,8 @@ def test_table_dataset(mock_image_folder, mock_table_label, use_polygons): # File existence check img_name, _ = ds.data[0] - move(os.path.join(mock_image_folder, img_name), os.path.join(mock_image_folder, "tmp_file")) - with pytest.raises(FileNotFoundError): + with _moved_away(os.path.join(mock_image_folder, img_name)), pytest.raises(FileNotFoundError): datasets.TableStructureDataset(mock_image_folder, mock_table_label) - move(os.path.join(mock_image_folder, "tmp_file"), os.path.join(mock_image_folder, img_name)) def test_recognition_dataset(mock_image_folder, mock_recognition_label, monkeypatch): @@ -372,10 +358,8 @@ def test_recognition_dataset(mock_image_folder, mock_recognition_label, monkeypa # File existence check img_name, _ = ds.data[0] - move(os.path.join(ds.root, img_name), os.path.join(ds.root, "tmp_file")) - with pytest.raises(FileNotFoundError): + with _moved_away(os.path.join(ds.root, img_name)), pytest.raises(FileNotFoundError): datasets.RecognitionDataset(mock_image_folder, mock_recognition_label) - move(os.path.join(ds.root, "tmp_file"), os.path.join(ds.root, img_name)) # Merge datasets with a relative image folder monkeypatch.chdir(os.path.dirname(mock_image_folder)) @@ -416,10 +400,8 @@ def test_ocrdataset(mock_ocrdataset, use_polygons): # File existence check img_name, _ = ds.data[0] - move(os.path.join(ds.root, img_name), os.path.join(ds.root, "tmp_file")) - with pytest.raises(FileNotFoundError): + with _moved_away(os.path.join(ds.root, img_name)), pytest.raises(FileNotFoundError): datasets.OCRDataset(*mock_ocrdataset) - move(os.path.join(ds.root, "tmp_file"), os.path.join(ds.root, img_name)) def test_charactergenerator(): diff --git a/tests/pytorch/test_io_image_pt.py b/tests/pytorch/test_io_image_pt.py index 37f81c8ce8..5f60297dd6 100644 --- a/tests/pytorch/test_io_image_pt.py +++ b/tests/pytorch/test_io_image_pt.py @@ -65,6 +65,16 @@ def test_tensor_from_numpy(mock_image_stream): assert torch.equal(tensor_from_pil(Image.fromarray(gray), dtype=torch.uint8)[0], torch.from_numpy(gray)) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.uint8]) +def test_tensor_from_numpy_views(dtype): + # Flipped / rotated views have negative strides, which torch.from_numpy rejects, and read-only arrays warn + img = np.random.default_rng(0).integers(0, 256, (16, 24, 3), dtype=np.uint8) + read_only = img.copy() + read_only.flags.writeable = False + for view in (img[::-1], img[:, ::-1], np.rot90(img), np.rot90(img[:1], 3), read_only): + assert torch.equal(tensor_from_numpy(view, dtype), tensor_from_numpy(view.copy(), dtype)) + + def test_read_img_as_tensor_exif_orientation(tmpdir_factory): folder = tmpdir_factory.mktemp("images") path = str(folder.join("exif_o6.jpg")) diff --git a/tests/pytorch/test_models_layout.py b/tests/pytorch/test_models_layout_pt.py similarity index 100% rename from tests/pytorch/test_models_layout.py rename to tests/pytorch/test_models_layout_pt.py diff --git a/tests/pytorch/test_models_preprocessor_pt.py b/tests/pytorch/test_models_preprocessor_pt.py index 9b4d9a668d..7264425214 100644 --- a/tests/pytorch/test_models_preprocessor_pt.py +++ b/tests/pytorch/test_models_preprocessor_pt.py @@ -59,3 +59,17 @@ def test_preprocessor(batch_size, output_size, input_tensor, expected_batches, e # mask sanity: should contain both 0 and 1 somewhere (padding depends on Resize logic) assert all(torch.is_tensor(m) for m in masks) assert len(repr(processor).split("\n")) == 4 + + +def test_preprocessor_views(): + # Rectified crops are rotated views: np.rot90 of a crop of 1 pixel in height has a negative stride on an axis of + # size 1, which numpy considers contiguous, but torch.from_numpy rejects + crop = np.random.default_rng(0).integers(0, 256, (1, 40, 3), dtype=np.uint8) + views = [np.rot90(crop, k) for k in range(4)] + [crop[:, ::-1], crop[..., ::-1]] + processor = PreProcessor((32, 128), batch_size=8, preserve_aspect_ratio=True) + expected = processor([view.copy() for view in views]) + out = processor(views) + assert len(out) == 1 and torch.equal(out[0], expected[0]) + # Same for a batch of pages passed as a single array + pages = np.rot90(np.repeat(crop[None], 2, axis=0), 3, axes=(1, 2)) + assert torch.equal(processor(pages)[0], processor(pages.copy())[0]) diff --git a/tests/pytorch/test_transforms_pt.py b/tests/pytorch/test_transforms_pt.py index d0336c3fdb..1ccb0200c2 100644 --- a/tests/pytorch/test_transforms_pt.py +++ b/tests/pytorch/test_transforms_pt.py @@ -31,180 +31,111 @@ from doctr.utils import Sample -def test_resize(): - output_size = (32, 32) - transfo = Resize(output_size) - input_t = Sample(image=torch.ones((3, 64, 64), dtype=torch.float32)) - out = transfo(input_t).image +def _ones(h, w, dtype=torch.float32): + return torch.ones((3, h, w), dtype=dtype) - assert torch.all(out == 1) - assert out.shape[-2:] == output_size - assert repr(transfo) == "Resize(output_size=(32, 32), interpolation='bilinear')" - # Test return_padding_mask without aspect ratio - transfo = Resize(output_size, return_padding_mask=True) - data = transfo(input_t) - out, mask = data.image, data.mask - assert out.shape[-2:] == output_size - assert mask.shape == output_size - assert mask.dtype == torch.bool - # No padding was added: every pixel is valid image content - assert torch.all(mask) +_BOXES = np.array([[0.1, 0.1, 0.3, 0.4], [0.2, 0.2, 0.8, 0.8]]) +_POLYGONS = np.array([ + [[0.1, 0.1], [0.9, 0.1], [0.9, 0.9], [0.1, 0.9]], + [[0.2, 0.2], [0.8, 0.2], [0.8, 0.8], [0.2, 0.8]], +]) - # Test with preserve_aspect_ratio - output_size = (32, 32) - input_t = Sample(image=torch.ones((3, 32, 64), dtype=torch.float32)) - # Asymmetric padding - transfo = Resize(output_size, preserve_aspect_ratio=True) - out = transfo(input_t).image - assert out.shape[-2:] == output_size - assert not torch.all(out == 1) - assert torch.all(out[:, -1] == 0) and torch.all(out[:, 0] == 1) +def test_resize(): + transfo = Resize((32, 32)) + out = transfo(Sample(image=_ones(64, 64))).image + assert torch.all(out == 1) + assert out.shape[-2:] == (32, 32) + assert repr(transfo) == "Resize(output_size=(32, 32), interpolation='bilinear')" - # Asymmetric padding mask: the (32, 64) image is scaled to (16, 32) and padded at the bottom - transfo = Resize(output_size, preserve_aspect_ratio=True, return_padding_mask=True) - data = transfo(input_t) - out, mask = data.image, data.mask - assert mask.shape == output_size - assert mask.dtype == torch.bool - assert torch.all(mask[:16]) # image content is marked as valid (True) - assert not mask[16:].any() # bottom padding is masked out (False) + # A plain (stretching) resize adds no padding: every pixel is valid image content + data = Resize((32, 32), return_padding_mask=True)(Sample(image=_ones(64, 64))) + assert isinstance(data, Sample) + assert data.image.shape[-2:] == (32, 32) + assert data.mask.shape == (32, 32) and data.mask.dtype == torch.bool + assert torch.all(data.mask) - # Symmetric padding - transfo = Resize(32, preserve_aspect_ratio=True, symmetric_pad=True) - out = transfo(input_t).image - assert out.shape[-2:] == output_size - assert torch.all(out[:, 0] == 0) and torch.all(out[:, -1] == 0) - # Symmetric padding mask: the (32, 64) image is scaled to (16, 32) and padded top & bottom (8 rows each) - transfo = Resize(32, preserve_aspect_ratio=True, symmetric_pad=True, return_padding_mask=True) - data = transfo(input_t) +@pytest.mark.parametrize( + "symmetric_pad, content_rows", + [ + # The (32, 64) image is scaled to (16, 32), then padded at the bottom... + [False, slice(0, 16)], + # ... or on both sides (8 rows each) + [True, slice(8, 24)], + ], +) +def test_resize_preserve_aspect_ratio(symmetric_pad, content_rows): + transfo = Resize((32, 32), preserve_aspect_ratio=True, symmetric_pad=symmetric_pad, return_padding_mask=True) + data = transfo(Sample(image=_ones(32, 64))) out, mask = data.image, data.mask - assert mask.shape == output_size - assert mask.dtype == torch.bool - assert torch.all(mask[8:24]) # centered image content is marked as valid (True) - assert not mask[:8].any() and not mask[24:].any() # symmetric padding is masked out (False) - - expected = "Resize(output_size=(32, 32), interpolation='bilinear', preserve_aspect_ratio=True, symmetric_pad=True)" - assert repr(transfo) == expected - # Test with inverse resize - input_t = Sample(image=torch.ones((3, 64, 32), dtype=torch.float32)) - transfo = Resize(32, preserve_aspect_ratio=True, symmetric_pad=True) - out = transfo(input_t).image assert out.shape[-2:] == (32, 32) + assert torch.all(out[:, content_rows] == 1) + assert torch.all(out[:, -1] == 0) + assert torch.all(out[:, 0] == (0 if symmetric_pad else 1)) + # Image content is marked as valid (True), padding is masked out (False) + assert mask.shape == (32, 32) and mask.dtype == torch.bool + expected_mask = torch.zeros((32, 32), dtype=torch.bool) + expected_mask[content_rows] = True + assert torch.equal(mask, expected_mask) - # Test resize with same ratio - transfo = Resize((32, 128), preserve_aspect_ratio=True) - out = transfo(Sample(image=torch.ones((3, 16, 64), dtype=torch.float32))).image - assert out.shape[-2:] == (32, 128) - # Test with fp16 input +def test_resize_repr_and_shapes(): + transfo = Resize(32, preserve_aspect_ratio=True, symmetric_pad=True) + expected = "Resize(output_size=(32, 32), interpolation='bilinear', preserve_aspect_ratio=True, symmetric_pad=True)" + assert repr(transfo) == expected + # Portrait input + assert transfo(Sample(image=_ones(64, 32))).image.shape[-2:] == (32, 32) + # Input with the same aspect ratio as the output transfo = Resize((32, 128), preserve_aspect_ratio=True) - input_t = Sample(image=torch.ones((3, 64, 64), dtype=torch.float16)) - out = transfo(input_t).image - assert out.dtype == torch.float16 + assert transfo(Sample(image=_ones(16, 64))).image.shape[-2:] == (32, 128) + # The data type is kept + assert transfo(Sample(image=_ones(64, 64, dtype=torch.float16))).image.dtype == torch.float16 - padding = [True, False] - for symmetric_pad in padding: - # Test with target boxes - target_boxes = np.array([[0.1, 0.1, 0.3, 0.4], [0.2, 0.2, 0.8, 0.8]]) - transfo = Resize((64, 64), preserve_aspect_ratio=True, symmetric_pad=symmetric_pad, return_padding_mask=True) - input_t = Sample(image=torch.ones((3, 32, 64), dtype=torch.float32), target=target_boxes) - data = transfo(input_t) - out, mask, new_target = data.image, data.mask, data.target - - assert out.shape[-2:] == (64, 64) - assert new_target.shape == target_boxes.shape - assert np.all((0 <= new_target) & (new_target <= 1)) - assert mask.shape == (64, 64) - assert mask.dtype == torch.bool - - # Test with target polygons - target_boxes = np.array([ - [[0.1, 0.1], [0.9, 0.1], [0.9, 0.9], [0.1, 0.9]], - [[0.2, 0.2], [0.8, 0.2], [0.8, 0.8], [0.2, 0.8]], - ]) - transfo = Resize((64, 64), preserve_aspect_ratio=True, symmetric_pad=symmetric_pad, return_padding_mask=True) - input_t = Sample(image=torch.ones((3, 32, 64), dtype=torch.float32), target=target_boxes) - data = transfo(input_t) - out, mask, new_target = data.image, data.mask, data.target - - assert out.shape[-2:] == (64, 64) - assert new_target.shape == target_boxes.shape - assert np.all((0 <= new_target) & (new_target <= 1)) - assert mask.shape == (64, 64) - assert mask.dtype == torch.bool - - # Test with included mask in input sample - input_t = Sample( - image=torch.ones((3, 32, 64), dtype=torch.float32), - mask=torch.zeros((32, 64), dtype=torch.bool), - target=target_boxes, - ) - data = transfo(input_t) - out, mask, new_target = data.image, data.mask, data.target - assert out.shape[-2:] == (64, 64) - assert new_target.shape == target_boxes.shape - assert np.all((0 <= new_target) & (new_target <= 1)) - assert mask.shape == (64, 64) - assert mask.dtype == torch.bool +@pytest.mark.parametrize("symmetric_pad", [True, False]) +@pytest.mark.parametrize("target", [_BOXES, _POLYGONS], ids=["boxes", "polygons"]) +@pytest.mark.parametrize("input_mask", [None, torch.zeros((32, 64), dtype=torch.bool)], ids=["no_mask", "mask"]) +def test_resize_targets(symmetric_pad, target, input_mask): + transfo = Resize((64, 64), preserve_aspect_ratio=True, symmetric_pad=symmetric_pad, return_padding_mask=True) + data = transfo(Sample(image=_ones(32, 64), mask=input_mask, target=target)) - # Test with invalid target shape - input_t = torch.ones((3, 32, 64), dtype=torch.float32) - target = np.ones((2, 5)) # Invalid shape + assert data.image.shape[-2:] == (64, 64) + assert data.target.shape == target.shape + assert np.all((0 <= data.target) & (data.target <= 1)) + assert data.mask.shape == (64, 64) and data.mask.dtype == torch.bool - transfo = Resize((64, 64), preserve_aspect_ratio=True) - with pytest.raises(AssertionError): - transfo(Sample(image=input_t, target=target)) - # Test dict targets - target_dict = { +def test_resize_dict_targets(): + target = { "boxes": np.array([[0.1, 0.1, 0.9, 0.9]], dtype=np.float32), "polygons": np.array([[[0.2, 0.2], [0.8, 0.2], [0.8, 0.8], [0.2, 0.8]]], dtype=np.float32), } - transfo = Resize( - (64, 64), - preserve_aspect_ratio=True, - symmetric_pad=True, - ) - new_target = transfo(Sample(image=input_t, target=target_dict)).target + new_target = Resize((64, 64), preserve_aspect_ratio=True, symmetric_pad=True)( + Sample(image=_ones(32, 64), target=target) + ).target assert isinstance(new_target, dict) assert set(new_target.keys()) == {"boxes", "polygons"} assert new_target["boxes"].shape == (1, 4) assert new_target["polygons"].shape == (1, 4, 2) - # Test return type combinations - transfo = Resize((32, 32)) - out = transfo(Sample(image=input_t)).image - assert isinstance(out, torch.Tensor) - transfo = Resize((32, 32), return_padding_mask=True) - out = transfo(Sample(image=input_t)) - assert isinstance(out, Sample) - assert hasattr(out, "image") and hasattr(out, "mask") - assert out.image.shape[-2:] == (32, 32) - assert out.mask.shape[-2:] == (32, 32) - - transfo = Resize((32, 32), preserve_aspect_ratio=True) - out = transfo(Sample(image=input_t, target=target_boxes)) - assert isinstance(out, Sample) - assert hasattr(out, "image") and hasattr(out, "target") - assert out.image.shape[-2:] == (32, 32) +def test_resize_invalid_target(): + with pytest.raises(AssertionError): + Resize((64, 64), preserve_aspect_ratio=True)(Sample(image=_ones(32, 64), target=np.ones((2, 5)))) - transfo = Resize( - (32, 32), - preserve_aspect_ratio=True, - return_padding_mask=True, - ) - out = transfo(Sample(image=input_t, target=target_boxes)) +@pytest.mark.parametrize("return_padding_mask", [True, False]) +def test_resize_returns_sample(return_padding_mask): + transfo = Resize((32, 32), preserve_aspect_ratio=True, return_padding_mask=return_padding_mask) + out = transfo(Sample(image=_ones(32, 64), target=_POLYGONS)) assert isinstance(out, Sample) - assert hasattr(out, "image") and hasattr(out, "mask") and hasattr(out, "target") assert out.image.shape[-2:] == (32, 32) - assert out.mask.shape[-2:] == (32, 32) + assert out.target is not None + if return_padding_mask: + assert out.mask.shape[-2:] == (32, 32) @pytest.mark.parametrize(