python-doctr 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

doctr/datasets/sroie.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2021-2024, Mindee.
+# Copyright (C) 2021-2025, Mindee.
 
 # This program is licensed under the Apache License 2.0.
 # See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
@@ -6,7 +6,7 @@
 import csv
 import os
 from pathlib import Path
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any
 
 import numpy as np
 from tqdm import tqdm
@@ -29,7 +29,6 @@ class SROIE(VisionDataset):
     >>> img, target = train_set[0]
 
     Args:
-    ----
         train: whether the subset should be the training one
         use_polygons: whether polygons should be considered as rotated bounding box (instead of straight ones)
         recognition_task: whether the dataset should be used for recognition task
@@ -74,10 +73,12 @@ class SROIE(VisionDataset):
         self.train = train
 
         tmp_root = os.path.join(self.root, "images")
-        self.data: List[Tuple[Union[str, np.ndarray], Union[str, Dict[str, Any], np.ndarray]]] = []
+        self.data: list[tuple[str | np.ndarray, str | dict[str, Any] | np.ndarray]] = []
         np_dtype = np.float32
 
-        for img_path in tqdm(iterable=os.listdir(tmp_root), desc="Unpacking SROIE", total=len(os.listdir(tmp_root))):
+        for img_path in tqdm(
+            iterable=os.listdir(tmp_root), desc="Preparing and Loading SROIE", total=len(os.listdir(tmp_root))
+        ):
             # File existence check
             if not os.path.exists(os.path.join(tmp_root, img_path)):
                 raise FileNotFoundError(f"unable to locate {os.path.join(tmp_root, img_path)}")

doctr/datasets/svhn.py

@@ -1,10 +1,10 @@
-# Copyright (C) 2021-2024, Mindee.
+# Copyright (C) 2021-2025, Mindee.
 
 # This program is licensed under the Apache License 2.0.
 # See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
 
 import os
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any
 
 import h5py
 import numpy as np
@@ -28,7 +28,6 @@ class SVHN(VisionDataset):
     >>> img, target = train_set[0]
 
     Args:
-    ----
         train: whether the subset should be the training one
         use_polygons: whether polygons should be considered as rotated bounding box (instead of straight ones)
         recognition_task: whether the dataset should be used for recognition task
@@ -72,7 +71,7 @@ class SVHN(VisionDataset):
             )
 
         self.train = train
-        self.data: List[Tuple[Union[str, np.ndarray], Union[str, Dict[str, Any], np.ndarray]]] = []
+        self.data: list[tuple[str | np.ndarray, str | dict[str, Any] | np.ndarray]] = []
         np_dtype = np.float32
 
         tmp_root = os.path.join(self.root, "train" if train else "test")
@@ -81,7 +80,9 @@ class SVHN(VisionDataset):
         with h5py.File(os.path.join(tmp_root, "digitStruct.mat"), "r") as f:
             img_refs = f["digitStruct/name"]
             box_refs = f["digitStruct/bbox"]
-            for img_ref, box_ref in tqdm(iterable=zip(img_refs, box_refs), desc="Unpacking SVHN", total=len(img_refs)):
+            for img_ref, box_ref in tqdm(
+                iterable=zip(img_refs, box_refs), desc="Preparing and Loading SVHN", total=len(img_refs)
+            ):
                 # convert ascii matrix to string
                 img_name = "".join(map(chr, f[img_ref[0]][()].flatten()))
 
@@ -128,7 +129,7 @@ class SVHN(VisionDataset):
                 if recognition_task:
                     crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, img_name), geoms=box_targets)
                     for crop, label in zip(crops, label_targets):
-                        if crop.shape[0] > 0 and crop.shape[1] > 0 and len(label) > 0:
+                        if crop.shape[0] > 0 and crop.shape[1] > 0 and len(label) > 0 and " " not in label:
                             self.data.append((crop, label))
                 elif detection_task:
                     self.data.append((img_name, box_targets))

doctr/datasets/svt.py

@@ -1,10 +1,10 @@
-# Copyright (C) 2021-2024, Mindee.
+# Copyright (C) 2021-2025, Mindee.
 
 # This program is licensed under the Apache License 2.0.
 # See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
 
 import os
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any
 
 import defusedxml.ElementTree as ET
 import numpy as np
@@ -28,7 +28,6 @@ class SVT(VisionDataset):
     >>> img, target = train_set[0]
 
     Args:
-    ----
         train: whether the subset should be the training one
         use_polygons: whether polygons should be considered as rotated bounding box (instead of straight ones)
         recognition_task: whether the dataset should be used for recognition task
@@ -36,7 +35,7 @@ class SVT(VisionDataset):
         **kwargs: keyword arguments from `VisionDataset`.
     """
 
-    URL = "http://vision.ucsd.edu/~kai/svt/svt.zip"
+    URL = "http://www.iapr-tc11.org/dataset/SVT/svt.zip"
     SHA256 = "63b3d55e6b6d1e036e2a844a20c034fe3af3c32e4d914d6e0c4a3cd43df3bebf"
 
     def __init__(
@@ -62,7 +61,7 @@ class SVT(VisionDataset):
             )
 
         self.train = train
-        self.data: List[Tuple[Union[str, np.ndarray], Union[str, Dict[str, Any], np.ndarray]]] = []
+        self.data: list[tuple[str | np.ndarray, str | dict[str, Any] | np.ndarray]] = []
         np_dtype = np.float32
 
         # Load xml data
@@ -74,7 +73,7 @@ class SVT(VisionDataset):
         )
         xml_root = xml_tree.getroot()
 
-        for image in tqdm(iterable=xml_root, desc="Unpacking SVT", total=len(xml_root)):
+        for image in tqdm(iterable=xml_root, desc="Preparing and Loading SVT", total=len(xml_root)):
             name, _, _, _resolution, rectangles = image
 
             # File existence check
@@ -114,7 +113,7 @@ class SVT(VisionDataset):
             if recognition_task:
                 crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, name.text), geoms=boxes)
                 for crop, label in zip(crops, labels):
-                    if crop.shape[0] > 0 and crop.shape[1] > 0 and len(label) > 0:
+                    if crop.shape[0] > 0 and crop.shape[1] > 0 and len(label) > 0 and " " not in label:
                         self.data.append((crop, label))
             elif detection_task:
                 self.data.append((name.text, boxes))

doctr/datasets/synthtext.py

@@ -1,11 +1,11 @@
-# Copyright (C) 2021-2024, Mindee.
+# Copyright (C) 2021-2025, Mindee.
 
 # This program is licensed under the Apache License 2.0.
 # See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
 
 import glob
 import os
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any
 
 import numpy as np
 from PIL import Image
@@ -31,7 +31,6 @@ class SynthText(VisionDataset):
     >>> img, target = train_set[0]
 
     Args:
-    ----
         train: whether the subset should be the training one
         use_polygons: whether polygons should be considered as rotated bounding box (instead of straight ones)
         recognition_task: whether the dataset should be used for recognition task
@@ -42,6 +41,12 @@ class SynthText(VisionDataset):
     URL = "https://thor.robots.ox.ac.uk/~vgg/data/scenetext/SynthText.zip"
     SHA256 = "28ab030485ec8df3ed612c568dd71fb2793b9afbfa3a9d9c6e792aef33265bf1"
 
+    # filter corrupted or missing images
+    BLACKLIST = (
+        "67/fruits_129_",
+        "194/window_19_",
+    )
+
     def __init__(
         self,
         train: bool = True,
@@ -65,7 +70,7 @@ class SynthText(VisionDataset):
             )
 
         self.train = train
-        self.data: List[Tuple[Union[str, np.ndarray], Union[str, Dict[str, Any], np.ndarray]]] = []
+        self.data: list[tuple[str | np.ndarray, str | dict[str, Any] | np.ndarray]] = []
         np_dtype = np.float32
 
         # Load mat data
@@ -91,7 +96,7 @@ class SynthText(VisionDataset):
         del mat_data
 
         for img_path, word_boxes, txt in tqdm(
-            iterable=zip(paths, boxes, labels), desc="Unpacking SynthText", total=len(paths)
+            iterable=zip(paths, boxes, labels), desc="Preparing and Loading SynthText", total=len(paths)
         ):
             # File existence check
             if not os.path.exists(os.path.join(tmp_root, img_path[0])):
@@ -112,7 +117,13 @@ class SynthText(VisionDataset):
             if recognition_task:
                 crops = crop_bboxes_from_image(img_path=os.path.join(tmp_root, img_path[0]), geoms=word_boxes)
                 for crop, label in zip(crops, labels):
-                    if crop.shape[0] > 0 and crop.shape[1] > 0 and len(label) > 0:
+                    if (
+                        crop.shape[0] > 0
+                        and crop.shape[1] > 0
+                        and len(label) > 0
+                        and len(label) < 30
+                        and " " not in label
+                    ):
                         # write data to disk
                         with open(os.path.join(reco_folder_path, f"{reco_images_counter}.txt"), "w") as f:
                             f.write(label)
@@ -133,6 +144,7 @@ class SynthText(VisionDataset):
         return f"train={self.train}"
 
     def _read_from_folder(self, path: str) -> None:
-        for img_path in glob.glob(os.path.join(path, "*.png")):
+        img_paths = glob.glob(os.path.join(path, "*.png"))
+        for img_path in tqdm(iterable=img_paths, desc="Preparing and Loading SynthText", total=len(img_paths)):
             with open(os.path.join(path, f"{os.path.basename(img_path)[:-4]}.txt"), "r") as f:
                 self.data.append((img_path, f.read()))

doctr/datasets/utils.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2021-2024, Mindee.
+# Copyright (C) 2021-2025, Mindee.
 
 # This program is licensed under the Apache License 2.0.
 # See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
@@ -6,10 +6,10 @@
 import string
 import unicodedata
 from collections.abc import Sequence
+from collections.abc import Sequence as SequenceType
 from functools import partial
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, TypeVar, Union
-from typing import Sequence as SequenceType
+from typing import Any, TypeVar
 
 import numpy as np
 from PIL import Image
@@ -19,7 +19,15 @@ from doctr.utils.geometry import convert_to_relative_coords, extract_crops, extr
 
 from .vocabs import VOCABS
 
-__all__ = ["translate", "encode_string", "decode_sequence", "encode_sequences", "pre_transform_multiclass"]
+__all__ = [
+    "translate",
+    "encode_string",
+    "decode_sequence",
+    "encode_sequences",
+    "pre_transform_multiclass",
+    "crop_bboxes_from_image",
+    "convert_target_to_relative",
+]
 
 ImageTensor = TypeVar("ImageTensor")
 
@@ -32,17 +40,15 @@ def translate(
     """Translate a string input in a given vocabulary
 
     Args:
-    ----
         input_string: input string to translate
         vocab_name: vocabulary to use (french, latin, ...)
         unknown_char: unknown character for non-translatable characters
 
     Returns:
-    -------
         A string translated in a given vocab
     """
     if VOCABS.get(vocab_name) is None:
-        raise KeyError("output vocabulary must be in vocabs dictionnary")
+        raise KeyError("output vocabulary must be in vocabs dictionary")
 
     translated = ""
     for char in input_string:
@@ -63,40 +69,37 @@ def translate(
 def encode_string(
     input_string: str,
     vocab: str,
-) -> List[int]:
+) -> list[int]:
     """Given a predefined mapping, encode the string to a sequence of numbers
 
     Args:
-    ----
         input_string: string to encode
         vocab: vocabulary (string), the encoding is given by the indexing of the character sequence
 
     Returns:
-    -------
         A list encoding the input_string
     """
     try:
         return list(map(vocab.index, input_string))
-    except ValueError:
+    except ValueError as e:
+        missing_chars = [char for char in input_string if char not in vocab]
         raise ValueError(
-            f"some characters cannot be found in 'vocab'. \
-                         Please check the input string {input_string} and the vocabulary {vocab}"
-        )
+            f"Some characters cannot be found in 'vocab': {set(missing_chars)}.\n"
+            f"Please check the input string `{input_string}` and the vocabulary `{vocab}`"
+        ) from e
 
 
 def decode_sequence(
-    input_seq: Union[np.ndarray, SequenceType[int]],
+    input_seq: np.ndarray | SequenceType[int],
     mapping: str,
 ) -> str:
     """Given a predefined mapping, decode the sequence of numbers to a string
 
     Args:
-    ----
         input_seq: array to decode
         mapping: vocabulary (string), the encoding is given by the indexing of the character sequence
 
     Returns:
-    -------
         A string, decoded from input_seq
     """
     if not isinstance(input_seq, (Sequence, np.ndarray)):
@@ -108,18 +111,17 @@ def decode_sequence(
 
 
 def encode_sequences(
-    sequences: List[str],
+    sequences: list[str],
     vocab: str,
-    target_size: Optional[int] = None,
+    target_size: int | None = None,
     eos: int = -1,
-    sos: Optional[int] = None,
-    pad: Optional[int] = None,
+    sos: int | None = None,
+    pad: int | None = None,
     dynamic_seq_length: bool = False,
 ) -> np.ndarray:
     """Encode character sequences using a given vocab as mapping
 
     Args:
-    ----
         sequences: the list of character sequences of size N
         vocab: the ordered vocab to use for encoding
         target_size: maximum length of the encoded data
@@ -129,7 +131,6 @@ def encode_sequences(
         dynamic_seq_length: if `target_size` is specified, uses it as upper bound and enables dynamic sequence size
 
     Returns:
-    -------
         the padded encoded data as a tensor
     """
     if 0 <= eos < len(vocab):
@@ -170,29 +171,36 @@ def encode_sequences(
 
 
 def convert_target_to_relative(
-    img: ImageTensor, target: Union[np.ndarray, Dict[str, Any]]
-) -> Tuple[ImageTensor, Union[Dict[str, Any], np.ndarray]]:
+    img: ImageTensor, target: np.ndarray | dict[str, Any]
+) -> tuple[ImageTensor, dict[str, Any] | np.ndarray]:
+    """Converts target to relative coordinates
+
+    Args:
+        img: tf.Tensor or torch.Tensor representing the image
+        target: target to convert to relative coordinates (boxes (N, 4) or polygons (N, 4, 2))
+
+    Returns:
+        The image and the target in relative coordinates
+    """
     if isinstance(target, np.ndarray):
-        target = convert_to_relative_coords(target, get_img_shape(img))
+        target = convert_to_relative_coords(target, get_img_shape(img))  # type: ignore[arg-type]
     else:
-        target["boxes"] = convert_to_relative_coords(target["boxes"], get_img_shape(img))
+        target["boxes"] = convert_to_relative_coords(target["boxes"], get_img_shape(img))  # type: ignore[arg-type]
     return img, target
 
 
-def crop_bboxes_from_image(img_path: Union[str, Path], geoms: np.ndarray) -> List[np.ndarray]:
+def crop_bboxes_from_image(img_path: str | Path, geoms: np.ndarray) -> list[np.ndarray]:
     """Crop a set of bounding boxes from an image
 
     Args:
-    ----
         img_path: path to the image
         geoms: a array of polygons of shape (N, 4, 2) or of straight boxes of shape (N, 4)
 
     Returns:
-    -------
         a list of cropped images
     """
     with Image.open(img_path) as pil_img:
-        img: np.ndarray = np.array(pil_img.convert("RGB"))
+        img: np.ndarray = np.asarray(pil_img.convert("RGB"))
     # Polygon
     if geoms.ndim == 3 and geoms.shape[1:] == (4, 2):
         return extract_rcrops(img, geoms.astype(dtype=int))
@@ -201,21 +209,19 @@ def crop_bboxes_from_image(img_path: Union[str, Path], geoms: np.ndarray) -> Lis
     raise ValueError("Invalid geometry format")
 
 
-def pre_transform_multiclass(img, target: Tuple[np.ndarray, List]) -> Tuple[np.ndarray, Dict[str, List]]:
+def pre_transform_multiclass(img, target: tuple[np.ndarray, list]) -> tuple[np.ndarray, dict[str, list]]:
     """Converts multiclass target to relative coordinates.
 
     Args:
-    ----
         img: Image
         target: tuple of target polygons and their classes names
 
     Returns:
-    -------
         Image and dictionary of boxes, with class names as keys
     """
     boxes = convert_to_relative_coords(target[0], get_img_shape(img))
     boxes_classes = target[1]
-    boxes_dict: Dict = {k: [] for k in sorted(set(boxes_classes))}
+    boxes_dict: dict = {k: [] for k in sorted(set(boxes_classes))}
     for k, poly in zip(boxes_classes, boxes):
         boxes_dict[k].append(poly)
     boxes_dict = {k: np.stack(v, axis=0) for k, v in boxes_dict.items()}