matrice-analytics 0.1.60__py3-none-any.whl

matrice_analytics/post_processing/ocr/fast_plate_ocr_py38/cli/validate_dataset.py

@@ -0,0 +1,240 @@
+"""
+Validate a `fast-plate-ocr` dataset before training.
+"""
+
+from __future__ import annotations
+
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Optional
+import click
+import pandas as pd
+from PIL import Image, UnidentifiedImageError
+from rich import box
+from rich.console import Console
+from rich.markup import escape
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn
+from rich.table import Table
+
+from fast_plate_ocr.train.model.config import load_plate_config_from_yaml
+
+# pylint: disable=too-many-locals
+
+console = Console()
+
+
+def partial_decode_ok(path: Path) -> tuple[bool, Optional[tuple[int, int]]]:
+    try:
+        with Image.open(path) as im:
+            im.verify()
+            w, h = im.size
+            return True, (h, w)
+    except (UnidentifiedImageError, OSError):
+        return False, None
+
+
+def _validate_dataset(
+    df: pd.DataFrame,
+    cfg,
+    min_h: int,
+    min_w: int,
+) -> tuple[list[tuple[str, str]], list[tuple[str, str]], pd.DataFrame]:
+    """
+    Iterate over the dataframe, collect errors and warnings, and return a cleaned df.
+    """
+    errors, warnings, ok_rows = [], [], []
+    char_counter: Counter[str] = Counter()
+    seen_paths: set[Path] = set()
+
+    progress = Progress(
+        SpinnerColumn(),
+        BarColumn(bar_width=None),
+        TextColumn("{task.completed}/{task.total}"),
+        TextColumn("[bold blue]{task.description}"),
+        transient=True,
+        console=console,
+    )
+    task: TaskID = progress.add_task("Scanning images", total=len(df))
+
+    with progress:
+        for row_idx, row in enumerate(df.itertuples(index=False)):
+            img_path = Path(str(row.image_path))
+            plate = str(row.plate_text)
+            char_counter.update(plate)
+            line_no = str(row_idx + 2)
+
+            # Check file exists
+            if not img_path.exists():
+                errors.append((line_no, f"Missing image file: {img_path}"))
+                progress.update(task, advance=1)
+                continue
+
+            # Check decodable & size
+            ok, shape = partial_decode_ok(img_path)
+            if not ok or shape is None:
+                errors.append((line_no, f"Corrupt or unreadable image: {img_path}"))
+                progress.update(task, advance=1)
+                continue
+
+            h, w = shape
+            if h < min_h or w < min_w:
+                warnings.append((line_no, f"Tiny image ({h}x{w} < {min_h}x{min_w}): {img_path}"))
+
+            # Check resize feasibility (>= 1 px in each axis)
+            r = min(cfg.img_height / h, cfg.img_width / w)
+            new_w, new_h = round(w * r), round(h * r)
+            if new_w == 0 or new_h == 0:
+                errors.append((line_no, f"Resize would give 0x0 ({new_h}x{new_w}) from {img_path}"))
+                progress.update(task, advance=1)
+                continue
+
+            # Check plate text length and alphabet
+            if len(plate) > cfg.max_plate_slots:
+                errors.append(
+                    (
+                        line_no,
+                        f"Plate too long ({len(plate)}>{cfg.max_plate_slots}):"
+                        f" '{plate}' [{img_path}]",
+                    )
+                )
+                progress.update(task, advance=1)
+                continue
+
+            bad_chars = set(plate) - set(cfg.alphabet)
+            if bad_chars:
+                errors.append(
+                    (line_no, f"Invalid chars {bad_chars} in plate '{plate}' [{img_path}]")
+                )
+                progress.update(task, advance=1)
+                continue
+
+            # Check duplicate paths
+            if img_path in seen_paths:
+                warnings.append((line_no, f"Duplicate image entry: {img_path}"))
+                progress.update(task, advance=1)
+                continue
+            seen_paths.add(img_path)
+
+            ok_rows.append(row)
+            progress.update(task, advance=1)
+
+    alphabet_no_pad = cfg.alphabet.replace(cfg.pad_char, "")
+    unused_chars = sorted(set(alphabet_no_pad) - set(char_counter))
+    if unused_chars:
+        warnings.append(("-", f"Character(s) not found in any plate: {', '.join(unused_chars)}"))
+    cleaned_df = pd.DataFrame(ok_rows)
+    return errors, warnings, cleaned_df
+
+
+def rich_report(errors, warnings):
+    summary = Table(title="Validation Summary", box=box.SQUARE, expand=False)
+    summary.add_column("Category", style="bold")
+    summary.add_column("Count", justify="right")
+    summary.add_row("Errors", str(len(errors)), style="red" if errors else "green")
+    summary.add_row("Warnings", str(len(warnings)), style="yellow" if warnings else "green")
+    console.print()
+    console.print(summary)
+
+    def dump(name, rows, style):
+        if not rows:
+            return
+        tbl = Table(title=name, box=box.SQUARE, header_style=style, show_lines=False, expand=False)
+        tbl.add_column("Line", justify="right", style=style)
+        tbl.add_column("Message", style=style, overflow="fold")
+
+        for line_no, msg in rows:
+            tbl.add_row(str(line_no), escape(msg))
+        console.print()
+        console.print(tbl)
+
+    dump("Errors", errors, "red")
+    dump("Warnings", warnings, "yellow")
+
+
+@click.command(context_settings={"max_content_width": 120})
+@click.option(
+    "--annotations-file",
+    "-a",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path, resolve_path=True),
+    help="CSV with image_path and plate_text columns.",
+)
+@click.option(
+    "--plate-config-file",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path),
+    help="Path to the OCR YAML config with image dimensions & alphabet.",
+)
+@click.option(
+    "--warn-only",
+    is_flag=True,
+    help="Exit 0 even if errors occur.",
+)
+@click.option(
+    "--export-fixed",
+    type=str,
+    help="Filename for the cleaned CSV written in the same directory as the annotations file.",
+)
+@click.option(
+    "--min-height",
+    default=2,
+    show_default=True,
+    type=int,
+    help="Minimum allowed image height.",
+)
+@click.option(
+    "--min-width",
+    default=2,
+    show_default=True,
+    type=int,
+    help="Minimum allowed image width.",
+)
+def validate_dataset(
+    annotations_file: Path,
+    plate_config_file: Path,
+    warn_only: bool,
+    export_fixed: Optional[str],
+    min_height: int,
+    min_width: int,
+):
+    """
+    Script to validate the dataset before training.
+    """
+    cfg = load_plate_config_from_yaml(plate_config_file)
+
+    df_annots = pd.read_csv(annotations_file)
+    csv_root = annotations_file.parent
+    df_annots["image_path"] = df_annots["image_path"].apply(lambda p: str((csv_root / p).resolve()))
+
+    errors, warnings, cleaned = _validate_dataset(df_annots, cfg, min_height, min_width)
+
+    # Make cleaned dataset img_path relative (expected format)
+    cleaned["image_path"] = cleaned["image_path"].apply(
+        lambda p: str(Path(p).relative_to(csv_root))
+    )
+
+    rich_report(errors, warnings)
+
+    if export_fixed:
+        export_path = csv_root / Path(export_fixed).name
+        if export_path.resolve() == annotations_file.resolve():
+            console.print(
+                "[yellow]⚠️ Skipping export: make sure you don't "
+                "overwrite original annotations file.[/]"
+            )
+        elif export_path.exists():
+            console.print(f"[yellow]⚠️ Skipping export: file already exists at {export_path}[/]")
+        else:
+            cleaned.to_csv(export_path, index=False)
+            console.print(
+                f"[green]✅ Wrote cleaned CSV with {len(cleaned)} rows at {export_path} [/]"
+            )
+
+    if errors and not warn_only:
+        sys.exit(1)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    validate_dataset()

matrice_analytics/post_processing/ocr/fast_plate_ocr_py38/cli/visualize_augmentation.py

@@ -0,0 +1,176 @@
+"""
+Script to visualize the augmented plates used during training.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import random
+from math import ceil
+from typing import Optional
+import albumentations as A
+import click
+import matplotlib.pyplot as plt
+import numpy as np
+import numpy.typing as npt
+
+from fast_plate_ocr.train.data.augmentation import (
+    default_train_augmentation,
+)
+from fast_plate_ocr.train.model.config import PlateOCRConfig, load_plate_config_from_yaml
+from fast_plate_ocr.train.utilities import utils
+
+
+def _set_seed(seed: Optional[int] = None) -> None:
+    """Set random seed for reproducing augmentations."""
+    if seed:
+        random.seed(seed)
+        np.random.seed(seed)
+
+
+def load_images(
+    img_dir: pathlib.Path,
+    num_images: int,
+    shuffle: bool,
+    plate_config: PlateOCRConfig,
+    augmentation: A.Compose,
+) -> tuple[list[npt.NDArray[np.uint8]], list[npt.NDArray[np.uint8]]]:
+    images = list(
+        utils.load_images_from_folder(
+            img_dir,
+            width=plate_config.img_width,
+            height=plate_config.img_height,
+            image_color_mode=plate_config.image_color_mode,
+            keep_aspect_ratio=plate_config.keep_aspect_ratio,
+            interpolation_method=plate_config.interpolation,
+            padding_color=plate_config.padding_color,
+            shuffle=shuffle,
+            limit=num_images,
+        )
+    )
+    augmented_images = [augmentation(image=i)["image"] for i in images]
+    return images, augmented_images
+
+
+def display_images(
+    images: list[npt.NDArray[np.uint8]],
+    augmented_images: list[npt.NDArray[np.uint8]],
+    columns: int,
+    rows: int,
+    show_original: bool,
+) -> None:
+    num_images = len(images)
+    total_plots = rows * columns
+    num_pages = ceil(num_images / total_plots)
+    for page in range(num_pages):
+        _, axs = plt.subplots(rows, columns, figsize=(8, 8))
+        axs = axs.flatten()
+        for i, ax in enumerate(axs):
+            idx = page * total_plots + i
+            if idx < num_images:
+                if show_original:
+                    img_to_show = np.concatenate((images[idx], augmented_images[idx]), axis=1)
+                else:
+                    img_to_show = augmented_images[idx]
+                ax.imshow(img_to_show, cmap="gray")
+                ax.axis("off")
+            else:
+                ax.axis("off")
+        plt.tight_layout()
+        plt.show()
+
+
+# ruff: noqa: PLR0913
+# pylint: disable=too-many-arguments,too-many-locals
+
+
+@click.command(context_settings={"max_content_width": 120})
+@click.option(
+    "--img-dir",
+    "-d",
+    required=True,
+    type=click.Path(exists=True, dir_okay=True, path_type=pathlib.Path),
+    help="Path to the images that will be augmented and visualized.",
+)
+@click.option(
+    "--plate-config-file",
+    required=True,
+    type=click.Path(exists=True, file_okay=True, path_type=pathlib.Path),
+    help="Path pointing to the model license plate OCR config.",
+)
+@click.option(
+    "--num-images",
+    "-n",
+    type=int,
+    default=250,
+    show_default=True,
+    help="Maximum number of images to visualize.",
+)
+@click.option(
+    "--augmentation-path",
+    type=click.Path(exists=True, file_okay=True, path_type=pathlib.Path),
+    help="YAML file pointing to the augmentation pipeline saved with Albumentations.save(...)",
+)
+@click.option(
+    "--shuffle",
+    "-s",
+    is_flag=True,
+    default=False,
+    help="Whether to shuffle the images before plotting them.",
+)
+@click.option(
+    "--columns",
+    "-c",
+    type=int,
+    default=3,
+    show_default=True,
+    help="Number of columns in the grid layout for displaying images.",
+)
+@click.option(
+    "--rows",
+    "-r",
+    type=int,
+    default=4,
+    show_default=True,
+    help="Number of rows in the grid layout for displaying images.",
+)
+@click.option(
+    "--show-original",
+    "-o",
+    is_flag=True,
+    help="Show the original image along with the augmented one.",
+)
+@click.option(
+    "--seed",
+    type=int,
+    help="Seed for reproducing augmentations.",
+)
+def visualize_augmentation(
+    img_dir: pathlib.Path,
+    plate_config_file: pathlib.Path,
+    num_images: int,
+    augmentation_path: Optional[pathlib.Path],
+    shuffle: bool,
+    columns: int,
+    rows: int,
+    seed: Optional[int],
+    show_original: bool,
+) -> None:
+    """
+    Visualize augmentation pipeline applied to raw images.
+    """
+    _set_seed(seed)
+    config = load_plate_config_from_yaml(plate_config_file)
+    aug = (
+        A.load(augmentation_path, data_format="yaml")
+        if augmentation_path
+        else default_train_augmentation(config.image_color_mode)
+    )
+    aug.set_random_seed(seed)
+    images, augmented_images = load_images(img_dir, num_images, shuffle, config, aug)
+    display_images(images, augmented_images, columns, rows, show_original)
+
+
+if __name__ == "__main__":
+    # pylint: disable = no-value-for-parameter
+    visualize_augmentation()

matrice_analytics/post_processing/ocr/fast_plate_ocr_py38/cli/visualize_predictions.py

@@ -0,0 +1,96 @@
+"""
+Script for displaying an image with the OCR model predictions.
+"""
+
+from __future__ import annotations
+
+import logging
+import pathlib
+from typing import Optional
+import click
+import cv2
+import keras
+import numpy as np
+
+from fast_plate_ocr.train.model.config import load_plate_config_from_yaml
+from fast_plate_ocr.train.utilities import utils
+from fast_plate_ocr.train.utilities.utils import postprocess_model_output
+
+
+
+@click.command(context_settings={"max_content_width": 120})
+@click.option(
+    "-m",
+    "--model",
+    "model_path",
+    required=True,
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=pathlib.Path),
+    help="Path to the saved .keras model.",
+)
+@click.option(
+    "--plate-config-file",
+    required=True,
+    type=click.Path(exists=True, file_okay=True, path_type=pathlib.Path),
+    help="Path pointing to the model license plate OCR config.",
+)
+@click.option(
+    "-d",
+    "--img-dir",
+    required=True,
+    type=click.Path(exists=True, dir_okay=True, file_okay=False, path_type=pathlib.Path),
+    help="Directory containing the images to make predictions from.",
+)
+@click.option(
+    "-l",
+    "--low-conf-thresh",
+    type=float,
+    default=0.35,
+    show_default=True,
+    help="Threshold for displaying low confidence characters.",
+)
+@click.option(
+    "-f",
+    "--filter-conf",
+    type=float,
+    help="Display plates that any of the plate characters are below this number.",
+)
+def visualize_predictions(
+    model_path: pathlib.Path,
+    plate_config_file: pathlib.Path,
+    img_dir: pathlib.Path,
+    low_conf_thresh: float,
+    filter_conf: Optional[float],
+):
+    """
+    Visualize OCR model predictions on unlabeled data.
+    """
+    plate_config = load_plate_config_from_yaml(plate_config_file)
+    model = utils.load_keras_model(model_path, plate_config)
+    images = utils.load_images_from_folder(
+        img_dir,
+        width=plate_config.img_width,
+        height=plate_config.img_height,
+        image_color_mode=plate_config.image_color_mode,
+        keep_aspect_ratio=plate_config.keep_aspect_ratio,
+        interpolation_method=plate_config.interpolation,
+        padding_color=plate_config.padding_color,
+    )
+    for image in images:
+        x = np.expand_dims(image, 0)
+        prediction = model(x, training=False)
+        prediction = keras.ops.stop_gradient(prediction).numpy()
+        plate, probs = postprocess_model_output(
+            prediction=prediction,
+            alphabet=plate_config.alphabet,
+            max_plate_slots=plate_config.max_plate_slots,
+            vocab_size=plate_config.vocabulary_size,
+        )
+        if not filter_conf or (filter_conf and np.any(probs < filter_conf)):
+            utils.display_predictions(
+                image=image, plate=plate, probs=probs, low_conf_thresh=low_conf_thresh
+            )
+    cv2.destroyAllWindows()
+
+
+if __name__ == "__main__":
+    visualize_predictions()

matrice_analytics/post_processing/ocr/fast_plate_ocr_py38/core/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+"""Core utilities and types for Fast Plate OCR (Python 3.8 compatible)."""