camera-ui-ml 1.2.21__py3-none-any.whl

camera_ui_ml/__init__.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+from .backend import (
+    DType,
+    InferenceBackend,
+    InputSpec,
+    Layout,
+    NDArray,
+    Normalize,
+    Outputs,
+)
+from .detectors import (
+    BaseDetector,
+    BoxDetector,
+    Embedder,
+    LandmarkDetector,
+    NormalizedDetection,
+    OcrResult,
+    ParseKind,
+    PersonEmbedder,
+    PlateOcr,
+    Segmenter,
+)
+from .geometry import (
+    Box,
+    NormalizedBox,
+    clamp_box,
+    cxcywh_to_xyxy,
+    normalize_box,
+    pad_box,
+    scale_box,
+)
+from .model_manager import BaseModelManager
+from .parsing import (
+    RawDetection,
+    channels_first,
+    decode_ocr,
+    l2_normalize,
+    nms,
+    parse_end2end,
+    parse_yolov9,
+)
+from .pipeline import prepare_executor, run_prepare
+from .pipelines import (
+    DETECTION_FLOOR,
+    detect_clip,
+    detect_faces,
+    detect_faces_in_image,
+    detect_objects,
+    detect_objects_in_image,
+    detect_plates,
+    detect_plates_in_image,
+    embed_face_images,
+    embed_faces,
+    embed_person_images,
+    embed_persons,
+    requested_threshold,
+    segment_images,
+    segment_objects,
+)
+from .preprocess import (
+    crop_rgb,
+    decode_image,
+    frame_to_rgb,
+    resize_rgb,
+    to_pil,
+    to_tensor,
+)
+from .runtime import model_runtime
+from .settings import reset_stored_settings
+
+__all__ = [
+    "model_runtime",
+    "InferenceBackend",
+    "InputSpec",
+    "Outputs",
+    "NDArray",
+    "Layout",
+    "Normalize",
+    "DType",
+    "BaseModelManager",
+    # ClipEncoder is opt-in via camera_ui_ml.detectors.clip
+    "BaseDetector",
+    "BoxDetector",
+    "NormalizedDetection",
+    "ParseKind",
+    "Embedder",
+    "LandmarkDetector",
+    "PersonEmbedder",
+    "Segmenter",
+    "PlateOcr",
+    "OcrResult",
+    "Box",
+    "NormalizedBox",
+    "cxcywh_to_xyxy",
+    "scale_box",
+    "clamp_box",
+    "normalize_box",
+    "pad_box",
+    "RawDetection",
+    "channels_first",
+    "parse_yolov9",
+    "parse_end2end",
+    "decode_ocr",
+    "l2_normalize",
+    "nms",
+    "frame_to_rgb",
+    "resize_rgb",
+    "to_pil",
+    "to_tensor",
+    "crop_rgb",
+    "decode_image",
+    "run_prepare",
+    "prepare_executor",
+    "DETECTION_FLOOR",
+    "requested_threshold",
+    "detect_objects",
+    "detect_faces",
+    "detect_plates",
+    "detect_objects_in_image",
+    "detect_faces_in_image",
+    "detect_plates_in_image",
+    "reset_stored_settings",
+    "detect_clip",
+    "embed_faces",
+    "embed_face_images",
+    "embed_persons",
+    "embed_person_images",
+    "segment_objects",
+    "segment_images",
+]

camera_ui_ml/align.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+from PIL import Image
+
+NDArray = np.ndarray[Any, Any]
+
+ARCFACE_DST_112 = np.array(
+    [
+        [38.2946, 51.6963],
+        [73.5318, 51.5014],
+        [56.0252, 71.7366],
+        [41.5493, 92.3655],
+        [70.7299, 92.2041],
+    ],
+    dtype=np.float64,
+)
+
+
+def umeyama_similarity(src: NDArray, dst: NDArray) -> NDArray:
+    src = np.asarray(src, dtype=np.float64)
+    dst = np.asarray(dst, dtype=np.float64)
+    n, dim = src.shape
+
+    src_mean, dst_mean = src.mean(axis=0), dst.mean(axis=0)
+    src_demean, dst_demean = src - src_mean, dst - dst_mean
+
+    cov = dst_demean.T @ src_demean / n
+    u, s, vt = np.linalg.svd(cov)
+
+    d = np.ones(dim)
+    if np.linalg.det(u) * np.linalg.det(vt) < 0:
+        d[-1] = -1.0
+    rotation = u @ np.diag(d) @ vt
+
+    variance = (src_demean**2).sum() / n
+    scale = 1.0 if variance == 0 else float((s * d).sum() / variance)
+
+    matrix = np.zeros((2, 3), dtype=np.float64)
+    matrix[:, :2] = scale * rotation
+    matrix[:, 2] = dst_mean - scale * rotation @ src_mean
+    return matrix
+
+
+def warp_face(rgb: NDArray, points: NDArray, size: int = 112) -> NDArray:
+    scale = size / 112.0
+    matrix = umeyama_similarity(np.asarray(points, dtype=np.float64), ARCFACE_DST_112 * scale)
+
+    # PIL maps destination pixels back to the source, so it wants the inverse
+    full = np.vstack([matrix, [0.0, 0.0, 1.0]])
+    inverse = np.linalg.inv(full)[:2]
+
+    warped = Image.fromarray(rgb).transform(
+        (size, size), Image.Transform.AFFINE, tuple(inverse.flatten()), resample=Image.Resampling.BILINEAR
+    )
+    return np.asarray(warped)

camera_ui_ml/backend.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Literal
+
+import numpy as np
+
+from .pipeline import run_prepare
+
+NDArray = np.ndarray[Any, Any]
+
+Layout = Literal["nchw", "nhwc"]
+Normalize = Literal["unit", "facenet", "arcface", "none"]
+Channels = Literal["rgb", "bgr"]
+DType = Literal["float32", "uint8"]
+
+Outputs = Sequence[NDArray]
+
+
+@dataclass(frozen=True)
+class InputSpec:
+    width: int
+    height: int
+    layout: Layout = "nchw"
+    normalize: Normalize = "unit"
+    channels: Channels = "rgb"
+    dtype: DType = "float32"
+
+
+class InferenceBackend(ABC):
+    @property
+    @abstractmethod
+    def input_size(self) -> tuple[int, int]: ...
+
+    @abstractmethod
+    def metadata(self) -> Mapping[str, str]: ...
+
+    @property
+    def device(self) -> str:
+        return "unknown"
+
+    @property
+    def runtime(self) -> str | None:
+        return None
+
+    @property
+    def precision(self) -> str | None:
+        return None
+
+    @abstractmethod
+    async def infer(self, inputs: Sequence[Any]) -> Outputs: ...
+
+    @abstractmethod
+    def close(self) -> None: ...
+
+    def adapt(self, image: NDArray, spec: InputSpec) -> Sequence[Any]:
+        from .preprocess import to_tensor
+
+        return [to_tensor(image, spec)]
+
+    async def run(self, image: NDArray, spec: InputSpec) -> Outputs:
+        prepared = await run_prepare(lambda: list(self.adapt(image, spec)))
+        return await self.infer(prepared)

camera_ui_ml/detectors/__init__.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from .base import BaseDetector
+from .box import BoxDetector, NormalizedDetection, ParseKind
+from .embedder import Embedder
+from .landmarks import LandmarkDetector
+from .ocr import DEFAULT_ALPHABET, OcrResult, PlateOcr
+from .person import PersonEmbedder
+from .segmenter import Segmenter
+
+__all__ = [
+    "BaseDetector",
+    "BoxDetector",
+    "NormalizedDetection",
+    "ParseKind",
+    "Embedder",
+    "LandmarkDetector",
+    "PersonEmbedder",
+    "Segmenter",
+    "PlateOcr",
+    "OcrResult",
+    "DEFAULT_ALPHABET",
+]

camera_ui_ml/detectors/base.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import TYPE_CHECKING
+
+from camera_ui_sdk import LoggerService
+
+from ..backend import InferenceBackend
+from ..model_manager import BaseModelManager
+
+if TYPE_CHECKING:
+    from camera_ui_sdk import LoadedModel
+
+
+class BaseDetector:
+    name: str = "detector"
+
+    def __init__(self, manager: BaseModelManager, logger: LoggerService) -> None:
+        self.manager = manager
+        self.logger = logger
+        self.backend: InferenceBackend | None = None
+        self.initialized = False
+        self.closed = False
+        self.model_name: str | None = None
+        self.load_ms = 0
+        self._init_task: asyncio.Task[None] | None = None
+
+    async def initialize(self, model_name: str) -> None:
+        if self.initialized:
+            return
+        if self._init_task is None:
+            self._init_task = asyncio.create_task(self._do_initialize(model_name))
+        await self._init_task
+
+    async def close(self) -> None:
+        self.closed = True
+        if self._init_task is not None:
+            self._init_task.cancel()
+            self._init_task = None
+        self.initialized = False
+        if self.backend is not None:
+            self.backend.close()
+            self.backend = None
+
+    async def _do_initialize(self, model_name: str) -> None:
+        try:
+            self.logger.log(f"Loading {self.name}: {model_name}...")
+            started = time.monotonic()
+            self.backend = await self.manager.ensure_backend(model_name)
+            if self.closed:
+                return
+            await self._configure(model_name)
+            self.load_ms = round((time.monotonic() - started) * 1000)
+            self.model_name = model_name
+            self.initialized = True
+            self.logger.success(f"Loaded {self.name}: {model_name}")
+        except Exception as error:
+            self.logger.error(f"Failed to initialize {self.name}: {error}")
+            raise
+        finally:
+            self._init_task = None
+
+    async def _configure(self, model_name: str) -> None: ...
+
+    def loaded_model(self, role: str | None = None) -> LoadedModel | None:
+        if not self._ready() or not self.model_name or self.backend is None:
+            return None
+
+        entry: LoadedModel = {"name": self.model_name}
+        if role:
+            entry["role"] = role
+        if self.backend.device and self.backend.device != "unknown":
+            entry["device"] = self.backend.device
+        if self.backend.precision:
+            entry["precision"] = self.backend.precision
+        if self.load_ms:
+            entry["loadMs"] = self.load_ms
+        return entry
+
+    def _ready(self) -> bool:
+        return self.initialized and self.backend is not None

camera_ui_ml/detectors/box.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import ast
+import re
+from collections.abc import Mapping
+from typing import Any, Literal
+
+import numpy as np
+from camera_ui_sdk import ImageMetadata, LoggerService, VideoFrameData
+
+from ..backend import InputSpec, NDArray, Normalize
+from ..geometry import NormalizedBox, normalize_box, scale_box
+from ..model_manager import BaseModelManager
+from ..parsing import RawDetection, channels_first, nms, parse_end2end, parse_yolov9
+from ..preprocess import decode_image, frame_to_rgb
+from .base import BaseDetector
+
+ParseKind = Literal["yolov9", "end2end"]
+NormalizedDetection = tuple[int, float, NormalizedBox]
+
+
+class BoxDetector(BaseDetector):
+    def __init__(
+        self,
+        manager: BaseModelManager,
+        logger: LoggerService,
+        *,
+        name: str = "detector",
+        parse: ParseKind = "yolov9",
+        size: tuple[int, int] = (320, 320),
+        normalize: Normalize = "unit",
+        multiclass: bool = False,
+        apply_nms: bool = True,
+        threshold: float = 0.5,
+    ) -> None:
+        super().__init__(manager, logger)
+        self.name = name
+        self.threshold = threshold
+        self.input_size = size
+        self.labels: dict[int, str] = {}
+
+        self._parse = parse
+        self._normalize = normalize
+        self._multiclass = multiclass
+        self._apply_nms = apply_nms
+
+    async def _configure(self, model_name: str) -> None:
+        assert self.backend is not None
+        self.input_size = self.backend.input_size
+        if self._multiclass:
+            self.labels = _parse_labels(self.backend.metadata())
+
+    @property
+    def _spec(self) -> InputSpec:
+        return InputSpec(
+            self.input_size[0],
+            self.input_size[1],
+            layout="nchw",
+            normalize=self._normalize,
+        )
+
+    async def detect(self, image: NDArray, threshold: float | None = None) -> list[RawDetection]:
+        if not self._ready():
+            return []
+        assert self.backend is not None
+
+        outputs = await self.backend.run(image, self._spec)
+        primary = np.asarray(outputs[0])
+        limit = self.threshold if threshold is None else threshold
+
+        if self._parse == "yolov9":
+            detections = parse_yolov9(channels_first(primary), limit)
+        else:
+            detections = parse_end2end(np.squeeze(primary), limit)
+
+        return nms(detections) if self._apply_nms else detections
+
+    async def detect_frame(self, frame: VideoFrameData, threshold: float | None = None) -> list[RawDetection]:
+        if not self._ready():
+            return []
+
+        rgb = frame_to_rgb(frame["data"], frame["width"], frame["height"], frame["format"])
+        raw = await self.detect(rgb, threshold)
+
+        scale_x = frame["width"] / self.input_size[0]
+        scale_y = frame["height"] / self.input_size[1]
+        return [(cid, conf, scale_box(box, scale_x, scale_y)) for cid, conf, box in raw]
+
+    async def detect_single(
+        self, image_data: bytes, metadata: ImageMetadata, threshold: float | None = None
+    ) -> list[NormalizedDetection]:
+        if not self._ready():
+            return []
+
+        raw = await self.detect(decode_image(image_data), threshold)
+        return [
+            (cid, conf, normalize_box(box, self.input_size[0], self.input_size[1])) for cid, conf, box in raw
+        ]
+
+
+def _parse_labels(metadata: Mapping[str, str]) -> dict[int, str]:
+    names = metadata.get("names") or metadata.get("yolo.names")
+    if names:
+        parsed: dict[Any, Any] = ast.literal_eval(names)
+        return {int(key): str(value) for key, value in parsed.items()}
+
+    classes = metadata.get("classes")
+    if not classes:
+        return {}
+
+    labels: dict[int, str] = {}
+    for row, line in enumerate(classes.split(",")):
+        pair = re.split(r"[:\s]+", line.strip(), maxsplit=1)
+        if len(pair) == 2 and pair[0].strip().isdigit():
+            labels[int(pair[0])] = pair[1].strip()
+        else:
+            labels[row] = line.strip()
+    return labels

camera_ui_ml/detectors/clip.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+from camera_ui_sdk import LoggerService, VideoFrameData
+from transformers import CLIPProcessor
+
+from ..backend import InferenceBackend, NDArray
+from ..model_manager import BaseModelManager
+from ..parsing import l2_normalize
+from ..pipeline import run_prepare
+from ..preprocess import frame_to_rgb, to_pil
+
+if TYPE_CHECKING:
+    from camera_ui_sdk import LoadedModel
+
+
+class ClipEncoder:
+    def __init__(
+        self,
+        manager: BaseModelManager,
+        logger: LoggerService,
+        *,
+        embedding_model: str = "clip-vit-base-patch32",
+    ) -> None:
+        self.manager = manager
+        self.logger = logger
+        self.embedding_model = embedding_model
+
+        self.vision: InferenceBackend | None = None
+        self.text: InferenceBackend | None = None
+        self.processor: Any = None
+
+        self.initialized = False
+        self.closed = False
+        self.vision_model: str | None = None
+        self.text_model: str | None = None
+        self.load_ms = 0
+        self._init_task: asyncio.Task[None] | None = None
+
+    async def initialize(self, vision_model: str, text_model: str) -> None:
+        if self.initialized:
+            return
+        if self._init_task is None:
+            self._init_task = asyncio.create_task(self._do_initialize(vision_model, text_model))
+        await self._init_task
+
+    async def close(self) -> None:
+        self.closed = True
+        if self._init_task is not None:
+            self._init_task.cancel()
+            self._init_task = None
+        self.initialized = False
+        if self.vision is not None:
+            self.vision.close()
+            self.vision = None
+        if self.text is not None:
+            self.text.close()
+            self.text = None
+        self.processor = None
+
+    async def embed_image(self, image: NDArray) -> list[float]:
+        if not self._ready():
+            return []
+        assert self.vision is not None
+
+        pil = to_pil(image)
+        pixel_values = await run_prepare(lambda: self._vision_input(pil))
+        outputs = await self.vision.infer([pixel_values])
+        return [float(value) for value in l2_normalize(outputs[0])]
+
+    async def embed_text(self, text: str) -> list[float]:
+        if not self._ready():
+            return []
+        assert self.text is not None
+
+        input_ids, attention_mask = await run_prepare(lambda: self._text_input(text))
+        outputs = await self.text.infer([input_ids, attention_mask])
+        return [float(value) for value in l2_normalize(outputs[0])]
+
+    async def embed_frame(self, width: int, height: int, data: bytes) -> list[float]:
+        return await self.embed_image(frame_to_rgb(data, width, height))
+
+    async def embed_frames(self, frames: list[VideoFrameData]) -> list[list[float]]:
+        if not self._ready():
+            return [[] for _ in frames]
+        return [await self.embed_frame(frame["width"], frame["height"], frame["data"]) for frame in frames]
+
+    def loaded_models(self, _role: str | None = None) -> list[LoadedModel]:
+        if not self._ready():
+            return []
+
+        entries: list[LoadedModel] = []
+        for backend, name, role in (
+            (self.vision, self.vision_model, "vision"),
+            (self.text, self.text_model, "text"),
+        ):
+            if backend is None or not name:
+                continue
+            entry: LoadedModel = {"name": name, "role": role}
+            if backend.device and backend.device != "unknown":
+                entry["device"] = backend.device
+            if backend.precision:
+                entry["precision"] = backend.precision
+            if self.load_ms:
+                entry["loadMs"] = self.load_ms
+            entries.append(entry)
+        return entries
+
+    def _ready(self) -> bool:
+        return (
+            self.initialized
+            and self.vision is not None
+            and self.text is not None
+            and self.processor is not None
+        )
+
+    async def _do_initialize(self, vision_model: str, text_model: str) -> None:
+        try:
+            self.logger.log(f"Loading CLIP: {vision_model} + {text_model}...")
+            started = time.monotonic()
+            self.vision = await self.manager.ensure_backend(vision_model)
+            self.text = await self.manager.ensure_backend(text_model)
+            processor_dir = await self.manager.ensure_clip_processor()
+            if processor_dir is None:
+                raise RuntimeError("CLIP processor files are not bundled (clip_processor_files is empty)")
+            self.processor = await asyncio.to_thread(CLIPProcessor.from_pretrained, processor_dir)
+            if self.closed:
+                return
+            self.load_ms = round((time.monotonic() - started) * 1000)
+            self.vision_model = vision_model
+            self.text_model = text_model
+            self.initialized = True
+            self.logger.success(f"Loaded CLIP: {vision_model} + {text_model}")
+        except Exception as error:
+            self.logger.error(f"Failed to initialize CLIP encoder: {error}")
+            raise
+        finally:
+            self._init_task = None
+
+    def _vision_input(self, pil: Any) -> NDArray:
+        inputs = self.processor(images=pil, return_tensors="np", padding="max_length", truncation=True)
+        return np.asarray(inputs["pixel_values"], dtype=np.float32)
+
+    def _text_input(self, text: str) -> tuple[NDArray, NDArray]:
+        inputs = self.processor(text=text, return_tensors="np", padding="max_length", truncation=True)
+        return (
+            np.asarray(inputs["input_ids"], dtype=np.int64),
+            np.asarray(inputs["attention_mask"], dtype=np.int64),
+        )

camera_ui_ml/detectors/embedder.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from camera_ui_sdk import LoggerService
+
+from ..align import warp_face
+from ..backend import InputSpec, NDArray, Normalize
+from ..geometry import Box, square_box
+from ..model_manager import BaseModelManager
+from ..parsing import FaceLandmarks, l2_normalize
+from ..preprocess import crop_rgb, frame_to_rgb
+from .base import BaseDetector
+
+# whole-picture enrolment against a live crop: cosine 0.14, located and cut like this: 1.00
+FACE_PADDING = 0.25
+
+
+class Embedder(BaseDetector):
+    def __init__(
+        self,
+        manager: BaseModelManager,
+        logger: LoggerService,
+        *,
+        size: int = 160,
+        normalize: Normalize = "facenet",
+        aligned: bool = False,
+        name: str = "face embedder",
+    ) -> None:
+        super().__init__(manager, logger)
+        self.name = name
+        self.input_size = (size, size)
+        self.normalize = normalize
+        # ArcFace-style heads need the 112x112 template, a plain box crop drops them from 51% to near zero
+        self.aligned = aligned
+
+    @property
+    def _spec(self) -> InputSpec:
+        return InputSpec(self.input_size[0], self.input_size[1], layout="nchw", normalize=self.normalize)
+
+    async def embed(self, image: NDArray) -> list[float]:
+        if not self._ready():
+            return []
+        assert self.backend is not None
+        outputs = await self.backend.run(image, self._spec)
+        return [float(value) for value in l2_normalize(outputs[0])]
+
+    async def embed_from_crop(self, frame_data: bytes, width: int, height: int, box: Box) -> list[float]:
+        if not self._ready():
+            return []
+
+        x1 = max(0, int(box[0]))
+        y1 = max(0, int(box[1]))
+        x2 = min(width, int(box[2]))
+        y2 = min(height, int(box[3]))
+        if x2 <= x1 or y2 <= y1:
+            return []
+
+        rgb = frame_to_rgb(frame_data, width, height)
+        return await self.embed(rgb[y1:y2, x1:x2])
+
+    async def embed_face(self, crop: NDArray, landmarks: FaceLandmarks | None) -> list[float]:
+        if not self._ready() or crop.size == 0 or landmarks is None:
+            return []
+        if self.aligned:
+            return await self.embed(warp_face(crop, landmarks.points, self.input_size[0]))
+        box = square_box(landmarks.box, crop.shape[1], crop.shape[0], FACE_PADDING)
+        return await self.embed(crop_rgb(crop, box))