eye-cv 1.0.0__py3-none-any.whl

eye/detection/lmm.py

@@ -0,0 +1,184 @@
+import re
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+
+from eye.detection.utils import polygon_to_mask, polygon_to_xyxy
+
+
+class LMM(Enum):
+    PALIGEMMA = "paligemma"
+    FLORENCE_2 = "florence_2"
+
+
+RESULT_TYPES: Dict[LMM, type] = {LMM.PALIGEMMA: str, LMM.FLORENCE_2: dict}
+
+REQUIRED_ARGUMENTS: Dict[LMM, List[str]] = {
+    LMM.PALIGEMMA: ["resolution_wh"],
+    LMM.FLORENCE_2: ["resolution_wh"],
+}
+
+ALLOWED_ARGUMENTS: Dict[LMM, List[str]] = {
+    LMM.PALIGEMMA: ["resolution_wh", "classes"],
+    LMM.FLORENCE_2: ["resolution_wh"],
+}
+
+SUPPORTED_TASKS_FLORENCE_2 = [
+    "<OD>",
+    "<CAPTION_TO_PHRASE_GROUNDING>",
+    "<DENSE_REGION_CAPTION>",
+    "<REGION_PROPOSAL>",
+    "<OCR_WITH_REGION>",
+    "<REFERRING_EXPRESSION_SEGMENTATION>",
+    "<REGION_TO_SEGMENTATION>",
+    "<OPEN_VOCABULARY_DETECTION>",
+    "<REGION_TO_CATEGORY>",
+    "<REGION_TO_DESCRIPTION>",
+]
+
+
+def validate_lmm_parameters(
+    lmm: Union[LMM, str], result: Any, kwargs: Dict[str, Any]
+) -> LMM:
+    if isinstance(lmm, str):
+        try:
+            lmm = LMM(lmm.lower())
+        except ValueError:
+            raise ValueError(
+                f"Invalid lmm value: {lmm}. Must be one of {[e.value for e in LMM]}"
+            )
+
+    if not isinstance(result, RESULT_TYPES[lmm]):
+        raise ValueError(
+            f"Invalid LMM result type: {type(result)}. Must be {RESULT_TYPES[lmm]}"
+        )
+
+    required_args = REQUIRED_ARGUMENTS.get(lmm, [])
+    for arg in required_args:
+        if arg not in kwargs:
+            raise ValueError(f"Missing required argument: {arg}")
+
+    allowed_args = ALLOWED_ARGUMENTS.get(lmm, [])
+    for arg in kwargs:
+        if arg not in allowed_args:
+            raise ValueError(f"Argument {arg} is not allowed for {lmm.name}")
+
+    return lmm
+
+
+def from_paligemma(
+    result: str, resolution_wh: Tuple[int, int], classes: Optional[List[str]] = None
+) -> Tuple[np.ndarray, Optional[np.ndarray], np.ndarray]:
+    w, h = resolution_wh
+    pattern = re.compile(
+        r"(?<!<loc\d{4}>)<loc(\d{4})><loc(\d{4})><loc(\d{4})><loc(\d{4})> ([\w\s\-]+)"
+    )
+    matches = pattern.findall(result)
+    matches = np.array(matches) if matches else np.empty((0, 5))
+
+    xyxy, class_name = matches[:, [1, 0, 3, 2]], matches[:, 4]
+    xyxy = xyxy.astype(int) / 1024 * np.array([w, h, w, h])
+    class_name = np.char.strip(class_name.astype(str))
+    class_id = None
+
+    if classes is not None:
+        mask = np.array([name in classes for name in class_name]).astype(bool)
+        xyxy, class_name = xyxy[mask], class_name[mask]
+        class_id = np.array([classes.index(name) for name in class_name])
+
+    return xyxy, class_id, class_name
+
+
+def from_florence_2(
+    result: dict, resolution_wh: Tuple[int, int]
+) -> Tuple[
+    np.ndarray, Optional[np.ndarray], Optional[np.ndarray], Optional[np.ndarray]
+]:
+    """
+    Parse results from the Florence 2 multi-model model.
+    https://huggingface.co/microsoft/Florence-2-large
+
+    Parameters:
+        result: dict containing the model output
+
+    Returns:
+        xyxy (np.ndarray): An array of shape `(n, 4)` containing
+            the bounding boxes coordinates in format `[x1, y1, x2, y2]`
+        labels: (Optional[np.ndarray]): An array of shape `(n,)` containing
+            the class labels for each bounding box
+        masks: (Optional[np.ndarray]): An array of shape `(n, h, w)` containing
+            the segmentation masks for each bounding box
+        obb_boxes: (Optional[np.ndarray]): An array of shape `(n, 4, 2)` containing
+            oriented bounding boxes.
+    """
+    assert len(result) == 1, f"Expected result with a single element. Got: {result}"
+    task = next(iter(result.keys()))
+    if task not in SUPPORTED_TASKS_FLORENCE_2:
+        raise ValueError(
+            f"{task} not supported. Supported tasks are: {SUPPORTED_TASKS_FLORENCE_2}"
+        )
+    result = result[task]
+
+    if task in ["<OD>", "<CAPTION_TO_PHRASE_GROUNDING>", "<DENSE_REGION_CAPTION>"]:
+        xyxy = np.array(result["bboxes"], dtype=np.float32)
+        labels = np.array(result["labels"])
+        return xyxy, labels, None, None
+
+    if task == "<REGION_PROPOSAL>":
+        xyxy = np.array(result["bboxes"], dtype=np.float32)
+        # provides labels, but they are ["", "", "", ...]
+        return xyxy, None, None, None
+
+    if task == "<OCR_WITH_REGION>":
+        xyxyxyxy = np.array(result["quad_boxes"], dtype=np.float32)
+        xyxyxyxy = xyxyxyxy.reshape(-1, 4, 2)
+        xyxy = np.array([polygon_to_xyxy(polygon) for polygon in xyxyxyxy])
+        labels = np.array(result["labels"])
+        return xyxy, labels, None, xyxyxyxy
+
+    if task in ["<REFERRING_EXPRESSION_SEGMENTATION>", "<REGION_TO_SEGMENTATION>"]:
+        xyxy_list = []
+        masks_list = []
+        for polygons_of_same_class in result["polygons"]:
+            for polygon in polygons_of_same_class:
+                polygon = np.reshape(polygon, (-1, 2)).astype(np.int32)
+                mask = polygon_to_mask(polygon, resolution_wh).astype(bool)
+                masks_list.append(mask)
+                xyxy = polygon_to_xyxy(polygon)
+                xyxy_list.append(xyxy)
+            # per-class labels also provided, but they are ["", "", "", ...]
+            # when we figure out how to set class names, we can do
+            # zip(result["labels"], result["polygons"])
+        xyxy = np.array(xyxy_list, dtype=np.float32)
+        masks = np.array(masks_list)
+        return xyxy, None, masks, None
+
+    if task == "<OPEN_VOCABULARY_DETECTION>":
+        xyxy = np.array(result["bboxes"], dtype=np.float32)
+        labels = np.array(result["bboxes_labels"])
+        # Also has "polygons" and "polygons_labels", but they don't seem to be used
+        return xyxy, labels, None, None
+
+    if task in ["<REGION_TO_CATEGORY>", "<REGION_TO_DESCRIPTION>"]:
+        assert isinstance(
+            result, str
+        ), f"Expected string as <REGION_TO_CATEGORY> result, got {type(result)}"
+
+        if result == "No object detected.":
+            return np.empty((0, 4), dtype=np.float32), np.array([]), None, None
+
+        pattern = re.compile(r"<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>")
+        match = pattern.search(result)
+        assert (
+            match is not None
+        ), f"Expected string to end in location tags, but got {result}"
+
+        w, h = resolution_wh
+        xyxy = np.array([match.groups()], dtype=np.float32)
+        xyxy *= np.array([w, h, w, h]) / 1000
+        result_string = result[: match.start()]
+        labels = np.array([result_string])
+        return xyxy, labels, None, None
+
+    assert False, f"Unimplemented task: {task}"

eye/detection/overlap_filter.py

@@ -0,0 +1,270 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import List, Union
+
+import numpy as np
+import numpy.typing as npt
+
+from eye.detection.utils import box_iou_batch, mask_iou_batch
+
+
+def resize_masks(masks: np.ndarray, max_dimension: int = 640) -> np.ndarray:
+    """
+    Resize all masks in the array to have a maximum dimension of max_dimension,
+    maintaining aspect ratio.
+
+    Args:
+        masks (np.ndarray): 3D array of binary masks with shape (N, H, W).
+        max_dimension (int): The maximum dimension for the resized masks.
+
+    Returns:
+        np.ndarray: Array of resized masks.
+    """
+    max_height = np.max(masks.shape[1])
+    max_width = np.max(masks.shape[2])
+    scale = min(max_dimension / max_height, max_dimension / max_width)
+
+    new_height = int(scale * max_height)
+    new_width = int(scale * max_width)
+
+    x = np.linspace(0, max_width - 1, new_width).astype(int)
+    y = np.linspace(0, max_height - 1, new_height).astype(int)
+    xv, yv = np.meshgrid(x, y)
+
+    resized_masks = masks[:, yv, xv]
+
+    resized_masks = resized_masks.reshape(masks.shape[0], new_height, new_width)
+    return resized_masks
+
+
+def mask_non_max_suppression(
+    predictions: np.ndarray,
+    masks: np.ndarray,
+    iou_threshold: float = 0.5,
+    mask_dimension: int = 640,
+) -> np.ndarray:
+    """
+    Perform Non-Maximum Suppression (NMS) on segmentation predictions.
+
+    Args:
+        predictions (np.ndarray): A 2D array of object detection predictions in
+            the format of `(x_min, y_min, x_max, y_max, score)`
+            or `(x_min, y_min, x_max, y_max, score, class)`. Shape: `(N, 5)` or
+            `(N, 6)`, where N is the number of predictions.
+        masks (np.ndarray): A 3D array of binary masks corresponding to the predictions.
+            Shape: `(N, H, W)`, where N is the number of predictions, and H, W are the
+            dimensions of each mask.
+        iou_threshold (float): The intersection-over-union threshold
+            to use for non-maximum suppression.
+        mask_dimension (int): The dimension to which the masks should be
+            resized before computing IOU values. Defaults to 640.
+
+    Returns:
+        np.ndarray: A boolean array indicating which predictions to keep after
+            non-maximum suppression.
+
+    Raises:
+        AssertionError: If `iou_threshold` is not within the closed
+            range from `0` to `1`.
+    """
+    assert 0 <= iou_threshold <= 1, (
+        "Value of `iou_threshold` must be in the closed range from 0 to 1, "
+        f"{iou_threshold} given."
+    )
+    rows, columns = predictions.shape
+
+    if columns == 5:
+        predictions = np.c_[predictions, np.zeros(rows)]
+
+    sort_index = predictions[:, 4].argsort()[::-1]
+    predictions = predictions[sort_index]
+    masks = masks[sort_index]
+    masks_resized = resize_masks(masks, mask_dimension)
+    ious = mask_iou_batch(masks_resized, masks_resized)
+    categories = predictions[:, 5]
+
+    keep = np.ones(rows, dtype=bool)
+    for i in range(rows):
+        if keep[i]:
+            condition = (ious[i] > iou_threshold) & (categories[i] == categories)
+            keep[i + 1 :] = np.where(condition[i + 1 :], False, keep[i + 1 :])
+
+    return keep[sort_index.argsort()]
+
+
+def box_non_max_suppression(
+    predictions: np.ndarray, iou_threshold: float = 0.5
+) -> np.ndarray:
+    """
+    Perform Non-Maximum Suppression (NMS) on object detection predictions.
+
+    Args:
+        predictions (np.ndarray): An array of object detection predictions in
+            the format of `(x_min, y_min, x_max, y_max, score)`
+            or `(x_min, y_min, x_max, y_max, score, class)`.
+        iou_threshold (float): The intersection-over-union threshold
+            to use for non-maximum suppression.
+
+    Returns:
+        np.ndarray: A boolean array indicating which predictions to keep after n
+            on-maximum suppression.
+
+    Raises:
+        AssertionError: If `iou_threshold` is not within the
+            closed range from `0` to `1`.
+    """
+    assert 0 <= iou_threshold <= 1, (
+        "Value of `iou_threshold` must be in the closed range from 0 to 1, "
+        f"{iou_threshold} given."
+    )
+    rows, columns = predictions.shape
+
+    # add column #5 - category filled with zeros for agnostic nms
+    if columns == 5:
+        predictions = np.c_[predictions, np.zeros(rows)]
+
+    # sort predictions column #4 - score
+    sort_index = np.flip(predictions[:, 4].argsort())
+    predictions = predictions[sort_index]
+
+    boxes = predictions[:, :4]
+    categories = predictions[:, 5]
+    ious = box_iou_batch(boxes, boxes)
+    ious = ious - np.eye(rows)
+
+    keep = np.ones(rows, dtype=bool)
+
+    for index, (iou, category) in enumerate(zip(ious, categories)):
+        if not keep[index]:
+            continue
+
+        # drop detections with iou > iou_threshold and
+        # same category as current detections
+        condition = (iou > iou_threshold) & (categories == category)
+        keep = keep & ~condition
+
+    return keep[sort_index.argsort()]
+
+
+def group_overlapping_boxes(
+    predictions: npt.NDArray[np.float64], iou_threshold: float = 0.5
+) -> List[List[int]]:
+    """
+    Apply greedy version of non-maximum merging to avoid detecting too many
+    overlapping bounding boxes for a given object.
+
+    Args:
+        predictions (npt.NDArray[np.float64]): An array of shape `(n, 5)` containing
+            the bounding boxes coordinates in format `[x1, y1, x2, y2]`
+            and the confidence scores.
+        iou_threshold (float): The intersection-over-union threshold
+            to use for non-maximum suppression. Defaults to 0.5.
+
+    Returns:
+        List[List[int]]: Groups of prediction indices be merged.
+            Each group may have 1 or more elements.
+    """
+    merge_groups: List[List[int]] = []
+
+    scores = predictions[:, 4]
+    order = scores.argsort()
+
+    while len(order) > 0:
+        idx = int(order[-1])
+
+        order = order[:-1]
+        if len(order) == 0:
+            merge_groups.append([idx])
+            break
+
+        merge_candidate = np.expand_dims(predictions[idx], axis=0)
+        ious = box_iou_batch(predictions[order][:, :4], merge_candidate[:, :4])
+        ious = ious.flatten()
+
+        above_threshold = ious >= iou_threshold
+        merge_group = [idx, *np.flip(order[above_threshold]).tolist()]
+        merge_groups.append(merge_group)
+        order = order[~above_threshold]
+    return merge_groups
+
+
+def box_non_max_merge(
+    predictions: npt.NDArray[np.float64],
+    iou_threshold: float = 0.5,
+) -> List[List[int]]:
+    """
+    Apply greedy version of non-maximum merging per category to avoid detecting
+    too many overlapping bounding boxes for a given object.
+
+    Args:
+        predictions (npt.NDArray[np.float64]): An array of shape `(n, 5)` or `(n, 6)`
+            containing the bounding boxes coordinates in format `[x1, y1, x2, y2]`,
+            the confidence scores and class_ids. Omit class_id column to allow
+            detections of different classes to be merged.
+        iou_threshold (float): The intersection-over-union threshold
+            to use for non-maximum suppression. Defaults to 0.5.
+
+    Returns:
+        List[List[int]]: Groups of prediction indices be merged.
+            Each group may have 1 or more elements.
+    """
+    if predictions.shape[1] == 5:
+        return group_overlapping_boxes(predictions, iou_threshold)
+
+    category_ids = predictions[:, 5]
+    merge_groups = []
+    for category_id in np.unique(category_ids):
+        curr_indices = np.where(category_ids == category_id)[0]
+        merge_class_groups = group_overlapping_boxes(
+            predictions[curr_indices], iou_threshold
+        )
+
+        for merge_class_group in merge_class_groups:
+            merge_groups.append(curr_indices[merge_class_group].tolist())
+
+    for merge_group in merge_groups:
+        if len(merge_group) == 0:
+            raise ValueError(
+                f"Empty group detected when non-max-merging "
+                f"detections: {merge_groups}"
+            )
+    return merge_groups
+
+
+class OverlapFilter(Enum):
+    """
+    Enum specifying the strategy for filtering overlapping detections.
+
+    Attributes:
+        NONE: Do not filter detections based on overlap.
+        NON_MAX_SUPPRESSION: Filter detections using non-max suppression. This means,
+            detections that overlap by more than a set threshold will be discarded,
+            except for the one with the highest confidence.
+        NON_MAX_MERGE: Merge detections with non-max merging. This means,
+            detections that overlap by more than a set threshold will be merged
+            into a single detection.
+    """
+
+    NONE = "none"
+    NON_MAX_SUPPRESSION = "non_max_suppression"
+    NON_MAX_MERGE = "non_max_merge"
+
+    @classmethod
+    def list(cls):
+        return list(map(lambda c: c.value, cls))
+
+    @classmethod
+    def from_value(cls, value: Union[OverlapFilter, str]) -> OverlapFilter:
+        if isinstance(value, cls):
+            return value
+        if isinstance(value, str):
+            value = value.lower()
+            try:
+                return cls(value)
+            except ValueError:
+                raise ValueError(f"Invalid value: {value}. Must be one of {cls.list()}")
+        raise ValueError(
+            f"Invalid value type: {type(value)}. Must be an instance of "
+            f"{cls.__name__} or str."
+        )

eye/detection/tools/csv_sink.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import csv
+import os
+from typing import Any, Dict, List, Optional
+
+from eye.detection.core import Detections
+
+BASE_HEADER = [
+    "x_min",
+    "y_min",
+    "x_max",
+    "y_max",
+    "class_id",
+    "confidence",
+    "tracker_id",
+]
+
+
+class CSVSink:
+    """
+    A utility class for saving detection data to a CSV file. This class is designed to
+    efficiently serialize detection objects into a CSV format, allowing for the
+    inclusion of bounding box coordinates and additional attributes like `confidence`,
+    `class_id`, and `tracker_id`.
+
+    !!! tip
+
+        CSVSink allow to pass custom data alongside the detection fields, providing
+        flexibility for logging various types of information.
+
+    Args:
+        file_name (str): The name of the CSV file where the detections will be stored.
+            Defaults to 'output.csv'.
+
+    Example:
+        ```python
+        import eye as sv
+        from ultralytics import YOLO
+
+        model = YOLO(<SOURCE_MODEL_PATH>)
+        csv_sink = sv.CSVSink(<RESULT_CSV_FILE_PATH>)
+        frames_generator = sv.get_video_frames_generator(<SOURCE_VIDEO_PATH>)
+
+        with csv_sink as sink:
+            for frame in frames_generator:
+                result = model(frame)[0]
+                detections = sv.Detections.from_ultralytics(result)
+                sink.append(detections, custom_data={'<CUSTOM_LABEL>':'<CUSTOM_DATA>'})
+        ```
+    """
+
+    def __init__(self, file_name: str = "output.csv") -> None:
+        """
+        Initialize the CSVSink instance.
+
+        Args:
+            file_name (str): The name of the CSV file.
+
+        Returns:
+            None
+        """
+        self.file_name = file_name
+        self.file: Optional[open] = None
+        self.writer: Optional[csv.writer] = None
+        self.header_written = False
+        self.field_names = []
+
+    def __enter__(self) -> CSVSink:
+        self.open()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[type],
+        exc_val: Optional[Exception],
+        exc_tb: Optional[Any],
+    ) -> None:
+        self.close()
+
+    def open(self) -> None:
+        """
+        Open the CSV file for writing.
+
+        Returns:
+            None
+        """
+        parent_directory = os.path.dirname(self.file_name)
+        if parent_directory and not os.path.exists(parent_directory):
+            os.makedirs(parent_directory)
+
+        self.file = open(self.file_name, "w", newline="")
+        self.writer = csv.writer(self.file)
+
+    def close(self) -> None:
+        """
+        Close the CSV file.
+
+        Returns:
+            None
+        """
+        if self.file:
+            self.file.close()
+
+    @staticmethod
+    def parse_detection_data(
+        detections: Detections, custom_data: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        parsed_rows = []
+        for i in range(len(detections.xyxy)):
+            row = {
+                "x_min": detections.xyxy[i][0],
+                "y_min": detections.xyxy[i][1],
+                "x_max": detections.xyxy[i][2],
+                "y_max": detections.xyxy[i][3],
+                "class_id": ""
+                if detections.class_id is None
+                else str(detections.class_id[i]),
+                "confidence": ""
+                if detections.confidence is None
+                else str(detections.confidence[i]),
+                "tracker_id": ""
+                if detections.tracker_id is None
+                else str(detections.tracker_id[i]),
+            }
+
+            if hasattr(detections, "data"):
+                for key, value in detections.data.items():
+                    if value.ndim == 0:
+                        row[key] = value
+                    else:
+                        row[key] = value[i]
+
+            if custom_data:
+                row.update(custom_data)
+            parsed_rows.append(row)
+        return parsed_rows
+
+    def append(
+        self, detections: Detections, custom_data: Optional[Dict[str, Any]] = None
+    ) -> None:
+        """
+        Append detection data to the CSV file.
+
+        Args:
+            detections (Detections): The detection data.
+            custom_data (Dict[str, Any]): Custom data to include.
+
+        Returns:
+            None
+        """
+        if not self.writer:
+            raise Exception(
+                f"Cannot append to CSV: The file '{self.file_name}' is not open."
+            )
+        field_names = CSVSink.parse_field_names(detections, custom_data)
+        if not self.header_written:
+            self.field_names = field_names
+            self.writer.writerow(field_names)
+            self.header_written = True
+
+        if field_names != self.field_names:
+            print(
+                f"Field names do not match the header. "
+                f"Expected: {self.field_names}, given: {field_names}"
+            )
+
+        parsed_rows = CSVSink.parse_detection_data(detections, custom_data)
+        for row in parsed_rows:
+            self.writer.writerow(
+                [row.get(field_name, "") for field_name in self.field_names]
+            )
+
+    @staticmethod
+    def parse_field_names(
+        detections: Detections, custom_data: Dict[str, Any]
+    ) -> List[str]:
+        dynamic_header = sorted(
+            set(custom_data.keys()) | set(getattr(detections, "data", {}).keys())
+        )
+        return BASE_HEADER + dynamic_header