valor-lite 0.37.1__py3-none-any.whl

valor_lite/semantic_segmentation/loader.py

@@ -0,0 +1,205 @@
+import numpy as np
+import pyarrow as pa
+from tqdm import tqdm
+
+from valor_lite.cache import FileCacheWriter, MemoryCacheWriter
+from valor_lite.semantic_segmentation.annotation import Segmentation
+from valor_lite.semantic_segmentation.computation import compute_intermediates
+from valor_lite.semantic_segmentation.evaluator import Builder
+
+
+class Loader(Builder):
+    def __init__(
+        self,
+        writer: MemoryCacheWriter | FileCacheWriter,
+        metadata_fields: list[tuple[str, str | pa.DataType]] | None = None,
+    ):
+        super().__init__(
+            writer=writer,
+            metadata_fields=metadata_fields,
+        )
+
+        # internal state
+        self._labels: dict[str, int] = {}
+        self._index_to_label: dict[int, str] = {}
+        self._datum_count = 0
+
+    def _add_label(self, value: str) -> int:
+        idx = self._labels.get(value, None)
+        if idx is None:
+            idx = len(self._labels)
+            self._labels[value] = idx
+            self._index_to_label[idx] = value
+        return idx
+
+    def add_data(
+        self,
+        segmentations: list[Segmentation],
+        show_progress: bool = False,
+    ):
+        """
+        Adds segmentations to the cache.
+
+        Parameters
+        ----------
+        segmentations : list[Segmentation]
+            A list of Segmentation objects.
+        show_progress : bool, default=False
+            Toggle for tqdm progress bar.
+        """
+
+        disable_tqdm = not show_progress
+        for segmentation in tqdm(segmentations, disable=disable_tqdm):
+
+            groundtruth_labels = -1 * np.ones(
+                len(segmentation.groundtruths), dtype=np.int64
+            )
+            for idx, groundtruth in enumerate(segmentation.groundtruths):
+                label_idx = self._add_label(groundtruth.label)
+                groundtruth_labels[idx] = label_idx
+
+            prediction_labels = -1 * np.ones(
+                len(segmentation.predictions), dtype=np.int64
+            )
+            for idx, prediction in enumerate(segmentation.predictions):
+                label_idx = self._add_label(prediction.label)
+                prediction_labels[idx] = label_idx
+
+            if segmentation.groundtruths:
+                combined_groundtruths = np.stack(
+                    [
+                        groundtruth.mask.flatten()
+                        for groundtruth in segmentation.groundtruths
+                    ],
+                    axis=0,
+                )
+            else:
+                combined_groundtruths = np.zeros(
+                    (1, segmentation.shape[0] * segmentation.shape[1]),
+                    dtype=np.bool_,
+                )
+
+            if segmentation.predictions:
+                combined_predictions = np.stack(
+                    [
+                        prediction.mask.flatten()
+                        for prediction in segmentation.predictions
+                    ],
+                    axis=0,
+                )
+            else:
+                combined_predictions = np.zeros(
+                    (1, segmentation.shape[0] * segmentation.shape[1]),
+                    dtype=np.bool_,
+                )
+
+            n_labels = len(self._labels)
+            counts = compute_intermediates(
+                groundtruths=combined_groundtruths,
+                predictions=combined_predictions,
+                groundtruth_labels=groundtruth_labels,
+                prediction_labels=prediction_labels,
+                n_labels=n_labels,
+            )
+
+            # prepare metadata
+            datum_metadata = (
+                segmentation.metadata if segmentation.metadata else {}
+            )
+            gt_metadata = {
+                self._labels[gt.label]: gt.metadata
+                for gt in segmentation.groundtruths
+                if gt.metadata
+            }
+            pd_metadata = {
+                self._labels[pd.label]: pd.metadata
+                for pd in segmentation.predictions
+                if pd.metadata
+            }
+
+            # cache formatting
+            rows = []
+            for idx in range(n_labels):
+                label = self._index_to_label[idx]
+                for pidx in range(n_labels):
+                    # write non-zero intersections to cache
+                    if counts[idx + 1, pidx + 1] > 0:
+                        plabel = self._index_to_label[pidx]
+                        rows.append(
+                            {
+                                # metadata
+                                **datum_metadata,
+                                **gt_metadata.get(idx, {}),
+                                **pd_metadata.get(pidx, {}),
+                                # datum
+                                "datum_uid": segmentation.uid,
+                                "datum_id": self._datum_count,
+                                # groundtruth
+                                "gt_label": label,
+                                "gt_label_id": idx,
+                                # prediction
+                                "pd_label": plabel,
+                                "pd_label_id": pidx,
+                                # pair
+                                "count": counts[idx + 1, pidx + 1],
+                            }
+                        )
+                # write all unmatched to preserve labels
+                rows.extend(
+                    [
+                        {
+                            # metadata
+                            **datum_metadata,
+                            **gt_metadata.get(idx, {}),
+                            # datum
+                            "datum_uid": segmentation.uid,
+                            "datum_id": self._datum_count,
+                            # groundtruth
+                            "gt_label": label,
+                            "gt_label_id": idx,
+                            # prediction
+                            "pd_label": None,
+                            "pd_label_id": -1,
+                            # pair
+                            "count": counts[idx + 1, 0],
+                        },
+                        {
+                            # metadata
+                            **datum_metadata,
+                            **gt_metadata.get(idx, {}),
+                            **pd_metadata.get(idx, {}),
+                            # datum
+                            "datum_uid": segmentation.uid,
+                            "datum_id": self._datum_count,
+                            # groundtruth
+                            "gt_label": None,
+                            "gt_label_id": -1,
+                            # prediction
+                            "pd_label": label,
+                            "pd_label_id": idx,
+                            # pair
+                            "count": counts[0, idx + 1],
+                        },
+                    ]
+                )
+            rows.append(
+                {
+                    # metadata
+                    **datum_metadata,
+                    # datum
+                    "datum_uid": segmentation.uid,
+                    "datum_id": self._datum_count,
+                    # groundtruth
+                    "gt_label": None,
+                    "gt_label_id": -1,
+                    # prediction
+                    "pd_label": None,
+                    "pd_label_id": -1,
+                    # pair
+                    "count": counts[0, 0],
+                }
+            )
+            self._writer.write_rows(rows)
+
+            # update datum count
+            self._datum_count += 1

valor_lite/semantic_segmentation/metric.py

@@ -0,0 +1,275 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from valor_lite.schemas import BaseMetric
+
+
+class MetricType(Enum):
+    Precision = "Precision"
+    Recall = "Recall"
+    Accuracy = "Accuracy"
+    F1 = "F1"
+    IOU = "IOU"
+    mIOU = "mIOU"
+    ConfusionMatrix = "ConfusionMatrix"
+
+
+@dataclass
+class Metric(BaseMetric):
+    """
+    Semantic Segmentation Metric.
+
+    Attributes
+    ----------
+    type : str
+        The metric type.
+    value : int | float | dict
+        The metric value.
+    parameters : dict[str, Any]
+        A dictionary containing metric parameters.
+    """
+
+    def __post_init__(self):
+        if not isinstance(self.type, str):
+            raise TypeError(
+                f"Metric type should be of type 'str': {self.type}"
+            )
+        elif not isinstance(self.value, (int, float, dict)):
+            raise TypeError(
+                f"Metric value must be of type 'int', 'float' or 'dict': {self.value}"
+            )
+        elif not isinstance(self.parameters, dict):
+            raise TypeError(
+                f"Metric parameters must be of type 'dict[str, Any]': {self.parameters}"
+            )
+        elif not all([isinstance(k, str) for k in self.parameters.keys()]):
+            raise TypeError(
+                f"Metric parameter dictionary should only have keys with type 'str': {self.parameters}"
+            )
+
+    @classmethod
+    def precision(
+        cls,
+        value: float,
+        label: str,
+    ):
+        """
+        Precision metric for a specific class label.
+
+        Precision is calulated using the number of true-positive pixels divided by
+        the sum of all true-positive and false-positive pixels.
+
+        Parameters
+        ----------
+        value : float
+            The computed precision value.
+        label : str
+            The label for which the precision is calculated.
+
+        Returns
+        -------
+        Metric
+        """
+        return cls(
+            type=MetricType.Precision.value,
+            value=value,
+            parameters={
+                "label": label,
+            },
+        )
+
+    @classmethod
+    def recall(
+        cls,
+        value: float,
+        label: str,
+    ):
+        """
+        Recall metric for a specific class label.
+
+        Recall is calulated using the number of true-positive pixels divided by
+        the sum of all true-positive and false-negative pixels.
+
+        Parameters
+        ----------
+        value : float
+            The computed recall value.
+        label : str
+            The label for which the recall is calculated.
+
+        Returns
+        -------
+        Metric
+        """
+        return cls(
+            type=MetricType.Recall.value,
+            value=value,
+            parameters={
+                "label": label,
+            },
+        )
+
+    @classmethod
+    def f1_score(
+        cls,
+        value: float,
+        label: str,
+    ):
+        """
+        F1 score for a specific class label.
+
+        Parameters
+        ----------
+        value : float
+            The computed F1 score.
+        label : str
+            The label for which the F1 score is calculated.
+
+        Returns
+        -------
+        Metric
+        """
+        return cls(
+            type=MetricType.F1.value,
+            value=value,
+            parameters={
+                "label": label,
+            },
+        )
+
+    @classmethod
+    def iou(
+        cls,
+        value: float,
+        label: str,
+    ):
+        """
+        Intersection over Union (IOU) ratio for a specific class label.
+
+        Parameters
+        ----------
+        value : float
+            The computed IOU ratio.
+        label : str
+            The label for which the IOU is calculated.
+
+        Returns
+        -------
+        Metric
+        """
+        return cls(
+            type=MetricType.IOU.value,
+            value=value,
+            parameters={
+                "label": label,
+            },
+        )
+
+    @classmethod
+    def mean_iou(cls, value: float):
+        """
+        Mean Intersection over Union (mIOU) ratio.
+
+        The mIOU value is computed by averaging IOU over all labels.
+
+        Parameters
+        ----------
+        value : float
+            The mIOU value.
+
+        Returns
+        -------
+        Metric
+        """
+        return cls(type=MetricType.mIOU.value, value=value, parameters={})
+
+    @classmethod
+    def accuracy(cls, value: float):
+        """
+        Accuracy metric computed over all labels.
+
+        Parameters
+        ----------
+        value : float
+            The accuracy value.
+
+        Returns
+        -------
+        Metric
+        """
+        return cls(type=MetricType.Accuracy.value, value=value, parameters={})
+
+    @classmethod
+    def confusion_matrix(
+        cls,
+        confusion_matrix: dict[
+            str,  # ground truth label value
+            dict[
+                str,  # prediction label value
+                dict[str, float],  # iou
+            ],
+        ],
+        unmatched_predictions: dict[
+            str,  # prediction label value
+            dict[str, float],  # pixel ratio
+        ],
+        unmatched_ground_truths: dict[
+            str,  # ground truth label value
+            dict[str, float],  # pixel ratio
+        ],
+    ):
+        """
+        The confusion matrix and related metrics for semantic segmentation tasks.
+
+        This class encapsulates detailed information about the model's performance, including correct
+        predictions, misclassifications, unmatched_predictions (subset of false positives), and unmatched ground truths
+        (subset of false negatives). It provides counts for each category to facilitate in-depth analysis.
+
+        Confusion Matrix Format:
+        {
+            <ground truth label>: {
+                <prediction label>: {
+                    'iou': <float>,
+                },
+            },
+        }
+
+        Unmatched Predictions Format:
+        {
+            <prediction label>: {
+                'iou': <float>,
+            },
+        }
+
+        Unmatched Ground Truths Format:
+        {
+            <ground truth label>: {
+                'iou': <float>,
+            },
+        }
+
+        Parameters
+        ----------
+        confusion_matrix : dict
+            Nested dictionaries representing the Intersection over Union (IOU) scores for each
+            ground truth label and prediction label pair.
+        unmatched_predictions : dict
+            Dictionary representing the pixel ratios for predicted labels that do not correspond
+            to any ground truth labels (false positives).
+        unmatched_ground_truths : dict
+            Dictionary representing the pixel ratios for ground truth labels that were not predicted
+            (false negatives).
+
+        Returns
+        -------
+        Metric
+        """
+        return cls(
+            type=MetricType.ConfusionMatrix.value,
+            value={
+                "confusion_matrix": confusion_matrix,
+                "unmatched_predictions": unmatched_predictions,
+                "unmatched_ground_truths": unmatched_ground_truths,
+            },
+            parameters={},
+        )

valor_lite/semantic_segmentation/shared.py

@@ -0,0 +1,149 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from valor_lite.cache import FileCacheReader, MemoryCacheReader
+
+
+@dataclass
+class EvaluatorInfo:
+    number_of_rows: int = 0
+    number_of_datums: int = 0
+    number_of_labels: int = 0
+    number_of_pixels: int = 0
+    number_of_groundtruth_pixels: int = 0
+    number_of_prediction_pixels: int = 0
+    metadata_fields: list[tuple[str, str | pa.DataType]] | None = None
+
+
+def generate_cache_path(path: str | Path) -> Path:
+    """Generate cache path from parent directory."""
+    return Path(path) / "counts"
+
+
+def generate_metadata_path(path: str | Path) -> Path:
+    """Generate metadata path from parent directory."""
+    return Path(path) / "metadata.json"
+
+
+def generate_schema(
+    metadata_fields: list[tuple[str, str | pa.DataType]] | None
+) -> pa.Schema:
+    """Generate PyArrow schema from metadata fields."""
+
+    metadata_fields = metadata_fields if metadata_fields else []
+    reserved_fields = [
+        ("datum_uid", pa.string()),
+        ("datum_id", pa.int64()),
+        # groundtruth
+        ("gt_label", pa.string()),
+        ("gt_label_id", pa.int64()),
+        # prediction
+        ("pd_label", pa.string()),
+        ("pd_label_id", pa.int64()),
+        # pair
+        ("count", pa.uint64()),
+    ]
+
+    # validate
+    reserved_field_names = {f[0] for f in reserved_fields}
+    metadata_field_names = {f[0] for f in metadata_fields}
+    if conflicting := reserved_field_names & metadata_field_names:
+        raise ValueError(
+            f"metadata fields {conflicting} conflict with reserved fields"
+        )
+
+    return pa.schema(
+        [
+            *reserved_fields,
+            *metadata_fields,
+        ]
+    )
+
+
+def encode_metadata_fields(
+    metadata_fields: list[tuple[str, str | pa.DataType]] | None
+) -> dict[str, str]:
+    """Encode metadata fields into JSON format."""
+    metadata_fields = metadata_fields if metadata_fields else []
+    return {k: str(v) for k, v in metadata_fields}
+
+
+def decode_metadata_fields(
+    encoded_metadata_fields: dict[str, str]
+) -> list[tuple[str, str | pa.DataType]]:
+    """Decode metadata fields from JSON format."""
+    return [(k, v) for k, v in encoded_metadata_fields.items()]
+
+
+def extract_labels(
+    reader: MemoryCacheReader | FileCacheReader,
+    index_to_label_override: dict[int, str] | None = None,
+) -> dict[int, str]:
+    if index_to_label_override is not None:
+        return index_to_label_override
+
+    index_to_label = {}
+    for tbl in reader.iterate_tables(
+        columns=[
+            "gt_label_id",
+            "gt_label",
+            "pd_label_id",
+            "pd_label",
+        ]
+    ):
+
+        # get gt labels
+        gt_label_ids = tbl["gt_label_id"].to_numpy()
+        gt_label_ids, gt_indices = np.unique(gt_label_ids, return_index=True)
+        gt_labels = tbl["gt_label"].take(gt_indices).to_pylist()
+        gt_labels = dict(zip(gt_label_ids.astype(int).tolist(), gt_labels))
+        gt_labels.pop(-1, None)
+        index_to_label.update(gt_labels)
+
+        # get pd labels
+        pd_label_ids = tbl["pd_label_id"].to_numpy()
+        pd_label_ids, pd_indices = np.unique(pd_label_ids, return_index=True)
+        pd_labels = tbl["pd_label"].take(pd_indices).to_pylist()
+        pd_labels = dict(zip(pd_label_ids.astype(int).tolist(), pd_labels))
+        pd_labels.pop(-1, None)
+        index_to_label.update(pd_labels)
+
+    return index_to_label
+
+
+def extract_counts(
+    reader: MemoryCacheReader | FileCacheReader,
+    datums: pc.Expression | None = None,
+    groundtruths: pc.Expression | None = None,
+    predictions: pc.Expression | None = None,
+):
+    n_dts, n_total, n_gts, n_pds = 0, 0, 0, 0
+    for tbl in reader.iterate_tables(filter=datums):
+
+        # count datums
+        n_dts += int(np.unique(tbl["datum_id"].to_numpy()).shape[0])
+
+        # count pixels
+        n_total += int(tbl["count"].to_numpy().sum())
+
+        # count groundtruth pixels
+        gt_tbl = tbl
+        gt_expr = pc.field("gt_label_id") >= 0
+        if groundtruths is not None:
+            gt_expr &= groundtruths
+        gt_tbl = tbl.filter(gt_expr)
+        n_gts += int(gt_tbl["count"].to_numpy().sum())
+
+        # count prediction pixels
+        pd_tbl = tbl
+        pd_expr = pc.field("pd_label_id") >= 0
+        if predictions is not None:
+            pd_expr &= predictions
+        pd_tbl = tbl.filter(pd_expr)
+        n_pds += int(pd_tbl["count"].to_numpy().sum())
+
+    return n_dts, n_total, n_gts, n_pds

valor_lite/semantic_segmentation/utilities.py

@@ -0,0 +1,88 @@
+from collections import defaultdict
+
+from valor_lite.semantic_segmentation.metric import Metric, MetricType
+
+
+def unpack_precision_recall_iou_into_metric_lists(
+    results: tuple,
+    index_to_label: dict[int, str],
+) -> dict[MetricType, list[Metric]]:
+
+    n_labels = len(index_to_label)
+    (
+        precision,
+        recall,
+        f1_score,
+        accuracy,
+        ious,
+        unmatched_prediction_ratios,
+        unmatched_ground_truth_ratios,
+    ) = results
+
+    metrics = defaultdict(list)
+
+    metrics[MetricType.Accuracy] = [
+        Metric.accuracy(
+            value=float(accuracy),
+        )
+    ]
+
+    metrics[MetricType.ConfusionMatrix] = [
+        Metric.confusion_matrix(
+            confusion_matrix={
+                index_to_label[gt_label_idx]: {
+                    index_to_label[pd_label_idx]: {
+                        "iou": float(ious[gt_label_idx, pd_label_idx])
+                    }
+                    for pd_label_idx in range(n_labels)
+                }
+                for gt_label_idx in range(n_labels)
+            },
+            unmatched_predictions={
+                index_to_label[pd_label_idx]: {
+                    "ratio": float(unmatched_prediction_ratios[pd_label_idx])
+                }
+                for pd_label_idx in range(n_labels)
+            },
+            unmatched_ground_truths={
+                index_to_label[gt_label_idx]: {
+                    "ratio": float(unmatched_ground_truth_ratios[gt_label_idx])
+                }
+                for gt_label_idx in range(n_labels)
+            },
+        )
+    ]
+
+    metrics[MetricType.mIOU] = [
+        Metric.mean_iou(
+            value=float(ious.diagonal().mean()),
+        )
+    ]
+
+    for label_idx, label in index_to_label.items():
+        metrics[MetricType.Precision].append(
+            Metric.precision(
+                value=float(precision[label_idx]),
+                label=label,
+            )
+        )
+        metrics[MetricType.Recall].append(
+            Metric.recall(
+                value=float(recall[label_idx]),
+                label=label,
+            )
+        )
+        metrics[MetricType.F1].append(
+            Metric.f1_score(
+                value=float(f1_score[label_idx]),
+                label=label,
+            )
+        )
+        metrics[MetricType.IOU].append(
+            Metric.iou(
+                value=float(ious[label_idx, label_idx]),
+                label=label,
+            )
+        )
+
+    return metrics