valor-lite 0.36.6__py3-none-any.whl → 0.37.5__py3-none-any.whl

valor_lite/classification/loader.py

@@ -0,0 +1,97 @@
+import numpy as np
+import pyarrow as pa
+from tqdm import tqdm
+
+from valor_lite.cache.ephemeral import MemoryCacheWriter
+from valor_lite.cache.persistent import FileCacheWriter
+from valor_lite.classification.annotation import Classification
+from valor_lite.classification.evaluator import Builder
+
+
+class Loader(Builder):
+    def __init__(
+        self,
+        writer: MemoryCacheWriter | FileCacheWriter,
+        roc_curve_writer: MemoryCacheWriter | FileCacheWriter,
+        intermediate_writer: MemoryCacheWriter | FileCacheWriter,
+        metadata_fields: list[tuple[str, str | pa.DataType]] | None = None,
+    ):
+        super().__init__(
+            writer=writer,
+            roc_curve_writer=roc_curve_writer,
+            intermediate_writer=intermediate_writer,
+            metadata_fields=metadata_fields,
+        )
+
+        # internal state
+        self._labels: dict[str, int] = {}
+        self._index_to_label: dict[int, str] = {}
+        self._datum_count = 0
+
+    def _add_label(self, value: str) -> int:
+        idx = self._labels.get(value, None)
+        if idx is None:
+            idx = len(self._labels)
+            self._labels[value] = idx
+            self._index_to_label[idx] = value
+        return idx
+
+    def add_data(
+        self,
+        classifications: list[Classification],
+        show_progress: bool = False,
+    ):
+        """
+        Adds classifications to the cache.
+
+        Parameters
+        ----------
+        classifications : list[Classification]
+            A list of Classification objects.
+        show_progress : bool, default=False
+            Toggle for tqdm progress bar.
+        """
+
+        disable_tqdm = not show_progress
+        for classification in tqdm(classifications, disable=disable_tqdm):
+            if len(classification.predictions) == 0:
+                raise ValueError(
+                    "Classifications must contain at least one prediction."
+                )
+
+            # prepare metadata
+            datum_metadata = (
+                classification.metadata if classification.metadata else {}
+            )
+
+            # write to cache
+            rows = []
+            gidx = self._add_label(classification.groundtruth)
+            max_score_idx = np.argmax(np.array(classification.scores))
+            for idx, (plabel, score) in enumerate(
+                zip(classification.predictions, classification.scores)
+            ):
+                pidx = self._add_label(plabel)
+                rows.append(
+                    {
+                        # metadata
+                        **datum_metadata,
+                        # datum
+                        "datum_uid": classification.uid,
+                        "datum_id": self._datum_count,
+                        # groundtruth
+                        "gt_label": classification.groundtruth,
+                        "gt_label_id": gidx,
+                        # prediction
+                        "pd_label": plabel,
+                        "pd_label_id": pidx,
+                        "pd_score": float(score),
+                        "pd_winner": max_score_idx == idx,
+                        # pair
+                        "match": (gidx == pidx) and pidx >= 0,
+                    }
+                )
+            self._writer.write_rows(rows)
+
+            # update datum count
+            self._datum_count += 1

valor_lite/classification/metric.py

@@ -13,6 +13,8 @@ class MetricType(Enum):
     Accuracy = "Accuracy"
     F1 = "F1"
     ConfusionMatrix = "ConfusionMatrix"
+    Examples = "Examples"
+    ConfusionMatrixWithExamples = "ConfusionMatrixWithExamples"
 
 
 @dataclass
@@ -304,6 +306,139 @@ class Metric(BaseMetric):
 
     @classmethod
     def confusion_matrix(
+        cls,
+        confusion_matrix: dict[str, dict[str, int]],
+        unmatched_ground_truths: dict[str, int],
+        score_threshold: float,
+        hardmax: bool,
+    ):
+        """
+        Confusion matrix for object detection task.
+
+        This class encapsulates detailed information about the model's performance, including correct
+        predictions, misclassifications and unmatched ground truths (subset of false negatives).
+
+        Confusion Matrix Format:
+        {
+            <ground truth label>: {
+                <prediction label>: 129
+                ...
+            },
+            ...
+        }
+
+        Unmatched Ground Truths Format:
+        {
+            <ground truth label>: 7
+            ...
+        }
+
+        Parameters
+        ----------
+        confusion_matrix : dict
+            A nested dictionary containing integer counts of occurences where the first key is the ground truth label value
+            and the second key is the prediction label value.
+        unmatched_ground_truths : dict
+            A dictionary where each key is a ground truth label value for which the model failed to predict
+            (subset of false negatives). The value is a dictionary containing counts.
+        score_threshold : float
+            The confidence score threshold used to filter predictions.
+        hardmax : bool
+            Indicates whether hardmax thresholding was used.
+
+        Returns
+        -------
+        Metric
+        """
+        return cls(
+            type=MetricType.ConfusionMatrix.value,
+            value={
+                "confusion_matrix": confusion_matrix,
+                "unmatched_ground_truths": unmatched_ground_truths,
+            },
+            parameters={
+                "score_threshold": score_threshold,
+                "hardmax": hardmax,
+            },
+        )
+
+    @classmethod
+    def examples(
+        cls,
+        datum_id: str,
+        true_positives: list[str],
+        false_positives: list[str],
+        false_negatives: list[str],
+        score_threshold: float,
+        hardmax: bool,
+    ):
+        """
+        Per-datum examples for object detection tasks.
+
+        This metric is per-datum and contains lists of annotation identifiers that categorize them
+        as true-positive, false-positive or false-negative. This is intended to be used with an
+        external database where the identifiers can be used for retrieval.
+
+        Examples Format:
+        {
+            "type": "Examples",
+            "value": {
+                "datum_id": "some string ID",
+                "true_positives": [
+                    "label A",
+                ],
+                "false_positives": [
+                    "label 25",
+                    "label 92",
+                    ...
+                ]
+                "false_negatives": [
+                    "groundtruth32",
+                    "groundtruth24",
+                    ...
+                ]
+            },
+            "parameters": {
+                "score_threshold": 0.5,
+                "hardmax": False,
+            }
+        }
+
+        Parameters
+        ----------
+        datum_id : str
+            A string identifier representing a datum.
+        true_positives : list[str]
+            A list of string identifier representing true positive labels.
+        false_positives : list[str]
+            A list of string identifiers representing false positive predictions.
+        false_negatives : list[str]
+            A list of string identifiers representing false negative ground truths.
+        score_threshold : float
+            The confidence score threshold used to filter predictions.
+        hardmax : bool
+            Indicates whether hardmax thresholding was used.
+
+        Returns
+        -------
+        Metric
+        """
+        return cls(
+            type=MetricType.Examples.value,
+            value={
+                "datum_id": datum_id,
+                "true_positives": true_positives,
+                "false_positives": false_positives,
+                "false_negatives": false_negatives,
+            },
+            parameters={
+                "score_threshold": score_threshold,
+                "hardmax": hardmax,
+            },
+        )
+
+    @classmethod
+    def confusion_matrix_with_examples(
         cls,
         confusion_matrix: dict[
             str,  # ground truth label value
@@ -329,9 +464,10 @@ class Metric(BaseMetric):
             ],
         ],
         score_threshold: float,
+        hardmax: bool,
     ):
         """
-        The confusion matrix and related metrics for the classification task.
+        The confusion matrix with examples for the classification task.
 
         This class encapsulates detailed information about the model's performance, including correct
         predictions, misclassifications and unmatched ground truths (subset of false negatives).
@@ -379,20 +515,21 @@ class Metric(BaseMetric):
             A dictionary where each key is a ground truth label value for which the model failed to predict
             (false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
             Each example includes the datum UID.
-        score_threshold : float
-            The confidence score threshold used to filter predictions.
+        hardmax : bool
+            Indicates whether hardmax thresholding was used.
 
         Returns
         -------
         Metric
         """
         return cls(
-            type=MetricType.ConfusionMatrix.value,
+            type=MetricType.ConfusionMatrixWithExamples.value,
             value={
                 "confusion_matrix": confusion_matrix,
                 "unmatched_ground_truths": unmatched_ground_truths,
             },
             parameters={
                 "score_threshold": score_threshold,
+                "hardmax": hardmax,
             },
         )

valor_lite/classification/shared.py

@@ -0,0 +1,184 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+from numpy.typing import NDArray
+
+from valor_lite.cache import FileCacheReader, MemoryCacheReader
+
+
+@dataclass
+class EvaluatorInfo:
+    number_of_rows: int = 0
+    number_of_datums: int = 0
+    number_of_labels: int = 0
+    metadata_fields: list[tuple[str, str]] | None = None
+
+
+def generate_cache_path(path: str | Path) -> Path:
+    return Path(path) / "cache"
+
+
+def generate_intermediate_cache_path(path: str | Path) -> Path:
+    return Path(path) / "intermediate"
+
+
+def generate_roc_curve_cache_path(path: str | Path) -> Path:
+    return Path(path) / "roc_curve"
+
+
+def generate_metadata_path(path: str | Path) -> Path:
+    return Path(path) / "metadata.json"
+
+
+def generate_schema(
+    metadata_fields: list[tuple[str, str | pa.DataType]] | None
+) -> pa.Schema:
+    metadata_fields = metadata_fields if metadata_fields else []
+    reserved_fields = [
+        ("datum_uid", pa.string()),
+        ("datum_id", pa.int64()),
+        # groundtruth
+        ("gt_label", pa.string()),
+        ("gt_label_id", pa.int64()),
+        # prediction
+        ("pd_label", pa.string()),
+        ("pd_label_id", pa.int64()),
+        ("pd_score", pa.float64()),
+        ("pd_winner", pa.bool_()),
+        # pair
+        ("match", pa.bool_()),
+    ]
+
+    # validate
+    reserved_field_names = {f[0] for f in reserved_fields}
+    metadata_field_names = {f[0] for f in metadata_fields}
+    if conflicting := reserved_field_names & metadata_field_names:
+        raise ValueError(
+            f"metadata fields {conflicting} conflict with reserved fields"
+        )
+    return pa.schema(reserved_fields + metadata_fields)
+
+
+def generate_intermediate_schema() -> pa.Schema:
+    return pa.schema(
+        [
+            ("pd_label_id", pa.int64()),
+            ("pd_score", pa.float64()),
+            ("match", pa.bool_()),
+        ]
+    )
+
+
+def generate_roc_curve_schema() -> pa.Schema:
+    return pa.schema(
+        [
+            ("pd_label_id", pa.int64()),
+            ("cumulative_fp", pa.uint64()),
+            ("cumulative_tp", pa.uint64()),
+        ]
+    )
+
+
+def encode_metadata_fields(
+    metadata_fields: list[tuple[str, str | pa.DataType]] | None
+) -> dict[str, str]:
+    metadata_fields = metadata_fields if metadata_fields else []
+    return {k: str(v) for k, v in metadata_fields}
+
+
+def decode_metadata_fields(
+    encoded_metadata_fields: dict[str, str]
+) -> list[tuple[str, str]]:
+    return [(k, v) for k, v in encoded_metadata_fields.items()]
+
+
+def extract_labels(
+    reader: MemoryCacheReader | FileCacheReader,
+    index_to_label_override: dict[int, str] | None = None,
+) -> dict[int, str]:
+    if index_to_label_override is not None:
+        return index_to_label_override
+
+    index_to_label = {}
+    for tbl in reader.iterate_tables(
+        columns=[
+            "gt_label_id",
+            "gt_label",
+            "pd_label_id",
+            "pd_label",
+        ]
+    ):
+
+        # get gt labels
+        gt_label_ids = tbl["gt_label_id"].to_numpy()
+        gt_label_ids, gt_indices = np.unique(gt_label_ids, return_index=True)
+        gt_labels = tbl["gt_label"].take(gt_indices).to_pylist()
+        gt_labels = dict(zip(gt_label_ids.astype(int).tolist(), gt_labels))
+        gt_labels.pop(-1, None)
+        index_to_label.update(gt_labels)
+
+        # get pd labels
+        pd_label_ids = tbl["pd_label_id"].to_numpy()
+        pd_label_ids, pd_indices = np.unique(pd_label_ids, return_index=True)
+        pd_labels = tbl["pd_label"].take(pd_indices).to_pylist()
+        pd_labels = dict(zip(pd_label_ids.astype(int).tolist(), pd_labels))
+        pd_labels.pop(-1, None)
+        index_to_label.update(pd_labels)
+
+    return index_to_label
+
+
+def extract_counts(
+    reader: MemoryCacheReader | FileCacheReader,
+    datums: pc.Expression | None = None,
+):
+    n_dts = 0
+    for tbl in reader.iterate_tables(filter=datums):
+        n_dts += int(np.unique(tbl["datum_id"].to_numpy()).shape[0])
+    return n_dts
+
+
+def extract_groundtruth_count_per_label(
+    reader: MemoryCacheReader | FileCacheReader,
+    number_of_labels: int,
+    datums: pc.Expression | None = None,
+    groundtruths: pc.Expression | None = None,
+    predictions: pc.Expression | None = None,
+) -> NDArray[np.uint64]:
+
+    # count ground truth and prediction label occurences
+    label_counts = np.zeros((number_of_labels, 2), dtype=np.uint64)
+    for tbl in reader.iterate_tables(filter=datums):
+
+        # count unique gt labels
+        gt_expr = pc.field("gt_label_id") >= 0
+        if groundtruths is not None:
+            gt_expr &= groundtruths
+        gt_tbl = tbl.filter(gt_expr)
+        gt_ids = np.column_stack(
+            [gt_tbl[col].to_numpy() for col in ["datum_id", "gt_label_id"]]
+        ).astype(np.int64)
+        unique_gts = np.unique(gt_ids, axis=0)
+        unique_gt_labels, gt_label_counts = np.unique(
+            unique_gts[:, 1], return_counts=True
+        )
+        label_counts[unique_gt_labels, 0] += gt_label_counts.astype(np.uint64)
+
+        # count unique pd labels
+        pd_expr = pc.field("pd_label_id") >= 0
+        if predictions is not None:
+            pd_expr &= predictions
+        pd_tbl = tbl.filter(pd_expr)
+        pd_ids = np.column_stack(
+            [pd_tbl[col].to_numpy() for col in ["datum_id", "pd_label_id"]]
+        ).astype(np.int64)
+        unique_pds = np.unique(pd_ids, axis=0)
+        unique_pd_labels, pd_label_counts = np.unique(
+            unique_pds[:, 1], return_counts=True
+        )
+        label_counts[unique_pd_labels, 1] += pd_label_counts.astype(np.uint64)
+
+    return label_counts