valor-lite 0.37.1__py3-none-any.whl

valor_lite/classification/shared.py

@@ -0,0 +1,184 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+from numpy.typing import NDArray
+
+from valor_lite.cache import FileCacheReader, MemoryCacheReader
+
+
+@dataclass
+class EvaluatorInfo:
+    number_of_rows: int = 0
+    number_of_datums: int = 0
+    number_of_labels: int = 0
+    metadata_fields: list[tuple[str, str]] | None = None
+
+
+def generate_cache_path(path: str | Path) -> Path:
+    return Path(path) / "cache"
+
+
+def generate_intermediate_cache_path(path: str | Path) -> Path:
+    return Path(path) / "intermediate"
+
+
+def generate_roc_curve_cache_path(path: str | Path) -> Path:
+    return Path(path) / "roc_curve"
+
+
+def generate_metadata_path(path: str | Path) -> Path:
+    return Path(path) / "metadata.json"
+
+
+def generate_schema(
+    metadata_fields: list[tuple[str, str | pa.DataType]] | None
+) -> pa.Schema:
+    metadata_fields = metadata_fields if metadata_fields else []
+    reserved_fields = [
+        ("datum_uid", pa.string()),
+        ("datum_id", pa.int64()),
+        # groundtruth
+        ("gt_label", pa.string()),
+        ("gt_label_id", pa.int64()),
+        # prediction
+        ("pd_label", pa.string()),
+        ("pd_label_id", pa.int64()),
+        ("pd_score", pa.float64()),
+        ("pd_winner", pa.bool_()),
+        # pair
+        ("match", pa.bool_()),
+    ]
+
+    # validate
+    reserved_field_names = {f[0] for f in reserved_fields}
+    metadata_field_names = {f[0] for f in metadata_fields}
+    if conflicting := reserved_field_names & metadata_field_names:
+        raise ValueError(
+            f"metadata fields {conflicting} conflict with reserved fields"
+        )
+    return pa.schema(reserved_fields + metadata_fields)
+
+
+def generate_intermediate_schema() -> pa.Schema:
+    return pa.schema(
+        [
+            ("pd_label_id", pa.int64()),
+            ("pd_score", pa.float64()),
+            ("match", pa.bool_()),
+        ]
+    )
+
+
+def generate_roc_curve_schema() -> pa.Schema:
+    return pa.schema(
+        [
+            ("pd_label_id", pa.int64()),
+            ("cumulative_fp", pa.uint64()),
+            ("cumulative_tp", pa.uint64()),
+        ]
+    )
+
+
+def encode_metadata_fields(
+    metadata_fields: list[tuple[str, str | pa.DataType]] | None
+) -> dict[str, str]:
+    metadata_fields = metadata_fields if metadata_fields else []
+    return {k: str(v) for k, v in metadata_fields}
+
+
+def decode_metadata_fields(
+    encoded_metadata_fields: dict[str, str]
+) -> list[tuple[str, str]]:
+    return [(k, v) for k, v in encoded_metadata_fields.items()]
+
+
+def extract_labels(
+    reader: MemoryCacheReader | FileCacheReader,
+    index_to_label_override: dict[int, str] | None = None,
+) -> dict[int, str]:
+    if index_to_label_override is not None:
+        return index_to_label_override
+
+    index_to_label = {}
+    for tbl in reader.iterate_tables(
+        columns=[
+            "gt_label_id",
+            "gt_label",
+            "pd_label_id",
+            "pd_label",
+        ]
+    ):
+
+        # get gt labels
+        gt_label_ids = tbl["gt_label_id"].to_numpy()
+        gt_label_ids, gt_indices = np.unique(gt_label_ids, return_index=True)
+        gt_labels = tbl["gt_label"].take(gt_indices).to_pylist()
+        gt_labels = dict(zip(gt_label_ids.astype(int).tolist(), gt_labels))
+        gt_labels.pop(-1, None)
+        index_to_label.update(gt_labels)
+
+        # get pd labels
+        pd_label_ids = tbl["pd_label_id"].to_numpy()
+        pd_label_ids, pd_indices = np.unique(pd_label_ids, return_index=True)
+        pd_labels = tbl["pd_label"].take(pd_indices).to_pylist()
+        pd_labels = dict(zip(pd_label_ids.astype(int).tolist(), pd_labels))
+        pd_labels.pop(-1, None)
+        index_to_label.update(pd_labels)
+
+    return index_to_label
+
+
+def extract_counts(
+    reader: MemoryCacheReader | FileCacheReader,
+    datums: pc.Expression | None = None,
+):
+    n_dts = 0
+    for tbl in reader.iterate_tables(filter=datums):
+        n_dts += int(np.unique(tbl["datum_id"].to_numpy()).shape[0])
+    return n_dts
+
+
+def extract_groundtruth_count_per_label(
+    reader: MemoryCacheReader | FileCacheReader,
+    number_of_labels: int,
+    datums: pc.Expression | None = None,
+    groundtruths: pc.Expression | None = None,
+    predictions: pc.Expression | None = None,
+) -> NDArray[np.uint64]:
+
+    # count ground truth and prediction label occurences
+    label_counts = np.zeros((number_of_labels, 2), dtype=np.uint64)
+    for tbl in reader.iterate_tables(filter=datums):
+
+        # count unique gt labels
+        gt_expr = pc.field("gt_label_id") >= 0
+        if groundtruths is not None:
+            gt_expr &= groundtruths
+        gt_tbl = tbl.filter(gt_expr)
+        gt_ids = np.column_stack(
+            [gt_tbl[col].to_numpy() for col in ["datum_id", "gt_label_id"]]
+        ).astype(np.int64)
+        unique_gts = np.unique(gt_ids, axis=0)
+        unique_gt_labels, gt_label_counts = np.unique(
+            unique_gts[:, 1], return_counts=True
+        )
+        label_counts[unique_gt_labels, 0] += gt_label_counts.astype(np.uint64)
+
+        # count unique pd labels
+        pd_expr = pc.field("pd_label_id") >= 0
+        if predictions is not None:
+            pd_expr &= predictions
+        pd_tbl = tbl.filter(pd_expr)
+        pd_ids = np.column_stack(
+            [pd_tbl[col].to_numpy() for col in ["datum_id", "pd_label_id"]]
+        ).astype(np.int64)
+        unique_pds = np.unique(pd_ids, axis=0)
+        unique_pd_labels, pd_label_counts = np.unique(
+            unique_pds[:, 1], return_counts=True
+        )
+        label_counts[unique_pd_labels, 1] += pd_label_counts.astype(np.uint64)
+
+    return label_counts

valor_lite/classification/utilities.py

@@ -0,0 +1,314 @@
+from collections import defaultdict
+
+import numpy as np
+import pyarrow as pa
+from numpy.typing import NDArray
+
+from valor_lite.classification.metric import Metric, MetricType
+
+
+def unpack_precision_recall(
+    counts: NDArray[np.uint64],
+    precision: NDArray[np.float64],
+    recall: NDArray[np.float64],
+    accuracy: NDArray[np.float64],
+    f1_score: NDArray[np.float64],
+    score_thresholds: list[float],
+    hardmax: bool,
+    index_to_label: dict[int, str],
+) -> dict[MetricType, list[Metric]]:
+
+    metrics = defaultdict(list)
+
+    metrics[MetricType.Accuracy] = [
+        Metric.accuracy(
+            value=float(accuracy[score_idx]),
+            score_threshold=score_threshold,
+            hardmax=hardmax,
+        )
+        for score_idx, score_threshold in enumerate(score_thresholds)
+    ]
+
+    for label_idx, label in index_to_label.items():
+        for score_idx, score_threshold in enumerate(score_thresholds):
+            kwargs = {
+                "label": label,
+                "hardmax": hardmax,
+                "score_threshold": score_threshold,
+            }
+            row = counts[:, label_idx]
+            metrics[MetricType.Counts].append(
+                Metric.counts(
+                    tp=int(row[score_idx, 0]),
+                    fp=int(row[score_idx, 1]),
+                    fn=int(row[score_idx, 2]),
+                    tn=int(row[score_idx, 3]),
+                    **kwargs,
+                )
+            )
+
+            metrics[MetricType.Precision].append(
+                Metric.precision(
+                    value=float(precision[score_idx, label_idx]),
+                    **kwargs,
+                )
+            )
+            metrics[MetricType.Recall].append(
+                Metric.recall(
+                    value=float(recall[score_idx, label_idx]),
+                    **kwargs,
+                )
+            )
+            metrics[MetricType.F1].append(
+                Metric.f1_score(
+                    value=float(f1_score[score_idx, label_idx]),
+                    **kwargs,
+                )
+            )
+    return metrics
+
+
+def unpack_rocauc(
+    rocauc: NDArray[np.float64],
+    mean_rocauc: float,
+    index_to_label: dict[int, str],
+) -> dict[MetricType, list[Metric]]:
+    metrics = {}
+    metrics[MetricType.ROCAUC] = [
+        Metric.roc_auc(
+            value=float(rocauc[label_idx]),
+            label=label,
+        )
+        for label_idx, label in index_to_label.items()
+    ]
+    metrics[MetricType.mROCAUC] = [
+        Metric.mean_roc_auc(
+            value=float(mean_rocauc),
+        )
+    ]
+    return metrics
+
+
+def unpack_confusion_matrix(
+    confusion_matrices: NDArray[np.uint64],
+    unmatched_groundtruths: NDArray[np.uint64],
+    index_to_label: dict[int, str],
+    score_thresholds: list[float],
+    hardmax: bool,
+) -> list[Metric]:
+    metrics = []
+    for score_idx, score_thresh in enumerate(score_thresholds):
+        cm_dict = {}
+        ugt_dict = {}
+        for idx, label in index_to_label.items():
+            ugt_dict[label] = int(unmatched_groundtruths[score_idx, idx])
+            for pidx, plabel in index_to_label.items():
+                if label not in cm_dict:
+                    cm_dict[label] = {}
+                cm_dict[label][plabel] = int(
+                    confusion_matrices[score_idx, idx, pidx]
+                )
+        metrics.append(
+            Metric.confusion_matrix(
+                confusion_matrix=cm_dict,
+                unmatched_ground_truths=ugt_dict,
+                score_threshold=score_thresh,
+                hardmax=hardmax,
+            )
+        )
+    return metrics
+
+
+def create_mapping(
+    tbl: pa.Table,
+    pairs: NDArray[np.float64],
+    index: int,
+    id_col: str,
+    uid_col: str,
+) -> dict[int, str]:
+    col = pairs[:, index].astype(np.int64)
+    values, indices = np.unique(col, return_index=True)
+    indices = indices[values >= 0]
+    return {
+        tbl[id_col][idx].as_py(): tbl[uid_col][idx].as_py() for idx in indices
+    }
+
+
+def unpack_examples(
+    ids: NDArray[np.int64],
+    mask_tp: NDArray[np.bool_],
+    mask_fn: NDArray[np.bool_],
+    mask_fp: NDArray[np.bool_],
+    score_thresholds: list[float],
+    hardmax: bool,
+    index_to_datum_id: dict[int, str],
+    index_to_label: dict[int, str],
+) -> list[Metric]:
+    metrics = []
+    unique_datums = np.unique(ids[:, 0])
+    for datum_index in unique_datums:
+        mask_datum = ids[:, 0] == datum_index
+        mask_datum_tp = mask_tp & mask_datum
+        mask_datum_fp = mask_fp & mask_datum
+        mask_datum_fn = mask_fn & mask_datum
+
+        datum_id = index_to_datum_id[datum_index]
+        for score_idx, score_thresh in enumerate(score_thresholds):
+
+            unique_tp = np.unique(
+                # extract true-positive (datum_id, gt_id, pd_id) pairs
+                ids[np.ix_(mask_datum_tp[score_idx], (0, 1, 2))],
+                axis=0,
+            )
+            unique_fp = np.unique(
+                # extract false-positive (datum_id, pd_id) pairs
+                ids[np.ix_(mask_datum_fp[score_idx], (0, 2))],
+                axis=0,
+            )
+            unique_fn = np.unique(
+                # extract false-negative (datum_id, gt_id)
+                ids[np.ix_(mask_datum_fn[score_idx], (0, 1))],
+                axis=0,
+            )
+
+            tp = [index_to_label[row[1]] for row in unique_tp]
+            fp = [
+                index_to_label[row[1]]
+                for row in unique_fp
+                if index_to_label[row[1]] not in tp
+            ]
+            fn = [
+                index_to_label[row[1]]
+                for row in unique_fn
+                if index_to_label[row[1]] not in tp
+            ]
+            metrics.append(
+                Metric.examples(
+                    datum_id=datum_id,
+                    true_positives=tp,
+                    false_negatives=fn,
+                    false_positives=fp,
+                    score_threshold=score_thresh,
+                    hardmax=hardmax,
+                )
+            )
+    return metrics
+
+
+def create_empty_confusion_matrix_with_examples(
+    score_threshold: float,
+    hardmax: bool,
+    index_to_label: dict[int, str],
+) -> Metric:
+    unmatched_groundtruths = dict()
+    confusion_matrix = dict()
+    for label in index_to_label.values():
+        unmatched_groundtruths[label] = {"count": 0, "examples": []}
+        confusion_matrix[label] = {}
+        for plabel in index_to_label.values():
+            confusion_matrix[label][plabel] = {"count": 0, "examples": []}
+
+    return Metric.confusion_matrix_with_examples(
+        confusion_matrix=confusion_matrix,
+        unmatched_ground_truths=unmatched_groundtruths,
+        score_threshold=score_threshold,
+        hardmax=hardmax,
+    )
+
+
+def _unpack_confusion_matrix_with_examples(
+    metric: Metric,
+    ids: NDArray[np.int64],
+    scores: NDArray[np.float64],
+    winners: NDArray[np.bool_],
+    mask_matched: NDArray[np.bool_],
+    mask_unmatched_fn: NDArray[np.bool_],
+    index_to_datum_id: dict[int, str],
+    index_to_label: dict[int, str],
+):
+    if not isinstance(metric.value, dict):
+        raise TypeError("expected metric to contain a dictionary value")
+
+    mask_valid_gts = ids[:, 1] >= 0
+    mask_valid_pds = ids[:, 2] >= 0
+
+    valid_matches = ids[mask_valid_gts & mask_valid_pds]
+    valid_gts = ids[mask_valid_gts]
+
+    n_matched = 0
+    unique_matches = np.empty((1, 3))
+    if valid_matches.size > 0:
+        unique_matches, unique_match_indices = np.unique(
+            # extract matched (datum_id, gt_id, pd_id) pairs
+            valid_matches[np.ix_(mask_matched, (0, 1, 2))],  # type: ignore[reportArgumentType]
+            axis=0,
+            return_index=True,
+        )
+        scores = scores[mask_matched][unique_match_indices]
+        n_matched = unique_matches.shape[0]
+
+    n_unmatched_groundtruths = 0
+    unique_unmatched_groundtruths = np.empty((1, 2))
+    if valid_gts.size > 0:
+        unique_unmatched_groundtruths = np.unique(
+            # extract unmatched false-negative (datum_id, gt_id) pairs
+            valid_gts[np.ix_(mask_unmatched_fn, (0, 1))],  # type: ignore[reportArgumentType]
+            axis=0,
+        )
+        unique_unmatched_groundtruths = unique_unmatched_groundtruths[
+            unique_unmatched_groundtruths[:, 1] >= 0
+        ]
+        n_unmatched_groundtruths = unique_unmatched_groundtruths.shape[0]
+
+    n_max = max(n_matched, n_unmatched_groundtruths)
+    for idx in range(n_max):
+        if idx < n_matched:
+            datum_id = index_to_datum_id[unique_matches[idx, 0]]
+            glabel = index_to_label[unique_matches[idx, 1]]
+            plabel = index_to_label[unique_matches[idx, 2]]
+            score = float(scores[idx])
+
+            metric.value["confusion_matrix"][glabel][plabel]["count"] += 1
+            metric.value["confusion_matrix"][glabel][plabel][
+                "examples"
+            ].append(
+                {
+                    "datum_id": datum_id,
+                    "score": score,
+                }
+            )
+        if idx < n_unmatched_groundtruths:
+            datum_id = index_to_datum_id[unique_unmatched_groundtruths[idx, 0]]
+            label = index_to_label[unique_unmatched_groundtruths[idx, 1]]
+
+            metric.value["unmatched_ground_truths"][label]["count"] += 1
+            metric.value["unmatched_ground_truths"][label]["examples"].append(
+                {"datum_id": datum_id}
+            )
+
+    return metric
+
+
+def unpack_confusion_matrix_with_examples(
+    metrics: dict[int, Metric],
+    ids: NDArray[np.int64],
+    scores: NDArray[np.float64],
+    winners: NDArray[np.bool_],
+    mask_matched: NDArray[np.bool_],
+    mask_unmatched_fn: NDArray[np.bool_],
+    index_to_datum_id: dict[int, str],
+    index_to_label: dict[int, str],
+) -> list[Metric]:
+    return [
+        _unpack_confusion_matrix_with_examples(
+            metric,
+            ids=ids,
+            scores=scores,
+            winners=winners,
+            mask_matched=mask_matched[score_idx, :],
+            mask_unmatched_fn=mask_unmatched_fn[score_idx, :],
+            index_to_datum_id=index_to_datum_id,
+            index_to_label=index_to_label,
+        )
+        for score_idx, metric in metrics.items()
+    ]

valor_lite/exceptions.py

@@ -0,0 +1,20 @@
+class EmptyEvaluatorError(Exception):
+    def __init__(self):
+        super().__init__(
+            "evaluator cannot be finalized as it contains no data"
+        )
+
+
+class EmptyCacheError(Exception):
+    def __init__(self):
+        super().__init__("cache contains no data")
+
+
+class EmptyFilterError(Exception):
+    def __init__(self, message: str):
+        super().__init__(message)
+
+
+class InternalCacheError(Exception):
+    def __init__(self, message: str):
+        super().__init__(message)

valor_lite/object_detection/__init__.py

@@ -0,0 +1,17 @@
+from .annotation import Bitmask, BoundingBox, Detection, Polygon
+from .evaluator import Evaluator
+from .loader import Loader
+from .metric import Metric, MetricType
+from .shared import EvaluatorInfo
+
+__all__ = [
+    "Bitmask",
+    "BoundingBox",
+    "Detection",
+    "Polygon",
+    "Metric",
+    "MetricType",
+    "Loader",
+    "Evaluator",
+    "EvaluatorInfo",
+]