orca-sdk 0.0.91__tar.gz → 0.0.92__tar.gz

{orca_sdk-0.0.91 → orca_sdk-0.0.92}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: orca_sdk
-Version: 0.0.91
+Version: 0.0.92
 Summary: SDK for interacting with Orca Services
 License: Apache-2.0
 Author: Orca DB Inc.
@@ -20,7 +20,9 @@ Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
 Requires-Dist: python-dateutil (>=2.8.0,<3.0.0)
 Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
+Requires-Dist: scikit-learn (>=1.6.1,<2.0.0)
 Requires-Dist: torch (>=2.5.1,<3.0.0)
+Requires-Dist: transformers (>=4.51.3,<5.0.0)
 Description-Content-Type: text/markdown
 
 <!--

{orca_sdk-0.0.91 → orca_sdk-0.0.92}/orca_sdk/_generated_api_client/models/base_label_prediction_result.py

@@ -10,7 +10,7 @@ The main change is:
 
 # flake8: noqa: C901
 
-from typing import Any, Type, TypeVar, Union, cast
+from typing import Any, List, Type, TypeVar, Union, cast
 
 from attrs import define as _attrs_define
 from attrs import field as _attrs_field
@@ -28,6 +28,7 @@ class BaseLabelPredictionResult:
         anomaly_score (Union[None, float]):
         label (int):
         label_name (Union[None, str]):
+        logits (List[float]):
     """
 
     prediction_id: Union[None, str]
@@ -35,6 +36,7 @@ class BaseLabelPredictionResult:
     anomaly_score: Union[None, float]
     label: int
     label_name: Union[None, str]
+    logits: List[float]
     additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
 
     def to_dict(self) -> dict[str, Any]:
@@ -51,6 +53,8 @@ class BaseLabelPredictionResult:
         label_name: Union[None, str]
         label_name = self.label_name
 
+        logits = self.logits
+
         field_dict: dict[str, Any] = {}
         field_dict.update(self.additional_properties)
         field_dict.update(
@@ -60,6 +64,7 @@ class BaseLabelPredictionResult:
                 "anomaly_score": anomaly_score,
                 "label": label,
                 "label_name": label_name,
+                "logits": logits,
             }
         )
 
@@ -94,12 +99,15 @@ class BaseLabelPredictionResult:
 
         label_name = _parse_label_name(d.pop("label_name"))
 
+        logits = cast(List[float], d.pop("logits"))
+
         base_label_prediction_result = cls(
             prediction_id=prediction_id,
             confidence=confidence,
             anomaly_score=anomaly_score,
             label=label,
             label_name=label_name,
+            logits=logits,
         )
 
         base_label_prediction_result.additional_properties = d

{orca_sdk-0.0.91 → orca_sdk-0.0.92}/orca_sdk/_generated_api_client/models/labeled_memoryset_metadata.py

@@ -43,6 +43,7 @@ class LabeledMemorysetMetadata:
         label_names (List[str]):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        memories_updated_at (datetime.datetime):
         insertion_task_id (str):
         insertion_status (TaskStatus): Status of task in the task queue
         metrics (MemorysetMetrics):
@@ -59,6 +60,7 @@ class LabeledMemorysetMetadata:
     label_names: List[str]
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    memories_updated_at: datetime.datetime
     insertion_task_id: str
     insertion_status: TaskStatus
     metrics: "MemorysetMetrics"
@@ -97,6 +99,8 @@ class LabeledMemorysetMetadata:
 
         updated_at = self.updated_at.isoformat()
 
+        memories_updated_at = self.memories_updated_at.isoformat()
+
         insertion_task_id = self.insertion_task_id
 
         insertion_status = (
@@ -120,6 +124,7 @@ class LabeledMemorysetMetadata:
                 "label_names": label_names,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "memories_updated_at": memories_updated_at,
                 "insertion_task_id": insertion_task_id,
                 "insertion_status": insertion_status,
                 "metrics": metrics,
@@ -180,6 +185,8 @@ class LabeledMemorysetMetadata:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        memories_updated_at = isoparse(d.pop("memories_updated_at"))
+
         insertion_task_id = d.pop("insertion_task_id")
 
         insertion_status = TaskStatus(d.pop("insertion_status"))
@@ -198,6 +205,7 @@ class LabeledMemorysetMetadata:
             label_names=label_names,
             created_at=created_at,
             updated_at=updated_at,
+            memories_updated_at=memories_updated_at,
             insertion_task_id=insertion_task_id,
             insertion_status=insertion_status,
             metrics=metrics,

orca_sdk-0.0.92/orca_sdk/_shared/__init__.py

@@ -0,0 +1 @@
+from .metrics import calculate_pr_curve, calculate_roc_curve, compute_classifier_metrics

orca_sdk-0.0.92/orca_sdk/_shared/metrics.py

@@ -0,0 +1,195 @@
+"""
+This module contains metrics for usage with the Hugging Face Trainer.
+
+IMPORTANT:
+- This is a shared file between OrcaLib and the Orca SDK.
+- Please ensure that it does not have any dependencies on the OrcaLib code.
+- Make sure to edit this file in orcalib/shared and NOT in orca_sdk, since it will be overwritten there.
+
+"""
+
+from typing import Literal, Tuple, TypedDict
+
+import numpy as np
+from numpy.typing import NDArray
+from scipy.special import softmax
+from sklearn.metrics import accuracy_score, auc, f1_score, log_loss
+from sklearn.metrics import precision_recall_curve as sklearn_precision_recall_curve
+from sklearn.metrics import roc_auc_score
+from sklearn.metrics import roc_curve as sklearn_roc_curve
+from transformers.trainer_utils import EvalPrediction
+
+
+class ClassificationMetrics(TypedDict):
+    accuracy: float
+    f1_score: float
+    roc_auc: float | None  # receiver operating characteristic area under the curve (if all classes are present)
+    pr_auc: float | None  # precision-recall area under the curve (only for binary classification)
+    log_loss: float  # cross-entropy loss for probabilities
+
+
+def compute_classifier_metrics(eval_pred: EvalPrediction) -> ClassificationMetrics:
+    """
+    Compute standard metrics for classifier with Hugging Face Trainer.
+
+    Args:
+        eval_pred: The predictions containing logits and expected labels as given by the Trainer.
+
+    Returns:
+        A dictionary containing the accuracy, f1 score, and ROC AUC score.
+    """
+    logits, references = eval_pred
+    if isinstance(logits, tuple):
+        logits = logits[0]
+    if not isinstance(logits, np.ndarray):
+        raise ValueError("Logits must be a numpy array")
+    if not isinstance(references, np.ndarray):
+        raise ValueError(
+            "Multiple label columns found, use the `label_names` training argument to specify which one to use"
+        )
+
+    if not (logits > 0).all():
+        # convert logits to probabilities with softmax if necessary
+        probabilities = softmax(logits)
+    elif not np.allclose(logits.sum(-1, keepdims=True), 1.0):
+        # convert logits to probabilities through normalization if necessary
+        probabilities = logits / logits.sum(-1, keepdims=True)
+    else:
+        probabilities = logits
+
+    return classification_scores(references, probabilities)
+
+
+def classification_scores(
+    references: NDArray[np.int64],
+    probabilities: NDArray[np.float32],
+    average: Literal["micro", "macro", "weighted", "binary"] | None = None,
+    multi_class: Literal["ovr", "ovo"] = "ovr",
+) -> ClassificationMetrics:
+    if probabilities.ndim == 1:
+        # convert 1D probabilities (binary) to 2D logits
+        probabilities = np.column_stack([1 - probabilities, probabilities])
+    elif probabilities.ndim == 2:
+        if probabilities.shape[1] < 2:
+            raise ValueError("Use a different metric function for regression tasks")
+    else:
+        raise ValueError("Probabilities must be 1 or 2 dimensional")
+
+    predictions = np.argmax(probabilities, axis=-1)
+
+    num_classes_references = len(set(references))
+    num_classes_predictions = len(set(predictions))
+
+    if average is None:
+        average = "binary" if num_classes_references == 2 else "weighted"
+
+    accuracy = accuracy_score(references, predictions)
+    f1 = f1_score(references, predictions, average=average)
+    loss = log_loss(references, probabilities)
+
+    if num_classes_references == num_classes_predictions:
+        # special case for binary classification: https://github.com/scikit-learn/scikit-learn/issues/20186
+        if num_classes_references == 2:
+            roc_auc = roc_auc_score(references, probabilities[:, 1])
+            precisions, recalls, _ = calculate_pr_curve(references, probabilities[:, 1])
+            pr_auc = auc(recalls, precisions)
+        else:
+            roc_auc = roc_auc_score(references, probabilities, multi_class=multi_class)
+            pr_auc = None
+    else:
+        roc_auc = None
+        pr_auc = None
+
+    return {
+        "accuracy": float(accuracy),
+        "f1_score": float(f1),
+        "roc_auc": float(roc_auc) if roc_auc is not None else None,
+        "pr_auc": float(pr_auc) if pr_auc is not None else None,
+        "log_loss": float(loss),
+    }
+
+
+def calculate_pr_curve(
+    references: NDArray[np.int64],
+    probabilities: NDArray[np.float32],
+    max_length: int = 100,
+) -> Tuple[NDArray[np.float32], NDArray[np.float32], NDArray[np.float32]]:
+    if probabilities.ndim == 1:
+        probabilities_slice = probabilities
+    elif probabilities.ndim == 2:
+        probabilities_slice = probabilities[:, 1]
+    else:
+        raise ValueError("Probabilities must be 1 or 2 dimensional")
+
+    if len(probabilities_slice) != len(references):
+        raise ValueError("Probabilities and references must have the same length")
+
+    precisions, recalls, thresholds = sklearn_precision_recall_curve(references, probabilities_slice)
+
+    # Convert all arrays to float32 immediately after getting them
+    precisions = precisions.astype(np.float32)
+    recalls = recalls.astype(np.float32)
+    thresholds = thresholds.astype(np.float32)
+
+    # Concatenate with 0 to include the lowest threshold
+    thresholds = np.concatenate(([0], thresholds))
+
+    # Sort by threshold
+    sorted_indices = np.argsort(thresholds)
+    thresholds = thresholds[sorted_indices]
+    precisions = precisions[sorted_indices]
+    recalls = recalls[sorted_indices]
+
+    if len(precisions) > max_length:
+        new_thresholds = np.linspace(0, 1, max_length, dtype=np.float32)
+        new_precisions = np.interp(new_thresholds, thresholds, precisions)
+        new_recalls = np.interp(new_thresholds, thresholds, recalls)
+        thresholds = new_thresholds
+        precisions = new_precisions
+        recalls = new_recalls
+
+    return precisions.astype(np.float32), recalls.astype(np.float32), thresholds.astype(np.float32)
+
+
+def calculate_roc_curve(
+    references: NDArray[np.int64],
+    probabilities: NDArray[np.float32],
+    max_length: int = 100,
+) -> Tuple[NDArray[np.float32], NDArray[np.float32], NDArray[np.float32]]:
+    if probabilities.ndim == 1:
+        probabilities_slice = probabilities
+    elif probabilities.ndim == 2:
+        probabilities_slice = probabilities[:, 1]
+    else:
+        raise ValueError("Probabilities must be 1 or 2 dimensional")
+
+    if len(probabilities_slice) != len(references):
+        raise ValueError("Probabilities and references must have the same length")
+
+    # Convert probabilities to float32 before calling sklearn_roc_curve
+    probabilities_slice = probabilities_slice.astype(np.float32)
+    fpr, tpr, thresholds = sklearn_roc_curve(references, probabilities_slice)
+
+    # Convert all arrays to float32 immediately after getting them
+    fpr = fpr.astype(np.float32)
+    tpr = tpr.astype(np.float32)
+    thresholds = thresholds.astype(np.float32)
+
+    # We set the first threshold to 1.0 instead of inf for reasonable values in interpolation
+    thresholds[0] = 1.0
+
+    # Sort by threshold
+    sorted_indices = np.argsort(thresholds)
+    thresholds = thresholds[sorted_indices]
+    fpr = fpr[sorted_indices]
+    tpr = tpr[sorted_indices]
+
+    if len(fpr) > max_length:
+        new_thresholds = np.linspace(0, 1, max_length, dtype=np.float32)
+        new_fpr = np.interp(new_thresholds, thresholds, fpr)
+        new_tpr = np.interp(new_thresholds, thresholds, tpr)
+        thresholds = new_thresholds
+        fpr = new_fpr
+        tpr = new_tpr
+
+    return fpr.astype(np.float32), tpr.astype(np.float32), thresholds.astype(np.float32)

orca_sdk-0.0.92/orca_sdk/_shared/metrics_test.py

@@ -0,0 +1,169 @@
+"""
+IMPORTANT:
+- This is a shared file between OrcaLib and the Orca SDK.
+- Please ensure that it does not have any dependencies on the OrcaLib code.
+- Make sure to edit this file in orcalib/shared and NOT in orca_sdk, since it will be overwritten there.
+"""
+
+from typing import Literal
+
+import numpy as np
+import pytest
+
+from .metrics import (
+    EvalPrediction,
+    calculate_pr_curve,
+    calculate_roc_curve,
+    classification_scores,
+    compute_classifier_metrics,
+    softmax,
+)
+
+
+def test_binary_metrics():
+    y_true = np.array([0, 1, 1, 0, 1])
+    y_score = np.array([0.1, 0.9, 0.8, 0.3, 0.2])
+
+    metrics = classification_scores(y_true, y_score)
+
+    assert metrics["accuracy"] == 0.8
+    assert metrics["f1_score"] == 0.8
+    assert metrics["roc_auc"] is not None
+    assert metrics["roc_auc"] > 0.8
+    assert metrics["roc_auc"] < 1.0
+    assert metrics["pr_auc"] is not None
+    assert metrics["pr_auc"] > 0.8
+    assert metrics["pr_auc"] < 1.0
+    assert metrics["log_loss"] is not None
+    assert metrics["log_loss"] > 0.0
+
+
+def test_multiclass_metrics_with_2_classes():
+    y_true = np.array([0, 1, 1, 0, 1])
+    y_score = np.array([[0.9, 0.1], [0.1, 0.9], [0.2, 0.8], [0.7, 0.3], [0.8, 0.2]])
+
+    metrics = classification_scores(y_true, y_score)
+
+    assert metrics["accuracy"] == 0.8
+    assert metrics["f1_score"] == 0.8
+    assert metrics["roc_auc"] is not None
+    assert metrics["roc_auc"] > 0.8
+    assert metrics["roc_auc"] < 1.0
+    assert metrics["pr_auc"] is not None
+    assert metrics["pr_auc"] > 0.8
+    assert metrics["pr_auc"] < 1.0
+    assert metrics["log_loss"] is not None
+    assert metrics["log_loss"] > 0.0
+
+
+@pytest.mark.parametrize(
+    "average, multiclass",
+    [("micro", "ovr"), ("macro", "ovr"), ("weighted", "ovr"), ("micro", "ovo"), ("macro", "ovo"), ("weighted", "ovo")],
+)
+def test_multiclass_metrics_with_3_classes(
+    average: Literal["micro", "macro", "weighted"], multiclass: Literal["ovr", "ovo"]
+):
+    y_true = np.array([0, 1, 1, 0, 2])
+    y_score = np.array([[0.9, 0.1, 0.0], [0.1, 0.9, 0.0], [0.2, 0.8, 0.0], [0.7, 0.3, 0.0], [0.0, 0.0, 1.0]])
+
+    metrics = classification_scores(y_true, y_score, average=average, multi_class=multiclass)
+
+    assert metrics["accuracy"] == 1.0
+    assert metrics["f1_score"] == 1.0
+    assert metrics["roc_auc"] is not None
+    assert metrics["roc_auc"] > 0.8
+    assert metrics["pr_auc"] is None
+    assert metrics["log_loss"] is not None
+    assert metrics["log_loss"] > 0.0
+
+
+def test_does_not_modify_logits_unless_necessary():
+    logits = np.array([[0.1, 0.9], [0.2, 0.8], [0.7, 0.3], [0.8, 0.2]])
+    references = np.array([0, 1, 0, 1])
+    metrics = compute_classifier_metrics(EvalPrediction(logits, references))
+    assert metrics["log_loss"] == classification_scores(references, logits)["log_loss"]
+
+
+def test_normalizes_logits_if_necessary():
+    logits = np.array([[1.2, 3.9], [1.2, 5.8], [1.2, 2.7], [1.2, 1.3]])
+    references = np.array([0, 1, 0, 1])
+    metrics = compute_classifier_metrics(EvalPrediction(logits, references))
+    assert (
+        metrics["log_loss"] == classification_scores(references, logits / logits.sum(axis=1, keepdims=True))["log_loss"]
+    )
+
+
+def test_softmaxes_logits_if_necessary():
+    logits = np.array([[-1.2, 3.9], [1.2, -5.8], [1.2, 2.7], [1.2, 1.3]])
+    references = np.array([0, 1, 0, 1])
+    metrics = compute_classifier_metrics(EvalPrediction(logits, references))
+    assert metrics["log_loss"] == classification_scores(references, softmax(logits))["log_loss"]
+
+
+def test_precision_recall_curve():
+    y_true = np.array([0, 1, 1, 0, 1])
+    y_score = np.array([0.1, 0.9, 0.8, 0.6, 0.2])
+
+    precision, recall, thresholds = calculate_pr_curve(y_true, y_score)
+    assert precision is not None
+    assert recall is not None
+    assert thresholds is not None
+
+    assert len(precision) == len(recall) == len(thresholds) == 6
+    assert precision[0] == 0.6
+    assert recall[0] == 1.0
+    assert precision[-1] == 1.0
+    assert recall[-1] == 0.0
+
+    # test that thresholds are sorted
+    assert np.all(np.diff(thresholds) >= 0)
+
+
+def test_roc_curve():
+    y_true = np.array([0, 1, 1, 0, 1])
+    y_score = np.array([0.1, 0.9, 0.8, 0.6, 0.2])
+
+    fpr, tpr, thresholds = calculate_roc_curve(y_true, y_score)
+    assert fpr is not None
+    assert tpr is not None
+    assert thresholds is not None
+
+    assert len(fpr) == len(tpr) == len(thresholds) == 6
+    assert fpr[0] == 1.0
+    assert tpr[0] == 1.0
+    assert fpr[-1] == 0.0
+    assert tpr[-1] == 0.0
+
+    # test that thresholds are sorted
+    assert np.all(np.diff(thresholds) >= 0)
+
+
+def test_precision_recall_curve_max_length():
+    y_true = np.array([0, 1, 1, 0, 1])
+    y_score = np.array([0.1, 0.9, 0.8, 0.6, 0.2])
+
+    precision, recall, thresholds = calculate_pr_curve(y_true, y_score, max_length=5)
+    assert len(precision) == len(recall) == len(thresholds) == 5
+
+    assert precision[0] == 0.6
+    assert recall[0] == 1.0
+    assert precision[-1] == 1.0
+    assert recall[-1] == 0.0
+
+    # test that thresholds are sorted
+    assert np.all(np.diff(thresholds) >= 0)
+
+
+def test_roc_curve_max_length():
+    y_true = np.array([0, 1, 1, 0, 1])
+    y_score = np.array([0.1, 0.9, 0.8, 0.6, 0.2])
+
+    fpr, tpr, thresholds = calculate_roc_curve(y_true, y_score, max_length=5)
+    assert len(fpr) == len(tpr) == len(thresholds) == 5
+    assert fpr[0] == 1.0
+    assert tpr[0] == 1.0
+    assert fpr[-1] == 0.0
+    assert tpr[-1] == 0.0
+
+    # test that thresholds are sorted
+    assert np.all(np.diff(thresholds) >= 0)