orca-sdk 0.0.91__py3-none-any.whl → 0.0.93__py3-none-any.whl

orca_sdk/_generated_api_client/models/label_prediction_memory_lookup.py

@@ -38,6 +38,7 @@ class LabelPredictionMemoryLookup:
         memory_version (int):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        edited_at (datetime.datetime):
         metrics (MemoryMetrics):
         label (int):
         label_name (Union[None, str]):
@@ -54,6 +55,7 @@ class LabelPredictionMemoryLookup:
     memory_version: int
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    edited_at: datetime.datetime
     metrics: "MemoryMetrics"
     label: int
     label_name: Union[None, str]
@@ -81,6 +83,8 @@ class LabelPredictionMemoryLookup:
 
         updated_at = self.updated_at.isoformat()
 
+        edited_at = self.edited_at.isoformat()
+
         metrics = self.metrics.to_dict()
 
         label = self.label
@@ -106,6 +110,7 @@ class LabelPredictionMemoryLookup:
                 "memory_version": memory_version,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "edited_at": edited_at,
                 "metrics": metrics,
                 "label": label,
                 "label_name": label_name,
@@ -148,6 +153,8 @@ class LabelPredictionMemoryLookup:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        edited_at = isoparse(d.pop("edited_at"))
+
         metrics = MemoryMetrics.from_dict(d.pop("metrics"))
 
         label = d.pop("label")
@@ -174,6 +181,7 @@ class LabelPredictionMemoryLookup:
             memory_version=memory_version,
             created_at=created_at,
             updated_at=updated_at,
+            edited_at=edited_at,
             metrics=metrics,
             label=label,
             label_name=label_name,

orca_sdk/_generated_api_client/models/label_prediction_with_memories_and_feedback.py

@@ -34,10 +34,10 @@ class LabelPredictionWithMemoriesAndFeedback:
         anomaly_score (Union[None, float]):
         label (int):
         label_name (Union[None, str]):
+        logits (List[float]):
         timestamp (datetime.datetime):
         input_value (str):
         input_embedding (List[float]):
-        logits (List[float]):
         expected_label (Union[None, int]):
         expected_label_name (Union[None, str]):
         memories (List['LabelPredictionMemoryLookup']):
@@ -56,10 +56,10 @@ class LabelPredictionWithMemoriesAndFeedback:
     anomaly_score: Union[None, float]
     label: int
     label_name: Union[None, str]
+    logits: List[float]
     timestamp: datetime.datetime
     input_value: str
     input_embedding: List[float]
-    logits: List[float]
     expected_label: Union[None, int]
     expected_label_name: Union[None, str]
     memories: List["LabelPredictionMemoryLookup"]
@@ -86,6 +86,8 @@ class LabelPredictionWithMemoriesAndFeedback:
         label_name: Union[None, str]
         label_name = self.label_name
 
+        logits = self.logits
+
         timestamp = self.timestamp.isoformat()
 
         input_value: str
@@ -93,8 +95,6 @@ class LabelPredictionWithMemoriesAndFeedback:
 
         input_embedding = self.input_embedding
 
-        logits = self.logits
-
         expected_label: Union[None, int]
         expected_label = self.expected_label
 
@@ -136,10 +136,10 @@ class LabelPredictionWithMemoriesAndFeedback:
                 "anomaly_score": anomaly_score,
                 "label": label,
                 "label_name": label_name,
+                "logits": logits,
                 "timestamp": timestamp,
                 "input_value": input_value,
                 "input_embedding": input_embedding,
-                "logits": logits,
                 "expected_label": expected_label,
                 "expected_label_name": expected_label_name,
                 "memories": memories,
@@ -182,6 +182,8 @@ class LabelPredictionWithMemoriesAndFeedback:
 
         label_name = _parse_label_name(d.pop("label_name"))
 
+        logits = cast(List[float], d.pop("logits"))
+
         timestamp = isoparse(d.pop("timestamp"))
 
         def _parse_input_value(data: object) -> str:
@@ -191,8 +193,6 @@ class LabelPredictionWithMemoriesAndFeedback:
 
         input_embedding = cast(List[float], d.pop("input_embedding"))
 
-        logits = cast(List[float], d.pop("logits"))
-
         def _parse_expected_label(data: object) -> Union[None, int]:
             if data is None:
                 return data
@@ -251,10 +251,10 @@ class LabelPredictionWithMemoriesAndFeedback:
             anomaly_score=anomaly_score,
             label=label,
             label_name=label_name,
+            logits=logits,
             timestamp=timestamp,
             input_value=input_value,
             input_embedding=input_embedding,
-            logits=logits,
             expected_label=expected_label,
             expected_label_name=expected_label_name,
             memories=memories,

orca_sdk/_generated_api_client/models/labeled_memory.py

@@ -38,6 +38,7 @@ class LabeledMemory:
         memory_version (int):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        edited_at (datetime.datetime):
         metrics (LabeledMemoryMetrics): Metrics computed for a labeled memory.
         label (int):
         label_name (Union[None, str]):
@@ -51,6 +52,7 @@ class LabeledMemory:
     memory_version: int
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    edited_at: datetime.datetime
     metrics: "LabeledMemoryMetrics"
     label: int
     label_name: Union[None, str]
@@ -75,6 +77,8 @@ class LabeledMemory:
 
         updated_at = self.updated_at.isoformat()
 
+        edited_at = self.edited_at.isoformat()
+
         metrics = self.metrics.to_dict()
 
         label = self.label
@@ -94,6 +98,7 @@ class LabeledMemory:
                 "memory_version": memory_version,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "edited_at": edited_at,
                 "metrics": metrics,
                 "label": label,
                 "label_name": label_name,
@@ -133,6 +138,8 @@ class LabeledMemory:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        edited_at = isoparse(d.pop("edited_at"))
+
         metrics = LabeledMemoryMetrics.from_dict(d.pop("metrics"))
 
         label = d.pop("label")
@@ -153,6 +160,7 @@ class LabeledMemory:
             memory_version=memory_version,
             created_at=created_at,
             updated_at=updated_at,
+            edited_at=edited_at,
             metrics=metrics,
             label=label,
             label_name=label_name,

orca_sdk/_generated_api_client/models/labeled_memory_lookup.py

@@ -38,6 +38,7 @@ class LabeledMemoryLookup:
         memory_version (int):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        edited_at (datetime.datetime):
         metrics (MemoryMetrics):
         label (int):
         label_name (Union[None, str]):
@@ -52,6 +53,7 @@ class LabeledMemoryLookup:
     memory_version: int
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    edited_at: datetime.datetime
     metrics: "MemoryMetrics"
     label: int
     label_name: Union[None, str]
@@ -77,6 +79,8 @@ class LabeledMemoryLookup:
 
         updated_at = self.updated_at.isoformat()
 
+        edited_at = self.edited_at.isoformat()
+
         metrics = self.metrics.to_dict()
 
         label = self.label
@@ -98,6 +102,7 @@ class LabeledMemoryLookup:
                 "memory_version": memory_version,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "edited_at": edited_at,
                 "metrics": metrics,
                 "label": label,
                 "label_name": label_name,
@@ -138,6 +143,8 @@ class LabeledMemoryLookup:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        edited_at = isoparse(d.pop("edited_at"))
+
         metrics = MemoryMetrics.from_dict(d.pop("metrics"))
 
         label = d.pop("label")
@@ -160,6 +167,7 @@ class LabeledMemoryLookup:
             memory_version=memory_version,
             created_at=created_at,
             updated_at=updated_at,
+            edited_at=edited_at,
             metrics=metrics,
             label=label,
             label_name=label_name,

orca_sdk/_generated_api_client/models/labeled_memory_with_feedback_metrics.py

@@ -40,6 +40,7 @@ class LabeledMemoryWithFeedbackMetrics:
         memory_version (int):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        edited_at (datetime.datetime):
         metrics (LabeledMemoryMetrics): Metrics computed for a labeled memory.
         label (int):
         label_name (Union[None, str]):
@@ -55,6 +56,7 @@ class LabeledMemoryWithFeedbackMetrics:
     memory_version: int
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    edited_at: datetime.datetime
     metrics: "LabeledMemoryMetrics"
     label: int
     label_name: Union[None, str]
@@ -81,6 +83,8 @@ class LabeledMemoryWithFeedbackMetrics:
 
         updated_at = self.updated_at.isoformat()
 
+        edited_at = self.edited_at.isoformat()
+
         metrics = self.metrics.to_dict()
 
         label = self.label
@@ -104,6 +108,7 @@ class LabeledMemoryWithFeedbackMetrics:
                 "memory_version": memory_version,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "edited_at": edited_at,
                 "metrics": metrics,
                 "label": label,
                 "label_name": label_name,
@@ -148,6 +153,8 @@ class LabeledMemoryWithFeedbackMetrics:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        edited_at = isoparse(d.pop("edited_at"))
+
         metrics = LabeledMemoryMetrics.from_dict(d.pop("metrics"))
 
         label = d.pop("label")
@@ -172,6 +179,7 @@ class LabeledMemoryWithFeedbackMetrics:
             memory_version=memory_version,
             created_at=created_at,
             updated_at=updated_at,
+            edited_at=edited_at,
             metrics=metrics,
             label=label,
             label_name=label_name,

orca_sdk/_generated_api_client/models/labeled_memoryset_metadata.py

@@ -43,6 +43,7 @@ class LabeledMemorysetMetadata:
         label_names (List[str]):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        memories_updated_at (datetime.datetime):
         insertion_task_id (str):
         insertion_status (TaskStatus): Status of task in the task queue
         metrics (MemorysetMetrics):
@@ -59,6 +60,7 @@ class LabeledMemorysetMetadata:
     label_names: List[str]
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    memories_updated_at: datetime.datetime
     insertion_task_id: str
     insertion_status: TaskStatus
     metrics: "MemorysetMetrics"
@@ -97,6 +99,8 @@ class LabeledMemorysetMetadata:
 
         updated_at = self.updated_at.isoformat()
 
+        memories_updated_at = self.memories_updated_at.isoformat()
+
         insertion_task_id = self.insertion_task_id
 
         insertion_status = (
@@ -120,6 +124,7 @@ class LabeledMemorysetMetadata:
                 "label_names": label_names,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "memories_updated_at": memories_updated_at,
                 "insertion_task_id": insertion_task_id,
                 "insertion_status": insertion_status,
                 "metrics": metrics,
@@ -180,6 +185,8 @@ class LabeledMemorysetMetadata:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        memories_updated_at = isoparse(d.pop("memories_updated_at"))
+
         insertion_task_id = d.pop("insertion_task_id")
 
         insertion_status = TaskStatus(d.pop("insertion_status"))
@@ -198,6 +205,7 @@ class LabeledMemorysetMetadata:
             label_names=label_names,
             created_at=created_at,
             updated_at=updated_at,
+            memories_updated_at=memories_updated_at,
             insertion_task_id=insertion_task_id,
             insertion_status=insertion_status,
             metrics=metrics,

orca_sdk/_generated_api_client/models/prediction_request.py

@@ -28,14 +28,16 @@ class PredictionRequest:
         expected_labels (Union[List[int], None, Unset]):
         tags (Union[Unset, List[str]]):
         memoryset_override_id (Union[None, Unset, str]):
-        disable_telemetry (Union[Unset, bool]):  Default: False.
+        save_telemetry (Union[Unset, bool]):  Default: True.
+        save_telemetry_synchronously (Union[Unset, bool]):  Default: False.
     """
 
     input_values: List[str]
     expected_labels: Union[List[int], None, Unset] = UNSET
     tags: Union[Unset, List[str]] = UNSET
     memoryset_override_id: Union[None, Unset, str] = UNSET
-    disable_telemetry: Union[Unset, bool] = False
+    save_telemetry: Union[Unset, bool] = True
+    save_telemetry_synchronously: Union[Unset, bool] = False
     additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
 
     def to_dict(self) -> dict[str, Any]:
@@ -62,7 +64,9 @@ class PredictionRequest:
         else:
             memoryset_override_id = self.memoryset_override_id
 
-        disable_telemetry = self.disable_telemetry
+        save_telemetry = self.save_telemetry
+
+        save_telemetry_synchronously = self.save_telemetry_synchronously
 
         field_dict: dict[str, Any] = {}
         field_dict.update(self.additional_properties)
@@ -77,8 +81,10 @@ class PredictionRequest:
             field_dict["tags"] = tags
         if memoryset_override_id is not UNSET:
             field_dict["memoryset_override_id"] = memoryset_override_id
-        if disable_telemetry is not UNSET:
-            field_dict["disable_telemetry"] = disable_telemetry
+        if save_telemetry is not UNSET:
+            field_dict["save_telemetry"] = save_telemetry
+        if save_telemetry_synchronously is not UNSET:
+            field_dict["save_telemetry_synchronously"] = save_telemetry_synchronously
 
         return field_dict
 
@@ -156,14 +162,17 @@ class PredictionRequest:
 
         memoryset_override_id = _parse_memoryset_override_id(d.pop("memoryset_override_id", UNSET))
 
-        disable_telemetry = d.pop("disable_telemetry", UNSET)
+        save_telemetry = d.pop("save_telemetry", UNSET)
+
+        save_telemetry_synchronously = d.pop("save_telemetry_synchronously", UNSET)
 
         prediction_request = cls(
             input_values=input_values,
             expected_labels=expected_labels,
             tags=tags,
             memoryset_override_id=memoryset_override_id,
-            disable_telemetry=disable_telemetry,
+            save_telemetry=save_telemetry,
+            save_telemetry_synchronously=save_telemetry_synchronously,
         )
 
         prediction_request.additional_properties = d

orca_sdk/_shared/__init__.py

@@ -0,0 +1 @@
+from .metrics import calculate_pr_curve, calculate_roc_curve, compute_classifier_metrics

orca_sdk/_shared/metrics.py

@@ -0,0 +1,195 @@
+"""
+This module contains metrics for usage with the Hugging Face Trainer.
+
+IMPORTANT:
+- This is a shared file between OrcaLib and the Orca SDK.
+- Please ensure that it does not have any dependencies on the OrcaLib code.
+- Make sure to edit this file in orcalib/shared and NOT in orca_sdk, since it will be overwritten there.
+
+"""
+
+from typing import Literal, Tuple, TypedDict
+
+import numpy as np
+from numpy.typing import NDArray
+from scipy.special import softmax
+from sklearn.metrics import accuracy_score, auc, f1_score, log_loss
+from sklearn.metrics import precision_recall_curve as sklearn_precision_recall_curve
+from sklearn.metrics import roc_auc_score
+from sklearn.metrics import roc_curve as sklearn_roc_curve
+from transformers.trainer_utils import EvalPrediction
+
+
+class ClassificationMetrics(TypedDict):
+    accuracy: float
+    f1_score: float
+    roc_auc: float | None  # receiver operating characteristic area under the curve (if all classes are present)
+    pr_auc: float | None  # precision-recall area under the curve (only for binary classification)
+    log_loss: float  # cross-entropy loss for probabilities
+
+
+def compute_classifier_metrics(eval_pred: EvalPrediction) -> ClassificationMetrics:
+    """
+    Compute standard metrics for classifier with Hugging Face Trainer.
+
+    Args:
+        eval_pred: The predictions containing logits and expected labels as given by the Trainer.
+
+    Returns:
+        A dictionary containing the accuracy, f1 score, and ROC AUC score.
+    """
+    logits, references = eval_pred
+    if isinstance(logits, tuple):
+        logits = logits[0]
+    if not isinstance(logits, np.ndarray):
+        raise ValueError("Logits must be a numpy array")
+    if not isinstance(references, np.ndarray):
+        raise ValueError(
+            "Multiple label columns found, use the `label_names` training argument to specify which one to use"
+        )
+
+    if not (logits > 0).all():
+        # convert logits to probabilities with softmax if necessary
+        probabilities = softmax(logits)
+    elif not np.allclose(logits.sum(-1, keepdims=True), 1.0):
+        # convert logits to probabilities through normalization if necessary
+        probabilities = logits / logits.sum(-1, keepdims=True)
+    else:
+        probabilities = logits
+
+    return classification_scores(references, probabilities)
+
+
+def classification_scores(
+    references: NDArray[np.int64],
+    probabilities: NDArray[np.float32],
+    average: Literal["micro", "macro", "weighted", "binary"] | None = None,
+    multi_class: Literal["ovr", "ovo"] = "ovr",
+) -> ClassificationMetrics:
+    if probabilities.ndim == 1:
+        # convert 1D probabilities (binary) to 2D logits
+        probabilities = np.column_stack([1 - probabilities, probabilities])
+    elif probabilities.ndim == 2:
+        if probabilities.shape[1] < 2:
+            raise ValueError("Use a different metric function for regression tasks")
+    else:
+        raise ValueError("Probabilities must be 1 or 2 dimensional")
+
+    predictions = np.argmax(probabilities, axis=-1)
+
+    num_classes_references = len(set(references))
+    num_classes_predictions = len(set(predictions))
+
+    if average is None:
+        average = "binary" if num_classes_references == 2 else "weighted"
+
+    accuracy = accuracy_score(references, predictions)
+    f1 = f1_score(references, predictions, average=average)
+    loss = log_loss(references, probabilities)
+
+    if num_classes_references == num_classes_predictions:
+        # special case for binary classification: https://github.com/scikit-learn/scikit-learn/issues/20186
+        if num_classes_references == 2:
+            roc_auc = roc_auc_score(references, probabilities[:, 1])
+            precisions, recalls, _ = calculate_pr_curve(references, probabilities[:, 1])
+            pr_auc = auc(recalls, precisions)
+        else:
+            roc_auc = roc_auc_score(references, probabilities, multi_class=multi_class)
+            pr_auc = None
+    else:
+        roc_auc = None
+        pr_auc = None
+
+    return {
+        "accuracy": float(accuracy),
+        "f1_score": float(f1),
+        "roc_auc": float(roc_auc) if roc_auc is not None else None,
+        "pr_auc": float(pr_auc) if pr_auc is not None else None,
+        "log_loss": float(loss),
+    }
+
+
+def calculate_pr_curve(
+    references: NDArray[np.int64],
+    probabilities: NDArray[np.float32],
+    max_length: int = 100,
+) -> Tuple[NDArray[np.float32], NDArray[np.float32], NDArray[np.float32]]:
+    if probabilities.ndim == 1:
+        probabilities_slice = probabilities
+    elif probabilities.ndim == 2:
+        probabilities_slice = probabilities[:, 1]
+    else:
+        raise ValueError("Probabilities must be 1 or 2 dimensional")
+
+    if len(probabilities_slice) != len(references):
+        raise ValueError("Probabilities and references must have the same length")
+
+    precisions, recalls, thresholds = sklearn_precision_recall_curve(references, probabilities_slice)
+
+    # Convert all arrays to float32 immediately after getting them
+    precisions = precisions.astype(np.float32)
+    recalls = recalls.astype(np.float32)
+    thresholds = thresholds.astype(np.float32)
+
+    # Concatenate with 0 to include the lowest threshold
+    thresholds = np.concatenate(([0], thresholds))
+
+    # Sort by threshold
+    sorted_indices = np.argsort(thresholds)
+    thresholds = thresholds[sorted_indices]
+    precisions = precisions[sorted_indices]
+    recalls = recalls[sorted_indices]
+
+    if len(precisions) > max_length:
+        new_thresholds = np.linspace(0, 1, max_length, dtype=np.float32)
+        new_precisions = np.interp(new_thresholds, thresholds, precisions)
+        new_recalls = np.interp(new_thresholds, thresholds, recalls)
+        thresholds = new_thresholds
+        precisions = new_precisions
+        recalls = new_recalls
+
+    return precisions.astype(np.float32), recalls.astype(np.float32), thresholds.astype(np.float32)
+
+
+def calculate_roc_curve(
+    references: NDArray[np.int64],
+    probabilities: NDArray[np.float32],
+    max_length: int = 100,
+) -> Tuple[NDArray[np.float32], NDArray[np.float32], NDArray[np.float32]]:
+    if probabilities.ndim == 1:
+        probabilities_slice = probabilities
+    elif probabilities.ndim == 2:
+        probabilities_slice = probabilities[:, 1]
+    else:
+        raise ValueError("Probabilities must be 1 or 2 dimensional")
+
+    if len(probabilities_slice) != len(references):
+        raise ValueError("Probabilities and references must have the same length")
+
+    # Convert probabilities to float32 before calling sklearn_roc_curve
+    probabilities_slice = probabilities_slice.astype(np.float32)
+    fpr, tpr, thresholds = sklearn_roc_curve(references, probabilities_slice)
+
+    # Convert all arrays to float32 immediately after getting them
+    fpr = fpr.astype(np.float32)
+    tpr = tpr.astype(np.float32)
+    thresholds = thresholds.astype(np.float32)
+
+    # We set the first threshold to 1.0 instead of inf for reasonable values in interpolation
+    thresholds[0] = 1.0
+
+    # Sort by threshold
+    sorted_indices = np.argsort(thresholds)
+    thresholds = thresholds[sorted_indices]
+    fpr = fpr[sorted_indices]
+    tpr = tpr[sorted_indices]
+
+    if len(fpr) > max_length:
+        new_thresholds = np.linspace(0, 1, max_length, dtype=np.float32)
+        new_fpr = np.interp(new_thresholds, thresholds, fpr)
+        new_tpr = np.interp(new_thresholds, thresholds, tpr)
+        thresholds = new_thresholds
+        fpr = new_fpr
+        tpr = new_tpr
+
+    return fpr.astype(np.float32), tpr.astype(np.float32), thresholds.astype(np.float32)