orca-sdk 0.0.101__tar.gz → 0.0.102__tar.gz

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: orca_sdk
-Version: 0.0.101
+Version: 0.0.102
 Summary: SDK for interacting with Orca Services
 License: Apache-2.0
 Author: Orca DB Inc.
@@ -10,6 +10,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: datasets (>=3.1.0,<4.0.0)
 Requires-Dist: gradio (==5.13.0)
 Requires-Dist: httpx (>=0.20.0,<0.29.0)

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/orca_sdk/_shared/metrics.py

@@ -238,7 +238,13 @@ def calculate_classification_metrics(
 
     accuracy = sklearn.metrics.accuracy_score(references, predictions)
     f1 = sklearn.metrics.f1_score(references, predictions, average=average)
-    loss = sklearn.metrics.log_loss(references, probabilities)
+    # Ensure sklearn sees the full class set corresponding to probability columns
+    # to avoid errors when y_true does not contain all classes.
+    loss = sklearn.metrics.log_loss(
+        references,
+        probabilities,
+        labels=list(range(probabilities.shape[1])),
+    )
 
     if num_classes_references == num_classes_predictions:
         # special case for binary classification: https://github.com/scikit-learn/scikit-learn/issues/20186

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/orca_sdk/_shared/metrics_test.py

@@ -138,6 +138,25 @@ def test_roc_curve():
     assert np.all(np.diff(roc_curve["thresholds"]) >= 0)
 
 
+def test_log_loss_handles_missing_classes_in_y_true():
+    # y_true contains only a subset of classes, but predictions include an extra class column
+    y_true = np.array([0, 1, 0, 1])
+    y_score = np.array(
+        [
+            [0.7, 0.2, 0.1],
+            [0.1, 0.8, 0.1],
+            [0.6, 0.3, 0.1],
+            [0.2, 0.7, 0.1],
+        ]
+    )
+
+    metrics = calculate_classification_metrics(y_true, y_score)
+    expected_loss = sklearn.metrics.log_loss(y_true, y_score, labels=[0, 1, 2])
+
+    assert np.isfinite(metrics.loss)
+    assert np.allclose(metrics.loss, expected_loss)
+
+
 def test_precision_recall_curve_max_length():
     y_true = np.array([0, 1, 1, 0, 1])
     y_score = np.array([0.1, 0.9, 0.8, 0.6, 0.2])

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/orca_sdk/_utils/prediction_result_ui.py

@@ -7,13 +7,13 @@ from typing import TYPE_CHECKING
 
 import gradio as gr
 
-from ..memoryset import LabeledMemoryLookup, ScoredMemoryLookup, LabeledMemoryset
+from ..memoryset import LabeledMemoryLookup, LabeledMemoryset, ScoredMemoryLookup
 
 if TYPE_CHECKING:
-    from ..telemetry import _Prediction
+    from ..telemetry import PredictionBase
 
 
-def inspect_prediction_result(prediction_result: _Prediction):
+def inspect_prediction_result(prediction_result: PredictionBase):
 
     def update_label(val: str, memory: LabeledMemoryLookup, progress=gr.Progress(track_tqdm=True)):
         progress(0)

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/orca_sdk/client.py

@@ -320,6 +320,7 @@ class MemorysetDuplicateAnalysisConfig(TypedDict):
 
 class MemorysetDuplicateMetrics(TypedDict):
     num_duplicates: int
+    num_potential_duplicates: int
 
 
 class MemorysetLabelAnalysisConfig(TypedDict):
@@ -332,6 +333,7 @@ class MemorysetLabelMetrics(TypedDict):
     mean_neighbor_label_confidence: float
     mean_neighbor_label_entropy: float
     mean_neighbor_predicted_label_ambiguity: float
+    num_potential_mislabels: int
 
 
 class MemorysetNeighborAnalysisConfig(TypedDict):
@@ -522,6 +524,7 @@ class RegressionPredictionRequest(TypedDict):
     save_telemetry_synchronously: NotRequired[bool]
     prompt: NotRequired[str | None]
     use_lookup_cache: NotRequired[bool]
+    consistency_level: NotRequired[Literal["Bounded", "Session", "Strong", "Eventual"] | None]
 
 
 class ScorePredictionMemoryLookup(TypedDict):
@@ -842,7 +845,7 @@ class GetDatasourceByNameOrIdDownloadParams(TypedDict):
     name_or_id: str
     file_type: NotRequired[Literal["hf_dataset", "json", "csv"]]
     """
-    File type to download: 
+    File type to download:
     * `hf_dataset`: Zipped HuggingFace dataset (default)
     * `json`: Row-oriented JSON array
     * `csv`: CSV file
@@ -1087,6 +1090,7 @@ class ClassificationPredictionRequest(TypedDict):
     save_telemetry_synchronously: NotRequired[bool]
     prompt: NotRequired[str | None]
     use_lookup_cache: NotRequired[bool]
+    consistency_level: NotRequired[Literal["Bounded", "Session", "Strong", "Eventual"] | None]
 
 
 class CloneMemorysetRequest(TypedDict):
@@ -1574,6 +1578,55 @@ class OrcaClient(Client):
     ) -> Any:
         pass
 
+    @overload
+    def GET(
+        self,
+        path: Literal["/check/alive"],
+        *,
+        params: None = None,
+        parse_as: Literal["json"] = "json",
+        headers: HeaderTypes | None = None,
+        cookies: CookieTypes | None = None,
+        auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
+        timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        extensions: RequestExtensions | None = None,
+    ) -> Any:
+        pass
+
+    @overload
+    def GET(
+        self,
+        path: Literal["/check/ready"],
+        *,
+        params: None = None,
+        parse_as: Literal["json"] = "json",
+        headers: HeaderTypes | None = None,
+        cookies: CookieTypes | None = None,
+        auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
+        timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        extensions: RequestExtensions | None = None,
+    ) -> Any:
+        pass
+
+    @overload
+    def GET(
+        self,
+        path: Literal["/auth/root"],
+        *,
+        params: None = None,
+        parse_as: Literal["json"] = "json",
+        headers: HeaderTypes | None = None,
+        cookies: CookieTypes | None = None,
+        auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
+        timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        extensions: RequestExtensions | None = None,
+    ) -> bool:
+        """Return true only when called with a valid root API key; otherwise 401 Unauthenticated."""
+        pass
+
     @overload
     def GET(
         self,
@@ -2308,6 +2361,38 @@ class OrcaClient(Client):
     ) -> Any:
         pass
 
+    @overload
+    def GET(
+        self,
+        path: Literal["/gpu/check/alive"],
+        *,
+        params: None = None,
+        parse_as: Literal["json"] = "json",
+        headers: HeaderTypes | None = None,
+        cookies: CookieTypes | None = None,
+        auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
+        timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        extensions: RequestExtensions | None = None,
+    ) -> Any:
+        pass
+
+    @overload
+    def GET(
+        self,
+        path: Literal["/gpu/check/ready"],
+        *,
+        params: None = None,
+        parse_as: Literal["json"] = "json",
+        headers: HeaderTypes | None = None,
+        cookies: CookieTypes | None = None,
+        auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
+        timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT,
+        extensions: RequestExtensions | None = None,
+    ) -> Any:
+        pass
+
     def GET(
         self,
         path: str,
@@ -3558,6 +3643,6 @@ logging.getLogger("httpcore").setLevel(logging.ERROR)
 orca_api = OrcaClient(
     event_hooks={"request": [_instrument_request], "response": [_raise_error_for_response]},
     follow_redirects=True,
-    timeout=Timeout(connect=3, read=10, write=2, pool=1),
+    timeout=Timeout(connect=3, read=20, write=10, pool=5),
 )
 """Typed client for the Orca API"""

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/orca_sdk/credentials.py

@@ -109,8 +109,8 @@ class OrcaCredentials:
         """
         Add or override default HTTP headers for all Orca API requests.
 
-        Args:
-            **kwargs: Header names with their string values
+        Params:
+            headers: Mapping of header names to their string values
 
         Notes:
             New keys are merged into the existing headers, this will overwrite headers with the

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/orca_sdk/embedding_model.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from abc import abstractmethod
+from abc import ABC, abstractmethod
 from datetime import datetime
 from typing import TYPE_CHECKING, Literal, Sequence, cast, get_args, overload
 
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
     from .memoryset import LabeledMemoryset
 
 
-class _EmbeddingModel:
+class EmbeddingModelBase(ABC):
     embedding_dim: int
     max_seq_length: int
     uses_context: bool
@@ -39,7 +39,7 @@ class _EmbeddingModel:
 
     @classmethod
     @abstractmethod
-    def all(cls) -> Sequence[_EmbeddingModel]:
+    def all(cls) -> Sequence[EmbeddingModelBase]:
         pass
 
     def _get_instruction_error_message(self) -> str:
@@ -291,7 +291,7 @@ class _ModelDescriptor:
         return self.model
 
 
-class PretrainedEmbeddingModel(_EmbeddingModel):
+class PretrainedEmbeddingModel(EmbeddingModelBase):
     """
     A pretrained embedding model
 
@@ -481,7 +481,7 @@ class PretrainedEmbeddingModel(_EmbeddingModel):
             label_column: Column name of the label
             value_column: Column name of the value
             training_method: Training method to use
-            training_args: Optional override for Hugging Face [`TrainingArguments`](transformers.TrainingArguments).
+            training_args: Optional override for Hugging Face [`TrainingArguments`][transformers.TrainingArguments].
                 If not provided, reasonable training arguments will be used for the specified training method
             if_exists: What to do if a finetuned embedding model with the same name already exists, defaults to
                 `"error"`. Other option is `"open"` to open the existing finetuned embedding model.
@@ -539,7 +539,7 @@ class PretrainedEmbeddingModel(_EmbeddingModel):
         return job if background else job.result()
 
 
-class FinetunedEmbeddingModel(_EmbeddingModel):
+class FinetunedEmbeddingModel(EmbeddingModelBase):
     """
     A finetuned embedding model in the OrcaCloud
 

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/orca_sdk/memoryset.py

@@ -47,9 +47,9 @@ from .client import (
 )
 from .datasource import Datasource
 from .embedding_model import (
+    EmbeddingModelBase,
     FinetunedEmbeddingModel,
     PretrainedEmbeddingModel,
-    _EmbeddingModel,
 )
 from .job import Job, Status
 
@@ -241,7 +241,7 @@ def _parse_memory_update(update: dict[str, Any], type: MemoryType) -> LabeledMem
             return cast(ScoredMemoryUpdate, payload)
 
 
-class _Memory(ABC):
+class MemoryBase(ABC):
     value: str
     embedding: list[float]
     source_id: str | None
@@ -303,8 +303,6 @@ class _Memory(ABC):
 
         Params:
             value: New value of the memory
-            label: New label of the memory
-            score: New score of the memory
             source_id: New source ID of the memory
             **metadata: New values for metadata properties
 
@@ -345,7 +343,7 @@ class _Memory(ABC):
         }
 
 
-class LabeledMemory(_Memory):
+class LabeledMemory(MemoryBase):
     """
     A row of the [`LabeledMemoryset`][orca_sdk.LabeledMemoryset]
 
@@ -486,7 +484,7 @@ class LabeledMemoryLookup(LabeledMemory):
         )
 
 
-class ScoredMemory(_Memory):
+class ScoredMemory(MemoryBase):
     """
     A row of the [`ScoredMemoryset`][orca_sdk.ScoredMemoryset]
 
@@ -617,11 +615,11 @@ class ScoredMemoryLookup(ScoredMemory):
         )
 
 
-MemoryT = TypeVar("MemoryT", bound=_Memory)
-MemoryLookupT = TypeVar("MemoryLookupT", bound=_Memory)
+MemoryT = TypeVar("MemoryT", bound=MemoryBase)
+MemoryLookupT = TypeVar("MemoryLookupT", bound=MemoryBase)
 
 
-class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
+class MemorysetBase(Generic[MemoryT, MemoryLookupT], ABC):
     """
     A Handle to a collection of memories with labels in the OrcaCloud
 
@@ -644,7 +642,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
     created_at: datetime
     updated_at: datetime
     insertion_status: Status
-    embedding_model: _EmbeddingModel
+    embedding_model: EmbeddingModelBase
     index_type: IndexType
     index_params: dict[str, Any]
 
@@ -669,7 +667,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
         self.memory_type = metadata["memory_type"]
 
     def __eq__(self, other) -> bool:
-        return isinstance(other, _Memoryset) and self.id == other.id
+        return isinstance(other, MemorysetBase) and self.id == other.id
 
     def __repr__(self) -> str:
         return (
@@ -794,7 +792,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
                 `"open"` and the params do not match those of the existing memoryset.
         """
         if embedding_model is None:
-            embedding_model = PretrainedEmbeddingModel.CDE_SMALL
+            embedding_model = PretrainedEmbeddingModel.GTE_BASE
 
         if label_column is None and score_column is None:
             raise ValueError("label_column or score_column must be provided")
@@ -862,7 +860,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
             name: Name for the new memoryset (must be unique)
             hf_dataset: Hugging Face dataset to create the memoryset from
             kwargs: Additional parameters for creating the memoryset. See
-                [`create`][orca_sdk.LabeledMemoryset.create] attributes for details.
+                [`create`][orca_sdk.memoryset.MemorysetBase.create] attributes for details.
 
         Returns:
             Handle to the new memoryset in the OrcaCloud
@@ -926,7 +924,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
                 argument must be provided to specify the names of the columns.
             background: Whether to run the operation in the background
             kwargs: Additional parameters for creating the memoryset. See
-                [`create`][orca_sdk.LabeledMemoryset.create] attributes for details.
+                [`create`][orca_sdk.memoryset.MemorysetBase.create] attributes for details.
 
         Returns:
             Handle to the new memoryset in the OrcaCloud
@@ -984,7 +982,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
             data: List of dictionaries to create the memoryset from
             background: Whether to run the operation in the background
             kwargs: Additional parameters for creating the memoryset. See
-                [`create`][orca_sdk.LabeledMemoryset.create] attributes for details.
+                [`create`][orca_sdk.memoryset.MemorysetBase.create] attributes for details.
 
         Returns:
             Handle to the new memoryset in the OrcaCloud
@@ -1046,7 +1044,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
             data: Dictionary of columns to create the memoryset from
             background: Whether to run the operation in the background
             kwargs: Additional parameters for creating the memoryset. See
-                [`create`][orca_sdk.LabeledMemoryset.create] attributes for details.
+                [`create`][orca_sdk.memoryset.MemorysetBase.create] attributes for details.
 
         Returns:
             Handle to the new memoryset in the OrcaCloud
@@ -1109,7 +1107,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
             dataframe: Dataframe to create the memoryset from
             background: Whether to run the operation in the background
             kwargs: Additional parameters for creating the memoryset. See
-                [`create`][orca_sdk.LabeledMemoryset.create] attributes for details.
+                [`create`][orca_sdk.memoryset.MemorysetBase.create] attributes for details.
 
         Returns:
             Handle to the new memoryset in the OrcaCloud
@@ -1165,7 +1163,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
             pyarrow_table: PyArrow table to create the memoryset from
             background: Whether to run the operation in the background
             kwargs: Additional parameters for creating the memoryset. See
-                [`create`][orca_sdk.LabeledMemoryset.create] attributes for details.
+                [`create`][orca_sdk.memoryset.MemorysetBase.create] attributes for details.
 
         Returns:
             Handle to the new memoryset in the OrcaCloud
@@ -1230,7 +1228,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
                 - dataset directory: Directory containing a saved HuggingFace [`Dataset`][datasets.Dataset]
             background: Whether to run the operation in the background
             kwargs: Additional parameters for creating the memoryset. See
-                [`create`][orca_sdk.LabeledMemoryset.create] attributes for details.
+                [`create`][orca_sdk.memoryset.MemorysetBase.create] attributes for details.
 
         Returns:
             Handle to the new memoryset in the OrcaCloud
@@ -1370,10 +1368,10 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
             embedding_model: Optional new embedding model to use for re-embedding the memory values
                 value is longer than this it will be truncated, will default to the model's max
                 sequence length if not provided
-            document_prompt_override: Optional custom document prompt to use for the cloned memoryset.
-                If not provided, will use the source memoryset's document prompt.
-            query_prompt_override: Optional custom query prompt to use for the cloned memoryset.
-                If not provided, will use the source memoryset's query prompt.
+            max_seq_length_override: Optional custom max sequence length to use for the cloned memoryset.
+                If not provided, will use the source memoryset's max sequence length.
+            prompt: Optional custom prompt to use for the cloned memoryset.
+                If not provided, will use the source memoryset's prompt.
             if_exists: What to do if a memoryset with the same name already exists, defaults to
                 `"error"`. Other option is `"open"` to open the existing memoryset.
 
@@ -1854,7 +1852,6 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
                 to be considered again.
             label_confirmation_cooldown_time: Minimum time (in seconds) since a neighbor's label was confirmed
                 to be considered for suggestions.
-            _current_time: Optional override for the current timestamp (useful for testing).
 
         Returns:
             A list of CascadingEditSuggestion objects, each containing a neighbor and the suggested new label.
@@ -2115,7 +2112,7 @@ class _Memoryset(Generic[MemoryT, MemoryLookupT], ABC):
         return job if background else job.result()
 
 
-class LabeledMemoryset(_Memoryset[LabeledMemory, LabeledMemoryLookup]):
+class LabeledMemoryset(MemorysetBase[LabeledMemory, LabeledMemoryLookup]):
     """
     A Handle to a collection of memories with labels in the OrcaCloud
 
@@ -2145,8 +2142,19 @@ class LabeledMemoryset(_Memoryset[LabeledMemory, LabeledMemoryLookup]):
     def create(cls, name: str, datasource: Datasource, *, label_column: str | None = "label", **kwargs):
         return super().create(name, datasource, label_column=label_column, score_column=None, **kwargs)
 
+    def display_label_analysis(self):
+        """
+        Display an interactive UI to review and act upon the label analysis results
+
+        Note:
+            This method is only available in Jupyter notebooks.
+        """
+        from ._utils.analysis_ui import display_suggested_memory_relabels
+
+        display_suggested_memory_relabels(self)
+
 
-class ScoredMemoryset(_Memoryset[ScoredMemory, ScoredMemoryLookup]):
+class ScoredMemoryset(MemorysetBase[ScoredMemory, ScoredMemoryLookup]):
     """
     A Handle to a collection of memories with scores in the OrcaCloud
 

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/orca_sdk/telemetry.py

@@ -152,7 +152,7 @@ class AddMemorySuggestions:
         )
 
 
-class _Prediction(ABC):
+class PredictionBase(ABC):
     prediction_id: str | None
     confidence: float
     anomaly_score: float | None
@@ -460,8 +460,19 @@ class _Prediction(ABC):
         )
         self.refresh()
 
+    def inspect(self) -> None:
+        """
+        Display an interactive UI with the details about this prediction
+
+        Note:
+            This method is only available in Jupyter notebooks.
+        """
+        from ._utils.prediction_result_ui import inspect_prediction_result
+
+        inspect_prediction_result(self)
+
 
-class ClassificationPrediction(_Prediction):
+class ClassificationPrediction(PredictionBase):
     """
     Labeled prediction result from a [`ClassificationModel`][orca_sdk.ClassificationModel]
 
@@ -614,7 +625,7 @@ class ClassificationPrediction(_Prediction):
         )
 
 
-class RegressionPrediction(_Prediction):
+class RegressionPrediction(PredictionBase):
     """
     Score-based prediction result from a [`RegressionModel`][orca_sdk.RegressionModel]
 

{orca_sdk-0.0.101 → orca_sdk-0.0.102}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "orca_sdk"
-version = "0.0.101" # Will be set by CI before building the wheel from the git tag. Do not set manually.
+version = "0.0.102" # Will be set by CI before building the wheel from the git tag. Do not set manually.
 description = "SDK for interacting with Orca Services"
 authors = ["Orca DB Inc. <dev-rel@orcadb.ai>"]
 license = "Apache-2.0"