orca-sdk 0.0.93__py3-none-any.whl → 0.0.95__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- orca_sdk/__init__.py +13 -4
- orca_sdk/_generated_api_client/api/__init__.py +84 -34
- orca_sdk/_generated_api_client/api/classification_model/create_classification_model_classification_model_post.py +170 -0
- orca_sdk/_generated_api_client/api/classification_model/{get_model_classification_model_name_or_id_get.py → delete_classification_model_classification_model_name_or_id_delete.py} +20 -20
- orca_sdk/_generated_api_client/api/classification_model/{delete_evaluation_classification_model_model_name_or_id_evaluation_task_id_delete.py → delete_classification_model_evaluation_classification_model_model_name_or_id_evaluation_task_id_delete.py} +4 -4
- orca_sdk/_generated_api_client/api/classification_model/{create_evaluation_classification_model_model_name_or_id_evaluation_post.py → evaluate_classification_model_classification_model_model_name_or_id_evaluation_post.py} +14 -14
- orca_sdk/_generated_api_client/api/classification_model/get_classification_model_classification_model_name_or_id_get.py +156 -0
- orca_sdk/_generated_api_client/api/classification_model/{get_evaluation_classification_model_model_name_or_id_evaluation_task_id_get.py → get_classification_model_evaluation_classification_model_model_name_or_id_evaluation_task_id_get.py} +16 -16
- orca_sdk/_generated_api_client/api/classification_model/{list_evaluations_classification_model_model_name_or_id_evaluation_get.py → list_classification_model_evaluations_classification_model_model_name_or_id_evaluation_get.py} +16 -16
- orca_sdk/_generated_api_client/api/classification_model/list_classification_models_classification_model_get.py +127 -0
- orca_sdk/_generated_api_client/api/classification_model/{predict_gpu_classification_model_name_or_id_prediction_post.py → predict_label_gpu_classification_model_name_or_id_prediction_post.py} +14 -14
- orca_sdk/_generated_api_client/api/classification_model/update_classification_model_classification_model_name_or_id_patch.py +183 -0
- orca_sdk/_generated_api_client/api/datasource/download_datasource_datasource_name_or_id_download_get.py +172 -0
- orca_sdk/_generated_api_client/api/memoryset/clone_memoryset_memoryset_name_or_id_clone_post.py +22 -22
- orca_sdk/_generated_api_client/api/memoryset/create_memoryset_memoryset_post.py +22 -22
- orca_sdk/_generated_api_client/api/memoryset/get_memories_memoryset_name_or_id_memories_get_post.py +38 -16
- orca_sdk/_generated_api_client/api/memoryset/get_memory_memoryset_name_or_id_memory_memory_id_get.py +29 -12
- orca_sdk/_generated_api_client/api/memoryset/get_memoryset_memoryset_name_or_id_get.py +12 -12
- orca_sdk/_generated_api_client/api/memoryset/insert_memories_gpu_memoryset_name_or_id_memory_post.py +17 -14
- orca_sdk/_generated_api_client/api/memoryset/list_memorysets_memoryset_get.py +72 -19
- orca_sdk/_generated_api_client/api/memoryset/memoryset_lookup_gpu_memoryset_name_or_id_lookup_post.py +31 -12
- orca_sdk/_generated_api_client/api/memoryset/potential_duplicate_groups_memoryset_name_or_id_potential_duplicate_groups_get.py +49 -20
- orca_sdk/_generated_api_client/api/memoryset/query_memoryset_memoryset_name_or_id_memories_post.py +38 -16
- orca_sdk/_generated_api_client/api/memoryset/update_memories_gpu_memoryset_name_or_id_memories_patch.py +54 -29
- orca_sdk/_generated_api_client/api/memoryset/update_memory_gpu_memoryset_name_or_id_memory_patch.py +44 -26
- orca_sdk/_generated_api_client/api/memoryset/update_memoryset_memoryset_name_or_id_patch.py +22 -22
- orca_sdk/_generated_api_client/api/predictive_model/__init__.py +0 -0
- orca_sdk/_generated_api_client/api/predictive_model/list_predictive_models_predictive_model_get.py +150 -0
- orca_sdk/_generated_api_client/api/regression_model/__init__.py +0 -0
- orca_sdk/_generated_api_client/api/{classification_model/create_model_classification_model_post.py → regression_model/create_regression_model_regression_model_post.py} +27 -27
- orca_sdk/_generated_api_client/api/regression_model/delete_regression_model_evaluation_regression_model_model_name_or_id_evaluation_task_id_delete.py +168 -0
- orca_sdk/_generated_api_client/api/{classification_model/delete_model_classification_model_name_or_id_delete.py → regression_model/delete_regression_model_regression_model_name_or_id_delete.py} +5 -5
- orca_sdk/_generated_api_client/api/regression_model/evaluate_regression_model_regression_model_model_name_or_id_evaluation_post.py +183 -0
- orca_sdk/_generated_api_client/api/regression_model/get_regression_model_evaluation_regression_model_model_name_or_id_evaluation_task_id_get.py +170 -0
- orca_sdk/_generated_api_client/api/regression_model/get_regression_model_regression_model_name_or_id_get.py +156 -0
- orca_sdk/_generated_api_client/api/regression_model/list_regression_model_evaluations_regression_model_model_name_or_id_evaluation_get.py +161 -0
- orca_sdk/_generated_api_client/api/{classification_model/list_models_classification_model_get.py → regression_model/list_regression_models_regression_model_get.py} +17 -17
- orca_sdk/_generated_api_client/api/regression_model/predict_score_gpu_regression_model_name_or_id_prediction_post.py +190 -0
- orca_sdk/_generated_api_client/api/{classification_model/update_model_classification_model_name_or_id_patch.py → regression_model/update_regression_model_regression_model_name_or_id_patch.py} +27 -27
- orca_sdk/_generated_api_client/api/task/get_task_task_task_id_get.py +156 -0
- orca_sdk/_generated_api_client/api/task/list_tasks_task_get.py +60 -10
- orca_sdk/_generated_api_client/api/telemetry/count_predictions_telemetry_prediction_count_post.py +10 -10
- orca_sdk/_generated_api_client/api/telemetry/get_prediction_telemetry_prediction_prediction_id_get.py +35 -12
- orca_sdk/_generated_api_client/api/telemetry/list_memories_with_feedback_telemetry_memories_post.py +20 -12
- orca_sdk/_generated_api_client/api/telemetry/list_predictions_telemetry_prediction_post.py +35 -12
- orca_sdk/_generated_api_client/models/__init__.py +90 -24
- orca_sdk/_generated_api_client/models/base_score_prediction_result.py +108 -0
- orca_sdk/_generated_api_client/models/{evaluation_request.py → classification_evaluation_request.py} +13 -45
- orca_sdk/_generated_api_client/models/{classification_evaluation_result.py → classification_metrics.py} +106 -56
- orca_sdk/_generated_api_client/models/{rac_model_metadata.py → classification_model_metadata.py} +51 -43
- orca_sdk/_generated_api_client/models/{prediction_request.py → classification_prediction_request.py} +31 -6
- orca_sdk/_generated_api_client/models/{clone_labeled_memoryset_request.py → clone_memoryset_request.py} +5 -5
- orca_sdk/_generated_api_client/models/column_info.py +31 -0
- orca_sdk/_generated_api_client/models/count_predictions_request.py +195 -0
- orca_sdk/_generated_api_client/models/{create_rac_model_request.py → create_classification_model_request.py} +25 -57
- orca_sdk/_generated_api_client/models/{create_labeled_memoryset_request.py → create_memoryset_request.py} +73 -56
- orca_sdk/_generated_api_client/models/create_memoryset_request_index_params.py +66 -0
- orca_sdk/_generated_api_client/models/create_memoryset_request_index_type.py +13 -0
- orca_sdk/_generated_api_client/models/create_regression_model_request.py +137 -0
- orca_sdk/_generated_api_client/models/embedding_evaluation_payload.py +187 -0
- orca_sdk/_generated_api_client/models/embedding_evaluation_response.py +10 -0
- orca_sdk/_generated_api_client/models/evaluation_response.py +22 -9
- orca_sdk/_generated_api_client/models/evaluation_response_classification_metrics.py +140 -0
- orca_sdk/_generated_api_client/models/evaluation_response_regression_metrics.py +140 -0
- orca_sdk/_generated_api_client/models/http_validation_error.py +86 -0
- orca_sdk/_generated_api_client/models/list_predictions_request.py +62 -0
- orca_sdk/_generated_api_client/models/memory_type.py +9 -0
- orca_sdk/_generated_api_client/models/memoryset_analysis_configs.py +0 -20
- orca_sdk/_generated_api_client/models/{labeled_memoryset_metadata.py → memoryset_metadata.py} +73 -13
- orca_sdk/_generated_api_client/models/memoryset_metadata_index_params.py +55 -0
- orca_sdk/_generated_api_client/models/memoryset_metadata_index_type.py +13 -0
- orca_sdk/_generated_api_client/models/{labeled_memoryset_update.py → memoryset_update.py} +19 -31
- orca_sdk/_generated_api_client/models/not_found_error_response_resource_type_0.py +1 -0
- orca_sdk/_generated_api_client/models/{paginated_labeled_memory_with_feedback_metrics.py → paginated_union_labeled_memory_with_feedback_metrics_scored_memory_with_feedback_metrics.py} +37 -10
- orca_sdk/_generated_api_client/models/{precision_recall_curve.py → pr_curve.py} +5 -13
- orca_sdk/_generated_api_client/models/{rac_model_update.py → predictive_model_update.py} +14 -5
- orca_sdk/_generated_api_client/models/pretrained_embedding_model_metadata.py +11 -1
- orca_sdk/_generated_api_client/models/pretrained_embedding_model_name.py +5 -0
- orca_sdk/_generated_api_client/models/rar_head_type.py +8 -0
- orca_sdk/_generated_api_client/models/regression_evaluation_request.py +148 -0
- orca_sdk/_generated_api_client/models/regression_metrics.py +172 -0
- orca_sdk/_generated_api_client/models/regression_model_metadata.py +177 -0
- orca_sdk/_generated_api_client/models/regression_prediction_request.py +195 -0
- orca_sdk/_generated_api_client/models/roc_curve.py +0 -8
- orca_sdk/_generated_api_client/models/score_prediction_memory_lookup.py +196 -0
- orca_sdk/_generated_api_client/models/score_prediction_memory_lookup_metadata.py +68 -0
- orca_sdk/_generated_api_client/models/score_prediction_with_memories_and_feedback.py +252 -0
- orca_sdk/_generated_api_client/models/scored_memory.py +172 -0
- orca_sdk/_generated_api_client/models/scored_memory_insert.py +128 -0
- orca_sdk/_generated_api_client/models/scored_memory_insert_metadata.py +68 -0
- orca_sdk/_generated_api_client/models/scored_memory_lookup.py +180 -0
- orca_sdk/_generated_api_client/models/scored_memory_lookup_metadata.py +68 -0
- orca_sdk/_generated_api_client/models/scored_memory_metadata.py +68 -0
- orca_sdk/_generated_api_client/models/scored_memory_update.py +171 -0
- orca_sdk/_generated_api_client/models/scored_memory_update_metadata_type_0.py +68 -0
- orca_sdk/_generated_api_client/models/scored_memory_with_feedback_metrics.py +193 -0
- orca_sdk/_generated_api_client/models/scored_memory_with_feedback_metrics_feedback_metrics.py +68 -0
- orca_sdk/_generated_api_client/models/scored_memory_with_feedback_metrics_metadata.py +68 -0
- orca_sdk/_generated_api_client/models/update_prediction_request.py +20 -0
- orca_sdk/_generated_api_client/models/validation_error.py +99 -0
- orca_sdk/_shared/__init__.py +9 -1
- orca_sdk/_shared/metrics.py +257 -87
- orca_sdk/_shared/metrics_test.py +136 -77
- orca_sdk/_utils/data_parsing.py +0 -3
- orca_sdk/_utils/data_parsing_test.py +0 -3
- orca_sdk/_utils/prediction_result_ui.py +55 -23
- orca_sdk/classification_model.py +184 -174
- orca_sdk/classification_model_test.py +178 -142
- orca_sdk/conftest.py +77 -26
- orca_sdk/datasource.py +34 -0
- orca_sdk/datasource_test.py +9 -1
- orca_sdk/embedding_model.py +136 -14
- orca_sdk/embedding_model_test.py +10 -6
- orca_sdk/job.py +329 -0
- orca_sdk/job_test.py +48 -0
- orca_sdk/memoryset.py +882 -161
- orca_sdk/memoryset_test.py +58 -23
- orca_sdk/regression_model.py +647 -0
- orca_sdk/regression_model_test.py +338 -0
- orca_sdk/telemetry.py +225 -106
- orca_sdk/telemetry_test.py +34 -30
- {orca_sdk-0.0.93.dist-info → orca_sdk-0.0.95.dist-info}/METADATA +2 -4
- {orca_sdk-0.0.93.dist-info → orca_sdk-0.0.95.dist-info}/RECORD +124 -74
- orca_sdk/_utils/task.py +0 -73
- {orca_sdk-0.0.93.dist-info → orca_sdk-0.0.95.dist-info}/WHEEL +0 -0
orca_sdk/telemetry.py
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
from abc import ABC
|
|
4
5
|
from datetime import datetime
|
|
5
|
-
from typing import TYPE_CHECKING, Any, Generator, Iterable, overload
|
|
6
|
+
from typing import TYPE_CHECKING, Any, Generator, Iterable, Self, overload
|
|
6
7
|
from uuid import UUID
|
|
7
8
|
|
|
8
9
|
from orca_sdk._utils.common import UNSET
|
|
@@ -23,14 +24,21 @@ from ._generated_api_client.models import (
|
|
|
23
24
|
ListPredictionsRequest,
|
|
24
25
|
PredictionFeedbackCategory,
|
|
25
26
|
PredictionFeedbackRequest,
|
|
27
|
+
ScorePredictionWithMemoriesAndFeedback,
|
|
26
28
|
UpdatePredictionRequest,
|
|
27
29
|
)
|
|
28
30
|
from ._generated_api_client.types import UNSET as CLIENT_UNSET
|
|
29
31
|
from ._utils.prediction_result_ui import inspect_prediction_result
|
|
30
|
-
from .memoryset import
|
|
32
|
+
from .memoryset import (
|
|
33
|
+
LabeledMemoryLookup,
|
|
34
|
+
LabeledMemoryset,
|
|
35
|
+
ScoredMemoryLookup,
|
|
36
|
+
ScoredMemoryset,
|
|
37
|
+
)
|
|
31
38
|
|
|
32
39
|
if TYPE_CHECKING:
|
|
33
40
|
from .classification_model import ClassificationModel
|
|
41
|
+
from .regression_model import RegressionModel
|
|
34
42
|
|
|
35
43
|
|
|
36
44
|
def _parse_feedback(feedback: dict[str, Any]) -> PredictionFeedbackRequest:
|
|
@@ -107,88 +115,61 @@ class FeedbackCategory:
|
|
|
107
115
|
return "FeedbackCategory({" + f"name: {self.name}, " + f"value_type: {self.value_type}" + "})"
|
|
108
116
|
|
|
109
117
|
|
|
110
|
-
class
|
|
111
|
-
"""
|
|
112
|
-
A prediction made by a model
|
|
113
|
-
|
|
114
|
-
Attributes:
|
|
115
|
-
prediction_id: Unique identifier for the prediction
|
|
116
|
-
label: Predicted label for the input value
|
|
117
|
-
label_name: Name of the predicted label
|
|
118
|
-
confidence: Confidence of the prediction
|
|
119
|
-
anomaly_score: The score for how anomalous the input is relative to the memories
|
|
120
|
-
memory_lookups: List of memories used to ground the prediction
|
|
121
|
-
input_value: Input value that this prediction was for
|
|
122
|
-
model: Model that was used to make the prediction
|
|
123
|
-
memoryset: Memoryset that was used to lookup memories to ground the prediction
|
|
124
|
-
expected_label: Optional expected label that was set for the prediction
|
|
125
|
-
expected_label_name: Name of the expected label
|
|
126
|
-
tags: tags that were set for the prediction
|
|
127
|
-
feedback: Feedback recorded, mapping from category name to value
|
|
128
|
-
explanation: Explanation why the model made the prediction generated by a reasoning agent
|
|
129
|
-
"""
|
|
130
|
-
|
|
118
|
+
class _Prediction(ABC):
|
|
131
119
|
prediction_id: str | None
|
|
132
|
-
label: int
|
|
133
|
-
label_name: str | None
|
|
134
120
|
confidence: float
|
|
135
121
|
anomaly_score: float | None
|
|
136
|
-
memoryset: LabeledMemoryset
|
|
137
|
-
model: ClassificationModel
|
|
138
|
-
logits: list[float] | None
|
|
139
122
|
|
|
140
123
|
def __init__(
|
|
141
124
|
self,
|
|
142
125
|
prediction_id: str | None,
|
|
143
126
|
*,
|
|
144
|
-
label: int,
|
|
127
|
+
label: int | None,
|
|
145
128
|
label_name: str | None,
|
|
129
|
+
score: float | None,
|
|
146
130
|
confidence: float,
|
|
147
131
|
anomaly_score: float | None,
|
|
148
|
-
memoryset: LabeledMemoryset |
|
|
149
|
-
model: ClassificationModel |
|
|
150
|
-
telemetry: LabelPredictionWithMemoriesAndFeedback | None = None,
|
|
132
|
+
memoryset: LabeledMemoryset | ScoredMemoryset,
|
|
133
|
+
model: ClassificationModel | RegressionModel,
|
|
134
|
+
telemetry: LabelPredictionWithMemoriesAndFeedback | ScorePredictionWithMemoriesAndFeedback | None = None,
|
|
151
135
|
logits: list[float] | None = None,
|
|
136
|
+
input_value: str | None = None,
|
|
152
137
|
):
|
|
153
|
-
# for internal use only, do not document
|
|
154
|
-
from .classification_model import ClassificationModel
|
|
155
|
-
|
|
156
138
|
self.prediction_id = prediction_id
|
|
157
139
|
self.label = label
|
|
158
140
|
self.label_name = label_name
|
|
141
|
+
self.score = score
|
|
159
142
|
self.confidence = confidence
|
|
160
143
|
self.anomaly_score = anomaly_score
|
|
161
|
-
self.memoryset =
|
|
162
|
-
self.model =
|
|
144
|
+
self.memoryset = memoryset
|
|
145
|
+
self.model = model
|
|
163
146
|
self.__telemetry = telemetry if telemetry else None
|
|
164
147
|
self.logits = logits
|
|
165
|
-
|
|
166
|
-
def __repr__(self):
|
|
167
|
-
return (
|
|
168
|
-
"LabelPrediction({"
|
|
169
|
-
+ f"label: <{self.label_name}: {self.label}>, "
|
|
170
|
-
+ f"confidence: {self.confidence:.2f}, "
|
|
171
|
-
+ f"anomaly_score: {self.anomaly_score:.2f}, "
|
|
172
|
-
if self.anomaly_score is not None
|
|
173
|
-
else ""
|
|
174
|
-
+ f"input_value: '{str(self.input_value)[:100] + '...' if len(str(self.input_value)) > 100 else self.input_value}'"
|
|
175
|
-
+ "})"
|
|
176
|
-
)
|
|
148
|
+
self._input_value = input_value
|
|
177
149
|
|
|
178
150
|
@property
|
|
179
|
-
def _telemetry(self) -> LabelPredictionWithMemoriesAndFeedback:
|
|
151
|
+
def _telemetry(self) -> LabelPredictionWithMemoriesAndFeedback | ScorePredictionWithMemoriesAndFeedback:
|
|
180
152
|
# for internal use only, do not document
|
|
181
153
|
if self.__telemetry is None:
|
|
182
154
|
self.__telemetry = get_prediction(prediction_id=UUID(self.prediction_id))
|
|
183
155
|
return self.__telemetry
|
|
184
156
|
|
|
185
157
|
@property
|
|
186
|
-
def
|
|
187
|
-
|
|
158
|
+
def input_value(self) -> str:
|
|
159
|
+
if self._input_value is not None:
|
|
160
|
+
return self._input_value
|
|
161
|
+
assert isinstance(self._telemetry.input_value, str)
|
|
162
|
+
return self._telemetry.input_value
|
|
188
163
|
|
|
189
164
|
@property
|
|
190
|
-
def
|
|
191
|
-
|
|
165
|
+
def memory_lookups(self) -> list[LabeledMemoryLookup] | list[ScoredMemoryLookup]:
|
|
166
|
+
match self._telemetry:
|
|
167
|
+
case LabelPredictionWithMemoriesAndFeedback():
|
|
168
|
+
return [
|
|
169
|
+
LabeledMemoryLookup(self._telemetry.memoryset_id, lookup) for lookup in self._telemetry.memories
|
|
170
|
+
]
|
|
171
|
+
case ScorePredictionWithMemoriesAndFeedback():
|
|
172
|
+
return [ScoredMemoryLookup(self._telemetry.memoryset_id, lookup) for lookup in self._telemetry.memories]
|
|
192
173
|
|
|
193
174
|
@property
|
|
194
175
|
def feedback(self) -> dict[str, bool | float]:
|
|
@@ -199,14 +180,6 @@ class LabelPrediction:
|
|
|
199
180
|
for f in self._telemetry.feedbacks
|
|
200
181
|
}
|
|
201
182
|
|
|
202
|
-
@property
|
|
203
|
-
def expected_label(self) -> int | None:
|
|
204
|
-
return self._telemetry.expected_label
|
|
205
|
-
|
|
206
|
-
@property
|
|
207
|
-
def expected_label_name(self) -> str | None:
|
|
208
|
-
return self._telemetry.expected_label_name
|
|
209
|
-
|
|
210
183
|
@property
|
|
211
184
|
def tags(self) -> set[str]:
|
|
212
185
|
return set(self._telemetry.tags)
|
|
@@ -244,16 +217,16 @@ class LabelPrediction:
|
|
|
244
217
|
|
|
245
218
|
@overload
|
|
246
219
|
@classmethod
|
|
247
|
-
def get(cls, prediction_id: str) ->
|
|
220
|
+
def get(cls, prediction_id: str) -> Self: # type: ignore -- this takes precedence
|
|
248
221
|
pass
|
|
249
222
|
|
|
250
223
|
@overload
|
|
251
224
|
@classmethod
|
|
252
|
-
def get(cls, prediction_id: Iterable[str]) -> list[
|
|
225
|
+
def get(cls, prediction_id: Iterable[str]) -> list[Self]:
|
|
253
226
|
pass
|
|
254
227
|
|
|
255
228
|
@classmethod
|
|
256
|
-
def get(cls, prediction_id: str | Iterable[str]) ->
|
|
229
|
+
def get(cls, prediction_id: str | Iterable[str]) -> Self | list[Self]:
|
|
257
230
|
"""
|
|
258
231
|
Fetch a prediction or predictions
|
|
259
232
|
|
|
@@ -301,30 +274,40 @@ class LabelPrediction:
|
|
|
301
274
|
}),
|
|
302
275
|
]
|
|
303
276
|
"""
|
|
304
|
-
|
|
305
|
-
|
|
277
|
+
from .classification_model import ClassificationModel
|
|
278
|
+
from .regression_model import RegressionModel
|
|
279
|
+
|
|
280
|
+
def create_prediction(
|
|
281
|
+
prediction: LabelPredictionWithMemoriesAndFeedback | ScorePredictionWithMemoriesAndFeedback,
|
|
282
|
+
) -> Self:
|
|
283
|
+
|
|
284
|
+
if isinstance(prediction, LabelPredictionWithMemoriesAndFeedback):
|
|
285
|
+
memoryset = LabeledMemoryset.open(prediction.memoryset_id)
|
|
286
|
+
model = ClassificationModel.open(prediction.model_id)
|
|
287
|
+
elif isinstance(prediction, ScorePredictionWithMemoriesAndFeedback):
|
|
288
|
+
memoryset = ScoredMemoryset.open(prediction.memoryset_id)
|
|
289
|
+
model = RegressionModel.open(prediction.model_id)
|
|
290
|
+
else:
|
|
291
|
+
raise ValueError(f"Invalid prediction type: {type(prediction)}")
|
|
292
|
+
|
|
306
293
|
return cls(
|
|
307
294
|
prediction_id=prediction.prediction_id,
|
|
308
|
-
label=prediction
|
|
309
|
-
label_name=prediction
|
|
295
|
+
label=getattr(prediction, "label", None),
|
|
296
|
+
label_name=getattr(prediction, "label_name", None),
|
|
297
|
+
score=getattr(prediction, "score", None),
|
|
310
298
|
confidence=prediction.confidence,
|
|
311
299
|
anomaly_score=prediction.anomaly_score,
|
|
312
|
-
memoryset=
|
|
313
|
-
model=
|
|
300
|
+
memoryset=memoryset,
|
|
301
|
+
model=model,
|
|
314
302
|
telemetry=prediction,
|
|
315
303
|
)
|
|
304
|
+
|
|
305
|
+
if isinstance(prediction_id, str):
|
|
306
|
+
prediction = get_prediction(prediction_id=UUID(prediction_id))
|
|
307
|
+
return create_prediction(prediction)
|
|
316
308
|
else:
|
|
317
309
|
return [
|
|
318
|
-
|
|
319
|
-
prediction_id=prediction.prediction_id,
|
|
320
|
-
label=prediction.label,
|
|
321
|
-
label_name=prediction.label_name,
|
|
322
|
-
confidence=prediction.confidence,
|
|
323
|
-
anomaly_score=prediction.anomaly_score,
|
|
324
|
-
memoryset=prediction.memoryset_id,
|
|
325
|
-
model=prediction.model_id,
|
|
326
|
-
telemetry=prediction,
|
|
327
|
-
)
|
|
310
|
+
create_prediction(prediction)
|
|
328
311
|
for prediction in list_predictions(body=ListPredictionsRequest(prediction_ids=list(prediction_id)))
|
|
329
312
|
]
|
|
330
313
|
|
|
@@ -332,30 +315,15 @@ class LabelPrediction:
|
|
|
332
315
|
"""Refresh the prediction data from the OrcaCloud"""
|
|
333
316
|
if self.prediction_id is None:
|
|
334
317
|
raise ValueError("Cannot refresh prediction with no prediction ID")
|
|
335
|
-
self.__dict__.update(
|
|
336
|
-
|
|
337
|
-
def inspect(self):
|
|
338
|
-
"""Open a UI to inspect the memories used by this prediction"""
|
|
339
|
-
inspect_prediction_result(self)
|
|
340
|
-
|
|
341
|
-
def update(self, *, expected_label: int | None = UNSET, tags: set[str] | None = UNSET) -> None:
|
|
342
|
-
"""
|
|
343
|
-
Update editable prediction properties.
|
|
344
|
-
|
|
345
|
-
Params:
|
|
346
|
-
expected_label: Value to set for the expected label, defaults to `[UNSET]` if not provided.
|
|
347
|
-
tags: Value to replace existing tags with, defaults to `[UNSET]` if not provided.
|
|
318
|
+
self.__dict__.update(self.get(self.prediction_id).__dict__)
|
|
348
319
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
Remove expected label and tags:
|
|
357
|
-
>>> prediction.update(expected_label=None, tags=None)
|
|
358
|
-
"""
|
|
320
|
+
def _update(
|
|
321
|
+
self,
|
|
322
|
+
*,
|
|
323
|
+
tags: set[str] | None = UNSET,
|
|
324
|
+
expected_label: int | None = UNSET,
|
|
325
|
+
expected_score: float | None = UNSET,
|
|
326
|
+
) -> None:
|
|
359
327
|
if self.prediction_id is None:
|
|
360
328
|
raise ValueError("Cannot update prediction with no prediction ID")
|
|
361
329
|
|
|
@@ -363,6 +331,7 @@ class LabelPrediction:
|
|
|
363
331
|
prediction_id=self.prediction_id,
|
|
364
332
|
body=UpdatePredictionRequest(
|
|
365
333
|
expected_label=expected_label if expected_label is not UNSET else CLIENT_UNSET,
|
|
334
|
+
expected_score=expected_score if expected_score is not UNSET else CLIENT_UNSET,
|
|
366
335
|
tags=[] if tags is None else list(tags) if tags is not UNSET else CLIENT_UNSET,
|
|
367
336
|
),
|
|
368
337
|
)
|
|
@@ -375,7 +344,7 @@ class LabelPrediction:
|
|
|
375
344
|
Params:
|
|
376
345
|
tag: Tag to add to the prediction
|
|
377
346
|
"""
|
|
378
|
-
self.
|
|
347
|
+
self._update(tags=self.tags | {tag})
|
|
379
348
|
|
|
380
349
|
def remove_tag(self, tag: str) -> None:
|
|
381
350
|
"""
|
|
@@ -384,7 +353,7 @@ class LabelPrediction:
|
|
|
384
353
|
Params:
|
|
385
354
|
tag: Tag to remove from the prediction
|
|
386
355
|
"""
|
|
387
|
-
self.
|
|
356
|
+
self._update(tags=self.tags - {tag})
|
|
388
357
|
|
|
389
358
|
def record_feedback(
|
|
390
359
|
self,
|
|
@@ -446,3 +415,153 @@ class LabelPrediction:
|
|
|
446
415
|
body=[PredictionFeedbackRequest(prediction_id=self.prediction_id, category_name=category, value=None)]
|
|
447
416
|
)
|
|
448
417
|
self.refresh()
|
|
418
|
+
|
|
419
|
+
def inspect(self) -> None:
|
|
420
|
+
"""
|
|
421
|
+
Display an interactive UI with the details about this prediction
|
|
422
|
+
|
|
423
|
+
Params:
|
|
424
|
+
**kwargs: Additional keyword arguments to pass to the display function
|
|
425
|
+
|
|
426
|
+
Note:
|
|
427
|
+
This method is only available in Jupyter notebooks.
|
|
428
|
+
"""
|
|
429
|
+
inspect_prediction_result(self) # type: ignore
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
class ClassificationPrediction(_Prediction):
|
|
433
|
+
"""
|
|
434
|
+
Labeled prediction result from a [`ClassificationModel`][orca_sdk.ClassificationModel]
|
|
435
|
+
|
|
436
|
+
Attributes:
|
|
437
|
+
prediction_id: Unique identifier of this prediction used for feedback
|
|
438
|
+
label: Label predicted by the model
|
|
439
|
+
label_name: Human-readable name of the label
|
|
440
|
+
confidence: Confidence of the prediction
|
|
441
|
+
anomaly_score: Anomaly score of the input
|
|
442
|
+
input_value: The input value used for the prediction
|
|
443
|
+
expected_label: Expected label for the prediction, useful when evaluating the model
|
|
444
|
+
expected_label_name: Human-readable name of the expected label
|
|
445
|
+
memory_lookups: Memories used by the model to make the prediction
|
|
446
|
+
explanation: Natural language explanation of the prediction, only available if the model
|
|
447
|
+
has the Explain API enabled
|
|
448
|
+
tags: Tags for the prediction, useful for filtering and grouping predictions
|
|
449
|
+
model: Model used to make the prediction
|
|
450
|
+
memoryset: Memoryset that was used to lookup memories to ground the prediction
|
|
451
|
+
"""
|
|
452
|
+
|
|
453
|
+
label: int
|
|
454
|
+
label_name: str
|
|
455
|
+
logits: list[float] | None
|
|
456
|
+
model: ClassificationModel
|
|
457
|
+
memoryset: LabeledMemoryset
|
|
458
|
+
|
|
459
|
+
def __repr__(self):
|
|
460
|
+
return (
|
|
461
|
+
"ClassificationPrediction({"
|
|
462
|
+
+ f"label: <{self.label_name}: {self.label}>, "
|
|
463
|
+
+ f"confidence: {self.confidence:.2f}, "
|
|
464
|
+
+ (f"anomaly_score: {self.anomaly_score:.2f}, " if self.anomaly_score is not None else "")
|
|
465
|
+
+ f"input_value: '{str(self.input_value)[:100] + '...' if len(str(self.input_value)) > 100 else self.input_value}'"
|
|
466
|
+
+ "})"
|
|
467
|
+
)
|
|
468
|
+
|
|
469
|
+
@property
|
|
470
|
+
def memory_lookups(self) -> list[LabeledMemoryLookup]:
|
|
471
|
+
assert isinstance(self._telemetry, LabelPredictionWithMemoriesAndFeedback)
|
|
472
|
+
return [LabeledMemoryLookup(self._telemetry.memoryset_id, lookup) for lookup in self._telemetry.memories]
|
|
473
|
+
|
|
474
|
+
@property
|
|
475
|
+
def expected_label(self) -> int | None:
|
|
476
|
+
assert isinstance(self._telemetry, LabelPredictionWithMemoriesAndFeedback)
|
|
477
|
+
return self._telemetry.expected_label
|
|
478
|
+
|
|
479
|
+
@property
|
|
480
|
+
def expected_label_name(self) -> str | None:
|
|
481
|
+
assert isinstance(self._telemetry, LabelPredictionWithMemoriesAndFeedback)
|
|
482
|
+
return self._telemetry.expected_label_name
|
|
483
|
+
|
|
484
|
+
def update(
|
|
485
|
+
self,
|
|
486
|
+
*,
|
|
487
|
+
tags: set[str] | None = UNSET,
|
|
488
|
+
expected_label: int | None = UNSET,
|
|
489
|
+
) -> None:
|
|
490
|
+
"""
|
|
491
|
+
Update the prediction.
|
|
492
|
+
|
|
493
|
+
Note:
|
|
494
|
+
If a field is not provided, it will default to [UNSET][orca_sdk.UNSET] and not be updated.
|
|
495
|
+
|
|
496
|
+
Params:
|
|
497
|
+
tags: New tags to set for the prediction. Set to `None` to remove all tags.
|
|
498
|
+
expected_label: New expected label to set for the prediction. Set to `None` to remove.
|
|
499
|
+
"""
|
|
500
|
+
self._update(tags=tags, expected_label=expected_label)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
class RegressionPrediction(_Prediction):
|
|
504
|
+
"""
|
|
505
|
+
Score-based prediction result from a [`RegressionModel`][orca_sdk.RegressionModel]
|
|
506
|
+
|
|
507
|
+
Attributes:
|
|
508
|
+
prediction_id: Unique identifier of this prediction used for feedback
|
|
509
|
+
score: Score predicted by the model
|
|
510
|
+
confidence: Confidence of the prediction
|
|
511
|
+
anomaly_score: Anomaly score of the input
|
|
512
|
+
input_value: The input value used for the prediction
|
|
513
|
+
expected_score: Expected score for the prediction, useful when evaluating the model
|
|
514
|
+
memory_lookups: Memories used by the model to make the prediction
|
|
515
|
+
explanation: Natural language explanation of the prediction, only available if the model
|
|
516
|
+
has the Explain API enabled
|
|
517
|
+
tags: Tags for the prediction, useful for filtering and grouping predictions
|
|
518
|
+
model: Model used to make the prediction
|
|
519
|
+
memoryset: Memoryset that was used to lookup memories to ground the prediction
|
|
520
|
+
"""
|
|
521
|
+
|
|
522
|
+
score: float
|
|
523
|
+
model: RegressionModel
|
|
524
|
+
memoryset: ScoredMemoryset
|
|
525
|
+
|
|
526
|
+
def __repr__(self):
|
|
527
|
+
return (
|
|
528
|
+
"RegressionPrediction({"
|
|
529
|
+
+ f"score: {self.score:.2f}, "
|
|
530
|
+
+ f"confidence: {self.confidence:.2f}, "
|
|
531
|
+
+ (f"anomaly_score: {self.anomaly_score:.2f}, " if self.anomaly_score is not None else "")
|
|
532
|
+
+ f"input_value: '{str(self.input_value)[:100] + '...' if len(str(self.input_value)) > 100 else self.input_value}'"
|
|
533
|
+
+ "})"
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
@property
|
|
537
|
+
def memory_lookups(self) -> list[ScoredMemoryLookup]:
|
|
538
|
+
assert isinstance(self._telemetry, ScorePredictionWithMemoriesAndFeedback)
|
|
539
|
+
return [ScoredMemoryLookup(self._telemetry.memoryset_id, lookup) for lookup in self._telemetry.memories]
|
|
540
|
+
|
|
541
|
+
@property
|
|
542
|
+
def explanation(self) -> str:
|
|
543
|
+
"""The explanation for this prediction. Requires `lighthouse_client_api_key` to be set."""
|
|
544
|
+
raise NotImplementedError("Explanation is not supported for regression predictions")
|
|
545
|
+
|
|
546
|
+
@property
|
|
547
|
+
def expected_score(self) -> float | None:
|
|
548
|
+
assert isinstance(self._telemetry, ScorePredictionWithMemoriesAndFeedback)
|
|
549
|
+
return self._telemetry.expected_score
|
|
550
|
+
|
|
551
|
+
def update(
|
|
552
|
+
self,
|
|
553
|
+
*,
|
|
554
|
+
tags: set[str] | None = UNSET,
|
|
555
|
+
expected_score: float | None = UNSET,
|
|
556
|
+
) -> None:
|
|
557
|
+
"""
|
|
558
|
+
Update the prediction.
|
|
559
|
+
|
|
560
|
+
Note:
|
|
561
|
+
If a field is not provided, it will default to [UNSET][orca_sdk.UNSET] and not be updated.
|
|
562
|
+
|
|
563
|
+
Params:
|
|
564
|
+
tags: New tags to set for the prediction. Set to `None` to remove all tags.
|
|
565
|
+
expected_score: New expected score to set for the prediction. Set to `None` to remove.
|
|
566
|
+
"""
|
|
567
|
+
self._update(tags=tags, expected_score=expected_score)
|
orca_sdk/telemetry_test.py
CHANGED
|
@@ -2,75 +2,79 @@ import pytest
|
|
|
2
2
|
|
|
3
3
|
from .classification_model import ClassificationModel
|
|
4
4
|
from .memoryset import LabeledMemoryLookup
|
|
5
|
-
from .telemetry import
|
|
5
|
+
from .telemetry import ClassificationPrediction, FeedbackCategory
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
def test_get_prediction(
|
|
9
|
-
predictions =
|
|
8
|
+
def test_get_prediction(classification_model: ClassificationModel):
|
|
9
|
+
predictions = classification_model.predict(["Do you love soup?", "Are cats cute?"])
|
|
10
10
|
assert len(predictions) == 2
|
|
11
11
|
assert predictions[0].prediction_id is not None
|
|
12
12
|
assert predictions[1].prediction_id is not None
|
|
13
|
-
prediction_with_telemetry =
|
|
13
|
+
prediction_with_telemetry = ClassificationPrediction.get(predictions[0].prediction_id)
|
|
14
14
|
assert prediction_with_telemetry is not None
|
|
15
15
|
assert prediction_with_telemetry.label == 0
|
|
16
16
|
assert prediction_with_telemetry.input_value == "Do you love soup?"
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
def test_get_predictions(
|
|
20
|
-
predictions =
|
|
19
|
+
def test_get_predictions(classification_model: ClassificationModel):
|
|
20
|
+
predictions = classification_model.predict(["Do you love soup?", "Are cats cute?"])
|
|
21
21
|
assert len(predictions) == 2
|
|
22
22
|
assert predictions[0].prediction_id is not None
|
|
23
23
|
assert predictions[1].prediction_id is not None
|
|
24
|
-
prediction_with_telemetry =
|
|
24
|
+
prediction_with_telemetry = ClassificationPrediction.get(
|
|
25
|
+
[predictions[0].prediction_id, predictions[1].prediction_id]
|
|
26
|
+
)
|
|
25
27
|
assert len(prediction_with_telemetry) == 2
|
|
26
28
|
assert prediction_with_telemetry[0].label == 0
|
|
27
29
|
assert prediction_with_telemetry[0].input_value == "Do you love soup?"
|
|
28
30
|
assert prediction_with_telemetry[1].label == 1
|
|
29
31
|
|
|
30
32
|
|
|
31
|
-
def test_get_predictions_with_expected_label_match(
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
assert len(
|
|
33
|
+
def test_get_predictions_with_expected_label_match(classification_model: ClassificationModel):
|
|
34
|
+
classification_model.predict(
|
|
35
|
+
["Do you love soup?", "Are cats cute?"], expected_labels=[0, 0], tags={"expected_label_match"}
|
|
36
|
+
)
|
|
37
|
+
classification_model.predict("no expectations", tags={"expected_label_match"})
|
|
38
|
+
assert len(classification_model.predictions(tag="expected_label_match")) == 3
|
|
39
|
+
assert len(classification_model.predictions(expected_label_match=True, tag="expected_label_match")) == 1
|
|
40
|
+
assert len(classification_model.predictions(expected_label_match=False, tag="expected_label_match")) == 1
|
|
37
41
|
|
|
38
42
|
|
|
39
|
-
def test_get_prediction_memory_lookups(
|
|
40
|
-
prediction =
|
|
43
|
+
def test_get_prediction_memory_lookups(classification_model: ClassificationModel):
|
|
44
|
+
prediction = classification_model.predict("Do you love soup?")
|
|
41
45
|
assert isinstance(prediction.memory_lookups, list)
|
|
42
46
|
assert len(prediction.memory_lookups) > 0
|
|
43
47
|
assert all(isinstance(lookup, LabeledMemoryLookup) for lookup in prediction.memory_lookups)
|
|
44
48
|
|
|
45
49
|
|
|
46
|
-
def test_record_feedback(
|
|
47
|
-
prediction =
|
|
50
|
+
def test_record_feedback(classification_model: ClassificationModel):
|
|
51
|
+
prediction = classification_model.predict("Do you love soup?")
|
|
48
52
|
assert "correct" not in prediction.feedback
|
|
49
53
|
prediction.record_feedback(category="correct", value=prediction.label == 0)
|
|
50
54
|
assert prediction.feedback["correct"] is True
|
|
51
55
|
|
|
52
56
|
|
|
53
|
-
def test_record_feedback_with_invalid_value(
|
|
57
|
+
def test_record_feedback_with_invalid_value(classification_model: ClassificationModel):
|
|
54
58
|
with pytest.raises(ValueError, match=r"Invalid input.*"):
|
|
55
|
-
|
|
59
|
+
classification_model.predict("Do you love soup?").record_feedback(category="correct", value="not a bool") # type: ignore
|
|
56
60
|
|
|
57
61
|
|
|
58
|
-
def test_record_feedback_with_inconsistent_value_for_category(
|
|
59
|
-
|
|
62
|
+
def test_record_feedback_with_inconsistent_value_for_category(classification_model: ClassificationModel):
|
|
63
|
+
classification_model.predict("Do you love soup?").record_feedback(category="correct", value=True)
|
|
60
64
|
with pytest.raises(ValueError, match=r"Invalid input.*"):
|
|
61
|
-
|
|
65
|
+
classification_model.predict("Do you love soup?").record_feedback(category="correct", value=-1.0)
|
|
62
66
|
|
|
63
67
|
|
|
64
|
-
def test_delete_feedback(
|
|
65
|
-
prediction =
|
|
68
|
+
def test_delete_feedback(classification_model: ClassificationModel):
|
|
69
|
+
prediction = classification_model.predict("Do you love soup?")
|
|
66
70
|
prediction.record_feedback(category="test_delete", value=True)
|
|
67
71
|
assert "test_delete" in prediction.feedback
|
|
68
72
|
prediction.delete_feedback("test_delete")
|
|
69
73
|
assert "test_delete" not in prediction.feedback
|
|
70
74
|
|
|
71
75
|
|
|
72
|
-
def test_list_feedback_categories(
|
|
73
|
-
prediction =
|
|
76
|
+
def test_list_feedback_categories(classification_model: ClassificationModel):
|
|
77
|
+
prediction = classification_model.predict("Do you love soup?")
|
|
74
78
|
prediction.record_feedback(category="correct", value=True)
|
|
75
79
|
prediction.record_feedback(category="confidence", value=0.8)
|
|
76
80
|
categories = FeedbackCategory.all()
|
|
@@ -79,8 +83,8 @@ def test_list_feedback_categories(model: ClassificationModel):
|
|
|
79
83
|
assert any(c.name == "confidence" and c.value_type == float for c in categories)
|
|
80
84
|
|
|
81
85
|
|
|
82
|
-
def test_drop_feedback_category(
|
|
83
|
-
prediction =
|
|
86
|
+
def test_drop_feedback_category(classification_model: ClassificationModel):
|
|
87
|
+
prediction = classification_model.predict("Do you love soup?")
|
|
84
88
|
prediction.record_feedback(category="test_category", value=True)
|
|
85
89
|
assert any(c.name == "test_category" for c in FeedbackCategory.all())
|
|
86
90
|
FeedbackCategory.drop("test_category")
|
|
@@ -89,8 +93,8 @@ def test_drop_feedback_category(model: ClassificationModel):
|
|
|
89
93
|
assert "test_category" not in prediction.feedback
|
|
90
94
|
|
|
91
95
|
|
|
92
|
-
def test_update_prediction(
|
|
93
|
-
prediction =
|
|
96
|
+
def test_update_prediction(classification_model: ClassificationModel):
|
|
97
|
+
prediction = classification_model.predict("Do you love soup?")
|
|
94
98
|
assert prediction.expected_label is None
|
|
95
99
|
assert prediction.tags == set()
|
|
96
100
|
# update expected label
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: orca_sdk
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.95
|
|
4
4
|
Summary: SDK for interacting with Orca Services
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Author: Orca DB Inc.
|
|
@@ -15,14 +15,12 @@ Requires-Dist: attrs (>=22.2.0)
|
|
|
15
15
|
Requires-Dist: datasets (>=3.1.0,<4.0.0)
|
|
16
16
|
Requires-Dist: gradio (==5.13.0)
|
|
17
17
|
Requires-Dist: httpx (>=0.20.0,<0.29.0)
|
|
18
|
-
Requires-Dist: networkx (>=3.4.2,<4.0.0)
|
|
19
18
|
Requires-Dist: pandas (>=2.2.3,<3.0.0)
|
|
20
19
|
Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
|
|
21
20
|
Requires-Dist: python-dateutil (>=2.8.0,<3.0.0)
|
|
22
21
|
Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
|
|
23
22
|
Requires-Dist: scikit-learn (>=1.6.1,<2.0.0)
|
|
24
23
|
Requires-Dist: torch (>=2.5.1,<3.0.0)
|
|
25
|
-
Requires-Dist: transformers (>=4.51.3,<5.0.0)
|
|
26
24
|
Description-Content-Type: text/markdown
|
|
27
25
|
|
|
28
26
|
<!--
|
|
@@ -75,7 +73,7 @@ model = ClassificationModel("my_model", memoryset)
|
|
|
75
73
|
prediction = model.predict("my input")
|
|
76
74
|
```
|
|
77
75
|
|
|
78
|
-
For a more detailed walkthrough, check out our [Quick Start Guide](https://docs.orcadb.ai/quickstart/).
|
|
76
|
+
For a more detailed walkthrough, check out our [Quick Start Guide](https://docs.orcadb.ai/quickstart-sdk/).
|
|
79
77
|
|
|
80
78
|
## Support
|
|
81
79
|
|