orca-sdk 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- orca_sdk/__init__.py +1 -1
- orca_sdk/_utils/auth.py +12 -8
- orca_sdk/async_client.py +3942 -0
- orca_sdk/classification_model.py +218 -20
- orca_sdk/classification_model_test.py +96 -28
- orca_sdk/client.py +899 -712
- orca_sdk/conftest.py +37 -36
- orca_sdk/credentials.py +54 -14
- orca_sdk/credentials_test.py +92 -28
- orca_sdk/datasource.py +64 -12
- orca_sdk/datasource_test.py +144 -18
- orca_sdk/embedding_model.py +54 -37
- orca_sdk/embedding_model_test.py +27 -20
- orca_sdk/job.py +27 -21
- orca_sdk/memoryset.py +823 -205
- orca_sdk/memoryset_test.py +315 -33
- orca_sdk/regression_model.py +59 -15
- orca_sdk/regression_model_test.py +35 -26
- orca_sdk/telemetry.py +76 -26
- {orca_sdk-0.1.2.dist-info → orca_sdk-0.1.4.dist-info}/METADATA +1 -1
- orca_sdk-0.1.4.dist-info/RECORD +41 -0
- orca_sdk-0.1.2.dist-info/RECORD +0 -40
- {orca_sdk-0.1.2.dist-info → orca_sdk-0.1.4.dist-info}/WHEEL +0 -0
orca_sdk/regression_model.py
CHANGED
|
@@ -10,10 +10,12 @@ from datasets import Dataset
|
|
|
10
10
|
from ._shared.metrics import RegressionMetrics, calculate_regression_metrics
|
|
11
11
|
from ._utils.common import UNSET, CreateMode, DropMode
|
|
12
12
|
from .client import (
|
|
13
|
+
OrcaClient,
|
|
14
|
+
PostRegressionModelByModelNameOrIdEvaluationParams,
|
|
13
15
|
PredictiveModelUpdate,
|
|
14
16
|
RARHeadType,
|
|
17
|
+
RegressionEvaluationRequest,
|
|
15
18
|
RegressionModelMetadata,
|
|
16
|
-
orca_api,
|
|
17
19
|
)
|
|
18
20
|
from .datasource import Datasource
|
|
19
21
|
from .job import Job
|
|
@@ -154,7 +156,8 @@ class RegressionModel:
|
|
|
154
156
|
|
|
155
157
|
return existing
|
|
156
158
|
|
|
157
|
-
|
|
159
|
+
client = OrcaClient._resolve_client()
|
|
160
|
+
metadata = client.POST(
|
|
158
161
|
"/regression_model",
|
|
159
162
|
json={
|
|
160
163
|
"name": name,
|
|
@@ -179,7 +182,8 @@ class RegressionModel:
|
|
|
179
182
|
Raises:
|
|
180
183
|
LookupError: If the regression model does not exist
|
|
181
184
|
"""
|
|
182
|
-
|
|
185
|
+
client = OrcaClient._resolve_client()
|
|
186
|
+
return cls(client.GET("/regression_model/{name_or_id}", params={"name_or_id": name}))
|
|
183
187
|
|
|
184
188
|
@classmethod
|
|
185
189
|
def exists(cls, name_or_id: str) -> bool:
|
|
@@ -206,7 +210,8 @@ class RegressionModel:
|
|
|
206
210
|
Returns:
|
|
207
211
|
List of handles to all regression models in the OrcaCloud
|
|
208
212
|
"""
|
|
209
|
-
|
|
213
|
+
client = OrcaClient._resolve_client()
|
|
214
|
+
return [cls(metadata) for metadata in client.GET("/regression_model")]
|
|
210
215
|
|
|
211
216
|
@classmethod
|
|
212
217
|
def drop(cls, name_or_id: str, if_not_exists: DropMode = "error"):
|
|
@@ -225,7 +230,8 @@ class RegressionModel:
|
|
|
225
230
|
LookupError: If the regression model does not exist and if_not_exists is `"error"`
|
|
226
231
|
"""
|
|
227
232
|
try:
|
|
228
|
-
|
|
233
|
+
client = OrcaClient._resolve_client()
|
|
234
|
+
client.DELETE("/regression_model/{name_or_id}", params={"name_or_id": name_or_id})
|
|
229
235
|
logging.info(f"Deleted model {name_or_id}")
|
|
230
236
|
except LookupError:
|
|
231
237
|
if if_not_exists == "error":
|
|
@@ -261,7 +267,8 @@ class RegressionModel:
|
|
|
261
267
|
update["description"] = description
|
|
262
268
|
if locked is not UNSET:
|
|
263
269
|
update["locked"] = locked
|
|
264
|
-
|
|
270
|
+
client = OrcaClient._resolve_client()
|
|
271
|
+
client.PATCH("/regression_model/{name_or_id}", params={"name_or_id": self.id}, json=update)
|
|
265
272
|
self.refresh()
|
|
266
273
|
|
|
267
274
|
def lock(self) -> None:
|
|
@@ -282,6 +289,8 @@ class RegressionModel:
|
|
|
282
289
|
prompt: str | None = None,
|
|
283
290
|
use_lookup_cache: bool = True,
|
|
284
291
|
timeout_seconds: int = 10,
|
|
292
|
+
ignore_unlabeled: bool = False,
|
|
293
|
+
use_gpu: bool = True,
|
|
285
294
|
) -> RegressionPrediction: ...
|
|
286
295
|
|
|
287
296
|
@overload
|
|
@@ -294,6 +303,8 @@ class RegressionModel:
|
|
|
294
303
|
prompt: str | None = None,
|
|
295
304
|
use_lookup_cache: bool = True,
|
|
296
305
|
timeout_seconds: int = 10,
|
|
306
|
+
ignore_unlabeled: bool = False,
|
|
307
|
+
use_gpu: bool = True,
|
|
297
308
|
) -> list[RegressionPrediction]: ...
|
|
298
309
|
|
|
299
310
|
# TODO: add filter support
|
|
@@ -306,6 +317,8 @@ class RegressionModel:
|
|
|
306
317
|
prompt: str | None = None,
|
|
307
318
|
use_lookup_cache: bool = True,
|
|
308
319
|
timeout_seconds: int = 10,
|
|
320
|
+
ignore_unlabeled: bool = False,
|
|
321
|
+
use_gpu: bool = True,
|
|
309
322
|
) -> RegressionPrediction | list[RegressionPrediction]:
|
|
310
323
|
"""
|
|
311
324
|
Make predictions using the regression model.
|
|
@@ -321,6 +334,9 @@ class RegressionModel:
|
|
|
321
334
|
prompt: Optional prompt for instruction-tuned embedding models
|
|
322
335
|
use_lookup_cache: Whether to use cached lookup results for faster predictions
|
|
323
336
|
timeout_seconds: Timeout in seconds for the request, defaults to 10 seconds
|
|
337
|
+
ignore_unlabeled: If True, only use memories with scores during lookup.
|
|
338
|
+
If False (default), allow memories without scores when necessary.
|
|
339
|
+
use_gpu: Whether to use GPU for the prediction (defaults to True)
|
|
324
340
|
|
|
325
341
|
Returns:
|
|
326
342
|
Single RegressionPrediction or list of RegressionPrediction objects
|
|
@@ -333,9 +349,15 @@ class RegressionModel:
|
|
|
333
349
|
if timeout_seconds <= 0:
|
|
334
350
|
raise ValueError("timeout_seconds must be a positive integer")
|
|
335
351
|
|
|
352
|
+
if use_gpu:
|
|
353
|
+
endpoint = "/gpu/regression_model/{name_or_id}/prediction"
|
|
354
|
+
else:
|
|
355
|
+
endpoint = "/regression_model/{name_or_id}/prediction"
|
|
356
|
+
|
|
336
357
|
telemetry_on, telemetry_sync = _get_telemetry_config(save_telemetry)
|
|
337
|
-
|
|
338
|
-
|
|
358
|
+
client = OrcaClient._resolve_client()
|
|
359
|
+
response = client.POST(
|
|
360
|
+
endpoint,
|
|
339
361
|
params={"name_or_id": self.id},
|
|
340
362
|
json={
|
|
341
363
|
"input_values": value if isinstance(value, list) else [value],
|
|
@@ -350,6 +372,7 @@ class RegressionModel:
|
|
|
350
372
|
"save_telemetry_synchronously": telemetry_sync,
|
|
351
373
|
"prompt": prompt,
|
|
352
374
|
"use_lookup_cache": use_lookup_cache,
|
|
375
|
+
"ignore_unlabeled": ignore_unlabeled,
|
|
353
376
|
},
|
|
354
377
|
timeout=timeout_seconds,
|
|
355
378
|
)
|
|
@@ -409,7 +432,8 @@ class RegressionModel:
|
|
|
409
432
|
>>> predictions = model.predictions(sort=[("confidence", "desc")], offset=1, limit=1)
|
|
410
433
|
[RegressionPrediction({score: 4.2, confidence: 0.90, anomaly_score: 0.1, input_value: 'Good service'})]
|
|
411
434
|
"""
|
|
412
|
-
|
|
435
|
+
client = OrcaClient._resolve_client()
|
|
436
|
+
predictions = client.POST(
|
|
413
437
|
"/telemetry/prediction",
|
|
414
438
|
json={
|
|
415
439
|
"model_id": self.id,
|
|
@@ -444,9 +468,12 @@ class RegressionModel:
|
|
|
444
468
|
score_column: str,
|
|
445
469
|
record_predictions: bool,
|
|
446
470
|
tags: set[str] | None,
|
|
471
|
+
subsample: int | float | None,
|
|
447
472
|
background: bool = False,
|
|
473
|
+
ignore_unlabeled: bool = False,
|
|
448
474
|
) -> RegressionMetrics | Job[RegressionMetrics]:
|
|
449
|
-
|
|
475
|
+
client = OrcaClient._resolve_client()
|
|
476
|
+
response = client.POST(
|
|
450
477
|
"/regression_model/{model_name_or_id}/evaluation",
|
|
451
478
|
params={"model_name_or_id": self.id},
|
|
452
479
|
json={
|
|
@@ -456,13 +483,16 @@ class RegressionModel:
|
|
|
456
483
|
"memoryset_override_name_or_id": self._memoryset_override_id,
|
|
457
484
|
"record_telemetry": record_predictions,
|
|
458
485
|
"telemetry_tags": list(tags) if tags else None,
|
|
486
|
+
"subsample": subsample,
|
|
487
|
+
"ignore_unlabeled": ignore_unlabeled,
|
|
459
488
|
},
|
|
460
489
|
)
|
|
461
490
|
|
|
462
491
|
def get_value():
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
492
|
+
client = OrcaClient._resolve_client()
|
|
493
|
+
res = client.GET(
|
|
494
|
+
"/regression_model/{model_name_or_id}/evaluation/{job_id}",
|
|
495
|
+
params={"model_name_or_id": self.id, "job_id": response["job_id"]},
|
|
466
496
|
)
|
|
467
497
|
assert res["result"] is not None
|
|
468
498
|
return RegressionMetrics(
|
|
@@ -478,7 +508,7 @@ class RegressionModel:
|
|
|
478
508
|
anomaly_score_variance=res["result"].get("anomaly_score_variance"),
|
|
479
509
|
)
|
|
480
510
|
|
|
481
|
-
job = Job(response["
|
|
511
|
+
job = Job(response["job_id"], get_value)
|
|
482
512
|
return job if background else job.result()
|
|
483
513
|
|
|
484
514
|
def _evaluate_dataset(
|
|
@@ -490,6 +520,7 @@ class RegressionModel:
|
|
|
490
520
|
tags: set[str],
|
|
491
521
|
batch_size: int,
|
|
492
522
|
prompt: str | None = None,
|
|
523
|
+
ignore_unlabeled: bool = False,
|
|
493
524
|
) -> RegressionMetrics:
|
|
494
525
|
if len(dataset) == 0:
|
|
495
526
|
raise ValueError("Evaluation dataset cannot be empty")
|
|
@@ -506,6 +537,7 @@ class RegressionModel:
|
|
|
506
537
|
tags=tags,
|
|
507
538
|
save_telemetry="sync" if record_predictions else "off",
|
|
508
539
|
prompt=prompt,
|
|
540
|
+
ignore_unlabeled=ignore_unlabeled,
|
|
509
541
|
)
|
|
510
542
|
]
|
|
511
543
|
|
|
@@ -526,7 +558,9 @@ class RegressionModel:
|
|
|
526
558
|
tags: set[str] = {"evaluation"},
|
|
527
559
|
batch_size: int = 100,
|
|
528
560
|
prompt: str | None = None,
|
|
561
|
+
subsample: int | float | None = None,
|
|
529
562
|
background: Literal[True],
|
|
563
|
+
ignore_unlabeled: bool = False,
|
|
530
564
|
) -> Job[RegressionMetrics]:
|
|
531
565
|
pass
|
|
532
566
|
|
|
@@ -541,7 +575,9 @@ class RegressionModel:
|
|
|
541
575
|
tags: set[str] = {"evaluation"},
|
|
542
576
|
batch_size: int = 100,
|
|
543
577
|
prompt: str | None = None,
|
|
578
|
+
subsample: int | float | None = None,
|
|
544
579
|
background: Literal[False] = False,
|
|
580
|
+
ignore_unlabeled: bool = False,
|
|
545
581
|
) -> RegressionMetrics:
|
|
546
582
|
pass
|
|
547
583
|
|
|
@@ -555,7 +591,9 @@ class RegressionModel:
|
|
|
555
591
|
tags: set[str] = {"evaluation"},
|
|
556
592
|
batch_size: int = 100,
|
|
557
593
|
prompt: str | None = None,
|
|
594
|
+
subsample: int | float | None = None,
|
|
558
595
|
background: bool = False,
|
|
596
|
+
ignore_unlabeled: bool = False,
|
|
559
597
|
) -> RegressionMetrics | Job[RegressionMetrics]:
|
|
560
598
|
"""
|
|
561
599
|
Evaluate the regression model on a given dataset or datasource
|
|
@@ -568,7 +606,9 @@ class RegressionModel:
|
|
|
568
606
|
tags: Optional tags to add to the recorded [`RegressionPrediction`][orca_sdk.telemetry.RegressionPrediction]s
|
|
569
607
|
batch_size: Batch size for processing Dataset inputs (only used when input is a Dataset)
|
|
570
608
|
prompt: Optional prompt for instruction-tuned embedding models
|
|
609
|
+
subsample: Optional number (int) of rows to sample or fraction (float in (0, 1]) of data to sample for evaluation.
|
|
571
610
|
background: Whether to run the operation in the background and return a job handle
|
|
611
|
+
ignore_unlabeled: If True, only use memories with scores during lookup. If False (default), allow memories without scores
|
|
572
612
|
|
|
573
613
|
Returns:
|
|
574
614
|
RegressionMetrics containing metrics including MAE, MSE, RMSE, R2, and anomaly score statistics
|
|
@@ -597,7 +637,9 @@ class RegressionModel:
|
|
|
597
637
|
score_column=score_column,
|
|
598
638
|
record_predictions=record_predictions,
|
|
599
639
|
tags=tags,
|
|
640
|
+
subsample=subsample,
|
|
600
641
|
background=background,
|
|
642
|
+
ignore_unlabeled=ignore_unlabeled,
|
|
601
643
|
)
|
|
602
644
|
elif isinstance(data, Dataset):
|
|
603
645
|
return self._evaluate_dataset(
|
|
@@ -608,6 +650,7 @@ class RegressionModel:
|
|
|
608
650
|
tags=tags,
|
|
609
651
|
batch_size=batch_size,
|
|
610
652
|
prompt=prompt,
|
|
653
|
+
ignore_unlabeled=ignore_unlabeled,
|
|
611
654
|
)
|
|
612
655
|
else:
|
|
613
656
|
raise ValueError(f"Invalid data type: {type(data)}")
|
|
@@ -676,7 +719,8 @@ class RegressionModel:
|
|
|
676
719
|
ValueError: If the value does not match previous value types for the category, or is a
|
|
677
720
|
[`float`][float] that is not between `-1.0` and `+1.0`.
|
|
678
721
|
"""
|
|
679
|
-
|
|
722
|
+
client = OrcaClient._resolve_client()
|
|
723
|
+
client.PUT(
|
|
680
724
|
"/telemetry/prediction/feedback",
|
|
681
725
|
json=[
|
|
682
726
|
_parse_feedback(f) for f in (cast(list[dict], [feedback]) if isinstance(feedback, dict) else feedback)
|
|
@@ -36,9 +36,10 @@ def test_create_model_already_exists_return(scored_memoryset, regression_model:
|
|
|
36
36
|
assert new_model.memory_lookup_count == 3
|
|
37
37
|
|
|
38
38
|
|
|
39
|
-
def test_create_model_unauthenticated(
|
|
40
|
-
with
|
|
41
|
-
|
|
39
|
+
def test_create_model_unauthenticated(unauthenticated_client, scored_memoryset: ScoredMemoryset):
|
|
40
|
+
with unauthenticated_client.use():
|
|
41
|
+
with pytest.raises(ValueError, match="Invalid API key"):
|
|
42
|
+
RegressionModel.create("test_regression_model", scored_memoryset)
|
|
42
43
|
|
|
43
44
|
|
|
44
45
|
def test_get_model(regression_model: RegressionModel):
|
|
@@ -50,9 +51,10 @@ def test_get_model(regression_model: RegressionModel):
|
|
|
50
51
|
assert fetched_model == regression_model
|
|
51
52
|
|
|
52
53
|
|
|
53
|
-
def test_get_model_unauthenticated(
|
|
54
|
-
with
|
|
55
|
-
|
|
54
|
+
def test_get_model_unauthenticated(unauthenticated_client):
|
|
55
|
+
with unauthenticated_client.use():
|
|
56
|
+
with pytest.raises(ValueError, match="Invalid API key"):
|
|
57
|
+
RegressionModel.open("test_regression_model")
|
|
56
58
|
|
|
57
59
|
|
|
58
60
|
def test_get_model_invalid_input():
|
|
@@ -65,9 +67,10 @@ def test_get_model_not_found():
|
|
|
65
67
|
RegressionModel.open(str(uuid4()))
|
|
66
68
|
|
|
67
69
|
|
|
68
|
-
def test_get_model_unauthorized(
|
|
69
|
-
with
|
|
70
|
-
|
|
70
|
+
def test_get_model_unauthorized(unauthorized_client, regression_model: RegressionModel):
|
|
71
|
+
with unauthorized_client.use():
|
|
72
|
+
with pytest.raises(LookupError):
|
|
73
|
+
RegressionModel.open(regression_model.name)
|
|
71
74
|
|
|
72
75
|
|
|
73
76
|
def test_list_models(regression_model: RegressionModel):
|
|
@@ -76,13 +79,15 @@ def test_list_models(regression_model: RegressionModel):
|
|
|
76
79
|
assert any(model.name == regression_model.name for model in models)
|
|
77
80
|
|
|
78
81
|
|
|
79
|
-
def test_list_models_unauthenticated(
|
|
80
|
-
with
|
|
81
|
-
|
|
82
|
+
def test_list_models_unauthenticated(unauthenticated_client):
|
|
83
|
+
with unauthenticated_client.use():
|
|
84
|
+
with pytest.raises(ValueError, match="Invalid API key"):
|
|
85
|
+
RegressionModel.all()
|
|
82
86
|
|
|
83
87
|
|
|
84
|
-
def test_list_models_unauthorized(
|
|
85
|
-
|
|
88
|
+
def test_list_models_unauthorized(unauthorized_client, regression_model: RegressionModel):
|
|
89
|
+
with unauthorized_client.use():
|
|
90
|
+
assert RegressionModel.all() == []
|
|
86
91
|
|
|
87
92
|
|
|
88
93
|
def test_update_model_attributes(regression_model: RegressionModel):
|
|
@@ -113,9 +118,10 @@ def test_delete_model(scored_memoryset: ScoredMemoryset):
|
|
|
113
118
|
RegressionModel.open("regression_model_to_delete")
|
|
114
119
|
|
|
115
120
|
|
|
116
|
-
def test_delete_model_unauthenticated(
|
|
117
|
-
with
|
|
118
|
-
|
|
121
|
+
def test_delete_model_unauthenticated(unauthenticated_client, regression_model: RegressionModel):
|
|
122
|
+
with unauthenticated_client.use():
|
|
123
|
+
with pytest.raises(ValueError, match="Invalid API key"):
|
|
124
|
+
RegressionModel.drop(regression_model.name)
|
|
119
125
|
|
|
120
126
|
|
|
121
127
|
def test_delete_model_not_found():
|
|
@@ -125,9 +131,10 @@ def test_delete_model_not_found():
|
|
|
125
131
|
RegressionModel.drop(str(uuid4()), if_not_exists="ignore")
|
|
126
132
|
|
|
127
133
|
|
|
128
|
-
def test_delete_model_unauthorized(
|
|
129
|
-
with
|
|
130
|
-
|
|
134
|
+
def test_delete_model_unauthorized(unauthorized_client, regression_model: RegressionModel):
|
|
135
|
+
with unauthorized_client.use():
|
|
136
|
+
with pytest.raises(LookupError):
|
|
137
|
+
RegressionModel.drop(regression_model.name)
|
|
131
138
|
|
|
132
139
|
|
|
133
140
|
def test_delete_memoryset_before_model_constraint_violation(hf_dataset):
|
|
@@ -204,14 +211,16 @@ def test_regression_prediction_has_no_score(regression_model: RegressionModel):
|
|
|
204
211
|
assert prediction.score is None
|
|
205
212
|
|
|
206
213
|
|
|
207
|
-
def test_predict_unauthenticated(
|
|
208
|
-
with
|
|
209
|
-
|
|
214
|
+
def test_predict_unauthenticated(unauthenticated_client, regression_model: RegressionModel):
|
|
215
|
+
with unauthenticated_client.use():
|
|
216
|
+
with pytest.raises(ValueError, match="Invalid API key"):
|
|
217
|
+
regression_model.predict(["This is excellent!", "This is terrible!"])
|
|
210
218
|
|
|
211
219
|
|
|
212
|
-
def test_predict_unauthorized(
|
|
213
|
-
with
|
|
214
|
-
|
|
220
|
+
def test_predict_unauthorized(unauthorized_client, regression_model: RegressionModel):
|
|
221
|
+
with unauthorized_client.use():
|
|
222
|
+
with pytest.raises(LookupError):
|
|
223
|
+
regression_model.predict(["This is excellent!", "This is terrible!"])
|
|
215
224
|
|
|
216
225
|
|
|
217
226
|
def test_predict_constraint_violation(scored_memoryset: ScoredMemoryset):
|
orca_sdk/telemetry.py
CHANGED
|
@@ -4,28 +4,28 @@ import logging
|
|
|
4
4
|
import os
|
|
5
5
|
from abc import ABC
|
|
6
6
|
from datetime import datetime
|
|
7
|
-
from typing import TYPE_CHECKING, Any, Iterable, Literal, Self, overload
|
|
7
|
+
from typing import TYPE_CHECKING, Any, Iterable, Literal, Self, cast, overload
|
|
8
8
|
|
|
9
9
|
from httpx import Timeout
|
|
10
10
|
|
|
11
11
|
from ._utils.common import UNSET
|
|
12
12
|
from .client import (
|
|
13
13
|
LabelPredictionWithMemoriesAndFeedback,
|
|
14
|
+
OrcaClient,
|
|
14
15
|
PredictionFeedbackCategory,
|
|
15
16
|
PredictionFeedbackRequest,
|
|
16
17
|
ScorePredictionWithMemoriesAndFeedback,
|
|
17
18
|
UpdatePredictionRequest,
|
|
18
|
-
orca_api,
|
|
19
|
-
)
|
|
20
|
-
from .memoryset import (
|
|
21
|
-
LabeledMemoryLookup,
|
|
22
|
-
LabeledMemoryset,
|
|
23
|
-
ScoredMemoryLookup,
|
|
24
|
-
ScoredMemoryset,
|
|
25
19
|
)
|
|
26
20
|
|
|
27
21
|
if TYPE_CHECKING:
|
|
28
22
|
from .classification_model import ClassificationModel
|
|
23
|
+
from .memoryset import (
|
|
24
|
+
LabeledMemoryLookup,
|
|
25
|
+
LabeledMemoryset,
|
|
26
|
+
ScoredMemoryLookup,
|
|
27
|
+
ScoredMemoryset,
|
|
28
|
+
)
|
|
29
29
|
from .regression_model import RegressionModel
|
|
30
30
|
|
|
31
31
|
TelemetryMode = Literal["off", "on", "sync", "async"]
|
|
@@ -98,7 +98,8 @@ class FeedbackCategory:
|
|
|
98
98
|
Returns:
|
|
99
99
|
List with information about all existing feedback categories.
|
|
100
100
|
"""
|
|
101
|
-
|
|
101
|
+
client = OrcaClient._resolve_client()
|
|
102
|
+
return [FeedbackCategory(category) for category in client.GET("/telemetry/feedback_category")]
|
|
102
103
|
|
|
103
104
|
@classmethod
|
|
104
105
|
def drop(cls, name: str) -> None:
|
|
@@ -115,7 +116,8 @@ class FeedbackCategory:
|
|
|
115
116
|
Raises:
|
|
116
117
|
LookupError: If the category is not found.
|
|
117
118
|
"""
|
|
118
|
-
|
|
119
|
+
client = OrcaClient._resolve_client()
|
|
120
|
+
client.DELETE("/telemetry/feedback_category/{name_or_id}", params={"name_or_id": name})
|
|
119
121
|
logging.info(f"Deleted feedback category {name} with all associated feedback")
|
|
120
122
|
|
|
121
123
|
def __repr__(self):
|
|
@@ -145,6 +147,8 @@ class AddMemorySuggestions:
|
|
|
145
147
|
)
|
|
146
148
|
|
|
147
149
|
def apply(self) -> None:
|
|
150
|
+
from .memoryset import LabeledMemoryset
|
|
151
|
+
|
|
148
152
|
memoryset = LabeledMemoryset.open(self.memoryset_id)
|
|
149
153
|
label_name_to_label = {label_name: label for label, label_name in enumerate(memoryset.label_names)}
|
|
150
154
|
memoryset.insert(
|
|
@@ -190,7 +194,8 @@ class PredictionBase(ABC):
|
|
|
190
194
|
if self.__telemetry is None:
|
|
191
195
|
if self.prediction_id is None:
|
|
192
196
|
raise ValueError("Cannot fetch telemetry with no prediction ID")
|
|
193
|
-
|
|
197
|
+
client = OrcaClient._resolve_client()
|
|
198
|
+
self.__telemetry = client.GET(
|
|
194
199
|
"/telemetry/prediction/{prediction_id}", params={"prediction_id": self.prediction_id}
|
|
195
200
|
)
|
|
196
201
|
return self.__telemetry
|
|
@@ -204,6 +209,8 @@ class PredictionBase(ABC):
|
|
|
204
209
|
|
|
205
210
|
@property
|
|
206
211
|
def memory_lookups(self) -> list[LabeledMemoryLookup] | list[ScoredMemoryLookup]:
|
|
212
|
+
from .memoryset import LabeledMemoryLookup, ScoredMemoryLookup
|
|
213
|
+
|
|
207
214
|
if "label" in self._telemetry:
|
|
208
215
|
return [
|
|
209
216
|
LabeledMemoryLookup(self._telemetry["memoryset_id"], lookup) for lookup in self._telemetry["memories"]
|
|
@@ -215,12 +222,42 @@ class PredictionBase(ABC):
|
|
|
215
222
|
|
|
216
223
|
@property
|
|
217
224
|
def feedback(self) -> dict[str, bool | float]:
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
225
|
+
feedbacks = self._telemetry.get("feedbacks", [])
|
|
226
|
+
if not feedbacks:
|
|
227
|
+
return {}
|
|
228
|
+
|
|
229
|
+
feedback_by_category: dict[str, bool | float] = {}
|
|
230
|
+
seen_categories: set[str] = set()
|
|
231
|
+
total_categories = len(set(f["category_name"] for f in feedbacks))
|
|
232
|
+
|
|
233
|
+
for f in feedbacks:
|
|
234
|
+
category_name = f["category_name"]
|
|
235
|
+
if category_name not in seen_categories:
|
|
236
|
+
# Convert BINARY (1/0) to boolean, CONTINUOUS to float
|
|
237
|
+
value = f["value"]
|
|
238
|
+
if f["category_type"] == "BINARY":
|
|
239
|
+
value = bool(value)
|
|
240
|
+
else:
|
|
241
|
+
value = float(value)
|
|
242
|
+
feedback_by_category[category_name] = value
|
|
243
|
+
seen_categories.add(category_name)
|
|
244
|
+
|
|
245
|
+
# Early exit once we've found the most recent value for all categories
|
|
246
|
+
if len(seen_categories) == total_categories:
|
|
247
|
+
break
|
|
248
|
+
|
|
249
|
+
return feedback_by_category
|
|
250
|
+
|
|
251
|
+
@property
|
|
252
|
+
def is_correct(self) -> bool:
|
|
253
|
+
if "label" in self._telemetry:
|
|
254
|
+
expected_label = self._telemetry.get("expected_label")
|
|
255
|
+
label = self._telemetry.get("label")
|
|
256
|
+
return expected_label is not None and label is not None and label == expected_label
|
|
257
|
+
else:
|
|
258
|
+
expected_score = self._telemetry.get("expected_score")
|
|
259
|
+
score = self._telemetry.get("score")
|
|
260
|
+
return expected_score is not None and score is not None and abs(score - expected_score) < 0.001
|
|
224
261
|
|
|
225
262
|
@property
|
|
226
263
|
def tags(self) -> set[str]:
|
|
@@ -229,7 +266,8 @@ class PredictionBase(ABC):
|
|
|
229
266
|
@property
|
|
230
267
|
def explanation(self) -> str:
|
|
231
268
|
if self._telemetry["explanation"] is None:
|
|
232
|
-
|
|
269
|
+
client = OrcaClient._resolve_client()
|
|
270
|
+
self._telemetry["explanation"] = client.GET(
|
|
233
271
|
"/telemetry/prediction/{prediction_id}/explanation",
|
|
234
272
|
params={"prediction_id": self._telemetry["prediction_id"]},
|
|
235
273
|
parse_as="text",
|
|
@@ -247,7 +285,8 @@ class PredictionBase(ABC):
|
|
|
247
285
|
if not refresh and self._telemetry["explanation"] is not None:
|
|
248
286
|
print(self._telemetry["explanation"])
|
|
249
287
|
else:
|
|
250
|
-
|
|
288
|
+
client = OrcaClient._resolve_client()
|
|
289
|
+
with client.stream(
|
|
251
290
|
"GET",
|
|
252
291
|
f"/telemetry/prediction/{self.prediction_id}/explanation?refresh={refresh}",
|
|
253
292
|
timeout=Timeout(connect=3, read=None),
|
|
@@ -321,6 +360,7 @@ class PredictionBase(ABC):
|
|
|
321
360
|
def create_prediction(
|
|
322
361
|
prediction: LabelPredictionWithMemoriesAndFeedback | ScorePredictionWithMemoriesAndFeedback,
|
|
323
362
|
) -> Self:
|
|
363
|
+
from .memoryset import LabeledMemoryset, ScoredMemoryset
|
|
324
364
|
|
|
325
365
|
if "label" in prediction:
|
|
326
366
|
memoryset = LabeledMemoryset.open(prediction["memoryset_id"])
|
|
@@ -341,14 +381,15 @@ class PredictionBase(ABC):
|
|
|
341
381
|
telemetry=prediction,
|
|
342
382
|
)
|
|
343
383
|
|
|
384
|
+
client = OrcaClient._resolve_client()
|
|
344
385
|
if isinstance(prediction_id, str):
|
|
345
386
|
return create_prediction(
|
|
346
|
-
|
|
387
|
+
client.GET("/telemetry/prediction/{prediction_id}", params={"prediction_id": prediction_id})
|
|
347
388
|
)
|
|
348
389
|
else:
|
|
349
390
|
return [
|
|
350
391
|
create_prediction(prediction)
|
|
351
|
-
for prediction in
|
|
392
|
+
for prediction in client.POST("/telemetry/prediction", json={"prediction_ids": list(prediction_id)})
|
|
352
393
|
]
|
|
353
394
|
|
|
354
395
|
def refresh(self):
|
|
@@ -374,7 +415,8 @@ class PredictionBase(ABC):
|
|
|
374
415
|
payload["expected_label"] = expected_label
|
|
375
416
|
if expected_score is not UNSET:
|
|
376
417
|
payload["expected_score"] = expected_score
|
|
377
|
-
|
|
418
|
+
client = OrcaClient._resolve_client()
|
|
419
|
+
client.PATCH(
|
|
378
420
|
"/telemetry/prediction/{prediction_id}", params={"prediction_id": self.prediction_id}, json=payload
|
|
379
421
|
)
|
|
380
422
|
self.refresh()
|
|
@@ -431,7 +473,8 @@ class PredictionBase(ABC):
|
|
|
431
473
|
ValueError: If the value does not match previous value types for the category, or is a
|
|
432
474
|
[`float`][float] that is not between `-1.0` and `+1.0`.
|
|
433
475
|
"""
|
|
434
|
-
|
|
476
|
+
client = OrcaClient._resolve_client()
|
|
477
|
+
client.PUT(
|
|
435
478
|
"/telemetry/prediction/feedback",
|
|
436
479
|
json=[
|
|
437
480
|
_parse_feedback(
|
|
@@ -454,7 +497,8 @@ class PredictionBase(ABC):
|
|
|
454
497
|
if self.prediction_id is None:
|
|
455
498
|
raise ValueError("Cannot delete feedback with no prediction ID")
|
|
456
499
|
|
|
457
|
-
|
|
500
|
+
client = OrcaClient._resolve_client()
|
|
501
|
+
client.PUT(
|
|
458
502
|
"/telemetry/prediction/feedback",
|
|
459
503
|
json=[PredictionFeedbackRequest(prediction_id=self.prediction_id, category_name=category, value=None)],
|
|
460
504
|
)
|
|
@@ -511,6 +555,8 @@ class ClassificationPrediction(PredictionBase):
|
|
|
511
555
|
|
|
512
556
|
@property
|
|
513
557
|
def memory_lookups(self) -> list[LabeledMemoryLookup]:
|
|
558
|
+
from .memoryset import LabeledMemoryLookup
|
|
559
|
+
|
|
514
560
|
assert "label" in self._telemetry
|
|
515
561
|
return [LabeledMemoryLookup(self._telemetry["memoryset_id"], lookup) for lookup in self._telemetry["memories"]]
|
|
516
562
|
|
|
@@ -571,7 +617,8 @@ class ClassificationPrediction(PredictionBase):
|
|
|
571
617
|
if self.prediction_id is None:
|
|
572
618
|
raise ValueError("Cannot get action recommendation with no prediction ID")
|
|
573
619
|
|
|
574
|
-
|
|
620
|
+
client = OrcaClient._resolve_client()
|
|
621
|
+
response = client.GET(
|
|
575
622
|
"/telemetry/prediction/{prediction_id}/action",
|
|
576
623
|
params={"prediction_id": self.prediction_id},
|
|
577
624
|
timeout=30,
|
|
@@ -611,7 +658,8 @@ class ClassificationPrediction(PredictionBase):
|
|
|
611
658
|
if self.prediction_id is None:
|
|
612
659
|
raise ValueError("Cannot generate memory suggestions with no prediction ID")
|
|
613
660
|
|
|
614
|
-
|
|
661
|
+
client = OrcaClient._resolve_client()
|
|
662
|
+
response = client.GET(
|
|
615
663
|
"/telemetry/prediction/{prediction_id}/memory_suggestions",
|
|
616
664
|
params={"prediction_id": self.prediction_id, "num_memories": num_memories},
|
|
617
665
|
timeout=30,
|
|
@@ -660,6 +708,8 @@ class RegressionPrediction(PredictionBase):
|
|
|
660
708
|
|
|
661
709
|
@property
|
|
662
710
|
def memory_lookups(self) -> list[ScoredMemoryLookup]:
|
|
711
|
+
from .memoryset import ScoredMemoryLookup
|
|
712
|
+
|
|
663
713
|
assert "score" in self._telemetry
|
|
664
714
|
return [ScoredMemoryLookup(self._telemetry["memoryset_id"], lookup) for lookup in self._telemetry["memories"]]
|
|
665
715
|
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
orca_sdk/__init__.py,sha256=xyjNwkLQXaX8A-UYgGwYDjv2btOXArT_yiMTfmW7KA8,1003
|
|
2
|
+
orca_sdk/_shared/__init__.py,sha256=3Kt0Hu3QLI5FEp9nqGTxqAm3hAoBJKcagfaGQZ-lbJQ,223
|
|
3
|
+
orca_sdk/_shared/metrics.py,sha256=LEZfAUWUtUWv_WWy9F_yjGLlUQHQpmR9WxG2fbKxa7U,14419
|
|
4
|
+
orca_sdk/_shared/metrics_test.py,sha256=Rw1MaH37FppNsMnW8Ir9vMd8xxnZt3eo2Iypx1igtBI,9440
|
|
5
|
+
orca_sdk/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
orca_sdk/_utils/analysis_ui.py,sha256=nT-M_YcNRCVPQzvuqYNFKnNHhYkADYBvq1GlIUePrWw,9232
|
|
7
|
+
orca_sdk/_utils/analysis_ui_style.css,sha256=q_ba_-_KtgztepHg829zLzypaxKayl7ySC1-oYDzV3k,836
|
|
8
|
+
orca_sdk/_utils/auth.py,sha256=nC252O171_3_wn4KBAN7kg8GNvoZFiQ5Xtzkrm5dWDo,2645
|
|
9
|
+
orca_sdk/_utils/auth_test.py,sha256=ygVWv1Ex53LaxIP7p2hzPHl8l9qYyBD5IGmEFJMps6s,1056
|
|
10
|
+
orca_sdk/_utils/common.py,sha256=wUm2pNDWytEecC5WiDWd02-yCZw3Akx0bIutG4lHsFA,805
|
|
11
|
+
orca_sdk/_utils/data_parsing.py,sha256=gkAwWEC8qRt3vRUObe7n7Pr0azOayNwc2yFY04WFp7E,5220
|
|
12
|
+
orca_sdk/_utils/data_parsing_test.py,sha256=fNEYzPzE1jt3KWE2Kj91KqIeuv-L5REHFAa98zkNGSQ,8962
|
|
13
|
+
orca_sdk/_utils/pagination.py,sha256=986z0QPZixrZeurJWorF6eMgnTRdDF84AagEA6qNbMw,4245
|
|
14
|
+
orca_sdk/_utils/pagination_test.py,sha256=BUylCrcHnwoKEBmMUzVr0lwLpA35ivcCwdBK4rMw9y8,4887
|
|
15
|
+
orca_sdk/_utils/prediction_result_ui.css,sha256=sqBlkRLnovb5X5EcUDdB6iGpH63nVRlTW4uAmXuD0WM,258
|
|
16
|
+
orca_sdk/_utils/prediction_result_ui.py,sha256=Ur_FY7dz3oWNmtPiP3Wl3yRlEMgK8q9UfT-SDu9UPxA,4805
|
|
17
|
+
orca_sdk/_utils/tqdm_file_reader.py,sha256=Lw7Cg1UgNuRUoN6jjqZb-IlV00H-kbRcrZLdudr1GxE,324
|
|
18
|
+
orca_sdk/_utils/value_parser.py,sha256=c3qMABCCDQcIjn9N1orYYnlRwDW9JWdGwW_2TDZPLdI,1286
|
|
19
|
+
orca_sdk/_utils/value_parser_test.py,sha256=OybsiC-Obi32RRi9NIuwrVBRAnlyPMV1xVAaevSrb7M,1079
|
|
20
|
+
orca_sdk/async_client.py,sha256=mBd8z5xuHpE8-7Zd0D7YjH2e1OHSO-sRm2tSddqsc9Q,130387
|
|
21
|
+
orca_sdk/classification_model.py,sha256=iYUrGjeYHvPvXwYXjXU_LGL7Dn2XxUcGCt6w93DlJO8,41702
|
|
22
|
+
orca_sdk/classification_model_test.py,sha256=_gaDg8QB0h0ByN4UwTk2fIIDXE4UzahuJBjz7NSPK28,23605
|
|
23
|
+
orca_sdk/client.py,sha256=voNo4NPsc-rsZQ3lZO2fsFuFLw4DC4Dl9REVJQEyKhY,129454
|
|
24
|
+
orca_sdk/conftest.py,sha256=RtINF1xea2iMycMkpMXIOOqRbfWeIZsceSAemhBmgNE,9761
|
|
25
|
+
orca_sdk/credentials.py,sha256=80_1r8n5jruEvN_E629SaRrRhKvF_NhWUEZyZzPXkqQ,6620
|
|
26
|
+
orca_sdk/credentials_test.py,sha256=TLbXJMz3IlThvtSrHeLM7jRsKnrncA_ahOTpHg15Ei4,4089
|
|
27
|
+
orca_sdk/datasource.py,sha256=6QaccghiyFEUSFcqnwjIJzpgIh9Id0snJk2EqViqPsU,22356
|
|
28
|
+
orca_sdk/datasource_test.py,sha256=sCk3IcQJbDut5oN4Wf7PXhTxyMwalxMuCXJekSxy9wk,16665
|
|
29
|
+
orca_sdk/embedding_model.py,sha256=bZhbNJBimWc9Ryklza3q9HS0MRWsiH5Lhn6p7pff0RI,28165
|
|
30
|
+
orca_sdk/embedding_model_test.py,sha256=-NItbNb3tTVj5jAvSi3WjV3FP448q08lmT5iObg9vwA,8133
|
|
31
|
+
orca_sdk/job.py,sha256=wHwVt-s7i-v8udhLGybB-90Kp4dwOLrY806bE4Tam5Q,13092
|
|
32
|
+
orca_sdk/job_test.py,sha256=nRSWxd_1UIfrj9oMVvrXjt6OBkBpddYAjb2y6P-DTUg,4327
|
|
33
|
+
orca_sdk/memoryset.py,sha256=QSnHA2SpAJkGdpVd8wQX2weAhLu9Iw-lfpeQvJxLedg,111690
|
|
34
|
+
orca_sdk/memoryset_test.py,sha256=7wGOtbVa3MEu91fN8DTjiYgB6QIObuA3cTchHmddTIk,33551
|
|
35
|
+
orca_sdk/regression_model.py,sha256=GIL-KgKtGzdb5dFraOKu6OD8yrcavc-CeXASPsKGLGM,28086
|
|
36
|
+
orca_sdk/regression_model_test.py,sha256=slwxbty_vL9d24OCn5xN61eKyri5GS7Jv2YmpEOMTrM,15856
|
|
37
|
+
orca_sdk/telemetry.py,sha256=ZyCMiyyo_SchjadWZH55TlLrC4Ucq5S316NbW26LL4Y,27834
|
|
38
|
+
orca_sdk/telemetry_test.py,sha256=eT66C5lFdNg-pQdo2I__BP7Tn5fTc9aTkVo9ZhWwhU0,5519
|
|
39
|
+
orca_sdk-0.1.4.dist-info/METADATA,sha256=AQBTSp780409HcaGN9ozHCYUCElgNqP30XP8u4fyBiw,3659
|
|
40
|
+
orca_sdk-0.1.4.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
41
|
+
orca_sdk-0.1.4.dist-info/RECORD,,
|