scale-gp-beta 0.1.0a12__py3-none-any.whl → 0.1.0a13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scale_gp_beta/__init__.py +5 -0
- scale_gp_beta/_utils/_proxy.py +4 -1
- scale_gp_beta/_utils/_resources_proxy.py +24 -0
- scale_gp_beta/_version.py +1 -1
- scale_gp_beta/resources/chat/completions.py +136 -2
- scale_gp_beta/resources/dataset_items.py +11 -11
- scale_gp_beta/resources/datasets.py +13 -5
- scale_gp_beta/resources/evaluation_items.py +6 -8
- scale_gp_beta/resources/evaluations.py +138 -21
- scale_gp_beta/resources/files/files.py +5 -5
- scale_gp_beta/resources/models.py +31 -35
- scale_gp_beta/resources/spans.py +42 -22
- scale_gp_beta/types/__init__.py +8 -1
- scale_gp_beta/types/chat/__init__.py +2 -0
- scale_gp_beta/types/chat/completion_models_params.py +29 -0
- scale_gp_beta/types/chat/model_definition.py +32 -0
- scale_gp_beta/types/component.py +18 -0
- scale_gp_beta/types/component_param.py +19 -0
- scale_gp_beta/types/container.py +35 -0
- scale_gp_beta/types/container_param.py +28 -0
- scale_gp_beta/types/dataset_item_list_params.py +4 -5
- scale_gp_beta/types/dataset_item_retrieve_params.py +1 -2
- scale_gp_beta/types/dataset_list_params.py +7 -3
- scale_gp_beta/types/evaluation.py +12 -2
- scale_gp_beta/types/evaluation_create_params.py +5 -5
- scale_gp_beta/types/{evaluation_archive_response.py → evaluation_delete_response.py} +2 -2
- scale_gp_beta/types/evaluation_item_list_params.py +3 -4
- scale_gp_beta/types/evaluation_list_params.py +7 -3
- scale_gp_beta/types/evaluation_task.py +139 -33
- scale_gp_beta/types/evaluation_task_param.py +88 -33
- scale_gp_beta/types/evaluation_update_params.py +17 -0
- scale_gp_beta/types/file_list_params.py +2 -3
- scale_gp_beta/types/inference_model.py +0 -4
- scale_gp_beta/types/item_locator.py +7 -0
- scale_gp_beta/types/item_locator_template.py +7 -0
- scale_gp_beta/types/model_list_params.py +15 -18
- scale_gp_beta/types/span.py +40 -1
- scale_gp_beta/types/span_create_params.py +13 -5
- scale_gp_beta/types/span_list_params.py +4 -5
- scale_gp_beta/types/span_update_params.py +5 -3
- {scale_gp_beta-0.1.0a12.dist-info → scale_gp_beta-0.1.0a13.dist-info}/METADATA +1 -1
- {scale_gp_beta-0.1.0a12.dist-info → scale_gp_beta-0.1.0a13.dist-info}/RECORD +44 -34
- {scale_gp_beta-0.1.0a12.dist-info → scale_gp_beta-0.1.0a13.dist-info}/WHEEL +0 -0
- {scale_gp_beta-0.1.0a12.dist-info → scale_gp_beta-0.1.0a13.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
2
|
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
3
5
|
from typing import TYPE_CHECKING, Dict, List, Union, Optional
|
|
4
6
|
from typing_extensions import Literal, Annotated, TypeAlias
|
|
5
7
|
|
|
6
8
|
from .._utils import PropertyInfo
|
|
9
|
+
from .._compat import PYDANTIC_V2
|
|
7
10
|
from .._models import BaseModel
|
|
11
|
+
from .item_locator import ItemLocator
|
|
8
12
|
|
|
9
13
|
__all__ = [
|
|
10
14
|
"EvaluationTask",
|
|
@@ -30,63 +34,67 @@ __all__ = [
|
|
|
30
34
|
"MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator",
|
|
31
35
|
"MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator",
|
|
32
36
|
"MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator",
|
|
37
|
+
"AutoEvaluationQuestionTask",
|
|
38
|
+
"AutoEvaluationQuestionTaskConfiguration",
|
|
39
|
+
"ContributorEvaluationQuestionTask",
|
|
40
|
+
"ContributorEvaluationQuestionTaskConfiguration",
|
|
33
41
|
]
|
|
34
42
|
|
|
35
43
|
|
|
36
44
|
class ChatCompletionEvaluationTaskConfiguration(BaseModel):
|
|
37
|
-
messages: Union[List[Dict[str, object]],
|
|
45
|
+
messages: Union[List[Dict[str, object]], ItemLocator]
|
|
38
46
|
|
|
39
47
|
model: str
|
|
40
48
|
|
|
41
|
-
audio: Union[Dict[str, object],
|
|
49
|
+
audio: Union[Dict[str, object], ItemLocator, None] = None
|
|
42
50
|
|
|
43
|
-
frequency_penalty: Union[float,
|
|
51
|
+
frequency_penalty: Union[float, ItemLocator, None] = None
|
|
44
52
|
|
|
45
|
-
function_call: Union[Dict[str, object],
|
|
53
|
+
function_call: Union[Dict[str, object], ItemLocator, None] = None
|
|
46
54
|
|
|
47
|
-
functions: Union[List[Dict[str, object]],
|
|
55
|
+
functions: Union[List[Dict[str, object]], ItemLocator, None] = None
|
|
48
56
|
|
|
49
|
-
logit_bias: Union[Dict[str, int],
|
|
57
|
+
logit_bias: Union[Dict[str, int], ItemLocator, None] = None
|
|
50
58
|
|
|
51
|
-
logprobs: Union[bool,
|
|
59
|
+
logprobs: Union[bool, ItemLocator, None] = None
|
|
52
60
|
|
|
53
|
-
max_completion_tokens: Union[int,
|
|
61
|
+
max_completion_tokens: Union[int, ItemLocator, None] = None
|
|
54
62
|
|
|
55
|
-
max_tokens: Union[int,
|
|
63
|
+
max_tokens: Union[int, ItemLocator, None] = None
|
|
56
64
|
|
|
57
|
-
metadata: Union[Dict[str, str],
|
|
65
|
+
metadata: Union[Dict[str, str], ItemLocator, None] = None
|
|
58
66
|
|
|
59
|
-
modalities: Union[List[str],
|
|
67
|
+
modalities: Union[List[str], ItemLocator, None] = None
|
|
60
68
|
|
|
61
|
-
n: Union[int,
|
|
69
|
+
n: Union[int, ItemLocator, None] = None
|
|
62
70
|
|
|
63
|
-
parallel_tool_calls: Union[bool,
|
|
71
|
+
parallel_tool_calls: Union[bool, ItemLocator, None] = None
|
|
64
72
|
|
|
65
|
-
prediction: Union[Dict[str, object],
|
|
73
|
+
prediction: Union[Dict[str, object], ItemLocator, None] = None
|
|
66
74
|
|
|
67
|
-
presence_penalty: Union[float,
|
|
75
|
+
presence_penalty: Union[float, ItemLocator, None] = None
|
|
68
76
|
|
|
69
77
|
reasoning_effort: Optional[str] = None
|
|
70
78
|
|
|
71
|
-
response_format: Union[Dict[str, object],
|
|
79
|
+
response_format: Union[Dict[str, object], ItemLocator, None] = None
|
|
72
80
|
|
|
73
|
-
seed: Union[int,
|
|
81
|
+
seed: Union[int, ItemLocator, None] = None
|
|
74
82
|
|
|
75
83
|
stop: Optional[str] = None
|
|
76
84
|
|
|
77
|
-
store: Union[bool,
|
|
85
|
+
store: Union[bool, ItemLocator, None] = None
|
|
78
86
|
|
|
79
|
-
temperature: Union[float,
|
|
87
|
+
temperature: Union[float, ItemLocator, None] = None
|
|
80
88
|
|
|
81
89
|
tool_choice: Optional[str] = None
|
|
82
90
|
|
|
83
|
-
tools: Union[List[Dict[str, object]],
|
|
91
|
+
tools: Union[List[Dict[str, object]], ItemLocator, None] = None
|
|
84
92
|
|
|
85
|
-
top_k: Union[int,
|
|
93
|
+
top_k: Union[int, ItemLocator, None] = None
|
|
86
94
|
|
|
87
|
-
top_logprobs: Union[int,
|
|
95
|
+
top_logprobs: Union[int, ItemLocator, None] = None
|
|
88
96
|
|
|
89
|
-
top_p: Union[float,
|
|
97
|
+
top_p: Union[float, ItemLocator, None] = None
|
|
90
98
|
|
|
91
99
|
if TYPE_CHECKING:
|
|
92
100
|
# Stub to indicate that arbitrary properties are accepted.
|
|
@@ -99,7 +107,7 @@ class ChatCompletionEvaluationTask(BaseModel):
|
|
|
99
107
|
configuration: ChatCompletionEvaluationTaskConfiguration
|
|
100
108
|
|
|
101
109
|
alias: Optional[str] = None
|
|
102
|
-
"""Alias to title the results column. Defaults to the `
|
|
110
|
+
"""Alias to title the results column. Defaults to the `chat_completion`"""
|
|
103
111
|
|
|
104
112
|
task_type: Optional[Literal["chat_completion"]] = None
|
|
105
113
|
|
|
@@ -111,14 +119,14 @@ class GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInf
|
|
|
111
119
|
|
|
112
120
|
|
|
113
121
|
GenericInferenceEvaluationTaskConfigurationInferenceConfiguration: TypeAlias = Union[
|
|
114
|
-
GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration,
|
|
122
|
+
GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, ItemLocator
|
|
115
123
|
]
|
|
116
124
|
|
|
117
125
|
|
|
118
126
|
class GenericInferenceEvaluationTaskConfiguration(BaseModel):
|
|
119
127
|
model: str
|
|
120
128
|
|
|
121
|
-
args: Union[Dict[str, object],
|
|
129
|
+
args: Union[Dict[str, object], ItemLocator, None] = None
|
|
122
130
|
|
|
123
131
|
inference_configuration: Optional[GenericInferenceEvaluationTaskConfigurationInferenceConfiguration] = None
|
|
124
132
|
|
|
@@ -127,7 +135,7 @@ class GenericInferenceEvaluationTask(BaseModel):
|
|
|
127
135
|
configuration: GenericInferenceEvaluationTaskConfiguration
|
|
128
136
|
|
|
129
137
|
alias: Optional[str] = None
|
|
130
|
-
"""Alias to title the results column. Defaults to the `
|
|
138
|
+
"""Alias to title the results column. Defaults to the `inference`"""
|
|
131
139
|
|
|
132
140
|
task_type: Optional[Literal["inference"]] = None
|
|
133
141
|
|
|
@@ -178,22 +186,24 @@ class ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplication
|
|
|
178
186
|
List[ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace]
|
|
179
187
|
] = None
|
|
180
188
|
|
|
189
|
+
return_span: Optional[bool] = None
|
|
190
|
+
|
|
181
191
|
use_channels: Optional[bool] = None
|
|
182
192
|
|
|
183
193
|
|
|
184
194
|
ApplicationVariantV1EvaluationTaskConfigurationOverrides: TypeAlias = Union[
|
|
185
|
-
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides,
|
|
195
|
+
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, ItemLocator
|
|
186
196
|
]
|
|
187
197
|
|
|
188
198
|
|
|
189
199
|
class ApplicationVariantV1EvaluationTaskConfiguration(BaseModel):
|
|
190
200
|
application_variant_id: str
|
|
191
201
|
|
|
192
|
-
inputs: Union[Dict[str, object],
|
|
202
|
+
inputs: Union[Dict[str, object], ItemLocator]
|
|
193
203
|
|
|
194
|
-
history: Union[List[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0],
|
|
204
|
+
history: Union[List[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], ItemLocator, None] = None
|
|
195
205
|
|
|
196
|
-
operation_metadata: Union[Dict[str, object],
|
|
206
|
+
operation_metadata: Union[Dict[str, object], ItemLocator, None] = None
|
|
197
207
|
|
|
198
208
|
overrides: Optional[ApplicationVariantV1EvaluationTaskConfigurationOverrides] = None
|
|
199
209
|
"""Execution override options for agentic applications"""
|
|
@@ -203,7 +213,7 @@ class ApplicationVariantV1EvaluationTask(BaseModel):
|
|
|
203
213
|
configuration: ApplicationVariantV1EvaluationTaskConfiguration
|
|
204
214
|
|
|
205
215
|
alias: Optional[str] = None
|
|
206
|
-
"""Alias to title the results column. Defaults to the `
|
|
216
|
+
"""Alias to title the results column. Defaults to the `application_variant`"""
|
|
207
217
|
|
|
208
218
|
task_type: Optional[Literal["application_variant"]] = None
|
|
209
219
|
|
|
@@ -282,17 +292,113 @@ class MetricEvaluationTask(BaseModel):
|
|
|
282
292
|
configuration: MetricEvaluationTaskConfiguration
|
|
283
293
|
|
|
284
294
|
alias: Optional[str] = None
|
|
285
|
-
"""Alias to title the results column.
|
|
295
|
+
"""Alias to title the results column.
|
|
296
|
+
|
|
297
|
+
Defaults to the metric type specified in the configuration
|
|
298
|
+
"""
|
|
286
299
|
|
|
287
300
|
task_type: Optional[Literal["metric"]] = None
|
|
288
301
|
|
|
289
302
|
|
|
303
|
+
class AutoEvaluationQuestionTaskConfiguration(BaseModel):
|
|
304
|
+
model: str
|
|
305
|
+
"""model specified as `model_vendor/model_name`"""
|
|
306
|
+
|
|
307
|
+
prompt: str
|
|
308
|
+
|
|
309
|
+
question_id: str
|
|
310
|
+
"""question to be evaluated"""
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
class AutoEvaluationQuestionTask(BaseModel):
|
|
314
|
+
configuration: AutoEvaluationQuestionTaskConfiguration
|
|
315
|
+
|
|
316
|
+
alias: Optional[str] = None
|
|
317
|
+
"""Alias to title the results column. Defaults to the `auto_evaluation_question`"""
|
|
318
|
+
|
|
319
|
+
task_type: Optional[Literal["auto_evaluation.question"]] = None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
class ContributorEvaluationQuestionTaskConfiguration(BaseModel):
|
|
323
|
+
layout: "Container"
|
|
324
|
+
|
|
325
|
+
question_id: str
|
|
326
|
+
|
|
327
|
+
queue_id: Optional[str] = None
|
|
328
|
+
"""The contributor annotation queue to include this task in. Defaults to `default`"""
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
class ContributorEvaluationQuestionTask(BaseModel):
|
|
332
|
+
configuration: ContributorEvaluationQuestionTaskConfiguration
|
|
333
|
+
|
|
334
|
+
alias: Optional[str] = None
|
|
335
|
+
"""Alias to title the results column.
|
|
336
|
+
|
|
337
|
+
Defaults to the `contributor_evaluation_question`
|
|
338
|
+
"""
|
|
339
|
+
|
|
340
|
+
task_type: Optional[Literal["contributor_evaluation.question"]] = None
|
|
341
|
+
|
|
342
|
+
|
|
290
343
|
EvaluationTask: TypeAlias = Annotated[
|
|
291
344
|
Union[
|
|
292
345
|
ChatCompletionEvaluationTask,
|
|
293
346
|
GenericInferenceEvaluationTask,
|
|
294
347
|
ApplicationVariantV1EvaluationTask,
|
|
295
348
|
MetricEvaluationTask,
|
|
349
|
+
AutoEvaluationQuestionTask,
|
|
350
|
+
ContributorEvaluationQuestionTask,
|
|
296
351
|
],
|
|
297
352
|
PropertyInfo(discriminator="task_type"),
|
|
298
353
|
]
|
|
354
|
+
|
|
355
|
+
from .container import Container
|
|
356
|
+
|
|
357
|
+
if PYDANTIC_V2:
|
|
358
|
+
ChatCompletionEvaluationTask.model_rebuild()
|
|
359
|
+
ChatCompletionEvaluationTaskConfiguration.model_rebuild()
|
|
360
|
+
GenericInferenceEvaluationTask.model_rebuild()
|
|
361
|
+
GenericInferenceEvaluationTaskConfiguration.model_rebuild()
|
|
362
|
+
GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration.model_rebuild()
|
|
363
|
+
ApplicationVariantV1EvaluationTask.model_rebuild()
|
|
364
|
+
ApplicationVariantV1EvaluationTaskConfiguration.model_rebuild()
|
|
365
|
+
ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0.model_rebuild()
|
|
366
|
+
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides.model_rebuild()
|
|
367
|
+
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState.model_rebuild()
|
|
368
|
+
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace.model_rebuild()
|
|
369
|
+
MetricEvaluationTask.model_rebuild()
|
|
370
|
+
MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator.model_rebuild()
|
|
371
|
+
MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator.model_rebuild()
|
|
372
|
+
MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator.model_rebuild()
|
|
373
|
+
MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator.model_rebuild()
|
|
374
|
+
MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator.model_rebuild()
|
|
375
|
+
MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator.model_rebuild()
|
|
376
|
+
MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator.model_rebuild()
|
|
377
|
+
AutoEvaluationQuestionTask.model_rebuild()
|
|
378
|
+
AutoEvaluationQuestionTaskConfiguration.model_rebuild()
|
|
379
|
+
ContributorEvaluationQuestionTask.model_rebuild()
|
|
380
|
+
ContributorEvaluationQuestionTaskConfiguration.model_rebuild()
|
|
381
|
+
else:
|
|
382
|
+
ChatCompletionEvaluationTask.update_forward_refs() # type: ignore
|
|
383
|
+
ChatCompletionEvaluationTaskConfiguration.update_forward_refs() # type: ignore
|
|
384
|
+
GenericInferenceEvaluationTask.update_forward_refs() # type: ignore
|
|
385
|
+
GenericInferenceEvaluationTaskConfiguration.update_forward_refs() # type: ignore
|
|
386
|
+
GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration.update_forward_refs() # type: ignore
|
|
387
|
+
ApplicationVariantV1EvaluationTask.update_forward_refs() # type: ignore
|
|
388
|
+
ApplicationVariantV1EvaluationTaskConfiguration.update_forward_refs() # type: ignore
|
|
389
|
+
ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0.update_forward_refs() # type: ignore
|
|
390
|
+
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides.update_forward_refs() # type: ignore
|
|
391
|
+
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState.update_forward_refs() # type: ignore
|
|
392
|
+
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace.update_forward_refs() # type: ignore
|
|
393
|
+
MetricEvaluationTask.update_forward_refs() # type: ignore
|
|
394
|
+
MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator.update_forward_refs() # type: ignore
|
|
395
|
+
MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator.update_forward_refs() # type: ignore
|
|
396
|
+
MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator.update_forward_refs() # type: ignore
|
|
397
|
+
MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator.update_forward_refs() # type: ignore
|
|
398
|
+
MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator.update_forward_refs() # type: ignore
|
|
399
|
+
MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator.update_forward_refs() # type: ignore
|
|
400
|
+
MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator.update_forward_refs() # type: ignore
|
|
401
|
+
AutoEvaluationQuestionTask.update_forward_refs() # type: ignore
|
|
402
|
+
AutoEvaluationQuestionTaskConfiguration.update_forward_refs() # type: ignore
|
|
403
|
+
ContributorEvaluationQuestionTask.update_forward_refs() # type: ignore
|
|
404
|
+
ContributorEvaluationQuestionTaskConfiguration.update_forward_refs() # type: ignore
|
|
@@ -5,6 +5,8 @@ from __future__ import annotations
|
|
|
5
5
|
from typing import Dict, List, Union, Iterable
|
|
6
6
|
from typing_extensions import Literal, Required, TypeAlias, TypedDict
|
|
7
7
|
|
|
8
|
+
from .item_locator import ItemLocator
|
|
9
|
+
|
|
8
10
|
__all__ = [
|
|
9
11
|
"EvaluationTaskParam",
|
|
10
12
|
"ChatCompletionEvaluationTask",
|
|
@@ -29,63 +31,67 @@ __all__ = [
|
|
|
29
31
|
"MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator",
|
|
30
32
|
"MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator",
|
|
31
33
|
"MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator",
|
|
34
|
+
"AutoEvaluationQuestionTask",
|
|
35
|
+
"AutoEvaluationQuestionTaskConfiguration",
|
|
36
|
+
"ContributorEvaluationQuestionTask",
|
|
37
|
+
"ContributorEvaluationQuestionTaskConfiguration",
|
|
32
38
|
]
|
|
33
39
|
|
|
34
40
|
|
|
35
41
|
class ChatCompletionEvaluationTaskConfigurationTyped(TypedDict, total=False):
|
|
36
|
-
messages: Required[Union[Iterable[Dict[str, object]],
|
|
42
|
+
messages: Required[Union[Iterable[Dict[str, object]], ItemLocator]]
|
|
37
43
|
|
|
38
44
|
model: Required[str]
|
|
39
45
|
|
|
40
|
-
audio: Union[Dict[str, object],
|
|
46
|
+
audio: Union[Dict[str, object], ItemLocator]
|
|
41
47
|
|
|
42
|
-
frequency_penalty: Union[float,
|
|
48
|
+
frequency_penalty: Union[float, ItemLocator]
|
|
43
49
|
|
|
44
|
-
function_call: Union[Dict[str, object],
|
|
50
|
+
function_call: Union[Dict[str, object], ItemLocator]
|
|
45
51
|
|
|
46
|
-
functions: Union[Iterable[Dict[str, object]],
|
|
52
|
+
functions: Union[Iterable[Dict[str, object]], ItemLocator]
|
|
47
53
|
|
|
48
|
-
logit_bias: Union[Dict[str, int],
|
|
54
|
+
logit_bias: Union[Dict[str, int], ItemLocator]
|
|
49
55
|
|
|
50
|
-
logprobs: Union[bool,
|
|
56
|
+
logprobs: Union[bool, ItemLocator]
|
|
51
57
|
|
|
52
|
-
max_completion_tokens: Union[int,
|
|
58
|
+
max_completion_tokens: Union[int, ItemLocator]
|
|
53
59
|
|
|
54
|
-
max_tokens: Union[int,
|
|
60
|
+
max_tokens: Union[int, ItemLocator]
|
|
55
61
|
|
|
56
|
-
metadata: Union[Dict[str, str],
|
|
62
|
+
metadata: Union[Dict[str, str], ItemLocator]
|
|
57
63
|
|
|
58
|
-
modalities: Union[List[str],
|
|
64
|
+
modalities: Union[List[str], ItemLocator]
|
|
59
65
|
|
|
60
|
-
n: Union[int,
|
|
66
|
+
n: Union[int, ItemLocator]
|
|
61
67
|
|
|
62
|
-
parallel_tool_calls: Union[bool,
|
|
68
|
+
parallel_tool_calls: Union[bool, ItemLocator]
|
|
63
69
|
|
|
64
|
-
prediction: Union[Dict[str, object],
|
|
70
|
+
prediction: Union[Dict[str, object], ItemLocator]
|
|
65
71
|
|
|
66
|
-
presence_penalty: Union[float,
|
|
72
|
+
presence_penalty: Union[float, ItemLocator]
|
|
67
73
|
|
|
68
74
|
reasoning_effort: str
|
|
69
75
|
|
|
70
|
-
response_format: Union[Dict[str, object],
|
|
76
|
+
response_format: Union[Dict[str, object], ItemLocator]
|
|
71
77
|
|
|
72
|
-
seed: Union[int,
|
|
78
|
+
seed: Union[int, ItemLocator]
|
|
73
79
|
|
|
74
80
|
stop: str
|
|
75
81
|
|
|
76
|
-
store: Union[bool,
|
|
82
|
+
store: Union[bool, ItemLocator]
|
|
77
83
|
|
|
78
|
-
temperature: Union[float,
|
|
84
|
+
temperature: Union[float, ItemLocator]
|
|
79
85
|
|
|
80
86
|
tool_choice: str
|
|
81
87
|
|
|
82
|
-
tools: Union[Iterable[Dict[str, object]],
|
|
88
|
+
tools: Union[Iterable[Dict[str, object]], ItemLocator]
|
|
83
89
|
|
|
84
|
-
top_k: Union[int,
|
|
90
|
+
top_k: Union[int, ItemLocator]
|
|
85
91
|
|
|
86
|
-
top_logprobs: Union[int,
|
|
92
|
+
top_logprobs: Union[int, ItemLocator]
|
|
87
93
|
|
|
88
|
-
top_p: Union[float,
|
|
94
|
+
top_p: Union[float, ItemLocator]
|
|
89
95
|
|
|
90
96
|
|
|
91
97
|
ChatCompletionEvaluationTaskConfiguration: TypeAlias = Union[
|
|
@@ -97,7 +103,7 @@ class ChatCompletionEvaluationTask(TypedDict, total=False):
|
|
|
97
103
|
configuration: Required[ChatCompletionEvaluationTaskConfiguration]
|
|
98
104
|
|
|
99
105
|
alias: str
|
|
100
|
-
"""Alias to title the results column. Defaults to the `
|
|
106
|
+
"""Alias to title the results column. Defaults to the `chat_completion`"""
|
|
101
107
|
|
|
102
108
|
task_type: Literal["chat_completion"]
|
|
103
109
|
|
|
@@ -111,14 +117,14 @@ class GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInf
|
|
|
111
117
|
|
|
112
118
|
|
|
113
119
|
GenericInferenceEvaluationTaskConfigurationInferenceConfiguration: TypeAlias = Union[
|
|
114
|
-
GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration,
|
|
120
|
+
GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, ItemLocator
|
|
115
121
|
]
|
|
116
122
|
|
|
117
123
|
|
|
118
124
|
class GenericInferenceEvaluationTaskConfiguration(TypedDict, total=False):
|
|
119
125
|
model: Required[str]
|
|
120
126
|
|
|
121
|
-
args: Union[Dict[str, object],
|
|
127
|
+
args: Union[Dict[str, object], ItemLocator]
|
|
122
128
|
|
|
123
129
|
inference_configuration: GenericInferenceEvaluationTaskConfigurationInferenceConfiguration
|
|
124
130
|
|
|
@@ -127,7 +133,7 @@ class GenericInferenceEvaluationTask(TypedDict, total=False):
|
|
|
127
133
|
configuration: Required[GenericInferenceEvaluationTaskConfiguration]
|
|
128
134
|
|
|
129
135
|
alias: str
|
|
130
|
-
"""Alias to title the results column. Defaults to the `
|
|
136
|
+
"""Alias to title the results column. Defaults to the `inference`"""
|
|
131
137
|
|
|
132
138
|
task_type: Literal["inference"]
|
|
133
139
|
|
|
@@ -180,22 +186,24 @@ class ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplication
|
|
|
180
186
|
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace
|
|
181
187
|
]
|
|
182
188
|
|
|
189
|
+
return_span: bool
|
|
190
|
+
|
|
183
191
|
use_channels: bool
|
|
184
192
|
|
|
185
193
|
|
|
186
194
|
ApplicationVariantV1EvaluationTaskConfigurationOverrides: TypeAlias = Union[
|
|
187
|
-
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides,
|
|
195
|
+
ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, ItemLocator
|
|
188
196
|
]
|
|
189
197
|
|
|
190
198
|
|
|
191
199
|
class ApplicationVariantV1EvaluationTaskConfiguration(TypedDict, total=False):
|
|
192
200
|
application_variant_id: Required[str]
|
|
193
201
|
|
|
194
|
-
inputs: Required[Union[Dict[str, object],
|
|
202
|
+
inputs: Required[Union[Dict[str, object], ItemLocator]]
|
|
195
203
|
|
|
196
|
-
history: Union[Iterable[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0],
|
|
204
|
+
history: Union[Iterable[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], ItemLocator]
|
|
197
205
|
|
|
198
|
-
operation_metadata: Union[Dict[str, object],
|
|
206
|
+
operation_metadata: Union[Dict[str, object], ItemLocator]
|
|
199
207
|
|
|
200
208
|
overrides: ApplicationVariantV1EvaluationTaskConfigurationOverrides
|
|
201
209
|
"""Execution override options for agentic applications"""
|
|
@@ -205,7 +213,7 @@ class ApplicationVariantV1EvaluationTask(TypedDict, total=False):
|
|
|
205
213
|
configuration: Required[ApplicationVariantV1EvaluationTaskConfiguration]
|
|
206
214
|
|
|
207
215
|
alias: str
|
|
208
|
-
"""Alias to title the results column. Defaults to the `
|
|
216
|
+
"""Alias to title the results column. Defaults to the `application_variant`"""
|
|
209
217
|
|
|
210
218
|
task_type: Literal["application_variant"]
|
|
211
219
|
|
|
@@ -281,14 +289,61 @@ class MetricEvaluationTask(TypedDict, total=False):
|
|
|
281
289
|
configuration: Required[MetricEvaluationTaskConfiguration]
|
|
282
290
|
|
|
283
291
|
alias: str
|
|
284
|
-
"""Alias to title the results column.
|
|
292
|
+
"""Alias to title the results column.
|
|
293
|
+
|
|
294
|
+
Defaults to the metric type specified in the configuration
|
|
295
|
+
"""
|
|
285
296
|
|
|
286
297
|
task_type: Literal["metric"]
|
|
287
298
|
|
|
288
299
|
|
|
300
|
+
class AutoEvaluationQuestionTaskConfiguration(TypedDict, total=False):
|
|
301
|
+
model: Required[str]
|
|
302
|
+
"""model specified as `model_vendor/model_name`"""
|
|
303
|
+
|
|
304
|
+
prompt: Required[str]
|
|
305
|
+
|
|
306
|
+
question_id: Required[str]
|
|
307
|
+
"""question to be evaluated"""
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
class AutoEvaluationQuestionTask(TypedDict, total=False):
|
|
311
|
+
configuration: Required[AutoEvaluationQuestionTaskConfiguration]
|
|
312
|
+
|
|
313
|
+
alias: str
|
|
314
|
+
"""Alias to title the results column. Defaults to the `auto_evaluation_question`"""
|
|
315
|
+
|
|
316
|
+
task_type: Literal["auto_evaluation.question"]
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
class ContributorEvaluationQuestionTaskConfiguration(TypedDict, total=False):
|
|
320
|
+
layout: Required["ContainerParam"]
|
|
321
|
+
|
|
322
|
+
question_id: Required[str]
|
|
323
|
+
|
|
324
|
+
queue_id: str
|
|
325
|
+
"""The contributor annotation queue to include this task in. Defaults to `default`"""
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
class ContributorEvaluationQuestionTask(TypedDict, total=False):
|
|
329
|
+
configuration: Required[ContributorEvaluationQuestionTaskConfiguration]
|
|
330
|
+
|
|
331
|
+
alias: str
|
|
332
|
+
"""Alias to title the results column.
|
|
333
|
+
|
|
334
|
+
Defaults to the `contributor_evaluation_question`
|
|
335
|
+
"""
|
|
336
|
+
|
|
337
|
+
task_type: Literal["contributor_evaluation.question"]
|
|
338
|
+
|
|
339
|
+
|
|
289
340
|
EvaluationTaskParam: TypeAlias = Union[
|
|
290
341
|
ChatCompletionEvaluationTask,
|
|
291
342
|
GenericInferenceEvaluationTask,
|
|
292
343
|
ApplicationVariantV1EvaluationTask,
|
|
293
344
|
MetricEvaluationTask,
|
|
345
|
+
AutoEvaluationQuestionTask,
|
|
346
|
+
ContributorEvaluationQuestionTask,
|
|
294
347
|
]
|
|
348
|
+
|
|
349
|
+
from .container_param import ContainerParam
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import List
|
|
6
|
+
from typing_extensions import TypedDict
|
|
7
|
+
|
|
8
|
+
__all__ = ["EvaluationUpdateParams"]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class EvaluationUpdateParams(TypedDict, total=False):
|
|
12
|
+
description: str
|
|
13
|
+
|
|
14
|
+
name: str
|
|
15
|
+
|
|
16
|
+
tags: List[str]
|
|
17
|
+
"""The tags associated with the entity"""
|
|
@@ -2,15 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from typing import Optional
|
|
6
5
|
from typing_extensions import TypedDict
|
|
7
6
|
|
|
8
7
|
__all__ = ["FileListParams"]
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
class FileListParams(TypedDict, total=False):
|
|
12
|
-
ending_before:
|
|
11
|
+
ending_before: str
|
|
13
12
|
|
|
14
13
|
limit: int
|
|
15
14
|
|
|
16
|
-
starting_after:
|
|
15
|
+
starting_after: str
|
|
@@ -170,10 +170,6 @@ class InferenceModel(BaseModel):
|
|
|
170
170
|
|
|
171
171
|
status: Literal["failed", "ready", "deploying"]
|
|
172
172
|
|
|
173
|
-
description: Optional[str] = None
|
|
174
|
-
|
|
175
|
-
display_name: Optional[str] = None
|
|
176
|
-
|
|
177
173
|
api_model_metadata: Optional[Dict[str, object]] = FieldInfo(alias="model_metadata", default=None)
|
|
178
174
|
|
|
179
175
|
object: Optional[Literal["model"]] = None
|
|
@@ -2,33 +2,30 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from typing import Optional
|
|
6
5
|
from typing_extensions import Literal, TypedDict
|
|
7
6
|
|
|
8
7
|
__all__ = ["ModelListParams"]
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
class ModelListParams(TypedDict, total=False):
|
|
12
|
-
ending_before:
|
|
11
|
+
ending_before: str
|
|
13
12
|
|
|
14
13
|
limit: int
|
|
15
14
|
|
|
16
|
-
model_vendor:
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
"xai",
|
|
29
|
-
]
|
|
15
|
+
model_vendor: Literal[
|
|
16
|
+
"openai",
|
|
17
|
+
"cohere",
|
|
18
|
+
"vertex_ai",
|
|
19
|
+
"anthropic",
|
|
20
|
+
"azure",
|
|
21
|
+
"gemini",
|
|
22
|
+
"launch",
|
|
23
|
+
"llmengine",
|
|
24
|
+
"model_zoo",
|
|
25
|
+
"bedrock",
|
|
26
|
+
"xai",
|
|
30
27
|
]
|
|
31
28
|
|
|
32
|
-
name:
|
|
29
|
+
name: str
|
|
33
30
|
|
|
34
|
-
starting_after:
|
|
31
|
+
starting_after: str
|