scale-gp-beta 0.1.0a12__py3-none-any.whl → 0.1.0a13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. scale_gp_beta/__init__.py +5 -0
  2. scale_gp_beta/_utils/_proxy.py +4 -1
  3. scale_gp_beta/_utils/_resources_proxy.py +24 -0
  4. scale_gp_beta/_version.py +1 -1
  5. scale_gp_beta/resources/chat/completions.py +136 -2
  6. scale_gp_beta/resources/dataset_items.py +11 -11
  7. scale_gp_beta/resources/datasets.py +13 -5
  8. scale_gp_beta/resources/evaluation_items.py +6 -8
  9. scale_gp_beta/resources/evaluations.py +138 -21
  10. scale_gp_beta/resources/files/files.py +5 -5
  11. scale_gp_beta/resources/models.py +31 -35
  12. scale_gp_beta/resources/spans.py +42 -22
  13. scale_gp_beta/types/__init__.py +8 -1
  14. scale_gp_beta/types/chat/__init__.py +2 -0
  15. scale_gp_beta/types/chat/completion_models_params.py +29 -0
  16. scale_gp_beta/types/chat/model_definition.py +32 -0
  17. scale_gp_beta/types/component.py +18 -0
  18. scale_gp_beta/types/component_param.py +19 -0
  19. scale_gp_beta/types/container.py +35 -0
  20. scale_gp_beta/types/container_param.py +28 -0
  21. scale_gp_beta/types/dataset_item_list_params.py +4 -5
  22. scale_gp_beta/types/dataset_item_retrieve_params.py +1 -2
  23. scale_gp_beta/types/dataset_list_params.py +7 -3
  24. scale_gp_beta/types/evaluation.py +12 -2
  25. scale_gp_beta/types/evaluation_create_params.py +5 -5
  26. scale_gp_beta/types/{evaluation_archive_response.py → evaluation_delete_response.py} +2 -2
  27. scale_gp_beta/types/evaluation_item_list_params.py +3 -4
  28. scale_gp_beta/types/evaluation_list_params.py +7 -3
  29. scale_gp_beta/types/evaluation_task.py +139 -33
  30. scale_gp_beta/types/evaluation_task_param.py +88 -33
  31. scale_gp_beta/types/evaluation_update_params.py +17 -0
  32. scale_gp_beta/types/file_list_params.py +2 -3
  33. scale_gp_beta/types/inference_model.py +0 -4
  34. scale_gp_beta/types/item_locator.py +7 -0
  35. scale_gp_beta/types/item_locator_template.py +7 -0
  36. scale_gp_beta/types/model_list_params.py +15 -18
  37. scale_gp_beta/types/span.py +40 -1
  38. scale_gp_beta/types/span_create_params.py +13 -5
  39. scale_gp_beta/types/span_list_params.py +4 -5
  40. scale_gp_beta/types/span_update_params.py +5 -3
  41. {scale_gp_beta-0.1.0a12.dist-info → scale_gp_beta-0.1.0a13.dist-info}/METADATA +1 -1
  42. {scale_gp_beta-0.1.0a12.dist-info → scale_gp_beta-0.1.0a13.dist-info}/RECORD +44 -34
  43. {scale_gp_beta-0.1.0a12.dist-info → scale_gp_beta-0.1.0a13.dist-info}/WHEEL +0 -0
  44. {scale_gp_beta-0.1.0a12.dist-info → scale_gp_beta-0.1.0a13.dist-info}/licenses/LICENSE +0 -0
@@ -1,10 +1,14 @@
1
1
  # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
2
 
3
+ from __future__ import annotations
4
+
3
5
  from typing import TYPE_CHECKING, Dict, List, Union, Optional
4
6
  from typing_extensions import Literal, Annotated, TypeAlias
5
7
 
6
8
  from .._utils import PropertyInfo
9
+ from .._compat import PYDANTIC_V2
7
10
  from .._models import BaseModel
11
+ from .item_locator import ItemLocator
8
12
 
9
13
  __all__ = [
10
14
  "EvaluationTask",
@@ -30,63 +34,67 @@ __all__ = [
30
34
  "MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator",
31
35
  "MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator",
32
36
  "MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator",
37
+ "AutoEvaluationQuestionTask",
38
+ "AutoEvaluationQuestionTaskConfiguration",
39
+ "ContributorEvaluationQuestionTask",
40
+ "ContributorEvaluationQuestionTaskConfiguration",
33
41
  ]
34
42
 
35
43
 
36
44
  class ChatCompletionEvaluationTaskConfiguration(BaseModel):
37
- messages: Union[List[Dict[str, object]], str]
45
+ messages: Union[List[Dict[str, object]], ItemLocator]
38
46
 
39
47
  model: str
40
48
 
41
- audio: Union[Dict[str, object], str, None] = None
49
+ audio: Union[Dict[str, object], ItemLocator, None] = None
42
50
 
43
- frequency_penalty: Union[float, str, None] = None
51
+ frequency_penalty: Union[float, ItemLocator, None] = None
44
52
 
45
- function_call: Union[Dict[str, object], str, None] = None
53
+ function_call: Union[Dict[str, object], ItemLocator, None] = None
46
54
 
47
- functions: Union[List[Dict[str, object]], str, None] = None
55
+ functions: Union[List[Dict[str, object]], ItemLocator, None] = None
48
56
 
49
- logit_bias: Union[Dict[str, int], str, None] = None
57
+ logit_bias: Union[Dict[str, int], ItemLocator, None] = None
50
58
 
51
- logprobs: Union[bool, str, None] = None
59
+ logprobs: Union[bool, ItemLocator, None] = None
52
60
 
53
- max_completion_tokens: Union[int, str, None] = None
61
+ max_completion_tokens: Union[int, ItemLocator, None] = None
54
62
 
55
- max_tokens: Union[int, str, None] = None
63
+ max_tokens: Union[int, ItemLocator, None] = None
56
64
 
57
- metadata: Union[Dict[str, str], str, None] = None
65
+ metadata: Union[Dict[str, str], ItemLocator, None] = None
58
66
 
59
- modalities: Union[List[str], str, None] = None
67
+ modalities: Union[List[str], ItemLocator, None] = None
60
68
 
61
- n: Union[int, str, None] = None
69
+ n: Union[int, ItemLocator, None] = None
62
70
 
63
- parallel_tool_calls: Union[bool, str, None] = None
71
+ parallel_tool_calls: Union[bool, ItemLocator, None] = None
64
72
 
65
- prediction: Union[Dict[str, object], str, None] = None
73
+ prediction: Union[Dict[str, object], ItemLocator, None] = None
66
74
 
67
- presence_penalty: Union[float, str, None] = None
75
+ presence_penalty: Union[float, ItemLocator, None] = None
68
76
 
69
77
  reasoning_effort: Optional[str] = None
70
78
 
71
- response_format: Union[Dict[str, object], str, None] = None
79
+ response_format: Union[Dict[str, object], ItemLocator, None] = None
72
80
 
73
- seed: Union[int, str, None] = None
81
+ seed: Union[int, ItemLocator, None] = None
74
82
 
75
83
  stop: Optional[str] = None
76
84
 
77
- store: Union[bool, str, None] = None
85
+ store: Union[bool, ItemLocator, None] = None
78
86
 
79
- temperature: Union[float, str, None] = None
87
+ temperature: Union[float, ItemLocator, None] = None
80
88
 
81
89
  tool_choice: Optional[str] = None
82
90
 
83
- tools: Union[List[Dict[str, object]], str, None] = None
91
+ tools: Union[List[Dict[str, object]], ItemLocator, None] = None
84
92
 
85
- top_k: Union[int, str, None] = None
93
+ top_k: Union[int, ItemLocator, None] = None
86
94
 
87
- top_logprobs: Union[int, str, None] = None
95
+ top_logprobs: Union[int, ItemLocator, None] = None
88
96
 
89
- top_p: Union[float, str, None] = None
97
+ top_p: Union[float, ItemLocator, None] = None
90
98
 
91
99
  if TYPE_CHECKING:
92
100
  # Stub to indicate that arbitrary properties are accepted.
@@ -99,7 +107,7 @@ class ChatCompletionEvaluationTask(BaseModel):
99
107
  configuration: ChatCompletionEvaluationTaskConfiguration
100
108
 
101
109
  alias: Optional[str] = None
102
- """Alias to title the results column. Defaults to the `task_type`"""
110
+ """Alias to title the results column. Defaults to the `chat_completion`"""
103
111
 
104
112
  task_type: Optional[Literal["chat_completion"]] = None
105
113
 
@@ -111,14 +119,14 @@ class GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInf
111
119
 
112
120
 
113
121
  GenericInferenceEvaluationTaskConfigurationInferenceConfiguration: TypeAlias = Union[
114
- GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, str
122
+ GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, ItemLocator
115
123
  ]
116
124
 
117
125
 
118
126
  class GenericInferenceEvaluationTaskConfiguration(BaseModel):
119
127
  model: str
120
128
 
121
- args: Union[Dict[str, object], str, None] = None
129
+ args: Union[Dict[str, object], ItemLocator, None] = None
122
130
 
123
131
  inference_configuration: Optional[GenericInferenceEvaluationTaskConfigurationInferenceConfiguration] = None
124
132
 
@@ -127,7 +135,7 @@ class GenericInferenceEvaluationTask(BaseModel):
127
135
  configuration: GenericInferenceEvaluationTaskConfiguration
128
136
 
129
137
  alias: Optional[str] = None
130
- """Alias to title the results column. Defaults to the `task_type`"""
138
+ """Alias to title the results column. Defaults to the `inference`"""
131
139
 
132
140
  task_type: Optional[Literal["inference"]] = None
133
141
 
@@ -178,22 +186,24 @@ class ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplication
178
186
  List[ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace]
179
187
  ] = None
180
188
 
189
+ return_span: Optional[bool] = None
190
+
181
191
  use_channels: Optional[bool] = None
182
192
 
183
193
 
184
194
  ApplicationVariantV1EvaluationTaskConfigurationOverrides: TypeAlias = Union[
185
- ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, str
195
+ ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, ItemLocator
186
196
  ]
187
197
 
188
198
 
189
199
  class ApplicationVariantV1EvaluationTaskConfiguration(BaseModel):
190
200
  application_variant_id: str
191
201
 
192
- inputs: Union[Dict[str, object], str]
202
+ inputs: Union[Dict[str, object], ItemLocator]
193
203
 
194
- history: Union[List[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], str, None] = None
204
+ history: Union[List[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], ItemLocator, None] = None
195
205
 
196
- operation_metadata: Union[Dict[str, object], str, None] = None
206
+ operation_metadata: Union[Dict[str, object], ItemLocator, None] = None
197
207
 
198
208
  overrides: Optional[ApplicationVariantV1EvaluationTaskConfigurationOverrides] = None
199
209
  """Execution override options for agentic applications"""
@@ -203,7 +213,7 @@ class ApplicationVariantV1EvaluationTask(BaseModel):
203
213
  configuration: ApplicationVariantV1EvaluationTaskConfiguration
204
214
 
205
215
  alias: Optional[str] = None
206
- """Alias to title the results column. Defaults to the `task_type`"""
216
+ """Alias to title the results column. Defaults to the `application_variant`"""
207
217
 
208
218
  task_type: Optional[Literal["application_variant"]] = None
209
219
 
@@ -282,17 +292,113 @@ class MetricEvaluationTask(BaseModel):
282
292
  configuration: MetricEvaluationTaskConfiguration
283
293
 
284
294
  alias: Optional[str] = None
285
- """Alias to title the results column. Defaults to the `task_type`"""
295
+ """Alias to title the results column.
296
+
297
+ Defaults to the metric type specified in the configuration
298
+ """
286
299
 
287
300
  task_type: Optional[Literal["metric"]] = None
288
301
 
289
302
 
303
+ class AutoEvaluationQuestionTaskConfiguration(BaseModel):
304
+ model: str
305
+ """model specified as `model_vendor/model_name`"""
306
+
307
+ prompt: str
308
+
309
+ question_id: str
310
+ """question to be evaluated"""
311
+
312
+
313
+ class AutoEvaluationQuestionTask(BaseModel):
314
+ configuration: AutoEvaluationQuestionTaskConfiguration
315
+
316
+ alias: Optional[str] = None
317
+ """Alias to title the results column. Defaults to the `auto_evaluation_question`"""
318
+
319
+ task_type: Optional[Literal["auto_evaluation.question"]] = None
320
+
321
+
322
+ class ContributorEvaluationQuestionTaskConfiguration(BaseModel):
323
+ layout: "Container"
324
+
325
+ question_id: str
326
+
327
+ queue_id: Optional[str] = None
328
+ """The contributor annotation queue to include this task in. Defaults to `default`"""
329
+
330
+
331
+ class ContributorEvaluationQuestionTask(BaseModel):
332
+ configuration: ContributorEvaluationQuestionTaskConfiguration
333
+
334
+ alias: Optional[str] = None
335
+ """Alias to title the results column.
336
+
337
+ Defaults to the `contributor_evaluation_question`
338
+ """
339
+
340
+ task_type: Optional[Literal["contributor_evaluation.question"]] = None
341
+
342
+
290
343
  EvaluationTask: TypeAlias = Annotated[
291
344
  Union[
292
345
  ChatCompletionEvaluationTask,
293
346
  GenericInferenceEvaluationTask,
294
347
  ApplicationVariantV1EvaluationTask,
295
348
  MetricEvaluationTask,
349
+ AutoEvaluationQuestionTask,
350
+ ContributorEvaluationQuestionTask,
296
351
  ],
297
352
  PropertyInfo(discriminator="task_type"),
298
353
  ]
354
+
355
+ from .container import Container
356
+
357
+ if PYDANTIC_V2:
358
+ ChatCompletionEvaluationTask.model_rebuild()
359
+ ChatCompletionEvaluationTaskConfiguration.model_rebuild()
360
+ GenericInferenceEvaluationTask.model_rebuild()
361
+ GenericInferenceEvaluationTaskConfiguration.model_rebuild()
362
+ GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration.model_rebuild()
363
+ ApplicationVariantV1EvaluationTask.model_rebuild()
364
+ ApplicationVariantV1EvaluationTaskConfiguration.model_rebuild()
365
+ ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0.model_rebuild()
366
+ ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides.model_rebuild()
367
+ ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState.model_rebuild()
368
+ ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace.model_rebuild()
369
+ MetricEvaluationTask.model_rebuild()
370
+ MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator.model_rebuild()
371
+ MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator.model_rebuild()
372
+ MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator.model_rebuild()
373
+ MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator.model_rebuild()
374
+ MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator.model_rebuild()
375
+ MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator.model_rebuild()
376
+ MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator.model_rebuild()
377
+ AutoEvaluationQuestionTask.model_rebuild()
378
+ AutoEvaluationQuestionTaskConfiguration.model_rebuild()
379
+ ContributorEvaluationQuestionTask.model_rebuild()
380
+ ContributorEvaluationQuestionTaskConfiguration.model_rebuild()
381
+ else:
382
+ ChatCompletionEvaluationTask.update_forward_refs() # type: ignore
383
+ ChatCompletionEvaluationTaskConfiguration.update_forward_refs() # type: ignore
384
+ GenericInferenceEvaluationTask.update_forward_refs() # type: ignore
385
+ GenericInferenceEvaluationTaskConfiguration.update_forward_refs() # type: ignore
386
+ GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration.update_forward_refs() # type: ignore
387
+ ApplicationVariantV1EvaluationTask.update_forward_refs() # type: ignore
388
+ ApplicationVariantV1EvaluationTaskConfiguration.update_forward_refs() # type: ignore
389
+ ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0.update_forward_refs() # type: ignore
390
+ ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides.update_forward_refs() # type: ignore
391
+ ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState.update_forward_refs() # type: ignore
392
+ ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace.update_forward_refs() # type: ignore
393
+ MetricEvaluationTask.update_forward_refs() # type: ignore
394
+ MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator.update_forward_refs() # type: ignore
395
+ MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator.update_forward_refs() # type: ignore
396
+ MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator.update_forward_refs() # type: ignore
397
+ MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator.update_forward_refs() # type: ignore
398
+ MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator.update_forward_refs() # type: ignore
399
+ MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator.update_forward_refs() # type: ignore
400
+ MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator.update_forward_refs() # type: ignore
401
+ AutoEvaluationQuestionTask.update_forward_refs() # type: ignore
402
+ AutoEvaluationQuestionTaskConfiguration.update_forward_refs() # type: ignore
403
+ ContributorEvaluationQuestionTask.update_forward_refs() # type: ignore
404
+ ContributorEvaluationQuestionTaskConfiguration.update_forward_refs() # type: ignore
@@ -5,6 +5,8 @@ from __future__ import annotations
5
5
  from typing import Dict, List, Union, Iterable
6
6
  from typing_extensions import Literal, Required, TypeAlias, TypedDict
7
7
 
8
+ from .item_locator import ItemLocator
9
+
8
10
  __all__ = [
9
11
  "EvaluationTaskParam",
10
12
  "ChatCompletionEvaluationTask",
@@ -29,63 +31,67 @@ __all__ = [
29
31
  "MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator",
30
32
  "MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator",
31
33
  "MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator",
34
+ "AutoEvaluationQuestionTask",
35
+ "AutoEvaluationQuestionTaskConfiguration",
36
+ "ContributorEvaluationQuestionTask",
37
+ "ContributorEvaluationQuestionTaskConfiguration",
32
38
  ]
33
39
 
34
40
 
35
41
  class ChatCompletionEvaluationTaskConfigurationTyped(TypedDict, total=False):
36
- messages: Required[Union[Iterable[Dict[str, object]], str]]
42
+ messages: Required[Union[Iterable[Dict[str, object]], ItemLocator]]
37
43
 
38
44
  model: Required[str]
39
45
 
40
- audio: Union[Dict[str, object], str]
46
+ audio: Union[Dict[str, object], ItemLocator]
41
47
 
42
- frequency_penalty: Union[float, str]
48
+ frequency_penalty: Union[float, ItemLocator]
43
49
 
44
- function_call: Union[Dict[str, object], str]
50
+ function_call: Union[Dict[str, object], ItemLocator]
45
51
 
46
- functions: Union[Iterable[Dict[str, object]], str]
52
+ functions: Union[Iterable[Dict[str, object]], ItemLocator]
47
53
 
48
- logit_bias: Union[Dict[str, int], str]
54
+ logit_bias: Union[Dict[str, int], ItemLocator]
49
55
 
50
- logprobs: Union[bool, str]
56
+ logprobs: Union[bool, ItemLocator]
51
57
 
52
- max_completion_tokens: Union[int, str]
58
+ max_completion_tokens: Union[int, ItemLocator]
53
59
 
54
- max_tokens: Union[int, str]
60
+ max_tokens: Union[int, ItemLocator]
55
61
 
56
- metadata: Union[Dict[str, str], str]
62
+ metadata: Union[Dict[str, str], ItemLocator]
57
63
 
58
- modalities: Union[List[str], str]
64
+ modalities: Union[List[str], ItemLocator]
59
65
 
60
- n: Union[int, str]
66
+ n: Union[int, ItemLocator]
61
67
 
62
- parallel_tool_calls: Union[bool, str]
68
+ parallel_tool_calls: Union[bool, ItemLocator]
63
69
 
64
- prediction: Union[Dict[str, object], str]
70
+ prediction: Union[Dict[str, object], ItemLocator]
65
71
 
66
- presence_penalty: Union[float, str]
72
+ presence_penalty: Union[float, ItemLocator]
67
73
 
68
74
  reasoning_effort: str
69
75
 
70
- response_format: Union[Dict[str, object], str]
76
+ response_format: Union[Dict[str, object], ItemLocator]
71
77
 
72
- seed: Union[int, str]
78
+ seed: Union[int, ItemLocator]
73
79
 
74
80
  stop: str
75
81
 
76
- store: Union[bool, str]
82
+ store: Union[bool, ItemLocator]
77
83
 
78
- temperature: Union[float, str]
84
+ temperature: Union[float, ItemLocator]
79
85
 
80
86
  tool_choice: str
81
87
 
82
- tools: Union[Iterable[Dict[str, object]], str]
88
+ tools: Union[Iterable[Dict[str, object]], ItemLocator]
83
89
 
84
- top_k: Union[int, str]
90
+ top_k: Union[int, ItemLocator]
85
91
 
86
- top_logprobs: Union[int, str]
92
+ top_logprobs: Union[int, ItemLocator]
87
93
 
88
- top_p: Union[float, str]
94
+ top_p: Union[float, ItemLocator]
89
95
 
90
96
 
91
97
  ChatCompletionEvaluationTaskConfiguration: TypeAlias = Union[
@@ -97,7 +103,7 @@ class ChatCompletionEvaluationTask(TypedDict, total=False):
97
103
  configuration: Required[ChatCompletionEvaluationTaskConfiguration]
98
104
 
99
105
  alias: str
100
- """Alias to title the results column. Defaults to the `task_type`"""
106
+ """Alias to title the results column. Defaults to the `chat_completion`"""
101
107
 
102
108
  task_type: Literal["chat_completion"]
103
109
 
@@ -111,14 +117,14 @@ class GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInf
111
117
 
112
118
 
113
119
  GenericInferenceEvaluationTaskConfigurationInferenceConfiguration: TypeAlias = Union[
114
- GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, str
120
+ GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, ItemLocator
115
121
  ]
116
122
 
117
123
 
118
124
  class GenericInferenceEvaluationTaskConfiguration(TypedDict, total=False):
119
125
  model: Required[str]
120
126
 
121
- args: Union[Dict[str, object], str]
127
+ args: Union[Dict[str, object], ItemLocator]
122
128
 
123
129
  inference_configuration: GenericInferenceEvaluationTaskConfigurationInferenceConfiguration
124
130
 
@@ -127,7 +133,7 @@ class GenericInferenceEvaluationTask(TypedDict, total=False):
127
133
  configuration: Required[GenericInferenceEvaluationTaskConfiguration]
128
134
 
129
135
  alias: str
130
- """Alias to title the results column. Defaults to the `task_type`"""
136
+ """Alias to title the results column. Defaults to the `inference`"""
131
137
 
132
138
  task_type: Literal["inference"]
133
139
 
@@ -180,22 +186,24 @@ class ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplication
180
186
  ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace
181
187
  ]
182
188
 
189
+ return_span: bool
190
+
183
191
  use_channels: bool
184
192
 
185
193
 
186
194
  ApplicationVariantV1EvaluationTaskConfigurationOverrides: TypeAlias = Union[
187
- ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, str
195
+ ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, ItemLocator
188
196
  ]
189
197
 
190
198
 
191
199
  class ApplicationVariantV1EvaluationTaskConfiguration(TypedDict, total=False):
192
200
  application_variant_id: Required[str]
193
201
 
194
- inputs: Required[Union[Dict[str, object], str]]
202
+ inputs: Required[Union[Dict[str, object], ItemLocator]]
195
203
 
196
- history: Union[Iterable[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], str]
204
+ history: Union[Iterable[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], ItemLocator]
197
205
 
198
- operation_metadata: Union[Dict[str, object], str]
206
+ operation_metadata: Union[Dict[str, object], ItemLocator]
199
207
 
200
208
  overrides: ApplicationVariantV1EvaluationTaskConfigurationOverrides
201
209
  """Execution override options for agentic applications"""
@@ -205,7 +213,7 @@ class ApplicationVariantV1EvaluationTask(TypedDict, total=False):
205
213
  configuration: Required[ApplicationVariantV1EvaluationTaskConfiguration]
206
214
 
207
215
  alias: str
208
- """Alias to title the results column. Defaults to the `task_type`"""
216
+ """Alias to title the results column. Defaults to the `application_variant`"""
209
217
 
210
218
  task_type: Literal["application_variant"]
211
219
 
@@ -281,14 +289,61 @@ class MetricEvaluationTask(TypedDict, total=False):
281
289
  configuration: Required[MetricEvaluationTaskConfiguration]
282
290
 
283
291
  alias: str
284
- """Alias to title the results column. Defaults to the `task_type`"""
292
+ """Alias to title the results column.
293
+
294
+ Defaults to the metric type specified in the configuration
295
+ """
285
296
 
286
297
  task_type: Literal["metric"]
287
298
 
288
299
 
300
+ class AutoEvaluationQuestionTaskConfiguration(TypedDict, total=False):
301
+ model: Required[str]
302
+ """model specified as `model_vendor/model_name`"""
303
+
304
+ prompt: Required[str]
305
+
306
+ question_id: Required[str]
307
+ """question to be evaluated"""
308
+
309
+
310
+ class AutoEvaluationQuestionTask(TypedDict, total=False):
311
+ configuration: Required[AutoEvaluationQuestionTaskConfiguration]
312
+
313
+ alias: str
314
+ """Alias to title the results column. Defaults to the `auto_evaluation_question`"""
315
+
316
+ task_type: Literal["auto_evaluation.question"]
317
+
318
+
319
+ class ContributorEvaluationQuestionTaskConfiguration(TypedDict, total=False):
320
+ layout: Required["ContainerParam"]
321
+
322
+ question_id: Required[str]
323
+
324
+ queue_id: str
325
+ """The contributor annotation queue to include this task in. Defaults to `default`"""
326
+
327
+
328
+ class ContributorEvaluationQuestionTask(TypedDict, total=False):
329
+ configuration: Required[ContributorEvaluationQuestionTaskConfiguration]
330
+
331
+ alias: str
332
+ """Alias to title the results column.
333
+
334
+ Defaults to the `contributor_evaluation_question`
335
+ """
336
+
337
+ task_type: Literal["contributor_evaluation.question"]
338
+
339
+
289
340
  EvaluationTaskParam: TypeAlias = Union[
290
341
  ChatCompletionEvaluationTask,
291
342
  GenericInferenceEvaluationTask,
292
343
  ApplicationVariantV1EvaluationTask,
293
344
  MetricEvaluationTask,
345
+ AutoEvaluationQuestionTask,
346
+ ContributorEvaluationQuestionTask,
294
347
  ]
348
+
349
+ from .container_param import ContainerParam
@@ -0,0 +1,17 @@
1
+ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import List
6
+ from typing_extensions import TypedDict
7
+
8
+ __all__ = ["EvaluationUpdateParams"]
9
+
10
+
11
+ class EvaluationUpdateParams(TypedDict, total=False):
12
+ description: str
13
+
14
+ name: str
15
+
16
+ tags: List[str]
17
+ """The tags associated with the entity"""
@@ -2,15 +2,14 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- from typing import Optional
6
5
  from typing_extensions import TypedDict
7
6
 
8
7
  __all__ = ["FileListParams"]
9
8
 
10
9
 
11
10
  class FileListParams(TypedDict, total=False):
12
- ending_before: Optional[str]
11
+ ending_before: str
13
12
 
14
13
  limit: int
15
14
 
16
- starting_after: Optional[str]
15
+ starting_after: str
@@ -170,10 +170,6 @@ class InferenceModel(BaseModel):
170
170
 
171
171
  status: Literal["failed", "ready", "deploying"]
172
172
 
173
- description: Optional[str] = None
174
-
175
- display_name: Optional[str] = None
176
-
177
173
  api_model_metadata: Optional[Dict[str, object]] = FieldInfo(alias="model_metadata", default=None)
178
174
 
179
175
  object: Optional[Literal["model"]] = None
@@ -0,0 +1,7 @@
1
+ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
+
3
+ from typing_extensions import TypeAlias
4
+
5
+ __all__ = ["ItemLocator"]
6
+
7
+ ItemLocator: TypeAlias = str
@@ -0,0 +1,7 @@
1
+ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
+
3
+ from typing_extensions import TypeAlias
4
+
5
+ __all__ = ["ItemLocatorTemplate"]
6
+
7
+ ItemLocatorTemplate: TypeAlias = str
@@ -2,33 +2,30 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- from typing import Optional
6
5
  from typing_extensions import Literal, TypedDict
7
6
 
8
7
  __all__ = ["ModelListParams"]
9
8
 
10
9
 
11
10
  class ModelListParams(TypedDict, total=False):
12
- ending_before: Optional[str]
11
+ ending_before: str
13
12
 
14
13
  limit: int
15
14
 
16
- model_vendor: Optional[
17
- Literal[
18
- "openai",
19
- "cohere",
20
- "vertex_ai",
21
- "anthropic",
22
- "azure",
23
- "gemini",
24
- "launch",
25
- "llmengine",
26
- "model_zoo",
27
- "bedrock",
28
- "xai",
29
- ]
15
+ model_vendor: Literal[
16
+ "openai",
17
+ "cohere",
18
+ "vertex_ai",
19
+ "anthropic",
20
+ "azure",
21
+ "gemini",
22
+ "launch",
23
+ "llmengine",
24
+ "model_zoo",
25
+ "bedrock",
26
+ "xai",
30
27
  ]
31
28
 
32
- name: Optional[str]
29
+ name: str
33
30
 
34
- starting_after: Optional[str]
31
+ starting_after: str