scale-gp-beta 0.1.0a3__py3-none-any.whl → 0.1.0a5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. scale_gp_beta/_base_client.py +1 -96
  2. scale_gp_beta/_client.py +17 -17
  3. scale_gp_beta/_models.py +5 -2
  4. scale_gp_beta/_version.py +1 -1
  5. scale_gp_beta/resources/__init__.py +27 -27
  6. scale_gp_beta/resources/chat/completions.py +86 -92
  7. scale_gp_beta/resources/datasets.py +569 -0
  8. scale_gp_beta/resources/{question_sets.py → evaluations.py} +259 -273
  9. scale_gp_beta/resources/models.py +2 -0
  10. scale_gp_beta/types/__init__.py +11 -11
  11. scale_gp_beta/types/dataset.py +27 -0
  12. scale_gp_beta/types/dataset_create_params.py +17 -0
  13. scale_gp_beta/types/{question_set_delete_response.py → dataset_delete_response.py} +3 -3
  14. scale_gp_beta/types/{question_list_params.py → dataset_list_params.py} +4 -2
  15. scale_gp_beta/types/dataset_retrieve_params.py +11 -0
  16. scale_gp_beta/types/dataset_update_params.py +13 -0
  17. scale_gp_beta/types/evaluation.py +30 -0
  18. scale_gp_beta/types/evaluation_archive_response.py +16 -0
  19. scale_gp_beta/types/evaluation_create_params.py +734 -0
  20. scale_gp_beta/types/{question_set_list_params.py → evaluation_list_params.py} +4 -6
  21. scale_gp_beta/types/evaluation_retrieve_params.py +11 -0
  22. scale_gp_beta/types/inference_model.py +16 -2
  23. scale_gp_beta/types/model_list_params.py +1 -0
  24. {scale_gp_beta-0.1.0a3.dist-info → scale_gp_beta-0.1.0a5.dist-info}/METADATA +24 -4
  25. {scale_gp_beta-0.1.0a3.dist-info → scale_gp_beta-0.1.0a5.dist-info}/RECORD +27 -27
  26. {scale_gp_beta-0.1.0a3.dist-info → scale_gp_beta-0.1.0a5.dist-info}/WHEEL +1 -1
  27. scale_gp_beta/resources/questions.py +0 -396
  28. scale_gp_beta/types/question.py +0 -68
  29. scale_gp_beta/types/question_create_params.py +0 -59
  30. scale_gp_beta/types/question_list.py +0 -27
  31. scale_gp_beta/types/question_set.py +0 -106
  32. scale_gp_beta/types/question_set_create_params.py +0 -115
  33. scale_gp_beta/types/question_set_list.py +0 -27
  34. scale_gp_beta/types/question_set_retrieve_params.py +0 -12
  35. scale_gp_beta/types/question_set_update_params.py +0 -23
  36. {scale_gp_beta-0.1.0a3.dist-info → scale_gp_beta-0.1.0a5.dist-info}/licenses/LICENSE +0 -0
@@ -2,19 +2,15 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- from typing import List, Optional
6
- from typing_extensions import Literal, overload
5
+ from typing import Dict, Iterable, Optional
6
+ from typing_extensions import overload
7
7
 
8
8
  import httpx
9
9
 
10
- from ..types import (
11
- question_set_list_params,
12
- question_set_create_params,
13
- question_set_update_params,
14
- question_set_retrieve_params,
15
- )
10
+ from ..types import evaluation_list_params, evaluation_create_params, evaluation_retrieve_params
16
11
  from .._types import NOT_GIVEN, Body, Query, Headers, NotGiven
17
12
  from .._utils import (
13
+ required_args,
18
14
  maybe_transform,
19
15
  async_maybe_transform,
20
16
  )
@@ -28,55 +24,54 @@ from .._response import (
28
24
  )
29
25
  from ..pagination import SyncCursorPage, AsyncCursorPage
30
26
  from .._base_client import AsyncPaginator, make_request_options
31
- from ..types.question_set import QuestionSet
32
- from ..types.question_set_delete_response import QuestionSetDeleteResponse
27
+ from ..types.evaluation import Evaluation
28
+ from ..types.evaluation_archive_response import EvaluationArchiveResponse
33
29
 
34
- __all__ = ["QuestionSetsResource", "AsyncQuestionSetsResource"]
30
+ __all__ = ["EvaluationsResource", "AsyncEvaluationsResource"]
35
31
 
36
32
 
37
- class QuestionSetsResource(SyncAPIResource):
33
+ class EvaluationsResource(SyncAPIResource):
38
34
  @cached_property
39
- def with_raw_response(self) -> QuestionSetsResourceWithRawResponse:
35
+ def with_raw_response(self) -> EvaluationsResourceWithRawResponse:
40
36
  """
41
37
  This property can be used as a prefix for any HTTP method call to return
42
38
  the raw response object instead of the parsed content.
43
39
 
44
40
  For more information, see https://www.github.com/scaleapi/sgp-python-beta#accessing-raw-response-data-eg-headers
45
41
  """
46
- return QuestionSetsResourceWithRawResponse(self)
42
+ return EvaluationsResourceWithRawResponse(self)
47
43
 
48
44
  @cached_property
49
- def with_streaming_response(self) -> QuestionSetsResourceWithStreamingResponse:
45
+ def with_streaming_response(self) -> EvaluationsResourceWithStreamingResponse:
50
46
  """
51
47
  An alternative to `.with_raw_response` that doesn't eagerly read the response body.
52
48
 
53
49
  For more information, see https://www.github.com/scaleapi/sgp-python-beta#with_streaming_response
54
50
  """
55
- return QuestionSetsResourceWithStreamingResponse(self)
51
+ return EvaluationsResourceWithStreamingResponse(self)
56
52
 
53
+ @overload
57
54
  def create(
58
55
  self,
59
56
  *,
57
+ data: Iterable[Dict[str, object]],
60
58
  name: str,
61
- questions: List[question_set_create_params.Question],
62
- instructions: str | NotGiven = NOT_GIVEN,
59
+ description: str | NotGiven = NOT_GIVEN,
60
+ tasks: Iterable[evaluation_create_params.EvaluationStandaloneCreateRequestTask] | NotGiven = NOT_GIVEN,
63
61
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
64
62
  # The extra values given here take precedence over values defined on the client or passed to this method.
65
63
  extra_headers: Headers | None = None,
66
64
  extra_query: Query | None = None,
67
65
  extra_body: Body | None = None,
68
66
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
69
- ) -> QuestionSet:
67
+ ) -> Evaluation:
70
68
  """
71
- Create Question Set
69
+ Create Evaluation
72
70
 
73
71
  Args:
74
- questions: IDs of existing questions in the question set or new questions to create. You
75
- can also optionally specify configurations for each question. Example:
76
- [`question_id`, {'id': 'question_id2', 'configuration': {...}}, {'title': 'New
77
- question', ...}]
72
+ data: Items to be evaluated
78
73
 
79
- instructions: Instructions to answer questions
74
+ tasks: Tasks allow you to augment and evaluate your data
80
75
 
81
76
  extra_headers: Send extra headers
82
77
 
@@ -86,38 +81,34 @@ class QuestionSetsResource(SyncAPIResource):
86
81
 
87
82
  timeout: Override the client-level default timeout for this request, in seconds
88
83
  """
89
- return self._post(
90
- "/v5/question-sets",
91
- body=maybe_transform(
92
- {
93
- "name": name,
94
- "questions": questions,
95
- "instructions": instructions,
96
- },
97
- question_set_create_params.QuestionSetCreateParams,
98
- ),
99
- options=make_request_options(
100
- extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
101
- ),
102
- cast_to=QuestionSet,
103
- )
84
+ ...
104
85
 
105
- def retrieve(
86
+ @overload
87
+ def create(
106
88
  self,
107
- question_set_id: str,
108
89
  *,
109
- views: List[Literal["questions"]] | NotGiven = NOT_GIVEN,
90
+ data: Iterable[evaluation_create_params.EvaluationFromDatasetCreateRequestData],
91
+ dataset_id: str,
92
+ name: str,
93
+ description: str | NotGiven = NOT_GIVEN,
94
+ tasks: Iterable[evaluation_create_params.EvaluationFromDatasetCreateRequestTask] | NotGiven = NOT_GIVEN,
110
95
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
111
96
  # The extra values given here take precedence over values defined on the client or passed to this method.
112
97
  extra_headers: Headers | None = None,
113
98
  extra_query: Query | None = None,
114
99
  extra_body: Body | None = None,
115
100
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
116
- ) -> QuestionSet:
101
+ ) -> Evaluation:
117
102
  """
118
- Get Question Set
103
+ Create Evaluation
119
104
 
120
105
  Args:
106
+ data: Items to be evaluated, including references to the input dataset items
107
+
108
+ dataset_id: The ID of the dataset containing the items referenced by the `data` field
109
+
110
+ tasks: Tasks allow you to augment and evaluate your data
111
+
121
112
  extra_headers: Send extra headers
122
113
 
123
114
  extra_query: Add additional query parameters to the request
@@ -126,39 +117,33 @@ class QuestionSetsResource(SyncAPIResource):
126
117
 
127
118
  timeout: Override the client-level default timeout for this request, in seconds
128
119
  """
129
- if not question_set_id:
130
- raise ValueError(f"Expected a non-empty value for `question_set_id` but received {question_set_id!r}")
131
- return self._get(
132
- f"/v5/question-sets/{question_set_id}",
133
- options=make_request_options(
134
- extra_headers=extra_headers,
135
- extra_query=extra_query,
136
- extra_body=extra_body,
137
- timeout=timeout,
138
- query=maybe_transform({"views": views}, question_set_retrieve_params.QuestionSetRetrieveParams),
139
- ),
140
- cast_to=QuestionSet,
141
- )
120
+ ...
142
121
 
143
122
  @overload
144
- def update(
123
+ def create(
145
124
  self,
146
- question_set_id: str,
147
125
  *,
148
- instructions: str | NotGiven = NOT_GIVEN,
149
- name: str | NotGiven = NOT_GIVEN,
126
+ data: Iterable[Dict[str, object]],
127
+ dataset: evaluation_create_params.EvaluationWithDatasetCreateRequestDataset,
128
+ name: str,
129
+ description: str | NotGiven = NOT_GIVEN,
130
+ tasks: Iterable[evaluation_create_params.EvaluationWithDatasetCreateRequestTask] | NotGiven = NOT_GIVEN,
150
131
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
151
132
  # The extra values given here take precedence over values defined on the client or passed to this method.
152
133
  extra_headers: Headers | None = None,
153
134
  extra_query: Query | None = None,
154
135
  extra_body: Body | None = None,
155
136
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
156
- ) -> QuestionSet:
137
+ ) -> Evaluation:
157
138
  """
158
- Update Question Set
139
+ Create Evaluation
159
140
 
160
141
  Args:
161
- instructions: Instructions to answer questions
142
+ data: Items to be evaluated
143
+
144
+ dataset: Create a reusable dataset from items in the `data` field
145
+
146
+ tasks: Tasks allow you to augment and evaluate your data
162
147
 
163
148
  extra_headers: Send extra headers
164
149
 
@@ -170,25 +155,58 @@ class QuestionSetsResource(SyncAPIResource):
170
155
  """
171
156
  ...
172
157
 
173
- @overload
174
- def update(
158
+ @required_args(["data", "name"], ["data", "dataset_id", "name"], ["data", "dataset", "name"])
159
+ def create(
160
+ self,
161
+ *,
162
+ data: Iterable[Dict[str, object]] | Iterable[evaluation_create_params.EvaluationFromDatasetCreateRequestData],
163
+ name: str,
164
+ description: str | NotGiven = NOT_GIVEN,
165
+ tasks: Iterable[evaluation_create_params.EvaluationStandaloneCreateRequestTask] | NotGiven = NOT_GIVEN,
166
+ dataset_id: str | NotGiven = NOT_GIVEN,
167
+ dataset: evaluation_create_params.EvaluationWithDatasetCreateRequestDataset | NotGiven = NOT_GIVEN,
168
+ # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
169
+ # The extra values given here take precedence over values defined on the client or passed to this method.
170
+ extra_headers: Headers | None = None,
171
+ extra_query: Query | None = None,
172
+ extra_body: Body | None = None,
173
+ timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
174
+ ) -> Evaluation:
175
+ return self._post(
176
+ "/v5/evaluations",
177
+ body=maybe_transform(
178
+ {
179
+ "data": data,
180
+ "name": name,
181
+ "description": description,
182
+ "tasks": tasks,
183
+ "dataset_id": dataset_id,
184
+ "dataset": dataset,
185
+ },
186
+ evaluation_create_params.EvaluationCreateParams,
187
+ ),
188
+ options=make_request_options(
189
+ extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
190
+ ),
191
+ cast_to=Evaluation,
192
+ )
193
+
194
+ def retrieve(
175
195
  self,
176
- question_set_id: str,
196
+ evaluation_id: str,
177
197
  *,
178
- restore: Literal[True],
198
+ include_archived: bool | NotGiven = NOT_GIVEN,
179
199
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
180
200
  # The extra values given here take precedence over values defined on the client or passed to this method.
181
201
  extra_headers: Headers | None = None,
182
202
  extra_query: Query | None = None,
183
203
  extra_body: Body | None = None,
184
204
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
185
- ) -> QuestionSet:
205
+ ) -> Evaluation:
186
206
  """
187
- Update Question Set
207
+ Get Evaluation
188
208
 
189
209
  Args:
190
- restore: Set to true to restore the entity from the database.
191
-
192
210
  extra_headers: Send extra headers
193
211
 
194
212
  extra_query: Add additional query parameters to the request
@@ -197,38 +215,20 @@ class QuestionSetsResource(SyncAPIResource):
197
215
 
198
216
  timeout: Override the client-level default timeout for this request, in seconds
199
217
  """
200
- ...
201
-
202
- def update(
203
- self,
204
- question_set_id: str,
205
- *,
206
- instructions: str | NotGiven = NOT_GIVEN,
207
- name: str | NotGiven = NOT_GIVEN,
208
- restore: Literal[True] | NotGiven = NOT_GIVEN,
209
- # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
210
- # The extra values given here take precedence over values defined on the client or passed to this method.
211
- extra_headers: Headers | None = None,
212
- extra_query: Query | None = None,
213
- extra_body: Body | None = None,
214
- timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
215
- ) -> QuestionSet:
216
- if not question_set_id:
217
- raise ValueError(f"Expected a non-empty value for `question_set_id` but received {question_set_id!r}")
218
- return self._patch(
219
- f"/v5/question-sets/{question_set_id}",
220
- body=maybe_transform(
221
- {
222
- "instructions": instructions,
223
- "name": name,
224
- "restore": restore,
225
- },
226
- question_set_update_params.QuestionSetUpdateParams,
227
- ),
218
+ if not evaluation_id:
219
+ raise ValueError(f"Expected a non-empty value for `evaluation_id` but received {evaluation_id!r}")
220
+ return self._get(
221
+ f"/v5/evaluations/{evaluation_id}",
228
222
  options=make_request_options(
229
- extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
223
+ extra_headers=extra_headers,
224
+ extra_query=extra_query,
225
+ extra_body=extra_body,
226
+ timeout=timeout,
227
+ query=maybe_transform(
228
+ {"include_archived": include_archived}, evaluation_retrieve_params.EvaluationRetrieveParams
229
+ ),
230
230
  ),
231
- cast_to=QuestionSet,
231
+ cast_to=Evaluation,
232
232
  )
233
233
 
234
234
  def list(
@@ -238,16 +238,15 @@ class QuestionSetsResource(SyncAPIResource):
238
238
  include_archived: bool | NotGiven = NOT_GIVEN,
239
239
  limit: int | NotGiven = NOT_GIVEN,
240
240
  starting_after: Optional[str] | NotGiven = NOT_GIVEN,
241
- views: List[Literal["questions"]] | NotGiven = NOT_GIVEN,
242
241
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
243
242
  # The extra values given here take precedence over values defined on the client or passed to this method.
244
243
  extra_headers: Headers | None = None,
245
244
  extra_query: Query | None = None,
246
245
  extra_body: Body | None = None,
247
246
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
248
- ) -> SyncCursorPage[QuestionSet]:
247
+ ) -> SyncCursorPage[Evaluation]:
249
248
  """
250
- List Question Sets
249
+ List Evaluations
251
250
 
252
251
  Args:
253
252
  extra_headers: Send extra headers
@@ -259,8 +258,8 @@ class QuestionSetsResource(SyncAPIResource):
259
258
  timeout: Override the client-level default timeout for this request, in seconds
260
259
  """
261
260
  return self._get_api_list(
262
- "/v5/question-sets",
263
- page=SyncCursorPage[QuestionSet],
261
+ "/v5/evaluations",
262
+ page=SyncCursorPage[Evaluation],
264
263
  options=make_request_options(
265
264
  extra_headers=extra_headers,
266
265
  extra_query=extra_query,
@@ -272,17 +271,16 @@ class QuestionSetsResource(SyncAPIResource):
272
271
  "include_archived": include_archived,
273
272
  "limit": limit,
274
273
  "starting_after": starting_after,
275
- "views": views,
276
274
  },
277
- question_set_list_params.QuestionSetListParams,
275
+ evaluation_list_params.EvaluationListParams,
278
276
  ),
279
277
  ),
280
- model=QuestionSet,
278
+ model=Evaluation,
281
279
  )
282
280
 
283
- def delete(
281
+ def archive(
284
282
  self,
285
- question_set_id: str,
283
+ evaluation_id: str,
286
284
  *,
287
285
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
288
286
  # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -290,9 +288,9 @@ class QuestionSetsResource(SyncAPIResource):
290
288
  extra_query: Query | None = None,
291
289
  extra_body: Body | None = None,
292
290
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
293
- ) -> QuestionSetDeleteResponse:
291
+ ) -> EvaluationArchiveResponse:
294
292
  """
295
- Archive Question Set
293
+ Archive Evaluation
296
294
 
297
295
  Args:
298
296
  extra_headers: Send extra headers
@@ -303,60 +301,59 @@ class QuestionSetsResource(SyncAPIResource):
303
301
 
304
302
  timeout: Override the client-level default timeout for this request, in seconds
305
303
  """
306
- if not question_set_id:
307
- raise ValueError(f"Expected a non-empty value for `question_set_id` but received {question_set_id!r}")
304
+ if not evaluation_id:
305
+ raise ValueError(f"Expected a non-empty value for `evaluation_id` but received {evaluation_id!r}")
308
306
  return self._delete(
309
- f"/v5/question-sets/{question_set_id}",
307
+ f"/v5/evaluations/{evaluation_id}",
310
308
  options=make_request_options(
311
309
  extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
312
310
  ),
313
- cast_to=QuestionSetDeleteResponse,
311
+ cast_to=EvaluationArchiveResponse,
314
312
  )
315
313
 
316
314
 
317
- class AsyncQuestionSetsResource(AsyncAPIResource):
315
+ class AsyncEvaluationsResource(AsyncAPIResource):
318
316
  @cached_property
319
- def with_raw_response(self) -> AsyncQuestionSetsResourceWithRawResponse:
317
+ def with_raw_response(self) -> AsyncEvaluationsResourceWithRawResponse:
320
318
  """
321
319
  This property can be used as a prefix for any HTTP method call to return
322
320
  the raw response object instead of the parsed content.
323
321
 
324
322
  For more information, see https://www.github.com/scaleapi/sgp-python-beta#accessing-raw-response-data-eg-headers
325
323
  """
326
- return AsyncQuestionSetsResourceWithRawResponse(self)
324
+ return AsyncEvaluationsResourceWithRawResponse(self)
327
325
 
328
326
  @cached_property
329
- def with_streaming_response(self) -> AsyncQuestionSetsResourceWithStreamingResponse:
327
+ def with_streaming_response(self) -> AsyncEvaluationsResourceWithStreamingResponse:
330
328
  """
331
329
  An alternative to `.with_raw_response` that doesn't eagerly read the response body.
332
330
 
333
331
  For more information, see https://www.github.com/scaleapi/sgp-python-beta#with_streaming_response
334
332
  """
335
- return AsyncQuestionSetsResourceWithStreamingResponse(self)
333
+ return AsyncEvaluationsResourceWithStreamingResponse(self)
336
334
 
335
+ @overload
337
336
  async def create(
338
337
  self,
339
338
  *,
339
+ data: Iterable[Dict[str, object]],
340
340
  name: str,
341
- questions: List[question_set_create_params.Question],
342
- instructions: str | NotGiven = NOT_GIVEN,
341
+ description: str | NotGiven = NOT_GIVEN,
342
+ tasks: Iterable[evaluation_create_params.EvaluationStandaloneCreateRequestTask] | NotGiven = NOT_GIVEN,
343
343
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
344
344
  # The extra values given here take precedence over values defined on the client or passed to this method.
345
345
  extra_headers: Headers | None = None,
346
346
  extra_query: Query | None = None,
347
347
  extra_body: Body | None = None,
348
348
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
349
- ) -> QuestionSet:
349
+ ) -> Evaluation:
350
350
  """
351
- Create Question Set
351
+ Create Evaluation
352
352
 
353
353
  Args:
354
- questions: IDs of existing questions in the question set or new questions to create. You
355
- can also optionally specify configurations for each question. Example:
356
- [`question_id`, {'id': 'question_id2', 'configuration': {...}}, {'title': 'New
357
- question', ...}]
354
+ data: Items to be evaluated
358
355
 
359
- instructions: Instructions to answer questions
356
+ tasks: Tasks allow you to augment and evaluate your data
360
357
 
361
358
  extra_headers: Send extra headers
362
359
 
@@ -366,38 +363,34 @@ class AsyncQuestionSetsResource(AsyncAPIResource):
366
363
 
367
364
  timeout: Override the client-level default timeout for this request, in seconds
368
365
  """
369
- return await self._post(
370
- "/v5/question-sets",
371
- body=await async_maybe_transform(
372
- {
373
- "name": name,
374
- "questions": questions,
375
- "instructions": instructions,
376
- },
377
- question_set_create_params.QuestionSetCreateParams,
378
- ),
379
- options=make_request_options(
380
- extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
381
- ),
382
- cast_to=QuestionSet,
383
- )
366
+ ...
384
367
 
385
- async def retrieve(
368
+ @overload
369
+ async def create(
386
370
  self,
387
- question_set_id: str,
388
371
  *,
389
- views: List[Literal["questions"]] | NotGiven = NOT_GIVEN,
372
+ data: Iterable[evaluation_create_params.EvaluationFromDatasetCreateRequestData],
373
+ dataset_id: str,
374
+ name: str,
375
+ description: str | NotGiven = NOT_GIVEN,
376
+ tasks: Iterable[evaluation_create_params.EvaluationFromDatasetCreateRequestTask] | NotGiven = NOT_GIVEN,
390
377
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
391
378
  # The extra values given here take precedence over values defined on the client or passed to this method.
392
379
  extra_headers: Headers | None = None,
393
380
  extra_query: Query | None = None,
394
381
  extra_body: Body | None = None,
395
382
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
396
- ) -> QuestionSet:
383
+ ) -> Evaluation:
397
384
  """
398
- Get Question Set
385
+ Create Evaluation
399
386
 
400
387
  Args:
388
+ data: Items to be evaluated, including references to the input dataset items
389
+
390
+ dataset_id: The ID of the dataset containing the items referenced by the `data` field
391
+
392
+ tasks: Tasks allow you to augment and evaluate your data
393
+
401
394
  extra_headers: Send extra headers
402
395
 
403
396
  extra_query: Add additional query parameters to the request
@@ -406,41 +399,33 @@ class AsyncQuestionSetsResource(AsyncAPIResource):
406
399
 
407
400
  timeout: Override the client-level default timeout for this request, in seconds
408
401
  """
409
- if not question_set_id:
410
- raise ValueError(f"Expected a non-empty value for `question_set_id` but received {question_set_id!r}")
411
- return await self._get(
412
- f"/v5/question-sets/{question_set_id}",
413
- options=make_request_options(
414
- extra_headers=extra_headers,
415
- extra_query=extra_query,
416
- extra_body=extra_body,
417
- timeout=timeout,
418
- query=await async_maybe_transform(
419
- {"views": views}, question_set_retrieve_params.QuestionSetRetrieveParams
420
- ),
421
- ),
422
- cast_to=QuestionSet,
423
- )
402
+ ...
424
403
 
425
404
  @overload
426
- async def update(
405
+ async def create(
427
406
  self,
428
- question_set_id: str,
429
407
  *,
430
- instructions: str | NotGiven = NOT_GIVEN,
431
- name: str | NotGiven = NOT_GIVEN,
408
+ data: Iterable[Dict[str, object]],
409
+ dataset: evaluation_create_params.EvaluationWithDatasetCreateRequestDataset,
410
+ name: str,
411
+ description: str | NotGiven = NOT_GIVEN,
412
+ tasks: Iterable[evaluation_create_params.EvaluationWithDatasetCreateRequestTask] | NotGiven = NOT_GIVEN,
432
413
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
433
414
  # The extra values given here take precedence over values defined on the client or passed to this method.
434
415
  extra_headers: Headers | None = None,
435
416
  extra_query: Query | None = None,
436
417
  extra_body: Body | None = None,
437
418
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
438
- ) -> QuestionSet:
419
+ ) -> Evaluation:
439
420
  """
440
- Update Question Set
421
+ Create Evaluation
441
422
 
442
423
  Args:
443
- instructions: Instructions to answer questions
424
+ data: Items to be evaluated
425
+
426
+ dataset: Create a reusable dataset from items in the `data` field
427
+
428
+ tasks: Tasks allow you to augment and evaluate your data
444
429
 
445
430
  extra_headers: Send extra headers
446
431
 
@@ -452,25 +437,58 @@ class AsyncQuestionSetsResource(AsyncAPIResource):
452
437
  """
453
438
  ...
454
439
 
455
- @overload
456
- async def update(
440
+ @required_args(["data", "name"], ["data", "dataset_id", "name"], ["data", "dataset", "name"])
441
+ async def create(
457
442
  self,
458
- question_set_id: str,
459
443
  *,
460
- restore: Literal[True],
444
+ data: Iterable[Dict[str, object]] | Iterable[evaluation_create_params.EvaluationFromDatasetCreateRequestData],
445
+ name: str,
446
+ description: str | NotGiven = NOT_GIVEN,
447
+ tasks: Iterable[evaluation_create_params.EvaluationStandaloneCreateRequestTask] | NotGiven = NOT_GIVEN,
448
+ dataset_id: str | NotGiven = NOT_GIVEN,
449
+ dataset: evaluation_create_params.EvaluationWithDatasetCreateRequestDataset | NotGiven = NOT_GIVEN,
461
450
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
462
451
  # The extra values given here take precedence over values defined on the client or passed to this method.
463
452
  extra_headers: Headers | None = None,
464
453
  extra_query: Query | None = None,
465
454
  extra_body: Body | None = None,
466
455
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
467
- ) -> QuestionSet:
456
+ ) -> Evaluation:
457
+ return await self._post(
458
+ "/v5/evaluations",
459
+ body=await async_maybe_transform(
460
+ {
461
+ "data": data,
462
+ "name": name,
463
+ "description": description,
464
+ "tasks": tasks,
465
+ "dataset_id": dataset_id,
466
+ "dataset": dataset,
467
+ },
468
+ evaluation_create_params.EvaluationCreateParams,
469
+ ),
470
+ options=make_request_options(
471
+ extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
472
+ ),
473
+ cast_to=Evaluation,
474
+ )
475
+
476
+ async def retrieve(
477
+ self,
478
+ evaluation_id: str,
479
+ *,
480
+ include_archived: bool | NotGiven = NOT_GIVEN,
481
+ # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
482
+ # The extra values given here take precedence over values defined on the client or passed to this method.
483
+ extra_headers: Headers | None = None,
484
+ extra_query: Query | None = None,
485
+ extra_body: Body | None = None,
486
+ timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
487
+ ) -> Evaluation:
468
488
  """
469
- Update Question Set
489
+ Get Evaluation
470
490
 
471
491
  Args:
472
- restore: Set to true to restore the entity from the database.
473
-
474
492
  extra_headers: Send extra headers
475
493
 
476
494
  extra_query: Add additional query parameters to the request
@@ -479,38 +497,20 @@ class AsyncQuestionSetsResource(AsyncAPIResource):
479
497
 
480
498
  timeout: Override the client-level default timeout for this request, in seconds
481
499
  """
482
- ...
483
-
484
- async def update(
485
- self,
486
- question_set_id: str,
487
- *,
488
- instructions: str | NotGiven = NOT_GIVEN,
489
- name: str | NotGiven = NOT_GIVEN,
490
- restore: Literal[True] | NotGiven = NOT_GIVEN,
491
- # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
492
- # The extra values given here take precedence over values defined on the client or passed to this method.
493
- extra_headers: Headers | None = None,
494
- extra_query: Query | None = None,
495
- extra_body: Body | None = None,
496
- timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
497
- ) -> QuestionSet:
498
- if not question_set_id:
499
- raise ValueError(f"Expected a non-empty value for `question_set_id` but received {question_set_id!r}")
500
- return await self._patch(
501
- f"/v5/question-sets/{question_set_id}",
502
- body=await async_maybe_transform(
503
- {
504
- "instructions": instructions,
505
- "name": name,
506
- "restore": restore,
507
- },
508
- question_set_update_params.QuestionSetUpdateParams,
509
- ),
500
+ if not evaluation_id:
501
+ raise ValueError(f"Expected a non-empty value for `evaluation_id` but received {evaluation_id!r}")
502
+ return await self._get(
503
+ f"/v5/evaluations/{evaluation_id}",
510
504
  options=make_request_options(
511
- extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
505
+ extra_headers=extra_headers,
506
+ extra_query=extra_query,
507
+ extra_body=extra_body,
508
+ timeout=timeout,
509
+ query=await async_maybe_transform(
510
+ {"include_archived": include_archived}, evaluation_retrieve_params.EvaluationRetrieveParams
511
+ ),
512
512
  ),
513
- cast_to=QuestionSet,
513
+ cast_to=Evaluation,
514
514
  )
515
515
 
516
516
  def list(
@@ -520,16 +520,15 @@ class AsyncQuestionSetsResource(AsyncAPIResource):
520
520
  include_archived: bool | NotGiven = NOT_GIVEN,
521
521
  limit: int | NotGiven = NOT_GIVEN,
522
522
  starting_after: Optional[str] | NotGiven = NOT_GIVEN,
523
- views: List[Literal["questions"]] | NotGiven = NOT_GIVEN,
524
523
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
525
524
  # The extra values given here take precedence over values defined on the client or passed to this method.
526
525
  extra_headers: Headers | None = None,
527
526
  extra_query: Query | None = None,
528
527
  extra_body: Body | None = None,
529
528
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
530
- ) -> AsyncPaginator[QuestionSet, AsyncCursorPage[QuestionSet]]:
529
+ ) -> AsyncPaginator[Evaluation, AsyncCursorPage[Evaluation]]:
531
530
  """
532
- List Question Sets
531
+ List Evaluations
533
532
 
534
533
  Args:
535
534
  extra_headers: Send extra headers
@@ -541,8 +540,8 @@ class AsyncQuestionSetsResource(AsyncAPIResource):
541
540
  timeout: Override the client-level default timeout for this request, in seconds
542
541
  """
543
542
  return self._get_api_list(
544
- "/v5/question-sets",
545
- page=AsyncCursorPage[QuestionSet],
543
+ "/v5/evaluations",
544
+ page=AsyncCursorPage[Evaluation],
546
545
  options=make_request_options(
547
546
  extra_headers=extra_headers,
548
547
  extra_query=extra_query,
@@ -554,17 +553,16 @@ class AsyncQuestionSetsResource(AsyncAPIResource):
554
553
  "include_archived": include_archived,
555
554
  "limit": limit,
556
555
  "starting_after": starting_after,
557
- "views": views,
558
556
  },
559
- question_set_list_params.QuestionSetListParams,
557
+ evaluation_list_params.EvaluationListParams,
560
558
  ),
561
559
  ),
562
- model=QuestionSet,
560
+ model=Evaluation,
563
561
  )
564
562
 
565
- async def delete(
563
+ async def archive(
566
564
  self,
567
- question_set_id: str,
565
+ evaluation_id: str,
568
566
  *,
569
567
  # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
570
568
  # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -572,9 +570,9 @@ class AsyncQuestionSetsResource(AsyncAPIResource):
572
570
  extra_query: Query | None = None,
573
571
  extra_body: Body | None = None,
574
572
  timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
575
- ) -> QuestionSetDeleteResponse:
573
+ ) -> EvaluationArchiveResponse:
576
574
  """
577
- Archive Question Set
575
+ Archive Evaluation
578
576
 
579
577
  Args:
580
578
  extra_headers: Send extra headers
@@ -585,96 +583,84 @@ class AsyncQuestionSetsResource(AsyncAPIResource):
585
583
 
586
584
  timeout: Override the client-level default timeout for this request, in seconds
587
585
  """
588
- if not question_set_id:
589
- raise ValueError(f"Expected a non-empty value for `question_set_id` but received {question_set_id!r}")
586
+ if not evaluation_id:
587
+ raise ValueError(f"Expected a non-empty value for `evaluation_id` but received {evaluation_id!r}")
590
588
  return await self._delete(
591
- f"/v5/question-sets/{question_set_id}",
589
+ f"/v5/evaluations/{evaluation_id}",
592
590
  options=make_request_options(
593
591
  extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
594
592
  ),
595
- cast_to=QuestionSetDeleteResponse,
593
+ cast_to=EvaluationArchiveResponse,
596
594
  )
597
595
 
598
596
 
599
- class QuestionSetsResourceWithRawResponse:
600
- def __init__(self, question_sets: QuestionSetsResource) -> None:
601
- self._question_sets = question_sets
597
+ class EvaluationsResourceWithRawResponse:
598
+ def __init__(self, evaluations: EvaluationsResource) -> None:
599
+ self._evaluations = evaluations
602
600
 
603
601
  self.create = to_raw_response_wrapper(
604
- question_sets.create,
602
+ evaluations.create,
605
603
  )
606
604
  self.retrieve = to_raw_response_wrapper(
607
- question_sets.retrieve,
608
- )
609
- self.update = to_raw_response_wrapper(
610
- question_sets.update,
605
+ evaluations.retrieve,
611
606
  )
612
607
  self.list = to_raw_response_wrapper(
613
- question_sets.list,
608
+ evaluations.list,
614
609
  )
615
- self.delete = to_raw_response_wrapper(
616
- question_sets.delete,
610
+ self.archive = to_raw_response_wrapper(
611
+ evaluations.archive,
617
612
  )
618
613
 
619
614
 
620
- class AsyncQuestionSetsResourceWithRawResponse:
621
- def __init__(self, question_sets: AsyncQuestionSetsResource) -> None:
622
- self._question_sets = question_sets
615
+ class AsyncEvaluationsResourceWithRawResponse:
616
+ def __init__(self, evaluations: AsyncEvaluationsResource) -> None:
617
+ self._evaluations = evaluations
623
618
 
624
619
  self.create = async_to_raw_response_wrapper(
625
- question_sets.create,
620
+ evaluations.create,
626
621
  )
627
622
  self.retrieve = async_to_raw_response_wrapper(
628
- question_sets.retrieve,
629
- )
630
- self.update = async_to_raw_response_wrapper(
631
- question_sets.update,
623
+ evaluations.retrieve,
632
624
  )
633
625
  self.list = async_to_raw_response_wrapper(
634
- question_sets.list,
626
+ evaluations.list,
635
627
  )
636
- self.delete = async_to_raw_response_wrapper(
637
- question_sets.delete,
628
+ self.archive = async_to_raw_response_wrapper(
629
+ evaluations.archive,
638
630
  )
639
631
 
640
632
 
641
- class QuestionSetsResourceWithStreamingResponse:
642
- def __init__(self, question_sets: QuestionSetsResource) -> None:
643
- self._question_sets = question_sets
633
+ class EvaluationsResourceWithStreamingResponse:
634
+ def __init__(self, evaluations: EvaluationsResource) -> None:
635
+ self._evaluations = evaluations
644
636
 
645
637
  self.create = to_streamed_response_wrapper(
646
- question_sets.create,
638
+ evaluations.create,
647
639
  )
648
640
  self.retrieve = to_streamed_response_wrapper(
649
- question_sets.retrieve,
650
- )
651
- self.update = to_streamed_response_wrapper(
652
- question_sets.update,
641
+ evaluations.retrieve,
653
642
  )
654
643
  self.list = to_streamed_response_wrapper(
655
- question_sets.list,
644
+ evaluations.list,
656
645
  )
657
- self.delete = to_streamed_response_wrapper(
658
- question_sets.delete,
646
+ self.archive = to_streamed_response_wrapper(
647
+ evaluations.archive,
659
648
  )
660
649
 
661
650
 
662
- class AsyncQuestionSetsResourceWithStreamingResponse:
663
- def __init__(self, question_sets: AsyncQuestionSetsResource) -> None:
664
- self._question_sets = question_sets
651
+ class AsyncEvaluationsResourceWithStreamingResponse:
652
+ def __init__(self, evaluations: AsyncEvaluationsResource) -> None:
653
+ self._evaluations = evaluations
665
654
 
666
655
  self.create = async_to_streamed_response_wrapper(
667
- question_sets.create,
656
+ evaluations.create,
668
657
  )
669
658
  self.retrieve = async_to_streamed_response_wrapper(
670
- question_sets.retrieve,
671
- )
672
- self.update = async_to_streamed_response_wrapper(
673
- question_sets.update,
659
+ evaluations.retrieve,
674
660
  )
675
661
  self.list = async_to_streamed_response_wrapper(
676
- question_sets.list,
662
+ evaluations.list,
677
663
  )
678
- self.delete = async_to_streamed_response_wrapper(
679
- question_sets.delete,
664
+ self.archive = async_to_streamed_response_wrapper(
665
+ evaluations.archive,
680
666
  )