scorable 1.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of scorable might be problematic. Click here for more details.
- root/__about__.py +4 -0
- root/__init__.py +17 -0
- root/client.py +207 -0
- root/datasets.py +231 -0
- root/execution_logs.py +162 -0
- root/generated/__init__.py +0 -0
- root/generated/openapi_aclient/__init__.py +1 -0
- root/generated/openapi_aclient/api/__init__.py +1 -0
- root/generated/openapi_aclient/api/datasets_api.py +1274 -0
- root/generated/openapi_aclient/api/evaluators_api.py +3641 -0
- root/generated/openapi_aclient/api/execution_logs_api.py +751 -0
- root/generated/openapi_aclient/api/judges_api.py +3794 -0
- root/generated/openapi_aclient/api/models_api.py +1473 -0
- root/generated/openapi_aclient/api/objectives_api.py +1767 -0
- root/generated/openapi_aclient/api_client.py +662 -0
- root/generated/openapi_aclient/api_response.py +22 -0
- root/generated/openapi_aclient/configuration.py +470 -0
- root/generated/openapi_aclient/exceptions.py +197 -0
- root/generated/openapi_aclient/models/__init__.py +122 -0
- root/generated/openapi_aclient/models/data_set_create.py +118 -0
- root/generated/openapi_aclient/models/data_set_create_request.py +105 -0
- root/generated/openapi_aclient/models/data_set_list.py +129 -0
- root/generated/openapi_aclient/models/data_set_type.py +36 -0
- root/generated/openapi_aclient/models/dataset_range_request.py +93 -0
- root/generated/openapi_aclient/models/evaluator.py +273 -0
- root/generated/openapi_aclient/models/evaluator_calibration_output.py +101 -0
- root/generated/openapi_aclient/models/evaluator_calibration_result.py +134 -0
- root/generated/openapi_aclient/models/evaluator_demonstrations.py +107 -0
- root/generated/openapi_aclient/models/evaluator_demonstrations_request.py +107 -0
- root/generated/openapi_aclient/models/evaluator_execution_function_parameter_property_request.py +86 -0
- root/generated/openapi_aclient/models/evaluator_execution_function_parameter_request.py +109 -0
- root/generated/openapi_aclient/models/evaluator_execution_function_parameter_type_enum.py +35 -0
- root/generated/openapi_aclient/models/evaluator_execution_function_request.py +99 -0
- root/generated/openapi_aclient/models/evaluator_execution_functions_request.py +98 -0
- root/generated/openapi_aclient/models/evaluator_execution_functions_type_enum.py +35 -0
- root/generated/openapi_aclient/models/evaluator_execution_request.py +134 -0
- root/generated/openapi_aclient/models/evaluator_execution_result.py +114 -0
- root/generated/openapi_aclient/models/evaluator_inputs_value.py +100 -0
- root/generated/openapi_aclient/models/evaluator_inputs_value_items.py +89 -0
- root/generated/openapi_aclient/models/evaluator_list_output.py +198 -0
- root/generated/openapi_aclient/models/evaluator_reference.py +90 -0
- root/generated/openapi_aclient/models/evaluator_reference_request.py +90 -0
- root/generated/openapi_aclient/models/evaluator_request.py +194 -0
- root/generated/openapi_aclient/models/evaluator_result.py +110 -0
- root/generated/openapi_aclient/models/execution_log_details.py +291 -0
- root/generated/openapi_aclient/models/execution_log_details_evaluation_context.py +83 -0
- root/generated/openapi_aclient/models/execution_log_details_evaluator_latencies_inner.py +83 -0
- root/generated/openapi_aclient/models/execution_log_list.py +217 -0
- root/generated/openapi_aclient/models/execution_log_list_evaluation_context.py +83 -0
- root/generated/openapi_aclient/models/generation_model_params_request.py +93 -0
- root/generated/openapi_aclient/models/id.py +87 -0
- root/generated/openapi_aclient/models/input_variable.py +121 -0
- root/generated/openapi_aclient/models/input_variable_request.py +82 -0
- root/generated/openapi_aclient/models/judge.py +178 -0
- root/generated/openapi_aclient/models/judge_execution_request.py +114 -0
- root/generated/openapi_aclient/models/judge_execution_response.py +97 -0
- root/generated/openapi_aclient/models/judge_files_inner.py +84 -0
- root/generated/openapi_aclient/models/judge_generator_request.py +142 -0
- root/generated/openapi_aclient/models/judge_generator_response.py +88 -0
- root/generated/openapi_aclient/models/judge_invite_request.py +87 -0
- root/generated/openapi_aclient/models/judge_list.py +156 -0
- root/generated/openapi_aclient/models/judge_rectifier_request_request.py +114 -0
- root/generated/openapi_aclient/models/judge_rectifier_response.py +121 -0
- root/generated/openapi_aclient/models/judge_request.py +108 -0
- root/generated/openapi_aclient/models/model.py +126 -0
- root/generated/openapi_aclient/models/model_list.py +115 -0
- root/generated/openapi_aclient/models/model_params.py +89 -0
- root/generated/openapi_aclient/models/model_params_request.py +89 -0
- root/generated/openapi_aclient/models/model_request.py +118 -0
- root/generated/openapi_aclient/models/nested_evaluator.py +110 -0
- root/generated/openapi_aclient/models/nested_evaluator_objective.py +87 -0
- root/generated/openapi_aclient/models/nested_evaluator_request.py +92 -0
- root/generated/openapi_aclient/models/nested_objective_evaluator.py +105 -0
- root/generated/openapi_aclient/models/nested_objective_evaluator_request.py +92 -0
- root/generated/openapi_aclient/models/nested_objective_list.py +111 -0
- root/generated/openapi_aclient/models/nested_user_details.py +88 -0
- root/generated/openapi_aclient/models/nested_user_details_request.py +82 -0
- root/generated/openapi_aclient/models/nested_vector_objective.py +88 -0
- root/generated/openapi_aclient/models/nested_vector_objective_request.py +82 -0
- root/generated/openapi_aclient/models/objective.py +157 -0
- root/generated/openapi_aclient/models/objective_list.py +128 -0
- root/generated/openapi_aclient/models/objective_request.py +113 -0
- root/generated/openapi_aclient/models/objective_validator.py +100 -0
- root/generated/openapi_aclient/models/objective_validator_request.py +90 -0
- root/generated/openapi_aclient/models/paginated_data_set_list_list.py +111 -0
- root/generated/openapi_aclient/models/paginated_evaluator_list.py +111 -0
- root/generated/openapi_aclient/models/paginated_evaluator_list_output_list.py +111 -0
- root/generated/openapi_aclient/models/paginated_execution_log_list_list.py +111 -0
- root/generated/openapi_aclient/models/paginated_judge_list_list.py +111 -0
- root/generated/openapi_aclient/models/paginated_model_list_list.py +111 -0
- root/generated/openapi_aclient/models/paginated_objective_list.py +111 -0
- root/generated/openapi_aclient/models/paginated_objective_list_list.py +111 -0
- root/generated/openapi_aclient/models/patched_evaluator_request.py +194 -0
- root/generated/openapi_aclient/models/patched_judge_request.py +110 -0
- root/generated/openapi_aclient/models/patched_model_request.py +118 -0
- root/generated/openapi_aclient/models/patched_objective_request.py +113 -0
- root/generated/openapi_aclient/models/provider.py +99 -0
- root/generated/openapi_aclient/models/reference_variable.py +123 -0
- root/generated/openapi_aclient/models/reference_variable_request.py +83 -0
- root/generated/openapi_aclient/models/skill_execution_validator_result.py +130 -0
- root/generated/openapi_aclient/models/skill_test_data_request.py +107 -0
- root/generated/openapi_aclient/models/skill_test_data_request_dataset_range.py +93 -0
- root/generated/openapi_aclient/models/skill_test_input_request.py +171 -0
- root/generated/openapi_aclient/models/skill_type_enum.py +36 -0
- root/generated/openapi_aclient/models/status_change.py +84 -0
- root/generated/openapi_aclient/models/status_change_request.py +84 -0
- root/generated/openapi_aclient/models/status_change_status_enum.py +36 -0
- root/generated/openapi_aclient/models/status_enum.py +38 -0
- root/generated/openapi_aclient/models/validation_result_status.py +36 -0
- root/generated/openapi_aclient/models/visibility_enum.py +38 -0
- root/generated/openapi_aclient/rest.py +166 -0
- root/generated/openapi_aclient_README.md +239 -0
- root/generated/openapi_client/__init__.py +1 -0
- root/generated/openapi_client/api/__init__.py +1 -0
- root/generated/openapi_client/api/datasets_api.py +1274 -0
- root/generated/openapi_client/api/evaluators_api.py +3641 -0
- root/generated/openapi_client/api/execution_logs_api.py +751 -0
- root/generated/openapi_client/api/judges_api.py +3794 -0
- root/generated/openapi_client/api/models_api.py +1473 -0
- root/generated/openapi_client/api/objectives_api.py +1767 -0
- root/generated/openapi_client/api_client.py +659 -0
- root/generated/openapi_client/api_response.py +22 -0
- root/generated/openapi_client/configuration.py +474 -0
- root/generated/openapi_client/exceptions.py +197 -0
- root/generated/openapi_client/models/__init__.py +120 -0
- root/generated/openapi_client/models/data_set_create.py +118 -0
- root/generated/openapi_client/models/data_set_create_request.py +105 -0
- root/generated/openapi_client/models/data_set_list.py +129 -0
- root/generated/openapi_client/models/data_set_type.py +36 -0
- root/generated/openapi_client/models/dataset_range_request.py +93 -0
- root/generated/openapi_client/models/evaluator.py +273 -0
- root/generated/openapi_client/models/evaluator_calibration_output.py +101 -0
- root/generated/openapi_client/models/evaluator_calibration_result.py +134 -0
- root/generated/openapi_client/models/evaluator_demonstrations.py +107 -0
- root/generated/openapi_client/models/evaluator_demonstrations_request.py +107 -0
- root/generated/openapi_client/models/evaluator_execution_function_parameter_property_request.py +86 -0
- root/generated/openapi_client/models/evaluator_execution_function_parameter_request.py +109 -0
- root/generated/openapi_client/models/evaluator_execution_function_parameter_type_enum.py +35 -0
- root/generated/openapi_client/models/evaluator_execution_function_request.py +99 -0
- root/generated/openapi_client/models/evaluator_execution_functions_request.py +98 -0
- root/generated/openapi_client/models/evaluator_execution_functions_type_enum.py +35 -0
- root/generated/openapi_client/models/evaluator_execution_request.py +134 -0
- root/generated/openapi_client/models/evaluator_execution_result.py +114 -0
- root/generated/openapi_client/models/evaluator_inputs_value.py +100 -0
- root/generated/openapi_client/models/evaluator_inputs_value_items.py +89 -0
- root/generated/openapi_client/models/evaluator_list_output.py +198 -0
- root/generated/openapi_client/models/evaluator_reference.py +90 -0
- root/generated/openapi_client/models/evaluator_reference_request.py +90 -0
- root/generated/openapi_client/models/evaluator_request.py +194 -0
- root/generated/openapi_client/models/evaluator_result.py +110 -0
- root/generated/openapi_client/models/execution_log_details.py +291 -0
- root/generated/openapi_client/models/execution_log_details_evaluation_context.py +83 -0
- root/generated/openapi_client/models/execution_log_details_evaluator_latencies_inner.py +83 -0
- root/generated/openapi_client/models/execution_log_list.py +215 -0
- root/generated/openapi_client/models/execution_log_list_evaluation_context.py +83 -0
- root/generated/openapi_client/models/generation_model_params_request.py +93 -0
- root/generated/openapi_client/models/id.py +87 -0
- root/generated/openapi_client/models/input_variable.py +121 -0
- root/generated/openapi_client/models/input_variable_request.py +82 -0
- root/generated/openapi_client/models/judge.py +178 -0
- root/generated/openapi_client/models/judge_execution_request.py +114 -0
- root/generated/openapi_client/models/judge_execution_response.py +97 -0
- root/generated/openapi_client/models/judge_files_inner.py +84 -0
- root/generated/openapi_client/models/judge_generator_request.py +142 -0
- root/generated/openapi_client/models/judge_generator_response.py +88 -0
- root/generated/openapi_client/models/judge_invite_request.py +87 -0
- root/generated/openapi_client/models/judge_list.py +156 -0
- root/generated/openapi_client/models/judge_rectifier_request_request.py +114 -0
- root/generated/openapi_client/models/judge_rectifier_response.py +121 -0
- root/generated/openapi_client/models/judge_request.py +108 -0
- root/generated/openapi_client/models/model.py +126 -0
- root/generated/openapi_client/models/model_list.py +115 -0
- root/generated/openapi_client/models/model_params.py +89 -0
- root/generated/openapi_client/models/model_params_request.py +89 -0
- root/generated/openapi_client/models/model_request.py +118 -0
- root/generated/openapi_client/models/nested_evaluator.py +110 -0
- root/generated/openapi_client/models/nested_evaluator_objective.py +87 -0
- root/generated/openapi_client/models/nested_evaluator_request.py +92 -0
- root/generated/openapi_client/models/nested_objective_evaluator.py +105 -0
- root/generated/openapi_client/models/nested_objective_evaluator_request.py +92 -0
- root/generated/openapi_client/models/nested_objective_list.py +111 -0
- root/generated/openapi_client/models/nested_user_details.py +88 -0
- root/generated/openapi_client/models/nested_user_details_request.py +82 -0
- root/generated/openapi_client/models/nested_vector_objective.py +88 -0
- root/generated/openapi_client/models/nested_vector_objective_request.py +82 -0
- root/generated/openapi_client/models/objective.py +157 -0
- root/generated/openapi_client/models/objective_list.py +128 -0
- root/generated/openapi_client/models/objective_request.py +113 -0
- root/generated/openapi_client/models/objective_validator.py +100 -0
- root/generated/openapi_client/models/objective_validator_request.py +90 -0
- root/generated/openapi_client/models/paginated_data_set_list_list.py +111 -0
- root/generated/openapi_client/models/paginated_evaluator_list.py +111 -0
- root/generated/openapi_client/models/paginated_evaluator_list_output_list.py +111 -0
- root/generated/openapi_client/models/paginated_execution_log_list_list.py +111 -0
- root/generated/openapi_client/models/paginated_judge_list_list.py +111 -0
- root/generated/openapi_client/models/paginated_model_list_list.py +111 -0
- root/generated/openapi_client/models/paginated_objective_list.py +111 -0
- root/generated/openapi_client/models/paginated_objective_list_list.py +111 -0
- root/generated/openapi_client/models/patched_evaluator_request.py +194 -0
- root/generated/openapi_client/models/patched_judge_request.py +110 -0
- root/generated/openapi_client/models/patched_model_request.py +118 -0
- root/generated/openapi_client/models/patched_objective_request.py +113 -0
- root/generated/openapi_client/models/provider.py +99 -0
- root/generated/openapi_client/models/reference_variable.py +123 -0
- root/generated/openapi_client/models/reference_variable_request.py +83 -0
- root/generated/openapi_client/models/skill_execution_validator_result.py +130 -0
- root/generated/openapi_client/models/skill_test_data_request.py +107 -0
- root/generated/openapi_client/models/skill_test_data_request_dataset_range.py +93 -0
- root/generated/openapi_client/models/skill_test_input_request.py +171 -0
- root/generated/openapi_client/models/skill_type_enum.py +36 -0
- root/generated/openapi_client/models/status_change.py +84 -0
- root/generated/openapi_client/models/status_change_request.py +84 -0
- root/generated/openapi_client/models/status_change_status_enum.py +36 -0
- root/generated/openapi_client/models/status_enum.py +38 -0
- root/generated/openapi_client/models/validation_result_status.py +36 -0
- root/generated/openapi_client/models/visibility_enum.py +38 -0
- root/generated/openapi_client/rest.py +203 -0
- root/generated/openapi_client_README.md +238 -0
- root/judges.py +681 -0
- root/models.py +197 -0
- root/objectives.py +343 -0
- root/py.typed +0 -0
- root/skills.py +1707 -0
- root/utils.py +90 -0
- scorable-1.6.4.dist-info/METADATA +395 -0
- scorable-1.6.4.dist-info/RECORD +228 -0
- scorable-1.6.4.dist-info/WHEEL +4 -0
- scorable-1.6.4.dist-info/licenses/LICENSE +202 -0
root/models.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
from contextlib import AbstractAsyncContextManager
|
|
2
|
+
from functools import partial
|
|
3
|
+
from typing import (
|
|
4
|
+
AsyncIterator,
|
|
5
|
+
Iterator,
|
|
6
|
+
List,
|
|
7
|
+
Optional,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
from pydantic import StrictStr
|
|
11
|
+
|
|
12
|
+
from .generated.openapi_aclient import ApiClient as AApiClient
|
|
13
|
+
from .generated.openapi_aclient.api.models_api import ModelsApi as AModelsApi
|
|
14
|
+
from .generated.openapi_aclient.models.model_list import ModelList as AModelItem
|
|
15
|
+
from .generated.openapi_aclient.models.model_request import (
|
|
16
|
+
ModelRequest as AModelRequest,
|
|
17
|
+
)
|
|
18
|
+
from .generated.openapi_aclient.models.paginated_model_list_list import (
|
|
19
|
+
PaginatedModelListList as APaginatedModelListList,
|
|
20
|
+
)
|
|
21
|
+
from .generated.openapi_client import ApiClient
|
|
22
|
+
from .generated.openapi_client.api.models_api import ModelsApi as ModelsApi
|
|
23
|
+
from .generated.openapi_client.models.model_list import ModelList as ModelItem
|
|
24
|
+
from .generated.openapi_client.models.model_request import ModelRequest
|
|
25
|
+
from .utils import (
|
|
26
|
+
ClientContextCallable,
|
|
27
|
+
iterate_cursor_list,
|
|
28
|
+
with_async_client,
|
|
29
|
+
with_sync_client,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Models:
|
|
34
|
+
"""Models (sub) API
|
|
35
|
+
|
|
36
|
+
Note:
|
|
37
|
+
|
|
38
|
+
The construction of the API instance should be handled by
|
|
39
|
+
accesing an attribute of a :class:`root.client.RootSignals` instance.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
client_context: ClientContextCallable,
|
|
45
|
+
):
|
|
46
|
+
self.client_context = client_context
|
|
47
|
+
|
|
48
|
+
@with_sync_client
|
|
49
|
+
def list(
|
|
50
|
+
self,
|
|
51
|
+
*,
|
|
52
|
+
capable_of: Optional[List[str]] = None,
|
|
53
|
+
limit: int = 100,
|
|
54
|
+
_client: ApiClient,
|
|
55
|
+
) -> Iterator[ModelItem]:
|
|
56
|
+
"""Iterate through the models.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
limit: Number of entries to iterate through at most.
|
|
60
|
+
capable_of: List of capabilities to filter the models by.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
api_instance = ModelsApi(_client)
|
|
64
|
+
yield from iterate_cursor_list(partial(api_instance.models_list, capable_of=capable_of), limit=limit)
|
|
65
|
+
|
|
66
|
+
async def alist(
|
|
67
|
+
self,
|
|
68
|
+
*,
|
|
69
|
+
capable_of: Optional[List[str]] = None,
|
|
70
|
+
limit: int = 100,
|
|
71
|
+
) -> AsyncIterator[AModelItem]:
|
|
72
|
+
"""Asynchronously iterate through the models.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
limit: Number of entries to iterate through at most.
|
|
76
|
+
capable_of: List of capabilities to filter the models by.
|
|
77
|
+
"""
|
|
78
|
+
context = self.client_context()
|
|
79
|
+
assert isinstance(context, AbstractAsyncContextManager), "This method is not available in synchronous mode"
|
|
80
|
+
|
|
81
|
+
async with context as client:
|
|
82
|
+
api_instance = AModelsApi(client)
|
|
83
|
+
partial_list = partial(api_instance.models_list, capable_of=capable_of)
|
|
84
|
+
|
|
85
|
+
cursor: Optional[StrictStr] = None
|
|
86
|
+
while limit > 0:
|
|
87
|
+
result: APaginatedModelListList = await partial_list(page_size=limit, cursor=cursor)
|
|
88
|
+
if not result.results:
|
|
89
|
+
return
|
|
90
|
+
used_results = result.results[:limit]
|
|
91
|
+
limit -= len(used_results)
|
|
92
|
+
for used_result in used_results:
|
|
93
|
+
yield used_result
|
|
94
|
+
|
|
95
|
+
if not (cursor := result.next):
|
|
96
|
+
return
|
|
97
|
+
|
|
98
|
+
@with_sync_client
|
|
99
|
+
def create(
|
|
100
|
+
self,
|
|
101
|
+
*,
|
|
102
|
+
name: str,
|
|
103
|
+
model: Optional[str] = None,
|
|
104
|
+
default_key: Optional[str] = None,
|
|
105
|
+
max_output_token_count: Optional[int] = None,
|
|
106
|
+
max_token_count: Optional[int] = None,
|
|
107
|
+
url: Optional[str] = None,
|
|
108
|
+
_request_timeout: Optional[int] = None,
|
|
109
|
+
_client: ApiClient,
|
|
110
|
+
) -> str:
|
|
111
|
+
"""Create a new model and return its ID.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
name: The unique identifier for the model instance (e.g. "google/gemma-2-9b").
|
|
115
|
+
model: The base model name to be used. Defaults to name.
|
|
116
|
+
default_key: The default API key required for the model, if applicable.
|
|
117
|
+
max_output_token_count: The maximum number of tokens to output.
|
|
118
|
+
max_token_count: The maximum number of tokens to process.
|
|
119
|
+
url: Optional URL pointing to the model's endpoint.
|
|
120
|
+
"""
|
|
121
|
+
request = ModelRequest(
|
|
122
|
+
name=name,
|
|
123
|
+
model=model,
|
|
124
|
+
default_key=default_key,
|
|
125
|
+
max_output_token_count=max_output_token_count,
|
|
126
|
+
max_token_count=max_token_count,
|
|
127
|
+
url=url,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
api_instance = ModelsApi(_client)
|
|
131
|
+
return api_instance.models_create(model_request=request, _request_timeout=_request_timeout).id
|
|
132
|
+
|
|
133
|
+
@with_async_client
|
|
134
|
+
async def acreate(
|
|
135
|
+
self,
|
|
136
|
+
*,
|
|
137
|
+
name: str,
|
|
138
|
+
model: Optional[str] = None,
|
|
139
|
+
default_key: Optional[str] = None,
|
|
140
|
+
max_output_token_count: Optional[int] = None,
|
|
141
|
+
max_token_count: Optional[int] = None,
|
|
142
|
+
url: Optional[str] = None,
|
|
143
|
+
_request_timeout: Optional[int] = None,
|
|
144
|
+
_client: AApiClient,
|
|
145
|
+
) -> str:
|
|
146
|
+
"""Asynchronously create a new model and return its ID.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
name: The unique identifier for the model instance (e.g. "google/gemma-2-9b").
|
|
150
|
+
model: The base model name to be used. Defaults to name.
|
|
151
|
+
default_key: The default API key required for the model, if applicable.
|
|
152
|
+
max_output_token_count: The maximum number of tokens to output.
|
|
153
|
+
max_token_count: The maximum number of tokens to process.
|
|
154
|
+
url: Optional URL pointing to the model's endpoint.
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
request = AModelRequest(
|
|
158
|
+
name=name,
|
|
159
|
+
model=model,
|
|
160
|
+
default_key=default_key,
|
|
161
|
+
max_output_token_count=max_output_token_count,
|
|
162
|
+
max_token_count=max_token_count,
|
|
163
|
+
url=url,
|
|
164
|
+
)
|
|
165
|
+
api_instance = AModelsApi(_client)
|
|
166
|
+
created_model = await api_instance.models_create(model_request=request, _request_timeout=_request_timeout)
|
|
167
|
+
return created_model.id
|
|
168
|
+
|
|
169
|
+
@with_sync_client
|
|
170
|
+
def delete(
|
|
171
|
+
self,
|
|
172
|
+
model_id: str,
|
|
173
|
+
*,
|
|
174
|
+
_request_timeout: Optional[int] = None,
|
|
175
|
+
_client: ApiClient,
|
|
176
|
+
) -> None:
|
|
177
|
+
"""
|
|
178
|
+
Delete the model.
|
|
179
|
+
"""
|
|
180
|
+
api_instance = ModelsApi(_client)
|
|
181
|
+
return api_instance.models_destroy(id=model_id, _request_timeout=_request_timeout)
|
|
182
|
+
|
|
183
|
+
@with_async_client
|
|
184
|
+
async def adelete(
|
|
185
|
+
self,
|
|
186
|
+
model_id: str,
|
|
187
|
+
*,
|
|
188
|
+
_request_timeout: Optional[int] = None,
|
|
189
|
+
_client: AApiClient,
|
|
190
|
+
) -> None:
|
|
191
|
+
"""
|
|
192
|
+
Asynchronously delete the model.
|
|
193
|
+
"""
|
|
194
|
+
api_instance = AModelsApi(_client)
|
|
195
|
+
return await api_instance.models_destroy(id=model_id, _request_timeout=_request_timeout)
|
|
196
|
+
|
|
197
|
+
# TODO: update
|
root/objectives.py
ADDED
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from contextlib import AbstractAsyncContextManager
|
|
4
|
+
from functools import partial
|
|
5
|
+
from typing import AsyncIterator, Iterator, Optional, cast
|
|
6
|
+
|
|
7
|
+
from pydantic import StrictStr
|
|
8
|
+
|
|
9
|
+
from .generated.openapi_aclient import ApiClient as AApiClient
|
|
10
|
+
from .generated.openapi_aclient.api.objectives_api import ObjectivesApi as AObjectivesApi
|
|
11
|
+
from .generated.openapi_aclient.models.objective import Objective as AOpenApiObjective
|
|
12
|
+
from .generated.openapi_aclient.models.objective_list import ObjectiveList as AObjectiveList
|
|
13
|
+
from .generated.openapi_aclient.models.objective_request import ObjectiveRequest as AObjectiveRequest
|
|
14
|
+
from .generated.openapi_aclient.models.paginated_objective_list import (
|
|
15
|
+
PaginatedObjectiveList as APaginatedObjectiveList,
|
|
16
|
+
)
|
|
17
|
+
from .generated.openapi_aclient.models.paginated_objective_list_list import (
|
|
18
|
+
PaginatedObjectiveListList as APaginatedObjectiveListList,
|
|
19
|
+
)
|
|
20
|
+
from .generated.openapi_aclient.models.patched_objective_request import (
|
|
21
|
+
PatchedObjectiveRequest as APatchedObjectiveRequest,
|
|
22
|
+
)
|
|
23
|
+
from .generated.openapi_client import ApiClient
|
|
24
|
+
from .generated.openapi_client.api.objectives_api import ObjectivesApi as ObjectivesApi
|
|
25
|
+
from .generated.openapi_client.models.objective import Objective as OpenApiObjective
|
|
26
|
+
from .generated.openapi_client.models.objective_list import ObjectiveList
|
|
27
|
+
from .generated.openapi_client.models.objective_request import ObjectiveRequest
|
|
28
|
+
from .generated.openapi_client.models.paginated_objective_list import PaginatedObjectiveList
|
|
29
|
+
from .generated.openapi_client.models.patched_objective_request import PatchedObjectiveRequest
|
|
30
|
+
from .utils import ClientContextCallable, iterate_cursor_list, with_async_client, with_sync_client
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Versions:
|
|
34
|
+
"""Version listing (sub)API
|
|
35
|
+
|
|
36
|
+
Note that this should not be directly instantiated.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(self, client_context: ClientContextCallable):
|
|
40
|
+
self.client_context = client_context
|
|
41
|
+
|
|
42
|
+
@with_sync_client
|
|
43
|
+
def list(self, objective_id: str, *, _client: ApiClient) -> PaginatedObjectiveList:
|
|
44
|
+
"""List all versions of an objective.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
objective_id: The objective to list the versions for
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
api_instance = ObjectivesApi(_client)
|
|
51
|
+
return api_instance.objectives_versions_list(id=objective_id)
|
|
52
|
+
|
|
53
|
+
async def alist(self, objective_id: str) -> APaginatedObjectiveList:
|
|
54
|
+
"""Asynchronously list all versions of an objective.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
objective_id: The objective to list the versions for
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
context = self.client_context()
|
|
61
|
+
assert isinstance(context, AbstractAsyncContextManager), "This method is not available in synchronous mode"
|
|
62
|
+
async with context as client:
|
|
63
|
+
api_instance = AObjectivesApi(client)
|
|
64
|
+
return await api_instance.objectives_versions_list(id=objective_id)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Objective(OpenApiObjective):
|
|
68
|
+
"""Wrapper for a single Objective.
|
|
69
|
+
|
|
70
|
+
For available attributes, please check the (automatically
|
|
71
|
+
generated) superclass documentation.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
client_context: ClientContextCallable
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
def _wrap(cls, apiobj: OpenApiObjective, client_context: ClientContextCallable) -> "Objective":
|
|
78
|
+
if not isinstance(apiobj, OpenApiObjective):
|
|
79
|
+
raise ValueError(f"Wrong instance in _wrap: {apiobj!r}")
|
|
80
|
+
obj = cast(Objective, apiobj)
|
|
81
|
+
obj.__class__ = cls
|
|
82
|
+
obj.client_context = client_context
|
|
83
|
+
return obj
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class AObjective(AOpenApiObjective):
|
|
87
|
+
"""
|
|
88
|
+
Wrapper for a single Objective.
|
|
89
|
+
|
|
90
|
+
For available attributes, please check the (automatically
|
|
91
|
+
generated) superclass documentation.
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
client_context: ClientContextCallable
|
|
95
|
+
|
|
96
|
+
@classmethod
|
|
97
|
+
async def _awrap(cls, apiobj: AOpenApiObjective, client_context: ClientContextCallable) -> "AObjective":
|
|
98
|
+
if not isinstance(apiobj, AOpenApiObjective):
|
|
99
|
+
raise ValueError(f"Wrong instance in _wrap: {apiobj!r}")
|
|
100
|
+
obj = cast(AObjective, apiobj)
|
|
101
|
+
obj.__class__ = cls
|
|
102
|
+
obj.client_context = client_context
|
|
103
|
+
return obj
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class Objectives:
|
|
107
|
+
"""
|
|
108
|
+
Objectives API
|
|
109
|
+
|
|
110
|
+
Note:
|
|
111
|
+
|
|
112
|
+
The construction of the API instance should be handled by
|
|
113
|
+
accesing an attribute of a :class:`root.client.RootSignals` instance.
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
def __init__(self, client_context: ClientContextCallable):
|
|
117
|
+
self.client_context = client_context
|
|
118
|
+
self.versions = Versions(client_context)
|
|
119
|
+
|
|
120
|
+
@with_sync_client
|
|
121
|
+
def create(
|
|
122
|
+
self,
|
|
123
|
+
*,
|
|
124
|
+
intent: Optional[str] = None,
|
|
125
|
+
test_dataset_id: Optional[str] = None,
|
|
126
|
+
_request_timeout: Optional[int] = None,
|
|
127
|
+
_client: ApiClient,
|
|
128
|
+
) -> Objective:
|
|
129
|
+
"""
|
|
130
|
+
Create a new objective and return its ID.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
intent: The intent of the objective.
|
|
134
|
+
test_dataset_id: The ID of the test dataset
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
request = ObjectiveRequest(
|
|
138
|
+
intent=intent,
|
|
139
|
+
test_dataset_id=test_dataset_id,
|
|
140
|
+
)
|
|
141
|
+
api_instance = ObjectivesApi(_client)
|
|
142
|
+
objective = api_instance.objectives_create(objective_request=request)
|
|
143
|
+
return self.get(objective.id, _request_timeout=_request_timeout)
|
|
144
|
+
|
|
145
|
+
@with_async_client
|
|
146
|
+
async def acreate(
|
|
147
|
+
self,
|
|
148
|
+
*,
|
|
149
|
+
intent: Optional[str] = None,
|
|
150
|
+
test_dataset_id: Optional[str] = None,
|
|
151
|
+
_request_timeout: Optional[int] = None,
|
|
152
|
+
_client: AApiClient,
|
|
153
|
+
) -> AObjective:
|
|
154
|
+
"""
|
|
155
|
+
Asynchronously create a new objective and return its ID.
|
|
156
|
+
|
|
157
|
+
Args:
|
|
158
|
+
intent: The intent of the objective.
|
|
159
|
+
test_dataset_id: The ID of the test dataset
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
request = AObjectiveRequest(
|
|
163
|
+
intent=intent,
|
|
164
|
+
test_dataset_id=test_dataset_id,
|
|
165
|
+
)
|
|
166
|
+
api_instance = AObjectivesApi(_client)
|
|
167
|
+
objective = await api_instance.objectives_create(objective_request=request)
|
|
168
|
+
return await self.aget(objective.id, _request_timeout=_request_timeout)
|
|
169
|
+
|
|
170
|
+
@with_sync_client
|
|
171
|
+
def get(
|
|
172
|
+
self,
|
|
173
|
+
objective_id: str,
|
|
174
|
+
*,
|
|
175
|
+
_client: ApiClient,
|
|
176
|
+
_request_timeout: Optional[int] = None,
|
|
177
|
+
) -> Objective:
|
|
178
|
+
"""
|
|
179
|
+
Get an objective by ID.
|
|
180
|
+
|
|
181
|
+
Args:
|
|
182
|
+
objective_id: The objective to be fetched.
|
|
183
|
+
"""
|
|
184
|
+
|
|
185
|
+
api_instance = ObjectivesApi(_client)
|
|
186
|
+
return Objective._wrap(
|
|
187
|
+
api_instance.objectives_retrieve(id=objective_id, _request_timeout=_request_timeout),
|
|
188
|
+
client_context=self.client_context,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
@with_async_client
|
|
192
|
+
async def aget(
|
|
193
|
+
self,
|
|
194
|
+
objective_id: str,
|
|
195
|
+
*,
|
|
196
|
+
_client: AApiClient,
|
|
197
|
+
_request_timeout: Optional[int] = None,
|
|
198
|
+
) -> AObjective:
|
|
199
|
+
"""
|
|
200
|
+
Asynchronously get an objective by ID.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
objective_id: The objective to be fetched.
|
|
204
|
+
"""
|
|
205
|
+
|
|
206
|
+
api_instance = AObjectivesApi(_client)
|
|
207
|
+
return await AObjective._awrap(
|
|
208
|
+
await api_instance.objectives_retrieve(id=objective_id, _request_timeout=_request_timeout),
|
|
209
|
+
client_context=self.client_context,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
@with_sync_client
|
|
213
|
+
def delete(self, objective_id: str, *, _request_timeout: Optional[int] = None, _client: ApiClient) -> None:
|
|
214
|
+
"""
|
|
215
|
+
Delete the objective from the registry.
|
|
216
|
+
|
|
217
|
+
Args:
|
|
218
|
+
objective_id: The objective to be deleted.
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
api_instance = ObjectivesApi(_client)
|
|
222
|
+
return api_instance.objectives_destroy(id=objective_id, _request_timeout=_request_timeout)
|
|
223
|
+
|
|
224
|
+
@with_async_client
|
|
225
|
+
async def adelete(self, objective_id: str, *, _request_timeout: Optional[int] = None, _client: AApiClient) -> None:
|
|
226
|
+
"""
|
|
227
|
+
Asynchronously delete the objective from the registry.
|
|
228
|
+
|
|
229
|
+
Args:
|
|
230
|
+
objective_id: The objective to be deleted.
|
|
231
|
+
"""
|
|
232
|
+
|
|
233
|
+
api_instance = AObjectivesApi(_client)
|
|
234
|
+
return await api_instance.objectives_destroy(id=objective_id, _request_timeout=_request_timeout)
|
|
235
|
+
|
|
236
|
+
@with_sync_client
|
|
237
|
+
def list(self, *, intent: Optional[str] = None, limit: int = 100, _client: ApiClient) -> Iterator[ObjectiveList]:
|
|
238
|
+
"""
|
|
239
|
+
Iterate through the objectives.
|
|
240
|
+
|
|
241
|
+
Args:
|
|
242
|
+
intent: Specific intent the returned objectives must match.
|
|
243
|
+
limit: Number of entries to iterate through at most.
|
|
244
|
+
"""
|
|
245
|
+
|
|
246
|
+
api_instance = ObjectivesApi(_client)
|
|
247
|
+
yield from iterate_cursor_list(partial(api_instance.objectives_list, intent=intent), limit=limit)
|
|
248
|
+
|
|
249
|
+
@with_async_client
|
|
250
|
+
async def alist(self, *, intent: Optional[str] = None, limit: int = 100) -> AsyncIterator[AObjectiveList]:
|
|
251
|
+
"""
|
|
252
|
+
Asynchronously iterate through the objectives.
|
|
253
|
+
|
|
254
|
+
Args:
|
|
255
|
+
intent: Specific intent the returned objectives must match.
|
|
256
|
+
limit: Number of entries to iterate through at most.
|
|
257
|
+
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
context = self.client_context()
|
|
261
|
+
assert isinstance(context, AbstractAsyncContextManager), "This method is not available in synchronous mode"
|
|
262
|
+
async with context as client:
|
|
263
|
+
api_instance = AObjectivesApi(client)
|
|
264
|
+
partial_list = partial(api_instance.objectives_list, intent=intent)
|
|
265
|
+
|
|
266
|
+
cursor: Optional[StrictStr] = None
|
|
267
|
+
while limit > 0:
|
|
268
|
+
result: APaginatedObjectiveListList = await partial_list(page_size=limit, cursor=cursor)
|
|
269
|
+
if not result.results:
|
|
270
|
+
return
|
|
271
|
+
|
|
272
|
+
used_results = result.results[:limit]
|
|
273
|
+
limit -= len(used_results)
|
|
274
|
+
for used_result in used_results:
|
|
275
|
+
yield used_result
|
|
276
|
+
|
|
277
|
+
if not (cursor := result.next):
|
|
278
|
+
return
|
|
279
|
+
|
|
280
|
+
@with_sync_client
|
|
281
|
+
def update(
|
|
282
|
+
self,
|
|
283
|
+
objective_id: str,
|
|
284
|
+
*,
|
|
285
|
+
intent: Optional[str] = None,
|
|
286
|
+
test_dataset_id: Optional[str] = None,
|
|
287
|
+
_request_timeout: Optional[int] = None,
|
|
288
|
+
_client: ApiClient,
|
|
289
|
+
) -> Objective:
|
|
290
|
+
"""
|
|
291
|
+
Update an existing objective.
|
|
292
|
+
|
|
293
|
+
Args:
|
|
294
|
+
objective_id: The objective to be updated.
|
|
295
|
+
intent: The intent of the objective.
|
|
296
|
+
"""
|
|
297
|
+
|
|
298
|
+
request = PatchedObjectiveRequest(
|
|
299
|
+
intent=intent,
|
|
300
|
+
test_dataset_id=test_dataset_id,
|
|
301
|
+
)
|
|
302
|
+
api_instance = ObjectivesApi(_client)
|
|
303
|
+
return Objective._wrap(
|
|
304
|
+
api_instance.objectives_partial_update(
|
|
305
|
+
id=objective_id,
|
|
306
|
+
patched_objective_request=request,
|
|
307
|
+
_request_timeout=_request_timeout,
|
|
308
|
+
),
|
|
309
|
+
client_context=self.client_context,
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
@with_async_client
|
|
313
|
+
async def aupdate(
|
|
314
|
+
self,
|
|
315
|
+
objective_id: str,
|
|
316
|
+
*,
|
|
317
|
+
intent: Optional[str] = None,
|
|
318
|
+
test_dataset_id: Optional[str] = None,
|
|
319
|
+
_request_timeout: Optional[int] = None,
|
|
320
|
+
_client: AApiClient,
|
|
321
|
+
) -> AObjective:
|
|
322
|
+
"""
|
|
323
|
+
Asynchronously update an existing objective.
|
|
324
|
+
|
|
325
|
+
Args:
|
|
326
|
+
objective_id: The objective to be updated.
|
|
327
|
+
intent: The intent of the objective.
|
|
328
|
+
|
|
329
|
+
"""
|
|
330
|
+
|
|
331
|
+
request = APatchedObjectiveRequest(
|
|
332
|
+
intent=intent,
|
|
333
|
+
test_dataset_id=test_dataset_id,
|
|
334
|
+
)
|
|
335
|
+
api_instance = AObjectivesApi(_client)
|
|
336
|
+
return await AObjective._awrap(
|
|
337
|
+
await api_instance.objectives_partial_update(
|
|
338
|
+
id=objective_id,
|
|
339
|
+
patched_objective_request=request,
|
|
340
|
+
_request_timeout=_request_timeout,
|
|
341
|
+
),
|
|
342
|
+
client_context=self.client_context,
|
|
343
|
+
)
|
root/py.typed
ADDED
|
File without changes
|