scorable 1.6.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of scorable might be problematic. Click here for more details.

Files changed (228) hide show
  1. root/__about__.py +4 -0
  2. root/__init__.py +17 -0
  3. root/client.py +207 -0
  4. root/datasets.py +231 -0
  5. root/execution_logs.py +162 -0
  6. root/generated/__init__.py +0 -0
  7. root/generated/openapi_aclient/__init__.py +1 -0
  8. root/generated/openapi_aclient/api/__init__.py +1 -0
  9. root/generated/openapi_aclient/api/datasets_api.py +1274 -0
  10. root/generated/openapi_aclient/api/evaluators_api.py +3641 -0
  11. root/generated/openapi_aclient/api/execution_logs_api.py +751 -0
  12. root/generated/openapi_aclient/api/judges_api.py +3794 -0
  13. root/generated/openapi_aclient/api/models_api.py +1473 -0
  14. root/generated/openapi_aclient/api/objectives_api.py +1767 -0
  15. root/generated/openapi_aclient/api_client.py +662 -0
  16. root/generated/openapi_aclient/api_response.py +22 -0
  17. root/generated/openapi_aclient/configuration.py +470 -0
  18. root/generated/openapi_aclient/exceptions.py +197 -0
  19. root/generated/openapi_aclient/models/__init__.py +122 -0
  20. root/generated/openapi_aclient/models/data_set_create.py +118 -0
  21. root/generated/openapi_aclient/models/data_set_create_request.py +105 -0
  22. root/generated/openapi_aclient/models/data_set_list.py +129 -0
  23. root/generated/openapi_aclient/models/data_set_type.py +36 -0
  24. root/generated/openapi_aclient/models/dataset_range_request.py +93 -0
  25. root/generated/openapi_aclient/models/evaluator.py +273 -0
  26. root/generated/openapi_aclient/models/evaluator_calibration_output.py +101 -0
  27. root/generated/openapi_aclient/models/evaluator_calibration_result.py +134 -0
  28. root/generated/openapi_aclient/models/evaluator_demonstrations.py +107 -0
  29. root/generated/openapi_aclient/models/evaluator_demonstrations_request.py +107 -0
  30. root/generated/openapi_aclient/models/evaluator_execution_function_parameter_property_request.py +86 -0
  31. root/generated/openapi_aclient/models/evaluator_execution_function_parameter_request.py +109 -0
  32. root/generated/openapi_aclient/models/evaluator_execution_function_parameter_type_enum.py +35 -0
  33. root/generated/openapi_aclient/models/evaluator_execution_function_request.py +99 -0
  34. root/generated/openapi_aclient/models/evaluator_execution_functions_request.py +98 -0
  35. root/generated/openapi_aclient/models/evaluator_execution_functions_type_enum.py +35 -0
  36. root/generated/openapi_aclient/models/evaluator_execution_request.py +134 -0
  37. root/generated/openapi_aclient/models/evaluator_execution_result.py +114 -0
  38. root/generated/openapi_aclient/models/evaluator_inputs_value.py +100 -0
  39. root/generated/openapi_aclient/models/evaluator_inputs_value_items.py +89 -0
  40. root/generated/openapi_aclient/models/evaluator_list_output.py +198 -0
  41. root/generated/openapi_aclient/models/evaluator_reference.py +90 -0
  42. root/generated/openapi_aclient/models/evaluator_reference_request.py +90 -0
  43. root/generated/openapi_aclient/models/evaluator_request.py +194 -0
  44. root/generated/openapi_aclient/models/evaluator_result.py +110 -0
  45. root/generated/openapi_aclient/models/execution_log_details.py +291 -0
  46. root/generated/openapi_aclient/models/execution_log_details_evaluation_context.py +83 -0
  47. root/generated/openapi_aclient/models/execution_log_details_evaluator_latencies_inner.py +83 -0
  48. root/generated/openapi_aclient/models/execution_log_list.py +217 -0
  49. root/generated/openapi_aclient/models/execution_log_list_evaluation_context.py +83 -0
  50. root/generated/openapi_aclient/models/generation_model_params_request.py +93 -0
  51. root/generated/openapi_aclient/models/id.py +87 -0
  52. root/generated/openapi_aclient/models/input_variable.py +121 -0
  53. root/generated/openapi_aclient/models/input_variable_request.py +82 -0
  54. root/generated/openapi_aclient/models/judge.py +178 -0
  55. root/generated/openapi_aclient/models/judge_execution_request.py +114 -0
  56. root/generated/openapi_aclient/models/judge_execution_response.py +97 -0
  57. root/generated/openapi_aclient/models/judge_files_inner.py +84 -0
  58. root/generated/openapi_aclient/models/judge_generator_request.py +142 -0
  59. root/generated/openapi_aclient/models/judge_generator_response.py +88 -0
  60. root/generated/openapi_aclient/models/judge_invite_request.py +87 -0
  61. root/generated/openapi_aclient/models/judge_list.py +156 -0
  62. root/generated/openapi_aclient/models/judge_rectifier_request_request.py +114 -0
  63. root/generated/openapi_aclient/models/judge_rectifier_response.py +121 -0
  64. root/generated/openapi_aclient/models/judge_request.py +108 -0
  65. root/generated/openapi_aclient/models/model.py +126 -0
  66. root/generated/openapi_aclient/models/model_list.py +115 -0
  67. root/generated/openapi_aclient/models/model_params.py +89 -0
  68. root/generated/openapi_aclient/models/model_params_request.py +89 -0
  69. root/generated/openapi_aclient/models/model_request.py +118 -0
  70. root/generated/openapi_aclient/models/nested_evaluator.py +110 -0
  71. root/generated/openapi_aclient/models/nested_evaluator_objective.py +87 -0
  72. root/generated/openapi_aclient/models/nested_evaluator_request.py +92 -0
  73. root/generated/openapi_aclient/models/nested_objective_evaluator.py +105 -0
  74. root/generated/openapi_aclient/models/nested_objective_evaluator_request.py +92 -0
  75. root/generated/openapi_aclient/models/nested_objective_list.py +111 -0
  76. root/generated/openapi_aclient/models/nested_user_details.py +88 -0
  77. root/generated/openapi_aclient/models/nested_user_details_request.py +82 -0
  78. root/generated/openapi_aclient/models/nested_vector_objective.py +88 -0
  79. root/generated/openapi_aclient/models/nested_vector_objective_request.py +82 -0
  80. root/generated/openapi_aclient/models/objective.py +157 -0
  81. root/generated/openapi_aclient/models/objective_list.py +128 -0
  82. root/generated/openapi_aclient/models/objective_request.py +113 -0
  83. root/generated/openapi_aclient/models/objective_validator.py +100 -0
  84. root/generated/openapi_aclient/models/objective_validator_request.py +90 -0
  85. root/generated/openapi_aclient/models/paginated_data_set_list_list.py +111 -0
  86. root/generated/openapi_aclient/models/paginated_evaluator_list.py +111 -0
  87. root/generated/openapi_aclient/models/paginated_evaluator_list_output_list.py +111 -0
  88. root/generated/openapi_aclient/models/paginated_execution_log_list_list.py +111 -0
  89. root/generated/openapi_aclient/models/paginated_judge_list_list.py +111 -0
  90. root/generated/openapi_aclient/models/paginated_model_list_list.py +111 -0
  91. root/generated/openapi_aclient/models/paginated_objective_list.py +111 -0
  92. root/generated/openapi_aclient/models/paginated_objective_list_list.py +111 -0
  93. root/generated/openapi_aclient/models/patched_evaluator_request.py +194 -0
  94. root/generated/openapi_aclient/models/patched_judge_request.py +110 -0
  95. root/generated/openapi_aclient/models/patched_model_request.py +118 -0
  96. root/generated/openapi_aclient/models/patched_objective_request.py +113 -0
  97. root/generated/openapi_aclient/models/provider.py +99 -0
  98. root/generated/openapi_aclient/models/reference_variable.py +123 -0
  99. root/generated/openapi_aclient/models/reference_variable_request.py +83 -0
  100. root/generated/openapi_aclient/models/skill_execution_validator_result.py +130 -0
  101. root/generated/openapi_aclient/models/skill_test_data_request.py +107 -0
  102. root/generated/openapi_aclient/models/skill_test_data_request_dataset_range.py +93 -0
  103. root/generated/openapi_aclient/models/skill_test_input_request.py +171 -0
  104. root/generated/openapi_aclient/models/skill_type_enum.py +36 -0
  105. root/generated/openapi_aclient/models/status_change.py +84 -0
  106. root/generated/openapi_aclient/models/status_change_request.py +84 -0
  107. root/generated/openapi_aclient/models/status_change_status_enum.py +36 -0
  108. root/generated/openapi_aclient/models/status_enum.py +38 -0
  109. root/generated/openapi_aclient/models/validation_result_status.py +36 -0
  110. root/generated/openapi_aclient/models/visibility_enum.py +38 -0
  111. root/generated/openapi_aclient/rest.py +166 -0
  112. root/generated/openapi_aclient_README.md +239 -0
  113. root/generated/openapi_client/__init__.py +1 -0
  114. root/generated/openapi_client/api/__init__.py +1 -0
  115. root/generated/openapi_client/api/datasets_api.py +1274 -0
  116. root/generated/openapi_client/api/evaluators_api.py +3641 -0
  117. root/generated/openapi_client/api/execution_logs_api.py +751 -0
  118. root/generated/openapi_client/api/judges_api.py +3794 -0
  119. root/generated/openapi_client/api/models_api.py +1473 -0
  120. root/generated/openapi_client/api/objectives_api.py +1767 -0
  121. root/generated/openapi_client/api_client.py +659 -0
  122. root/generated/openapi_client/api_response.py +22 -0
  123. root/generated/openapi_client/configuration.py +474 -0
  124. root/generated/openapi_client/exceptions.py +197 -0
  125. root/generated/openapi_client/models/__init__.py +120 -0
  126. root/generated/openapi_client/models/data_set_create.py +118 -0
  127. root/generated/openapi_client/models/data_set_create_request.py +105 -0
  128. root/generated/openapi_client/models/data_set_list.py +129 -0
  129. root/generated/openapi_client/models/data_set_type.py +36 -0
  130. root/generated/openapi_client/models/dataset_range_request.py +93 -0
  131. root/generated/openapi_client/models/evaluator.py +273 -0
  132. root/generated/openapi_client/models/evaluator_calibration_output.py +101 -0
  133. root/generated/openapi_client/models/evaluator_calibration_result.py +134 -0
  134. root/generated/openapi_client/models/evaluator_demonstrations.py +107 -0
  135. root/generated/openapi_client/models/evaluator_demonstrations_request.py +107 -0
  136. root/generated/openapi_client/models/evaluator_execution_function_parameter_property_request.py +86 -0
  137. root/generated/openapi_client/models/evaluator_execution_function_parameter_request.py +109 -0
  138. root/generated/openapi_client/models/evaluator_execution_function_parameter_type_enum.py +35 -0
  139. root/generated/openapi_client/models/evaluator_execution_function_request.py +99 -0
  140. root/generated/openapi_client/models/evaluator_execution_functions_request.py +98 -0
  141. root/generated/openapi_client/models/evaluator_execution_functions_type_enum.py +35 -0
  142. root/generated/openapi_client/models/evaluator_execution_request.py +134 -0
  143. root/generated/openapi_client/models/evaluator_execution_result.py +114 -0
  144. root/generated/openapi_client/models/evaluator_inputs_value.py +100 -0
  145. root/generated/openapi_client/models/evaluator_inputs_value_items.py +89 -0
  146. root/generated/openapi_client/models/evaluator_list_output.py +198 -0
  147. root/generated/openapi_client/models/evaluator_reference.py +90 -0
  148. root/generated/openapi_client/models/evaluator_reference_request.py +90 -0
  149. root/generated/openapi_client/models/evaluator_request.py +194 -0
  150. root/generated/openapi_client/models/evaluator_result.py +110 -0
  151. root/generated/openapi_client/models/execution_log_details.py +291 -0
  152. root/generated/openapi_client/models/execution_log_details_evaluation_context.py +83 -0
  153. root/generated/openapi_client/models/execution_log_details_evaluator_latencies_inner.py +83 -0
  154. root/generated/openapi_client/models/execution_log_list.py +215 -0
  155. root/generated/openapi_client/models/execution_log_list_evaluation_context.py +83 -0
  156. root/generated/openapi_client/models/generation_model_params_request.py +93 -0
  157. root/generated/openapi_client/models/id.py +87 -0
  158. root/generated/openapi_client/models/input_variable.py +121 -0
  159. root/generated/openapi_client/models/input_variable_request.py +82 -0
  160. root/generated/openapi_client/models/judge.py +178 -0
  161. root/generated/openapi_client/models/judge_execution_request.py +114 -0
  162. root/generated/openapi_client/models/judge_execution_response.py +97 -0
  163. root/generated/openapi_client/models/judge_files_inner.py +84 -0
  164. root/generated/openapi_client/models/judge_generator_request.py +142 -0
  165. root/generated/openapi_client/models/judge_generator_response.py +88 -0
  166. root/generated/openapi_client/models/judge_invite_request.py +87 -0
  167. root/generated/openapi_client/models/judge_list.py +156 -0
  168. root/generated/openapi_client/models/judge_rectifier_request_request.py +114 -0
  169. root/generated/openapi_client/models/judge_rectifier_response.py +121 -0
  170. root/generated/openapi_client/models/judge_request.py +108 -0
  171. root/generated/openapi_client/models/model.py +126 -0
  172. root/generated/openapi_client/models/model_list.py +115 -0
  173. root/generated/openapi_client/models/model_params.py +89 -0
  174. root/generated/openapi_client/models/model_params_request.py +89 -0
  175. root/generated/openapi_client/models/model_request.py +118 -0
  176. root/generated/openapi_client/models/nested_evaluator.py +110 -0
  177. root/generated/openapi_client/models/nested_evaluator_objective.py +87 -0
  178. root/generated/openapi_client/models/nested_evaluator_request.py +92 -0
  179. root/generated/openapi_client/models/nested_objective_evaluator.py +105 -0
  180. root/generated/openapi_client/models/nested_objective_evaluator_request.py +92 -0
  181. root/generated/openapi_client/models/nested_objective_list.py +111 -0
  182. root/generated/openapi_client/models/nested_user_details.py +88 -0
  183. root/generated/openapi_client/models/nested_user_details_request.py +82 -0
  184. root/generated/openapi_client/models/nested_vector_objective.py +88 -0
  185. root/generated/openapi_client/models/nested_vector_objective_request.py +82 -0
  186. root/generated/openapi_client/models/objective.py +157 -0
  187. root/generated/openapi_client/models/objective_list.py +128 -0
  188. root/generated/openapi_client/models/objective_request.py +113 -0
  189. root/generated/openapi_client/models/objective_validator.py +100 -0
  190. root/generated/openapi_client/models/objective_validator_request.py +90 -0
  191. root/generated/openapi_client/models/paginated_data_set_list_list.py +111 -0
  192. root/generated/openapi_client/models/paginated_evaluator_list.py +111 -0
  193. root/generated/openapi_client/models/paginated_evaluator_list_output_list.py +111 -0
  194. root/generated/openapi_client/models/paginated_execution_log_list_list.py +111 -0
  195. root/generated/openapi_client/models/paginated_judge_list_list.py +111 -0
  196. root/generated/openapi_client/models/paginated_model_list_list.py +111 -0
  197. root/generated/openapi_client/models/paginated_objective_list.py +111 -0
  198. root/generated/openapi_client/models/paginated_objective_list_list.py +111 -0
  199. root/generated/openapi_client/models/patched_evaluator_request.py +194 -0
  200. root/generated/openapi_client/models/patched_judge_request.py +110 -0
  201. root/generated/openapi_client/models/patched_model_request.py +118 -0
  202. root/generated/openapi_client/models/patched_objective_request.py +113 -0
  203. root/generated/openapi_client/models/provider.py +99 -0
  204. root/generated/openapi_client/models/reference_variable.py +123 -0
  205. root/generated/openapi_client/models/reference_variable_request.py +83 -0
  206. root/generated/openapi_client/models/skill_execution_validator_result.py +130 -0
  207. root/generated/openapi_client/models/skill_test_data_request.py +107 -0
  208. root/generated/openapi_client/models/skill_test_data_request_dataset_range.py +93 -0
  209. root/generated/openapi_client/models/skill_test_input_request.py +171 -0
  210. root/generated/openapi_client/models/skill_type_enum.py +36 -0
  211. root/generated/openapi_client/models/status_change.py +84 -0
  212. root/generated/openapi_client/models/status_change_request.py +84 -0
  213. root/generated/openapi_client/models/status_change_status_enum.py +36 -0
  214. root/generated/openapi_client/models/status_enum.py +38 -0
  215. root/generated/openapi_client/models/validation_result_status.py +36 -0
  216. root/generated/openapi_client/models/visibility_enum.py +38 -0
  217. root/generated/openapi_client/rest.py +203 -0
  218. root/generated/openapi_client_README.md +238 -0
  219. root/judges.py +681 -0
  220. root/models.py +197 -0
  221. root/objectives.py +343 -0
  222. root/py.typed +0 -0
  223. root/skills.py +1707 -0
  224. root/utils.py +90 -0
  225. scorable-1.6.4.dist-info/METADATA +395 -0
  226. scorable-1.6.4.dist-info/RECORD +228 -0
  227. scorable-1.6.4.dist-info/WHEEL +4 -0
  228. scorable-1.6.4.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,751 @@
1
+ # coding: utf-8
2
+
3
+ """
4
+ Root Signals API
5
+
6
+ Root Signals JSON API provides a way to access Root Signals using provisioned API token
7
+
8
+ The version of the OpenAPI document: 1.0.0 (latest)
9
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
10
+
11
+ Do not edit the class manually.
12
+ """ # noqa: E501
13
+
14
+ import warnings
15
+ from datetime import datetime
16
+ from typing import Any, Dict, List, Optional, Tuple, Union
17
+
18
+ from pydantic import Field, StrictBool, StrictFloat, StrictInt, StrictStr, field_validator, validate_call
19
+ from typing_extensions import Annotated
20
+
21
+ from root.generated.openapi_client.api_client import ApiClient, RequestSerialized
22
+ from root.generated.openapi_client.api_response import ApiResponse
23
+ from root.generated.openapi_client.models.execution_log_details import ExecutionLogDetails
24
+ from root.generated.openapi_client.models.paginated_execution_log_list_list import PaginatedExecutionLogListList
25
+ from root.generated.openapi_client.rest import RESTResponseType
26
+
27
+
28
+ class ExecutionLogsApi:
29
+ """NOTE: This class is auto generated by OpenAPI Generator
30
+ Ref: https://openapi-generator.tech
31
+
32
+ Do not edit the class manually.
33
+ """
34
+
35
+ def __init__(self, api_client=None) -> None:
36
+ if api_client is None:
37
+ api_client = ApiClient.get_default()
38
+ self.api_client = api_client
39
+
40
+ @validate_call
41
+ def execution_logs_list(
42
+ self,
43
+ cursor: Annotated[Optional[StrictStr], Field(description="The pagination cursor value.")] = None,
44
+ date_from: Annotated[
45
+ Optional[datetime], Field(description="Filter logs created after or at this datetime (inclusive)")
46
+ ] = None,
47
+ date_to: Annotated[
48
+ Optional[datetime], Field(description="Filter logs created before or at this datetime (inclusive)")
49
+ ] = None,
50
+ executed_item_id: Optional[StrictStr] = None,
51
+ execution_type: Annotated[
52
+ Optional[StrictStr],
53
+ Field(
54
+ description="* `skill` - skill * `evaluator` - evaluator * `judge` - judge * `experiment` - experiment * `test` - test * `proxy` - proxy"
55
+ ),
56
+ ] = None,
57
+ include: Annotated[
58
+ Optional[StrictStr],
59
+ Field(
60
+ description="Comma-separated list of additional fields to include in the response. Supports: llm_output, variables, evaluation_context"
61
+ ),
62
+ ] = None,
63
+ max_score: Annotated[
64
+ Optional[Union[StrictFloat, StrictInt]],
65
+ Field(description="Filter logs by maximum score, inclusive, excludes null scores"),
66
+ ] = None,
67
+ min_score: Annotated[
68
+ Optional[Union[StrictFloat, StrictInt]],
69
+ Field(description="Filter logs by minimum score, inclusive, excludes null scores"),
70
+ ] = None,
71
+ ordering: Annotated[
72
+ Optional[StrictStr], Field(description="Which field to use when ordering the results.")
73
+ ] = None,
74
+ page_size: Annotated[Optional[StrictInt], Field(description="Number of results to return per page.")] = None,
75
+ search: Annotated[Optional[StrictStr], Field(description="Filter with skill name or id")] = None,
76
+ tags: Annotated[Optional[StrictStr], Field(description="Filter logs by tag names (comma-separated)")] = None,
77
+ unique_skills: Annotated[Optional[StrictBool], Field(description="Return only unique skills")] = None,
78
+ _request_timeout: Union[
79
+ None,
80
+ Annotated[StrictFloat, Field(gt=0)],
81
+ Tuple[Annotated[StrictFloat, Field(gt=0)], Annotated[StrictFloat, Field(gt=0)]],
82
+ ] = None,
83
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
84
+ _content_type: Optional[StrictStr] = None,
85
+ _headers: Optional[Dict[StrictStr, Any]] = None,
86
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
87
+ ) -> PaginatedExecutionLogListList:
88
+ """execution_logs_list
89
+
90
+ Get a list of execution logs. An execution log is created every time a skill or an evaluator is executed.
91
+
92
+ :param cursor: The pagination cursor value.
93
+ :type cursor: str
94
+ :param date_from: Filter logs created after or at this datetime (inclusive)
95
+ :type date_from: datetime
96
+ :param date_to: Filter logs created before or at this datetime (inclusive)
97
+ :type date_to: datetime
98
+ :param executed_item_id:
99
+ :type executed_item_id: str
100
+ :param execution_type: * `skill` - skill * `evaluator` - evaluator * `judge` - judge * `experiment` - experiment * `test` - test * `proxy` - proxy
101
+ :type execution_type: str
102
+ :param include: Comma-separated list of additional fields to include in the response. Supports: llm_output, variables, evaluation_context
103
+ :type include: str
104
+ :param max_score: Filter logs by maximum score, inclusive, excludes null scores
105
+ :type max_score: float
106
+ :param min_score: Filter logs by minimum score, inclusive, excludes null scores
107
+ :type min_score: float
108
+ :param ordering: Which field to use when ordering the results.
109
+ :type ordering: str
110
+ :param page_size: Number of results to return per page.
111
+ :type page_size: int
112
+ :param search: Filter with skill name or id
113
+ :type search: str
114
+ :param tags: Filter logs by tag names (comma-separated)
115
+ :type tags: str
116
+ :param unique_skills: Return only unique skills
117
+ :type unique_skills: bool
118
+ :param _request_timeout: timeout setting for this request. If one
119
+ number provided, it will be total request
120
+ timeout. It can also be a pair (tuple) of
121
+ (connection, read) timeouts.
122
+ :type _request_timeout: int, tuple(int, int), optional
123
+ :param _request_auth: set to override the auth_settings for an a single
124
+ request; this effectively ignores the
125
+ authentication in the spec for a single request.
126
+ :type _request_auth: dict, optional
127
+ :param _content_type: force content-type for the request.
128
+ :type _content_type: str, Optional
129
+ :param _headers: set to override the headers for a single
130
+ request; this effectively ignores the headers
131
+ in the spec for a single request.
132
+ :type _headers: dict, optional
133
+ :param _host_index: set to override the host_index for a single
134
+ request; this effectively ignores the host_index
135
+ in the spec for a single request.
136
+ :type _host_index: int, optional
137
+ :return: Returns the result object.
138
+ """ # noqa: E501
139
+
140
+ _param = self._execution_logs_list_serialize(
141
+ cursor=cursor,
142
+ date_from=date_from,
143
+ date_to=date_to,
144
+ executed_item_id=executed_item_id,
145
+ execution_type=execution_type,
146
+ include=include,
147
+ max_score=max_score,
148
+ min_score=min_score,
149
+ ordering=ordering,
150
+ page_size=page_size,
151
+ search=search,
152
+ tags=tags,
153
+ unique_skills=unique_skills,
154
+ _request_auth=_request_auth,
155
+ _content_type=_content_type,
156
+ _headers=_headers,
157
+ _host_index=_host_index,
158
+ )
159
+
160
+ _response_types_map: Dict[str, Optional[str]] = {
161
+ "200": "PaginatedExecutionLogListList",
162
+ }
163
+ response_data = self.api_client.call_api(*_param, _request_timeout=_request_timeout)
164
+ response_data.read()
165
+ return self.api_client.response_deserialize(
166
+ response_data=response_data,
167
+ response_types_map=_response_types_map,
168
+ ).data
169
+
170
+ @validate_call
171
+ def execution_logs_list_with_http_info(
172
+ self,
173
+ cursor: Annotated[Optional[StrictStr], Field(description="The pagination cursor value.")] = None,
174
+ date_from: Annotated[
175
+ Optional[datetime], Field(description="Filter logs created after or at this datetime (inclusive)")
176
+ ] = None,
177
+ date_to: Annotated[
178
+ Optional[datetime], Field(description="Filter logs created before or at this datetime (inclusive)")
179
+ ] = None,
180
+ executed_item_id: Optional[StrictStr] = None,
181
+ execution_type: Annotated[
182
+ Optional[StrictStr],
183
+ Field(
184
+ description="* `skill` - skill * `evaluator` - evaluator * `judge` - judge * `experiment` - experiment * `test` - test * `proxy` - proxy"
185
+ ),
186
+ ] = None,
187
+ include: Annotated[
188
+ Optional[StrictStr],
189
+ Field(
190
+ description="Comma-separated list of additional fields to include in the response. Supports: llm_output, variables, evaluation_context"
191
+ ),
192
+ ] = None,
193
+ max_score: Annotated[
194
+ Optional[Union[StrictFloat, StrictInt]],
195
+ Field(description="Filter logs by maximum score, inclusive, excludes null scores"),
196
+ ] = None,
197
+ min_score: Annotated[
198
+ Optional[Union[StrictFloat, StrictInt]],
199
+ Field(description="Filter logs by minimum score, inclusive, excludes null scores"),
200
+ ] = None,
201
+ ordering: Annotated[
202
+ Optional[StrictStr], Field(description="Which field to use when ordering the results.")
203
+ ] = None,
204
+ page_size: Annotated[Optional[StrictInt], Field(description="Number of results to return per page.")] = None,
205
+ search: Annotated[Optional[StrictStr], Field(description="Filter with skill name or id")] = None,
206
+ tags: Annotated[Optional[StrictStr], Field(description="Filter logs by tag names (comma-separated)")] = None,
207
+ unique_skills: Annotated[Optional[StrictBool], Field(description="Return only unique skills")] = None,
208
+ _request_timeout: Union[
209
+ None,
210
+ Annotated[StrictFloat, Field(gt=0)],
211
+ Tuple[Annotated[StrictFloat, Field(gt=0)], Annotated[StrictFloat, Field(gt=0)]],
212
+ ] = None,
213
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
214
+ _content_type: Optional[StrictStr] = None,
215
+ _headers: Optional[Dict[StrictStr, Any]] = None,
216
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
217
+ ) -> ApiResponse[PaginatedExecutionLogListList]:
218
+ """execution_logs_list
219
+
220
+ Get a list of execution logs. An execution log is created every time a skill or an evaluator is executed.
221
+
222
+ :param cursor: The pagination cursor value.
223
+ :type cursor: str
224
+ :param date_from: Filter logs created after or at this datetime (inclusive)
225
+ :type date_from: datetime
226
+ :param date_to: Filter logs created before or at this datetime (inclusive)
227
+ :type date_to: datetime
228
+ :param executed_item_id:
229
+ :type executed_item_id: str
230
+ :param execution_type: * `skill` - skill * `evaluator` - evaluator * `judge` - judge * `experiment` - experiment * `test` - test * `proxy` - proxy
231
+ :type execution_type: str
232
+ :param include: Comma-separated list of additional fields to include in the response. Supports: llm_output, variables, evaluation_context
233
+ :type include: str
234
+ :param max_score: Filter logs by maximum score, inclusive, excludes null scores
235
+ :type max_score: float
236
+ :param min_score: Filter logs by minimum score, inclusive, excludes null scores
237
+ :type min_score: float
238
+ :param ordering: Which field to use when ordering the results.
239
+ :type ordering: str
240
+ :param page_size: Number of results to return per page.
241
+ :type page_size: int
242
+ :param search: Filter with skill name or id
243
+ :type search: str
244
+ :param tags: Filter logs by tag names (comma-separated)
245
+ :type tags: str
246
+ :param unique_skills: Return only unique skills
247
+ :type unique_skills: bool
248
+ :param _request_timeout: timeout setting for this request. If one
249
+ number provided, it will be total request
250
+ timeout. It can also be a pair (tuple) of
251
+ (connection, read) timeouts.
252
+ :type _request_timeout: int, tuple(int, int), optional
253
+ :param _request_auth: set to override the auth_settings for an a single
254
+ request; this effectively ignores the
255
+ authentication in the spec for a single request.
256
+ :type _request_auth: dict, optional
257
+ :param _content_type: force content-type for the request.
258
+ :type _content_type: str, Optional
259
+ :param _headers: set to override the headers for a single
260
+ request; this effectively ignores the headers
261
+ in the spec for a single request.
262
+ :type _headers: dict, optional
263
+ :param _host_index: set to override the host_index for a single
264
+ request; this effectively ignores the host_index
265
+ in the spec for a single request.
266
+ :type _host_index: int, optional
267
+ :return: Returns the result object.
268
+ """ # noqa: E501
269
+
270
+ _param = self._execution_logs_list_serialize(
271
+ cursor=cursor,
272
+ date_from=date_from,
273
+ date_to=date_to,
274
+ executed_item_id=executed_item_id,
275
+ execution_type=execution_type,
276
+ include=include,
277
+ max_score=max_score,
278
+ min_score=min_score,
279
+ ordering=ordering,
280
+ page_size=page_size,
281
+ search=search,
282
+ tags=tags,
283
+ unique_skills=unique_skills,
284
+ _request_auth=_request_auth,
285
+ _content_type=_content_type,
286
+ _headers=_headers,
287
+ _host_index=_host_index,
288
+ )
289
+
290
+ _response_types_map: Dict[str, Optional[str]] = {
291
+ "200": "PaginatedExecutionLogListList",
292
+ }
293
+ response_data = self.api_client.call_api(*_param, _request_timeout=_request_timeout)
294
+ response_data.read()
295
+ return self.api_client.response_deserialize(
296
+ response_data=response_data,
297
+ response_types_map=_response_types_map,
298
+ )
299
+
300
+ @validate_call
301
+ def execution_logs_list_without_preload_content(
302
+ self,
303
+ cursor: Annotated[Optional[StrictStr], Field(description="The pagination cursor value.")] = None,
304
+ date_from: Annotated[
305
+ Optional[datetime], Field(description="Filter logs created after or at this datetime (inclusive)")
306
+ ] = None,
307
+ date_to: Annotated[
308
+ Optional[datetime], Field(description="Filter logs created before or at this datetime (inclusive)")
309
+ ] = None,
310
+ executed_item_id: Optional[StrictStr] = None,
311
+ execution_type: Annotated[
312
+ Optional[StrictStr],
313
+ Field(
314
+ description="* `skill` - skill * `evaluator` - evaluator * `judge` - judge * `experiment` - experiment * `test` - test * `proxy` - proxy"
315
+ ),
316
+ ] = None,
317
+ include: Annotated[
318
+ Optional[StrictStr],
319
+ Field(
320
+ description="Comma-separated list of additional fields to include in the response. Supports: llm_output, variables, evaluation_context"
321
+ ),
322
+ ] = None,
323
+ max_score: Annotated[
324
+ Optional[Union[StrictFloat, StrictInt]],
325
+ Field(description="Filter logs by maximum score, inclusive, excludes null scores"),
326
+ ] = None,
327
+ min_score: Annotated[
328
+ Optional[Union[StrictFloat, StrictInt]],
329
+ Field(description="Filter logs by minimum score, inclusive, excludes null scores"),
330
+ ] = None,
331
+ ordering: Annotated[
332
+ Optional[StrictStr], Field(description="Which field to use when ordering the results.")
333
+ ] = None,
334
+ page_size: Annotated[Optional[StrictInt], Field(description="Number of results to return per page.")] = None,
335
+ search: Annotated[Optional[StrictStr], Field(description="Filter with skill name or id")] = None,
336
+ tags: Annotated[Optional[StrictStr], Field(description="Filter logs by tag names (comma-separated)")] = None,
337
+ unique_skills: Annotated[Optional[StrictBool], Field(description="Return only unique skills")] = None,
338
+ _request_timeout: Union[
339
+ None,
340
+ Annotated[StrictFloat, Field(gt=0)],
341
+ Tuple[Annotated[StrictFloat, Field(gt=0)], Annotated[StrictFloat, Field(gt=0)]],
342
+ ] = None,
343
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
344
+ _content_type: Optional[StrictStr] = None,
345
+ _headers: Optional[Dict[StrictStr, Any]] = None,
346
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
347
+ ) -> RESTResponseType:
348
+ """execution_logs_list
349
+
350
+ Get a list of execution logs. An execution log is created every time a skill or an evaluator is executed.
351
+
352
+ :param cursor: The pagination cursor value.
353
+ :type cursor: str
354
+ :param date_from: Filter logs created after or at this datetime (inclusive)
355
+ :type date_from: datetime
356
+ :param date_to: Filter logs created before or at this datetime (inclusive)
357
+ :type date_to: datetime
358
+ :param executed_item_id:
359
+ :type executed_item_id: str
360
+ :param execution_type: * `skill` - skill * `evaluator` - evaluator * `judge` - judge * `experiment` - experiment * `test` - test * `proxy` - proxy
361
+ :type execution_type: str
362
+ :param include: Comma-separated list of additional fields to include in the response. Supports: llm_output, variables, evaluation_context
363
+ :type include: str
364
+ :param max_score: Filter logs by maximum score, inclusive, excludes null scores
365
+ :type max_score: float
366
+ :param min_score: Filter logs by minimum score, inclusive, excludes null scores
367
+ :type min_score: float
368
+ :param ordering: Which field to use when ordering the results.
369
+ :type ordering: str
370
+ :param page_size: Number of results to return per page.
371
+ :type page_size: int
372
+ :param search: Filter with skill name or id
373
+ :type search: str
374
+ :param tags: Filter logs by tag names (comma-separated)
375
+ :type tags: str
376
+ :param unique_skills: Return only unique skills
377
+ :type unique_skills: bool
378
+ :param _request_timeout: timeout setting for this request. If one
379
+ number provided, it will be total request
380
+ timeout. It can also be a pair (tuple) of
381
+ (connection, read) timeouts.
382
+ :type _request_timeout: int, tuple(int, int), optional
383
+ :param _request_auth: set to override the auth_settings for an a single
384
+ request; this effectively ignores the
385
+ authentication in the spec for a single request.
386
+ :type _request_auth: dict, optional
387
+ :param _content_type: force content-type for the request.
388
+ :type _content_type: str, Optional
389
+ :param _headers: set to override the headers for a single
390
+ request; this effectively ignores the headers
391
+ in the spec for a single request.
392
+ :type _headers: dict, optional
393
+ :param _host_index: set to override the host_index for a single
394
+ request; this effectively ignores the host_index
395
+ in the spec for a single request.
396
+ :type _host_index: int, optional
397
+ :return: Returns the result object.
398
+ """ # noqa: E501
399
+
400
+ _param = self._execution_logs_list_serialize(
401
+ cursor=cursor,
402
+ date_from=date_from,
403
+ date_to=date_to,
404
+ executed_item_id=executed_item_id,
405
+ execution_type=execution_type,
406
+ include=include,
407
+ max_score=max_score,
408
+ min_score=min_score,
409
+ ordering=ordering,
410
+ page_size=page_size,
411
+ search=search,
412
+ tags=tags,
413
+ unique_skills=unique_skills,
414
+ _request_auth=_request_auth,
415
+ _content_type=_content_type,
416
+ _headers=_headers,
417
+ _host_index=_host_index,
418
+ )
419
+
420
+ _response_types_map: Dict[str, Optional[str]] = {
421
+ "200": "PaginatedExecutionLogListList",
422
+ }
423
+ response_data = self.api_client.call_api(*_param, _request_timeout=_request_timeout)
424
+ return response_data.response
425
+
426
+ def _execution_logs_list_serialize(
427
+ self,
428
+ cursor,
429
+ date_from,
430
+ date_to,
431
+ executed_item_id,
432
+ execution_type,
433
+ include,
434
+ max_score,
435
+ min_score,
436
+ ordering,
437
+ page_size,
438
+ search,
439
+ tags,
440
+ unique_skills,
441
+ _request_auth,
442
+ _content_type,
443
+ _headers,
444
+ _host_index,
445
+ ) -> RequestSerialized:
446
+ _host = None
447
+
448
+ _collection_formats: Dict[str, str] = {}
449
+
450
+ _path_params: Dict[str, str] = {}
451
+ _query_params: List[Tuple[str, str]] = []
452
+ _header_params: Dict[str, Optional[str]] = _headers or {}
453
+ _form_params: List[Tuple[str, str]] = []
454
+ _files: Dict[str, str] = {}
455
+ _body_params: Optional[bytes] = None
456
+
457
+ # process the path parameters
458
+ # process the query parameters
459
+ if cursor is not None:
460
+ _query_params.append(("cursor", cursor))
461
+
462
+ if date_from is not None:
463
+ if isinstance(date_from, datetime):
464
+ _query_params.append(("date_from", date_from.strftime(self.api_client.configuration.datetime_format)))
465
+ else:
466
+ _query_params.append(("date_from", date_from))
467
+
468
+ if date_to is not None:
469
+ if isinstance(date_to, datetime):
470
+ _query_params.append(("date_to", date_to.strftime(self.api_client.configuration.datetime_format)))
471
+ else:
472
+ _query_params.append(("date_to", date_to))
473
+
474
+ if executed_item_id is not None:
475
+ _query_params.append(("executed_item_id", executed_item_id))
476
+
477
+ if execution_type is not None:
478
+ _query_params.append(("execution_type", execution_type))
479
+
480
+ if include is not None:
481
+ _query_params.append(("include", include))
482
+
483
+ if max_score is not None:
484
+ _query_params.append(("max_score", max_score))
485
+
486
+ if min_score is not None:
487
+ _query_params.append(("min_score", min_score))
488
+
489
+ if ordering is not None:
490
+ _query_params.append(("ordering", ordering))
491
+
492
+ if page_size is not None:
493
+ _query_params.append(("page_size", page_size))
494
+
495
+ if search is not None:
496
+ _query_params.append(("search", search))
497
+
498
+ if tags is not None:
499
+ _query_params.append(("tags", tags))
500
+
501
+ if unique_skills is not None:
502
+ _query_params.append(("unique_skills", unique_skills))
503
+
504
+ # process the header parameters
505
+ # process the form parameters
506
+ # process the body parameter
507
+
508
+ # set the HTTP header `Accept`
509
+ _header_params["Accept"] = self.api_client.select_header_accept(["application/json"])
510
+
511
+ # authentication setting
512
+ _auth_settings: List[str] = ["publicApiKey"]
513
+
514
+ return self.api_client.param_serialize(
515
+ method="GET",
516
+ resource_path="/v1/execution-logs/",
517
+ path_params=_path_params,
518
+ query_params=_query_params,
519
+ header_params=_header_params,
520
+ body=_body_params,
521
+ post_params=_form_params,
522
+ files=_files,
523
+ auth_settings=_auth_settings,
524
+ collection_formats=_collection_formats,
525
+ _host=_host,
526
+ _request_auth=_request_auth,
527
+ )
528
+
529
+ @validate_call
530
+ def execution_logs_retrieve(
531
+ self,
532
+ log_id: StrictStr,
533
+ _request_timeout: Union[
534
+ None,
535
+ Annotated[StrictFloat, Field(gt=0)],
536
+ Tuple[Annotated[StrictFloat, Field(gt=0)], Annotated[StrictFloat, Field(gt=0)]],
537
+ ] = None,
538
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
539
+ _content_type: Optional[StrictStr] = None,
540
+ _headers: Optional[Dict[StrictStr, Any]] = None,
541
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
542
+ ) -> ExecutionLogDetails:
543
+ """execution_logs_retrieve
544
+
545
+ Get the full details of a single execution log.
546
+
547
+ :param log_id: (required)
548
+ :type log_id: str
549
+ :param _request_timeout: timeout setting for this request. If one
550
+ number provided, it will be total request
551
+ timeout. It can also be a pair (tuple) of
552
+ (connection, read) timeouts.
553
+ :type _request_timeout: int, tuple(int, int), optional
554
+ :param _request_auth: set to override the auth_settings for an a single
555
+ request; this effectively ignores the
556
+ authentication in the spec for a single request.
557
+ :type _request_auth: dict, optional
558
+ :param _content_type: force content-type for the request.
559
+ :type _content_type: str, Optional
560
+ :param _headers: set to override the headers for a single
561
+ request; this effectively ignores the headers
562
+ in the spec for a single request.
563
+ :type _headers: dict, optional
564
+ :param _host_index: set to override the host_index for a single
565
+ request; this effectively ignores the host_index
566
+ in the spec for a single request.
567
+ :type _host_index: int, optional
568
+ :return: Returns the result object.
569
+ """ # noqa: E501
570
+
571
+ _param = self._execution_logs_retrieve_serialize(
572
+ log_id=log_id,
573
+ _request_auth=_request_auth,
574
+ _content_type=_content_type,
575
+ _headers=_headers,
576
+ _host_index=_host_index,
577
+ )
578
+
579
+ _response_types_map: Dict[str, Optional[str]] = {
580
+ "200": "ExecutionLogDetails",
581
+ }
582
+ response_data = self.api_client.call_api(*_param, _request_timeout=_request_timeout)
583
+ response_data.read()
584
+ return self.api_client.response_deserialize(
585
+ response_data=response_data,
586
+ response_types_map=_response_types_map,
587
+ ).data
588
+
589
+ @validate_call
590
+ def execution_logs_retrieve_with_http_info(
591
+ self,
592
+ log_id: StrictStr,
593
+ _request_timeout: Union[
594
+ None,
595
+ Annotated[StrictFloat, Field(gt=0)],
596
+ Tuple[Annotated[StrictFloat, Field(gt=0)], Annotated[StrictFloat, Field(gt=0)]],
597
+ ] = None,
598
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
599
+ _content_type: Optional[StrictStr] = None,
600
+ _headers: Optional[Dict[StrictStr, Any]] = None,
601
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
602
+ ) -> ApiResponse[ExecutionLogDetails]:
603
+ """execution_logs_retrieve
604
+
605
+ Get the full details of a single execution log.
606
+
607
+ :param log_id: (required)
608
+ :type log_id: str
609
+ :param _request_timeout: timeout setting for this request. If one
610
+ number provided, it will be total request
611
+ timeout. It can also be a pair (tuple) of
612
+ (connection, read) timeouts.
613
+ :type _request_timeout: int, tuple(int, int), optional
614
+ :param _request_auth: set to override the auth_settings for an a single
615
+ request; this effectively ignores the
616
+ authentication in the spec for a single request.
617
+ :type _request_auth: dict, optional
618
+ :param _content_type: force content-type for the request.
619
+ :type _content_type: str, Optional
620
+ :param _headers: set to override the headers for a single
621
+ request; this effectively ignores the headers
622
+ in the spec for a single request.
623
+ :type _headers: dict, optional
624
+ :param _host_index: set to override the host_index for a single
625
+ request; this effectively ignores the host_index
626
+ in the spec for a single request.
627
+ :type _host_index: int, optional
628
+ :return: Returns the result object.
629
+ """ # noqa: E501
630
+
631
+ _param = self._execution_logs_retrieve_serialize(
632
+ log_id=log_id,
633
+ _request_auth=_request_auth,
634
+ _content_type=_content_type,
635
+ _headers=_headers,
636
+ _host_index=_host_index,
637
+ )
638
+
639
+ _response_types_map: Dict[str, Optional[str]] = {
640
+ "200": "ExecutionLogDetails",
641
+ }
642
+ response_data = self.api_client.call_api(*_param, _request_timeout=_request_timeout)
643
+ response_data.read()
644
+ return self.api_client.response_deserialize(
645
+ response_data=response_data,
646
+ response_types_map=_response_types_map,
647
+ )
648
+
649
+ @validate_call
650
+ def execution_logs_retrieve_without_preload_content(
651
+ self,
652
+ log_id: StrictStr,
653
+ _request_timeout: Union[
654
+ None,
655
+ Annotated[StrictFloat, Field(gt=0)],
656
+ Tuple[Annotated[StrictFloat, Field(gt=0)], Annotated[StrictFloat, Field(gt=0)]],
657
+ ] = None,
658
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
659
+ _content_type: Optional[StrictStr] = None,
660
+ _headers: Optional[Dict[StrictStr, Any]] = None,
661
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
662
+ ) -> RESTResponseType:
663
+ """execution_logs_retrieve
664
+
665
+ Get the full details of a single execution log.
666
+
667
+ :param log_id: (required)
668
+ :type log_id: str
669
+ :param _request_timeout: timeout setting for this request. If one
670
+ number provided, it will be total request
671
+ timeout. It can also be a pair (tuple) of
672
+ (connection, read) timeouts.
673
+ :type _request_timeout: int, tuple(int, int), optional
674
+ :param _request_auth: set to override the auth_settings for an a single
675
+ request; this effectively ignores the
676
+ authentication in the spec for a single request.
677
+ :type _request_auth: dict, optional
678
+ :param _content_type: force content-type for the request.
679
+ :type _content_type: str, Optional
680
+ :param _headers: set to override the headers for a single
681
+ request; this effectively ignores the headers
682
+ in the spec for a single request.
683
+ :type _headers: dict, optional
684
+ :param _host_index: set to override the host_index for a single
685
+ request; this effectively ignores the host_index
686
+ in the spec for a single request.
687
+ :type _host_index: int, optional
688
+ :return: Returns the result object.
689
+ """ # noqa: E501
690
+
691
+ _param = self._execution_logs_retrieve_serialize(
692
+ log_id=log_id,
693
+ _request_auth=_request_auth,
694
+ _content_type=_content_type,
695
+ _headers=_headers,
696
+ _host_index=_host_index,
697
+ )
698
+
699
+ _response_types_map: Dict[str, Optional[str]] = {
700
+ "200": "ExecutionLogDetails",
701
+ }
702
+ response_data = self.api_client.call_api(*_param, _request_timeout=_request_timeout)
703
+ return response_data.response
704
+
705
+ def _execution_logs_retrieve_serialize(
706
+ self,
707
+ log_id,
708
+ _request_auth,
709
+ _content_type,
710
+ _headers,
711
+ _host_index,
712
+ ) -> RequestSerialized:
713
+ _host = None
714
+
715
+ _collection_formats: Dict[str, str] = {}
716
+
717
+ _path_params: Dict[str, str] = {}
718
+ _query_params: List[Tuple[str, str]] = []
719
+ _header_params: Dict[str, Optional[str]] = _headers or {}
720
+ _form_params: List[Tuple[str, str]] = []
721
+ _files: Dict[str, str] = {}
722
+ _body_params: Optional[bytes] = None
723
+
724
+ # process the path parameters
725
+ if log_id is not None:
726
+ _path_params["log_id"] = log_id
727
+ # process the query parameters
728
+ # process the header parameters
729
+ # process the form parameters
730
+ # process the body parameter
731
+
732
+ # set the HTTP header `Accept`
733
+ _header_params["Accept"] = self.api_client.select_header_accept(["application/json"])
734
+
735
+ # authentication setting
736
+ _auth_settings: List[str] = ["publicApiKey"]
737
+
738
+ return self.api_client.param_serialize(
739
+ method="GET",
740
+ resource_path="/v1/execution-logs/{log_id}/",
741
+ path_params=_path_params,
742
+ query_params=_query_params,
743
+ header_params=_header_params,
744
+ body=_body_params,
745
+ post_params=_form_params,
746
+ files=_files,
747
+ auth_settings=_auth_settings,
748
+ collection_formats=_collection_formats,
749
+ _host=_host,
750
+ _request_auth=_request_auth,
751
+ )