eval-studio-client 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. eval_studio_client/api/__init__.py +43 -0
  2. eval_studio_client/api/api/__init__.py +5 -0
  3. eval_studio_client/api/api/human_calibration_service_api.py +304 -0
  4. eval_studio_client/api/api/perturbator_service_api.py +268 -1
  5. eval_studio_client/api/api/prompt_library_service_api.py +669 -0
  6. eval_studio_client/api/api/test_service_api.py +568 -0
  7. eval_studio_client/api/api/workflow_edge_service_api.py +296 -0
  8. eval_studio_client/api/api/workflow_node_service_api.py +1634 -0
  9. eval_studio_client/api/api/workflow_service_api.py +1609 -0
  10. eval_studio_client/api/docs/HumanCalibrationServiceApi.md +77 -0
  11. eval_studio_client/api/docs/PerturbationServiceCreatePerturbationRequest.md +1 -0
  12. eval_studio_client/api/docs/PerturbatorServiceApi.md +33 -3
  13. eval_studio_client/api/docs/PromptGenerationServiceAutoGeneratePromptsRequest.md +2 -1
  14. eval_studio_client/api/docs/PromptLibraryServiceApi.md +155 -0
  15. eval_studio_client/api/docs/ProtobufNullValue.md +12 -0
  16. eval_studio_client/api/docs/RequiredTheTestCaseToUpdate.md +1 -0
  17. eval_studio_client/api/docs/RequiredTheUpdatedWorkflow.md +44 -0
  18. eval_studio_client/api/docs/RequiredTheUpdatedWorkflowNode.md +44 -0
  19. eval_studio_client/api/docs/TestServiceApi.md +140 -0
  20. eval_studio_client/api/docs/TestServiceGenerateTestCasesRequest.md +1 -0
  21. eval_studio_client/api/docs/TestServiceImportTestCasesFromLibraryRequest.md +32 -0
  22. eval_studio_client/api/docs/TestServiceListTestCaseLibraryItemsRequest.md +35 -0
  23. eval_studio_client/api/docs/TestServicePerturbTestRequest.md +1 -0
  24. eval_studio_client/api/docs/V1BatchDeleteWorkflowsRequest.md +29 -0
  25. eval_studio_client/api/docs/V1BatchDeleteWorkflowsResponse.md +29 -0
  26. eval_studio_client/api/docs/V1BatchGetWorkflowEdgesResponse.md +29 -0
  27. eval_studio_client/api/docs/V1BatchGetWorkflowNodesResponse.md +29 -0
  28. eval_studio_client/api/docs/V1CreateEvaluationRequest.md +1 -0
  29. eval_studio_client/api/docs/V1CreateWorkflowResponse.md +29 -0
  30. eval_studio_client/api/docs/V1DeleteWorkflowNodeResponse.md +29 -0
  31. eval_studio_client/api/docs/V1DeleteWorkflowResponse.md +29 -0
  32. eval_studio_client/api/docs/V1EstimateThresholdRequest.md +33 -0
  33. eval_studio_client/api/docs/V1GetWorkflowNodePrerequisitesResponse.md +30 -0
  34. eval_studio_client/api/docs/V1GetWorkflowNodeResponse.md +29 -0
  35. eval_studio_client/api/docs/V1GetWorkflowResponse.md +29 -0
  36. eval_studio_client/api/docs/V1ImportEvaluationRequest.md +1 -0
  37. eval_studio_client/api/docs/V1ImportTestCasesFromLibraryResponse.md +29 -0
  38. eval_studio_client/api/docs/V1ImportTestCasesRequest.md +33 -0
  39. eval_studio_client/api/docs/V1LabeledTestCase.md +31 -0
  40. eval_studio_client/api/docs/V1ListPromptLibraryItemsResponse.md +29 -0
  41. eval_studio_client/api/docs/V1ListTestCaseLibraryItemsResponse.md +29 -0
  42. eval_studio_client/api/docs/V1ListWorkflowsResponse.md +29 -0
  43. eval_studio_client/api/docs/V1ProcessWorkflowNodeResponse.md +29 -0
  44. eval_studio_client/api/docs/V1PromptLibraryItem.md +42 -0
  45. eval_studio_client/api/docs/V1TestCase.md +1 -0
  46. eval_studio_client/api/docs/V1TestSuiteEvaluates.md +11 -0
  47. eval_studio_client/api/docs/V1UpdateWorkflowNodeResponse.md +29 -0
  48. eval_studio_client/api/docs/V1UpdateWorkflowResponse.md +29 -0
  49. eval_studio_client/api/docs/V1Workflow.md +46 -0
  50. eval_studio_client/api/docs/V1WorkflowEdge.md +40 -0
  51. eval_studio_client/api/docs/V1WorkflowEdgeType.md +12 -0
  52. eval_studio_client/api/docs/V1WorkflowNode.md +46 -0
  53. eval_studio_client/api/docs/V1WorkflowNodeArtifact.md +40 -0
  54. eval_studio_client/api/docs/V1WorkflowNodeArtifacts.md +29 -0
  55. eval_studio_client/api/docs/V1WorkflowNodeAttributes.md +30 -0
  56. eval_studio_client/api/docs/V1WorkflowNodeStatus.md +12 -0
  57. eval_studio_client/api/docs/V1WorkflowNodeType.md +12 -0
  58. eval_studio_client/api/docs/V1WorkflowNodeView.md +12 -0
  59. eval_studio_client/api/docs/V1WorkflowType.md +12 -0
  60. eval_studio_client/api/docs/WorkflowEdgeServiceApi.md +76 -0
  61. eval_studio_client/api/docs/WorkflowNodeServiceApi.md +423 -0
  62. eval_studio_client/api/docs/WorkflowServiceApi.md +417 -0
  63. eval_studio_client/api/models/__init__.py +38 -0
  64. eval_studio_client/api/models/perturbation_service_create_perturbation_request.py +8 -2
  65. eval_studio_client/api/models/prompt_generation_service_auto_generate_prompts_request.py +5 -3
  66. eval_studio_client/api/models/protobuf_null_value.py +36 -0
  67. eval_studio_client/api/models/required_the_test_case_to_update.py +6 -2
  68. eval_studio_client/api/models/required_the_updated_workflow.py +152 -0
  69. eval_studio_client/api/models/required_the_updated_workflow_node.py +152 -0
  70. eval_studio_client/api/models/test_service_generate_test_cases_request.py +4 -2
  71. eval_studio_client/api/models/test_service_import_test_cases_from_library_request.py +93 -0
  72. eval_studio_client/api/models/test_service_list_test_case_library_items_request.py +99 -0
  73. eval_studio_client/api/models/test_service_perturb_test_request.py +4 -2
  74. eval_studio_client/api/models/v1_batch_delete_workflows_request.py +87 -0
  75. eval_studio_client/api/models/v1_batch_delete_workflows_response.py +95 -0
  76. eval_studio_client/api/models/v1_batch_get_workflow_edges_response.py +95 -0
  77. eval_studio_client/api/models/v1_batch_get_workflow_nodes_response.py +95 -0
  78. eval_studio_client/api/models/v1_create_evaluation_request.py +7 -2
  79. eval_studio_client/api/models/v1_create_workflow_response.py +91 -0
  80. eval_studio_client/api/models/v1_delete_workflow_node_response.py +91 -0
  81. eval_studio_client/api/models/v1_delete_workflow_response.py +91 -0
  82. eval_studio_client/api/models/v1_estimate_threshold_request.py +103 -0
  83. eval_studio_client/api/models/v1_get_workflow_node_prerequisites_response.py +89 -0
  84. eval_studio_client/api/models/v1_get_workflow_node_response.py +91 -0
  85. eval_studio_client/api/models/v1_get_workflow_response.py +91 -0
  86. eval_studio_client/api/models/v1_import_evaluation_request.py +7 -2
  87. eval_studio_client/api/models/v1_import_test_cases_from_library_response.py +91 -0
  88. eval_studio_client/api/models/v1_import_test_cases_request.py +95 -0
  89. eval_studio_client/api/models/v1_labeled_test_case.py +91 -0
  90. eval_studio_client/api/models/v1_list_prompt_library_items_response.py +95 -0
  91. eval_studio_client/api/models/v1_list_test_case_library_items_response.py +95 -0
  92. eval_studio_client/api/models/v1_list_workflows_response.py +95 -0
  93. eval_studio_client/api/models/v1_process_workflow_node_response.py +91 -0
  94. eval_studio_client/api/models/v1_prompt_library_item.py +129 -0
  95. eval_studio_client/api/models/v1_test_case.py +6 -2
  96. eval_studio_client/api/models/v1_test_suite_evaluates.py +39 -0
  97. eval_studio_client/api/models/v1_update_workflow_node_response.py +91 -0
  98. eval_studio_client/api/models/v1_update_workflow_response.py +91 -0
  99. eval_studio_client/api/models/v1_workflow.py +156 -0
  100. eval_studio_client/api/models/v1_workflow_edge.py +123 -0
  101. eval_studio_client/api/models/v1_workflow_edge_type.py +37 -0
  102. eval_studio_client/api/models/v1_workflow_node.py +156 -0
  103. eval_studio_client/api/models/v1_workflow_node_artifact.py +122 -0
  104. eval_studio_client/api/models/v1_workflow_node_artifacts.py +97 -0
  105. eval_studio_client/api/models/v1_workflow_node_attributes.py +87 -0
  106. eval_studio_client/api/models/v1_workflow_node_status.py +40 -0
  107. eval_studio_client/api/models/v1_workflow_node_type.py +41 -0
  108. eval_studio_client/api/models/v1_workflow_node_view.py +38 -0
  109. eval_studio_client/api/models/v1_workflow_type.py +37 -0
  110. eval_studio_client/api/test/test_human_calibration_service_api.py +38 -0
  111. eval_studio_client/api/test/test_perturbation_service_create_perturbation_request.py +20 -2
  112. eval_studio_client/api/test/test_prompt_generation_service_auto_generate_prompts_request.py +4 -1
  113. eval_studio_client/api/test/test_prompt_library_service_api.py +43 -0
  114. eval_studio_client/api/test/test_protobuf_null_value.py +33 -0
  115. eval_studio_client/api/test/test_required_the_test_case_to_update.py +4 -1
  116. eval_studio_client/api/test/test_required_the_updated_workflow.py +88 -0
  117. eval_studio_client/api/test/test_required_the_updated_workflow_node.py +80 -0
  118. eval_studio_client/api/test/test_test_service_api.py +12 -0
  119. eval_studio_client/api/test/test_test_service_generate_test_cases_request.py +4 -1
  120. eval_studio_client/api/test/test_test_service_import_test_cases_from_library_request.py +56 -0
  121. eval_studio_client/api/test/test_test_service_list_test_case_library_items_request.py +63 -0
  122. eval_studio_client/api/test/test_test_service_perturb_test_request.py +4 -1
  123. eval_studio_client/api/test/test_v1_batch_delete_test_cases_response.py +4 -1
  124. eval_studio_client/api/test/test_v1_batch_delete_workflows_request.py +53 -0
  125. eval_studio_client/api/test/test_v1_batch_delete_workflows_response.py +92 -0
  126. eval_studio_client/api/test/test_v1_batch_get_workflow_edges_response.py +64 -0
  127. eval_studio_client/api/test/test_v1_batch_get_workflow_nodes_response.py +84 -0
  128. eval_studio_client/api/test/test_v1_create_evaluation_request.py +20 -2
  129. eval_studio_client/api/test/test_v1_create_test_case_response.py +4 -1
  130. eval_studio_client/api/test/test_v1_create_workflow_response.py +90 -0
  131. eval_studio_client/api/test/test_v1_delete_test_case_response.py +4 -1
  132. eval_studio_client/api/test/test_v1_delete_workflow_node_response.py +82 -0
  133. eval_studio_client/api/test/test_v1_delete_workflow_response.py +90 -0
  134. eval_studio_client/api/test/test_v1_estimate_threshold_request.py +60 -0
  135. eval_studio_client/api/test/test_v1_evaluation_test.py +4 -1
  136. eval_studio_client/api/test/test_v1_find_all_test_cases_by_id_response.py +4 -1
  137. eval_studio_client/api/test/test_v1_get_test_case_response.py +4 -1
  138. eval_studio_client/api/test/test_v1_get_workflow_node_prerequisites_response.py +56 -0
  139. eval_studio_client/api/test/test_v1_get_workflow_node_response.py +82 -0
  140. eval_studio_client/api/test/test_v1_get_workflow_response.py +90 -0
  141. eval_studio_client/api/test/test_v1_import_evaluation_request.py +16 -1
  142. eval_studio_client/api/test/test_v1_import_test_cases_from_library_response.py +71 -0
  143. eval_studio_client/api/test/test_v1_import_test_cases_request.py +57 -0
  144. eval_studio_client/api/test/test_v1_labeled_test_case.py +53 -0
  145. eval_studio_client/api/test/test_v1_list_prompt_library_items_response.py +71 -0
  146. eval_studio_client/api/test/test_v1_list_test_case_library_items_response.py +71 -0
  147. eval_studio_client/api/test/test_v1_list_test_cases_response.py +4 -1
  148. eval_studio_client/api/test/test_v1_list_workflows_response.py +92 -0
  149. eval_studio_client/api/test/test_v1_process_workflow_node_response.py +71 -0
  150. eval_studio_client/api/test/test_v1_prompt_library_item.py +68 -0
  151. eval_studio_client/api/test/test_v1_test_case.py +4 -1
  152. eval_studio_client/api/test/test_v1_test_suite_evaluates.py +33 -0
  153. eval_studio_client/api/test/test_v1_update_test_case_response.py +4 -1
  154. eval_studio_client/api/test/test_v1_update_workflow_node_response.py +82 -0
  155. eval_studio_client/api/test/test_v1_update_workflow_response.py +90 -0
  156. eval_studio_client/api/test/test_v1_workflow.py +89 -0
  157. eval_studio_client/api/test/test_v1_workflow_edge.py +61 -0
  158. eval_studio_client/api/test/test_v1_workflow_edge_type.py +33 -0
  159. eval_studio_client/api/test/test_v1_workflow_node.py +81 -0
  160. eval_studio_client/api/test/test_v1_workflow_node_artifact.py +61 -0
  161. eval_studio_client/api/test/test_v1_workflow_node_artifacts.py +64 -0
  162. eval_studio_client/api/test/test_v1_workflow_node_attributes.py +51 -0
  163. eval_studio_client/api/test/test_v1_workflow_node_status.py +33 -0
  164. eval_studio_client/api/test/test_v1_workflow_node_type.py +33 -0
  165. eval_studio_client/api/test/test_v1_workflow_node_view.py +33 -0
  166. eval_studio_client/api/test/test_v1_workflow_type.py +33 -0
  167. eval_studio_client/api/test/test_workflow_edge_service_api.py +38 -0
  168. eval_studio_client/api/test/test_workflow_node_service_api.py +73 -0
  169. eval_studio_client/api/test/test_workflow_service_api.py +73 -0
  170. eval_studio_client/client.py +7 -0
  171. eval_studio_client/dashboards.py +29 -0
  172. eval_studio_client/gen/openapiv2/eval_studio.swagger.json +2665 -794
  173. eval_studio_client/leaderboards.py +123 -0
  174. eval_studio_client/models.py +3 -42
  175. eval_studio_client/test_labs.py +49 -21
  176. eval_studio_client/tests.py +188 -1
  177. {eval_studio_client-1.0.0.dist-info → eval_studio_client-1.0.1.dist-info}/METADATA +1 -2
  178. {eval_studio_client-1.0.0.dist-info → eval_studio_client-1.0.1.dist-info}/RECORD +179 -50
  179. {eval_studio_client-1.0.0.dist-info → eval_studio_client-1.0.1.dist-info}/WHEEL +1 -1
@@ -21,6 +21,7 @@ from eval_studio_client.api.api.dashboard_service_api import DashboardServiceApi
21
21
  from eval_studio_client.api.api.document_service_api import DocumentServiceApi
22
22
  from eval_studio_client.api.api.evaluation_service_api import EvaluationServiceApi
23
23
  from eval_studio_client.api.api.evaluator_service_api import EvaluatorServiceApi
24
+ from eval_studio_client.api.api.human_calibration_service_api import HumanCalibrationServiceApi
24
25
  from eval_studio_client.api.api.info_service_api import InfoServiceApi
25
26
  from eval_studio_client.api.api.leaderboard_service_api import LeaderboardServiceApi
26
27
  from eval_studio_client.api.api.model_service_api import ModelServiceApi
@@ -29,11 +30,15 @@ from eval_studio_client.api.api.operation_service_api import OperationServiceApi
29
30
  from eval_studio_client.api.api.perturbation_service_api import PerturbationServiceApi
30
31
  from eval_studio_client.api.api.perturbator_service_api import PerturbatorServiceApi
31
32
  from eval_studio_client.api.api.prompt_generation_service_api import PromptGenerationServiceApi
33
+ from eval_studio_client.api.api.prompt_library_service_api import PromptLibraryServiceApi
32
34
  from eval_studio_client.api.api.test_case_service_api import TestCaseServiceApi
33
35
  from eval_studio_client.api.api.test_class_service_api import TestClassServiceApi
34
36
  from eval_studio_client.api.api.test_lab_service_api import TestLabServiceApi
35
37
  from eval_studio_client.api.api.test_service_api import TestServiceApi
36
38
  from eval_studio_client.api.api.who_am_i_service_api import WhoAmIServiceApi
39
+ from eval_studio_client.api.api.workflow_edge_service_api import WorkflowEdgeServiceApi
40
+ from eval_studio_client.api.api.workflow_node_service_api import WorkflowNodeServiceApi
41
+ from eval_studio_client.api.api.workflow_service_api import WorkflowServiceApi
37
42
 
38
43
  # import ApiClient
39
44
  from eval_studio_client.api.api_response import ApiResponse
@@ -50,6 +55,7 @@ from eval_studio_client.api.exceptions import ApiException
50
55
  from eval_studio_client.api.models.perturbation_service_create_perturbation_request import PerturbationServiceCreatePerturbationRequest
51
56
  from eval_studio_client.api.models.prompt_generation_service_auto_generate_prompts_request import PromptGenerationServiceAutoGeneratePromptsRequest
52
57
  from eval_studio_client.api.models.protobuf_any import ProtobufAny
58
+ from eval_studio_client.api.models.protobuf_null_value import ProtobufNullValue
53
59
  from eval_studio_client.api.models.required_the_dashboard_to_update import RequiredTheDashboardToUpdate
54
60
  from eval_studio_client.api.models.required_the_document_to_update import RequiredTheDocumentToUpdate
55
61
  from eval_studio_client.api.models.required_the_leaderboard_to_update import RequiredTheLeaderboardToUpdate
@@ -58,9 +64,13 @@ from eval_studio_client.api.models.required_the_operation_to_finalize import Req
58
64
  from eval_studio_client.api.models.required_the_operation_to_update import RequiredTheOperationToUpdate
59
65
  from eval_studio_client.api.models.required_the_test_case_to_update import RequiredTheTestCaseToUpdate
60
66
  from eval_studio_client.api.models.required_the_test_to_update import RequiredTheTestToUpdate
67
+ from eval_studio_client.api.models.required_the_updated_workflow import RequiredTheUpdatedWorkflow
68
+ from eval_studio_client.api.models.required_the_updated_workflow_node import RequiredTheUpdatedWorkflowNode
61
69
  from eval_studio_client.api.models.rpc_status import RpcStatus
62
70
  from eval_studio_client.api.models.test_case_service_batch_delete_test_cases_request import TestCaseServiceBatchDeleteTestCasesRequest
63
71
  from eval_studio_client.api.models.test_service_generate_test_cases_request import TestServiceGenerateTestCasesRequest
72
+ from eval_studio_client.api.models.test_service_import_test_cases_from_library_request import TestServiceImportTestCasesFromLibraryRequest
73
+ from eval_studio_client.api.models.test_service_list_test_case_library_items_request import TestServiceListTestCaseLibraryItemsRequest
64
74
  from eval_studio_client.api.models.test_service_perturb_test_request import TestServicePerturbTestRequest
65
75
  from eval_studio_client.api.models.v1_batch_create_leaderboards_request import V1BatchCreateLeaderboardsRequest
66
76
  from eval_studio_client.api.models.v1_batch_create_leaderboards_response import V1BatchCreateLeaderboardsResponse
@@ -77,12 +87,16 @@ from eval_studio_client.api.models.v1_batch_delete_models_response import V1Batc
77
87
  from eval_studio_client.api.models.v1_batch_delete_test_cases_response import V1BatchDeleteTestCasesResponse
78
88
  from eval_studio_client.api.models.v1_batch_delete_tests_request import V1BatchDeleteTestsRequest
79
89
  from eval_studio_client.api.models.v1_batch_delete_tests_response import V1BatchDeleteTestsResponse
90
+ from eval_studio_client.api.models.v1_batch_delete_workflows_request import V1BatchDeleteWorkflowsRequest
91
+ from eval_studio_client.api.models.v1_batch_delete_workflows_response import V1BatchDeleteWorkflowsResponse
80
92
  from eval_studio_client.api.models.v1_batch_get_dashboards_response import V1BatchGetDashboardsResponse
81
93
  from eval_studio_client.api.models.v1_batch_get_documents_response import V1BatchGetDocumentsResponse
82
94
  from eval_studio_client.api.models.v1_batch_get_leaderboards_response import V1BatchGetLeaderboardsResponse
83
95
  from eval_studio_client.api.models.v1_batch_get_models_response import V1BatchGetModelsResponse
84
96
  from eval_studio_client.api.models.v1_batch_get_operations_response import V1BatchGetOperationsResponse
85
97
  from eval_studio_client.api.models.v1_batch_get_tests_response import V1BatchGetTestsResponse
98
+ from eval_studio_client.api.models.v1_batch_get_workflow_edges_response import V1BatchGetWorkflowEdgesResponse
99
+ from eval_studio_client.api.models.v1_batch_get_workflow_nodes_response import V1BatchGetWorkflowNodesResponse
86
100
  from eval_studio_client.api.models.v1_batch_import_leaderboard_request import V1BatchImportLeaderboardRequest
87
101
  from eval_studio_client.api.models.v1_batch_import_leaderboard_response import V1BatchImportLeaderboardResponse
88
102
  from eval_studio_client.api.models.v1_batch_import_tests_request import V1BatchImportTestsRequest
@@ -101,6 +115,7 @@ from eval_studio_client.api.models.v1_create_perturbation_response import V1Crea
101
115
  from eval_studio_client.api.models.v1_create_test_case_response import V1CreateTestCaseResponse
102
116
  from eval_studio_client.api.models.v1_create_test_lab_response import V1CreateTestLabResponse
103
117
  from eval_studio_client.api.models.v1_create_test_response import V1CreateTestResponse
118
+ from eval_studio_client.api.models.v1_create_workflow_response import V1CreateWorkflowResponse
104
119
  from eval_studio_client.api.models.v1_dashboard import V1Dashboard
105
120
  from eval_studio_client.api.models.v1_dashboard_status import V1DashboardStatus
106
121
  from eval_studio_client.api.models.v1_delete_dashboard_response import V1DeleteDashboardResponse
@@ -110,7 +125,10 @@ from eval_studio_client.api.models.v1_delete_leaderboard_response import V1Delet
110
125
  from eval_studio_client.api.models.v1_delete_model_response import V1DeleteModelResponse
111
126
  from eval_studio_client.api.models.v1_delete_test_case_response import V1DeleteTestCaseResponse
112
127
  from eval_studio_client.api.models.v1_delete_test_response import V1DeleteTestResponse
128
+ from eval_studio_client.api.models.v1_delete_workflow_node_response import V1DeleteWorkflowNodeResponse
129
+ from eval_studio_client.api.models.v1_delete_workflow_response import V1DeleteWorkflowResponse
113
130
  from eval_studio_client.api.models.v1_document import V1Document
131
+ from eval_studio_client.api.models.v1_estimate_threshold_request import V1EstimateThresholdRequest
114
132
  from eval_studio_client.api.models.v1_evaluation_test import V1EvaluationTest
115
133
  from eval_studio_client.api.models.v1_evaluator import V1Evaluator
116
134
  from eval_studio_client.api.models.v1_evaluator_param_type import V1EvaluatorParamType
@@ -132,11 +150,17 @@ from eval_studio_client.api.models.v1_get_perturbator_response import V1GetPertu
132
150
  from eval_studio_client.api.models.v1_get_test_case_response import V1GetTestCaseResponse
133
151
  from eval_studio_client.api.models.v1_get_test_class_response import V1GetTestClassResponse
134
152
  from eval_studio_client.api.models.v1_get_test_response import V1GetTestResponse
153
+ from eval_studio_client.api.models.v1_get_workflow_node_prerequisites_response import V1GetWorkflowNodePrerequisitesResponse
154
+ from eval_studio_client.api.models.v1_get_workflow_node_response import V1GetWorkflowNodeResponse
155
+ from eval_studio_client.api.models.v1_get_workflow_response import V1GetWorkflowResponse
135
156
  from eval_studio_client.api.models.v1_import_evaluation_request import V1ImportEvaluationRequest
136
157
  from eval_studio_client.api.models.v1_import_leaderboard_request import V1ImportLeaderboardRequest
137
158
  from eval_studio_client.api.models.v1_import_leaderboard_response import V1ImportLeaderboardResponse
159
+ from eval_studio_client.api.models.v1_import_test_cases_from_library_response import V1ImportTestCasesFromLibraryResponse
160
+ from eval_studio_client.api.models.v1_import_test_cases_request import V1ImportTestCasesRequest
138
161
  from eval_studio_client.api.models.v1_info import V1Info
139
162
  from eval_studio_client.api.models.v1_insight import V1Insight
163
+ from eval_studio_client.api.models.v1_labeled_test_case import V1LabeledTestCase
140
164
  from eval_studio_client.api.models.v1_leaderboard import V1Leaderboard
141
165
  from eval_studio_client.api.models.v1_leaderboard_status import V1LeaderboardStatus
142
166
  from eval_studio_client.api.models.v1_leaderboard_type import V1LeaderboardType
@@ -155,10 +179,13 @@ from eval_studio_client.api.models.v1_list_most_recent_models_response import V1
155
179
  from eval_studio_client.api.models.v1_list_most_recent_tests_response import V1ListMostRecentTestsResponse
156
180
  from eval_studio_client.api.models.v1_list_operations_response import V1ListOperationsResponse
157
181
  from eval_studio_client.api.models.v1_list_perturbators_response import V1ListPerturbatorsResponse
182
+ from eval_studio_client.api.models.v1_list_prompt_library_items_response import V1ListPromptLibraryItemsResponse
158
183
  from eval_studio_client.api.models.v1_list_rag_collections_response import V1ListRAGCollectionsResponse
184
+ from eval_studio_client.api.models.v1_list_test_case_library_items_response import V1ListTestCaseLibraryItemsResponse
159
185
  from eval_studio_client.api.models.v1_list_test_cases_response import V1ListTestCasesResponse
160
186
  from eval_studio_client.api.models.v1_list_test_classes_response import V1ListTestClassesResponse
161
187
  from eval_studio_client.api.models.v1_list_tests_response import V1ListTestsResponse
188
+ from eval_studio_client.api.models.v1_list_workflows_response import V1ListWorkflowsResponse
162
189
  from eval_studio_client.api.models.v1_model import V1Model
163
190
  from eval_studio_client.api.models.v1_model_type import V1ModelType
164
191
  from eval_studio_client.api.models.v1_operation import V1Operation
@@ -168,6 +195,8 @@ from eval_studio_client.api.models.v1_perturbator import V1Perturbator
168
195
  from eval_studio_client.api.models.v1_perturbator_configuration import V1PerturbatorConfiguration
169
196
  from eval_studio_client.api.models.v1_perturbator_intensity import V1PerturbatorIntensity
170
197
  from eval_studio_client.api.models.v1_problem_and_action import V1ProblemAndAction
198
+ from eval_studio_client.api.models.v1_process_workflow_node_response import V1ProcessWorkflowNodeResponse
199
+ from eval_studio_client.api.models.v1_prompt_library_item import V1PromptLibraryItem
171
200
  from eval_studio_client.api.models.v1_test import V1Test
172
201
  from eval_studio_client.api.models.v1_test_case import V1TestCase
173
202
  from eval_studio_client.api.models.v1_test_case_relationship import V1TestCaseRelationship
@@ -175,6 +204,7 @@ from eval_studio_client.api.models.v1_test_cases_generator import V1TestCasesGen
175
204
  from eval_studio_client.api.models.v1_test_class import V1TestClass
176
205
  from eval_studio_client.api.models.v1_test_class_type import V1TestClassType
177
206
  from eval_studio_client.api.models.v1_test_lab import V1TestLab
207
+ from eval_studio_client.api.models.v1_test_suite_evaluates import V1TestSuiteEvaluates
178
208
  from eval_studio_client.api.models.v1_update_dashboard_response import V1UpdateDashboardResponse
179
209
  from eval_studio_client.api.models.v1_update_document_response import V1UpdateDocumentResponse
180
210
  from eval_studio_client.api.models.v1_update_leaderboard_response import V1UpdateLeaderboardResponse
@@ -182,4 +212,17 @@ from eval_studio_client.api.models.v1_update_model_response import V1UpdateModel
182
212
  from eval_studio_client.api.models.v1_update_operation_response import V1UpdateOperationResponse
183
213
  from eval_studio_client.api.models.v1_update_test_case_response import V1UpdateTestCaseResponse
184
214
  from eval_studio_client.api.models.v1_update_test_response import V1UpdateTestResponse
215
+ from eval_studio_client.api.models.v1_update_workflow_node_response import V1UpdateWorkflowNodeResponse
216
+ from eval_studio_client.api.models.v1_update_workflow_response import V1UpdateWorkflowResponse
185
217
  from eval_studio_client.api.models.v1_who_am_i_response import V1WhoAmIResponse
218
+ from eval_studio_client.api.models.v1_workflow import V1Workflow
219
+ from eval_studio_client.api.models.v1_workflow_edge import V1WorkflowEdge
220
+ from eval_studio_client.api.models.v1_workflow_edge_type import V1WorkflowEdgeType
221
+ from eval_studio_client.api.models.v1_workflow_node import V1WorkflowNode
222
+ from eval_studio_client.api.models.v1_workflow_node_artifact import V1WorkflowNodeArtifact
223
+ from eval_studio_client.api.models.v1_workflow_node_artifacts import V1WorkflowNodeArtifacts
224
+ from eval_studio_client.api.models.v1_workflow_node_attributes import V1WorkflowNodeAttributes
225
+ from eval_studio_client.api.models.v1_workflow_node_status import V1WorkflowNodeStatus
226
+ from eval_studio_client.api.models.v1_workflow_node_type import V1WorkflowNodeType
227
+ from eval_studio_client.api.models.v1_workflow_node_view import V1WorkflowNodeView
228
+ from eval_studio_client.api.models.v1_workflow_type import V1WorkflowType
@@ -5,6 +5,7 @@ from eval_studio_client.api.api.dashboard_service_api import DashboardServiceApi
5
5
  from eval_studio_client.api.api.document_service_api import DocumentServiceApi
6
6
  from eval_studio_client.api.api.evaluation_service_api import EvaluationServiceApi
7
7
  from eval_studio_client.api.api.evaluator_service_api import EvaluatorServiceApi
8
+ from eval_studio_client.api.api.human_calibration_service_api import HumanCalibrationServiceApi
8
9
  from eval_studio_client.api.api.info_service_api import InfoServiceApi
9
10
  from eval_studio_client.api.api.leaderboard_service_api import LeaderboardServiceApi
10
11
  from eval_studio_client.api.api.model_service_api import ModelServiceApi
@@ -13,9 +14,13 @@ from eval_studio_client.api.api.operation_service_api import OperationServiceApi
13
14
  from eval_studio_client.api.api.perturbation_service_api import PerturbationServiceApi
14
15
  from eval_studio_client.api.api.perturbator_service_api import PerturbatorServiceApi
15
16
  from eval_studio_client.api.api.prompt_generation_service_api import PromptGenerationServiceApi
17
+ from eval_studio_client.api.api.prompt_library_service_api import PromptLibraryServiceApi
16
18
  from eval_studio_client.api.api.test_case_service_api import TestCaseServiceApi
17
19
  from eval_studio_client.api.api.test_class_service_api import TestClassServiceApi
18
20
  from eval_studio_client.api.api.test_lab_service_api import TestLabServiceApi
19
21
  from eval_studio_client.api.api.test_service_api import TestServiceApi
20
22
  from eval_studio_client.api.api.who_am_i_service_api import WhoAmIServiceApi
23
+ from eval_studio_client.api.api.workflow_edge_service_api import WorkflowEdgeServiceApi
24
+ from eval_studio_client.api.api.workflow_node_service_api import WorkflowNodeServiceApi
25
+ from eval_studio_client.api.api.workflow_service_api import WorkflowServiceApi
21
26
 
@@ -0,0 +1,304 @@
1
+ # coding: utf-8
2
+
3
+ """
4
+ ai/h2o/eval_studio/v1/collection.proto
5
+
6
+ No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
7
+
8
+ The version of the OpenAPI document: version not set
9
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
10
+
11
+ Do not edit the class manually.
12
+ """ # noqa: E501
13
+
14
+ import warnings
15
+ from pydantic import validate_call, Field, StrictFloat, StrictStr, StrictInt
16
+ from typing import Any, Dict, List, Optional, Tuple, Union
17
+ from typing_extensions import Annotated
18
+
19
+ from eval_studio_client.api.models.v1_estimate_threshold_request import V1EstimateThresholdRequest
20
+ from eval_studio_client.api.models.v1_operation import V1Operation
21
+
22
+ from eval_studio_client.api.api_client import ApiClient, RequestSerialized
23
+ from eval_studio_client.api.api_response import ApiResponse
24
+ from eval_studio_client.api.rest import RESTResponseType
25
+
26
+
27
+ class HumanCalibrationServiceApi:
28
+ """NOTE: This class is auto generated by OpenAPI Generator
29
+ Ref: https://openapi-generator.tech
30
+
31
+ Do not edit the class manually.
32
+ """
33
+
34
+ def __init__(self, api_client=None) -> None:
35
+ if api_client is None:
36
+ api_client = ApiClient.get_default()
37
+ self.api_client = api_client
38
+
39
+
40
+ @validate_call
41
+ def human_calibration_service_estimate_threshold(
42
+ self,
43
+ body: V1EstimateThresholdRequest,
44
+ _request_timeout: Union[
45
+ None,
46
+ Annotated[StrictFloat, Field(gt=0)],
47
+ Tuple[
48
+ Annotated[StrictFloat, Field(gt=0)],
49
+ Annotated[StrictFloat, Field(gt=0)]
50
+ ]
51
+ ] = None,
52
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
53
+ _content_type: Optional[StrictStr] = None,
54
+ _headers: Optional[Dict[StrictStr, Any]] = None,
55
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
56
+ ) -> V1Operation:
57
+ """EstimateThreshold runs a threshold estimation process based on human labeling of randomly sampled test-cases.
58
+
59
+
60
+ :param body: (required)
61
+ :type body: V1EstimateThresholdRequest
62
+ :param _request_timeout: timeout setting for this request. If one
63
+ number provided, it will be total request
64
+ timeout. It can also be a pair (tuple) of
65
+ (connection, read) timeouts.
66
+ :type _request_timeout: int, tuple(int, int), optional
67
+ :param _request_auth: set to override the auth_settings for an a single
68
+ request; this effectively ignores the
69
+ authentication in the spec for a single request.
70
+ :type _request_auth: dict, optional
71
+ :param _content_type: force content-type for the request.
72
+ :type _content_type: str, Optional
73
+ :param _headers: set to override the headers for a single
74
+ request; this effectively ignores the headers
75
+ in the spec for a single request.
76
+ :type _headers: dict, optional
77
+ :param _host_index: set to override the host_index for a single
78
+ request; this effectively ignores the host_index
79
+ in the spec for a single request.
80
+ :type _host_index: int, optional
81
+ :return: Returns the result object.
82
+ """ # noqa: E501
83
+
84
+ _param = self._human_calibration_service_estimate_threshold_serialize(
85
+ body=body,
86
+ _request_auth=_request_auth,
87
+ _content_type=_content_type,
88
+ _headers=_headers,
89
+ _host_index=_host_index
90
+ )
91
+
92
+ _response_types_map: Dict[str, Optional[str]] = {
93
+ '200': "V1Operation",
94
+ }
95
+ response_data = self.api_client.call_api(
96
+ *_param,
97
+ _request_timeout=_request_timeout
98
+ )
99
+ response_data.read()
100
+ return self.api_client.response_deserialize(
101
+ response_data=response_data,
102
+ response_types_map=_response_types_map,
103
+ ).data
104
+
105
+
106
+ @validate_call
107
+ def human_calibration_service_estimate_threshold_with_http_info(
108
+ self,
109
+ body: V1EstimateThresholdRequest,
110
+ _request_timeout: Union[
111
+ None,
112
+ Annotated[StrictFloat, Field(gt=0)],
113
+ Tuple[
114
+ Annotated[StrictFloat, Field(gt=0)],
115
+ Annotated[StrictFloat, Field(gt=0)]
116
+ ]
117
+ ] = None,
118
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
119
+ _content_type: Optional[StrictStr] = None,
120
+ _headers: Optional[Dict[StrictStr, Any]] = None,
121
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
122
+ ) -> ApiResponse[V1Operation]:
123
+ """EstimateThreshold runs a threshold estimation process based on human labeling of randomly sampled test-cases.
124
+
125
+
126
+ :param body: (required)
127
+ :type body: V1EstimateThresholdRequest
128
+ :param _request_timeout: timeout setting for this request. If one
129
+ number provided, it will be total request
130
+ timeout. It can also be a pair (tuple) of
131
+ (connection, read) timeouts.
132
+ :type _request_timeout: int, tuple(int, int), optional
133
+ :param _request_auth: set to override the auth_settings for an a single
134
+ request; this effectively ignores the
135
+ authentication in the spec for a single request.
136
+ :type _request_auth: dict, optional
137
+ :param _content_type: force content-type for the request.
138
+ :type _content_type: str, Optional
139
+ :param _headers: set to override the headers for a single
140
+ request; this effectively ignores the headers
141
+ in the spec for a single request.
142
+ :type _headers: dict, optional
143
+ :param _host_index: set to override the host_index for a single
144
+ request; this effectively ignores the host_index
145
+ in the spec for a single request.
146
+ :type _host_index: int, optional
147
+ :return: Returns the result object.
148
+ """ # noqa: E501
149
+
150
+ _param = self._human_calibration_service_estimate_threshold_serialize(
151
+ body=body,
152
+ _request_auth=_request_auth,
153
+ _content_type=_content_type,
154
+ _headers=_headers,
155
+ _host_index=_host_index
156
+ )
157
+
158
+ _response_types_map: Dict[str, Optional[str]] = {
159
+ '200': "V1Operation",
160
+ }
161
+ response_data = self.api_client.call_api(
162
+ *_param,
163
+ _request_timeout=_request_timeout
164
+ )
165
+ response_data.read()
166
+ return self.api_client.response_deserialize(
167
+ response_data=response_data,
168
+ response_types_map=_response_types_map,
169
+ )
170
+
171
+
172
+ @validate_call
173
+ def human_calibration_service_estimate_threshold_without_preload_content(
174
+ self,
175
+ body: V1EstimateThresholdRequest,
176
+ _request_timeout: Union[
177
+ None,
178
+ Annotated[StrictFloat, Field(gt=0)],
179
+ Tuple[
180
+ Annotated[StrictFloat, Field(gt=0)],
181
+ Annotated[StrictFloat, Field(gt=0)]
182
+ ]
183
+ ] = None,
184
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
185
+ _content_type: Optional[StrictStr] = None,
186
+ _headers: Optional[Dict[StrictStr, Any]] = None,
187
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
188
+ ) -> RESTResponseType:
189
+ """EstimateThreshold runs a threshold estimation process based on human labeling of randomly sampled test-cases.
190
+
191
+
192
+ :param body: (required)
193
+ :type body: V1EstimateThresholdRequest
194
+ :param _request_timeout: timeout setting for this request. If one
195
+ number provided, it will be total request
196
+ timeout. It can also be a pair (tuple) of
197
+ (connection, read) timeouts.
198
+ :type _request_timeout: int, tuple(int, int), optional
199
+ :param _request_auth: set to override the auth_settings for an a single
200
+ request; this effectively ignores the
201
+ authentication in the spec for a single request.
202
+ :type _request_auth: dict, optional
203
+ :param _content_type: force content-type for the request.
204
+ :type _content_type: str, Optional
205
+ :param _headers: set to override the headers for a single
206
+ request; this effectively ignores the headers
207
+ in the spec for a single request.
208
+ :type _headers: dict, optional
209
+ :param _host_index: set to override the host_index for a single
210
+ request; this effectively ignores the host_index
211
+ in the spec for a single request.
212
+ :type _host_index: int, optional
213
+ :return: Returns the result object.
214
+ """ # noqa: E501
215
+
216
+ _param = self._human_calibration_service_estimate_threshold_serialize(
217
+ body=body,
218
+ _request_auth=_request_auth,
219
+ _content_type=_content_type,
220
+ _headers=_headers,
221
+ _host_index=_host_index
222
+ )
223
+
224
+ _response_types_map: Dict[str, Optional[str]] = {
225
+ '200': "V1Operation",
226
+ }
227
+ response_data = self.api_client.call_api(
228
+ *_param,
229
+ _request_timeout=_request_timeout
230
+ )
231
+ return response_data.response
232
+
233
+
234
+ def _human_calibration_service_estimate_threshold_serialize(
235
+ self,
236
+ body,
237
+ _request_auth,
238
+ _content_type,
239
+ _headers,
240
+ _host_index,
241
+ ) -> RequestSerialized:
242
+
243
+ _host = None
244
+
245
+ _collection_formats: Dict[str, str] = {
246
+ }
247
+
248
+ _path_params: Dict[str, str] = {}
249
+ _query_params: List[Tuple[str, str]] = []
250
+ _header_params: Dict[str, Optional[str]] = _headers or {}
251
+ _form_params: List[Tuple[str, str]] = []
252
+ _files: Dict[str, Union[str, bytes]] = {}
253
+ _body_params: Optional[bytes] = None
254
+
255
+ # process the path parameters
256
+ # process the query parameters
257
+ # process the header parameters
258
+ # process the form parameters
259
+ # process the body parameter
260
+ if body is not None:
261
+ _body_params = body
262
+
263
+
264
+ # set the HTTP header `Accept`
265
+ _header_params['Accept'] = self.api_client.select_header_accept(
266
+ [
267
+ 'application/json'
268
+ ]
269
+ )
270
+
271
+ # set the HTTP header `Content-Type`
272
+ if _content_type:
273
+ _header_params['Content-Type'] = _content_type
274
+ else:
275
+ _default_content_type = (
276
+ self.api_client.select_header_content_type(
277
+ [
278
+ 'application/json'
279
+ ]
280
+ )
281
+ )
282
+ if _default_content_type is not None:
283
+ _header_params['Content-Type'] = _default_content_type
284
+
285
+ # authentication setting
286
+ _auth_settings: List[str] = [
287
+ ]
288
+
289
+ return self.api_client.param_serialize(
290
+ method='POST',
291
+ resource_path='/v1:estimateThreshold',
292
+ path_params=_path_params,
293
+ query_params=_query_params,
294
+ header_params=_header_params,
295
+ body=_body_params,
296
+ post_params=_form_params,
297
+ files=_files,
298
+ auth_settings=_auth_settings,
299
+ collection_formats=_collection_formats,
300
+ _host=_host,
301
+ _request_auth=_request_auth
302
+ )
303
+
304
+