eval-studio-client 1.0.0a1__py3-none-any.whl → 1.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. eval_studio_client/api/__init__.py +43 -0
  2. eval_studio_client/api/api/__init__.py +5 -0
  3. eval_studio_client/api/api/human_calibration_service_api.py +304 -0
  4. eval_studio_client/api/api/perturbator_service_api.py +268 -1
  5. eval_studio_client/api/api/prompt_library_service_api.py +669 -0
  6. eval_studio_client/api/api/test_service_api.py +568 -0
  7. eval_studio_client/api/api/workflow_edge_service_api.py +296 -0
  8. eval_studio_client/api/api/workflow_node_service_api.py +1634 -0
  9. eval_studio_client/api/api/workflow_service_api.py +1609 -0
  10. eval_studio_client/api/docs/HumanCalibrationServiceApi.md +77 -0
  11. eval_studio_client/api/docs/PerturbationServiceCreatePerturbationRequest.md +1 -0
  12. eval_studio_client/api/docs/PerturbatorServiceApi.md +33 -3
  13. eval_studio_client/api/docs/PromptGenerationServiceAutoGeneratePromptsRequest.md +2 -1
  14. eval_studio_client/api/docs/PromptLibraryServiceApi.md +155 -0
  15. eval_studio_client/api/docs/ProtobufNullValue.md +12 -0
  16. eval_studio_client/api/docs/RequiredTheTestCaseToUpdate.md +1 -0
  17. eval_studio_client/api/docs/RequiredTheUpdatedWorkflow.md +44 -0
  18. eval_studio_client/api/docs/RequiredTheUpdatedWorkflowNode.md +44 -0
  19. eval_studio_client/api/docs/TestServiceApi.md +140 -0
  20. eval_studio_client/api/docs/TestServiceGenerateTestCasesRequest.md +1 -0
  21. eval_studio_client/api/docs/TestServiceImportTestCasesFromLibraryRequest.md +32 -0
  22. eval_studio_client/api/docs/TestServiceListTestCaseLibraryItemsRequest.md +35 -0
  23. eval_studio_client/api/docs/TestServicePerturbTestRequest.md +1 -0
  24. eval_studio_client/api/docs/V1BatchDeleteWorkflowsRequest.md +29 -0
  25. eval_studio_client/api/docs/V1BatchDeleteWorkflowsResponse.md +29 -0
  26. eval_studio_client/api/docs/V1BatchGetWorkflowEdgesResponse.md +29 -0
  27. eval_studio_client/api/docs/V1BatchGetWorkflowNodesResponse.md +29 -0
  28. eval_studio_client/api/docs/V1CreateEvaluationRequest.md +1 -0
  29. eval_studio_client/api/docs/V1CreateWorkflowResponse.md +29 -0
  30. eval_studio_client/api/docs/V1DeleteWorkflowNodeResponse.md +29 -0
  31. eval_studio_client/api/docs/V1DeleteWorkflowResponse.md +29 -0
  32. eval_studio_client/api/docs/V1EstimateThresholdRequest.md +33 -0
  33. eval_studio_client/api/docs/V1GetWorkflowNodePrerequisitesResponse.md +30 -0
  34. eval_studio_client/api/docs/V1GetWorkflowNodeResponse.md +29 -0
  35. eval_studio_client/api/docs/V1GetWorkflowResponse.md +29 -0
  36. eval_studio_client/api/docs/V1ImportEvaluationRequest.md +1 -0
  37. eval_studio_client/api/docs/V1ImportTestCasesFromLibraryResponse.md +29 -0
  38. eval_studio_client/api/docs/V1ImportTestCasesRequest.md +33 -0
  39. eval_studio_client/api/docs/V1LabeledTestCase.md +31 -0
  40. eval_studio_client/api/docs/V1ListPromptLibraryItemsResponse.md +29 -0
  41. eval_studio_client/api/docs/V1ListTestCaseLibraryItemsResponse.md +29 -0
  42. eval_studio_client/api/docs/V1ListWorkflowsResponse.md +29 -0
  43. eval_studio_client/api/docs/V1ProcessWorkflowNodeResponse.md +29 -0
  44. eval_studio_client/api/docs/V1PromptLibraryItem.md +42 -0
  45. eval_studio_client/api/docs/V1TestCase.md +1 -0
  46. eval_studio_client/api/docs/V1TestSuiteEvaluates.md +11 -0
  47. eval_studio_client/api/docs/V1UpdateWorkflowNodeResponse.md +29 -0
  48. eval_studio_client/api/docs/V1UpdateWorkflowResponse.md +29 -0
  49. eval_studio_client/api/docs/V1Workflow.md +46 -0
  50. eval_studio_client/api/docs/V1WorkflowEdge.md +40 -0
  51. eval_studio_client/api/docs/V1WorkflowEdgeType.md +12 -0
  52. eval_studio_client/api/docs/V1WorkflowNode.md +46 -0
  53. eval_studio_client/api/docs/V1WorkflowNodeArtifact.md +40 -0
  54. eval_studio_client/api/docs/V1WorkflowNodeArtifacts.md +29 -0
  55. eval_studio_client/api/docs/V1WorkflowNodeAttributes.md +30 -0
  56. eval_studio_client/api/docs/V1WorkflowNodeStatus.md +12 -0
  57. eval_studio_client/api/docs/V1WorkflowNodeType.md +12 -0
  58. eval_studio_client/api/docs/V1WorkflowNodeView.md +12 -0
  59. eval_studio_client/api/docs/V1WorkflowType.md +12 -0
  60. eval_studio_client/api/docs/WorkflowEdgeServiceApi.md +76 -0
  61. eval_studio_client/api/docs/WorkflowNodeServiceApi.md +423 -0
  62. eval_studio_client/api/docs/WorkflowServiceApi.md +417 -0
  63. eval_studio_client/api/models/__init__.py +38 -0
  64. eval_studio_client/api/models/perturbation_service_create_perturbation_request.py +8 -2
  65. eval_studio_client/api/models/prompt_generation_service_auto_generate_prompts_request.py +5 -3
  66. eval_studio_client/api/models/protobuf_null_value.py +36 -0
  67. eval_studio_client/api/models/required_the_test_case_to_update.py +6 -2
  68. eval_studio_client/api/models/required_the_updated_workflow.py +152 -0
  69. eval_studio_client/api/models/required_the_updated_workflow_node.py +152 -0
  70. eval_studio_client/api/models/test_service_generate_test_cases_request.py +4 -2
  71. eval_studio_client/api/models/test_service_import_test_cases_from_library_request.py +93 -0
  72. eval_studio_client/api/models/test_service_list_test_case_library_items_request.py +99 -0
  73. eval_studio_client/api/models/test_service_perturb_test_request.py +4 -2
  74. eval_studio_client/api/models/v1_batch_delete_workflows_request.py +87 -0
  75. eval_studio_client/api/models/v1_batch_delete_workflows_response.py +95 -0
  76. eval_studio_client/api/models/v1_batch_get_workflow_edges_response.py +95 -0
  77. eval_studio_client/api/models/v1_batch_get_workflow_nodes_response.py +95 -0
  78. eval_studio_client/api/models/v1_create_evaluation_request.py +7 -2
  79. eval_studio_client/api/models/v1_create_workflow_response.py +91 -0
  80. eval_studio_client/api/models/v1_delete_workflow_node_response.py +91 -0
  81. eval_studio_client/api/models/v1_delete_workflow_response.py +91 -0
  82. eval_studio_client/api/models/v1_estimate_threshold_request.py +103 -0
  83. eval_studio_client/api/models/v1_get_workflow_node_prerequisites_response.py +89 -0
  84. eval_studio_client/api/models/v1_get_workflow_node_response.py +91 -0
  85. eval_studio_client/api/models/v1_get_workflow_response.py +91 -0
  86. eval_studio_client/api/models/v1_import_evaluation_request.py +7 -2
  87. eval_studio_client/api/models/v1_import_test_cases_from_library_response.py +91 -0
  88. eval_studio_client/api/models/v1_import_test_cases_request.py +95 -0
  89. eval_studio_client/api/models/v1_labeled_test_case.py +91 -0
  90. eval_studio_client/api/models/v1_list_prompt_library_items_response.py +95 -0
  91. eval_studio_client/api/models/v1_list_test_case_library_items_response.py +95 -0
  92. eval_studio_client/api/models/v1_list_workflows_response.py +95 -0
  93. eval_studio_client/api/models/v1_process_workflow_node_response.py +91 -0
  94. eval_studio_client/api/models/v1_prompt_library_item.py +129 -0
  95. eval_studio_client/api/models/v1_test_case.py +6 -2
  96. eval_studio_client/api/models/v1_test_suite_evaluates.py +39 -0
  97. eval_studio_client/api/models/v1_update_workflow_node_response.py +91 -0
  98. eval_studio_client/api/models/v1_update_workflow_response.py +91 -0
  99. eval_studio_client/api/models/v1_workflow.py +156 -0
  100. eval_studio_client/api/models/v1_workflow_edge.py +123 -0
  101. eval_studio_client/api/models/v1_workflow_edge_type.py +37 -0
  102. eval_studio_client/api/models/v1_workflow_node.py +156 -0
  103. eval_studio_client/api/models/v1_workflow_node_artifact.py +122 -0
  104. eval_studio_client/api/models/v1_workflow_node_artifacts.py +97 -0
  105. eval_studio_client/api/models/v1_workflow_node_attributes.py +87 -0
  106. eval_studio_client/api/models/v1_workflow_node_status.py +40 -0
  107. eval_studio_client/api/models/v1_workflow_node_type.py +41 -0
  108. eval_studio_client/api/models/v1_workflow_node_view.py +38 -0
  109. eval_studio_client/api/models/v1_workflow_type.py +37 -0
  110. eval_studio_client/api/test/test_human_calibration_service_api.py +38 -0
  111. eval_studio_client/api/test/test_perturbation_service_create_perturbation_request.py +20 -2
  112. eval_studio_client/api/test/test_prompt_generation_service_auto_generate_prompts_request.py +4 -1
  113. eval_studio_client/api/test/test_prompt_library_service_api.py +43 -0
  114. eval_studio_client/api/test/test_protobuf_null_value.py +33 -0
  115. eval_studio_client/api/test/test_required_the_test_case_to_update.py +4 -1
  116. eval_studio_client/api/test/test_required_the_updated_workflow.py +88 -0
  117. eval_studio_client/api/test/test_required_the_updated_workflow_node.py +80 -0
  118. eval_studio_client/api/test/test_test_service_api.py +12 -0
  119. eval_studio_client/api/test/test_test_service_generate_test_cases_request.py +4 -1
  120. eval_studio_client/api/test/test_test_service_import_test_cases_from_library_request.py +56 -0
  121. eval_studio_client/api/test/test_test_service_list_test_case_library_items_request.py +63 -0
  122. eval_studio_client/api/test/test_test_service_perturb_test_request.py +4 -1
  123. eval_studio_client/api/test/test_v1_batch_delete_test_cases_response.py +4 -1
  124. eval_studio_client/api/test/test_v1_batch_delete_workflows_request.py +53 -0
  125. eval_studio_client/api/test/test_v1_batch_delete_workflows_response.py +92 -0
  126. eval_studio_client/api/test/test_v1_batch_get_workflow_edges_response.py +64 -0
  127. eval_studio_client/api/test/test_v1_batch_get_workflow_nodes_response.py +84 -0
  128. eval_studio_client/api/test/test_v1_create_evaluation_request.py +20 -2
  129. eval_studio_client/api/test/test_v1_create_test_case_response.py +4 -1
  130. eval_studio_client/api/test/test_v1_create_workflow_response.py +90 -0
  131. eval_studio_client/api/test/test_v1_delete_test_case_response.py +4 -1
  132. eval_studio_client/api/test/test_v1_delete_workflow_node_response.py +82 -0
  133. eval_studio_client/api/test/test_v1_delete_workflow_response.py +90 -0
  134. eval_studio_client/api/test/test_v1_estimate_threshold_request.py +60 -0
  135. eval_studio_client/api/test/test_v1_evaluation_test.py +4 -1
  136. eval_studio_client/api/test/test_v1_find_all_test_cases_by_id_response.py +4 -1
  137. eval_studio_client/api/test/test_v1_get_test_case_response.py +4 -1
  138. eval_studio_client/api/test/test_v1_get_workflow_node_prerequisites_response.py +56 -0
  139. eval_studio_client/api/test/test_v1_get_workflow_node_response.py +82 -0
  140. eval_studio_client/api/test/test_v1_get_workflow_response.py +90 -0
  141. eval_studio_client/api/test/test_v1_import_evaluation_request.py +16 -1
  142. eval_studio_client/api/test/test_v1_import_test_cases_from_library_response.py +71 -0
  143. eval_studio_client/api/test/test_v1_import_test_cases_request.py +57 -0
  144. eval_studio_client/api/test/test_v1_labeled_test_case.py +53 -0
  145. eval_studio_client/api/test/test_v1_list_prompt_library_items_response.py +71 -0
  146. eval_studio_client/api/test/test_v1_list_test_case_library_items_response.py +71 -0
  147. eval_studio_client/api/test/test_v1_list_test_cases_response.py +4 -1
  148. eval_studio_client/api/test/test_v1_list_workflows_response.py +92 -0
  149. eval_studio_client/api/test/test_v1_process_workflow_node_response.py +71 -0
  150. eval_studio_client/api/test/test_v1_prompt_library_item.py +68 -0
  151. eval_studio_client/api/test/test_v1_test_case.py +4 -1
  152. eval_studio_client/api/test/test_v1_test_suite_evaluates.py +33 -0
  153. eval_studio_client/api/test/test_v1_update_test_case_response.py +4 -1
  154. eval_studio_client/api/test/test_v1_update_workflow_node_response.py +82 -0
  155. eval_studio_client/api/test/test_v1_update_workflow_response.py +90 -0
  156. eval_studio_client/api/test/test_v1_workflow.py +89 -0
  157. eval_studio_client/api/test/test_v1_workflow_edge.py +61 -0
  158. eval_studio_client/api/test/test_v1_workflow_edge_type.py +33 -0
  159. eval_studio_client/api/test/test_v1_workflow_node.py +81 -0
  160. eval_studio_client/api/test/test_v1_workflow_node_artifact.py +61 -0
  161. eval_studio_client/api/test/test_v1_workflow_node_artifacts.py +64 -0
  162. eval_studio_client/api/test/test_v1_workflow_node_attributes.py +51 -0
  163. eval_studio_client/api/test/test_v1_workflow_node_status.py +33 -0
  164. eval_studio_client/api/test/test_v1_workflow_node_type.py +33 -0
  165. eval_studio_client/api/test/test_v1_workflow_node_view.py +33 -0
  166. eval_studio_client/api/test/test_v1_workflow_type.py +33 -0
  167. eval_studio_client/api/test/test_workflow_edge_service_api.py +38 -0
  168. eval_studio_client/api/test/test_workflow_node_service_api.py +73 -0
  169. eval_studio_client/api/test/test_workflow_service_api.py +73 -0
  170. eval_studio_client/client.py +7 -0
  171. eval_studio_client/dashboards.py +66 -18
  172. eval_studio_client/gen/openapiv2/eval_studio.swagger.json +2665 -794
  173. eval_studio_client/leaderboards.py +125 -0
  174. eval_studio_client/models.py +3 -42
  175. eval_studio_client/test_labs.py +49 -21
  176. eval_studio_client/tests.py +221 -51
  177. eval_studio_client/utils.py +26 -0
  178. {eval_studio_client-1.0.0a1.dist-info → eval_studio_client-1.0.1.dist-info}/METADATA +1 -2
  179. {eval_studio_client-1.0.0a1.dist-info → eval_studio_client-1.0.1.dist-info}/RECORD +180 -50
  180. {eval_studio_client-1.0.0a1.dist-info → eval_studio_client-1.0.1.dist-info}/WHEEL +1 -1
@@ -0,0 +1,95 @@
1
+ # coding: utf-8
2
+
3
+ """
4
+ ai/h2o/eval_studio/v1/collection.proto
5
+
6
+ No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
7
+
8
+ The version of the OpenAPI document: version not set
9
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
10
+
11
+ Do not edit the class manually.
12
+ """ # noqa: E501
13
+
14
+
15
+ from __future__ import annotations
16
+ import pprint
17
+ import re # noqa: F401
18
+ import json
19
+
20
+ from pydantic import BaseModel, ConfigDict, Field
21
+ from typing import Any, ClassVar, Dict, List, Optional
22
+ from eval_studio_client.api.models.v1_prompt_library_item import V1PromptLibraryItem
23
+ from typing import Optional, Set
24
+ from typing_extensions import Self
25
+
26
+ class V1ListPromptLibraryItemsResponse(BaseModel):
27
+ """
28
+ V1ListPromptLibraryItemsResponse
29
+ """ # noqa: E501
30
+ prompt_library_items: Optional[List[V1PromptLibraryItem]] = Field(default=None, description="Prompt library items (test suites).", alias="promptLibraryItems")
31
+ __properties: ClassVar[List[str]] = ["promptLibraryItems"]
32
+
33
+ model_config = ConfigDict(
34
+ populate_by_name=True,
35
+ validate_assignment=True,
36
+ protected_namespaces=(),
37
+ )
38
+
39
+
40
+ def to_str(self) -> str:
41
+ """Returns the string representation of the model using alias"""
42
+ return pprint.pformat(self.model_dump(by_alias=True))
43
+
44
+ def to_json(self) -> str:
45
+ """Returns the JSON representation of the model using alias"""
46
+ # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
47
+ return json.dumps(self.to_dict())
48
+
49
+ @classmethod
50
+ def from_json(cls, json_str: str) -> Optional[Self]:
51
+ """Create an instance of V1ListPromptLibraryItemsResponse from a JSON string"""
52
+ return cls.from_dict(json.loads(json_str))
53
+
54
+ def to_dict(self) -> Dict[str, Any]:
55
+ """Return the dictionary representation of the model using alias.
56
+
57
+ This has the following differences from calling pydantic's
58
+ `self.model_dump(by_alias=True)`:
59
+
60
+ * `None` is only added to the output dict for nullable fields that
61
+ were set at model initialization. Other fields with value `None`
62
+ are ignored.
63
+ """
64
+ excluded_fields: Set[str] = set([
65
+ ])
66
+
67
+ _dict = self.model_dump(
68
+ by_alias=True,
69
+ exclude=excluded_fields,
70
+ exclude_none=True,
71
+ )
72
+ # override the default output from pydantic by calling `to_dict()` of each item in prompt_library_items (list)
73
+ _items = []
74
+ if self.prompt_library_items:
75
+ for _item in self.prompt_library_items:
76
+ if _item:
77
+ _items.append(_item.to_dict())
78
+ _dict['promptLibraryItems'] = _items
79
+ return _dict
80
+
81
+ @classmethod
82
+ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
83
+ """Create an instance of V1ListPromptLibraryItemsResponse from a dict"""
84
+ if obj is None:
85
+ return None
86
+
87
+ if not isinstance(obj, dict):
88
+ return cls.model_validate(obj)
89
+
90
+ _obj = cls.model_validate({
91
+ "promptLibraryItems": [V1PromptLibraryItem.from_dict(_item) for _item in obj["promptLibraryItems"]] if obj.get("promptLibraryItems") is not None else None
92
+ })
93
+ return _obj
94
+
95
+
@@ -0,0 +1,95 @@
1
+ # coding: utf-8
2
+
3
+ """
4
+ ai/h2o/eval_studio/v1/collection.proto
5
+
6
+ No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
7
+
8
+ The version of the OpenAPI document: version not set
9
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
10
+
11
+ Do not edit the class manually.
12
+ """ # noqa: E501
13
+
14
+
15
+ from __future__ import annotations
16
+ import pprint
17
+ import re # noqa: F401
18
+ import json
19
+
20
+ from pydantic import BaseModel, ConfigDict, Field
21
+ from typing import Any, ClassVar, Dict, List, Optional
22
+ from eval_studio_client.api.models.v1_prompt_library_item import V1PromptLibraryItem
23
+ from typing import Optional, Set
24
+ from typing_extensions import Self
25
+
26
+ class V1ListTestCaseLibraryItemsResponse(BaseModel):
27
+ """
28
+ V1ListTestCaseLibraryItemsResponse
29
+ """ # noqa: E501
30
+ prompt_library_items: Optional[List[V1PromptLibraryItem]] = Field(default=None, description="Test suites library items.", alias="promptLibraryItems")
31
+ __properties: ClassVar[List[str]] = ["promptLibraryItems"]
32
+
33
+ model_config = ConfigDict(
34
+ populate_by_name=True,
35
+ validate_assignment=True,
36
+ protected_namespaces=(),
37
+ )
38
+
39
+
40
+ def to_str(self) -> str:
41
+ """Returns the string representation of the model using alias"""
42
+ return pprint.pformat(self.model_dump(by_alias=True))
43
+
44
+ def to_json(self) -> str:
45
+ """Returns the JSON representation of the model using alias"""
46
+ # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
47
+ return json.dumps(self.to_dict())
48
+
49
+ @classmethod
50
+ def from_json(cls, json_str: str) -> Optional[Self]:
51
+ """Create an instance of V1ListTestCaseLibraryItemsResponse from a JSON string"""
52
+ return cls.from_dict(json.loads(json_str))
53
+
54
+ def to_dict(self) -> Dict[str, Any]:
55
+ """Return the dictionary representation of the model using alias.
56
+
57
+ This has the following differences from calling pydantic's
58
+ `self.model_dump(by_alias=True)`:
59
+
60
+ * `None` is only added to the output dict for nullable fields that
61
+ were set at model initialization. Other fields with value `None`
62
+ are ignored.
63
+ """
64
+ excluded_fields: Set[str] = set([
65
+ ])
66
+
67
+ _dict = self.model_dump(
68
+ by_alias=True,
69
+ exclude=excluded_fields,
70
+ exclude_none=True,
71
+ )
72
+ # override the default output from pydantic by calling `to_dict()` of each item in prompt_library_items (list)
73
+ _items = []
74
+ if self.prompt_library_items:
75
+ for _item in self.prompt_library_items:
76
+ if _item:
77
+ _items.append(_item.to_dict())
78
+ _dict['promptLibraryItems'] = _items
79
+ return _dict
80
+
81
+ @classmethod
82
+ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
83
+ """Create an instance of V1ListTestCaseLibraryItemsResponse from a dict"""
84
+ if obj is None:
85
+ return None
86
+
87
+ if not isinstance(obj, dict):
88
+ return cls.model_validate(obj)
89
+
90
+ _obj = cls.model_validate({
91
+ "promptLibraryItems": [V1PromptLibraryItem.from_dict(_item) for _item in obj["promptLibraryItems"]] if obj.get("promptLibraryItems") is not None else None
92
+ })
93
+ return _obj
94
+
95
+
@@ -0,0 +1,95 @@
1
+ # coding: utf-8
2
+
3
+ """
4
+ ai/h2o/eval_studio/v1/collection.proto
5
+
6
+ No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
7
+
8
+ The version of the OpenAPI document: version not set
9
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
10
+
11
+ Do not edit the class manually.
12
+ """ # noqa: E501
13
+
14
+
15
+ from __future__ import annotations
16
+ import pprint
17
+ import re # noqa: F401
18
+ import json
19
+
20
+ from pydantic import BaseModel, ConfigDict, Field
21
+ from typing import Any, ClassVar, Dict, List, Optional
22
+ from eval_studio_client.api.models.v1_workflow import V1Workflow
23
+ from typing import Optional, Set
24
+ from typing_extensions import Self
25
+
26
+ class V1ListWorkflowsResponse(BaseModel):
27
+ """
28
+ V1ListWorkflowsResponse
29
+ """ # noqa: E501
30
+ workflows: Optional[List[V1Workflow]] = Field(default=None, description="The Workflows requested.")
31
+ __properties: ClassVar[List[str]] = ["workflows"]
32
+
33
+ model_config = ConfigDict(
34
+ populate_by_name=True,
35
+ validate_assignment=True,
36
+ protected_namespaces=(),
37
+ )
38
+
39
+
40
+ def to_str(self) -> str:
41
+ """Returns the string representation of the model using alias"""
42
+ return pprint.pformat(self.model_dump(by_alias=True))
43
+
44
+ def to_json(self) -> str:
45
+ """Returns the JSON representation of the model using alias"""
46
+ # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
47
+ return json.dumps(self.to_dict())
48
+
49
+ @classmethod
50
+ def from_json(cls, json_str: str) -> Optional[Self]:
51
+ """Create an instance of V1ListWorkflowsResponse from a JSON string"""
52
+ return cls.from_dict(json.loads(json_str))
53
+
54
+ def to_dict(self) -> Dict[str, Any]:
55
+ """Return the dictionary representation of the model using alias.
56
+
57
+ This has the following differences from calling pydantic's
58
+ `self.model_dump(by_alias=True)`:
59
+
60
+ * `None` is only added to the output dict for nullable fields that
61
+ were set at model initialization. Other fields with value `None`
62
+ are ignored.
63
+ """
64
+ excluded_fields: Set[str] = set([
65
+ ])
66
+
67
+ _dict = self.model_dump(
68
+ by_alias=True,
69
+ exclude=excluded_fields,
70
+ exclude_none=True,
71
+ )
72
+ # override the default output from pydantic by calling `to_dict()` of each item in workflows (list)
73
+ _items = []
74
+ if self.workflows:
75
+ for _item in self.workflows:
76
+ if _item:
77
+ _items.append(_item.to_dict())
78
+ _dict['workflows'] = _items
79
+ return _dict
80
+
81
+ @classmethod
82
+ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
83
+ """Create an instance of V1ListWorkflowsResponse from a dict"""
84
+ if obj is None:
85
+ return None
86
+
87
+ if not isinstance(obj, dict):
88
+ return cls.model_validate(obj)
89
+
90
+ _obj = cls.model_validate({
91
+ "workflows": [V1Workflow.from_dict(_item) for _item in obj["workflows"]] if obj.get("workflows") is not None else None
92
+ })
93
+ return _obj
94
+
95
+
@@ -0,0 +1,91 @@
1
+ # coding: utf-8
2
+
3
+ """
4
+ ai/h2o/eval_studio/v1/collection.proto
5
+
6
+ No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
7
+
8
+ The version of the OpenAPI document: version not set
9
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
10
+
11
+ Do not edit the class manually.
12
+ """ # noqa: E501
13
+
14
+
15
+ from __future__ import annotations
16
+ import pprint
17
+ import re # noqa: F401
18
+ import json
19
+
20
+ from pydantic import BaseModel, ConfigDict
21
+ from typing import Any, ClassVar, Dict, List, Optional
22
+ from eval_studio_client.api.models.v1_operation import V1Operation
23
+ from typing import Optional, Set
24
+ from typing_extensions import Self
25
+
26
+ class V1ProcessWorkflowNodeResponse(BaseModel):
27
+ """
28
+ V1ProcessWorkflowNodeResponse
29
+ """ # noqa: E501
30
+ operation: Optional[V1Operation] = None
31
+ __properties: ClassVar[List[str]] = ["operation"]
32
+
33
+ model_config = ConfigDict(
34
+ populate_by_name=True,
35
+ validate_assignment=True,
36
+ protected_namespaces=(),
37
+ )
38
+
39
+
40
+ def to_str(self) -> str:
41
+ """Returns the string representation of the model using alias"""
42
+ return pprint.pformat(self.model_dump(by_alias=True))
43
+
44
+ def to_json(self) -> str:
45
+ """Returns the JSON representation of the model using alias"""
46
+ # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
47
+ return json.dumps(self.to_dict())
48
+
49
+ @classmethod
50
+ def from_json(cls, json_str: str) -> Optional[Self]:
51
+ """Create an instance of V1ProcessWorkflowNodeResponse from a JSON string"""
52
+ return cls.from_dict(json.loads(json_str))
53
+
54
+ def to_dict(self) -> Dict[str, Any]:
55
+ """Return the dictionary representation of the model using alias.
56
+
57
+ This has the following differences from calling pydantic's
58
+ `self.model_dump(by_alias=True)`:
59
+
60
+ * `None` is only added to the output dict for nullable fields that
61
+ were set at model initialization. Other fields with value `None`
62
+ are ignored.
63
+ """
64
+ excluded_fields: Set[str] = set([
65
+ ])
66
+
67
+ _dict = self.model_dump(
68
+ by_alias=True,
69
+ exclude=excluded_fields,
70
+ exclude_none=True,
71
+ )
72
+ # override the default output from pydantic by calling `to_dict()` of operation
73
+ if self.operation:
74
+ _dict['operation'] = self.operation.to_dict()
75
+ return _dict
76
+
77
+ @classmethod
78
+ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
79
+ """Create an instance of V1ProcessWorkflowNodeResponse from a dict"""
80
+ if obj is None:
81
+ return None
82
+
83
+ if not isinstance(obj, dict):
84
+ return cls.model_validate(obj)
85
+
86
+ _obj = cls.model_validate({
87
+ "operation": V1Operation.from_dict(obj["operation"]) if obj.get("operation") is not None else None
88
+ })
89
+ return _obj
90
+
91
+
@@ -0,0 +1,129 @@
1
+ # coding: utf-8
2
+
3
+ """
4
+ ai/h2o/eval_studio/v1/collection.proto
5
+
6
+ No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
7
+
8
+ The version of the OpenAPI document: version not set
9
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
10
+
11
+ Do not edit the class manually.
12
+ """ # noqa: E501
13
+
14
+
15
+ from __future__ import annotations
16
+ import pprint
17
+ import re # noqa: F401
18
+ import json
19
+
20
+ from datetime import datetime
21
+ from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr
22
+ from typing import Any, ClassVar, Dict, List, Optional
23
+ from eval_studio_client.api.models.v1_test_suite_evaluates import V1TestSuiteEvaluates
24
+ from typing import Optional, Set
25
+ from typing_extensions import Self
26
+
27
+ class V1PromptLibraryItem(BaseModel):
28
+ """
29
+ V1PromptLibraryItem
30
+ """ # noqa: E501
31
+ name: Optional[StrictStr] = None
32
+ create_time: Optional[datetime] = Field(default=None, description="Output only. Timestamp when the item was created.", alias="createTime")
33
+ creator: Optional[StrictStr] = Field(default=None, description="Output only. Name of the user or service that requested creation of the item.")
34
+ update_time: Optional[datetime] = Field(default=None, description="Output only. Optional. Timestamp when the item was last updated.", alias="updateTime")
35
+ updater: Optional[StrictStr] = Field(default=None, description="Output only. Optional. Name of the user or service that requested update of the item.")
36
+ delete_time: Optional[datetime] = Field(default=None, description="Output only. Optional. Set when the item is deleted. When set, item should be considered as deleted.", alias="deleteTime")
37
+ deleter: Optional[StrictStr] = Field(default=None, description="Output only. Optional. Name of the user or service that requested deletion of the item.")
38
+ display_name: Optional[StrictStr] = Field(default=None, description="Human readable name of the item.", alias="displayName")
39
+ description: Optional[StrictStr] = Field(default=None, description="Optional. Arbitrary description of the item.")
40
+ test_suite_url: Optional[StrictStr] = Field(default=None, description="URL of the test suite which is represented by this library item.", alias="testSuiteUrl")
41
+ test_count: Optional[StrictInt] = Field(default=None, description="Number of tests in the test suite.", alias="testCount")
42
+ test_case_count: Optional[StrictInt] = Field(default=None, description="Number of test cases in the test suite.", alias="testCaseCount")
43
+ evaluates: Optional[List[V1TestSuiteEvaluates]] = Field(default=None, description="Types of systems evaluated by this item - like RAG (has corpus), LLM (no corpus) or agents.")
44
+ categories: Optional[List[StrictStr]] = Field(default=None, description="Categories of test cases in the item - like question_answering or summarization.")
45
+ __properties: ClassVar[List[str]] = ["name", "createTime", "creator", "updateTime", "updater", "deleteTime", "deleter", "displayName", "description", "testSuiteUrl", "testCount", "testCaseCount", "evaluates", "categories"]
46
+
47
+ model_config = ConfigDict(
48
+ populate_by_name=True,
49
+ validate_assignment=True,
50
+ protected_namespaces=(),
51
+ )
52
+
53
+
54
+ def to_str(self) -> str:
55
+ """Returns the string representation of the model using alias"""
56
+ return pprint.pformat(self.model_dump(by_alias=True))
57
+
58
+ def to_json(self) -> str:
59
+ """Returns the JSON representation of the model using alias"""
60
+ # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
61
+ return json.dumps(self.to_dict())
62
+
63
+ @classmethod
64
+ def from_json(cls, json_str: str) -> Optional[Self]:
65
+ """Create an instance of V1PromptLibraryItem from a JSON string"""
66
+ return cls.from_dict(json.loads(json_str))
67
+
68
+ def to_dict(self) -> Dict[str, Any]:
69
+ """Return the dictionary representation of the model using alias.
70
+
71
+ This has the following differences from calling pydantic's
72
+ `self.model_dump(by_alias=True)`:
73
+
74
+ * `None` is only added to the output dict for nullable fields that
75
+ were set at model initialization. Other fields with value `None`
76
+ are ignored.
77
+ * OpenAPI `readOnly` fields are excluded.
78
+ * OpenAPI `readOnly` fields are excluded.
79
+ * OpenAPI `readOnly` fields are excluded.
80
+ * OpenAPI `readOnly` fields are excluded.
81
+ * OpenAPI `readOnly` fields are excluded.
82
+ * OpenAPI `readOnly` fields are excluded.
83
+ * OpenAPI `readOnly` fields are excluded.
84
+ """
85
+ excluded_fields: Set[str] = set([
86
+ "name",
87
+ "create_time",
88
+ "creator",
89
+ "update_time",
90
+ "updater",
91
+ "delete_time",
92
+ "deleter",
93
+ ])
94
+
95
+ _dict = self.model_dump(
96
+ by_alias=True,
97
+ exclude=excluded_fields,
98
+ exclude_none=True,
99
+ )
100
+ return _dict
101
+
102
+ @classmethod
103
+ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
104
+ """Create an instance of V1PromptLibraryItem from a dict"""
105
+ if obj is None:
106
+ return None
107
+
108
+ if not isinstance(obj, dict):
109
+ return cls.model_validate(obj)
110
+
111
+ _obj = cls.model_validate({
112
+ "name": obj.get("name"),
113
+ "createTime": obj.get("createTime"),
114
+ "creator": obj.get("creator"),
115
+ "updateTime": obj.get("updateTime"),
116
+ "updater": obj.get("updater"),
117
+ "deleteTime": obj.get("deleteTime"),
118
+ "deleter": obj.get("deleter"),
119
+ "displayName": obj.get("displayName"),
120
+ "description": obj.get("description"),
121
+ "testSuiteUrl": obj.get("testSuiteUrl"),
122
+ "testCount": obj.get("testCount"),
123
+ "testCaseCount": obj.get("testCaseCount"),
124
+ "evaluates": obj.get("evaluates"),
125
+ "categories": obj.get("categories")
126
+ })
127
+ return _obj
128
+
129
+
@@ -39,7 +39,8 @@ class V1TestCase(BaseModel):
39
39
  answer: Optional[StrictStr] = Field(default=None, description="Expected answer text. Model output.")
40
40
  constraints: Optional[List[StrictStr]] = Field(default=None, description="Constraints on the model output.")
41
41
  condition: Optional[StrictStr] = Field(default=None, description="Optional. Test case output condition, in a form of AIP-160 compliant filter expression.")
42
- __properties: ClassVar[List[str]] = ["name", "createTime", "creator", "updateTime", "updater", "deleteTime", "deleter", "parent", "prompt", "answer", "constraints", "condition"]
42
+ perturbed_by: Optional[List[StrictStr]] = Field(default=None, description="Output only. The list of perturbators applied to this test case.", alias="perturbedBy")
43
+ __properties: ClassVar[List[str]] = ["name", "createTime", "creator", "updateTime", "updater", "deleteTime", "deleter", "parent", "prompt", "answer", "constraints", "condition", "perturbedBy"]
43
44
 
44
45
  model_config = ConfigDict(
45
46
  populate_by_name=True,
@@ -78,6 +79,7 @@ class V1TestCase(BaseModel):
78
79
  * OpenAPI `readOnly` fields are excluded.
79
80
  * OpenAPI `readOnly` fields are excluded.
80
81
  * OpenAPI `readOnly` fields are excluded.
82
+ * OpenAPI `readOnly` fields are excluded.
81
83
  """
82
84
  excluded_fields: Set[str] = set([
83
85
  "name",
@@ -87,6 +89,7 @@ class V1TestCase(BaseModel):
87
89
  "updater",
88
90
  "delete_time",
89
91
  "deleter",
92
+ "perturbed_by",
90
93
  ])
91
94
 
92
95
  _dict = self.model_dump(
@@ -117,7 +120,8 @@ class V1TestCase(BaseModel):
117
120
  "prompt": obj.get("prompt"),
118
121
  "answer": obj.get("answer"),
119
122
  "constraints": obj.get("constraints"),
120
- "condition": obj.get("condition")
123
+ "condition": obj.get("condition"),
124
+ "perturbedBy": obj.get("perturbedBy")
121
125
  })
122
126
  return _obj
123
127
 
@@ -0,0 +1,39 @@
1
+ # coding: utf-8
2
+
3
+ """
4
+ ai/h2o/eval_studio/v1/collection.proto
5
+
6
+ No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
7
+
8
+ The version of the OpenAPI document: version not set
9
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
10
+
11
+ Do not edit the class manually.
12
+ """ # noqa: E501
13
+
14
+
15
+ from __future__ import annotations
16
+ import json
17
+ from enum import Enum
18
+ from typing_extensions import Self
19
+
20
+
21
+ class V1TestSuiteEvaluates(str, Enum):
22
+ """
23
+ V1TestSuiteEvaluates
24
+ """
25
+
26
+ """
27
+ allowed enum values
28
+ """
29
+ TEST_SUITE_EVALUATES_UNSPECIFIED = 'TEST_SUITE_EVALUATES_UNSPECIFIED'
30
+ TEST_SUITE_EVALUATES_RAG = 'TEST_SUITE_EVALUATES_RAG'
31
+ TEST_SUITE_EVALUATES_LLM = 'TEST_SUITE_EVALUATES_LLM'
32
+ TEST_SUITE_EVALUATES_AGENT = 'TEST_SUITE_EVALUATES_AGENT'
33
+
34
+ @classmethod
35
+ def from_json(cls, json_str: str) -> Self:
36
+ """Create an instance of V1TestSuiteEvaluates from a JSON string"""
37
+ return cls(json.loads(json_str))
38
+
39
+
@@ -0,0 +1,91 @@
1
+ # coding: utf-8
2
+
3
+ """
4
+ ai/h2o/eval_studio/v1/collection.proto
5
+
6
+ No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
7
+
8
+ The version of the OpenAPI document: version not set
9
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
10
+
11
+ Do not edit the class manually.
12
+ """ # noqa: E501
13
+
14
+
15
+ from __future__ import annotations
16
+ import pprint
17
+ import re # noqa: F401
18
+ import json
19
+
20
+ from pydantic import BaseModel, ConfigDict
21
+ from typing import Any, ClassVar, Dict, List, Optional
22
+ from eval_studio_client.api.models.v1_workflow_node import V1WorkflowNode
23
+ from typing import Optional, Set
24
+ from typing_extensions import Self
25
+
26
+ class V1UpdateWorkflowNodeResponse(BaseModel):
27
+ """
28
+ V1UpdateWorkflowNodeResponse
29
+ """ # noqa: E501
30
+ node: Optional[V1WorkflowNode] = None
31
+ __properties: ClassVar[List[str]] = ["node"]
32
+
33
+ model_config = ConfigDict(
34
+ populate_by_name=True,
35
+ validate_assignment=True,
36
+ protected_namespaces=(),
37
+ )
38
+
39
+
40
+ def to_str(self) -> str:
41
+ """Returns the string representation of the model using alias"""
42
+ return pprint.pformat(self.model_dump(by_alias=True))
43
+
44
+ def to_json(self) -> str:
45
+ """Returns the JSON representation of the model using alias"""
46
+ # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
47
+ return json.dumps(self.to_dict())
48
+
49
+ @classmethod
50
+ def from_json(cls, json_str: str) -> Optional[Self]:
51
+ """Create an instance of V1UpdateWorkflowNodeResponse from a JSON string"""
52
+ return cls.from_dict(json.loads(json_str))
53
+
54
+ def to_dict(self) -> Dict[str, Any]:
55
+ """Return the dictionary representation of the model using alias.
56
+
57
+ This has the following differences from calling pydantic's
58
+ `self.model_dump(by_alias=True)`:
59
+
60
+ * `None` is only added to the output dict for nullable fields that
61
+ were set at model initialization. Other fields with value `None`
62
+ are ignored.
63
+ """
64
+ excluded_fields: Set[str] = set([
65
+ ])
66
+
67
+ _dict = self.model_dump(
68
+ by_alias=True,
69
+ exclude=excluded_fields,
70
+ exclude_none=True,
71
+ )
72
+ # override the default output from pydantic by calling `to_dict()` of node
73
+ if self.node:
74
+ _dict['node'] = self.node.to_dict()
75
+ return _dict
76
+
77
+ @classmethod
78
+ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
79
+ """Create an instance of V1UpdateWorkflowNodeResponse from a dict"""
80
+ if obj is None:
81
+ return None
82
+
83
+ if not isinstance(obj, dict):
84
+ return cls.model_validate(obj)
85
+
86
+ _obj = cls.model_validate({
87
+ "node": V1WorkflowNode.from_dict(obj["node"]) if obj.get("node") is not None else None
88
+ })
89
+ return _obj
90
+
91
+