eval-studio-client 1.2.4a2__py3-none-any.whl → 1.3.0__py3-none-any.whl

eval_studio_client/api/models/v1_batch_create_leaderboards_without_cache_response.py

@@ -0,0 +1,91 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict
+from typing import Any, ClassVar, Dict, List, Optional
+from eval_studio_client.api.models.v1_operation import V1Operation
+from typing import Optional, Set
+from typing_extensions import Self
+
+class V1BatchCreateLeaderboardsWithoutCacheResponse(BaseModel):
+    """
+    V1BatchCreateLeaderboardsWithoutCacheResponse
+    """ # noqa: E501
+    operation: Optional[V1Operation] = None
+    __properties: ClassVar[List[str]] = ["operation"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of V1BatchCreateLeaderboardsWithoutCacheResponse from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        # override the default output from pydantic by calling `to_dict()` of operation
+        if self.operation:
+            _dict['operation'] = self.operation.to_dict()
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of V1BatchCreateLeaderboardsWithoutCacheResponse from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj, strict=False)
+
+        _obj = cls.model_validate({
+            "operation": V1Operation.from_dict(obj["operation"]) if obj.get("operation") is not None else None
+        }, strict=False)
+        return _obj
+
+

eval_studio_client/api/models/v1_batch_mark_operation_seen_by_creator_response.py

@@ -0,0 +1,95 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, Field
+from typing import Any, ClassVar, Dict, List, Optional
+from eval_studio_client.api.models.v1_operation import V1Operation
+from typing import Optional, Set
+from typing_extensions import Self
+
+class V1BatchMarkOperationSeenByCreatorResponse(BaseModel):
+    """
+    V1BatchMarkOperationSeenByCreatorResponse
+    """ # noqa: E501
+    operations: Optional[List[V1Operation]] = Field(default=None, description="The updated Operations.")
+    __properties: ClassVar[List[str]] = ["operations"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of V1BatchMarkOperationSeenByCreatorResponse from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        # override the default output from pydantic by calling `to_dict()` of each item in operations (list)
+        _items = []
+        if self.operations:
+            for _item_operations in self.operations:
+                if _item_operations:
+                    _items.append(_item_operations.to_dict())
+            _dict['operations'] = _items
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of V1BatchMarkOperationSeenByCreatorResponse from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj, strict=False)
+
+        _obj = cls.model_validate({
+            "operations": [V1Operation.from_dict(_item) for _item in obj["operations"]] if obj.get("operations") is not None else None
+        }, strict=False)
+        return _obj
+
+

eval_studio_client/api/models/v1_cmp_leaderboard_reports_request.py

@@ -0,0 +1,96 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, Field, StrictStr
+from typing import Any, ClassVar, Dict, List, Optional
+from eval_studio_client.api.models.v1_text_similarity_metric import V1TextSimilarityMetric
+from typing import Optional, Set
+from typing_extensions import Self
+
+class V1CmpLeaderboardReportsRequest(BaseModel):
+    """
+    V1CmpLeaderboardReportsRequest
+    """ # noqa: E501
+    leaderboard_report_baseline: Optional[StrictStr] = Field(default=None, description="Required. The baseline Leaderboard Report to compare against (JSON string).", alias="leaderboardReportBaseline")
+    leaderboard_report_current: Optional[StrictStr] = Field(default=None, description="Required. The current Leaderboard Report to compare (JSON string).", alias="leaderboardReportCurrent")
+    text_similarity_metric: Optional[V1TextSimilarityMetric] = Field(default=None, alias="textSimilarityMetric")
+    llm_model_name_baseline: Optional[StrictStr] = Field(default=None, description="Required. The baseline LLM model name to compare.", alias="llmModelNameBaseline")
+    llm_model_name_current: Optional[StrictStr] = Field(default=None, description="Required. The current LLM model name to compare.", alias="llmModelNameCurrent")
+    __properties: ClassVar[List[str]] = ["leaderboardReportBaseline", "leaderboardReportCurrent", "textSimilarityMetric", "llmModelNameBaseline", "llmModelNameCurrent"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of V1CmpLeaderboardReportsRequest from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of V1CmpLeaderboardReportsRequest from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj, strict=False)
+
+        _obj = cls.model_validate({
+            "leaderboardReportBaseline": obj.get("leaderboardReportBaseline"),
+            "leaderboardReportCurrent": obj.get("leaderboardReportCurrent"),
+            "textSimilarityMetric": obj.get("textSimilarityMetric"),
+            "llmModelNameBaseline": obj.get("llmModelNameBaseline"),
+            "llmModelNameCurrent": obj.get("llmModelNameCurrent")
+        }, strict=False)
+        return _obj
+
+

eval_studio_client/api/models/v1_cmp_leaderboard_reports_response.py

@@ -0,0 +1,91 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, Field
+from typing import Any, ClassVar, Dict, List, Optional
+from eval_studio_client.api.models.v1_leaderboard_cmp_report import V1LeaderboardCmpReport
+from typing import Optional, Set
+from typing_extensions import Self
+
+class V1CmpLeaderboardReportsResponse(BaseModel):
+    """
+    V1CmpLeaderboardReportsResponse
+    """ # noqa: E501
+    leaderboard_cmp_report: Optional[V1LeaderboardCmpReport] = Field(default=None, alias="leaderboardCmpReport")
+    __properties: ClassVar[List[str]] = ["leaderboardCmpReport"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of V1CmpLeaderboardReportsResponse from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        # override the default output from pydantic by calling `to_dict()` of leaderboard_cmp_report
+        if self.leaderboard_cmp_report:
+            _dict['leaderboardCmpReport'] = self.leaderboard_cmp_report.to_dict()
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of V1CmpLeaderboardReportsResponse from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj, strict=False)
+
+        _obj = cls.model_validate({
+            "leaderboardCmpReport": V1LeaderboardCmpReport.from_dict(obj["leaderboardCmpReport"]) if obj.get("leaderboardCmpReport") is not None else None
+        }, strict=False)
+        return _obj
+
+

eval_studio_client/api/models/v1_comparison_item.py

@@ -0,0 +1,130 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, Field, StrictStr
+from typing import Any, ClassVar, Dict, List, Optional
+from eval_studio_client.api.models.v1_actual_output_meta_diff import V1ActualOutputMetaDiff
+from eval_studio_client.api.models.v1_flipped_metric import V1FlippedMetric
+from eval_studio_client.api.models.v1_retrieved_context_diff import V1RetrievedContextDiff
+from eval_studio_client.api.models.v1_test_case_result import V1TestCaseResult
+from typing import Optional, Set
+from typing_extensions import Self
+
+class V1ComparisonItem(BaseModel):
+    """
+    V1ComparisonItem
+    """ # noqa: E501
+    question: Optional[StrictStr] = Field(default=None, description="The question being evaluated.")
+    diff_flipped_metrics: Optional[List[V1FlippedMetric]] = Field(default=None, description="List of metrics that flipped between baseline and current.", alias="diffFlippedMetrics")
+    baseline_test_case_result: Optional[V1TestCaseResult] = Field(default=None, alias="baselineTestCaseResult")
+    baseline_diff_actual_output_meta: Optional[V1ActualOutputMetaDiff] = Field(default=None, alias="baselineDiffActualOutputMeta")
+    baseline_diff_retrieved_context: Optional[V1RetrievedContextDiff] = Field(default=None, alias="baselineDiffRetrievedContext")
+    current_test_case_result: Optional[V1TestCaseResult] = Field(default=None, alias="currentTestCaseResult")
+    current_diff_actual_output_meta: Optional[V1ActualOutputMetaDiff] = Field(default=None, alias="currentDiffActualOutputMeta")
+    current_diff_retrieved_context: Optional[V1RetrievedContextDiff] = Field(default=None, alias="currentDiffRetrievedContext")
+    __properties: ClassVar[List[str]] = ["question", "diffFlippedMetrics", "baselineTestCaseResult", "baselineDiffActualOutputMeta", "baselineDiffRetrievedContext", "currentTestCaseResult", "currentDiffActualOutputMeta", "currentDiffRetrievedContext"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of V1ComparisonItem from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        # override the default output from pydantic by calling `to_dict()` of each item in diff_flipped_metrics (list)
+        _items = []
+        if self.diff_flipped_metrics:
+            for _item_diff_flipped_metrics in self.diff_flipped_metrics:
+                if _item_diff_flipped_metrics:
+                    _items.append(_item_diff_flipped_metrics.to_dict())
+            _dict['diffFlippedMetrics'] = _items
+        # override the default output from pydantic by calling `to_dict()` of baseline_test_case_result
+        if self.baseline_test_case_result:
+            _dict['baselineTestCaseResult'] = self.baseline_test_case_result.to_dict()
+        # override the default output from pydantic by calling `to_dict()` of baseline_diff_actual_output_meta
+        if self.baseline_diff_actual_output_meta:
+            _dict['baselineDiffActualOutputMeta'] = self.baseline_diff_actual_output_meta.to_dict()
+        # override the default output from pydantic by calling `to_dict()` of baseline_diff_retrieved_context
+        if self.baseline_diff_retrieved_context:
+            _dict['baselineDiffRetrievedContext'] = self.baseline_diff_retrieved_context.to_dict()
+        # override the default output from pydantic by calling `to_dict()` of current_test_case_result
+        if self.current_test_case_result:
+            _dict['currentTestCaseResult'] = self.current_test_case_result.to_dict()
+        # override the default output from pydantic by calling `to_dict()` of current_diff_actual_output_meta
+        if self.current_diff_actual_output_meta:
+            _dict['currentDiffActualOutputMeta'] = self.current_diff_actual_output_meta.to_dict()
+        # override the default output from pydantic by calling `to_dict()` of current_diff_retrieved_context
+        if self.current_diff_retrieved_context:
+            _dict['currentDiffRetrievedContext'] = self.current_diff_retrieved_context.to_dict()
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of V1ComparisonItem from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj, strict=False)
+
+        _obj = cls.model_validate({
+            "question": obj.get("question"),
+            "diffFlippedMetrics": [V1FlippedMetric.from_dict(_item) for _item in obj["diffFlippedMetrics"]] if obj.get("diffFlippedMetrics") is not None else None,
+            "baselineTestCaseResult": V1TestCaseResult.from_dict(obj["baselineTestCaseResult"]) if obj.get("baselineTestCaseResult") is not None else None,
+            "baselineDiffActualOutputMeta": V1ActualOutputMetaDiff.from_dict(obj["baselineDiffActualOutputMeta"]) if obj.get("baselineDiffActualOutputMeta") is not None else None,
+            "baselineDiffRetrievedContext": V1RetrievedContextDiff.from_dict(obj["baselineDiffRetrievedContext"]) if obj.get("baselineDiffRetrievedContext") is not None else None,
+            "currentTestCaseResult": V1TestCaseResult.from_dict(obj["currentTestCaseResult"]) if obj.get("currentTestCaseResult") is not None else None,
+            "currentDiffActualOutputMeta": V1ActualOutputMetaDiff.from_dict(obj["currentDiffActualOutputMeta"]) if obj.get("currentDiffActualOutputMeta") is not None else None,
+            "currentDiffRetrievedContext": V1RetrievedContextDiff.from_dict(obj["currentDiffRetrievedContext"]) if obj.get("currentDiffRetrievedContext") is not None else None
+        }, strict=False)
+        return _obj
+
+

eval_studio_client/api/models/v1_comparison_metric_score.py

@@ -0,0 +1,89 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr
+from typing import Any, ClassVar, Dict, List, Optional, Union
+from typing import Optional, Set
+from typing_extensions import Self
+
+class V1ComparisonMetricScore(BaseModel):
+    """
+    V1ComparisonMetricScore
+    """ # noqa: E501
+    metric_name: Optional[StrictStr] = Field(default=None, description="Metric name.", alias="metricName")
+    metric_score: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="Metric score value.", alias="metricScore")
+    __properties: ClassVar[List[str]] = ["metricName", "metricScore"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of V1ComparisonMetricScore from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of V1ComparisonMetricScore from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj, strict=False)
+
+        _obj = cls.model_validate({
+            "metricName": obj.get("metricName"),
+            "metricScore": obj.get("metricScore")
+        }, strict=False)
+        return _obj
+
+