agenta 0.8.4__py3-none-any.whl → 0.10.0__py3-none-any.whl

agenta/client/backend/types/__init__.py

@@ -3,37 +3,46 @@
 from .add_variant_from_base_and_config_response import (
     AddVariantFromBaseAndConfigResponse,
 )
+from .aggregated_result import AggregatedResult
 from .app import App
 from .app_variant_output import AppVariantOutput
+from .app_variant_output_extended import AppVariantOutputExtended
+from .app_variant_revision import AppVariantRevision
 from .base_output import BaseOutput
 from .body_import_testset import BodyImportTestset
+from .config_db import ConfigDb
 from .container_templates_response import ContainerTemplatesResponse
 from .create_app_output import CreateAppOutput
-from .create_custom_evaluation import CreateCustomEvaluation
-from .custom_evaluation_detail import CustomEvaluationDetail
-from .custom_evaluation_names import CustomEvaluationNames
-from .custom_evaluation_output import CustomEvaluationOutput
+from .delete_evaluation import DeleteEvaluation
 from .docker_env_vars import DockerEnvVars
 from .environment_output import EnvironmentOutput
 from .evaluation import Evaluation
 from .evaluation_scenario import EvaluationScenario
 from .evaluation_scenario_input import EvaluationScenarioInput
 from .evaluation_scenario_output import EvaluationScenarioOutput
-from .evaluation_scenario_score import EvaluationScenarioScore
-from .evaluation_scenario_update_score import EvaluationScenarioUpdateScore
+from .evaluation_scenario_result import EvaluationScenarioResult
 from .evaluation_status_enum import EvaluationStatusEnum
 from .evaluation_type import EvaluationType
-from .evaluation_type_settings import EvaluationTypeSettings
 from .evaluation_webhook import EvaluationWebhook
+from .evaluator import Evaluator
+from .evaluator_config import EvaluatorConfig
 from .feedback import Feedback
 from .get_config_reponse import GetConfigReponse
 from .http_validation_error import HttpValidationError
+from .human_evaluation import HumanEvaluation
+from .human_evaluation_scenario import HumanEvaluationScenario
+from .human_evaluation_scenario_input import HumanEvaluationScenarioInput
+from .human_evaluation_scenario_output import HumanEvaluationScenarioOutput
+from .human_evaluation_scenario_score import HumanEvaluationScenarioScore
+from .human_evaluation_scenario_update_score import HumanEvaluationScenarioUpdateScore
 from .image import Image
 from .invite_request import InviteRequest
 from .list_api_keys_output import ListApiKeysOutput
+from .llm_run_rate_limit import LlmRunRateLimit
 from .new_testset import NewTestset
 from .organization import Organization
 from .organization_output import OrganizationOutput
+from .result import Result
 from .simple_evaluation_output import SimpleEvaluationOutput
 from .span import Span
 from .template import Template
@@ -49,37 +58,46 @@ from .variant_action_enum import VariantActionEnum
 
 __all__ = [
     "AddVariantFromBaseAndConfigResponse",
+    "AggregatedResult",
     "App",
     "AppVariantOutput",
+    "AppVariantOutputExtended",
+    "AppVariantRevision",
     "BaseOutput",
     "BodyImportTestset",
+    "ConfigDb",
     "ContainerTemplatesResponse",
     "CreateAppOutput",
-    "CreateCustomEvaluation",
-    "CustomEvaluationDetail",
-    "CustomEvaluationNames",
-    "CustomEvaluationOutput",
+    "DeleteEvaluation",
     "DockerEnvVars",
     "EnvironmentOutput",
     "Evaluation",
     "EvaluationScenario",
     "EvaluationScenarioInput",
     "EvaluationScenarioOutput",
-    "EvaluationScenarioScore",
-    "EvaluationScenarioUpdateScore",
+    "EvaluationScenarioResult",
     "EvaluationStatusEnum",
     "EvaluationType",
-    "EvaluationTypeSettings",
     "EvaluationWebhook",
+    "Evaluator",
+    "EvaluatorConfig",
     "Feedback",
     "GetConfigReponse",
     "HttpValidationError",
+    "HumanEvaluation",
+    "HumanEvaluationScenario",
+    "HumanEvaluationScenarioInput",
+    "HumanEvaluationScenarioOutput",
+    "HumanEvaluationScenarioScore",
+    "HumanEvaluationScenarioUpdateScore",
     "Image",
     "InviteRequest",
     "ListApiKeysOutput",
+    "LlmRunRateLimit",
     "NewTestset",
     "Organization",
     "OrganizationOutput",
+    "Result",
     "SimpleEvaluationOutput",
     "Span",
     "Template",

agenta/client/backend/types/aggregated_result.py

@@ -0,0 +1,39 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .evaluator_config import EvaluatorConfig
+from .result import Result
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class AggregatedResult(pydantic.BaseModel):
+    evaluator_config: EvaluatorConfig
+    result: Result
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

agenta/client/backend/types/app_variant_output.py

@@ -23,7 +23,6 @@ class AppVariantOutput(pydantic.BaseModel):
     base_name: str
     base_id: str
     config_name: str
-    config_id: str
     uri: typing.Optional[str]
 
     def json(self, **kwargs: typing.Any) -> str:

agenta/client/backend/types/app_variant_output_extended.py

@@ -0,0 +1,50 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .app_variant_revision import AppVariantRevision
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class AppVariantOutputExtended(pydantic.BaseModel):
+    app_id: str
+    app_name: str
+    variant_id: str
+    variant_name: str
+    parameters: typing.Optional[typing.Dict[str, typing.Any]]
+    previous_variant_name: typing.Optional[str]
+    organization_id: str
+    user_id: str
+    base_name: str
+    base_id: str
+    config_name: str
+    uri: typing.Optional[str]
+    revision: int
+    revisions: typing.List[AppVariantRevision]
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

agenta/client/backend/types/app_variant_revision.py

@@ -0,0 +1,40 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .config_db import ConfigDb
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class AppVariantRevision(pydantic.BaseModel):
+    revision: int
+    modified_by: str
+    config: ConfigDb
+    created_at: dt.datetime
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

agenta/client/backend/types/{custom_evaluation_output.py → config_db.py}

@@ -11,11 +11,9 @@ except ImportError:
     import pydantic  # type: ignore
 
 
-class CustomEvaluationOutput(pydantic.BaseModel):
-    id: str
-    app_id: str
-    evaluation_name: str
-    created_at: dt.datetime
+class ConfigDb(pydantic.BaseModel):
+    config_name: str
+    parameters: typing.Optional[typing.Dict[str, typing.Any]]
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {

agenta/client/backend/types/{custom_evaluation_names.py → delete_evaluation.py}

@@ -11,9 +11,8 @@ except ImportError:
     import pydantic  # type: ignore
 
 
-class CustomEvaluationNames(pydantic.BaseModel):
-    id: str
-    evaluation_name: str
+class DeleteEvaluation(pydantic.BaseModel):
+    evaluations_ids: typing.List[str]
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {

agenta/client/backend/types/environment_output.py

@@ -16,6 +16,8 @@ class EnvironmentOutput(pydantic.BaseModel):
     app_id: str
     deployed_app_variant_id: typing.Optional[str]
     deployed_variant_name: typing.Optional[str]
+    deployed_app_variant_revision_id: typing.Optional[str]
+    revision: typing.Optional[str]
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {

agenta/client/backend/types/evaluation.py

@@ -4,8 +4,7 @@ import datetime as dt
 import typing
 
 from ..core.datetime_utils import serialize_datetime
-from .evaluation_type import EvaluationType
-from .evaluation_type_settings import EvaluationTypeSettings
+from .aggregated_result import AggregatedResult
 
 try:
     import pydantic.v1 as pydantic  # type: ignore
@@ -18,13 +17,14 @@ class Evaluation(pydantic.BaseModel):
     app_id: str
     user_id: str
     user_username: str
-    evaluation_type: EvaluationType
-    evaluation_type_settings: typing.Optional[EvaluationTypeSettings]
     variant_ids: typing.List[str]
     variant_names: typing.List[str]
+    variant_revision_ids: typing.List[str]
+    revisions: typing.List[str]
     testset_id: str
     testset_name: str
     status: str
+    aggregated_results: typing.List[AggregatedResult]
     created_at: dt.datetime
     updated_at: dt.datetime
 

agenta/client/backend/types/evaluation_scenario.py

@@ -6,7 +6,7 @@ import typing
 from ..core.datetime_utils import serialize_datetime
 from .evaluation_scenario_input import EvaluationScenarioInput
 from .evaluation_scenario_output import EvaluationScenarioOutput
-from .evaluation_scenario_score import EvaluationScenarioScore
+from .evaluation_scenario_result import EvaluationScenarioResult
 
 try:
     import pydantic.v1 as pydantic  # type: ignore
@@ -19,12 +19,11 @@ class EvaluationScenario(pydantic.BaseModel):
     evaluation_id: str
     inputs: typing.List[EvaluationScenarioInput]
     outputs: typing.List[EvaluationScenarioOutput]
-    vote: typing.Optional[str]
-    score: typing.Optional[EvaluationScenarioScore]
     evaluation: typing.Optional[str]
     correct_answer: typing.Optional[str]
     is_pinned: typing.Optional[bool]
     note: typing.Optional[str]
+    results: typing.List[EvaluationScenarioResult]
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {

agenta/client/backend/types/evaluation_scenario_input.py

@@ -12,8 +12,9 @@ except ImportError:
 
 
 class EvaluationScenarioInput(pydantic.BaseModel):
-    input_name: str
-    input_value: str
+    name: str
+    type: str
+    value: typing.Optional[typing.Any]
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {

agenta/client/backend/types/evaluation_scenario_output.py

@@ -12,8 +12,8 @@ except ImportError:
 
 
 class EvaluationScenarioOutput(pydantic.BaseModel):
-    variant_id: str
-    variant_output: str
+    type: str
+    value: typing.Optional[typing.Any]
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {

agenta/client/backend/types/evaluation_scenario_result.py

@@ -0,0 +1,38 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .result import Result
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class EvaluationScenarioResult(pydantic.BaseModel):
+    evaluator_config: str
+    result: Result
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

agenta/client/backend/types/evaluation_status_enum.py

@@ -13,21 +13,21 @@ class EvaluationStatusEnum(str, enum.Enum):
 
     EVALUATION_INITIALIZED = "EVALUATION_INITIALIZED"
     EVALUATION_STARTED = "EVALUATION_STARTED"
-    COMPARISON_RUN_STARTED = "COMPARISON_RUN_STARTED"
     EVALUATION_FINISHED = "EVALUATION_FINISHED"
+    EVALUATION_FAILED = "EVALUATION_FAILED"
 
     def visit(
         self,
         evaluation_initialized: typing.Callable[[], T_Result],
         evaluation_started: typing.Callable[[], T_Result],
-        comparison_run_started: typing.Callable[[], T_Result],
         evaluation_finished: typing.Callable[[], T_Result],
+        evaluation_failed: typing.Callable[[], T_Result],
     ) -> T_Result:
         if self is EvaluationStatusEnum.EVALUATION_INITIALIZED:
             return evaluation_initialized()
         if self is EvaluationStatusEnum.EVALUATION_STARTED:
             return evaluation_started()
-        if self is EvaluationStatusEnum.COMPARISON_RUN_STARTED:
-            return comparison_run_started()
         if self is EvaluationStatusEnum.EVALUATION_FINISHED:
             return evaluation_finished()
+        if self is EvaluationStatusEnum.EVALUATION_FAILED:
+            return evaluation_failed()

agenta/client/backend/types/evaluation_type.py

@@ -11,43 +11,15 @@ class EvaluationType(str, enum.Enum):
     An enumeration.
     """
 
-    AUTO_EXACT_MATCH = "auto_exact_match"
-    AUTO_SIMILARITY_MATCH = "auto_similarity_match"
-    AUTO_REGEX_TEST = "auto_regex_test"
-    AUTO_WEBHOOK_TEST = "auto_webhook_test"
-    AUTO_AI_CRITIQUE = "auto_ai_critique"
     HUMAN_A_B_TESTING = "human_a_b_testing"
-    HUMAN_SCORING = "human_scoring"
-    CUSTOM_CODE_RUN = "custom_code_run"
     SINGLE_MODEL_TEST = "single_model_test"
 
     def visit(
         self,
-        auto_exact_match: typing.Callable[[], T_Result],
-        auto_similarity_match: typing.Callable[[], T_Result],
-        auto_regex_test: typing.Callable[[], T_Result],
-        auto_webhook_test: typing.Callable[[], T_Result],
-        auto_ai_critique: typing.Callable[[], T_Result],
         human_a_b_testing: typing.Callable[[], T_Result],
-        human_scoring: typing.Callable[[], T_Result],
-        custom_code_run: typing.Callable[[], T_Result],
         single_model_test: typing.Callable[[], T_Result],
     ) -> T_Result:
-        if self is EvaluationType.AUTO_EXACT_MATCH:
-            return auto_exact_match()
-        if self is EvaluationType.AUTO_SIMILARITY_MATCH:
-            return auto_similarity_match()
-        if self is EvaluationType.AUTO_REGEX_TEST:
-            return auto_regex_test()
-        if self is EvaluationType.AUTO_WEBHOOK_TEST:
-            return auto_webhook_test()
-        if self is EvaluationType.AUTO_AI_CRITIQUE:
-            return auto_ai_critique()
         if self is EvaluationType.HUMAN_A_B_TESTING:
             return human_a_b_testing()
-        if self is EvaluationType.HUMAN_SCORING:
-            return human_scoring()
-        if self is EvaluationType.CUSTOM_CODE_RUN:
-            return custom_code_run()
         if self is EvaluationType.SINGLE_MODEL_TEST:
             return single_model_test()

agenta/client/backend/types/evaluator.py

@@ -0,0 +1,39 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class Evaluator(pydantic.BaseModel):
+    name: str
+    key: str
+    direct_use: bool
+    settings_template: typing.Dict[str, typing.Any]
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

agenta/client/backend/types/{custom_evaluation_detail.py → evaluator_config.py}

@@ -11,11 +11,11 @@ except ImportError:
     import pydantic  # type: ignore
 
 
-class CustomEvaluationDetail(pydantic.BaseModel):
+class EvaluatorConfig(pydantic.BaseModel):
     id: str
-    app_id: str
-    evaluation_name: str
-    python_code: str
+    name: str
+    evaluator_key: str
+    settings_values: typing.Optional[typing.Dict[str, typing.Any]]
     created_at: dt.datetime
     updated_at: dt.datetime
 

agenta/client/backend/types/human_evaluation.py

@@ -0,0 +1,49 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class HumanEvaluation(pydantic.BaseModel):
+    id: str
+    app_id: str
+    user_id: str
+    user_username: str
+    evaluation_type: str
+    variant_ids: typing.List[str]
+    variant_names: typing.List[str]
+    variants_revision_ids: typing.List[str]
+    revisions: typing.List[str]
+    testset_id: str
+    testset_name: str
+    status: str
+    created_at: dt.datetime
+    updated_at: dt.datetime
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

agenta/client/backend/types/human_evaluation_scenario.py

@@ -0,0 +1,48 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .human_evaluation_scenario_input import HumanEvaluationScenarioInput
+from .human_evaluation_scenario_output import HumanEvaluationScenarioOutput
+from .human_evaluation_scenario_score import HumanEvaluationScenarioScore
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class HumanEvaluationScenario(pydantic.BaseModel):
+    id: typing.Optional[str]
+    evaluation_id: str
+    inputs: typing.List[HumanEvaluationScenarioInput]
+    outputs: typing.List[HumanEvaluationScenarioOutput]
+    vote: typing.Optional[str]
+    score: typing.Optional[HumanEvaluationScenarioScore]
+    evaluation: typing.Optional[str]
+    correct_answer: typing.Optional[str]
+    is_pinned: typing.Optional[bool]
+    note: typing.Optional[str]
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {
+            "by_alias": True,
+            "exclude_unset": True,
+            **kwargs,
+        }
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

agenta/client/backend/types/{create_custom_evaluation.py → human_evaluation_scenario_input.py}

@@ -11,10 +11,9 @@ except ImportError:
     import pydantic  # type: ignore
 
 
-class CreateCustomEvaluation(pydantic.BaseModel):
-    evaluation_name: str
-    python_code: str
-    app_id: str
+class HumanEvaluationScenarioInput(pydantic.BaseModel):
+    input_name: str
+    input_value: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {