agenta 0.57.0__py3-none-any.whl → 0.63.2__py3-none-any.whl

agenta/__init__.py

@@ -27,8 +27,12 @@ from .sdk.utils.costs import calculate_token_usage
 from .sdk.tracing import Tracing, get_tracer
 from .sdk.tracing.conventions import Reference
 from .sdk.decorators.tracing import instrument
-from .sdk.decorators.running import workflow, workflows
-from .sdk.decorators.serving import entrypoint, app, route
+from .sdk.decorators.running import (
+    workflow,
+    application,
+    evaluator,
+)
+from .sdk.decorators.serving import route, app
 from .sdk.context.running import workflow_mode_enabled
 from .sdk.litellm import litellm as callbacks
 from .sdk.managers.apps import AppManager
@@ -40,6 +44,10 @@ from .sdk.managers.deployment import DeploymentManager
 from .sdk import assets as assets
 from .sdk import tracer
 
+# evaluations
+from .sdk import testsets as testsets
+
+
 config = PreInitObject("agenta.config", Config)
 DEFAULT_AGENTA_SINGLETON_INSTANCE = AgentaSingleton()
 
@@ -65,7 +73,8 @@ def init(
     global api, async_api, tracing, tracer  # pylint: disable=global-statement
 
     _init(
-        host=host or api_url,
+        host=host,
+        api_url=api_url,
         api_key=api_key,
         config_fname=config_fname,
         redact=redact,

agenta/client/__init__.py

@@ -160,8 +160,8 @@ from .backend.types import (
     StandardProviderSettingsDto,
     StatusCode,
     StatusDto,
-    TestSetOutputResponse,
-    TestSetSimpleResponse,
+    TestsetOutputResponse,
+    TestsetSimpleResponse,
     TestcaseResponse,
     Testset,
     TestsetRequest,
@@ -403,8 +403,8 @@ __all__ = [
     "StandardProviderSettingsDto",
     "StatusCode",
     "StatusDto",
-    "TestSetOutputResponse",
-    "TestSetSimpleResponse",
+    "TestsetOutputResponse",
+    "TestsetSimpleResponse",
     "TestcaseResponse",
     "Testset",
     "TestsetRequest",

agenta/client/backend/__init__.py

@@ -155,8 +155,8 @@ from .types import (
     StandardProviderSettingsDto,
     StatusCode,
     StatusDto,
-    TestSetOutputResponse,
-    TestSetSimpleResponse,
+    TestsetOutputResponse,
+    TestsetSimpleResponse,
     TestcaseResponse,
     Testset,
     TestsetRequest,
@@ -398,8 +398,8 @@ __all__ = [
     "StandardProviderSettingsDto",
     "StatusCode",
     "StatusDto",
-    "TestSetOutputResponse",
-    "TestSetSimpleResponse",
+    "TestsetOutputResponse",
+    "TestsetSimpleResponse",
     "TestcaseResponse",
     "Testset",
     "TestsetRequest",

agenta/client/backend/api_keys/client.py

@@ -95,7 +95,7 @@ class ApiKeysClient:
         self,
         key_prefix: str,
         *,
-        request_options: typing.Optional[RequestOptions] = None
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> typing.Dict[str, typing.Optional[typing.Any]]:
         """
         Delete an API key with the given key prefix for the authenticated user.
@@ -246,7 +246,7 @@ class AsyncApiKeysClient:
         self,
         key_prefix: str,
         *,
-        request_options: typing.Optional[RequestOptions] = None
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> typing.Dict[str, typing.Optional[typing.Any]]:
         """
         Delete an API key with the given key prefix for the authenticated user.

agenta/client/backend/billing/client.py

@@ -80,7 +80,7 @@ class BillingClient:
         *,
         plan: Plan,
         success_url: str,
-        request_options: typing.Optional[RequestOptions] = None
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> typing.Optional[typing.Any]:
         """
         Parameters
@@ -343,7 +343,7 @@ class AsyncBillingClient:
         *,
         plan: Plan,
         success_url: str,
-        request_options: typing.Optional[RequestOptions] = None
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> typing.Optional[typing.Any]:
         """
         Parameters

agenta/client/backend/billing/raw_client.py

@@ -110,7 +110,7 @@ class RawBillingClient:
         *,
         plan: Plan,
         success_url: str,
-        request_options: typing.Optional[RequestOptions] = None
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> HttpResponse[typing.Optional[typing.Any]]:
         """
         Parameters
@@ -506,7 +506,7 @@ class AsyncRawBillingClient:
         *,
         plan: Plan,
         success_url: str,
-        request_options: typing.Optional[RequestOptions] = None
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> AsyncHttpResponse[typing.Optional[typing.Any]]:
         """
         Parameters

agenta/client/backend/client.py

@@ -98,13 +98,17 @@ class AgentaApi:
         self._client_wrapper = SyncClientWrapper(
             base_url=_get_base_url(base_url=base_url, environment=environment),
             api_key=api_key,
-            httpx_client=httpx_client
-            if httpx_client is not None
-            else httpx.Client(
-                timeout=_defaulted_timeout, follow_redirects=follow_redirects
-            )
-            if follow_redirects is not None
-            else httpx.Client(timeout=_defaulted_timeout),
+            httpx_client=(
+                httpx_client
+                if httpx_client is not None
+                else (
+                    httpx.Client(
+                        timeout=_defaulted_timeout, follow_redirects=follow_redirects
+                    )
+                    if follow_redirects is not None
+                    else httpx.Client(timeout=_defaulted_timeout)
+                )
+            ),
             timeout=_defaulted_timeout,
         )
         self._raw_client = RawAgentaApi(client_wrapper=self._client_wrapper)
@@ -172,7 +176,7 @@ class AgentaApi:
 
     def update_organization(
         self,
-        org_id: str,
+        organization_id: str,
         *,
         name: typing.Optional[str] = OMIT,
         description: typing.Optional[str] = OMIT,
@@ -182,7 +186,7 @@ class AgentaApi:
         """
         Parameters
         ----------
-        org_id : str
+        organization_id : str
 
         name : typing.Optional[str]
 
@@ -206,11 +210,11 @@ class AgentaApi:
             api_key="YOUR_API_KEY",
         )
         client.update_organization(
-            org_id="org_id",
+            organization_id="organization_id",
         )
         """
         _response = self._raw_client.update_organization(
-            org_id,
+            organization_id,
             name=name,
             description=description,
             updated_at=updated_at,
@@ -220,7 +224,7 @@ class AgentaApi:
 
     def create_workspace(
         self,
-        org_id: str,
+        organization_id: str,
         *,
         name: str,
         description: typing.Optional[str] = OMIT,
@@ -230,7 +234,7 @@ class AgentaApi:
         """
         Parameters
         ----------
-        org_id : str
+        organization_id : str
 
         name : str
 
@@ -254,12 +258,12 @@ class AgentaApi:
             api_key="YOUR_API_KEY",
         )
         client.create_workspace(
-            org_id="org_id",
+            organization_id="organization_id",
             name="name",
         )
         """
         _response = self._raw_client.create_workspace(
-            org_id,
+            organization_id,
             name=name,
             description=description,
             type=type,
@@ -269,7 +273,7 @@ class AgentaApi:
 
     def update_workspace(
         self,
-        org_id: str,
+        organization_id: str,
         workspace_id: str,
         *,
         name: typing.Optional[str] = OMIT,
@@ -280,7 +284,7 @@ class AgentaApi:
         """
         Parameters
         ----------
-        org_id : str
+        organization_id : str
 
         workspace_id : str
 
@@ -306,12 +310,12 @@ class AgentaApi:
             api_key="YOUR_API_KEY",
         )
         client.update_workspace(
-            org_id="org_id",
+            organization_id="organization_id",
             workspace_id="workspace_id",
         )
         """
         _response = self._raw_client.update_workspace(
-            org_id,
+            organization_id,
             workspace_id,
             name=name,
             description=description,
@@ -427,7 +431,7 @@ class AgentaApi:
         workspace_id: str,
         *,
         email: str,
-        org_id: str,
+        organization_id: str,
         role: str,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> typing.Optional[typing.Any]:
@@ -437,7 +441,7 @@ class AgentaApi:
         Args:
             workspace_id (str): The ID of the workspace.
             email (str): The email of the user to remove the role from.
-            org_id (str): The ID of the organization.
+            organization_id (str): The ID of the organization.
             role (str): The role to remove from the user.
             request (Request): The FastAPI request object.
 
@@ -454,7 +458,7 @@ class AgentaApi:
 
         email : str
 
-        org_id : str
+        organization_id : str
 
         role : str
 
@@ -476,14 +480,14 @@ class AgentaApi:
         client.unassign_role_from_user(
             workspace_id="workspace_id",
             email="email",
-            org_id="org_id",
+            organization_id="organization_id",
             role="role",
         )
         """
         _response = self._raw_client.unassign_role_from_user(
             workspace_id,
             email=email,
-            org_id=org_id,
+            organization_id=organization_id,
             role=role,
             request_options=request_options,
         )
@@ -657,13 +661,17 @@ class AsyncAgentaApi:
         self._client_wrapper = AsyncClientWrapper(
             base_url=_get_base_url(base_url=base_url, environment=environment),
             api_key=api_key,
-            httpx_client=httpx_client
-            if httpx_client is not None
-            else httpx.AsyncClient(
-                timeout=_defaulted_timeout, follow_redirects=follow_redirects
-            )
-            if follow_redirects is not None
-            else httpx.AsyncClient(timeout=_defaulted_timeout),
+            httpx_client=(
+                httpx_client
+                if httpx_client is not None
+                else (
+                    httpx.AsyncClient(
+                        timeout=_defaulted_timeout, follow_redirects=follow_redirects
+                    )
+                    if follow_redirects is not None
+                    else httpx.AsyncClient(timeout=_defaulted_timeout)
+                )
+            ),
             timeout=_defaulted_timeout,
         )
         self._raw_client = AsyncRawAgentaApi(client_wrapper=self._client_wrapper)
@@ -743,7 +751,7 @@ class AsyncAgentaApi:
 
     async def update_organization(
         self,
-        org_id: str,
+        organization_id: str,
         *,
         name: typing.Optional[str] = OMIT,
         description: typing.Optional[str] = OMIT,
@@ -753,7 +761,7 @@ class AsyncAgentaApi:
         """
         Parameters
         ----------
-        org_id : str
+        organization_id : str
 
         name : typing.Optional[str]
 
@@ -782,14 +790,14 @@ class AsyncAgentaApi:
 
         async def main() -> None:
             await client.update_organization(
-                org_id="org_id",
+                organization_id="organization_id",
             )
 
 
         asyncio.run(main())
         """
         _response = await self._raw_client.update_organization(
-            org_id,
+            organization_id,
             name=name,
             description=description,
             updated_at=updated_at,
@@ -799,7 +807,7 @@ class AsyncAgentaApi:
 
     async def create_workspace(
         self,
-        org_id: str,
+        organization_id: str,
         *,
         name: str,
         description: typing.Optional[str] = OMIT,
@@ -809,7 +817,7 @@ class AsyncAgentaApi:
         """
         Parameters
         ----------
-        org_id : str
+        organization_id : str
 
         name : str
 
@@ -838,7 +846,7 @@ class AsyncAgentaApi:
 
         async def main() -> None:
             await client.create_workspace(
-                org_id="org_id",
+                organization_id="organization_id",
                 name="name",
             )
 
@@ -846,7 +854,7 @@ class AsyncAgentaApi:
         asyncio.run(main())
         """
         _response = await self._raw_client.create_workspace(
-            org_id,
+            organization_id,
             name=name,
             description=description,
             type=type,
@@ -856,7 +864,7 @@ class AsyncAgentaApi:
 
     async def update_workspace(
         self,
-        org_id: str,
+        organization_id: str,
         workspace_id: str,
         *,
         name: typing.Optional[str] = OMIT,
@@ -867,7 +875,7 @@ class AsyncAgentaApi:
         """
         Parameters
         ----------
-        org_id : str
+        organization_id : str
 
         workspace_id : str
 
@@ -898,7 +906,7 @@ class AsyncAgentaApi:
 
         async def main() -> None:
             await client.update_workspace(
-                org_id="org_id",
+                organization_id="organization_id",
                 workspace_id="workspace_id",
             )
 
@@ -906,7 +914,7 @@ class AsyncAgentaApi:
         asyncio.run(main())
         """
         _response = await self._raw_client.update_workspace(
-            org_id,
+            organization_id,
             workspace_id,
             name=name,
             description=description,
@@ -1038,7 +1046,7 @@ class AsyncAgentaApi:
         workspace_id: str,
         *,
         email: str,
-        org_id: str,
+        organization_id: str,
         role: str,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> typing.Optional[typing.Any]:
@@ -1048,7 +1056,7 @@ class AsyncAgentaApi:
         Args:
             workspace_id (str): The ID of the workspace.
             email (str): The email of the user to remove the role from.
-            org_id (str): The ID of the organization.
+            organization_id (str): The ID of the organization.
             role (str): The role to remove from the user.
             request (Request): The FastAPI request object.
 
@@ -1065,7 +1073,7 @@ class AsyncAgentaApi:
 
         email : str
 
-        org_id : str
+        organization_id : str
 
         role : str
 
@@ -1092,7 +1100,7 @@ class AsyncAgentaApi:
             await client.unassign_role_from_user(
                 workspace_id="workspace_id",
                 email="email",
-                org_id="org_id",
+                organization_id="organization_id",
                 role="role",
             )
 
@@ -1102,7 +1110,7 @@ class AsyncAgentaApi:
         _response = await self._raw_client.unassign_role_from_user(
             workspace_id,
             email=email,
-            org_id=org_id,
+            organization_id=organization_id,
             role=role,
             request_options=request_options,
         )

agenta/client/backend/core/client_wrapper.py

@@ -35,7 +35,7 @@ class SyncClientWrapper(BaseClientWrapper):
         api_key: str,
         base_url: str,
         timeout: typing.Optional[float] = None,
-        httpx_client: httpx.Client
+        httpx_client: httpx.Client,
     ):
         super().__init__(api_key=api_key, base_url=base_url, timeout=timeout)
         self.httpx_client = HttpClient(
@@ -53,7 +53,7 @@ class AsyncClientWrapper(BaseClientWrapper):
         api_key: str,
         base_url: str,
         timeout: typing.Optional[float] = None,
-        httpx_client: httpx.AsyncClient
+        httpx_client: httpx.AsyncClient,
     ):
         super().__init__(api_key=api_key, base_url=base_url, timeout=timeout)
         self.httpx_client = AsyncHttpClient(

agenta/client/backend/core/file.py

@@ -53,7 +53,9 @@ def with_content_type(*, file: File, default_content_type: str) -> File:
             filename, content = cast(Tuple[Optional[str], FileContent], file)  # type: ignore
             return (filename, content, default_content_type)
         elif len(file) == 3:
-            filename, content, file_content_type = cast(Tuple[Optional[str], FileContent, Optional[str]], file)  # type: ignore
+            filename, content, file_content_type = cast(
+                Tuple[Optional[str], FileContent, Optional[str]], file
+            )  # type: ignore
             out_content_type = file_content_type or default_content_type
             return (filename, content, out_content_type)
         elif len(file) == 4:

agenta/client/backend/core/http_client.py

@@ -148,9 +148,9 @@ def get_request_body(
         json_body = maybe_filter_request_body(json, request_options, omit)
 
     # If you have an empty JSON body, you should just send None
-    return (
-        json_body if json_body != {} else None
-    ), data_body if data_body != {} else None
+    return (json_body if json_body != {} else None), (
+        data_body if data_body != {} else None
+    )
 
 
 class HttpClient:

agenta/client/backend/core/pydantic_utilities.py

@@ -78,7 +78,9 @@ class UniversalBaseModel(pydantic.BaseModel):
         )
 
         @pydantic.model_serializer(mode="wrap", when_used="json")  # type: ignore[attr-defined]
-        def serialize_model(self, handler: pydantic.SerializerFunctionWrapHandler) -> Any:  # type: ignore[name-defined]
+        def serialize_model(
+            self, handler: pydantic.SerializerFunctionWrapHandler
+        ) -> Any:  # type: ignore[name-defined]
             serialized = handler(self)
             data = {
                 k: serialize_datetime(v) if isinstance(v, dt.datetime) else v
@@ -260,7 +262,10 @@ def universal_root_validator(
 ) -> Callable[[AnyCallable], AnyCallable]:
     def decorator(func: AnyCallable) -> AnyCallable:
         if IS_PYDANTIC_V2:
-            return cast(AnyCallable, pydantic.model_validator(mode="before" if pre else "after")(func))  # type: ignore[attr-defined]
+            return cast(
+                AnyCallable,
+                pydantic.model_validator(mode="before" if pre else "after")(func),
+            )  # type: ignore[attr-defined]
         return cast(AnyCallable, pydantic.root_validator(pre=pre)(func))  # type: ignore[call-overload]
 
     return decorator
@@ -271,7 +276,12 @@ def universal_field_validator(
 ) -> Callable[[AnyCallable], AnyCallable]:
     def decorator(func: AnyCallable) -> AnyCallable:
         if IS_PYDANTIC_V2:
-            return cast(AnyCallable, pydantic.field_validator(field_name, mode="before" if pre else "after")(func))  # type: ignore[attr-defined]
+            return cast(
+                AnyCallable,
+                pydantic.field_validator(field_name, mode="before" if pre else "after")(
+                    func
+                ),
+            )  # type: ignore[attr-defined]
         return cast(AnyCallable, pydantic.validator(field_name, pre=pre)(func))
 
     return decorator

agenta/client/backend/human_evaluations/client.py

@@ -237,7 +237,7 @@ class HumanEvaluationsClient:
         Updates an evaluation's status.
 
         Raises:
-            HTTPException: If the columns in the test set do not match with the inputs in the variant.
+            HTTPException: If the columns in the testset do not match with the inputs in the variant.
 
         Returns:
             None: A 204 No Content status code, indicating that the update was successful.
@@ -785,7 +785,7 @@ class AsyncHumanEvaluationsClient:
         Updates an evaluation's status.
 
         Raises:
-            HTTPException: If the columns in the test set do not match with the inputs in the variant.
+            HTTPException: If the columns in the testset do not match with the inputs in the variant.
 
         Returns:
             None: A 204 No Content status code, indicating that the update was successful.

agenta/client/backend/human_evaluations/raw_client.py

@@ -336,7 +336,7 @@ class RawHumanEvaluationsClient:
         Updates an evaluation's status.
 
         Raises:
-            HTTPException: If the columns in the test set do not match with the inputs in the variant.
+            HTTPException: If the columns in the testset do not match with the inputs in the variant.
 
         Returns:
             None: A 204 No Content status code, indicating that the update was successful.
@@ -1112,7 +1112,7 @@ class AsyncRawHumanEvaluationsClient:
         Updates an evaluation's status.
 
         Raises:
-            HTTPException: If the columns in the test set do not match with the inputs in the variant.
+            HTTPException: If the columns in the testset do not match with the inputs in the variant.
 
         Returns:
             None: A 204 No Content status code, indicating that the update was successful.