agenta 0.14.7a0__py3-none-any.whl → 0.14.7a1__py3-none-any.whl

agenta/sdk/agenta_init.py

@@ -1,8 +1,9 @@
 import os
 import logging
-from typing import Optional
+from typing import Any, Optional
+
+from .utils.globals import set_global
 
-from agenta.sdk.utils.globals import set_global
 from agenta.client.backend.client import AgentaApi
 from agenta.sdk.tracing.llm_tracing import Tracing
 from agenta.client.exceptions import APIRequestError
@@ -12,6 +13,18 @@ logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
 
+BACKEND_URL_SUFFIX = os.environ.get("BACKEND_URL_SUFFIX", "api")
+CLIENT_API_KEY = os.environ.get("AGENTA_API_KEY")
+CLIENT_HOST = os.environ.get("AGENTA_HOST", "http://localhost")
+
+# initialize the client with the backend url and api key
+backend_url = f"{CLIENT_HOST}/{BACKEND_URL_SUFFIX}"
+client = AgentaApi(
+    base_url=backend_url,
+    api_key=CLIENT_API_KEY if CLIENT_API_KEY else "",
+)
+
+
 class AgentaSingleton:
     """Singleton class to save all the "global variables" for the sdk."""
 
@@ -24,21 +37,15 @@ class AgentaSingleton:
             cls._instance = super(AgentaSingleton, cls).__new__(cls)
         return cls._instance
 
-    @property
-    def client(self):
-        """Builds sdk client instance.
-
-        Returns:
-            AgentaAPI: instance of agenta api backend
-        """
-
-        return AgentaApi(base_url=self.host + "/api", api_key=self.api_key)
-
     def init(
         self,
+        app_name: Optional[str] = None,
+        base_name: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_id: Optional[str] = None,
         app_id: Optional[str] = None,
         host: Optional[str] = None,
-        api_key: Optional[str] = None,
+        **kwargs: Any,
     ) -> None:
         """Main function to initialize the singleton.
 
@@ -46,44 +53,57 @@ class AgentaSingleton:
         the function will look for them in environment variables.
 
         Args:
-            app_id (Optional[str]): ID of the Agenta application. Defaults to None. If not provided, will look for "AGENTA_APP_NAME" in environment variables.
+            app_name (Optional[str]): Name of the Agenta application. Defaults to None. If not provided, will look for "AGENTA_APP_NAME" in environment variables.
+            base_name (Optional[str]): Base name for the Agenta setup. Defaults to None. If not provided, will look for "AGENTA_BASE_NAME" in environment variables.
             host (Optional[str]): Host name of the backend server. Defaults to None. If not provided, will look for "AGENTA_HOST" in environment variables.
-            api_key (Optional[str]): API Key to use with the host of the backend server.
             kwargs (Any): Additional keyword arguments.
 
         Raises:
             ValueError: If `app_name`, `base_name`, or `host` are not specified either as arguments or in the environment variables.
         """
-
-        self.api_key = api_key or os.environ.get("AGENTA_API_KEY")
-        self.host = host or os.environ.get("AGENTA_HOST", "http://localhost")
-
-        app_id = app_id or os.environ.get("AGENTA_APP_ID")
-        if not app_id:
-            raise ValueError("App ID must be specified.")
-
-        base_id = os.environ.get("AGENTA_BASE_ID")
-        base_name = os.environ.get("AGENTA_BASE_NAME")
-        if base_id is None and (app_id is None or base_name is None):
-            print(
-                f"Warning: Your configuration will not be saved permanently since app_name and base_name are not provided."
-            )
-        else:
-            try:
-                base_id = self.get_app_base(app_id, base_name)  # type: ignore
-            except Exception as ex:
-                raise APIRequestError(
-                    f"Failed to get base id and/or app_id from the server with error: {ex}"
+        if app_name is None:
+            app_name = os.environ.get("AGENTA_APP_NAME")
+        if base_name is None:
+            base_name = os.environ.get("AGENTA_BASE_NAME")
+        if api_key is None:
+            api_key = os.environ.get("AGENTA_API_KEY")
+        if base_id is None:
+            base_id = os.environ.get("AGENTA_BASE_ID")
+        if host is None:
+            host = os.environ.get("AGENTA_HOST", "http://localhost")
+
+        if base_id is None:
+            if app_name is None or base_name is None:
+                print(
+                    f"Warning: Your configuration will not be saved permanently since app_name and base_name are not provided."
                 )
+            else:
+                try:
+                    app_id = self.get_app(app_name)
+                    base_id = self.get_app_base(app_id, base_name)
+                except Exception as ex:
+                    raise APIRequestError(
+                        f"Failed to get base id and/or app_id from the server with error: {ex}"
+                    )
 
-        self.app_id = app_id
         self.base_id = base_id
+        self.host = host
+        self.app_id = os.environ.get("AGENTA_APP_ID") if app_id is None else app_id
         self.variant_id = os.environ.get("AGENTA_VARIANT_ID")
         self.variant_name = os.environ.get("AGENTA_VARIANT_NAME")
+        self.api_key = api_key
         self.config = Config(base_id=base_id, host=host)
 
+    def get_app(self, app_name: str) -> str:
+        apps = client.apps.list_apps(app_name=app_name)
+        if len(apps) == 0:
+            raise APIRequestError(f"App with name {app_name} not found")
+
+        app_id = apps[0].app_id
+        return app_id
+
     def get_app_base(self, app_id: str, base_name: str) -> str:
-        bases = self.client.bases.list_bases(app_id=app_id, base_name=base_name)
+        bases = client.bases.list_bases(app_id=app_id, base_name=base_name)
         if len(bases) == 0:
             raise APIRequestError(f"No base was found for the app {app_id}")
         return bases[0].base_id
@@ -102,23 +122,11 @@ class Config:
     def __init__(self, base_id, host):
         self.base_id = base_id
         self.host = host
-
         if base_id is None or host is None:
             self.persist = False
         else:
             self.persist = True
 
-    @property
-    def client(self):
-        """Builds sdk client instance.
-
-        Returns:
-            AgentaAPI: instance of agenta api backend
-        """
-
-        sdk_client = SDKClient(api_key=self.api_key, host=self.host)  # type: ignore
-        return sdk_client._build_sdk_client()
-
     def register_default(self, overwrite=False, **kwargs):
         """alias for default"""
         return self.default(overwrite=overwrite, **kwargs)
@@ -149,7 +157,7 @@ class Config:
         if not self.persist:
             return
         try:
-            self.client.configs.save_config(
+            client.configs.save_config(
                 base_id=self.base_id,
                 config_name=config_name,
                 parameters=kwargs,
@@ -160,9 +168,7 @@ class Config:
                 "Failed to push the configuration to the server with error: " + str(ex)
             )
 
-    def pull(
-        self, config_name: str = "default", environment_name: Optional[str] = None
-    ):
+    def pull(self, config_name: str = "default", environment_name: str = None):
         """Pulls the parameters for the app variant from the server and sets them to the config"""
         if not self.persist and (
             config_name != "default" or environment_name is not None
@@ -173,12 +179,12 @@ class Config:
         if self.persist:
             try:
                 if environment_name:
-                    config = self.client.configs.get_config(
+                    config = client.configs.get_config(
                         base_id=self.base_id, environment_name=environment_name
                     )
 
                 else:
-                    config = self.client.configs.get_config(
+                    config = client.configs.get_config(
                         base_id=self.base_id,
                         config_name=config_name,
                     )
@@ -212,22 +218,15 @@ class Config:
             setattr(self, key, value)
 
 
-def init(
-    app_id: Optional[str] = None,
-    host: Optional[str] = None,
-    api_key: Optional[str] = None,
-):
+def init(app_name=None, base_name=None, **kwargs):
     """Main function to be called by the user to initialize the sdk.
 
     Args:
-        app_id (str): The Id of the app.
-        host (str): The host of the backend server.
-        api_key (str): The API key to use for the backend server.
+        app_name: _description_. Defaults to None.
+        base_name: _description_. Defaults to None.
     """
-
     singleton = AgentaSingleton()
-
-    singleton.init(app_id=app_id, host=host, api_key=api_key)
+    singleton.init(app_name=app_name, base_name=base_name, **kwargs)
     set_global(setup=singleton.setup, config=singleton.config)
 
 
@@ -236,7 +235,7 @@ def llm_tracing(max_workers: Optional[int] = None) -> Tracing:
 
     singleton = AgentaSingleton()
     return Tracing(
-        base_url=singleton.host,  # type: ignore
+        base_url=singleton.host,
         app_id=singleton.app_id,  # type: ignore
         variant_id=singleton.variant_id,  # type: ignore
         variant_name=singleton.variant_name,

agenta/sdk/tracing/callbacks.py

@@ -0,0 +1,129 @@
+# Own Imports
+from agenta.sdk import llm_tracing
+
+# Third Party Imports
+from litellm.utils import ModelResponse
+from litellm.integrations.custom_logger import CustomLogger as LitellmCustomLogger
+
+
+class AgentaLiteLLMHandler(LitellmCustomLogger):
+    """This handler is responsible for logging certain events when using litellm to call LLMs.
+
+    Args:
+        LitellmCustomLogger (object): custom logger that allows us to override the events to capture.
+    """
+
+    @property
+    def _trace(self):
+        return llm_tracing()
+
+    def log_pre_api_call(self, model, messages, kwargs):
+        self._trace.start_span(
+            name="pre_api_call",
+            input=(
+                {"messages": messages}
+                if isinstance(messages, list)
+                else {"inputs": messages}
+            ),
+            spankind=(
+                "llm"
+                if kwargs.get("call_type") in ["completion", "acompletion"]
+                else (
+                    "embedding"
+                    if kwargs.get("call_type") in ["embedding", "aembedding"]
+                    else "unset"
+                )
+            ),
+        )
+        self._trace.set_span_attribute(
+            "model_config",
+            {
+                "model": kwargs.get("model"),
+                "temperature": kwargs["optional_params"]["temperature"],
+            },
+        )
+
+    def log_stream_event(self, kwargs, response_obj, start_time, end_time):
+        self._trace.update_span_status(span=self._trace.active_span, value="OK")
+        self._trace.end_span(
+            outputs={
+                "message": kwargs("complete_streaming_response"),
+                "usage": kwargs.get("usage"),
+                "cost": kwargs.get("response_cost"),
+            },
+            span=self._trace.active_span,
+        )
+
+    def log_success_event(
+        self, kwargs, response_obj: ModelResponse, start_time, end_time
+    ):
+        self._trace.update_span_status(span=self._trace.active_span, value="OK")
+        self._trace.end_span(
+            outputs={
+                "message": kwargs["message"],
+                "usage": kwargs.get("usage"),
+                "cost": kwargs.get("response_cost"),
+            },
+            span=self._trace.active_span,
+        )
+
+    def log_failure_event(
+        self, kwargs, response_obj: ModelResponse, start_time, end_time
+    ):
+        self._trace.update_span_status(span=self._trace.active_span, value="ERROR")
+        self._trace.set_span_attribute(
+            attributes={
+                "traceback_exception": kwargs["traceback_exception"],
+                "call_end_time": kwargs["end_time"],
+            },
+        )
+        self._trace.end_span(
+            outputs={
+                "message": kwargs["exception"],
+                "usage": kwargs.get("usage"),
+                "cost": kwargs.get("response_cost"),
+            },
+            span=self._trace.active_span,
+        )
+
+    async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time):
+        self._trace.update_span_status(span=self._trace.active_span, value="OK")
+        self._trace.end_span(
+            outputs={
+                "message": kwargs("complete_streaming_response"),
+                "usage": kwargs.get("usage"),
+                "cost": kwargs.get("response_cost"),
+            },
+            span=self._trace.active_span,
+        )
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        self._trace.update_span_status(span=self._trace.active_span, value="OK")
+        self._trace.end_span(
+            outputs={
+                "message": kwargs["message"],
+                "usage": kwargs.get("usage"),
+                "cost": kwargs.get("response_cost"),
+            },
+            span=self._trace.active_span,
+        )
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        self._trace.update_span_status(span=self._trace.active_span, value="ERROR")
+        self._trace.set_span_attribute(
+            attributes={
+                "traceback_exception": kwargs["traceback_exception"],
+                "call_end_time": kwargs["end_time"],
+            },
+        )
+        self._trace.end_span(
+            outputs={
+                "message": kwargs["exception"],
+                "usage": kwargs.get("usage"),
+                "cost": kwargs.get("response_cost"),
+            },
+            span=self._trace.active_span,
+        )
+
+
+agenta_litellm_handler = AgentaLiteLLMHandler()

agenta/sdk/tracing/decorators.py

@@ -0,0 +1,41 @@
+# Stdlib Imports
+import inspect
+from functools import wraps
+
+# Own Imports
+import agenta as ag
+
+
+def span(type: str):
+    """Decorator to automatically start and end spans."""
+
+    tracing = ag.llm_tracing()
+
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            result = None
+            span = tracing.start_span(
+                name=func.__name__,
+                input=kwargs,
+                spankind=type,
+            )
+            try:
+                is_coroutine_function = inspect.iscoroutinefunction(func)
+                if is_coroutine_function:
+                    result = await func(*args, **kwargs)
+                else:
+                    result = func(*args, **kwargs)
+                tracing.update_span_status(span=span, value="OK")
+            except Exception as e:
+                result = str(e)
+                tracing.update_span_status(span=span, value="ERROR")
+            finally:
+                if not isinstance(result, dict):
+                    result = {"message": result}
+                tracing.end_span(outputs=result, span=span)
+            return result
+
+        return wrapper
+
+    return decorator

agenta/sdk/tracing/llm_tracing.py

@@ -19,7 +19,7 @@ class SingletonMeta(type):
     Thread-safe implementation of Singleton.
     """
 
-    _instances = {}  # type: ignore
+    _instances = {}
 
     # We need the lock mechanism to synchronize threads \
     # during the initial access to the Singleton object.
@@ -62,7 +62,7 @@ class Tracing(metaclass=SingletonMeta):
         self,
         base_url: str,
         app_id: str,
-        variant_id: Optional[str] = None,
+        variant_id: str,
         variant_name: Optional[str] = None,
         api_key: Optional[str] = None,
         max_workers: Optional[int] = None,
@@ -121,7 +121,7 @@ class Tracing(metaclass=SingletonMeta):
         self.tags.extend(tags)
 
     def start_parent_span(
-        self, name: str, inputs: Dict[str, Any], config: Dict[str, Any] = {}, **kwargs
+        self, name: str, inputs: Dict[str, Any], config: Dict[str, Any], **kwargs
     ):
         trace_id = self._create_trace_id()
         span_id = self._create_span_id()

agenta/sdk/tracing/tasks_manager.py

@@ -106,7 +106,9 @@ class TaskQueue(object):
                 future.result()
             except Exception as exc:
                 self._logger.error(f"Error running task: {str(exc)}")
-                self._logger.error(f"Recording {task.coroutine_type} status to ERROR.")
+                self._logger.error(
+                    f"Recording trace {task.coroutine_type} status to ERROR."
+                )
                 break
             finally:
                 self.tasks.task_done()

{agenta-0.14.7a0.dist-info → agenta-0.14.7a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: agenta
-Version: 0.14.7a0
+Version: 0.14.7a1
 Summary: The SDK for agenta is an open-source LLMOps platform.
 Home-page: https://agenta.ai
 Keywords: LLMOps,LLM,evaluation,prompt engineering
@@ -22,6 +22,7 @@ Requires-Dist: fastapi (>=0.96.1)
 Requires-Dist: httpx (>=0.24,<0.28)
 Requires-Dist: importlib-metadata (>=6.7.0,<7.0.0)
 Requires-Dist: ipdb (>=0.13)
+Requires-Dist: litellm (>=1.35.33,<2.0.0)
 Requires-Dist: posthog (>=3.1.0,<4.0.0)
 Requires-Dist: pydantic (==1.10.13)
 Requires-Dist: pymongo (>=4.6.3,<5.0.0)

{agenta-0.14.7a0.dist-info → agenta-0.14.7a1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-agenta/__init__.py,sha256=q83im-TuPtSRJVOQIAH59EuH_qNIGXpbcNugNfXI4mc,666
+agenta/__init__.py,sha256=ZJZlhn56j2rjl-QKTuNU6HHlJLVlmd_NcU67VIBvGGI,668
 agenta/cli/evaluation_commands.py,sha256=fs6492tprPId9p8eGO02Xy-NCBm2RZNJLZWcUxugwd8,474
 agenta/cli/helper.py,sha256=vRxHyeNaltzNIGrfU2vO0H28_rXDzx9QqIZ_S-W6zL4,6212
 agenta/cli/main.py,sha256=GgYu6UsrnHbqPV7zPlO14b61IyaDiTIjGMYQS9DlqC4,9551
@@ -126,18 +126,18 @@ agenta/docker/docker-assets/entrypoint.sh,sha256=29XK8VQjQsx4hN2j-4JDy-6kQb5y4LC
 agenta/docker/docker-assets/lambda_function.py,sha256=h4UZSSfqwpfsCgERv6frqwm_4JrYu9rLz3I-LxCfeEg,83
 agenta/docker/docker-assets/main.py,sha256=7MI-21n81U7N7A0GxebNi0cmGWtJKcR2sPB6FcH2QfA,251
 agenta/docker/docker_utils.py,sha256=5uHMCzXkCvIsDdEiwbnnn97KkzsFbBvyMwogCsv_Z5U,3509
-agenta/sdk/__init__.py,sha256=TyVUf1LEKLenwS_QAtqVDPCW1s8kLHipD6WYjMD4T48,632
-agenta/sdk/agenta_init.py,sha256=r28xgllqHe0Zs3Xz4UU7DoBMLF1tbPEbaqDi5irSiC8,8638
+agenta/sdk/__init__.py,sha256=3At8FvbuYFoP5IIYs4FZWYQK8l4XlVD1zv4E6QsgQv0,702
+agenta/sdk/agenta_decorator.py,sha256=ucoGffoIefsYaxniMfMgsepKDo8jrlVMA7gGoTYyQUc,17805
+agenta/sdk/agenta_init.py,sha256=wDfStpe8_3ZXRLtikarwDKI_VpA1YW4eIz_3fXq39is,9044
 agenta/sdk/client.py,sha256=trKyBOYFZRk0v5Eptxvh87yPf50Y9CqY6Qgv4Fy-VH4,2142
 agenta/sdk/context.py,sha256=q-PxL05-I84puunUAs9LGsffEXcYhDxhQxjuOz2vK90,901
-agenta/sdk/decorators/base.py,sha256=9aNdX5h8a2mFweuhdO-BQPwXGKY9ONPIdLRhSGAGMfY,217
-agenta/sdk/decorators/llm_entrypoint.py,sha256=RU_Se5RcS_JBrpd4Sst54WLrEzxxVHhmkrXwv6nZxW8,18849
-agenta/sdk/decorators/tracing.py,sha256=9-NZPo52pC9i6vbGaoAcmU4J-fFYA1U6eyPWdbm6esU,3300
 agenta/sdk/router.py,sha256=0sbajvn5C7t18anH6yNo7-oYxldHnYfwcbmQnIXBePw,269
+agenta/sdk/tracing/callbacks.py,sha256=OePIUSTNG4HqMp9mOx5QtyeHWarG1opv00gg_oVwceE,4525
 agenta/sdk/tracing/context_manager.py,sha256=HskDaiORoOhjeN375gm05wYnieQzh5UnoIsnSAHkAyc,252
-agenta/sdk/tracing/llm_tracing.py,sha256=s8sHojV1ncYEg1uAWcm3BsfPd_6yaOPvB03Tq1FnEMU,8709
+agenta/sdk/tracing/decorators.py,sha256=ujtU8gf3GDoHYuLTfEYK_2eIYZ-1oX5dpv02Mf4l_II,1191
+agenta/sdk/tracing/llm_tracing.py,sha256=f69FtUIA187gNob8N6IZ-eiF6QQyBVl2JUMJ4Nnnp7M,8671
 agenta/sdk/tracing/logger.py,sha256=4zG9c51p8xPdKA5SL8MOgBfkpCnBSuV6JfWiXO0A7oc,473
-agenta/sdk/tracing/tasks_manager.py,sha256=ROrWIaqS2J2HHiJtRWiHKlLY8CCsqToP5VeXu7mamck,3748
+agenta/sdk/tracing/tasks_manager.py,sha256=XVGBEOwmHa6KcCC0PApk0_bZ0Ilk2ESuduNObB1rw2s,3792
 agenta/sdk/types.py,sha256=Mn0yBlHh_Yr_5oQXUfsYI3V7sJAVWkJgkxEOBDOOMS0,5852
 agenta/sdk/utils/globals.py,sha256=lpgflY8xovZJtHfJf41dbNCZGwx07YNkG9ldruv6xoI,360
 agenta/sdk/utils/helper/openai_cost.py,sha256=1VkgvucDnNZm1pTfcVLz9icWunntp1d7zwMmnviy3Uw,5877
@@ -157,7 +157,7 @@ agenta/templates/simple_prompt/app.py,sha256=kODgF6lhzsaJPdgL5b21bUki6jkvqjWZzWR
 agenta/templates/simple_prompt/env.example,sha256=g9AE5bYcGPpxawXMJ96gh8oenEPCHTabsiOnfQo3c5k,70
 agenta/templates/simple_prompt/requirements.txt,sha256=ywRglRy7pPkw8bljmMEJJ4aOOQKrt9FGKULZ-DGkoBU,23
 agenta/templates/simple_prompt/template.toml,sha256=DQBtRrF4GU8LBEXOZ-GGuINXMQDKGTEG5y37tnvIUIE,60
-agenta-0.14.7a0.dist-info/METADATA,sha256=FZ1CBOL2QI1sj6R650zq7V9fbHfIGSs7zGE79opaooU,26472
-agenta-0.14.7a0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-agenta-0.14.7a0.dist-info/entry_points.txt,sha256=PDiu8_8AsL7ibU9v4iNoOKR1S7F2rdxjlEprjM9QOgo,46
-agenta-0.14.7a0.dist-info/RECORD,,
+agenta-0.14.7a1.dist-info/METADATA,sha256=nTeaKtltqrlaVnwP6NNqcDvSQHeulJNunKIbjyOIPEc,26514
+agenta-0.14.7a1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+agenta-0.14.7a1.dist-info/entry_points.txt,sha256=PDiu8_8AsL7ibU9v4iNoOKR1S7F2rdxjlEprjM9QOgo,46
+agenta-0.14.7a1.dist-info/RECORD,,

agenta/sdk/decorators/base.py

@@ -1,10 +0,0 @@
-# Stdlib Imports
-from typing import Any, Callable
-
-
-class BaseDecorator:
-    def __init__(self):
-        pass
-
-    def __call__(self, func: Callable[..., Any]) -> Callable[..., Any]:
-        raise NotImplementedError