agenta 0.20.0a0__tar.gz → 0.20.0a3__tar.gz

{agenta-0.20.0a0 → agenta-0.20.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: agenta
-Version: 0.20.0a0
+Version: 0.20.0a3
 Summary: The SDK for agenta is an open-source LLMOps platform.
 Home-page: https://agenta.ai
 Keywords: LLMOps,LLM,evaluation,prompt engineering

{agenta-0.20.0a0 → agenta-0.20.0a3}/agenta/client/backend/resources/observability/client.py

@@ -17,6 +17,9 @@ from ...types.span_detail import SpanDetail
 from ...types.trace_detail import TraceDetail
 from ...types.with_pagination import WithPagination
 
+from agenta.sdk.tracing.logger import llm_logger as logging
+
+
 try:
     import pydantic.v1 as pydantic  # type: ignore
 except ImportError:
@@ -769,6 +772,14 @@ class AsyncObservabilityClient:
             ],
         )
         """
+
+        logging.debug("----")
+        logging.debug(
+            urllib.parse.urljoin(
+                f"{self._client_wrapper.get_base_url()}/", "observability/trace/"
+            ),
+        )
+        logging.debug(self._client_wrapper.get_headers())
         _response = await self._client_wrapper.httpx_client.request(
             "POST",
             urllib.parse.urljoin(

{agenta-0.20.0a0 → agenta-0.20.0a3}/agenta/client/backend/types/span.py

@@ -23,7 +23,7 @@ class Span(pydantic.BaseModel):
     created_at: dt.datetime
     variant: SpanVariant
     environment: typing.Optional[str]
-    spankind: SpanKind
+    spankind: str
     status: SpanStatusCode
     metadata: typing.Dict[str, typing.Any]
     user_id: typing.Optional[str]

{agenta-0.20.0a0 → agenta-0.20.0a3}/agenta/client/backend/types/span_detail.py

@@ -22,7 +22,7 @@ class SpanDetail(pydantic.BaseModel):
     created_at: dt.datetime
     variant: SpanVariant
     environment: typing.Optional[str]
-    spankind: SpanKind
+    spankind: str
     status: SpanStatusCode
     metadata: typing.Dict[str, typing.Any]
     user_id: typing.Optional[str]

{agenta-0.20.0a0 → agenta-0.20.0a3}/agenta/client/backend/types/trace_detail.py

@@ -22,7 +22,7 @@ class TraceDetail(pydantic.BaseModel):
     created_at: dt.datetime
     variant: SpanVariant
     environment: typing.Optional[str]
-    spankind: SpanKind
+    spankind: str
     status: SpanStatusCode
     metadata: typing.Dict[str, typing.Any]
     user_id: typing.Optional[str]

{agenta-0.20.0a0 → agenta-0.20.0a3}/agenta/sdk/agenta_init.py

@@ -68,6 +68,12 @@ class AgentaSingleton:
             api_key or config.get("api_key") or os.environ.get("AGENTA_API_KEY")
         )
 
+        print(api_key)
+        print(config.get("api_key"))
+        print(os.environ.get("AGENTA_API_KEY"))
+        print(self.api_key)
+        print(os.environ)
+
         if not self.app_id:
             raise ValueError(
                 "App ID must be specified. You can provide it in one of the following ways:\n"

{agenta-0.20.0a0 → agenta-0.20.0a3}/agenta/sdk/decorators/llm_entrypoint.py

@@ -141,7 +141,11 @@ class entrypoint(BaseDecorator):
             )
 
             entrypoint_result = await self.execute_function(
-                func, *args, params=func_params, config_params=config_params
+                func,
+                True,  # inline trace: True
+                *args,
+                params=func_params,
+                config_params=config_params,
             )
 
             return entrypoint_result
@@ -194,7 +198,11 @@ class entrypoint(BaseDecorator):
             )
 
             entrypoint_result = await self.execute_function(
-                func, *args, params=func_params, config_params=config_params
+                func,
+                False,  # inline trace: False
+                *args,
+                params=func_params,
+                config_params=config_params,
             )
 
             return entrypoint_result
@@ -205,7 +213,7 @@ class entrypoint(BaseDecorator):
             ingestible_files,
         )
 
-        if route_path == "/":
+        if route_path == "":
             route_deployed = f"/{DEFAULT_PATH}_deployed"
             app.post(route_deployed, response_model=BaseResponse)(wrapper_deployed)
 
@@ -226,6 +234,8 @@ class entrypoint(BaseDecorator):
             )
         ### ---------------------- #
 
+        print(entrypoint.routes)
+
         if self.is_main_script(func) and route_path == "":
             self.handle_terminal_run(
                 func,
@@ -272,7 +282,9 @@ class entrypoint(BaseDecorator):
             if name in func_params and func_params[name] is not None:
                 func_params[name] = self.ingest_file(func_params[name])
 
-    async def execute_function(self, func: Callable[..., Any], *args, **func_params):
+    async def execute_function(
+        self, func: Callable[..., Any], inline_trace, *args, **func_params
+    ):
         """Execute the function and handle any exceptions."""
 
         try:
@@ -281,6 +293,11 @@ class entrypoint(BaseDecorator):
             For synchronous functions, it calls them directly, while for asynchronous functions,
             it awaits their execution.
             """
+            WAIT_FOR_SPANS = True
+            TIMEOUT = 10
+            TIMESTEP = 0.01
+            NOFSTEPS = TIMEOUT / TIMESTEP
+
             data = None
             trace = None
 
@@ -296,18 +313,41 @@ class entrypoint(BaseDecorator):
                 result = func(*args, **func_params["params"])
 
             if token is not None:
+                if WAIT_FOR_SPANS:
+                    remaining_steps = NOFSTEPS
+
+                    while not ag.tracing.is_trace_ready() and remaining_steps > 0:
+                        await asyncio.sleep(0.01)
+                        remaining_steps -= 1
+
                 trace = ag.tracing.dump_trace()
+
+                if not inline_trace:
+                    trace = {"trace_id": trace["trace_id"]}
+
+                ag.tracing.flush_spans()
                 tracing_context.reset(token)
 
             if isinstance(result, Context):
                 save_context(result)
 
+            DEFAULT_KEY = "message"
+
             if isinstance(result, Dict):
                 data = result
+
+                # EVENTUALLY THIS PATCH SHOULD BE REMOVED
+                # PATCH: if message in result then only keep message key/value
+                # DEFAULT_KEY = "message"
+
+                if "message" in result.keys():
+                    data = {DEFAULT_KEY: result["message"]}
+                # END OF PATCH
+
             elif isinstance(result, str):
-                data = {"message": result}
+                data = {DEFAULT_KEY: result}
             elif isinstance(result, int) or isinstance(result, float):
-                data = {"message": str(result)}
+                data = {DEFAULT_KEY: str(result)}
 
             if data is None:
                 warning = (
@@ -515,6 +555,7 @@ class entrypoint(BaseDecorator):
         result = loop.run_until_complete(
             self.execute_function(
                 func,
+                True,  # inline trace: True
                 **{"params": args_func_params, "config_params": args_config_params},
             )
         )
@@ -524,7 +565,7 @@ class entrypoint(BaseDecorator):
         print("-> data")
         print(json.dumps(result.data, indent=2))
         print("-> trace")
-        print(json.dumps(result.trace, indent=2))
+        # print(json.dumps(result.trace, indent=2))
 
     def override_schema(
         self, openapi_schema: dict, func: str, endpoint: str, params: dict
@@ -598,6 +639,7 @@ class entrypoint(BaseDecorator):
         schema_to_override = openapi_schema["components"]["schemas"][
             f"Body_{func}_{endpoint}_post"
         ]["properties"]
+
         for param_name, param_val in params.items():
             if isinstance(param_val, GroupedMultipleChoiceParam):
                 subschema = find_in_schema(

{agenta-0.20.0a0 → agenta-0.20.0a3}/agenta/sdk/decorators/tracing.py

@@ -65,7 +65,22 @@ class instrument(BaseDecorator):
                 ):
                     result = await func(*args, **kwargs)
 
-                    ag.tracing.store_outputs(result)
+                    outputs = result
+
+                    # EVENTUALLY THIS PATCH SHOULD BE REMOVED
+                    # PATCH : if result is not a dict, make it a dict, in span
+                    DEFAULT_KEY = "default"
+
+                    if not isinstance(result, dict):
+                        value = result
+
+                        if result.__class__.__module__ != "__builtin__":
+                            value = repr(value)
+
+                        outputs = {DEFAULT_KEY: result}
+                    # END OF PATH
+
+                    ag.tracing.store_outputs(outputs)
 
                     return result
 
@@ -82,7 +97,22 @@ class instrument(BaseDecorator):
                 ):
                     result = func(*args, **kwargs)
 
-                    ag.tracing.store_outputs(result)
+                    outputs = result
+
+                    # EVENTUALLY THIS PATCH SHOULD BE REMOVED
+                    # PATCH : if result is not a dict, make it a dict, in span
+                    DEFAULT_KEY = "default"
+
+                    if not isinstance(result, dict):
+                        value = result
+
+                        if result.__class__.__module__ != "__builtin__":
+                            value = repr(value)
+
+                        outputs = {DEFAULT_KEY: result}
+                    # END OF PATH
+
+                    ag.tracing.store_outputs(outputs)
 
                     return result
 

{agenta-0.20.0a0 → agenta-0.20.0a3}/agenta/sdk/tracing/callbacks.py

@@ -1,5 +1,9 @@
 import agenta as ag
 
+from agenta.sdk.tracing.tracing_context import tracing_context, TracingContext
+
+from agenta.sdk.utils.debug import debug
+
 
 def litellm_handler():
     try:
@@ -23,20 +27,25 @@ def litellm_handler():
             LitellmCustomLogger (object): custom logger that allows us to override the events to capture.
         """
 
+        def __init__(self):
+            self.span = None
+
         @property
         def _trace(self):
             return ag.tracing
 
+        @debug()
         def log_pre_api_call(self, model, messages, kwargs):
             call_type = kwargs.get("call_type")
             span_kind = (
                 "llm" if call_type in ["completion", "acompletion"] else "embedding"
             )
 
-            ag.tracing.start_span(
+            self.span = ag.tracing.open_span(
                 name=f"{span_kind}_call",
                 input={"messages": kwargs["messages"]},
                 spankind=span_kind,
+                active=False,
             )
             ag.tracing.set_attributes(
                 {
@@ -49,9 +58,10 @@ def litellm_handler():
                 }
             )
 
+        @debug()
         def log_stream_event(self, kwargs, response_obj, start_time, end_time):
-            ag.tracing.set_status(status="OK")
-            ag.tracing.end_span(
+            ag.tracing.set_status(status="OK", span_id=self.span.id)
+            ag.tracing.store_outputs(
                 outputs={
                     "message": kwargs.get(
                         "complete_streaming_response"
@@ -65,13 +75,16 @@ def litellm_handler():
                         "response_cost"
                     ),  # litellm calculates response cost
                 },
+                span_id=self.span.id,
             )
+            ag.tracing.close_span(span_id=self.span.id)
 
+        @debug()
         def log_success_event(
             self, kwargs, response_obj: ModelResponse, start_time, end_time
         ):
-            ag.tracing.set_status(status="OK")
-            ag.tracing.end_span(
+            ag.tracing.set_status(status="OK", span_id=self.span.id)
+            ag.tracing.store_outputs(
                 outputs={
                     "message": response_obj.choices[0].message.content,
                     "usage": (
@@ -83,12 +96,15 @@ def litellm_handler():
                         "response_cost"
                     ),  # litellm calculates response cost
                 },
+                span_id=self.span.id,
             )
+            ag.tracing.close_span(span_id=self.span.id)
 
+        @debug()
         def log_failure_event(
             self, kwargs, response_obj: ModelResponse, start_time, end_time
         ):
-            ag.tracing.set_status(status="ERROR")
+            ag.tracing.set_status(status="ERROR", span_id=self.span.id)
             ag.tracing.set_attributes(
                 {
                     "traceback_exception": repr(
@@ -98,10 +114,11 @@ def litellm_handler():
                         "end_time"
                     ],  # datetime object of when call was completed
                 },
+                span_id=self.span.id,
             )
-            ag.tracing.end_span(
+            ag.tracing.store_outputs(
                 outputs={
-                    "message": kwargs["exception"],  # the Exception raised
+                    "message": repr(kwargs["exception"]),  # the Exception raised
                     "usage": (
                         response_obj.usage.dict()
                         if hasattr(response_obj, "usage")
@@ -111,13 +128,16 @@ def litellm_handler():
                         "response_cost"
                     ),  # litellm calculates response cost
                 },
+                span_id=self.span.id,
             )
+            ag.tracing.close_span(span_id=self.span.id)
 
+        @debug()
         async def async_log_stream_event(
             self, kwargs, response_obj, start_time, end_time
         ):
-            ag.tracing.set_status(status="OK")
-            ag.tracing.end_span(
+            ag.tracing.set_status(status="OK", span_id=self.span.id)
+            ag.tracing.store_outputs(
                 outputs={
                     "message": kwargs.get(
                         "complete_streaming_response"
@@ -131,13 +151,16 @@ def litellm_handler():
                         "response_cost"
                     ),  # litellm calculates response cost
                 },
+                span_id=self.span.id,
             )
+            ag.tracing.close_span(span_id=self.span.id)
 
+        @debug()
         async def async_log_success_event(
             self, kwargs, response_obj, start_time, end_time
         ):
-            ag.tracing.set_status(status="OK")
-            ag.tracing.end_span(
+            ag.tracing.set_status(status="OK", span_id=self.span.id)
+            ag.tracing.store_outputs(
                 outputs={
                     "message": response_obj.choices[0].message.content,
                     "usage": (
@@ -149,12 +172,15 @@ def litellm_handler():
                         "response_cost"
                     ),  # litellm calculates response cost
                 },
+                span_id=self.span.id,
             )
+            ag.tracing.close_span(span_id=self.span.id)
 
+        @debug()
         async def async_log_failure_event(
             self, kwargs, response_obj, start_time, end_time
         ):
-            ag.tracing.set_status(status="ERROR")
+            ag.tracing.set_status(status="ERROR", span_id=self.span.id)
             ag.tracing.set_attributes(
                 {
                     "traceback_exception": kwargs[
@@ -164,8 +190,9 @@ def litellm_handler():
                         "end_time"
                     ],  # datetime object of when call was completed
                 },
+                span_id=self.span.id,
             )
-            ag.tracing.end_span(
+            ag.tracing.store_outputs(
                 outputs={
                     "message": repr(kwargs["exception"]),  # the Exception raised
                     "usage": (
@@ -177,6 +204,8 @@ def litellm_handler():
                         "response_cost"
                     ),  # litellm calculates response cost
                 },
+                span_id=self.span.id,
             )
+            ag.tracing.close_span(span_id=self.span.id)
 
     return LitellmHandler()