agenta 0.27.0a0__tar.gz → 0.27.0a1__tar.gz

{agenta-0.27.0a0 → agenta-0.27.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: agenta
-Version: 0.27.0a0
+Version: 0.27.0a1
 Summary: The SDK for agenta is an open-source LLMOps platform.
 Home-page: https://agenta.ai
 Keywords: LLMOps,LLM,evaluation,prompt engineering

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/__init__.py

@@ -16,6 +16,7 @@ from .sdk.types import (
 from .sdk.utils.logging import log as logging
 from .sdk.tracing import Tracing
 from .sdk.decorators.tracing import instrument
+from .sdk.tracing.conventions import Reference
 from .sdk.decorators.routing import entrypoint, app, route
 from .sdk.agenta_init import Config, AgentaSingleton, init
 from .sdk.utils.costs import calculate_token_usage

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/sdk/__init__.py

@@ -17,6 +17,7 @@ from .types import (
 
 from .tracing import Tracing, get_tracer
 from .decorators.tracing import instrument
+from .tracing.conventions import Reference
 from .decorators.routing import entrypoint, app, route
 from .agenta_init import Config, AgentaSingleton, init as _init
 from .utils.costs import calculate_token_usage

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/sdk/agenta_init.py

@@ -37,9 +37,9 @@ class AgentaSingleton:
         #
         app_id: Optional[str] = None,
     ) -> None:
-        log.info(f"\n--------------------------------")
-        log.info(f"Using Agenta Python SDK version: {version('agenta')}")
-        log.info(f"--------------------------------\n")
+        log.info(f"---------------------------")
+        log.info(f"Agenta SDK - using version: {version('agenta')}")
+        log.info(f"---------------------------")
 
         """Main function to initialize the singleton.
 
@@ -126,9 +126,10 @@ class Config:
         self.base_id = base_id
 
         if self.base_id is None:
-            print(
-                "Warning: Your configuration will not be saved permanently since base_id is not provided.\n"
-            )
+            # print(
+            #    "Warning: Your configuration will not be saved permanently since base_id is not provided.\n"
+            # )
+            pass
 
         if base_id is None or host is None:
             self.persist = False

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/sdk/decorators/routing.py

@@ -1,8 +1,3 @@
-"""The code for the Agenta SDK"""
-
-import os
-import sys
-import time
 import json
 import inspect
 import argparse
@@ -12,7 +7,6 @@ import functools
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Any, Callable, Dict, Optional, Tuple, List
-from importlib.metadata import version
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi import Body, FastAPI, UploadFile, HTTPException
 
@@ -44,8 +38,8 @@ from typing import Type
 from annotated_types import Ge, Le, Gt, Lt
 
 from pydantic import BaseModel, HttpUrl
-import contextvars
-from contextlib import contextmanager
+
+from traceback import format_exc
 
 
 app = FastAPI()
@@ -369,11 +363,11 @@ class entrypoint:
         *args,
         **func_params,
     ):
-        log.info(f"\n--------------------------")
+        log.info(f"---------------------------")
         log.info(
-            f"Running application route: {repr(self.route_path if self.route_path != '' else '/')}"
+            f"Agenta SDK - running route: {repr(self.route_path if self.route_path != '' else '/')}"
         )
-        log.info(f"--------------------------\n")
+        log.info(f"---------------------------")
 
         tracing_context.set(routing_context.get())
 
@@ -394,9 +388,11 @@ class entrypoint:
             )
             data = self.patch_result(result)
         except Exception as e:
-            log.error(f"Agenta SDK - Routing Exception")
-
-            traceback.print_exc()
+            log.error("--------------------------------------------------")
+            log.error("Agenta SDK - handling application exception below:")
+            log.error("--------------------------------------------------")
+            log.error(format_exc().strip("\n"))
+            log.error("--------------------------------------------------")
 
             self.handle_exception(e)
 
@@ -426,6 +422,10 @@ class entrypoint:
 
         response = BaseResponse(data=data, trace=trace)
 
+        log.info(f"----------------------------------")
+        log.info(f"Agenta SDK - exiting successfully: 200")
+        log.info(f"----------------------------------")
+
         return response
 
     def handle_exception(self, e: Exception):
@@ -434,6 +434,10 @@ class entrypoint:
         stacktrace = traceback.format_exception(e, value=e, tb=e.__traceback__)  # type: ignore
         detail = {"message": message, "stacktrace": stacktrace}
 
+        log.error(f"----------------------------------------")
+        log.error(f"Agenta SDK - exiting with HTTPException: {status_code}")
+        log.error(f"----------------------------------------")
+
         raise HTTPException(
             status_code=status_code,
             detail=detail,
@@ -658,27 +662,28 @@ class entrypoint:
         SHOW_DATA = False
         SHOW_TRACE = False
 
-        log.info("\n========= Result =========\n")
+        if result.trace:
+            log.info("\n========= Result =========\n")
 
-        log.info(f"trace_id: {result.trace['trace_id']}")
-        if SHOW_DETAILS:
-            log.info(f"latency:  {result.trace.get('latency')}")
-            log.info(f"cost:     {result.trace.get('cost')}")
-            log.info(f"usage:   {list(result.trace.get('usage', {}).values())}")
+            log.info(f"trace_id: {result.trace['trace_id']}")
+            if SHOW_DETAILS:
+                log.info(f"latency:  {result.trace.get('latency')}")
+                log.info(f"cost:     {result.trace.get('cost')}")
+                log.info(f"usage:   {list(result.trace.get('usage', {}).values())}")
 
-        if SHOW_DATA:
-            log.info(" ")
-            log.info(f"data:")
-            log.info(json.dumps(result.data, indent=2))
+            if SHOW_DATA:
+                log.info(" ")
+                log.info(f"data:")
+                log.info(json.dumps(result.data, indent=2))
 
-        if SHOW_TRACE:
-            log.info(" ")
-            log.info(f"trace:")
-            log.info(f"----------------")
-            log.info(json.dumps(result.trace.get("spans", []), indent=2))
-            log.info(f"----------------")
+            if SHOW_TRACE:
+                log.info(" ")
+                log.info(f"trace:")
+                log.info(f"----------------")
+                log.info(json.dumps(result.trace.get("spans", []), indent=2))
+                log.info(f"----------------")
 
-        log.info("\n==========================\n")
+            log.info("\n==========================\n")
 
     def override_config_in_schema(
         self,

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/sdk/decorators/tracing.py

@@ -147,6 +147,9 @@ class instrument:
                             cost = result.get("cost", None)
                             usage = result.get("usage", {})
 
+                        if isinstance(usage, (int, float)):
+                            usage = {"total_tokens": usage}
+
                         span.set_attributes(
                             attributes={"total": cost},
                             namespace="metrics.unit.costs",
@@ -240,6 +243,9 @@ class instrument:
                             cost = result.get("cost", None)
                             usage = result.get("usage", {})
 
+                        if isinstance(usage, (int, float)):
+                            usage = {"total_tokens": usage}
+
                         span.set_attributes(
                             attributes={"total": cost},
                             namespace="metrics.unit.costs",

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/sdk/tracing/attributes.py

@@ -134,48 +134,3 @@ def serialize(
     }
 
     return _attributes
-
-
-"""
-def _decode_key(namespace: Optional[str] = None, key: str = "") -> str:
-    if namespace is None:
-        return key
-
-    return key.replace(f"ag.{namespace}.", "")
-"""
-
-"""
-def _decode_value(value: Attribute) -> Any:
-    if isinstance(value, (int, float, bool, bytes)):
-        return value
-
-    if isinstance(value, str):
-        if value == "@ag.type=none:":
-            return None
-
-        if value.startswith("@ag.type=json:"):
-            encoded = value[len("@ag.type=json:") :]
-            value = loads(encoded)
-            return value
-
-        return value
-
-    return value
-"""
-
-"""
-def deserialize(
-    *,
-    namespace: str,
-    attributes: Dict[str, Any],
-    max_depth: Optional[int] = None,
-) -> Dict[str, Any]:
-    if not isinstance(attributes, dict):
-        return {}
-
-    return {
-        _decode_key(namespace, key): _decode_value(value)
-        for key, value in attributes
-        if key != _decode_key(namespace, key)
-    }
-"""

agenta-0.27.0a1/agenta/sdk/tracing/conventions.py

@@ -0,0 +1,47 @@
+from opentelemetry.trace import SpanKind
+
+from enum import Enum
+
+from re import fullmatch
+
+_PATTERN = r"[A-Za-z0-9._-]+"
+
+
+def is_valid_attribute_key(string):
+    return bool(fullmatch(_PATTERN, string))
+
+
+class Reference(str, Enum):
+    #
+    VARIANT_ID = "variant.id"
+    VARIANT_SLUG = "variant.slug"
+    VARIANT_VERSION = "variant.version"
+    #
+    ENVIRONMENT_ID = "environment.id"
+    ENVIRONMENT_SLUG = "environment.slug"
+    ENVIRONMENT_VERSION = "environment.version"
+    #
+    APPLICATION_ID = "application.id"
+    APPLICATION_SLUG = "application.slug"
+    #
+
+
+def parse_span_kind(type: str) -> SpanKind:
+    kind = SpanKind.INTERNAL
+    if type in [
+        "agent",
+        "chain",
+        "workflow",
+    ]:
+        kind = SpanKind.SERVER
+    elif type in [
+        "tool",
+        "embedding",
+        "query",
+        "completion",
+        "chat",
+        "rerank",
+    ]:
+        kind = SpanKind.CLIENT
+
+    return kind

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/sdk/tracing/exporters.py

@@ -48,6 +48,9 @@ class InlineTraceExporter(SpanExporter):
         return trace
 
 
+OTLPSpanExporter._MAX_RETRY_TIMEOUT = 2
+
+
 ConsoleExporter = ConsoleSpanExporter
 InlineExporter = InlineTraceExporter
 OTLPExporter = OTLPSpanExporter

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/sdk/tracing/inline.py

@@ -131,14 +131,6 @@ AttributeValueType = Any
 Attributes = Dict[str, AttributeValueType]
 
 
-class AttributesDTO(DisplayBase):
-    data: Optional[Attributes] = None
-    metrics: Optional[Attributes] = None
-    meta: Optional[Attributes] = None
-    tags: Optional[Attributes] = None
-    semconv: Optional[Attributes] = None
-
-
 class TreeType(Enum):
     # --- VARIANTS --- #
     INVOCATION = "invocation"
@@ -242,7 +234,7 @@ class OTelExtraDTO(DisplayBase):
 class SpanDTO(DisplayBase):
     scope: ProjectScopeDTO
 
-    lifecycle: LifecycleDTO
+    lifecycle: Optional[LifecycleDTO] = None
 
     root: RootDTO
     tree: TreeDTO
@@ -266,29 +258,6 @@ class SpanDTO(DisplayBase):
     nodes: Optional[Dict[str, Union["SpanDTO", List["SpanDTO"]]]] = None
 
 
-class SpanCreateDTO(DisplayBase):
-    scope: ProjectScopeDTO
-
-    root: RootDTO
-    tree: TreeDTO
-    node: NodeDTO
-
-    parent: Optional[ParentDTO] = None
-
-    time: TimeDTO
-    status: StatusDTO
-
-    data: Optional[Data] = None
-    metrics: Optional[Metrics] = None
-    meta: Optional[Metadata] = None
-    tags: Optional[Tags] = None
-    refs: Optional[Refs] = None
-
-    links: Optional[List[LinkDTO]] = None
-
-    otel: Optional[OTelExtraDTO] = None
-
-
 class OTelSpanDTO(DisplayBase):
     context: OTelContextDTO
 
@@ -321,20 +290,20 @@ from typing import List, Dict, OrderedDict
 
 
 def parse_span_dtos_to_span_idx(
-    span_dtos: List[SpanCreateDTO],
-) -> Dict[str, SpanCreateDTO]:
+    span_dtos: List[SpanDTO],
+) -> Dict[str, SpanDTO]:
     span_idx = {span_dto.node.id: span_dto for span_dto in span_dtos}
 
     return span_idx
 
 
 def parse_span_idx_to_span_id_tree(
-    span_idx: Dict[str, SpanCreateDTO],
+    span_idx: Dict[str, SpanDTO],
 ) -> OrderedDict:
     span_id_tree = OrderedDict()
     index = {}
 
-    def push(span_dto: SpanCreateDTO) -> None:
+    def push(span_dto: SpanDTO) -> None:
         if span_dto.parent is None:
             span_id_tree[span_dto.node.id] = OrderedDict()
             index[span_dto.node.id] = span_id_tree[span_dto.node.id]
@@ -350,15 +319,15 @@ def parse_span_idx_to_span_id_tree(
 
 def cumulate_costs(
     spans_id_tree: OrderedDict,
-    spans_idx: Dict[str, SpanCreateDTO],
+    spans_idx: Dict[str, SpanDTO],
 ) -> None:
-    def _get_unit(span: SpanCreateDTO):
+    def _get_unit(span: SpanDTO):
         if span.metrics is not None:
             return span.metrics.get("unit.costs.total", 0.0)
 
         return 0.0
 
-    def _get_acc(span: SpanCreateDTO):
+    def _get_acc(span: SpanDTO):
         if span.metrics is not None:
             return span.metrics.get("acc.costs.total", 0.0)
 
@@ -367,7 +336,7 @@ def cumulate_costs(
     def _acc(a: float, b: float):
         return a + b
 
-    def _set(span: SpanCreateDTO, cost: float):
+    def _set(span: SpanDTO, cost: float):
         if span.metrics is None:
             span.metrics = {}
 
@@ -381,7 +350,7 @@ def cumulate_tokens(
     spans_id_tree: OrderedDict,
     spans_idx: Dict[str, dict],
 ) -> None:
-    def _get_unit(span: SpanCreateDTO):
+    def _get_unit(span: SpanDTO):
         _tokens = {
             "prompt": 0.0,
             "completion": 0.0,
@@ -397,7 +366,7 @@ def cumulate_tokens(
 
         return _tokens
 
-    def _get_acc(span: SpanCreateDTO):
+    def _get_acc(span: SpanDTO):
         _tokens = {
             "prompt": 0.0,
             "completion": 0.0,
@@ -420,7 +389,7 @@ def cumulate_tokens(
             "total": a.get("total", 0.0) + b.get("total", 0.0),
         }
 
-    def _set(span: SpanCreateDTO, tokens: dict):
+    def _set(span: SpanDTO, tokens: dict):
         if span.metrics is None:
             span.metrics = {}
 
@@ -442,7 +411,7 @@ def cumulate_tokens(
 
 def _cumulate_tree_dfs(
     spans_id_tree: OrderedDict,
-    spans_idx: Dict[str, SpanCreateDTO],
+    spans_idx: Dict[str, SpanDTO],
     get_unit_metric,
     get_acc_metric,
     accumulate_metric,
@@ -1032,7 +1001,7 @@ def parse_inline_trace(
     ###############################################
     ### services.observability.service.ingest() ###
     ### --------------------------------------- ###
-    calculate_cost(span_idx)
+    calculate_costs(span_idx)
     cumulate_costs(span_id_tree, span_idx)
     cumulate_tokens(span_id_tree, span_idx)
     ### --------------------------------------- ###
@@ -1254,7 +1223,7 @@ def _parse_to_legacy_span(span: SpanDTO) -> CreateSpan:
         ),
         #
         app_id=(
-            span.refs.get("application_id", "missing-app-id")
+            span.refs.get("application.id", "missing-app-id")
             if span.refs
             else "missing-app-id"
         ),
@@ -1271,7 +1240,7 @@ def _parse_to_legacy_span(span: SpanDTO) -> CreateSpan:
     return legacy_span
 
 
-PAYING_TYPES = [
+TYPES_WITH_COSTS = [
     "embedding",
     "query",
     "completion",
@@ -1280,9 +1249,14 @@ PAYING_TYPES = [
 ]
 
 
-def calculate_cost(span_idx: Dict[str, SpanCreateDTO]):
+def calculate_costs(span_idx: Dict[str, SpanDTO]):
     for span in span_idx.values():
-        if span.node.type.name.lower() in PAYING_TYPES and span.meta and span.metrics:
+        if (
+            span.node.type
+            and span.node.type.name.lower() in TYPES_WITH_COSTS
+            and span.meta
+            and span.metrics
+        ):
             try:
                 costs = cost_calculator.cost_per_token(
                     model=span.meta.get("response.model"),

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/sdk/tracing/processors.py

@@ -1,4 +1,5 @@
-from typing import Optional, Any, Dict
+from typing import Optional, Dict
+from traceback import format_exc
 
 from opentelemetry.context import Context
 from opentelemetry.sdk.trace import Span
@@ -10,6 +11,8 @@ from opentelemetry.sdk.trace.export import (
     _DEFAULT_MAX_QUEUE_SIZE,
 )
 
+from agenta.sdk.utils.logging import log
+
 # LOAD CONTEXT, HERE
 
 
@@ -28,7 +31,7 @@ class TraceProcessor(BatchSpanProcessor):
             _DEFAULT_MAX_QUEUE_SIZE,
             60 * 60 * 1000,  # 1 hour
             _DEFAULT_MAX_QUEUE_SIZE,
-            _DEFAULT_EXPORT_TIMEOUT_MILLIS,
+            500,  # < 1 second (0.5 seconds)
         )
 
         self._registry = dict()
@@ -54,6 +57,14 @@ class TraceProcessor(BatchSpanProcessor):
         if self.is_ready(span.get_span_context().trace_id):
             self.force_flush()
 
+    def force_flush(self, timeout_millis: int = None) -> bool:
+        ret = super().force_flush(timeout_millis)
+
+        if not ret:
+            log.error("--------------------------------------------")
+            log.error("Agenta SDK - skipping export due to timeout.")
+            log.error("--------------------------------------------")
+
     def is_ready(self, trace_id: Optional[int] = None) -> bool:
         is_ready = not len(self._registry.get(trace_id, {}))
 

{agenta-0.27.0a0 → agenta-0.27.0a1}/agenta/sdk/tracing/tracing.py

@@ -1,4 +1,5 @@
 from typing import Optional, Any, Dict
+from enum import Enum
 
 from httpx import get as check
 
@@ -13,15 +14,16 @@ from opentelemetry.sdk.trace import Span, Tracer, TracerProvider
 from opentelemetry.sdk.resources import Resource
 
 from agenta.sdk.utils.singleton import Singleton
-from agenta.sdk.utils.exceptions import suppress  # USE IT !
+from agenta.sdk.utils.exceptions import suppress
 from agenta.sdk.utils.logging import log
 
 from agenta.sdk.tracing.processors import TraceProcessor
 from agenta.sdk.tracing.exporters import ConsoleExporter, InlineExporter, OTLPExporter
 from agenta.sdk.tracing.spans import CustomSpan
-from agenta.sdk.tracing.context import tracing_context
 from agenta.sdk.tracing.inline import parse_inline_trace
 
+from agenta.sdk.tracing.conventions import Reference, is_valid_attribute_key
+
 
 class Tracing(metaclass=Singleton):
     VERSION = "0.1.0"
@@ -74,7 +76,7 @@ class Tracing(metaclass=Singleton):
         if api_key:
             self.headers.update(**{"Authorization": self.api_key})
         # REFERENCES
-        self.references = {"application_id": app_id}
+        self.references = {"application.id": app_id}
 
         # TRACER PROVIDER
         self.tracer_provider = TracerProvider(
@@ -120,10 +122,13 @@ class Tracing(metaclass=Singleton):
     def get_current_span(
         self,
     ):
-        _span = get_current_span()
+        _span = None
+
+        with suppress():
+            _span = get_current_span()
 
-        if _span.is_recording():
-            return CustomSpan(_span)
+            if _span.is_recording():
+                return CustomSpan(_span)
 
         return _span
 
@@ -132,39 +137,89 @@ class Tracing(metaclass=Singleton):
         attributes: Dict[str, Any],
         span: Optional[Span] = None,
     ):
-        if span is None:
-            span = self.get_current_span()
+        with suppress():
+            if span is None:
+                span = self.get_current_span()
+
+            span.set_attributes(attributes={"internals": attributes}, namespace="data")
+
+    def store_refs(
+        self,
+        refs: Dict[str, str],
+        span: Optional[Span] = None,
+    ):
+        with suppress():
+            if span is None:
+                span = self.get_current_span()
+
+            for key in refs.keys():
+                if key in Reference:
+                    # ADD REFERENCE TO THIS SPAN
+                    span.set_attribute(
+                        key.value if isinstance(key, Enum) else key,
+                        refs[key],
+                        namespace="refs",
+                    )
+                    # AND TO ALL SPANS CREATED AFTER THIS ONE
+                    self.references[key] = refs[key]
+                    # TODO: THIS SHOULD BE REPLACED BY A TRACE CONTEXT !!!
+
+    def store_meta(
+        self,
+        meta: Dict[str, Any],
+        span: Optional[Span] = None,
+    ):
+        with suppress():
+            if span is None:
+                span = self.get_current_span()
+
+            for key in meta.keys():
+                if is_valid_attribute_key(key):
+                    span.set_attribute(key, meta[key], namespace="meta")
 
-        span.set_attributes(attributes={"internals": attributes}, namespace="data")
+    def store_metrics(
+        self,
+        metrics: Dict[str, Any],
+        span: Optional[Span] = None,
+    ):
+        with suppress():
+            if span is None:
+                span = self.get_current_span()
+
+            for key in metrics.keys():
+                if is_valid_attribute_key(key):
+                    span.set_attribute(key, metrics[key], namespace="metrics")
 
     def is_inline_trace_ready(
         self,
-        trace_id: int,
+        trace_id: Optional[int] = None,
     ) -> bool:
-        is_ready = self.inline.is_ready(trace_id)
+        is_ready = True
+
+        with suppress():
+            if trace_id is not None:
+                is_ready = self.inline.is_ready(trace_id)
 
         return is_ready
 
     def get_inline_trace(
         self,
-        trace_id: int,
+        trace_id: Optional[int] = None,
     ) -> Dict[str, Any]:
-        if trace_id is None:
-            return {}
-
-        is_ready = self.inline.is_ready(trace_id)
-
-        if is_ready is False:
-            return {}
+        _inline_trace = {}
 
-        otel_spans = self.inline.fetch(trace_id)
+        with suppress():
+            is_ready = self.inline.is_ready(trace_id)
 
-        if not otel_spans:
-            return {}
+            if is_ready is True:
+                otel_spans = self.inline.fetch(trace_id)
 
-        inline_trace = parse_inline_trace(self.project_id or self.app_id, otel_spans)
+                if otel_spans:
+                    _inline_trace = parse_inline_trace(
+                        self.project_id or self.app_id, otel_spans
+                    )
 
-        return inline_trace
+        return _inline_trace
 
 
 def get_tracer(tracing: Tracing) -> Tracer: