synth-ai 0.2.9.dev5__py3-none-any.whl → 0.2.10__py3-none-any.whl

synth_ai/tracing_v3/abstractions.py

@@ -37,7 +37,7 @@ Concepts:
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass, field
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import Any
 
 from .lm_call_record_abstractions import LLMCallRecord
@@ -61,6 +61,23 @@ class TimeRecord:
     message_time: int | None = None
 
 
+@dataclass(frozen=True)
+class SessionMessageContent:
+    """Normalized payload stored alongside session messages."""
+
+    text: str | None = None
+    json_payload: str | None = None
+
+    def as_text(self) -> str:
+        return self.text or (self.json_payload or "")
+
+    def has_json(self) -> bool:
+        return self.json_payload is not None
+
+    def __str__(self) -> str:  # pragma: no cover - convenience for logging
+        return self.as_text()
+
+
 @dataclass
 class SessionEventMarkovBlanketMessage:
     """Message crossing Markov blanket boundaries between systems in a session.
@@ -97,7 +114,7 @@ class SessionEventMarkovBlanketMessage:
                   - 'causal_influence': Direction of causal flow
     """
 
-    content: str
+    content: SessionMessageContent
     message_type: str
     time_record: TimeRecord
     metadata: dict[str, Any] = field(default_factory=dict)
@@ -232,7 +249,7 @@ class SessionTimeStep:
 
     step_id: str = ""
     step_index: int = 0
-    timestamp: datetime = field(default_factory=datetime.utcnow)
+    timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
     turn_number: int | None = None
     events: list[BaseEvent] = field(default_factory=list)
     markov_blanket_messages: list[SessionEventMarkovBlanketMessage] = field(default_factory=list)
@@ -266,7 +283,7 @@ class SessionTrace:
     """
 
     session_id: str = ""
-    created_at: datetime = field(default_factory=datetime.utcnow)
+    created_at: datetime = field(default_factory=lambda: datetime.now(UTC))
     session_time_steps: list[SessionTimeStep] = field(default_factory=list)
     event_history: list[BaseEvent] = field(default_factory=list)
     markov_blanket_message_history: list[SessionEventMarkovBlanketMessage] = field(

synth_ai/tracing_v3/db_config.py

@@ -4,6 +4,7 @@ Centralized database configuration for v3 tracing.
 
 import logging
 import os
+import shutil
 from typing import TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
@@ -30,7 +31,7 @@ class DatabaseConfig:
             http_port: HTTP port for sqld daemon. If None, uses DEFAULT_HTTP_PORT from serve.sh.
             use_sqld: Whether to use sqld daemon or direct SQLite.
         """
-        self.use_sqld = use_sqld
+        self.use_sqld = use_sqld and self._sqld_binary_available()
         self.http_port = http_port or int(os.getenv("SQLD_HTTP_PORT", self.DEFAULT_HTTP_PORT))
         self._daemon: SqldDaemon | None = None
 
@@ -70,6 +71,30 @@ class DatabaseConfig:
         # SQLite URLs need 3 slashes for absolute paths
         return f"sqlite+aiosqlite:///{actual_db_path}"
 
+    def _sqld_binary_available(self) -> bool:
+        """Check if the sqld (Turso) binary is available on PATH."""
+        # Respect explicit SQLD_BINARY override when present
+        binary_override = os.getenv("SQLD_BINARY")
+        candidates = [binary_override, "sqld", "libsql-server"]
+
+        for candidate in candidates:
+            if candidate and shutil.which(candidate):
+                return True
+
+        if binary_override:
+            logger.warning(
+                "Configured SQLD_BINARY='%s' but the executable was not found on PATH. "
+                "Falling back to direct SQLite.",
+                binary_override,
+            )
+        else:
+            logger.warning(
+                "sqld binary not detected; falling back to SQLite-only mode. "
+                "Install Turso's sqld or set SQLD_BINARY to enable the Turso daemon."
+            )
+
+        return False
+
     def start_daemon(self, wait_time: float = 2.0):
         """
         Start the sqld daemon if configured.

synth_ai/tracing_v3/decorators.py

@@ -1,4 +1,3 @@
-from __future__ import annotations
 """Async-aware decorators for tracing v3.
 
 This module provides decorators and context management utilities for the tracing
@@ -23,6 +22,8 @@ The decorators support both sync and async functions where appropriate,
 though async is preferred for consistency with the rest of the system.
 """
 
+from __future__ import annotations
+
 import asyncio
 import contextvars
 import functools
@@ -36,12 +37,8 @@ from .utils import calculate_cost, detect_provider
 # Context variables for session and turn tracking
 # These variables automatically propagate across async call boundaries,
 # allowing deeply nested code to access tracing context without explicit passing
-_session_id_ctx: contextvars.ContextVar[str | None] = contextvars.ContextVar(
-    "session_id", default=None
-)
-_turn_number_ctx: contextvars.ContextVar[int | None] = contextvars.ContextVar(
-    "turn_number", default=None
-)
+_session_id_ctx: contextvars.ContextVar[str | None] = contextvars.ContextVar("session_id")
+_turn_number_ctx: contextvars.ContextVar[int | None] = contextvars.ContextVar("turn_number")
 _session_tracer_ctx: contextvars.ContextVar[Any | None] = contextvars.ContextVar(
     "session_tracer", default=None
 )
@@ -119,7 +116,9 @@ def with_session(require: bool = True):
             async def async_wrapper(*args, **kwargs):
                 session_id = get_session_id()
                 if require and session_id is None:
-                    raise RuntimeError(f"No active session for {fn.__name__}")
+                    raise RuntimeError(
+                        f"No active session for {getattr(fn, '__name__', 'unknown')}"
+                    )
                 return await fn(*args, **kwargs)
 
             return async_wrapper
@@ -129,7 +128,9 @@ def with_session(require: bool = True):
             def sync_wrapper(*args, **kwargs):
                 session_id = get_session_id()
                 if require and session_id is None:
-                    raise RuntimeError(f"No active session for {fn.__name__}")
+                    raise RuntimeError(
+                        f"No active session for {getattr(fn, '__name__', 'unknown')}"
+                    )
                 return fn(*args, **kwargs)
 
             return sync_wrapper
@@ -138,7 +139,7 @@ def with_session(require: bool = True):
 
 
 def trace_llm_call(
-    model_name: str = None,
+    model_name: str | None = None,
     system_id: str = "llm",
     extract_tokens: bool = True,
     extract_cost: bool = True,
@@ -208,14 +209,16 @@ def trace_llm_call(
                         input_tokens=input_tokens,
                         output_tokens=output_tokens,
                         total_tokens=total_tokens,
-                        cost_usd=calculate_cost(actual_model, input_tokens or 0, output_tokens or 0)
+                        cost_usd=calculate_cost(
+                            actual_model or "unknown", input_tokens or 0, output_tokens or 0
+                        )
                         if extract_cost
                         else None,
                         latency_ms=latency_ms,
                         system_state_before=system_state_before,
                         system_state_after=kwargs.get("state_after", {}),
                         metadata={
-                            "function": fn.__name__,
+                            "function": getattr(fn, "__name__", "unknown"),
                             "step_id": kwargs.get("step_id"),
                         },
                     )
@@ -234,7 +237,7 @@ def trace_llm_call(
                             provider=detect_provider(model_name),
                             latency_ms=int((time.time() - start_time) * 1000),
                             metadata={
-                                "function": fn.__name__,
+                                "function": getattr(fn, "__name__", "unknown"),
                                 "error": str(e),
                                 "error_type": type(e).__name__,
                             },
@@ -249,7 +252,7 @@ def trace_llm_call(
     return decorator
 
 
-def trace_method(event_type: str = "runtime", system_id: str = None):
+def trace_method(event_type: str = "runtime", system_id: str | None = None):
     """Generic method tracing decorator.
 
     Traces any method call by recording it as a RuntimeEvent. Supports both
@@ -288,7 +291,7 @@ def trace_method(event_type: str = "runtime", system_id: str = None):
                     time_record=TimeRecord(event_time=time.time()),
                     actions=[],  # Can be overridden in metadata
                     metadata={
-                        "method": fn.__name__,
+                        "method": getattr(fn, "__name__", "unknown"),
                         "args": str(args)[:100],  # Truncate for safety
                         "step_id": kwargs.get("step_id"),
                     },

synth_ai/tracing_v3/examples/basic_usage.py

@@ -166,8 +166,9 @@ async def main():
 
         tracer.hooks.register("event_recorded", count_events, name="event_counter")
 
-        async with tracer.session(metadata={"example": "hooks"}) as session_id, tracer.timestep(
-            "hook_test"
+        async with (
+            tracer.session(metadata={"example": "hooks"}) as session_id,
+            tracer.timestep("hook_test"),
         ):
             for i in range(3):
                 event = RuntimeEvent(

synth_ai/tracing_v3/hooks.py

@@ -1,4 +1,3 @@
-from __future__ import annotations
 """Hook system for extending tracing functionality.
 
 The hook system provides a flexible way to extend the tracing system without
@@ -33,6 +32,8 @@ Common Use Cases:
 - Custom filtering and sampling
 """
 
+from __future__ import annotations
+
 import asyncio
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -88,9 +89,9 @@ class HookManager:
         self,
         event: str,
         callback: Callable,
-        name: str = None,
+        name: str | None = None,
         priority: int = 0,
-        event_types: list[str] = None,
+        event_types: list[str] | None = None,
     ) -> Hook:
         """Register a new hook.
 
@@ -114,7 +115,7 @@ class HookManager:
             raise ValueError(f"Unknown hook event: {event}")
 
         hook = Hook(
-            name=name or callback.__name__,
+            name=name or getattr(callback, "__name__", "unknown"),
             callback=callback,
             event_types=event_types,
             priority=priority,
@@ -202,6 +203,7 @@ def create_default_hooks() -> HookManager:
     # Example: Log session starts - useful for debugging and monitoring
     async def log_session_start(session_id: str, metadata: dict[str, Any]):
         import os
+
         if os.getenv("SYNTH_TRACE_VERBOSE", "0") in ("1", "true", "True"):
             print(f"Session started: {session_id}")
 

synth_ai/tracing_v3/llm_call_record_helpers.py

@@ -5,10 +5,9 @@ format and compute aggregates from call records.
 """
 
 import uuid
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import Any
 
-from synth_ai.lm.vendors.base import BaseLMResponse
 from synth_ai.tracing_v3.lm_call_record_abstractions import (
     LLMCallRecord,
     LLMChunk,
@@ -18,6 +17,7 @@ from synth_ai.tracing_v3.lm_call_record_abstractions import (
     LLMUsage,
     ToolCallSpec,
 )
+from synth_ai.v0.lm.vendors.base import BaseLMResponse
 
 
 def create_llm_call_record_from_response(
@@ -161,8 +161,8 @@ def create_llm_call_record_from_response(
         api_type=api_type,
         provider=provider,
         model_name=model_name,
-        started_at=started_at or datetime.utcnow(),
-        completed_at=completed_at or datetime.utcnow(),
+        started_at=started_at or datetime.now(UTC),
+        completed_at=completed_at or datetime.now(UTC),
         latency_ms=latency_ms,
         request_params=params,
         input_messages=input_messages,
@@ -322,8 +322,8 @@ def create_llm_call_record_from_streaming(
         api_type="responses",  # Streaming typically from Responses API
         provider=provider,
         model_name=model_name,
-        started_at=started_at or datetime.utcnow(),
-        completed_at=completed_at or datetime.utcnow(),
+        started_at=started_at or datetime.now(UTC),
+        completed_at=completed_at or datetime.now(UTC),
         latency_ms=latency_ms,
         request_params=params,
         input_messages=input_messages,

synth_ai/tracing_v3/replica_sync.py

@@ -180,6 +180,7 @@ class ReplicaSync:
             # Request cancellation
             self._sync_task.cancel()
             import contextlib
+
             with contextlib.suppress(asyncio.CancelledError):
                 # Wait for the task to finish
                 await self._sync_task

synth_ai/tracing_v3/session_tracer.py

@@ -1,14 +1,17 @@
-from __future__ import annotations
 """Main SessionTracer class for tracing v3."""
 
+from __future__ import annotations
+
 import asyncio
+import json
 from contextlib import asynccontextmanager
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import Any
 
 from .abstractions import (
     BaseEvent,
     SessionEventMarkovBlanketMessage,
+    SessionMessageContent,
     SessionTimeStep,
     SessionTrace,
     TimeRecord,
@@ -16,7 +19,9 @@ from .abstractions import (
 from .config import CONFIG
 from .decorators import set_session_id, set_session_tracer, set_turn_number
 from .hooks import GLOBAL_HOOKS, HookManager
-from .turso.manager import AsyncSQLTraceManager
+from .storage.base import TraceStorage
+from .storage.config import StorageConfig
+from .storage.factory import create_storage
 from .utils import generate_session_id
 
 
@@ -28,6 +33,8 @@ class SessionTracer:
         hooks: HookManager | None = None,
         db_url: str | None = None,
         auto_save: bool = True,
+        storage: TraceStorage | None = None,
+        storage_config: StorageConfig | None = None,
     ):
         """Initialize session tracer.
 
@@ -40,7 +47,8 @@ class SessionTracer:
         self._current_trace: SessionTrace | None = None
         self._lock = asyncio.Lock()
         self.db_url = db_url or CONFIG.db_url
-        self.db: AsyncSQLTraceManager | None = None
+        self._storage_config = storage_config
+        self.db: TraceStorage | None = storage
         self.auto_save = auto_save
         self._current_step: SessionTimeStep | None = None
 
@@ -57,7 +65,8 @@ class SessionTracer:
     async def initialize(self):
         """Initialize the database connection."""
         if self.db is None:
-            self.db = AsyncSQLTraceManager(self.db_url)
+            config = self._storage_config or StorageConfig(connection_string=self.db_url)
+            self.db = create_storage(config)
             await self.db.initialize()
 
     async def start_session(
@@ -97,7 +106,7 @@ class SessionTracer:
 
             self._current_trace = SessionTrace(
                 session_id=session_id,
-                created_at=datetime.utcnow(),
+                created_at=datetime.now(UTC),
                 session_time_steps=[],
                 event_history=[],
                 markov_blanket_message_history=[],
@@ -110,7 +119,9 @@ class SessionTracer:
 
             # Ensure session row exists for incremental writes
             if self.db:
-                await self.db.ensure_session(session_id, created_at=self._current_trace.created_at, metadata=metadata or {})
+                await self.db.ensure_session(
+                    session_id, created_at=self._current_trace.created_at, metadata=metadata or {}
+                )
 
             # Trigger hooks
             await self.hooks.trigger(
@@ -141,7 +152,7 @@ class SessionTracer:
         step = SessionTimeStep(
             step_id=step_id,
             step_index=len(self._current_trace.session_time_steps),
-            timestamp=datetime.utcnow(),
+            timestamp=datetime.now(UTC),
             turn_number=turn_number,
             step_metadata=metadata or {},
         )
@@ -186,7 +197,7 @@ class SessionTracer:
             step = self._current_step
 
         if step and step.completed_at is None:
-            step.completed_at = datetime.utcnow()
+            step.completed_at = datetime.now(UTC)
 
             # Trigger hooks
             await self.hooks.trigger(
@@ -259,7 +270,7 @@ class SessionTracer:
 
     async def record_message(
         self,
-        content: str,
+        content: Any,
         message_type: str,
         event_time: float | None = None,
         message_time: int | None = None,
@@ -277,11 +288,13 @@ class SessionTracer:
         if self._current_trace is None:
             raise RuntimeError("No active session")
 
+        normalised_content, content_str = self._normalise_message_content(content)
+
         msg = SessionEventMarkovBlanketMessage(
-            content=content,
+            content=normalised_content,
             message_type=message_type,
             time_record=TimeRecord(
-                event_time=event_time or datetime.utcnow().timestamp(), message_time=message_time
+                event_time=event_time or datetime.now(UTC).timestamp(), message_time=message_time
             ),
             metadata=metadata or {},
         )
@@ -315,7 +328,7 @@ class SessionTracer:
                 self._current_trace.session_id,
                 timestep_db_id=timestep_db_id,
                 message_type=message_type,
-                content=content,
+                content=content_str,
                 event_time=msg.time_record.event_time,
                 message_time=msg.time_record.message_time,
                 metadata=msg.metadata,
@@ -323,6 +336,22 @@ class SessionTracer:
             return message_id
         return None
 
+    @staticmethod
+    def _normalise_message_content(content: Any) -> tuple[SessionMessageContent, str]:
+        if isinstance(content, SessionMessageContent):
+            return content, content.as_text()
+        if isinstance(content, str):
+            payload = SessionMessageContent(text=content)
+            return payload, payload.as_text()
+        try:
+            serialized = json.dumps(content, ensure_ascii=False)
+            payload = SessionMessageContent(json_payload=serialized)
+            return payload, serialized
+        except (TypeError, ValueError):
+            text = str(content)
+            payload = SessionMessageContent(text=text)
+            return payload, text
+
     async def end_session(self, save: bool | None = None) -> SessionTrace:
         """End the current session.
 
@@ -339,7 +368,7 @@ class SessionTracer:
             # End any open timesteps
             for step in self._current_trace.session_time_steps:
                 if step.completed_at is None:
-                    step.completed_at = datetime.utcnow()
+                    step.completed_at = datetime.now(UTC)
 
             # Trigger pre-save hooks
             await self.hooks.trigger("before_save", session=self._current_trace)
@@ -435,7 +464,14 @@ class SessionTracer:
     # Reward recording helpers
     # -------------------------------
 
-    async def record_outcome_reward(self, *, total_reward: int, achievements_count: int, total_steps: int, reward_metadata: dict[str, Any] | None = None) -> int | None:
+    async def record_outcome_reward(
+        self,
+        *,
+        total_reward: int,
+        achievements_count: int,
+        total_steps: int,
+        reward_metadata: dict[str, Any] | None = None,
+    ) -> int | None:
         """Record an episode-level outcome reward for the current session."""
         if self._current_trace is None:
             raise RuntimeError("No active session")
@@ -462,7 +498,18 @@ class SessionTracer:
 
     # StepMetrics removed in favor of event_rewards; use record_event_reward for per-turn shaped values
 
-    async def record_event_reward(self, *, event_id: int, message_id: int | None = None, turn_number: int | None = None, reward_value: float = 0.0, reward_type: str | None = None, key: str | None = None, annotation: dict[str, Any] | None = None, source: str | None = None) -> int | None:
+    async def record_event_reward(
+        self,
+        *,
+        event_id: int,
+        message_id: int | None = None,
+        turn_number: int | None = None,
+        reward_value: float = 0.0,
+        reward_type: str | None = None,
+        key: str | None = None,
+        annotation: dict[str, Any] | None = None,
+        source: str | None = None,
+    ) -> int | None:
         """Record a first-class event-level reward with optional annotations."""
         if self._current_trace is None:
             raise RuntimeError("No active session")

synth_ai/tracing_v3/storage/base.py

@@ -54,7 +54,10 @@ class TraceStorage(ABC):
 
     @abstractmethod
     async def get_model_usage(
-        self, start_date: datetime | None = None, end_date: datetime | None = None, model_name: str | None = None
+        self,
+        start_date: datetime | None = None,
+        end_date: datetime | None = None,
+        model_name: str | None = None,
     ) -> Any:
         """Get model usage statistics.
 
@@ -85,6 +88,91 @@ class TraceStorage(ABC):
         """Close the storage connection."""
         pass
 
+    # Incremental helpers -------------------------------------------------
+
+    @abstractmethod
+    async def ensure_session(
+        self,
+        session_id: str,
+        *,
+        created_at: datetime | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Ensure a session row exists for the given session id."""
+        pass
+
+    @abstractmethod
+    async def ensure_timestep(
+        self,
+        session_id: str,
+        *,
+        step_id: str,
+        step_index: int,
+        turn_number: int | None = None,
+        started_at: datetime | None = None,
+        completed_at: datetime | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> int:
+        """Ensure a timestep row exists and return its database id."""
+        pass
+
+    @abstractmethod
+    async def insert_event_row(
+        self,
+        session_id: str,
+        *,
+        timestep_db_id: int | None,
+        event: Any,
+        metadata_override: dict[str, Any] | None = None,
+    ) -> int:
+        """Insert an event and return its database id."""
+        pass
+
+    @abstractmethod
+    async def insert_message_row(
+        self,
+        session_id: str,
+        *,
+        timestep_db_id: int | None,
+        message_type: str,
+        content: Any,
+        event_time: float | None = None,
+        message_time: int | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> int:
+        """Insert a message row linked to a session/timestep."""
+        pass
+
+    @abstractmethod
+    async def insert_outcome_reward(
+        self,
+        session_id: str,
+        *,
+        total_reward: int,
+        achievements_count: int,
+        total_steps: int,
+        reward_metadata: dict | None = None,
+    ) -> int:
+        """Record an outcome reward for a session."""
+        pass
+
+    @abstractmethod
+    async def insert_event_reward(
+        self,
+        session_id: str,
+        *,
+        event_id: int,
+        message_id: int | None = None,
+        turn_number: int | None = None,
+        reward_value: float = 0.0,
+        reward_type: str | None = None,
+        key: str | None = None,
+        annotation: dict[str, Any] | None = None,
+        source: str | None = None,
+    ) -> int:
+        """Record a reward tied to a specific event."""
+        pass
+
     # Optional experiment management methods
     async def create_experiment(
         self,

synth_ai/tracing_v3/storage/config.py

@@ -9,16 +9,22 @@ from typing import Any
 class StorageBackend(str, Enum):
     """Supported storage backends."""
 
-    TURSO = "turso"
+    TURSO_NATIVE = "turso_native"
     SQLITE = "sqlite"
     POSTGRES = "postgres"  # Future support
 
 
+def _is_enabled(value: str | None) -> bool:
+    if value is None:
+        return False
+    return value.lower() in {"1", "true", "yes", "on"}
+
+
 @dataclass
 class StorageConfig:
     """Configuration for storage backend."""
 
-    backend: StorageBackend = StorageBackend.TURSO
+    backend: StorageBackend = StorageBackend.TURSO_NATIVE
     connection_string: str | None = None
 
     # Turso-specific settings
@@ -34,23 +40,30 @@ class StorageConfig:
     enable_compression: bool = os.getenv("STORAGE_COMPRESSION", "false").lower() == "true"
     max_content_length: int = int(os.getenv("STORAGE_MAX_CONTENT_LENGTH", "1000000"))  # 1MB
 
+    def __post_init__(self):
+        # Allow legacy override while keeping compatibility with existing TURSO_NATIVE env flag
+        native_env = os.getenv("TURSO_NATIVE")
+        native_flag = _is_enabled(native_env) if native_env is not None else None
+
+        if native_flag is False:
+            self.backend = StorageBackend.SQLITE
+
     def get_connection_string(self) -> str:
         """Get the appropriate connection string for the backend."""
         if self.connection_string:
             return self.connection_string
 
-        if self.backend == StorageBackend.TURSO:
+        if self.backend == StorageBackend.TURSO_NATIVE:
             return self.turso_url
-        elif self.backend == StorageBackend.SQLITE:
+        if self.backend == StorageBackend.SQLITE:
             return "sqlite+aiosqlite:///traces.db"
-        elif self.backend == StorageBackend.POSTGRES:
+        if self.backend == StorageBackend.POSTGRES:
             return os.getenv("POSTGRES_URL", "postgresql+asyncpg://localhost/traces")
-        else:
-            raise ValueError(f"Unknown backend: {self.backend}")
+        raise ValueError(f"Unknown backend: {self.backend}")
 
     def get_backend_config(self) -> dict[str, Any]:
         """Get backend-specific configuration."""
-        if self.backend == StorageBackend.TURSO:
+        if self.backend == StorageBackend.TURSO_NATIVE:
             config = {}
             if self.turso_auth_token:
                 config["auth_token"] = self.turso_auth_token