synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/core/tracing_v3/__init__.py

@@ -0,0 +1,99 @@
+"""Tracing v3 - Turso/sqld based tracing implementation.
+
+This module provides a modern, async-first tracing system for capturing and storing
+detailed execution traces from AI systems. It's designed to handle high-throughput
+scenarios with proper async/await patterns throughout.
+
+Architecture Overview:
+---------------------
+The v3 tracing system is built on several key components:
+
+1. **Data Abstractions** (`abstractions.py`):
+   - Dataclass-based models for traces, events, and messages
+   - Type-safe representations of session data
+   - Support for multiple event types (Runtime, Environment, LM/CAIS)
+
+2. **Session Tracer** (`session_tracer.py`):
+   - Main interface for creating and managing trace sessions
+   - Async context managers for session and timestep management
+   - Automatic event and message recording with proper ordering
+
+3. **Async Storage** (`turso/manager.py`):
+   - SQLAlchemy async engine with Turso/sqld backend
+   - Batch insert capabilities for high-throughput scenarios
+   - Analytics views for querying trace data
+
+4. **Decorators** (`decorators.py`):
+   - Context-aware decorators using asyncio's ContextVar
+   - Automatic LLM call tracing with token/cost tracking
+   - Session and turn number propagation across async boundaries
+
+5. **Hook System** (`hooks.py`):
+   - Extensible hook points throughout the tracing lifecycle
+   - Support for both sync and async hook callbacks
+   - Pre/post processing of events and messages
+
+6. **Replica Sync** (`replica_sync.py`):
+   - Optional background sync with remote Turso database
+   - Local embedded SQLite for low-latency writes
+   - Configurable sync intervals
+
+Key Features:
+------------
+- **Async-First**: All database operations are async, preventing blocking
+- **Context Propagation**: Session/turn info flows through async call chains
+- **Type Safety**: Full typing support with dataclasses and type hints
+- **Extensibility**: Hook system allows custom processing logic
+- **Performance**: Batch operations and connection pooling for efficiency
+
+Usage Example:
+-------------
+    from synth_ai.core.tracing_v3 import SessionTracer
+
+    tracer = SessionTracer()
+    await tracer.initialize()
+
+    async with tracer.session() as session_id:
+        async with tracer.timestep("step1", turn_number=1):
+            # Record events during execution
+            await tracer.record_event(RuntimeEvent(...))
+            await tracer.record_message("User input", "user")
+
+Configuration:
+-------------
+The system uses environment variables for configuration:
+- TURSO_LOCAL_DB_URL: Local SQLite database URL
+- TURSO_POOL_SIZE: Connection pool size (default: 8)
+- TURSO_ECHO_SQL: Enable SQL logging (default: false)
+- SQLD_DB_PATH: Path to SQLite database file
+
+See `config.py` for full configuration options.
+"""
+
+from .abstractions import (
+    BaseEvent,
+    EnvironmentEvent,
+    RuntimeEvent,
+    SessionEventMarkovBlanketMessage,
+    SessionMessageContent,
+    SessionTimeStep,
+    SessionTrace,
+    TimeRecord,
+)
+from .config import TursoConfig
+from .llm_call_record_helpers import BaseLMResponse
+from .session_tracer import SessionTracer
+
+__all__ = [
+    "SessionTracer",
+    "SessionTrace",
+    "SessionTimeStep",
+    "BaseEvent",
+    "RuntimeEvent",
+    "EnvironmentEvent",
+    "SessionEventMarkovBlanketMessage",
+    "SessionMessageContent",
+    "TimeRecord",
+    "TursoConfig",
+    "BaseLMResponse",
+]

synth_ai/core/tracing_v3/config.py

@@ -0,0 +1,229 @@
+"""Configuration helpers for tracing v3.
+
+This module centralises the logic for discovering which datastore the tracer
+should use. Historically the project defaulted to a local SQLite file which
+breaks under parallel load. The new resolver inspects environment variables
+and defaults to Turso/libSQL whenever credentials are supplied, while keeping a
+SQLite fallback for contributors without remote access.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+from synth_ai.core.tracing_v3.constants import canonical_trace_db_path
+
+# STARTUP DIAGNOSTIC - Commented out to reduce noise
+# print(f"[TRACING_V3_CONFIG_LOADED] Python={sys.version_info.major}.{sys.version_info.minor} MODAL_IS_REMOTE={os.getenv('MODAL_IS_REMOTE')}", flush=True)
+
+# ---------------------------------------------------------------------------
+# DSN resolution helpers
+# ---------------------------------------------------------------------------
+
+_CANONICAL_DB_PATH = canonical_trace_db_path()
+_DEFAULT_TRACE_DIR = Path(os.getenv("SYNTH_TRACES_DIR", _CANONICAL_DB_PATH.parent))
+
+
+def _normalise_path(path: Path) -> Path:
+    """Resolve relative paths and expand user/home markers."""
+    path = path.expanduser()
+    if not path.is_absolute():
+        path = (Path.cwd() / path).resolve()
+    return path
+
+
+def _is_modal_environment() -> bool:
+    """Detect if running in Modal container.
+    
+    Modal automatically sets MODAL_IS_REMOTE=1 in all deployed containers.
+    We check this first, then fall back to other Modal env vars.
+    """
+    # Modal sets this in all deployed containers
+    if os.getenv("MODAL_IS_REMOTE") == "1":
+        return True
+    
+    # Additional Modal env vars as fallback
+    return bool(
+        os.getenv("MODAL_TASK_ID")
+        or os.getenv("MODAL_ENVIRONMENT")
+        or os.getenv("SERVICE", "").upper() == "MODAL"
+    )
+
+
+def _split_auth_from_url(url: str) -> tuple[str, str | None]:
+    """Strip any auth_token query parameter from a DSN."""
+    parsed = urlparse(url)
+    if not parsed.query:
+        return url, None
+
+    params = dict(parse_qsl(parsed.query, keep_blank_values=True))
+    token = params.pop("auth_token", None)
+    query = urlencode(params, doseq=True)
+    # urlunparse will omit the '?' automatically when query is empty
+    sanitised = urlunparse(parsed._replace(query=query))
+    return sanitised, token
+
+
+def _default_sqlite_url(*, ensure_dir: bool = True) -> tuple[str, str | None]:
+    """Generate a SQLite URL from SYNTH_TRACES_DIR if set, otherwise raise."""
+    traces_dir = os.getenv("SYNTH_TRACES_DIR")
+    if traces_dir:
+        dir_path = _normalise_path(Path(traces_dir))
+        if ensure_dir:
+            dir_path.mkdir(parents=True, exist_ok=True)
+        db_path = dir_path / "synth_traces.db"
+        sqlite_url = f"sqlite+aiosqlite:///{db_path}"
+        return sqlite_url, None
+    raise RuntimeError("SQLite fallback is disabled; configure LIBSQL_URL or run sqld locally.")
+
+
+def resolve_trace_db_settings(*, ensure_dir: bool = True) -> tuple[str, str | None]:
+    """Resolve the tracing database URL and optional auth token.
+
+    Resolution order:
+      1. `SYNTH_TRACES_DB` (explicit DSN override)
+      2. `LIBSQL_URL` / `TURSO_DATABASE_URL` (remote libSQL endpoints)
+      3. `TURSO_LOCAL_DB_URL` (legacy env for local sqld)
+      4. Modal environment: plain SQLite file (no sqld, no auth)
+      5. Local dev: sqld default
+    """
+    import logging
+    logger = logging.getLogger(__name__)
+
+    explicit = os.getenv("SYNTH_TRACES_DB")
+    if explicit:
+        logger.info(f"[TRACE_CONFIG] Using explicit SYNTH_TRACES_DB: {explicit}")
+        return _split_auth_from_url(explicit)
+
+    remote = os.getenv("LIBSQL_URL") or os.getenv("TURSO_DATABASE_URL")
+    if remote:
+        logger.info(f"[TRACE_CONFIG] Using remote Turso: {remote}")
+        url, token = _split_auth_from_url(remote)
+        if token:
+            return url, token
+        env_token = os.getenv("LIBSQL_AUTH_TOKEN") or os.getenv("TURSO_AUTH_TOKEN")
+        return url, env_token
+
+    local_override = os.getenv("TURSO_LOCAL_DB_URL")
+    if local_override:
+        logger.info(f"[TRACE_CONFIG] Using TURSO_LOCAL_DB_URL: {local_override}")
+        url, token = _split_auth_from_url(local_override)
+        if token:
+            return url, token
+        env_token = os.getenv("LIBSQL_AUTH_TOKEN") or os.getenv("TURSO_AUTH_TOKEN")
+        return url, env_token
+
+    # Check for SYNTH_TRACES_DIR to generate SQLite URL
+    traces_dir = os.getenv("SYNTH_TRACES_DIR")
+    if traces_dir:
+        try:
+            sqlite_url, _ = _default_sqlite_url(ensure_dir=ensure_dir)
+            logger.info(f"[TRACE_CONFIG] Using SQLite from SYNTH_TRACES_DIR: {sqlite_url}")
+            return sqlite_url, None
+        except RuntimeError:
+            pass  # Fall through to other options
+
+    # Modal environment: use plain SQLite file (no sqld daemon, no auth required)
+    is_modal = _is_modal_environment()
+    logger.info(f"[TRACE_CONFIG] Modal detection: {is_modal} (MODAL_IS_REMOTE={os.getenv('MODAL_IS_REMOTE')})")
+    if is_modal:
+        logger.info("[TRACE_CONFIG] Using Modal SQLite: file:/tmp/synth_traces.db")
+        return "file:/tmp/synth_traces.db", None
+
+    # Local dev: default to sqld HTTP API
+    default_url = os.getenv("LIBSQL_DEFAULT_URL", "http://127.0.0.1:8081")
+    logger.info(f"[TRACE_CONFIG] Using local sqld: {default_url}")
+    return default_url, None
+
+
+def resolve_trace_db_url(*, ensure_dir: bool = True) -> str:
+    """Return just the DSN, discarding any auth token."""
+    url, _ = resolve_trace_db_settings(ensure_dir=ensure_dir)
+    return url
+
+
+def resolve_trace_db_auth_token() -> str | None:
+    """Return the resolved auth token for the tracing datastore."""
+    _, token = resolve_trace_db_settings()
+    return token
+
+
+# ---------------------------------------------------------------------------
+# Config dataclasses
+# ---------------------------------------------------------------------------
+
+DEFAULT_DB_FILE = str(_normalise_path(_DEFAULT_TRACE_DIR) / _CANONICAL_DB_PATH.name)
+
+
+@dataclass
+class TursoConfig:
+    """Configuration for Turso/sqld connection."""
+
+    # Default values matching serve.sh
+    DEFAULT_DB_FILE = DEFAULT_DB_FILE
+    DEFAULT_HTTP_PORT = 8080
+
+    # Resolve DB URL and auth token from environment (libSQL preferred)
+    db_url: str = field(default_factory=resolve_trace_db_url)
+
+    # Remote database sync configuration
+    sync_url: str = os.getenv("LIBSQL_SYNC_URL") or os.getenv("TURSO_SYNC_URL", "")
+    auth_token: str = resolve_trace_db_auth_token() or ""
+    sync_interval: int = int(
+        os.getenv("TURSO_SYNC_SECONDS", "2")
+    )  # 2 seconds for responsive local development
+
+    # Connection pool settings
+    pool_size: int = int(os.getenv("TURSO_POOL_SIZE", "8"))
+    max_overflow: int = int(os.getenv("TURSO_MAX_OVERFLOW", "16"))
+    pool_timeout: float = float(os.getenv("TURSO_POOL_TIMEOUT", "30.0"))
+    pool_recycle: int = int(os.getenv("TURSO_POOL_RECYCLE", "3600"))
+
+    # SQLite settings
+    foreign_keys: bool = os.getenv("TURSO_FOREIGN_KEYS", "true").lower() == "true"
+    journal_mode: str = os.getenv("TURSO_JOURNAL_MODE", "WAL")
+
+    # Performance settings
+    echo_sql: bool = os.getenv("TURSO_ECHO_SQL", "false").lower() == "true"
+    batch_size: int = int(os.getenv("TURSO_BATCH_SIZE", "1000"))
+
+    # Daemon settings (for local sqld) - match serve.sh defaults
+    sqld_binary: str = os.getenv("SQLD_BINARY", "sqld")
+    sqld_db_path: str = os.getenv("SQLD_DB_PATH", DEFAULT_DB_FILE)
+    sqld_http_port: int = int(os.getenv("SQLD_HTTP_PORT", "8080"))
+    sqld_idle_shutdown: int = int(os.getenv("SQLD_IDLE_SHUTDOWN", "0"))  # 0 = no idle shutdown
+
+    def get_connect_args(self) -> dict[str, str]:
+        """Get SQLAlchemy connection arguments."""
+        args: dict[str, str] = {}
+        if self.auth_token:
+            args["auth_token"] = self.auth_token
+        return args
+
+    def get_engine_kwargs(self) -> dict[str, Any]:
+        """Get SQLAlchemy engine creation kwargs."""
+        kwargs: dict[str, Any] = {
+            "echo": self.echo_sql,
+            "future": True,
+        }
+
+        # Only add pool settings for non-SQLite URLs
+        if not (self.db_url.startswith("sqlite") or ":memory:" in self.db_url):
+            kwargs.update(
+                {
+                    "pool_size": self.pool_size,
+                    "max_overflow": self.max_overflow,
+                    "pool_timeout": self.pool_timeout,
+                    "pool_recycle": self.pool_recycle,
+                }
+            )
+
+        return kwargs
+
+
+# Global config instance
+CONFIG = TursoConfig()

synth_ai/core/tracing_v3/constants.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+TRACE_DB_DIR = Path("traces")
+TRACE_DB_BASENAME = "turso_task_app_traces"
+
+
+def canonical_trace_db_name(*, timestamp: datetime | None = None) -> str:
+    """Return the canonical trace database filename (with optional timestamp suffix)."""
+
+    if timestamp is None:
+        return f"{TRACE_DB_BASENAME}.db"
+    return f"{TRACE_DB_BASENAME}_{timestamp.strftime('%Y-%m-%d_%H-%M-%S')}.db"
+
+
+def canonical_trace_db_path(*, timestamp: datetime | None = None) -> Path:
+    """Return the canonical trace database path within the default trace directory."""
+
+    return TRACE_DB_DIR / canonical_trace_db_name(timestamp=timestamp)

synth_ai/core/tracing_v3/db_config.py

@@ -0,0 +1,182 @@
+"""
+Centralized database configuration for v3 tracing.
+"""
+
+import logging
+import os
+import shutil
+from typing import TYPE_CHECKING, Optional
+
+from synth_ai.core.tracing_v3.constants import canonical_trace_db_path
+
+if TYPE_CHECKING:
+    from .turso.daemon import SqldDaemon
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseConfig:
+    """Centralized database configuration management."""
+
+    # Default values from serve.sh
+    DEFAULT_DB_FILE = str(canonical_trace_db_path())
+    DEFAULT_HTTP_PORT = 8080
+
+    def __init__(
+        self, db_path: str | None = None, http_port: int | None = None, use_sqld: bool = True
+    ):
+        """
+        Initialize database configuration.
+
+        Args:
+            db_path: Path to database file. If None, uses DEFAULT_DB_FILE from serve.sh.
+            http_port: Hrana WebSocket port for sqld daemon (env: SQLD_HTTP_PORT). If None, uses DEFAULT_HTTP_PORT.
+            use_sqld: Whether to use sqld daemon or direct SQLite.
+        """
+        self.use_sqld = use_sqld and self._sqld_binary_available()
+        # Note: SQLD_HTTP_PORT is actually the hrana port (8080), not the HTTP API port
+        self.hrana_port = http_port or int(os.getenv("SQLD_HTTP_PORT", self.DEFAULT_HTTP_PORT))
+        self._daemon: SqldDaemon | None = None
+
+        # Set up database path to match serve.sh configuration
+        if db_path is None:
+            # Use the same database file as serve.sh
+            self.db_file = os.getenv("SQLD_DB_PATH", self.DEFAULT_DB_FILE)
+            # For sqld, db_base_path is just the filename without directory
+            self.db_base_path = self.db_file
+        else:
+            self.db_file = db_path
+            self.db_base_path = db_path
+
+    @property
+    def database_url(self) -> str:
+        """Get the SQLAlchemy database URL."""
+        # Always use direct file access with aiosqlite
+        # (sqld HTTP interface is not compatible with SQLAlchemy)
+
+        # Check if sqld is running - it creates a directory structure
+        abs_path = os.path.abspath(self.db_file)
+        sqld_data_path = os.path.join(abs_path, "dbs", "default", "data")
+
+        if not os.path.exists(sqld_data_path) and not os.path.exists(abs_path):
+            raise RuntimeError(
+                "sqld data directory not found. Run `sqld --db-path <path>` before using the tracing database."
+            )
+
+        # Use http:// for local sqld HTTP API port
+        # sqld has two ports: hrana_port (Hrana WebSocket) and hrana_port+1 (HTTP API)
+        # Python libsql client uses HTTP API with http:// URLs
+        http_api_port = self.hrana_port + 1
+        return f"http://127.0.0.1:{http_api_port}"
+
+    def _sqld_binary_available(self) -> bool:
+        """Check if the sqld (Turso) binary is available on PATH."""
+        # Respect explicit SQLD_BINARY override when present
+        binary_override = os.getenv("SQLD_BINARY")
+        candidates = [binary_override, "sqld", "libsql-server"]
+
+        for candidate in candidates:
+            if candidate and shutil.which(candidate):
+                return True
+
+        if binary_override:
+            raise RuntimeError(
+                f"Configured SQLD_BINARY='{binary_override}' but the executable was not found on PATH."
+            )
+        raise RuntimeError(
+            "sqld binary not detected; install Turso's sqld or set SQLD_BINARY so that libSQL can be used."
+        )
+
+    def start_daemon(self, wait_time: float = 2.0):
+        """
+        Start the sqld daemon if configured.
+
+        Args:
+            wait_time: Time to wait for daemon startup.
+
+        Returns:
+            The daemon instance.
+        """
+        if not self.use_sqld:
+            raise RuntimeError("Database not configured to use sqld daemon")
+
+        if self._daemon is None:
+            # Import here to avoid circular dependency
+            from .turso.daemon import SqldDaemon
+
+            self._daemon = SqldDaemon(db_path=self.db_base_path, hrana_port=self.hrana_port)
+
+        self._daemon.start()
+
+        # Wait for daemon to be ready
+        import time
+
+        time.sleep(wait_time)
+
+        return self._daemon
+
+    def stop_daemon(self):
+        """Stop the sqld daemon if running."""
+        if self._daemon:
+            self._daemon.stop()
+            self._daemon = None
+
+    def get_daemon_and_url(self, wait_time: float = 2.0) -> tuple[Optional["SqldDaemon"], str]:
+        """
+        Get daemon (starting if needed) and database URL.
+
+        Returns:
+            Tuple of (daemon or None, database_url)
+        """
+        daemon = None
+        if self.use_sqld:
+            daemon = self.start_daemon(wait_time)
+
+        return daemon, self.database_url
+
+
+# Global default configuration
+_default_config: DatabaseConfig | None = None
+
+
+def get_default_db_config() -> DatabaseConfig:
+    """Get the default database configuration."""
+    global _default_config
+    if _default_config is None:
+        # Check environment variable for database path
+        db_path = os.environ.get("SYNTH_AI_V3_DB_PATH")
+        use_sqld = os.environ.get("SYNTH_AI_V3_USE_SQLD", "true").lower() == "true"
+
+        # Check if sqld is already running (started by serve.sh)
+        import subprocess
+
+        sqld_hrana_port = int(os.getenv("SQLD_HTTP_PORT", DatabaseConfig.DEFAULT_HTTP_PORT))
+        sqld_http_port = sqld_hrana_port + 1
+        sqld_running = False
+        try:
+            # Check for either hrana or http port in the process command line
+            result = subprocess.run(
+                ["pgrep", "-f", f"sqld.*(--hrana-listen-addr.*:{sqld_hrana_port}|--http-listen-addr.*:{sqld_http_port})"],
+                capture_output=True,
+                text=True,
+            )
+            if result.returncode == 0:
+                # sqld is already running, don't start a new one
+                sqld_running = True
+                use_sqld = False
+                logger.debug(f"✅ Detected sqld already running on ports {sqld_hrana_port} (hrana) and {sqld_http_port} (http)")
+        except Exception as e:
+            logger.debug(f"Could not check for sqld process: {e}")
+
+        if not sqld_running and use_sqld:
+            logger.warning("sqld service not detected. Start the Turso daemon (./serve.sh) before running tracing workloads.")
+
+        _default_config = DatabaseConfig(db_path=db_path, use_sqld=use_sqld)
+
+    return _default_config
+
+
+def set_default_db_config(config: DatabaseConfig):
+    """Set the default database configuration."""
+    global _default_config
+    _default_config = config