synth-ai 0.2.6.dev1__py3-none-any.whl → 0.4.3__py3-none-any.whl

synth_ai/{tracing_v3 → core/tracing_v3}/__init__.py

@@ -48,7 +48,7 @@ Key Features:
 
 Usage Example:
 -------------
-    from synth_ai.tracing_v3 import SessionTracer
+    from synth_ai.core.tracing_v3 import SessionTracer
 
     tracer = SessionTracer()
     await tracer.initialize()
@@ -75,11 +75,13 @@ from .abstractions import (
     EnvironmentEvent,
     RuntimeEvent,
     SessionEventMarkovBlanketMessage,
+    SessionMessageContent,
     SessionTimeStep,
     SessionTrace,
     TimeRecord,
 )
 from .config import TursoConfig
+from .llm_call_record_helpers import BaseLMResponse
 from .session_tracer import SessionTracer
 
 __all__ = [
@@ -90,6 +92,8 @@ __all__ = [
     "RuntimeEvent",
     "EnvironmentEvent",
     "SessionEventMarkovBlanketMessage",
+    "SessionMessageContent",
     "TimeRecord",
     "TursoConfig",
+    "BaseLMResponse",
 ]

synth_ai/{tracing_v3 → core/tracing_v3}/abstractions.py

@@ -37,7 +37,7 @@ Concepts:
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass, field
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import Any
 
 from .lm_call_record_abstractions import LLMCallRecord
@@ -61,6 +61,23 @@ class TimeRecord:
     message_time: int | None = None
 
 
+@dataclass(frozen=True)
+class SessionMessageContent:
+    """Normalized payload stored alongside session messages."""
+
+    text: str | None = None
+    json_payload: str | None = None
+
+    def as_text(self) -> str:
+        return self.text or (self.json_payload or "")
+
+    def has_json(self) -> bool:
+        return self.json_payload is not None
+
+    def __str__(self) -> str:  # pragma: no cover - convenience for logging
+        return self.as_text()
+
+
 @dataclass
 class SessionEventMarkovBlanketMessage:
     """Message crossing Markov blanket boundaries between systems in a session.
@@ -97,7 +114,7 @@ class SessionEventMarkovBlanketMessage:
                   - 'causal_influence': Direction of causal flow
     """
 
-    content: str
+    content: SessionMessageContent
     message_type: str
     time_record: TimeRecord
     metadata: dict[str, Any] = field(default_factory=dict)
@@ -232,7 +249,7 @@ class SessionTimeStep:
 
     step_id: str = ""
     step_index: int = 0
-    timestamp: datetime = field(default_factory=datetime.utcnow)
+    timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
     turn_number: int | None = None
     events: list[BaseEvent] = field(default_factory=list)
     markov_blanket_messages: list[SessionEventMarkovBlanketMessage] = field(default_factory=list)
@@ -266,7 +283,7 @@ class SessionTrace:
     """
 
     session_id: str = ""
-    created_at: datetime = field(default_factory=datetime.utcnow)
+    created_at: datetime = field(default_factory=lambda: datetime.now(UTC))
     session_time_steps: list[SessionTimeStep] = field(default_factory=list)
     event_history: list[BaseEvent] = field(default_factory=list)
     markov_blanket_message_history: list[SessionEventMarkovBlanketMessage] = field(

synth_ai/core/tracing_v3/config.py

@@ -0,0 +1,229 @@
+"""Configuration helpers for tracing v3.
+
+This module centralises the logic for discovering which datastore the tracer
+should use. Historically the project defaulted to a local SQLite file which
+breaks under parallel load. The new resolver inspects environment variables
+and defaults to Turso/libSQL whenever credentials are supplied, while keeping a
+SQLite fallback for contributors without remote access.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+from synth_ai.core.tracing_v3.constants import canonical_trace_db_path
+
+# STARTUP DIAGNOSTIC - Commented out to reduce noise
+# print(f"[TRACING_V3_CONFIG_LOADED] Python={sys.version_info.major}.{sys.version_info.minor} MODAL_IS_REMOTE={os.getenv('MODAL_IS_REMOTE')}", flush=True)
+
+# ---------------------------------------------------------------------------
+# DSN resolution helpers
+# ---------------------------------------------------------------------------
+
+_CANONICAL_DB_PATH = canonical_trace_db_path()
+_DEFAULT_TRACE_DIR = Path(os.getenv("SYNTH_TRACES_DIR", _CANONICAL_DB_PATH.parent))
+
+
+def _normalise_path(path: Path) -> Path:
+    """Resolve relative paths and expand user/home markers."""
+    path = path.expanduser()
+    if not path.is_absolute():
+        path = (Path.cwd() / path).resolve()
+    return path
+
+
+def _is_modal_environment() -> bool:
+    """Detect if running in Modal container.
+    
+    Modal automatically sets MODAL_IS_REMOTE=1 in all deployed containers.
+    We check this first, then fall back to other Modal env vars.
+    """
+    # Modal sets this in all deployed containers
+    if os.getenv("MODAL_IS_REMOTE") == "1":
+        return True
+    
+    # Additional Modal env vars as fallback
+    return bool(
+        os.getenv("MODAL_TASK_ID")
+        or os.getenv("MODAL_ENVIRONMENT")
+        or os.getenv("SERVICE", "").upper() == "MODAL"
+    )
+
+
+def _split_auth_from_url(url: str) -> tuple[str, str | None]:
+    """Strip any auth_token query parameter from a DSN."""
+    parsed = urlparse(url)
+    if not parsed.query:
+        return url, None
+
+    params = dict(parse_qsl(parsed.query, keep_blank_values=True))
+    token = params.pop("auth_token", None)
+    query = urlencode(params, doseq=True)
+    # urlunparse will omit the '?' automatically when query is empty
+    sanitised = urlunparse(parsed._replace(query=query))
+    return sanitised, token
+
+
+def _default_sqlite_url(*, ensure_dir: bool = True) -> tuple[str, str | None]:
+    """Generate a SQLite URL from SYNTH_TRACES_DIR if set, otherwise raise."""
+    traces_dir = os.getenv("SYNTH_TRACES_DIR")
+    if traces_dir:
+        dir_path = _normalise_path(Path(traces_dir))
+        if ensure_dir:
+            dir_path.mkdir(parents=True, exist_ok=True)
+        db_path = dir_path / "synth_traces.db"
+        sqlite_url = f"sqlite+aiosqlite:///{db_path}"
+        return sqlite_url, None
+    raise RuntimeError("SQLite fallback is disabled; configure LIBSQL_URL or run sqld locally.")
+
+
+def resolve_trace_db_settings(*, ensure_dir: bool = True) -> tuple[str, str | None]:
+    """Resolve the tracing database URL and optional auth token.
+
+    Resolution order:
+      1. `SYNTH_TRACES_DB` (explicit DSN override)
+      2. `LIBSQL_URL` / `TURSO_DATABASE_URL` (remote libSQL endpoints)
+      3. `TURSO_LOCAL_DB_URL` (legacy env for local sqld)
+      4. Modal environment: plain SQLite file (no sqld, no auth)
+      5. Local dev: sqld default
+    """
+    import logging
+    logger = logging.getLogger(__name__)
+
+    explicit = os.getenv("SYNTH_TRACES_DB")
+    if explicit:
+        logger.info(f"[TRACE_CONFIG] Using explicit SYNTH_TRACES_DB: {explicit}")
+        return _split_auth_from_url(explicit)
+
+    remote = os.getenv("LIBSQL_URL") or os.getenv("TURSO_DATABASE_URL")
+    if remote:
+        logger.info(f"[TRACE_CONFIG] Using remote Turso: {remote}")
+        url, token = _split_auth_from_url(remote)
+        if token:
+            return url, token
+        env_token = os.getenv("LIBSQL_AUTH_TOKEN") or os.getenv("TURSO_AUTH_TOKEN")
+        return url, env_token
+
+    local_override = os.getenv("TURSO_LOCAL_DB_URL")
+    if local_override:
+        logger.info(f"[TRACE_CONFIG] Using TURSO_LOCAL_DB_URL: {local_override}")
+        url, token = _split_auth_from_url(local_override)
+        if token:
+            return url, token
+        env_token = os.getenv("LIBSQL_AUTH_TOKEN") or os.getenv("TURSO_AUTH_TOKEN")
+        return url, env_token
+
+    # Check for SYNTH_TRACES_DIR to generate SQLite URL
+    traces_dir = os.getenv("SYNTH_TRACES_DIR")
+    if traces_dir:
+        try:
+            sqlite_url, _ = _default_sqlite_url(ensure_dir=ensure_dir)
+            logger.info(f"[TRACE_CONFIG] Using SQLite from SYNTH_TRACES_DIR: {sqlite_url}")
+            return sqlite_url, None
+        except RuntimeError:
+            pass  # Fall through to other options
+
+    # Modal environment: use plain SQLite file (no sqld daemon, no auth required)
+    is_modal = _is_modal_environment()
+    logger.info(f"[TRACE_CONFIG] Modal detection: {is_modal} (MODAL_IS_REMOTE={os.getenv('MODAL_IS_REMOTE')})")
+    if is_modal:
+        logger.info("[TRACE_CONFIG] Using Modal SQLite: file:/tmp/synth_traces.db")
+        return "file:/tmp/synth_traces.db", None
+
+    # Local dev: default to sqld HTTP API
+    default_url = os.getenv("LIBSQL_DEFAULT_URL", "http://127.0.0.1:8081")
+    logger.info(f"[TRACE_CONFIG] Using local sqld: {default_url}")
+    return default_url, None
+
+
+def resolve_trace_db_url(*, ensure_dir: bool = True) -> str:
+    """Return just the DSN, discarding any auth token."""
+    url, _ = resolve_trace_db_settings(ensure_dir=ensure_dir)
+    return url
+
+
+def resolve_trace_db_auth_token() -> str | None:
+    """Return the resolved auth token for the tracing datastore."""
+    _, token = resolve_trace_db_settings()
+    return token
+
+
+# ---------------------------------------------------------------------------
+# Config dataclasses
+# ---------------------------------------------------------------------------
+
+DEFAULT_DB_FILE = str(_normalise_path(_DEFAULT_TRACE_DIR) / _CANONICAL_DB_PATH.name)
+
+
+@dataclass
+class TursoConfig:
+    """Configuration for Turso/sqld connection."""
+
+    # Default values matching serve.sh
+    DEFAULT_DB_FILE = DEFAULT_DB_FILE
+    DEFAULT_HTTP_PORT = 8080
+
+    # Resolve DB URL and auth token from environment (libSQL preferred)
+    db_url: str = field(default_factory=resolve_trace_db_url)
+
+    # Remote database sync configuration
+    sync_url: str = os.getenv("LIBSQL_SYNC_URL") or os.getenv("TURSO_SYNC_URL", "")
+    auth_token: str = resolve_trace_db_auth_token() or ""
+    sync_interval: int = int(
+        os.getenv("TURSO_SYNC_SECONDS", "2")
+    )  # 2 seconds for responsive local development
+
+    # Connection pool settings
+    pool_size: int = int(os.getenv("TURSO_POOL_SIZE", "8"))
+    max_overflow: int = int(os.getenv("TURSO_MAX_OVERFLOW", "16"))
+    pool_timeout: float = float(os.getenv("TURSO_POOL_TIMEOUT", "30.0"))
+    pool_recycle: int = int(os.getenv("TURSO_POOL_RECYCLE", "3600"))
+
+    # SQLite settings
+    foreign_keys: bool = os.getenv("TURSO_FOREIGN_KEYS", "true").lower() == "true"
+    journal_mode: str = os.getenv("TURSO_JOURNAL_MODE", "WAL")
+
+    # Performance settings
+    echo_sql: bool = os.getenv("TURSO_ECHO_SQL", "false").lower() == "true"
+    batch_size: int = int(os.getenv("TURSO_BATCH_SIZE", "1000"))
+
+    # Daemon settings (for local sqld) - match serve.sh defaults
+    sqld_binary: str = os.getenv("SQLD_BINARY", "sqld")
+    sqld_db_path: str = os.getenv("SQLD_DB_PATH", DEFAULT_DB_FILE)
+    sqld_http_port: int = int(os.getenv("SQLD_HTTP_PORT", "8080"))
+    sqld_idle_shutdown: int = int(os.getenv("SQLD_IDLE_SHUTDOWN", "0"))  # 0 = no idle shutdown
+
+    def get_connect_args(self) -> dict[str, str]:
+        """Get SQLAlchemy connection arguments."""
+        args: dict[str, str] = {}
+        if self.auth_token:
+            args["auth_token"] = self.auth_token
+        return args
+
+    def get_engine_kwargs(self) -> dict[str, Any]:
+        """Get SQLAlchemy engine creation kwargs."""
+        kwargs: dict[str, Any] = {
+            "echo": self.echo_sql,
+            "future": True,
+        }
+
+        # Only add pool settings for non-SQLite URLs
+        if not (self.db_url.startswith("sqlite") or ":memory:" in self.db_url):
+            kwargs.update(
+                {
+                    "pool_size": self.pool_size,
+                    "max_overflow": self.max_overflow,
+                    "pool_timeout": self.pool_timeout,
+                    "pool_recycle": self.pool_recycle,
+                }
+            )
+
+        return kwargs
+
+
+# Global config instance
+CONFIG = TursoConfig()

synth_ai/core/tracing_v3/constants.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+TRACE_DB_DIR = Path("traces")
+TRACE_DB_BASENAME = "turso_task_app_traces"
+
+
+def canonical_trace_db_name(*, timestamp: datetime | None = None) -> str:
+    """Return the canonical trace database filename (with optional timestamp suffix)."""
+
+    if timestamp is None:
+        return f"{TRACE_DB_BASENAME}.db"
+    return f"{TRACE_DB_BASENAME}_{timestamp.strftime('%Y-%m-%d_%H-%M-%S')}.db"
+
+
+def canonical_trace_db_path(*, timestamp: datetime | None = None) -> Path:
+    """Return the canonical trace database path within the default trace directory."""
+
+    return TRACE_DB_DIR / canonical_trace_db_name(timestamp=timestamp)

synth_ai/{tracing_v3 → core/tracing_v3}/db_config.py

@@ -4,8 +4,11 @@ Centralized database configuration for v3 tracing.
 
 import logging
 import os
+import shutil
 from typing import TYPE_CHECKING, Optional
 
+from synth_ai.core.tracing_v3.constants import canonical_trace_db_path
+
 if TYPE_CHECKING:
     from .turso.daemon import SqldDaemon
 
@@ -16,7 +19,7 @@ class DatabaseConfig:
     """Centralized database configuration management."""
 
     # Default values from serve.sh
-    DEFAULT_DB_FILE = "traces/v3/synth_ai.db"
+    DEFAULT_DB_FILE = str(canonical_trace_db_path())
     DEFAULT_HTTP_PORT = 8080
 
     def __init__(
@@ -27,11 +30,12 @@ class DatabaseConfig:
 
         Args:
             db_path: Path to database file. If None, uses DEFAULT_DB_FILE from serve.sh.
-            http_port: HTTP port for sqld daemon. If None, uses DEFAULT_HTTP_PORT from serve.sh.
+            http_port: Hrana WebSocket port for sqld daemon (env: SQLD_HTTP_PORT). If None, uses DEFAULT_HTTP_PORT.
             use_sqld: Whether to use sqld daemon or direct SQLite.
         """
-        self.use_sqld = use_sqld
-        self.http_port = http_port or int(os.getenv("SQLD_HTTP_PORT", self.DEFAULT_HTTP_PORT))
+        self.use_sqld = use_sqld and self._sqld_binary_available()
+        # Note: SQLD_HTTP_PORT is actually the hrana port (8080), not the HTTP API port
+        self.hrana_port = http_port or int(os.getenv("SQLD_HTTP_PORT", self.DEFAULT_HTTP_PORT))
         self._daemon: SqldDaemon | None = None
 
         # Set up database path to match serve.sh configuration
@@ -54,21 +58,34 @@ class DatabaseConfig:
         abs_path = os.path.abspath(self.db_file)
         sqld_data_path = os.path.join(abs_path, "dbs", "default", "data")
 
-        if os.path.exists(sqld_data_path):
-            # sqld is managing the database
-            logger.debug(f"✅ Using sqld-managed database at: {sqld_data_path}")
-            actual_db_path = sqld_data_path
-        else:
-            # Direct SQLite file
-            if not os.path.exists(abs_path):
-                logger.debug(f"⚠️  Database file not found at: {abs_path}")
-                logger.debug("🔧 Make sure to run './serve.sh' to start the turso/sqld service")
-            else:
-                logger.debug(f"📁 Using direct SQLite file at: {abs_path}")
-            actual_db_path = abs_path
+        if not os.path.exists(sqld_data_path) and not os.path.exists(abs_path):
+            raise RuntimeError(
+                "sqld data directory not found. Run `sqld --db-path <path>` before using the tracing database."
+            )
 
-        # SQLite URLs need 3 slashes for absolute paths
-        return f"sqlite+aiosqlite:///{actual_db_path}"
+        # Use http:// for local sqld HTTP API port
+        # sqld has two ports: hrana_port (Hrana WebSocket) and hrana_port+1 (HTTP API)
+        # Python libsql client uses HTTP API with http:// URLs
+        http_api_port = self.hrana_port + 1
+        return f"http://127.0.0.1:{http_api_port}"
+
+    def _sqld_binary_available(self) -> bool:
+        """Check if the sqld (Turso) binary is available on PATH."""
+        # Respect explicit SQLD_BINARY override when present
+        binary_override = os.getenv("SQLD_BINARY")
+        candidates = [binary_override, "sqld", "libsql-server"]
+
+        for candidate in candidates:
+            if candidate and shutil.which(candidate):
+                return True
+
+        if binary_override:
+            raise RuntimeError(
+                f"Configured SQLD_BINARY='{binary_override}' but the executable was not found on PATH."
+            )
+        raise RuntimeError(
+            "sqld binary not detected; install Turso's sqld or set SQLD_BINARY so that libSQL can be used."
+        )
 
     def start_daemon(self, wait_time: float = 2.0):
         """
@@ -87,7 +104,7 @@ class DatabaseConfig:
             # Import here to avoid circular dependency
             from .turso.daemon import SqldDaemon
 
-            self._daemon = SqldDaemon(db_path=self.db_base_path, http_port=self.http_port)
+            self._daemon = SqldDaemon(db_path=self.db_base_path, hrana_port=self.hrana_port)
 
         self._daemon.start()
 
@@ -133,11 +150,13 @@ def get_default_db_config() -> DatabaseConfig:
         # Check if sqld is already running (started by serve.sh)
         import subprocess
 
-        sqld_port = int(os.getenv("SQLD_HTTP_PORT", DatabaseConfig.DEFAULT_HTTP_PORT))
+        sqld_hrana_port = int(os.getenv("SQLD_HTTP_PORT", DatabaseConfig.DEFAULT_HTTP_PORT))
+        sqld_http_port = sqld_hrana_port + 1
         sqld_running = False
         try:
+            # Check for either hrana or http port in the process command line
             result = subprocess.run(
-                ["pgrep", "-f", f"sqld.*--http-listen-addr.*:{sqld_port}"],
+                ["pgrep", "-f", f"sqld.*(--hrana-listen-addr.*:{sqld_hrana_port}|--http-listen-addr.*:{sqld_http_port})"],
                 capture_output=True,
                 text=True,
             )
@@ -145,18 +164,12 @@ def get_default_db_config() -> DatabaseConfig:
                 # sqld is already running, don't start a new one
                 sqld_running = True
                 use_sqld = False
-                logger.debug(f"✅ Detected sqld already running on port {sqld_port}")
+                logger.debug(f"✅ Detected sqld already running on ports {sqld_hrana_port} (hrana) and {sqld_http_port} (http)")
         except Exception as e:
             logger.debug(f"Could not check for sqld process: {e}")
 
         if not sqld_running and use_sqld:
-            logger.warning("⚠️  sqld service not detected!")
-            logger.warning("🔧 Please start the turso/sqld service by running:")
-            logger.warning("   ./serve.sh")
-            logger.warning("")
-            logger.warning("This will start:")
-            logger.warning("  - sqld daemon (SQLite server) on port 8080")
-            logger.warning("  - Environment service on port 8901")
+            logger.warning("sqld service not detected. Start the Turso daemon (./serve.sh) before running tracing workloads.")
 
         _default_config = DatabaseConfig(db_path=db_path, use_sqld=use_sqld)