synth-ai 0.2.8.dev2__py3-none-any.whl → 0.4.3__py3-none-any.whl

synth_ai/core/tracing_v3/trace_utils.py

@@ -0,0 +1,326 @@
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any
+
+Row = sqlite3.Row
+
+
+def connect(db_path: str | bytes | os.PathLike[str] | os.PathLike[bytes]) -> sqlite3.Connection:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def _json_load(value: Any) -> Any:
+    if value is None:
+        return None
+    if isinstance(value, dict | list):
+        return value
+    if isinstance(value, bytes | bytearray):
+        value = value.decode("utf-8", errors="ignore")
+    try:
+        return json.loads(value)
+    except Exception:
+        return value
+
+
+def _row_get(row: Row, key: str) -> Any:
+    try:
+        return row[key]
+    except Exception:
+        return None
+
+
+def fetch_crafter_sessions(
+    conn: sqlite3.Connection,
+    *,
+    limit: int,
+    metadata_filter: str | None = None,
+    session_ids: Sequence[str] | None = None,
+    min_event_count: int = 0,
+) -> list[str]:
+    if session_ids:
+        placeholders = ",".join("?" for _ in session_ids)
+        rows = conn.execute(
+            f"""
+            SELECT session_id
+            FROM session_traces
+            WHERE session_id IN ({placeholders})
+            ORDER BY created_at DESC
+            """,
+            tuple(session_ids),
+        ).fetchall()
+        return [row["session_id"] for row in rows]
+
+    params: list[Any] = []
+    where_clauses: list[str] = []
+    if metadata_filter:
+        where_clauses.append("session_traces.metadata LIKE ?")
+        params.append(f"%{metadata_filter}%")
+    where_sql = ""
+    if where_clauses:
+        where_sql = "WHERE " + " AND ".join(where_clauses)
+
+    having_sql = ""
+    if min_event_count > 0:
+        having_sql = "HAVING COUNT(events.id) >= ?"
+        params.append(min_event_count)
+
+    query = f"""
+        SELECT session_traces.session_id
+        FROM session_traces
+        LEFT JOIN events ON session_traces.session_id = events.session_id
+        {where_sql}
+        GROUP BY session_traces.session_id
+        {having_sql}
+        ORDER BY session_traces.created_at DESC
+        LIMIT ?
+    """
+    rows = conn.execute(query, (*params, limit)).fetchall()
+    return [row["session_id"] for row in rows]
+
+
+def load_session_trace(conn: sqlite3.Connection, session_id: str) -> dict[str, Any]:
+    session_row = conn.execute(
+        """
+        SELECT session_id, created_at, metadata
+        FROM session_traces
+        WHERE session_id = ?
+        """,
+        (session_id,),
+    ).fetchone()
+    if not session_row:
+        raise ValueError(f"Session {session_id} not found")
+
+    timesteps = conn.execute(
+        """
+        SELECT step_id,
+               step_index,
+               turn_number,
+               started_at,
+               completed_at,
+               step_metadata
+        FROM session_timesteps
+        WHERE session_id = ?
+        ORDER BY step_index ASC
+        """,
+        (session_id,),
+    ).fetchall()
+
+    event_rows = conn.execute(
+        """
+        SELECT *
+        FROM events
+        WHERE session_id = ?
+        ORDER BY event_time ASC, id ASC
+        """,
+        (session_id,),
+    ).fetchall()
+
+    message_rows = conn.execute(
+        """
+        SELECT *
+        FROM messages
+        WHERE session_id = ?
+        ORDER BY event_time ASC, id ASC
+        """,
+        (session_id,),
+    ).fetchall()
+
+    event_rewards = conn.execute(
+        """
+        SELECT *
+        FROM event_rewards
+        WHERE session_id = ?
+        ORDER BY turn_number ASC, id ASC
+        """,
+        (session_id,),
+    ).fetchall()
+
+    outcome_rewards = conn.execute(
+        """
+        SELECT *
+        FROM outcome_rewards
+        WHERE session_id = ?
+        ORDER BY created_at ASC
+        """,
+        (session_id,),
+    ).fetchall()
+
+    metadata = _json_load(session_row["metadata"]) or {}
+    if isinstance(metadata, dict):
+        episode_id = metadata.get("episode_id")
+        if episode_id is not None and not isinstance(episode_id, str):
+            metadata["episode_id"] = str(episode_id)
+
+    events_payload = [
+        {
+            "id": row["id"],
+            "event_type": row["event_type"],
+            "system_instance_id": row["system_instance_id"],
+            "time_record": {
+                "event_time": row["event_time"],
+                "message_time": row["message_time"],
+                "created_at": row["created_at"],
+            },
+            "model_name": row["model_name"],
+            "provider": row["provider"],
+            "input_tokens": row["input_tokens"],
+            "output_tokens": row["output_tokens"],
+            "total_tokens": row["total_tokens"],
+            "cost_usd": row["cost_usd"],
+            "latency_ms": row["latency_ms"],
+            "span_id": row["span_id"],
+            "trace_id": row["trace_id"],
+            "call_records": _json_load(row["call_records"]) or [],
+            "reward": row["reward"],
+            "terminated": row["terminated"],
+            "truncated": row["truncated"],
+            "system_state_before": _json_load(row["system_state_before"]),
+            "system_state_after": _json_load(row["system_state_after"]),
+            "metadata": _json_load(row["metadata"]) or {},
+            "event_metadata": _json_load(row["event_metadata"]),
+        }
+        for row in event_rows
+    ]
+
+    messages_payload = [
+        {
+            "id": row["id"],
+            "message_type": row["message_type"],
+            "content": row["content"],
+            "time_record": {
+                "event_time": row["event_time"],
+                "message_time": row["message_time"],
+                "timestamp": row["timestamp"],
+            },
+            "metadata": _json_load(row["metadata"]) or {},
+        }
+        for row in message_rows
+    ]
+
+    trace: dict[str, Any] = {
+        "session_id": session_row["session_id"],
+        "created_at": session_row["created_at"],
+        "metadata": metadata,
+        "session_time_steps": [
+            {
+                "step_id": row["step_id"],
+                "step_index": row["step_index"],
+                "turn_number": row["turn_number"],
+                "started_at": row["started_at"],
+                "completed_at": row["completed_at"],
+                "metadata": _json_load(row["step_metadata"]) or {},
+            }
+            for row in timesteps
+        ],
+        "event_history": events_payload,
+        "events": events_payload,
+        "markov_blanket_message_history": messages_payload,
+        "messages": messages_payload,
+        "event_rewards": [
+            {
+                "id": row["id"],
+                "event_id": row["event_id"],
+                "turn_number": row["turn_number"],
+                "reward_value": row["reward_value"],
+                "reward_type": row["reward_type"],
+                "key": row["key"],
+                "annotation": _json_load(row["annotation"]) or {},
+                "source": row["source"],
+                "created_at": row["created_at"],
+            }
+            for row in event_rewards
+        ],
+        "outcome_rewards": [
+            {
+                "id": row["id"],
+                "total_reward": row["total_reward"],
+                "achievements_count": row["achievements_count"],
+                "total_steps": row["total_steps"],
+                "reward_metadata": _json_load(row["reward_metadata"]) or {},
+                "annotation": _json_load(_row_get(row, "annotation")) or {},
+                "created_at": row["created_at"],
+            }
+            for row in outcome_rewards
+        ],
+    }
+    return trace
+
+
+@dataclass
+class DeterministicMetrics:
+    session_id: str
+    unique_achievement_reward: float
+    achievement_reward: float
+    outcome_total_reward: float
+    unique_achievement_count: int
+    final_achievement_count: int
+
+
+def compute_deterministic_metrics(conn: sqlite3.Connection, session_id: str) -> DeterministicMetrics:
+    event_rows = conn.execute(
+        """
+        SELECT reward_type, reward_value, annotation
+        FROM event_rewards
+        WHERE session_id = ?
+        """,
+        (session_id,),
+    ).fetchall()
+
+    unique_total = 0.0
+    all_total = 0.0
+    unique_achievements: set[str] = set()
+
+    for row in event_rows:
+        reward_type = row["reward_type"]
+        value = float(row["reward_value"] or 0.0)
+        if reward_type == "unique_achievement_delta":
+            unique_total += value
+            annotation = _json_load(row["annotation"]) or {}
+            for name in annotation.get("new_unique") or []:
+                if isinstance(name, str):
+                    unique_achievements.add(name)
+        elif reward_type == "achievement_delta":
+            all_total += value
+
+    outcome_rows = conn.execute(
+        """
+        SELECT total_reward, reward_metadata
+        FROM outcome_rewards
+        WHERE session_id = ?
+        """,
+        (session_id,),
+    ).fetchall()
+
+    outcome_total = 0.0
+    final_achievements: set[str] = set()
+    for row in outcome_rows:
+        outcome_total += float(row["total_reward"] or 0.0)
+        metadata = _json_load(row["reward_metadata"]) or {}
+        for name in metadata.get("achievements") or []:
+            if isinstance(name, str):
+                final_achievements.add(name)
+
+    return DeterministicMetrics(
+        session_id=session_id,
+        unique_achievement_reward=unique_total,
+        achievement_reward=all_total,
+        outcome_total_reward=outcome_total,
+        unique_achievement_count=len(unique_achievements),
+        final_achievement_count=len(final_achievements),
+    )
+
+
+__all__ = [
+    "DeterministicMetrics",
+    "compute_deterministic_metrics",
+    "connect",
+    "fetch_crafter_sessions",
+    "load_session_trace",
+]

synth_ai/core/tracing_v3/turso/__init__.py

@@ -0,0 +1,12 @@
+"""Turso integration package for tracing v3."""
+
+from .daemon import SqldDaemon, get_daemon, start_sqld, stop_sqld
+from .native_manager import NativeLibsqlTraceManager
+
+__all__ = [
+    "SqldDaemon",
+    "NativeLibsqlTraceManager",
+    "get_daemon",
+    "start_sqld",
+    "stop_sqld",
+]

synth_ai/core/tracing_v3/turso/daemon.py

@@ -0,0 +1,278 @@
+"""sqld daemon management utilities."""
+
+import logging
+import os
+import pathlib
+import shutil
+import socket
+import subprocess
+import sys
+import time
+
+import requests
+from requests import RequestException
+
+from ..config import CONFIG
+
+logger = logging.getLogger(__name__)
+
+
+class SqldDaemon:
+    """Manages local sqld daemon lifecycle."""
+
+    def __init__(
+        self,
+        db_path: str | None = None,
+        http_port: int | None = None,
+        hrana_port: int | None = None,
+        binary_path: str | None = None,
+    ):
+        """Initialize sqld daemon manager.
+
+        Args:
+            db_path: Path to database file (uses config default if not provided)
+            http_port: HTTP port for health/API (uses config default + 1 if not provided)
+            hrana_port: Hrana WebSocket port for libsql connections (uses config default if not provided)
+            binary_path: Path to sqld binary (auto-detected if not provided)
+        """
+        self.db_path = db_path or CONFIG.sqld_db_path
+        self.hrana_port = hrana_port or CONFIG.sqld_http_port  # Main port for libsql://
+        self.http_port = http_port or (self.hrana_port + 1)  # HTTP API on next port
+        self.binary_path = binary_path or self._find_binary()
+        self.process: subprocess.Popen[str] | None = None
+
+    def _find_binary(self) -> str:
+        """Find sqld binary in PATH, auto-installing if needed.
+        
+        Search order:
+        1. CONFIG.sqld_binary in PATH
+        2. libsql-server in PATH
+        3. Common install locations (~/.turso/bin, /usr/local/bin, etc.)
+        4. Auto-install via synth_ai.utils.sqld (if interactive terminal)
+        
+        Returns:
+            Path to sqld binary
+            
+        Raises:
+            RuntimeError: If binary not found and auto-install fails/disabled
+        """
+        # Check PATH first
+        binary = shutil.which(CONFIG.sqld_binary) or shutil.which("libsql-server")
+        if binary:
+            logger.debug(f"Found sqld binary in PATH: {binary}")
+            return binary
+        
+        # Check common install locations
+        try:
+            from synth_ai.cli.lib.sqld import find_sqld_binary
+            binary = find_sqld_binary()
+            if binary:
+                logger.debug(f"Found sqld binary in common location: {binary}")
+                return binary
+        except ImportError:
+            logger.debug("synth_ai.utils.sqld not available, skipping common location check")
+        
+        # Try auto-install if enabled and interactive
+        auto_install_enabled = os.getenv("SYNTH_AI_AUTO_INSTALL_SQLD", "true").lower() == "true"
+        
+        if auto_install_enabled and sys.stdin.isatty():
+            try:
+                from synth_ai.cli.lib.sqld import install_sqld
+                logger.info("sqld binary not found. Attempting automatic installation...")
+                
+                # Use click if available for better UX, otherwise proceed automatically
+                try:
+                    import click
+                    if not click.confirm(
+                        "sqld not found. Install automatically via Homebrew?",
+                        default=True
+                    ):
+                        raise RuntimeError("User declined automatic installation")
+                except ImportError:
+                    # click not available, auto-install without prompt
+                    logger.info("Installing sqld automatically (non-interactive mode)")
+                
+                binary = install_sqld()
+                logger.info(f"Successfully installed sqld to: {binary}")
+                return binary
+                
+            except Exception as exc:
+                logger.warning(f"Auto-install failed: {exc}")
+                # Fall through to error message below
+        elif not auto_install_enabled:
+            logger.debug("Auto-install disabled via SYNTH_AI_AUTO_INSTALL_SQLD=false")
+        elif not sys.stdin.isatty():
+            logger.debug("Non-interactive terminal, skipping auto-install prompt")
+        
+        # If we get here, all methods failed
+        raise RuntimeError(
+            "sqld binary not found. Install using one of these methods:\n"
+            "\n"
+            "Quick install (recommended):\n"
+            "  synth-ai turso\n"
+            "\n"
+            "Manual install:\n"
+            "  brew install turso-tech/tools/sqld\n"
+            "  # or\n"
+            "  curl -sSfL https://get.tur.so/install.sh | bash && turso dev\n"
+            "\n"
+            "For CI/CD environments:\n"
+            "  Set SYNTH_AI_AUTO_INSTALL_SQLD=false and pre-install sqld"
+        )
+
+    def start(self, wait_for_ready: bool = True) -> subprocess.Popen:
+        """Start the sqld daemon."""
+        if self.process and self.process.poll() is None:
+            return self.process
+
+        # Avoid port conflicts by selecting free ports when needed
+        if not self._port_available(self.hrana_port):
+            self.hrana_port = self._find_free_port()
+        if not self._port_available(self.http_port) or self.http_port == self.hrana_port:
+            self.http_port = self._find_free_port()
+            # Ensure distinct ports
+            if self.http_port == self.hrana_port:
+                self.http_port = self._find_free_port()
+
+        db_file = pathlib.Path(self.db_path).resolve()
+        db_file.parent.mkdir(parents=True, exist_ok=True)
+
+        args = [
+            self.binary_path,
+            "--db-path",
+            str(db_file),
+            "--hrana-listen-addr",
+            f"127.0.0.1:{self.hrana_port}",
+            "--http-listen-addr",
+            f"127.0.0.1:{self.http_port}",
+        ]
+
+        # No replication for local-only mode
+        if CONFIG.sqld_idle_shutdown > 0:
+            args.extend(["--idle-shutdown-timeout-s", str(CONFIG.sqld_idle_shutdown)])
+
+        self.process = subprocess.Popen(
+            args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+        )
+
+        if wait_for_ready:
+            self._wait_for_ready()
+
+        return self.process
+
+    def _wait_for_ready(self, timeout: float = 10.0):
+        """Wait for daemon to be ready to accept connections."""
+        start_time = time.time()
+        health_url = f"http://127.0.0.1:{self.http_port}/health"
+
+        while time.time() - start_time < timeout:
+            try:
+                response = requests.get(health_url, timeout=1)
+                if response.status_code == 200:
+                    return
+            except RequestException:
+                pass
+
+            # Check if process crashed
+            if self.process and self.process.poll() is not None:
+                stdout, stderr = self.process.communicate()
+                raise RuntimeError(
+                    f"sqld daemon failed to start:\nstdout: {stdout}\nstderr: {stderr}"
+                )
+
+            time.sleep(0.1)
+
+        raise TimeoutError(f"sqld daemon did not become ready within {timeout} seconds")
+
+    @staticmethod
+    def _port_available(port: int) -> bool:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            try:
+                s.bind(("127.0.0.1", port))
+                return True
+            except OSError:
+                return False
+
+    @staticmethod
+    def _find_free_port() -> int:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind(("127.0.0.1", 0))
+            return s.getsockname()[1]
+
+    def stop(self, timeout: float = 5.0):
+        """Stop the sqld daemon gracefully."""
+        if not self.process:
+            return
+
+        self.process.terminate()
+        try:
+            self.process.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            self.process.kill()
+            self.process.wait()
+
+        self.process = None
+
+    def is_running(self) -> bool:
+        """Check if daemon is running."""
+        return self.process is not None and self.process.poll() is None
+
+    def get_hrana_port(self) -> int:
+        """Get the Hrana WebSocket port for libsql:// connections."""
+        return self.hrana_port
+
+    def get_http_port(self) -> int:
+        """Get the HTTP API port for health checks."""
+        return self.http_port
+
+    def __enter__(self):
+        """Context manager entry."""
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit."""
+        self.stop()
+
+
+# Convenience functions
+_daemon: SqldDaemon | None = None
+
+
+def start_sqld(
+    db_path: str | None = None,
+    port: int | None = None,
+    hrana_port: int | None = None,
+    http_port: int | None = None,
+) -> SqldDaemon:
+    """Start a global sqld daemon instance.
+    
+    Args:
+        db_path: Path to database file
+        port: Legacy parameter - used as hrana_port if hrana_port not specified
+        hrana_port: Hrana WebSocket port for libsql:// connections
+        http_port: HTTP API port for health checks
+    """
+    global _daemon
+    if _daemon and _daemon.is_running():
+        return _daemon
+
+    # Support legacy 'port' parameter by using it as hrana_port
+    final_hrana_port = hrana_port or port
+    _daemon = SqldDaemon(db_path=db_path, hrana_port=final_hrana_port, http_port=http_port)
+    _daemon.start()
+    return _daemon
+
+
+def stop_sqld():
+    """Stop the global sqld daemon instance."""
+    global _daemon
+    if _daemon:
+        _daemon.stop()
+        _daemon = None
+
+
+def get_daemon() -> SqldDaemon | None:
+    """Get the global daemon instance."""
+    return _daemon

synth_ai/{tracing_v3 → core/tracing_v3}/turso/models.py

@@ -1,6 +1,7 @@
-from __future__ import annotations
 """SQLAlchemy declarative models for tracing v3."""
 
+from __future__ import annotations
+
 import json
 
 from sqlalchemy import (
@@ -425,12 +426,13 @@ class OutcomeReward(Base):
 
     id = Column(Integer, primary_key=True, autoincrement=True)
     session_id = Column(String, ForeignKey("session_traces.session_id"), nullable=False)
-    total_reward = Column(Integer, nullable=False)
+    total_reward = Column(Float, nullable=False)
     achievements_count = Column(Integer, nullable=False, default=0)
     total_steps = Column(Integer, nullable=False, default=0)
     created_at = Column(DateTime, default=func.current_timestamp(), nullable=False)
     # Store additional structured metadata about the outcome (e.g., achievements list)
     reward_metadata = Column(JSONText)
+    annotation = Column(JSONText)
 
     __table_args__ = (
         Index("idx_outcome_rewards_session", "session_id"),
@@ -452,7 +454,9 @@ class EventReward(Base):
     message_id = Column(Integer, ForeignKey("messages.id"), nullable=True)
     turn_number = Column(Integer, nullable=True)
     reward_value = Column(Float, nullable=False, default=0.0)
-    reward_type = Column(String, nullable=True)  # shaped | sparse | achievement | penalty | evaluator | human
+    reward_type = Column(
+        String, nullable=True
+    )  # shaped | sparse | achievement | penalty | evaluator | human
     key = Column(String, nullable=True)  # e.g., achievement name
     annotation = Column(JSONText)  # free-form JSON
     source = Column(String, nullable=True)  # environment | runner | evaluator | human