synth-ai 0.2.17__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/tracing_v3/turso/daemon.py

@@ -1,8 +1,11 @@
 """sqld daemon management utilities."""
 
+import logging
+import os
 import pathlib
 import shutil
 import subprocess
+import sys
 import time
 
 import requests
@@ -10,6 +13,8 @@ from requests import RequestException
 
 from ..config import CONFIG
 
+logger = logging.getLogger(__name__)
+
 
 class SqldDaemon:
     """Manages local sqld daemon lifecycle."""
@@ -18,28 +23,101 @@ class SqldDaemon:
         self,
         db_path: str | None = None,
         http_port: int | None = None,
+        hrana_port: int | None = None,
         binary_path: str | None = None,
     ):
         """Initialize sqld daemon manager.
 
         Args:
             db_path: Path to database file (uses config default if not provided)
-            http_port: HTTP port for daemon (uses config default if not provided)
+            http_port: HTTP port for health/API (uses config default + 1 if not provided)
+            hrana_port: Hrana WebSocket port for libsql connections (uses config default if not provided)
             binary_path: Path to sqld binary (auto-detected if not provided)
         """
         self.db_path = db_path or CONFIG.sqld_db_path
-        self.http_port = http_port or CONFIG.sqld_http_port
+        self.hrana_port = hrana_port or CONFIG.sqld_http_port  # Main port for libsql://
+        self.http_port = http_port or (self.hrana_port + 1)  # HTTP API on next port
         self.binary_path = binary_path or self._find_binary()
         self.process: subprocess.Popen[str] | None = None
 
     def _find_binary(self) -> str:
-        """Find sqld binary in PATH."""
+        """Find sqld binary in PATH, auto-installing if needed.
+        
+        Search order:
+        1. CONFIG.sqld_binary in PATH
+        2. libsql-server in PATH
+        3. Common install locations (~/.turso/bin, /usr/local/bin, etc.)
+        4. Auto-install via synth_ai.utils.sqld (if interactive terminal)
+        
+        Returns:
+            Path to sqld binary
+            
+        Raises:
+            RuntimeError: If binary not found and auto-install fails/disabled
+        """
+        # Check PATH first
         binary = shutil.which(CONFIG.sqld_binary) or shutil.which("libsql-server")
-        if not binary:
-            raise RuntimeError(
-                "sqld binary not found in PATH. Install with: brew install turso-tech/tools/sqld"
-            )
-        return binary
+        if binary:
+            logger.debug(f"Found sqld binary in PATH: {binary}")
+            return binary
+        
+        # Check common install locations
+        try:
+            from synth_ai.utils.sqld import find_sqld_binary
+            binary = find_sqld_binary()
+            if binary:
+                logger.debug(f"Found sqld binary in common location: {binary}")
+                return binary
+        except ImportError:
+            logger.debug("synth_ai.utils.sqld not available, skipping common location check")
+        
+        # Try auto-install if enabled and interactive
+        auto_install_enabled = os.getenv("SYNTH_AI_AUTO_INSTALL_SQLD", "true").lower() == "true"
+        
+        if auto_install_enabled and sys.stdin.isatty():
+            try:
+                from synth_ai.utils.sqld import install_sqld
+                logger.info("sqld binary not found. Attempting automatic installation...")
+                
+                # Use click if available for better UX, otherwise proceed automatically
+                try:
+                    import click
+                    if not click.confirm(
+                        "sqld not found. Install automatically via Homebrew?",
+                        default=True
+                    ):
+                        raise RuntimeError("User declined automatic installation")
+                except ImportError:
+                    # click not available, auto-install without prompt
+                    logger.info("Installing sqld automatically (non-interactive mode)")
+                
+                binary = install_sqld()
+                logger.info(f"Successfully installed sqld to: {binary}")
+                return binary
+                
+            except Exception as exc:
+                logger.warning(f"Auto-install failed: {exc}")
+                # Fall through to error message below
+        elif not auto_install_enabled:
+            logger.debug("Auto-install disabled via SYNTH_AI_AUTO_INSTALL_SQLD=false")
+        elif not sys.stdin.isatty():
+            logger.debug("Non-interactive terminal, skipping auto-install prompt")
+        
+        # If we get here, all methods failed
+        raise RuntimeError(
+            "sqld binary not found. Install using one of these methods:\n"
+            "\n"
+            "Quick install (recommended):\n"
+            "  synth-ai turso\n"
+            "\n"
+            "Manual install:\n"
+            "  brew install turso-tech/tools/sqld\n"
+            "  # or\n"
+            "  curl -sSfL https://get.tur.so/install.sh | bash && turso dev\n"
+            "\n"
+            "For CI/CD environments:\n"
+            "  Set SYNTH_AI_AUTO_INSTALL_SQLD=false and pre-install sqld"
+        )
 
     def start(self, wait_for_ready: bool = True) -> subprocess.Popen:
         """Start the sqld daemon."""
@@ -53,6 +131,8 @@ class SqldDaemon:
             self.binary_path,
             "--db-path",
             str(db_file),
+            "--hrana-listen-addr",
+            f"127.0.0.1:{self.hrana_port}",
             "--http-listen-addr",
             f"127.0.0.1:{self.http_port}",
         ]
@@ -112,6 +192,14 @@ class SqldDaemon:
         """Check if daemon is running."""
         return self.process is not None and self.process.poll() is None
 
+    def get_hrana_port(self) -> int:
+        """Get the Hrana WebSocket port for libsql:// connections."""
+        return self.hrana_port
+
+    def get_http_port(self) -> int:
+        """Get the HTTP API port for health checks."""
+        return self.http_port
+
     def __enter__(self):
         """Context manager entry."""
         self.start()
@@ -126,13 +214,27 @@ class SqldDaemon:
 _daemon: SqldDaemon | None = None
 
 
-def start_sqld(db_path: str | None = None, port: int | None = None) -> SqldDaemon:
-    """Start a global sqld daemon instance."""
+def start_sqld(
+    db_path: str | None = None,
+    port: int | None = None,
+    hrana_port: int | None = None,
+    http_port: int | None = None,
+) -> SqldDaemon:
+    """Start a global sqld daemon instance.
+    
+    Args:
+        db_path: Path to database file
+        port: Legacy parameter - used as hrana_port if hrana_port not specified
+        hrana_port: Hrana WebSocket port for libsql:// connections
+        http_port: HTTP API port for health checks
+    """
     global _daemon
     if _daemon and _daemon.is_running():
         return _daemon
 
-    _daemon = SqldDaemon(db_path=db_path, http_port=port)
+    # Support legacy 'port' parameter by using it as hrana_port
+    final_hrana_port = hrana_port or port
+    _daemon = SqldDaemon(db_path=db_path, hrana_port=final_hrana_port, http_port=http_port)
     _daemon.start()
     return _daemon
 

synth_ai/tracing_v3/turso/native_manager.py

@@ -14,8 +14,11 @@ import re
 from collections.abc import Callable
 from dataclasses import asdict, dataclass
 from datetime import UTC, datetime
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 
+import httpx
 import libsql
 from sqlalchemy.engine import make_url
 
@@ -60,36 +63,70 @@ class _ConnectionTarget:
     auth_token: str | None = None
 
 
+def _strip_auth_component(url: str) -> tuple[str, str | None]:
+    """Remove auth_token query parameter from URL, returning the token separately."""
+    parsed = urlparse(url)
+    if not parsed.query:
+        return url, None
+
+    params = dict(parse_qsl(parsed.query, keep_blank_values=True))
+    token = params.pop("auth_token", None)
+    query = urlencode(params, doseq=True)
+    sanitised = urlunparse(parsed._replace(query=query))
+    return sanitised, token
+
+
 def _resolve_connection_target(db_url: str | None, auth_token: str | None) -> _ConnectionTarget:
     """Normalise the configured database URL."""
     url = db_url or CONFIG.db_url
-
-    # Fast-path local SQLite URLs (`sqlite+aiosqlite:///path/to/db`)
-    if url.startswith("sqlite+aiosqlite:///"):
-        return _ConnectionTarget(database=url.replace("sqlite+aiosqlite:///", ""), auth_token=auth_token)
+    sanitised, token_from_url = _strip_auth_component(url)
+    effective_token = auth_token or token_from_url or CONFIG.auth_token
 
     # SQLAlchemy-compatible libsql scheme (`sqlite+libsql://<endpoint or path>`)
-    if url.startswith("sqlite+libsql://"):
-        target = url.replace("sqlite+libsql://", "", 1)
-        return _ConnectionTarget(database=target, sync_url=target if target.startswith("libsql://") else None, auth_token=auth_token)
+    if sanitised.startswith("sqlite+libsql://"):
+        raise RuntimeError("sqlite+libsql scheme is no longer supported; use libsql://")
+
+    # Plain SQLite files: file://, /absolute/path, or relative path
+    # libsql.connect() handles these without sync_url or auth_token
+    if sanitised.startswith("file://") or sanitised.startswith("/") or "://" not in sanitised:
+        # Strip file:// prefix if present, libsql.connect handles both formats
+        db_path = sanitised.replace("file://", "") if sanitised.startswith("file://") else sanitised
+        return _ConnectionTarget(database=db_path, sync_url=None, auth_token=None)
 
     # Native libsql URLs (`libsql://...`).
-    if url.startswith("libsql://"):
-        return _ConnectionTarget(database=url, sync_url=url, auth_token=auth_token)
+    if sanitised.startswith("libsql://"):
+        return _ConnectionTarget(database=sanitised, sync_url=sanitised, auth_token=effective_token)
 
     # Fallback to SQLAlchemy URL parsing for anything else we missed.
     try:
-        parsed = make_url(url)
-        if parsed.drivername.startswith("sqlite") and parsed.database:
-            return _ConnectionTarget(database=parsed.database, auth_token=auth_token)
-        if parsed.drivername.startswith("libsql"):
+        parsed = make_url(sanitised)
+        driver = parsed.drivername.lower()
+        if driver.startswith("sqlite"):
+            database = parsed.database or ""
+            if database and database not in {":memory:", ":memory"}:
+                # Absolute paths are passed through; relative paths are resolved to cwd
+                if database.startswith("/"):
+                    db_path = database
+                else:
+                    db_path = str(Path(database).expanduser().resolve())
+            elif database in {":memory:", ":memory"}:
+                db_path = ":memory:"
+            else:
+                raise RuntimeError("SQLite URL missing database path.")
+            return _ConnectionTarget(database=db_path, sync_url=None, auth_token=None)
+        if driver.startswith("libsql"):
             database = parsed.render_as_string(hide_password=False)
-            return _ConnectionTarget(database=database, sync_url=database, auth_token=auth_token)
+            return _ConnectionTarget(database=database, sync_url=database, auth_token=effective_token)
     except Exception:  # pragma: no cover - defensive guardrail
         logger.debug("Unable to parse db_url via SQLAlchemy", exc_info=True)
 
-    # As a last resort use the raw value (libsql.connect can handle absolute paths).
-    return _ConnectionTarget(database=url, auth_token=auth_token)
+    # Python libsql client uses HTTP API for http:// URLs, not Hrana WebSocket
+    # For local sqld with http:// URL, we need to ensure it points to the HTTP API port
+    # sqld uses two ports: Hrana WebSocket (e.g. 8080) and HTTP API (e.g. 8081)
+    # libsql.connect() with http:// uses HTTP API, so URL should point to HTTP API port
+    if sanitised.startswith(("http://", "https://", "libsql://")):
+        return _ConnectionTarget(database=sanitised, sync_url=sanitised, auth_token=effective_token)
+    raise RuntimeError(f"Unsupported tracing database URL: {sanitised}")
 
 
 def _json_dumps(value: Any) -> str | None:
@@ -350,6 +387,45 @@ class NativeLibsqlTraceManager(TraceStorage):
             if self._initialized:
                 return
 
+            # Fast-fail preflight: if using remote endpoint or local sqld, check health
+            # Skip health check for plain SQLite files (sync_url is None)
+            if self._target.sync_url:
+                try:
+                    parsed = urlparse(self._target.database or "")
+                    # Check for local sqld: http://, https://, or libsql://
+                    if parsed.scheme in ("http", "https", "libsql"):
+                        host_port = parsed.netloc or ""
+                        host = (host_port.split(":", 1)[0] or "").strip().lower()
+                        if host in {"127.0.0.1", "localhost"} and host_port:
+                            # For http:// URLs, the port should already be the HTTP API port
+                            # For libsql:// URLs, we need to calculate health check port
+                            if ":" in host_port:
+                                port = int(host_port.split(":", 1)[1])
+                                if parsed.scheme == "libsql":
+                                    # libsql:// uses Hrana port, health check is on HTTP API port (Hrana + 1)
+                                    health_url = f"http://{host}:{port + 1}/health"
+                                else:
+                                    # http:// already points to HTTP API port
+                                    health_url = f"http://{host}:{port}/health"
+                            else:
+                                health_url = f"http://{host_port}/health"
+                            try:
+                                async with httpx.AsyncClient(timeout=httpx.Timeout(1.0)) as client:
+                                    resp = await client.get(health_url)
+                                if resp.status_code != 200:
+                                    raise RuntimeError(
+                                        f"Tracing backend unhealthy at {health_url} (status={resp.status_code})"
+                                    )
+                            except Exception as exc:  # pragma: no cover - network env dependent
+                                raise RuntimeError(
+                                    f"Tracing backend not reachable at {health_url}. "
+                                    f"Start sqld with both ports: sqld --db-path <path> --hrana-listen-addr {host}:HRANA_PORT --http-listen-addr {host}:HTTP_PORT "
+                                    f"or disable tracing (TASKAPP_TRACING_ENABLED=0)."
+                                ) from exc
+                except Exception:
+                    # Propagate any preflight failure to abort early
+                    raise
+
             # Establish a libsql connection for future native operations.
             self._conn = self._open_connection()
             self._ensure_schema()

synth_ai/types.py

@@ -0,0 +1,8 @@
+import typing
+from typing import Literal
+
+ModelName = Literal[
+    "synth-small",
+    "synth-medium"
+]
+MODEL_NAMES = list(typing.get_args(ModelName))

synth_ai/urls.py

@@ -0,0 +1,11 @@
+# Base URL for all backends
+BACKEND_URL_BASE = "https://agent-learning.onrender.com"
+
+# Synth Research API base (supports OpenAI, Anthropic, and custom formats)
+# Real routes: /api/synth-research/chat/completions, /api/synth-research/messages
+# V1 routes: /api/synth-research/v1/chat/completions, /api/synth-research/v1/messages
+BACKEND_URL_SYNTH_RESEARCH_BASE = BACKEND_URL_BASE + "/api/synth-research"
+
+# Provider-specific URLs (for SDKs that expect standard paths)
+BACKEND_URL_SYNTH_RESEARCH_OPENAI = BACKEND_URL_SYNTH_RESEARCH_BASE + "/v1"  # For OpenAI SDKs (appends /chat/completions)
+BACKEND_URL_SYNTH_RESEARCH_ANTHROPIC = BACKEND_URL_SYNTH_RESEARCH_BASE  # For Anthropic SDKs (appends /v1/messages)

synth_ai/utils/__init__.py

@@ -1,8 +1,18 @@
 from . import task_app_state
+from .agents import write_agents_md
 from .base_url import PROD_BASE_URL_DEFAULT, get_backend_from_env, get_learning_v2_base_url
-from .cli import PromptedChoiceOption, PromptedChoiceType, print_next_step
+from .bin import install_bin, verify_bin
+from .cli import (
+    PromptedChoiceOption,
+    PromptedChoiceType,
+    PromptedPathOption,
+    print_next_step,
+    prompt_choice,
+    prompt_for_path,
+)
 from .env import mask_str, resolve_env_var, write_env_var_to_dotenv, write_env_var_to_json
 from .http import AsyncHttpClient, HTTPError, http_request
+from .json import create_and_write_json, load_json_to_dict, strip_json_comments
 from .modal import (
     ensure_modal_installed,
     ensure_task_app_ready,
@@ -11,6 +21,12 @@ from .modal import (
     is_modal_public_url,
     normalize_endpoint_url,
 )
+from .paths import (
+    find_bin_path,
+    find_config_path,
+    get_env_file_paths,
+    get_home_config_file_paths,
+)
 from .process import ensure_local_port_available, popen_capture, popen_stream, popen_stream_capture
 from .sqld import SQLD_VERSION, find_sqld_binary, install_sqld
 from .task_app_discovery import AppChoice, discover_eval_config_paths, select_app_choice
@@ -50,8 +66,11 @@ __all__ = [
     "PROD_BASE_URL_DEFAULT",
     "PromptedChoiceOption",
     "PromptedChoiceType",
+    "PromptedPathOption",
+    "prompt_for_path",
     "SQLD_VERSION",
     "USER_CONFIG_PATH",
+    "create_and_write_json",
     "current_task_app_id",
     "discover_eval_config_paths",
     "ensure_env_credentials",
@@ -60,14 +79,20 @@ __all__ = [
     "ensure_port_free",
     "ensure_task_app_ready",
     "find_asgi_apps",
+    "find_bin_path",
+    "find_config_path",
     "find_sqld_binary",
     "get_backend_from_env",
+    "get_env_file_paths",
+    "get_home_config_file_paths",
     "get_learning_v2_base_url",
     "http_request",
+    "install_bin",
     "install_sqld",
     "is_local_demo_url",
     "is_modal_public_url",
     "load_demo_dir",
+    "load_json_to_dict",
     "load_template_id",
     "load_user_config",
     "load_user_env",
@@ -84,17 +109,21 @@ __all__ = [
     "popen_stream_capture",
     "preflight_env_key",
     "print_next_step",
+    "prompt_choice",
     "read_task_app_config",
     "record_task_app",
     "resolve_env_var",
     "resolve_task_app_entry",
     "save_user_config",
     "select_app_choice",
+    "strip_json_comments",
     "task_app_config_path",
     "task_app_id_from_path",
     "task_app_state",
     "update_task_app_entry",
     "update_user_config",
+    "verify_bin",
+    'write_agents_md',
     "write_env_var_to_dotenv",
     "write_env_var_to_json",
     "write_task_app_config",

synth_ai/utils/agents.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+
+AGENTS_TEXT = """
+sinf
+"""
+SYNTH_DIV_START = "### --- SYNTH SECTION START ---"
+SYNTH_DIV_END = "### ---- SYNTH SECTION END ----"
+
+
+def _render_block() -> str:
+    return f"{SYNTH_DIV_START}\n{AGENTS_TEXT}\n{SYNTH_DIV_END}"
+
+
+def _append_block(prefix: str) -> str:
+    prefix = prefix.rstrip()
+    block = _render_block()
+    if prefix:
+        return f"{prefix}\n\n{block}\n"
+    return f"{block}\n"
+
+
+def write_agents_md() -> None:
+    path = Path.cwd() / "AGENTS.md"
+    if not path.exists():
+        path.write_text(_append_block(""), encoding="utf-8")
+        return
+
+    file_text = path.read_text(encoding="utf-8")
+
+    # Remove orphan end markers first (end markers without a preceding start marker)
+    cleaned = file_text
+    end_pos = cleaned.find(SYNTH_DIV_END)
+    start_pos = cleaned.find(SYNTH_DIV_START)
+    
+    # If there's an end marker before any start marker
+    if end_pos != -1 and (start_pos == -1 or end_pos < start_pos):
+        if start_pos == -1:
+            # No start markers at all - remove everything including content before orphan
+            cleaned = cleaned[end_pos + len(SYNTH_DIV_END):].lstrip()
+        else:
+            # There are start markers after the orphan - preserve content before orphan
+            before_orphan = cleaned[:end_pos].rstrip()
+            after_orphan = cleaned[end_pos + len(SYNTH_DIV_END):].lstrip()
+            cleaned = "\n\n".join(filter(None, [before_orphan, after_orphan]))
+    
+    # Find the first start and last end marker to consolidate multiple sections
+    first_start = cleaned.find(SYNTH_DIV_START)
+    last_end = cleaned.rfind(SYNTH_DIV_END)
+    
+    if first_start != -1 and last_end != -1 and last_end > first_start:
+        # We have at least one valid section, consolidate all into one
+        before = cleaned[:first_start].rstrip()
+        after = cleaned[last_end + len(SYNTH_DIV_END):].lstrip()
+        
+        parts: list[str] = []
+        if before:
+            parts.append(before)
+        parts.append(_render_block())
+        if after:
+            parts.append(after)
+        
+        new_text = "\n\n".join(parts)
+        if not new_text.endswith("\n"):
+            new_text += "\n"
+        path.write_text(new_text, encoding="utf-8")
+        return
+    
+    # No valid sections found, remove any remaining orphan markers
+    cleaned = cleaned.replace(SYNTH_DIV_END, "")
+    cleaned = cleaned.replace(AGENTS_TEXT, "")
+    cleaned = cleaned.strip()
+    if cleaned:
+        cleaned += "\n\n"
+    path.write_text(f"{cleaned}{_render_block()}\n", encoding="utf-8")

synth_ai/utils/bin.py

@@ -0,0 +1,39 @@
+import shlex
+import subprocess
+from pathlib import Path
+
+from .cli import prompt_choice
+
+
+def install_bin(name: str, install_options: list[str]) -> bool:
+    cmd = prompt_choice(
+        f"How would you like to install {name}?",
+        install_options
+    )
+    div_start = f"{'-' * 29} INSTALL START {'-' * 29}"
+    div_end = f"{'-' * 30} INSTALL END {'-' * 30}"
+    try:
+        print(f"Installing {name} via `{cmd}`")
+        print('\n' + div_start)
+        subprocess.run(shlex.split(cmd), check=True)
+        print(div_end + '\n')
+        return True
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to install {name}: {e}")
+        print(div_end + '\n')
+        return False
+    
+
+def verify_bin(bin_path: Path) -> bool:
+    try:
+        result = subprocess.run(
+            [str(bin_path), "--version"],
+            capture_output=True,
+            text=True,
+            timeout=3,
+            check=False
+        )
+        return result.returncode == 0
+    except (OSError, subprocess.SubprocessError) as e:
+        print(e)
+        return False