synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

synth_ai/utils/log_filter.py

@@ -0,0 +1,99 @@
+"""Utilities for filtering noisy logs from stdout/stderr."""
+
+from __future__ import annotations
+
+import re
+import sys
+from typing import TextIO
+
+# Patterns for noisy logs that should be filtered out
+# Using simple substring matching for better performance and broader matching
+_NOISY_LOG_SUBSTRINGS = [
+    'codex_otel::otel_event_manager',
+    'event.kind=response.reasoning_summary_text.delta',
+    'event.name="codex.sse_event"',
+    'codex_otel',
+]
+
+# Compiled regex patterns for more complex matching
+_NOISY_LOG_PATTERNS = [
+    # Filter all codex_otel verbose SSE event logs (most aggressive - catches everything from this module)
+    re.compile(r'.*codex_otel::otel_event_manager.*', re.IGNORECASE),
+    # Filter reasoning summary delta events
+    re.compile(r'.*event\.kind=response\.reasoning_summary_text\.delta.*', re.IGNORECASE),
+    # Filter any codex.sse_event logs
+    re.compile(r'.*event\.name="codex\.sse_event".*', re.IGNORECASE),
+    # Also catch logs that start with timestamp and have codex_otel
+    re.compile(r'^\d{4}-\d{2}-\d{2}T.*codex_otel.*', re.IGNORECASE),
+]
+
+
+def should_filter_log_line(line: str) -> bool:
+    """Check if a log line should be filtered out.
+    
+    Filters out noisy logs like verbose codex_otel SSE event logs.
+    Uses both substring matching (faster) and regex (more precise).
+    """
+    if not line.strip():
+        return False
+    
+    # Fast substring check first
+    line_lower = line.lower()
+    for substr in _NOISY_LOG_SUBSTRINGS:
+        if substr.lower() in line_lower:
+            return True
+    
+    # Fallback to regex for edge cases
+    return any(pattern.search(line) for pattern in _NOISY_LOG_PATTERNS)
+
+
+class FilteredStream:
+    """A file-like object that filters noisy log lines before writing."""
+    
+    def __init__(self, stream: TextIO):
+        self._stream = stream
+        self._buffer = ""
+        # Store original write method to avoid recursion
+        self._original_write = stream.write
+    
+    def write(self, text: str) -> int:
+        """Write text, filtering out noisy log lines."""
+        # Check if entire text matches filter (for unbuffered writes)
+        if should_filter_log_line(text.strip()):
+            return len(text)
+        
+        # Buffer incomplete lines
+        self._buffer += text
+        lines = self._buffer.split('\n')
+        # Keep the last incomplete line in buffer
+        self._buffer = lines[-1]
+        # Process complete lines
+        written = 0
+        for line in lines[:-1]:
+            if not should_filter_log_line(line):
+                written += self._original_write(line + '\n')
+        return len(text)
+    
+    def flush(self) -> None:
+        """Flush any remaining buffered content."""
+        if self._buffer and not should_filter_log_line(self._buffer):
+            self._original_write(self._buffer)
+            self._buffer = ""
+        self._stream.flush()
+    
+    def __getattr__(self, name: str):
+        """Delegate other attributes to the underlying stream."""
+        return getattr(self._stream, name)
+
+
+def install_log_filter() -> None:
+    """Install stdout/stderr filters to suppress noisy logs.
+    
+    This wraps sys.stdout and sys.stderr to filter out verbose
+    codex_otel logs before they're printed.
+    """
+    if not isinstance(sys.stdout, FilteredStream):
+        sys.stdout = FilteredStream(sys.stdout)  # type: ignore[assignment]
+    if not isinstance(sys.stderr, FilteredStream):
+        sys.stderr = FilteredStream(sys.stderr)  # type: ignore[assignment]
+

synth_ai/utils/logging.py

@@ -0,0 +1,198 @@
+import threading
+import time
+from contextlib import suppress
+import uuid
+from datetime import UTC, datetime
+from typing import Any, Mapping, Sequence
+
+from synth_ai import __version__ as sdk_version
+
+from .base_url import get_backend_from_env
+from .http import http_request
+
+Severity = str
+
+_ENDPOINT_SUFFIX = "/api/v1/sdk-logs"
+_MAX_MESSAGE_LENGTH = 4_096
+_ALLOWED_LEVELS = {"debug", "info", "warning", "error", "critical"}
+_ACTIVE_THREADS: set[threading.Thread] = set()
+_ACTIVE_LOCK = threading.Lock()
+
+
+def log_event(
+    level: Severity,
+    message: str,
+    *,
+    attributes: Mapping[str, Any] | None = None,
+    ctx: Mapping[str, Any] | None = None,
+    context: Mapping[str, Any] | None = None,
+    request_id: str | None = None,
+    batch_id: str | None = None,
+) -> None:
+    """Send a single log entry to the backend BetterStack proxy (best effort)."""
+
+    try:
+        payload_context = context if context is not None else ctx
+        entry = _build_entry(level, message, attributes=attributes, context=payload_context, request_id=request_id)
+        if entry is None:
+            return
+        _dispatch((entry,), batch_id=batch_id)
+    except Exception:
+        # Logging must never raise for callers
+        return
+
+
+def log_info(message: str, **kwargs: Any) -> None:
+    log_event("info", message, **kwargs)
+
+
+def log_warning(message: str, **kwargs: Any) -> None:
+    log_event("warning", message, **kwargs)
+
+
+def log_error(message: str, **kwargs: Any) -> None:
+    log_event("error", message, **kwargs)
+
+
+def log_batch(
+    entries: Sequence[Mapping[str, Any]],
+    *,
+    batch_id: str | None = None
+) -> None:
+    """Send a batch of pre-built entries. Invalid entries are skipped silently."""
+
+    try:
+        prepared = [
+            _build_entry(
+                entry.get("level", "info"),
+                entry.get("message", ""),
+                attributes=entry.get("attributes"),
+                context=entry.get("context"),
+                request_id=entry.get("request_id"),
+            )
+            for entry in entries
+            if isinstance(entry, Mapping)
+        ]
+        filtered = tuple(item for item in prepared if item)
+        if filtered:
+            _dispatch(filtered, batch_id=batch_id)
+    except Exception:
+        return
+
+
+def _build_entry(
+    level: Severity,
+    message: str,
+    *,
+    attributes: Mapping[str, Any] | None,
+    context: Mapping[str, Any] | None,
+    request_id: str | None,
+) -> dict[str, Any] | None:
+    normalized_level = str(level or "").lower()
+    if normalized_level not in _ALLOWED_LEVELS:
+        normalized_level = "info"
+    safe_message = (message or "").strip()
+    if not safe_message:
+        return None
+    safe_message = safe_message[:_MAX_MESSAGE_LENGTH]
+    return {
+        "level": normalized_level,
+        "message": safe_message,
+        "timestamp": datetime.now(UTC).isoformat(),
+        "sdk_version": sdk_version,
+        "request_id": request_id or uuid.uuid4().hex,
+        "attributes": _normalize_mapping(attributes),
+        "context": _normalize_mapping(context),
+    }
+
+
+def _dispatch(
+    entries: Sequence[Mapping[str, Any]],
+    *,
+    batch_id: str | None
+) -> None:
+    try:
+        thread = threading.Thread(
+            target=_thread_target,
+            args=(tuple(entries), batch_id),
+            daemon=True,
+            name="sdk-logger",
+        )
+        with _ACTIVE_LOCK:
+            _ACTIVE_THREADS.add(thread)
+        thread.start()
+    except Exception:
+        return
+
+
+def _thread_target(entries: Sequence[Mapping[str, Any]], batch_id: str | None) -> None:
+    try:
+        _post_entries(entries, batch_id)
+    finally:
+        with _ACTIVE_LOCK:
+            _ACTIVE_THREADS.discard(threading.current_thread())
+
+
+def _post_entries(
+    entries: Sequence[Mapping[str, Any]],
+    batch_id: str | None
+) -> None:
+    try:
+        base_url, api_key = get_backend_from_env()
+        base = (base_url or "").rstrip("/")
+        key = (api_key or "").strip()
+        if not base or not key:
+            return
+        url = f"{base}{_ENDPOINT_SUFFIX}"
+        headers = {
+            "authorization": f"Bearer {key}",
+            "accept": "application/json",
+            "content-type": "application/json",
+        }
+        payload: dict[str, Any] = {"entries": list(entries)}
+        if batch_id:
+            payload["batch_id"] = batch_id
+        http_request("POST", url, headers=headers, body=payload)
+    except Exception:
+        return
+
+
+def _normalize_mapping(values: Mapping[str, Any] | None) -> dict[str, Any]:
+    if not values:
+        return {}
+    normalized: dict[str, Any] = {}
+    for key, value in values.items():
+        try:
+            normalized[str(key)] = _coerce(value)
+        except Exception:
+            continue
+    return normalized
+
+
+def _coerce(value: Any) -> Any:
+    if isinstance(value, str | int | float | bool) or value is None:
+        return value
+    if isinstance(value, Mapping):
+        return {str(k): _coerce(v) for k, v in value.items()}
+    if isinstance(value, list | tuple | set):
+        return [_coerce(v) for v in value]
+    return str(value)
+
+
+def flush_logger(timeout: float | None = None) -> None:
+    """Wait for any in-flight logging threads to finish."""
+
+    deadline = time.time() + timeout if timeout is not None else None
+    while True:
+        with _ACTIVE_LOCK:
+            threads = [t for t in _ACTIVE_THREADS if t.is_alive()]
+        if not threads:
+            return
+        remaining = None
+        if deadline is not None:
+            remaining = max(0.0, deadline - time.time())
+            if remaining == 0:
+                return
+        for thread in threads:
+            with suppress(Exception):
+                thread.join(remaining)

synth_ai/utils/modal.py

@@ -0,0 +1,299 @@
+
+import contextlib
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse, urlunparse
+
+from synth_ai.demos import core as demo_core
+from synth_ai.demos.core import DEFAULT_TASK_APP_SECRET_NAME, DemoEnv
+from synth_ai.utils.env import mask_str
+from synth_ai.utils.http import http_request
+from synth_ai.utils.process import popen_capture
+from synth_ai.utils.user_config import load_user_config
+
+
+def is_modal_public_url(url: str | None) -> bool:
+    try:
+        candidate = (url or "").strip().lower()
+        if not candidate or not (candidate.startswith("http://") or candidate.startswith("https://")):
+            return False
+        return (".modal.run" in candidate) and ("modal.local" not in candidate) and ("pypi-mirror" not in candidate)
+    except Exception:
+        return False
+
+
+def is_local_demo_url(url: str | None) -> bool:
+    try:
+        candidate = (url or "").strip().lower()
+        if not candidate:
+            return False
+        return candidate.startswith("http://127.0.0.1") or candidate.startswith("http://localhost")
+    except Exception:
+        return False
+
+
+def normalize_endpoint_url(url: str) -> str:
+    """Convert loopback URLs to forms accepted by the backend."""
+    if not url:
+        return url
+    try:
+        parsed = urlparse(url)
+        host = parsed.hostname or ""
+        if host in {"127.0.0.1", "::1"}:
+            new_host = "localhost"
+            netloc = new_host
+            if parsed.port:
+                netloc = f"{new_host}:{parsed.port}"
+            if parsed.username:
+                creds = parsed.username
+                if parsed.password:
+                    creds += f":{parsed.password}"
+                netloc = f"{creds}@{netloc}"
+            parsed = parsed._replace(netloc=netloc)
+            return urlunparse(parsed)
+    except Exception:
+        pass
+    return url
+
+
+def find_asgi_apps(root: Path) -> list[Path]:
+    """Recursively search for Python files that declare a Modal ASGI app."""
+    results: list[Path] = []
+    skip_dirs = {
+        ".git",
+        ".hg",
+        ".svn",
+        "node_modules",
+        "dist",
+        "build",
+        "__pycache__",
+        ".ruff_cache",
+        ".mypy_cache",
+        "venv",
+        ".venv",
+    }
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in skip_dirs]
+        for name in filenames:
+            if not name.endswith(".py"):
+                continue
+            path = Path(dirpath) / name
+            try:
+                with path.open("r", encoding="utf-8", errors="ignore") as fh:
+                    txt = fh.read()
+                if ("@asgi_app()" in txt) or ("@modal.asgi_app()" in txt):
+                    results.append(path)
+            except Exception:
+                continue
+
+    def _priority(path: Path) -> tuple[int, str]:
+        rel = str(path.resolve())
+        in_demo = "/synth_demo/" in rel or rel.endswith("/synth_demo/task_app.py")
+        return (0 if in_demo else 1, rel)
+
+    results.sort(key=_priority)
+    return results
+
+
+def ensure_task_app_ready(env: DemoEnv, synth_key: str, *, label: str) -> DemoEnv:
+    persist_path = demo_core.load_demo_dir() or os.getcwd()
+    user_config_map = load_user_config()
+
+    env_key = (env.env_api_key or "").strip()
+    if not env_key:
+        raise RuntimeError(
+            f"[{label}] ENVIRONMENT_API_KEY missing. Run `uvx synth-ai demo deploy` first."
+        )
+
+    template_id = demo_core.load_template_id()
+    allow_local = template_id == "crafter-local"
+
+    task_url = env.task_app_base_url
+    url_ok = is_modal_public_url(task_url) or (allow_local and is_local_demo_url(task_url or ""))
+    if not task_url or not url_ok:
+        resolved = task_url or ""
+        dynamic_lookup_allowed = env.task_app_name and not (
+            allow_local and is_local_demo_url(task_url or "")
+        )
+        if dynamic_lookup_allowed and not is_modal_public_url(resolved):
+            code, out = popen_capture(
+                [
+                    "uv",
+                    "run",
+                    "python",
+                    "-m",
+                    "modal",
+                    "app",
+                    "url",
+                    env.task_app_name,
+                ]
+            )
+            if code == 0 and out:
+                for token in out.split():
+                    if is_modal_public_url(token):
+                        resolved = token.strip().rstrip("/")
+                        break
+        if dynamic_lookup_allowed and not is_modal_public_url(resolved):
+            try:
+                choice = (
+                    input(
+                        f"Resolve URL from Modal for app '{env.task_app_name}'? [Y/n]: "
+                    ).strip().lower()
+                    or "y"
+                )
+            except Exception:
+                choice = "y"
+            if choice.startswith("y"):
+                code, out = popen_capture(
+                    [
+                        "uv",
+                        "run",
+                        "python",
+                        "-m",
+                        "modal",
+                        "app",
+                        "url",
+                        env.task_app_name,
+                    ]
+                )
+                if code == 0 and out:
+                    for token in out.split():
+                        if is_modal_public_url(token):
+                            resolved = token.strip().rstrip("/")
+                            break
+        if not is_modal_public_url(resolved):
+            hint = "Examples: https://<app-name>-fastapi-app.modal.run"
+            if allow_local:
+                hint += " or http://127.0.0.1:8001"
+            print(f"[{label}] Task app URL not configured or not a valid target.")
+            print(hint)
+            entered = input(
+                "Enter Task App base URL (must contain '.modal.run'), or press Enter to abort: "
+            ).strip()
+            if not entered:
+                raise RuntimeError(f"[{label}] Task App URL is required.")
+            entered_clean = entered.rstrip("/")
+            if not (
+                is_modal_public_url(entered_clean)
+                or (allow_local and is_local_demo_url(entered_clean))
+            ):
+                raise RuntimeError(f"[{label}] Valid Task App URL is required.")
+            task_url = entered_clean
+        else:
+            task_url = resolved
+        demo_core.persist_task_url(task_url, name=(env.task_app_name or None), path=persist_path)
+
+    app_name = (env.task_app_name or "").strip()
+    requires_modal_name = is_modal_public_url(task_url)
+    if requires_modal_name and not app_name:
+        fallback = input("Enter Modal app name for the task app (required): ").strip()
+        if not fallback:
+            raise RuntimeError(f"[{label}] Task app name is required.")
+        app_name = fallback
+        demo_core.persist_task_url(task_url, name=app_name, path=persist_path)
+
+    demo_core.persist_task_url(task_url, name=app_name if requires_modal_name else None, path=persist_path)
+    if synth_key:
+        os.environ["SYNTH_API_KEY"] = synth_key
+
+    openai_key = (
+        os.environ.get("OPENAI_API_KEY")
+        or str(user_config_map.get("OPENAI_API_KEY") or "")
+    ).strip()
+    if openai_key:
+        os.environ["OPENAI_API_KEY"] = openai_key
+
+    print(f"[{label}] Verifying rollout health:")
+    try:
+        preview = mask_str(env_key)
+        print(f"[{label}] {preview}")
+    except Exception:
+        pass
+    health_base = task_url.rstrip("/")
+    health_urls = [f"{health_base}/health/rollout", f"{health_base}/health"]
+    rc = 0
+    body: Any = ""
+    for h in health_urls:
+        print(f"[{label}] GET", h)
+        rc, body = http_request("GET", h, headers={"X-API-Key": env_key})
+        if rc == 200:
+            break
+    print(f"[{label}] status: {rc}")
+    try:
+        preview_body = json.dumps(body)[:800] if isinstance(body, dict) else str(body)[:800]
+    except Exception:
+        preview_body = str(body)[:800]
+    print(f"[{label}] body:", preview_body)
+    if rc != 200:
+        print(f"[{label}] Warning: rollout health check failed ({rc}). Response: {body}")
+        with contextlib.suppress(Exception):
+            print(f"[{label}] Sent header X-API-Key → {mask_str(env_key)}")
+    else:
+        print(f"[{label}] Task app rollout health check OK.")
+
+    os.environ["TASK_APP_BASE_URL"] = task_url
+    os.environ["ENVIRONMENT_API_KEY"] = env_key
+    os.environ["TASK_APP_SECRET_NAME"] = DEFAULT_TASK_APP_SECRET_NAME
+    updated_env = demo_core.load_env()
+    updated_env.env_api_key = env_key
+    updated_env.task_app_base_url = task_url
+    updated_env.task_app_name = app_name if requires_modal_name else ""
+    updated_env.task_app_secret_name = DEFAULT_TASK_APP_SECRET_NAME
+    return updated_env
+
+
+def ensure_modal_installed() -> None:
+    """Install the modal package if it is not already available and check authentication."""
+    modal_installed = False
+    try:
+        import importlib.util as import_util
+
+        if import_util.find_spec("modal") is not None:
+            modal_installed = True
+    except Exception:
+        pass
+
+    if not modal_installed:
+        print("modal not found; installing…")
+        try:
+            if shutil.which("uv"):
+                code, out = popen_capture(["uv", "pip", "install", "modal>=1.1.4"])
+            else:
+                code, out = popen_capture([sys.executable, "-m", "pip", "install", "modal>=1.1.4"])
+            if code != 0:
+                print(out)
+                print("Failed to install modal; continuing may fail.")
+                return
+            print("✓ modal installed successfully")
+            modal_installed = True
+        except Exception as exc:
+            print(f"modal install error: {exc}")
+            return
+
+    if modal_installed:
+        try:
+            import importlib.util as import_util
+
+            if import_util.find_spec("modal") is None:
+                print("Warning: modal is still not importable after install attempt.")
+                return
+        except Exception:
+            print("Warning: unable to verify modal installation.")
+            return
+
+    auth_ok, auth_msg = demo_core.modal_auth_status()
+    if auth_ok:
+        print(f"✓ Modal authenticated: {auth_msg}")
+    else:
+        print("\n⚠️  Modal authentication required")
+        print(f"   Status: {auth_msg}")
+        print("\n   To authenticate Modal, run:")
+        print("     modal setup")
+        print("\n   Or set environment variables:")
+        print("     export MODAL_TOKEN_ID=your-token-id")
+        print("     export MODAL_TOKEN_SECRET=your-token-secret")
+        print("\n   You can deploy later after authenticating.\n")

synth_ai/utils/paths.py

@@ -0,0 +1,95 @@
+import os
+import shutil
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def is_py_file(path: Path) -> bool:
+    path = path.resolve()
+    if not path.is_file():
+        return False
+    return path.suffix == ".py"
+
+
+def get_bin_path(name: str) -> Path | None:
+    path = shutil.which(name)
+    return Path(path) if path else None
+
+
+def get_env_file_paths(base_dir: str | Path = '.') -> list[Path]:
+    base = Path(base_dir).resolve()
+    return [path for path in base.rglob(".env*") if path.is_file()]
+
+
+def get_home_config_file_paths(
+    dir_name: str,
+    file_extension: str = "json"
+) -> list[Path]:
+    dir = Path.home() / dir_name
+    if not dir.exists():
+        return []
+    return [path for path in dir.glob(f"*.{file_extension}") if path.is_file()]
+
+
+def find_config_path(
+    bin: Path,
+    home_subdir: str,
+    filename: str,
+) -> Path | None:
+    """
+    Return a config file located in the user's home directory or alongside the binary.
+
+    Args:
+        bin_path: Resolved path to the executable.
+        home_subdir: Directory under the user's home to inspect (e.g., ".codex").
+        filename: Name of the config file to locate.
+    """
+    home_candidate = Path.home() / home_subdir / filename
+    if home_candidate.exists():
+        return home_candidate
+
+    local_candidate = Path(bin).parent / home_subdir / filename
+    if local_candidate.exists():
+        return local_candidate
+
+    return None
+
+
+def configure_import_paths(
+    app: Path,
+    repo_root: Path | None = REPO_ROOT
+) -> None:
+    app_dir = app.parent.resolve()
+
+    initial_dirs: list[Path] = [app_dir]
+    if (app_dir / "__init__.py").exists():
+        initial_dirs.append(app_dir.parent.resolve())
+    if repo_root:
+        initial_dirs.append(repo_root)
+
+    unique_dirs: list[str] = []
+    for dir in initial_dirs:
+        dir_str = str(dir)
+        if dir_str and dir_str not in unique_dirs:
+            unique_dirs.append(dir_str)
+
+    existing_pythonpath_dirs = os.environ.get("PYTHONPATH")
+    if existing_pythonpath_dirs:
+        for segment in existing_pythonpath_dirs.split(os.pathsep):
+            if segment and segment not in unique_dirs:
+                unique_dirs.append(segment)
+
+    os.environ["PYTHONPATH"] = os.pathsep.join(unique_dirs)
+
+    for dir in reversed(unique_dirs):
+        if dir and dir not in sys.path:
+            sys.path.insert(0, dir)
+
+
+def cleanup_paths(*, file: Path, dir: Path) -> None:
+    if not file.is_relative_to(dir):
+        raise ValueError(f"{file} is not inside {dir}")
+    file.unlink(missing_ok=True)
+    shutil.rmtree(dir, ignore_errors=True)