synth-ai 0.2.16__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/streaming/streamer.py

@@ -0,0 +1,320 @@
+from __future__ import annotations
+
+import contextlib
+import random
+from dataclasses import dataclass
+from typing import Any, Iterable, Sequence
+
+from synth_ai.http import AsyncHttpClient, sleep
+
+from .config import StreamConfig
+from .handlers import StreamHandler
+from .types import StreamMessage, StreamType
+
+TERMINAL_STATUSES = {"succeeded", "failed", "cancelled", "canceled", "completed"}
+TERMINAL_EVENT_SUCCESS = {
+    "sft.job.completed",
+    "rl.train.completed",
+    "rl.job.completed",
+    "workflow.completed",
+    "training.completed",
+}
+TERMINAL_EVENT_FAILURE = {
+    "sft.job.failed",
+    "rl.train.failed",
+    "rl.job.failed",
+    "workflow.failed",
+    "training.failed",
+}
+
+
+@dataclass(slots=True)
+class StreamEndpoints:
+    """Collection of endpoint paths (with optional fallbacks) to poll for a job."""
+
+    status: str | None
+    events: str | None = None
+    metrics: str | None = None
+    timeline: str | None = None
+    status_fallbacks: tuple[str, ...] = ()
+    event_fallbacks: tuple[str, ...] = ()
+    metric_fallbacks: tuple[str, ...] = ()
+    timeline_fallbacks: tuple[str, ...] = ()
+
+    @classmethod
+    def learning(cls, job_id: str) -> StreamEndpoints:
+        base = f"/learning/jobs/{job_id}"
+        return cls(
+            status=base,
+            events=f"{base}/events",
+            metrics=f"{base}/metrics",
+            timeline=f"{base}/timeline",
+        )
+
+    @classmethod
+    def prompt_learning(cls, job_id: str) -> StreamEndpoints:
+        """Endpoints for prompt learning jobs (MIPRO/GEPA)."""
+        base = f"/prompt-learning/online/jobs/{job_id}"
+        return cls(
+            status=base,
+            events=f"{base}/events",
+            metrics=f"{base}/metrics",
+            timeline=None,
+            status_fallbacks=(
+                f"/learning/jobs/{job_id}",
+                f"/orchestration/jobs/{job_id}",
+            ),
+            event_fallbacks=(
+                f"/learning/jobs/{job_id}/events",
+                f"/orchestration/jobs/{job_id}/events",
+            ),
+        )
+
+    @classmethod
+    def rl(cls, job_id: str) -> StreamEndpoints:
+        base = f"/rl/jobs/{job_id}"
+        return cls(
+            status=base,
+            events=f"{base}/events",
+            metrics=f"{base}/metrics",
+            timeline=f"{base}/timeline",
+            status_fallbacks=(
+                f"/learning/jobs/{job_id}",
+                f"/orchestration/jobs/{job_id}",
+            ),
+            event_fallbacks=(
+                f"/learning/jobs/{job_id}/events",
+                f"/orchestration/jobs/{job_id}/events",
+            ),
+            metric_fallbacks=(
+                f"/learning/jobs/{job_id}/metrics",
+            ),
+            timeline_fallbacks=(
+                f"/learning/jobs/{job_id}/timeline",
+            ),
+        )
+
+
+class JobStreamer:
+    """Poll job endpoints and dispatch messages to configured handlers."""
+
+    def __init__(
+        self,
+        *,
+        base_url: str,
+        api_key: str,
+        job_id: str,
+        endpoints: StreamEndpoints | None = None,
+        config: StreamConfig | None = None,
+        handlers: Sequence[StreamHandler] | None = None,
+        interval_seconds: float = 2.0,
+        timeout_seconds: float | None = None,
+        http_timeout: float = 60.0,
+        http_client: AsyncHttpClient | None = None,
+        sleep_fn= sleep,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.job_id = job_id
+        self.endpoints = endpoints or StreamEndpoints.learning(job_id)
+        self.config = config or StreamConfig.default()
+        self.handlers: list[StreamHandler] = list(handlers or [])
+        self.interval_seconds = interval_seconds
+        self.timeout_seconds = timeout_seconds
+        self.http_timeout = http_timeout
+        self._http = http_client
+        self._sleep = sleep_fn
+
+        status_sources: list[str | None] = [self.endpoints.status]
+        status_sources.extend(self.endpoints.status_fallbacks)
+        self._status_paths = [p for p in status_sources if p]
+
+        event_sources: list[str | None] = [self.endpoints.events]
+        event_sources.extend(self.endpoints.event_fallbacks)
+        self._event_paths = [p for p in event_sources if p]
+
+        metric_sources: list[str | None] = [self.endpoints.metrics]
+        metric_sources.extend(self.endpoints.metric_fallbacks)
+        self._metric_paths = [p for p in metric_sources if p]
+
+        timeline_sources: list[str | None] = [self.endpoints.timeline]
+        timeline_sources.extend(self.endpoints.timeline_fallbacks)
+        self._timeline_paths = [p for p in timeline_sources if p]
+
+        self._last_seq_by_stream: dict[str, int] = {}
+        self._last_step_by_metric: dict[str, int] = {}
+        self._seen_messages: set[str] = set()
+        self._last_status_payload: dict[str, Any] | None = None
+        self._last_status_value: str | None = None
+        self._terminal_seen = False
+        self._terminal_event_status: str | None = None
+
+        if not self.handlers:
+            from .handlers import CLIHandler
+
+            self.handlers = [CLIHandler()]
+
+    async def stream_until_terminal(self) -> dict[str, Any]:
+        """Stream configured endpoints until the job reaches a terminal state."""
+        http_cm = self._http or AsyncHttpClient(self.base_url, self.api_key, timeout=self.http_timeout)
+        async with http_cm as http:
+            while True:
+                status = await self._refresh_status(http)
+
+                event_messages = await self._poll_events(http)
+                metric_messages = await self._poll_metrics(http)
+                timeline_messages = await self._poll_timeline(http)
+
+                self._dispatch(event_messages + metric_messages + timeline_messages)
+
+                if self._terminal_seen or (status and status in TERMINAL_STATUSES):
+                    break
+
+                await self._sleep(self.interval_seconds)
+
+        for handler in self.handlers:
+            with contextlib.suppress(Exception):
+                handler.flush()
+
+        final_status = self._terminal_event_status or self._last_status_value or "unknown"
+        if self._last_status_payload:
+            self._last_status_payload["status"] = final_status
+            return self._last_status_payload
+        return {"job_id": self.job_id, "status": final_status}
+
+    async def _refresh_status(self, http: AsyncHttpClient) -> str:
+        status_payload = await self._poll_status(http)
+        if status_payload:
+            self._last_status_payload = status_payload
+            status = str(status_payload.get("status") or status_payload.get("state") or "").lower()
+            if status:
+                self._last_status_value = status
+                if status in TERMINAL_STATUSES:
+                    self._terminal_seen = True
+            return status
+        return self._last_status_value or ""
+
+    async def _poll_status(self, http: AsyncHttpClient) -> dict[str, Any] | None:
+        if StreamType.STATUS not in self.config.enabled_streams or not self._status_paths:
+            return None
+
+        for path in self._status_paths:
+            try:
+                data = await http.get(path)
+            except Exception:
+                continue
+            if isinstance(data, dict):
+                message = StreamMessage.from_status(self.job_id, data)
+                self._dispatch([message])
+                return data
+        return None
+
+    async def _poll_events(self, http: AsyncHttpClient) -> list[StreamMessage]:
+        if StreamType.EVENTS not in self.config.enabled_streams or not self._event_paths:
+            return []
+        messages: list[StreamMessage] = []
+        total = 0
+        for path in self._event_paths:
+            since = self._last_seq_by_stream.get(path, 0)
+            params = {"since_seq": since, "limit": 200}
+            try:
+                data = await http.get(path, params=params)
+            except Exception:
+                continue
+            raw_events = _extract_list(data, "events")
+            for event in raw_events:
+                seq = int(event.get("seq") or 0)
+                if seq <= self._last_seq_by_stream.get(path, 0):
+                    continue
+                if not self.config.should_include_event(event):
+                    continue
+                self._last_seq_by_stream[path] = seq
+                event_job_id = event.get("job_id") or self.job_id
+                event_message = StreamMessage.from_event(event_job_id, event)
+                event_type = str(event.get("type") or "").lower()
+                if event_type in TERMINAL_EVENT_SUCCESS:
+                    self._terminal_seen = True
+                    self._terminal_event_status = "succeeded"
+                elif event_type in TERMINAL_EVENT_FAILURE:
+                    self._terminal_seen = True
+                    self._terminal_event_status = "failed"
+                messages.append(event_message)
+                total += 1
+                if self.config.max_events_per_poll and total >= self.config.max_events_per_poll:
+                    return messages
+        return messages
+
+    async def _poll_metrics(self, http: AsyncHttpClient) -> list[StreamMessage]:
+        if StreamType.METRICS not in self.config.enabled_streams or not self._metric_paths:
+            return []
+        messages: list[StreamMessage] = []
+        for path in self._metric_paths:
+            after = max(self._last_step_by_metric.values()) if self._last_step_by_metric else -1
+            params = {"after_step": after, "limit": 200}
+            try:
+                data = await http.get(path, params=params)
+            except Exception:
+                continue
+            points = _extract_list(data, "points")
+            for point in points:
+                name = point.get("name", "")
+                step = int(point.get("step") or -1)
+                if step <= self._last_step_by_metric.get(name, -1):
+                    continue
+                if not self.config.should_include_metric(point):
+                    continue
+                self._last_step_by_metric[name] = step
+                metric_job_id = point.get("job_id") or self.job_id
+                messages.append(StreamMessage.from_metric(metric_job_id, point))
+        return messages
+
+    async def _poll_timeline(self, http: AsyncHttpClient) -> list[StreamMessage]:
+        if StreamType.TIMELINE not in self.config.enabled_streams or not self._timeline_paths:
+            return []
+        messages: list[StreamMessage] = []
+        for path in self._timeline_paths:
+            try:
+                data = await http.get(path)
+            except Exception:
+                continue
+
+            timeline_entries = _extract_list(data, "events")
+            for entry in timeline_entries:
+                if not self.config.should_include_timeline(entry):
+                    continue
+                timeline_job_id = entry.get("job_id") or self.job_id
+                phase = str(entry.get("phase") or "").lower()
+                if phase in TERMINAL_STATUSES:
+                    self._terminal_seen = True
+                    if phase in {"failed", "cancelled", "canceled"}:
+                        self._terminal_event_status = "failed"
+                    elif phase:
+                        self._terminal_event_status = "succeeded"
+                messages.append(StreamMessage.from_timeline(timeline_job_id, entry))
+        return messages
+
+    def _dispatch(self, messages: Iterable[StreamMessage]) -> None:
+        for message in messages:
+            if self.config.deduplicate and message.key in self._seen_messages:
+                continue
+            if self.config.sample_rate < 1.0 and random.random() > self.config.sample_rate:
+                continue
+            if self.config.deduplicate:
+                self._seen_messages.add(message.key)
+
+            for handler in self.handlers:
+                try:
+                    if handler.should_handle(message):
+                        handler.handle(message)
+                except Exception:
+                    pass
+
+
+def _extract_list(data: Any, field: str) -> list[dict[str, Any]]:
+    raw = (data or {}).get(field) if isinstance(data, dict) else None
+    if isinstance(raw, list):
+        return [item for item in raw if isinstance(item, dict)]
+    return []
+
+
+__all__ = ["JobStreamer", "StreamEndpoints"]

synth_ai/streaming/types.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import Any
+
+
+class StreamType(Enum):
+    """Categories of streaming payloads emitted by training jobs."""
+
+    STATUS = auto()
+    EVENTS = auto()
+    METRICS = auto()
+    TIMELINE = auto()
+
+    @property
+    def endpoint_path(self) -> str:
+        """Return the endpoint suffix used when polling this stream."""
+        return {
+            StreamType.STATUS: "",
+            StreamType.EVENTS: "/events",
+            StreamType.METRICS: "/metrics",
+            StreamType.TIMELINE: "/timeline",
+        }[self]
+
+
+@dataclass(slots=True)
+class StreamMessage:
+    """Unified representation of a streaming payload."""
+
+    stream_type: StreamType
+    timestamp: str
+    job_id: str
+    data: dict[str, Any]
+    seq: int | None = None
+    step: int | None = None
+    phase: str | None = None
+
+    @property
+    def key(self) -> str:
+        """Return a unique identifier used for deduplication."""
+        if self.stream_type is StreamType.EVENTS:
+            return f"event:{self.seq}"
+        if self.stream_type is StreamType.METRICS:
+            name = self.data.get("name", "")
+            return f"metric:{name}:{self.step}"
+        if self.stream_type is StreamType.TIMELINE:
+            return f"timeline:{self.phase}:{self.timestamp}"
+        return f"status:{self.timestamp}"
+
+    @classmethod
+    def from_status(cls, job_id: str, status_data: dict[str, Any]) -> StreamMessage:
+        """Create a message representing a job status payload."""
+        return cls(
+            stream_type=StreamType.STATUS,
+            timestamp=status_data.get("updated_at", "") or status_data.get("created_at", ""),
+            job_id=job_id,
+            data=status_data,
+        )
+
+    @classmethod
+    def from_event(cls, job_id: str, event_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a job event."""
+        return cls(
+            stream_type=StreamType.EVENTS,
+            timestamp=event_data.get("created_at", ""),
+            job_id=job_id,
+            data=event_data,
+            seq=event_data.get("seq"),
+        )
+
+    @classmethod
+    def from_metric(cls, job_id: str, metric_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a metric point."""
+        return cls(
+            stream_type=StreamType.METRICS,
+            timestamp=metric_data.get("created_at", ""),
+            job_id=job_id,
+            data=metric_data,
+            step=metric_data.get("step"),
+        )
+
+    @classmethod
+    def from_timeline(cls, job_id: str, timeline_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a status timeline entry."""
+        return cls(
+            stream_type=StreamType.TIMELINE,
+            timestamp=timeline_data.get("created_at", ""),
+            job_id=job_id,
+            data=timeline_data,
+            phase=timeline_data.get("phase"),
+        )
+
+
+__all__ = ["StreamMessage", "StreamType"]

synth_ai/task/apps/__init__.py

@@ -22,6 +22,7 @@ class ModalDeploymentConfig:
     extra_local_dirs: Sequence[tuple[str, str]] = field(default_factory=tuple)
     secret_names: Sequence[str] = field(default_factory=tuple)
     volume_mounts: Sequence[tuple[str, str]] = field(default_factory=tuple)
+    env_vars: dict[str, str] = field(default_factory=dict)
     timeout: int = 600
     memory: int = 4096
     cpu: float = 2.0

synth_ai/task/config.py

@@ -257,3 +257,5 @@ class FilterConfig:
         output_path.parent.mkdir(parents=True, exist_ok=True)
         return output_path
 
+
+

synth_ai/task/tracing_utils.py

@@ -26,34 +26,34 @@ def tracing_env_enabled(default: bool = False) -> bool:
 
 
 def resolve_tracing_db_url() -> str | None:
-    """Resolve tracing database URL and prefer async drivers for SQLite."""
-
-    db_url = os.getenv("TURSO_LOCAL_DB_URL")
-    if db_url:
+    """Resolve tracing database URL using centralized tracing_v3 config logic.
+    
+    This delegates to synth_ai.tracing_v3.config.resolve_trace_db_settings() which
+    handles Modal detection, remote Turso, local sqld, and SQLite fallbacks.
+    """
+    try:
+        from synth_ai.tracing_v3.config import resolve_trace_db_settings
+        db_url, _ = resolve_trace_db_settings(ensure_dir=True)
         return db_url
-
-    sqld_path = os.getenv("SQLD_DB_PATH")
-    if sqld_path:
-        path = Path(sqld_path).expanduser()
-        if path.is_dir():
-            candidate = path / "dbs" / "default" / "data"
-            candidate.parent.mkdir(parents=True, exist_ok=True)
-            return f"sqlite+aiosqlite:///{candidate}"
-        else:
-            path.parent.mkdir(parents=True, exist_ok=True)
-            return f"sqlite+aiosqlite:///{path}"
-
-    existing = os.getenv("TASKAPP_TRACE_DB_PATH")
-    if existing:
-        path = Path(existing).expanduser()
-    else:
+    except ImportError:
+        # Fallback if tracing_v3 is not available (shouldn't happen in normal usage)
+        db_url = (
+            os.getenv("TURSO_LOCAL_DB_URL")
+            or os.getenv("LIBSQL_URL")
+            or os.getenv("SYNTH_TRACES_DB")
+        )
+        if db_url:
+            return db_url
+        
+        # Auto-provision local sqld location for callers that rely on trace directories.
         base_dir = TRACE_DB_DIR.expanduser()
         base_dir.mkdir(parents=True, exist_ok=True)
-        path = base_dir / canonical_trace_db_name(timestamp=datetime.now())
-        os.environ["TASKAPP_TRACE_DB_PATH"] = str(path)
-        os.environ.setdefault("SQLD_DB_PATH", str(path))
-    path.parent.mkdir(parents=True, exist_ok=True)
-    return f"sqlite+aiosqlite:///{path}"
+        candidate = base_dir / canonical_trace_db_name(timestamp=datetime.now())
+        os.environ["TASKAPP_TRACE_DB_PATH"] = str(candidate)
+        os.environ.setdefault("SQLD_DB_PATH", str(candidate))
+        
+        default_url = os.getenv("LIBSQL_DEFAULT_URL", "http://127.0.0.1:8081")
+        return default_url
 
 
 def build_tracer_factory(

synth_ai/task/validators.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import re
-from typing import Any, cast
+from typing import Any
 from urllib.parse import urlparse, urlunparse
 
 import click
@@ -133,13 +133,46 @@ def normalize_inference_url(url: str | None, *, default: str = "https://api.open
     if not candidate:
         candidate = default
     
-    # Parse the URL to separate path and query components
     parsed = urlparse(candidate)
-    
+    path = (parsed.path or "").rstrip("/")
+    query = parsed.query or ""
+
+    # Repair malformed URLs where the completions path ended up in the query string.
+    # Example: https://host?cid=trace/v1/chat/completions  ->  https://host/v1/chat/completions?cid=trace
+    if query and "/" in query:
+        base_query, remainder = query.split("/", 1)
+        remainder_path = remainder
+        extra_query = ""
+        for separator in ("&", "?"):
+            idx = remainder_path.find(separator)
+            if idx != -1:
+                extra_query = remainder_path[idx + 1 :]
+                remainder_path = remainder_path[:idx]
+                break
+
+        query_path = "/" + remainder_path.lstrip("/")
+        merged_query_parts: list[str] = []
+        if base_query:
+            merged_query_parts.append(base_query)
+        if extra_query:
+            merged_query_parts.append(extra_query)
+        merged_query = "&".join(part for part in merged_query_parts if part)
+
+        if query_path and query_path != "/":
+            combined_path = f"{path.rstrip('/')}{query_path}" if path else query_path
+        else:
+            combined_path = path
+
+        parsed = parsed._replace(path=combined_path or "", query=merged_query)
+        path = (parsed.path or "").rstrip("/")
+        query = parsed.query or ""
+
     # Check if path already ends with a completions endpoint
-    path = parsed.path.rstrip('/')
     if path.endswith("/v1/chat/completions") or path.endswith("/chat/completions"):
-        return candidate
+        final_query = parsed.query or ""
+        if final_query and "/" in final_query:
+            parsed = parsed._replace(query=final_query.split("/", 1)[0])
+        return urlunparse(parsed)
     
     # Determine what to append based on existing path
     if path.endswith("/v1"):
@@ -147,11 +180,14 @@ def normalize_inference_url(url: str | None, *, default: str = "https://api.open
     elif path.endswith("/chat"):
         new_path = f"{path}/completions"
     else:
-        # Default: append full path
         new_path = f"{path}/v1/chat/completions" if path else "/v1/chat/completions"
-    
-    # Reconstruct URL with new path and original query/fragment
-    return cast(str, urlunparse(parsed._replace(path=new_path)))
+
+    parsed = parsed._replace(path=new_path)
+    final_query = parsed.query or ""
+    if final_query and "/" in final_query:
+        parsed = parsed._replace(query=final_query.split("/", 1)[0])
+
+    return urlunparse(parsed)
 
 
 def validate_task_app_url(url: str | None) -> str:

synth_ai/task_app_cfgs.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+from typing import Literal, Optional
+
+from pydantic import BaseModel
+
+
+class LocalTaskAppConfig(BaseModel):
+    task_app_path: Path
+    trace: bool = True
+    host: str = "127.0.0.1"
+    port: int = 8000
+
+
+
+class ModalTaskAppConfig(BaseModel):
+    task_app_path: Path
+    modal_app_path: Path
+    modal_bin_path: Path
+    cmd_arg: Literal["deploy", "serve"] = "deploy"
+    task_app_name: Optional[str] = None
+    dry_run: bool = False