synth-ai 0.2.13.dev1__py3-none-any.whl → 0.2.14__py3-none-any.whl

synth_ai/task/validators.py

@@ -1,11 +1,418 @@
+"""Task app validation utilities."""
+
 from __future__ import annotations
 
-from urllib.parse import urlparse
+import re
+from typing import Any
+from urllib.parse import urlparse, urlunparse
+
+import click
+import httpx
+
+from synth_ai.task.contracts import TaskAppEndpoints  # type: ignore[attr-defined]
+
+
+def validate_rollout_response_for_rl(response_data: dict[str, Any], *, warn_only: bool = False) -> list[str]:
+    """Validate that a task app rollout response has required fields for RL training.
+    
+    The backend RL trainer requires:
+    1. pipeline_metadata["inference_url"] at top level (with ?cid= for trace correlation)
+    2. Each step's info.meta["inference_url"] must be present (nested structure!)
+    
+    Args:
+        response_data: The rollout response dict from task app
+        warn_only: If True, return warnings instead of raising exceptions
+        
+    Returns:
+        List of validation warnings/errors
+        
+    Raises:
+        ValueError: If critical fields are missing (unless warn_only=True)
+    """
+    issues = []
+    
+    # Check pipeline_metadata
+    pipeline_metadata = response_data.get("pipeline_metadata")
+    if not isinstance(pipeline_metadata, dict):
+        issues.append("Missing or invalid 'pipeline_metadata' (required for RL training)")
+    else:
+        inference_url = pipeline_metadata.get("inference_url")
+        if not inference_url:
+            issues.append(
+                "pipeline_metadata['inference_url'] is missing. "
+                "RL trainer requires this field to extract traces."
+            )
+        elif not isinstance(inference_url, str):
+            issues.append(
+                f"pipeline_metadata['inference_url'] must be a string, got: {type(inference_url).__name__}"
+            )
+        elif "?cid=" not in inference_url:
+            issues.append(
+                f"pipeline_metadata['inference_url'] should contain '?cid=' for trace correlation. "
+                f"Got: {inference_url[:80]}..."
+            )
+    
+    # Check trajectories and steps
+    trajectories = response_data.get("trajectories", [])
+    if not trajectories:
+        issues.append("No trajectories found in response")
+    
+    for traj_idx, trajectory in enumerate(trajectories):
+        if not isinstance(trajectory, dict):
+            continue
+        
+        steps = trajectory.get("steps", [])
+        for step_idx, step in enumerate(steps):
+            if not isinstance(step, dict):
+                continue
+            
+            step_info = step.get("info", {})
+            if not isinstance(step_info, dict):
+                issues.append(
+                    f"trajectory[{traj_idx}].steps[{step_idx}].info is not a dict"
+                )
+                continue
+            
+            # Check for nested meta.inference_url (backend expects this structure!)
+            step_meta = step_info.get("meta", {})
+            if not isinstance(step_meta, dict):
+                issues.append(
+                    f"trajectory[{traj_idx}].steps[{step_idx}].info.meta is missing or not a dict. "
+                    f"RL trainer expects nested structure: info.meta.inference_url"
+                )
+                continue
+            
+            step_inference_url = step_meta.get("inference_url")
+            if not step_inference_url:
+                issues.append(
+                    f"trajectory[{traj_idx}].steps[{step_idx}].info.meta['inference_url'] is missing. "
+                    f"RL trainer needs this for trace extraction (nested structure required!)"
+                )
+            elif not isinstance(step_inference_url, str):
+                issues.append(
+                    f"trajectory[{traj_idx}].steps[{step_idx}].info.meta['inference_url'] must be a string, "
+                    f"got: {type(step_inference_url).__name__}"
+                )
+    
+    if issues and not warn_only:
+        error_msg = "Task app response validation failed for RL training:\n" + "\n".join(
+            f"  - {issue}" for issue in issues
+        )
+        raise ValueError(error_msg)
+    
+    return issues
+
+
+def normalize_inference_url(url: str | None, *, default: str = "https://api.openai.com/v1/chat/completions") -> str:
+    """Normalize an inference URL to include the /v1/chat/completions path.
+    
+    This utility ensures inference URLs have the correct path structure for OpenAI-compatible
+    chat completions endpoints, while preserving query parameters (e.g., ?cid=trace_123) 
+    that may be added for tracing.
+    
+    Args:
+        url: The inference URL to normalize (may be None or incomplete)
+        default: Default URL to use if url is None/empty
+        
+    Returns:
+        Normalized URL with proper path and preserved query parameters
+        
+    Examples:
+        >>> normalize_inference_url("https://api.groq.com")
+        'https://api.groq.com/v1/chat/completions'
+        
+        >>> normalize_inference_url("https://modal.host?cid=trace_123")
+        'https://modal.host/v1/chat/completions?cid=trace_123'
+        
+        >>> normalize_inference_url("https://api.openai.com/v1")
+        'https://api.openai.com/v1/chat/completions'
+        
+        >>> normalize_inference_url("https://api.groq.com/openai/v1/chat/completions")
+        'https://api.groq.com/openai/v1/chat/completions'
+    """
+    candidate = (url or default).strip()
+    if not candidate:
+        candidate = default
+    
+    # Parse the URL to separate path and query components
+    parsed = urlparse(candidate)
+    
+    # Check if path already ends with a completions endpoint
+    path = parsed.path.rstrip('/')
+    if path.endswith("/v1/chat/completions") or path.endswith("/chat/completions"):
+        return candidate
+    
+    # Determine what to append based on existing path
+    if path.endswith("/v1"):
+        new_path = f"{path}/chat/completions"
+    elif path.endswith("/chat"):
+        new_path = f"{path}/completions"
+    else:
+        # Default: append full path
+        new_path = f"{path}/v1/chat/completions" if path else "/v1/chat/completions"
+    
+    # Reconstruct URL with new path and original query/fragment
+    return urlunparse(parsed._replace(path=new_path))
+
+
+def validate_task_app_url(url: str | None) -> str:
+    """Validate and normalize a task app URL.
+    
+    Args:
+        url: URL to validate
+        
+    Returns:
+        Normalized URL
+        
+    Raises:
+        ValueError: If URL is invalid
+    """
+    if not url:
+        raise ValueError("Task app URL is required")
+    
+    url = url.strip().rstrip("/")
+    
+    # Basic URL validation
+    url_pattern = re.compile(
+        r"^https?://"  # http:// or https://
+        r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?|"  # domain...
+        r"localhost|"  # localhost...
+        r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"  # ...or ip
+        r"(?::\d+)?"  # optional port
+        r"(?:/?|[/?]\S+)$",
+        re.IGNORECASE,
+    )
+    
+    if not url_pattern.match(url):
+        raise ValueError(f"Invalid task app URL: {url}")
+    
+    return url
+
+
+def _print_success(msg: str) -> None:
+    """Print success message in green."""
+    click.echo(click.style(f"✓ {msg}", fg="green"))
+
+
+def _print_error(msg: str) -> None:
+    """Print error message in red."""
+    click.echo(click.style(f"✗ {msg}", fg="red"), err=True)
+
+
+def _print_warning(msg: str) -> None:
+    """Print warning message in yellow."""
+    click.echo(click.style(f"⚠ {msg}", fg="yellow"))
+
 
+def _print_info(msg: str) -> None:
+    """Print info message."""
+    click.echo(f"  {msg}")
 
-def validate_task_app_url(url: str, *, name: str = "TASK_APP_BASE_URL") -> None:
-    """Validate a Task App base URL (scheme + host present)."""
 
-    p = urlparse(url)
-    if p.scheme not in ("http", "https") or not p.netloc:
-        raise ValueError(f"Invalid {name}: malformed: {url}")
+async def validate_task_app_endpoint(
+    url: str,
+    api_key: str | None = None,
+    min_instances: int = 10,
+    verbose: bool = False,
+) -> tuple[bool, dict[str, Any]]:
+    """Validate a task app deployment.
+    
+    Returns:
+        (success: bool, results: dict)
+    """
+    results: dict[str, Any] = {
+        "url": url,
+        "endpoints": {},
+        "auth": {},
+        "task_instances": {},
+        "overall": False,
+    }
+    
+    all_passed = True
+    endpoints = TaskAppEndpoints()
+    
+    # Set up headers
+    headers = {}
+    if api_key:
+        headers["X-API-Key"] = api_key
+    
+    click.echo(f"\n{'='*60}")
+    click.echo(f"Validating Task App: {url}")
+    click.echo(f"{'='*60}\n")
+    
+    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+        # 1. Check root endpoint
+        click.echo("1. Checking root endpoint...")
+        try:
+            resp = await client.get(f"{url}{endpoints.root}")
+            if resp.status_code == 200:
+                data = resp.json()
+                _print_success(f"Root endpoint responds (status: {data.get('status')})")
+                results["endpoints"]["root"] = {"passed": True, "data": data}
+                if verbose:
+                    _print_info(f"Service: {data.get('service', 'N/A')}")
+            else:
+                _print_error(f"Root endpoint returned {resp.status_code}")
+                results["endpoints"]["root"] = {"passed": False, "status": resp.status_code}
+                all_passed = False
+        except Exception as e:
+            _print_error(f"Root endpoint failed: {e}")
+            results["endpoints"]["root"] = {"passed": False, "error": str(e)}
+            all_passed = False
+        
+        # 2. Check health endpoint
+        click.echo("\n2. Checking health endpoint...")
+        try:
+            resp = await client.get(f"{url}{endpoints.health}", headers=headers)
+            if resp.status_code == 200:
+                data = resp.json()
+                _print_success(f"Health endpoint responds (healthy: {data.get('healthy')})")
+                results["endpoints"]["health"] = {"passed": True, "data": data}
+                
+                # Check auth configuration
+                auth_info = data.get("auth", {})
+                if auth_info.get("required"):
+                    _print_info(f"Auth required: {auth_info.get('required')}")
+                    _print_info(f"Expected key prefix: {auth_info.get('expected_prefix', 'N/A')}")
+                    
+                    if api_key:
+                        _print_success("API key provided and accepted")
+                        results["auth"]["provided"] = True
+                        results["auth"]["accepted"] = True
+                    else:
+                        _print_warning("No API key provided but may be required")
+                        results["auth"]["provided"] = False
+                        results["auth"]["required"] = True
+            else:
+                _print_error(f"Health endpoint returned {resp.status_code}")
+                results["endpoints"]["health"] = {"passed": False, "status": resp.status_code}
+                all_passed = False
+                
+                if resp.status_code == 403:
+                    _print_error("Authentication failed - provide API key with --api-key")
+                    results["auth"]["error"] = "Authentication failed"
+                    
+        except Exception as e:
+            _print_error(f"Health endpoint failed: {e}")
+            results["endpoints"]["health"] = {"passed": False, "error": str(e)}
+            all_passed = False
+        
+        # 3. Check info endpoint
+        click.echo("\n3. Checking info endpoint...")
+        try:
+            resp = await client.get(f"{url}{endpoints.info}", headers=headers)
+            if resp.status_code == 200:
+                data = resp.json()
+                _print_success("Info endpoint responds")
+                results["endpoints"]["info"] = {"passed": True, "data": data}
+                
+                if verbose:
+                    service = data.get("service", {})
+                    task_info = service.get("task", {})
+                    if isinstance(task_info, dict):
+                        _print_info(f"Task: {task_info.get('name', 'N/A')}")
+                    _print_info(f"Version: {service.get('version', 'N/A')}")
+                    
+                    dataset = data.get("dataset", {})
+                    if isinstance(dataset, dict):
+                        _print_info(f"Dataset: {dataset.get('id', 'N/A')}")
+            else:
+                _print_error(f"Info endpoint returned {resp.status_code}")
+                results["endpoints"]["info"] = {"passed": False, "status": resp.status_code}
+                all_passed = False
+        except Exception as e:
+            _print_error(f"Info endpoint failed: {e}")
+            results["endpoints"]["info"] = {"passed": False, "error": str(e)}
+            all_passed = False
+        
+        # 4. Check task_info endpoint and instance count
+        click.echo("\n4. Checking task_info endpoint and instance availability...")
+        try:
+            # Get taskset descriptor first
+            resp = await client.get(f"{url}{endpoints.task_info}", headers=headers)
+            if resp.status_code == 200:
+                data = resp.json()
+                _print_success("Task info endpoint responds")
+                results["endpoints"]["task_info"] = {"passed": True}
+                
+                taskset = data.get("taskset", {})
+                if verbose and taskset:
+                    if isinstance(taskset, dict):
+                        _print_info(f"Taskset: {taskset.get('id', 'N/A')}")
+                    else:
+                        _print_info(f"Taskset: {taskset}")
+                
+                # Try to get specific task instances (seeds 0-19)
+                # Fetch instances one by one to verify we can get at least min_instances
+                instances = []
+                for seed in range(min_instances + 5):  # Try a few extra
+                    try:
+                        resp_seed = await client.get(
+                            f"{url}{endpoints.task_info}",
+                            params={"seed": seed},
+                            headers=headers,
+                        )
+                        if resp_seed.status_code == 200:
+                            instance = resp_seed.json()
+                            instances.append(instance)
+                        else:
+                            break  # Stop if we hit an invalid seed
+                    except Exception:
+                        break
+                
+                instance_count = len(instances)
+                results["task_instances"]["count"] = instance_count
+                results["task_instances"]["requested"] = min_instances
+                
+                if instance_count >= min_instances:
+                    _print_success(f"Found {instance_count} task instances (≥ {min_instances} required)")
+                    results["task_instances"]["passed"] = True
+                    
+                    if verbose and instances:
+                        sample = instances[0]
+                        task_info_sample = sample.get('task', {})
+                        if isinstance(task_info_sample, dict):
+                            _print_info(f"Sample task: {task_info_sample.get('name', 'N/A')}")
+                        _print_info(f"Environment: {sample.get('environment', 'N/A')}")
+                else:
+                    _print_error(f"Only {instance_count} task instances available (need ≥ {min_instances})")
+                    results["task_instances"]["passed"] = False
+                    all_passed = False
+            else:
+                _print_error(f"Task info endpoint returned {resp.status_code}")
+                results["endpoints"]["task_info"] = {"passed": False, "status": resp.status_code}
+                all_passed = False
+        except Exception as e:
+            _print_error(f"Task info endpoint failed: {e}")
+            results["endpoints"]["task_info"] = {"passed": False, "error": str(e)}
+            results["task_instances"]["passed"] = False
+            all_passed = False
+        
+        # 5. Check rollout endpoint structure (don't actually run a rollout)
+        click.echo("\n5. Checking rollout endpoint availability...")
+        try:
+            # Just check if it's registered (OPTIONS or a lightweight probe)
+            resp = await client.options(f"{url}{endpoints.rollout}", headers=headers)
+            # Many servers return 200 for OPTIONS, some return 405
+            if resp.status_code in (200, 204, 405):
+                _print_success("Rollout endpoint is registered")
+                results["endpoints"]["rollout"] = {"passed": True}
+            else:
+                _print_warning(f"Rollout endpoint returned unexpected status: {resp.status_code}")
+                results["endpoints"]["rollout"] = {"passed": True, "note": "endpoint exists"}
+        except Exception as e:
+            # OPTIONS might not be supported, that's okay
+            _print_info(f"Rollout endpoint check skipped (OPTIONS not supported): {e}")
+            results["endpoints"]["rollout"] = {"passed": True, "note": "assumed present"}
+    
+    # Summary
+    click.echo(f"\n{'='*60}")
+    if all_passed:
+        _print_success("All validations passed!")
+        click.echo(f"{'='*60}\n")
+    else:
+        _print_error("Some validations failed. See errors above.")
+        click.echo(f"{'='*60}\n")
+    
+    results["overall"] = all_passed
+    return all_passed, results

synth_ai/tracing_v3/abstractions.py

@@ -37,7 +37,7 @@ Concepts:
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass, field
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from typing import Any
 
 from .lm_call_record_abstractions import LLMCallRecord
@@ -249,7 +249,7 @@ class SessionTimeStep:
 
     step_id: str = ""
     step_index: int = 0
-    timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
+    timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
     turn_number: int | None = None
     events: list[BaseEvent] = field(default_factory=list)
     markov_blanket_messages: list[SessionEventMarkovBlanketMessage] = field(default_factory=list)
@@ -283,7 +283,7 @@ class SessionTrace:
     """
 
     session_id: str = ""
-    created_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
     session_time_steps: list[SessionTimeStep] = field(default_factory=list)
     event_history: list[BaseEvent] = field(default_factory=list)
     markov_blanket_message_history: list[SessionEventMarkovBlanketMessage] = field(

synth_ai/tracing_v3/decorators.py

@@ -37,10 +37,14 @@ from .utils import calculate_cost, detect_provider
 # Context variables for session and turn tracking
 # These variables automatically propagate across async call boundaries,
 # allowing deeply nested code to access tracing context without explicit passing
-_session_id_ctx: contextvars.ContextVar[str | None] = contextvars.ContextVar("session_id")
-_turn_number_ctx: contextvars.ContextVar[int | None] = contextvars.ContextVar("turn_number")
+_session_id_ctx: contextvars.ContextVar[str | None] = contextvars.ContextVar(
+    "session_id"
+)
+_turn_number_ctx: contextvars.ContextVar[int | None] = contextvars.ContextVar(
+    "turn_number"
+)
 _session_tracer_ctx: contextvars.ContextVar[Any | None] = contextvars.ContextVar(
-    "session_tracer", default=None
+    "session_tracer"
 )
 
 

synth_ai/tracing_v3/llm_call_record_helpers.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import uuid
 from dataclasses import dataclass, field
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from typing import Any, TypedDict, cast
 
 from .lm_call_record_abstractions import (
@@ -180,8 +180,8 @@ def create_llm_call_record_from_response(
         api_type=api_type,
         provider=provider,
         model_name=model_name,
-        started_at=started_at or datetime.now(UTC),
-        completed_at=completed_at or datetime.now(UTC),
+        started_at=started_at or datetime.now(timezone.utc),
+        completed_at=completed_at or datetime.now(timezone.utc),
         latency_ms=latency_ms,
         request_params=params,
         input_messages=input_messages,
@@ -376,8 +376,8 @@ def create_llm_call_record_from_streaming(
         api_type="responses",  # Streaming typically from Responses API
         provider=provider,
         model_name=model_name,
-        started_at=started_at or datetime.now(UTC),
-        completed_at=completed_at or datetime.now(UTC),
+        started_at=started_at or datetime.now(timezone.utc),
+        completed_at=completed_at or datetime.now(timezone.utc),
         latency_ms=latency_ms,
         request_params=params,
         input_messages=input_messages,

synth_ai/tracing_v3/replica_sync.py

@@ -25,15 +25,15 @@ application to continue without blocking on sync operations.
 """
 
 import asyncio
+import importlib
 import logging
-from typing import Any
-
-import libsql
+from typing import Any, cast
 
 from .config import CONFIG
 
 logger = logging.getLogger(__name__)
 
+libsql = cast(Any, importlib.import_module("libsql"))
 
 class ReplicaSync:
     """Manages synchronization of embedded SQLite replica with remote Turso database.
@@ -53,7 +53,7 @@ class ReplicaSync:
         db_path: str = "embedded.db",
         sync_url: str | None = None,
         auth_token: str | None = None,
-        sync_interval: int | None = None,
+        sync_interval: float | None = None,
     ):
         """Initialize replica sync manager.
 

synth_ai/tracing_v3/serialization.py

@@ -55,11 +55,11 @@ def normalize_for_json(value: Any) -> Any:
         return {str(k): normalize_for_json(v) for k, v in value.items()}
 
     # Sequences
-    if isinstance(value, (list, tuple, set)):
+    if isinstance(value, list | tuple | set):
         return [normalize_for_json(v) for v in value]
 
     # Datetime / Date
-    if isinstance(value, (datetime, date)):
+    if isinstance(value, datetime | date):
         return value.isoformat()
 
     # Decimal
@@ -73,7 +73,7 @@ def normalize_for_json(value: Any) -> Any:
             return str(value)
 
     # Bytes-like
-    if isinstance(value, (bytes, bytearray)):
+    if isinstance(value, bytes | bytearray):
         return base64.b64encode(bytes(value)).decode("ascii")
 
     # Enum
@@ -82,9 +82,9 @@ def normalize_for_json(value: Any) -> Any:
 
     # Numpy scalars / arrays
     if _np is not None:
-        if isinstance(value, (_np.generic,)):  # type: ignore[attr-defined]
+        if isinstance(value, _np.generic):  # type: ignore[attr-defined]
             return normalize_for_json(value.item())
-        if isinstance(value, (_np.ndarray,)):
+        if isinstance(value, _np.ndarray):
             return normalize_for_json(value.tolist())
 
     # Floats: sanitize NaN / Infinity to None

synth_ai/tracing_v3/session_tracer.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import asyncio
 import json
 from contextlib import asynccontextmanager
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from typing import Any
 
 from .abstractions import (
@@ -106,7 +106,7 @@ class SessionTracer:
 
             self._current_trace = SessionTrace(
                 session_id=session_id,
-                created_at=datetime.now(UTC),
+                created_at=datetime.now(timezone.utc),
                 session_time_steps=[],
                 event_history=[],
                 markov_blanket_message_history=[],
@@ -152,7 +152,7 @@ class SessionTracer:
         step = SessionTimeStep(
             step_id=step_id,
             step_index=len(self._current_trace.session_time_steps),
-            timestamp=datetime.now(UTC),
+            timestamp=datetime.now(timezone.utc),
             turn_number=turn_number,
             step_metadata=metadata or {},
         )
@@ -197,7 +197,7 @@ class SessionTracer:
             step = self._current_step
 
         if step and step.completed_at is None:
-            step.completed_at = datetime.now(UTC)
+            step.completed_at = datetime.now(timezone.utc)
 
             # Trigger hooks
             await self.hooks.trigger(
@@ -294,7 +294,7 @@ class SessionTracer:
             content=normalised_content,
             message_type=message_type,
             time_record=TimeRecord(
-                event_time=event_time or datetime.now(UTC).timestamp(), message_time=message_time
+                event_time=event_time or datetime.now(timezone.utc).timestamp(), message_time=message_time
             ),
             metadata=metadata or {},
         )
@@ -368,18 +368,28 @@ class SessionTracer:
             # End any open timesteps
             for step in self._current_trace.session_time_steps:
                 if step.completed_at is None:
-                    step.completed_at = datetime.now(UTC)
+                    step.completed_at = datetime.now(timezone.utc)
 
             # Trigger pre-save hooks
             await self.hooks.trigger("before_save", session=self._current_trace)
 
             # Save if requested
             should_save = save if save is not None else self.auto_save
+            
+            # Debug logging
+            import logging
+            _logger = logging.getLogger(__name__)
+            _logger.info(f"[TRACE_DEBUG] end_session: should_save={should_save}, self.db={self.db is not None}, auto_save={self.auto_save}")
+            
             if should_save and self.db:
+                _logger.info(f"[TRACE_DEBUG] Calling insert_session_trace with {len(self._current_trace.markov_blanket_message_history)} messages")
                 await self.db.insert_session_trace(self._current_trace)
+                _logger.info(f"[TRACE_DEBUG] insert_session_trace completed")
 
                 # Trigger post-save hooks
                 await self.hooks.trigger("after_save", session=self._current_trace)
+            else:
+                _logger.warning(f"[TRACE_DEBUG] Skipping save: should_save={should_save}, self.db={self.db is not None}")
 
             # Trigger session end hooks
             await self.hooks.trigger("session_end", session=self._current_trace)