synth-ai 0.2.13.dev2__py3-none-any.whl → 0.2.14__py3-none-any.whl

synth_ai/cli/traces.py

@@ -11,7 +11,7 @@ from rich import box
 from rich.console import Console
 from rich.table import Table
 
-from ._storage import load_storage
+from synth_ai.cli._storage import load_storage
 
 
 def register(cli):

synth_ai/cli/tui.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+CLI: Interactive TUI dashboard for Synth AI.
+"""
+
+import os
+
+import click
+from rich.console import Console
+
+
+def register(cli):
+    @cli.command()
+    @click.option(
+        "--url",
+        "db_url",
+        default="sqlite+libsql://http://127.0.0.1:8080",
+        help="Database URL (default: sqlite+libsql://http://127.0.0.1:8080)",
+    )
+    @click.option("--debug", is_flag=True, help="Enable debug logging")
+    def tui(db_url: str, debug: bool):
+        """Launch interactive TUI dashboard showing experiments, balance, and active runs."""
+        console = Console()
+
+        # Import here to avoid circular imports and handle optional dependencies
+        try:
+            from synth_ai.tui.dashboard import main as tui_main
+        except (ImportError, ModuleNotFoundError) as e:
+            console.print("[red]Error:[/red] TUI dashboard not available.")
+            console.print(f"Missing dependencies: {e}")
+            console.print("Install with: pip install textual")
+            return
+        except Exception as e:
+            # Handle other import errors (like missing libsql, type annotation issues, etc.)
+            console.print("[red]Error:[/red] TUI dashboard not available.")
+            console.print("This may be due to missing dependencies or Python version compatibility.")
+            console.print("Try: pip install textual libsql")
+            console.print("If using Python < 3.10, you may need to update Python or install eval_type_backport.")
+            return
+
+        # Set environment variables for the TUI to use
+        os.environ.setdefault("TUI_DB_URL", db_url)
+        if debug:
+            os.environ["TUI_DEBUG"] = "1"
+
+        # Run the TUI by calling the module directly with sanitized argv
+        try:
+            tui_args = ["--url", db_url]
+            if debug:
+                tui_args.append("--debug")
+            tui_main(tui_args)
+        except KeyboardInterrupt:
+            console.print("\n[blue]TUI closed.[/blue]")
+        except Exception as e:
+            console.print(f"\n[red]Error running TUI:[/red] {e}")
+            if debug:
+                raise

synth_ai/cli/turso.py

@@ -6,7 +6,7 @@ import subprocess
 
 import click
 
-from .root import SQLD_VERSION, find_sqld_binary, install_sqld
+from synth_ai.cli.root import SQLD_VERSION, find_sqld_binary, install_sqld
 
 
 def register(cli: click.Group) -> None:

synth_ai/cli/watch.py

@@ -16,7 +16,7 @@ from rich.console import Console, Group
 from rich.panel import Panel
 from rich.table import Table
 
-from ._storage import load_storage
+from synth_ai.cli._storage import load_storage
 
 
 def _open_db(db_url: str):

synth_ai/demos/demo_task_apps/crafter/grpo_crafter_task_app.py

@@ -57,7 +57,7 @@ def _load_build_config():
 build_config = _load_build_config()
 
 
-APP_ID = "grpo-crafter"
+APP_ID = "grpo-crafter-task-app"
 
 
 def _build_base_config() -> TaskAppConfig:

synth_ai/environments/examples/crafter_classic/environment.py

@@ -397,7 +397,7 @@ class CrafterClassicEnvironment(StatefulEnvironment, ReproducibleEnvironment[Cra
             priv_state, pub_state, self.custom_step_observation_callable
         )
         total_step_time = time.time() - step_start_time
-        logger.info(
+        logger.debug(
             f"CrafterClassic step completed in {total_step_time:.3f}s (interact: {interact_time:.3f}s)"
         )
         return obs

synth_ai/environments/examples/verilog/engine.py

@@ -46,7 +46,7 @@ class VerilogCompileSuccessComponent(RewardComponent):
         if hasattr(action, "get") and action.get("type") == "compile":
             # Check if compilation was successful (returncode 0)
             if action.get("returncode") == 0:
-                return 0.1
+                return 0.01  # Normalized: 0.1 / 10.0 = 0.01
         return 0.0
 
 
@@ -55,12 +55,12 @@ class VerilogSimulationPassComponent(RewardComponent):
         if hasattr(action, "get") and action.get("type") == "simulate":
             # Check if simulation passed
             if action.get("passed", False):
-                return 1.0
+                return 0.1  # Normalized: 1.0 / 10.0 = 0.1
         return 0.0
 
 
 class VerilogStepPenaltyComponent(RewardComponent):
-    def __init__(self, penalty: float = -0.01):
+    def __init__(self, penalty: float = 0.0):  # No per-step reward - only reward accomplishments
         self.penalty = penalty
 
     async def score(self, state: Any, action: Any) -> float:
@@ -68,12 +68,12 @@ class VerilogStepPenaltyComponent(RewardComponent):
 
 
 class VerilogSubmitSuccessComponent(RewardComponent):
-    """Reward for successful submission (tests passed)."""
+    """Reward for successful submission (tests passed). Max reward = 1.0 (normalized)."""
     async def score(self, state: VerilogPublicState, action: Any) -> float:
         if hasattr(action, "get") and action.get("type") == "submit":
             # Check if submission passed
             if action.get("passed", False):
-                return 10.0  # Large reward for completing the task correctly
+                return 1.0  # Normalized: Maximum reward is now 1.0
         return 0.0
 
 
@@ -83,6 +83,9 @@ class VerilogEngine(StatefulEngine):
     """
 
     def __init__(self, task_instance: TaskInstance):
+        # Validate required Verilog tools are available
+        self._validate_verilog_tools()
+        
         self.task_instance = task_instance
         self._total_reward = 0.0
         self._current_action_for_reward: Optional[Dict[str, Any]] = None
@@ -92,7 +95,7 @@ class VerilogEngine(StatefulEngine):
                 VerilogCompileSuccessComponent(),
                 VerilogSimulationPassComponent(),
                 VerilogSubmitSuccessComponent(),
-                VerilogStepPenaltyComponent(penalty=-0.01),
+                VerilogStepPenaltyComponent(penalty=0.0),  # No per-step reward
             ]
         )
 
@@ -103,6 +106,39 @@ class VerilogEngine(StatefulEngine):
         # Track last compile/simulate outputs
         self._last_compile_output: Optional[str] = None
         self._last_simulate_output: Optional[str] = None
+    
+    @staticmethod
+    def _validate_verilog_tools() -> None:
+        """Validate that required Verilog tools (iverilog, vvp) are available."""
+        missing_tools = []
+        
+        if not shutil.which("iverilog"):
+            missing_tools.append("iverilog")
+        if not shutil.which("vvp"):
+            missing_tools.append("vvp")
+        
+        if missing_tools:
+            error_msg = (
+                f"🚨🚨🚨 CRITICAL CONFIGURATION ERROR 🚨🚨🚨\n"
+                f"\n"
+                f"Missing required Verilog tools: {', '.join(missing_tools)}\n"
+                f"\n"
+                f"The Verilog environment CANNOT function without these tools.\n"
+                f"ALL compile/simulate operations will FAIL.\n"
+                f"ALL rewards will be ZERO.\n"
+                f"Training or evaluation will be COMPLETELY BROKEN.\n"
+                f"\n"
+                f"🔧 FIX THIS NOW:\n"
+                f"1. Add 'iverilog' to apt_packages in Modal deployment config\n"
+                f"2. Location: examples/task_apps/verilog/task_app/grpo_verilog.py\n"
+                f"3. Look for: modal=ModalDeploymentConfig(\n"
+                f"4. Add: apt_packages=('iverilog',)  # Provides both iverilog and vvp\n"
+                f"5. Redeploy: uvx synth-ai modal-serve grpo-verilog\n"
+                f"\n"
+                f"{'='*80}"
+            )
+            print(f"\n{'='*80}\n{error_msg}\n{'='*80}\n", flush=True)
+            raise RuntimeError(error_msg)
 
     async def _reset_engine(
         self, *, seed: Optional[int] = None
@@ -133,6 +169,13 @@ class VerilogEngine(StatefulEngine):
     ) -> Tuple[VerilogPrivateState, VerilogPublicState]:
         """Process an action result and update engine state."""
         self._current_action_for_reward = action_result
+        
+        # DEBUG: Print action_result
+        print(f"\n[ENGINE DEBUG] _step_engine called")
+        print(f"  action_result: {action_result}")
+        print(f"  action_result.type: {action_result.get('type')}")
+        print(f"  action_result.returncode: {action_result.get('returncode')}")
+        print(f"  action_result.ok: {action_result.get('ok')}")
 
         # Update last outputs if this is a compile or simulate action
         if action_result.get("type") == "compile":
@@ -147,18 +190,21 @@ class VerilogEngine(StatefulEngine):
         current_pub_state = VerilogPublicState(
             files=self._get_file_contents(),
             build_dir=str(self.build_dir),
-            task_completed=action_result.get("passed", False),
+            task_completed=action_result.get("submitted", False) and action_result.get("passed", False),
         )
 
         reward_from_stack = await self.reward_stack.step_reward(
             state=current_pub_state, action=self._current_action_for_reward
         )
         self._current_action_for_reward = None
+        
+        # DEBUG: Print reward
+        print(f"[ENGINE DEBUG] reward_from_stack: {reward_from_stack}")
 
         self._total_reward += reward_from_stack
 
-        # Check termination conditions
-        terminated = action_result.get("passed", False) or action_result.get("submitted", False)
+        # Check termination conditions - only terminate if submitted (regardless of pass/fail)
+        terminated = action_result.get("submitted", False)
 
         priv = VerilogPrivateState(
             reward_last=reward_from_stack,
@@ -170,7 +216,7 @@ class VerilogEngine(StatefulEngine):
         pub = VerilogPublicState(
             files=self._get_file_contents(),
             build_dir=str(self.build_dir),
-            task_completed=action_result.get("passed", False),
+            task_completed=action_result.get("submitted", False) and action_result.get("passed", False),
             last_compile_output=self._last_compile_output,
             last_simulate_output=self._last_simulate_output,
         )
@@ -259,6 +305,16 @@ class VerilogEngine(StatefulEngine):
             }
         except subprocess.TimeoutExpired:
             return {"ok": False, "error": "Compilation timeout", "type": "compile"}
+        except FileNotFoundError:
+            error_msg = (
+                "🚨 CRITICAL ERROR: 'iverilog' executable not found! 🚨\n"
+                "The Verilog compiler (iverilog) is not installed in this environment.\n"
+                "This will cause ALL compile operations to fail and result in ZERO rewards.\n"
+                "Fix: Add 'iverilog' to apt_packages in the Modal deployment config.\n"
+                "Location: examples/task_apps/verilog/task_app/grpo_verilog.py -> modal=ModalDeploymentConfig(apt_packages=('iverilog',))"
+            )
+            print(f"\n{'='*80}\n{error_msg}\n{'='*80}\n", flush=True)
+            raise RuntimeError(error_msg) from None
         except Exception as e:
             return {"ok": False, "error": str(e), "type": "compile"}
 
@@ -290,6 +346,16 @@ class VerilogEngine(StatefulEngine):
             }
         except subprocess.TimeoutExpired:
             return {"ok": False, "error": "Simulation timeout", "type": "simulate"}
+        except FileNotFoundError:
+            error_msg = (
+                "🚨 CRITICAL ERROR: 'vvp' executable not found! 🚨\n"
+                "The Verilog simulator (vvp) is not installed in this environment.\n"
+                "This will cause ALL simulate operations to fail and result in ZERO rewards.\n"
+                "Fix: Add 'iverilog' to apt_packages in the Modal deployment config (provides both iverilog and vvp).\n"
+                "Location: examples/task_apps/verilog/task_app/grpo_verilog.py -> modal=ModalDeploymentConfig(apt_packages=('iverilog',))"
+            )
+            print(f"\n{'='*80}\n{error_msg}\n{'='*80}\n", flush=True)
+            raise RuntimeError(error_msg) from None
         except Exception as e:
             return {"ok": False, "error": str(e), "type": "simulate"}
 

synth_ai/judge_schemas.py

@@ -9,7 +9,7 @@ This is the canonical contract that the backend MUST conform to.
 
 from __future__ import annotations
 
-from typing import Any, Literal
+from typing import Any, Dict, Literal, Optional
 
 from pydantic import BaseModel, Field
 
@@ -31,7 +31,7 @@ class ReviewPayload(BaseModel):
         description="Map of criterion keys to their scores"
     )
     total: float = Field(default=0.0, description="Aggregated total score")
-    summary: str | None = Field(None, description="Optional text summary")
+    summary: Optional[str] = Field(None, description="Optional text summary")
 
 
 class JudgeScoreResponse(BaseModel):
@@ -46,7 +46,7 @@ class JudgeScoreResponse(BaseModel):
         default_factory=list,
         description="List of per-event rubric reviews (one per step)"
     )
-    outcome_review: ReviewPayload | None = Field(
+    outcome_review: Optional[ReviewPayload] = Field(
         None,
         description="Optional outcome-level rubric review"
     )
@@ -92,15 +92,15 @@ class JudgeTaskApp(BaseModel):
     """Task application metadata."""
     
     id: str = Field(..., description="Task app identifier")
-    base_url: str | None = Field(None, description="Optional base URL for task app")
+    base_url: Optional[str] = Field(None, description="Optional base URL for task app")
 
 
 class JudgeOptions(BaseModel):
     """Judge provider and configuration options."""
     
-    provider: str | None = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
-    model: str | None = Field(None, description="Model identifier")
-    rubric_id: str | None = Field(None, description="Rubric identifier")
+    provider: Optional[str] = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
+    model: Optional[str] = Field(None, description="Model identifier")
+    rubric_id: Optional[str] = Field(None, description="Rubric identifier")
     event: bool = Field(True, description="Enable event-level judging")
     outcome: bool = Field(True, description="Enable outcome-level judging")
 
@@ -123,5 +123,5 @@ class JudgeScoreRequest(BaseModel):
     task_app: JudgeTaskApp = Field(..., description="Task application metadata")
     trace: JudgeTracePayload = Field(..., description="Trajectory trace to evaluate")
     options: JudgeOptions = Field(default_factory=lambda: JudgeOptions(), description="Judge options")
-    rubric: dict[str, Any] | None = Field(None, description="Optional explicit rubric criteria")
+    rubric: Optional[Dict[str, Any]] = Field(None, description="Optional explicit rubric criteria")
 

synth_ai/task/__init__.py

@@ -4,6 +4,7 @@ from .auth import (
     require_api_key_dependency,
 )
 from .client import TaskAppClient
+from .config import EvalConfig, FilterConfig
 from .contracts import (
     DatasetInfo,
     InferenceInfo,
@@ -50,7 +51,12 @@ from .server import (
     create_task_app,
     run_task_app,
 )
-from .validators import validate_task_app_endpoint, validate_task_app_url
+from .validators import (
+    normalize_inference_url,
+    validate_rollout_response_for_rl,
+    validate_task_app_endpoint,
+    validate_task_app_url,
+)
 from .vendors import (
     get_groq_key_or_503,
     get_openai_key_or_503,
@@ -58,9 +64,13 @@ from .vendors import (
 )
 
 __all__ = [
+    "normalize_inference_url",
+    "validate_rollout_response_for_rl",
     "validate_task_app_url",
     "validate_task_app_endpoint",
     "task_app_health",
+    "EvalConfig",
+    "FilterConfig",
     "TaskAppEndpoints",
     "RolloutEnvSpec",
     "RolloutPolicySpec",

synth_ai/task/apps/__init__.py

@@ -18,6 +18,7 @@ class ModalDeploymentConfig:
     app_name: str
     python_version: str = "3.11"
     pip_packages: Sequence[str] = field(default_factory=tuple)
+    apt_packages: Sequence[str] = field(default_factory=tuple)
     extra_local_dirs: Sequence[tuple[str, str]] = field(default_factory=tuple)
     secret_names: Sequence[str] = field(default_factory=tuple)
     volume_mounts: Sequence[tuple[str, str]] = field(default_factory=tuple)

synth_ai/task/config.py

@@ -0,0 +1,257 @@
+"""Configuration dataclasses for task app CLI commands (eval, filter)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+
+@dataclass(slots=True)
+class EvalConfig:
+    """Configuration for 'synth-ai eval' command.
+    
+    Validates and provides defaults for evaluation runs against task apps.
+    """
+    
+    # Required: Task app identifier
+    app_id: str
+    
+    # Required: Model to evaluate
+    model: str
+    
+    # Required: Seeds to run
+    seeds: list[int]
+    
+    # Optional: Task app URL (None = spawn in-process)
+    task_app_url: str | None = None
+    
+    # Optional: Data split to use
+    split: str = "train"
+    
+    # Optional: Maximum turns/steps per episode
+    max_turns: int | None = None
+    
+    # Optional: Maximum LLM calls per episode
+    max_llm_calls: int = 10
+    
+    # Optional: Concurrency for parallel rollouts
+    concurrency: int = 1
+    
+    # Optional: Environment name
+    env_name: str | None = None
+    
+    # Optional: Policy name
+    policy_name: str | None = None
+    
+    # Optional: Trace format ("compact", "full", "structured")
+    trace_format: Literal["compact", "full", "structured"] = "compact"
+    
+    # Optional: Whether to return traces in response
+    return_trace: bool = False
+    
+    # Optional: Operations sequence (if not provided, generates default)
+    ops: list[str] | None = None
+    
+    # Optional: Environment config overrides
+    env_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Policy config overrides
+    policy_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for traces
+    metadata: dict[str, str] = field(default_factory=dict)
+    
+    # Optional: SQL query for metadata filtering
+    metadata_sql: str | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.app_id:
+            raise ValueError("app_id is required")
+        
+        if not self.model:
+            raise ValueError("model is required")
+        
+        if not self.seeds:
+            raise ValueError("seeds list cannot be empty")
+        
+        if not isinstance(self.seeds, list):
+            raise ValueError("seeds must be a list of integers")
+        
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be >= 1")
+        
+        if self.max_llm_calls < 1:
+            raise ValueError("max_llm_calls must be >= 1")
+        
+        if self.max_turns is not None and self.max_turns < 1:
+            raise ValueError("max_turns must be >= 1")
+        
+        if self.trace_format not in ("compact", "full", "structured"):
+            raise ValueError(f"trace_format must be 'compact', 'full', or 'structured', got: {self.trace_format}")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvalConfig:
+        """Create EvalConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with eval configuration
+            
+        Returns:
+            Validated EvalConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "app_id": data.get("app_id"),
+            "model": data.get("model"),
+            "seeds": data.get("seeds", []),
+            "task_app_url": data.get("task_app_url"),
+            "split": data.get("split", "train"),
+            "max_turns": data.get("max_turns"),
+            "max_llm_calls": data.get("max_llm_calls", 10),
+            "concurrency": data.get("concurrency", 1),
+            "env_name": data.get("env_name"),
+            "policy_name": data.get("policy_name"),
+            "trace_format": data.get("trace_format", "compact"),
+            "return_trace": data.get("return_trace", False),
+            "ops": data.get("ops"),
+            "env_config": data.get("env_config", {}),
+            "policy_config": data.get("policy_config", {}),
+            "metadata": data.get("metadata", {}),
+            "metadata_sql": data.get("metadata_sql"),
+        }
+        
+        return cls(**config_dict)
+
+
+@dataclass(slots=True)
+class FilterConfig:
+    """Configuration for 'synth-ai filter' command.
+    
+    Validates and provides defaults for filtering traces into SFT datasets.
+    """
+    
+    # Required: Database path or URL
+    db: str
+    
+    # Required: Output JSONL path
+    output: str
+    
+    # Optional: Filter by data splits
+    splits: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by task IDs
+    task_ids: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by models
+    models: list[str] = field(default_factory=list)
+    
+    # Optional: Minimum official score threshold
+    min_official_score: float | None = None
+    
+    # Optional: Maximum official score threshold
+    max_official_score: float | None = None
+    
+    # Optional: Minimum judge scores (judge_name -> min_score)
+    min_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Maximum judge scores (judge_name -> max_score)
+    max_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Limit number of examples
+    limit: int | None = None
+    
+    # Optional: Offset for pagination
+    offset: int | None = None
+    
+    # Optional: Whether to shuffle results
+    shuffle: bool = False
+    
+    # Optional: Random seed for shuffling
+    shuffle_seed: int | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.db:
+            raise ValueError("db (database path or URL) is required")
+        
+        if not self.output:
+            raise ValueError("output (JSONL file path) is required")
+        
+        # Validate output has .jsonl extension
+        output_path = Path(self.output)
+        if output_path.suffix.lower() not in (".jsonl", ".json"):
+            raise ValueError(f"output must be a .jsonl or .json file, got: {self.output}")
+        
+        # Validate score thresholds
+        if self.min_official_score is not None and self.max_official_score is not None:
+            if self.min_official_score > self.max_official_score:
+                raise ValueError("min_official_score cannot be greater than max_official_score")
+        
+        # Validate limit/offset
+        if self.limit is not None and self.limit < 1:
+            raise ValueError("limit must be >= 1")
+        
+        if self.offset is not None and self.offset < 0:
+            raise ValueError("offset must be >= 0")
+        
+        # Validate shuffle seed requires shuffle
+        if self.shuffle_seed is not None and not self.shuffle:
+            raise ValueError("shuffle_seed requires shuffle=true")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FilterConfig:
+        """Create FilterConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with filter configuration
+            
+        Returns:
+            Validated FilterConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "db": data.get("db"),
+            "output": data.get("output"),
+            "splits": data.get("splits", []),
+            "task_ids": data.get("task_ids", []),
+            "models": data.get("models", []),
+            "min_official_score": data.get("min_official_score"),
+            "max_official_score": data.get("max_official_score"),
+            "min_judge_scores": data.get("min_judge_scores", {}),
+            "max_judge_scores": data.get("max_judge_scores", {}),
+            "limit": data.get("limit"),
+            "offset": data.get("offset"),
+            "shuffle": data.get("shuffle", False),
+            "shuffle_seed": data.get("shuffle_seed"),
+        }
+        
+        return cls(**config_dict)
+    
+    def get_db_url(self) -> str:
+        """Convert db path to proper SQLite URL if needed.
+        
+        Returns:
+            Database URL suitable for SQLAlchemy/aiosqlite
+        """
+        db_value = self.db.strip()
+        if "://" in db_value:
+            return db_value
+        else:
+            db_path = Path(db_value).expanduser().resolve()
+            # Ensure parent directory exists
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+            return f"sqlite+aiosqlite:///{db_path}"
+    
+    def get_output_path(self) -> Path:
+        """Get resolved output path with parent directory created.
+        
+        Returns:
+            Resolved Path object with parent directory created
+        """
+        output_path = Path(self.output).expanduser().resolve()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        return output_path
+
+