synth-ai 0.2.9.dev5__py3-none-any.whl → 0.2.10__py3-none-any.whl

examples/swe/task_app/hosted/storage/__init__.py

@@ -0,0 +1,5 @@
+"""Storage module for Modal Volume operations."""
+
+from .volume import VolumeStorage, storage
+
+__all__ = ["VolumeStorage", "storage"]

examples/swe/task_app/hosted/storage/volume.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+import os
+import tarfile
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+
+class VolumeStorage:
+    """Helpers for Modal Volume storage operations."""
+
+    def __init__(self, base_path: str = "/data/state") -> None:
+        self.base_path = Path(base_path)
+
+    def get_snapshot_path(
+        self,
+        rl_run_id: str,
+        kind: str,
+        snapshot_id: str,
+    ) -> Path:
+        """Build the path for a snapshot file."""
+        # Use first 2 chars of snapshot_id for sharding
+        shard1 = snapshot_id[:2] if len(snapshot_id) >= 2 else "00"
+        shard2 = snapshot_id[2:4] if len(snapshot_id) >= 4 else "00"
+
+        return (
+            self.base_path / "runs" / rl_run_id / kind / shard1 / shard2 / f"{snapshot_id}.tar.gz"
+        )
+
+    def get_index_path(self, rl_run_id: str) -> Path:
+        """Get the index file path for a run."""
+        return self.base_path / "runs" / rl_run_id / "index" / "meta.jsonl"
+
+    def write_snapshot_atomic(
+        self,
+        path: Path,
+        archive_bytes: bytes,
+    ) -> None:
+        """Atomically write a snapshot archive to disk."""
+        # Ensure parent directory exists
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write to temp file first
+        tmp_path = path.with_suffix(".tmp")
+        with open(tmp_path, "wb") as f:
+            f.write(archive_bytes)
+            f.flush()
+            os.fsync(f.fileno())
+
+        # Atomic rename
+        os.replace(tmp_path, path)
+
+    def create_archive(
+        self,
+        state_dict: dict[str, Any],
+        meta: dict[str, Any],
+    ) -> bytes:
+        """Create a tar.gz archive with state and metadata."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmppath = Path(tmpdir)
+
+            # Write state.json
+            state_path = tmppath / "state.json"
+            with open(state_path, "w") as f:
+                json.dump(state_dict, f, sort_keys=True, indent=2)
+
+            # Write meta.json
+            meta_path = tmppath / "meta.json"
+            with open(meta_path, "w") as f:
+                json.dump(meta, f, sort_keys=True, indent=2)
+
+            # Create tar archive
+            tar_path = tmppath / "archive.tar"
+            with tarfile.open(tar_path, "w") as tar:
+                tar.add(state_path, arcname="state.json")
+                tar.add(meta_path, arcname="meta.json")
+
+            # Compress with gzip
+            with open(tar_path, "rb") as f:
+                tar_bytes = f.read()
+
+            compressed = gzip.compress(tar_bytes, compresslevel=6)
+
+            return compressed
+
+    def extract_archive(self, archive_bytes: bytes) -> tuple[dict[str, Any], dict[str, Any]]:
+        """Extract state and metadata from a tar.gz archive."""
+        # Decompress
+        tar_bytes = gzip.decompress(archive_bytes)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmppath = Path(tmpdir)
+
+            # Write tar bytes to temp file
+            tar_path = tmppath / "archive.tar"
+            with open(tar_path, "wb") as f:
+                f.write(tar_bytes)
+
+            # Extract tar
+            with tarfile.open(tar_path, "r") as tar:
+                tar.extractall(tmppath)
+
+            # Read state and meta
+            with open(tmppath / "state.json") as f:
+                state = json.load(f)
+
+            with open(tmppath / "meta.json") as f:
+                meta = json.load(f)
+
+            return state, meta
+
+    def compute_snapshot_id(self, archive_bytes: bytes) -> str:
+        """Compute content-addressed snapshot ID."""
+        return hashlib.sha256(archive_bytes).hexdigest()
+
+    def save_snapshot(
+        self,
+        rl_run_id: str,
+        kind: str,
+        state_dict: dict[str, Any],
+        config: dict[str, Any] | None = None,
+        parent_snapshot_id: str | None = None,
+    ) -> tuple[str, str, int]:
+        """Save a snapshot and return (snapshot_id, path, size)."""
+        # Build metadata
+        meta = {
+            "kind": kind,
+            "rl_run_id": rl_run_id,
+            "schema_version": "1.0",
+            "created_at": datetime.utcnow().isoformat(),
+        }
+
+        if parent_snapshot_id:
+            meta["parent_snapshot_id"] = parent_snapshot_id
+
+        if config:
+            config_str = json.dumps(config, sort_keys=True)
+            meta["config_hash"] = hashlib.sha256(config_str.encode()).hexdigest()
+
+        # Create archive
+        archive_bytes = self.create_archive(state_dict, meta)
+
+        # Compute snapshot ID
+        snapshot_id = self.compute_snapshot_id(archive_bytes)
+        meta["snapshot_id"] = snapshot_id
+
+        # Recreate archive with snapshot_id in metadata
+        archive_bytes = self.create_archive(state_dict, meta)
+
+        # Get path and write
+        path = self.get_snapshot_path(rl_run_id, kind, snapshot_id)
+        self.write_snapshot_atomic(path, archive_bytes)
+
+        # Append to index
+        self.append_to_index(rl_run_id, meta)
+
+        return snapshot_id, str(path), len(archive_bytes)
+
+    def load_snapshot(
+        self,
+        rl_run_id: str,
+        kind: str,
+        snapshot_id: str,
+    ) -> tuple[dict[str, Any], dict[str, Any]]:
+        """Load a snapshot and return (state_dict, meta)."""
+        path = self.get_snapshot_path(rl_run_id, kind, snapshot_id)
+
+        if not path.exists():
+            raise FileNotFoundError(f"Snapshot not found: {path}")
+
+        with open(path, "rb") as f:
+            archive_bytes = f.read()
+
+        state, meta = self.extract_archive(archive_bytes)
+        return state, meta
+
+    def append_to_index(
+        self,
+        rl_run_id: str,
+        meta: dict[str, Any],
+    ) -> None:
+        """Append metadata to the run's index file."""
+        index_path = self.get_index_path(rl_run_id)
+        index_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(index_path, "a") as f:
+            f.write(json.dumps(meta) + "\n")
+
+    def read_index(self, rl_run_id: str) -> list[dict[str, Any]]:
+        """Read all entries from a run's index file."""
+        index_path = self.get_index_path(rl_run_id)
+
+        if not index_path.exists():
+            return []
+
+        entries = []
+        with open(index_path) as f:
+            for line in f:
+                if line.strip():
+                    entries.append(json.loads(line))
+
+        return entries
+
+
+# Global storage instance
+storage = VolumeStorage()

examples/swe/task_app/hosted/test_agents.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""
+Smoke test for Wordle and Sokoban ReAct agents using the hosted service.
+
+Prereqs:
+- Run the service: python examples/swe/task_app/hosted/main.py
+- Run an OpenAI-compatible inference server (e.g., Flash/vLLM) at VLLM_BASE_URL
+  that serves model "gpt-5-nano" or adjust MODEL below.
+
+This script will:
+- Create a Wordle/Sokoban env
+- Create corresponding *-react policy with tools
+- Ask the policy for tool_calls via /policy/step (which calls the model)
+- Apply tool_calls to the env via /env/step
+"""
+
+import asyncio
+import os
+
+import httpx
+
+BASE_URL = os.environ.get("SYNTH_ENVS_HOSTED_URL", "http://localhost:8000")
+INFER_URL = os.environ.get("VLLM_BASE_URL", "http://localhost:8001")
+MODEL = os.environ.get("MODEL", "gpt-5-nano")
+
+
+async def run_wordle(rounds: int = 3) -> None:
+    async with httpx.AsyncClient() as client:
+        # Create env
+        resp = await client.post(
+            f"{BASE_URL}/env/create",
+            json={
+                "env_name": "Wordle",
+                "config": {"word_length": 5, "max_guesses": 6},
+                "seed": 0,
+                "rl_run_id": "agents-smoke",
+            },
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        env_id = data["env_id"]
+        obs = data["observation"]
+        print("Wordle env created:", env_id)
+
+        # Create policy
+        resp = await client.post(
+            f"{BASE_URL}/policy/create",
+            json={
+                "policy_name": "wordle-react",
+                "config": {
+                    "inference_url": INFER_URL,
+                    "model": MODEL,
+                    "use_tools": True,
+                    "word_length": 5,
+                    "max_guesses": 6,
+                },
+                "rl_run_id": "agents-smoke",
+                "bound_env_id": env_id,
+            },
+        )
+        resp.raise_for_status()
+        policy_id = resp.json()["policy_id"]
+        print("Wordle policy:", policy_id)
+
+        # Loop a few rounds
+        for i in range(rounds):
+            print(f"[Wordle] Round {i + 1}")
+            step_req = {"policy_id": policy_id, "observation": obs, "dry_run": False}
+            resp = await client.post(f"{BASE_URL}/policy/step", json=step_req)
+            resp.raise_for_status()
+            step_out = resp.json()
+            tool_calls = step_out.get("tool_calls", [])
+            print(" tool_calls:", tool_calls)
+            if not tool_calls:
+                break
+            resp = await client.post(
+                f"{BASE_URL}/env/step",
+                json={"env_id": env_id, "tool_calls": tool_calls},
+            )
+            resp.raise_for_status()
+            env_step = resp.json()
+            obs = env_step["observation"]
+            print(" done:", env_step.get("done"), "reward:", env_step.get("reward"))
+            if env_step.get("done"):
+                break
+
+
+async def run_sokoban(rounds: int = 3) -> None:
+    async with httpx.AsyncClient() as client:
+        # Create env (no initial_state provided; relies on env default)
+        resp = await client.post(
+            f"{BASE_URL}/env/create",
+            json={
+                "env_name": "Sokoban",
+                "config": {"difficulty": "easy"},
+                "seed": 0,
+                "rl_run_id": "agents-smoke",
+            },
+        )
+        if resp.status_code != 200:
+            print("Sokoban create failed:", resp.status_code, resp.text)
+            return
+        data = resp.json()
+        env_id = data["env_id"]
+        obs = data["observation"]
+        print("Sokoban env created:", env_id)
+
+        resp = await client.post(
+            f"{BASE_URL}/policy/create",
+            json={
+                "policy_name": "sokoban-react",
+                "config": {
+                    "inference_url": INFER_URL,
+                    "model": MODEL,
+                    "use_tools": True,
+                },
+                "rl_run_id": "agents-smoke",
+                "bound_env_id": env_id,
+            },
+        )
+        if resp.status_code != 200:
+            print("Sokoban policy create failed:", resp.status_code, resp.text)
+            return
+        policy_id = resp.json()["policy_id"]
+        print("Sokoban policy:", policy_id)
+
+        for i in range(rounds):
+            print(f"[Sokoban] Round {i + 1}")
+            step_req = {"policy_id": policy_id, "observation": obs, "dry_run": False}
+            resp = await client.post(f"{BASE_URL}/policy/step", json=step_req)
+            if resp.status_code != 200:
+                print(" policy step failed:", resp.status_code, resp.text)
+                break
+            step_out = resp.json()
+            tool_calls = step_out.get("tool_calls", [])
+            print(" tool_calls:", tool_calls)
+            if not tool_calls:
+                break
+            resp = await client.post(
+                f"{BASE_URL}/env/step",
+                json={"env_id": env_id, "tool_calls": tool_calls},
+            )
+            if resp.status_code != 200:
+                print(" env step failed:", resp.status_code, resp.text)
+                break
+            env_step = resp.json()
+            obs = env_step["observation"]
+            print(" done:", env_step.get("done"), "reward:", env_step.get("reward"))
+            if env_step.get("done"):
+                break
+
+
+async def main():
+    print("Testing Wordle agent with model:", MODEL)
+    await run_wordle(rounds=3)
+    print("\nTesting Sokoban agent with model:", MODEL)
+    await run_sokoban(rounds=3)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/swe/task_app/hosted/test_service.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""
+Simple test script for the GRPO Synth Envs Hosted Service.
+
+Run this after starting the service with:
+    python main.py
+"""
+
+import asyncio
+import json
+
+import httpx
+
+
+async def test_service():
+    """Test basic service functionality."""
+    base_url = "http://localhost:8000"
+
+    async with httpx.AsyncClient() as client:
+        # Test 1: Service info
+        print("1. Testing /info endpoint...")
+        response = await client.get(f"{base_url}/info")
+        assert response.status_code == 200
+        info = response.json()
+        print(f"   Service info: {json.dumps(info, indent=2)}")
+
+        # Test 2: Health check
+        print("\n2. Testing /health endpoint...")
+        response = await client.get(f"{base_url}/health")
+        assert response.status_code == 200
+        print(f"   Health: {response.json()}")
+
+        # Test 3: Create environment
+        print("\n3. Creating environment...")
+        response = await client.post(
+            f"{base_url}/env/create",
+            json={
+                "env_name": "crafter",
+                "config": {},
+                "seed": 42,
+                "rl_run_id": "test-run-001",
+            },
+        )
+        if response.status_code != 200:
+            print(f"   Error: {response.status_code} - {response.text}")
+            return
+        env_data = response.json()
+        env_id = env_data["env_id"]
+        print(f"   Created env: {env_id}")
+        print(f"   Initial observation keys: {list(env_data['observation'].keys())}")
+
+        # Test 4: Create policy
+        print("\n4. Creating policy...")
+        response = await client.post(
+            f"{base_url}/policy/create",
+            json={
+                "policy_name": "crafter-react",
+                "config": {
+                    "inference_url": "http://localhost:8001",
+                    "model": "test-model",
+                },
+                "rl_run_id": "test-run-001",
+                "bound_env_id": env_id,
+            },
+        )
+        if response.status_code != 200:
+            print(f"   Error: {response.status_code} - {response.text}")
+            return
+        policy_data = response.json()
+        policy_id = policy_data["policy_id"]
+        print(f"   Created policy: {policy_id}")
+
+        # Test 5: Environment step with dummy tool calls
+        print("\n5. Testing environment step...")
+        response = await client.post(
+            f"{base_url}/env/step",
+            json={
+                "env_id": env_id,
+                "tool_calls": [{"tool": "interact", "args": {"action": "move_left"}}],
+            },
+        )
+        if response.status_code != 200:
+            print(f"   Error: {response.status_code} - {response.text}")
+        else:
+            step_data = response.json()
+            print(f"   Step result - done: {step_data['done']}, reward: {step_data.get('reward')}")
+
+        # Test 6: Environment snapshot
+        print("\n6. Creating environment snapshot...")
+        response = await client.post(f"{base_url}/env/snapshot", json={"env_id": env_id})
+        if response.status_code != 200:
+            print(f"   Error: {response.status_code} - {response.text}")
+        else:
+            snapshot_data = response.json()
+            print(f"   Snapshot ID: {snapshot_data['snapshot_id']}")
+            print(f"   Size: {snapshot_data['size']} bytes")
+
+        # Test 7: Policy snapshot
+        print("\n7. Creating policy snapshot...")
+        response = await client.post(f"{base_url}/policy/snapshot", json={"policy_id": policy_id})
+        if response.status_code != 200:
+            print(f"   Error: {response.status_code} - {response.text}")
+        else:
+            snapshot_data = response.json()
+            print(f"   Snapshot ID: {snapshot_data['snapshot_id']}")
+            print(f"   Size: {snapshot_data['size']} bytes")
+
+        # Test 8: Run status
+        print("\n8. Testing run status...")
+        response = await client.get(f"{base_url}/run/status/test-run-001")
+        if response.status_code != 200:
+            print(f"   Error: {response.status_code} - {response.text}")
+        else:
+            status_data = response.json()
+            print(f"   Run status: {status_data['status']}")
+
+        # Test 9: Terminate environment
+        print("\n9. Terminating environment...")
+        response = await client.post(f"{base_url}/env/terminate", json={"env_id": env_id})
+        if response.status_code != 200:
+            print(f"   Error: {response.status_code} - {response.text}")
+        else:
+            print(f"   Environment terminated: {response.json()['ok']}")
+
+        # Test 10: Terminate policy
+        print("\n10. Terminating policy...")
+        response = await client.post(f"{base_url}/policy/terminate", json={"policy_id": policy_id})
+        if response.status_code != 200:
+            print(f"   Error: {response.status_code} - {response.text}")
+        else:
+            print(f"   Policy terminated: {response.json()['ok']}")
+
+        print("\n✅ All basic tests completed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(test_service())

examples/swe/task_app/hosted/utils.py

@@ -0,0 +1,62 @@
+"""Utility functions for the task service."""
+
+from typing import Any
+
+import numpy as np
+
+
+def convert_numpy_to_python(obj: Any) -> Any:
+    """
+    Recursively convert numpy types to Python native types for JSON serialization.
+
+    Args:
+        obj: Object that may contain numpy types
+
+    Returns:
+        Object with numpy types converted to Python native types
+    """
+    if isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, dict):
+        return {key: convert_numpy_to_python(value) for key, value in obj.items()}
+    elif isinstance(obj, list | tuple):
+        return [convert_numpy_to_python(item) for item in obj]
+    else:
+        return obj
+
+
+def sanitize_observation(observation: dict[str, Any]) -> dict[str, Any]:
+    """
+    Sanitize observation data for JSON serialization.
+
+    Converts numpy types and removes non-serializable objects.
+
+    Args:
+        observation: Raw observation from environment
+
+    Returns:
+        Sanitized observation safe for JSON serialization
+    """
+    if not isinstance(observation, dict):
+        return observation
+
+    sanitized = {}
+    for key, value in observation.items():
+        # Skip non-serializable keys or convert them
+        if key in ["semantic_map", "world_material_map", "observation_image"]:
+            # These are likely numpy arrays - convert to lists or skip
+            if isinstance(value, np.ndarray):
+                # For large arrays, we might want to skip or compress
+                # For now, skip them as they're likely debug info
+                continue
+        elif key == "player_position" and isinstance(value, tuple):
+            # Convert tuple with potential numpy types
+            sanitized[key] = [convert_numpy_to_python(v) for v in value]
+        else:
+            sanitized[key] = convert_numpy_to_python(value)
+
+    return sanitized

examples/vlm/PROPOSAL.md

@@ -0,0 +1,53 @@
+# Crafter VLM Fine-Tuning Plan
+
+## Why
+
+Crafter observations already expose 64×64 RGB frames, but until now the stack only
+surfaced text summaries. The new multimodal trace format allows us to:
+
+- Retrieve and persist `observation_image_base64` on every step.
+- Thread data URLs through policy prompts so LLMs see the raw frame.
+- Emit multimodal content parts inside LLM call records (and therefore SFT datasets).
+- Capture the same information inside the session trace (`policy_user_prompt` entries).
+
+This unlocks a supervised dataset for vision‑language models that treat the frame as a
+first-class token alongside the textual state summary.
+
+## What Changed
+
+1. **Environment wrappers** now attach `observation_image_base64`, `..._data_url`,
+   and image metadata to every observation (initialise + step).
+2. **Crafter policy** augments the user prompt with `{"type":"image_url"}` segments
+   while still emitting the textual summary for language-only models.
+3. **Tracing** stores structured prompt content and serialises it to JSON so prompts
+   with images survive the round-trip to SQLite/Turso.
+4. **Dataset exporter** preserves multimodal content, flags rows with images, and
+   carries this metadata through SFT JSONL.
+5. **Utility scripts** under `examples/vlm/` make it easy to sanity check frames,
+   filter datasets, and spin up an image-aware training job.
+
+## Proposed Workflow
+
+1. **Collect rollouts** with tracing enabled (either via the task app or scripted
+   runs) to populate `traces/v3/synth_ai.db`.
+2. **Export** using `export_trace_sft.py` — images are embedded automatically.
+3. **Filter** to rows with user images (see `filter_image_rows.py`) and optionally
+   build validation splits.
+4. **Upload** the JSONL via `run_fft_and_save.py` or the Synth CLI.
+5. **Train** with a VLM-capable base model (e.g. `openai/gpt-4o-mini-2024-07-18`)
+   using `configs/crafter_vlm_gpt4o.toml`.
+6. **Evaluate** the resulting checkpoint on Crafter tasks (reuse the evaluation
+   harness from `examples/warming_up_to_rl` but now with multimodal prompts).
+
+## Open Questions / Future Work
+
+- **Longer context**: investigate packing multiple sequential frames per turn
+  (e.g. last N frames) as either separate image parts or a stitched sprite sheet.
+- **Reward shaping**: extend metadata so SFT rows carry frame-level reward deltas,
+  enabling hybrid BC + value regression objectives.
+- **Assistant images**: currently most assistant messages are textual; we could
+  experiment with returning thumbnails explaining the plan.
+- **Automated filtering**: add CLI helpers to keep only turns whose tool calls led
+  to high-reward outcomes (e.g. achievements).
+- **Evaluation**: define a reference set of human-labelled “vision checkpoints”
+  (e.g. recognise nearby resource, detect threats) to quantify multimodal progress.