synth-ai 0.2.14__py3-none-any.whl → 0.2.16__py3-none-any.whl

examples/qwen_vl/collect_vision_traces.py

@@ -0,0 +1,368 @@
+#!/usr/bin/env python3
+"""
+Collect Crafter vision traces for SFT dataset creation.
+
+Supports both:
+1. OpenAI models (gpt-5-nano, gpt-4o-mini) via OpenAI API
+2. Qwen-VL models via synth-ai hosted inference
+
+Traces are stored in SQLite with full multimodal messages (text + base64 images)
+ready for export to SFT JSONL format.
+
+Requirements:
+  - For OpenAI: OPENAI_API_KEY environment variable
+  - For synth-ai: SYNTH_API_KEY environment variable
+
+Usage:
+  # Collect with gpt-5-nano
+  uv run python examples/qwen_vl/collect_vision_traces.py \
+      --model gpt-5-nano \
+      --provider openai \
+      --episodes 100 \
+      --max-steps 50 \
+      --output-dir traces/gpt5nano_vision
+
+  # Collect with Qwen2-VL via synth
+  uv run python examples/qwen_vl/collect_vision_traces.py \
+      --model Qwen/Qwen2-VL-7B-Instruct \
+      --provider synth \
+      --episodes 100 \
+      --max-steps 50 \
+      --output-dir traces/qwen2vl_vision
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, cast
+from uuid import uuid4
+
+from examples.task_apps.crafter.task_app.synth_envs_hosted.envs.crafter.environment import (
+    CrafterEnvironmentWrapper,
+)
+from examples.task_apps.crafter.task_app.synth_envs_hosted.envs.crafter.policy import CrafterPolicy
+from synth_ai.environments.examples.crafter_classic.environment import CrafterClassicEnvironment
+from synth_ai.environments.examples.crafter_classic.taskset import (
+    CrafterTaskInstance,
+    CrafterTaskInstanceMetadata,
+)
+from synth_ai.environments.tasks.core import Impetus, Intent
+
+# Try importing trace storage
+try:
+    from synth_ai.tracing_v3.storage import create_storage
+    from synth_ai.tracing_v3.storage.config import StorageBackend, StorageConfig
+    TRACING_AVAILABLE = True
+except ImportError:
+    print("Warning: Tracing storage not available. Traces will not be persisted.")
+    TRACING_AVAILABLE = False
+
+
+def _get_openai_client():
+    """Get OpenAI client."""
+    from openai import OpenAI
+    
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError("OPENAI_API_KEY not set")
+    return OpenAI(api_key=api_key)
+
+
+def _default_backend_base_url() -> str:
+    raw = os.getenv("BACKEND_BASE_URL", "https://agent-learning.onrender.com/api").strip()
+    return raw if raw.endswith("/api") else f"{raw}/api"
+
+
+def _get_synth_client():
+    """Get synth-ai inference client."""
+    from synth_ai.inference.client import InferenceClient
+    
+    api_key = os.getenv("SYNTH_API_KEY")
+    if not api_key:
+        raise RuntimeError("SYNTH_API_KEY not set")
+    base_url = os.getenv("SYNTH_BASE_URL", _default_backend_base_url())
+    return InferenceClient(base_url=base_url, api_key=api_key)
+
+
+def _build_task_instance(seed: int) -> CrafterTaskInstance:
+    """Create Crafter task instance."""
+    impetus = Impetus(instructions="Explore, survive, and unlock achievements.")
+    intent = Intent(
+        rubric={"goal": "Maximise Crafter achievements."},
+        gold_trajectories=None,
+        gold_state_diff={},
+    )
+    metadata = CrafterTaskInstanceMetadata(
+        difficulty="custom",
+        seed=seed,
+        num_trees_radius=0,
+        num_cows_radius=0,
+        num_hostiles_radius=0,
+    )
+    instance = CrafterTaskInstance(
+        id=uuid4(),
+        impetus=impetus,
+        intent=intent,
+        metadata=metadata,
+        is_reproducible=True,
+        initial_engine_snapshot=None,
+    )
+    setattr(instance, "config", {"seed": seed, "length": 256, "area": [64, 64]})
+    return instance
+
+
+def _normalise_openai_request(payload: dict[str, Any], model: str, temperature: float) -> dict[str, Any]:
+    """Normalize inference request for OpenAI API."""
+    request = dict(payload)
+    request["model"] = model
+    
+    # Remove vendor-specific knobs
+    request.pop("stop_after_tool_calls", None)
+    request.pop("thinking_mode", None)
+    request.pop("thinking_budget", None)
+    
+    # gpt-5 models have specific requirements
+    if "gpt-5" in model.lower():
+        # gpt-5-nano only supports temperature=1 (default)
+        request.pop("temperature", None)  # Remove custom temperature
+        request.setdefault("max_completion_tokens", 512)
+        request.pop("max_tokens", None)  # Remove if present
+    else:
+        # Older models use max_tokens and support custom temperature
+        request.setdefault("temperature", temperature)
+        max_completion = request.pop("max_completion_tokens", None)
+        if max_completion is not None:
+            request["max_tokens"] = max_completion
+        else:
+            request.setdefault("max_tokens", 512)
+    
+    return request
+
+
+async def collect_traces(
+    model: str,
+    provider: str,
+    num_episodes: int,
+    max_steps: int,
+    seed_start: int,
+    output_dir: Path,
+    temperature: float,
+):
+    """Collect vision traces for SFT."""
+    # Setup tracing store
+    if not TRACING_AVAILABLE:
+        raise RuntimeError("Tracing storage not available. Cannot persist traces.")
+    
+    output_dir.mkdir(parents=True, exist_ok=True)
+    db_path = output_dir / "rollouts.db"
+    storage_config = StorageConfig(
+        backend=StorageBackend.SQLITE,
+        connection_string=f"sqlite+aiosqlite:///{db_path}",
+    )
+    tracing_store = create_storage(storage_config)
+    await tracing_store.initialize()
+    
+    # Setup inference client
+    if provider == "openai":
+        client = _get_openai_client()
+        inference_url = "openai://chat-completions"
+    elif provider == "synth":
+        client = _get_synth_client()
+        inference_url = "synth://inference"
+    else:
+        raise ValueError(f"Unknown provider: {provider}")
+    
+    print(f"🎮 Collecting {num_episodes} episodes with {model}")
+    print(f"   Provider: {provider}")
+    print(f"   Max steps: {max_steps}")
+    print(f"   Output: {output_dir}")
+    print(f"   Database: {db_path}")
+    print()
+    
+    total_steps = 0
+    total_achievements = 0
+    
+    for episode_id in range(num_episodes):
+        seed = seed_start + episode_id
+        
+        # Build task instance
+        task_instance = _build_task_instance(seed)
+        env = CrafterClassicEnvironment(task_instance)
+        wrapper = CrafterEnvironmentWrapper(env, seed=seed)
+        
+        # Initialize policy (vision auto-detected from model name)
+        policy = CrafterPolicy(inference_url=inference_url, model=model)
+        await policy.initialize({
+            "use_tools": True,
+            "model": model,
+            "temperature": temperature,
+            "max_tokens": 512,
+        })
+        
+        observation_packet = await wrapper.initialize()
+        
+        steps_taken = 0
+        achievements = set()
+        
+        # Run episode
+        for step_idx in range(max_steps):
+            obs_dict = observation_packet.get("observation")
+            if not isinstance(obs_dict, dict):
+                break
+            
+            # Format observation
+            obs_text = policy._format_observation_for_llm(observation_packet)  # noqa: SLF001
+            
+            # Get tool calls from policy
+            tool_calls, meta = await policy.step(
+                observation_text=obs_text,
+                metadata={"raw_observation": observation_packet},
+            )
+            if "inference_request" not in meta:
+                break
+            
+            inference_request = meta["inference_request"]
+            
+            # Call inference
+            if provider == "openai":
+                normalized_request = _normalise_openai_request(
+                    inference_request,
+                    model=model,
+                    temperature=temperature,
+                )
+                response = client.chat.completions.create(**normalized_request)
+                response_dict = response.model_dump()
+            else:  # synth
+                response_dict = await client.create_chat_completion(
+                    model=model,
+                    messages=inference_request["messages"],
+                    temperature=temperature,
+                    max_tokens=512,
+                    tools=inference_request.get("tools"),
+                )
+            
+            # Parse tool calls
+            assistant_tool_calls = CrafterPolicy.parse_response_to_tool_calls(
+                response_dict,
+                use_tools=policy.use_tools,
+            )
+            if not assistant_tool_calls:
+                break
+            
+            # Store trace
+            assistant_message = response_dict["choices"][0].get("message", {})
+            trace_messages = inference_request["messages"] + [assistant_message]
+            
+            tracing_store_any = cast(Any, tracing_store)
+            if hasattr(tracing_store_any, "store_trace"):
+                await tracing_store_any.store_trace(
+                    session_id=f"ep{episode_id:04d}",
+                    step=step_idx,
+                    messages=trace_messages,
+                    model=model,
+                    metadata={
+                        "seed": seed,
+                        "has_image": policy.use_vision,
+                        "provider": provider,
+                    },
+                )
+            else:
+                logging.warning(
+                    "Tracing backend does not expose store_trace(); skipping persistence for episode %s",
+                    episode_id,
+                )
+            
+            # Execute action
+            assistant_text = assistant_message.get("content")
+            env_response = await wrapper.step(assistant_tool_calls)
+            if not isinstance(env_response, dict):
+                break
+            
+            # Update policy history
+            policy._append_assistant_turn(  # noqa: SLF001
+                assistant_text,
+                assistant_tool_calls,
+                env_response,
+            )
+            
+            steps_taken += 1
+            
+            # Track achievements
+            obs = env_response.get("observation", {})
+            ach_status = obs.get("achievements_status", {})
+            for name, unlocked in ach_status.items():
+                if unlocked:
+                    achievements.add(name)
+            
+            if env_response.get("done"):
+                break
+            observation_packet = env_response
+        
+        await wrapper.terminate()
+        
+        total_steps += steps_taken
+        total_achievements += len(achievements)
+        
+        print(
+            f"✓ Episode {episode_id:3d} (seed={seed}): {steps_taken} steps, "
+            f"{len(achievements)} achievements"
+        )
+    
+    print()
+    print(f"✅ Collection complete!")
+    print(f"   Total episodes: {num_episodes}")
+    print(f"   Total steps: {total_steps}")
+    print(f"   Avg achievements: {total_achievements / num_episodes:.2f}")
+    print(f"   Database: {db_path}")
+    print()
+    print("Next steps:")
+    print("  1. Export traces to SFT JSONL format")
+    print("  2. Split into train/val datasets")
+    print("  3. Train VLM with LoRA")
+    
+    return db_path
+
+
+async def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--model",
+        required=True,
+        help="Model name (e.g., gpt-5-nano, Qwen/Qwen2-VL-7B-Instruct)",
+    )
+    parser.add_argument(
+        "--provider",
+        choices=["openai", "synth"],
+        required=True,
+        help="Inference provider",
+    )
+    parser.add_argument("--episodes", type=int, default=100, help="Number of episodes")
+    parser.add_argument("--max-steps", type=int, default=50, help="Max steps per episode")
+    parser.add_argument("--seed-start", type=int, default=0, help="Starting seed")
+    parser.add_argument("--temperature", type=float, default=0.7, help="Sampling temperature")
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=Path("traces/vision_traces"),
+        help="Output directory for traces",
+    )
+    args = parser.parse_args()
+    
+    await collect_traces(
+        model=args.model,
+        provider=args.provider,
+        num_episodes=args.episodes,
+        max_steps=args.max_steps,
+        seed_start=args.seed_start,
+        output_dir=args.output_dir,
+        temperature=args.temperature,
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/qwen_vl/configs/crafter_rl_vision_qwen3vl4b.toml

@@ -0,0 +1,127 @@
+# Crafter RL with Vision - Qwen3-VL-4B
+#
+# This configuration runs online RL (GRPO/GSPO) with a vision-language model
+# using the same Crafter task app that generates image observations for SFT data.
+#
+# Model: Qwen/Qwen3-VL-4B (smaller, faster for testing)
+# Task App: grpo-crafter-task-app (Modal deployed, supports vision)
+# Policy: crafter-react with use_vision=true, image_only_mode=true
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 4096
+# Vision-specific settings
+limit_mm_per_prompt = { "image": 1 }  # Max 1 image per prompt
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-VL-4B-Instruct"
+trainer_mode = "lora"
+label = "crafter-rl-vision-qwen3vl4b"
+supports_vision = true  # Enable vision support
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+# Note: will automatically include mm_projector for vision models
+
+[rollout]
+env_name = "crafter"
+max_turns = 10  # 10 steps per episode for faster testing
+episodes_per_batch = 2
+policy_name = "crafter-react"
+max_concurrent_rollouts = 4  # Lower for vision models (memory)
+batches_per_step = 2
+ops = ["agent", "env"]
+
+  [rollout.env_config]
+  difficulty = "easy"
+  
+    [rollout.env_config.step_rewards]
+    enabled = true
+    mode = "decision_stepwise"
+    strategy = "consistent"
+    indicator_lambda = 1.0
+    step_beta = 0.0
+
+  [rollout.policy_config]
+  # Vision-specific policy settings
+  use_vision = true  # Enable vision input
+  image_only_mode = true  # Use only images, no text observations
+  temperature = 0.6  # Slightly higher for exploration
+  top_p = 0.95
+  max_tokens = 512
+  max_llm_calls = 10
+
+[evaluation]
+instances = 8  # Lower for faster vision evals
+every_n_iters = 5
+seeds = [0, 1, 2, 3, 4, 5, 6, 7]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 3  # Shorter for integration test
+gradient_accumulation_steps = 2
+max_accumulated_minibatch = 1
+max_turns = 10
+batch_size = 2  # Smaller for vision models
+group_size = 2
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+event_rewards_kind = "unique"
+async_semaphore_max = 2  # Lower concurrency for vision
+
+# Enable dense decision rewards
+step_rewards_enabled = true
+step_rewards_mode = "decision_stepwise"
+step_rewards_indicator_lambda = 1.0
+step_rewards_beta = 0.0
+step_rewards_strategy = "consistent"
+
+# Vision-specific training settings
+max_images_per_message = 1  # Limit images for memory
+supports_vision = true  # Enable vision training path
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0
+
+[judge]
+type = "env"  # Use environment rewards only (simpler for testing)
+timeout_s = 30
+
+[tags]
+experiment = "crafter_rl_vision_qwen3vl4b"
+task = "crafter_agent_vision"
+model_size = "4b"
+vision_enabled = true
+image_only = true
+

examples/qwen_vl/configs/crafter_vlm_sft_example.toml

@@ -0,0 +1,60 @@
+# Example Vision SFT Config for Crafter
+# Train Qwen-VL on collected vision traces
+
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "lora"
+
+[job]
+model = "Qwen/Qwen2-VL-7B-Instruct"  # or Qwen/Qwen3-VL-8B
+# Dataset from collect_vision_traces.py → export_to_sft.py
+data = "traces/gpt5nano_vision/train.jsonl"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2          # 2x H200 (282GB total)
+nodes = 1
+
+[training]
+mode = "lora"          # SFT with LoRA
+use_qlora = true       # Quantized LoRA for memory efficiency
+
+[hyperparameters]
+n_epochs = 2           # 2 epochs over collected samples
+per_device_batch = 1   # Batch size 1 (images are memory-intensive)
+gradient_accumulation_steps = 32
+sequence_length = 2048 # Shorter context (images dominate memory)
+learning_rate = 5e-06
+warmup_ratio = 0.03
+train_kind = "peft"
+
+# LoRA config
+lora_rank = 16
+lora_alpha = 32
+lora_dropout = 0.05
+lora_target_modules = ["all-linear"]  # Full linear layer adaptation
+
+# Training optimizations
+[hyperparameters.parallelism]
+use_deepspeed = true
+deepspeed_stage = 2
+fsdp = false
+bf16 = true
+fp16 = false
+activation_checkpointing = true
+
+# Evaluation
+evaluation_strategy = "steps"
+eval_steps = 100
+save_best_model_at_end = true
+metric_for_best_model = "val.loss"
+greater_is_better = false
+load_best_model_at_end = true
+
+[tags]
+task = "crafter"
+modality = "vision"
+data_source = "collected_traces"
+model_family = "qwen_vl"
+

examples/qwen_vl/configs/eval_gpt4o_mini_vision.toml

@@ -0,0 +1,43 @@
+# Evaluation config for gpt-4o-mini with vision
+# Stronger teacher than gpt-5-nano, use for high-quality distillation
+
+[eval]
+model = "gpt-4o-mini-2024-07-18"
+provider = "openai"  # Use OpenAI API
+
+# Task app endpoint
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app.modal.run"
+
+# Vision settings (auto-detected from "gpt-4o" in model name)
+use_vision = true
+image_only_mode = false  # Include both text + images
+
+# Rollout settings
+num_episodes = 100
+max_steps_per_episode = 50
+seeds = "200-299"  # Different seeds for comparison
+
+# Sampling parameters
+temperature = 0.6  # Lower temperature for more consistent behavior
+max_tokens = 512
+
+# Trace collection
+collect_traces = true
+trace_db = "traces/gpt4o_mini_vision/rollouts.db"
+
+# Tools
+use_tools = true
+
+# Parallel rollouts
+parallel_episodes = 5
+
+[task]
+name = "crafter"
+environment = "crafter-classic"
+
+# Task-specific settings
+[task.config]
+seed_start = 200
+max_episode_length = 256
+render_size = [64, 64]  # 64x64 PNG images
+

examples/qwen_vl/configs/eval_gpt4o_vision_proper.toml

@@ -0,0 +1,29 @@
+# Proper synth-ai eval config for Crafter with gpt-4o-mini vision
+# Collects traces with images to database for synth-ai filter
+
+[eval]
+app_id = "grpo-crafter-task-app"  # Modal deployed task app
+model = "gpt-4o-mini-2024-07-18"
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]  # 10 episodes for test
+max_turns = 50  # 50 steps per episode
+concurrency = 2  # 2 parallel episodes
+env_name = "crafter"
+policy_name = "crafter-react"
+trace_format = "structured"  # Required for synth-ai eval
+return_trace = true
+
+[eval.env_config]
+env_params = {max_steps_per_episode = 50}
+
+[eval.policy_config]
+provider = "openai"
+model = "gpt-4o-mini-2024-07-18"
+inference_url = "https://api.openai.com"  # Base URL
+# Note: Don't set temperature for gpt-4o-mini, use default
+top_p = 0.95
+max_tokens = 512
+use_vision = true  # Enable vision
+image_only_mode = false  # Use both text + images
+max_llm_calls = 50
+use_tools = true  # Enable tool calling
+

examples/qwen_vl/configs/eval_gpt5nano_vision.toml

@@ -0,0 +1,45 @@
+# Evaluation config for gpt-4o-mini with vision
+# Collects vision traces for SFT training
+# Note: gpt-5-nano doesn't support tool calling yet, use gpt-4o-mini instead
+
+[eval]
+model = "gpt-4o-mini-2024-07-18"  # Changed from gpt-5-nano (no tool support)
+provider = "openai"  # Use OpenAI API
+
+# Task app endpoint (local or hosted)
+# task_app_url = "http://localhost:8000"  # Local
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app.modal.run"  # Hosted
+
+# Vision settings (auto-detected from "gpt-5" in model name)
+use_vision = true
+image_only_mode = false  # Include both text + images
+
+# Rollout settings
+num_episodes = 100
+max_steps_per_episode = 50
+seeds = "0-99"  # Seeds 0 through 99
+
+# Sampling parameters
+temperature = 0.7
+max_tokens = 512
+
+# Trace collection
+collect_traces = true
+trace_db = "traces/gpt5nano_vision/rollouts.db"
+
+# Tools
+use_tools = true
+
+# Parallel rollouts (speeds up collection)
+parallel_episodes = 5  # Run 5 episodes in parallel
+
+[task]
+name = "crafter"
+environment = "crafter-classic"
+
+# Task-specific settings
+[task.config]
+seed_start = 0
+max_episode_length = 256
+render_size = [64, 64]  # 64x64 PNG images
+