synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/kick_off_ft_oai.py

@@ -1,362 +0,0 @@
-#!/usr/bin/env python3
-"""
-OpenAI Fine-Tuning Script
-========================
-Uploads a JSONL file, kicks off a fine-tuning job, and polls until completion.
-"""
-
-import argparse
-import json
-import os
-import random
-import sys
-import time
-from pathlib import Path
-from typing import Optional
-
-try:
-    import openai
-except ImportError:
-    print("❌ OpenAI package not found. Installing...")
-    os.system("pip install openai")
-    import openai
-
-try:
-    import tiktoken
-except ImportError:
-    print("❌ tiktoken package not found. Installing...")
-    os.system("pip install tiktoken")
-    import tiktoken
-
-
-def encoding_for(model: str = "gpt-4.1-mini"):
-    """Return a tiktoken encoding for any GPT‑4.1 family model."""
-    try:
-        return tiktoken.encoding_for_model(model)
-    except KeyError:                 # 4.1 isn't mapped yet
-        return tiktoken.get_encoding("o200k_base")  # same BPE as 4o/modern models
-
-
-def analyze_jsonl_tokens(file_path: Path, model: str) -> tuple[int, int, float]:
-    """Analyze JSONL file to estimate token usage."""
-    print(f"🔍 Analyzing {file_path.name} for token usage...")
-    
-    # Get the appropriate encoding for the model (handles GPT-4.1 properly)
-    encoding = encoding_for(model)
-    print(f"   🔤 Using encoding: {encoding.name}")
-    
-    total_input_tokens = 0
-    total_output_tokens = 0
-    line_count = 0
-    
-    with open(file_path, 'r') as f:
-        for line in f:
-            try:
-                data = json.loads(line.strip())
-                messages = data.get('messages', [])
-                
-                # Count input tokens (all messages except the last assistant message)
-                input_messages = []
-                output_message = None
-                
-                for msg in messages:
-                    if msg.get('role') == 'assistant' and msg == messages[-1]:
-                        # This is the target output
-                        output_message = msg
-                    else:
-                        # This is input context
-                        input_messages.append(msg)
-                
-                # Estimate input tokens
-                input_text = ""
-                for msg in input_messages:
-                    content = msg.get('content', '')
-                    if content:
-                        input_text += content + " "
-                    
-                    # Include tool calls in input if present
-                    tool_calls = msg.get('tool_calls', [])
-                    for tc in tool_calls:
-                        if tc.get('function', {}).get('arguments'):
-                            input_text += tc['function']['arguments'] + " "
-                
-                input_tokens = len(encoding.encode(input_text))
-                total_input_tokens += input_tokens
-                
-                # Estimate output tokens
-                output_tokens = 0
-                if output_message:
-                    content = output_message.get('content', '') or ''
-                    output_tokens += len(encoding.encode(content))
-                    
-                    # Include tool calls in output
-                    tool_calls = output_message.get('tool_calls', [])
-                    for tc in tool_calls:
-                        if tc.get('function', {}).get('arguments'):
-                            output_tokens += len(encoding.encode(tc['function']['arguments']))
-                
-                total_output_tokens += output_tokens
-                line_count += 1
-                
-            except json.JSONDecodeError:
-                print(f"   ⚠️  Skipping invalid JSON line {line_count + 1}")
-                continue
-            except Exception as e:
-                print(f"   ⚠️  Error processing line {line_count + 1}: {e}")
-                continue
-    
-    avg_tokens_per_line = (total_input_tokens + total_output_tokens) / line_count if line_count > 0 else 0
-    
-    print(f"   📊 Analysis complete:")
-    print(f"      Lines: {line_count:,}")
-    print(f"      Input tokens: {total_input_tokens:,}")
-    print(f"      Output tokens: {total_output_tokens:,}")
-    print(f"      Total tokens: {total_input_tokens + total_output_tokens:,}")
-    print(f"      Avg tokens/line: {avg_tokens_per_line:.1f}")
-    
-    return line_count, total_input_tokens + total_output_tokens, avg_tokens_per_line
-
-
-def create_subset_file(original_path: Path, num_lines: int) -> Path:
-    """Create a subset of the original JSONL file with specified number of lines."""
-    subset_path = original_path.parent / f"{original_path.stem}_subset_{num_lines}.jsonl"
-    
-    print(f"📝 Creating subset with {num_lines} lines...")
-    
-    # Read all lines
-    with open(original_path, 'r') as f:
-        all_lines = [line.strip() for line in f if line.strip()]
-    
-    # Randomly sample lines
-    if num_lines >= len(all_lines):
-        selected_lines = all_lines
-        print(f"   ⚠️  Requested {num_lines} lines, but file only has {len(all_lines)}. Using all lines.")
-    else:
-        selected_lines = random.sample(all_lines, num_lines)
-    
-    # Write subset
-    with open(subset_path, 'w') as f:
-        for line in selected_lines:
-            f.write(line + '\n')
-    
-    print(f"   ✅ Subset saved to: {subset_path.name}")
-    return subset_path
-
-
-def upload_file(client: openai.OpenAI, file_path: Path) -> str:
-    """Upload training file to OpenAI."""
-    print(f"📤 Uploading {file_path.name} ({file_path.stat().st_size / 1024 / 1024:.1f} MB)...")
-    
-    with open(file_path, 'rb') as f:
-        file_obj = client.files.create(
-            file=f,
-            purpose="fine-tune"
-        )
-    
-    print(f"✅ File uploaded: {file_obj.id}")
-    return file_obj.id
-
-
-def create_fine_tune_job(client: openai.OpenAI, file_id: str, model: str = "gpt-4.1-nano-2025-04-14", 
-                        suffix: Optional[str] = None) -> str:
-    """Create a fine-tuning job."""
-    print(f"🚀 Starting fine-tune job for {model}...")
-    
-    kwargs = {
-        "training_file": file_id,
-        "model": model,
-    }
-    
-    if suffix:
-        kwargs["suffix"] = suffix
-    
-    job = client.fine_tuning.jobs.create(**kwargs)
-    
-    print(f"✅ Fine-tune job created: {job.id}")
-    print(f"   Model: {job.model}")
-    print(f"   Status: {job.status}")
-    
-    return job.id
-
-
-def poll_job_status(client: openai.OpenAI, job_id: str, poll_interval: int = 30) -> str:
-    """Poll job status until completion."""
-    print(f"⏳ Polling job {job_id} every {poll_interval}s...")
-    
-    start_time = time.time()
-    last_status = None
-    
-    while True:
-        try:
-            job = client.fine_tuning.jobs.retrieve(job_id)
-            
-            if job.status != last_status:
-                elapsed = time.time() - start_time
-                print(f"   Status: {job.status} (elapsed: {elapsed/60:.1f}m)")
-                last_status = job.status
-                
-                # Show training progress if available
-                if hasattr(job, 'trained_tokens') and job.trained_tokens:
-                    print(f"   Trained tokens: {job.trained_tokens:,}")
-            
-            # Terminal states
-            if job.status == "succeeded":
-                print(f"🎉 Fine-tuning completed successfully!")
-                print(f"   Final model: {job.fine_tuned_model}")
-                return job.fine_tuned_model
-            
-            elif job.status == "failed":
-                print(f"❌ Fine-tuning failed!")
-                if hasattr(job, 'error') and job.error:
-                    print(f"   Error: {job.error}")
-                return None
-            
-            elif job.status == "cancelled":
-                print(f"⚠️  Fine-tuning was cancelled")
-                return None
-            
-            # Continue polling for running states
-            elif job.status in ["validating_files", "queued", "running"]:
-                time.sleep(poll_interval)
-                continue
-            
-            else:
-                print(f"⚠️  Unknown status: {job.status}")
-                time.sleep(poll_interval)
-                continue
-                
-        except KeyboardInterrupt:
-            print(f"\n⚠️  Interrupted by user. Job {job_id} is still running on OpenAI.")
-            print(f"   Check status with: openai api fine_tunes.get -i {job_id}")
-            return None
-        
-        except Exception as e:
-            print(f"❌ Error polling job: {e}")
-            time.sleep(poll_interval)
-            continue
-
-
-def main():
-    parser = argparse.ArgumentParser(description="OpenAI Fine-Tuning Script")
-    parser.add_argument("jsonl_file", type=Path, help="Path to JSONL training file")
-    parser.add_argument("--model", default="gpt-4.1-nano-2025-04-14", 
-                       help="Base model to fine-tune (default: gpt-4.1-nano-2025-04-14)")
-    parser.add_argument("--suffix", type=str, help="Suffix for the fine-tuned model name")
-    parser.add_argument("--poll-interval", type=int, default=30, 
-                       help="Polling interval in seconds (default: 30)")
-    parser.add_argument("--api-key", type=str, help="OpenAI API key (or set OPENAI_API_KEY env var)")
-    
-    args = parser.parse_args()
-    
-    # Validate file
-    if not args.jsonl_file.exists():
-        print(f"❌ File not found: {args.jsonl_file}")
-        sys.exit(1)
-    
-    if not args.jsonl_file.suffix == '.jsonl':
-        print(f"⚠️  Warning: File doesn't have .jsonl extension: {args.jsonl_file}")
-    
-    # Setup API key
-    api_key = args.api_key or os.getenv("OPENAI_API_KEY")
-    if not api_key:
-        print("❌ OpenAI API key required. Set OPENAI_API_KEY env var or use --api-key")
-        sys.exit(1)
-    
-    # Initialize client
-    client = openai.OpenAI(api_key=api_key)
-    
-    # Analyze tokens first
-    line_count, total_tokens, avg_tokens = analyze_jsonl_tokens(args.jsonl_file, args.model)
-    
-    # Calculate estimated cost (rough estimate for fine-tuning)
-    # OpenAI pricing is approximately $8 per 1M tokens for gpt-3.5-turbo fine-tuning
-    # For gpt-4 models, it's higher (varies by model)
-    estimated_cost = total_tokens / 1_000_000 * 8  # Rough estimate
-    
-    print(f"\n💰 Estimated fine-tuning cost: ~${estimated_cost:.2f}")
-    print(f"   (Based on $8/1M tokens - actual cost may vary by model)")
-    
-    # Ask if user wants to use a subset
-    print(f"\n🤔 Do you want to fine-tune on all {line_count:,} lines?")
-    print(f"   Total tokens: {total_tokens:,}")
-    print(f"   Average tokens per line: {avg_tokens:.1f}")
-    
-    use_subset = input("\nUse a subset instead? (y/n): ").lower().strip()
-    
-    training_file = args.jsonl_file
-    if use_subset == 'y':
-        while True:
-            try:
-                subset_size = input(f"How many lines to use? (1-{line_count}): ").strip()
-                subset_size = int(subset_size)
-                
-                if subset_size < 1:
-                    print("   ❌ Number must be at least 1")
-                    continue
-                elif subset_size > line_count:
-                    print(f"   ❌ Number cannot exceed {line_count}")
-                    continue
-                
-                # Show updated estimates
-                subset_tokens = int(avg_tokens * subset_size)
-                subset_cost = subset_tokens / 1_000_000 * 8
-                
-                print(f"\n📊 Subset estimates:")
-                print(f"   Lines: {subset_size:,}")
-                print(f"   Estimated tokens: {subset_tokens:,}")
-                print(f"   Estimated cost: ~${subset_cost:.2f}")
-                
-                confirm = input(f"\nProceed with {subset_size} lines? (y/n): ").lower().strip()
-                if confirm == 'y':
-                    training_file = create_subset_file(args.jsonl_file, subset_size)
-                    break
-                else:
-                    continue
-                    
-            except ValueError:
-                print("   ❌ Please enter a valid number")
-                continue
-    
-    print("🤖 OpenAI Fine-Tuning Pipeline")
-    print("=" * 50)
-    print(f"Training file: {training_file}")
-    print(f"Base model: {args.model}")
-    if args.suffix:
-        print(f"Model suffix: {args.suffix}")
-    print("=" * 50)
-    
-    try:
-        # Step 1: Upload file
-        file_id = upload_file(client, training_file)
-        
-        # Step 2: Create fine-tune job
-        job_id = create_fine_tune_job(client, file_id, args.model, args.suffix)
-        
-        # Step 3: Poll until completion
-        final_model = poll_job_status(client, job_id, args.poll_interval)
-        
-        if final_model:
-            print("\n" + "=" * 50)
-            print(f"🎯 SUCCESS! Fine-tuned model ready: {final_model}")
-            print("=" * 50)
-            
-            # Show usage example
-            print("\n📝 Usage example:")
-            print(f'client = openai.OpenAI()')
-            print(f'response = client.chat.completions.create(')
-            print(f'    model="{final_model}",')
-            print(f'    messages=[{{"role": "user", "content": "Hello!"}}]')
-            print(f')')
-        else:
-            print("\n❌ Fine-tuning did not complete successfully")
-            sys.exit(1)
-            
-    except Exception as e:
-        print(f"\n❌ Unexpected error: {e}")
-        import traceback
-        traceback.print_exc()
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/multi_model_config.toml

@@ -1,49 +0,0 @@
-# Configuration for Multi-Model Crafter Evaluation
-
-[experiment]
-# Number of episodes to run per model
-episodes = 10
-
-# Maximum turns per episode
-max_turns = 100
-
-# Difficulty level (easy, medium, hard)
-difficulty = "easy"
-
-# Models to test (can be overridden with --models CLI argument)
-models = [
-    "gpt-4o-mini",
-    "gpt-4.1-mini", 
-    "gpt-4.1-nano",
-    "gemini-1.5-flash",
-    "gemini-2.5-flash-lite",
-    "qwen3/32b"
-]
-
-# Concurrency settings
-concurrent_models = false  # Whether to run models concurrently
-max_concurrent_models = 3  # Maximum number of models to run at once
-
-[services]
-# Crafter environment service URL
-crafter_service_url = "http://localhost:8901"
-
-# Database path for storing traces
-database_path = "crafter_multi_model_traces.duckdb"
-
-[output]
-# Directory for saving traces and results
-trace_dir = "./traces_multi_model"
-
-# Whether to save detailed traces
-save_traces = true
-
-# Whether to enable verbose output
-verbose = true
-
-[http]
-# HTTP timeout in seconds
-timeout = 30.0
-
-# Maximum retries for failed requests
-max_retries = 3