synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/crafter_custom/old/compare_worlds.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-"""
-Compare agent performance across different world configurations.
-"""
-
-import subprocess
-import json
-from pathlib import Path
-import sys
-
-def run_evaluation(world_config, episodes=2, max_turns=20):
-    """Run evaluation with a specific world configuration."""
-    print(f"\n{'='*60}")
-    print(f"Testing {world_config.upper()} world configuration")
-    print(f"{'='*60}")
-    
-    cmd = [
-        sys.executable, "-m",
-        "synth_ai.environments.examples.crafter_custom.agent_demos.test_crafter_custom_agent",
-        "--model", "gpt-4.1-nano",
-        "--world-config", world_config,
-        "--episodes", str(episodes),
-        "--max-turns", str(max_turns),
-        "--evaluate-traces"
-    ]
-    
-    result = subprocess.run(cmd, capture_output=True, text=True)
-    
-    # Extract key metrics from output
-    lines = result.stdout.split('\n')
-    metrics = {}
-    
-    for line in lines:
-        if "Mean Score" in line:
-            metrics["mean_score"] = float(line.split()[-1])
-        elif "Avg Achievements/Episode" in line:
-            metrics["avg_achievements"] = float(line.split()[-1])
-        elif "Average Score:" in line and "📊" not in line:
-            metrics["trace_score"] = float(line.split()[-1])
-    
-    return metrics
-
-def main():
-    print("🎮 Crafter World Configuration Comparison")
-    print("=" * 60)
-    
-    configs = ["peaceful", "easy", "normal", "hard"]
-    results = {}
-    
-    for config in configs:
-        results[config] = run_evaluation(config, episodes=2, max_turns=15)
-        print(f"\nResults for {config}: {results[config]}")
-    
-    # Summary comparison
-    print("\n" + "="*60)
-    print("📊 COMPARISON SUMMARY")
-    print("="*60)
-    print(f"{'Config':<10} {'Mean Score':<12} {'Avg Achievements':<18} {'Trace Score':<12}")
-    print("-"*60)
-    
-    for config in configs:
-        r = results[config]
-        print(f"{config:<10} {r.get('mean_score', 0):<12.2f} {r.get('avg_achievements', 0):<18.2f} {r.get('trace_score', 0):<12.2f}")
-    
-    print("\n💡 Analysis:")
-    print("- Peaceful worlds should have highest scores (no enemies)")
-    print("- Hard worlds should have lowest scores (many enemies, few resources)")
-    print("- Resource availability directly impacts achievement unlocking")
-
-if __name__ == "__main__":
-    main()

synth_ai/environments/examples/crafter_custom/old/dataset_stats.py

@@ -1,105 +0,0 @@
-#!/usr/bin/env python3
-"""
-Display statistics about Crafter datasets
-"""
-
-import json
-from pathlib import Path
-from collections import Counter
-import sys
-
-
-def analyze_dataset(dataset_path: Path):
-    """Analyze a single dataset"""
-    # Load metadata
-    with open(dataset_path / "metadata.json", "r") as f:
-        metadata = json.load(f)
-    
-    # Load instances
-    with open(dataset_path / "instances.json", "r") as f:
-        instances = json.load(f)
-    
-    print(f"\nDataset: {metadata['name']}")
-    print(f"Description: {metadata['description']}")
-    print(f"Total instances: {metadata['num_instances']}")
-    print(f"Train/Val/Test split: {len(instances) - len(metadata['split_info']['val_instance_ids']) - len(metadata['split_info']['test_instance_ids'])}/{len(metadata['split_info']['val_instance_ids'])}/{len(metadata['split_info']['test_instance_ids'])}")
-    
-    # Analyze by difficulty
-    difficulties = Counter(inst['metadata']['difficulty'] for inst in instances)
-    print(f"\nInstances by difficulty:")
-    for diff, count in sorted(difficulties.items()):
-        print(f"  {diff}: {count} ({count/len(instances)*100:.1f}%)")
-    
-    # Analyze by impetus type
-    impetus_types = Counter()
-    for inst in instances:
-        instructions = inst['impetus']['instructions'].lower()
-        if 'speedrun' in instructions:
-            impetus_types['speedrun'] += 1
-        elif 'focus on' in instructions:
-            impetus_types['focused'] += 1
-        else:
-            impetus_types['general'] += 1
-    
-    print(f"\nInstances by type:")
-    for type_name, count in sorted(impetus_types.items()):
-        print(f"  {type_name}: {count} ({count/len(instances)*100:.1f}%)")
-    
-    # Analyze achievement focuses
-    focus_counts = Counter()
-    speedrun_targets = Counter()
-    
-    for inst in instances:
-        if inst['impetus'].get('achievement_focus'):
-            for ach in inst['impetus']['achievement_focus']:
-                focus_counts[ach] += 1
-        
-        if 'speedrun' in inst['impetus']['instructions'].lower():
-            # Extract speedrun target
-            instructions = inst['impetus']['instructions']
-            if ':' in instructions:
-                target = instructions.split(':')[1].strip().split(' ')[0]
-                speedrun_targets[target] += 1
-    
-    if focus_counts:
-        print(f"\nTop achievement focuses:")
-        for ach, count in focus_counts.most_common(10):
-            print(f"  {ach}: {count}")
-    
-    if speedrun_targets:
-        print(f"\nSpeedrun targets:")
-        for target, count in speedrun_targets.most_common():
-            print(f"  {target}: {count}")
-    
-    # Sample some instances
-    print(f"\nSample instances:")
-    for i, inst in enumerate(instances[:3]):
-        print(f"\n  Instance {i+1}:")
-        print(f"    ID: {inst['id']}")
-        print(f"    Difficulty: {inst['metadata']['difficulty']}")
-        print(f"    Seed: {inst['metadata']['world_seed']}")
-        print(f"    Instructions: {inst['impetus']['instructions'][:80]}...")
-
-
-def main():
-    dataset_dir = Path("dataset")
-    
-    print("Crafter Dataset Statistics")
-    print("=" * 60)
-    
-    # Find all datasets
-    datasets = [d for d in dataset_dir.iterdir() if d.is_dir() and (d / "metadata.json").exists()]
-    
-    if not datasets:
-        print("No datasets found in dataset/")
-        return
-    
-    print(f"Found {len(datasets)} dataset(s)")
-    
-    for dataset_path in sorted(datasets):
-        analyze_dataset(dataset_path)
-        print("\n" + "-" * 60)
-
-
-if __name__ == "__main__":
-    main()

synth_ai/environments/examples/crafter_custom/old/diamond_spawning_summary.py

@@ -1,119 +0,0 @@
-"""
-Summary of the diamond spawning issue in Crafter.
-"""
-
-def explain_probability_condition():
-    """Explain how the probability condition works."""
-    print("DIAMOND SPAWNING PROBABILITY EXPLANATION")
-    print("=" * 60)
-    print()
-    print("The code uses this condition to spawn diamonds:")
-    print("  uniform() > (1 - config.diamond_probability)")
-    print()
-    print("This means:")
-    print("  - uniform() generates a random number between 0 and 1")
-    print("  - We spawn a diamond if this random number is GREATER than (1 - diamond_probability)")
-    print()
-    print("Examples:")
-    print("-" * 60)
-    
-    probabilities = [0.006, 0.1, 0.5, 0.8, 0.95]
-    
-    for prob in probabilities:
-        threshold = 1 - prob
-        actual_chance = 1 - threshold
-        print(f"\nWith diamond_probability = {prob}:")
-        print(f"  Condition: uniform() > {threshold:.3f}")
-        print(f"  Actual spawn chance: {actual_chance:.1%}")
-        print(f"  Description: {'VERY LOW' if prob < 0.1 else 'LOW' if prob < 0.3 else 'MODERATE' if prob < 0.7 else 'HIGH' if prob < 0.9 else 'VERY HIGH'}")
-
-def show_cascading_conditions():
-    """Show how conditions cascade in the world generation."""
-    print("\n\nCASCADING CONDITIONS IN WORLD GENERATION")
-    print("=" * 60)
-    print()
-    print("For a position in a mountain area, materials are checked in this order:")
-    print()
-    print("1. Cave check (two conditions)")
-    print("   └─ If true → place 'path'")
-    print()
-    print("2. Horizontal tunnel check") 
-    print("   └─ If true → place 'path'")
-    print()
-    print("3. Vertical tunnel check")
-    print("   └─ If true → place 'path'")
-    print()
-    print("4. Coal check (noise threshold AND probability)")
-    print("   └─ If true → place 'coal'")
-    print()
-    print("5. Iron check (noise threshold AND probability)")
-    print("   └─ If true → place 'iron'")
-    print()
-    print("6. Diamond check (mountain threshold AND probability)")
-    print("   └─ If true → place 'diamond'")
-    print()
-    print("7. Lava check")
-    print("   └─ If true → place 'lava'")
-    print()
-    print("8. Default: place 'stone'")
-    print()
-    print("IMPORTANT: Once any condition is met, no further checks are done!")
-    print("This means diamonds can be blocked by caves, tunnels, coal, or iron.")
-
-def calculate_effective_spawn_rate():
-    """Calculate the effective diamond spawn rate."""
-    print("\n\nEFFECTIVE DIAMOND SPAWN RATE")
-    print("=" * 60)
-    print()
-    print("Based on the analysis of 10,000 positions:")
-    print()
-    
-    # Data from our analysis
-    total_positions = 10000
-    mountain_positions = 2402
-    blocked_caves = 431
-    blocked_tunnels = 445
-    blocked_coal = 104
-    blocked_iron = 41
-    diamond_threshold_met = 1185
-    diamond_probability = 0.006
-    
-    print(f"Total positions: {total_positions}")
-    print(f"Mountain positions: {mountain_positions} ({100*mountain_positions/total_positions:.1f}%)")
-    print(f"  Blocked by caves: {blocked_caves}")
-    print(f"  Blocked by tunnels: {blocked_tunnels}")
-    print(f"  Blocked by coal: {blocked_coal}")
-    print(f"  Blocked by iron: {blocked_iron}")
-    print(f"  Reached diamond check: {diamond_threshold_met}")
-    print()
-    print(f"Diamond spawn probability: {diamond_probability} ({diamond_probability*100:.1f}%)")
-    print()
-    
-    expected_diamonds = diamond_threshold_met * diamond_probability
-    overall_rate = expected_diamonds / total_positions
-    
-    print(f"Expected diamonds: {expected_diamonds:.1f}")
-    print(f"Overall spawn rate: {overall_rate:.4%}")
-    print()
-    print("This explains why diamonds are so rare!")
-
-if __name__ == "__main__":
-    explain_probability_condition()
-    show_cascading_conditions()
-    calculate_effective_spawn_rate()
-    
-    print("\n\nRECOMMENDATION")
-    print("=" * 60)
-    print()
-    print("To make diamonds appear with 'high probability', you should:")
-    print()
-    print("1. Increase diamond_probability to at least 0.5 (50% chance)")
-    print("   or even 0.8-0.9 for truly 'high' probability")
-    print()
-    print("2. Consider adjusting other thresholds to reduce blocking:")
-    print("   - Increase coal_threshold (currently 0.0)")
-    print("   - Increase iron_threshold (currently 0.4)")
-    print("   - Decrease coal_probability and iron_probability")
-    print()
-    print("3. Optionally lower diamond_threshold slightly (currently 0.18)")
-    print("   to increase the number of positions that can spawn diamonds")

synth_ai/environments/examples/crafter_custom/old/example_dataset_usage.py

@@ -1,52 +0,0 @@
-#!/usr/bin/env python3
-"""
-Example usage of Crafter datasets
-"""
-
-import os
-import sys
-sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-from run_dataset import CrafterDatasetRunner
-
-
-def main():
-    runner = CrafterDatasetRunner()
-    
-    print("=== Example 1: Run 5 easy instances ===")
-    runner.run_batch(
-        dataset_name="crafter_balanced_v1",
-        num_instances=5,
-        difficulties=["easy"],
-        max_steps=500
-    )
-    
-    print("\n\n=== Example 2: Run validation set instances ===")
-    runner.run_batch(
-        dataset_name="crafter_balanced_v1",
-        num_instances=5,
-        split="val",
-        max_steps=500
-    )
-    
-    print("\n\n=== Example 3: Run speedrun challenges ===")
-    runner.run_batch(
-        dataset_name="crafter_balanced_v1",
-        num_instances=5,
-        impetus_types=["speedrun"],
-        max_steps=500
-    )
-    
-    print("\n\n=== Example 4: Compare difficulties ===")
-    for difficulty in ["easy", "normal", "hard"]:
-        print(f"\n--- Testing {difficulty} ---")
-        runner.run_batch(
-            dataset_name="crafter_progression_v1",
-            num_instances=3,
-            difficulties=[difficulty],
-            max_steps=300
-        )
-
-
-if __name__ == "__main__":
-    main()

synth_ai/environments/examples/crafter_custom/run_dataset.py

@@ -1,305 +0,0 @@
-#!/usr/bin/env python3
-"""
-Run script for Crafter dataset instances
-"""
-
-import argparse
-import json
-import os
-import random
-import sys
-import uuid
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-from crafter import Env
-
-
-class CrafterDatasetRunner:
-    """Run Crafter instances from a dataset"""
-
-    def __init__(self, dataset_path: Path = Path("dataset")):
-        self.dataset_path = dataset_path
-
-    def load_dataset(self, dataset_name: str) -> Dict[str, Any]:
-        """Load a dataset from disk"""
-        dataset_dir = self.dataset_path / dataset_name
-
-        # Load metadata
-        with open(dataset_dir / "metadata.json", "r") as f:
-            metadata = json.load(f)
-
-        # Load instances
-        with open(dataset_dir / "instances.json", "r") as f:
-            instances = json.load(f)
-
-        return {"metadata": metadata, "instances": instances}
-
-    def filter_instances(
-        self,
-        instances: List[Dict[str, Any]],
-        difficulties: Optional[List[str]] = None,
-        impetus_types: Optional[List[str]] = None,
-        split: Optional[str] = None,
-        split_info: Optional[Dict[str, Any]] = None,
-    ) -> List[Dict[str, Any]]:
-        """Filter instances based on criteria"""
-        filtered = instances
-
-        # Filter by difficulty
-        if difficulties:
-            filtered = [inst for inst in filtered if inst["metadata"]["difficulty"] in difficulties]
-
-        # Filter by impetus type
-        if impetus_types:
-            filtered = [inst for inst in filtered if self._get_impetus_type(inst) in impetus_types]
-
-        # Filter by split
-        if split and split_info:
-            if split == "train":
-                val_ids = set(split_info["val_instance_ids"])
-                test_ids = set(split_info["test_instance_ids"])
-                filtered = [
-                    inst
-                    for inst in filtered
-                    if inst["id"] not in val_ids and inst["id"] not in test_ids
-                ]
-            elif split == "val":
-                val_ids = set(split_info["val_instance_ids"])
-                filtered = [inst for inst in filtered if inst["id"] in val_ids]
-            elif split == "test":
-                test_ids = set(split_info["test_instance_ids"])
-                filtered = [inst for inst in filtered if inst["id"] in test_ids]
-
-        return filtered
-
-    def _get_impetus_type(self, instance: Dict[str, Any]) -> str:
-        """Determine impetus type from instructions"""
-        instructions = instance["impetus"]["instructions"].lower()
-        if "speedrun" in instructions:
-            return "speedrun"
-        elif "focus on" in instructions:
-            return "focused"
-        else:
-            return "general"
-
-    def run_instance(
-        self, instance: Dict[str, Any], render: bool = False, max_steps: int = 1000, agent_fn=None
-    ):
-        """Run a single instance"""
-
-        # Extract parameters
-        difficulty = instance["metadata"]["difficulty"]
-        seed = instance["metadata"]["world_seed"]
-
-        print(f"\n{'=' * 60}")
-        print(f"Running instance: {instance['id']}")
-        print(f"Difficulty: {difficulty}")
-        print(f"Seed: {seed}")
-        print(f"Instructions: {instance['impetus']['instructions']}")
-        if instance["impetus"].get("achievement_focus"):
-            print(f"Focus: {', '.join(instance['impetus']['achievement_focus'])}")
-        print(f"{'=' * 60}")
-
-        # Create environment
-        env = Env(seed=seed, world_config=difficulty)
-
-        obs = env.reset()
-
-        # Run agent or random policy
-        total_reward = 0
-        achievements = set()
-
-        for step in range(max_steps):
-            if agent_fn:
-                action = agent_fn(obs, instance)
-            else:
-                action = env.action_space.sample()
-
-            obs, reward, done, info = env.step(action)
-            total_reward += reward
-
-            # Track achievements
-            if "achievements" in info:
-                for ach, unlocked in info["achievements"].items():
-                    if unlocked:
-                        achievements.add(ach)
-
-            if done:
-                break
-
-        # Evaluate based on intent
-        success = self._evaluate_instance(instance, achievements, total_reward, step)
-
-        print(f"\nResults:")
-        print(f"Steps: {step}")
-        print(f"Total reward: {total_reward}")
-        print(f"Achievements: {len(achievements)} - {list(achievements)}")
-        print(f"Success: {success}")
-
-        return {
-            "instance_id": instance["id"],
-            "difficulty": difficulty,
-            "seed": seed,
-            "steps": step,
-            "total_reward": total_reward,
-            "achievements": list(achievements),
-            "success": success,
-        }
-
-    def _evaluate_instance(
-        self, instance: Dict[str, Any], achievements: set, total_reward: float, steps: int
-    ) -> bool:
-        """Evaluate if instance was successful based on intent"""
-        intent = instance["intent"]
-
-        # Check minimum score
-        if intent.get("minimum_score"):
-            if len(achievements) < intent["minimum_score"]:
-                return False
-
-        # Check target achievements
-        if intent.get("target_achievements"):
-            targets = set(intent["target_achievements"])
-            if not achievements.intersection(targets):
-                return False
-
-        return True
-
-    def run_batch(
-        self,
-        dataset_name: str,
-        num_instances: int = 10,
-        difficulties: Optional[List[str]] = None,
-        impetus_types: Optional[List[str]] = None,
-        split: Optional[str] = None,
-        render: bool = False,
-        max_steps: int = 1000,
-        agent_fn=None,
-    ):
-        """Run a batch of instances"""
-
-        # Load dataset
-        dataset = self.load_dataset(dataset_name)
-        instances = dataset["instances"]
-
-        # Filter instances
-        filtered = self.filter_instances(
-            instances,
-            difficulties=difficulties,
-            impetus_types=impetus_types,
-            split=split,
-            split_info=dataset["metadata"].get("split_info"),
-        )
-
-        if not filtered:
-            print("No instances match the filter criteria!")
-            return []
-
-        # Sample instances
-        if num_instances > len(filtered):
-            print(f"Only {len(filtered)} instances available, running all")
-            selected = filtered
-        else:
-            selected = random.sample(filtered, num_instances)
-
-        print(f"\nRunning {len(selected)} instances from {dataset_name}")
-        print(f"Difficulties: {difficulties or 'all'}")
-        print(f"Impetus types: {impetus_types or 'all'}")
-        print(f"Split: {split or 'all'}")
-
-        # Run instances
-        results = []
-        for instance in selected:
-            result = self.run_instance(
-                instance, render=render, max_steps=max_steps, agent_fn=agent_fn
-            )
-            results.append(result)
-
-        # Summary statistics
-        self._print_summary(results)
-
-        return results
-
-    def _print_summary(self, results: List[Dict[str, Any]]):
-        """Print summary statistics"""
-        print(f"\n{'=' * 60}")
-        print("SUMMARY")
-        print(f"{'=' * 60}")
-
-        # Group by difficulty
-        by_difficulty = {}
-        for result in results:
-            diff = result["difficulty"]
-            if diff not in by_difficulty:
-                by_difficulty[diff] = []
-            by_difficulty[diff].append(result)
-
-        print(f"\nResults by difficulty:")
-        print(
-            f"{'Difficulty':<15} {'Count':<8} {'Success':<10} {'Avg Steps':<12} {'Avg Achievements'}"
-        )
-        print("-" * 60)
-
-        for diff in sorted(by_difficulty.keys()):
-            diff_results = by_difficulty[diff]
-            count = len(diff_results)
-            success_rate = sum(1 for r in diff_results if r["success"]) / count
-            avg_steps = sum(r["steps"] for r in diff_results) / count
-            avg_achievements = sum(len(r["achievements"]) for r in diff_results) / count
-
-            print(
-                f"{diff:<15} {count:<8} {success_rate:<10.1%} {avg_steps:<12.1f} {avg_achievements:.1f}"
-            )
-
-        # Overall stats
-        total_success = sum(1 for r in results if r["success"])
-        print(
-            f"\nOverall success rate: {total_success}/{len(results)} ({total_success / len(results):.1%})"
-        )
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Run Crafter dataset instances")
-    parser.add_argument("dataset", help="Dataset name")
-    parser.add_argument(
-        "-n", "--num-instances", type=int, default=10, help="Number of instances to run"
-    )
-    parser.add_argument(
-        "-d",
-        "--difficulties",
-        nargs="+",
-        choices=["easy", "normal", "hard", "peaceful", "resource_rich"],
-        help="Filter by difficulties",
-    )
-    parser.add_argument(
-        "-t",
-        "--impetus-types",
-        nargs="+",
-        choices=["general", "focused", "speedrun"],
-        help="Filter by impetus types",
-    )
-    parser.add_argument(
-        "-s", "--split", choices=["train", "val", "test"], help="Filter by dataset split"
-    )
-    parser.add_argument("--render", action="store_true", help="Render the environment")
-    parser.add_argument("--max-steps", type=int, default=1000, help="Maximum steps per episode")
-
-    args = parser.parse_args()
-
-    runner = CrafterDatasetRunner()
-    runner.run_batch(
-        dataset_name=args.dataset,
-        num_instances=args.num_instances,
-        difficulties=args.difficulties,
-        impetus_types=args.impetus_types,
-        split=args.split,
-        render=args.render,
-        max_steps=args.max_steps,
-    )
-
-
-if __name__ == "__main__":
-    main()