themis-eval 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

themis/cli/commands/config_commands.py

@@ -0,0 +1,244 @@
+"""Configuration-related commands."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Literal
+
+from cyclopts import Parameter
+
+from themis.cli.utils import effective_total, export_outputs
+from themis.config import (
+    load_dataset_from_config,
+    load_experiment_config,
+    run_experiment_from_config,
+    summarize_report_for_config,
+)
+from themis.utils.logging_utils import configure_logging
+from themis.utils.progress import ProgressReporter
+
+
+def run_configured_experiment(
+    *,
+    config: Annotated[
+        Path, Parameter(help="Path to a Hydra/OmegaConf experiment config file")
+    ],
+    overrides: Annotated[
+        tuple[str, ...],
+        Parameter(
+            help="Optional Hydra-style overrides (e.g. generation.sampling.temperature=0.2)",
+            show_default=False,
+        ),
+    ] = (),
+    log_level: Annotated[
+        str, Parameter(help="Logging level (critical/error/warning/info/debug/trace)")
+    ] = "info",
+    csv_output: Annotated[
+        Path | None, Parameter(help="Write CSV export to this path")
+    ] = None,
+    html_output: Annotated[
+        Path | None, Parameter(help="Write HTML summary to this path")
+    ] = None,
+    json_output: Annotated[
+        Path | None, Parameter(help="Write JSON export to this path")
+    ] = None,
+) -> int:
+    """Execute an experiment described via config file."""
+    configure_logging(log_level)
+    experiment_config = load_experiment_config(config, overrides)
+    dataset = load_dataset_from_config(experiment_config)
+    total = effective_total(len(dataset), experiment_config.max_samples)
+    with ProgressReporter(total=total, description="Generating") as progress:
+        report = run_experiment_from_config(
+            experiment_config,
+            dataset=dataset,
+            on_result=progress.on_result,
+        )
+    print(summarize_report_for_config(experiment_config, report))
+    export_outputs(
+        report,
+        csv_output=csv_output,
+        html_output=html_output,
+        json_output=json_output,
+        title=f"{experiment_config.name} experiment",
+    )
+    return 0
+
+
+def validate_config(
+    *,
+    config: Annotated[Path, Parameter(help="Path to config file to validate")],
+) -> int:
+    """Validate a configuration file without running the experiment."""
+    if not config.exists():
+        print(f"❌ Error: Config file not found: {config}")
+        return 1
+
+    print(f"Validating config: {config}")
+    print("-" * 60)
+
+    try:
+        # Try to load as experiment config
+        experiment_config = load_experiment_config(config, overrides=())
+        print("✓ Config file is valid")
+        print(f"\nExperiment: {experiment_config.name}")
+        print(f"Run ID: {experiment_config.run_id or '(auto-generated)'}")
+        print(f"Resume: {experiment_config.resume}")
+        print(f"Max samples: {experiment_config.max_samples or '(unlimited)'}")
+
+        print("\nDataset:")
+        print(f"  Source: {experiment_config.dataset.source}")
+        print(f"  Split: {experiment_config.dataset.split}")
+        if experiment_config.dataset.limit:
+            print(f"  Limit: {experiment_config.dataset.limit}")
+        if experiment_config.dataset.subjects:
+            print(f"  Subjects: {', '.join(experiment_config.dataset.subjects)}")
+
+        print("\nGeneration:")
+        print(f"  Model: {experiment_config.generation.model_identifier}")
+        print(f"  Provider: {experiment_config.generation.provider.name}")
+        print(f"  Temperature: {experiment_config.generation.sampling.temperature}")
+        print(f"  Max tokens: {experiment_config.generation.sampling.max_tokens}")
+
+        if experiment_config.storage.path:
+            print(f"\nStorage: {experiment_config.storage.path}")
+
+        return 0
+    except Exception as e:
+        print(f"❌ Config validation failed: {e}")
+        return 1
+
+
+def init_config(
+    *,
+    output: Annotated[Path, Parameter(help="Output path for config file")] = Path(
+        "themis_config.yaml"
+    ),
+    template: Annotated[
+        Literal["basic", "math500", "inline"],
+        Parameter(help="Config template to generate"),
+    ] = "basic",
+) -> int:
+    """Generate a sample configuration file for use with run-config."""
+    templates = {
+        "basic": """name: my_experiment
+task: math500
+dataset:
+  source: huggingface
+  dataset_id: math500
+  limit: 50
+generation:
+  model_identifier: fake-math-llm
+  provider:
+    name: fake
+  sampling:
+    temperature: 0.0
+    top_p: 0.95
+    max_tokens: 512
+  runner:
+    max_parallel: 1
+    max_retries: 3
+storage:
+  path: .cache/my_experiment
+run_id: my-experiment-001
+resume: true
+""",
+        "math500": """name: math500_evaluation
+task: math500
+dataset:
+  source: huggingface
+  dataset_id: math500
+  limit: null  # No limit, run full dataset
+  subjects:
+    - algebra
+    - geometry
+generation:
+  model_identifier: my-model
+  provider:
+    name: openai-compatible
+    options:
+      base_url: http://localhost:1234/v1
+      api_key: not-needed
+      model_name: qwen2.5-7b-instruct
+      timeout: 60
+  sampling:
+    temperature: 0.0
+    top_p: 0.95
+    max_tokens: 512
+  runner:
+    max_parallel: 4
+    max_retries: 3
+    retry_initial_delay: 0.5
+    retry_backoff_multiplier: 2.0
+    retry_max_delay: 2.0
+storage:
+  path: .cache/math500
+run_id: math500-run-001
+resume: true
+max_samples: null
+""",
+        "inline": """name: inline_dataset_experiment
+task: math500
+dataset:
+  source: inline
+  inline_samples:
+    - unique_id: sample-1
+      problem: "What is 2 + 2?"
+      answer: "4"
+      subject: arithmetic
+      level: 1
+    - unique_id: sample-2
+      problem: "Solve for x: 2x + 5 = 13"
+      answer: "4"
+      subject: algebra
+      level: 2
+generation:
+  model_identifier: fake-math-llm
+  provider:
+    name: fake
+  sampling:
+    temperature: 0.0
+    top_p: 0.95
+    max_tokens: 512
+storage:
+  path: .cache/inline_experiment
+run_id: inline-001
+resume: true
+""",
+    }
+
+    if output.exists():
+        print(f"❌ Error: File already exists: {output}")
+        print("   Use a different --output path or delete the existing file")
+        return 1
+
+    config_content = templates[template]
+
+    try:
+        output.parent.mkdir(parents=True, exist_ok=True)
+        with open(output, "w") as f:
+            f.write(config_content)
+
+        print(f"✓ Created config file: {output}")
+        print(f"  Template: {template}")
+        print("\n📝 Next steps:")
+        print(f"  1. Edit {output} to customize settings")
+        print(
+            f"  2. Validate: uv run python -m themis.cli validate-config --config {output}"
+        )
+        print(f"  3. Run: uv run python -m themis.cli run-config --config {output}")
+
+        if template == "math500":
+            print("\n⚠️  Remember to:")
+            print("  • Update provider.options.base_url with your LLM server endpoint")
+            print("  • Update provider.options.model_name with your actual model")
+            print("  • Set provider.options.api_key if required by your server")
+        elif template == "inline":
+            print("\n💡 Tip:")
+            print("  • Add more samples to dataset.inline_samples list")
+            print("  • Each sample needs: unique_id, problem, answer")
+
+        return 0
+    except Exception as e:
+        print(f"❌ Error creating config file: {e}")
+        return 1

themis/cli/commands/cost.py

@@ -0,0 +1,214 @@
+"""Cost estimation and tracking commands."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from cyclopts import Parameter
+
+from themis.experiment.cost import estimate_experiment_cost
+from themis.experiment.pricing import (
+    compare_provider_costs,
+    get_all_models,
+    get_provider_pricing,
+)
+
+
+def estimate_cost_command(
+    *,
+    model: Annotated[
+        str, Parameter(help="Model identifier (e.g., gpt-4, claude-3-5-sonnet)")
+    ],
+    dataset_size: Annotated[int, Parameter(help="Number of samples in dataset")],
+    avg_prompt_tokens: Annotated[
+        int, Parameter(help="Average prompt tokens per sample")
+    ] = 500,
+    avg_completion_tokens: Annotated[
+        int, Parameter(help="Average completion tokens per sample")
+    ] = 300,
+) -> int:
+    """Estimate cost for an experiment before running.
+
+    Examples:
+        # Estimate cost for 100 samples with GPT-4
+        uv run python -m themis.cli estimate-cost \\
+          --model gpt-4 \\
+          --dataset-size 100
+
+        # Custom token estimates
+        uv run python -m themis.cli estimate-cost \\
+          --model claude-3-5-sonnet-20241022 \\
+          --dataset-size 1000 \\
+          --avg-prompt-tokens 800 \\
+          --avg-completion-tokens 400
+    """
+    try:
+        estimate = estimate_experiment_cost(
+            model=model,
+            dataset_size=dataset_size,
+            avg_prompt_tokens=avg_prompt_tokens,
+            avg_completion_tokens=avg_completion_tokens,
+        )
+
+        print("=" * 80)
+        print("Cost Estimate")
+        print("=" * 80)
+        print(f"\nModel: {model}")
+        print(f"Dataset size: {dataset_size} samples")
+        print(
+            f"Avg tokens per sample: {avg_prompt_tokens} prompt + {avg_completion_tokens} completion"
+        )
+
+        print("\n💰 Estimated Cost")
+        print(f"  Total: ${estimate.estimated_cost:.4f}")
+        print(f"  Per sample: ${estimate.assumptions['cost_per_sample']:.6f}")
+        print(f"  95% CI: ${estimate.lower_bound:.4f} - ${estimate.upper_bound:.4f}")
+
+        print("\n📊 Breakdown")
+        for phase, cost in estimate.breakdown_by_phase.items():
+            print(f"  {phase.capitalize()}: ${cost:.4f}")
+
+        print("\n" + "=" * 80)
+
+        # Warning if cost is high
+        if estimate.estimated_cost > 10.0:
+            print(
+                f"\n⚠️  Warning: Estimated cost is ${estimate.estimated_cost:.2f}. "
+                "Consider using --limit for initial testing."
+            )
+
+        return 0
+
+    except Exception as e:
+        print(f"Error estimating cost: {e}")
+        return 1
+
+
+def show_pricing_command(
+    *,
+    model: Annotated[
+        str | None, Parameter(help="Show pricing for specific model")
+    ] = None,
+    list_all: Annotated[bool, Parameter(help="List all available models")] = False,
+    compare_models: Annotated[
+        list[str] | None, Parameter(help="Compare costs for multiple models")
+    ] = None,
+) -> int:
+    """Show pricing information for LLM models.
+
+    Examples:
+        # Show pricing for a specific model
+        uv run python -m themis.cli show-pricing --model gpt-4
+
+        # List all models with pricing
+        uv run python -m themis.cli show-pricing --list-all
+
+        # Compare pricing across models (use repeated --compare-models flags)
+        uv run python -m themis.cli show-pricing \\
+          --compare-models gpt-4 \\
+          --compare-models gpt-3.5-turbo \\
+          --compare-models claude-3-haiku-20240307
+    """
+    try:
+        if list_all:
+            models = get_all_models()
+            print("=" * 80)
+            print(f"Available Models ({len(models)} total)")
+            print("=" * 80)
+            print("\nModel pricing (per 1M tokens):\n")
+
+            for model_name in sorted(models):
+                pricing = get_provider_pricing(model_name)
+                prompt_price = pricing["prompt_tokens"] * 1_000_000
+                completion_price = pricing["completion_tokens"] * 1_000_000
+                print(
+                    f"  {model_name:40s} | "
+                    f"Prompt: ${prompt_price:6.2f} | "
+                    f"Completion: ${completion_price:6.2f}"
+                )
+
+            print("\n" + "=" * 80)
+            return 0
+
+        if compare_models:
+            # Compare costs for standard workload
+            prompt_tokens = 1000
+            completion_tokens = 500
+
+            costs = compare_provider_costs(
+                prompt_tokens, completion_tokens, compare_models
+            )
+
+            print("=" * 80)
+            print(
+                f"Cost Comparison ({prompt_tokens} prompt + {completion_tokens} completion tokens)"
+            )
+            print("=" * 80)
+            print()
+
+            # Sort by cost
+            sorted_costs = sorted(costs.items(), key=lambda x: x[1])
+
+            for model_name, cost in sorted_costs:
+                # Calculate cost per 1M tokens for comparison
+                pricing = get_provider_pricing(model_name)
+                prompt_price = pricing["prompt_tokens"] * 1_000_000
+                completion_price = pricing["completion_tokens"] * 1_000_000
+
+                print(f"  {model_name:40s} | ${cost:.6f}")
+                print(
+                    f"    {'':40s} | (${prompt_price:.2f} / ${completion_price:.2f} per 1M)"
+                )
+
+            # Show relative costs
+            if sorted_costs:
+                cheapest_cost = sorted_costs[0][1]
+                print(f"\nRelative costs (vs {sorted_costs[0][0]}):")
+                for model_name, cost in sorted_costs[1:]:
+                    multiplier = cost / cheapest_cost if cheapest_cost > 0 else 0
+                    print(f"  {model_name:40s} | {multiplier:.1f}x more expensive")
+
+            print("\n" + "=" * 80)
+            return 0
+
+        if model:
+            pricing = get_provider_pricing(model)
+            prompt_price = pricing["prompt_tokens"] * 1_000_000
+            completion_price = pricing["completion_tokens"] * 1_000_000
+
+            print("=" * 80)
+            print(f"Pricing for {model}")
+            print("=" * 80)
+            print(f"\nPrompt tokens: ${prompt_price:.2f} per 1M tokens")
+            print(f"Completion tokens: ${completion_price:.2f} per 1M tokens")
+
+            # Show example costs
+            print("\nExample costs:")
+            examples = [
+                (100, 50, "Short query"),
+                (500, 300, "Medium query"),
+                (1000, 500, "Long query"),
+            ]
+
+            for prompt_tok, completion_tok, label in examples:
+                from themis.experiment.pricing import calculate_cost
+
+                cost = calculate_cost(model, prompt_tok, completion_tok)
+                print(
+                    f"  {label:15s} ({prompt_tok:4d} + {completion_tok:4d} tokens): ${cost:.6f}"
+                )
+
+            print("\n" + "=" * 80)
+            return 0
+
+        # No options provided
+        print("Error: Must specify --model, --list-all, or --compare-models")
+        print("Use --help for usage information")
+        return 1
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+
+
+__all__ = ["estimate_cost_command", "show_pricing_command"]

themis/cli/commands/demo.py

@@ -0,0 +1,68 @@
+"""Demo command implementation."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+from cyclopts import Parameter
+
+from themis.cli.utils import effective_total, export_outputs
+from themis.experiment import math as math_experiment
+from themis.utils.logging_utils import configure_logging
+from themis.utils.progress import ProgressReporter
+
+
+def demo_command(
+    *,
+    max_samples: Annotated[
+        int | None, Parameter(help="Limit number of demo samples")
+    ] = None,
+    log_level: Annotated[
+        str, Parameter(help="Logging level (critical/error/warning/info/debug/trace)")
+    ] = "info",
+    csv_output: Annotated[
+        Path | None, Parameter(help="Write CSV export to this path")
+    ] = None,
+    html_output: Annotated[
+        Path | None, Parameter(help="Write HTML summary to this path")
+    ] = None,
+    json_output: Annotated[
+        Path | None, Parameter(help="Write JSON export to this path")
+    ] = None,
+) -> int:
+    """Run the built-in demo dataset."""
+    configure_logging(log_level)
+    dataset = [
+        {
+            "unique_id": "demo-1",
+            "problem": "Convert the point (0,3) in rectangular coordinates to polar coordinates.",
+            "answer": "\\left( 3, \\frac{\\pi}{2} \\right)",
+            "subject": "precalculus",
+            "level": 2,
+        },
+        {
+            "unique_id": "demo-2",
+            "problem": "What is 7 + 5?",
+            "answer": "12",
+            "subject": "arithmetic",
+            "level": 1,
+        },
+    ]
+    experiment = math_experiment.build_math500_zero_shot_experiment()
+    total = effective_total(len(dataset), max_samples)
+    with ProgressReporter(total=total, description="Generating") as progress:
+        report = experiment.run(
+            dataset,
+            max_samples=max_samples,
+            on_result=progress.on_result,
+        )
+    print(math_experiment.summarize_report(report))
+    export_outputs(
+        report,
+        csv_output=csv_output,
+        html_output=html_output,
+        json_output=json_output,
+        title="Demo experiment",
+    )
+    return 0

themis/cli/commands/info.py

@@ -0,0 +1,90 @@
+"""System information and listing commands."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Annotated
+
+from cyclopts import Parameter
+
+from themis.providers.registry import _REGISTRY
+
+
+def show_info() -> int:
+    """Show system information and installed components."""
+    import themis
+    from themis import _version
+
+    print("Themis Information")
+    print("=" * 60)
+    print(f"Version: {getattr(_version, '__version__', 'unknown')}")
+    print(f"Python: {sys.version.split()[0]}")
+    print(f"Platform: {sys.platform}")
+
+    print("\n📦 Installed Providers:")
+    providers = sorted(_REGISTRY._factories.keys())
+    for provider in providers:
+        print(f"  ✓ {provider}")
+
+    print("\n📊 Available Benchmarks:")
+    benchmarks = [
+        "demo",
+        "math500",
+        "aime24",
+        "aime25",
+        "amc23",
+        "olympiadbench",
+        "beyondaime",
+        "supergpqa",
+        "mmlu-pro",
+        "inline (via config)",
+    ]
+    for bench in benchmarks:
+        print(f"  ✓ {bench}")
+
+    print("\n📁 Example Locations:")
+    examples_dir = Path(themis.__file__).parent.parent / "examples"
+    if examples_dir.exists():
+        print(f"  {examples_dir}")
+        example_dirs = sorted(
+            [
+                d.name
+                for d in examples_dir.iterdir()
+                if d.is_dir() and not d.name.startswith("_")
+            ]
+        )
+        for ex in example_dirs:
+            print(f"    • {ex}/")
+
+    print("\n📚 Documentation:")
+    print("  examples/README.md - Comprehensive tutorial cookbook")
+    print("  COOKBOOK.md - Quick reference guide")
+    print("  docs/ - Detailed documentation")
+
+    print("\n🚀 Quick Start:")
+    print("  uv run python -m themis.cli demo")
+    print("  uv run python -m themis.cli list-providers")
+    print("  uv run python -m themis.cli list-benchmarks")
+
+    return 0
+
+
+def new_project(
+    *,
+    project_name: Annotated[str, Parameter(help="The name of the new project")],
+    project_path: Annotated[
+        Path,
+        Parameter(help="The path where the new project will be created"),
+    ] = Path("."),
+) -> int:
+    """Create a new Themis project."""
+    from themis.cli.new_project import create_project
+
+    try:
+        create_project(project_name, project_path)
+        print(f"Successfully created new project '{project_name}' in {project_path}")
+        return 0
+    except FileExistsError as e:
+        print(f"Error: {e}")
+        return 1