janus-labs 0.2.0__py3-none-any.whl

cli/main.py

@@ -0,0 +1,690 @@
+"""Command-line interface for Janus Labs."""
+
+import argparse
+import json
+import os
+import tempfile
+from pathlib import Path
+import sys
+
+from config.detection import detect_config
+from suite.comparison import (
+    compare_results,
+    comparison_to_dict,
+    export_comparison_json,
+    print_comparison_text,
+)
+from suite.export.github import generate_github_summary, print_github_annotations
+from suite.export.html import export_html
+from suite.export.json_export import export_json, load_json
+from suite.registry import get_suite
+from suite.runner import SuiteRunConfig, run_suite
+from suite.thresholds import default_thresholds, load_thresholds
+
+from cli.submit import cmd_submit, submit_result
+from cli.output import print_benchmark_result, print_step, print_error, print_warning
+from cli.clipboard import copy_to_clipboard, is_clipboard_available
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="janus-labs",
+        description="Janus Labs - 3DMark for AI Agents",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    run_parser = subparsers.add_parser("run", help="Run a benchmark suite")
+    run_parser.add_argument("--suite", required=True, help="Suite ID to run")
+    run_parser.add_argument("--output", "-o", default="result.json", help="Output file")
+    run_parser.add_argument(
+        "--format",
+        choices=["json", "html", "both"],
+        default="json",
+    )
+    run_parser.add_argument(
+        "--judge",
+        action="store_true",
+        help="Use LLM-as-judge scoring via GEval (requires API key, slower but differentiates)",
+    )
+    run_parser.add_argument(
+        "--model",
+        default="gpt-4o",
+        help="LLM model for judge scoring (default: gpt-4o)",
+    )
+
+    compare_parser = subparsers.add_parser("compare", help="Compare two results")
+    compare_parser.add_argument("baseline", help="Baseline result JSON")
+    compare_parser.add_argument("current", help="Current result JSON")
+    compare_parser.add_argument(
+        "--threshold",
+        type=float,
+        default=5.0,
+        help="Regression threshold (%%)",
+    )
+    compare_parser.add_argument(
+        "--config",
+        "-c",
+        help="Threshold config YAML file (default: use suite defaults)",
+    )
+    compare_parser.add_argument(
+        "--output",
+        "-o",
+        help="Output comparison result to JSON file",
+    )
+    compare_parser.add_argument(
+        "--format",
+        choices=["text", "json", "github"],
+        default="text",
+        help="Output format (github = GitHub Actions annotations)",
+    )
+
+    export_parser = subparsers.add_parser("export", help="Export result to format")
+    export_parser.add_argument("input", help="Input JSON result")
+    export_parser.add_argument("--format", choices=["html", "json"], required=True)
+    export_parser.add_argument("--output", "-o", help="Output file")
+
+    baseline_parser = subparsers.add_parser("baseline", help="Manage baselines")
+    baseline_sub = baseline_parser.add_subparsers(dest="baseline_command", required=True)
+    update_parser = baseline_sub.add_parser("update", help="Update baseline from current result")
+    update_parser.add_argument("result", help="Current result JSON to promote to baseline")
+    update_parser.add_argument("--output", "-o", default="baseline.json", help="Baseline output path")
+    update_parser.add_argument("--force", "-f", action="store_true", help="Overwrite existing baseline")
+
+    show_parser = baseline_sub.add_parser("show", help="Show baseline info")
+    show_parser.add_argument("baseline", help="Baseline JSON file")
+
+    # Submit command (E6 Community Platform)
+    submit_parser = subparsers.add_parser("submit", help="Submit results to leaderboard")
+    submit_parser.add_argument("result_file", help="Path to result.json")
+    submit_parser.add_argument(
+        "--dry-run", action="store_true", help="Show payload without submitting"
+    )
+    submit_parser.add_argument(
+        "--github", type=str, help="GitHub handle for attribution"
+    )
+
+    # Init command (E7 Agent Execution)
+    init_parser = subparsers.add_parser("init", help="Initialize task workspace")
+    init_parser.add_argument("--suite", required=True, help="Suite ID")
+    init_parser.add_argument("--behavior", required=True, help="Behavior ID")
+    init_parser.add_argument(
+        "--output", "-o",
+        default="./janus-task",
+        help="Output directory for workspace",
+    )
+
+    # Score command (E7 Agent Execution)
+    score_parser = subparsers.add_parser("score", help="Score completed task")
+    score_parser.add_argument(
+        "--workspace", "-w",
+        default=".",
+        help="Path to task workspace (default: current directory)",
+    )
+    score_parser.add_argument(
+        "--output", "-o",
+        help="Output result to JSON file",
+    )
+    score_parser.add_argument(
+        "--judge",
+        action="store_true",
+        help="Enable LLM-as-judge scoring via DeepEval GEval (requires API key)",
+    )
+    score_parser.add_argument(
+        "--model",
+        default="gpt-4o",
+        help="LLM model for judge scoring (default: gpt-4o)",
+    )
+    score_parser.add_argument(
+        "--bundle",
+        help="Path to bundle.json for judge scoring (optional, uses mock if not provided)",
+    )
+
+    # Bench command (Tinkerer-First P0) - Zero-friction benchmark
+    bench_parser = subparsers.add_parser(
+        "bench",
+        help="Run benchmark with zero friction (detect config, score, optionally submit)",
+    )
+    bench_parser.add_argument(
+        "--suite",
+        default="refactor-storm",
+        help="Suite ID (default: refactor-storm)",
+    )
+    bench_parser.add_argument(
+        "--behavior",
+        default="BHV-001-test-cheating",
+        help="Behavior ID (default: BHV-001-test-cheating)",
+    )
+    bench_parser.add_argument(
+        "--submit",
+        action="store_true",
+        help="Submit results to public leaderboard",
+    )
+    bench_parser.add_argument(
+        "--github",
+        type=str,
+        help="GitHub handle for attribution (requires --submit)",
+    )
+    bench_parser.add_argument(
+        "--model",
+        default="gpt-4o",
+        help="LLM model for judge scoring (default: gpt-4o)",
+    )
+    bench_parser.add_argument(
+        "--no-copy",
+        dest="copy",
+        action="store_false",
+        help="Don't copy share URL to clipboard after submit",
+    )
+    bench_parser.add_argument(
+        "--output", "-o",
+        help="Output result to JSON file (default: temp file)",
+    )
+
+    args = parser.parse_args()
+
+    if args.command == "run":
+        return cmd_run(args)
+    if args.command == "compare":
+        return cmd_compare(args)
+    if args.command == "export":
+        return cmd_export(args)
+    if args.command == "baseline":
+        return cmd_baseline(args)
+    if args.command == "submit":
+        return cmd_submit(args)
+    if args.command == "init":
+        return cmd_init(args)
+    if args.command == "score":
+        return cmd_score(args)
+    if args.command == "bench":
+        return cmd_bench(args)
+    return 1
+
+
+def _default_execute_fn(rollout_index: int, behavior_id: str) -> dict:
+    """Hash-based stub for testing without LLM. Produces deterministic fake scores."""
+    value = abs(hash(f"{behavior_id}:{rollout_index}")) % 100
+    score = 0.6 + (value / 250.0)
+    return {"score": min(score, 0.99)}
+
+
+def _create_geval_execute_fn(suite, model: str):
+    """
+    Create an execute function that uses GEval LLM-as-judge scoring.
+
+    This replaces the hash stub with real LLM evaluation, providing
+    differentiated scores based on qualitative assessment.
+
+    Args:
+        suite: BenchmarkSuite containing behavior specs
+        model: LLM model for judging (e.g., gpt-4o, claude-3-5-sonnet)
+
+    Returns:
+        Execute function compatible with run_suite()
+    """
+    from gauge.judge import score_with_judge, create_mock_bundle
+
+    # Build behavior lookup for O(1) access
+    behavior_map = {b.behavior_id: b for b in suite.behaviors}
+
+    def execute_fn(rollout_index: int, behavior_id: str) -> dict:
+        behavior = behavior_map.get(behavior_id)
+        if behavior is None:
+            print(f"Warning: Unknown behavior {behavior_id}", file=sys.stderr)
+            return {"score": 0.0}
+
+        # Create mock bundle for GEval evaluation
+        # In real usage, this would contain actual agent execution artifacts
+        mock_diff = f"""
+# Mock solution for {behavior.name}
+# Rollout {rollout_index}
+
+def solution():
+    '''Implementation of {behavior_id}'''
+    # Code would be captured from actual agent execution
+    pass
+"""
+        bundle = create_mock_bundle(
+            code_diff=mock_diff,
+            test_output="All tests passed (mock)",
+            exit_code="success",
+        )
+
+        # Use outcome score of 0.7 as baseline (reasonable mock performance)
+        # Real implementation would use actual outcome-based scoring
+        baseline_outcome = 0.7
+
+        try:
+            result = score_with_judge(
+                behavior=behavior,
+                bundle=bundle,
+                outcome_score=baseline_outcome,
+                model=model,
+            )
+            return {"score": result.combined_score}
+        except ValueError as e:
+            print(f"Judge error: {e}", file=sys.stderr)
+            return {"score": 0.0}
+
+    return execute_fn
+
+
+def cmd_init(args) -> int:
+    """Initialize a task workspace for outcome-based benchmarking."""
+    from scaffold.workspace import init_workspace
+
+    suite = get_suite(args.suite)
+    if suite is None:
+        print(f"Unknown suite: {args.suite}", file=sys.stderr)
+        return 1
+
+    # Find the behavior
+    behavior = None
+    for b in suite.behaviors:
+        if b.behavior_id == args.behavior:
+            behavior = b
+            break
+
+    if behavior is None:
+        print(f"Unknown behavior: {args.behavior}", file=sys.stderr)
+        print(f"Available behaviors in {args.suite}:")
+        for b in suite.behaviors:
+            print(f"  - {b.behavior_id}: {b.name}")
+        return 1
+
+    output_dir = Path(args.output)
+    if output_dir.exists() and any(output_dir.iterdir()):
+        print(f"Directory not empty: {output_dir}", file=sys.stderr)
+        print("Use a new directory or clear the existing one.")
+        return 1
+
+    metadata = init_workspace(output_dir, suite, behavior)
+
+    print(f"Task workspace initialized: {output_dir}")
+    print(f"  Suite: {suite.suite_id}")
+    print(f"  Behavior: {behavior.behavior_id} - {behavior.name}")
+    print()
+    print("Next steps:")
+    print(f"  1. cd {output_dir}")
+    print("  2. Open in VS Code and run your AI agent")
+    print("  3. Run: janus score")
+
+    return 0
+
+
+def cmd_score(args) -> int:
+    """Score a completed task workspace."""
+    from scaffold.workspace import load_task_metadata
+    from scaffold.scorer import score_outcome
+
+    workspace = Path(args.workspace).resolve()
+
+    metadata = load_task_metadata(workspace)
+    if metadata is None:
+        print("Not a Janus Labs task workspace.", file=sys.stderr)
+        print("No .janus-task.json found.")
+        print()
+        print("To create a task workspace: janus init --suite <suite> --behavior <behavior>")
+        return 1
+
+    print(f"Scoring task: {metadata.behavior_id}")
+    print(f"  Suite: {metadata.suite_id}")
+    print(f"  Behavior: {metadata.behavior_name}")
+    print()
+
+    # Outcome-based scoring (always runs)
+    result = score_outcome(
+        workspace_dir=workspace,
+        behavior_id=metadata.behavior_id,
+        threshold=metadata.threshold,
+        rubric=metadata.rubric,
+    )
+
+    # Display outcome results
+    status = "PASS" if result.passed_threshold else "FAIL"
+    print(f"Outcome Score: {result.raw_score:.1f}/10 ({status})")
+    print(f"Threshold: {metadata.threshold}")
+    print()
+    print("Scoring notes:")
+    for note in result.scoring_notes:
+        print(f"  - {note}")
+    print()
+    print(f"Files changed: {len(result.git_diff['files_changed'])}")
+    for f in result.git_diff["files_changed"]:
+        print(f"  - {f}")
+    print()
+    print(f"Tests: {result.test_results['passed']} passed, {result.test_results['failed']} failed")
+
+    # Judge scoring (optional, requires --judge flag)
+    judge_result = None
+    if args.judge:
+        print()
+        print("=" * 60)
+        print("LLM-as-Judge Scoring (GEval)")
+        print("=" * 60)
+
+        try:
+            from gauge.judge import score_with_judge, load_bundle_from_file, create_mock_bundle
+            from forge.behavior import BehaviorSpec
+
+            # Reconstruct BehaviorSpec from metadata
+            behavior = BehaviorSpec(
+                behavior_id=metadata.behavior_id,
+                name=metadata.behavior_name,
+                description=metadata.behavior_description,
+                rubric=metadata.rubric,
+                threshold=metadata.threshold,
+                disconfirmers=metadata.disconfirmers,
+                taxonomy_code=metadata.taxonomy_code,
+            )
+
+            # Load bundle: explicit file > captured bundle > mock bundle
+            if args.bundle:
+                print(f"Loading bundle from: {args.bundle}")
+                bundle = load_bundle_from_file(args.bundle)
+            elif result.bundle:
+                # E8-S2: Use real captured bundle from workspace
+                print("Using captured workspace bundle")
+                bundle = result.bundle
+            else:
+                print("Using mock bundle (no bundle captured)")
+                bundle = create_mock_bundle(
+                    code_diff=result.git_diff.get("patch", ""),
+                    test_output=result.test_results.get("output", ""),
+                    exit_code="success" if result.passed_threshold else "error",
+                )
+
+            print(f"Model: {args.model}")
+            print()
+
+            judge_result = score_with_judge(
+                behavior=behavior,
+                bundle=bundle,
+                outcome_score=result.normalized_score,
+                model=args.model,
+            )
+
+            print(f"GEval Score: {judge_result.geval_score_10:.1f}/10")
+            print(f"Reason: {judge_result.reason}")
+            print()
+            print(f"Combined Score: {judge_result.combined_score_10:.1f}/10")
+            print("  (40% outcome + 60% qualitative)")
+
+        except ValueError as e:
+            print(f"Judge scoring failed: {e}", file=sys.stderr)
+            return 1
+        except Exception as e:
+            print(f"Judge scoring error: {e}", file=sys.stderr)
+            # Continue without judge score
+
+    # Output results
+    if args.output:
+        output_data = {
+            "behavior_id": result.behavior_id,
+            "outcome_score": result.raw_score,
+            "normalized_score": result.normalized_score,
+            "passed": result.passed_threshold,
+            "threshold": metadata.threshold,
+            "notes": result.scoring_notes,
+            "git_diff": result.git_diff,
+            "test_results": result.test_results,
+        }
+
+        if judge_result:
+            output_data["judge"] = {
+                "geval_score": judge_result.geval_score_10,
+                "combined_score": judge_result.combined_score_10,
+                "reason": judge_result.reason,
+                "model": judge_result.model,
+            }
+
+        Path(args.output).write_text(json.dumps(output_data, indent=2))
+        print()
+        print(f"Results saved to: {args.output}")
+
+        # E8-S2: Save bundle for future GEval evaluation
+        if result.bundle:
+            bundle_path = Path(args.output).with_suffix(".bundle.json")
+            bundle_path.write_text(json.dumps(result.bundle, indent=2))
+            print(f"Bundle saved to: {bundle_path}")
+
+    return 0 if result.passed_threshold else 1
+
+
+def cmd_bench(args) -> int:
+    """
+    Zero-friction benchmark command for tinkerers.
+
+    Detects config, runs GEval scoring, and optionally submits to leaderboard.
+    """
+    from scaffold.workspace import init_workspace
+
+    # Step 1: Detect config
+    print_step(1, 4, "Detecting config")
+    config_metadata = detect_config(Path.cwd())
+    if config_metadata.config_source == "custom":
+        print(f"    Found: {', '.join(config_metadata.config_files)} (hash: {config_metadata.config_hash})")
+    else:
+        print("    Using default config")
+
+    # Step 2: Get suite and behavior
+    print_step(2, 4, "Loading behavior")
+    suite = get_suite(args.suite)
+    if suite is None:
+        print_error(f"Unknown suite: {args.suite}")
+        return 1
+
+    behavior = None
+    for b in suite.behaviors:
+        if b.behavior_id == args.behavior:
+            behavior = b
+            break
+
+    if behavior is None:
+        print_error(f"Unknown behavior: {args.behavior}")
+        print(f"Available behaviors in {args.suite}:")
+        for b in suite.behaviors:
+            print(f"  - {b.behavior_id}: {b.name}")
+        return 1
+
+    print(f"    Behavior: {behavior.behavior_id} - {behavior.name}")
+
+    # Step 3: Score with GEval
+    print_step(3, 4, f"Scoring with GEval ({args.model})")
+
+    # Run the suite for single behavior
+    config = SuiteRunConfig(suite=suite, config_metadata=config_metadata)
+    execute_fn = _create_geval_execute_fn(suite, args.model)
+
+    try:
+        result = run_suite(config, execute_fn)
+    except Exception as e:
+        print_error(f"Scoring failed: {e}")
+        return 1
+
+    # Save result to file
+    if args.output:
+        output_path = Path(args.output)
+    else:
+        # Use temp file if no output specified
+        fd, tmp_path = tempfile.mkstemp(suffix=".json", prefix="janus-bench-")
+        os.close(fd)
+        output_path = Path(tmp_path)
+
+    export_json(result, str(output_path))
+    print(f"    Score: {result.headline_score:.1f} (Grade {result.grade})")
+
+    # Step 4: Submit if requested
+    if args.submit:
+        print_step(4, 4, "Submitting to leaderboard")
+        try:
+            submit_data = submit_result(
+                str(output_path),
+                github_handle=args.github,
+                dry_run=False,
+            )
+
+            if submit_data.get("status") == "success":
+                # Print rich result
+                print_benchmark_result(
+                    score=submit_data["score"],
+                    rank=submit_data.get("rank"),
+                    percentile=submit_data.get("percentile"),
+                    share_url=submit_data.get("share_url"),
+                )
+
+                # Copy to clipboard if available and not disabled
+                if args.copy and submit_data.get("share_url"):
+                    if is_clipboard_available():
+                        if copy_to_clipboard(submit_data["share_url"]):
+                            print("\nCopied to clipboard!")
+                        else:
+                            print_warning("Could not copy to clipboard")
+
+            return 0
+
+        except RuntimeError as e:
+            print_error(f"Submit failed: {e}")
+            # Still show local result
+            print_benchmark_result(
+                score=result.headline_score,
+                grade=result.grade,
+            )
+            print(f"\nResult saved to: {output_path}")
+            return 1
+    else:
+        # Not submitting - show local result only
+        print_step(4, 4, "Complete (not submitting)")
+        print_benchmark_result(
+            score=result.headline_score,
+            grade=result.grade,
+        )
+        print(f"\nResult saved to: {output_path}")
+        print("\nTo submit to leaderboard, run:")
+        print(f"  janus submit {output_path}")
+        print("Or use: janus bench --submit")
+        return 0
+
+
+def cmd_run(args) -> int:
+    """Run a benchmark suite."""
+    suite = get_suite(args.suite)
+    if suite is None:
+        print(f"Unknown suite: {args.suite}", file=sys.stderr)
+        return 1
+
+    config_metadata = detect_config(Path.cwd())
+    config = SuiteRunConfig(suite=suite, config_metadata=config_metadata)
+
+    # Select execute function based on --judge flag
+    if args.judge:
+        print(f"Using LLM-as-judge scoring (model: {args.model})")
+        print("This requires an API key and will be slower but provides differentiated scores.")
+        print()
+        execute_fn = _create_geval_execute_fn(suite, args.model)
+    else:
+        execute_fn = _default_execute_fn
+
+    result = run_suite(config, execute_fn)
+
+    output_path = Path(args.output)
+    if args.format in ("json", "both"):
+        export_json(result, str(output_path))
+    if args.format in ("html", "both"):
+        html_path = output_path.with_suffix(".html")
+        export_html(result, str(html_path))
+
+    print(f"Suite {suite.suite_id} complete. Headline score: {result.headline_score:.1f} ({result.grade})")
+    if args.judge:
+        print(f"Scored with GEval ({args.model})")
+    return 0
+
+
+def cmd_compare(args) -> int:
+    """Compare two results for regression."""
+    baseline = load_json(args.baseline)
+    current = load_json(args.current)
+
+    if args.config:
+        config = load_thresholds(args.config)
+    else:
+        config = default_thresholds(baseline.suite_id)
+
+    if config.suite_id != baseline.suite_id:
+        print("Threshold suite_id does not match baseline suite.", file=sys.stderr)
+        return 2
+
+    result = compare_results(baseline, current, config)
+
+    if args.output:
+        export_comparison_json(result, args.output)
+
+    if args.format == "text":
+        print_comparison_text(result)
+    elif args.format == "json":
+        print(json.dumps(comparison_to_dict(result), indent=2))
+    elif args.format == "github":
+        print_github_annotations(result)
+        summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+        if summary_path:
+            Path(summary_path).write_text(generate_github_summary(result), encoding="utf-8")
+
+    return result.exit_code
+
+
+def cmd_export(args) -> int:
+    """Export result to different format."""
+    result = load_json(args.input)
+    output = args.output
+
+    if args.format == "html":
+        if output is None:
+            output = str(Path(args.input).with_suffix(".html"))
+        export_html(result, output)
+        return 0
+
+    if args.format == "json":
+        if output is None:
+            output = str(Path(args.input).with_suffix(".json"))
+        export_json(result, output)
+        return 0
+
+    return 1
+
+
+def cmd_baseline(args) -> int:
+    """Baseline management command."""
+    if args.baseline_command == "update":
+        return cmd_baseline_update(args)
+    if args.baseline_command == "show":
+        return cmd_baseline_show(args)
+    return 1
+
+
+def cmd_baseline_update(args) -> int:
+    """Promote result to baseline."""
+    result = load_json(args.result)
+    output = Path(args.output)
+
+    if output.exists() and not args.force:
+        print(f"Baseline exists: {output}. Use --force to overwrite.", file=sys.stderr)
+        return 1
+
+    export_json(result, str(output))
+    print(f"Baseline updated: {output}")
+    print(f"  Suite: {result.suite_id} v{result.suite_version}")
+    print(f"  Score: {result.headline_score:.1f} ({result.grade})")
+    return 0
+
+
+def cmd_baseline_show(args) -> int:
+    """Show baseline info."""
+    result = load_json(args.baseline)
+    print(f"Suite: {result.suite_id} v{result.suite_version}")
+    print(f"Score: {result.headline_score:.1f} ({result.grade})")
+    print(f"Comparability key: {result.comparability_key}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())