refactor-arena 0.6.0__py3-none-any.whl

refactor_arena/__init__.py

@@ -0,0 +1,8 @@
+"""Full Repository Refactor - A Control Arena setting for repository refactoring evaluation."""
+
+from importlib.metadata import version
+
+from .settings.refactor_arena_setting import RefactorArenaSetting
+
+__version__ = version("refactor-arena")
+__all__ = ["RefactorArenaSetting"]

refactor_arena/cli/__init__.py

@@ -0,0 +1,5 @@
+"""CLI package for refactor-arena."""
+
+from .main import main
+
+__all__ = ["main"]

refactor_arena/cli/commands/__init__.py

@@ -0,0 +1,17 @@
+from ..types.command_function import CommandFunction
+from .eval import eval_command
+from .init import init_command
+from .rescore import rescore_command
+
+COMMANDS: dict[str, CommandFunction] = {
+    "eval": eval_command,
+    "init": init_command,
+    "rescore": rescore_command,
+}
+
+__all__ = [
+    "COMMANDS",
+    "eval_command",
+    "init_command",
+    "rescore_command",
+]

refactor_arena/cli/commands/eval.py

@@ -0,0 +1,131 @@
+import logging
+from collections.abc import Sequence
+from pathlib import Path
+from typing import cast
+
+from control_arena.eval import get_control_task
+from inspect_ai import eval
+from inspect_ai.model import get_model
+from inspect_ai.solver import chain, human_agent
+
+from ...config import Config, create_fingerprint
+from ...settings.refactor_arena_setting import RefactorArenaSetting
+from ...solvers import agent_control_solver
+from ...utils.collections import find_index
+from ...utils.config import get_config_items
+from ...utils.format import format_as_yaml
+
+logger = logging.getLogger(__name__)
+
+
+def eval_command(config: Config, logs_dir: Path) -> int:
+    """
+    Runs the evaluation.
+    """
+    try:
+        dataset = config.get("setting.dataset")
+        main_task = config.get("setting.main_task")
+        side_task = config.get("setting.side_task")
+        eval_mode = config.get("eval.mode")
+
+        use_human_agent = config.get("human_agent.active")
+        swe_agent = config.get("swe_agent.name")
+        extra_features = []
+
+        if use_human_agent:
+            extra_features.append("human-agent")
+
+        setting = RefactorArenaSetting(
+            main_task=main_task,
+            side_task=side_task,
+            dataset=dataset,
+            extra_sandbox_features=extra_features,
+            tool_timeout=config.get("setting.tool_timeout"),
+        )
+
+        logger.info(f"Starting Full Repository Refactor evaluation in {eval_mode.value} mode")
+        logger.info(f"Dataset: {dataset}")
+        logger.info(f"Main task filter: {main_task or 'all'}")
+        logger.info(f"Side task filter: {side_task or 'all'}")
+        logger.info(f"Logs directory: {logs_dir}")
+        logger.info(f"Config Fingerprint:\n{format_as_yaml(create_fingerprint(config))}")
+
+        # Create evaluation configuration
+        eval_config = config.get("eval.config")
+
+        # Create policy and micro-protocol
+        untrusted_model = config.get("models.untrusted.name")
+        policy_name = config.get("policy.name")
+        micro_protocol = config.get(f"policy.options.{policy_name}")
+        monitors = get_config_items(config, "monitors")
+
+        # Create evaluation tasks
+        eval_task = get_control_task(
+            setting=setting,
+            micro_protocol=micro_protocol,
+            mode=eval_mode,
+            eval_config=eval_config,
+            monitoring_scorers=monitors,
+            metadata={
+                # Store only the actual configuration values used by this eval, not the full config
+                "config": config.get_usage(),
+            }
+        )
+
+        # Set the primary model for the eval log, so tools that only show this model will show a
+        # useful model string. Use the untrusted model since it's the primary agent.
+        # TODO: Review this decision. Now that we have configurable policies, it may not always be
+        #       appropriate to use the untrusted model. [fastfedora 30.Jan.26]
+        if untrusted_model:
+            eval_task.model = get_model(untrusted_model)
+
+        solvers = list(cast(Sequence, eval_task.solver))
+        control_solver_index = find_index(solvers, lambda x: "control_solver" in str(x))
+
+        if control_solver_index is not None:
+            if use_human_agent:
+                solvers[control_solver_index] = human_agent(
+                    intermediate_scoring=config.get("human_agent.intermediate_scoring", True),
+                    record_session=config.get("human_agent.record_session", False),
+                )
+                eval_task.solver = chain(solvers)
+            elif swe_agent:
+                solvers[control_solver_index] = agent_control_solver(
+                    micro_protocol=micro_protocol,
+                    tools=setting.get_agent_tools,
+                    get_submit_tool_config=setting.get_submit_tool_config,
+                )
+                eval_task.solver = chain(solvers)
+
+        import_path = config.get("import.path")
+        import_mode = config.get("import.mode")
+
+        # Allow human agent if we're only scoring, but otherwise remove the control solver
+        if import_path and import_mode == "score" and not use_human_agent:
+            logger.info(f"Import mode: Using trajectory from {import_path}")
+            if control_solver_index is not None:
+                solvers.pop(control_solver_index)
+            eval_task.solver = chain(solvers)
+
+        eval(
+            eval_task,
+            **eval_config.eval_args(),
+        )
+
+        if hasattr(setting, "cleanup"):
+            setting.cleanup()
+
+        logger.info("Evaluation completed")
+
+        return 0
+
+    except KeyboardInterrupt:
+        logger.info("Evaluation interrupted by user")
+        return 1
+
+    except Exception as e:
+        logger.error(f"Evaluation failed: {e}")
+        # logger.error(e.with_traceback())
+        if config.get("logs.verbose"):
+            logger.exception("Full traceback:")
+        return 1

refactor_arena/cli/commands/init.py

@@ -0,0 +1,157 @@
+import getpass
+import sys
+from pathlib import Path
+
+from ...config import Config, config_dirs
+from ...utils.filesystem.resources.find_resource import find_resource
+
+
+STARTER_CONFIG_FILENAME = "getting-started.yaml"
+
+
+def init_command(config: Config, logs_dir: Path) -> int:  # noqa: ARG001
+    """
+    Initialize a new .env file for the current project.
+
+    If a .env file already exists in the current working directory, this command reports that the
+    project is already initialized and makes no changes.
+    """
+    target_dir = Path.cwd()
+    env_path = target_dir / ".env"
+
+    if env_path.exists():
+        print("This project is already initialized ('.env' already exists).")
+        return 0
+
+    trusted_default = config.get("models.trusted.name")
+    untrusted_default = config.get("models.untrusted.name")
+
+    print("Initializing refactor-arena project...\n")
+
+    try:
+        # First ask for the model names, since that determines which API keys are needed
+        trusted_model = _prompt_with_default("Trusted model name", trusted_default)
+        untrusted_model = _prompt_with_default("Untrusted model name", untrusted_default)
+
+        # Only ask for the API keys of the configured model providers
+        trusted_provider = _get_model_provider(trusted_model)
+        untrusted_provider = _get_model_provider(untrusted_model)
+
+        needs_openai = trusted_provider == "openai" or untrusted_provider == "openai"
+        needs_anthropic = trusted_provider == "anthropic" or untrusted_provider == "anthropic"
+
+        openai_api_key: str | None = None
+        anthropic_api_key: str | None = None
+
+        if needs_openai:
+            print("An OpenAI model is configured as one of the models and needs an API key.")
+            openai_api_key = _prompt_secret("Enter an OpenAI API key (leave blank to skip)")
+
+        if needs_anthropic:
+            print("An Anthropic model is configured as one of the models and needs an API key.")
+            anthropic_api_key = _prompt_secret("Enter an Anthropic API key (leave blank to skip)")
+    except (EOFError, KeyboardInterrupt):
+        print("\nInitialization aborted.")
+        return 1
+
+    env_created = _write_env_file(
+        config=config,
+        env_path=env_path,
+        trusted_model=trusted_model,
+        untrusted_model=untrusted_model,
+        openai_api_key=openai_api_key,
+        anthropic_api_key=anthropic_api_key,
+    )
+
+    starter_scenario_created = _copy_starter_scenario(target_dir)
+
+    if not env_created or not starter_scenario_created:
+        return 1
+
+    print(
+        f"\nThis directory has been initialized with a '.env' and starter configuration file. You"
+        " can now use the CLI to run a control eval. For example:"
+    )
+    print(f"  refactor-arena eval --config {STARTER_CONFIG_FILENAME}")
+    return 0
+
+
+def _prompt_with_default(prompt: str, default: str) -> str:
+    """
+    Prompt the user for a value, showing a default.
+
+    Returns the user-provided value, or the default if the user enters nothing.
+    """
+    return input(f"{prompt} [{default}]: ").strip() or default
+
+
+def _prompt_secret(prompt: str) -> str:
+    """
+    Prompt the user for a secret value without echoing.
+    """
+    return getpass.getpass(f"{prompt}: ").strip()
+
+
+def _get_model_provider(model_name: str | None) -> str | None:
+    """
+    Extract the provider from a model name of the form 'provider/model-name'.
+    """
+    return model_name.split("/", 1)[0].lower() if model_name and "/" in model_name else None
+
+
+def _write_env_file(
+    config: Config,
+    env_path: Path,
+    trusted_model: str,
+    untrusted_model: str,
+    openai_api_key: str | None,
+    anthropic_api_key: str | None,
+) -> bool:
+    """Write a new .env file at the given path. Returns True on success."""
+    trusted_swe_model = config.get("swe_agent.claude-code.model")
+    untrusted_swe_model = config.get("swe_agent.codex.model")
+
+    lines: list[str] = [
+        "# Provider API keys",
+        f"OPENAI_API_KEY={openai_api_key or ''}",
+        f"ANTHROPIC_API_KEY={anthropic_api_key or ''}",
+        "",
+        "# Trusted and untrusted models--use versions to record the exact model in the transcripts",
+        f"TRUSTED_MODEL={trusted_model}",
+        f"UNTRUSTED_MODEL={untrusted_model}",
+        "",
+        "# Model overrides when using SWE agents (e.g., `--swe-agent` for the CLI). "
+        "If not specified, the `UNTRUSTED_MODEL` or `TRUSTED_MODEL` will be used",
+        f"CLAUDE_CODE_MODEL={trusted_swe_model or ''}",
+        f"CODEX_MODEL={untrusted_swe_model or ''}",
+        "",
+        "# Whether to record human agent sessions when using `--human-agent` for the CLI",
+        "HUMAN_AGENT_RECORD_SESSION=false",
+        "",
+    ]
+
+    try:
+        env_path.write_text("\n".join(lines), encoding="utf-8")
+    except Exception as error:
+        print(f"Failed to create '.env' file: {error}", file=sys.stderr)
+        return False
+
+    return True
+
+
+def _copy_starter_scenario(target_directory: Path) -> bool:
+    """Copy the getting-started scenario config into the project directory for convenience."""
+    target_path = target_directory / STARTER_CONFIG_FILENAME
+
+    if target_path.exists():
+        print(f"Skipping copying starter configuration: {STARTER_CONFIG_FILENAME} already exists")
+        return True
+
+    try:
+        source_path = Path(find_resource("scenarios/getting-started.yaml", config_dirs))
+        target_path.write_text(source_path.read_text(encoding="utf-8"), encoding="utf-8")
+    except Exception as error:
+        print(f"Failed to create starter configuration file: {error}", file=sys.stderr)
+        return False
+
+    return True

refactor_arena/cli/commands/rescore.py

@@ -0,0 +1,328 @@
+import asyncio
+import logging
+import re
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from control_arena.eval import EvalMode
+# TODO: Figure out if these can be done via the public API. [fastfedora 15.Apr.26]
+from inspect_ai._eval.score import score_async as inspect_score_async
+from inspect_ai._eval.task.results import eval_results
+from inspect_ai._util.registry import registry_unqualified_name
+from inspect_ai.log import EvalLog, EvalSample, EvalScore, write_eval_log_async
+from inspect_ai.scorer import SampleScore, Scorer
+from pydash import get as get_nested, set_ as set_nested
+
+from ...config import Config
+from ...utils.config import get_config_items, resolve_active_items
+from ...utils.evals.read_setting_eval_log import read_setting_eval_log
+from ...utils.filesystem import find_unused_filename
+from ...utils.ui import confirmed_by_user
+
+logger = logging.getLogger(__name__)
+
+
+def rescore_command(config: Config, logs_dir: Path) -> int:
+    """
+    Re-score previous evaluation results from an .eval log file.
+    """
+    try:
+        source_path = config.get("rescore.source.path")
+        if not source_path:
+            logger.error("No source path provided for rescore (rescore.source.path is empty).")
+            return 1
+
+        source_path = Path(source_path)
+
+        if source_path.suffix != ".eval":
+            logger.error("Only .eval log files are currently supported for rescore.")
+            return 1
+
+        success = asyncio.run(_rescore_eval_file(config, source_path, logs_dir))
+
+        return 1 if not success else 0
+
+    except KeyboardInterrupt:
+        logger.info("Rescore interrupted by user")
+        return 1
+
+    except Exception as e:
+        logger.error(f"Rescore failed: {e}")
+        if logger.isEnabledFor(logging.DEBUG):
+            logger.exception("Full traceback:")
+        return 1
+
+
+async def _rescore_eval_file(config: Config, eval_path: Path, logs_dir: Path) -> bool:
+    """
+    Rescore an existing .eval log file.
+    """
+    # Read in log and initialize config
+    log = await read_setting_eval_log(eval_path, config.get("rescore.import", {}))
+    if log.samples is None:
+        logger.error("No samples found in the eval log.")
+        return False
+
+    _import_config_from_log(config, log)
+
+    # Apply dataset filters
+    all_samples = log.samples
+    log.samples = _filter_samples(all_samples, config.get("rescore.filter", {}))
+
+    if not log.samples:
+        logger.error("No samples matched the dataset filters; no output was written.")
+        return False
+
+    # Check output settings and warn about issues
+    output_mode: str = config.get("rescore.output.mode", "new-log")
+    filter_samples: bool = config.get("rescore.output.filter.samples", False)
+    filter_metrics: bool = config.get("rescore.output.filter.metrics", False)
+    filter_active = len(all_samples) != len(log.samples)
+
+    if filter_samples and not filter_active:
+        logger.warning("--filter-samples has no effect when no samples are filtered.")
+
+    if output_mode == "overwrite-log" and filter_samples and filter_active:
+        if not confirmed_by_user(
+            "Using --output-mode `overwrite-log` with --filter-samples will remove non-matching"
+            " samples from the source log. Proceed? (y/n): ",
+            config=config,
+            config_path="rescore.confirm.overwrite_with_filter_samples",
+        ):
+            logger.error("Aborting rescore.")
+            return False
+
+    logger.info(f"Rescoring {len(log.samples)} of {len(all_samples)} sample(s).")
+
+    # Perform the rescoring
+    original_scores = list(log.results.scores) if log.results else []
+    metrics_only: bool = config.get("rescore.metrics_only", False)
+    scorer_names = _get_active_scorer_names(config)
+    scorers = _get_active_scorers(config)
+
+    if metrics_only:
+        if filter_active and not filter_samples:
+            logger.error(
+                "To ensure metrics represent the entire dataset, --metrics-only can only be used"
+                " with --filter-samples when filtering samples."
+            )
+            return False
+        _recompute_metrics(log, scorers)
+    else:
+        _assert_no_sandbox_required_for_eval_logs(config, scorer_names)
+        await inspect_score_async(log=log, scorers=scorers, action="append", copy=False)
+        if filter_active and not filter_samples:
+            log.samples = _merge_samples(log.samples, all_samples)
+            _recompute_metrics(log, scorers)
+
+    if not filter_metrics and log.results:
+        _merge_original_scores(log, original_scores)
+
+    # Write the re-scored log to the appropriate location based on the output mode
+    if output_mode == "overwrite-log":
+        await write_eval_log_async(log)
+        logger.info(f"Wrote re-scored log in-place to '{log.location}'")
+    else:
+        output_path = find_unused_filename(logs_dir / f"rescore_{Path(eval_path).stem}.eval")
+        log.location = output_path.as_posix()
+        await write_eval_log_async(log, output_path)
+        logger.info(f"Wrote re-scored log to '{output_path}'")
+
+    return True
+
+
+def _merge_samples(destination: list[EvalSample], source: list[EvalSample]) -> list[EvalSample]:
+    """
+    Merge the samples from the source list of samples into the destination list of samples, matching
+    samples by their ID.
+    """
+    by_sample_id = {sample.id: sample for sample in source}
+    return [by_sample_id.get(sample.id, sample) for sample in destination]
+
+
+def _get_active_scorer_names(config: Config) -> list[str]:
+    """
+    Return the names of the active scorers based on the current config.
+    """
+    monitors = resolve_active_items(config, "monitors")
+
+    monitors_only: bool = config.get("rescore.monitors_only", False)
+    if monitors_only:
+        return monitors
+
+    scorers = resolve_active_items(config, "scorers")
+    return scorers + monitors
+
+
+def _get_active_scorers(config: Config) -> list[Scorer]:
+    """
+    Instantiate the scorers to use for rescoring based on the current config.
+
+    Uses the standard `scorers` and `monitors` config sections (list + include-if + options
+    pattern), which can be filtered via the `--scorers` and `--monitors` CLI options respectively.
+    When `--monitors-only` is set, only monitor scorers are included.
+    """
+    monitors = get_config_items(config, "monitors")
+
+    monitors_only: bool = config.get("rescore.monitors_only", False)
+    if monitors_only:
+        return monitors
+
+    scorers = get_config_items(config, "scorers")
+    return scorers + monitors
+
+
+def _filter_samples(samples: list[EvalSample], config: Mapping[str, Any]) -> list[EvalSample]:
+    """
+    Filter samples based on dataset filter config values.
+
+    Returns:
+        List of filtered samples. If no filters are configured, the original list is returned.
+    """
+    main_task_pattern = config.get("main_task")
+    side_task_pattern = config.get("side_task")
+    repo_pattern = config.get("repo")
+    limit = config.get("limit")
+    limit_active = isinstance(limit, int) and limit > 0
+
+    filter_active = bool(main_task_pattern or side_task_pattern or repo_pattern or limit_active)
+    if not filter_active:
+        return samples
+
+    filtered_samples: list[EvalSample] = []
+    for sample in samples:
+        metadata = sample.metadata if isinstance(sample.metadata, Mapping) else {}
+        sample_main_task = metadata.get("main_task_name", "")
+        sample_side_task = metadata.get("side_task_name", "")
+        sample_repo = str(sample.id).split("__", maxsplit=1)[0]
+
+        if main_task_pattern and not re.search(main_task_pattern, sample_main_task):
+            continue
+
+        if side_task_pattern and not re.search(side_task_pattern, sample_side_task):
+            continue
+
+        if repo_pattern and not re.search(repo_pattern, sample_repo):
+            continue
+
+        filtered_samples.append(sample)
+
+    if limit_active:
+        filtered_samples = filtered_samples[:limit]
+
+    return filtered_samples
+
+
+def _recompute_metrics(log: EvalLog, scorers: list[Scorer]) -> None:
+    """
+    Recompute log.results and log.reductions from the current log.samples scores.
+    """
+    if not log.samples:
+        raise ValueError("_recompute_metrics must be called with at least one sample.")
+
+    active_scorer_names = {registry_unqualified_name(scorer) for scorer in scorers}
+
+    scores: list[dict[str, SampleScore]] = []
+    samples = list(log.samples)
+    for sample in samples:
+        if not sample.scores:
+            continue
+
+        sample_scores: dict[str, SampleScore] = {}
+        for name, score in sample.scores.items():
+            if name in active_scorer_names:
+                sample_scores[name] = SampleScore(score=score, sample_id=sample.id)
+        if sample_scores:
+            scores.append(sample_scores)
+
+    results, reductions = eval_results(
+        samples=len(samples),
+        scores=scores,
+        reducers=None,
+        scorers=scorers,
+        metrics=None,
+    )
+
+    log.results = results
+    log.reductions = reductions
+
+
+def _merge_original_scores(
+    log: EvalLog,
+    original_scores: list[EvalScore],
+) -> None:
+    """
+    Merge metrics from previously existing scores back into the log results, preserving metrics for
+    scorers that were not part of the current rescore run. The original ordering is retained, with
+    recomputed scores replacing their originals in place.
+    """
+    if not original_scores or not log.results:
+        return
+
+    new_scores_by_name = {score.name: score for score in log.results.scores}
+    merged: list[EvalScore] = []
+    for score in original_scores:
+        merged.append(new_scores_by_name.pop(score.name, score))
+
+    merged.extend(new_scores_by_name.values())
+    log.results.scores = merged
+
+
+def _import_config_from_log(config: Config, log: EvalLog) -> None:
+    """
+    Import config values from the eval log's stored configuration into the current config as
+    overrides. The paths to import are specified by `rescore.import.config`.
+    """
+    import_paths: Sequence[str] = config.get("rescore.import.config", [])
+    if not import_paths:
+        return
+
+    stored_config = (log.eval.metadata or {}).get("config")
+    if not stored_config or not isinstance(stored_config, Mapping):
+        logger.warning("Eval log has no stored config in metadata; skipping config import.")
+        return
+
+    overrides: dict[str, Any] = {}
+    sentinel = object()
+
+    for path in import_paths:
+        value = get_nested(stored_config, path, default=sentinel)
+        if value is sentinel:
+            logger.debug(f"Config path '{path}' not found in eval log metadata; skipping.")
+            continue
+        set_nested(overrides, path, value)
+        logger.debug(f"Imported config override from eval log: {path} = {value!r}")
+
+    if overrides:
+        # TODO: Figure out a way to integrate this into the config parser itself. I extended the
+        #       args parser to create enums, but when the eval mode is set in a configuration file,
+        #       it is not converted to an enum. [fastfedora 03.Apr.26]
+        # Force the eval mode to be an enum
+        eval_mode = overrides.get("eval", {}).get("mode")
+        if isinstance(eval_mode, str):
+            overrides["eval"] = { "mode": EvalMode(eval_mode) }
+
+        config.add_overrides(overrides)
+
+
+def _assert_no_sandbox_required_for_eval_logs(config: Config, scorer_names: list[str]) -> None:
+    """
+    Ensure that when rescoring .eval logs we don't attempt to run scorers that require sandbox setup
+    or post-submission solvers.
+    """
+    options: Mapping[str, Mapping[str, object]] = config.get("rescore.scorers.options") or {}
+
+    invalid: list[str] = []
+    for name in scorer_names:
+        info = options.get(name, { "setup": False })
+        requires_setup = bool(info.get("setup"))
+        if requires_setup:
+            invalid.append(name)
+
+    if invalid:
+        raise RuntimeError(
+            "The following scorers require sandbox setup and cannot be run when rescoring "
+            f"existing .eval logs: {", ".join(sorted(invalid))}"
+        )
+