evalrx 0.1.2__py3-none-any.whl

evalrx/datasets/web_search_qa.py

@@ -0,0 +1,52 @@
+"""Web-search-augmented QA loader → CaseBatch (multi-hop / retrieval-dependent QA).
+
+Same shape as PureQA, but records may carry a ``context`` (retrieved passages) and
+a ``requires_retrieval`` flag; cases are tagged ``web_search`` so analyzers /
+A/B strategies can condition on them.  Records / JSONL / built-in sample.
+"""
+
+from __future__ import annotations
+
+from evalrx.core.case import CaseBatch
+from evalrx.datasets.base import Dataset, cases_from_records, read_jsonl
+
+_SAMPLE = [
+    {"question": "In what year did the author of 'The Origin of Species' publish it?",
+     "answer": "1859", "requires_retrieval": True, "hops": 2},
+    {"question": "What is the population of the capital of Japan?",
+     "answer": "Tokyo", "requires_retrieval": True, "hops": 2},
+]
+
+
+class WebSearchQADataset(Dataset):
+    """QA with (optional) retrieved web context."""
+
+    def __init__(
+        self,
+        records: list[dict] | None = None,
+        path: str | None = None,
+        *,
+        prompt_key: str = "question",
+        answer_key: str = "answer",
+    ) -> None:
+        self._records = records
+        self._path = path
+        self._keys = dict(prompt_key=prompt_key, answer_key=answer_key)
+
+    @classmethod
+    def from_records(cls, records: list[dict], **keys) -> "WebSearchQADataset":
+        return cls(records=records, **keys)
+
+    @classmethod
+    def from_jsonl(cls, path: str, **keys) -> "WebSearchQADataset":
+        return cls(path=path, **keys)
+
+    @classmethod
+    def sample(cls) -> "WebSearchQADataset":
+        return cls(records=_SAMPLE)
+
+    def load(self) -> CaseBatch:
+        records = self._records if self._records is not None else (
+            read_jsonl(self._path) if self._path else _SAMPLE
+        )
+        return cases_from_records(records, tags={"web_search"}, **self._keys)

evalrx/eval_agent/__init__.py

@@ -0,0 +1,341 @@
+"""Self-evolving evaluation agent — automated failure discovery and diagnosis.
+
+Two loops are available:
+
+  AutoDiagnoseLoop   M1 → M2 → M3 → M5 (legacy, four-stage sweep)
+  VLDiagnoseLoop     M1 → M2 → M3 → M4 inner loop, M5 called post-loop (Plan A)
+                     Stops when M4 finds a statistically supported,
+                     protocol-consistent hypothesis.
+
+Stage modules live in ``stages/``; shared infrastructure stays at the top level.
+The CLI-agent runtime (sandboxing, code generation, providers, judge models,
+skills) lives in ``evalrx.agent_runtime`` — shared with ``evalrx.analysis``
+so neither package depends on the other. ``cli_agent.py`` / ``cli_skills.py``
+remain here as compatibility facades over that runtime.
+
+Top-level (shared / orchestration):
+  loop.py              AutoDiagnoseLoop, VLDiagnoseLoop, SelfEvolveLoop
+  run_logger_v2.py     RunLoggerV2 — run.json + per-stage M<n>/log.json event log
+  hypothesis.py        Hypothesis, HypothesisStatus, serialization helpers
+  cli_agent.py         compatibility facade over agent_runtime CLI providers/judges
+  cli_skills.py        compatibility facade over agent_runtime.skills.installer
+  store.py             Store / InMemoryStore / JsonlStore — persistent memory
+  orchestrator.py      thin facade over the loop (pre-registered A/B)
+  ab_runner.py         A/B execution across prompting strategies
+  report.py            DiagnosticReport — final diagnostic conclusions
+  evolution.py         EvolutionStore — JSONL lesson store, 30-day half-life decay
+  preregister.py       pre-registration helpers (DataSplit, PreregisteredHypothesis)
+  git_manager.py       git-native experiment versioning (eval/{run_id} branches)
+
+evalrx.agent_runtime (shared CLI-agent runtime, imports neither analysis nor eval_agent):
+  cli_types.py         CliAgentConfig / CliAgentResult
+  cli_runtime.py       SubprocessRunner, ProcessRun, collect_py_files
+  sandbox.py           ExperimentSandbox, SandboxProtocol
+  experiment_harness.py immutable evaluation harness injected into projects
+  factory.py           sandbox factory (subprocess / docker backends)
+  providers/           CLI coding-provider adapters and registry
+  judges/              CLI-backed judge model wrappers (agy / Claude Code)
+  codegen/             shared code-generation runner used by stages
+  skills/              skill resolution, installation, and prompt policy
+
+stages/ (M1–M4 implementation):
+  probe.py             M1 — StrategyProbe: model-kind detection + analyzer ranking
+  probe_agent.py       M1 — ProbeAgent: execute ranked analyzers (direct or Docker);
+                              protocol-guided via ExperimentProtocol.probe_hints();
+                              tier(b) generates a bespoke probe when no analyzer fits
+  probe_generator.py   M1 tier(b) — ProbeGenerator: host collects model outputs,
+                              an LLM/CLI writes a probe over them, run in a sandbox,
+                              parse PROBE_RESULT_JSON into a Result (per_case findings)
+  protocol.py          M1 — ExperimentProtocol (NL description → probe hints);
+                              ProbingSchema (records M1 selection rationale)
+  analysis.py          M2 — AnalysisModule: threshold rules → AnalysisReport
+  stats_agent.py       M2 — StatsAnalysisAgent: extends AnalysisModule with a
+                              statistical-tool layer (select tools from the catalog,
+                              run them, e-BH FDR-correct, plot) + LLM-guided
+                              conclusion/evidence chain (StatsAnalysisReport)
+  stats_tools.py       M2 — statistical tool catalog wrapping evalrx.stats
+                              (signal/label association, McNemar+e-value, Friedman,
+                              single-rate e-value, rank corr) + StatsInput/fdr_correct
+  stats_tool_agent.py  M2 — legacy deterministic exploratory stats tools
+  stats_tool_generator.py M2 tier(b) — StatsToolGenerator: LLM/CLI writes a new
+                              stats script, runs it in a sandbox, parses a
+                              STATS_RESULT_JSON contract (never mutates repo source)
+  diagnosis.py         M3 — DiagnosisAgent: judge reads report → Hypothesis list
+  case_discovery.py    Data — run candidate prompts and label PASS/FAIL cases
+  surgery.py           M5 — SurgeryAgent: correlate / param-sweep / ExperimentWriter
+                              → InterventionResult (SUPPORTED / REFUTED / INCONCLUSIVE)
+  experiment_writer.py M5 — multi-phase LLM/CLI agent writes + executes fix scripts
+  fix_tiers.py         Fix — FixTier intervention-space ladder (L0 runtime /
+                              L1 prompt / L2 scaffold / L3a read / L3b write /
+                              L4 params)
+                              + hypothesis -> minimum-tier routing
+  fix_tools.py         Fix — L2 tool catalog (zoom/contrast/equalize/upscale)
+                              + PipelineSpec executor around the unchanged model
+  fix_agent.py         Fix — FixAgent: tiered candidates -> paired McNemar
+                              validation -> FixOutcome (+ tier recommendation)
+  fix_pipeline.py      Fix — L2 coded pipelines: sandboxed agent code with
+                              bridged model access (model_generate/model_attend)
+  fix_internals.py     Fix — L3a attention-guided crop, L3b intervention
+                              primitives (visual embedding boost); L4
+                              FinetuneSpec + run_lora_repair (v1: LoRA on
+                              target="llm" only, trained on a caller-supplied
+                              finetune_pool; other recipe shapes recorded only)
+  hypothesis_tester.py M4 — HypothesisTester: statistical test + protocol consistency;
+                              stopping_criteria_met() drives the VLDiagnoseLoop exit
+"""
+
+from evalrx.agent_runtime.cli_types import CliAgentConfig, CliAgentResult
+from evalrx.agent_runtime.factory import SandboxConfig, SandboxFactoryConfig, create_sandbox
+from evalrx.agent_runtime.judges import AgyModel, ClaudeModel, CodexModel
+from evalrx.agent_runtime.providers import create_cli_agent
+from evalrx.agent_runtime.sandbox import (
+    ExperimentSandbox,
+    SandboxProtocol,
+    SandboxResult,
+    parse_metrics,
+    validate_entry_point,
+    validate_entry_point_resolved,
+)
+from evalrx.analysis.analysis_module import AnalysisModule, AnalysisReport
+from evalrx.analysis.planner import AnalysisPlanItem, plan_stats_input, ranked_signal_names
+from evalrx.analysis.profile import (
+    ColumnProfile,
+    DatasetProfile,
+    describe_outcome,
+    profile_records,
+    profile_stats_input,
+)
+from evalrx.analysis.stats_agent import StatsAnalysisAgent, StatsAnalysisReport
+from evalrx.analysis.stats_tool_agent import StatsToolAgent
+from evalrx.analysis.stats_tool_generator import (
+    GeneratedStatsTool,
+    StatsToolGenerator,
+)
+from evalrx.analysis.stats_tools import (
+    STATS_TOOL_CATALOG,
+    EvidenceResult,
+    StatsInput,
+    StatsToolResult,
+    build_stats_input,
+    build_stats_input_from_records,
+    default_plan,
+    fdr_correct,
+    run_stats_tool,
+)
+from evalrx.eval_agent.ab_runner import ABResult, ABRunner
+from evalrx.eval_agent.agentic import (
+    AgenticDiagnoseLoop,
+    ToolOutcome,
+    ToolRegistry,
+    ToolSpec,
+)
+from evalrx.eval_agent.evolution import EvolutionStore, LessonEntry, extract_lessons
+from evalrx.eval_agent.git_manager import ExperimentGitManager
+from evalrx.eval_agent.hypothesis import (
+    Hypothesis,
+    HypothesisGenerator,
+    HypothesisStatus,
+    ManualHypothesisGenerator,
+    hypothesis_from_dict,
+    hypothesis_to_dict,
+)
+from evalrx.eval_agent.legacy import AutoDiagnoseLoop, SelfEvolveLoop
+from evalrx.eval_agent.log_schema import (
+    RUN_LOG_SCHEMA_VERSION,
+    SCHEMA_PATH,
+    build_schema,
+    iter_log_errors,
+    load_schema,
+    validate_event,
+)
+from evalrx.eval_agent.loop import VLDiagnoseLoop
+from evalrx.eval_agent.loop_reports import AutoDiagnoseReport, VLDiagnoseReport
+from evalrx.eval_agent.nl_runner import scaffold_from_description
+from evalrx.eval_agent.orchestrator import EvalOrchestrator
+from evalrx.eval_agent.preregister import (
+    DataSplit,
+    PreregisteredHypothesis,
+    PreregistrationLog,
+    Split,
+)
+from evalrx.eval_agent.report import DiagnosticReport
+from evalrx.eval_agent.run_context import RunContext
+from evalrx.eval_agent.run_logger_v2 import RunLoggerV2
+from evalrx.eval_agent.stages.case_discovery import (
+    CaseDiscoveryAgent,
+    CaseDiscoveryReport,
+)
+from evalrx.eval_agent.stages.diagnosis import DiagnosisAgent, DiagnosisResult
+from evalrx.eval_agent.stages.experiment_writer import (
+    ExperimentWriter,
+    ExperimentWriterConfig,
+    ExperimentWriterResult,
+    SolutionNode,
+    build_model_context,
+)
+from evalrx.eval_agent.stages.fix_agent import (
+    FixAgent,
+    FixCandidate,
+    FixOutcome,
+    FixValidation,
+)
+from evalrx.eval_agent.stages.fix_internals import (
+    INTERNALS_PRIMITIVES,
+    FinetuneSpec,
+    InternalsPrimitive,
+)
+from evalrx.eval_agent.stages.fix_tiers import FixTier, parse_tier, route_min_tier
+from evalrx.eval_agent.stages.fix_tools import PipelineSpec
+from evalrx.eval_agent.stages.hypothesis_tester import HypothesisTester, HypothesisTestResult
+from evalrx.eval_agent.stages.probe import ModelKind, StrategyProbe
+from evalrx.eval_agent.stages.probe_agent import ProbeAgent
+from evalrx.eval_agent.stages.probe_candidate_generator import VLMProbeCandidateGenerator
+from evalrx.eval_agent.stages.probe_generator import GeneratedProbe, ProbeGenerator
+from evalrx.eval_agent.stages.probe_search_agent import ProbeSearchAgent
+from evalrx.eval_agent.stages.protocol import ExperimentProtocol, ProbingSchema
+from evalrx.eval_agent.stages.surgery import InterventionResult, SurgeryAgent
+from evalrx.eval_agent.stages.whitebox_probe_generator import (
+    GeneratedWhiteboxProbe,
+    WhiteboxProbeGenerator,
+)
+from evalrx.eval_agent.store import InMemoryStore, JsonlStore, Store
+
+__all__ = [
+    # Judge
+    "AgyModel",
+    "ClaudeModel",
+    "CodexModel",
+    # M1
+    "ProbeAgent",
+    "StrategyProbe",
+    "ModelKind",
+    # M1 tier (b) probe generation
+    "ProbeGenerator",
+    "GeneratedProbe",
+    "WhiteboxProbeGenerator",
+    "GeneratedWhiteboxProbe",
+    # ProbeLLM-style hierarchical MCTS probe search
+    "ProbeSearchAgent",
+    "VLMProbeCandidateGenerator",
+    # Fix module (post-loop tiered repair)
+    "FixAgent",
+    "FixCandidate",
+    "FixOutcome",
+    "FixValidation",
+    "FixTier",
+    "parse_tier",
+    "route_min_tier",
+    "PipelineSpec",
+    "INTERNALS_PRIMITIVES",
+    "InternalsPrimitive",
+    "FinetuneSpec",
+    # M2
+    "AnalysisModule",
+    "AnalysisReport",
+    # M3
+    "DiagnosisAgent",
+    "DiagnosisResult",
+    # Case discovery / labeling
+    "CaseDiscoveryAgent",
+    "CaseDiscoveryReport",
+    # M5
+    "SurgeryAgent",
+    "InterventionResult",
+    # Loop
+    "AutoDiagnoseLoop",
+    "AutoDiagnoseReport",
+    "SelfEvolveLoop",
+    "VLDiagnoseLoop",
+    "VLDiagnoseReport",
+    # Agentic loop (judge-decided M1-M4, alternative to VLDiagnoseLoop's fixed cycle)
+    "AgenticDiagnoseLoop",
+    "ToolSpec",
+    "ToolOutcome",
+    "ToolRegistry",
+    # Protocol
+    "ExperimentProtocol",
+    "ProbingSchema",
+    # M2 stats agent
+    "StatsAnalysisAgent",
+    "StatsAnalysisReport",
+    "StatsToolAgent",
+    # M2 stats tools
+    "StatsInput",
+    "StatsToolResult",
+    "EvidenceResult",
+    "STATS_TOOL_CATALOG",
+    "build_stats_input",
+    "build_stats_input_from_records",
+    "default_plan",
+    "fdr_correct",
+    "run_stats_tool",
+    "ColumnProfile",
+    "DatasetProfile",
+    "describe_outcome",
+    "profile_records",
+    "profile_stats_input",
+    "AnalysisPlanItem",
+    "ranked_signal_names",
+    "plan_stats_input",
+    # M2 tier (b) code generation
+    "StatsToolGenerator",
+    "GeneratedStatsTool",
+    # M4 hypothesis tester
+    "HypothesisTester",
+    "HypothesisTestResult",
+    # Shared
+    "EvalOrchestrator",
+    "Hypothesis",
+    "HypothesisGenerator",
+    "ManualHypothesisGenerator",
+    "HypothesisStatus",
+    "hypothesis_to_dict",
+    "hypothesis_from_dict",
+    "Store",
+    "InMemoryStore",
+    "JsonlStore",
+    "ABRunner",
+    "ABResult",
+    "DataSplit",
+    "Split",
+    "PreregisteredHypothesis",
+    "PreregistrationLog",
+    "DiagnosticReport",
+    "RunContext",
+    "RunLoggerV2",
+    # Published event schema
+    "RUN_LOG_SCHEMA_VERSION",
+    "build_schema",
+    "load_schema",
+    "validate_event",
+    "iter_log_errors",
+    "SCHEMA_PATH",
+    # Experiment execution
+    "ExperimentWriter",
+    "ExperimentWriterConfig",
+    "ExperimentWriterResult",
+    "SolutionNode",
+    "build_model_context",
+    "ExperimentSandbox",
+    "SandboxResult",
+    "SandboxProtocol",
+    "parse_metrics",
+    "validate_entry_point",
+    "validate_entry_point_resolved",
+    # Sandbox factory
+    "SandboxConfig",
+    "SandboxFactoryConfig",
+    "create_sandbox",
+    # NL scaffold
+    "scaffold_from_description",
+    # Git versioning
+    "ExperimentGitManager",
+    # Evolution store
+    "EvolutionStore",
+    "LessonEntry",
+    "extract_lessons",
+    # CLI agents
+    "CliAgentConfig",
+    "CliAgentResult",
+    "create_cli_agent",
+]

evalrx/eval_agent/_tools.py

@@ -0,0 +1,81 @@
+"""The agent's action space over the EvalRX package.
+
+The self-evolving agent doesn't reach into internals — it acts through this
+thin tool layer, which wraps the registry, the analyzer/run machinery, and the
+store.  Each function is one "skill" the agent can call.  Functions that only
+forward to the core are implemented now; the ones needing the Stage-2 store or
+case-generation raise ``NotImplementedError`` but already have their final
+signatures, so the loop can be wired against them.
+
+Design intent: this module *is* the contract between the agent and the package.
+Keeping it small and uniform is what makes the agent able to drive the package
+without bespoke glue.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from evalrx.core.registry import registry
+
+if TYPE_CHECKING:
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+    from evalrx.core.result import Result
+
+
+# ----------------------------------------------------------------------
+# Discovery — "what can I do?"  (implemented: pure registry reads)
+# ----------------------------------------------------------------------
+
+def list_models() -> list[str]:
+    """All known model spec keys (the agent's model-selection surface)."""
+    from evalrx.specs import list_specs
+
+    return list_specs()
+
+
+def list_analyses() -> list[str]:
+    """All registered analyzer names."""
+    return registry.analyzers.list()
+
+
+def compatible_analyses(model: "Model") -> list[str]:
+    """Analyzer names whose capability requirements *model* satisfies.
+
+    This is the agent's key planning primitive: given a model, what analyses
+    are even runnable on it?
+    """
+    return registry.analyzers.names_compatible_with(model)
+
+
+# ----------------------------------------------------------------------
+# Action — "run an analysis"  (implemented: forwards to the analyzer)
+# ----------------------------------------------------------------------
+
+def run_analysis(model: "Model", analysis: str, data: Any, **params) -> "Result":
+    """Instantiate analyzer *analysis* with *params* and run it on *model*/*data*."""
+    analyzer_cls = registry.analyzers.get(analysis)
+    return analyzer_cls(**params).run(model, data)
+
+
+# ----------------------------------------------------------------------
+# Case construction & memory  (Stage 2 — needs generators + store)
+# ----------------------------------------------------------------------
+
+def make_cases(spec: Any) -> "CaseBatch":
+    """Construct/sample a batch of candidate failure cases from a spec.
+
+    Stage 2: backed by dataset samplers and agent-driven case synthesis.
+    """
+    raise NotImplementedError("make_cases is planned for Stage 2.")
+
+
+def record(result: "Result") -> None:
+    """Persist a result (and its cases) into the store for self-evolution."""
+    raise NotImplementedError("record is planned for Stage 2 (needs Store).")
+
+
+def query_store(query: Any) -> Any:
+    """Query accumulated cases/results/hypotheses from the store."""
+    raise NotImplementedError("query_store is planned for Stage 2 (needs Store).")

evalrx/eval_agent/ab_runner.py

@@ -0,0 +1,50 @@
+"""A/B execution across two strategies, with statistics done right.
+
+A *strategy* is a callable ``FailureCase -> bool`` (did this strategy succeed on
+this case) — typically wrapping a model + a prompt transform + a verifier.  The
+runner produces paired per-example success vectors and hands them to
+``stats.compare`` (McNemar + e-value + clustered bootstrap CI), so you get an
+effect size + corrected decision, never a bare p.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+from evalrx.stats import StatResult, compare
+
+
+@dataclass
+class ABResult:
+    success_a: list
+    success_b: list
+    stat: StatResult
+    n: int
+
+
+class ABRunner:
+    """Run two strategies over a CaseBatch and compare them statistically."""
+
+    def __init__(
+        self,
+        strategy_a: Callable,
+        strategy_b: Callable,
+        *,
+        cluster_fn: Optional[Callable] = None,  # case -> cluster id (e.g. task); default per-example
+        label_a: str = "A",
+        label_b: str = "B",
+    ) -> None:
+        self.strategy_a = strategy_a
+        self.strategy_b = strategy_b
+        self.cluster_fn = cluster_fn
+        self.label_a = label_a
+        self.label_b = label_b
+
+    def run(self, cases, *, paired: bool = True, alpha: float = 0.05, min_effect: float = 0.0) -> ABResult:
+        cases = list(cases)
+        sa = [bool(self.strategy_a(c)) for c in cases]
+        sb = [bool(self.strategy_b(c)) for c in cases]
+        clusters = [self.cluster_fn(c) for c in cases] if self.cluster_fn else None
+        stat = compare(sa, sb, paired=paired, alpha=alpha, min_effect=min_effect, cluster_by=clusters)
+        return ABResult(success_a=sa, success_b=sb, stat=stat, n=len(cases))

evalrx/eval_agent/agentic/__init__.py

@@ -0,0 +1,43 @@
+"""Agentic diagnosis loop — a judge-decided alternative to VLDiagnoseLoop's
+fixed M1->M2->M3->M4 cycle. See ``loop.py`` for the full picture.
+
+  actions.py  Action, parse_action, decide — judge output -> validated action
+  board.py    EvidenceBoard, BudgetState — what the judge sees each turn
+  tools.py    ToolSpec, ToolRegistry, build_default_registry — the M1-M4 tools
+  loop.py     AgenticDiagnoseLoop
+"""
+
+from __future__ import annotations
+
+from evalrx.eval_agent.agentic.actions import (
+    ACTION_SCHEMA,
+    Action,
+    ActionParseError,
+    decide,
+    parse_action,
+    validate_json_shape,
+)
+from evalrx.eval_agent.agentic.board import BudgetState, EvidenceBoard
+from evalrx.eval_agent.agentic.loop import AgenticDiagnoseLoop
+from evalrx.eval_agent.agentic.tools import (
+    ToolOutcome,
+    ToolRegistry,
+    ToolSpec,
+    build_default_registry,
+)
+
+__all__ = [
+    "AgenticDiagnoseLoop",
+    "Action",
+    "ActionParseError",
+    "ACTION_SCHEMA",
+    "parse_action",
+    "decide",
+    "validate_json_shape",
+    "EvidenceBoard",
+    "BudgetState",
+    "ToolSpec",
+    "ToolOutcome",
+    "ToolRegistry",
+    "build_default_registry",
+]