evalrx 0.1.2__py3-none-any.whl

evalrx/agent_runtime/skills/installer.py

@@ -0,0 +1,68 @@
+"""Skill installation strategies for CLI coding-provider adapters."""
+
+from __future__ import annotations
+
+import logging
+import shutil
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class SkillInstaller:
+    """Vendor skill directories into a run-local Claude-compatible layout."""
+
+    def __init__(self, skills: list[str] | tuple[str, ...]) -> None:
+        self._skills = list(skills)
+
+    @property
+    def skills(self) -> list[str]:
+        return list(self._skills)
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self._skills)
+
+    def install(self, workdir: Path) -> None:
+        if not self._skills:
+            return
+        dest_root = workdir / ".claude" / "skills"
+        for src in self._skills:
+            src_path = Path(src)
+            if not src_path.exists() or not src_path.is_dir():
+                logger.warning("skill dir not found, skipping: %s", src)
+                continue
+            try:
+                shutil.copytree(src_path, dest_root / src_path.name, dirs_exist_ok=True)
+            except OSError as exc:
+                logger.warning("could not vendor skill %s: %s", src, exc)
+
+
+class CodexSkillInstaller(SkillInstaller):
+    """Vendor skills and expose them through AGENTS.md for Codex."""
+
+    def install(self, workdir: Path) -> None:
+        super().install(workdir)
+        names = [Path(s).name for s in self._skills if Path(s).is_dir()]
+        if not names:
+            return
+        section = "\n".join([
+            "# Agent Skills (vendored)",
+            "",
+            "Before writing any figure/plot, read and APPLY these style guides:",
+            "",
+            *[f"- `.claude/skills/{n}/SKILL.md`" for n in names],
+            "",
+            "They govern chart-type choice and styling only -- never change the "
+            "data, the analysis, or the required output format.",
+            "",
+        ])
+        agents_md = workdir / "AGENTS.md"
+        try:
+            existing = (
+                agents_md.read_text(encoding="utf-8").rstrip() + "\n\n"
+                if agents_md.exists() else ""
+            )
+            agents_md.write_text(existing + section, encoding="utf-8")
+        except OSError as exc:
+            logger.warning("could not write AGENTS.md for codex skills: %s", exc)

evalrx/agent_runtime/skills/prompt_policy.py

@@ -0,0 +1,86 @@
+"""Prompt addenda for Agent Skill usage."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from evalrx.agent_runtime.cli_types import CliAgentConfig
+
+
+def fences_hint(cli_config: "CliAgentConfig | None") -> str:
+    if cli_config is not None and cli_config.provider != "llm":
+        return ", written to a file named analysis.py"
+    return " inside a ```python code block"
+
+
+def skills_hint(cli_config: "CliAgentConfig | None") -> str:
+    """Prompt addendum steering the agent to apply available Agent Skills."""
+    if cli_config is None or not getattr(cli_config, "skills_enabled", False):
+        return ""
+
+    names = [Path(path).name for path in (cli_config.skills or [])]
+    is_codex = getattr(cli_config, "provider", "") == "codex"
+
+    def _use(nlist: list[str]) -> str:
+        if not nlist:
+            if is_codex:
+                return (
+                    "read the vendored guides under `.claude/skills/` "
+                    "(also listed in AGENTS.md) and apply them"
+                )
+            return "invoke any installed Agent Skills via the Skill tool and follow them"
+        if is_codex:
+            refs = ", ".join(f"`.claude/skills/{name}/SKILL.md`" for name in nlist)
+            return f"read the vendored guides ({refs} -- also listed in AGENTS.md) and apply them"
+        listed = ", ".join(f"`/{name}`" for name in nlist)
+        return f"invoke the {listed} skill(s) via the Skill tool and follow them"
+
+    parts: list[str] = []
+
+    if "outcome-driver-analysis" in names:
+        parts.append(
+            "ANALYSIS METHOD: BEFORE writing any analysis code, "
+            + _use(["outcome-driver-analysis"])
+            + " to choose justified statistical methods for the outcome analysis: "
+            "explanatory-variable EDA, per-variable tests WITH effect sizes, "
+            "conditioning/confounding (Simpson's) checks, marginal screening, a "
+            "justified regression model (state the GLM-vs-mixed-effects reasoning "
+            "from the ACTUAL clustering structure -- with very few clusters prefer "
+            "a fixed effect over a random effect), and fit diagnostics "
+            "(collinearity/VIF, discrimination, calibration). Adopt its "
+            "METHODOLOGY, not its file layout or report template -- every output "
+            "still flows into the required result-JSON contract, and intake "
+            "answers are inferred from the data profile and the question (never "
+            "stop to ask). Keep all wording DESCRIPTIVE: takeaways carry effect "
+            "sizes + confidence intervals + direction; test statistics/p-values "
+            "belong in tables/artifacts and must NOT be phrased as significance "
+            "or confirmation verdicts -- validity is adjudicated downstream."
+        )
+
+    style = [name for name in names if name != "outcome-driver-analysis"]
+    roles = []
+    if "eval-chart-style" in style:
+        roles.append(
+            "`eval-chart-style` governs chart-TYPE choice (distribution-first: "
+            "violin/ECDF/heatmap/forest/paired-slope -- never a mean as a bar) and "
+            "the palette: FAIL/PASS semantic hues, a categorical series order for "
+            "non-outcome dimensions, and single-hue ramps for ordered dimensions "
+            "(e.g. model sizes). When any other skill suggests a specific chart "
+            "type or color, eval-chart-style's policy and palette take precedence"
+        )
+    if "nature-figure" in style:
+        roles.append("`nature-figure` adds publication-grade matplotlib polish")
+    role_line = ("; ".join(roles) + ". ") if roles else ""
+    parts.append(
+        "FIGURE STYLING: Agent Skills are available -- apply them BY DEFAULT to "
+        f"every plot you write under figures/: BEFORE plotting, {_use(style)}. "
+        f"{role_line}"
+        "This is a non-interactive PYTHON analysis: if a skill asks you to choose "
+        "a plotting backend, choose Python and proceed without pausing -- never "
+        "stop to ask a question. Use skills for styling only -- they must not "
+        "change the data, the analysis, the sandbox workflow, or the final "
+        "result JSON."
+    )
+    return "\n" + "\n\n".join(parts) + "\n"

evalrx/agent_runtime/skills/resolver.py

@@ -0,0 +1,19 @@
+"""Resolve bundled and explicitly configured Agent Skills."""
+
+from __future__ import annotations
+
+from evalrx.agent_assets.skills import SKILL_BACKENDS, bundled_skill_paths
+
+
+def resolve_skill_paths(
+    *,
+    provider: str,
+    explicit: list[str] | tuple[str, ...] = (),
+    use_bundled: bool = True,
+) -> tuple[str, ...]:
+    """Return ordered skill dirs for a provider, de-duplicating explicit repeats."""
+    skill_dirs = list(explicit or [])
+    if use_bundled and provider in SKILL_BACKENDS:
+        bundled = [path for path in bundled_skill_paths() if path not in skill_dirs]
+        skill_dirs = bundled + skill_dirs
+    return tuple(skill_dirs)

evalrx/analysis/__init__.py

@@ -0,0 +1,132 @@
+"""Public data-analysis API: exploratory analysis, hypothesis proposal, and
+statistical confirmation.
+
+This package exposes EvalRX' data-analysis layer as a standalone
+capability — ``ExploratoryAnalysisAgent`` (M2) for descriptive EDA (no
+hypothesis generation or validation itself), ``HypothesisAgent`` (M3) for
+proposing falsifiable hypotheses from M2's takeaways (proposal only, no
+validation), and ``StatsAnalysisAgent`` for confirmatory effect/CI/e-value/FDR
+verdicts. The eval-agent loops still use the same implementation internally,
+but callers do not need to import from ``evalrx.eval_agent.stages``.
+"""
+
+from evalrx.analysis.adjudicate import adjudicate_report, adjudicate_signals
+from evalrx.analysis.api import ExploreRunResult, explore
+from evalrx.analysis.explorer import (
+    CandidateSignal,
+    ExploratoryAnalysisAgent,
+    ExploratoryAnalysisReport,
+    Takeaway,
+    load_records_from_path,
+    scan_folder,
+)
+from evalrx.analysis.failure_modes import FailureMode, FailureModeReport, cluster_failures
+from evalrx.analysis.fused_pipeline import (
+    FusedReport,
+    FusedSignal,
+    run_fused_analysis,
+)
+from evalrx.analysis.hypothesis_agent import Hypothesis, HypothesisAgent
+from evalrx.analysis.operationalize import (
+    RecipeError,
+    SignalRecipe,
+    bridge_recipes_to_result,
+    compile_recipe,
+    compile_recipes,
+    per_case_finding,
+    per_case_to_records,
+    safe_ident,
+)
+from evalrx.analysis.planner import AnalysisPlanItem, plan_stats_input, ranked_signal_names
+from evalrx.analysis.profile import (
+    ColumnProfile,
+    DatasetProfile,
+    describe_outcome,
+    profile_records,
+    profile_stats_input,
+)
+from evalrx.analysis.run_codebase import CodebaseRunResult, run_codebase, run_codebase_cli
+from evalrx.analysis.stats_agent import StatsAnalysisAgent, StatsAnalysisReport
+from evalrx.analysis.stats_tools import (
+    STATS_TOOL_CATALOG,
+    EvidenceResult,
+    StatsInput,
+    StatsToolResult,
+    build_stats_input,
+    build_stats_input_from_records,
+    default_plan,
+    fdr_correct,
+    run_stats_tool,
+)
+from evalrx.analysis.workbench import (
+    DatasetBundle,
+    EventSink,
+    ThreadStore,
+    UploadLimits,
+    extract_archive,
+    ingest_directory,
+)
+from evalrx.reporting.compiler import compile_diagnostic_report
+from evalrx.reporting.model import Claim, DiagnosticReport, Evidence, ReportStep
+
+__all__ = [
+    "explore",
+    "ExploreRunResult",
+    "run_codebase",
+    "run_codebase_cli",
+    "CodebaseRunResult",
+    "cluster_failures",
+    "FailureMode",
+    "FailureModeReport",
+    "StatsAnalysisAgent",
+    "StatsAnalysisReport",
+    "ExploratoryAnalysisAgent",
+    "ExploratoryAnalysisReport",
+    "Takeaway",
+    "CandidateSignal",
+    "Hypothesis",
+    "HypothesisAgent",
+    "adjudicate_report",
+    "adjudicate_signals",
+    "SignalRecipe",
+    "compile_recipe",
+    "compile_recipes",
+    "per_case_finding",
+    "per_case_to_records",
+    "bridge_recipes_to_result",
+    "safe_ident",
+    "RecipeError",
+    "ColumnProfile",
+    "DatasetProfile",
+    "describe_outcome",
+    "profile_records",
+    "profile_stats_input",
+    "AnalysisPlanItem",
+    "ranked_signal_names",
+    "plan_stats_input",
+    "run_fused_analysis",
+    "FusedReport",
+    "FusedSignal",
+    "DiagnosticReport",
+    "Claim",
+    "Evidence",
+    "ReportStep",
+    "compile_diagnostic_report",
+    "load_records_from_path",
+    "scan_folder",
+    "StatsInput",
+    "StatsToolResult",
+    "EvidenceResult",
+    "STATS_TOOL_CATALOG",
+    "build_stats_input",
+    "build_stats_input_from_records",
+    "default_plan",
+    "fdr_correct",
+    "run_stats_tool",
+    "DatasetBundle",
+    "EventSink",
+    "ThreadStore",
+    "UploadLimits",
+    "extract_archive",
+    "ingest_directory",
+]

evalrx/analysis/adjudicate.py

@@ -0,0 +1,154 @@
+"""Host-side adjudication of explorer candidate signals (the firewall).
+
+The :mod:`~evalrx.analysis.explorer` PROPOSES candidate signals; it has no
+authority to declare significance. This module is the host firewall: it takes the
+host-adjudicable ``sufficient`` statistics an explorer attached to a candidate and
+recomputes the verdict with the SAME validated, multiplicity-aware core the M2
+stats engine uses — never trusting a ``reject`` / ``e_value`` / ``p_value`` the
+explorer self-declared (the explorer schema does not even carry one).
+
+Mechanics (mirror ``stats_tools`` / ``stats_tool_generator`` exactly):
+
+- ``_reconstruct_decision`` turns sufficient statistics into
+  ``(reject, e_value, effect, ci)`` via the validated core.
+- ``fdr_correct`` applies e-BH across every candidate that produced an e-value
+  (the ``paired_binary`` shape), so proposing more candidates pays more
+  multiplicity — exactly one e-BH family, no per-candidate self-rejection.
+- A ``two_group`` candidate yields a bootstrap-CI reject but no e-value (matching
+  the ``signal_label_assoc`` catalog tool). It is host-adjudicated but sits
+  OUTSIDE the e-BH family and is flagged ``fdr_corrected=False`` — honest, not
+  over-claimed.
+- A candidate with no adjudicable ``sufficient`` is ``descriptive_only`` and never
+  rejects.
+
+IMPORTANT — split discipline: in the standalone chat path the candidate's
+``sufficient`` is computed on the SAME rows used to discover the signal. That is
+IN-SAMPLE; ``split_label`` records it so nothing over-claims, and an honest caveat
+is appended. Held-out CONFIRM-split adjudication (selecting on EXPLORE, computing
+each e-value on a disjoint CONFIRM split) is the job of the fused pipeline
+(Phase B). This module supplies only the verdict mechanism.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from evalrx.analysis.stats_tool_generator import _reconstruct_decision
+from evalrx.analysis.stats_tools import StatsToolResult, fdr_correct
+
+if TYPE_CHECKING:
+    from evalrx.analysis.explorer import CandidateSignal, ExploratoryAnalysisReport
+
+
+def _family_key(candidate: "CandidateSignal", index: int) -> str:
+    """Stable, unique key so an e-BH rejection maps back to the right candidate
+    even when two candidates share a name."""
+    return f"{candidate.name or 'signal'}#{index}"
+
+
+def adjudicate_signals(
+    candidates: "list[CandidateSignal]",
+    *,
+    alpha: float = 0.05,
+    split_label: str = "in_sample",
+) -> dict[str, Any]:
+    """Annotate each candidate IN PLACE with a host verdict; return family metadata.
+
+    Reads only ``candidate.sufficient`` — never a self-declared verdict. Returns a
+    metadata dict describing the single e-BH family the candidates competed in.
+    """
+    # 1) Host reconstruction per candidate; assemble the e-BH family from the
+    #    candidates that produced an e-value (paired_binary).
+    raw_reject: dict[int, bool] = {}
+    family: list[StatsToolResult] = []
+    family_key: dict[int, str] = {}
+    for i, c in enumerate(candidates):
+        recon = (
+            _reconstruct_decision(c.sufficient, alpha)
+            if isinstance(c.sufficient, dict)
+            else None
+        )
+        if recon is None:
+            c.host_adjudicated = False
+            c.descriptive_only = True
+            c.fdr_corrected = False
+            c.e_value = None
+            c.reject = False
+            continue
+
+        reject, e_value, effect, ci = recon
+        c.host_adjudicated = True
+        c.descriptive_only = False
+        c.e_value = e_value
+        if effect is not None:
+            c.effect = effect
+        if ci is not None:
+            c.ci = ci
+        raw_reject[i] = bool(reject)
+        if e_value is not None:
+            key = _family_key(c, i)
+            family_key[i] = key
+            family.append(
+                StatsToolResult(
+                    tool=key, e_value=e_value, effect=effect, ci=ci, reject=bool(reject)
+                )
+            )
+
+    # 2) ONE e-BH family across the e-value-bearing candidates (the multiplicity
+    #    firewall). `fdr_correct` only considers results with an e-value.
+    fdr = fdr_correct(family, alpha=alpha)
+    rejected_keys = set(fdr.get("rejected_tools", []))
+
+    # 3) Final reject per candidate.
+    for i, c in enumerate(candidates):
+        if not c.host_adjudicated:
+            continue
+        if c.e_value is not None:
+            # In the e-BH family. `reject` IS the e-BH outcome; `fdr_corrected`
+            # means "this rejection is FDR-controlled" — True ONLY when it survived
+            # e-BH (matches fused_pipeline; a family member that did not survive is
+            # reject=False, fdr_corrected=False). Family membership is recoverable
+            # from `e_value is not None`, so no information is lost.
+            c.reject = family_key.get(i) in rejected_keys
+            c.fdr_corrected = bool(c.reject)
+        else:
+            # two_group: CI-based reject, outside the e-BH family (mirrors M2's
+            # signal_label_assoc). Honest about not being FDR-corrected.
+            c.fdr_corrected = False
+            c.reject = raw_reject.get(i, False)
+
+    return {
+        "method": "e-BH",
+        "alpha": alpha,
+        "split": split_label,
+        "n_candidates": len(candidates),
+        "n_host_adjudicated": sum(1 for c in candidates if c.host_adjudicated),
+        "n_in_family": len(family),  # e-value-bearing candidates under e-BH
+        "n_rejected": sum(1 for c in candidates if c.reject),
+        "rejected": [c.name for c in candidates if c.reject],
+    }
+
+
+def adjudicate_report(
+    report: "ExploratoryAnalysisReport",
+    *,
+    alpha: float = 0.05,
+    split_label: str = "in_sample",
+) -> "ExploratoryAnalysisReport":
+    """Run :func:`adjudicate_signals` over a report's candidates, record metadata.
+
+    Mutates and returns *report*. Appends an honest caveat when any verdict is
+    in-sample (no held-out CONFIRM split was used to compute the e-values).
+    """
+    meta = adjudicate_signals(
+        report.candidate_signals, alpha=alpha, split_label=split_label
+    )
+    report.adjudication = meta
+    if meta["n_host_adjudicated"] and split_label == "in_sample":
+        caveat = (
+            "host verdicts are IN-SAMPLE (computed on the rows used to discover "
+            "the signal); confirm on a held-out split before trusting"
+        )
+        if caveat not in report.caveats:
+            report.caveats.append(caveat)
+    return report