agent-attrib 0.1.0__py3-none-any.whl

agent_attrib/__init__.py

@@ -0,0 +1,124 @@
+"""agent-attrib: multi-factor attribution for agent release changes."""
+
+from .model import MetricResult, analyze_runset
+from .schema import Design, Run, RunSet, load_runs
+
+__version__ = "0.1.0"
+__all__ = [
+    "plan",
+    "analyze",
+    "load_runs",
+    "RunSet",
+    "Run",
+    "Design",
+    "MetricResult",
+    "__version__",
+]
+
+
+def plan(
+    factors: list[str],
+    tasks: int = 30,
+    trials: int = 2,
+    seed: int = 42,
+    resolution: str = "IV",
+    sd: float | None = None,
+) -> "Design":
+    """Generate a factorial experiment design.
+
+    Args:
+        factors: list of factor names.
+        tasks: number of distinct tasks.
+        trials: trials per configuration per task.
+        seed: random seed for run-order randomization.
+        resolution: minimum design resolution for fractional designs ("IV" or "V").
+        sd: assumed SD for MDE calculation on continuous metrics.
+
+    Returns:
+        Design object with run order and summary statistics.
+    """
+    from .design import build_design, generate_run_order
+    from .power import mde_binary, mde_continuous
+    from .schema import Design, DesignRow
+
+    dm_matrix, dm = build_design(factors, tasks=tasks, trials=trials, seed=seed, resolution=resolution)
+    task_ids = [f"t{i:04d}" for i in range(tasks)]
+    df = generate_run_order(dm_matrix, factors, task_ids, trials, seed)
+
+    n_configs = dm_matrix.shape[0]
+    n_total = n_configs * tasks * trials
+
+    mde_val = mde_continuous(n_configs, tasks, trials, sd=sd) if sd else mde_binary(n_configs, tasks, trials)
+
+    rows: list[DesignRow] = []
+    for _, row in df.iterrows():
+        factor_levels = {f: str(row[f"f_{f}"]) for f in factors}
+        rows.append(
+            DesignRow(
+                config_id=str(row["config_id"]),
+                task_id=str(row["task_id"]),
+                trial=int(row["trial"]),
+                run_order=int(row["run_order"]),
+                factors=factor_levels,  # type: ignore[arg-type]
+            )
+        )
+
+    return Design(
+        factors=factors,
+        tasks=tasks,
+        trials=trials,
+        seed=seed,
+        design_type=dm.design_type,
+        resolution=dm.resolution,
+        alias_structure=dm.alias_structure,
+        runs=rows,
+        n_configs=n_configs,
+        n_total=n_total,
+        mde=mde_val,
+        mde_metric="success (binary, p=0.5)" if sd is None else "continuous",
+    )
+
+
+def analyze(
+    runset: "RunSet",
+    metrics: list[str] | None = None,
+    margin: float = 0.05,
+    binary_metrics: list[str] | None = None,
+) -> "AnalysisResult":
+    """Analyze a RunSet and return an AnalysisResult.
+
+    Args:
+        runset: loaded runs from load_runs().
+        metrics: metrics to analyze (default: all in runset).
+        margin: equivalence margin for EQUIVALENT/INCONCLUSIVE verdicts.
+        binary_metrics: metric names to treat as binary (default: ["success"]).
+
+    Returns:
+        AnalysisResult with per-metric results and reporting methods.
+    """
+    metrics = metrics or runset.metrics
+    results = analyze_runset(runset, metrics=metrics, margin=margin, binary_metrics=binary_metrics)
+    return AnalysisResult(results)
+
+
+class AnalysisResult:
+    """Container for per-metric analysis results with reporting methods."""
+
+    def __init__(self, results: list["MetricResult"]) -> None:
+        self.results = results
+
+    def to_markdown(self) -> str:
+        from .report import to_markdown
+        return to_markdown(self.results)
+
+    def to_json(self) -> str:
+        from .report import to_json
+        return to_json(self.results)
+
+    def print(self) -> None:
+        from .report import print_rich
+        print_rich(self.results)
+
+    def __repr__(self) -> str:
+        metrics = [r.metric for r in self.results]
+        return f"AnalysisResult(metrics={metrics})"

agent_attrib/cli.py

@@ -0,0 +1,250 @@
+"""Typer CLI: plan, analyze, demo, version."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+app = typer.Typer(
+    name="agent-attrib",
+    help="Multi-factor attribution for agent release changes.",
+    no_args_is_help=True,
+)
+console = Console()
+
+
+@app.command()
+def plan(
+    factors: Annotated[
+        str,
+        typer.Option("--factors", "-f", help="Comma-separated factor names"),
+    ],
+    tasks: Annotated[int, typer.Option("--tasks", "-t", help="Number of tasks")] = 30,
+    trials: Annotated[
+        int, typer.Option("--trials", help="Trials per config per task")
+    ] = 2,
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", "-o", help="Write design CSV to this path"),
+    ] = None,
+    seed: Annotated[int, typer.Option("--seed", help="Random seed")] = 42,
+    resolution: Annotated[
+        str,
+        typer.Option("--resolution", help="Minimum resolution for fractional designs (IV or V)"),
+    ] = "IV",
+    sd: Annotated[
+        float | None,
+        typer.Option("--sd", help="Assumed SD for MDE (continuous metrics)"),
+    ] = None,
+    pilot: Annotated[
+        Path | None,
+        typer.Option("--pilot", help="Pilot runs JSONL for SD estimation"),
+    ] = None,
+) -> None:
+    """Generate a factorial experiment design."""
+    from .design import build_design, generate_run_order
+    from .power import mde_binary, mde_continuous
+    from .schema import Design, DesignRow, load_runs
+
+    factor_list = [f.strip() for f in factors.split(",") if f.strip()]
+    if not factor_list:
+        console.print("[red]Error: --factors must specify at least one factor[/red]")
+        raise typer.Exit(1)
+
+    try:
+        dm_matrix, dm = build_design(factor_list, tasks=tasks, trials=trials, seed=seed, resolution=resolution)
+    except ValueError as e:
+        console.print(f"[red]Error: {e}[/red]")
+        raise typer.Exit(1) from None
+
+    task_ids = [f"t{i:04d}" for i in range(tasks)]
+    df = generate_run_order(dm_matrix, factor_list, task_ids, trials, seed)
+
+    n_configs = dm_matrix.shape[0]
+    n_total = n_configs * tasks * trials
+
+    # MDE calculation
+    mde_val: float | None = None
+    mde_metric_name = "success (binary, p=0.5)"
+    if pilot is not None:
+        try:
+            rs = load_runs(pilot)
+            import numpy as np
+            vals = [r.metrics.get("success", 0) for r in rs.runs]
+            sd_est = float(np.std(vals))
+            mde_val = mde_continuous(n_configs, tasks, trials, sd=sd_est)
+            mde_metric_name = "success (from pilot)"
+        except Exception as e:
+            console.print(f"[yellow]Warning: could not read pilot file: {e}[/yellow]")
+    elif sd is not None:
+        mde_val = mde_continuous(n_configs, tasks, trials, sd=sd)
+        mde_metric_name = f"continuous metric (SD={sd})"
+    else:
+        mde_val = mde_binary(n_configs, tasks, trials)
+
+    rows: list[DesignRow] = []
+    for _, row in df.iterrows():
+        factor_levels = {f: str(row[f"f_{f}"]) for f in factor_list}
+        rows.append(
+            DesignRow(
+                config_id=str(row["config_id"]),
+                task_id=str(row["task_id"]),
+                trial=int(row["trial"]),
+                run_order=int(row["run_order"]),
+                factors=factor_levels,  # type: ignore[arg-type]
+            )
+        )
+
+    design = Design(
+        factors=factor_list,
+        tasks=tasks,
+        trials=trials,
+        seed=seed,
+        design_type=dm.design_type,
+        resolution=dm.resolution,
+        alias_structure=dm.alias_structure,
+        runs=rows,
+        n_configs=n_configs,
+        n_total=n_total,
+        mde=mde_val,
+        mde_metric=mde_metric_name,
+    )
+
+    console.print(design.summary())
+
+    if output is not None:
+        design.to_csv(output)
+        console.print(f"\nDesign written to [cyan]{output}[/cyan]")
+
+
+@app.command()
+def analyze(
+    runs_file: Annotated[Path, typer.Argument(help="JSONL or CSV runs file")],
+    metric: Annotated[
+        list[str] | None,
+        typer.Option("--metric", "-m", help="Metric to analyze (repeatable)"),
+    ] = None,
+    margin: Annotated[
+        float,
+        typer.Option("--margin", help="Equivalence margin for verdict"),
+    ] = 0.05,
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", "-o", help="Write markdown report to this path"),
+    ] = None,
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output JSON instead of markdown")
+    ] = False,
+    binary: Annotated[
+        list[str] | None,
+        typer.Option("--binary", help="Metric names to treat as binary (default: success)"),
+    ] = None,
+) -> None:
+    """Analyze agent runs and report effect attributions."""
+    from .model import analyze_runset
+    from .report import print_rich, to_json, to_markdown
+    from .schema import load_runs
+
+    try:
+        runset = load_runs(runs_file)
+    except Exception as e:
+        console.print(f"[red]Error loading runs: {e}[/red]")
+        raise typer.Exit(1) from None
+
+    metrics_to_analyze = metric or runset.metrics
+    binary_metrics = binary or ["success"]
+
+    try:
+        results = analyze_runset(
+            runset,
+            metrics=metrics_to_analyze,
+            margin=margin,
+            binary_metrics=binary_metrics,
+        )
+    except Exception as e:
+        console.print(f"[red]Analysis error: {e}[/red]")
+        raise typer.Exit(1) from None
+
+    if json_output:
+        from .report import to_json
+        text = to_json(results)
+        if output:
+            output.write_text(text)
+            console.print(f"JSON report written to [cyan]{output}[/cyan]")
+        else:
+            print(text)
+    elif output:
+        md = to_markdown(results)
+        output.write_text(md)
+        console.print(f"Markdown report written to [cyan]{output}[/cyan]")
+    else:
+        print_rich(results, console=console)
+
+
+@app.command()
+def demo(
+    tasks: Annotated[int, typer.Option("--tasks", "-t")] = 50,
+    seed: Annotated[int, typer.Option("--seed")] = 7,
+    output: Annotated[Path | None, typer.Option("--output", "-o")] = None,
+) -> None:
+    """Run a zero-cost demo with synthetic runs and planted effects."""
+    from .model import analyze_runset
+    from .report import print_rich, to_markdown
+    from .simulate import simulate_multi_metric
+
+    console.print("[bold cyan]agent-attrib demo[/bold cyan]")
+    console.print(
+        "Generating synthetic runs with planted effects:\n"
+        "  prompt:    +0.15 on success\n"
+        "  model:     +0.08 on success, −20% on cost\n"
+        "  retriever: no effect on success, +10% on cost\n"
+        "  prompt*model interaction: +0.05 on success\n"
+    )
+
+    factors = ["prompt", "model", "retriever"]
+    runset = simulate_multi_metric(
+        factors=factors,
+        tasks=tasks,
+        trials=2,
+        seed=seed,
+        success_effects={"prompt": 0.3, "model": 0.16},
+        cost_effects={"model": -0.22, "retriever": 0.095},
+        success_interaction={("prompt", "model"): 0.10},
+    )
+
+    console.print(
+        f"Generated {len(runset.runs)} runs "
+        f"({tasks} tasks × {2**3} configs × 2 trials)\n"
+    )
+
+    results = analyze_runset(
+        runset,
+        metrics=["success", "cost_usd"],
+        margin=0.05,
+        binary_metrics=["success"],
+    )
+
+    print_rich(results, console=console)
+
+    if output:
+        md = to_markdown(results)
+        output.write_text(md)
+        console.print(f"Report written to [cyan]{output}[/cyan]")
+
+
+@app.command()
+def version() -> None:
+    """Print the version."""
+    from importlib.metadata import version as pkg_version
+    try:
+        v = pkg_version("agent-attrib")
+    except Exception:
+        v = "0.1.0 (development)"
+    console.print(v)
+
+
+if __name__ == "__main__":
+    app()

agent_attrib/design.py

@@ -0,0 +1,214 @@
+"""Full and fractional 2-level factorial design generation."""
+
+from __future__ import annotations
+
+import itertools
+from dataclasses import dataclass
+
+import numpy as np
+import pandas as pd
+
+# ---------------------------------------------------------------------------
+# Generator tables for regular 2^(k-p) minimum-aberration designs
+# Source: Wu & Hamada "Experiments", Appendix 4B / Table 5A.1
+# Format: {k: {p: (resolution, {added_col_1based: generator_string})}}
+# Generator string: 1-based indices of base columns whose product gives this col.
+# len(added_map) must equal p.
+# ---------------------------------------------------------------------------
+#
+# 2^(5-1): only one design exists; E = ABCD → resolution V (word length 5).
+# 2^(6-2): E = ABC, F = BCD → I = ABCE = BCDF = ACDEF → resolution IV.
+# 2^(7-3): E = ABC, F = BCD, G = ACD → all words length 4 → resolution IV.
+_GENERATORS: dict[int, dict[int, tuple[str, dict[int, str]]]] = {
+    5: {
+        1: (
+            "V",
+            {5: "1234"},   # E = ABCD → resolution V
+        ),
+    },
+    6: {
+        2: (
+            "IV",
+            {5: "123", 6: "234"},  # E = ABC, F = BCD
+        ),
+    },
+    7: {
+        3: (
+            "IV",
+            {5: "123", 6: "234", 7: "134"},  # E = ABC, F = BCD, G = ACD
+        ),
+    },
+}
+
+# Resolution V override table (k=5 already gives V; k=6/7 not supported in v0.1)
+_GENERATORS_RES_V: dict[int, dict[int, tuple[str, dict[int, str]]]] = {
+    5: {
+        1: (
+            "V",
+            {5: "1234"},  # E = ABCD → resolution V
+        ),
+    },
+}
+
+
+@dataclass
+class _DesignMatrix:
+    matrix: np.ndarray  # shape (n_configs, k), values ±1
+    factors: list[str]
+    resolution: str
+    design_type: str
+    alias_structure: list[str]
+
+
+def _full_factorial(factors: list[str]) -> _DesignMatrix:
+    k = len(factors)
+    n = 2**k
+    cols = []
+    for i in range(k):
+        period = 2 ** (k - i - 1)
+        col = np.array([(-1) ** (j // period) for j in range(n)], dtype=float)
+        cols.append(col)
+    matrix = np.column_stack(cols)
+    return _DesignMatrix(
+        matrix=matrix,
+        factors=factors,
+        resolution="full",
+        design_type=f"2^{k} full factorial",
+        alias_structure=[],
+    )
+
+
+def _parse_generator(gen_str: str, base_cols: np.ndarray) -> np.ndarray:
+    """Compute added column as product of base columns specified by 1-based indices."""
+    col = np.ones(base_cols.shape[0], dtype=float)
+    for ch in gen_str:
+        col = col * base_cols[:, int(ch) - 1]
+    return col
+
+
+def _fractional_factorial(
+    factors: list[str], resolution_req: str = "IV"
+) -> _DesignMatrix:
+    k = len(factors)
+
+    # Pick generator table
+    if resolution_req.upper() == "V" and k in _GENERATORS_RES_V:
+        gen_table = _GENERATORS_RES_V
+    else:
+        gen_table = _GENERATORS
+
+    if k not in gen_table:
+        raise ValueError(f"No generator available for k={k}. Supported: {sorted(gen_table)}")
+
+    p_choices = sorted(gen_table[k].keys())
+    res, added_map = gen_table[k][p_choices[0]]
+
+    # Base design: full factorial on first (k - p) factors
+    p = p_choices[0]
+    base_k = k - p
+    base_dm = _full_factorial(factors[:base_k])
+    base_cols = base_dm.matrix  # shape (2^base_k, base_k)
+
+    added_cols = []
+    for col_idx in sorted(added_map.keys()):
+        gen_str = added_map[col_idx]
+        added_cols.append(_parse_generator(gen_str, base_cols))
+
+    matrix = np.column_stack([base_cols] + [c.reshape(-1, 1) for c in added_cols])
+
+    alias_structure = _compute_aliases(matrix, factors)
+
+    return _DesignMatrix(
+        matrix=matrix,
+        factors=factors,
+        resolution=res,
+        design_type=f"2^({k}-{p}) fractional factorial",
+        alias_structure=alias_structure,
+    )
+
+
+def _compute_aliases(matrix: np.ndarray, factors: list[str]) -> list[str]:
+    """Return human-readable alias chains for a fractional design."""
+    k = len(factors)
+    aliases: list[str] = []
+
+    # Build all two-factor interactions
+    interactions: dict[str, np.ndarray] = {}
+    for i, j in itertools.combinations(range(k), 2):
+        name = f"{factors[i]}*{factors[j]}"
+        interactions[name] = matrix[:, i] * matrix[:, j]
+
+    # Check which main effects alias with which interactions
+    seen: set[str] = set()
+    for i in range(k):
+        chain = [factors[i]]
+        for name, col in interactions.items():
+            if np.allclose(matrix[:, i], col) or np.allclose(matrix[:, i], -col):
+                chain.append(name)
+                seen.add(name)
+        if len(chain) > 1:
+            aliases.append(" = ".join(chain))
+
+    # Three-factor interactions could also appear; skip for display brevity
+    return aliases
+
+
+def build_design(
+    factors: list[str],
+    tasks: int,
+    trials: int,
+    seed: int,
+    resolution: str = "IV",
+) -> tuple[np.ndarray, _DesignMatrix]:
+    """Return (design_matrix ±1, _DesignMatrix metadata)."""
+    if len(factors) < 1:
+        raise ValueError("Need at least 1 factor")
+    if len(factors) > 7:
+        raise ValueError(
+            "agent-attrib v0.1 supports up to 7 factors. "
+            "For k>7, split into independent sub-experiments."
+        )
+    if tasks < 1:
+        raise ValueError("tasks must be >= 1")
+    if trials < 1:
+        raise ValueError("trials must be >= 1")
+
+    if len(factors) <= 4:
+        dm = _full_factorial(factors)
+    else:
+        dm = _fractional_factorial(factors, resolution_req=resolution)
+
+    return dm.matrix, dm
+
+
+def generate_run_order(
+    dm_matrix: np.ndarray,
+    factors: list[str],
+    task_ids: list[str],
+    trials: int,
+    seed: int,
+) -> pd.DataFrame:
+    """Expand design matrix to per-run rows and randomize order."""
+    rng = np.random.default_rng(seed)
+    n_configs = dm_matrix.shape[0]
+    rows = []
+    for ci in range(n_configs):
+        config_id = f"c{ci:03d}"
+        factor_levels = {
+            f: "candidate" if dm_matrix[ci, fi] == 1 else "baseline"
+            for fi, f in enumerate(factors)
+        }
+        for task_id in task_ids:
+            for trial in range(1, trials + 1):
+                rows.append(
+                    {
+                        "config_id": config_id,
+                        "task_id": task_id,
+                        "trial": trial,
+                        **{f"f_{f}": level for f, level in factor_levels.items()},
+                    }
+                )
+    df = pd.DataFrame(rows)
+    order = rng.permutation(len(df))
+    df["run_order"] = order + 1
+    return df.sort_values("run_order").reset_index(drop=True)