agent-self-edit 0.2.0__py3-none-any.whl

agent_self_edit/__init__.py

@@ -0,0 +1,8 @@
+"""AgentSelfEdit — An agent that rewrites its own system prompt from execution feedback."""
+
+from importlib.metadata import version as _version
+
+try:
+    __version__ = _version("agent-self-edit")
+except Exception:
+    __version__ = "0.0.0"

agent_self_edit/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    main()

agent_self_edit/ab_test.py

@@ -0,0 +1,333 @@
+"""Task runner, A/B test engine, statistics, and cost tracking."""
+
+from __future__ import annotations
+
+import random
+import statistics
+import time
+from dataclasses import dataclass, field
+from typing import Literal
+
+from .config import ABTestConfig, Config
+from .llm.base import LLMProvider, ProviderError
+from .scorers import Scorer
+from .tasks import Task, TaskSet
+from .types import utc_now_iso  # noqa: F401  (re-export convenience)
+
+_COST_PER_1K_TOKENS = 0.0033  # approx gpt-4o-mini blend ($ per 1K tokens)
+
+
+@dataclass(frozen=True)
+class TaskResult:
+    """Result of running one task against one prompt."""
+
+    output: str
+    success: bool
+    latency_ms: float
+    token_count: int
+    error: str | None = None
+
+
+@dataclass(frozen=True)
+class PerTask:
+    """Paired per-task comparison for the A/B result."""
+
+    task_id: str
+    task_input: str
+    expected_output: str
+    output_a: str
+    score_a: float
+    output_b: str
+    score_b: float
+    delta: float
+    latency_a_ms: float
+    latency_b_ms: float
+    tokens_a: int
+    tokens_b: int
+    error_a: str | None = None
+    error_b: str | None = None
+
+
+@dataclass(frozen=True)
+class ABResult:
+    """Statistical result of comparing prompt B against prompt A."""
+
+    winner: Literal["a", "b", "tie", "inconclusive"]
+    mean_delta: float
+    ci_low: float
+    ci_high: float
+    p_value: float
+    effect_size: float
+    n_trials: int
+    per_task: list[PerTask] = field(default_factory=list)
+    cost_usd: float = 0.0
+    token_count: int = 0
+
+
+@dataclass(frozen=True)
+class BootstrapResult:
+    mean: float
+    ci_low: float
+    ci_high: float
+    std: float
+
+
+def estimate_tokens(text: str) -> int:
+    """Rough token estimate: ~4 characters per token."""
+    return max(1, len(text) // 4)
+
+
+def estimate_cost(token_count: int, price_per_1k: float = _COST_PER_1K_TOKENS) -> float:
+    """Estimate USD cost for ``token_count`` tokens."""
+    return token_count * price_per_1k / 1000.0
+
+
+# ---------------------------------------------------------------------------
+# Task runner (#16)
+# ---------------------------------------------------------------------------
+
+
+def run_task(task: Task, prompt: str, llm: LLMProvider) -> TaskResult:
+    """Run ``task`` against one ``prompt`` and measure latency + tokens."""
+    if not prompt.strip():
+        return TaskResult(output="", success=False, latency_ms=0.0, token_count=0,
+                          error="empty prompt")
+
+    full_prompt = f"{prompt}\n\n---\n\nTask: {task.input}"
+    start = time.monotonic()
+    try:
+        output = llm.complete(prompt=full_prompt, system_prompt="", temperature=0.0)
+    except ProviderError as e:
+        return TaskResult(output="", success=False, latency_ms=0.0, token_count=0,
+                          error=str(e))
+    latency_ms = (time.monotonic() - start) * 1000.0
+    tokens = estimate_tokens(full_prompt) + estimate_tokens(output)
+    return TaskResult(
+        output=output,
+        success=True,
+        latency_ms=latency_ms,
+        token_count=tokens,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Statistics (#19, #20, #21)
+# ---------------------------------------------------------------------------
+
+
+def bootstrap_ci(
+    scores_a: list[float],
+    scores_b: list[float],
+    n_resamples: int = 10000,
+    ci_level: float = 0.95,
+) -> BootstrapResult:
+    """Bootstrap CI for the mean delta = mean(score_b - score_a).
+
+    Uses a deterministic seed for reproducibility.
+    """
+    n = len(scores_a)
+    if n == 0:
+        return BootstrapResult(mean=0.0, ci_low=0.0, ci_high=0.0, std=0.0)
+    if n < 2:
+        return BootstrapResult(mean=0.0, ci_low=0.0, ci_high=0.0, std=0.0)
+
+    deltas = [b - a for a, b in zip(scores_a, scores_b)]
+    mean_delta = sum(deltas) / n
+
+    if n_resamples <= 0:
+        return BootstrapResult(mean=mean_delta, ci_low=mean_delta, ci_high=mean_delta, std=0.0)
+
+    rng = random.Random(0)
+    means: list[float] = []
+    for _ in range(n_resamples):
+        sample = [rng.choice(deltas) for _ in range(n)]
+        means.append(sum(sample) / n)
+
+    means.sort()
+    tail = (1.0 - ci_level) / 2.0
+    low_idx = int(tail * n_resamples)
+    high_idx = int((1.0 - tail) * n_resamples) - 1
+    ci_low = means[low_idx]
+    ci_high = means[high_idx]
+    std = statistics.stdev(means) if len(means) > 1 else 0.0
+    return BootstrapResult(mean=mean_delta, ci_low=ci_low, ci_high=ci_high, std=std)
+
+
+def permutation_test(
+    scores_a: list[float],
+    scores_b: list[float],
+    n_permutations: int = 1000,
+) -> float:
+    """One-tailed permutation p-value: how often a random split beats observed.
+
+    Deterministic seed for reproducibility.
+    """
+    n = len(scores_a)
+    if n == 0:
+        return 1.0
+    observed_diff = sum(scores_b) / n - sum(scores_a) / n
+    pooled = list(scores_a) + list(scores_b)
+    rng = random.Random(0)
+
+    count = 0
+    for _ in range(n_permutations):
+        rng.shuffle(pooled)
+        fake_a = pooled[:n]
+        fake_b = pooled[n:]
+        fake_diff = sum(fake_b) / n - sum(fake_a) / n
+        if fake_diff >= observed_diff:
+            count += 1
+    return count / n_permutations
+
+
+def effect_size(scores_a: list[float], scores_b: list[float]) -> float:
+    """Relative improvement = (mean_b - mean_a) / mean_a.
+
+    Baseline of 0 renders ``inf`` for any positive improvement (handled by
+    the caller), and 0.0 for no change.
+    """
+    if not scores_a or not scores_b:
+        return 0.0
+    mean_a = sum(scores_a) / len(scores_a)
+    mean_b = sum(scores_b) / len(scores_b)
+    if mean_a == 0:
+        if mean_b > 0:
+            return float("inf")
+        return 0.0
+    return (mean_b - mean_a) / mean_a
+
+
+# ---------------------------------------------------------------------------
+# A/B test runner (#18)
+# ---------------------------------------------------------------------------
+
+
+def _resolve_ab_config(config: Config | None) -> ABTestConfig:
+    return config.ab_test if config is not None else ABTestConfig()
+
+
+def run_ab_test(
+    prompt_a: str,
+    prompt_b: str,
+    task_set: TaskSet,
+    llm: LLMProvider,
+    scorer: Scorer,
+    config: Config | None = None,
+) -> ABResult:
+    """Run the paired A/B comparison of ``prompt_b`` vs ``prompt_a``."""
+    ab_config = _resolve_ab_config(config)
+    tasks = task_set.list_tasks()
+    results: list[PerTask] = []
+    total_tokens = 0
+    failures = 0
+
+    for task in tasks:
+        result_a = run_task(task, prompt_a, llm)
+        result_b = run_task(task, prompt_b, llm)
+
+        score_a = (
+            scorer.score(task.expected_output, result_a.output)[1]
+            if not result_a.error
+            else 0.0
+        )
+        score_b = (
+            scorer.score(task.expected_output, result_b.output)[1]
+            if not result_b.error
+            else 0.0
+        )
+
+        if result_a.error or result_b.error:
+            failures += 1
+
+        total_tokens += result_a.token_count + result_b.token_count
+        results.append(
+            PerTask(
+                task_id=task.id,
+                task_input=task.input,
+                expected_output=task.expected_output,
+                output_a=result_a.output,
+                score_a=score_a,
+                output_b=result_b.output,
+                score_b=score_b,
+                delta=score_b - score_a,
+                latency_a_ms=result_a.latency_ms,
+                latency_b_ms=result_b.latency_ms,
+                tokens_a=result_a.token_count,
+                tokens_b=result_b.token_count,
+                error_a=result_a.error,
+                error_b=result_b.error,
+            )
+        )
+
+        # Cost ceiling abort (D3 §8.1)
+        if estimate_cost(total_tokens) > ab_config.cost_ceiling_usd:
+            return _inconclusive(results, total_tokens)
+
+    if tasks and failures / len(tasks) > 0.2:
+        return _inconclusive(results, total_tokens)
+
+    scores_a = [r.score_a for r in results]
+    scores_b = [r.score_b for r in results]
+    mean_a = sum(scores_a) / len(scores_a) if scores_a else 0.0
+    mean_b = sum(scores_b) / len(scores_b) if scores_b else 0.0
+
+    if not scores_a:
+        return _inconclusive([], 0)
+
+    deltas = [r.delta for r in results]
+    if all(d == 0.0 for d in deltas):
+        winner: Literal["a", "b", "tie", "inconclusive"] = "tie"
+        ci = BootstrapResult(mean=0.0, ci_low=0.0, ci_high=0.0, std=0.0)
+        p_value = 1.0
+    else:
+        ci = bootstrap_ci(
+            scores_a, scores_b,
+            n_resamples=ab_config.n_resamples,
+        )
+        p_value = permutation_test(
+            scores_a, scores_b,
+            n_permutations=ab_config.n_permutations,
+        )
+        effect = effect_size(scores_a, scores_b)
+        alpha = 1.0 - ab_config.confidence_level
+        if (
+            ci.ci_low > 0
+            and p_value < alpha
+            and effect >= ab_config.min_effect_size
+        ):
+            winner = "b"
+        elif ci.ci_high < 0 and p_value < alpha:
+            winner = "a"
+        else:
+            winner = "inconclusive"
+
+    return ABResult(
+        winner=winner,
+        mean_delta=mean_b - mean_a,
+        ci_low=ci.ci_low,
+        ci_high=ci.ci_high,
+        p_value=p_value,
+        effect_size=effect_size(scores_a, scores_b),
+        n_trials=len(results),
+        per_task=results,
+        cost_usd=estimate_cost(total_tokens),
+        token_count=total_tokens,
+    )
+
+
+def _inconclusive(
+    per_task: list[PerTask],
+    token_count: int,
+) -> ABResult:
+    return ABResult(
+        winner="inconclusive",
+        mean_delta=0.0,
+        ci_low=0.0,
+        ci_high=0.0,
+        p_value=1.0,
+        effect_size=0.0,
+        n_trials=len(per_task),
+        per_task=per_task,
+        cost_usd=estimate_cost(token_count),
+        token_count=token_count,
+    )

agent_self_edit/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""Trace adapter imports."""
+
+from .base import TraceAdapter
+from .file import FileAdapter
+from .stdin import StdinAdapter
+
+__all__ = ["TraceAdapter", "StdinAdapter", "FileAdapter"]

agent_self_edit/adapters/base.py

@@ -0,0 +1,27 @@
+"""Trace adapter interface."""
+
+from __future__ import annotations
+
+import threading
+from abc import ABC, abstractmethod
+
+from ..trace import TraceStore
+
+
+class TraceAdapter(ABC):
+    """Abstract adapter that ingests traces from a specific source."""
+
+    def __init__(self, store: TraceStore) -> None:
+        self._store = store
+        self._stop_event = threading.Event()
+
+    @abstractmethod
+    def run(self) -> None:
+        """Blocking run loop; calls ``store.ingest()`` per trace."""
+
+    def stop(self) -> None:
+        """Signal the run loop to exit."""
+        self._stop_event.set()
+
+    def _stopped(self) -> bool:
+        return self._stop_event.is_set()

agent_self_edit/adapters/file.py

@@ -0,0 +1,63 @@
+"""Directory-watching trace adapter."""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from pathlib import Path
+
+from ..trace import TraceStore
+from .base import TraceAdapter
+
+logger = logging.getLogger("agent_self_edit.adapters.file")
+
+
+class FileAdapter(TraceAdapter):
+    """Watches a directory for new ``.json`` trace files and ingests each.
+
+    Each file must be a JSON object (one trace per file). After successful
+    ingestion the file is moved to a ``.done`` sibling so it is not re-read.
+    Uses the ``.done`` rename as the sole dedup mechanism — no in-memory
+    filename cache, so repeated filenames are safe (ref #142).
+    """
+
+    def __init__(
+        self, store: TraceStore, watch_dir: str | Path, poll_interval: float = 1.0
+    ) -> None:
+        super().__init__(store)
+        self._watch_dir = Path(watch_dir)
+        self._watch_dir.mkdir(parents=True, exist_ok=True)
+        self._poll_interval = poll_interval
+
+    def run(self) -> None:
+        while not self._stopped():
+            self._process_once()
+            time.sleep(self._poll_interval)
+
+    def _process_once(self) -> int:
+        processed = 0
+        for path in sorted(self._watch_dir.glob("*.json")):
+            self._ingest_file(path)
+            processed += 1
+        return processed
+
+    def _ingest_file(self, path: Path) -> None:
+        try:
+            data = json.loads(path.read_text())
+        except (json.JSONDecodeError, OSError) as e:
+            logger.warning("Skipping malformed trace file %s: %s", path, e)
+            return
+        if not isinstance(data, dict):
+            logger.warning("Skipping trace file %s: expected a JSON object", path)
+            return
+        try:
+            self._store.ingest(data)
+        except ValueError as e:
+            logger.warning("Skipping invalid trace %s: %s", path, e)
+            return
+        try:
+            done = path.with_suffix(path.suffix + ".done")
+            path.rename(done)
+        except OSError as e:
+            logger.warning("Could not move %s to .done: %s", path, e)

agent_self_edit/adapters/stdin.py

@@ -0,0 +1,48 @@
+"""Stdin-aware trace adapter for JSON-lines input."""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import TextIO
+
+from ..trace import TraceStore
+from .base import TraceAdapter
+
+
+class StdinAdapter(TraceAdapter):
+    """Reads JSON-lines from stdin, one trace object per line, and ingests each."""
+
+    def __init__(
+        self, store: TraceStore, stream: TextIO | None = None, batch_size: int = 1000
+    ) -> None:
+        super().__init__(store)
+        self._stream = stream if stream is not None else sys.stdin
+        self._batch_size = batch_size
+        self._ingested = 0
+
+    def run(self) -> None:
+        for line in self._stream:
+            if self._stopped():
+                break
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                trace = json.loads(line)
+            except json.JSONDecodeError as e:
+                print(f"agent-self-edit: skipping malformed line: {e}", file=sys.stderr)
+                continue
+            try:
+                self._store.ingest(trace)
+                self._ingested += 1
+            except ValueError as e:
+                print(
+                    f"agent-self-edit: skipping invalid trace: {e}",
+                    file=sys.stderr,
+                )
+            if self._ingested % self._batch_size == 0:
+                print(
+                    f"agent-self-edit: ingested {self._ingested} traces",
+                    file=sys.stderr,
+                )