quantlab-factor-agent 0.1.0__py3-none-any.whl

factor_agent/ledger.py

@@ -0,0 +1,85 @@
+"""The trial ledger: one JSON line per candidate, accepted or not."""
+
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class TrialRecord:
+    """One candidate's outcome.
+
+    ``status`` is ``"accepted"``, ``"rejected"`` (refused by the reviewer,
+    unevaluated, or evaluated and refused by admission) or ``"failed"``
+    (never evaluated: refused by the parser or the Gate after every retry
+    or, for the rewrite of a candidate the reviewer sent back, once, or its
+    graph did not compile). Only an evaluated trial, one with
+    ``metrics``, counts toward the evaluated-trial count. ``stage`` is
+    where it ended: ``"parse"``, ``"gate"``, ``"review"``, ``"evaluate"``
+    or ``"admission"``. ``candidate`` is the last version the proposer
+    wrote, ``attempts`` every refused version before it (``item``,
+    ``stage``, ``reasons``), the one the reviewer sent back to be rewritten
+    with stage ``"review"`` and the reviewer's reason. ``direction`` is the planned direction the
+    candidate was written for (``mechanism_type``, ``action``, ``parent``,
+    ``direction``). ``stats`` holds the Gate's statistics, ``graph`` the
+    graph it built (``Graph.to_json``) and ``metrics`` the evaluation, once
+    there are any; ``confirmation`` the evaluation on the confirmation
+    segment, which no prompt shows, when the campaign has one.
+    """
+
+    round: int
+    candidate: dict
+    status: str
+    stage: str
+    reasons: list[str]
+    fingerprint: str | None = None
+    metrics: dict | None = None
+    stats: dict | None = None
+    attempts: list[dict] = field(default_factory=list)
+    direction: dict | None = None
+    graph: list | None = None
+    confirmation: dict | None = None
+
+    @property
+    def evaluated(self) -> bool:
+        """Whether the Evaluator scored the candidate: it counts toward the evaluated-trial count."""
+        return self.metrics is not None
+
+
+class Ledger:
+    """Append and read the trial records of a run directory."""
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+
+    def append(self, records: list[TrialRecord]) -> None:
+        """Append ``records``; the file and its directory are created when missing."""
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self.path.open("a") as out:
+            for record in records:
+                out.write(json.dumps(asdict(record)) + "\n")
+
+    def keep(self, rounds: int) -> None:
+        """Drop the records of every round after ``rounds``, such as a round a crash left half written."""
+        if not self.path.exists():
+            return
+        lines = [line for line in self.path.read_text().splitlines() if line]
+        kept = []
+        for line in lines:
+            try:
+                record = TrialRecord(**json.loads(line))
+            except (json.JSONDecodeError, TypeError):  # a line a crash cut short belongs to an unfinished round
+                continue
+            if record.round <= rounds:
+                kept.append(record)
+        if len(kept) == len(lines):
+            return
+        tmp = self.path.with_suffix(".tmp")
+        tmp.write_text("".join(json.dumps(asdict(r)) + "\n" for r in kept))
+        tmp.replace(self.path)
+
+    def records(self) -> list[TrialRecord]:
+        """Every record written so far, in order."""
+        if not self.path.exists():
+            return []
+        return [TrialRecord(**json.loads(line)) for line in self.path.read_text().splitlines() if line]

factor_agent/llm/_common.py

@@ -0,0 +1,18 @@
+"""What the provider adapters share: the key from its environment variable."""
+
+import os
+
+
+def key_from_environment(name: str) -> str:
+    """The value of the environment variable ``name``.
+
+    Raises
+    ------
+    ValueError
+        If the variable is unset or empty.
+    """
+    value = os.environ.get(name, "")
+    if not value:
+        raise ValueError(f"the environment variable {name} holding the API key is not set")
+    return value
+

factor_agent/llm/_raw.py

@@ -0,0 +1,78 @@
+"""What a provider's raw reply says beyond the reply's text and usage: cache hits, reasoning, why it stopped.
+
+Every backend keeps the provider's reply as it came (``Reply.raw``): an
+Anthropic message, an OpenAI-compatible chat completion (DeepSeek's among
+them) or the Claude Code CLI's result. ``raw_facts`` reads the few facts
+the events file and the frontend need from any of them, and nothing of
+the reply is read anywhere else. A raw reply that is empty, not JSON or of
+another shape gives no fact.
+"""
+
+import json
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RawFacts:
+    """What a provider's raw reply reports; ``None`` for anything it does not.
+
+    ``cache_hits`` are input tokens read from the provider's cache,
+    ``reasoning_tokens`` output tokens the model spent reasoning, and
+    ``reasoning`` the model's reasoning itself. ``truncated`` is true when
+    the reply was cut off at the output limit, false when it stopped for
+    another reason.
+    """
+
+    cache_hits: int | None = None
+    reasoning_tokens: int | None = None
+    truncated: bool | None = None
+    reasoning: str | None = None
+
+
+def raw_facts(raw: str) -> RawFacts:
+    """The facts of the provider's raw reply ``raw`` (see the module's notes)."""
+    try:
+        value = json.loads(raw) if raw else {}
+    except json.JSONDecodeError:
+        return RawFacts()
+    if not isinstance(value, dict):
+        return RawFacts()
+    usage = value.get("usage") if isinstance(value.get("usage"), dict) else {}
+    choices = value.get("choices")
+    choice = choices[0] if isinstance(choices, list) and choices and isinstance(choices[0], dict) else {}
+    stop = _first(value.get("stop_reason"), choice.get("finish_reason"))
+    return RawFacts(
+        cache_hits=_first(
+            usage.get("cache_read_input_tokens"),
+            _dig(usage, "prompt_tokens_details", "cached_tokens"),
+            usage.get("prompt_cache_hit_tokens"),
+        ),
+        reasoning_tokens=_dig(usage, "completion_tokens_details", "reasoning_tokens"),
+        truncated=None if stop is None else stop in ("max_tokens", "length"),
+        reasoning=_reasoning(value, choice),
+    )
+
+
+def _reasoning(value: dict, choice: dict) -> str | None:
+    """The model's reasoning: an OpenAI-compatible message's, or an Anthropic message's thinking blocks."""
+    if choice:
+        message = choice.get("message")
+        text = (message.get("reasoning_content") or message.get("reasoning")) if isinstance(message, dict) else None
+        return text if isinstance(text, str) and text else None
+    blocks = value.get("content")
+    if isinstance(blocks, list):
+        thinking = [b.get("thinking") for b in blocks if isinstance(b, dict) and b.get("type") == "thinking"]
+        return "\n\n".join(t for t in thinking if isinstance(t, str) and t) or None
+    return None
+
+
+def _first(*values):
+    """The first of ``values`` that is not ``None``."""
+    return next((v for v in values if v is not None), None)
+
+
+def _dig(value, *keys):
+    """``value[keys[0]][keys[1]]...``, ``None`` where a level is not a dict."""
+    for key in keys:
+        value = value.get(key) if isinstance(value, dict) else None
+    return value

factor_agent/llm/anthropic.py

@@ -0,0 +1,100 @@
+"""The Anthropic API backend: the Messages API through the ``anthropic`` SDK, its key from ``ANTHROPIC_API_KEY``."""
+
+import anthropic
+
+from factor_agent.llm._common import key_from_environment
+from factor_agent.llm.base import ProviderQuotaExhausted, ProviderReplyMalformed, Reply, Request
+
+
+class AnthropicBackend:
+    """Complete a request with one Messages API call.
+
+    Parameters
+    ----------
+    model : str
+        The model of every call.
+    max_tokens : int
+        The ``max_tokens`` of every call.
+    effort : str, optional
+        The ``output_config.effort`` of every call; the model's default if None.
+    base_url : str, optional
+        Another endpoint than Anthropic's.
+    timeout_seconds : float
+        Seconds a call may take.
+    """
+
+    def __init__(
+        self, model: str, max_tokens: int = 16000, effort: str | None = None,
+        base_url: str | None = None, timeout_seconds: float = 600.0,
+    ):
+        self.model = model
+        self.max_tokens = max_tokens
+        self.effort = effort
+        self._client = anthropic.Anthropic(
+            api_key=key_from_environment("ANTHROPIC_API_KEY"), base_url=base_url, timeout=timeout_seconds,
+        )
+
+    @classmethod
+    def from_config(cls, spec: dict) -> "AnthropicBackend":
+        """Build from a role's config: ``model`` and optionally ``max_tokens``, ``effort``, ``base_url``, ``timeout_seconds``."""
+        options = {k: spec[k] for k in ("max_tokens", "effort", "base_url", "timeout_seconds") if k in spec}
+        return cls(spec["model"], **options)
+
+    @property
+    def params(self) -> dict:
+        """The parameters every call sends with the prompt."""
+        return {"max_tokens": self.max_tokens, **({"effort": self.effort} if self.effort else {})}
+
+    def complete(self, request: Request) -> Reply:
+        """Send ``request``'s prompt as one user message.
+
+        Raises
+        ------
+        ProviderQuotaExhausted
+            If Anthropic refuses the call for billing: an error of type
+            ``billing_error`` (HTTP 402), or the request error saying the
+            credit balance is too low.
+        anthropic.APIError
+            If the call fails otherwise, after the SDK's retries.
+        ProviderReplyMalformed
+            If Anthropic answers something that is not a message: no body
+            (``null``), no content or no token usage.
+        """
+        extra = {"output_config": {"effort": self.effort}} if self.effort else {}
+        try:
+            message = self._client.messages.create(
+                model=self.model, max_tokens=self.max_tokens,
+                messages=[{"role": "user", "content": request.prompt}], **extra,
+            )
+        except anthropic.APIStatusError as exc:
+            if _is_billing(exc):
+                raise ProviderQuotaExhausted(f"anthropic: {self.model}: {exc.message}") from exc
+            raise
+        lacking = _lacking(message)
+        if lacking:
+            raise ProviderReplyMalformed(f"anthropic: {self.model} answered a message without {lacking}")
+        usage = message.usage
+        input_tokens = usage.input_tokens + (usage.cache_creation_input_tokens or 0) + (usage.cache_read_input_tokens or 0)
+        return Reply(
+            text="".join(block.text for block in message.content if block.type == "text"),
+            usage={"input_tokens": input_tokens, "output_tokens": usage.output_tokens},
+            model_id=message.model, params=self.params, raw=message.to_json(indent=None),
+        )
+
+
+def _is_billing(exc: "anthropic.APIStatusError") -> bool:
+    """Whether Anthropic refused the call for billing: a ``billing_error``, or a request error on the credit balance."""
+    error = exc.body.get("error", {}) if isinstance(exc.body, dict) else {}
+    kind, message = error.get("type"), str(error.get("message", ""))
+    return kind == "billing_error" or (kind == "invalid_request_error" and "credit balance" in message.lower())
+
+
+def _lacking(message) -> str | None:
+    """What ``message`` lacks to be a reply (its body, the content, the usage), or None."""
+    if message is None:
+        return "a body"
+    if getattr(message, "content", None) is None:
+        return "content"
+    if getattr(message, "usage", None) is None:
+        return "token usage"
+    return None

factor_agent/llm/base.py

@@ -0,0 +1,124 @@
+"""The one external seam: an LLM backend turns a request into a reply."""
+
+import time
+from dataclasses import dataclass, field
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class Request:
+    """One call: the role making it, the model, the prompt and the sampling parameters."""
+
+    role: str
+    model: str
+    prompt: str
+    params: dict = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class Reply:
+    """The provider's answer: its text, token usage as the provider reports it and the model that answered.
+
+    ``params`` are the parameters the backend sent with the prompt (such as
+    ``max_tokens``), and ``raw`` is the provider's reply as it came, before
+    the text was taken out of it.
+    """
+
+    text: str
+    usage: dict
+    model_id: str
+    params: dict = field(default_factory=dict)
+    raw: str = ""
+
+
+class ProviderQuotaExhausted(RuntimeError):
+    """The provider refuses calls because the balance or quota is exhausted; its message says so in its words.
+
+    Each backend recognises its own provider's refusal. A campaign stops on
+    it cleanly, dropping the round in progress, and resumes once topped up.
+    """
+
+
+class ProviderReplyMalformed(RuntimeError):
+    """The provider answered, without an error, something that is not a reply; its message says what it lacked.
+
+    Such as a body of ``null`` (DeepSeek, after the blank lines it sends
+    while a call waits), a reply without its text or its usage, or output
+    that is not the JSON it should be. The SDKs do not retry it, as the
+    answer came back as a success; a campaign asks again, a few times
+    (``AskingAgain``), before raising it.
+    """
+
+
+class LLMBackend(Protocol):
+    """Anything that completes a request."""
+
+    def complete(self, request: Request) -> Reply: ...
+
+
+#: Seconds waited before each call asked again after an answer that was not a reply: two calls again, then it raises.
+ASK_AGAIN_WAITS = (5.0, 30.0)
+
+
+class AskingAgain:
+    """A backend that asks ``backend`` again when it answers something that is not a reply.
+
+    A campaign calls every role's backend through one, its own backends and
+    those passed in alike.
+
+    ``ProviderReplyMalformed`` is retried after each wait of
+    ``ASK_AGAIN_WAITS`` and raised once they are spent; every other error
+    is raised at once.
+    """
+
+    def __init__(self, backend: LLMBackend):
+        self.backend = backend
+
+    def complete(self, request: Request) -> Reply:
+        """Complete ``request`` with ``backend``, asking again after an answer that was not a reply."""
+        for wait in ASK_AGAIN_WAITS:
+            try:
+                return self.backend.complete(request)
+            except ProviderReplyMalformed:
+                time.sleep(wait)
+        return self.backend.complete(request)
+
+
+def make_backend(spec: dict) -> LLMBackend:
+    """Build the backend a role's config names in ``spec["backend"]``.
+
+    Raises
+    ------
+    ValueError
+        If the backend name is unknown, or the config holds a key: keys come
+        only from environment variables (``api_key_env`` names one).
+    """
+    from factor_agent.llm.anthropic import AnthropicBackend
+    from factor_agent.llm.claude_code import ClaudeCodeBackend
+    from factor_agent.llm.openai_compatible import OpenAICompatibleBackend
+    from factor_agent.llm.replay import ReplayBackend
+    from factor_agent.llm.scripted import ScriptedBackend
+
+    backends = {
+        "scripted": ScriptedBackend.from_config, "replay": ReplayBackend.from_config,
+        "claude_code": ClaudeCodeBackend.from_config, "anthropic": AnthropicBackend.from_config,
+        "openai_compatible": OpenAICompatibleBackend.from_config,
+    }
+    name = spec.get("backend")
+    if name not in backends:
+        raise ValueError(f"unknown LLM backend {name!r}; known: {sorted(backends)}")
+    secrets = [option for option in spec if _is_secret(option)]
+    if secrets:
+        raise ValueError(
+            f"LLM backend {name!r}: {secrets} look like keys; a key is read only from an environment "
+            f"variable (name it with api_key_env), never from the config"
+        )
+    return backends[name](spec)
+
+
+def _is_secret(option: str) -> bool:
+    """Whether a config option's name says it holds a credential."""
+    name = option.lower()
+    if name == "api_key_env":
+        return False
+    return name in {"key", "token", "secret", "password"} or name.endswith(("_key", "_token", "_secret", "_password"))

factor_agent/llm/claude_code.py

@@ -0,0 +1,101 @@
+"""The Claude Code backend: ``claude -p`` headless, JSON output, every tool off, the local login.
+
+Every built-in tool, MCP server, skill, slash command and settings file is
+turned off and the call runs from an empty directory, so the model only
+writes text and cannot read the data. ``ANTHROPIC_API_KEY`` and
+``ANTHROPIC_AUTH_TOKEN`` are kept out of the child's environment, so the
+call goes through the Claude Code login. Managed (enterprise) settings
+cannot be turned off from the command line.
+"""
+
+import json
+import os
+import subprocess
+import tempfile
+
+from factor_agent.llm.base import ProviderReplyMalformed, Reply, Request
+
+
+class ClaudeCodeBackend:
+    """Complete a request with one headless Claude Code call.
+
+    Parameters
+    ----------
+    model : str
+        The ``--model`` of every call.
+    effort : str, optional
+        The ``--effort`` of every call; Claude Code's default if None.
+    executable : str
+        The Claude Code command.
+    timeout_seconds : float
+        Seconds a call may take.
+    """
+
+    def __init__(
+        self, model: str, effort: str | None = None, executable: str = "claude", timeout_seconds: float = 900.0,
+    ):
+        self.model = model
+        self.effort = effort
+        self.executable = executable
+        self.timeout_seconds = timeout_seconds
+
+    @classmethod
+    def from_config(cls, spec: dict) -> "ClaudeCodeBackend":
+        """Build from a role's config: ``model`` and optionally ``effort``, ``executable`` and ``timeout_seconds``."""
+        options = {k: spec[k] for k in ("effort", "executable", "timeout_seconds") if k in spec}
+        return cls(spec["model"], **options)
+
+    def command(self) -> list[str]:
+        """The command line of every call; the prompt goes on stdin."""
+        command = [
+            self.executable, "-p", "--output-format", "json", "--model", self.model,
+            "--tools", "", "--strict-mcp-config", "--mcp-config", json.dumps({"mcpServers": {}}),
+            "--disable-slash-commands", "--setting-sources", "", "--no-session-persistence",
+        ]
+        if self.effort:
+            command += ["--effort", self.effort]
+        return command
+
+    def complete(self, request: Request) -> Reply:
+        """Run the call and read its JSON reply.
+
+        Raises
+        ------
+        RuntimeError
+            If Claude Code exits with an error or reports one.
+        subprocess.TimeoutExpired
+            If the call takes longer than ``timeout_seconds``.
+        ProviderReplyMalformed
+            If Claude Code succeeds but writes no JSON object with a ``result``.
+        """
+        env = {k: v for k, v in os.environ.items() if k not in _CREDENTIALS}
+        with tempfile.TemporaryDirectory(prefix="factor_agent_claude_") as empty:
+            finished = subprocess.run(
+                self.command(), input=request.prompt, capture_output=True, text=True, cwd=empty, env=env,
+                timeout=self.timeout_seconds,
+            )
+        if finished.returncode != 0:
+            raise RuntimeError(f"claude_code: exit {finished.returncode}: {(finished.stderr or finished.stdout)[-2000:]}")
+        try:
+            reply = json.loads(finished.stdout)
+        except json.JSONDecodeError:
+            reply = None
+        if not isinstance(reply, dict):
+            raise ProviderReplyMalformed(f"claude_code: the output is not a JSON object: {finished.stdout[-2000:]!r}")
+        if reply.get("is_error"):
+            raise RuntimeError(f"claude_code: the call failed: {str(reply.get('result'))[:2000]}")
+        if not isinstance(reply.get("result"), str):
+            raise ProviderReplyMalformed(f"claude_code: the reply has no result: {finished.stdout[-2000:]!r}")
+        usage = reply.get("usage") or {}
+        models = reply.get("modelUsage") or {}
+        return Reply(
+            text=reply["result"],
+            usage={k: usage[k] for k in ("input_tokens", "output_tokens") if k in usage},
+            model_id=self.model if self.model in models else next(iter(models), self.model),
+            params={"effort": self.effort} if self.effort else {},
+            raw=finished.stdout.strip(),
+        )
+
+
+#: Environment variables kept from the child, so it answers through the Claude Code login.
+_CREDENTIALS = ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")

factor_agent/llm/log.py

@@ -0,0 +1,74 @@
+"""The LLM log: every call of a run, in full, one JSON file per call under ``llm_log/``.
+
+A file ``<number>-<role>.json`` holds the request (role, model, prompt and
+the parameters sent) and the reply (text, usage, the model that
+answered and the provider's raw reply); the numbers count the calls of the
+run from 1, across every role, in the order they finish (calls sent side by
+side are written one at a time); the calls of a round a resume dropped are
+kept in ``dropped/``, out of the sequence. No key is ever part of a request
+or a reply, so none is written.
+"""
+
+import json
+import threading
+from dataclasses import asdict
+from pathlib import Path
+
+from factor_agent.llm.base import LLMBackend, Reply, Request
+
+
+class LLMLog:
+    """The call files of one run directory's ``llm_log``."""
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self._writing = threading.Lock()
+
+    def files(self) -> list[Path]:
+        """The call files, in call order."""
+        return sorted(self.path.glob("*.json"))
+
+    def drop_after(self, count: int) -> None:
+        """Move every call after the first ``count`` to ``dropped/``: calls of a round a crash cut off."""
+        for file in self.files()[count:]:
+            (self.path / "dropped").mkdir(exist_ok=True)
+            file.replace(self.path / "dropped" / file.name)
+
+    def calls(self) -> list[dict]:
+        """Every recorded call, in call order, as ``{"request": ..., "reply": ...}``."""
+        return [json.loads(file.read_text()) for file in self.files()]
+
+    def write(self, request: Request, reply: Reply) -> Path:
+        """Write the next call file and return its path; the file appears whole or not at all.
+
+        Safe to call from several threads: one call is written at a time, each under the next number.
+        """
+        call = {
+            "request": {**asdict(request), "params": {**request.params, **reply.params}},
+            "reply": {k: v for k, v in asdict(reply).items() if k != "params"},
+        }
+        with self._writing:
+            self.path.mkdir(parents=True, exist_ok=True)
+            number = len(self.files()) + 1
+            path = self.path / f"{number:06d}-{request.role}.json"
+            tmp = path.with_suffix(".tmp")
+            tmp.write_text(json.dumps(call, indent=2, ensure_ascii=False) + "\n")
+            tmp.replace(path)
+        return path
+
+
+class RecordingBackend:
+    """A backend that writes every call it passes on to ``log``."""
+
+    def __init__(self, backend: LLMBackend, log: LLMLog):
+        self.backend = backend
+        self.log = log
+
+    def complete(self, request: Request) -> Reply:
+        """Pass ``request`` on and write the call once its reply is back."""
+        return self.record(request)[0]
+
+    def record(self, request: Request) -> tuple[Reply, Path]:
+        """Pass ``request`` on, write the call once its reply is back, and return the reply and the call file."""
+        reply = self.backend.complete(request)
+        return reply, self.log.write(request, reply)