tuningfork-lab 0.6.0__py3-none-any.whl

tuningfork/__init__.py

@@ -0,0 +1,3 @@
+"""tuningfork — a step debugger and experiment lab for agent skills on local LLMs."""
+
+__version__ = "0.6.0"

tuningfork/__main__.py

@@ -0,0 +1,6 @@
+"""`python -m tuningfork`: the CLI, for a client config that names a Python, not a script."""
+
+from tuningfork.cli import app
+
+if __name__ == "__main__":
+    app(prog_name="tuningfork")

tuningfork/access.py

@@ -0,0 +1,129 @@
+"""The access token: what a request needs to change tuningfork's state.
+
+Reading stays open to the person at the machine. Anything that changes state — pausing,
+a breakpoint, a decision at a held call, and from Phase 5 starting work that launches
+harnesses — needs the token a running proxy was started with. A web page open in the same
+browser can send requests to a loopback port, but it cannot read the token: it is in the
+address `tuningfork serve` printed, in this browser tab's session, and in a file only this
+user can read.
+
+An MCP client that connects to the proxy (`/_tuningfork/mcp`) holds a token of its own, the
+MCP token, which opens that endpoint and nothing else: an agent offered tools there cannot
+use its client's token to call what the tools leave out, such as approving an edit.
+
+The token is not a sandbox for the harness. A launched harness runs shell commands as this
+user, and can read anything this user can; what the token keeps out is everything that is
+not this user at this machine.
+"""
+
+from __future__ import annotations
+
+import hmac
+import os
+import secrets
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Mapping
+
+#: Where a request carries the token.
+HEADER = "x-tuningfork-token"
+#: Pins the token a proxy starts with, and gives the CLI one without the file.
+ENV = "TUNINGFORK_TOKEN"
+#: The proxy's current token, in its recordings folder, readable by this user only.
+TOKEN_FILE = ".access-token"
+#: The MCP endpoint's own token, beside it: pinned by `MCP_ENV`, else new each start.
+MCP_ENV = "TUNINGFORK_MCP_TOKEN"
+MCP_TOKEN_FILE = ".mcp-token"
+#: The built-in proposer's API key, for an endpoint that needs one (`ops.proposer`); else
+#: the file `PROPOSER_KEY_FILE` in the recordings folder.
+PROPOSER_KEY_ENV = "TUNINGFORK_PROPOSER_KEY"
+PROPOSER_KEY_FILE = ".proposer-key"
+#: Never handed to anything tuningfork runs for a task: a check runs as this user, in a
+#: workspace a model wrote.
+SECRET_ENVS = (ENV, MCP_ENV, PROPOSER_KEY_ENV)
+
+
+def new_token() -> str:
+    return secrets.token_urlsafe(24)
+
+
+def token_path(home: Path | str) -> Path:
+    return Path(home) / TOKEN_FILE
+
+
+def mcp_token_path(home: Path | str) -> Path:
+    return Path(home) / MCP_TOKEN_FILE
+
+
+def write_token(home: Path | str, token: str) -> Path:
+    """Write the token where the CLI looks for it."""
+    return write_secret(token_path(home), token)
+
+
+def write_secret(path: Path | str, text: str) -> Path:
+    """Write a secret atomically, mode 0600 from the moment the file exists — a secret
+    briefly readable by others would be a secret given away."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp")
+    descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    try:
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(f"{text}\n")
+        os.chmod(temporary, 0o600)  # whatever the umask made of the mode above
+        os.replace(temporary, path)
+    except BaseException:
+        temporary.unlink(missing_ok=True)
+        raise
+    return path
+
+
+def read_token(path: Path | str) -> str | None:
+    """The token in a token file, or None when there is none to read."""
+    try:
+        text = Path(path).read_text(encoding="utf-8").strip()
+    except OSError:
+        return None
+    return text or None
+
+
+def matches(given: str | None, expected: str | None) -> bool:
+    """Compared in constant time: how long a refusal takes says nothing about the token."""
+    if not given or not expected:
+        return False
+    return hmac.compare_digest(given.encode("utf-8"), expected.encode("utf-8"))
+
+
+@dataclass(frozen=True, slots=True)
+class Access:
+    """How a server decides who may change its state."""
+
+    token: str
+    #: Where the token was written, for the CLI to find; reported by `/health`.
+    file: Path | None = None
+    #: The MCP endpoint's token: it opens `/_tuningfork/mcp`, and nothing else.
+    mcp_token: str | None = None
+    mcp_file: Path | None = None
+
+    @classmethod
+    def start(cls, home: Path | str, token: str | None = None, mcp_token: str | None = None) -> Access:
+        """A proxy's access: the tokens it was given, or new ones, written into its home."""
+        token = token or new_token()
+        mcp_token = mcp_token or new_token()
+        if matches(mcp_token, token):
+            raise ValueError(f"{MCP_ENV} must differ from {ENV}: the MCP token opens the MCP endpoint alone")
+        return cls(token=token, file=write_token(home, token), mcp_token=mcp_token,
+                   mcp_file=write_secret(mcp_token_path(home), mcp_token))
+
+    def allows(self, given: str | None) -> bool:
+        """Whether `given` is the access token, which every change but MCP needs."""
+        return matches(given, self.token)
+
+    def allows_mcp(self, given: str | None) -> bool:
+        """Whether `given` is the MCP token, which the MCP endpoint alone takes."""
+        return matches(given, self.mcp_token)
+
+    def status(self, headers: Mapping[str, Any]) -> dict[str, Any]:
+        """What `/status` says about the asking page's token: whether one is needed, and
+        whether the one it sent (if any) is it."""
+        return {"required": True, "valid": self.allows(headers.get(HEADER))}

tuningfork/analysis/__init__.py

@@ -0,0 +1,53 @@
+"""Outcome normalization, statistics, distributions, and divergence localization."""
+
+from tuningfork.analysis.distribution import (
+    ABComparison,
+    Cluster,
+    ClusterShift,
+    Distribution,
+    TargetTest,
+    compare_arms,
+    distribution,
+)
+from tuningfork.analysis.divergence import (
+    DivergenceReport,
+    OutcomeGroup,
+    StepComparison,
+    compare_runs,
+)
+from tuningfork.analysis.normalize import (
+    KNOWN_NORMALIZERS,
+    Outcome,
+    outcome_for,
+    step_outcome,
+)
+from tuningfork.analysis.stats import (
+    Rate,
+    fisher_exact,
+    rate,
+    samples_needed,
+    wilson_interval,
+)
+
+__all__ = [
+    "KNOWN_NORMALIZERS",
+    "ABComparison",
+    "Cluster",
+    "ClusterShift",
+    "Distribution",
+    "DivergenceReport",
+    "Outcome",
+    "OutcomeGroup",
+    "Rate",
+    "StepComparison",
+    "TargetTest",
+    "compare_arms",
+    "compare_runs",
+    "distribution",
+    "fisher_exact",
+    "outcome_for",
+    "rate",
+    "samples_needed",
+    "step_outcome",
+    "wilson_interval",
+]

tuningfork/analysis/conditions.py

@@ -0,0 +1,149 @@
+"""Pass rates by condition: runs of one task, under one harness, model, skill version and
+server.
+
+A rate pooled across wordings or models measures none of them. Once a task has been run
+before and after an edit — which is the whole workflow — its runs are two samples, not one,
+and each gets its own rate. So do runs answered by different servers: LM Studio and
+llama-server can both report the same model name for one GGUF, with chat templates that
+need not agree.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Iterable
+
+from tuningfork.analysis.stats import Rate, rate
+from tuningfork.store.db import Run
+
+
+def pass_rate(results: Iterable[dict[str, Any]]) -> Rate | None:
+    """The check pass rate over stored run results (`runner.RunOutcome.to_result`).
+
+    None when no run has a verdict. Runs the batch marked unusable are left out: a run
+    whose skill changed underneath it, that timed out, was edited at a breakpoint or whose
+    model calls failed describes neither wording.
+    """
+    verdicts = [
+        result["passed"]
+        for result in results
+        if result.get("usable") and result.get("passed") is not None
+    ]
+    return rate(sum(verdicts), len(verdicts)) if verdicts else None
+
+
+#: What `exclusion_from` says of a run that timed out.
+TIMED_OUT = "timed out"
+
+
+def exclusion_from(result: dict[str, Any] | None) -> str | None:
+    """Why a run with this stored result (`runner.RunOutcome.to_result`) is not counted, in
+    words; None when nothing in it says so."""
+    result = result or {}
+
+    def steps(key: str) -> str:
+        return ", ".join(map(str, result[key]))
+
+    if result.get("skills_changed"):
+        return f"skills changed: {', '.join(result['skills_changed'])}"
+    if result.get("timed_out"):
+        return TIMED_OUT
+    if result.get("edited_steps"):
+        return f"edited at a breakpoint (step {steps('edited_steps')})"
+    if result.get("failed_steps"):
+        return f"model call failed (step {steps('failed_steps')})"
+    if result.get("thread_trouble"):
+        return f"a call failed or was edited in {', '.join(result['thread_trouble'])}"
+    departed = result.get("departed")
+    if departed:
+        where = f"step {departed.get('step')}"
+        if departed.get("thread"):
+            where = f"thread {departed['thread']} {where}"
+        return f"the replay departed from the recording at {where}"
+    if result.get("leaked"):
+        return f"the original wording reached the model (step {steps('leaked')})"
+    if result.get("reached") is False:
+        return f"the harness never reached step {result.get('fork_step')}"
+    return None
+
+
+def fork_counted(passed: bool | None, exclusion: str | None) -> bool | None:
+    """How a fork run to the end counts in its arm: as its check said; failed when it timed
+    out, as the gate counts a timeout (`analysis.validation.COUNTING`) — an edit that makes the
+    model loop must not drop out of its own count; None, left out, when something else kept it
+    from measuring the wording, or no check judged it."""
+    if exclusion == TIMED_OUT:
+        return False
+    if exclusion is not None or passed is None:
+        return None
+    return bool(passed)
+
+
+def counted_verdict(result: dict[str, Any] | None) -> str | None:
+    """How one run fared, as `pass_rate` counts it: "passed" or "failed"; "not counted" for a
+    verdict every rate leaves out; None for a run with no verdict."""
+    result = result or {}
+    if result.get("passed") is None:
+        return None
+    if not result.get("usable"):
+        return "not counted"
+    return "passed" if result["passed"] else "failed"
+
+
+@dataclass(slots=True)
+class Condition:
+    task_id: str | None
+    harness: str | None
+    model: str | None
+    #: `Run.skill_version`: a fingerprint of the skill files the runs were given.
+    skill_version: str | None
+    #: The server the runs were answered by (`Store.server`); None for runs recorded before
+    #: servers were, which form a condition of their own.
+    server_id: str | None = None
+    runs: list[Run] = field(default_factory=list)
+
+    @property
+    def run_ids(self) -> list[str]:
+        return [run.id for run in self.runs]
+
+    @property
+    def skill_file(self) -> str | None:
+        """The file a batch installed as SKILL.md in place of the spec's own, if every run had
+        it. Runs given the same text another way share the fingerprint, and the condition: a
+        validation's original arm installs a copy of the skill's own file."""
+        files = {run.parity.get("skill_file") or None for run in self.runs}
+        return files.pop() if len(files) == 1 else None
+
+    @property
+    def passed(self) -> Rate | None:
+        return pass_rate(run.result for run in self.runs if run.result)
+
+    @property
+    def uncounted(self) -> int:
+        """Runs with a verdict that may not be counted: timed out, skill changed, edited."""
+        return sum(1 for run in self.runs if run.result and not run.result.get("usable"))
+
+    @property
+    def first(self) -> float:
+        return min(run.started_at for run in self.runs)
+
+    @property
+    def latest(self) -> float:
+        return max(run.started_at for run in self.runs)
+
+
+def conditions(runs: Iterable[Run]) -> list[Condition]:
+    """Runs grouped by task, harness, model, skill version and server, oldest first.
+
+    The skill version is the content the runs were given, not how it got there: a variant
+    installed with `batch --skill` and the same text later copied into SKILL.md are one
+    condition. A launch's subagent threads and an experiment's forks are not runs of the
+    task, and are expected to have been left out already (`Store.runs` does).
+    """
+    groups: dict[tuple[str | None, ...], Condition] = {}
+    for run in runs:
+        key = (run.task_id, run.harness, run.model, run.skill_version, run.server_id)
+        if key not in groups:
+            groups[key] = Condition(*key)
+        groups[key].runs.append(run)
+    return sorted(groups.values(), key=lambda condition: condition.first)