rlmness-cli 0.2.0__py3-none-any.whl

rlmagent/refinement/evidence.py

@@ -0,0 +1,131 @@
+"""What is known about a saved change: that it was saved, reached the model, and what followed."""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+from pathlib import Path
+
+#: How many tasks either side of a change are compared.
+WINDOW = 20
+VERDICTS = ("better", "same", "worse")
+#: A task with this many failed tool calls is worth a look for something to keep.
+TROUBLE = 3
+
+
+class EvidenceLog:
+    """An append-only record per project. Nothing here changes what the agent does."""
+
+    def __init__(self, path: Path | None):
+        self.path = path
+        self.rows: list[dict] = []
+        self.last_failed = 0
+        self._seen = {"transcript": 0, "events": 0}
+        if path is not None:
+            try:
+                for line in Path(path).read_text(encoding="utf-8").splitlines():
+                    try:
+                        self.rows.append(json.loads(line))
+                    except ValueError:
+                        continue
+            except OSError:
+                pass
+
+    def _add(self, row: dict) -> None:
+        row = {"time": time.time(), **row}
+        self.rows.append(row)
+        if self.path is None:
+            return
+        try:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            with self.path.open("a", encoding="utf-8") as stream:
+                stream.write(json.dumps(row, ensure_ascii=False) + "\n")
+        except OSError:
+            pass  # a record about the run must never stop the run
+
+    def saved(self, change: str, source: str, entries: list[str], summary: str = "") -> None:
+        self._add({"type": "saved", "change": change, "source": source,
+                   "entries": entries, "summary": summary})
+
+    def rate(self, change: str, verdict: str) -> str:
+        if verdict not in VERDICTS:
+            return f"A verdict is one of {', '.join(VERDICTS)}."
+        if not any(r["type"] == "saved" and r["change"] == change for r in self.rows):
+            return f"No saved change {change}."
+        self._add({"type": "verdict", "change": change, "verdict": verdict})
+        return f"Change {change} rated {verdict}."
+
+    def task_finished(self, session) -> None:
+        """One finished task: which saved entries the model could read, and how the tools went."""
+        calls, failed = self._tool_counts(session)
+        self.last_failed = failed
+        self._add({"type": "task", "calls": calls, "failed": failed,
+                   "reached": sorted(self._reached(session))})
+
+    def _tool_counts(self, session) -> tuple[int, int]:
+        calls = failed = 0
+        transcript = list(session.transcript)
+        for message in transcript[self._seen["transcript"]:]:
+            if hasattr(message, "is_error") and hasattr(message, "tool_call_id"):
+                calls += 1
+                failed += bool(message.is_error)
+        self._seen["transcript"] = len(transcript)
+        events = list(getattr(getattr(session, "backend", None), "events", None) or [])
+        for event in events[self._seen["events"]:]:
+            if event.get("kind") == "host_result":
+                calls += 1
+                failed += str(event.get("result", "")).startswith("Error:")
+            elif event.get("kind") == "step" and event.get("step", 0) > 0 and event.get("code"):
+                calls += 1
+                failed += bool(event.get("error"))
+        self._seen["events"] = len(events)
+        return calls, failed
+
+    def _reached(self, session) -> set[str]:
+        """Entries actually in what the model is sent, read back rather than assumed."""
+        names = {name for r in self.rows if r["type"] == "saved" for name in r["entries"]}
+        if not names:
+            return set()
+        backend = getattr(session, "backend", None)
+        sent = [getattr(session, "_system", "") or "", getattr(backend, "instructions", "") or ""]
+        sent += [getattr(m, "text", "") or "" for m in session.transcript]
+        text = "\n".join(sent)
+        return {
+            name for name in names
+            if re.search(rf"(?:\] |Workflow |prompt |subagent ){re.escape(name)}\b", text)
+        }
+
+    def report(self) -> str:
+        changes = [(n, r) for n, r in enumerate(self.rows) if r["type"] == "saved"]
+        if not changes:
+            return "No saved changes have been recorded for this project."
+        # By position: clocks can give two neighbouring rows the same time.
+        tasks = [(n, r) for n, r in enumerate(self.rows) if r["type"] == "task"]
+        lines = []
+        for position, change in changes[-10:]:
+            before = [t for n, t in tasks if n < position][-WINDOW:]
+            later = [t for n, t in tasks if n > position]
+            after = later[:WINDOW]
+            names = set(change["entries"])
+            reached = sum(1 for t in later if names & set(t["reached"]))
+            verdicts = [r["verdict"] for r in self.rows
+                        if r["type"] == "verdict" and r["change"] == change["change"]]
+            lines += [
+                f"{change['change']} [{change['source']}] {', '.join(change['entries'])}",
+                "  saved: " + time.strftime("%Y-%m-%d %H:%M", time.localtime(change["time"])),
+                f"  reached the model: in {reached} of {len(later)} later tasks",
+                f"  failed tool calls per task: {_rate(before)} before, {_rate(after)} after",
+                f"  your verdict: {verdicts[-1] if verdicts else 'none yet'}",
+            ]
+        lines.append(
+            "These counts are signals, not proof: tasks differ, and nothing here was a "
+            "controlled test. Record a judgement with /refine rate <change> better|same|worse."
+        )
+        return "\n".join(lines)
+
+
+def _rate(tasks: list[dict]) -> str:
+    if not tasks:
+        return "no data"
+    return f"{sum(t['failed'] for t in tasks) / len(tasks):.2f} over {len(tasks)} tasks"

rlmagent/refinement/scoped.py

@@ -0,0 +1,90 @@
+"""Branch-local snapshots inherited along the saved conversation ancestry."""
+
+from __future__ import annotations
+
+import json
+from uuid import uuid4
+
+from rlmagent.harness.session.store import deserialize_records
+from rlmagent.refinement.store import HarnessStore, Snapshot
+
+
+class ScopedStore(HarnessStore):
+    def __init__(self, path, session, inherited: Snapshot):
+        super().__init__(path)
+        self.session = session
+        self.initial = inherited.model_copy(deep=True)
+        self._document = {"base": self.initial.model_dump(), "anchors": {}}
+
+    def _data(self):
+        if self.path is None:
+            return self._document
+        try:
+            return json.loads(self.path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            return self._document
+
+    def _save(self, data):
+        if self.path is None:
+            self._document = data
+            return
+        temporary = self.path.with_suffix(f".{uuid4().hex}.tmp")
+        try:
+            temporary.write_text(json.dumps(data), encoding="utf-8")
+            temporary.replace(self.path)
+        finally:
+            temporary.unlink(missing_ok=True)
+
+    def _lineage(self):
+        tip = self.session._tip_id
+        vault = self.session._vault
+        parents = {}
+        if vault is not None and vault.path.exists():
+            records = deserialize_records(vault.path.read_text(encoding="utf-8").split("\n"))
+            parents = {record.id: record.parent_id for record in records}
+        seen = set()
+        while tip is not None and tip not in seen:
+            seen.add(tip)
+            yield tip
+            tip = parents.get(tip)
+        yield "base"
+
+    def _read(self):
+        data = self._data()
+        for anchor in self._lineage():
+            versions = data["anchors"].get(anchor, [])
+            if versions:
+                return Snapshot.model_validate(versions[-1])
+        return Snapshot.model_validate(data["base"])
+
+    def initialize(self):
+        with self.transaction():
+            self._save(self._data())
+
+    def commit(self, baseline: Snapshot, candidate: Snapshot):
+        candidate = Snapshot.model_validate(candidate.model_dump())
+        with self.transaction():
+            if self._read() != baseline:
+                raise ValueError("Branch guidance changed during review; start a fresh review")
+            if candidate.revision != baseline.revision + 1:
+                raise ValueError("Candidate must advance the harness revision once")
+            data = self._data()
+            anchor = self.session._tip_id or "base"
+            history = data["anchors"].setdefault(anchor, [])
+            if not history:
+                history.append(baseline.model_dump())
+            history.append(candidate.model_dump())
+            self._save(data)
+        return candidate.model_copy(deep=True)
+
+    def version(self, revision: int):
+        with self.transaction():
+            data = self._data()
+            for anchor in self._lineage():
+                for snapshot in reversed(data["anchors"].get(anchor, [])):
+                    if snapshot["revision"] == revision:
+                        return Snapshot.model_validate(snapshot)
+            baseline = Snapshot.model_validate(data["base"])
+            if baseline.revision == revision:
+                return baseline
+        raise ValueError("No saved snapshot for this revision in the current branch")

rlmagent/refinement/store.py

@@ -0,0 +1,193 @@
+"""Validated harness snapshots and compare-and-swap persistence."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import tempfile
+import threading
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+MAX_CONTENT = 32 * 1024
+MAX_ENTRIES = 128
+_LOCKS: dict[str, threading.RLock] = {}
+_LOCK_GUARD = threading.Lock()
+
+
+class Record(BaseModel):
+    model_config = ConfigDict(extra="forbid", strict=True)
+
+
+class Guidance(Record):
+    id: str = Field(pattern=r"^[a-z][a-z0-9-]{0,63}$")
+    kind: Literal["prompt", "skill", "subagent"]
+    title: str = Field(min_length=1, max_length=160)
+    content: str = Field(min_length=1)
+
+    @model_validator(mode="after")
+    def check_content(self):
+        if not self.title.strip() or not self.content.strip():
+            raise ValueError("Guidance needs a title and nonblank content")
+        if len(self.content.encode("utf-8")) > MAX_CONTENT:
+            raise ValueError("Guidance exceeds the byte limit")
+        if self.kind == "skill":
+            skill_parts(self.id, self.content)
+        return self
+
+
+def skill_parts(name: str, content: str) -> tuple[str, str]:
+    match = re.fullmatch(r"---\r?\n(.*?)\r?\n---\r?\n(.*)", content, re.S)
+    if match is None:
+        raise ValueError("A workflow must begin with YAML frontmatter")
+    # Reject duplicate metadata keys rather than silently choosing the last value.
+    try:
+        node = yaml.compose(match[1], Loader=yaml.SafeLoader)
+        meta = yaml.safe_load(match[1])
+    except yaml.YAMLError as error:
+        raise ValueError("Invalid workflow metadata") from error
+    if not isinstance(node, yaml.MappingNode):
+        raise ValueError("Workflow metadata must be a mapping")
+    keys = [key.value for key, _ in node.value]
+    if sorted(keys) != ["description", "name"]:
+        raise ValueError("Workflow metadata needs exactly name and description")
+    if meta["name"] != name or not isinstance(meta["description"], str):
+        raise ValueError("Workflow name must match its id; description must be text")
+    if not meta["description"].strip() or not match[2].strip():
+        raise ValueError("Workflow description and procedure must not be blank")
+    return meta["description"], match[2]
+
+
+class ChangeNote(Record):
+    id: str
+    summary: str = Field(min_length=1, max_length=2000)
+    evidence: str = Field(min_length=1, max_length=8000)
+    expected: str = Field(min_length=1, max_length=2000)
+    edits: list[dict]
+
+
+class Snapshot(Record):
+    format: Literal["rlmagent.harness/1"] = "rlmagent.harness/1"
+    revision: int = Field(default=0, ge=0)
+    entries: dict[str, Guidance] = Field(default_factory=dict)
+    history: list[ChangeNote] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def check_ids(self):
+        if len(self.entries) > MAX_ENTRIES:
+            raise ValueError("Too many guidance entries")
+        if any(key != entry.id for key, entry in self.entries.items()):
+            raise ValueError("Stored guidance id does not match its key")
+        return self
+
+
+@contextmanager
+def _lock(path: Path):
+    key = str(path.resolve())
+    with _LOCK_GUARD:
+        mutex = _LOCKS.setdefault(key, threading.RLock())
+    with mutex:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open("a+b") as handle:
+            handle.seek(0, 2)
+            if handle.tell() == 0:
+                handle.write(b"0")
+                handle.flush()
+            handle.seek(0)
+            if os.name == "nt":
+                import msvcrt
+
+                msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
+            else:
+                import fcntl
+
+                fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
+            try:
+                yield
+            finally:
+                handle.seek(0)
+                if os.name == "nt":
+                    msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+                else:
+                    fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+
+
+class HarnessStore:
+    def __init__(self, path: Path | None):
+        self.path = path
+        self._memory = Snapshot()
+        self._mutex = threading.RLock()
+        self._versions: dict[int, Snapshot] = {}
+
+    def _read(self) -> Snapshot:
+        if self.path is None:
+            return self._memory.model_copy(deep=True)
+        try:
+            raw = self.path.read_text(encoding="utf-8")
+        except FileNotFoundError:
+            return Snapshot()
+        return Snapshot.model_validate(json.loads(raw))
+
+    @contextmanager
+    def transaction(self):
+        with self._mutex:
+            if self.path is None:
+                yield
+            else:
+                with _lock(self.path.with_suffix(".lock")):
+                    yield
+
+    def read(self) -> Snapshot:
+        with self.transaction():
+            return self._read()
+
+    def commit(self, baseline: Snapshot, candidate: Snapshot) -> Snapshot:
+        candidate = Snapshot.model_validate(candidate.model_dump())
+        with self.transaction():
+            if self._read() != baseline:
+                raise ValueError("Harness changed during refinement; start a fresh review")
+            if candidate.revision != baseline.revision + 1:
+                raise ValueError("Candidate must advance the harness revision once")
+            if self.path is None:
+                self._versions[baseline.revision] = baseline.model_copy(deep=True)
+                self._memory = candidate.model_copy(deep=True)
+            else:
+                versions = self.path.parent / "snapshots"
+                versions.mkdir(exist_ok=True)
+                saved = versions / f"{baseline.revision}.json"
+                if not saved.exists():
+                    with saved.open("x", encoding="utf-8") as output:
+                        output.write(baseline.model_dump_json(indent=2))
+                descriptor, name = tempfile.mkstemp(prefix=".harness-", dir=self.path.parent)
+                temporary = Path(name)
+                try:
+                    with os.fdopen(descriptor, "w", encoding="utf-8") as output:
+                        output.write(candidate.model_dump_json(indent=2) + "\n")
+                        output.flush()
+                        os.fsync(output.fileno())
+                    os.replace(temporary, self.path)
+                finally:
+                    temporary.unlink(missing_ok=True)
+            return candidate.model_copy(deep=True)
+
+    def version(self, revision: int) -> Snapshot:
+        if revision < 0:
+            raise ValueError("revision must be nonnegative")
+        current = self.read()
+        if current.revision == revision:
+            return current
+        if self.path is None:
+            if revision not in self._versions:
+                raise ValueError("no saved snapshot for this revision")
+            return self._versions[revision].model_copy(deep=True)
+        try:
+            return Snapshot.model_validate_json(
+                (self.path.parent / "snapshots" / f"{revision}.json").read_text(encoding="utf-8")
+            )
+        except FileNotFoundError:
+            raise ValueError("no saved snapshot for this revision") from None

rlmagent/refresh.py

@@ -0,0 +1,202 @@
+"""Reload diagnostics: structured JSONL logging for hot-reload operations."""
+
+from __future__ import annotations
+
+import json
+import re
+import traceback
+import uuid
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+# ── constants ──────────────────────────────────────────────────────────────
+
+_LOG_FILENAME = "reload_diag.jsonl"
+_MAX_ERROR_LENGTH = 300
+_MAX_TRACEBACK_LINES = 20
+
+# Patterns that look like secrets — replaced with [REDACTED] before logging.
+_SECRET_RE = re.compile(
+    r"(sk-[A-Za-z0-9]{8,})"          # Anthropic / OpenAI API keys
+    r"|(key-[A-Za-z0-9]{8,})"        # generic key- prefixed tokens
+    r"|(Bearer\s+\S+)"               # Authorization headers
+    r"|(token[=:]\s*\S+)"            # token= or token: values
+    r"|(password[=:]\s*\S+)"         # password= or password: values
+    r"|(secret[=:]\s*\S+)",          # secret= or secret: values
+    re.IGNORECASE,
+)
+
+
+# ── scrubbing ──────────────────────────────────────────────────────────────
+
+
+def _scrub(text: str) -> str:
+    """Replace substrings that look like secrets with ``[REDACTED]``."""
+    return _SECRET_RE.sub("[REDACTED]", text)
+
+
+def _scrub_error(message: str) -> str:
+    """Scrub and truncate an error message for safe logging."""
+    scrubbed = _scrub(message)
+    if len(scrubbed) > _MAX_ERROR_LENGTH:
+        scrubbed = scrubbed[:_MAX_ERROR_LENGTH] + "…"
+    return scrubbed
+
+
+def _scrub_traceback(exc: BaseException) -> list[str]:
+    """Format an exception traceback and scrub each line of secrets."""
+    raw = traceback.format_exception(type(exc), exc, exc.__traceback__)
+    flat: list[str] = []
+    for chunk in raw:
+        flat.extend(chunk.rstrip().splitlines())
+
+    scrubbed = [_scrub(line) for line in flat]
+    if len(scrubbed) > _MAX_TRACEBACK_LINES:
+        overflow = len(flat) - _MAX_TRACEBACK_LINES
+        scrubbed = scrubbed[:_MAX_TRACEBACK_LINES] + [f"… ({overflow} more lines)"]
+    return scrubbed
+
+
+# ── timestamps & run IDs ──────────────────────────────────────────────────
+
+
+def _utc_now() -> datetime:
+    """Return the current UTC time (timezone-aware)."""
+    return datetime.now(UTC)
+
+
+def _iso_utc(dt: datetime) -> str:
+    """Format *dt* as a compact ISO-8601 UTC string."""
+    return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def new_run_id() -> str:
+    """Generate a unique 12-character hex identifier for a reload run."""
+    return uuid.uuid4().hex[:12]
+
+
+# ── ReloadDiagnosticContext ───────────────────────────────────────────────
+
+
+@dataclass(frozen=True, slots=True)
+class ReloadDiagnosticContext:
+    """Immutable metadata stamp for one reload operation."""
+
+    run_id: str
+    started_at: str
+
+    @staticmethod
+    def create() -> ReloadDiagnosticContext:
+        """Build a fresh context with a new run ID and current UTC time."""
+        return ReloadDiagnosticContext(
+            run_id=new_run_id(),
+            started_at=_iso_utc(_utc_now()),
+        )
+
+
+# ── log entry helpers ─────────────────────────────────────────────────────
+
+
+def build_base_entry(ctx: ReloadDiagnosticContext) -> dict[str, Any]:
+    """Return the fields common to every diagnostic log line."""
+    return {
+        "run_id": ctx.run_id,
+        "ts": _iso_utc(_utc_now()),
+    }
+
+
+def build_failure_entry(
+    ctx: ReloadDiagnosticContext,
+    category: str,
+    name: str,
+    error: str,
+    *,
+    path: str = "",
+) -> dict[str, Any]:
+    """Build a log entry for an expected reload failure."""
+    entry = build_base_entry(ctx)
+    entry.update(level="failure", category=category, name=name,
+                 error=_scrub_error(error))
+    if path:
+        entry["path"] = path
+    return entry
+
+
+def build_exception_entry(
+    ctx: ReloadDiagnosticContext,
+    category: str,
+    name: str,
+    exc: BaseException,
+    *,
+    path: str = "",
+) -> dict[str, Any]:
+    """Build a log entry for an unexpected exception."""
+    entry = build_base_entry(ctx)
+    entry.update(level="error", category=category, name=name,
+                 error=_scrub_error(str(exc)),
+                 traceback=_scrub_traceback(exc))
+    if path:
+        entry["path"] = path
+    return entry
+
+
+# ── ReloadDiagnosticLogger ────────────────────────────────────────────────
+
+
+class ReloadDiagnosticLogger:
+    """Appends structured JSONL diagnostic entries for reload operations."""
+
+    def __init__(self, log_dir: Path, *, filename: str = _LOG_FILENAME) -> None:
+        self._log_dir = log_dir
+        self._filename = filename
+
+    @property
+    def log_path(self) -> Path:
+        """Full path to the diagnostic log file."""
+        return self._log_dir / self._filename
+
+    # ── writing ───────────────────────────────────────────────────────
+
+    def _append(self, entry: dict[str, Any]) -> None:
+        """Serialize *entry* as compact JSON and append it as one line."""
+        self._log_dir.mkdir(parents=True, exist_ok=True)
+        with self.log_path.open("a", encoding="utf-8") as fh:
+            fh.write(json.dumps(entry, separators=(",", ":")) + "\n")
+
+    # ── public API ────────────────────────────────────────────────────
+
+    def log_failure(
+        self,
+        ctx: ReloadDiagnosticContext,
+        category: str,
+        name: str,
+        error: str,
+        *,
+        path: str = "",
+    ) -> None:
+        """Log an expected reload failure (bad file, missing field, etc.)."""
+        self._append(build_failure_entry(ctx, category, name, error, path=path))
+
+    def log_exception(
+        self,
+        ctx: ReloadDiagnosticContext,
+        category: str,
+        name: str,
+        exc: BaseException,
+        *,
+        path: str = "",
+    ) -> None:
+        """Log an unexpected exception with a scrubbed traceback."""
+        self._append(build_exception_entry(ctx, category, name, exc, path=path))
+
+
+__all__ = [
+    "ReloadDiagnosticContext",
+    "ReloadDiagnosticLogger",
+    "build_base_entry",
+    "build_exception_entry",
+    "build_failure_entry",
+    "new_run_id",
+]

rlmagent/routing.py

@@ -0,0 +1,52 @@
+"""Inference-route pinning for the Hugging Face router."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+ROUTED_PROVIDER = "huggingface"
+
+RouteMode = Literal["automatic", "fixed"]
+
+
+@dataclass(frozen=True, slots=True)
+class RoutePin:
+    """Which backend requests are sent to, and whether it may change."""
+
+    mode: RouteMode = "automatic"
+    backend: str | None = None
+
+    def model_for(self, model: str) -> str:
+        """The model id to send: suffixed with the backend when pinned."""
+        return f"{model}:{self.backend}" if self.backend else model
+
+    def describe(self) -> str:
+        if self.mode == "fixed" and self.backend:
+            return f"{self.backend} (fixed)"
+        if self.backend:
+            return f"automatic (currently {self.backend})"
+        return "automatic"
+
+
+def pin_from_storage(backend: str | None, mode: str | None) -> RoutePin:
+    """Rebuild a pin from saved metadata."""
+    if mode in ("automatic", "fixed"):
+        return RoutePin(mode, backend)  # type: ignore[arg-type]
+    return RoutePin("fixed", backend) if backend else RoutePin()
+
+
+def is_route_failure(status: int | None) -> bool:
+    """Statuses worth abandoning a learned backend for (after its own retries)."""
+    if status is None:
+        return False
+    return status in (408, 409, 425, 429) or status >= 500
+
+
+__all__ = [
+    "ROUTED_PROVIDER",
+    "RouteMode",
+    "RoutePin",
+    "is_route_failure",
+    "pin_from_storage",
+]