hyperagents 1.2.3rc5.post3__py3-none-any.whl

hyperclaw/tools/ledger.py

@@ -0,0 +1,339 @@
+"""
+The effect ledger (tool standard S3) and activity log (spec rule 6).
+
+A local, append-only SQLite table at $HYPERCLAW_ROOT/ledger/effects.sqlite3
+(default ~/.hyperclaw). Every non-read tool call leaves rows here:
+
+- an intent row (status "pending") written BEFORE the tool runs, in the same
+  transaction as the idempotency check (BEGIN IMMEDIATE, so two processes —
+  the terminal and the Telegram bridge — cannot both pass the check);
+- an outcome row (ok / failed / uncertain / ...) after it ran, pointing at
+  the intent (`ref`);
+- a single row for a call the runtime refused (duplicate, needs_approval,
+  budget_exhausted).
+
+UPDATE and DELETE are blocked by triggers. A pending row with no outcome
+(the process died mid-call) is treated as uncertain: at-most-once.
+
+Decisions (per conversation, within the window):
+- same key, latest row ok              -> duplicate (original receipt), not run
+  (file targets: only while the file still has the content that was written)
+- same key, latest row uncertain/pending -> needs_approval ("may already have
+  been sent"), not run (file targets: checked on disk instead)
+- same target, different payload, send/spend/delete, inside the shorter
+  conflict window                       -> needs_approval, not run
+- otherwise                             -> intent row, run
+
+Broker mapping: `key` is the agent-mode proposal `payloadDigest`; this table
+is the local equivalent of `agent_action_ledger` (append-only, one row per
+state change, receipt on the result).
+"""
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+DEFAULT_WINDOW_S = 24 * 3600
+DEFAULT_CONFLICT_WINDOW_S = 600
+
+# Statuses that mean "an effect happened or may have happened".
+DONE = ("ok",)
+MAYBE = ("uncertain", "pending")
+REFUSALS = ("duplicate", "needs_approval", "budget_exhausted")
+CONFLICT_EFFECTS = ("send", "spend", "delete")
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS effect_ledger (
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    ts            REAL    NOT NULL,
+    conversation  TEXT    NOT NULL,
+    turn          TEXT,
+    surface       TEXT,
+    tool          TEXT    NOT NULL,
+    effect        TEXT    NOT NULL,
+    target        TEXT,
+    payload_hash  TEXT,
+    key           TEXT,
+    status        TEXT    NOT NULL,
+    receipt_id    TEXT,
+    ref           INTEGER,
+    post_hash     TEXT,
+    args          TEXT,
+    message       TEXT
+);
+CREATE INDEX IF NOT EXISTS effect_ledger_key ON effect_ledger (conversation, key, id);
+CREATE INDEX IF NOT EXISTS effect_ledger_target ON effect_ledger (conversation, target, id);
+CREATE INDEX IF NOT EXISTS effect_ledger_ref ON effect_ledger (ref);
+CREATE TRIGGER IF NOT EXISTS effect_ledger_no_update BEFORE UPDATE ON effect_ledger
+BEGIN SELECT RAISE(ABORT, 'effect_ledger is append-only'); END;
+CREATE TRIGGER IF NOT EXISTS effect_ledger_no_delete BEFORE DELETE ON effect_ledger
+BEGIN SELECT RAISE(ABORT, 'effect_ledger is append-only'); END;
+"""
+
+_COLUMNS = ("id", "ts", "conversation", "turn", "surface", "tool", "effect", "target",
+            "payload_hash", "key", "status", "receipt_id", "ref", "post_hash", "args", "message")
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        return float(os.environ.get(name, default))
+    except (TypeError, ValueError):
+        return float(default)
+
+
+def default_path() -> Path:
+    """$HYPERCLAW_EFFECT_LEDGER if set, else $HYPERCLAW_ROOT/ledger/effects.sqlite3
+    (HYPERCLAW_ROOT defaults to ~/.hyperclaw)."""
+    explicit = os.environ.get("HYPERCLAW_EFFECT_LEDGER")
+    if explicit:
+        return Path(explicit).expanduser()
+    root = Path(os.environ.get("HYPERCLAW_ROOT", Path.home() / ".hyperclaw")).expanduser()
+    return root / "ledger" / "effects.sqlite3"
+
+
+@dataclass
+class Decision:
+    action: str                      # "proceed" | "duplicate" | "needs_approval"
+    intent_id: Optional[int] = None
+    prior: Optional[Dict[str, Any]] = None
+    reason: str = ""                 # "duplicate" | "uncertain" | "conflict"
+
+
+class EffectLedger:
+    def __init__(self, path: Optional[os.PathLike] = None):
+        self.path = Path(path) if path else default_path()
+        self._init_lock = threading.Lock()
+        self._ready = False
+
+    # ── connection ──
+    def _connect(self) -> sqlite3.Connection:
+        if not self._ready:
+            with self._init_lock:
+                if not self._ready:
+                    self.path.parent.mkdir(parents=True, exist_ok=True)
+                    try:
+                        os.chmod(self.path.parent, 0o700)
+                    except OSError:
+                        pass
+                    con = sqlite3.connect(self.path, timeout=10, isolation_level=None)
+                    try:
+                        con.execute("PRAGMA journal_mode=WAL")
+                        con.executescript(_SCHEMA)
+                    finally:
+                        con.close()
+                    try:
+                        os.chmod(self.path, 0o600)
+                    except OSError:
+                        pass
+                    self._ready = True
+        con = sqlite3.connect(self.path, timeout=10, isolation_level=None)
+        con.row_factory = sqlite3.Row
+        return con
+
+    @staticmethod
+    def window_s() -> float:
+        return _env_float("HYPERCLAW_EFFECT_WINDOW_S", DEFAULT_WINDOW_S)
+
+    @staticmethod
+    def conflict_window_s() -> float:
+        return _env_float("HYPERCLAW_EFFECT_CONFLICT_WINDOW_S", DEFAULT_CONFLICT_WINDOW_S)
+
+    @staticmethod
+    def _insert(con, row: Dict[str, Any]) -> int:
+        row = dict(row)
+        row.setdefault("ts", time.time())
+        cols = [c for c in _COLUMNS if c != "id" and c in row]
+        cur = con.execute(
+            f"INSERT INTO effect_ledger ({','.join(cols)}) VALUES ({','.join('?' * len(cols))})",
+            [row[c] for c in cols])
+        return int(cur.lastrowid)
+
+    # ── the check + intent, in one transaction ──
+    def begin(self, *, conversation: str, turn: str, surface: str, tool: str, effect: str,
+              target: str, payload_hash: str, key: Optional[str], args: str = "",
+              current_file_hash: Optional[str] = None,
+              expected_post_hash: Optional[str] = None,
+              is_file: bool = False, allow_conflict: bool = False) -> Decision:
+        """allow_conflict: the owner approved this exact payload (approval
+        layer), so a different earlier payload to the same target does not
+        hold it. Exact duplicates and uncertain repeats are still held."""
+        now = time.time()
+        base = dict(ts=now, conversation=conversation, turn=turn, surface=surface, tool=tool,
+                    effect=effect, target=target, payload_hash=payload_hash, key=key, args=args)
+        con = self._connect()
+        try:
+            con.execute("BEGIN IMMEDIATE")
+            decision = self._decide(con, now, conversation, effect, target, key,
+                                    current_file_hash, expected_post_hash, is_file)
+            if allow_conflict and decision.reason == "conflict":
+                decision = Decision("proceed")
+            if decision.action == "proceed":
+                decision.intent_id = self._insert(con, {**base, "status": "pending"})
+            else:
+                prior = decision.prior or {}
+                self._insert(con, {**base, "status": decision.action,
+                                   "receipt_id": prior.get("receipt_id"),
+                                   "ref": prior.get("id"),
+                                   "message": decision.reason})
+            con.execute("COMMIT")
+            return decision
+        except Exception:
+            try:
+                con.execute("ROLLBACK")
+            except sqlite3.Error:
+                pass
+            raise
+        finally:
+            con.close()
+
+    def check(self, *, conversation: str, effect: str, target: str, key: Optional[str],
+              current_file_hash: Optional[str] = None,
+              expected_post_hash: Optional[str] = None, is_file: bool = False) -> Decision:
+        """The same decision as begin(), without writing anything."""
+        con = self._connect()
+        try:
+            return self._decide(con, time.time(), conversation, effect, target, key,
+                                current_file_hash, expected_post_hash, is_file)
+        finally:
+            con.close()
+
+    def _decide(self, con, now, conversation, effect, target, key, current_file_hash,
+                expected_post_hash, is_file) -> Decision:
+        if key:
+            row = con.execute(
+                "SELECT * FROM effect_ledger WHERE conversation=? AND key=? AND ts>=? "
+                f"AND status NOT IN ({','.join('?' * len(REFUSALS))}) ORDER BY id DESC LIMIT 1",
+                (conversation, key, now - self.window_s(), *REFUSALS)).fetchone()
+            if row is not None:
+                prior = dict(row)
+                st = prior["status"]
+                if st in DONE:
+                    if not is_file:
+                        return Decision("duplicate", prior=prior, reason="duplicate")
+                    # A file effect is "already done" only while the file still
+                    # holds what was written; edits since then make it new.
+                    if prior.get("post_hash") and prior["post_hash"] == current_file_hash:
+                        return Decision("duplicate", prior=prior, reason="duplicate")
+                elif st in MAYBE:
+                    if is_file:
+                        if expected_post_hash and expected_post_hash == current_file_hash:
+                            return Decision("duplicate", prior=prior, reason="duplicate")
+                    else:
+                        return Decision("needs_approval", prior=prior, reason="uncertain")
+        if effect in CONFLICT_EFFECTS and target:
+            row = con.execute(
+                "SELECT * FROM effect_ledger r WHERE r.conversation=? AND r.target=? "
+                "AND r.effect=? AND r.ts>=? AND (r.key IS NULL OR r.key<>?) AND ("
+                " r.status IN ('ok','uncertain') OR (r.status='pending' AND NOT EXISTS "
+                " (SELECT 1 FROM effect_ledger o WHERE o.ref=r.id AND o.status NOT IN "
+                f"({','.join('?' * len(REFUSALS))})))) "
+                "ORDER BY r.id DESC LIMIT 1",
+                (conversation, target, effect, now - self.conflict_window_s(),
+                 key or "", *REFUSALS)).fetchone()
+            if row is not None:
+                return Decision("needs_approval", prior=dict(row), reason="conflict")
+        return Decision("proceed")
+
+    # ── outcomes and plain records ──
+    def finish(self, intent_id: int, *, status: str, receipt_id: Optional[str] = None,
+               post_hash: Optional[str] = None, message: str = "") -> int:
+        con = self._connect()
+        try:
+            con.execute("BEGIN IMMEDIATE")
+            row = con.execute("SELECT * FROM effect_ledger WHERE id=?", (intent_id,)).fetchone()
+            if row is None:
+                con.execute("ROLLBACK")
+                raise KeyError(intent_id)
+            base = {k: row[k] for k in ("conversation", "turn", "surface", "tool", "effect",
+                                        "target", "payload_hash", "key", "args")}
+            rid = self._insert(con, {**base, "status": status, "receipt_id": receipt_id,
+                                     "ref": intent_id, "post_hash": post_hash,
+                                     "message": (message or "")[:500]})
+            con.execute("COMMIT")
+            return rid
+        finally:
+            con.close()
+
+    def record(self, **row) -> int:
+        """Append one finished row (bash file effects, budget refusals)."""
+        row.setdefault("message", "")
+        row["message"] = (row["message"] or "")[:500]
+        con = self._connect()
+        try:
+            return self._insert(con, row)
+        finally:
+            con.close()
+
+    # ── readers ──
+    def rows(self, conversation: Optional[str] = None, limit: int = 1000) -> List[Dict[str, Any]]:
+        con = self._connect()
+        try:
+            if conversation:
+                cur = con.execute("SELECT * FROM effect_ledger WHERE conversation=? "
+                                  "ORDER BY id DESC LIMIT ?", (conversation, limit))
+            else:
+                cur = con.execute("SELECT * FROM effect_ledger ORDER BY id DESC LIMIT ?", (limit,))
+            return [dict(r) for r in cur.fetchall()][::-1]
+        finally:
+            con.close()
+
+    def activity(self, conversation: Optional[str] = None, limit: int = 20) -> List[Dict[str, Any]]:
+        """The activity log: one entry per action (an intent and its outcome
+        collapsed), newest last. An intent with no outcome shows as
+        'pending (outcome unknown)'."""
+        con = self._connect()
+        try:
+            where, params = ("WHERE r.conversation=?", [conversation]) if conversation else ("", [])
+            cur = con.execute(
+                "SELECT r.* FROM effect_ledger r "
+                f"{where} {'AND' if where else 'WHERE'} NOT (r.status='pending' AND EXISTS "
+                "(SELECT 1 FROM effect_ledger o WHERE o.ref=r.id AND o.status NOT IN "
+                f"({','.join('?' * len(REFUSALS))}))) "
+                "ORDER BY r.id DESC LIMIT ?", [*params, *REFUSALS, limit])
+            return [dict(r) for r in cur.fetchall()][::-1]
+        finally:
+            con.close()
+
+
+def format_activity(rows: List[Dict[str, Any]]) -> str:
+    if not rows:
+        return "No activity recorded."
+    lines = []
+    for r in rows:
+        when = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(r["ts"]))
+        status = r["status"] if r["status"] != "pending" else "pending (outcome unknown)"
+        target = r.get("target") or ""
+        if target.startswith(r["tool"] + ":"):
+            target = target[len(r["tool"]) + 1:]
+        line = f"{when}  {r['conversation']:<14} {r['tool']:<14} {r['effect']:<7} {status:<15}"
+        if target:
+            line += f" {target[:70]}"
+        if r.get("receipt_id"):
+            line += f"  [{r['receipt_id']}]"
+        if r.get("message") and r["status"] != "ok":
+            line += f"  — {r['message'][:120]}"
+        lines.append(line)
+    return "\n".join(lines)
+
+
+def activity_json(rows: List[Dict[str, Any]]) -> str:
+    return json.dumps(rows, ensure_ascii=False, default=str, indent=1)
+
+
+_LEDGERS: Dict[str, EffectLedger] = {}
+_LEDGERS_LOCK = threading.Lock()
+
+
+def get_ledger() -> EffectLedger:
+    """The ledger for the current HYPERCLAW_ROOT (read at call time)."""
+    p = str(default_path())
+    with _LEDGERS_LOCK:
+        if p not in _LEDGERS:
+            _LEDGERS[p] = EffectLedger(p)
+        return _LEDGERS[p]

hyperclaw/tools/receipts.py

@@ -0,0 +1,70 @@
+"""
+Send receipts (benchmark 20.3, rc3).
+
+A send tool reports success only with a provider-confirmed receipt:
+
+- Telegram: HTTP 200 AND ``ok: true`` AND ``result.message_id`` -> the
+  message id is the receipt. ``ok: false`` (e.g. "Bad Request: chat not
+  found") or a missing id is a failure carrying the provider's description.
+- Discord (webhook with ``?wait=true``): the returned message ``id``.
+- Gmail: the sent message ``id``.
+
+Success strings carry the provider id as ``[id:<id>]``; the classifier
+(tools/classify.py) requires it for these channels and returns ``failed``
+without it, so "Message sent" can never stand without a receipt.
+Provider descriptions are redacted (no tokens or keys) and clipped.
+"""
+from __future__ import annotations
+
+import re
+from typing import Optional, Tuple
+
+RECEIPT_RE = r"\[id:([^\]\s]+)\]"
+_BOT_URL = re.compile(r"/bot[^/\s]+/")
+
+
+def redact(text: str, limit: int = 200) -> str:
+    """Provider text safe to show and log: secrets removed, clipped."""
+    from hyperclaw.truth_guard import redact_secrets
+    t = _BOT_URL.sub("/bot[REDACTED]/", str(text or ""))
+    return redact_secrets(t).strip()[:limit]
+
+
+def _json(resp) -> Optional[dict]:
+    try:
+        body = resp.json()
+    except Exception:
+        return None
+    return body if isinstance(body, dict) else None
+
+
+def telegram_receipt(resp) -> Tuple[Optional[str], str]:
+    """(message_id, "") when Telegram confirmed delivery, else (None, why)."""
+    body = _json(resp)
+    status = getattr(resp, "status_code", None)
+    if body is None:
+        return None, redact(f"HTTP {status}: response was not Telegram JSON")
+    if body.get("ok") is True and status == 200:
+        res = body.get("result")
+        mid = res.get("message_id") if isinstance(res, dict) else None
+        if mid not in (None, ""):
+            return str(mid), ""
+        return None, "Telegram returned no message_id, so delivery is not confirmed"
+    desc = body.get("description") or f"HTTP {status}"
+    return None, redact(desc)
+
+
+def discord_receipt(resp) -> Tuple[Optional[str], str]:
+    body = _json(resp)
+    status = getattr(resp, "status_code", None)
+    if status == 200 and body and body.get("id"):
+        return str(body["id"]), ""
+    if status == 204:
+        return None, "Discord accepted the message but returned no message id (use ?wait=true)"
+    desc = (body or {}).get("message") or getattr(resp, "text", "") or f"HTTP {status}"
+    return None, redact(f"HTTP {status}: {desc}")
+
+
+def has_receipt(text: str) -> Optional[str]:
+    m = re.search(RECEIPT_RE, text or "")
+    return m.group(1) if m else None