interlockd 0.1.0__py3-none-any.whl

interlockd/store/drainer.py

@@ -0,0 +1,74 @@
+"""Background drainer: moves undrained spool records to the repository in batches, off the decision path."""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Any
+
+from . import Repository
+from .spool import Spool
+
+log = logging.getLogger("interlockd.drainer")
+
+
+class Drainer:
+    def __init__(self, spool: Spool, repo: Repository, *, interval_s: float = 2.0, batch: int = 25):
+        self.spool, self.repo, self.interval, self.batch = spool, repo, interval_s, batch
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+        self.drained_total = 0
+        self.last_error: str | None = None
+
+    def drain_once(self) -> int:
+        n = 0
+        pending: list[tuple[Any, int, dict[str, Any]]] = []
+
+        def flush() -> None:
+            nonlocal n
+            if not pending:
+                return
+            records = [r for _, _, r in pending]
+            if hasattr(self.repo, "put_events"):
+                self.repo.put_events(records)  # type: ignore[attr-defined]
+            else:
+                for r in records:
+                    self.repo.put_event(r)
+            seg, off, _ = pending[-1]
+            self.spool.mark_drained(seg, off)
+            n += len(pending)
+            pending.clear()
+
+        current_seg = None
+        for seg, off, rec in self.spool.undrained():
+            if current_seg is not None and seg != current_seg:
+                flush()
+            current_seg = seg
+            pending.append((seg, off, rec))
+            if len(pending) >= self.batch:
+                flush()
+        flush()
+        self.drained_total += n
+        return n
+
+    def _run(self) -> None:
+        backoff = self.interval
+        while not self._stop.is_set():
+            try:
+                self.drain_once()
+                self.last_error = None
+                backoff = self.interval
+            except Exception as e:  # noqa: BLE001  boundary: network, credentials, throttling; keep the spool, retry
+                self.last_error = f"{type(e).__name__}: {e}"
+                log.warning("drain failed, will retry: %s", self.last_error)
+                backoff = min(backoff * 2, 60.0)
+            self._stop.wait(backoff)
+
+    def start(self) -> None:
+        self._thread = threading.Thread(target=self._run, name="interlock-drainer", daemon=True)
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+        if self._thread:
+            self._thread.join(timeout=5)

interlockd/store/dynamodb.py

@@ -0,0 +1,144 @@
+"""DynamoDB single-table repository (design §7). boto3 is imported lazily so the daemon runs without it.
+
+Items:
+  Event            PK=EVT#<event_id>  SK=EVT
+  Explanation      PK=EVT#<event_id>  SK=EXPL
+  Feedback         PK=EVT#<event_id>  SK=FDBK#<ulid>
+  Session summary  PK=SES#<session_id> SK=META
+Indexes:
+  GSI1-session         session_id / event_id
+  GSI2-workspace-time  GSI2PK (WS#<workspace>#<yyyy-mm-dd-hh>) / event_id
+  GSI3-review          GSI3PK (DISP#<disposition>#<yyyy-mm-dd>) / event_id   (sparse)
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from typing import Any
+
+from . import hour_bucket
+
+REVIEW_DISPOSITIONS = {"ask", "block", "withhold", "warn"}
+
+
+def _to_ddb(value: Any) -> Any:
+    if isinstance(value, float):
+        return Decimal(str(value))
+    if isinstance(value, dict):
+        return {k: _to_ddb(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_to_ddb(v) for v in value]
+    return value
+
+
+def _from_ddb(value: Any) -> Any:
+    if isinstance(value, Decimal):
+        return int(value) if value == value.to_integral_value() else float(value)
+    if isinstance(value, dict):
+        return {k: _from_ddb(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_from_ddb(v) for v in value]
+    return value
+
+
+TABLE_DEFINITION: dict[str, Any] = {
+    "AttributeDefinitions": [
+        {"AttributeName": "PK", "AttributeType": "S"},
+        {"AttributeName": "SK", "AttributeType": "S"},
+        {"AttributeName": "session_id", "AttributeType": "S"},
+        {"AttributeName": "event_id", "AttributeType": "S"},
+        {"AttributeName": "GSI2PK", "AttributeType": "S"},
+        {"AttributeName": "GSI3PK", "AttributeType": "S"},
+    ],
+    "KeySchema": [{"AttributeName": "PK", "KeyType": "HASH"}, {"AttributeName": "SK", "KeyType": "RANGE"}],
+    "BillingMode": "PAY_PER_REQUEST",
+    "GlobalSecondaryIndexes": [
+        {"IndexName": "GSI1-session", "Projection": {"ProjectionType": "ALL"},
+         "KeySchema": [{"AttributeName": "session_id", "KeyType": "HASH"}, {"AttributeName": "event_id", "KeyType": "RANGE"}]},
+        {"IndexName": "GSI2-workspace-time", "Projection": {"ProjectionType": "ALL"},
+         "KeySchema": [{"AttributeName": "GSI2PK", "KeyType": "HASH"}, {"AttributeName": "event_id", "KeyType": "RANGE"}]},
+        {"IndexName": "GSI3-review", "Projection": {"ProjectionType": "ALL"},
+         "KeySchema": [{"AttributeName": "GSI3PK", "KeyType": "HASH"}, {"AttributeName": "event_id", "KeyType": "RANGE"}]},
+    ],
+}
+
+
+class DynamoRepository:
+    def __init__(self, table_name: str, *, region: str | None = None, profile: str | None = None, retention_days: int = 90, resource=None):
+        import boto3  # lazy
+
+        self.ddb = resource or boto3.Session(profile_name=profile, region_name=region).resource("dynamodb")
+        self.table = self.ddb.Table(table_name)
+        self.table_name = table_name
+        self.retention_days = retention_days
+
+    @classmethod
+    def ensure_table(cls, table_name: str, *, region: str | None = None, resource=None) -> DynamoRepository:
+        import boto3  # lazy
+
+        ddb = resource or boto3.resource("dynamodb", region_name=region)
+        existing = [t.name for t in ddb.tables.all()]
+        if table_name not in existing:
+            t = ddb.create_table(TableName=table_name, **TABLE_DEFINITION)
+            t.wait_until_exists()
+            ddb.meta.client.update_time_to_live(
+                TableName=table_name, TimeToLiveSpecification={"Enabled": True, "AttributeName": "expires_at"}
+            )
+        return cls(table_name, region=region, resource=ddb)
+
+    def _event_item(self, record: dict[str, Any]) -> dict[str, Any]:
+        item = _to_ddb(dict(record))
+        item["PK"] = f"EVT#{record['event_id']}"
+        item["SK"] = "EVT"
+        ws = record.get("agent_workspace") or ""
+        item["GSI2PK"] = f"WS#{ws}#{hour_bucket(record['received_at'])}"
+        if record.get("disposition") in REVIEW_DISPOSITIONS:
+            item["GSI3PK"] = f"DISP#{record['disposition']}#{record['received_at'][:10]}"
+        if "expires_at" not in item:
+            exp = datetime.now(UTC) + timedelta(days=self.retention_days)
+            item["expires_at"] = int(exp.timestamp())
+        return item
+
+    def put_event(self, record: dict[str, Any]) -> None:
+        self.table.put_item(Item=self._event_item(record))
+
+    def put_events(self, records: list[dict[str, Any]]) -> None:
+        with self.table.batch_writer() as bw:
+            for r in records:
+                bw.put_item(Item=self._event_item(r))
+
+    def get_event(self, event_id: str) -> dict[str, Any] | None:
+        r = self.table.get_item(Key={"PK": f"EVT#{event_id}", "SK": "EVT"})
+        item = r.get("Item")
+        return _from_ddb(item) if item else None
+
+    def tail(self, workspace: str | None = None, limit: int = 50, *, hours_back: int = 48) -> list[dict[str, Any]]:
+        from boto3.dynamodb.conditions import Key  # lazy
+
+        out: list[dict[str, Any]] = []
+        now = datetime.now(UTC).replace(minute=0, second=0, microsecond=0)
+        ws = workspace or ""
+        for h in range(hours_back):
+            bucket = (now - timedelta(hours=h)).strftime("%Y-%m-%d-%H")
+            resp = self.table.query(
+                IndexName="GSI2-workspace-time",
+                KeyConditionExpression=Key("GSI2PK").eq(f"WS#{ws}#{bucket}"),
+                ScanIndexForward=False,
+                Limit=limit - len(out),
+            )
+            out.extend(_from_ddb(i) for i in resp.get("Items", []))
+            if len(out) >= limit:
+                break
+        return out[:limit]
+
+    def put_session(self, summary: dict[str, Any]) -> None:
+        item = _to_ddb(dict(summary))
+        item["PK"] = f"SES#{summary['session_id']}"
+        item["SK"] = "META"
+        self.table.put_item(Item=item)
+
+    def get_session(self, session_id: str) -> dict[str, Any] | None:
+        r = self.table.get_item(Key={"PK": f"SES#{session_id}", "SK": "META"})
+        item = r.get("Item")
+        return _from_ddb(item) if item else None

interlockd/store/memory.py

@@ -0,0 +1,28 @@
+"""In-memory repository for tests and the spool-only mode."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class MemoryRepository:
+    def __init__(self) -> None:
+        self.events: dict[str, dict[str, Any]] = {}
+        self.sessions: dict[str, dict[str, Any]] = {}
+
+    def put_event(self, record: dict[str, Any]) -> None:
+        self.events[record["event_id"]] = record
+
+    def get_event(self, event_id: str) -> dict[str, Any] | None:
+        return self.events.get(event_id)
+
+    def tail(self, workspace: str | None = None, limit: int = 50) -> list[dict[str, Any]]:
+        rows = [r for r in self.events.values() if workspace is None or r.get("agent_workspace") == workspace]
+        rows.sort(key=lambda r: r["event_id"], reverse=True)
+        return rows[:limit]
+
+    def put_session(self, summary: dict[str, Any]) -> None:
+        self.sessions[summary["session_id"]] = summary
+
+    def get_session(self, session_id: str) -> dict[str, Any] | None:
+        return self.sessions.get(session_id)

interlockd/store/spool.py

@@ -0,0 +1,159 @@
+"""Append-only JSONL spool with per-segment drain offsets, hash-chained for tamper evidence.
+
+Segments are one file per UTC day. `append` writes one line and fsyncs so the record is durable on the
+host before the verdict is returned. A drainer reads undrained bytes and advances the offset file after
+the remote write is confirmed.
+
+Every record carries `prev_hash` and `hash = sha256(prev_hash + canonical JSON of the record without `hash`)`,
+chained across segments, so editing, deleting, reordering, or inserting a record breaks `verify()`. Truncating
+the newest records is only detectable against an anchored head (`interlock audit head`). Records written before
+chaining existed are reported as legacy, not as breaks.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import json
+import os
+import threading
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+
+class Spool:
+    def __init__(self, directory: Path, *, fsync: bool = True):
+        self.dir = Path(directory)
+        self.dir.mkdir(parents=True, exist_ok=True)
+        self.fsync = fsync
+        self._lock = threading.Lock()
+
+    def _segment(self, received_at: str | None = None) -> Path:
+        day = (received_at or datetime.now(UTC).isoformat())[:10]
+        return self.dir / f"{day}.jsonl"
+
+    def append(self, record: dict[str, Any]) -> None:
+        seg = self._segment()      # by write time, not received_at: the chain must follow file order
+        with self._lock:
+            lock_fd = os.open(self.dir / ".lock", os.O_WRONLY | os.O_CREAT, 0o600)
+            try:
+                fcntl.flock(lock_fd, fcntl.LOCK_EX)         # one chain even with several writers on this spool
+                rec = {k: v for k, v in record.items() if k not in ("hash", "prev_hash")}
+                rec["prev_hash"] = self.head()
+                rec["hash"] = record_hash(rec)
+                line = json.dumps(rec, separators=(",", ":"), ensure_ascii=False) + "\n"
+                fd = os.open(seg, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
+                try:
+                    os.write(fd, line.encode("utf-8"))
+                    if self.fsync:
+                        os.fsync(fd)
+                finally:
+                    os.close(fd)
+            finally:
+                fcntl.flock(lock_fd, fcntl.LOCK_UN)
+                os.close(lock_fd)
+
+    def head(self) -> str:
+        """Hash of the newest chained record ("genesis" for an empty or legacy-only spool)."""
+        for seg in reversed(self.segments()):
+            last = _last_line(seg)
+            if last:
+                try:
+                    return json.loads(last).get("hash") or GENESIS
+                except ValueError:
+                    return GENESIS
+        return GENESIS
+
+    def verify(self) -> dict[str, Any]:
+        """Walk every segment in order. Returns {ok, records, legacy, head, break: {segment, line, reason} | None}."""
+        expected, n, legacy = None, 0, 0
+        for seg in self.segments():
+            for i, raw in enumerate(seg.read_text(encoding="utf-8").splitlines(), 1):
+                if not raw.strip():
+                    continue
+                try:
+                    rec = json.loads(raw)
+                except ValueError:
+                    return _broken(seg, i, "not valid JSON", n, legacy)
+                if "hash" not in rec:
+                    if expected is not None:
+                        return _broken(seg, i, "unchained record after the chain started (inserted?)", n, legacy)
+                    legacy += 1
+                    continue
+                prev = rec.get("prev_hash")
+                if expected is None and prev != GENESIS and legacy == 0:
+                    return _broken(seg, i, "first chained record does not start the chain (records removed before it?)", n, legacy)
+                if expected is not None and prev != expected:
+                    return _broken(seg, i, "prev_hash does not match the previous record (removed or reordered?)", n, legacy)
+                if record_hash({k: v for k, v in rec.items() if k != "hash"}) != rec["hash"]:
+                    return _broken(seg, i, "content does not match its hash (edited)", n, legacy)
+                expected, n = rec["hash"], n + 1
+        return {"ok": True, "records": n, "legacy": legacy, "head": expected or GENESIS, "break": None}
+
+    def segments(self) -> list[Path]:
+        return sorted(self.dir.glob("*.jsonl"))
+
+    def iter_records(self, *, limit: int | None = None, reverse: bool = True) -> Iterator[dict[str, Any]]:
+        n = 0
+        segs = self.segments()
+        if reverse:
+            segs = list(reversed(segs))
+        for seg in segs:
+            lines = seg.read_text(encoding="utf-8").splitlines()
+            if reverse:
+                lines = list(reversed(lines))
+            for line in lines:
+                if not line.strip():
+                    continue
+                yield json.loads(line)
+                n += 1
+                if limit is not None and n >= limit:
+                    return
+
+    def _offset_path(self, seg: Path) -> Path:
+        return seg.with_suffix(".drained")
+
+    def drained_offset(self, seg: Path) -> int:
+        p = self._offset_path(seg)
+        return int(p.read_text().strip() or 0) if p.exists() else 0
+
+    def undrained(self) -> Iterator[tuple[Path, int, dict[str, Any]]]:
+        """Yield (segment, end_offset_after_record, record) for every record not yet drained."""
+        for seg in self.segments():
+            start = self.drained_offset(seg)
+            with open(seg, "rb") as f:
+                f.seek(start)
+                pos = start
+                for raw in f:
+                    pos += len(raw)
+                    if raw.strip():
+                        yield seg, pos, json.loads(raw.decode("utf-8"))
+
+    def mark_drained(self, seg: Path, offset: int) -> None:
+        tmp = self._offset_path(seg).with_suffix(".drained.tmp")
+        tmp.write_text(str(offset))
+        os.replace(tmp, self._offset_path(seg))
+
+
+GENESIS = "genesis"
+
+
+def record_hash(rec_without_hash: dict[str, Any]) -> str:
+    body = json.dumps(rec_without_hash, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(f"{rec_without_hash.get('prev_hash', GENESIS)}\n{body}".encode()).hexdigest()
+
+
+def _last_line(path: Path) -> str | None:
+    with open(path, "rb") as f:
+        f.seek(0, os.SEEK_END)
+        size = f.tell()
+        block = min(size, 1 << 20)
+        f.seek(size - block)
+        lines = [ln for ln in f.read().splitlines() if ln.strip()]
+        return lines[-1].decode("utf-8") if lines else None
+
+
+def _broken(seg: Path, line: int, reason: str, n: int, legacy: int) -> dict[str, Any]:
+    return {"ok": False, "records": n, "legacy": legacy, "head": None, "break": {"segment": seg.name, "line": line, "reason": reason}}