thinkstack-core 4.0.0__py3-none-any.whl

thinkstack_core/rep.py

@@ -0,0 +1,134 @@
+"""
+Sprint 5 – REP (Reasoning Exchange Protocol) envelope schema and local transport.
+
+Message envelope: agent_id, round, trust, payload hashes. Optional signatures.
+Local transport: file-based ledger with deterministic ordering for record/replay.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+REP_VERSION = "rep-v1"
+REP_LEDGER_NAME = "rep_ledger.jsonl"
+REP_DIR_NAME = "rep"
+
+
+def _hash_payload(payload: Any) -> str:
+    """SHA-256 hex digest of JSON-serialized payload."""
+    return hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()
+
+
+@dataclass
+class REPEnvelope:
+    """REP message envelope for sensitivity exchange."""
+    agent_id: str
+    round_id: int
+    trust: float  # [0, 1] optional trust score
+    payload_hash: str
+    payload_type: str = "sensitivity"  # sensitivity | control | ack
+    payload_ref: Optional[str] = None  # optional pointer to full payload
+    signature: Optional[str] = None  # optional crypto signature
+
+    def to_dict(self) -> dict:
+        d = {
+            "version": REP_VERSION,
+            "agent_id": self.agent_id,
+            "round_id": self.round_id,
+            "trust": self.trust,
+            "payload_hash": self.payload_hash,
+            "payload_type": self.payload_type,
+        }
+        if self.payload_ref is not None:
+            d["payload_ref"] = self.payload_ref
+        if self.signature is not None:
+            d["signature"] = self.signature
+        return d
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "REPEnvelope":
+        return cls(
+            agent_id=str(d["agent_id"]),
+            round_id=int(d["round_id"]),
+            trust=float(d.get("trust", 0.5)),
+            payload_hash=str(d["payload_hash"]),
+            payload_type=str(d.get("payload_type", "sensitivity")),
+            payload_ref=d.get("payload_ref"),
+            signature=d.get("signature"),
+        )
+
+
+class REPLedger:
+    """
+    Append-only REP ledger for local transport. Deterministic ordering by
+    sequence id and timestamp for record/replay.
+    """
+    def __init__(self, gcc_dir: Path) -> None:
+        self.rep_dir = gcc_dir / REP_DIR_NAME
+        self.ledger_path = self.rep_dir / REP_LEDGER_NAME
+
+    def ensure_dir(self) -> None:
+        self.rep_dir.mkdir(parents=True, exist_ok=True)
+
+    def append(self, envelope: REPEnvelope, timestamp: str, sequence_id: Optional[int] = None) -> int:
+        """Append envelope to ledger; return sequence id (1-based)."""
+        self.ensure_dir()
+        if sequence_id is None:
+            sequence_id = self._next_sequence_id()
+        record = {
+            "sequence_id": sequence_id,
+            "timestamp": timestamp,
+            "envelope": envelope.to_dict(),
+        }
+        with self.ledger_path.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(record, sort_keys=True) + "\n")
+        return sequence_id
+
+    def _next_sequence_id(self) -> int:
+        if not self.ledger_path.exists():
+            return 1
+        last = 0
+        with self.ledger_path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    rec = json.loads(line)
+                    last = max(last, rec.get("sequence_id", 0))
+                except json.JSONDecodeError:
+                    continue
+        return last + 1
+
+    def read_all(self) -> List[dict]:
+        """Read all records in deterministic order (sequence_id, timestamp)."""
+        if not self.ledger_path.exists():
+            return []
+        records = []
+        with self.ledger_path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    records.append(json.loads(line))
+                except json.JSONDecodeError:
+                    continue
+        records.sort(key=lambda r: (r.get("sequence_id", 0), r.get("timestamp", "")))
+        return records
+
+    def replay_checksum(self) -> str:
+        """Deterministic checksum of ledger for replay verification."""
+        records = self.read_all()
+        content = json.dumps(records, sort_keys=True)
+        return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def create_envelope(agent_id: str, round_id: int, payload: Any, trust: float = 0.5) -> REPEnvelope:
+    """Build REP envelope from payload (hash stored; payload not stored in envelope)."""
+    payload_hash = _hash_payload(payload)
+    return REPEnvelope(agent_id=agent_id, round_id=round_id, trust=trust, payload_hash=payload_hash)

thinkstack_core/rep_network/__init__.py

@@ -0,0 +1,25 @@
+"""
+thinkstack_core.rep_network — S15 multi-node REP ledger synchronization.
+"""
+
+from thinkstack_core.rep_network.node import NodeIdentity, NodeRegistry, PeerEntry
+from thinkstack_core.rep_network.sync import (
+    SyncResult,
+    pull_from_peer,
+    push_to_peer,
+    sync_all_peers,
+    sync_with_peer,
+)
+from thinkstack_core.rep_network.merge import merge_theta
+
+__all__ = [
+    "NodeIdentity",
+    "PeerEntry",
+    "NodeRegistry",
+    "SyncResult",
+    "push_to_peer",
+    "pull_from_peer",
+    "sync_with_peer",
+    "sync_all_peers",
+    "merge_theta",
+]

thinkstack_core/rep_network/merge.py

@@ -0,0 +1,70 @@
+"""
+theta.json merge function for REP network synchronisation.
+"""
+from __future__ import annotations
+from typing import Dict
+
+_LEVEL_RANK = {"PUBLIC": 0, "PROTECTED": 1, "PRIVATE": 2}
+_RANK_LEVEL = {0: "PUBLIC", 1: "PROTECTED", 2: "PRIVATE"}
+
+
+def merge_theta(local: dict, remote: dict) -> dict:
+    """
+    Merge two theta.json dicts (remote into local) and return the merged result.
+
+    Merge rules per concept:
+    - event_count: sum of both
+    - mean_confidence: weighted mean (weighted by event_count from each side)
+    - disclosure_level: max (most restrictive) of both sides
+    - last_updated: take the more recent of both (ISO string comparison)
+
+    The returned dict has the same shape as ThetaStore.load() output:
+    {
+      "version": 1,
+      "updated_at": <now ISO UTC>,
+      "coordination_vector": { concept: { ... } }
+    }
+    """
+    import datetime as _dt
+
+    local_cv = local.get("coordination_vector", {})
+    remote_cv = remote.get("coordination_vector", {})
+    all_concepts = set(local_cv) | set(remote_cv)
+
+    merged_cv: Dict[str, dict] = {}
+    for concept in all_concepts:
+        l = local_cv.get(concept, {})
+        r = remote_cv.get(concept, {})
+
+        l_count = int(l.get("event_count", 0))
+        r_count = int(r.get("event_count", 0))
+        total = l_count + r_count
+
+        if total == 0:
+            merged_conf = 0.0
+        else:
+            l_conf = float(l.get("mean_confidence", 0.0))
+            r_conf = float(r.get("mean_confidence", 0.0))
+            merged_conf = round((l_conf * l_count + r_conf * r_count) / total, 6)
+
+        l_rank = _LEVEL_RANK.get(l.get("disclosure_level", "PUBLIC"), 0)
+        r_rank = _LEVEL_RANK.get(r.get("disclosure_level", "PUBLIC"), 0)
+        merged_level = _RANK_LEVEL[max(l_rank, r_rank)]
+
+        l_ts = l.get("last_updated", "")
+        r_ts = r.get("last_updated", "")
+        merged_ts = l_ts if l_ts >= r_ts else r_ts
+
+        merged_cv[concept] = {
+            "event_count": total,
+            "mean_confidence": merged_conf,
+            "disclosure_level": merged_level,
+            "last_updated": merged_ts,
+        }
+
+    now = _dt.datetime.now(tz=_dt.timezone.utc).isoformat()
+    return {
+        "version": local.get("version", 1),
+        "updated_at": now,
+        "coordination_vector": merged_cv,
+    }

thinkstack_core/rep_network/node.py

@@ -0,0 +1,137 @@
+"""
+REP network node identity and peer discovery.
+
+Nodes are identified by a UUID generated once and stored in .GCC/rep/node_id.
+Peer discovery is file-based (no mDNS/DNS-SD in S15): peers are listed in
+.GCC/rep/peers.json as [{id: str, url: str, name: str}].
+"""
+
+from __future__ import annotations
+
+import json
+import socket
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List
+
+
+@dataclass
+class NodeIdentity:
+    """Identity of this node."""
+    node_id: str
+    name: str
+    url: str
+
+
+@dataclass
+class PeerEntry:
+    """A known remote peer."""
+    node_id: str
+    name: str
+    url: str
+    last_seen: str = ""
+
+
+class NodeRegistry:
+    """Manages node identity and peer list backed by .GCC/rep/ files."""
+
+    def __init__(self, gcc_root: str) -> None:
+        self._rep_dir = Path(gcc_root) / "rep"
+
+    # ------------------------------------------------------------------
+    # Identity
+    # ------------------------------------------------------------------
+
+    def get_or_create_identity(self, url: str = "") -> NodeIdentity:
+        """Read .GCC/rep/node_id; create and persist one if missing."""
+        self._rep_dir.mkdir(parents=True, exist_ok=True)
+        node_id_path = self._rep_dir / "node_id"
+        if node_id_path.exists():
+            try:
+                node_id = node_id_path.read_text(encoding="utf-8").strip()
+            except OSError:
+                node_id = str(uuid.uuid4())
+                node_id_path.write_text(node_id, encoding="utf-8")
+        else:
+            node_id = str(uuid.uuid4())
+            node_id_path.write_text(node_id, encoding="utf-8")
+
+        name = socket.gethostname()
+        return NodeIdentity(node_id=node_id, name=name, url=url)
+
+    # ------------------------------------------------------------------
+    # Peers
+    # ------------------------------------------------------------------
+
+    def _peers_path(self) -> Path:
+        return self._rep_dir / "peers.json"
+
+    def list_peers(self) -> List[PeerEntry]:
+        """Read .GCC/rep/peers.json; return empty list if missing or corrupt."""
+        path = self._peers_path()
+        if not path.exists():
+            return []
+        try:
+            raw = json.loads(path.read_text(encoding="utf-8"))
+            if not isinstance(raw, list):
+                return []
+            peers = []
+            for item in raw:
+                if not isinstance(item, dict):
+                    continue
+                try:
+                    peers.append(PeerEntry(
+                        node_id=str(item["node_id"]),
+                        name=str(item.get("name", "")),
+                        url=str(item.get("url", "")),
+                        last_seen=str(item.get("last_seen", "")),
+                    ))
+                except (KeyError, TypeError):
+                    continue
+            return peers
+        except (json.JSONDecodeError, OSError):
+            return []
+
+    def _save_peers(self, peers: List[PeerEntry]) -> None:
+        self._rep_dir.mkdir(parents=True, exist_ok=True)
+        data = [
+            {
+                "node_id": p.node_id,
+                "name": p.name,
+                "url": p.url,
+                "last_seen": p.last_seen,
+            }
+            for p in peers
+        ]
+        self._peers_path().write_text(json.dumps(data, indent=2), encoding="utf-8")
+
+    def add_peer(self, peer: PeerEntry) -> None:
+        """Append peer; replace existing entry with the same node_id."""
+        peers = self.list_peers()
+        peers = [p for p in peers if p.node_id != peer.node_id]
+        peers.append(peer)
+        self._save_peers(peers)
+
+    def remove_peer(self, node_id: str) -> None:
+        """Remove peer by node_id; no-op if not found."""
+        peers = [p for p in self.list_peers() if p.node_id != node_id]
+        self._save_peers(peers)
+
+    def update_peer_last_seen(self, node_id: str) -> None:
+        """Set last_seen to current UTC ISO timestamp for the given peer."""
+        peers = self.list_peers()
+        ts = datetime.now(timezone.utc).isoformat()
+        updated = []
+        for p in peers:
+            if p.node_id == node_id:
+                updated.append(PeerEntry(
+                    node_id=p.node_id,
+                    name=p.name,
+                    url=p.url,
+                    last_seen=ts,
+                ))
+            else:
+                updated.append(p)
+        self._save_peers(updated)

thinkstack_core/rep_network/server.py

@@ -0,0 +1,140 @@
+"""
+FastAPI server exposing REP ledger to peers.
+Runs on a different port from the proxy (default 8767).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+from thinkstack_core.rep import REPEnvelope, REPLedger
+from thinkstack_core.rep_network.node import NodeRegistry, PeerEntry
+
+try:
+    from fastapi import Body, FastAPI
+    from pydantic import BaseModel
+
+    HAS_FASTAPI = True
+
+    class PushEntriesRequest(BaseModel):
+        entries: List[Dict[str, Any]]
+        sender_id: str = ""
+
+    class PeerEntryModel(BaseModel):
+        node_id: str
+        name: str = ""
+        url: str = ""
+        last_seen: str = ""
+
+except ImportError:
+    HAS_FASTAPI = False
+
+
+def create_rep_server_app(ledger: REPLedger, registry: NodeRegistry) -> "FastAPI":
+    """Build and return a FastAPI application exposing the REP ledger.
+
+    Raises RuntimeError if FastAPI is not installed.
+    """
+    if not HAS_FASTAPI:
+        raise RuntimeError("FastAPI is not installed; cannot create REP server app.")
+
+    app = FastAPI(title="ThinkStack REP Network Node", version="0.1.0")
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    from thinkstack_core.rep_network.sync import _envelope_ids_in_ledger, _persist_pulled_id
+
+    def _existing_envelope_ids() -> set:
+        return _envelope_ids_in_ledger(ledger)
+
+    def _get_node_id() -> str:
+        try:
+            identity = registry.get_or_create_identity()
+            return identity.node_id
+        except Exception:  # noqa: BLE001
+            return ""
+
+    # ------------------------------------------------------------------
+    # Routes
+    # ------------------------------------------------------------------
+
+    @app.get("/rep/entries")
+    def get_entries(since: int = 0) -> Dict[str, Any]:
+        """Return ledger entries with sequence_id > since."""
+        all_records = ledger.read_all()
+        filtered = [r for r in all_records if r.get("sequence_id", 0) > since]
+        entries = [r.get("envelope", {}) for r in filtered]
+        return {
+            "node_id": _get_node_id(),
+            "entries": entries,
+            "count": len(entries),
+        }
+
+    @app.post("/rep/entries")
+    def post_entries(request: PushEntriesRequest = Body(...)) -> Dict[str, Any]:
+        """Accept pushed entries; skip duplicates by envelope_id."""
+        existing_ids = _existing_envelope_ids()
+        accepted = 0
+        duplicate = 0
+        import datetime as _dt
+
+        for entry_dict in request.entries:
+            eid = entry_dict.get("envelope_id")
+            if eid and eid in existing_ids:
+                duplicate += 1
+                continue
+            try:
+                envelope = REPEnvelope.from_dict(entry_dict)
+            except Exception:  # noqa: BLE001
+                continue
+            ts = entry_dict.get("timestamp", _dt.datetime.now(_dt.timezone.utc).isoformat())
+            ledger.append(envelope, timestamp=ts)
+            if eid:
+                existing_ids.add(eid)
+                _persist_pulled_id(ledger, eid)
+            accepted += 1
+
+        # Update last_seen for sender
+        if request.sender_id:
+            try:
+                registry.update_peer_last_seen(request.sender_id)
+            except Exception:  # noqa: BLE001
+                pass
+
+        return {"accepted": accepted, "duplicate": duplicate}
+
+    @app.get("/rep/peers")
+    def get_peers() -> List[Dict[str, Any]]:
+        peers = registry.list_peers()
+        return [
+            {
+                "node_id": p.node_id,
+                "name": p.name,
+                "url": p.url,
+                "last_seen": p.last_seen,
+            }
+            for p in peers
+        ]
+
+    @app.post("/rep/peers")
+    def post_peer(body: PeerEntryModel = Body(...)) -> Dict[str, str]:
+        peer = PeerEntry(
+            node_id=body.node_id,
+            name=body.name,
+            url=body.url,
+            last_seen=body.last_seen,
+        )
+        registry.add_peer(peer)
+        return {"status": "ok"}
+
+    @app.get("/health")
+    def health() -> Dict[str, Any]:
+        return {
+            "status": "ok",
+            "node_id": _get_node_id(),
+            "entry_count": len(ledger.read_all()),
+        }
+
+    return app