contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/context/webhooks.py

@@ -0,0 +1,194 @@
+"""
+Webhook Registry & Dispatcher
+==============================
+Register outbound webhook URLs to receive HTTP POST notifications
+when context events occur in sessions.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import logging
+import sqlite3
+import time
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Webhook:
+    webhook_id: str
+    tenant_id: str
+    url: str
+    event_types: List[str]
+    secret: str
+    created_at: float
+    status: str = "active"
+
+    def to_dict(self) -> dict:
+        return {
+            "webhook_id": self.webhook_id,
+            "url": self.url,
+            "event_types": self.event_types,
+            "status": self.status,
+            "created_at": self.created_at,
+        }
+
+
+class WebhookRegistry:
+    """SQLite-backed webhook registration."""
+
+    def __init__(self, db_path: Optional[str] = None):
+        if db_path is None:
+            db_path = str(Path("contextcore_data") / "context.db")
+        self._db_path = db_path
+        self._init_db()
+
+    def _init_db(self):
+        Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
+        with sqlite3.connect(self._db_path) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS webhooks (
+                    webhook_id TEXT PRIMARY KEY,
+                    tenant_id TEXT NOT NULL,
+                    url TEXT NOT NULL,
+                    event_types TEXT NOT NULL DEFAULT '[]',
+                    secret TEXT NOT NULL,
+                    status TEXT NOT NULL DEFAULT 'active',
+                    created_at REAL NOT NULL
+                )
+            """)
+
+    def register(
+        self,
+        tenant_id: str,
+        url: str,
+        event_types: Optional[List[str]] = None,
+    ) -> Webhook:
+        webhook = Webhook(
+            webhook_id=str(uuid.uuid4()),
+            tenant_id=tenant_id,
+            url=url,
+            event_types=event_types or ["context_added", "agent_joined"],
+            secret=uuid.uuid4().hex,
+            created_at=time.time(),
+        )
+        with sqlite3.connect(self._db_path) as conn:
+            conn.execute(
+                """INSERT INTO webhooks
+                   (webhook_id, tenant_id, url, event_types, secret, status, created_at)
+                   VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    webhook.webhook_id,
+                    webhook.tenant_id,
+                    webhook.url,
+                    json.dumps(webhook.event_types),
+                    webhook.secret,
+                    webhook.status,
+                    webhook.created_at,
+                ),
+            )
+        return webhook
+
+    def list_webhooks(self, tenant_id: str) -> List[Webhook]:
+        with sqlite3.connect(self._db_path) as conn:
+            conn.row_factory = sqlite3.Row
+            rows = conn.execute(
+                "SELECT * FROM webhooks WHERE tenant_id = ? ORDER BY created_at DESC",
+                (tenant_id,),
+            ).fetchall()
+        return [
+            Webhook(
+                webhook_id=r["webhook_id"],
+                tenant_id=r["tenant_id"],
+                url=r["url"],
+                event_types=json.loads(r["event_types"]),
+                secret=r["secret"],
+                created_at=r["created_at"],
+                status=r["status"],
+            )
+            for r in rows
+        ]
+
+    def delete(self, webhook_id: str) -> bool:
+        with sqlite3.connect(self._db_path) as conn:
+            c = conn.execute("DELETE FROM webhooks WHERE webhook_id = ?", (webhook_id,))
+            return c.rowcount > 0
+
+    def get(self, webhook_id: str) -> Optional[Webhook]:
+        with sqlite3.connect(self._db_path) as conn:
+            conn.row_factory = sqlite3.Row
+            r = conn.execute(
+                "SELECT * FROM webhooks WHERE webhook_id = ?", (webhook_id,)
+            ).fetchone()
+        if not r:
+            return None
+        return Webhook(
+            webhook_id=r["webhook_id"],
+            tenant_id=r["tenant_id"],
+            url=r["url"],
+            event_types=json.loads(r["event_types"]),
+            secret=r["secret"],
+            created_at=r["created_at"],
+            status=r["status"],
+        )
+
+    @staticmethod
+    def sign_payload(payload: dict, secret: str) -> str:
+        """HMAC-SHA256 signature for webhook payload."""
+        body = json.dumps(payload, sort_keys=True).encode()
+        return hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
+
+    def dispatch(self, tenant_id: str, event: dict):
+        """Fire webhooks matching event type (best-effort, non-blocking)."""
+        hooks = self.list_webhooks(tenant_id)
+        event_type = event.get("event_type", "")
+
+        for hook in hooks:
+            if hook.status != "active":
+                continue
+            if event_type and hook.event_types and event_type not in hook.event_types:
+                continue
+
+            try:
+                self._deliver(hook, event)
+            except Exception as e:
+                logger.warning("Webhook dispatch failed for %s: %s", hook.webhook_id, e)
+
+    def _deliver(self, hook: Webhook, event: dict, max_retries: int = 3):
+        """Deliver a webhook with retry + exponential backoff."""
+        import urllib.request
+
+        sig = self.sign_payload(event, hook.secret)
+        body = json.dumps(event, default=str).encode("utf-8")
+
+        for attempt in range(max_retries):
+            try:
+                req = urllib.request.Request(
+                    hook.url,
+                    data=body,
+                    headers={
+                        "Content-Type": "application/json",
+                        "X-Webhook-Signature": sig,
+                        "X-Webhook-Event": event.get("event_type", ""),
+                        "X-Webhook-ID": hook.webhook_id,
+                    },
+                    method="POST",
+                )
+                with urllib.request.urlopen(req, timeout=10) as resp:
+                    if resp.status < 300:
+                        logger.info("Webhook delivered: %s → %s (attempt %d)",
+                                     event.get("event_type"), hook.url, attempt + 1)
+                        return
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    logger.error("Webhook delivery failed after %d attempts: %s → %s: %s",
+                                  max_retries, event.get("event_type"), hook.url, e)
+                else:
+                    time.sleep(2 ** attempt)  # 1s, 2s, 4s backoff

contextsynapse/context/working_memory.py

@@ -0,0 +1,224 @@
+"""
+WorkingMemory — Per-Agent Context Cache
+========================================
+Redis-backed (with in-memory fallback) short-lived cache that stores
+pre-built context payloads per agent+namespace.  Designed to make
+repeated ``orient()`` calls return in sub-5 ms when the underlying
+graph has not changed.
+
+Key layout (Redis):
+    contextcore:wm:{namespace}:{agent_id}  →  Hash
+        content         JSON-encoded payload
+        topic_keywords  JSON list of strings
+        task_id         optional current task id
+        _ts             Unix timestamp (float)
+
+TTL is enforced both via Redis EXPIRE and a client-side timestamp
+check so the in-memory fallback behaves identically.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+_PREFIX = "contextcore:wm"
+
+# ---------------------------------------------------------------------------
+# Singleton
+# ---------------------------------------------------------------------------
+
+_instance: Optional["WorkingMemory"] = None
+
+
+def get_working_memory(redis_client=None, ttl_s: int = 60) -> "WorkingMemory":
+    """Return (or create) the singleton WorkingMemory instance."""
+    global _instance
+    if _instance is None:
+        _instance = WorkingMemory(redis_client=redis_client, ttl_s=ttl_s)
+    return _instance
+
+
+# ---------------------------------------------------------------------------
+# WorkingMemory
+# ---------------------------------------------------------------------------
+
+
+class WorkingMemory:
+    """Per-agent working-memory cache with Redis or in-memory backend."""
+
+    def __init__(self, redis_client=None, ttl_s: int = 60):
+        self._redis = redis_client
+        self._ttl_s = ttl_s
+        # In-memory fallback: {redis_key: {content, topic_keywords, task_id, _ts}}
+        self._local: Dict[str, Dict[str, Any]] = {}
+        self._hits = 0
+        self._misses = 0
+
+    # -- helpers -------------------------------------------------------------
+
+    @staticmethod
+    def _key(namespace: str, agent_id: str) -> str:
+        return f"{_PREFIX}:{namespace}:{agent_id}"
+
+    def _is_expired(self, ts: float) -> bool:
+        return (time.time() - ts) > self._ttl_s
+
+    # -- public API ----------------------------------------------------------
+
+    def get(self, agent_id: str, namespace: str) -> Optional[Any]:
+        """Return cached content or ``None`` on miss / expiry."""
+        key = self._key(namespace, agent_id)
+
+        if self._redis is not None:
+            return self._get_redis(key)
+        return self._get_local(key)
+
+    def set(
+        self,
+        agent_id: str,
+        namespace: str,
+        content: Any,
+        topic_keywords: List[str],
+        task_id: Optional[str] = None,
+    ) -> None:
+        """Store *content* in the cache with the configured TTL."""
+        key = self._key(namespace, agent_id)
+        entry = {
+            "content": json.dumps(content),
+            "topic_keywords": json.dumps(topic_keywords),
+            "task_id": task_id or "",
+            "_ts": str(time.time()),
+        }
+
+        if self._redis is not None:
+            self._redis.hset(key, mapping=entry)
+            self._redis.expire(key, self._ttl_s)
+        else:
+            self._local[key] = entry
+
+    def invalidate(self, agent_id: str, namespace: str) -> None:
+        """Remove a specific agent's cached context."""
+        key = self._key(namespace, agent_id)
+        if self._redis is not None:
+            self._redis.delete(key)
+        else:
+            self._local.pop(key, None)
+
+    def invalidate_by_keywords(self, namespace: str, keywords: List[str]) -> None:
+        """Invalidate all agents in *namespace* whose topic_keywords overlap.
+
+        Also emits ``context_invalidated`` events so connected agents
+        re-orient immediately (CMA Phase 2 — reactive push).
+        """
+        kw_set = set(keywords)
+        prefix = f"{_PREFIX}:{namespace}:"
+        invalidated_agents: List[str] = []
+
+        if self._redis is not None:
+            cursor = "0"
+            while True:
+                cursor, keys = self._redis.scan(
+                    cursor=cursor, match=f"{prefix}*", count=200
+                )
+                for k in keys:
+                    raw = self._redis.hget(k, "topic_keywords")
+                    if raw:
+                        stored = set(json.loads(raw))
+                        if stored & kw_set:
+                            self._redis.delete(k)
+                            # Extract agent_id from key: contextcore:wm:{ns}:{agent_id}
+                            k_str = k.decode() if isinstance(k, bytes) else k
+                            parts = k_str.split(":")
+                            if len(parts) >= 4:
+                                invalidated_agents.append(parts[-1])
+                if cursor == 0 or cursor == b"0":
+                    break
+        else:
+            to_remove = []
+            for k, entry in self._local.items():
+                if not k.startswith(prefix):
+                    continue
+                stored = set(json.loads(entry["topic_keywords"]))
+                if stored & kw_set:
+                    to_remove.append(k)
+                    parts = k.split(":")
+                    if len(parts) >= 4:
+                        invalidated_agents.append(parts[-1])
+            for k in to_remove:
+                del self._local[k]
+
+        # Reactive push: notify invalidated agents to re-orient
+        if invalidated_agents:
+            self._emit_invalidation(namespace, invalidated_agents, keywords)
+
+    def invalidate_namespace(self, namespace: str) -> None:
+        """Remove all cached entries for a namespace."""
+        prefix = f"{_PREFIX}:{namespace}:"
+
+        if self._redis is not None:
+            cursor = "0"
+            while True:
+                cursor, keys = self._redis.scan(
+                    cursor=cursor, match=f"{prefix}*", count=200
+                )
+                if keys:
+                    self._redis.delete(*keys)
+                if cursor == 0 or cursor == b"0":
+                    break
+        else:
+            self._local = {
+                k: v for k, v in self._local.items() if not k.startswith(prefix)
+            }
+
+    def _emit_invalidation(self, namespace: str, agent_ids: List[str], keywords: List[str]) -> None:
+        """Emit context_invalidated event so agents re-orient immediately."""
+        try:
+            from ..api.events import event_bus
+            for agent_id in agent_ids:
+                event_bus.emit("context_invalidated", {
+                    "namespace": namespace,
+                    "agent_id": agent_id,
+                    "keywords": keywords[:10],
+                    "reason": "graph_write",
+                })
+            logger.debug("[WM] Reactive push: %d agent(s) notified in %s", len(agent_ids), namespace)
+        except Exception:
+            pass  # event bus not available (e.g. during tests)
+
+    def stats(self) -> Dict[str, int]:
+        """Return hit/miss counters."""
+        return {"hits": self._hits, "misses": self._misses}
+
+    # -- internals -----------------------------------------------------------
+
+    def _get_redis(self, key: str) -> Optional[Any]:
+        raw = self._redis.hgetall(key)
+        if not raw:
+            self._misses += 1
+            return None
+        ts = float(raw.get(b"_ts") or raw.get("_ts") or 0)
+        if self._is_expired(ts):
+            self._redis.delete(key)
+            self._misses += 1
+            return None
+        self._hits += 1
+        content_raw = raw.get(b"content") or raw.get("content")
+        return json.loads(content_raw)
+
+    def _get_local(self, key: str) -> Optional[Any]:
+        entry = self._local.get(key)
+        if entry is None:
+            self._misses += 1
+            return None
+        ts = float(entry["_ts"])
+        if self._is_expired(ts):
+            del self._local[key]
+            self._misses += 1
+            return None
+        self._hits += 1
+        return json.loads(entry["content"])

contextsynapse/core/__init__.py

@@ -0,0 +1,18 @@
+"""
+Core data structures for AIContextDB.
+"""
+
+# Only import what actually exists
+from .graph_structures import GraphNode, GraphEdge
+from .hybrid_graph_storage import AIContextDB
+from .registry import GraphRegistry
+from .registry_metadata import GraphRegistryMetadata, registry_metadata
+
+__all__ = [
+    "GraphNode",
+    "GraphEdge", 
+    "AIContextDB",
+    "GraphRegistry",
+    "GraphRegistryMetadata",
+    "registry_metadata"
+]

contextsynapse/core/archiver.py

@@ -0,0 +1,237 @@
+"""
+Graph Archiver — TTL-based node archival to cold storage.
+
+Moves old nodes from active graph to archive HDF5 files based on _created_at.
+Archived nodes are saved separately and removed from the active graph.
+
+Usage:
+    from contextsynapse.core.archiver import GraphArchiver
+    archiver = GraphArchiver(graph_registry)
+    result = archiver.archive_old_nodes("my_graph", ttl_days=30)
+    # {"archived": 150, "remaining": 850, "archive_path": "..."}
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+ARCHIVE_DIR = "contextcore_data/archive"
+
+
+class GraphArchiver:
+    """TTL-based graph archival — moves old nodes to cold HDF5 storage."""
+
+    def __init__(self, graph_registry=None):
+        self._registry = graph_registry
+        Path(ARCHIVE_DIR).mkdir(parents=True, exist_ok=True)
+
+    def archive_old_nodes(
+        self,
+        graph_name: str,
+        ttl_days: int = 30,
+        dry_run: bool = False,
+    ) -> Dict[str, Any]:
+        """Archive nodes older than ttl_days to a separate HDF5 file.
+
+        Args:
+            graph_name: Name of the graph to archive from
+            ttl_days: Nodes older than this many days get archived
+            dry_run: If True, report what would be archived without doing it
+
+        Returns:
+            {archived, remaining, archive_path, cutoff_date}
+        """
+        if not self._registry:
+            return {"error": "No graph registry"}
+
+        db = self._registry.get_graph(graph_name)
+        if not db:
+            return {"error": f"Graph '{graph_name}' not found"}
+
+        cutoff = (datetime.now(timezone.utc) - timedelta(days=ttl_days)).isoformat()
+        all_nodes = db.get_all_nodes()
+
+        # Partition nodes by age
+        old_nodes = []
+        keep_nodes = []
+        for node in all_nodes:
+            props = node.properties if hasattr(node, "properties") else {}
+            created = props.get("_created_at", "")
+            if created and created < cutoff:
+                old_nodes.append(node)
+            else:
+                keep_nodes.append(node)
+
+        result = {
+            "graph": graph_name,
+            "total": len(all_nodes),
+            "to_archive": len(old_nodes),
+            "remaining": len(keep_nodes),
+            "cutoff_date": cutoff[:10],
+            "ttl_days": ttl_days,
+            "dry_run": dry_run,
+        }
+
+        if not old_nodes:
+            result["archived"] = 0
+            result["message"] = "No nodes older than cutoff"
+            return result
+
+        if dry_run:
+            result["archived"] = 0
+            result["message"] = f"Would archive {len(old_nodes)} nodes"
+            return result
+
+        # Save old nodes to archive HDF5
+        timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+        safe_name = graph_name.replace(":", "_").replace("/", "_")
+        archive_path = os.path.join(ARCHIVE_DIR, f"{safe_name}_archive_{timestamp}.json")
+
+        archived_data = []
+        for node in old_nodes:
+            archived_data.append({
+                "id": node.id if hasattr(node, "id") else str(node),
+                "label": node.label if hasattr(node, "label") else "",
+                "properties": node.properties if hasattr(node, "properties") else {},
+            })
+
+        # Also archive edges connected to archived nodes
+        archived_node_ids = {n["id"] for n in archived_data}
+        all_edges = db.get_all_edges()
+        archived_edges = []
+        for edge in all_edges:
+            src = edge.source if hasattr(edge, "source") else ""
+            tgt = edge.target if hasattr(edge, "target") else ""
+            if src in archived_node_ids or tgt in archived_node_ids:
+                archived_edges.append({
+                    "id": edge.id if hasattr(edge, "id") else "",
+                    "source": src,
+                    "target": tgt,
+                    "label": edge.label if hasattr(edge, "label") else "",
+                    "properties": edge.properties if hasattr(edge, "properties") else {},
+                })
+
+        # Write archive file
+        archive_record = {
+            "graph": graph_name,
+            "archived_at": datetime.now(timezone.utc).isoformat(),
+            "cutoff_date": cutoff,
+            "ttl_days": ttl_days,
+            "node_count": len(archived_data),
+            "edge_count": len(archived_edges),
+            "nodes": archived_data,
+            "edges": archived_edges,
+        }
+        with open(archive_path, "w") as f:
+            json.dump(archive_record, f, default=str)
+
+        # Remove archived nodes from active graph
+        removed = 0
+        for node_data in archived_data:
+            try:
+                db.remove_node(node_data["id"])
+                removed += 1
+            except Exception as e:
+                logger.debug("Failed to remove node %s: %s", node_data["id"][:12], e)
+
+        # Save the active graph
+        try:
+            self._registry.save_graph(graph_name, create_checkpoint=True)
+        except Exception as e:
+            logger.warning("Failed to save graph after archival: %s", e)
+
+        result["archived"] = removed
+        result["archive_path"] = archive_path
+        result["edges_archived"] = len(archived_edges)
+        result["message"] = f"Archived {removed} nodes + {len(archived_edges)} edges"
+
+        logger.info("Archived %d nodes from %s (cutoff: %s) → %s",
+                     removed, graph_name, cutoff[:10], archive_path)
+        return result
+
+    def list_archives(self, graph_name: Optional[str] = None) -> List[Dict[str, Any]]:
+        """List archive files, optionally filtered by graph name."""
+        archives = []
+        for fname in sorted(os.listdir(ARCHIVE_DIR), reverse=True):
+            if not fname.endswith(".json"):
+                continue
+            if graph_name:
+                safe = graph_name.replace(":", "_").replace("/", "_")
+                if not fname.startswith(safe):
+                    continue
+            fpath = os.path.join(ARCHIVE_DIR, fname)
+            try:
+                with open(fpath) as f:
+                    header = json.load(f)
+                archives.append({
+                    "filename": fname,
+                    "graph": header.get("graph", ""),
+                    "archived_at": header.get("archived_at", ""),
+                    "node_count": header.get("node_count", 0),
+                    "edge_count": header.get("edge_count", 0),
+                    "ttl_days": header.get("ttl_days", 0),
+                    "size_bytes": os.path.getsize(fpath),
+                })
+            except Exception:
+                pass
+        return archives
+
+    def restore_archive(self, filename: str, graph_name: Optional[str] = None) -> Dict[str, Any]:
+        """Restore archived nodes back into a graph."""
+        fpath = os.path.join(ARCHIVE_DIR, filename)
+        if not os.path.exists(fpath):
+            return {"error": f"Archive file not found: {filename}"}
+
+        with open(fpath) as f:
+            archive = json.load(f)
+
+        target_graph = graph_name or archive.get("graph", "")
+        if not target_graph or not self._registry:
+            return {"error": "No target graph specified"}
+
+        db = self._registry.get_graph(target_graph)
+        if not db:
+            return {"error": f"Graph '{target_graph}' not found"}
+
+        from .graph_structures import GraphNode, GraphEdge
+
+        restored_nodes = 0
+        for nd in archive.get("nodes", []):
+            try:
+                node = GraphNode(id=nd["id"], label=nd["label"], properties=nd.get("properties", {}))
+                db.add_node(node)
+                restored_nodes += 1
+            except Exception:
+                pass
+
+        restored_edges = 0
+        for ed in archive.get("edges", []):
+            try:
+                edge = GraphEdge(
+                    id=ed.get("id", ""),
+                    source=ed["source"], target=ed["target"],
+                    label=ed["label"], properties=ed.get("properties", {}),
+                )
+                db.add_edge(edge)
+                restored_edges += 1
+            except Exception:
+                pass
+
+        try:
+            self._registry.save_graph(target_graph)
+        except Exception:
+            pass
+
+        return {
+            "restored_nodes": restored_nodes,
+            "restored_edges": restored_edges,
+            "from_archive": filename,
+            "to_graph": target_graph,
+        }