contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/storage/redis_graph_adapter.py

@@ -0,0 +1,186 @@
+"""
+Redis Graph Adapter — wraps RedisGraphStorage to match the CSRGraphStorageAdapter interface.
+
+This allows AIContextDB to use Redis as its storage backend without
+changing any calling code. All 51 files that call add_node, get_node, etc.
+continue to work unchanged.
+
+Usage:
+    from contextsynapse.storage.redis_graph_adapter import RedisGraphAdapter
+
+    adapter = RedisGraphAdapter("my_graph")
+    adapter.add_node(node_id, label, properties)
+    node = adapter.get_node(node_id)
+    neighbors = adapter.get_neighbors(node_id)
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+from ..core.graph_structures import GraphNode, GraphEdge
+
+logger = logging.getLogger(__name__)
+
+
+class _CompatEdge:
+    """Edge wrapper with both GraphEdge attrs (source/target) and CSREdge attrs (source_id/target_id)."""
+    def __init__(self, source: str, target: str, label: str, properties: dict, edge_id: str = ""):
+        self.id = edge_id
+        self.source = source
+        self.target = target
+        self.label = label
+        self.properties = properties
+        # CSREdge compatibility
+        self.source_id = source
+        self.target_id = target
+        self.edge_type = label
+        self.weight = 1.0
+
+
+class RedisGraphAdapter:
+    """Adapter that makes RedisGraphStorage compatible with AIContextDB's CSR interface."""
+
+    def __init__(self, graph_name: str, redis_url: str = None):
+        from .redis_graph_storage import RedisGraphStorage
+        self._store = RedisGraphStorage(graph_name, redis_url)
+        self._graph_name = graph_name
+
+    # ------------------------------------------------------------------
+    # Node operations (match CSRGraphStorageAdapter interface)
+    # ------------------------------------------------------------------
+
+    def add_node(self, node_id: str, label: str, properties: Dict[str, Any] = None, alias: str = None) -> bool:
+        return self._store.add_node(node_id, label, properties or {})
+
+    def get_node(self, node_id: str):
+        rn = self._store.get_node(node_id)
+        if not rn:
+            return None
+        node = GraphNode(id=rn.id, label=rn.node_type, properties=rn.properties)
+        node.node_type = rn.node_type  # CSRNode compatibility
+        return node
+
+    def get_all_nodes(self, node_type: str = None) -> List[GraphNode]:
+        nodes = []
+        for n in self._store.get_all_nodes(node_type=node_type):
+            gn = GraphNode(id=n.id, label=n.node_type, properties=n.properties)
+            gn.node_type = n.node_type  # CSRNode compatibility
+            nodes.append(gn)
+        return nodes
+
+    def remove_node(self, node_id: str) -> bool:
+        return self._store.remove_node(node_id)
+
+    def update_node_properties(self, node_id: str, properties: Dict[str, Any]) -> bool:
+        return self._store.update_node_properties(node_id, properties)
+
+    def update_node(self, node_id: str, properties: Dict[str, Any]) -> bool:
+        """Alias for update_node_properties — used by PortfolioManager.update_prices."""
+        return self._store.update_node_properties(node_id, properties)
+
+    def get_node_count(self) -> int:
+        return self._store.get_node_count()
+
+    def get_nodes_by_type(self, node_type: str) -> List[str]:
+        return self._store.get_nodes_by_type(node_type)
+
+    # ------------------------------------------------------------------
+    # Edge operations
+    # ------------------------------------------------------------------
+
+    def add_edge(self, source_or_edge, target: str = None, label: str = None,
+                 properties: Dict[str, Any] = None) -> bool:
+        """Accept both GraphEdge object or positional args (source, target, label, props)."""
+        if isinstance(source_or_edge, GraphEdge):
+            edge = source_or_edge
+            return self._store.add_edge(
+                source_id=edge.source, target_id=edge.target,
+                edge_type=edge.label, properties=edge.properties, edge_id=edge.id,
+            )
+        # Positional: add_edge(source, target, label, properties)
+        return self._store.add_edge(
+            source_id=source_or_edge, target_id=target or "",
+            edge_type=label or "", properties=properties or {},
+        )
+
+    def get_all_edges(self) -> list:
+        return [
+            _CompatEdge(e.source_id, e.target_id, e.edge_type, e.properties)
+            for e in self._store.get_all_edges()
+        ]
+
+    def get_edge_count(self) -> int:
+        return self._store.get_edge_count()
+
+    # ------------------------------------------------------------------
+    # Adjacency
+    # ------------------------------------------------------------------
+
+    def get_neighbors(self, node_id: str, edge_type: str = None) -> List[Tuple[str, Any]]:
+        """Returns [(neighbor_id, edge)] matching CSR interface."""
+        results = self._store.get_neighbors(node_id, edge_type)
+        return [
+            (nid, _CompatEdge(node_id, nid, e.edge_type, e.properties))
+            for nid, e in results
+        ]
+
+    def get_incoming_neighbors(self, node_id: str, edge_type: str = None) -> List[Tuple[str, Any]]:
+        results = self._store.get_incoming_neighbors(node_id, edge_type)
+        return [
+            (src_id, _CompatEdge(src_id, node_id, e.edge_type, e.properties))
+            for src_id, e in results
+        ]
+
+    # ------------------------------------------------------------------
+    # Traversal + property lookup
+    # ------------------------------------------------------------------
+
+    def traverse(self, start_node_id: str, edge_label: str = None,
+                 max_depth: int = 3, direction: str = "OUTGOING") -> List[Tuple[str, int]]:
+        """BFS traversal from start node. Returns [(node_id, depth)]."""
+        visited = set()
+        result = []
+        queue = [(start_node_id, 0)]
+        while queue:
+            nid, depth = queue.pop(0)
+            if nid in visited or depth > max_depth:
+                continue
+            visited.add(nid)
+            result.append((nid, depth))
+            if depth < max_depth:
+                if direction == "OUTGOING":
+                    neighbors = self.get_neighbors(nid, edge_label)
+                else:
+                    neighbors = self.get_incoming_neighbors(nid, edge_label)
+                for neighbor_id, _ in neighbors:
+                    if neighbor_id not in visited:
+                        queue.append((neighbor_id, depth + 1))
+        return result
+
+    def get_nodes_by_property(self, property_name: str, property_value: Any) -> List[str]:
+        """Find nodes by property value. Scans all nodes (no index)."""
+        result = []
+        for node in self._store.get_all_nodes():
+            if node.properties.get(property_name) == property_value:
+                result.append(node.id)
+        return result
+
+    # ------------------------------------------------------------------
+    # Graph management
+    # ------------------------------------------------------------------
+
+    def delete_graph(self) -> int:
+        return self._store.delete_graph()
+
+    def export_json(self) -> Dict[str, Any]:
+        return self._store.export_json()
+
+    @property
+    def node_count(self) -> int:
+        return self._store.node_count
+
+    @property
+    def edge_count(self) -> int:
+        return self._store.edge_count

contextsynapse/storage/redis_graph_storage.py

@@ -0,0 +1,375 @@
+"""
+Redis Graph Storage — drop-in replacement for CSRGraphStorage.
+
+Stores nodes, edges, and adjacency in Redis hashes and sets.
+Workers never load the full graph into memory. All operations
+are O(1) or O(degree) via Redis.
+
+Data model:
+  node:{graph}:{id}       → HASH (label, properties JSON)
+  edge:{graph}:{id}       → HASH (source, target, label, properties JSON)
+  adj:{graph}:{id}:out    → SET of edge IDs (outgoing)
+  adj:{graph}:{id}:in     → SET of edge IDs (incoming)
+  nodes:{graph}           → SET of all node IDs
+  edges:{graph}           → SET of all edge IDs
+  ntype:{graph}:{label}   → SET of node IDs with this label
+  etype:{graph}:{label}   → SET of edge IDs with this label
+
+Disk backup: periodic full export to graph.json (same format as before).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class RedisNode:
+    id: str
+    node_type: str
+    properties: Dict[str, Any]
+
+
+@dataclass
+class RedisEdge:
+    source_id: str
+    target_id: str
+    edge_type: str
+    properties: Dict[str, Any]
+    weight: float = 1.0
+
+
+class RedisGraphStorage:
+    """Graph storage backed by Redis — no in-memory loading required."""
+
+    def __init__(self, graph_name: str, redis_url: str = None):
+        import redis
+        self._url = redis_url or os.environ.get("CONTEXTSYNAPSE_REDIS_URL") or os.environ.get("AICONTEXTDB_REDIS_URL", "redis://localhost:6379/0")
+        # On Windows, "localhost" resolves to IPv6 (::1) first, adding ~2s latency on first connect.
+        # Force IPv4 by rewriting localhost → 127.0.0.1.
+        _connect_url = self._url.replace("://localhost:", "://127.0.0.1:")
+        self._r = redis.from_url(_connect_url, decode_responses=True)
+        self._graph = graph_name
+        self._prefix = f"g:{graph_name}:"
+
+    # Key helpers
+    def _nk(self, node_id: str) -> str:
+        return f"{self._prefix}n:{node_id}"
+
+    def _ek(self, edge_id: str) -> str:
+        return f"{self._prefix}e:{edge_id}"
+
+    def _adj_out(self, node_id: str) -> str:
+        return f"{self._prefix}adj:{node_id}:out"
+
+    def _adj_in(self, node_id: str) -> str:
+        return f"{self._prefix}adj:{node_id}:in"
+
+    # ------------------------------------------------------------------
+    # Node operations
+    # ------------------------------------------------------------------
+
+    def add_node(self, node_id: str, node_type: str, properties: Dict[str, Any] = None) -> bool:
+        from datetime import datetime, timezone
+        props = properties or {}
+        if "_created_at" not in props:
+            props["_created_at"] = datetime.now(timezone.utc).isoformat()
+        pipe = self._r.pipeline()
+        pipe.hset(self._nk(node_id), mapping={
+            "id": node_id,
+            "label": node_type,
+            "props": json.dumps(props, default=str),
+        })
+        pipe.sadd(f"{self._prefix}nodes", node_id)
+        pipe.sadd(f"{self._prefix}ntype:{node_type}", node_id)
+        pipe.execute()
+        return True
+
+    def get_node(self, node_id: str) -> Optional[RedisNode]:
+        data = self._r.hgetall(self._nk(node_id))
+        if not data:
+            return None
+        try:
+            props = json.loads(data.get("props", "{}"))
+        except Exception:
+            props = {}
+        return RedisNode(
+            id=data.get("id", node_id),
+            node_type=data.get("label", ""),
+            properties=props,
+        )
+
+    def get_node_count(self) -> int:
+        return self._r.scard(f"{self._prefix}nodes")
+
+    def get_all_node_ids(self) -> Set[str]:
+        return self._r.smembers(f"{self._prefix}nodes")
+
+    def get_nodes_by_type(self, node_type: str) -> List[str]:
+        return list(self._r.smembers(f"{self._prefix}ntype:{node_type}"))
+
+    def update_node_properties(self, node_id: str, properties: Dict[str, Any]) -> bool:
+        from datetime import datetime, timezone
+        existing = self.get_node(node_id)
+        if not existing:
+            return False
+        properties["_updated_at"] = datetime.now(timezone.utc).isoformat()
+        merged = {**existing.properties, **properties}
+        self._r.hset(self._nk(node_id), "props", json.dumps(merged, default=str))
+        return True
+
+    def remove_node(self, node_id: str) -> bool:
+        data = self._r.hgetall(self._nk(node_id))
+        if not data:
+            return False
+        label = data.get("label", "")
+
+        # Remove edges connected to this node
+        for edge_id in self._r.smembers(self._adj_out(node_id)):
+            self._remove_edge_internal(edge_id)
+        for edge_id in self._r.smembers(self._adj_in(node_id)):
+            self._remove_edge_internal(edge_id)
+
+        pipe = self._r.pipeline()
+        pipe.delete(self._nk(node_id))
+        pipe.delete(self._adj_out(node_id))
+        pipe.delete(self._adj_in(node_id))
+        pipe.srem(f"{self._prefix}nodes", node_id)
+        if label:
+            pipe.srem(f"{self._prefix}ntype:{label}", node_id)
+        pipe.execute()
+        return True
+
+    # ------------------------------------------------------------------
+    # Edge operations
+    # ------------------------------------------------------------------
+
+    def add_edge(self, source_id: str, target_id: str, edge_type: str,
+                 properties: Dict[str, Any] = None, weight: float = 1.0,
+                 edge_id: str = None) -> bool:
+        import uuid
+        from datetime import datetime, timezone
+        eid = edge_id or str(uuid.uuid4())
+        props = properties or {}
+        if "_created_at" not in props:
+            props["_created_at"] = datetime.now(timezone.utc).isoformat()
+
+        pipe = self._r.pipeline()
+        pipe.hset(self._ek(eid), mapping={
+            "id": eid,
+            "source": source_id,
+            "target": target_id,
+            "label": edge_type,
+            "weight": str(weight),
+            "props": json.dumps(props, default=str),
+        })
+        pipe.sadd(f"{self._prefix}edges", eid)
+        pipe.sadd(f"{self._prefix}etype:{edge_type}", eid)
+        pipe.sadd(self._adj_out(source_id), eid)
+        pipe.sadd(self._adj_in(target_id), eid)
+        pipe.execute()
+        return True
+
+    def get_edge(self, edge_id: str) -> Optional[RedisEdge]:
+        data = self._r.hgetall(self._ek(edge_id))
+        if not data:
+            return None
+        try:
+            props = json.loads(data.get("props", "{}"))
+        except Exception:
+            props = {}
+        return RedisEdge(
+            source_id=data.get("source", ""),
+            target_id=data.get("target", ""),
+            edge_type=data.get("label", ""),
+            properties=props,
+            weight=float(data.get("weight", 1.0)),
+        )
+
+    def get_edge_count(self) -> int:
+        return self._r.scard(f"{self._prefix}edges")
+
+    def _remove_edge_internal(self, edge_id: str):
+        data = self._r.hgetall(self._ek(edge_id))
+        if not data:
+            return
+        source = data.get("source", "")
+        target = data.get("target", "")
+        label = data.get("label", "")
+
+        pipe = self._r.pipeline()
+        pipe.delete(self._ek(edge_id))
+        pipe.srem(f"{self._prefix}edges", edge_id)
+        if label:
+            pipe.srem(f"{self._prefix}etype:{label}", edge_id)
+        if source:
+            pipe.srem(self._adj_out(source), edge_id)
+        if target:
+            pipe.srem(self._adj_in(target), edge_id)
+        pipe.execute()
+
+    # ------------------------------------------------------------------
+    # Adjacency / neighbor queries
+    # ------------------------------------------------------------------
+
+    def get_neighbors(self, node_id: str, edge_type: str = None) -> List[Tuple[str, RedisEdge]]:
+        """Get outgoing neighbors: [(neighbor_id, edge)]"""
+        edge_ids = self._r.smembers(self._adj_out(node_id))
+        result = []
+        for eid in edge_ids:
+            edge = self.get_edge(eid)
+            if edge and (edge_type is None or edge.edge_type == edge_type):
+                result.append((edge.target_id, edge))
+        return result
+
+    def get_incoming_neighbors(self, node_id: str, edge_type: str = None) -> List[Tuple[str, RedisEdge]]:
+        """Get incoming neighbors: [(source_id, edge)]"""
+        edge_ids = self._r.smembers(self._adj_in(node_id))
+        result = []
+        for eid in edge_ids:
+            edge = self.get_edge(eid)
+            if edge and (edge_type is None or edge.edge_type == edge_type):
+                result.append((edge.source_id, edge))
+        return result
+
+    # ------------------------------------------------------------------
+    # Bulk operations
+    # ------------------------------------------------------------------
+
+    def get_all_nodes(self, node_type: str = None) -> List[RedisNode]:
+        """Get all nodes (or filtered by type). Use sparingly — O(n)."""
+        if node_type:
+            ids = self._r.smembers(f"{self._prefix}ntype:{node_type}")
+        else:
+            ids = self._r.smembers(f"{self._prefix}nodes")
+        if not ids:
+            return []
+        # Batch fetch with pipeline
+        pipe = self._r.pipeline()
+        for nid in ids:
+            pipe.hgetall(self._nk(nid))
+        results = pipe.execute()
+        nodes = []
+        for data in results:
+            if data:
+                try:
+                    props = json.loads(data.get("props", "{}"))
+                except Exception:
+                    props = {}
+                nodes.append(RedisNode(
+                    id=data.get("id", ""),
+                    node_type=data.get("label", ""),
+                    properties=props,
+                ))
+        return nodes
+
+    def get_all_edges(self, edge_type: str = None) -> List[RedisEdge]:
+        """Get all edges (or filtered by type). Use sparingly — O(m)."""
+        if edge_type:
+            ids = self._r.smembers(f"{self._prefix}etype:{edge_type}")
+        else:
+            ids = self._r.smembers(f"{self._prefix}edges")
+        if not ids:
+            return []
+        pipe = self._r.pipeline()
+        for eid in ids:
+            pipe.hgetall(self._ek(eid))
+        results = pipe.execute()
+        edges = []
+        for data in results:
+            if data:
+                try:
+                    props = json.loads(data.get("props", "{}"))
+                except Exception:
+                    props = {}
+                edges.append(RedisEdge(
+                    source_id=data.get("source", ""),
+                    target_id=data.get("target", ""),
+                    edge_type=data.get("label", ""),
+                    properties=props,
+                    weight=float(data.get("weight", 1.0)),
+                ))
+        return edges
+
+    # ------------------------------------------------------------------
+    # Graph deletion
+    # ------------------------------------------------------------------
+
+    def delete_graph(self) -> int:
+        """Delete all keys for this graph. Returns count of keys deleted."""
+        pattern = f"{self._prefix}*"
+        count = 0
+        cursor = 0
+        while True:
+            cursor, keys = self._r.scan(cursor, match=pattern, count=200)
+            if keys:
+                self._r.delete(*keys)
+                count += len(keys)
+            if cursor == 0:
+                break
+        logger.info("[REDIS-GRAPH] Deleted graph '%s': %d keys", self._graph, count)
+        return count
+
+    # ------------------------------------------------------------------
+    # Export (for disk backup)
+    # ------------------------------------------------------------------
+
+    def export_json(self) -> Dict[str, Any]:
+        """Export entire graph as JSON dict (for disk backup)."""
+        nodes = []
+        for n in self.get_all_nodes():
+            nodes.append({"id": n.id, "label": n.node_type, "properties": n.properties})
+        edges = []
+        for e in self.get_all_edges():
+            edges.append({
+                "source": e.source_id, "target": e.target_id,
+                "label": e.edge_type, "properties": e.properties,
+            })
+        return {
+            "nodes": nodes,
+            "edges": edges,
+            "metadata": {
+                "name": self._graph,
+                "node_count": len(nodes),
+                "edge_count": len(edges),
+                "storage_backend": "redis",
+            },
+        }
+
+    @property
+    def nodes(self) -> Dict[str, RedisNode]:
+        """Compatibility: returns dict-like {node_id: node}. Lazy — fetches from Redis."""
+        result = {}
+        for n in self.get_all_nodes():
+            result[n.id] = n
+        return result
+
+    @property
+    def node_count(self) -> int:
+        return self.get_node_count()
+
+    @property
+    def edge_count(self) -> int:
+        return self.get_edge_count()
+
+    @property
+    def node_types(self) -> Dict[str, Set[str]]:
+        """Compatibility: returns {label: {node_ids}}."""
+        result: Dict[str, Set[str]] = {}
+        # Scan ntype keys
+        pattern = f"{self._prefix}ntype:*"
+        cursor = 0
+        while True:
+            cursor, keys = self._r.scan(cursor, match=pattern, count=100)
+            for key in keys:
+                label = key.replace(f"{self._prefix}ntype:", "")
+                result[label] = self._r.smembers(key)
+            if cursor == 0:
+                break
+        return result

contextsynapse/storage/router/__init__.py

@@ -0,0 +1,29 @@
+"""Storage Router -- routes data to the right store based on type.
+
+Graph stays thin (entities + edges). Content goes to the configured
+content store (DuckDB default, PostgreSQL for production).
+Vectors go to Qdrant. Blobs go to filesystem.
+
+Configure backend via env:
+    CONTEXTCORE_CONTENT_STORE=duckdb     (default, embedded)
+    CONTEXTCORE_CONTENT_STORE=postgres   (requires DATABASE_URL)
+"""
+from contextsynapse.storage.router.interface import ContentStoreInterface
+from contextsynapse.storage.router.router import StorageRouter, get_storage_router
+from contextsynapse.storage.router.resolver import ContentResolver, get_content_resolver
+from contextsynapse.storage.router.factory import (
+    create_content_store,
+    get_content_store,
+    set_content_store,
+)
+
+__all__ = [
+    "ContentStoreInterface",
+    "StorageRouter",
+    "get_storage_router",
+    "ContentResolver",
+    "get_content_resolver",
+    "create_content_store",
+    "get_content_store",
+    "set_content_store",
+]