contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/search/rag_cache.py

@@ -0,0 +1,167 @@
+"""
+RAG Answer Cache — cache LLM-generated answers to avoid redundant API calls.
+
+Two tiers:
+  1. Redis (shared across workers, 5-min TTL)
+  2. In-process LRU (per-worker fallback, 2-min TTL)
+
+Cache key = hash(graph_name + question_normalised + top_node_ids)
+This means the cache invalidates when:
+  - The question changes
+  - The graph data changes (different nodes retrieved)
+  - TTL expires
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import re
+import time
+from collections import OrderedDict
+from threading import Lock
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# In-process LRU cache (fallback when Redis unavailable)
+# ---------------------------------------------------------------------------
+
+class _InProcessRAGCache:
+    def __init__(self, max_size: int = 200, ttl: float = 120.0):
+        self._max = max_size
+        self._ttl = ttl
+        self._cache: OrderedDict = OrderedDict()
+        self._lock = Lock()
+        self._hits = 0
+        self._misses = 0
+
+    def get(self, key: str) -> Optional[Dict]:
+        with self._lock:
+            if key in self._cache:
+                val, ts = self._cache[key]
+                if time.time() - ts < self._ttl:
+                    self._cache.move_to_end(key)
+                    self._hits += 1
+                    return val
+                del self._cache[key]
+        self._misses += 1
+        return None
+
+    def put(self, key: str, value: Dict):
+        with self._lock:
+            self._cache[key] = (value, time.time())
+            self._cache.move_to_end(key)
+            while len(self._cache) > self._max:
+                self._cache.popitem(last=False)
+
+    def invalidate(self, graph_name: str):
+        with self._lock:
+            keys = [k for k in self._cache if k.startswith(f"{graph_name}:")]
+            for k in keys:
+                del self._cache[k]
+
+    @property
+    def stats(self):
+        return {"size": len(self._cache), "hits": self._hits, "misses": self._misses}
+
+
+# ---------------------------------------------------------------------------
+# Redis-backed RAG cache
+# ---------------------------------------------------------------------------
+
+class _RedisRAGCache:
+    PREFIX = "rag:answer:"
+
+    def __init__(self, redis_url: str, ttl: int = 300):
+        import redis
+        self._r = redis.from_url(redis_url, decode_responses=True)
+        self._ttl = ttl
+        self._hits = 0
+        self._misses = 0
+
+    def get(self, key: str) -> Optional[Dict]:
+        try:
+            data = self._r.get(f"{self.PREFIX}{key}")
+            if data:
+                self._hits += 1
+                return json.loads(data)
+        except Exception:
+            pass
+        self._misses += 1
+        return None
+
+    def put(self, key: str, value: Dict):
+        try:
+            self._r.setex(f"{self.PREFIX}{key}", self._ttl, json.dumps(value, default=str))
+        except Exception:
+            pass
+
+    def invalidate(self, graph_name: str):
+        try:
+            cursor = 0
+            while True:
+                cursor, keys = self._r.scan(cursor, match=f"{self.PREFIX}{graph_name}:*", count=100)
+                if keys:
+                    self._r.delete(*keys)
+                if cursor == 0:
+                    break
+        except Exception:
+            pass
+
+    @property
+    def stats(self):
+        return {"hits": self._hits, "misses": self._misses}
+
+
+# ---------------------------------------------------------------------------
+# Cache key generation
+# ---------------------------------------------------------------------------
+
+def _normalise_question(q: str) -> str:
+    """Normalise question for cache key — lowercase, strip punctuation, collapse whitespace."""
+    q = q.lower().strip()
+    q = re.sub(r"[^\w\s]", "", q)
+    q = re.sub(r"\s+", " ", q)
+    return q
+
+
+def make_cache_key(graph_name: str, question: str, top_node_ids: list = None) -> str:
+    """Build a deterministic cache key from graph + question + retrieved nodes."""
+    parts = [graph_name or "", _normalise_question(question)]
+    if top_node_ids:
+        parts.append(",".join(sorted(top_node_ids[:10])))
+    raw = "|".join(parts)
+    return f"{graph_name}:{hashlib.sha256(raw.encode()).hexdigest()[:16]}"
+
+
+# ---------------------------------------------------------------------------
+# Singleton
+# ---------------------------------------------------------------------------
+
+_cache = None
+
+
+def get_rag_cache():
+    """Get or create the RAG answer cache (Redis if available, in-process fallback)."""
+    global _cache
+    if _cache is not None:
+        return _cache
+
+    url = os.environ.get("CONTEXTSYNAPSE_REDIS_URL") or os.environ.get("AICONTEXTDB_REDIS_URL")
+    if url:
+        try:
+            cache = _RedisRAGCache(url, ttl=300)
+            cache._r.ping()
+            logger.info("RAG answer cache: Redis (TTL=300s)")
+            _cache = cache
+            return _cache
+        except Exception:
+            pass
+
+    _cache = _InProcessRAGCache(max_size=200, ttl=120)
+    logger.info("RAG answer cache: in-process (TTL=120s)")
+    return _cache

contextsynapse/search/reasoning_chain.py

@@ -0,0 +1,231 @@
+"""
+Graph Reasoning Chain — LLM follows edges to answer multi-hop questions.
+
+Unlike flat RAG (retrieve chunks → answer), this traverses the graph:
+  Q: "Who is responsible for billing security?"
+  → Find: Billing --MAINTAINED_BY--> Alice --REPORTS_TO--> Bob (Security Lead)
+  → Answer: "Bob, via Alice who maintains Billing"
+
+The LLM decides which edges to follow at each hop, building a reasoning
+chain that explains HOW it arrived at the answer.
+
+Usage:
+    from contextsynapse.search.reasoning_chain import reason_over_graph
+    result = reason_over_graph(db, "Who maintains the auth module?")
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, List, Optional, Set
+
+logger = logging.getLogger(__name__)
+
+MAX_HOPS = 4
+MAX_NEIGHBORS = 15
+
+
+def reason_over_graph(
+    db,
+    question: str,
+    max_hops: int = MAX_HOPS,
+    use_llm: bool = True,
+) -> Dict[str, Any]:
+    """Multi-hop graph reasoning to answer a question.
+
+    1. Find seed nodes matching the question
+    2. At each hop, decide which edges to follow (LLM or heuristic)
+    3. Build reasoning chain explaining the path
+    4. Generate final answer from collected evidence
+
+    Returns:
+        {answer, chain, nodes_visited, hops, evidence}
+    """
+    # Build adjacency
+    all_nodes = db.get_all_nodes()
+    all_edges = db.get_all_edges()
+
+    node_map = {}
+    for n in all_nodes:
+        nid = n.id if hasattr(n, "id") else ""
+        props = n.properties if hasattr(n, "properties") else {}
+        label = n.label if hasattr(n, "label") else ""
+        node_map[nid] = {"label": label, "props": props, "node": n}
+
+    adj = {}  # node_id → [(edge_label, neighbor_id, edge_props)]
+    for e in all_edges:
+        src = e.source if hasattr(e, "source") else ""
+        tgt = e.target if hasattr(e, "target") else ""
+        lbl = e.label if hasattr(e, "label") else ""
+        eprops = e.properties if hasattr(e, "properties") else {}
+        adj.setdefault(src, []).append((lbl, tgt, eprops))
+        adj.setdefault(tgt, []).append((lbl, src, eprops))
+
+    # Step 1: Find seed nodes
+    keywords = _extract_keywords(question)
+    seeds = _find_seeds(node_map, keywords, max_seeds=5)
+
+    if not seeds:
+        return {"answer": "No relevant nodes found in the graph.", "chain": [], "nodes_visited": 0, "hops": 0}
+
+    # Step 2: Multi-hop traversal
+    visited: Set[str] = set()
+    chain = []  # list of (hop, from_node, edge_label, to_node, relevance)
+    evidence = []  # collected text from nodes along the path
+    frontier = [(nid, 0) for nid in seeds]
+
+    for nid in seeds:
+        info = node_map.get(nid, {})
+        props = info.get("props", {})
+        name = props.get("name") or props.get("title") or props.get("content", "")[:60]
+        evidence.append(f"[{info.get('label', '')}] {name}")
+
+    while frontier and len(chain) < max_hops * len(seeds):
+        current_id, hop = frontier.pop(0)
+        if current_id in visited or hop >= max_hops:
+            continue
+        visited.add(current_id)
+
+        neighbors = adj.get(current_id, [])[:MAX_NEIGHBORS]
+        if not neighbors:
+            continue
+
+        # Score neighbors by keyword relevance
+        scored_neighbors = []
+        for edge_label, neighbor_id, eprops in neighbors:
+            if neighbor_id in visited or neighbor_id not in node_map:
+                continue
+            n_info = node_map[neighbor_id]
+            n_text = _node_text(n_info).lower()
+            relevance = sum(1 for kw in keywords if kw in n_text) + (0.5 if any(kw in edge_label.lower() for kw in keywords) else 0)
+            if relevance > 0 or hop == 0:  # always explore first hop
+                scored_neighbors.append((relevance, edge_label, neighbor_id, eprops))
+
+        scored_neighbors.sort(key=lambda x: x[0], reverse=True)
+
+        for relevance, edge_label, neighbor_id, eprops in scored_neighbors[:5]:
+            n_info = node_map[neighbor_id]
+            from_info = node_map.get(current_id, {})
+            from_name = _node_name(from_info)
+            to_name = _node_name(n_info)
+
+            chain.append({
+                "hop": hop + 1,
+                "from": from_name,
+                "from_type": from_info.get("label", ""),
+                "edge": edge_label,
+                "to": to_name,
+                "to_type": n_info.get("label", ""),
+                "relevance": relevance,
+            })
+
+            # Collect evidence (resolve text for vector-ref nodes)
+            props = n_info.get("props", {})
+            content = props.get("content") or props.get("description") or props.get("statement") or ""
+            if not content and props.get("vector_ref"):
+                try:
+                    from .text_resolver import resolve_text
+                    content = resolve_text(neighbor_id, props, "")
+                except Exception:
+                    pass
+            if content:
+                evidence.append(f"[{n_info.get('label', '')}] {to_name}: {content[:150]}")
+
+            if relevance > 0:
+                frontier.append((neighbor_id, hop + 1))
+
+    # Step 3: Generate answer
+    if use_llm and chain:
+        answer = _llm_answer(question, chain, evidence)
+    else:
+        answer = _heuristic_answer(question, chain, evidence)
+
+    return {
+        "answer": answer,
+        "chain": chain[:20],
+        "evidence": evidence[:10],
+        "nodes_visited": len(visited),
+        "hops": max(c["hop"] for c in chain) if chain else 0,
+        "seeds": [_node_name(node_map.get(s, {})) for s in seeds],
+    }
+
+
+def _extract_keywords(question: str) -> List[str]:
+    """Extract meaningful keywords from the question."""
+    stop = {"what", "who", "how", "why", "when", "where", "which", "is", "are",
+            "was", "the", "a", "an", "does", "did", "has", "have", "been",
+            "for", "from", "with", "about", "that", "this", "show", "find",
+            "tell", "get", "me", "all", "can", "do", "to", "of", "in", "on"}
+    words = re.split(r'\W+', question.lower())
+    return [w for w in words if len(w) > 2 and w not in stop]
+
+
+def _find_seeds(node_map: dict, keywords: List[str], max_seeds: int = 5) -> List[str]:
+    """Find starting nodes that match the question keywords."""
+    scored = []
+    for nid, info in node_map.items():
+        text = _node_text(info).lower()
+        score = sum(2 for kw in keywords if kw in text)
+        if score > 0:
+            scored.append((score, nid))
+    scored.sort(key=lambda x: x[0], reverse=True)
+    return [nid for _, nid in scored[:max_seeds]]
+
+
+def _node_text(info: dict) -> str:
+    props = info.get("props", {})
+    parts = [info.get("label", "")]
+    for k in ("name", "title", "content", "description", "statement"):
+        v = props.get(k, "")
+        if v:
+            parts.append(str(v))
+    return " ".join(parts)
+
+
+def _node_name(info: dict) -> str:
+    props = info.get("props", {})
+    return props.get("name") or props.get("title") or props.get("content", "")[:40] or info.get("label", "?")
+
+
+def _llm_answer(question: str, chain: list, evidence: list) -> str:
+    """Use LLM to synthesize answer from reasoning chain."""
+    try:
+        from contextsynapse.llm.client import LLMClient
+        llm = LLMClient()
+
+        chain_text = "\n".join(
+            f"  Hop {c['hop']}: [{c['from_type']}] {c['from']} --{c['edge']}--> [{c['to_type']}] {c['to']}"
+            for c in chain[:15]
+        )
+        evidence_text = "\n".join(f"  - {e}" for e in evidence[:8])
+
+        prompt = (
+            f"Answer the question using ONLY the graph traversal evidence below.\n"
+            f"Explain the reasoning path (which nodes and edges you followed).\n\n"
+            f"Question: {question}\n\n"
+            f"Graph traversal path:\n{chain_text}\n\n"
+            f"Evidence collected:\n{evidence_text}\n\n"
+            f"Answer (include the reasoning path):"
+        )
+        answer = llm.generate(prompt, system="Answer based on graph evidence. Explain the path.", max_tokens=500)
+        if answer and answer.strip():
+            return answer.strip()
+    except Exception as e:
+        logger.debug("LLM reasoning failed: %s", e)
+
+    return _heuristic_answer(question, chain, evidence)
+
+
+def _heuristic_answer(question: str, chain: list, evidence: list) -> str:
+    """Build answer from chain without LLM."""
+    if not chain:
+        return "No reasoning path found."
+    lines = ["Reasoning path:"]
+    for c in chain[:10]:
+        lines.append(f"  [{c['from_type']}] {c['from']} --{c['edge']}--> [{c['to_type']}] {c['to']}")
+    if evidence:
+        lines.append("\nEvidence:")
+        for e in evidence[:5]:
+            lines.append(f"  {e}")
+    return "\n".join(lines)

contextsynapse/search/redis_search.py

@@ -0,0 +1,207 @@
+"""
+Redis Search integration — shared full-text index across all workers.
+
+Replaces the per-worker in-process keyword cache and single-process Whoosh
+index with RediSearch (FT module). Falls back to the in-process inverted
+index if RediSearch is not available.
+
+Each graph gets its own search index: idx:{graph_name}
+Documents are indexed with: name, content, statement, label fields.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class RedisSearchIndex:
+    """Full-text search index backed by RediSearch (FT module)."""
+
+    def __init__(self, redis_url: str = None):
+        import redis
+        self._url = redis_url or os.environ.get("CONTEXTSYNAPSE_REDIS_URL") or os.environ.get("AICONTEXTDB_REDIS_URL", "redis://localhost:6379/0")
+        self._r = redis.from_url(self._url, decode_responses=True)
+        self._available = self._check_ft_module()
+
+    def _check_ft_module(self) -> bool:
+        """Check if RediSearch module is loaded."""
+        try:
+            modules = self._r.module_list()
+            return any(m.get("name", b"").lower() in (b"search", "search", b"ft", "ft") for m in modules)
+        except Exception:
+            return False
+
+    @property
+    def available(self) -> bool:
+        return self._available
+
+    def _index_name(self, graph: str) -> str:
+        safe = hashlib.sha256(graph.encode()).hexdigest()[:12]
+        return f"idx:{safe}"
+
+    def ensure_index(self, graph_name: str):
+        """Create the search index for a graph if it doesn't exist."""
+        if not self._available:
+            return
+        idx = self._index_name(graph_name)
+        try:
+            self._r.execute_command("FT.INFO", idx)
+            return  # already exists
+        except Exception:
+            pass
+        try:
+            self._r.execute_command(
+                "FT.CREATE", idx,
+                "ON", "HASH",
+                "PREFIX", "1", f"doc:{graph_name}:",
+                "SCHEMA",
+                "name", "TEXT", "WEIGHT", "2.0",
+                "content", "TEXT", "WEIGHT", "1.0",
+                "statement", "TEXT", "WEIGHT", "1.5",
+                "label", "TAG",
+                "node_id", "TAG",
+            )
+            logger.info("[REDIS-FT] Created search index for '%s'", graph_name)
+        except Exception as e:
+            logger.debug("[REDIS-FT] Index creation failed: %s", e)
+
+    def index_node(self, graph_name: str, node_id: str, label: str, props: Dict[str, Any]):
+        """Index a single node's searchable fields."""
+        if not self._available:
+            return
+        key = f"doc:{graph_name}:{node_id}"
+        # Resolve content (handles vector-ref pointer-only nodes)
+        content = props.get("content") or props.get("description") or ""
+        if not content and props.get("vector_ref"):
+            try:
+                from .text_resolver import resolve_text
+                content = resolve_text(node_id, props, graph_name)
+            except Exception:
+                pass
+        doc = {
+            "node_id": node_id,
+            "label": label,
+            "name": str(props.get("name", "") or ""),
+            "content": str(content or "")[:2000],
+            "statement": str(props.get("statement", "") or "")[:1000],
+        }
+        try:
+            self._r.hset(key, mapping=doc)
+        except Exception:
+            pass
+
+    def index_graph(self, graph_name: str, db):
+        """Bulk-index all nodes from a graph."""
+        if not self._available:
+            return 0
+        self.ensure_index(graph_name)
+        count = 0
+        try:
+            pipe = self._r.pipeline()
+            for node in db.get_all_nodes():
+                if isinstance(node, dict):
+                    props = node.get("properties", {})
+                    nid = str(node.get("id", ""))
+                    nlabel = node.get("label", "")
+                else:
+                    props = getattr(node, "properties", {}) or {}
+                    nid = str(getattr(node, "id", ""))
+                    nlabel = getattr(node, "label", "")
+
+                # Resolve content for vector-ref nodes
+                content = props.get("content") or props.get("description") or ""
+                if not content and props.get("vector_ref"):
+                    try:
+                        from .text_resolver import resolve_text
+                        content = resolve_text(nid, props, graph_name)
+                    except Exception:
+                        pass
+
+                key = f"doc:{graph_name}:{nid}"
+                pipe.hset(key, mapping={
+                    "node_id": nid,
+                    "label": nlabel,
+                    "name": str(props.get("name", "") or "")[:500],
+                    "content": str(content or "")[:2000],
+                    "statement": str(props.get("statement", "") or "")[:1000],
+                })
+                count += 1
+                if count % 200 == 0:
+                    pipe.execute()
+                    pipe = self._r.pipeline()
+            pipe.execute()
+            logger.info("[REDIS-FT] Indexed %d nodes for '%s'", count, graph_name)
+        except Exception as e:
+            logger.error("[REDIS-FT] Bulk index failed: %s", e)
+        return count
+
+    def search(self, graph_name: str, query: str, limit: int = 20) -> List[Dict[str, Any]]:
+        """Full-text search across a graph's indexed nodes."""
+        if not self._available:
+            return []
+        idx = self._index_name(graph_name)
+        try:
+            # Escape special RediSearch characters
+            safe_q = query.replace("@", " ").replace("{", " ").replace("}", " ")
+            result = self._r.execute_command(
+                "FT.SEARCH", idx, safe_q,
+                "LIMIT", "0", str(limit),
+                "RETURN", "4", "node_id", "label", "name", "statement",
+            )
+            # Parse FT.SEARCH result: [total, key1, [field, val, ...], key2, ...]
+            total = result[0]
+            hits = []
+            i = 1
+            while i < len(result):
+                key = result[i]
+                fields = result[i + 1] if i + 1 < len(result) else []
+                doc = {}
+                for j in range(0, len(fields), 2):
+                    doc[fields[j]] = fields[j + 1]
+                hits.append(doc)
+                i += 2
+            return hits
+        except Exception as e:
+            logger.debug("[REDIS-FT] Search failed: %s", e)
+            return []
+
+    def drop_index(self, graph_name: str):
+        """Drop a graph's search index."""
+        if not self._available:
+            return
+        idx = self._index_name(graph_name)
+        try:
+            self._r.execute_command("FT.DROPINDEX", idx, "DD")
+        except Exception:
+            pass
+
+
+# Singleton
+_index = None
+
+
+def get_search_index() -> Optional[RedisSearchIndex]:
+    """Get or create the Redis Search index (None if unavailable)."""
+    global _index
+    if _index is not None:
+        return _index if _index.available else None
+
+    url = os.environ.get("CONTEXTSYNAPSE_REDIS_URL") or os.environ.get("AICONTEXTDB_REDIS_URL")
+    if url:
+        try:
+            idx = RedisSearchIndex(url)
+            if idx.available:
+                logger.info("Redis Search (FT): available")
+                _index = idx
+                return _index
+            else:
+                logger.info("Redis Search (FT): not available (module not loaded)")
+                _index = idx  # cache the negative result
+        except Exception:
+            pass
+    return None