contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/context/token.py

@@ -0,0 +1,236 @@
+"""ContextToken — blockchain-inspired per-entity hash chain.
+
+Each atomic context (entity) maintains a chain of versioned tokens.
+Every append creates a new block linked to its predecessor via SHA-256.
+Multiple pipelines feed the same entity — each pipeline writes to its
+own namespace within the content dict.
+
+Redis keys:
+  chain:{entity}:head  → int (latest version number)
+  chain:{entity}:v{N}  → JSON (ContextToken)
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, Tuple
+
+
+def canonical_json(data: Any) -> str:
+    """Deterministic JSON serialization for hashing."""
+    return json.dumps(data, sort_keys=True, default=str)
+
+
+def content_hash(data: Any) -> str:
+    """SHA-256 hash of canonical JSON."""
+    return hashlib.sha256(canonical_json(data).encode("utf-8")).hexdigest()
+
+
+def compute_block_hash(
+    version: int,
+    content_hash_val: str,
+    prev_hash: Optional[str],
+    timestamp: str,
+) -> str:
+    """Compute the block hash for a context token version.
+
+    block_hash = SHA-256(version || content_hash || prev_hash || timestamp)
+    Genesis blocks use the literal string 'genesis' in place of prev_hash.
+    """
+    block_data = f"{version}:{content_hash_val}:{prev_hash or 'genesis'}:{timestamp}"
+    return hashlib.sha256(block_data.encode("utf-8")).hexdigest()
+
+
+@dataclass
+class ContextToken:
+    """One version (block) in an entity's hash chain."""
+    token_id: str
+    entity: str
+    version: int
+
+    content_hash: str
+    content_summary: Dict[str, Any]
+
+    prev_hash: Optional[str]
+    block_hash: str
+
+    pipelines: List[str]
+    pipeline_freshness: Dict[str, str]
+
+    created_at: str
+    freshness_score: float
+
+    times_assembled: int = 0
+    decisions_influenced: int = 0
+
+
+def _token_to_json(token: ContextToken) -> str:
+    return json.dumps({
+        "token_id": token.token_id,
+        "entity": token.entity,
+        "version": token.version,
+        "content_hash": token.content_hash,
+        "content_summary": token.content_summary,
+        "prev_hash": token.prev_hash,
+        "block_hash": token.block_hash,
+        "pipelines": token.pipelines,
+        "pipeline_freshness": token.pipeline_freshness,
+        "created_at": token.created_at,
+        "freshness_score": token.freshness_score,
+        "times_assembled": token.times_assembled,
+        "decisions_influenced": token.decisions_influenced,
+    }, default=str)
+
+
+def _token_from_json(raw: str) -> ContextToken:
+    data = json.loads(raw)
+    return ContextToken(**data)
+
+
+class ContextChain:
+    """Manages the hash chain for an atomic context (entity)."""
+
+    def __init__(self, redis_client: Any, entity: str) -> None:
+        self._r = redis_client
+        self._entity = entity
+
+    def append(self, content: Dict[str, Any], pipeline: str) -> ContextToken:
+        """Append a new version to the chain.
+
+        Merges incoming content under the pipeline's namespace, computes a
+        content hash over the merged state, then seals the block with a
+        SHA-256 block hash that chains back to the previous block.
+        """
+        head_version = self._get_head_version()
+        prev_token = self.get_version(head_version) if head_version > 0 else None
+
+        merged = self._merge_content(prev_token, content, pipeline)
+        new_version = head_version + 1
+        c_hash = content_hash(merged)
+        prev_hash = prev_token.block_hash if prev_token else None
+        timestamp = datetime.now(timezone.utc).isoformat()
+        block_hash = compute_block_hash(new_version, c_hash, prev_hash, timestamp)
+
+        pipeline_freshness: Dict[str, str] = {}
+        if prev_token:
+            pipeline_freshness = dict(prev_token.pipeline_freshness)
+        pipeline_freshness[pipeline] = timestamp
+
+        pipelines = sorted(set(
+            (prev_token.pipelines if prev_token else []) + [pipeline]
+        ))
+
+        token = ContextToken(
+            token_id=f"ct:{self._entity}:v{new_version}",
+            entity=self._entity,
+            version=new_version,
+            content_hash=c_hash,
+            content_summary=merged,
+            prev_hash=prev_hash,
+            block_hash=block_hash,
+            pipelines=pipelines,
+            pipeline_freshness=pipeline_freshness,
+            created_at=timestamp,
+            freshness_score=self._compute_freshness(pipeline_freshness),
+            times_assembled=0,
+            decisions_influenced=0,
+        )
+
+        self._r.set(f"chain:{self._entity}:v{new_version}", _token_to_json(token))
+        self._r.set(f"chain:{self._entity}:head", str(new_version))
+        return token
+
+    def get_head(self) -> Optional[ContextToken]:
+        """Return the latest token in the chain, or None if empty."""
+        head = self._get_head_version()
+        if head == 0:
+            return None
+        return self.get_version(head)
+
+    def get_version(self, version: int) -> Optional[ContextToken]:
+        """Return a specific version from the chain, or None if not found."""
+        raw = self._r.get(f"chain:{self._entity}:v{version}")
+        if raw is None:
+            return None
+        return _token_from_json(raw)
+
+    def verify(self) -> Tuple[bool, int]:
+        """Walk the chain from genesis to head, verifying each link.
+
+        Checks:
+        - block_hash matches recomputed value (tamper detection)
+        - prev_hash correctly references the previous block's hash
+
+        Returns:
+            (is_valid, verified_count) where verified_count is the number of
+            blocks successfully verified before a failure (or all of them).
+        """
+        head = self._get_head_version()
+        if head == 0:
+            return (True, 0)
+
+        prev_hash: Optional[str] = None
+        for v in range(1, head + 1):
+            token = self.get_version(v)
+            if token is None:
+                return (False, v - 1)
+            expected = compute_block_hash(
+                token.version, token.content_hash, prev_hash, token.created_at
+            )
+            if token.block_hash != expected:
+                return (False, v - 1)
+            if token.prev_hash != prev_hash:
+                return (False, v - 1)
+            prev_hash = token.block_hash
+
+        return (True, head)
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    def _get_head_version(self) -> int:
+        raw = self._r.get(f"chain:{self._entity}:head")
+        if raw is None:
+            return 0
+        return int(raw)
+
+    def _merge_content(
+        self,
+        prev_token: Optional[ContextToken],
+        new_content: Dict[str, Any],
+        pipeline: str,
+    ) -> Dict[str, Any]:
+        """Merge new pipeline data into entity content.
+
+        Each pipeline writes to its own namespace within the content dict,
+        so contributions from different pipelines never collide.
+        """
+        if prev_token is None:
+            return {pipeline: new_content}
+        merged = dict(prev_token.content_summary)
+        merged[pipeline] = new_content
+        return merged
+
+    def _compute_freshness(self, pipeline_freshness: Dict[str, str]) -> float:
+        """Compute a freshness score in [0.0, 1.0] based on pipeline timestamps.
+
+        Score decays linearly from 1.0 (just written) to 0.0 over 1 hour.
+        Returns the mean score across all contributing pipelines.
+        """
+        if not pipeline_freshness:
+            return 0.0
+        now = time.time()
+        scores = []
+        for ts_str in pipeline_freshness.values():
+            try:
+                ts = datetime.fromisoformat(ts_str).timestamp()
+                age_seconds = now - ts
+                score = max(0.0, 1.0 - (age_seconds / 3600))  # decay over 1 hour
+                scores.append(score)
+            except (ValueError, TypeError):
+                scores.append(0.0)
+        return sum(scores) / len(scores) if scores else 0.0

contextsynapse/context/vector_integration.py

@@ -0,0 +1,297 @@
+"""
+Vector Integration for Context Sessions
+========================================
+Bridges the CAS session layer with the vector DB stack.
+
+Each session gets its own vector collection. Text ingested into a session
+is auto-embedded and stored, enabling semantic search across context.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# Default embedding dimension (OpenAI ada-002 / fallback)
+DEFAULT_DIMENSION = 1536
+
+# Singleton — one SVS instance shared across all callers
+_svs_singleton: Optional["SessionVectorStore"] = None
+
+
+def get_session_vector_store() -> "SessionVectorStore":
+    """Get the singleton SessionVectorStore. Creates and warms up on first call."""
+    global _svs_singleton
+    if _svs_singleton is None:
+        _svs_singleton = SessionVectorStore()
+        # Warm up Ollama embedding model — first call loads model into memory (5s),
+        # subsequent calls are <100ms. keep_alive=60m prevents unloading.
+        if _svs_singleton._embedding_service:
+            try:
+                import time
+                t0 = time.time()
+                result = _svs_singleton._embedding_service.embed_text("warmup embedding model")
+                emb = result.get("embedding") if isinstance(result, dict) else None
+                elapsed = int((time.time() - t0) * 1000)
+                if emb:
+                    logger.info("Embedding model warmed up: dim=%d (%dms)", len(emb), elapsed)
+                else:
+                    logger.info("Embedding warmup completed (%dms)", elapsed)
+            except Exception as e:
+                logger.debug("Embedding warmup failed: %s", e)
+    return _svs_singleton
+
+
+class SessionVectorStore:
+    """
+    Manages per-session vector collections.
+
+    Wraps ``VectorDBManager`` so the rest of CAS never touches vector
+    internals directly.
+    """
+
+    def __init__(self, dimension: int = DEFAULT_DIMENSION, backend: Optional[str] = None):
+        self._dimension = dimension
+        self._stores: Dict[str, Any] = {}  # session_id -> vector store instance
+        self._embedding_service = None
+        self._manager = None
+
+        # Lazy-init embedding service first (to detect dimension)
+        try:
+            from ..models.embedding_service import get_embedding_service
+            self._embedding_service = get_embedding_service()
+            # Auto-detect dimension from embedding service
+            try:
+                test = self._embedding_service.embed_text("test")
+                emb = test.get("embedding")
+                if emb is not None:
+                    detected = len(emb) if isinstance(emb, list) else (emb.shape[0] if hasattr(emb, 'shape') else len(list(emb)))
+                    if detected > 0:
+                        self._dimension = detected
+                        logger.info(f"Auto-detected embedding dimension: {detected}")
+            except Exception:
+                pass
+            logger.info("Embedding service available for auto-embed")
+        except Exception as e:
+            logger.warning(f"Embedding service unavailable: {e}")
+
+        # Lazy-init vector backend
+        try:
+            from ..vector.vector_db_manager import get_vector_db_manager
+            self._manager = get_vector_db_manager(backend=backend)
+            logger.info(f"SessionVectorStore using backend: {self._manager.backend_name}")
+        except Exception as e:
+            logger.warning(f"Vector DB manager unavailable: {e}")
+
+    @property
+    def available(self) -> bool:
+        """True if both vector store and embeddings are functional."""
+        return self._manager is not None and self._embedding_service is not None
+
+    def _get_store(self, session_id: str):
+        """Get or create the vector store for a session/graph namespace.
+
+        Collection naming convention matches the ingestion pipeline:
+        ``{namespace}_passages`` — so vectors written during ingestion
+        are findable by search_nodes, and vice versa.
+        """
+        if session_id not in self._stores:
+            if self._manager is None:
+                return None
+            # Use {namespace}_passages to match ingestion STORE_VECTORS stage
+            safe_ns = session_id.replace(":", "_")
+            collection = f"{safe_ns}_passages"
+            store = self._manager.create_store(
+                dimension=self._dimension,
+                metric="cosine",
+                collection_name=collection,
+            )
+            self._stores[session_id] = store
+            logger.info(f"Created vector store for session {session_id} (collection: {collection})")
+        return self._stores[session_id]
+
+    def embed_text(self, text: str) -> Optional[List[float]]:
+        """Generate an embedding for a single text string. Uses LMDB cache."""
+        if self._embedding_service is None:
+            return None
+
+        # Check cache first (microseconds vs 3-5s Ollama call)
+        try:
+            from ..search.embedding_cache import get_embedding_cache
+            cache = get_embedding_cache()
+            cached = cache.get(text)
+            if cached is not None:
+                return cached
+        except Exception:
+            cache = None
+
+        try:
+            result = self._embedding_service.embed_text(text)
+            # Reject fallback/mock embeddings — they are not semantically meaningful
+            if result.get("provider") == "fallback":
+                logger.warning("Embedding returned fallback (mock) — treating as unavailable")
+                return None
+            embedding = result.get("embedding")
+            if embedding is not None:
+                vec = list(embedding) if not isinstance(embedding, list) else embedding
+                # Cache for next time
+                if cache:
+                    try:
+                        cache.put(text, vec)
+                    except Exception:
+                        pass
+                return vec
+        except Exception as e:
+            logger.warning(f"Embedding failed: {e}")
+        return None
+
+    def embed_batch(self, texts: List[str]) -> Optional[List[List[float]]]:
+        """Generate embeddings for multiple texts."""
+        if self._embedding_service is None:
+            return None
+        try:
+            result = self._embedding_service.embed_batch(texts)
+            embeddings = result.get("embeddings", [])
+            return [list(e) if not isinstance(e, list) else e for e in embeddings]
+        except Exception as e:
+            logger.warning(f"Batch embedding failed: {e}")
+        return None
+
+    def add_text(
+        self,
+        session_id: str,
+        text: str,
+        node_id: str,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> bool:
+        """Embed and store a single text for a session."""
+        store = self._get_store(session_id)
+        if store is None:
+            return False
+
+        embedding = self.embed_text(text)
+        if embedding is None:
+            return False
+
+        try:
+            store.add_vectors(
+                node_ids=[node_id],
+                vectors=[embedding],
+                metadata_list=[{
+                    "text": text[:500],  # store truncated text for retrieval
+                    **(metadata or {}),
+                }],
+            )
+            return True
+        except Exception as e:
+            logger.warning(f"Vector add failed: {e}")
+            return False
+
+    def add_texts(
+        self,
+        session_id: str,
+        texts: List[str],
+        node_ids: List[str],
+        metadata_list: Optional[List[Dict[str, Any]]] = None,
+    ) -> bool:
+        """Embed and store multiple texts for a session."""
+        store = self._get_store(session_id)
+        if store is None:
+            return False
+
+        embeddings = self.embed_batch(texts)
+        if embeddings is None or len(embeddings) != len(texts):
+            return False
+
+        meta = metadata_list or [{}] * len(texts)
+        # Inject truncated text into each metadata entry
+        enriched_meta = [
+            {"text": t[:500], **m}
+            for t, m in zip(texts, meta)
+        ]
+
+        try:
+            store.add_vectors(
+                node_ids=node_ids,
+                vectors=embeddings,
+                metadata_list=enriched_meta,
+            )
+            return True
+        except Exception as e:
+            logger.warning(f"Vector batch add failed: {e}")
+            return False
+
+    def search(
+        self,
+        session_id: str,
+        query: str,
+        k: int = 10,
+        filter_ids: Optional[List[str]] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        Semantic search across a session's vectors.
+
+        Returns list of ``{node_id, score, metadata}`` dicts sorted by relevance.
+        """
+        store = self._get_store(session_id)
+        if store is None:
+            return []
+
+        query_embedding = self.embed_text(query)
+        if query_embedding is None:
+            return []
+
+        try:
+            results = store.search(
+                query_vector=query_embedding,
+                k=k,
+                filter_ids=filter_ids,
+            )
+            return results
+        except Exception as e:
+            logger.warning(f"Vector search failed: {e}")
+            return []
+
+    def has_vectors(self, session_id: str) -> bool:
+        """Return True if this namespace has any indexed vectors.
+
+        Cheap check — avoids embedding the query when the collection is empty
+        (e.g. test graphs that were never passed through the ingestion pipeline).
+        """
+        if self._manager is None:
+            return False
+        safe_ns = session_id.replace(":", "_")
+        collection = f"{safe_ns}_passages"
+        try:
+            # Ask the backend whether the collection exists and is non-empty
+            client = getattr(self._manager, "_client", None)
+            if client is not None and hasattr(client, "collection_exists"):
+                if not client.collection_exists(collection):
+                    return False
+                info = client.get_collection(collection)
+                count = getattr(info, "vectors_count", None)
+                if count is not None:
+                    return count > 0
+        except Exception:
+            pass
+        # If we can't check, assume vectors may exist (safe default)
+        return True
+
+    def delete_session(self, session_id: str):
+        """Remove a session's vector store from memory."""
+        self._stores.pop(session_id, None)
+
+    def get_stats(self, session_id: str) -> Dict[str, Any]:
+        """Get stats for a session's vector store."""
+        store = self._get_store(session_id)
+        if store is None:
+            return {"available": False}
+        return {
+            "available": True,
+            "backend": self._manager.backend_name if self._manager else "none",
+            "vector_count": getattr(store, "vector_count", 0),
+            "dimension": self._dimension,
+            "embedding_service": self._embedding_service is not None,
+        }