contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/ingestion/cleanse.py

@@ -0,0 +1,252 @@
+"""Context cleansing — improve quality of existing context without re-ingesting.
+
+Runs 5 steps:
+1. DEDUP    — merge duplicate entities
+2. PRUNE    — remove low-confidence nodes (< threshold)
+3. ORPHAN   — remove disconnected nodes (zero edges)
+4. VALIDATE — re-run schema validation on all nodes
+5. REINDEX  — rebuild BM25 + vector indexes
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CleanseResult:
+    """Result of cleansing a context."""
+    duplicates_merged: int = 0
+    low_quality_pruned: int = 0
+    orphans_removed: int = 0
+    nodes_revalidated: int = 0
+    nodes_reindexed: int = 0
+    errors: List[str] = field(default_factory=list)
+
+    def to_dict(self) -> Dict:
+        return {
+            "duplicates_merged": self.duplicates_merged,
+            "low_quality_pruned": self.low_quality_pruned,
+            "orphans_removed": self.orphans_removed,
+            "nodes_revalidated": self.nodes_revalidated,
+            "nodes_reindexed": self.nodes_reindexed,
+            "errors": self.errors,
+        }
+
+    @property
+    def summary(self) -> str:
+        parts = []
+        if self.duplicates_merged:
+            parts.append(f"merged {self.duplicates_merged} duplicates")
+        if self.low_quality_pruned:
+            parts.append(f"pruned {self.low_quality_pruned} low-quality")
+        if self.orphans_removed:
+            parts.append(f"removed {self.orphans_removed} orphans")
+        if self.nodes_revalidated:
+            parts.append(f"re-validated {self.nodes_revalidated} nodes")
+        if self.nodes_reindexed:
+            parts.append(f"re-indexed {self.nodes_reindexed} nodes")
+        return "Cleansed: " + ", ".join(parts) if parts else "No changes needed"
+
+
+# Node types that should never be pruned or removed
+PROTECTED_TYPES = {"ContextMeta", "VectorIndex", "BM25Index", "Session", "ContextRef", "Agent", "AgentRef"}
+
+
+def cleanse_context(db, schema=None, min_confidence: float = 0.3) -> CleanseResult:
+    """Cleanse an existing context — improve quality without re-ingesting.
+
+    Args:
+        db: AIContextDB instance.
+        schema: Optional IngestionSchema for re-validation.
+        min_confidence: Nodes below this confidence get pruned.
+
+    Returns:
+        CleanseResult with counts of what was changed.
+    """
+    result = CleanseResult()
+
+    try:
+        all_nodes = db.csr_adapter.get_all_nodes()
+    except Exception as e:
+        result.errors.append(f"Failed to get nodes: {e}")
+        return result
+
+    # ── Step 1: DEDUP — merge duplicate entities ──────────────────
+    # Three levels of dedup:
+    #   a) Exact name match (same label) — "Tehran" + "Tehran"
+    #   b) Cross-label match — "Iran" as Location + "Iran" as Organization
+    #   c) Substring containment — "Netanyahu" matches "Benjamin Netanyahu"
+    try:
+        entity_labels = {"Person", "Organization", "Location", "Event"}
+        seen = {}           # "name_lower" -> (label, node)
+        to_merge = []       # [(keep_id, remove_id)]
+        junk_ids = []       # nodes that are clearly not entities
+
+        # Junk entity patterns — not real entities
+        import re as _re
+        _JUNK_PATTERNS = [
+            _re.compile(r'^(edition|auto|watch|talk|class|world her|what has|look|give|story)$', _re.I),
+            _re.compile(r'^.{1,2}$'),  # 1-2 char names
+            _re.compile(r'^\d+$'),     # pure numbers
+            _re.compile(r'^(the|a|an|this|that|http|www)\b', _re.I),  # starts with article/url
+        ]
+
+        for node in all_nodes:
+            label = getattr(node, 'node_type', getattr(node, 'label', ''))
+            if label not in entity_labels:
+                continue
+            name = node.properties.get('name', '').strip()
+            if not name:
+                continue
+            name_lower = name.lower()
+
+            # Remove junk entities
+            if any(p.search(name_lower) for p in _JUNK_PATTERNS):
+                junk_ids.append(node.id)
+                continue
+
+            # Dedup key: name only (cross-label merge)
+            if name_lower in seen:
+                existing_label, existing = seen[name_lower]
+                existing_props = len([v for v in existing.properties.values() if v])
+                current_props = len([v for v in node.properties.values() if v])
+                # Prefer the one with more properties, or the better label
+                label_priority = {"Location": 3, "Organization": 2, "Event": 2, "Person": 1}
+                existing_priority = label_priority.get(existing_label, 0)
+                current_priority = label_priority.get(label, 0)
+                if current_props > existing_props or (current_props == existing_props and current_priority > existing_priority):
+                    to_merge.append((node.id, existing.id))
+                    seen[name_lower] = (label, node)
+                else:
+                    to_merge.append((existing.id, node.id))
+            else:
+                seen[name_lower] = (label, node)
+
+        # Substring containment dedup — "Netanyahu" + "Benjamin Netanyahu" → keep longer
+        seen_names = sorted(seen.keys(), key=len, reverse=True)  # longest first
+        for i, long_name in enumerate(seen_names):
+            if len(long_name) < 5:
+                continue
+            long_tokens = set(long_name.split())
+            for short_name in seen_names[i+1:]:
+                if len(short_name) < 3 or short_name == long_name:
+                    continue
+                # Short name's tokens are a subset of long name's tokens
+                short_tokens = set(short_name.split())
+                if short_tokens and short_tokens.issubset(long_tokens) and len(short_tokens) < len(long_tokens):
+                    long_label, long_node = seen[long_name]
+                    short_label, short_node = seen[short_name]
+                    # Only merge if same entity type (or both people/orgs)
+                    if long_label == short_label or {long_label, short_label} <= entity_labels:
+                        to_merge.append((long_node.id, short_node.id))  # keep longer name
+                        logger.debug("[CLEANSE] Substring merge: '%s' ← '%s'", long_name, short_name)
+
+        # Execute merges
+        for keep_id, remove_id in to_merge:
+            try:
+                db.csr_adapter.remove_node(remove_id)
+                result.duplicates_merged += 1
+            except Exception:
+                pass
+
+        # Remove junk entities
+        for junk_id in junk_ids:
+            try:
+                db.csr_adapter.remove_node(junk_id)
+                result.duplicates_merged += 1
+            except Exception:
+                pass
+
+        if junk_ids:
+            logger.info("[CLEANSE] Removed %d junk entities", len(junk_ids))
+    except Exception as e:
+        result.errors.append(f"Dedup failed: {e}")
+
+    # ── Step 2: PRUNE — remove low-confidence nodes ──────────────
+    try:
+        # Re-fetch after dedup
+        all_nodes = db.csr_adapter.get_all_nodes()
+        for node in all_nodes:
+            label = getattr(node, 'node_type', getattr(node, 'label', ''))
+            if label in PROTECTED_TYPES or label in ('Document', 'Passage'):
+                continue  # never prune documents or passages
+            confidence = node.properties.get('confidence', 1.0)
+            if isinstance(confidence, (int, float)) and confidence < min_confidence:
+                try:
+                    db.csr_adapter.remove_node(node.id)
+                    result.low_quality_pruned += 1
+                except Exception:
+                    pass
+    except Exception as e:
+        result.errors.append(f"Prune failed: {e}")
+
+    # ── Step 3: ORPHAN — remove disconnected nodes ────────────────
+    try:
+        all_nodes = db.csr_adapter.get_all_nodes()
+        for node in all_nodes:
+            label = getattr(node, 'node_type', getattr(node, 'label', ''))
+            if label in PROTECTED_TYPES or label in ('Document', 'ContextMeta'):
+                continue
+            # Check if node has any edges
+            outgoing = db.csr_adapter.get_neighbors(node.id)
+            incoming = db.csr_adapter.get_incoming_neighbors(node.id) if hasattr(db.csr_adapter, 'get_incoming_neighbors') else []
+            if not outgoing and not incoming:
+                try:
+                    db.csr_adapter.remove_node(node.id)
+                    result.orphans_removed += 1
+                except Exception:
+                    pass
+    except Exception as e:
+        result.errors.append(f"Orphan removal failed: {e}")
+
+    # ── Step 4: VALIDATE — re-run schema validation ───────────────
+    try:
+        if schema:
+            from .schema_validator import validate_and_flag
+            all_nodes = db.csr_adapter.get_all_nodes()
+            for node in all_nodes:
+                label = getattr(node, 'node_type', getattr(node, 'label', ''))
+                if label in PROTECTED_TYPES:
+                    continue
+                node_dict = {"label": label, "properties": dict(node.properties)}
+                validated = validate_and_flag(node_dict, schema)
+                for vk in ("validation_status", "validation_errors"):
+                    if vk in validated["properties"]:
+                        node.properties[vk] = validated["properties"][vk]
+                db.csr_adapter.update_node_properties(node.id, node.properties)
+                result.nodes_revalidated += 1
+    except Exception as e:
+        result.errors.append(f"Validation failed: {e}")
+
+    # ── Step 5: REINDEX — rebuild search indexes ──────────────────
+    try:
+        all_nodes = db.csr_adapter.get_all_nodes()
+        indexable = [n for n in all_nodes
+                     if getattr(n, 'node_type', getattr(n, 'label', '')) not in PROTECTED_TYPES]
+        if indexable:
+            from ..search.whoosh_search import WhooshSearchEngine
+            engine = WhooshSearchEngine(db.name)
+            engine.index_nodes(indexable)
+            result.nodes_reindexed = len(indexable)
+    except Exception as e:
+        result.errors.append(f"Reindex failed: {e}")
+
+    # ── Update ContextMeta ────────────────────────────────────────
+    try:
+        meta = db.csr_adapter.get_node("_context_meta")
+        if meta:
+            all_nodes = db.csr_adapter.get_all_nodes()
+            content = [n for n in all_nodes if getattr(n, 'node_type', getattr(n, 'label', '')) not in PROTECTED_TYPES]
+            meta.properties["node_count"] = len(content)
+            labels = set(getattr(n, 'node_type', getattr(n, 'label', '')) for n in content)
+            meta.properties["schema_summary"] = ", ".join(sorted(labels))
+            db.csr_adapter.update_node_properties("_context_meta", meta.properties)
+    except Exception:
+        pass
+
+    logger.info("Cleanse complete: %s", result.summary)
+    return result

contextsynapse/ingestion/connectors/__init__.py

@@ -0,0 +1,38 @@
+"""
+Connector Ingestor Registry — maps connector_type to ConnectorIngestor subclass.
+"""
+
+from typing import Dict, Optional, Type
+from .base import ConnectorIngestor, ConnectorGraphResult
+
+__all__ = ["ConnectorIngestor", "ConnectorGraphResult", "get_ingestor", "CONNECTOR_INGEST_MAP"]
+
+CONNECTOR_INGEST_MAP: Dict[str, Type[ConnectorIngestor]] = {}
+
+
+def register_ingestor(connector_type: str, cls: Type[ConnectorIngestor]):
+    CONNECTOR_INGEST_MAP[connector_type] = cls
+
+
+def get_ingestor(connector_type: str, config: dict, credentials: dict) -> Optional[ConnectorIngestor]:
+    cls = CONNECTOR_INGEST_MAP.get(connector_type)
+    if cls:
+        return cls(config, credentials)
+    return None
+
+
+# Register implemented connectors only
+_CONNECTORS = [
+    ("sap", "sap_ingestor", "SAPIngestor"),
+    ("chatgpt", "chatgpt_memory", "ChatGPTMemoryIngestor"),
+    ("claude", "claude_memory", "ClaudeMemoryIngestor"),
+]
+
+import importlib as _importlib
+for _ctype, _module, _cls_name in _CONNECTORS:
+    try:
+        _mod = _importlib.import_module(f".{_module}", package=__name__)
+        _cls = getattr(_mod, _cls_name)
+        register_ingestor(_ctype, _cls)
+    except (ImportError, ModuleNotFoundError, AttributeError):
+        pass

contextsynapse/ingestion/connectors/base.py

@@ -0,0 +1,300 @@
+"""
+ConnectorIngestor — Base class for pulling external system metadata + data into the graph.
+
+Two-phase protocol:
+  Phase 1: discover_metadata() → structural graph nodes (Table, Column, Issue, etc.)
+  Phase 2: pull_data() → content items fed into the extraction pipeline
+"""
+
+from __future__ import annotations
+
+import uuid
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ConnectorGraphResult:
+    """Result of a connector ingestion phase."""
+    nodes: List[Dict[str, Any]] = field(default_factory=list)
+    edges: List[Dict[str, Any]] = field(default_factory=list)
+    data_items: List[Dict[str, Any]] = field(default_factory=list)  # Phase 2: text content for pipeline
+    stats: Dict[str, Any] = field(default_factory=dict)
+
+
+class ConnectorIngestor(ABC):
+    """
+    Base class for connector-to-graph ingestion.
+
+    Subclasses implement discover_metadata() for structural schema discovery
+    and pull_data() for content extraction.
+    """
+
+    connector_type: str = ""
+
+    def __init__(self, config: Dict[str, Any], credentials: Dict[str, Any]):
+        self.config = config
+        self.credentials = credentials
+
+    @abstractmethod
+    def test_connection(self) -> Dict[str, Any]:
+        """Test connectivity. Returns {success: bool, message: str}."""
+        ...
+
+    @abstractmethod
+    def discover_metadata(self, context_id: str) -> ConnectorGraphResult:
+        """Phase 1: Discover structural metadata and return graph nodes/edges."""
+        ...
+
+    @abstractmethod
+    def pull_data(self, context_id: str, options: Optional[Dict[str, Any]] = None) -> ConnectorGraphResult:
+        """Phase 2: Pull content for extraction pipeline. Returns data_items."""
+        ...
+
+    # ------------------------------------------------------------------
+    # Helper methods for building graph nodes
+    # ------------------------------------------------------------------
+
+    def _uid(self) -> str:
+        return str(uuid.uuid4())
+
+    def _make_connector_node(
+        self, context_id: str, *, name: Optional[str] = None,
+        extra_props: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[Dict, Dict]:
+        """Create root Connector node + HAS_CONNECTOR edge from Context."""
+        node_id = self._uid()
+        props = {
+            "name": name or f"{self.connector_type} connector",
+            "connector_type": self.connector_type,
+            "connected_at": datetime.now(timezone.utc).isoformat(),
+        }
+        if extra_props:
+            props.update(extra_props)
+
+        node = {"id": node_id, "label": "Connector", "properties": props}
+        edge = {
+            "id": self._uid(),
+            "source": context_id,
+            "target": node_id,
+            "label": "HAS_CONNECTOR",
+            "properties": {"connector_type": self.connector_type},
+        }
+        return node, edge
+
+    def _make_sync_run_node(
+        self, connector_node_id: str, status: str,
+        nodes_created: int, edges_created: int, duration_ms: int,
+        error: Optional[str] = None,
+    ) -> Tuple[Dict, Dict]:
+        """Create a SyncRun node linked to the Connector."""
+        node_id = self._uid()
+        props = {
+            "name": f"Sync {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M')}",
+            "status": status,
+            "nodes_created": nodes_created,
+            "edges_created": edges_created,
+            "duration_ms": duration_ms,
+            "synced_at": datetime.now(timezone.utc).isoformat(),
+        }
+        if error:
+            props["error"] = error[:500]
+
+        node = {"id": node_id, "label": "SyncRun", "properties": props}
+        edge = {
+            "id": self._uid(),
+            "source": connector_node_id,
+            "target": node_id,
+            "label": "HAS_SYNC_RUN",
+            "properties": {},
+        }
+        return node, edge
+
+    # ------------------------------------------------------------------
+    # Phase 2 helpers: smart text formatting for extraction pipeline
+    # ------------------------------------------------------------------
+
+    # Fields to skip when formatting records as text
+    _SYSTEM_FIELD_PREFIXES = ("_", "sys_", "link", "href")
+    _SYSTEM_FIELDS = {
+        "sys_id", "sys_created_on", "sys_updated_on", "sys_mod_count",
+        "sys_class_name", "sys_domain", "sys_tags", "sys_scope",
+        "self", "expand", "fields", "transitions", "editmeta",
+    }
+
+    @staticmethod
+    def _format_record_as_text(
+        record: Dict[str, Any],
+        skip_keys: Optional[set] = None,
+        key_fields: Optional[List[str]] = None,
+    ) -> str:
+        """Format a JSON record as readable text for LLM extraction.
+
+        Prioritizes key_fields first, then appends remaining fields.
+        Skips internal/system fields automatically.
+        """
+        skip = (skip_keys or set()) | ConnectorIngestor._SYSTEM_FIELDS
+        parts = []
+
+        # Key fields first (headline)
+        if key_fields:
+            headline = []
+            for k in key_fields:
+                v = record.get(k)
+                if v is not None and str(v).strip():
+                    headline.append(f"{k}: {v}")
+            if headline:
+                parts.append(" | ".join(headline))
+
+        # Remaining fields
+        details = []
+        seen = set(key_fields or [])
+        for k, v in record.items():
+            if k in seen:
+                continue
+            if k.lower() in skip:
+                continue
+            if any(k.lower().startswith(p) for p in ConnectorIngestor._SYSTEM_FIELD_PREFIXES):
+                continue
+            if v is None or (isinstance(v, str) and not v.strip()):
+                continue
+            if isinstance(v, (dict, list)):
+                v = str(v)[:200]
+            details.append(f"{k}: {v}")
+            seen.add(k)
+
+        if details:
+            parts.append(". ".join(details))
+
+        return "\n".join(parts)
+
+    @staticmethod
+    def _table_stats_text(
+        table_name: str,
+        columns: List[Dict[str, Any]],
+        row_count: int,
+        sample_rows: List[Dict[str, Any]],
+        column_stats: Optional[Dict[str, Dict]] = None,
+    ) -> str:
+        """Generate a natural language summary of a database table.
+
+        Returns text optimized for LLM entity/fact extraction.
+        """
+        lines = [f"Database table '{table_name}' contains {row_count:,} rows.\n"]
+
+        # Column descriptions
+        col_descs = []
+        for col in columns:
+            name = col.get("name", "")
+            dtype = col.get("data_type", "unknown")
+            nullable = "nullable" if col.get("nullable") else "required"
+            primary = " (primary key)" if col.get("is_primary") else ""
+            desc = f"  - {name} ({dtype}, {nullable}{primary})"
+
+            # Add stats if available
+            if column_stats and name in column_stats:
+                stats = column_stats[name]
+                stat_parts = []
+                if "distinct" in stats:
+                    stat_parts.append(f"{stats['distinct']} distinct values")
+                if "min" in stats and "max" in stats:
+                    stat_parts.append(f"range: {stats['min']} to {stats['max']}")
+                if "null_count" in stats and stats["null_count"] > 0:
+                    stat_parts.append(f"{stats['null_count']} nulls")
+                if stat_parts:
+                    desc += f" — {', '.join(stat_parts)}"
+            col_descs.append(desc)
+
+        if col_descs:
+            lines.append(f"Columns ({len(col_descs)}):")
+            lines.extend(col_descs)
+
+        # Sample data
+        if sample_rows:
+            lines.append(f"\nSample data ({len(sample_rows)} rows):")
+            for i, row in enumerate(sample_rows[:10]):
+                row_text = " | ".join(f"{k}={v}" for k, v in row.items() if v is not None)
+                lines.append(f"  Row {i+1}: {row_text}")
+
+        return "\n".join(lines)
+
+    # ------------------------------------------------------------------
+    # Structured data helpers: direct record → typed node mapping
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _table_name_to_label(table_name: str) -> str:
+        """Convert a table/collection name to a PascalCase node label.
+
+        'order_items' → 'OrderItem', 'customers' → 'Customer',
+        'user_sessions' → 'UserSession'
+        """
+        parts = table_name.replace("-", "_").split("_")
+        result = []
+        for p in parts:
+            if not p:
+                continue
+            # Simple singularize: strip trailing 's' (unless too short or ends in 'ss')
+            word = p.capitalize()
+            if len(word) > 3 and word.endswith("s") and not word.endswith("ss"):
+                word = word[:-1]
+            result.append(word)
+        return "".join(result) or "Record"
+
+    def _records_to_nodes(
+        self,
+        records: List[Dict[str, Any]],
+        label: str,
+        key_field: Optional[str] = None,
+        source_node_id: Optional[str] = None,
+    ) -> ConnectorGraphResult:
+        """Convert a list of records directly to typed graph nodes.
+
+        Args:
+            records: List of dicts (rows/records)
+            label: Node label (e.g., 'Customer', 'Order')
+            key_field: Field to use as deterministic node ID (e.g., 'id')
+            source_node_id: Parent node to link via HAS_RECORD edge
+
+        Returns:
+            ConnectorGraphResult with nodes and edges
+        """
+        import hashlib
+        result = ConnectorGraphResult()
+
+        for record in records:
+            # Deterministic ID from key field if available
+            if key_field and key_field in record:
+                key_val = str(record[key_field])
+                node_id = f"{label.lower()}_{hashlib.sha256(key_val.encode()).hexdigest()[:12]}"
+            else:
+                node_id = self._uid()
+
+            # Clean properties: skip None, convert complex types
+            props = {"name": str(record.get(key_field, record.get("name", node_id[:20])))}
+            for k, v in record.items():
+                if v is None:
+                    continue
+                if isinstance(v, (dict, list)):
+                    props[k] = str(v)[:500]
+                else:
+                    props[k] = v
+            props["_source_connector"] = self.connector_type
+
+            result.nodes.append(self._node(label, props, node_id=node_id))
+
+            if source_node_id:
+                result.edges.append(self._edge(source_node_id, node_id, "HAS_RECORD"))
+
+        return result
+
+    def _node(self, label: str, properties: Dict[str, Any], node_id: Optional[str] = None) -> Dict:
+        return {"id": node_id or self._uid(), "label": label, "properties": properties}
+
+    def _edge(self, source: str, target: str, label: str, properties: Optional[Dict[str, Any]] = None) -> Dict:
+        return {"id": self._uid(), "source": source, "target": target, "label": label, "properties": properties or {}}