contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/connectors/registry.py

@@ -0,0 +1,116 @@
+"""ConnectorRegistry — YAML-driven connector auto-instantiation.
+
+Loads connector configurations from YAML files and creates connector
+instances using registered connector classes.
+
+Usage:
+    reg = ConnectorRegistry()
+    reg.register_connector_class("amfi_nav", AmfiNavConnector)
+    reg.load_from_directory("config/connectors/")
+    connectors = reg.create_all()
+"""
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Type
+
+import yaml
+
+from .base import BaseConnector, ConnectorConfig
+
+logger = logging.getLogger(__name__)
+
+
+class ConnectorRegistry:
+    """Loads YAML configs and instantiates connectors by type."""
+
+    def __init__(self):
+        self._classes: Dict[str, Type[BaseConnector]] = {}
+        self._configs: List[ConnectorConfig] = []
+
+    @property
+    def configs(self) -> List[ConnectorConfig]:
+        return list(self._configs)
+
+    def register_connector_class(
+        self, connector_type: str, cls: Type[BaseConnector]
+    ) -> None:
+        """Register a connector class for a given type string."""
+        self._classes[connector_type] = cls
+
+    def get_connector_class(self, connector_type: str) -> Type[BaseConnector]:
+        """Get the registered class for a connector type."""
+        if connector_type not in self._classes:
+            raise KeyError(
+                f"Unknown connector type: {connector_type!r}. "
+                f"Registered: {list(self._classes.keys())}"
+            )
+        return self._classes[connector_type]
+
+    def load_from_yaml(self, path: str) -> ConnectorConfig:
+        """Load and validate a single YAML connector config."""
+        with open(path, "r") as f:
+            data = yaml.safe_load(f) or {}
+
+        if not data.get("name"):
+            raise ValueError(
+                f"Connector config {path} missing required field: name"
+            )
+        if not data.get("connector_type"):
+            raise ValueError(
+                f"Connector config {path} missing required field: connector_type"
+            )
+
+        config = ConnectorConfig(
+            connector_id=data.get("connector_id", ""),
+            connector_type=data["connector_type"],
+            name=data["name"],
+            target_context_id=data.get("target_context_id", ""),
+            pipeline=data.get("pipeline", "builtin:knowledge-graph"),
+            llm_model=data.get("llm_model", ""),
+            embedding_model=data.get("embedding_model", ""),
+            poll_interval_minutes=data.get("poll_interval_minutes", 5),
+            active=data.get("active", True),
+            config=data.get("config", {}),
+        )
+        self._configs.append(config)
+        return config
+
+    def load_from_directory(self, config_dir: str) -> int:
+        """Load all YAML configs from a directory. Returns count loaded."""
+        loaded = 0
+        config_path = Path(config_dir)
+        if not config_path.is_dir():
+            logger.warning("[REGISTRY] Config dir not found: %s", config_dir)
+            return 0
+
+        for f in sorted(config_path.iterdir()):
+            if f.suffix not in (".yaml", ".yml"):
+                continue
+            try:
+                self.load_from_yaml(str(f))
+                loaded += 1
+            except Exception as e:
+                logger.warning("[REGISTRY] Skipping %s: %s", f.name, e)
+        return loaded
+
+    def create_connector(self, config: ConnectorConfig) -> BaseConnector:
+        """Create a connector instance from a config."""
+        cls = self.get_connector_class(config.connector_type)
+        return cls(config)
+
+    def create_all(self) -> List[BaseConnector]:
+        """Create connector instances for all loaded configs."""
+        connectors = []
+        for config in self._configs:
+            try:
+                connector = self.create_connector(config)
+                connectors.append(connector)
+            except KeyError:
+                logger.warning(
+                    "[REGISTRY] No class registered for type %r, skipping %s",
+                    config.connector_type, config.name,
+                )
+        return connectors

contextsynapse/connectors/scheduler.py

@@ -0,0 +1,87 @@
+"""ConnectorScheduler — runs connectors on their configured intervals.
+
+Usage:
+    scheduler = ConnectorScheduler(registry, manager)
+    results = scheduler.run_due()   # runs only due connectors
+    results = scheduler.run_all()   # runs all connectors
+"""
+from __future__ import annotations
+
+import logging
+import time
+from typing import Dict, List
+
+from .base import ConnectorManager
+from .registry import ConnectorRegistry
+
+logger = logging.getLogger(__name__)
+
+
+class ConnectorScheduler:
+    """Runs connectors based on their poll_interval_minutes."""
+
+    def __init__(self, registry: ConnectorRegistry, manager: ConnectorManager):
+        self._registry = registry
+        self._manager = manager
+        self._last_run: Dict[str, float] = {}  # connector_id → timestamp
+        self._intervals: Dict[str, float] = {}  # connector_id → seconds
+
+        # Initialize intervals from registry configs
+        for config in registry.configs:
+            cid = config.connector_id or config.name
+            self._intervals[cid] = config.poll_interval_minutes * 60
+
+        # Also pull intervals directly from manager's registered connectors
+        # (when connectors are registered manually without going through registry YAML load)
+        for cid, connector in manager._connectors.items():
+            if cid not in self._intervals:
+                self._intervals[cid] = connector.config.poll_interval_minutes * 60
+
+        # Seed last_run=now for connectors with interval > 0 so they don't
+        # all fire immediately on first run_due() call. Connectors with
+        # interval==0 are left unseeded so they always fire immediately.
+        now = time.time()
+        for cid, interval in self._intervals.items():
+            if interval > 0 and cid not in self._last_run:
+                self._last_run[cid] = now
+
+    def run_due(self) -> Dict[str, int]:
+        """Run connectors whose interval has elapsed. Returns {id: doc_count}."""
+        now = time.time()
+        results = {}
+
+        for cid, connector in self._manager._connectors.items():
+            interval = self._intervals.get(cid, 300)  # default 5 min
+            last = self._last_run.get(cid, 0)
+
+            if (now - last) >= interval:
+                docs = self._manager.poll_one(cid)
+                results[cid] = len(docs)
+                self._last_run[cid] = now
+                if docs:
+                    logger.info("[SCHEDULER] %s: %d new docs", cid, len(docs))
+
+        return results
+
+    def run_all(self) -> Dict[str, int]:
+        """Run all connectors regardless of schedule."""
+        results = self._manager.poll_all()
+        now = time.time()
+        for cid in results:
+            self._last_run[cid] = now
+        return results
+
+    def get_status(self) -> List[Dict]:
+        """Get status of all connectors."""
+        statuses = self._manager.list_status()
+        result = []
+        for s in statuses:
+            d = s.to_dict()
+            # Enrich with connector name from config
+            connector = self._manager._connectors.get(s.connector_id)
+            if connector:
+                d["name"] = connector.config.name
+            d["last_scheduled_run"] = self._last_run.get(s.connector_id)
+            d["interval_minutes"] = self._intervals.get(s.connector_id, 300) / 60
+            result.append(d)
+        return result

contextsynapse/connectors/universal.py

@@ -0,0 +1,360 @@
+"""Universal Connectors -- plug-and-play data sources for ContextCore.
+
+5 built-in connectors that any vertical can use:
+  1. RSS/Atom feed connector
+  2. REST API connector (any JSON API)
+  3. Web scraper connector (URL -> passages)
+  4. File watcher connector (local directory)
+  5. Webhook receiver connector (push-based)
+
+Each connector implements collect() -> List[dict] that returns
+items ready for the StorageRouter.
+
+Usage:
+    from contextsynapse.connectors.universal import RSSConnector, APIConnector
+
+    rss = RSSConnector(feeds=["https://news.ycombinator.com/rss"])
+    items = await rss.collect()
+    engine.ingest(items)
+"""
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+import time
+from abc import ABC, abstractmethod
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from dataclasses import dataclass, field
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ConnectorConfig:
+    """Configuration for a connector."""
+    name: str = ""
+    interval_seconds: int = 300
+    max_items: int = 100
+    namespace: str = "default"
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+class UniversalConnector(ABC):
+    """Base class for universal connectors.
+
+    Subclasses implement collect() which returns items
+    ready for engine.ingest().
+    """
+
+    def __init__(self, config: ConnectorConfig = None):
+        self.config = config or ConnectorConfig()
+        self._last_poll = 0
+        self._seen_ids: set = set()
+
+    @property
+    @abstractmethod
+    def connector_type(self) -> str:
+        """e.g. 'rss', 'api', 'scraper', 'file', 'webhook'"""
+
+    @abstractmethod
+    async def collect(self) -> List[Dict[str, Any]]:
+        """Collect data from the source. Returns items for engine.ingest()."""
+
+    async def health_check(self) -> Dict[str, Any]:
+        return {"healthy": True, "type": self.connector_type, "name": self.config.name}
+
+    def _dedup_id(self, content: str) -> str:
+        return hashlib.md5(content.encode()).hexdigest()[:12]
+
+
+class RSSConnector(UniversalConnector):
+    """RSS/Atom feed connector. Polls feeds and returns articles as Document + Passage items."""
+
+    connector_type = "rss"
+
+    def __init__(self, feeds: List[str], config: ConnectorConfig = None):
+        super().__init__(config)
+        self.feeds = feeds
+
+    async def collect(self) -> List[Dict[str, Any]]:
+        try:
+            import feedparser
+        except ImportError:
+            logger.error("feedparser not installed: pip install feedparser")
+            return []
+
+        items = []
+        for feed_url in self.feeds:
+            try:
+                feed = feedparser.parse(feed_url)
+                for entry in feed.entries[:self.config.max_items]:
+                    entry_id = self._dedup_id(entry.get("link", entry.get("title", "")))
+                    if entry_id in self._seen_ids:
+                        continue
+                    self._seen_ids.add(entry_id)
+
+                    doc_id = f"rss_{entry_id}"
+                    title = entry.get("title", "")
+                    summary = entry.get("summary", entry.get("description", ""))
+                    link = entry.get("link", "")
+                    published = entry.get("published", "")
+
+                    # Document node
+                    items.append({
+                        "id": doc_id, "type": "Document",
+                        "properties": {
+                            "title": title, "url": link,
+                            "date": published, "source": "rss",
+                            "feed_url": feed_url,
+                        },
+                    })
+
+                    # Passage from summary
+                    if summary and len(summary) > 20:
+                        items.append({
+                            "id": f"p_{entry_id}", "type": "Passage",
+                            "properties": {
+                                "text": f"{title}. {summary}",
+                                "doc_id": doc_id, "position": 0,
+                            },
+                        })
+
+            except Exception as e:
+                logger.warning("RSS feed error %s: %s", feed_url, e)
+
+        return items
+
+
+class APIConnector(UniversalConnector):
+    """REST API connector. Polls any JSON API and maps response to items."""
+
+    connector_type = "api"
+
+    def __init__(self, url: str, headers: Dict[str, str] = None,
+                 item_path: str = "", item_type: str = "Document",
+                 text_field: str = "content", id_field: str = "id",
+                 config: ConnectorConfig = None):
+        super().__init__(config)
+        self.url = url
+        self.headers = headers or {}
+        self.item_path = item_path  # JSON path to items array (e.g. "data.articles")
+        self.item_type = item_type
+        self.text_field = text_field
+        self.id_field = id_field
+
+    async def collect(self) -> List[Dict[str, Any]]:
+        try:
+            import aiohttp
+        except ImportError:
+            # Fallback to urllib
+            import urllib.request
+            import json
+            req = urllib.request.Request(self.url, headers=self.headers)
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                data = json.loads(resp.read())
+            return self._parse_response(data)
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(self.url, headers=self.headers, timeout=aiohttp.ClientTimeout(total=30)) as resp:
+                data = await resp.json()
+                return self._parse_response(data)
+
+    def _parse_response(self, data: Any) -> List[Dict[str, Any]]:
+        # Navigate to item_path
+        items_raw = data
+        if self.item_path:
+            for key in self.item_path.split("."):
+                if isinstance(items_raw, dict):
+                    items_raw = items_raw.get(key, [])
+                else:
+                    break
+
+        if not isinstance(items_raw, list):
+            items_raw = [items_raw]
+
+        items = []
+        for raw in items_raw[:self.config.max_items]:
+            if not isinstance(raw, dict):
+                continue
+            item_id = str(raw.get(self.id_field, self._dedup_id(str(raw))))
+            text = str(raw.get(self.text_field, ""))
+
+            items.append({
+                "id": f"api_{item_id}", "type": self.item_type,
+                "properties": {"text": text, "source": "api", "api_url": self.url, **raw},
+            })
+        return items
+
+
+class WebScraperConnector(UniversalConnector):
+    """Web scraper connector. Fetches a URL, extracts text, returns as passages."""
+
+    connector_type = "scraper"
+
+    def __init__(self, urls: List[str], config: ConnectorConfig = None):
+        super().__init__(config)
+        self.urls = urls
+
+    async def collect(self) -> List[Dict[str, Any]]:
+        items = []
+        for url in self.urls:
+            try:
+                import urllib.request
+                req = urllib.request.Request(url, headers={"User-Agent": "ContextCore/1.0"})
+                with urllib.request.urlopen(req, timeout=30) as resp:
+                    html = resp.read().decode("utf-8", errors="ignore")
+
+                # Extract text (try trafilatura, fallback to regex)
+                text = ""
+                try:
+                    import trafilatura
+                    text = trafilatura.extract(html) or ""
+                except ImportError:
+                    import re
+                    text = re.sub(r"<[^>]+>", " ", html)
+                    text = re.sub(r"\s+", " ", text).strip()
+
+                if not text or len(text) < 50:
+                    continue
+
+                url_id = self._dedup_id(url)
+                doc_id = f"web_{url_id}"
+
+                # Extract title
+                title = ""
+                import re
+                title_match = re.search(r"<title>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+                if title_match:
+                    title = title_match.group(1).strip()
+
+                items.append({
+                    "id": doc_id, "type": "Document",
+                    "properties": {"title": title or url, "url": url, "source": "web"},
+                })
+
+                # Chunk into passages (~500 chars each)
+                chunks = [text[i:i+500] for i in range(0, len(text), 450)]
+                for idx, chunk in enumerate(chunks[:20]):  # max 20 passages per page
+                    items.append({
+                        "id": f"p_{url_id}_{idx}", "type": "Passage",
+                        "properties": {"text": chunk.strip(), "doc_id": doc_id, "position": idx},
+                    })
+
+            except Exception as e:
+                logger.warning("Scrape error %s: %s", url, e)
+
+        return items
+
+
+class FileWatcherConnector(UniversalConnector):
+    """File watcher connector. Monitors a directory for new/changed files."""
+
+    connector_type = "file"
+
+    def __init__(self, directory: str, extensions: List[str] = None,
+                 config: ConnectorConfig = None):
+        super().__init__(config)
+        self.directory = Path(directory)
+        self.extensions = set(extensions or [".txt", ".md", ".csv", ".json"])
+        self._file_mtimes: Dict[str, float] = {}
+
+    async def collect(self) -> List[Dict[str, Any]]:
+        if not self.directory.exists():
+            return []
+
+        items = []
+        for fpath in self.directory.rglob("*"):
+            if not fpath.is_file():
+                continue
+            if fpath.suffix.lower() not in self.extensions:
+                continue
+
+            mtime = fpath.stat().st_mtime
+            str_path = str(fpath)
+            if str_path in self._file_mtimes and self._file_mtimes[str_path] >= mtime:
+                continue  # unchanged
+            self._file_mtimes[str_path] = mtime
+
+            try:
+                content = fpath.read_text(encoding="utf-8", errors="ignore")
+                if len(content) < 10:
+                    continue
+
+                file_id = self._dedup_id(str_path)
+                doc_id = f"file_{file_id}"
+
+                items.append({
+                    "id": doc_id, "type": "Document",
+                    "properties": {
+                        "title": fpath.name, "url": str(fpath),
+                        "source": "file", "file_path": str_path,
+                        "file_size": fpath.stat().st_size,
+                    },
+                })
+
+                # Chunk into passages
+                chunks = [content[i:i+500] for i in range(0, len(content), 450)]
+                for idx, chunk in enumerate(chunks[:50]):
+                    items.append({
+                        "id": f"p_{file_id}_{idx}", "type": "Passage",
+                        "properties": {"text": chunk.strip(), "doc_id": doc_id, "position": idx},
+                    })
+
+            except Exception as e:
+                logger.warning("File read error %s: %s", fpath, e)
+
+        return items
+
+
+class WebhookConnector(UniversalConnector):
+    """Webhook receiver connector. Receives push data via HTTP POST.
+
+    Call webhook.receive(data) when a webhook arrives.
+    collect() returns buffered items.
+    """
+
+    connector_type = "webhook"
+
+    def __init__(self, item_type: str = "Document", text_field: str = "content",
+                 config: ConnectorConfig = None):
+        super().__init__(config)
+        self.item_type = item_type
+        self.text_field = text_field
+        self._buffer: List[Dict] = []
+
+    def receive(self, data: Dict[str, Any]) -> str:
+        """Buffer an incoming webhook payload. Returns item ID."""
+        item_id = f"wh_{self._dedup_id(str(data))}"
+        text = str(data.get(self.text_field, data.get("text", data.get("message", ""))))
+        self._buffer.append({
+            "id": item_id, "type": self.item_type,
+            "properties": {"text": text, "source": "webhook", **data},
+        })
+        return item_id
+
+    async def collect(self) -> List[Dict[str, Any]]:
+        """Return and clear the buffer."""
+        items = self._buffer[:self.config.max_items]
+        self._buffer = self._buffer[self.config.max_items:]
+        return items
+
+
+# ── Connector Registry ──
+
+BUILTIN_CONNECTORS = {
+    "rss": RSSConnector,
+    "api": APIConnector,
+    "scraper": WebScraperConnector,
+    "file": FileWatcherConnector,
+    "webhook": WebhookConnector,
+}
+
+def create_connector(connector_type: str, **kwargs) -> UniversalConnector:
+    """Factory to create a connector by type."""
+    cls = BUILTIN_CONNECTORS.get(connector_type)
+    if not cls:
+        raise ValueError(f"Unknown connector type: {connector_type}. Available: {list(BUILTIN_CONNECTORS.keys())}")
+    return cls(**kwargs)

contextsynapse/context/__init__.py

@@ -0,0 +1,96 @@
+"""
+Context-as-a-Service — Build, share, and export structured context for LLMs.
+
+Usage:
+    from contextsynapse.context import ContextHub, AgentRegistry, ContextSessionManager
+
+    # Register an agent
+    registry = AgentRegistry()
+    agent, api_key = registry.register("my-agent", role="researcher")
+
+    # Create a shared session
+    sessions = ContextSessionManager(graph_registry=my_registry)
+    session = sessions.create_session("project-alpha", owner_agent_id=agent.agent_id)
+
+    # Build context
+    hub = ContextHub(system_prompt="You are a graph analyst.")
+    hub.add_text("Some background", role="background")
+    messages = hub.to_messages()
+"""
+
+# Only import the most commonly used items eagerly
+from .hub import ContextHub, ContextItem, ContextRole, ContextFormat, Sensitivity, SlotConfig
+
+
+def __getattr__(name):
+    """Lazy import for less commonly used classes."""
+    _lazy_imports = {
+        "AgentIdentity": ".agents",
+        "AgentRegistry": ".agents",
+        "ProvenanceRecord": ".agents",
+        "ContextSession": ".session",
+        "ContextSessionManager": ".session",
+        "Context": ".context_manager",
+        "ContextManager": ".context_manager",
+        "BlobStore": ".blob",
+        "UnifiedIngestor": ".ingest",
+        "DocumentProcessor": ".document_processor",
+        "ContextEvent": ".pubsub",
+        "PubSubHub": ".pubsub",
+        "pubsub_hub": ".pubsub",
+        "SessionVectorStore": ".vector_integration",
+        "ContextScoper": ".scoping",
+        "ScopingConfig": ".scoping",
+        "ConversationStore": ".conversation",
+        "Conversation": ".conversation",
+        "ConversationMessage": ".conversation",
+        "EmbeddingHooks": ".embedding_hooks",
+        "HybridSync": ".sync",
+        "CompressionConfig": ".compression",
+        "CompressionTier": ".compression",
+        "InteractionGraphifier": ".interaction_graphifier",
+        "GraphifyResult": ".interaction_graphifier",
+        "SessionGraphBuilder": ".session_graph",
+        "AgentMemory": ".agent_memory",
+    }
+    if name in _lazy_imports:
+        import importlib
+        module = importlib.import_module(_lazy_imports[name], __package__)
+        return getattr(module, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    # Hub (original)
+    "ContextHub", "ContextItem", "ContextRole", "ContextFormat", "Sensitivity",
+    # Agents
+    "AgentIdentity", "AgentRegistry", "ProvenanceRecord",
+    # Sessions
+    "ContextSession", "ContextSessionManager",
+    # Contexts (first-class)
+    "Context", "ContextManager",
+    # Storage
+    "BlobStore",
+    # Ingestion
+    "UnifiedIngestor",
+    # Document processing
+    "DocumentProcessor",
+    # PubSub
+    "ContextEvent", "PubSubHub", "pubsub_hub",
+    # Vector
+    "SessionVectorStore",
+    # Scoping (Phase 2)
+    "ContextScoper", "ScopingConfig",
+    # Conversations (Phase 2)
+    "ConversationStore", "Conversation", "ConversationMessage",
+    # Embedding hooks (Phase 2)
+    "EmbeddingHooks",
+    # Sync (Phase 2)
+    "HybridSync",
+    # Compression & Slots (Phase 3)
+    "CompressionConfig", "CompressionTier", "SlotConfig",
+    # Session Graph (unified knowledge + conversation layer)
+    "InteractionGraphifier", "GraphifyResult", "SessionGraphBuilder",
+    # Agent Memory
+    "AgentMemory",
+]

contextsynapse/context/__main__.py

@@ -0,0 +1,4 @@
+"""Allow running the context CLI via: python -m contextsynapse.context"""
+from .cli import main
+
+main()