contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/plugins/domain_loader.py

@@ -0,0 +1,241 @@
+"""Domain Plugin Loader — discovers and loads domain plugins.
+
+Searches for domain.yaml files in:
+  1. plugins/ directory (built-in)
+  2. verticals/*/config/ (vertical-specific)
+  3. ~/.contextsynapse/plugins/ (user-installed)
+
+Usage:
+    loader = PluginLoader()
+    finance = loader.get_domain("finance")
+    life_sci = loader.get_domain("life_science")
+    all_domains = loader.list_domains()
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from contextsynapse.plugins.domain_interface import DomainPlugin
+
+logger = logging.getLogger(__name__)
+
+# Project root — two levels up from this file (contextsynapse/plugins/ -> project root)
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+
+# Search paths for domain.yaml files, in priority order
+_SEARCH_DIRS = [
+    _PROJECT_ROOT / "plugins",                       # top-level domain plugins
+    _PROJECT_ROOT / "contextsynapse" / "plugins",    # built-in
+    _PROJECT_ROOT / "verticals",                     # vertical-specific (*/config/)
+    Path.home() / ".contextsynapse" / "plugins",     # user-installed
+]
+
+
+class PluginLoader:
+    """Discovers, loads, and caches domain plugins.
+
+    The loader scans known directories for ``domain.yaml`` files,
+    parses them into ``DomainPlugin`` instances, and caches them
+    by ``domain_id``.  Plugins can also be registered at runtime.
+
+    The active domain is determined by the ``CONTEXTSYNAPSE_DOMAIN``
+    environment variable, defaulting to ``"finance"``.
+    """
+
+    def __init__(self) -> None:
+        self._domains: Dict[str, DomainPlugin] = {}
+        self._scan_complete = False
+        self._scan_and_load()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def get_domain(self, domain_id: str) -> Optional[DomainPlugin]:
+        """Get a loaded domain plugin by its ID.
+
+        Args:
+            domain_id: The domain identifier (e.g. "finance", "life_science").
+
+        Returns:
+            DomainPlugin instance or None if not found.
+        """
+        return self._domains.get(domain_id)
+
+    def list_domains(self) -> List[Dict[str, Any]]:
+        """List all loaded domain plugins with summary metadata.
+
+        Returns:
+            List of dicts with keys: domain_id, domain_name, version,
+            entity_count, sensor_count.
+        """
+        return [
+            {
+                "domain_id": plugin.domain_id,
+                "domain_name": plugin.domain_name,
+                "version": plugin.version,
+                "entity_count": len(plugin.entity_types),
+                "sensor_count": len(plugin.sensors),
+            }
+            for plugin in self._domains.values()
+        ]
+
+    def get_active_domain(self) -> Optional[DomainPlugin]:
+        """Get the currently active domain plugin.
+
+        Determined by the ``CONTEXTSYNAPSE_DOMAIN`` environment variable.
+        Falls back to ``"finance"`` if not set.  Returns None if the
+        requested domain is not loaded.
+
+        Returns:
+            The active DomainPlugin, or None.
+        """
+        domain_id = os.environ.get("CONTEXTSYNAPSE_DOMAIN", "finance")
+        plugin = self.get_domain(domain_id)
+        if plugin is None:
+            logger.warning(
+                "Active domain '%s' not found among loaded domains: %s",
+                domain_id,
+                list(self._domains.keys()),
+            )
+        return plugin
+
+    def register_domain(self, plugin: DomainPlugin) -> None:
+        """Register a domain plugin at runtime.
+
+        If a plugin with the same ``domain_id`` is already loaded it
+        will be replaced (with a warning).
+
+        Args:
+            plugin: A fully configured DomainPlugin instance.
+
+        Raises:
+            ValueError: If the plugin has no domain_id.
+        """
+        if not plugin.domain_id:
+            raise ValueError("Cannot register a domain plugin without a domain_id")
+
+        if plugin.domain_id in self._domains:
+            logger.warning(
+                "Replacing already-loaded domain '%s' (v%s -> v%s)",
+                plugin.domain_id,
+                self._domains[plugin.domain_id].version,
+                plugin.version,
+            )
+
+        self._domains[plugin.domain_id] = plugin
+        logger.info(
+            "Registered domain '%s' v%s (%d entity types, %d sensors)",
+            plugin.domain_id,
+            plugin.version,
+            len(plugin.entity_types),
+            len(plugin.sensors),
+        )
+
+    def load_yaml(self, yaml_path: str) -> DomainPlugin:
+        """Explicitly load a domain YAML file and register it.
+
+        Args:
+            yaml_path: Path to a domain.yaml file.
+
+        Returns:
+            The loaded DomainPlugin instance.
+        """
+        plugin = DomainPlugin.from_yaml(yaml_path)
+        self.register_domain(plugin)
+        return plugin
+
+    @property
+    def domain_count(self) -> int:
+        """Number of loaded domain plugins."""
+        return len(self._domains)
+
+    # ------------------------------------------------------------------
+    # Internal — scanning and loading
+    # ------------------------------------------------------------------
+
+    def _scan_and_load(self) -> None:
+        """Scan all search directories for domain.yaml files and load them."""
+        if self._scan_complete:
+            return
+
+        yaml_files = self._discover_yaml_files()
+
+        for yf in yaml_files:
+            try:
+                plugin = DomainPlugin.from_yaml(str(yf))
+                if plugin.domain_id in self._domains:
+                    existing = self._domains[plugin.domain_id]
+                    logger.debug(
+                        "Domain '%s' already loaded from earlier path (keeping first, "
+                        "skipping %s)",
+                        plugin.domain_id,
+                        yf,
+                    )
+                    continue
+                self._domains[plugin.domain_id] = plugin
+            except Exception:
+                logger.exception("Failed to load domain YAML: %s", yf)
+
+        self._scan_complete = True
+
+        if self._domains:
+            logger.info(
+                "Domain loader ready: %d domain(s) loaded — %s",
+                len(self._domains),
+                ", ".join(sorted(self._domains.keys())),
+            )
+        else:
+            logger.info("Domain loader ready: no domain YAML files found")
+
+    def _discover_yaml_files(self) -> List[Path]:
+        """Find all domain.yaml files across search directories.
+
+        Search order:
+          1. contextsynapse/plugins/domain.yaml, contextsynapse/plugins/*/domain.yaml
+          2. verticals/*/config/domain.yaml
+          3. ~/.contextsynapse/plugins/domain.yaml, ~/.contextsynapse/plugins/*/domain.yaml
+
+        Returns:
+            Deduplicated list of Path objects.
+        """
+        found: List[Path] = []
+        seen: set = set()
+
+        for search_dir in _SEARCH_DIRS:
+            if not search_dir.exists():
+                continue
+
+            # Direct domain.yaml in the directory
+            direct = search_dir / "domain.yaml"
+            if direct.exists() and direct.resolve() not in seen:
+                found.append(direct)
+                seen.add(direct.resolve())
+
+            # Subdirectories: look for domain.yaml or config/domain.yaml
+            try:
+                for child in sorted(search_dir.iterdir()):
+                    if not child.is_dir():
+                        continue
+
+                    candidates = [
+                        child / "domain.yaml",
+                        child / "config" / "domain.yaml",
+                    ]
+                    for candidate in candidates:
+                        if candidate.exists() and candidate.resolve() not in seen:
+                            found.append(candidate)
+                            seen.add(candidate.resolve())
+            except PermissionError:
+                logger.debug("Permission denied scanning: %s", search_dir)
+
+        logger.debug("Discovered %d domain YAML file(s): %s", len(found), found)
+        return found
+
+    def __repr__(self) -> str:
+        domains = ", ".join(sorted(self._domains.keys())) or "none"
+        return f"PluginLoader(domains=[{domains}])"

contextsynapse/plugins/loader.py

@@ -0,0 +1,148 @@
+"""Plugin discovery and loading via Python entry points.
+
+Plugins register themselves in pyproject.toml:
+
+    [project.entry-points."contextsynapse.plugins"]
+    finance = "qgraph_finance.plugin:FinanceVertical"
+
+The loader discovers all installed plugins and instantiates them.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Dict, List, Optional, Type
+
+from contextsynapse.plugins.base import PluginState, VerticalPlugin
+
+logger = logging.getLogger(__name__)
+
+ENTRY_POINT_GROUP = "contextsynapse.plugins"
+
+
+def discover_plugins(
+    exclude: Optional[List[str]] = None,
+) -> List[VerticalPlugin]:
+    """Discover and instantiate all installed plugins.
+
+    Uses Python entry points (PEP 621) to find plugins registered
+    under the 'contextsynapse.plugins' group.
+
+    Args:
+        exclude: List of plugin names to skip.
+
+    Returns:
+        List of instantiated plugin objects.
+    """
+    exclude = set(exclude or [])
+    plugins: List[VerticalPlugin] = []
+
+    try:
+        from importlib.metadata import entry_points
+    except ImportError:
+        from importlib_metadata import entry_points  # Python 3.9 fallback
+
+    try:
+        eps = entry_points(group=ENTRY_POINT_GROUP)
+    except TypeError:
+        # Python 3.9: entry_points() returns a dict
+        all_eps = entry_points()
+        eps = all_eps.get(ENTRY_POINT_GROUP, [])
+
+    for ep in eps:
+        if ep.name in exclude:
+            logger.info("Skipping excluded plugin: %s", ep.name)
+            continue
+
+        try:
+            t0 = time.monotonic()
+            cls = ep.load()
+            if not (isinstance(cls, type) and issubclass(cls, VerticalPlugin)):
+                logger.warning(
+                    "Entry point '%s' does not point to a VerticalPlugin subclass, skipping",
+                    ep.name,
+                )
+                continue
+
+            instance = cls()
+            elapsed = (time.monotonic() - t0) * 1000
+            logger.info(
+                "Discovered plugin '%s' v%s (loaded in %.1fms)",
+                instance.name,
+                instance.version,
+                elapsed,
+            )
+            plugins.append(instance)
+
+        except Exception:
+            logger.exception("Failed to load plugin '%s'", ep.name)
+
+    return plugins
+
+
+def load_plugin(
+    module_path: str, class_name: str
+) -> Optional[VerticalPlugin]:
+    """Load a single plugin by module path and class name.
+
+    For explicit loading without entry points (e.g. dev/testing).
+
+    Args:
+        module_path: e.g. "qgraph_finance.plugin"
+        class_name: e.g. "FinanceVertical"
+
+    Returns:
+        Instantiated plugin or None on failure.
+    """
+    try:
+        import importlib
+
+        module = importlib.import_module(module_path)
+        cls = getattr(module, class_name)
+
+        if not (isinstance(cls, type) and issubclass(cls, VerticalPlugin)):
+            logger.error("%s.%s is not a VerticalPlugin subclass", module_path, class_name)
+            return None
+
+        instance = cls()
+        logger.info("Loaded plugin '%s' v%s", instance.name, instance.version)
+        return instance
+
+    except Exception:
+        logger.exception("Failed to load plugin %s.%s", module_path, class_name)
+        return None
+
+
+def validate_plugin(plugin: VerticalPlugin) -> List[str]:
+    """Validate a plugin's configuration. Returns list of issues (empty = valid)."""
+    issues: List[str] = []
+
+    if not plugin.name or not plugin.name.strip():
+        issues.append("Plugin name is empty")
+    elif not plugin.name.replace("_", "").replace("-", "").isalnum():
+        issues.append(f"Plugin name '{plugin.name}' contains invalid characters")
+    if not plugin.version:
+        issues.append("Plugin version is empty")
+
+    seen_sensor_names = set()
+    for sensor in plugin.sensors():
+        if sensor.name in seen_sensor_names:
+            issues.append(f"Duplicate sensor name: '{sensor.name}'")
+        seen_sensor_names.add(sensor.name)
+        if sensor.interval_seconds < 10:
+            issues.append(f"Sensor '{sensor.name}' interval too short ({sensor.interval_seconds}s, min 10s)")
+
+    seen_schema_names = set()
+    for schema in plugin.schemas():
+        if schema.name in seen_schema_names:
+            issues.append(f"Duplicate schema name: '{schema.name}'")
+        seen_schema_names.add(schema.name)
+
+    for card in plugin.dashboard_cards():
+        if not card.id:
+            issues.append("Dashboard card has empty id")
+        if not card.route:
+            issues.append(f"Dashboard card '{card.id}' has empty route")
+
+    return issues

contextsynapse/plugins/registry.py

@@ -0,0 +1,265 @@
+"""Plugin registry — manages plugin lifecycle and provides access to loaded plugins."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import threading
+import time
+from typing import Any, Dict, List, Optional
+
+from contextsynapse.plugins.base import (
+    DashboardCard,
+    PluginMCPTool,
+    PluginRoute,
+    PluginSchema,
+    PluginState,
+    Sensor,
+    VerticalPlugin,
+)
+from contextsynapse.plugins.loader import discover_plugins, validate_plugin
+
+logger = logging.getLogger(__name__)
+
+_global_registry: Optional["PluginRegistry"] = None
+
+
+class PluginRegistry:
+    """Central registry for all loaded plugins.
+
+    The registry manages plugin lifecycle (discovery → validation →
+    startup → shutdown) and provides aggregated access to all
+    schemas, sensors, routes, MCP tools, and dashboard cards
+    across all plugins.
+    """
+
+    def __init__(self):
+        self._plugins: Dict[str, VerticalPlugin] = {}
+        self._sensor_tasks: Dict[str, asyncio.Task] = {}
+        self._started = False
+
+    @property
+    def plugins(self) -> Dict[str, VerticalPlugin]:
+        return dict(self._plugins)
+
+    def register(self, plugin: VerticalPlugin) -> bool:
+        """Register a plugin manually (without entry point discovery)."""
+        issues = validate_plugin(plugin)
+        if issues:
+            logger.error(
+                "Plugin '%s' validation failed: %s", plugin.name, "; ".join(issues)
+            )
+            return False
+
+        if plugin.name in self._plugins:
+            logger.warning("Plugin '%s' already registered, replacing", plugin.name)
+
+        self._plugins[plugin.name] = plugin
+        logger.info("Registered plugin '%s' v%s", plugin.name, plugin.version)
+        return True
+
+    def discover_and_register(self, exclude: Optional[List[str]] = None) -> int:
+        """Discover plugins via entry points and register them.
+
+        Returns:
+            Number of plugins successfully registered.
+        """
+        plugins = discover_plugins(exclude=exclude)
+        count = 0
+        for plugin in plugins:
+            if self.register(plugin):
+                count += 1
+        return count
+
+    async def startup_all(self, db: Any) -> None:
+        """Start all registered plugins.
+
+        Args:
+            db: AIContextDB instance or GraphRegistry.
+        """
+        for name, plugin in self._plugins.items():
+            try:
+                plugin._state = PluginState.STARTING
+                t0 = time.monotonic()
+                await plugin.on_startup(db)
+                elapsed = (time.monotonic() - t0) * 1000
+                plugin._state = PluginState.RUNNING
+                logger.info("Started plugin '%s' in %.1fms", name, elapsed)
+            except Exception:
+                plugin._state = PluginState.ERROR
+                logger.exception("Failed to start plugin '%s'", name)
+
+        self._started = True
+        self._start_sensors(db)
+
+    async def shutdown_all(self) -> None:
+        """Shut down all plugins and cancel sensor tasks."""
+        # Cancel sensor tasks first
+        for task_name, task in self._sensor_tasks.items():
+            task.cancel()
+            try:
+                await asyncio.wait_for(asyncio.shield(task), timeout=5.0)
+            except (asyncio.CancelledError, asyncio.TimeoutError):
+                pass
+        self._sensor_tasks.clear()
+
+        for name, plugin in self._plugins.items():
+            try:
+                await plugin.on_shutdown()
+                logger.info("Stopped plugin '%s'", name)
+            except Exception:
+                logger.exception("Error stopping plugin '%s'", name)
+
+        self._started = False
+
+    def _start_sensors(self, db: Any) -> None:
+        """Start background sensor collection tasks."""
+        for plugin in self._plugins.values():
+            if plugin.state != PluginState.RUNNING:
+                continue
+            for sensor in plugin.sensors():
+                if not sensor.enabled:
+                    continue
+                task_name = f"{plugin.name}.{sensor.name}"
+                task = asyncio.create_task(
+                    self._sensor_loop(plugin, sensor, db),
+                    name=task_name,
+                )
+                self._sensor_tasks[task_name] = task
+                logger.info(
+                    "Started sensor '%s' (every %ds)",
+                    task_name,
+                    sensor.interval_seconds,
+                )
+
+    async def _sensor_loop(
+        self, plugin: VerticalPlugin, sensor: Sensor, db: Any
+    ) -> None:
+        """Run a sensor's collect() on its interval.
+
+        Uses the StorageRouter to route data to the right store:
+        - Entities + edges -> Graph (thin nodes)
+        - Passages, facts -> DuckDB (content)
+        - Prices -> DuckDB (time-series, no graph nodes)
+        """
+        from contextsynapse.storage.router import get_storage_router
+
+        router = get_storage_router(graph=db, namespace=plugin.name)
+
+        while True:
+            try:
+                if not sensor.enabled:
+                    await asyncio.sleep(sensor.interval_seconds)
+                    continue
+
+                t0 = time.monotonic()
+                results = await sensor.collect(db)
+                elapsed_collect = (time.monotonic() - t0) * 1000
+
+                # Route all items through the storage router
+                t1 = time.monotonic()
+                items = results[:sensor.batch_size]
+                counts = router.ingest(items, namespace=plugin.name)
+                elapsed_store = (time.monotonic() - t1) * 1000
+
+                logger.debug(
+                    "Sensor '%s.%s' collected %d items in %.1fms, stored in %.1fms "
+                    "(graph: %d nodes + %d edges, content: %d rows, prices: %d)",
+                    plugin.name, sensor.name, len(items),
+                    elapsed_collect, elapsed_store,
+                    counts["graph_nodes"], counts["graph_edges"],
+                    counts["content_rows"], counts["prices"],
+                )
+
+                # Sleep AFTER collection so first run is immediate
+                await asyncio.sleep(sensor.interval_seconds)
+
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                logger.exception(
+                    "Sensor '%s.%s' collection failed", plugin.name, sensor.name
+                )
+                await asyncio.sleep(60)
+
+    # ── Aggregated access ──
+
+    def get_plugin(self, name: str) -> Optional[VerticalPlugin]:
+        return self._plugins.get(name)
+
+    def all_schemas(self) -> List[PluginSchema]:
+        """All schemas from all plugins."""
+        schemas = []
+        for plugin in self._plugins.values():
+            schemas.extend(plugin.schemas())
+        return schemas
+
+    def all_sensors(self) -> List[tuple]:
+        """All sensors as (plugin_name, sensor) tuples."""
+        result = []
+        for plugin in self._plugins.values():
+            for sensor in plugin.sensors():
+                result.append((plugin.name, sensor))
+        return result
+
+    def all_api_routers(self) -> List[tuple]:
+        """All API routers as (plugin_name, PluginRoute) tuples."""
+        result = []
+        for plugin in self._plugins.values():
+            for route in plugin.api_routers():
+                result.append((plugin.name, route))
+        return result
+
+    def all_mcp_tools(self) -> List[tuple]:
+        """All MCP tools as (plugin_name, PluginMCPTool) tuples."""
+        result = []
+        for plugin in self._plugins.values():
+            for tool in plugin.mcp_tools():
+                result.append((plugin.name, tool))
+        return result
+
+    def all_dashboard_cards(self) -> List[DashboardCard]:
+        """All dashboard cards from all plugins, sorted by order."""
+        cards = []
+        for plugin in self._plugins.values():
+            cards.extend(plugin.dashboard_cards())
+        return sorted(cards, key=lambda c: c.order)
+
+    def status(self) -> Dict[str, Any]:
+        """Return registry status."""
+        return {
+            "plugins_registered": len(self._plugins),
+            "plugins_running": sum(
+                1 for p in self._plugins.values() if p.state == PluginState.RUNNING
+            ),
+            "sensors_active": len(self._sensor_tasks),
+            "plugins": {
+                name: plugin.info() for name, plugin in self._plugins.items()
+            },
+        }
+
+    def emit_graph_event(self, event_type: str, payload: Dict[str, Any]) -> None:
+        """Forward a graph event to all running plugins."""
+        for plugin in self._plugins.values():
+            if plugin.state == PluginState.RUNNING:
+                try:
+                    plugin.on_graph_event(event_type, payload)
+                except Exception:
+                    logger.exception(
+                        "Plugin '%s' failed handling event '%s'",
+                        plugin.name,
+                        event_type,
+                    )
+
+
+_registry_lock = threading.Lock()
+
+
+def get_plugin_registry() -> PluginRegistry:
+    """Get or create the global plugin registry singleton."""
+    global _global_registry
+    if _global_registry is None:
+        with _registry_lock:
+            if _global_registry is None:
+                _global_registry = PluginRegistry()
+    return _global_registry

contextsynapse/project/__init__.py

@@ -0,0 +1,11 @@
+"""
+Project Knowledge Graph
+========================
+Structures a codebase as a knowledge graph with tasks, documents,
+decisions, and code files — enabling multi-agent collaboration
+where Claude and Codex work as a team.
+"""
+
+from .graph import ProjectGraph
+
+__all__ = ["ProjectGraph"]