ciris-agent 1.7.7__py3-none-any.whl

ciris_engine/logic/config/__init__.py

@@ -0,0 +1,25 @@
+"""
+Configuration module for CIRIS Engine.
+
+Provides graph-based configuration with bootstrap support.
+"""
+
+from .bootstrap import ConfigBootstrap
+from .config_accessor import ConfigAccessor
+from .db_paths import (
+    get_audit_db_full_path,
+    get_graph_memory_full_path,
+    get_secrets_db_full_path,
+    get_sqlite_db_full_path,
+)
+from .env_utils import get_env_var
+
+__all__ = [
+    "ConfigBootstrap",
+    "ConfigAccessor",
+    "get_env_var",
+    "get_sqlite_db_full_path",
+    "get_secrets_db_full_path",
+    "get_audit_db_full_path",
+    "get_graph_memory_full_path",
+]

ciris_engine/logic/config/bootstrap.py

@@ -0,0 +1,255 @@
+"""
+Configuration Bootstrap for Graph-Based Config System.
+
+Loads essential configuration from multiple sources in priority order,
+then migrates to graph-based configuration for runtime management.
+"""
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import aiofiles
+import yaml
+
+from ciris_engine.schemas.config.essential import EssentialConfig
+from ciris_engine.schemas.types import ConfigDict, JSONDict
+
+from .env_utils import get_env_var
+
+logger = logging.getLogger(__name__)
+
+
+class ConfigBootstrap:
+    """Load essential config from multiple sources in priority order."""
+
+    @staticmethod
+    def _deep_merge(base: ConfigDict, update: ConfigDict) -> ConfigDict:
+        """Recursively merge two configuration dictionaries."""
+        for key, value in update.items():
+            if key in base and isinstance(base[key], dict) and isinstance(value, dict):
+                base[key] = ConfigBootstrap._deep_merge(base[key], value)  # type: ignore[arg-type]
+            else:
+                base[key] = value
+        return base
+
+    @staticmethod
+    def _apply_env_overrides(config_data: ConfigDict) -> ConfigDict:
+        """Apply environment variable overrides to config data."""
+        from typing import cast
+
+        # Database URL (overrides main_db path if set)
+        db_url = get_env_var("CIRIS_DB_URL")
+        if db_url:
+            db_section = cast(JSONDict, config_data.setdefault("database", {}))
+            db_section["database_url"] = db_url
+
+        # Database paths
+        db_path = get_env_var("CIRIS_DB_PATH")
+        if db_path:
+            db_section = cast(JSONDict, config_data.setdefault("database", {}))
+            db_section["main_db"] = db_path
+
+        secrets_db = get_env_var("CIRIS_SECRETS_DB_PATH")
+        if secrets_db:
+            db_section = cast(JSONDict, config_data.setdefault("database", {}))
+            db_section["secrets_db"] = secrets_db
+
+        audit_db = get_env_var("CIRIS_AUDIT_DB_PATH")
+        if audit_db:
+            db_section = cast(JSONDict, config_data.setdefault("database", {}))
+            db_section["audit_db"] = audit_db
+
+        # Service endpoints
+        llm_endpoint = get_env_var("OPENAI_API_BASE") or get_env_var("LLM_ENDPOINT")
+        if llm_endpoint:
+            services_section = cast(JSONDict, config_data.setdefault("services", {}))
+            services_section["llm_endpoint"] = llm_endpoint
+
+        llm_model = get_env_var("OPENAI_MODEL_NAME") or get_env_var("OPENAI_MODEL") or get_env_var("LLM_MODEL")
+        if llm_model:
+            services_section = cast(JSONDict, config_data.setdefault("services", {}))
+            services_section["llm_model"] = llm_model
+
+        # Security settings
+        retention_days = get_env_var("AUDIT_RETENTION_DAYS")
+        if retention_days:
+            try:
+                security_section = cast(JSONDict, config_data.setdefault("security", {}))
+                security_section["audit_retention_days"] = int(retention_days)
+            except ValueError:
+                logger.warning(f"Invalid AUDIT_RETENTION_DAYS value: {retention_days}")
+
+        # Operational limits
+        max_tasks = get_env_var("MAX_ACTIVE_TASKS")
+        if max_tasks:
+            try:
+                limits_section = cast(JSONDict, config_data.setdefault("limits", {}))
+                limits_section["max_active_tasks"] = int(max_tasks)
+            except ValueError:
+                logger.warning(f"Invalid MAX_ACTIVE_TASKS value: {max_tasks}")
+
+        max_depth = get_env_var("MAX_THOUGHT_DEPTH")
+        if max_depth:
+            try:
+                security_section = cast(JSONDict, config_data.setdefault("security", {}))
+                security_section["max_thought_depth"] = int(max_depth)
+            except ValueError:
+                logger.warning(f"Invalid MAX_THOUGHT_DEPTH value: {max_depth}")
+
+        # Runtime settings
+        log_level = get_env_var("LOG_LEVEL")
+        if log_level:
+            config_data["log_level"] = log_level.upper()
+
+        debug_mode = get_env_var("DEBUG_MODE")
+        if debug_mode:
+            config_data["debug_mode"] = debug_mode.lower() in ("true", "1", "yes", "on")
+
+        # Agent template (only used for first-time identity creation)
+        template = get_env_var("CIRIS_TEMPLATE")
+        if template:
+            config_data["default_template"] = template
+
+        return config_data
+
+    @staticmethod
+    def _resolve_database_paths(config_data: ConfigDict) -> ConfigDict:
+        """Resolve relative database paths to use proper data directory.
+
+        If database paths are relative (not absolute), resolve them against
+        get_data_dir() to ensure they use the correct location based on
+        runtime environment (development/installed/managed mode).
+        """
+        from typing import cast
+
+        from ciris_engine.logic.utils.path_resolution import get_data_dir
+
+        db_section = cast(JSONDict, config_data.get("database", {}))
+        data_dir = get_data_dir()
+
+        # Set or resolve main_db
+        if "main_db" not in db_section:
+            # Use schema default but resolved to proper data directory
+            db_section["main_db"] = str(data_dir / "ciris_engine.db")
+        else:
+            main_db = Path(str(db_section["main_db"]))
+            if not main_db.is_absolute():
+                # For relative paths, use just the filename (strip any data/ prefix)
+                db_section["main_db"] = str(data_dir / main_db.name)
+
+        # Set or resolve secrets_db
+        if "secrets_db" not in db_section:
+            db_section["secrets_db"] = str(data_dir / "secrets.db")
+        else:
+            secrets_db = Path(str(db_section["secrets_db"]))
+            if not secrets_db.is_absolute():
+                db_section["secrets_db"] = str(data_dir / secrets_db.name)
+
+        # Set or resolve audit_db
+        if "audit_db" not in db_section:
+            db_section["audit_db"] = str(data_dir / "ciris_audit.db")
+        else:
+            audit_db = Path(str(db_section["audit_db"]))
+            if not audit_db.is_absolute():
+                db_section["audit_db"] = str(data_dir / audit_db.name)
+
+        config_data["database"] = db_section
+        return config_data
+
+    @staticmethod
+    async def load_essential_config(
+        config_path: Optional[Path] = None, cli_overrides: Optional[ConfigDict] = None
+    ) -> EssentialConfig:
+        """
+        Load essential configuration from multiple sources.
+
+        Priority order (highest to lowest):
+        1. CLI arguments (if provided)
+        2. Environment variables
+        3. Configuration file (YAML)
+        4. Schema defaults
+
+        Args:
+            config_path: Optional path to YAML config file
+            cli_overrides: Optional CLI argument overrides
+
+        Returns:
+            Validated EssentialConfig instance
+        """
+        # Start with empty config data
+        config_data: ConfigDict = {}
+
+        # Load from YAML file if exists
+        yaml_path = config_path or Path("config/essential.yaml")
+        if yaml_path.exists():
+            try:
+                async with aiofiles.open(yaml_path, "r") as f:
+                    yaml_content = await f.read()
+                    yaml_data = yaml.safe_load(yaml_content) or {}
+                config_data = ConfigBootstrap._deep_merge(config_data, yaml_data)
+                logger.info(f"Loaded configuration from {yaml_path}")
+            except Exception as e:
+                logger.warning(f"Failed to load YAML config from {yaml_path}: {e}")
+
+        # Apply environment variable overrides
+        config_data = ConfigBootstrap._apply_env_overrides(config_data)
+
+        # Apply CLI overrides (highest priority)
+        if cli_overrides:
+            config_data = ConfigBootstrap._deep_merge(config_data, cli_overrides)
+
+        # Ensure database section exists with schema defaults if not present
+        if "database" not in config_data:
+            config_data["database"] = {}
+
+        # Resolve relative database paths to use proper data directory
+        config_data = ConfigBootstrap._resolve_database_paths(config_data)
+
+        # Create and validate config
+        try:
+            essential_config = EssentialConfig(**config_data)
+            logger.info("Essential configuration loaded and validated")
+            return essential_config
+        except Exception as e:
+            logger.error(f"Configuration validation failed: {e}")
+            raise ValueError(f"Invalid configuration: {e}") from e
+
+    @staticmethod
+    def get_config_metadata(config: EssentialConfig, yaml_path: Optional[Path] = None) -> JSONDict:
+        """
+        Generate metadata about config sources for migration to graph.
+
+        Returns dict mapping config keys to their source information
+        (source, env_var/file, bootstrap_phase flag).
+        """
+        metadata: JSONDict = {}
+
+        # Check which values came from environment
+        env_sources = {
+            "database.main_db": "CIRIS_DB_PATH",
+            "database.secrets_db": "CIRIS_SECRETS_DB_PATH",
+            "database.audit_db": "CIRIS_AUDIT_DB_PATH",
+            "services.llm_endpoint": "OPENAI_API_BASE",
+            "services.llm_model": "OPENAI_MODEL",
+            "security.audit_retention_days": "AUDIT_RETENTION_DAYS",
+            "limits.max_active_tasks": "MAX_ACTIVE_TASKS",
+            "security.max_thought_depth": "MAX_THOUGHT_DEPTH",
+            "log_level": "LOG_LEVEL",
+            "debug_mode": "DEBUG_MODE",
+        }
+
+        for key, env_var in env_sources.items():
+            if get_env_var(env_var):
+                metadata[key] = {"source": "env_var", "env_var": env_var, "bootstrap_phase": True}
+
+        # Mark file-sourced configs
+        if yaml_path and yaml_path.exists():
+            # Would need to track which specific values came from file
+            # For now, mark all non-env values as file-sourced
+            for key in ["database", "services", "security", "limits"]:
+                if key not in metadata:
+                    metadata[key] = {"source": "config_file", "file": str(yaml_path), "bootstrap_phase": True}
+
+        # Everything else is from defaults
+        return metadata

ciris_engine/logic/config/config_accessor.py

@@ -0,0 +1,202 @@
+"""
+Unified Configuration Access with Graph + Bootstrap Fallback.
+
+Provides a consistent interface for services to access configuration,
+whether from the graph or bootstrap phase.
+"""
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from ciris_engine.protocols.services.graph.config import GraphConfigServiceProtocol
+from ciris_engine.schemas.config.essential import EssentialConfig
+from ciris_engine.schemas.types import ConfigDict
+
+logger = logging.getLogger(__name__)
+
+
+class ConfigAccessor:
+    """
+    Unified config access with graph + fallback.
+
+    Services use this instead of direct config objects, enabling:
+    - Runtime config updates without restart
+    - Graceful fallback during bootstrap
+    - Type-safe access patterns
+    """
+
+    def __init__(self, graph_config: Optional[GraphConfigServiceProtocol], bootstrap_config: EssentialConfig):
+        """
+        Initialize config accessor.
+
+        Args:
+            graph_config: GraphConfigService (may be None during bootstrap)
+            bootstrap_config: Essential config loaded at startup
+        """
+        self.graph = graph_config
+        self.bootstrap = bootstrap_config
+        self._graph_available = graph_config is not None
+
+    def set_graph_service(self, graph_config: GraphConfigServiceProtocol) -> None:
+        """Set graph service after it becomes available."""
+        self.graph = graph_config
+        self._graph_available = True
+        logger.info("GraphConfigService now available for configuration")
+
+    async def get(self, key: str, default: Any = None) -> Any:
+        """
+        Get configuration value.
+
+        Args:
+            key: Dot-separated config key (e.g., "database.main_db")
+            default: Default value if not found
+
+        Returns:
+            Configuration value or default
+        """
+        # Try graph first if available
+        if self._graph_available and self.graph:
+            try:
+                node = await self.graph.get_config(key)
+                if node and node.value:
+                    # Handle ConfigValue wrapper
+                    if hasattr(node.value, "string_value"):
+                        # It's a ConfigValue object
+                        if node.value.string_value is not None:
+                            return node.value.string_value
+                        elif node.value.int_value is not None:
+                            return node.value.int_value
+                        elif node.value.float_value is not None:
+                            return node.value.float_value
+                        elif node.value.bool_value is not None:
+                            return node.value.bool_value
+                        elif node.value.list_value is not None:
+                            return node.value.list_value
+                        elif node.value.dict_value is not None:
+                            return node.value.dict_value
+                    else:
+                        # Direct value
+                        return node.value
+            except Exception as e:
+                logger.warning(f"Failed to get config '{key}' from graph: {e}")
+
+        # Fall back to bootstrap config
+        return self._get_from_bootstrap(key, default)
+
+    def _get_from_bootstrap(self, key: str, default: Any = None) -> Any:
+        """Get value from bootstrap config."""
+        parts = key.split(".")
+        value = self.bootstrap
+
+        try:
+            for part in parts:
+                if hasattr(value, part):
+                    value = getattr(value, part)
+                elif hasattr(value, "__getitem__"):
+                    try:
+                        value = value[part]
+                    except (KeyError, TypeError):
+                        return default
+                else:
+                    return default
+
+            # Convert Path objects to strings for compatibility
+            if isinstance(value, Path):
+                return str(value)
+
+            return value
+        except Exception as e:
+            logger.debug(f"Failed to get '{key}' from bootstrap: {e}")
+            return default
+
+    async def get_int(self, key: str, default: int = 0) -> int:
+        """Get integer configuration value."""
+        value = await self.get(key, default)
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            logger.warning(f"Config value '{key}' is not a valid integer: {value}")
+            return default
+
+    async def get_float(self, key: str, default: float = 0.0) -> float:
+        """Get float configuration value."""
+        value = await self.get(key, default)
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            logger.warning(f"Config value '{key}' is not a valid float: {value}")
+            return default
+
+    async def get_bool(self, key: str, default: bool = False) -> bool:
+        """Get boolean configuration value."""
+        value = await self.get(key, default)
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            return value.lower() in ("true", "1", "yes", "on")
+        return bool(value)
+
+    async def get_str(self, key: str, default: str = "") -> str:
+        """Get string configuration value."""
+        value = await self.get(key, default)
+        return str(value) if value is not None else default
+
+    async def get_path(self, key: str, default: Optional[Path] = None) -> Path:
+        """Get Path configuration value."""
+        value = await self.get(key, default)
+        if value is None:
+            return default or Path()
+        if isinstance(value, Path):
+            return value
+        return Path(str(value))
+
+    async def exists(self, key: str) -> bool:
+        """Check if configuration key exists."""
+        value = await self.get(key, sentinel := object())
+        return value is not sentinel
+
+    async def get_section(self, prefix: str) -> ConfigDict:
+        """
+        Get all config values under a prefix.
+
+        Args:
+            prefix: Key prefix (e.g., "database")
+
+        Returns:
+            Dict of config values under that prefix
+        """
+        if self._graph_available and self.graph:
+            try:
+                configs = await self.graph.list_configs(prefix=prefix)
+                return configs
+            except Exception as e:
+                logger.warning(f"Failed to get config section '{prefix}' from graph: {e}")
+
+        # Fall back to bootstrap
+        return self._get_section_from_bootstrap(prefix)
+
+    def _get_section_from_bootstrap(self, prefix: str) -> ConfigDict:
+        """Get config section from bootstrap config."""
+        parts = prefix.split(".")
+        value = self.bootstrap
+
+        try:
+            for part in parts:
+                if hasattr(value, part):
+                    value = getattr(value, part)
+                else:
+                    return {}
+
+            # Convert to dict if it's a Pydantic model
+            if hasattr(value, "model_dump"):
+                return value.model_dump()
+            elif hasattr(value, "dict"):
+                return value.dict()
+            elif hasattr(value, "__getitem__") and hasattr(value, "keys"):
+                return dict(value)
+            else:
+                return {}
+        except Exception as e:
+            logger.debug(f"Failed to get section '{prefix}' from bootstrap: {e}")
+            return {}

ciris_engine/logic/config/db_paths.py

@@ -0,0 +1,194 @@
+"""
+Database path utilities for the new config system.
+
+Provides compatibility functions for getting database paths.
+"""
+
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlparse, urlunparse
+
+from ciris_engine.schemas.config.essential import EssentialConfig
+
+
+def _get_config_from_registry() -> Optional[EssentialConfig]:
+    """Get EssentialConfig from the service registry.
+
+    Returns:
+        EssentialConfig instance or None if not available
+    """
+    try:
+        from ciris_engine.logic.registries.base import ServiceRegistry
+        from ciris_engine.schemas.runtime.enums import ServiceType
+
+        registry = ServiceRegistry()
+        config_services = registry.get_services_by_type(ServiceType.CONFIG)
+
+        if config_services:
+            config_service = config_services[0]
+            if hasattr(config_service, "essential_config"):
+                config: Optional[EssentialConfig] = getattr(config_service, "essential_config", None)
+                return config
+            elif hasattr(config_service, "_config"):
+                config = getattr(config_service, "_config", None)
+                return config
+
+        return None
+    except (ImportError, AttributeError):
+        return None
+
+
+def _modify_database_name_in_url(base_url: str, suffix: str) -> str:
+    """Modify the database name in a PostgreSQL URL while preserving query parameters.
+
+    Args:
+        base_url: Original PostgreSQL URL (e.g., postgresql://user:pass@host:port/dbname?sslmode=require)
+        suffix: Suffix to append to database name (e.g., "_secrets", "_auth")
+
+    Returns:
+        Modified URL with suffix added to database name, query params preserved
+
+    Example:
+        >>> _modify_database_name_in_url("postgresql://user:pass@host:5432/db?sslmode=require", "_secrets")
+        "postgresql://user:pass@host:5432/db_secrets?sslmode=require"
+    """
+    parsed = urlparse(base_url)
+
+    # Extract database name from path (remove leading /)
+    db_name = parsed.path.lstrip("/")
+
+    # Add suffix to database name
+    new_db_name = f"{db_name}{suffix}"
+
+    # Reconstruct path with leading /
+    new_path = f"/{new_db_name}"
+
+    # Reconstruct URL with new path, preserving all other components
+    return urlunparse(
+        (
+            parsed.scheme,  # scheme (postgresql)
+            parsed.netloc,  # netloc (user:pass@host:port)
+            new_path,  # path (/db_secrets)
+            parsed.params,  # params (empty for PostgreSQL URLs)
+            parsed.query,  # query (sslmode=require)
+            parsed.fragment,  # fragment (empty for PostgreSQL URLs)
+        )
+    )
+
+
+def get_sqlite_db_full_path(config: Optional[EssentialConfig] = None) -> str:
+    """
+    Get the full path to the main SQLite database.
+
+    Args:
+        config: Optional EssentialConfig instance. If not provided, will attempt
+                to get from the config service via ServiceRegistry.
+
+    Returns:
+        Full path to the SQLite database file
+
+    Raises:
+        RuntimeError: If no config is available from any source
+    """
+    if config is None:
+        # Try to get config from the service registry
+        try:
+            from ciris_engine.logic.registries.base import get_global_registry
+            from ciris_engine.schemas.runtime.enums import ServiceType
+
+            # Use the global registry singleton
+            registry = get_global_registry()
+            config_services = registry.get_services_by_type(ServiceType.CONFIG)
+
+            if config_services:
+                config_service = config_services[0]
+                if hasattr(config_service, "essential_config"):
+                    config = config_service.essential_config
+                elif hasattr(config_service, "_config"):
+                    config = config_service._config
+
+            if config is None:
+                # FAIL FAST AND LOUD - no fallback to defaults
+                raise RuntimeError(
+                    "No configuration available from config service. "
+                    "The system must be properly initialized with configuration."
+                )
+        except (ImportError, AttributeError) as e:
+            # FAIL FAST AND LOUD - no fallback to defaults
+            raise RuntimeError(
+                f"Cannot access config service: {e}. " "The system must be properly initialized with configuration."
+            )
+
+    # Prefer database_url if set (supports PostgreSQL), otherwise use main_db path
+    if config.database.database_url:
+        return config.database.database_url
+
+    db_path = Path(config.database.main_db)
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    return str(db_path.resolve())
+
+
+def get_secrets_db_full_path(config: Optional[EssentialConfig] = None) -> str:
+    """
+    Get the full path to the secrets database.
+
+    For PostgreSQL: Modifies the database name in the URL (e.g., ciris_db -> ciris_db_secrets)
+    For SQLite: Returns the configured secrets_db path
+
+    Args:
+        config: Optional EssentialConfig instance. If not provided, will attempt
+                to get from the config service via ServiceRegistry.
+
+    Returns:
+        Full path to the secrets database file or modified PostgreSQL URL
+    """
+    if config is None:
+        config = _get_config_from_registry()
+        if config is None:
+            # For secrets db, we can fall back to defaults as it's less critical
+            config = EssentialConfig()
+
+    # If using PostgreSQL (database_url is set), modify the database name
+    if config.database.database_url:
+        return _modify_database_name_in_url(config.database.database_url, "_secrets")
+
+    # SQLite: use configured path
+    db_path = Path(config.database.secrets_db)
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    return str(db_path.resolve())
+
+
+def get_audit_db_full_path(config: Optional[EssentialConfig] = None) -> str:
+    """
+    Get the full path to the audit database.
+
+    For PostgreSQL: Modifies the database name in the URL (e.g., ciris_db -> ciris_db_auth)
+    For SQLite: Returns the configured audit_db path
+
+    Args:
+        config: Optional EssentialConfig instance. If not provided, will attempt
+                to get from the config service via ServiceRegistry.
+
+    Returns:
+        Full path to the audit database file or modified PostgreSQL URL
+    """
+    if config is None:
+        config = _get_config_from_registry()
+        if config is None:
+            # For audit db, we can fall back to defaults as it's less critical
+            config = EssentialConfig()
+
+    # If using PostgreSQL (database_url is set), modify the database name
+    if config.database.database_url:
+        return _modify_database_name_in_url(config.database.database_url, "_auth")
+
+    # SQLite: use configured path
+    db_path = Path(config.database.audit_db)
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    return str(db_path.resolve())
+
+
+# For backward compatibility - uses defaults
+def get_graph_memory_full_path() -> str:
+    """Legacy function - graph memory is now in the main database."""
+    return get_sqlite_db_full_path()