ciris-agent 1.7.7__py3-none-any.whl

ciris_adapters/external_data_sql/adapter.py

@@ -0,0 +1,144 @@
+"""
+External Data SQL Adapter for CIRIS.
+
+Provides the BaseAdapterProtocol-compliant wrapper around SQLToolService
+so it can be loaded dynamically via RuntimeAdapterManager.load_adapter().
+"""
+
+import asyncio
+import logging
+from typing import Any, List, Optional
+
+from ciris_engine.logic.adapters.base import Service
+from ciris_engine.logic.registries.base import Priority
+from ciris_engine.schemas.adapters import AdapterServiceRegistration
+from ciris_engine.schemas.runtime.adapter_management import AdapterConfig, RuntimeAdapterStatus
+from ciris_engine.schemas.runtime.enums import ServiceType
+
+from .service import SQLToolService
+
+logger = logging.getLogger(__name__)
+
+
+class ExternalDataSQLAdapter(Service):
+    """
+    External Data SQL adapter platform for CIRIS.
+
+    Wraps SQLToolService to provide the BaseAdapterProtocol interface
+    required for dynamic adapter loading via RuntimeAdapterManager.
+    """
+
+    def __init__(self, runtime: Any, context: Optional[Any] = None, **kwargs: Any) -> None:
+        """Initialize External Data SQL adapter."""
+        super().__init__(config=kwargs.get("adapter_config"))
+        self.runtime = runtime
+        self.context = context
+
+        # Create the underlying SQL tool service
+        # It reads config from environment variables set by SQLConfigurableAdapter.apply_config()
+        self.sql_service = SQLToolService()
+
+        # Track adapter state
+        self._running = False
+        self._lifecycle_task: Optional[asyncio.Task[None]] = None
+
+        logger.info("External Data SQL adapter initialized")
+
+    def get_services_to_register(self) -> List[AdapterServiceRegistration]:
+        """Get services provided by this adapter."""
+        registrations = []
+
+        # Register the SQL service as a tool service with its capabilities
+        registrations.append(
+            AdapterServiceRegistration(
+                service_type=ServiceType.TOOL,
+                provider=self.sql_service,
+                priority=Priority.NORMAL,
+                capabilities=[
+                    "tool:sql",
+                    "tool:sql:find_user_data",
+                    "tool:sql:export_user",
+                    "tool:sql:delete_user",
+                    "tool:sql:anonymize_user",
+                    "tool:sql:verify_deletion",
+                    "tool:sql:get_stats",
+                    "tool:sql:query",
+                    "provider:sql",
+                    "dsar:sql",
+                ],
+            )
+        )
+
+        return registrations
+
+    async def start(self) -> None:
+        """Start the External Data SQL adapter."""
+        logger.info("Starting External Data SQL adapter")
+        await self.sql_service.initialize()
+        self._running = True
+        logger.info("External Data SQL adapter started")
+
+    async def stop(self) -> None:
+        """Stop the External Data SQL adapter."""
+        logger.info("Stopping External Data SQL adapter")
+        self._running = False
+
+        if self._lifecycle_task and not self._lifecycle_task.done():
+            self._lifecycle_task.cancel()
+            try:
+                await self._lifecycle_task
+            except asyncio.CancelledError:
+                pass
+
+        # Stop the SQL service if it has a stop method
+        if hasattr(self.sql_service, "stop"):
+            await self.sql_service.stop()
+        elif hasattr(self.sql_service, "shutdown"):
+            await self.sql_service.shutdown()
+
+        logger.info("External Data SQL adapter stopped")
+
+    async def run_lifecycle(self, agent_task: Any) -> None:
+        """Run the adapter lifecycle.
+
+        For SQL data access, we just wait for the agent task to complete
+        since SQL operations are request-driven and don't need continuous polling.
+        """
+        logger.info("External Data SQL adapter lifecycle started")
+        try:
+            # Wait for the agent task to signal shutdown
+            await agent_task
+        except asyncio.CancelledError:
+            logger.info("External Data SQL adapter lifecycle cancelled")
+        finally:
+            await self.stop()
+
+    def get_config(self) -> AdapterConfig:
+        """Get adapter configuration."""
+        config_obj = self.sql_service._config
+        connector_id = config_obj.connector_id if config_obj else "sql"
+        dialect = config_obj.dialect.value if config_obj and config_obj.dialect else "unknown"
+
+        return AdapterConfig(
+            adapter_type="external_data_sql",
+            enabled=self._running,
+            settings={
+                "connector_id": connector_id,
+                "dialect": dialect,
+                "has_privacy_schema": bool(config_obj and config_obj.privacy_schema),
+            },
+        )
+
+    def get_status(self) -> RuntimeAdapterStatus:
+        """Get adapter status."""
+        return RuntimeAdapterStatus(
+            adapter_id="external_data_sql",
+            adapter_type="external_data_sql",
+            is_running=self._running,
+            loaded_at=None,
+            error=None,
+        )
+
+
+# Export as Adapter for load_adapter() compatibility
+Adapter = ExternalDataSQLAdapter

ciris_adapters/external_data_sql/configurable.py

@@ -0,0 +1,315 @@
+"""
+External Data SQL ConfigurableAdapterProtocol implementation.
+
+Provides interactive configuration workflow for SQL database connections:
+1. Database Type - Select dialect (PostgreSQL, MySQL, SQLite, MSSQL)
+2. Connection - Enter connection details (host, port, database, credentials)
+3. Privacy Schema - Optional privacy schema for GDPR/DSAR compliance
+4. Features - Select enabled features (read-only mode, etc.)
+5. Confirm - Review and apply configuration
+"""
+
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from .schemas import SQLConnectorConfig, SQLDialect
+
+logger = logging.getLogger(__name__)
+
+
+class SQLConfigurableAdapter:
+    """SQL database configurable adapter.
+
+    Implements ConfigurableAdapterProtocol for SQL database connections
+    with support for multiple dialects and GDPR/DSAR compliance features.
+    """
+
+    # Supported SQL dialects
+    SUPPORTED_DIALECTS = {
+        "postgres": {
+            "label": "PostgreSQL",
+            "description": "PostgreSQL database",
+            "default_port": 5432,
+            "connection_template": "postgresql://{user}:{password}@{host}:{port}/{database}",
+        },
+        "mysql": {
+            "label": "MySQL/MariaDB",
+            "description": "MySQL or MariaDB database",
+            "default_port": 3306,
+            "connection_template": "mysql+pymysql://{user}:{password}@{host}:{port}/{database}",
+        },
+        "sqlite": {
+            "label": "SQLite",
+            "description": "SQLite database file",
+            "default_port": None,
+            "connection_template": "sqlite:///{path}",
+        },
+        "mssql": {
+            "label": "Microsoft SQL Server",
+            "description": "Microsoft SQL Server database",
+            "default_port": 1433,
+            "connection_template": "mssql+pyodbc://{user}:{password}@{host}:{port}/{database}?driver=ODBC+Driver+17+for+SQL+Server",
+        },
+    }
+
+    # Available features
+    AVAILABLE_FEATURES = {
+        "read_only": {
+            "label": "Read-Only Mode",
+            "description": "Restrict to SELECT queries only (no modifications)",
+            "default": True,
+        },
+        "dsar_operations": {
+            "label": "DSAR Operations",
+            "description": "Enable GDPR/DSAR operations (find, export, delete, anonymize)",
+            "default": True,
+        },
+        "query_validation": {
+            "label": "Query Validation",
+            "description": "Validate queries against privacy schema",
+            "default": True,
+        },
+    }
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
+        """Initialize the SQL configurable adapter.
+
+        Args:
+            config: Optional existing configuration
+        """
+        self.config = config or {}
+        self._applied_config: Optional[Dict[str, Any]] = None
+
+        logger.info("SQLConfigurableAdapter initialized")
+
+    async def get_config_options(self, step_id: str, context: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Get options for a selection step.
+
+        Args:
+            step_id: ID of the configuration step
+            context: Current configuration context
+
+        Returns:
+            List of available options
+        """
+        logger.info(f"Getting config options for step: {step_id}")
+
+        if step_id == "select_dialect":
+            # Return supported SQL dialects
+            return [
+                {
+                    "id": dialect_id,
+                    "label": dialect["label"],
+                    "description": dialect["description"],
+                    "metadata": {
+                        "default_port": dialect["default_port"],
+                        "connection_template": dialect["connection_template"],
+                    },
+                }
+                for dialect_id, dialect in self.SUPPORTED_DIALECTS.items()
+            ]
+
+        elif step_id == "select_features":
+            # Return available features
+            return [
+                {
+                    "id": feature_id,
+                    "label": feature["label"],
+                    "description": feature["description"],
+                    "metadata": {"default": feature["default"]},
+                }
+                for feature_id, feature in self.AVAILABLE_FEATURES.items()
+            ]
+
+        return []
+
+    async def validate_config(self, config: Dict[str, Any]) -> Tuple[bool, Optional[str]]:
+        """Validate SQL configuration before applying.
+
+        Performs:
+        - Required field validation
+        - Connection string format validation
+        - Dialect validation
+        - Optional connectivity test
+
+        Args:
+            config: Complete configuration to validate
+
+        Returns:
+            (is_valid, error_message) tuple
+        """
+        logger.info("Validating SQL configuration")
+
+        if not config:
+            return False, "Configuration is empty"
+
+        # Check required fields
+        dialect = config.get("dialect")
+        if not dialect:
+            return False, "dialect is required"
+
+        if dialect not in self.SUPPORTED_DIALECTS:
+            return False, f"Invalid dialect: {dialect}. Must be one of: {list(self.SUPPORTED_DIALECTS.keys())}"
+
+        # Validate connection parameters based on dialect
+        if dialect == "sqlite":
+            # SQLite needs path
+            db_path = config.get("database_path")
+            if not db_path:
+                return False, "database_path is required for SQLite"
+        else:
+            # Other databases need host, database, credentials
+            host = config.get("host")
+            database = config.get("database")
+            user = config.get("user")
+            password = config.get("password")
+
+            if not host:
+                return False, "host is required"
+            if not database:
+                return False, "database is required"
+            if not user:
+                return False, "user is required"
+            if not password:
+                return False, "password is required"
+
+        # Optional: Test connection if requested
+        test_connection = config.get("test_connection", False)
+        if test_connection:
+            connection_string = self._build_connection_string(config)
+            if not connection_string:
+                return False, "Failed to build connection string"
+
+            try:
+                from sqlalchemy import create_engine, text
+
+                engine = create_engine(connection_string, connect_args={"timeout": 10})
+                with engine.connect() as conn:
+                    conn.execute(text("SELECT 1"))
+                engine.dispose()
+                logger.info("SQL connection test successful")
+            except Exception as e:
+                return False, f"Connection test failed: {e}"
+
+        logger.info("SQL configuration validated successfully")
+        return True, None
+
+    def _build_connection_string(self, config: Dict[str, Any]) -> Optional[str]:
+        """Build SQLAlchemy connection string from config.
+
+        Args:
+            config: Configuration dictionary
+
+        Returns:
+            Connection string or None if invalid
+        """
+        dialect = config.get("dialect")
+        if not dialect or dialect not in self.SUPPORTED_DIALECTS:
+            return None
+
+        template = self.SUPPORTED_DIALECTS[dialect]["connection_template"]
+
+        try:
+            if dialect == "sqlite":
+                # SQLite uses path
+                db_path = config.get("database_path", "")
+                return template.format(path=db_path)
+            else:
+                # Other databases use host/port/user/password/database
+                host = config.get("host", "localhost")
+                port = config.get("port", self.SUPPORTED_DIALECTS[dialect]["default_port"])
+                user = config.get("user", "")
+                password = config.get("password", "")
+                database = config.get("database", "")
+
+                return template.format(
+                    host=host,
+                    port=port,
+                    user=user,
+                    password=password,
+                    database=database,
+                )
+        except KeyError as e:
+            logger.error(f"Missing template parameter: {e}")
+            return None
+
+    async def apply_config(self, config: Dict[str, Any]) -> bool:
+        """Apply the configuration.
+
+        Stores configuration and sets up environment for the service.
+
+        Args:
+            config: Validated configuration to apply
+
+        Returns:
+            True if applied successfully
+        """
+        logger.info("Applying SQL configuration")
+
+        self._applied_config = config.copy()
+
+        # Build connection string
+        connection_string = self._build_connection_string(config)
+        if not connection_string:
+            logger.error("Failed to build connection string")
+            return False
+
+        # Create SQL connector config
+        dialect = config.get("dialect", "sqlite")
+        connector_id = config.get("connector_id", "sql")
+        privacy_schema_path = config.get("privacy_schema_path")
+
+        try:
+            sql_dialect = SQLDialect(dialect)
+        except ValueError:
+            logger.error(f"Invalid dialect: {dialect}")
+            return False
+
+        # Build config dict for JSON serialization
+        config_dict = {
+            "connector_id": connector_id,
+            "connection_string": connection_string,
+            "dialect": sql_dialect.value,
+            "privacy_schema_path": privacy_schema_path,
+            "connection_timeout": config.get("connection_timeout", 30),
+            "query_timeout": config.get("query_timeout", 60),
+            "max_retries": config.get("max_retries", 3),
+        }
+
+        # Save config to file
+        config_path = os.environ.get("CIRIS_DATA_DIR", "data")
+        config_file = Path(config_path) / "sql_config.json"
+        config_file.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            with open(config_file, "w") as f:
+                json.dump(config_dict, f, indent=2)
+            logger.info(f"SQL configuration saved to {config_file}")
+        except Exception as e:
+            logger.error(f"Failed to save config file: {e}")
+            return False
+
+        # Set environment variable for the SQL service
+        os.environ["CIRIS_SQL_EXTERNAL_DATA_CONFIG"] = str(config_file)
+
+        # Set feature flags
+        enabled_features = config.get("enabled_features", [])
+        if enabled_features:
+            os.environ["CIRIS_SQL_ENABLED_FEATURES"] = ",".join(enabled_features)
+
+        # Log sanitized config
+        safe_config = {k: ("***" if "password" in k.lower() or "token" in k.lower() else v) for k, v in config.items()}
+        logger.info(f"SQL configuration applied: {safe_config}")
+
+        return True
+
+    def get_applied_config(self) -> Optional[Dict[str, Any]]:
+        """Get the currently applied configuration.
+
+        Returns:
+            Applied configuration or None if not configured
+        """
+        return self._applied_config

ciris_adapters/external_data_sql/dialects/__init__.py

@@ -0,0 +1,37 @@
+"""SQL dialect implementations."""
+
+from typing import Union
+
+from .base import BaseSQLDialect
+from .mysql import MySQLDialect
+from .postgresql import PostgreSQLDialect
+from .sqlite import SQLiteDialect
+
+__all__ = ["BaseSQLDialect", "SQLiteDialect", "PostgreSQLDialect", "MySQLDialect", "get_dialect"]
+
+
+DialectType = Union[SQLiteDialect, PostgreSQLDialect, MySQLDialect]
+
+
+def get_dialect(dialect_name: str) -> DialectType:
+    """Get dialect instance by name.
+
+    Args:
+        dialect_name: Dialect name (sqlite, postgresql, mysql)
+
+    Returns:
+        Dialect instance
+
+    Raises:
+        ValueError: If dialect not supported
+    """
+    normalized_name = dialect_name.lower()
+
+    if normalized_name == "sqlite":
+        return SQLiteDialect()
+    elif normalized_name == "postgresql":
+        return PostgreSQLDialect()
+    elif normalized_name == "mysql":
+        return MySQLDialect()
+    else:
+        raise ValueError(f"Unsupported dialect: {dialect_name}. Supported: sqlite, postgresql, mysql")

ciris_adapters/external_data_sql/dialects/base.py

@@ -0,0 +1,133 @@
+"""Base SQL dialect interface."""
+
+from abc import ABC, abstractmethod
+from typing import List
+
+from ..schemas import PrivacyColumnMapping
+
+
+class BaseSQLDialect(ABC):
+    """Base class for SQL dialect-specific operations."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Dialect name."""
+        ...
+
+    @property
+    @abstractmethod
+    def supports_md5(self) -> bool:
+        """Whether dialect supports MD5 function."""
+        ...
+
+    @property
+    @abstractmethod
+    def supports_sha256(self) -> bool:
+        """Whether dialect supports SHA256 function."""
+        ...
+
+    @abstractmethod
+    def build_count_query(self, table_name: str, identifier_col: str) -> str:
+        """Build a COUNT query for finding user data.
+
+        Args:
+            table_name: Table to query
+            identifier_col: Column to match on
+
+        Returns:
+            SQL query string with :user_id parameter
+        """
+        ...
+
+    @abstractmethod
+    def build_select_query(self, table_name: str, identifier_col: str) -> str:
+        """Build a SELECT query for exporting user data.
+
+        Args:
+            table_name: Table to query
+            identifier_col: Column to match on
+
+        Returns:
+            SQL query string with :user_id parameter
+        """
+        ...
+
+    @abstractmethod
+    def build_delete_query(self, table_name: str, identifier_col: str) -> str:
+        """Build a DELETE query for removing user data.
+
+        Args:
+            table_name: Table to delete from
+            identifier_col: Column to match on
+
+        Returns:
+            SQL query string with :user_id parameter
+        """
+        ...
+
+    @abstractmethod
+    def build_anonymization_clause(self, column_mapping: PrivacyColumnMapping, user_identifier: str) -> str:
+        """Build anonymization clause for UPDATE statement.
+
+        Args:
+            column_mapping: Column with anonymization strategy
+            user_identifier: User identifier for deterministic pseudonyms
+
+        Returns:
+            SQL fragment like "column_name = 'ANONYMIZED'" or "column_name = md5(column_name)"
+        """
+        ...
+
+    @abstractmethod
+    def build_substring_function(self, column_name: str, start: int, length: int) -> str:
+        """Build substring function (dialect-specific).
+
+        Args:
+            column_name: Column to substring
+            start: Start position (1-indexed)
+            length: Length of substring
+
+        Returns:
+            SQL fragment like "SUBSTR(col, 1, 3)" or "SUBSTRING(col, 1, 3)"
+        """
+        ...
+
+    def build_update_query(
+        self,
+        table_name: str,
+        identifier_col: str,
+        column_mappings: List[PrivacyColumnMapping],
+        user_identifier: str,
+    ) -> str:
+        """Build UPDATE query for anonymization.
+
+        Args:
+            table_name: Table to update
+            identifier_col: Column to match on
+            column_mappings: Columns to anonymize
+            user_identifier: User identifier
+
+        Returns:
+            Complete UPDATE SQL statement
+        """
+        updates = []
+        for col_mapping in column_mappings:
+            if col_mapping.anonymization_strategy == "delete":
+                continue  # Skip deletion columns in anonymization
+            clause = self.build_anonymization_clause(col_mapping, user_identifier)
+            updates.append(clause)
+
+        if not updates:
+            return ""
+
+        update_clause = ", ".join(updates)
+        return f"UPDATE {table_name} SET {update_clause} WHERE {identifier_col} = :user_id"
+
+    def get_database_size_query(self) -> str:
+        """Get database size query (dialect-specific, may not be supported).
+
+        Returns:
+            SQL query to get database size in MB, or empty string if not supported
+        """
+        return ""  # Default: not supported

ciris_adapters/external_data_sql/dialects/mysql.py

@@ -0,0 +1,63 @@
+"""MySQL dialect implementation."""
+
+import hashlib
+
+from ..schemas import PrivacyColumnMapping
+from .base import BaseSQLDialect
+
+
+class MySQLDialect(BaseSQLDialect):
+    """MySQL-specific SQL dialect."""
+
+    @property
+    def name(self) -> str:
+        return "mysql"
+
+    @property
+    def supports_md5(self) -> bool:
+        return True  # MySQL has MD5
+
+    @property
+    def supports_sha256(self) -> bool:
+        return True  # MySQL has SHA2
+
+    def build_count_query(self, table_name: str, identifier_col: str) -> str:
+        return f"SELECT COUNT(*) as cnt FROM {table_name} WHERE {identifier_col} = :user_id"
+
+    def build_select_query(self, table_name: str, identifier_col: str) -> str:
+        return f"SELECT * FROM {table_name} WHERE {identifier_col} = :user_id"
+
+    def build_delete_query(self, table_name: str, identifier_col: str) -> str:
+        return f"DELETE FROM {table_name} WHERE {identifier_col} = :user_id"
+
+    def build_anonymization_clause(self, column_mapping: PrivacyColumnMapping, user_identifier: str) -> str:
+        """Build anonymization clause for MySQL (has MD5, SHA2)."""
+        col_name = column_mapping.column_name
+        strategy = column_mapping.anonymization_strategy
+
+        if strategy == "null":
+            return f"{col_name} = NULL"
+        elif strategy == "hash":
+            # MySQL has built-in MD5
+            return f"{col_name} = MD5({col_name})"
+        elif strategy == "pseudonymize":
+            # Generate deterministic pseudonym (done at Python level for consistency)
+            pseudonym = hashlib.sha256(f"{user_identifier}_{col_name}".encode()).hexdigest()[:16]
+            return f"{col_name} = '{pseudonym}'"
+        elif strategy == "truncate":
+            # MySQL uses SUBSTRING
+            return f"{col_name} = {self.build_substring_function(col_name, 1, 3)} || '***'"
+        else:
+            return f"{col_name} = 'ANONYMIZED'"
+
+    def build_substring_function(self, column_name: str, start: int, length: int) -> str:
+        """MySQL uses SUBSTRING function."""
+        return f"SUBSTRING({column_name}, {start}, {length})"
+
+    def get_database_size_query(self) -> str:
+        """MySQL: Get database size from information_schema."""
+        return """
+        SELECT SUM(data_length + index_length) / 1024.0 / 1024.0 as size_mb
+        FROM information_schema.tables
+        WHERE table_schema = DATABASE()
+        """