ciris-agent 1.7.7__py3-none-any.whl

ciris_engine/logic/secrets/__init__.py

@@ -0,0 +1,30 @@
+"""
+CIRIS Agent Secrets Management System
+
+Provides secure detection, storage, and access control for sensitive information.
+All secrets are encrypted at rest and access is audited.
+
+Key components:
+- SecretsFilter: Detects and filters secrets from content
+- SecretsStore: Encrypted storage and retrieval of secrets
+- SecretsService: Main service coordinating secrets management
+- SecretTools: Agent tools for managing secrets
+"""
+
+from ciris_engine.schemas.secrets.core import DetectedSecret, SecretAccessLog, SecretPattern, SecretRecord
+
+from .encryption import SecretsEncryption
+from .filter import SecretsFilter
+from .service import SecretsService
+from .store import SecretsStore
+
+__all__ = [
+    "SecretsFilter",
+    "SecretPattern",
+    "DetectedSecret",
+    "SecretsStore",
+    "SecretsEncryption",
+    "SecretRecord",
+    "SecretAccessLog",
+    "SecretsService",
+]

ciris_engine/logic/secrets/encryption.py

@@ -0,0 +1,175 @@
+"""
+Cryptographic functions for CIRIS secrets management.
+
+Provides AES-256-GCM encryption with per-secret keys derived from a master key.
+Implements secure key derivation, rotation, and forward secrecy.
+"""
+
+import logging
+import secrets
+from typing import Optional, Tuple
+
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+
+logger = logging.getLogger(__name__)
+
+
+class SecretsEncryption:
+    """Handles encryption/decryption of secrets using AES-256-GCM"""
+
+    def __init__(self, master_key: Optional[bytes] = None) -> None:
+        """
+        Initialize with a master key. If not provided, generates a new one.
+
+        Args:
+            master_key: 32-byte master key for deriving per-secret keys
+        """
+        if master_key is None:
+            self.master_key = self._generate_master_key()
+            logger.warning("Generated new master key - ensure this is persisted securely")
+        else:
+            if len(master_key) != 32:
+                raise ValueError("Master key must be exactly 32 bytes")
+            self.master_key = master_key
+
+    def _generate_master_key(self) -> bytes:
+        """Generate a new 256-bit master key"""
+        return secrets.token_bytes(32)
+
+    def _derive_key(self, salt: bytes) -> bytes:
+        """
+        Derive a per-secret key from master key + salt using PBKDF2
+
+        Args:
+            salt: 16-byte cryptographic salt
+
+        Returns:
+            32-byte derived key
+        """
+        kdf = PBKDF2HMAC(
+            algorithm=hashes.SHA256(),
+            length=32,
+            salt=salt,
+            iterations=100000,
+        )
+        return kdf.derive(self.master_key)
+
+    def encrypt_secret(self, value: str) -> Tuple[bytes, bytes, bytes]:
+        """
+        Encrypt a secret value using AES-256-GCM
+
+        Args:
+            value: The secret string to encrypt
+
+        Returns:
+            Tuple of (encrypted_value, salt, nonce)
+        """
+        salt = secrets.token_bytes(16)
+        nonce = secrets.token_bytes(12)
+
+        key = self._derive_key(salt)
+
+        aesgcm = AESGCM(key)
+        encrypted_value = aesgcm.encrypt(nonce, value.encode("utf-8"), None)
+
+        logger.debug(f"Encrypted secret of length {len(value)} characters")
+        return encrypted_value, salt, nonce
+
+    def decrypt_secret(self, encrypted_value: bytes, salt: bytes, nonce: bytes) -> str:
+        """
+        Decrypt a secret value using AES-256-GCM
+
+        Args:
+            encrypted_value: The encrypted secret data
+            salt: The salt used for key derivation
+            nonce: The nonce used for encryption
+
+        Returns:
+            The decrypted secret string
+
+        Raises:
+            InvalidSignature: If decryption fails (wrong key, corrupted data, etc.)
+        """
+        key = self._derive_key(salt)
+
+        aesgcm = AESGCM(key)
+        decrypted_bytes = aesgcm.decrypt(nonce, encrypted_value, None)
+
+        logger.debug("Successfully decrypted secret")
+        return decrypted_bytes.decode("utf-8")
+
+    def rotate_master_key(self, new_master_key: Optional[bytes] = None) -> bytes:
+        """
+        Rotate the master key. This should be used with SecretsStore.reencrypt_all()
+
+        Args:
+            new_master_key: New master key, or None to generate one
+
+        Returns:
+            The new master key that was set
+        """
+        _old_key = self.master_key
+
+        if new_master_key is None:
+            self.master_key = self._generate_master_key()
+        else:
+            if len(new_master_key) != 32:
+                raise ValueError("New master key must be exactly 32 bytes")
+            self.master_key = new_master_key
+
+        logger.info("Master key rotated successfully")
+        return self.master_key
+
+    def get_master_key(self) -> bytes:
+        """Get the current master key (for backup/persistence)"""
+        return self.master_key
+
+    @staticmethod
+    def generate_key_from_password(password: str, salt: Optional[bytes] = None) -> Tuple[bytes, bytes]:
+        """
+        Generate a master key from a password (for human-memorable keys)
+
+        Args:
+            password: The password to derive key from
+            salt: Optional salt, generates new one if not provided
+
+        Returns:
+            Tuple of (key, salt)
+        """
+        if salt is None:
+            salt = secrets.token_bytes(16)
+
+        kdf = PBKDF2HMAC(
+            algorithm=hashes.SHA256(),
+            length=32,
+            salt=salt,
+            iterations=100000,
+        )
+
+        key = kdf.derive(password.encode("utf-8"))
+        return key, salt
+
+    def test_encryption(self) -> bool:
+        """
+        Test that encryption/decryption is working correctly
+
+        Returns:
+            True if test passes, False otherwise
+        """
+        try:
+            test_secret = "test_secret_value_123"
+            encrypted, salt, nonce = self.encrypt_secret(test_secret)
+            decrypted = self.decrypt_secret(encrypted, salt, nonce)
+
+            if decrypted == test_secret:
+                logger.debug("Encryption test passed")
+                return True
+            else:
+                logger.error("Encryption test failed: decrypted value doesn't match")
+                return False
+
+        except Exception as e:
+            logger.error(f"Encryption test failed with exception: {e}")
+            return False

ciris_engine/logic/secrets/filter.py

@@ -0,0 +1,295 @@
+"""
+Secrets Detection and Filtering System for CIRIS Agent.
+
+Automatically detects and protects sensitive information while maintaining
+the agent's ability to reason about and use secrets safely.
+"""
+
+import logging
+import re
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+
+from ciris_engine.schemas.secrets.core import ConfigExport, DetectedSecret, PatternStats
+from ciris_engine.schemas.secrets.core import SecretPattern as ConfigSecretPattern
+from ciris_engine.schemas.secrets.core import SecretsDetectionConfig, SecretsFilterResult
+from ciris_engine.schemas.types import JSONDict
+
+logger = logging.getLogger(__name__)
+
+
+class SecretsFilter:
+    """
+    Automatic secrets detection and filtering system.
+
+    Detects secrets in text and replaces them with secure UUID references
+    while maintaining context for the agent.
+    """
+
+    def __init__(self, detection_config: Optional[SecretsDetectionConfig] = None) -> None:
+        self.detection_config = detection_config or SecretsDetectionConfig()
+        self._compiled_patterns: Dict[str, re.Pattern[str]] = {}
+        self._compile_patterns()
+
+    def _compile_patterns(self) -> None:
+        """Compile all active patterns for efficient matching."""
+        self._compiled_patterns.clear()
+
+        # Add all patterns from config
+        if self.detection_config.enabled:
+            for pattern in self.detection_config.patterns:
+                try:
+                    self._compiled_patterns[pattern.name] = re.compile(pattern.pattern)
+                except re.error as e:
+                    logger.error(f"Failed to compile pattern {pattern.name}: {e}")
+
+    def detect_secrets(self, text: str, context_hint: str = "") -> List[DetectedSecret]:
+        """
+        Detect secrets in the given text.
+
+        Args:
+            text: Text to scan for secrets
+            context_hint: Safe context description for logging
+
+        Returns:
+            List of detected secrets with metadata
+        """
+        detected_secrets = []
+
+        for pattern_name, compiled_pattern in self._compiled_patterns.items():
+            try:
+                matches = compiled_pattern.finditer(text)
+                for match in matches:
+                    # Get pattern metadata
+                    pattern_info = self._get_pattern_info(pattern_name)
+                    if not pattern_info:
+                        continue
+
+                    # Generate UUID for this secret
+                    secret_uuid = str(uuid.uuid4())
+
+                    # Create replacement text
+                    replacement_text = f"{{SECRET:{secret_uuid}:{pattern_info.description}}}"
+
+                    # Create detected secret record
+                    detected_secret = DetectedSecret(
+                        secret_uuid=secret_uuid,
+                        original_value=match.group(0),
+                        replacement_text=replacement_text,
+                        pattern_name=pattern_name,
+                        description=pattern_info.description,
+                        sensitivity=pattern_info.sensitivity,  # Use pattern sensitivity directly
+                        context_hint=context_hint,
+                    )
+
+                    detected_secrets.append(detected_secret)
+
+            except Exception as e:
+                logger.error(f"Error processing pattern {pattern_name}: {e}")
+
+        return detected_secrets
+
+    def filter_text(self, text: str, context_hint: str = "") -> Tuple[str, List[DetectedSecret]]:
+        """
+        Filter text by detecting and replacing secrets with UUID references.
+
+        Args:
+            text: Original text containing potential secrets
+            context_hint: Safe context description for logging
+
+        Returns:
+            Tuple of (filtered_text, detected_secrets_list)
+        """
+        detected_secrets = self.detect_secrets(text, context_hint)
+
+        if not detected_secrets:
+            return text, []
+
+        # Replace secrets with UUID references using simple string replacement
+        filtered_text = text
+        for secret in detected_secrets:
+            filtered_text = filtered_text.replace(secret.original_value, secret.replacement_text)
+
+        logger.info(f"Filtered {len(detected_secrets)} secrets from text. Context: {context_hint}")
+
+        return filtered_text, detected_secrets
+
+    def _get_pattern_info(self, pattern_name: str) -> Optional[ConfigSecretPattern]:
+        """Get pattern information by name."""
+        for pattern in self.detection_config.patterns:
+            if pattern.name == pattern_name:
+                return pattern
+
+        return None
+
+    def add_custom_pattern(self, pattern: ConfigSecretPattern) -> None:
+        """Add a new custom pattern."""
+        # Remove existing pattern with same name
+        self.detection_config.patterns = [p for p in self.detection_config.patterns if p.name != pattern.name]
+        self.detection_config.patterns.append(pattern)
+        self._compile_patterns()
+
+        logger.info(f"Added custom secret pattern: {pattern.name}")
+
+    def remove_custom_pattern(self, pattern_name: str) -> bool:
+        """Remove a custom pattern by name."""
+        original_count = len(self.detection_config.patterns)
+        self.detection_config.patterns = [p for p in self.detection_config.patterns if p.name != pattern_name]
+
+        if len(self.detection_config.patterns) < original_count:
+            self._compile_patterns()
+            logger.info(f"Removed custom secret pattern: {pattern_name}")
+            return True
+        return False
+
+    def disable_pattern(self, pattern_name: str) -> None:
+        """Disable a pattern (default or custom)."""
+        # Remove pattern from active patterns
+        self.detection_config.patterns = [p for p in self.detection_config.patterns if p.name != pattern_name]
+        self._compile_patterns()
+        logger.info(f"Disabled secret pattern: {pattern_name}")
+
+    def enable_pattern(self, pattern_name: str) -> None:
+        """Re-enable a previously disabled pattern."""
+        # This method would need to store disabled patterns separately
+        # For now, just log a warning
+        logger.warning(f"Pattern re-enabling not supported without storage: {pattern_name}")
+
+    def get_pattern_stats(self) -> PatternStats:
+        """Get statistics about active patterns."""
+        # Count patterns by type based on sensitivity level
+        default_count = 0
+        custom_count = 0
+        for pattern in self.detection_config.patterns:
+            # Assume patterns with standard sensitivity levels are defaults
+            if pattern.sensitivity.value in ["low", "medium", "high"]:
+                default_count += 1
+            else:
+                custom_count += 1
+
+        return PatternStats(
+            total_patterns=len(self._compiled_patterns),
+            default_patterns=default_count,
+            custom_patterns=custom_count,
+            disabled_patterns=0,  # No disabled patterns in current schema
+            builtin_patterns=True,  # Assume builtin patterns are always enabled
+            filter_version="v1.0",
+        )
+
+    def export_config(self) -> ConfigExport:
+        """Export current configuration for persistence."""
+        # Convert patterns to appropriate categories
+        custom_patterns = []
+        default_patterns = []
+
+        for pattern in self.detection_config.patterns:
+            pattern_dict = pattern.model_dump()
+            if pattern.sensitivity.value in ["low", "medium", "high"]:
+                default_patterns.append(pattern_dict)
+            else:
+                custom_patterns.append(pattern_dict)
+
+        return ConfigExport(
+            filter_id="config_based",
+            version=1,
+            builtin_patterns_enabled=self.detection_config.enabled,
+            custom_patterns=custom_patterns,
+            disabled_patterns=[],  # No disabled patterns in current schema
+            sensitivity_overrides={},
+            require_confirmation_for=["CRITICAL"],
+            auto_decrypt_for_actions=["speak", "tool"],
+        )
+
+    def import_config(self, config: ConfigExport) -> None:
+        """Import configuration from dictionary."""
+        # Convert from ConfigExport back to SecretsDetectionConfig
+        patterns = []
+
+        # Add custom patterns from config
+        # Note: ConfigExport doesn't have default_patterns, only custom_patterns
+        # The builtin patterns are controlled by builtin_patterns_enabled flag
+
+        # Add custom patterns
+        for pattern_obj in config.custom_patterns:
+            # pattern_obj is already a SecretPattern (alias for ConfigSecretPattern)
+            patterns.append(pattern_obj)
+
+        # Create new config
+        # Note: SecretsDetectionConfig only has 'enabled' and 'patterns' attributes
+        self.detection_config = SecretsDetectionConfig(enabled=config.builtin_patterns_enabled, patterns=patterns)
+        self._compile_patterns()
+        logger.info(f"Imported secrets detection config with {len(patterns)} patterns")
+
+    # Implement SecretsFilterInterface methods
+    def filter_content(self, content: str, source_id: Optional[str] = None) -> SecretsFilterResult:
+        """Filter content for secrets using the text filtering method."""
+        filtered_text, detected_secrets = self.filter_text(content)
+
+        schema_secrets = []
+        for secret in detected_secrets:
+            schema_secret = DetectedSecret(
+                original_value=secret.original_value,
+                secret_uuid=secret.secret_uuid,
+                pattern_name=secret.pattern_name,
+                description=secret.description,
+                sensitivity=secret.sensitivity,
+                context_hint=secret.context_hint,
+                replacement_text=secret.replacement_text,
+            )
+            schema_secrets.append(schema_secret)
+
+        return SecretsFilterResult(
+            filtered_content=filtered_text,
+            detected_secrets=schema_secrets,
+            secrets_found=len(detected_secrets),
+            patterns_matched=[s.pattern_name for s in detected_secrets],
+        )
+
+    def add_pattern(self, pattern: ConfigSecretPattern) -> bool:
+        """Add a new secret detection pattern."""
+        try:
+            # Pattern is already a ConfigSecretPattern, just add it
+            self.add_custom_pattern(pattern)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to add pattern: {e}")
+            return False
+
+    def remove_pattern(self, pattern_name: str) -> bool:
+        """Remove a secret detection pattern."""
+        return self.remove_custom_pattern(pattern_name)
+
+    def get_filter_config(self) -> ConfigExport:  # pragma: no cover - thin wrapper
+        """Get the current filter configuration."""
+        # Convert detection config to schema format
+        # Note: SecretsDetectionConfig just has 'patterns', not separate builtin/custom lists
+        return ConfigExport(
+            filter_id="config_based",
+            version=1,
+            builtin_patterns_enabled=self.detection_config.enabled,
+            custom_patterns=[
+                {
+                    "name": p.name,
+                    "pattern": p.pattern,
+                    "description": p.description if hasattr(p, "description") else "",
+                    "sensitivity": p.sensitivity.value if hasattr(p, "sensitivity") else "medium",
+                    "enabled": True,  # All patterns in the list are considered enabled
+                }
+                for p in self.detection_config.patterns
+            ],
+            disabled_patterns=[],  # Not tracked separately in SecretsDetectionConfig
+            sensitivity_overrides={},  # Not used in new system
+            require_confirmation_for=["CRITICAL"],  # Default
+            auto_decrypt_for_actions=["speak", "tool"],  # Default
+        )
+
+    def update_filter_config(self, updates: JSONDict) -> bool:  # pragma: no cover - rarely used
+        """Update filter configuration settings."""
+        try:
+            for key, value in updates.items():
+                if hasattr(self.detection_config, key):
+                    setattr(self.detection_config, key, value)
+            self._compile_patterns()
+            return True
+        except Exception:
+            return False