empathy-framework 5.1.1__py3-none-any.whl → 5.2.1__py3-none-any.whl

empathy_os/memory/long_term.py

@@ -27,381 +27,40 @@ Copyright 2025 Smart AI Memory, LLC
 Licensed under Fair Source 0.9
 """
 
-import base64
-import binascii
 import concurrent.futures
 import hashlib
-import json
 import os
-from dataclasses import dataclass, field
 from datetime import datetime, timedelta
-from enum import Enum
-from pathlib import Path
 from typing import Any
 
 import structlog
 
-from empathy_os.config import _validate_file_path
-
+from .encryption import HAS_ENCRYPTION, EncryptionManager
+
+# Import extracted modules for backward compatibility
+from .long_term_types import (
+    DEFAULT_CLASSIFICATION_RULES,
+    Classification,
+    ClassificationRules,
+    PatternMetadata,
+    PermissionError,
+    SecurePattern,
+    SecurityError,
+)
 from .security.audit_logger import AuditEvent, AuditLogger
 from .security.pii_scrubber import PIIScrubber
 from .security.secrets_detector import SecretsDetector
+from .simple_storage import LongTermMemory
+from .storage_backend import MemDocsStorage
 
 logger = structlog.get_logger(__name__)
 
-# Check for cryptography library
-try:
-    from cryptography.exceptions import InvalidTag
-    from cryptography.hazmat.primitives.ciphers.aead import AESGCM
-
-    HAS_ENCRYPTION = True
-except ImportError:
-    HAS_ENCRYPTION = False
-    logger.warning("cryptography library not available - encryption disabled")
-
-
-class Classification(Enum):
-    """Three-tier classification system for MemDocs patterns"""
-
-    PUBLIC = "PUBLIC"  # Shareable across organization, anonymized
-    INTERNAL = "INTERNAL"  # Team/project only, no PII or secrets
-    SENSITIVE = "SENSITIVE"  # Encrypted at rest, access-controlled (HIPAA, finance)
-
-
-@dataclass
-class ClassificationRules:
-    """Security rules for each classification level"""
-
-    classification: Classification
-    encryption_required: bool
-    retention_days: int
-    access_level: str  # "all_users", "project_team", "explicit_permission"
-    audit_all_access: bool = False
-
-
-# Default classification rules based on enterprise security policy
-DEFAULT_CLASSIFICATION_RULES: dict[Classification, ClassificationRules] = {
-    Classification.PUBLIC: ClassificationRules(
-        classification=Classification.PUBLIC,
-        encryption_required=False,
-        retention_days=365,
-        access_level="all_users",
-        audit_all_access=False,
-    ),
-    Classification.INTERNAL: ClassificationRules(
-        classification=Classification.INTERNAL,
-        encryption_required=False,
-        retention_days=180,
-        access_level="project_team",
-        audit_all_access=False,
-    ),
-    Classification.SENSITIVE: ClassificationRules(
-        classification=Classification.SENSITIVE,
-        encryption_required=True,
-        retention_days=90,
-        access_level="explicit_permission",
-        audit_all_access=True,
-    ),
-}
-
-
-@dataclass
-class PatternMetadata:
-    """Metadata for stored MemDocs patterns"""
-
-    pattern_id: str
-    created_by: str
-    created_at: str
-    classification: str
-    retention_days: int
-    encrypted: bool
-    pattern_type: str
-    sanitization_applied: bool
-    pii_removed: int
-    secrets_detected: int
-    access_control: dict[str, Any] = field(default_factory=dict)
-    custom_metadata: dict[str, Any] = field(default_factory=dict)
-
-
-@dataclass
-class SecurePattern:
-    """Represents a securely stored pattern"""
-
-    pattern_id: str
-    content: str
-    metadata: PatternMetadata
-
-
-class SecurityError(Exception):
-    """Raised when security policy is violated"""
-
-
-class PermissionError(Exception):
-    """Raised when access is denied"""
-
-
-class EncryptionManager:
-    """Manages encryption/decryption for SENSITIVE patterns.
-
-    Uses AES-256-GCM (Galois/Counter Mode) for authenticated encryption.
-    Keys are derived from a master key using HKDF.
-    """
-
-    def __init__(self, master_key: bytes | None = None):
-        """Initialize encryption manager.
-
-        Args:
-            master_key: 32-byte master key (or None to generate/load)
-
-        """
-        if not HAS_ENCRYPTION:
-            logger.warning("Encryption not available - install cryptography library")
-            self.enabled = False
-            return
-
-        self.enabled = True
-        self.master_key = master_key or self._load_or_generate_key()
-
-    def _load_or_generate_key(self) -> bytes:
-        """Load master key from environment or generate new one.
-
-        Production: Set EMPATHY_MASTER_KEY environment variable
-        Development: Generates ephemeral key (warning logged)
-        """
-        # Check environment variable first
-        if env_key := os.getenv("EMPATHY_MASTER_KEY"):
-            try:
-                return base64.b64decode(env_key)
-            except (binascii.Error, ValueError) as e:
-                logger.error("invalid_master_key_in_env", error=str(e))
-                raise ValueError("Invalid EMPATHY_MASTER_KEY format") from e
-
-        # Check key file
-        key_file = Path.home() / ".empathy" / "master.key"
-        if key_file.exists():
-            try:
-                return key_file.read_bytes()
-            except (OSError, PermissionError) as e:
-                logger.error("failed_to_load_key_file", error=str(e))
-
-        # Generate ephemeral key (NOT for production)
-        logger.warning(
-            "no_master_key_found",
-            message="Generating ephemeral encryption key - set EMPATHY_MASTER_KEY for production",
-        )
-        return AESGCM.generate_key(bit_length=256)
-
-    def encrypt(self, plaintext: str) -> str:
-        """Encrypt plaintext using AES-256-GCM.
-
-        Args:
-            plaintext: Content to encrypt
-
-        Returns:
-            Base64-encoded ciphertext with format: nonce||ciphertext||tag
-
-        Raises:
-            SecurityError: If encryption fails
-
-        """
-        if not self.enabled:
-            raise SecurityError("Encryption not available - install cryptography library")
-
-        try:
-            # Generate random 96-bit nonce (12 bytes)
-            nonce = os.urandom(12)
-
-            # Create AESGCM cipher
-            aesgcm = AESGCM(self.master_key)
-
-            # Encrypt and authenticate
-            ciphertext = aesgcm.encrypt(nonce, plaintext.encode("utf-8"), None)
-
-            # Combine nonce + ciphertext for storage
-            encrypted_data = nonce + ciphertext
-
-            # Return base64-encoded
-            return base64.b64encode(encrypted_data).decode("utf-8")
-
-        except (ValueError, TypeError, UnicodeEncodeError) as e:
-            logger.error("encryption_failed", error=str(e))
-            raise SecurityError(f"Encryption failed: {e}") from e
-
-    def decrypt(self, ciphertext_b64: str) -> str:
-        """Decrypt ciphertext using AES-256-GCM.
-
-        Args:
-            ciphertext_b64: Base64-encoded encrypted data
-
-        Returns:
-            Decrypted plaintext
-
-        Raises:
-            SecurityError: If decryption fails (invalid key, corrupted data, etc.)
-
-        """
-        if not self.enabled:
-            raise SecurityError("Encryption not available - install cryptography library")
-
-        try:
-            # Decode from base64
-            encrypted_data = base64.b64decode(ciphertext_b64)
-
-            # Extract nonce (first 12 bytes) and ciphertext (rest)
-            nonce = encrypted_data[:12]
-            ciphertext = encrypted_data[12:]
-
-            # Create AESGCM cipher
-            aesgcm = AESGCM(self.master_key)
-
-            # Decrypt and verify
-            plaintext_bytes = aesgcm.decrypt(nonce, ciphertext, None)
-
-            return plaintext_bytes.decode("utf-8")
-
-        except (ValueError, TypeError, UnicodeDecodeError, binascii.Error, InvalidTag) as e:
-            logger.error("decryption_failed", error=str(e))
-            raise SecurityError(f"Decryption failed: {e}") from e
-
-
-class MemDocsStorage:
-    """Mock/Simple MemDocs storage backend.
-
-    In production, this would integrate with the actual MemDocs library.
-    For now, provides a simple file-based storage for testing.
-    """
-
-    def __init__(self, storage_dir: str = "./memdocs_storage"):
-        """Initialize storage backend.
-
-        Args:
-            storage_dir: Directory for pattern storage
-
-        """
-        self.storage_dir = Path(storage_dir)
-        self.storage_dir.mkdir(parents=True, exist_ok=True)
-        logger.info("memdocs_storage_initialized", storage_dir=str(self.storage_dir))
-
-    def store(self, pattern_id: str, content: str, metadata: dict[str, Any]) -> bool:
-        """Store a pattern.
-
-        Args:
-            pattern_id: Unique pattern identifier
-            content: Pattern content (may be encrypted)
-            metadata: Pattern metadata
-
-        Returns:
-            True if successful
-
-        Raises:
-            IOError: If storage fails
-
-        """
-        try:
-            pattern_file = self.storage_dir / f"{pattern_id}.json"
-
-            # Ensure parent directory exists
-            pattern_file.parent.mkdir(parents=True, exist_ok=True)
-
-            pattern_data = {"pattern_id": pattern_id, "content": content, "metadata": metadata}
-
-            validated_pattern_file = _validate_file_path(str(pattern_file))
-            with open(validated_pattern_file, "w", encoding="utf-8") as f:
-                json.dump(pattern_data, f, indent=2)
-
-            logger.debug("pattern_stored", pattern_id=pattern_id)
-            return True
-
-        except (OSError, PermissionError, json.JSONDecodeError) as e:
-            logger.error("pattern_storage_failed", pattern_id=pattern_id, error=str(e))
-            raise
-
-    def retrieve(self, pattern_id: str) -> dict[str, Any] | None:
-        """Retrieve a pattern.
-
-        Args:
-            pattern_id: Unique pattern identifier
-
-        Returns:
-            Pattern data dictionary or None if not found
-
-        """
-        try:
-            pattern_file = self.storage_dir / f"{pattern_id}.json"
-
-            if not pattern_file.exists():
-                logger.warning("pattern_not_found", pattern_id=pattern_id)
-                return None
-
-            with open(pattern_file, encoding="utf-8") as f:
-                pattern_data: dict[str, Any] = json.load(f)
-
-            logger.debug("pattern_retrieved", pattern_id=pattern_id)
-            return pattern_data
-
-        except (OSError, PermissionError, json.JSONDecodeError) as e:
-            logger.error("pattern_retrieval_failed", pattern_id=pattern_id, error=str(e))
-            return None
+# HAS_ENCRYPTION is now imported from encryption module
 
-    def delete(self, pattern_id: str) -> bool:
-        """Delete a pattern.
-
-        Args:
-            pattern_id: Unique pattern identifier
-
-        Returns:
-            True if deleted, False if not found
-
-        """
-        try:
-            pattern_file = self.storage_dir / f"{pattern_id}.json"
-
-            if not pattern_file.exists():
-                return False
-
-            pattern_file.unlink()
-            logger.info("pattern_deleted", pattern_id=pattern_id)
-            return True
-
-        except (OSError, PermissionError) as e:
-            logger.error("pattern_deletion_failed", pattern_id=pattern_id, error=str(e))
-            return False
-
-    def list_patterns(
-        self,
-        classification: str | None = None,
-        created_by: str | None = None,
-    ) -> list[str]:
-        """List pattern IDs matching criteria.
-
-        Args:
-            classification: Filter by classification
-            created_by: Filter by creator
-
-        Returns:
-            List of pattern IDs
-
-        """
-        pattern_ids = []
-
-        for pattern_file in self.storage_dir.glob("*.json"):
-            try:
-                with open(pattern_file, encoding="utf-8") as f:
-                    data = json.load(f)
-                    metadata = data.get("metadata", {})
-
-                    # Apply filters
-                    if classification and metadata.get("classification") != classification:
-                        continue
-                    if created_by and metadata.get("created_by") != created_by:
-                        continue
-
-                    pattern_ids.append(data.get("pattern_id"))
-
-            except Exception:
-                continue
-
-        return pattern_ids
+# NOTE: Classification, ClassificationRules, PatternMetadata, SecurePattern,
+# EncryptionManager, MemDocsStorage, and LongTermMemory have been extracted to
+# separate modules for better modularity. They are imported above and re-exported
+# below for backward compatibility.
 
 
 class SecureMemDocsIntegration:
@@ -1227,272 +886,28 @@ class SecureMemDocsIntegration:
 # ============================================================================
 
 
-class LongTermMemory:
-    """Simplified long-term persistent storage interface.
-
-    Provides basic CRUD operations for long-term memory storage without
-    the full security pipeline of SecureMemDocsIntegration. Suitable for
-    simple persistent storage needs where PII scrubbing and encryption
-    are not required.
-
-    Features:
-    - JSON file-based storage
-    - Classification support (PUBLIC/INTERNAL/SENSITIVE)
-    - Simple key-value interface
-    - List keys by classification
-
-    Example:
-        >>> memory = LongTermMemory(storage_path="./data")
-        >>> memory.store("config", {"setting": "value"}, classification="INTERNAL")
-        >>> data = memory.retrieve("config")
-        >>> keys = memory.list_keys(classification="INTERNAL")
-
-    Note:
-        For enterprise features (PII scrubbing, encryption, audit logging),
-        use SecureMemDocsIntegration instead.
-    """
-
-    def __init__(self, storage_path: str = "./long_term_storage"):
-        """Initialize long-term memory storage.
-
-        Args:
-            storage_path: Directory path for JSON storage
-
-        """
-        self.storage_path = Path(storage_path)
-        self.storage_path.mkdir(parents=True, exist_ok=True)
-        logger.info("long_term_memory_initialized", storage_path=str(self.storage_path))
-
-    def store(
-        self,
-        key: str,
-        data: Any,
-        classification: str | Classification | None = None,
-    ) -> bool:
-        """Store data in long-term memory.
-
-        Args:
-            key: Storage key
-            data: Data to store (must be JSON-serializable)
-            classification: Data classification (PUBLIC/INTERNAL/SENSITIVE)
-
-        Returns:
-            True if stored successfully, False otherwise
-
-        Raises:
-            ValueError: If key is empty or data is not JSON-serializable
-            TypeError: If data cannot be serialized to JSON
-
-        Example:
-            >>> memory = LongTermMemory()
-            >>> memory.store("user_prefs", {"theme": "dark"}, "INTERNAL")
-            True
-
-        """
-        if not key or not key.strip():
-            raise ValueError("key cannot be empty")
-
-        # Validate key for path traversal attacks
-        if ".." in key or key.startswith("/") or "\x00" in key:
-            logger.error("path_traversal_attempt", key=key)
-            return False
-
-        try:
-            # Convert classification to string
-            classification_str = "INTERNAL"  # Default
-            if classification is not None:
-                if isinstance(classification, Classification):
-                    classification_str = classification.value
-                elif isinstance(classification, str):
-                    # Validate classification string
-                    try:
-                        Classification[classification.upper()]
-                        classification_str = classification.upper()
-                    except KeyError:
-                        logger.warning(
-                            "invalid_classification",
-                            classification=classification,
-                            using_default="INTERNAL",
-                        )
-
-            # Create storage record
-            record = {
-                "key": key,
-                "data": data,
-                "classification": classification_str,
-                "created_at": datetime.utcnow().isoformat() + "Z",
-                "updated_at": datetime.utcnow().isoformat() + "Z",
-            }
-
-            # Store to JSON file
-            file_path = self.storage_path / f"{key}.json"
-            validated_file_path = _validate_file_path(str(file_path))
-            with validated_file_path.open("w", encoding="utf-8") as f:
-                json.dump(record, f, indent=2)
 
-            logger.debug("data_stored", key=key, classification=classification_str)
-            return True
-
-        except (TypeError, ValueError) as e:
-            logger.error("store_failed", key=key, error=str(e))
-            raise
-        except (OSError, PermissionError) as e:
-            logger.error("storage_io_error", key=key, error=str(e))
-            return False
-
-    def retrieve(self, key: str) -> Any | None:
-        """Retrieve data from long-term memory.
-
-        Args:
-            key: Storage key
-
-        Returns:
-            Stored data or None if not found
-
-        Raises:
-            ValueError: If key is empty
-
-        Example:
-            >>> memory = LongTermMemory()
-            >>> memory.store("config", {"value": 42})
-            >>> data = memory.retrieve("config")
-            >>> print(data["value"])
-            42
 
-        """
-        if not key or not key.strip():
-            raise ValueError("key cannot be empty")
-
-        try:
-            file_path = self.storage_path / f"{key}.json"
-
-            if not file_path.exists():
-                logger.debug("key_not_found", key=key)
-                return None
-
-            with file_path.open(encoding="utf-8") as f:
-                record = json.load(f)
-
-            logger.debug("data_retrieved", key=key)
-            return record.get("data")
-
-        except (OSError, PermissionError, json.JSONDecodeError) as e:
-            logger.error("retrieve_failed", key=key, error=str(e))
-            return None
-
-    def delete(self, key: str) -> bool:
-        """Delete data from long-term memory.
-
-        Args:
-            key: Storage key
-
-        Returns:
-            True if deleted, False if not found or error
-
-        Raises:
-            ValueError: If key is empty
-
-        Example:
-            >>> memory = LongTermMemory()
-            >>> memory.store("temp", {"data": "value"})
-            >>> memory.delete("temp")
-            True
-
-        """
-        if not key or not key.strip():
-            raise ValueError("key cannot be empty")
-
-        try:
-            file_path = self.storage_path / f"{key}.json"
-
-            if not file_path.exists():
-                logger.debug("key_not_found_for_deletion", key=key)
-                return False
-
-            file_path.unlink()
-            logger.info("data_deleted", key=key)
-            return True
-
-        except (OSError, PermissionError) as e:
-            logger.error("delete_failed", key=key, error=str(e))
-            return False
-
-    def list_keys(self, classification: str | Classification | None = None) -> list[str]:
-        """List all keys in long-term memory, optionally filtered by classification.
-
-        Args:
-            classification: Filter by classification (PUBLIC/INTERNAL/SENSITIVE)
-
-        Returns:
-            List of storage keys
-
-        Example:
-            >>> memory = LongTermMemory()
-            >>> memory.store("public_data", {"x": 1}, "PUBLIC")
-            >>> memory.store("internal_data", {"y": 2}, "INTERNAL")
-            >>> keys = memory.list_keys(classification="PUBLIC")
-            >>> print(keys)
-            ['public_data']
-
-        """
-        keys = []
-
-        # Convert classification to string if needed
-        filter_classification = None
-        if classification is not None:
-            if isinstance(classification, Classification):
-                filter_classification = classification.value
-            elif isinstance(classification, str):
-                try:
-                    Classification[classification.upper()]
-                    filter_classification = classification.upper()
-                except KeyError:
-                    logger.warning("invalid_classification_filter", classification=classification)
-                    return []
-
-        try:
-            for file_path in self.storage_path.glob("*.json"):
-                try:
-                    with file_path.open(encoding="utf-8") as f:
-                        record = json.load(f)
-
-                    # Apply classification filter if specified
-                    if filter_classification is not None:
-                        if record.get("classification") != filter_classification:
-                            continue
-
-                    keys.append(record.get("key", file_path.stem))
-
-                except (OSError, json.JSONDecodeError):
-                    continue
-
-        except (OSError, PermissionError) as e:
-            logger.error("list_keys_failed", error=str(e))
-
-        return keys
-
-    def clear(self) -> int:
-        """Clear all data from long-term memory.
-
-        Returns:
-            Number of keys deleted
-
-        Warning:
-            This operation cannot be undone!
-
-        """
-        count = 0
-        try:
-            for file_path in self.storage_path.glob("*.json"):
-                try:
-                    file_path.unlink()
-                    count += 1
-                except (OSError, PermissionError):
-                    continue
-
-            logger.warning("long_term_memory_cleared", count=count)
-            return count
+# ============================================================================
+# Backward Compatibility Exports
+# ============================================================================
 
-        except (OSError, PermissionError) as e:
-            logger.error("clear_failed", error=str(e))
-            return count
+__all__ = [
+    # Types (from long_term_types.py)
+    "Classification",
+    "ClassificationRules",
+    "DEFAULT_CLASSIFICATION_RULES",
+    "PatternMetadata",
+    "SecurePattern",
+    "SecurityError",
+    "PermissionError",
+    # Encryption (from encryption.py)
+    "EncryptionManager",
+    "HAS_ENCRYPTION",
+    # Storage (from storage_backend.py)
+    "MemDocsStorage",
+    # Simple storage (from simple_storage.py)
+    "LongTermMemory",
+    # Main integration (defined in this file)
+    "SecureMemDocsIntegration",
+]