empathy-framework 5.1.1__py3-none-any.whl → 5.2.1__py3-none-any.whl

empathy_os/memory/mixins/promotion_mixin.py

@@ -0,0 +1,109 @@
+"""Pattern promotion mixin for UnifiedMemory.
+
+Handles promotion of patterns from short-term to long-term memory.
+
+Copyright 2025 Smart AI Memory, LLC
+Licensed under Fair Source 0.9
+"""
+
+from typing import TYPE_CHECKING, Any
+
+import structlog
+
+if TYPE_CHECKING:
+    from ..long_term import Classification
+    from ..short_term import AgentCredentials, RedisShortTermMemory
+
+logger = structlog.get_logger(__name__)
+
+
+class PatternPromotionMixin:
+    """Mixin providing pattern promotion capabilities for UnifiedMemory."""
+
+    # Type hints for attributes that will be provided by UnifiedMemory
+    _short_term: "RedisShortTermMemory | None"
+    _long_term: Any  # SecureMemDocsIntegration | None
+
+    # Needs access to methods from other mixins
+    @property
+    def credentials(self) -> "AgentCredentials":
+        """Get credentials - provided by ShortTermOperationsMixin."""
+        ...
+
+    def get_staged_patterns(self) -> list[dict]:
+        """Get staged patterns - provided by ShortTermOperationsMixin."""
+        ...
+
+    def persist_pattern(
+        self,
+        content: str,
+        pattern_type: str,
+        classification: "Classification | str | None" = None,
+        auto_classify: bool = True,
+        metadata: dict[str, Any] | None = None,
+    ) -> dict[str, Any] | None:
+        """Persist pattern - provided by LongTermOperationsMixin."""
+        ...
+
+    # =========================================================================
+    # PATTERN PROMOTION (SHORT-TERM → LONG-TERM)
+    # =========================================================================
+
+    def promote_pattern(
+        self,
+        staged_pattern_id: str,
+        classification: "Classification | str | None" = None,
+        auto_classify: bool = True,
+    ) -> dict[str, Any] | None:
+        """Promote a staged pattern from short-term to long-term memory.
+
+        Args:
+            staged_pattern_id: ID of staged pattern to promote
+            classification: Override classification (or auto-detect)
+            auto_classify: Auto-detect classification from content
+
+        Returns:
+            Long-term storage result, or None if failed
+
+        """
+        if not self._short_term or not self._long_term:
+            logger.error("memory_backends_unavailable")
+            return None
+
+        # Retrieve staged pattern
+        staged_patterns = self.get_staged_patterns()
+        staged = next(
+            (p for p in staged_patterns if p.get("pattern_id") == staged_pattern_id),
+            None,
+        )
+
+        if not staged:
+            logger.warning("staged_pattern_not_found", pattern_id=staged_pattern_id)
+            return None
+
+        # Persist to long-term storage
+        # Content is stored in context dict by stage_pattern
+        context = staged.get("context", {})
+        content = context.get("content", "") or staged.get("description", "")
+        result = self.persist_pattern(
+            content=content,
+            pattern_type=staged.get("pattern_type", "general"),
+            classification=classification,
+            auto_classify=auto_classify,
+            metadata=context,
+        )
+
+        if result:
+            # Remove from staging (use promote_pattern which handles deletion)
+            try:
+                self._short_term.promote_pattern(staged_pattern_id, self.credentials)
+            except PermissionError:
+                # If we can't promote (delete from staging), just log it
+                logger.warning("could_not_remove_from_staging", pattern_id=staged_pattern_id)
+            logger.info(
+                "pattern_promoted",
+                staged_id=staged_pattern_id,
+                long_term_id=result.get("pattern_id"),
+            )
+
+        return result

empathy_os/memory/mixins/short_term_mixin.py

@@ -0,0 +1,182 @@
+"""Short-term memory operations mixin for UnifiedMemory.
+
+Provides working memory operations (stash/retrieve) and pattern staging.
+
+Copyright 2025 Smart AI Memory, LLC
+Licensed under Fair Source 0.9
+"""
+
+import uuid
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+import structlog
+
+if TYPE_CHECKING:
+    from ..file_session import FileSessionMemory
+    from ..redis_bootstrap import RedisStatus
+    from ..short_term import (
+        AccessTier,
+        AgentCredentials,
+        RedisShortTermMemory,
+    )
+
+logger = structlog.get_logger(__name__)
+
+
+class ShortTermOperationsMixin:
+    """Mixin providing short-term memory operations for UnifiedMemory."""
+
+    # Type hints for attributes that will be provided by UnifiedMemory
+    user_id: str
+    access_tier: "AccessTier"
+    _file_session: "FileSessionMemory | None"
+    _short_term: "RedisShortTermMemory | None"
+    _redis_status: "RedisStatus | None"
+    config: Any  # MemoryConfig
+
+    @property
+    def credentials(self) -> "AgentCredentials":
+        """Get agent credentials for short-term memory operations."""
+        from ..short_term import AgentCredentials
+
+        return AgentCredentials(agent_id=self.user_id, tier=self.access_tier)
+
+    # =========================================================================
+    # SHORT-TERM MEMORY OPERATIONS (Working Memory)
+    # =========================================================================
+
+    def stash(self, key: str, value: Any, ttl_seconds: int | None = None) -> bool:
+        """Store data in working memory with TTL.
+
+        Uses file-based session as primary storage, with optional Redis for
+        real-time features. Data is persisted to disk automatically.
+
+        Args:
+            key: Storage key
+            value: Data to store (must be JSON-serializable)
+            ttl_seconds: Time-to-live in seconds (default from config)
+
+        Returns:
+            True if stored successfully
+
+        """
+        from ..short_term import TTLStrategy
+
+        ttl = ttl_seconds or self.config.default_ttl_seconds
+
+        # Primary: File session memory (always available)
+        if self._file_session:
+            self._file_session.stash(key, value, ttl=ttl)
+
+        # Optional: Redis for real-time sync
+        if self._short_term and self._redis_status and self._redis_status.available:
+            # Map ttl_seconds to TTLStrategy
+            ttl_strategy = TTLStrategy.WORKING_RESULTS
+            if ttl_seconds is not None:
+                # COORDINATION removed in v5.0 - use SESSION for short-lived data
+                if ttl_seconds <= TTLStrategy.SESSION.value:
+                    ttl_strategy = TTLStrategy.SESSION
+                elif ttl_seconds <= TTLStrategy.WORKING_RESULTS.value:
+                    ttl_strategy = TTLStrategy.WORKING_RESULTS
+                elif ttl_seconds <= TTLStrategy.STAGED_PATTERNS.value:
+                    ttl_strategy = TTLStrategy.STAGED_PATTERNS
+                else:
+                    ttl_strategy = TTLStrategy.CONFLICT_CONTEXT
+
+            try:
+                self._short_term.stash(key, value, self.credentials, ttl_strategy)
+            except Exception as e:
+                logger.debug("redis_stash_failed", key=key, error=str(e))
+
+        # Return True if at least one backend succeeded
+        return self._file_session is not None
+
+    def retrieve(self, key: str) -> Any | None:
+        """Retrieve data from working memory.
+
+        Checks Redis first (if available) for faster access, then falls back
+        to file-based session storage.
+
+        Args:
+            key: Storage key
+
+        Returns:
+            Stored data or None if not found
+
+        """
+        # Try Redis first (faster, if available)
+        if self._short_term and self._redis_status and self._redis_status.available:
+            try:
+                result = self._short_term.retrieve(key, self.credentials)
+                if result is not None:
+                    return result
+            except Exception as e:
+                logger.debug("redis_retrieve_failed", key=key, error=str(e))
+
+        # Fall back to file session (primary storage)
+        if self._file_session:
+            return self._file_session.retrieve(key)
+
+        return None
+
+    # =========================================================================
+    # PATTERN STAGING
+    # =========================================================================
+
+    def stage_pattern(
+        self,
+        pattern_data: dict[str, Any],
+        pattern_type: str = "general",
+        ttl_hours: int = 24,
+    ) -> str | None:
+        """Stage a pattern for validation before long-term storage.
+
+        Args:
+            pattern_data: Pattern content and metadata
+            pattern_type: Type of pattern (algorithm, protocol, etc.)
+            ttl_hours: Hours before staged pattern expires (not used in current impl)
+
+        Returns:
+            Staged pattern ID or None if failed
+
+        """
+        from ..short_term import StagedPattern
+
+        if not self._short_term:
+            logger.warning("short_term_memory_unavailable")
+            return None
+
+        # Create a StagedPattern object from the pattern_data dict
+        pattern_id = f"staged_{uuid.uuid4().hex[:12]}"
+        staged_pattern = StagedPattern(
+            pattern_id=pattern_id,
+            agent_id=self.user_id,
+            pattern_type=pattern_type,
+            name=pattern_data.get("name", f"Pattern {pattern_id[:8]}"),
+            description=pattern_data.get("description", ""),
+            code=pattern_data.get("code"),
+            context=pattern_data.get("context", {}),
+            confidence=pattern_data.get("confidence", 0.5),
+            staged_at=datetime.now(),
+            interests=pattern_data.get("interests", []),
+        )
+        # Store content in context if provided
+        if "content" in pattern_data:
+            staged_pattern.context["content"] = pattern_data["content"]
+
+        success = self._short_term.stage_pattern(staged_pattern, self.credentials)
+        return pattern_id if success else None
+
+    def get_staged_patterns(self) -> list[dict]:
+        """Get all staged patterns awaiting validation.
+
+        Returns:
+            List of staged patterns with metadata
+
+        """
+        if not self._short_term:
+            return []
+
+        staged_list = self._short_term.list_staged_patterns(self.credentials)
+        return [p.to_dict() for p in staged_list]

empathy_os/memory/short_term.py

@@ -2034,6 +2034,9 @@ class RedisShortTermMemory:
                     f"Confidence {pattern.confidence} below threshold {min_confidence}",
                 )
             del self._mock_storage[key]
+            # Also invalidate local cache
+            if key in self._local_cache:
+                del self._local_cache[key]
             return True, pattern, "Pattern promoted successfully"
 
         if self._client is None:
@@ -2063,6 +2066,10 @@ class RedisShortTermMemory:
             pipe.delete(key)
             pipe.execute()
 
+            # Also invalidate local cache
+            if key in self._local_cache:
+                del self._local_cache[key]
+
             return True, pattern, "Pattern promoted successfully"
 
         except redis.WatchError:

empathy_os/memory/simple_storage.py

@@ -0,0 +1,302 @@
+"""Simplified long-term storage without security pipeline
+
+Provides basic CRUD operations for long-term persistent storage without
+the full security pipeline of SecureMemDocsIntegration. Suitable for
+simple storage needs where PII scrubbing and encryption are not required.
+
+Extracted from long_term.py for better modularity and testability.
+
+Features:
+- JSON file-based storage
+- Classification support (PUBLIC/INTERNAL/SENSITIVE)
+- Simple key-value interface
+- List keys by classification
+- Path validation for security
+
+Copyright 2025 Smart AI Memory, LLC
+Licensed under Fair Source 0.9
+"""
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from empathy_os.config import _validate_file_path
+
+from .long_term_types import Classification
+
+logger = structlog.get_logger(__name__)
+
+
+class LongTermMemory:
+    """Simplified long-term persistent storage interface.
+
+    Provides basic CRUD operations for long-term memory storage without
+    the full security pipeline of SecureMemDocsIntegration. Suitable for
+    simple persistent storage needs where PII scrubbing and encryption
+    are not required.
+
+    Features:
+    - JSON file-based storage
+    - Classification support (PUBLIC/INTERNAL/SENSITIVE)
+    - Simple key-value interface
+    - List keys by classification
+
+    Example:
+        >>> memory = LongTermMemory(storage_path="./data")
+        >>> memory.store("config", {"setting": "value"}, classification="INTERNAL")
+        >>> data = memory.retrieve("config")
+        >>> keys = memory.list_keys(classification="INTERNAL")
+
+    Note:
+        For enterprise features (PII scrubbing, encryption, audit logging),
+        use SecureMemDocsIntegration instead.
+    """
+
+    def __init__(self, storage_path: str = "./long_term_storage"):
+        """Initialize long-term memory storage.
+
+        Args:
+            storage_path: Directory path for JSON storage
+
+        """
+        self.storage_path = Path(storage_path)
+        self.storage_path.mkdir(parents=True, exist_ok=True)
+        logger.info("long_term_memory_initialized", storage_path=str(self.storage_path))
+
+    def store(
+        self,
+        key: str,
+        data: Any,
+        classification: str | Classification | None = None,
+    ) -> bool:
+        """Store data in long-term memory.
+
+        Args:
+            key: Storage key
+            data: Data to store (must be JSON-serializable)
+            classification: Data classification (PUBLIC/INTERNAL/SENSITIVE)
+
+        Returns:
+            True if stored successfully, False otherwise
+
+        Raises:
+            ValueError: If key is empty or data is not JSON-serializable
+            TypeError: If data cannot be serialized to JSON
+
+        Example:
+            >>> memory = LongTermMemory()
+            >>> memory.store("user_prefs", {"theme": "dark"}, "INTERNAL")
+            True
+
+        """
+        if not key or not key.strip():
+            raise ValueError("key cannot be empty")
+
+        # Validate key for path traversal attacks
+        if ".." in key or key.startswith("/") or "\x00" in key:
+            logger.error("path_traversal_attempt", key=key)
+            return False
+
+        try:
+            # Convert classification to string
+            classification_str = "INTERNAL"  # Default
+            if classification is not None:
+                if isinstance(classification, Classification):
+                    classification_str = classification.value
+                elif isinstance(classification, str):
+                    # Validate classification string
+                    try:
+                        Classification[classification.upper()]
+                        classification_str = classification.upper()
+                    except KeyError:
+                        logger.warning(
+                            "invalid_classification",
+                            classification=classification,
+                            using_default="INTERNAL",
+                        )
+
+            # Create storage record
+            record = {
+                "key": key,
+                "data": data,
+                "classification": classification_str,
+                "created_at": datetime.utcnow().isoformat() + "Z",
+                "updated_at": datetime.utcnow().isoformat() + "Z",
+            }
+
+            # Store to JSON file
+            file_path = self.storage_path / f"{key}.json"
+            validated_file_path = _validate_file_path(str(file_path))
+            with validated_file_path.open("w", encoding="utf-8") as f:
+                json.dump(record, f, indent=2)
+
+            logger.debug("data_stored", key=key, classification=classification_str)
+            return True
+
+        except (TypeError, ValueError) as e:
+            logger.error("store_failed", key=key, error=str(e))
+            raise
+        except (OSError, PermissionError) as e:
+            logger.error("storage_io_error", key=key, error=str(e))
+            return False
+
+    def retrieve(self, key: str) -> Any | None:
+        """Retrieve data from long-term memory.
+
+        Args:
+            key: Storage key
+
+        Returns:
+            Stored data or None if not found
+
+        Raises:
+            ValueError: If key is empty
+
+        Example:
+            >>> memory = LongTermMemory()
+            >>> memory.store("config", {"value": 42})
+            >>> data = memory.retrieve("config")
+            >>> print(data["value"])
+            42
+
+        """
+        if not key or not key.strip():
+            raise ValueError("key cannot be empty")
+
+        try:
+            file_path = self.storage_path / f"{key}.json"
+
+            if not file_path.exists():
+                logger.debug("key_not_found", key=key)
+                return None
+
+            with file_path.open(encoding="utf-8") as f:
+                record = json.load(f)
+
+            logger.debug("data_retrieved", key=key)
+            return record.get("data")
+
+        except (OSError, PermissionError, json.JSONDecodeError) as e:
+            logger.error("retrieve_failed", key=key, error=str(e))
+            return None
+
+    def delete(self, key: str) -> bool:
+        """Delete data from long-term memory.
+
+        Args:
+            key: Storage key
+
+        Returns:
+            True if deleted, False if not found or error
+
+        Raises:
+            ValueError: If key is empty
+
+        Example:
+            >>> memory = LongTermMemory()
+            >>> memory.store("temp", {"data": "value"})
+            >>> memory.delete("temp")
+            True
+
+        """
+        if not key or not key.strip():
+            raise ValueError("key cannot be empty")
+
+        try:
+            file_path = self.storage_path / f"{key}.json"
+
+            if not file_path.exists():
+                logger.debug("key_not_found_for_deletion", key=key)
+                return False
+
+            file_path.unlink()
+            logger.info("data_deleted", key=key)
+            return True
+
+        except (OSError, PermissionError) as e:
+            logger.error("delete_failed", key=key, error=str(e))
+            return False
+
+    def list_keys(self, classification: str | Classification | None = None) -> list[str]:
+        """List all keys in long-term memory, optionally filtered by classification.
+
+        Args:
+            classification: Filter by classification (PUBLIC/INTERNAL/SENSITIVE)
+
+        Returns:
+            List of storage keys
+
+        Example:
+            >>> memory = LongTermMemory()
+            >>> memory.store("public_data", {"x": 1}, "PUBLIC")
+            >>> memory.store("internal_data", {"y": 2}, "INTERNAL")
+            >>> keys = memory.list_keys(classification="PUBLIC")
+            >>> print(keys)
+            ['public_data']
+
+        """
+        keys = []
+
+        # Convert classification to string if needed
+        filter_classification = None
+        if classification is not None:
+            if isinstance(classification, Classification):
+                filter_classification = classification.value
+            elif isinstance(classification, str):
+                try:
+                    Classification[classification.upper()]
+                    filter_classification = classification.upper()
+                except KeyError:
+                    logger.warning("invalid_classification_filter", classification=classification)
+                    return []
+
+        try:
+            for file_path in self.storage_path.glob("*.json"):
+                try:
+                    with file_path.open(encoding="utf-8") as f:
+                        record = json.load(f)
+
+                    # Apply classification filter if specified
+                    if filter_classification is not None:
+                        if record.get("classification") != filter_classification:
+                            continue
+
+                    keys.append(record.get("key", file_path.stem))
+
+                except (OSError, json.JSONDecodeError):
+                    continue
+
+        except (OSError, PermissionError) as e:
+            logger.error("list_keys_failed", error=str(e))
+
+        return keys
+
+    def clear(self) -> int:
+        """Clear all data from long-term memory.
+
+        Returns:
+            Number of keys deleted
+
+        Warning:
+            This operation cannot be undone!
+
+        """
+        count = 0
+        try:
+            for file_path in self.storage_path.glob("*.json"):
+                try:
+                    file_path.unlink()
+                    count += 1
+                except (OSError, PermissionError):
+                    continue
+
+            logger.warning("long_term_memory_cleared", count=count)
+            return count
+
+        except (OSError, PermissionError) as e:
+            logger.error("clear_failed", error=str(e))
+            return count