empathy-framework 4.7.1__py3-none-any.whl → 4.9.0__py3-none-any.whl

empathy_os/memory/short_term.py

@@ -26,9 +26,7 @@ import json
 import threading
 import time
 from collections.abc import Callable
-from dataclasses import dataclass, field
 from datetime import datetime
-from enum import Enum
 from typing import Any
 
 import structlog
@@ -37,6 +35,20 @@ from .security.pii_scrubber import PIIScrubber
 from .security.secrets_detector import SecretsDetector
 from .security.secrets_detector import Severity as SecretSeverity
 
+# Import types from dedicated module
+from .types import (
+    AccessTier,
+    AgentCredentials,
+    ConflictContext,
+    PaginatedResult,
+    RedisConfig,
+    RedisMetrics,
+    SecurityError,
+    StagedPattern,
+    TimeWindowQuery,
+    TTLStrategy,
+)
+
 logger = structlog.get_logger(__name__)
 
 try:
@@ -51,408 +63,6 @@ except ImportError:
     RedisTimeoutError = Exception  # type: ignore
 
 
-class AccessTier(Enum):
-    """Role-based access tiers per EMPATHY_PHILOSOPHY.md
-
-    Tier 1 - Observer: Read-only access to validated patterns
-    Tier 2 - Contributor: Can stage patterns for validation
-    Tier 3 - Validator: Can promote staged patterns to active
-    Tier 4 - Steward: Full access including deprecation and audit
-    """
-
-    OBSERVER = 1
-    CONTRIBUTOR = 2
-    VALIDATOR = 3
-    STEWARD = 4
-
-
-class TTLStrategy(Enum):
-    """TTL strategies for different memory types
-
-    Per EMPATHY_PHILOSOPHY.md Section 9.3:
-    - Working results: 1 hour
-    - Staged patterns: 24 hours
-    - Coordination signals: 5 minutes
-    - Conflict context: Until resolution
-    """
-
-    WORKING_RESULTS = 3600  # 1 hour
-    STAGED_PATTERNS = 86400  # 24 hours
-    COORDINATION = 300  # 5 minutes
-    CONFLICT_CONTEXT = 604800  # 7 days (fallback for unresolved)
-    SESSION = 1800  # 30 minutes
-    STREAM_ENTRY = 86400 * 7  # 7 days for audit stream entries
-    TASK_QUEUE = 3600 * 4  # 4 hours for task queue items
-
-
-@dataclass
-class RedisConfig:
-    """Enhanced Redis configuration with SSL and retry support.
-
-    Supports:
-    - Standard connections (host:port)
-    - URL-based connections (redis://...)
-    - SSL/TLS for managed services (rediss://...)
-    - Sentinel for high availability
-    - Connection pooling
-    - Retry with exponential backoff
-    """
-
-    host: str = "localhost"
-    port: int = 6379
-    db: int = 0
-    password: str | None = None
-    use_mock: bool = False
-
-    # Security settings
-    pii_scrub_enabled: bool = True  # Scrub PII before storing (HIPAA/GDPR compliance)
-    secrets_detection_enabled: bool = True  # Block storage of detected secrets
-
-    # SSL/TLS settings
-    ssl: bool = False
-    ssl_cert_reqs: str | None = None  # "required", "optional", "none"
-    ssl_ca_certs: str | None = None
-    ssl_certfile: str | None = None
-    ssl_keyfile: str | None = None
-
-    # Connection pool settings
-    max_connections: int = 10
-    socket_timeout: float = 5.0
-    socket_connect_timeout: float = 5.0
-
-    # Retry settings
-    retry_on_timeout: bool = True
-    retry_max_attempts: int = 3
-    retry_base_delay: float = 0.1  # seconds
-    retry_max_delay: float = 2.0  # seconds
-
-    # Sentinel settings (for HA)
-    sentinel_hosts: list[tuple[str, int]] | None = None
-    sentinel_master_name: str | None = None
-
-    def to_redis_kwargs(self) -> dict:
-        """Convert to redis.Redis constructor kwargs."""
-        kwargs: dict[str, Any] = {
-            "host": self.host,
-            "port": self.port,
-            "db": self.db,
-            "password": self.password,
-            "decode_responses": True,
-            "socket_timeout": self.socket_timeout,
-            "socket_connect_timeout": self.socket_connect_timeout,
-            "retry_on_timeout": self.retry_on_timeout,
-        }
-
-        if self.ssl:
-            kwargs["ssl"] = True
-            if self.ssl_cert_reqs:
-                kwargs["ssl_cert_reqs"] = self.ssl_cert_reqs
-            if self.ssl_ca_certs:
-                kwargs["ssl_ca_certs"] = self.ssl_ca_certs
-            if self.ssl_certfile:
-                kwargs["ssl_certfile"] = self.ssl_certfile
-            if self.ssl_keyfile:
-                kwargs["ssl_keyfile"] = self.ssl_keyfile
-
-        return kwargs
-
-
-@dataclass
-class RedisMetrics:
-    """Metrics for Redis operations."""
-
-    operations_total: int = 0
-    operations_success: int = 0
-    operations_failed: int = 0
-    retries_total: int = 0
-    latency_sum_ms: float = 0.0
-    latency_max_ms: float = 0.0
-
-    # Per-operation metrics
-    stash_count: int = 0
-    retrieve_count: int = 0
-    publish_count: int = 0
-    stream_append_count: int = 0
-
-    # Security metrics
-    pii_scrubbed_total: int = 0  # Total PII instances scrubbed
-    pii_scrub_operations: int = 0  # Operations that had PII scrubbed
-    secrets_blocked_total: int = 0  # Total secrets blocked from storage
-
-    def record_operation(self, operation: str, latency_ms: float, success: bool = True) -> None:
-        """Record an operation metric."""
-        self.operations_total += 1
-        self.latency_sum_ms += latency_ms
-        self.latency_max_ms = max(self.latency_max_ms, latency_ms)
-
-        if success:
-            self.operations_success += 1
-        else:
-            self.operations_failed += 1
-
-        # Track by operation type
-        if operation == "stash":
-            self.stash_count += 1
-        elif operation == "retrieve":
-            self.retrieve_count += 1
-        elif operation == "publish":
-            self.publish_count += 1
-        elif operation == "stream_append":
-            self.stream_append_count += 1
-
-    @property
-    def latency_avg_ms(self) -> float:
-        """Average latency in milliseconds."""
-        if self.operations_total == 0:
-            return 0.0
-        return self.latency_sum_ms / self.operations_total
-
-    @property
-    def success_rate(self) -> float:
-        """Success rate as percentage."""
-        if self.operations_total == 0:
-            return 100.0
-        return (self.operations_success / self.operations_total) * 100
-
-    def to_dict(self) -> dict:
-        """Convert metrics to dictionary for reporting and serialization.
-
-        Returns:
-            Dictionary with keys: operations_total, operations_success,
-            operations_failed, retries_total, latency_avg_ms, latency_max_ms,
-            success_rate, by_operation, security.
-        """
-        return {
-            "operations_total": self.operations_total,
-            "operations_success": self.operations_success,
-            "operations_failed": self.operations_failed,
-            "retries_total": self.retries_total,
-            "latency_avg_ms": round(self.latency_avg_ms, 2),
-            "latency_max_ms": round(self.latency_max_ms, 2),
-            "success_rate": round(self.success_rate, 2),
-            "by_operation": {
-                "stash": self.stash_count,
-                "retrieve": self.retrieve_count,
-                "publish": self.publish_count,
-                "stream_append": self.stream_append_count,
-            },
-            "security": {
-                "pii_scrubbed_total": self.pii_scrubbed_total,
-                "pii_scrub_operations": self.pii_scrub_operations,
-                "secrets_blocked_total": self.secrets_blocked_total,
-            },
-        }
-
-
-@dataclass
-class PaginatedResult:
-    """Result of a paginated query."""
-
-    items: list[Any]
-    cursor: str
-    has_more: bool
-    total_scanned: int = 0
-
-
-@dataclass
-class TimeWindowQuery:
-    """Query parameters for time-window operations."""
-
-    start_time: datetime | None = None
-    end_time: datetime | None = None
-    limit: int = 100
-    offset: int = 0
-
-    @property
-    def start_score(self) -> float:
-        """Start timestamp as Redis score."""
-        if self.start_time is None:
-            return float("-inf")
-        return self.start_time.timestamp()
-
-    @property
-    def end_score(self) -> float:
-        """End timestamp as Redis score."""
-        if self.end_time is None:
-            return float("+inf")
-        return self.end_time.timestamp()
-
-
-@dataclass
-class AgentCredentials:
-    """Agent identity and access permissions"""
-
-    agent_id: str
-    tier: AccessTier
-    roles: list[str] = field(default_factory=list)
-    created_at: datetime = field(default_factory=datetime.now)
-
-    def can_read(self) -> bool:
-        """All tiers can read"""
-        return True
-
-    def can_stage(self) -> bool:
-        """Contributor+ can stage patterns"""
-        return self.tier.value >= AccessTier.CONTRIBUTOR.value
-
-    def can_validate(self) -> bool:
-        """Validator+ can promote patterns"""
-        return self.tier.value >= AccessTier.VALIDATOR.value
-
-    def can_administer(self) -> bool:
-        """Only Stewards have full admin access"""
-        return self.tier.value >= AccessTier.STEWARD.value
-
-
-@dataclass
-class StagedPattern:
-    """Pattern awaiting validation"""
-
-    pattern_id: str
-    agent_id: str
-    pattern_type: str
-    name: str
-    description: str
-    code: str | None = None
-    context: dict = field(default_factory=dict)
-    confidence: float = 0.5
-    staged_at: datetime = field(default_factory=datetime.now)
-    interests: list[str] = field(default_factory=list)  # For negotiation
-
-    def __post_init__(self):
-        """Validate fields after initialization"""
-        # Pattern 1: String ID validation
-        if not self.pattern_id or not self.pattern_id.strip():
-            raise ValueError("pattern_id cannot be empty")
-        if not self.agent_id or not self.agent_id.strip():
-            raise ValueError("agent_id cannot be empty")
-        if not self.pattern_type or not self.pattern_type.strip():
-            raise ValueError("pattern_type cannot be empty")
-
-        # Pattern 4: Range validation for confidence
-        if not 0.0 <= self.confidence <= 1.0:
-            raise ValueError(f"confidence must be between 0.0 and 1.0, got {self.confidence}")
-
-        # Pattern 5: Type validation
-        if not isinstance(self.context, dict):
-            raise TypeError(f"context must be dict, got {type(self.context).__name__}")
-        if not isinstance(self.interests, list):
-            raise TypeError(f"interests must be list, got {type(self.interests).__name__}")
-
-    def to_dict(self) -> dict:
-        """Convert staged pattern to dictionary for serialization.
-
-        Returns:
-            Dictionary with keys: pattern_id, agent_id, pattern_type, name,
-            description, code, context, confidence, staged_at, interests.
-        """
-        return {
-            "pattern_id": self.pattern_id,
-            "agent_id": self.agent_id,
-            "pattern_type": self.pattern_type,
-            "name": self.name,
-            "description": self.description,
-            "code": self.code,
-            "context": self.context,
-            "confidence": self.confidence,
-            "staged_at": self.staged_at.isoformat(),
-            "interests": self.interests,
-        }
-
-    @classmethod
-    def from_dict(cls, data: dict) -> "StagedPattern":
-        """Reconstruct StagedPattern from dictionary.
-
-        Args:
-            data: Dictionary with required keys: pattern_id, agent_id,
-                pattern_type, name, description, staged_at.
-
-        Returns:
-            Reconstructed StagedPattern instance.
-
-        Raises:
-            KeyError: If required keys are missing.
-            ValueError: If data format is invalid.
-        """
-        return cls(
-            pattern_id=data["pattern_id"],
-            agent_id=data["agent_id"],
-            pattern_type=data["pattern_type"],
-            name=data["name"],
-            description=data["description"],
-            code=data.get("code"),
-            context=data.get("context", {}),
-            confidence=data.get("confidence", 0.5),
-            staged_at=datetime.fromisoformat(data["staged_at"]),
-            interests=data.get("interests", []),
-        )
-
-
-@dataclass
-class ConflictContext:
-    """Context for principled negotiation
-
-    Per Getting to Yes framework:
-    - Positions: What each party says they want
-    - Interests: Why they want it (underlying needs)
-    - BATNA: Best Alternative to Negotiated Agreement
-    """
-
-    conflict_id: str
-    positions: dict[str, Any]  # agent_id -> stated position
-    interests: dict[str, list[str]]  # agent_id -> underlying interests
-    batna: str | None = None  # Fallback strategy
-    created_at: datetime = field(default_factory=datetime.now)
-    resolved: bool = False
-    resolution: str | None = None
-
-    def to_dict(self) -> dict:
-        """Convert conflict context to dictionary for serialization.
-
-        Returns:
-            Dictionary with keys: conflict_id, positions, interests,
-            batna, created_at, resolved, resolution.
-        """
-        return {
-            "conflict_id": self.conflict_id,
-            "positions": self.positions,
-            "interests": self.interests,
-            "batna": self.batna,
-            "created_at": self.created_at.isoformat(),
-            "resolved": self.resolved,
-            "resolution": self.resolution,
-        }
-
-    @classmethod
-    def from_dict(cls, data: dict) -> "ConflictContext":
-        """Reconstruct ConflictContext from dictionary.
-
-        Args:
-            data: Dictionary with required keys: conflict_id, positions,
-                interests, created_at.
-
-        Returns:
-            Reconstructed ConflictContext instance.
-
-        Raises:
-            KeyError: If required keys are missing.
-            ValueError: If data format is invalid.
-        """
-        return cls(
-            conflict_id=data["conflict_id"],
-            positions=data["positions"],
-            interests=data["interests"],
-            batna=data.get("batna"),
-            created_at=datetime.fromisoformat(data["created_at"]),
-            resolved=data.get("resolved", False),
-            resolution=data.get("resolution"),
-        )
-
-
-class SecurityError(Exception):
-    """Raised when a security policy is violated (e.g., secrets detected in data)."""
-
-
 class RedisShortTermMemory:
     """Redis-backed short-term memory for agent coordination
 
@@ -554,6 +164,14 @@ class RedisShortTermMemory:
         self._mock_streams: dict[str, list[tuple[str, dict]]] = {}
         self._mock_pubsub_handlers: dict[str, list[Callable[[dict], None]]] = {}
 
+        # Local LRU cache for two-tier caching (memory + Redis)
+        # Reduces network I/O from 37ms to <0.001ms for frequently accessed keys
+        self._local_cache_enabled = self._config.local_cache_enabled
+        self._local_cache_max_size = self._config.local_cache_size
+        self._local_cache: dict[str, tuple[str, float, float]] = {}  # key -> (value, timestamp, last_access)
+        self._local_cache_hits = 0
+        self._local_cache_misses = 0
+
         # Security: Initialize PII scrubber and secrets detector
         self._pii_scrubber: PIIScrubber | None = None
         self._secrets_detector: SecretsDetector | None = None
@@ -652,43 +270,105 @@ class RedisShortTermMemory:
         raise last_error if last_error else ConnectionError("Redis operation failed")
 
     def _get(self, key: str) -> str | None:
-        """Get value from Redis or mock"""
+        """Get value from Redis or mock with two-tier caching (local + Redis)"""
+        # Check local cache first (0.001ms vs 37ms for Redis/mock)
+        # This works for BOTH mock and real Redis modes
+        if self._local_cache_enabled and key in self._local_cache:
+            value, timestamp, last_access = self._local_cache[key]
+            now = time.time()
+
+            # Update last access time for LRU
+            self._local_cache[key] = (value, timestamp, now)
+            self._local_cache_hits += 1
+
+            return value
+
+        # Cache miss - fetch from storage (mock or Redis)
+        self._local_cache_misses += 1
+
+        # Mock mode path
         if self.use_mock:
             if key in self._mock_storage:
                 value, expires = self._mock_storage[key]
                 if expires is None or datetime.now().timestamp() < expires:
-                    return str(value) if value is not None else None
+                    result = str(value) if value is not None else None
+                    # Add to local cache for next access
+                    if result and self._local_cache_enabled:
+                        self._add_to_local_cache(key, result)
+                    return result
                 del self._mock_storage[key]
             return None
+
+        # Real Redis path
         if self._client is None:
             return None
+
         result = self._client.get(key)
+
+        # Add to local cache if successful
+        if result and self._local_cache_enabled:
+            self._add_to_local_cache(key, str(result))
+
         return str(result) if result else None
 
     def _set(self, key: str, value: str, ttl: int | None = None) -> bool:
-        """Set value in Redis or mock"""
+        """Set value in Redis or mock with two-tier caching"""
+        # Mock mode path
         if self.use_mock:
             expires = datetime.now().timestamp() + ttl if ttl else None
             self._mock_storage[key] = (value, expires)
+
+            # Update local cache in mock mode too
+            if self._local_cache_enabled:
+                self._add_to_local_cache(key, value)
+
             return True
+
+        # Real Redis path
         if self._client is None:
             return False
+
+        # Set in Redis
         if ttl:
             self._client.setex(key, ttl, value)
-            return True
-        result = self._client.set(key, value)
-        return bool(result)
+        else:
+            result = self._client.set(key, value)
+            if not result:
+                return False
+
+        # Update local cache if enabled
+        if self._local_cache_enabled:
+            self._add_to_local_cache(key, value)
+
+        return True
 
     def _delete(self, key: str) -> bool:
-        """Delete key from Redis or mock"""
+        """Delete key from Redis or mock and local cache"""
+        # Mock mode path
         if self.use_mock:
+            deleted = False
             if key in self._mock_storage:
                 del self._mock_storage[key]
-                return True
-            return False
+                deleted = True
+
+            # Remove from local cache if present
+            if self._local_cache_enabled and key in self._local_cache:
+                del self._local_cache[key]
+
+            return deleted
+
+        # Real Redis path
         if self._client is None:
             return False
-        return bool(self._client.delete(key) > 0)
+
+        # Delete from Redis
+        result = bool(self._client.delete(key) > 0)
+
+        # Also remove from local cache if present
+        if self._local_cache_enabled and key in self._local_cache:
+            del self._local_cache[key]
+
+        return result
 
     def _keys(self, pattern: str) -> list[str]:
         """Get keys matching pattern"""
@@ -703,6 +383,58 @@ class RedisShortTermMemory:
         # Convert bytes to strings - needed for API return type
         return [k.decode() if isinstance(k, bytes) else str(k) for k in keys]
 
+    # === Local LRU Cache Methods ===
+
+    def _add_to_local_cache(self, key: str, value: str) -> None:
+        """Add entry to local cache with LRU eviction.
+
+        Args:
+            key: Cache key
+            value: Value to cache
+        """
+        now = time.time()
+
+        # Evict oldest entry if cache is full
+        if len(self._local_cache) >= self._local_cache_max_size:
+            # Find key with oldest last_access time
+            oldest_key = min(self._local_cache, key=lambda k: self._local_cache[k][2])
+            del self._local_cache[oldest_key]
+
+        # Add new entry: (value, timestamp, last_access)
+        self._local_cache[key] = (value, now, now)
+
+    def clear_local_cache(self) -> int:
+        """Clear all entries from local cache.
+
+        Returns:
+            Number of entries cleared
+        """
+        count = len(self._local_cache)
+        self._local_cache.clear()
+        self._local_cache_hits = 0
+        self._local_cache_misses = 0
+        logger.info("local_cache_cleared", entries_cleared=count)
+        return count
+
+    def get_local_cache_stats(self) -> dict:
+        """Get local cache performance statistics.
+
+        Returns:
+            Dict with cache stats (hits, misses, hit_rate, size)
+        """
+        total = self._local_cache_hits + self._local_cache_misses
+        hit_rate = (self._local_cache_hits / total * 100) if total > 0 else 0.0
+
+        return {
+            "enabled": self._local_cache_enabled,
+            "size": len(self._local_cache),
+            "max_size": self._local_cache_max_size,
+            "hits": self._local_cache_hits,
+            "misses": self._local_cache_misses,
+            "hit_rate": hit_rate,
+            "total_requests": total,
+        }
+
     # === Security Methods ===
 
     def _sanitize_data(self, data: Any) -> tuple[Any, int]: