attune-ai 2.1.5__py3-none-any.whl → 2.2.0__py3-none-any.whl

attune/memory/short_term/pagination.py

@@ -0,0 +1,215 @@
+"""SCAN-based pagination for large key sets.
+
+This module provides cursor-based pagination using Redis SCAN:
+- Paginated pattern listing
+- Generic key scanning with filters
+
+Benefits:
+- Memory-efficient for large datasets
+- Non-blocking (unlike KEYS command)
+- Cursor-based for consistent iteration
+
+Classes:
+    Pagination: SCAN-based pagination operations
+
+Example:
+    >>> from attune.memory.short_term.pagination import Pagination
+    >>> from attune.memory.types import AgentCredentials, AccessTier
+    >>> pagination = Pagination(base_ops)
+    >>> creds = AgentCredentials("agent_1", AccessTier.CONTRIBUTOR)
+    >>> result = pagination.list_staged_patterns_paginated(creds, "0", 10)
+    >>> for pattern in result.items:
+    ...     print(pattern.name)
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+import structlog
+
+from attune.memory.types import (
+    AgentCredentials,
+    PaginatedResult,
+    StagedPattern,
+)
+
+if TYPE_CHECKING:
+    from attune.memory.short_term.base import BaseOperations
+
+logger = structlog.get_logger(__name__)
+
+
+class Pagination:
+    """SCAN-based pagination operations.
+
+    Provides memory-efficient pagination using Redis SCAN command
+    instead of the blocking KEYS command. Suitable for large datasets.
+
+    The class is designed to be composed with BaseOperations
+    for dependency injection.
+
+    Attributes:
+        PREFIX_STAGED: Key prefix for staged patterns namespace
+
+    Example:
+        >>> pagination = Pagination(base_ops)
+        >>> result = pagination.list_staged_patterns_paginated(creds, "0", 10)
+        >>> while result.has_more:
+        ...     for pattern in result.items:
+        ...         process(pattern)
+        ...     result = pagination.list_staged_patterns_paginated(creds, result.cursor, 10)
+    """
+
+    PREFIX_STAGED = "empathy:staged:"
+
+    def __init__(self, base: BaseOperations) -> None:
+        """Initialize pagination operations.
+
+        Args:
+            base: BaseOperations instance for storage access
+        """
+        self._base = base
+
+    def list_staged_patterns_paginated(
+        self,
+        credentials: AgentCredentials,
+        cursor: str = "0",
+        count: int = 100,
+    ) -> PaginatedResult:
+        """List staged patterns with pagination using SCAN.
+
+        More efficient than list_staged_patterns() for large datasets.
+
+        Args:
+            credentials: Agent credentials
+            cursor: Pagination cursor (start with "0")
+            count: Maximum items per page
+
+        Returns:
+            PaginatedResult with items, cursor, and has_more flag
+
+        Example:
+            >>> result = pagination.list_staged_patterns_paginated(creds, "0", 10)
+            >>> for pattern in result.items:
+            ...     print(pattern.name)
+            >>> if result.has_more:
+            ...     next_result = pagination.list_staged_patterns_paginated(
+            ...         creds, result.cursor, 10
+            ...     )
+        """
+        start_time = time.perf_counter()
+        pattern = f"{self.PREFIX_STAGED}*"
+
+        # Handle mock storage mode
+        if self._base.use_mock:
+            import fnmatch
+
+            all_keys = [
+                k for k in self._base._mock_storage.keys() if fnmatch.fnmatch(k, pattern)
+            ]
+            start_idx = int(cursor)
+            end_idx = start_idx + count
+            page_keys = all_keys[start_idx:end_idx]
+
+            patterns = []
+            for key in page_keys:
+                raw_value, expires = self._base._mock_storage[key]
+                if expires is None or datetime.now().timestamp() < expires:
+                    patterns.append(StagedPattern.from_dict(json.loads(str(raw_value))))
+
+            new_cursor = str(end_idx) if end_idx < len(all_keys) else "0"
+            has_more = end_idx < len(all_keys)
+
+            latency_ms = (time.perf_counter() - start_time) * 1000
+            self._base._metrics.record_operation("list_paginated", latency_ms)
+
+            return PaginatedResult(
+                items=patterns,
+                cursor=new_cursor,
+                has_more=has_more,
+                total_scanned=len(page_keys),
+            )
+
+        # Handle real Redis client
+        if self._base._client is None:
+            return PaginatedResult(items=[], cursor="0", has_more=False)
+
+        # Use SCAN for efficient iteration
+        new_cursor, keys = self._base._client.scan(
+            cursor=int(cursor), match=pattern, count=count
+        )
+
+        patterns = []
+        for key in keys:
+            raw = self._base._client.get(key)
+            if raw:
+                patterns.append(StagedPattern.from_dict(json.loads(raw)))
+
+        has_more = new_cursor != 0
+
+        latency_ms = (time.perf_counter() - start_time) * 1000
+        self._base._metrics.record_operation("list_paginated", latency_ms)
+
+        return PaginatedResult(
+            items=patterns,
+            cursor=str(new_cursor),
+            has_more=has_more,
+            total_scanned=len(keys),
+        )
+
+    def scan_keys(
+        self,
+        pattern: str,
+        cursor: str = "0",
+        count: int = 100,
+    ) -> PaginatedResult:
+        """Scan keys matching a pattern with pagination.
+
+        Generic key scanning that can be used for any key namespace.
+
+        Args:
+            pattern: Key pattern (e.g., "empathy:working:*")
+            cursor: Pagination cursor
+            count: Items per page
+
+        Returns:
+            PaginatedResult with key strings
+
+        Example:
+            >>> result = pagination.scan_keys("empathy:session:*", "0", 50)
+            >>> for key in result.items:
+            ...     print(key)
+        """
+        # Handle mock storage mode
+        if self._base.use_mock:
+            import fnmatch
+
+            all_keys = [
+                k for k in self._base._mock_storage.keys() if fnmatch.fnmatch(k, pattern)
+            ]
+            start_idx = int(cursor)
+            end_idx = start_idx + count
+            page_keys = all_keys[start_idx:end_idx]
+            new_cursor = str(end_idx) if end_idx < len(all_keys) else "0"
+            has_more = end_idx < len(all_keys)
+            return PaginatedResult(items=page_keys, cursor=new_cursor, has_more=has_more)
+
+        # Handle real Redis client
+        if self._base._client is None:
+            return PaginatedResult(items=[], cursor="0", has_more=False)
+
+        new_cursor, keys = self._base._client.scan(
+            cursor=int(cursor), match=pattern, count=count
+        )
+        return PaginatedResult(
+            items=[str(k) for k in keys],
+            cursor=str(new_cursor),
+            has_more=new_cursor != 0,
+        )

attune/memory/short_term/patterns.py

@@ -0,0 +1,271 @@
+"""Pattern staging workflow - stage, validate, promote/reject.
+
+This module provides the pattern staging lifecycle:
+- Stage: Store patterns for validation (CONTRIBUTOR+)
+- Get/List: Retrieve staged patterns (any tier)
+- Promote: Move pattern to active library (VALIDATOR+)
+- Reject: Remove pattern from staging (VALIDATOR+)
+
+Key Prefix: PREFIX_STAGED = "empathy:staged:"
+
+Classes:
+    PatternStaging: Pattern staging lifecycle operations
+
+Example:
+    >>> from attune.memory.short_term.patterns import PatternStaging
+    >>> from attune.memory.types import AgentCredentials, AccessTier, StagedPattern
+    >>> staging = PatternStaging(base_ops)
+    >>> creds = AgentCredentials("agent_1", AccessTier.CONTRIBUTOR)
+    >>> pattern = StagedPattern(pattern_id="p1", name="Test", ...)
+    >>> staging.stage_pattern(pattern, creds)
+    >>> staged = staging.list_staged_patterns(creds)
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+import structlog
+
+from attune.memory.types import (
+    AgentCredentials,
+    StagedPattern,
+    TTLStrategy,
+)
+
+if TYPE_CHECKING:
+    from attune.memory.short_term.base import BaseOperations
+
+logger = structlog.get_logger(__name__)
+
+
+class PatternStaging:
+    """Pattern staging lifecycle operations.
+
+    Implements the pattern validation workflow per EMPATHY_PHILOSOPHY.md:
+    - Patterns must be staged before being promoted to active library
+    - CONTRIBUTOR tier can stage patterns
+    - VALIDATOR tier can promote or reject patterns
+
+    The class is designed to be composed with BaseOperations
+    for dependency injection.
+
+    Attributes:
+        PREFIX_STAGED: Key prefix for staged patterns namespace
+
+    Example:
+        >>> staging = PatternStaging(base_ops)
+        >>> creds = AgentCredentials("agent_1", AccessTier.CONTRIBUTOR)
+        >>> staging.stage_pattern(pattern, creds)
+        True
+        >>> staging.list_staged_patterns(creds)
+        [StagedPattern(...)]
+    """
+
+    PREFIX_STAGED = "empathy:staged:"
+
+    def __init__(self, base: BaseOperations) -> None:
+        """Initialize pattern staging operations.
+
+        Args:
+            base: BaseOperations instance for storage access
+        """
+        self._base = base
+
+    def stage_pattern(
+        self,
+        pattern: StagedPattern,
+        credentials: AgentCredentials,
+    ) -> bool:
+        """Stage a pattern for validation.
+
+        Per EMPATHY_PHILOSOPHY.md: Patterns must be staged before
+        being promoted to the active library.
+
+        Args:
+            pattern: Pattern to stage
+            credentials: Must be CONTRIBUTOR or higher
+
+        Returns:
+            True if staged successfully
+
+        Raises:
+            TypeError: If pattern is not StagedPattern
+            PermissionError: If credentials lack staging access
+
+        Example:
+            >>> pattern = StagedPattern(pattern_id="p1", name="Test", ...)
+            >>> staging.stage_pattern(pattern, creds)
+            True
+        """
+        # Pattern 5: Type validation
+        if not isinstance(pattern, StagedPattern):
+            raise TypeError(f"pattern must be StagedPattern, got {type(pattern).__name__}")
+
+        if not credentials.can_stage():
+            raise PermissionError(
+                f"Agent {credentials.agent_id} cannot stage patterns. "
+                "Requires CONTRIBUTOR tier or higher.",
+            )
+
+        key = f"{self.PREFIX_STAGED}{pattern.pattern_id}"
+        return self._base._set(
+            key,
+            json.dumps(pattern.to_dict()),
+            TTLStrategy.STAGED_PATTERNS.value,
+        )
+
+    def get_staged_pattern(
+        self,
+        pattern_id: str,
+        credentials: AgentCredentials,
+    ) -> StagedPattern | None:
+        """Retrieve a staged pattern.
+
+        Args:
+            pattern_id: Pattern ID
+            credentials: Any tier can read
+
+        Returns:
+            StagedPattern or None if not found
+
+        Raises:
+            ValueError: If pattern_id is empty
+
+        Example:
+            >>> pattern = staging.get_staged_pattern("p1", creds)
+            >>> if pattern:
+            ...     print(f"Found: {pattern.name}")
+        """
+        # Pattern 1: String ID validation
+        if not pattern_id or not pattern_id.strip():
+            raise ValueError(f"pattern_id cannot be empty. Got: {pattern_id!r}")
+
+        key = f"{self.PREFIX_STAGED}{pattern_id}"
+        raw = self._base._get(key)
+
+        if raw is None:
+            return None
+
+        return StagedPattern.from_dict(json.loads(raw))
+
+    def list_staged_patterns(
+        self,
+        credentials: AgentCredentials,
+    ) -> list[StagedPattern]:
+        """List all staged patterns awaiting validation.
+
+        Args:
+            credentials: Any tier can read
+
+        Returns:
+            List of staged patterns
+
+        Example:
+            >>> patterns = staging.list_staged_patterns(creds)
+            >>> for p in patterns:
+            ...     print(f"{p.pattern_id}: {p.name}")
+        """
+        pattern = f"{self.PREFIX_STAGED}*"
+        keys = self._base._keys(pattern)
+        patterns = []
+
+        for key in keys:
+            raw = self._base._get(key)
+            if raw:
+                patterns.append(StagedPattern.from_dict(json.loads(raw)))
+
+        return patterns
+
+    def promote_pattern(
+        self,
+        pattern_id: str,
+        credentials: AgentCredentials,
+    ) -> StagedPattern | None:
+        """Promote staged pattern (remove from staging for library add).
+
+        Args:
+            pattern_id: Pattern to promote
+            credentials: Must be VALIDATOR or higher
+
+        Returns:
+            The promoted pattern (for adding to PatternLibrary)
+
+        Raises:
+            PermissionError: If credentials lack validation access
+
+        Example:
+            >>> pattern = staging.promote_pattern("p1", validator_creds)
+            >>> if pattern:
+            ...     pattern_library.add(pattern)
+        """
+        if not credentials.can_validate():
+            raise PermissionError(
+                f"Agent {credentials.agent_id} cannot promote patterns. "
+                "Requires VALIDATOR tier or higher.",
+            )
+
+        pattern = self.get_staged_pattern(pattern_id, credentials)
+        if pattern:
+            key = f"{self.PREFIX_STAGED}{pattern_id}"
+            self._base._delete(key)
+        return pattern
+
+    def reject_pattern(
+        self,
+        pattern_id: str,
+        credentials: AgentCredentials,
+        reason: str = "",
+    ) -> bool:
+        """Reject a staged pattern.
+
+        Args:
+            pattern_id: Pattern to reject
+            credentials: Must be VALIDATOR or higher
+            reason: Rejection reason (for audit)
+
+        Returns:
+            True if rejected
+
+        Raises:
+            PermissionError: If credentials lack validation access
+
+        Example:
+            >>> staging.reject_pattern("p1", validator_creds, "Not applicable")
+            True
+        """
+        if not credentials.can_validate():
+            raise PermissionError(
+                f"Agent {credentials.agent_id} cannot reject patterns. "
+                "Requires VALIDATOR tier or higher.",
+            )
+
+        key = f"{self.PREFIX_STAGED}{pattern_id}"
+        deleted = self._base._delete(key)
+
+        if deleted and reason:
+            logger.info(
+                "pattern_rejected",
+                pattern_id=pattern_id,
+                agent_id=credentials.agent_id,
+                reason=reason,
+            )
+
+        return deleted
+
+    def count_staged(self) -> int:
+        """Count the number of staged patterns.
+
+        Returns:
+            Number of staged patterns
+
+        Example:
+            >>> count = staging.count_staged()
+            >>> print(f"{count} patterns awaiting validation")
+        """
+        pattern = f"{self.PREFIX_STAGED}*"
+        return len(self._base._keys(pattern))