voxagent 0.1.0__py3-none-any.whl

voxagent/security/filter.py

@@ -0,0 +1,169 @@
+"""Redaction filters for voxagent."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from voxagent.security.events import SecurityEvent, SecurityEventEmitter
+from voxagent.security.registry import SecretRegistry
+
+if TYPE_CHECKING:
+    from voxagent.types.messages import Message
+
+
+class RedactionFilter:
+    """Filters secrets from text."""
+
+    def __init__(
+        self,
+        registry: SecretRegistry,
+        placeholder: str = "[REDACTED]",
+        event_emitter: SecurityEventEmitter | None = None,
+    ) -> None:
+        """Initialize a redaction filter.
+
+        Args:
+            registry: The secret registry to use for detection.
+            placeholder: The text to replace secrets with.
+            event_emitter: Optional event emitter for security events.
+        """
+        self.registry = registry
+        self.placeholder = placeholder
+        self.event_emitter = event_emitter
+
+    def redact(self, text: str) -> str:
+        """Redact all secrets from text.
+
+        Args:
+            text: The text to redact secrets from.
+
+        Returns:
+            Text with all secrets replaced by the placeholder.
+        """
+        secrets = self.registry.find_secrets(text)
+        if not secrets:
+            return text
+
+        # Emit events for each secret found
+        if self.event_emitter:
+            for match, start, end in secrets:
+                # Emit SECRET_REDACTED event
+                self.event_emitter.emit(
+                    SecurityEvent.SECRET_REDACTED,
+                    {"match": match, "start": start, "end": end},
+                )
+                # Emit PATTERN_MATCHED event
+                self.event_emitter.emit(
+                    SecurityEvent.PATTERN_MATCHED,
+                    {"match": match, "start": start, "end": end},
+                )
+
+        # Replace from end to start to preserve positions
+        result = text
+        for match, start, end in reversed(secrets):
+            result = result[:start] + self.placeholder + result[end:]
+
+        return result
+
+    def redact_message(self, message: Message) -> Message:
+        """Redact secrets from a message.
+
+        Args:
+            message: The message to redact secrets from.
+
+        Returns:
+            A new message with redacted content, or the original if unchanged.
+        """
+        from voxagent.types.messages import Message as MessageClass
+
+        # Handle None or non-string content
+        if message.content is None or not isinstance(message.content, str):
+            return message
+
+        redacted_content = self.redact(message.content)
+        if redacted_content == message.content:
+            return message
+
+        # Create new message with redacted content
+        return MessageClass(
+            role=message.role,
+            content=redacted_content,
+            tool_calls=message.tool_calls,
+        )
+
+
+class StreamFilter:
+    """Streaming-aware redaction with buffering."""
+
+    def __init__(
+        self,
+        registry: SecretRegistry,
+        buffer_size: int = 100,
+    ) -> None:
+        """Initialize a stream filter.
+
+        Args:
+            registry: The secret registry to use for detection.
+            buffer_size: Number of characters to buffer for partial match detection.
+        """
+        self.registry = registry
+        self.buffer_size = buffer_size
+        self._buffer: str = ""
+
+    def process_chunk(self, chunk: str) -> str:
+        """Process a streaming chunk, buffering for potential secrets.
+
+        Args:
+            chunk: The chunk of text to process.
+
+        Returns:
+            The safe portion of text that can be output.
+        """
+        self._buffer += chunk
+
+        # If buffer is smaller than buffer_size, hold everything
+        if len(self._buffer) <= self.buffer_size:
+            return ""
+
+        # Find safe output point (keep buffer_size chars for potential matches)
+        safe_end = len(self._buffer) - self.buffer_size
+        output = self._buffer[:safe_end]
+        self._buffer = self._buffer[safe_end:]
+
+        # Redact the output portion
+        return self._redact_text(output)
+
+    def flush(self) -> str:
+        """Flush remaining buffer.
+
+        Returns:
+            The remaining buffered text, redacted as needed.
+        """
+        output = self._buffer
+        self._buffer = ""
+
+        # Redact remaining buffer
+        return self._redact_text(output)
+
+    def _redact_text(self, text: str) -> str:
+        """Redact secrets from text.
+
+        Args:
+            text: The text to redact.
+
+        Returns:
+            The redacted text.
+        """
+        if not text:
+            return text
+
+        secrets = self.registry.find_secrets(text)
+        if not secrets:
+            return text
+
+        result = text
+        for match, start, end in reversed(secrets):
+            result = result[:start] + "[REDACTED]" + result[end:]
+
+        return result
+

voxagent/security/registry.py

@@ -0,0 +1,87 @@
+"""Secret registry for voxagent."""
+
+from __future__ import annotations
+
+import re
+import threading
+from typing import Pattern
+
+
+class SecretRegistry:
+    """Thread-safe registry for secret patterns and values."""
+
+    def __init__(self) -> None:
+        """Initialize an empty registry."""
+        self._patterns: list[Pattern[str]] = []
+        self._values: dict[str, str] = {}  # name -> value
+        self._lock = threading.Lock()
+
+    def register_pattern(self, pattern: str) -> None:
+        """Register a regex pattern to detect secrets.
+
+        Args:
+            pattern: A regex pattern string to match secrets.
+        """
+        with self._lock:
+            compiled = re.compile(pattern)
+            self._patterns.append(compiled)
+
+    def register_value(self, name: str, value: str) -> None:
+        """Register a specific value to redact.
+
+        Args:
+            name: A name/key for the secret.
+            value: The actual secret value to detect.
+        """
+        with self._lock:
+            self._values[name] = value
+
+    def contains_secret(self, text: str) -> bool:
+        """Check if text contains any registered secrets.
+
+        Args:
+            text: The text to check for secrets.
+
+        Returns:
+            True if any registered secret pattern or value is found.
+        """
+        with self._lock:
+            # Check patterns
+            for pattern in self._patterns:
+                if pattern.search(text):
+                    return True
+            # Check values
+            for value in self._values.values():
+                if value in text:
+                    return True
+            return False
+
+    def find_secrets(self, text: str) -> list[tuple[str, int, int]]:
+        """Find all secrets in text.
+
+        Args:
+            text: The text to search for secrets.
+
+        Returns:
+            List of (match, start, end) tuples for each found secret.
+        """
+        results: list[tuple[str, int, int]] = []
+        with self._lock:
+            # Find pattern matches
+            for pattern in self._patterns:
+                for match in pattern.finditer(text):
+                    results.append((match.group(), match.start(), match.end()))
+            # Find value matches
+            for value in self._values.values():
+                start = 0
+                while True:
+                    idx = text.find(value, start)
+                    if idx == -1:
+                        break
+                    results.append((value, idx, idx + len(value)))
+                    start = idx + 1
+
+        # Sort by position
+        results.sort(key=lambda x: x[1])
+        return results
+

voxagent/session/__init__.py

@@ -0,0 +1,39 @@
+"""Session management and persistence.
+
+This subpackage provides:
+- Session model with messages and metadata
+- File-based session storage (JSONL format)
+- Session locking for concurrent access
+- Context compaction and token management
+"""
+
+from voxagent.session.compaction import (
+    CompactionStrategy,
+    compact_context,
+    count_message_tokens,
+    count_tokens,
+    needs_compaction,
+)
+from voxagent.session.lock import LockTimeoutError, SessionLock, SessionLockManager
+from voxagent.session.model import Session, resolve_session_key
+from voxagent.session.storage import (
+    FileSessionStorage,
+    InMemorySessionStorage,
+    SessionStorage,
+)
+
+__all__ = [
+    "CompactionStrategy",
+    "FileSessionStorage",
+    "InMemorySessionStorage",
+    "LockTimeoutError",
+    "Session",
+    "SessionLock",
+    "SessionLockManager",
+    "SessionStorage",
+    "compact_context",
+    "count_message_tokens",
+    "count_tokens",
+    "needs_compaction",
+    "resolve_session_key",
+]

voxagent/session/compaction.py

@@ -0,0 +1,237 @@
+"""Context compaction for voxagent.
+
+This module provides functionality to manage token limits by:
+- Counting tokens in messages and text
+- Detecting when compaction is needed
+- Compacting context using various strategies
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+from voxagent.types.messages import Message, ToolResultBlock
+
+
+class CompactionStrategy(Enum):
+    """Available compaction strategies."""
+
+    REMOVE_TOOL_RESULTS = "remove_tool_results"
+    TRUNCATE_OLDEST = "truncate_oldest"
+    SUMMARIZE = "summarize"  # Requires LLM call (placeholder)
+
+
+def count_tokens(text: str, model: str = "gpt-4") -> int:
+    """Count tokens in text.
+
+    Uses tiktoken for OpenAI models, estimates for others.
+
+    Args:
+        text: The text to count tokens for.
+        model: The model name for tokenization.
+
+    Returns:
+        Number of tokens in the text.
+    """
+    if not text:
+        return 0
+
+    try:
+        import tiktoken
+
+        # Map model names to tiktoken encodings
+        if "gpt-4" in model or "gpt-3.5" in model:
+            try:
+                encoding = tiktoken.encoding_for_model(model)
+            except KeyError:
+                encoding = tiktoken.get_encoding("cl100k_base")
+        else:
+            # Default to cl100k_base for other models
+            encoding = tiktoken.get_encoding("cl100k_base")
+
+        return len(encoding.encode(text))
+    except ImportError:
+        # Fallback: estimate ~4 chars per token
+        return len(text) // 4
+
+
+def count_message_tokens(messages: list[Message], model: str = "gpt-4") -> int:
+    """Count total tokens in a list of messages.
+
+    Args:
+        messages: List of messages to count tokens for.
+        model: The model name for tokenization.
+
+    Returns:
+        Total token count across all messages.
+    """
+    if not messages:
+        return 0
+
+    total = 0
+    for msg in messages:
+        # Role overhead (tokens for role, separators, message framing, metadata)
+        # Include overhead for message structure in API calls
+        total += 20
+
+        # Content tokens
+        if isinstance(msg.content, str):
+            total += count_tokens(msg.content, model)
+        elif isinstance(msg.content, list):
+            # Content blocks
+            for block in msg.content:
+                if hasattr(block, "text"):
+                    total += count_tokens(block.text, model)
+                elif hasattr(block, "content"):
+                    total += count_tokens(block.content, model)
+                elif isinstance(block, dict):
+                    if "text" in block:
+                        total += count_tokens(block["text"], model)
+                    elif "content" in block:
+                        total += count_tokens(block["content"], model)
+
+        # Tool calls
+        if msg.tool_calls:
+            for tc in msg.tool_calls:
+                total += count_tokens(tc.name, model)
+                if isinstance(tc.params, str):
+                    total += count_tokens(tc.params, model)
+                elif isinstance(tc.params, dict):
+                    import json
+
+                    total += count_tokens(json.dumps(tc.params), model)
+
+    return total
+
+
+def needs_compaction(
+    messages: list[Message],
+    system_prompt: str,
+    max_tokens: int,
+    reserve_tokens: int = 0,
+    model: str = "gpt-4",
+) -> bool:
+    """Check if context needs compaction.
+
+    Args:
+        messages: List of messages in the conversation.
+        system_prompt: The system prompt text.
+        max_tokens: Maximum allowed tokens.
+        reserve_tokens: Tokens to reserve for response.
+        model: The model name for tokenization.
+
+    Returns:
+        True if current tokens > (max_tokens - reserve_tokens).
+    """
+    available = max_tokens - reserve_tokens
+
+    # Count system prompt tokens
+    current = count_tokens(system_prompt, model)
+
+    # Count message tokens
+    current += count_message_tokens(messages, model)
+
+    return current > available
+
+
+def _has_tool_result_content(msg: Message) -> bool:
+    """Check if a message contains tool result content blocks."""
+    if not isinstance(msg.content, list):
+        return False
+    return any(isinstance(block, ToolResultBlock) for block in msg.content)
+
+
+def compact_context(
+    messages: list[Message],
+    target_tokens: int,
+    preserve_recent: int = 4,
+    model: str = "gpt-4",
+    aggressive: bool = False,
+) -> list[Message]:
+    """Compact messages to fit within target token count.
+
+    Strategies (in order):
+    1. Remove messages with tool result content blocks
+    2. Truncate from oldest messages
+    3. If aggressive: more aggressive truncation
+
+    Always preserves the most recent `preserve_recent` turns.
+
+    Args:
+        messages: List of messages to compact.
+        target_tokens: Target maximum token count.
+        preserve_recent: Number of recent messages to preserve.
+        model: The model name for tokenization.
+        aggressive: Whether to use aggressive compaction.
+
+    Returns:
+        Compacted list of messages.
+    """
+    if not messages:
+        return []
+
+    # Handle negative or zero target with preserve_recent=0
+    if target_tokens <= 0 and preserve_recent == 0:
+        return []
+
+    # Check if already under target
+    current_tokens = count_message_tokens(messages, model)
+    if current_tokens <= target_tokens:
+        return list(messages)
+
+    result = list(messages)
+
+    # Strategy 1: Remove messages with tool result content blocks from outside preserve window
+    if len(result) > preserve_recent:
+        if preserve_recent > 0:
+            compactable = result[:-preserve_recent]
+            preserved = result[-preserve_recent:]
+        else:
+            compactable = result
+            preserved = []
+
+        # Remove messages with tool result content blocks from compactable portion
+        compactable = [m for m in compactable if not _has_tool_result_content(m)]
+        result = compactable + preserved
+
+        current_tokens = count_message_tokens(result, model)
+        if current_tokens <= target_tokens:
+            return result
+
+    # If still over target, also remove tool results from preserved portion
+    if count_message_tokens(result, model) > target_tokens:
+        result = [m for m in result if not _has_tool_result_content(m)]
+
+        current_tokens = count_message_tokens(result, model)
+        if current_tokens <= target_tokens:
+            return result
+
+    # Strategy 2: Truncate from oldest messages (respecting preserve_recent)
+    while len(result) > preserve_recent and count_message_tokens(result, model) > target_tokens:
+        # Skip system messages at the beginning
+        if result[0].role == "system":
+            if len(result) > 1:
+                result = [result[0]] + result[2:]
+            else:
+                break
+        else:
+            result = result[1:]  # Remove oldest message
+
+    # Strategy 3: If aggressive and still over, truncate more aggressively
+    if aggressive:
+        while len(result) > 1 and count_message_tokens(result, model) > target_tokens:
+            # In aggressive mode, we can remove from preserved messages too
+            # But keep at least one message
+            if result[0].role == "system":
+                if len(result) > 1:
+                    result = [result[0]] + result[2:]
+                else:
+                    break
+            else:
+                result = result[1:]
+
+        # If still over and only one message left, keep it (graceful handling)
+
+    return result
+

voxagent/session/lock.py

@@ -0,0 +1,103 @@
+"""Session locking for voxagent."""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+from typing import Any
+
+
+class LockTimeoutError(Exception):
+    """Raised when lock acquisition times out."""
+
+    def __init__(self, session_key: str, timeout: float) -> None:
+        self.session_key = session_key
+        self.timeout = timeout
+        super().__init__(f"Lock timeout for session {session_key} after {timeout}s")
+
+
+# Module-level lock registry keyed by (event_loop_id, session_key)
+_lock_registry: dict[tuple[int, str], asyncio.Lock] = {}
+_registry_thread_lock = threading.Lock()
+
+
+def _get_lock_for_session(session_key: str) -> asyncio.Lock:
+    """Get or create the asyncio.Lock for a session key in the current event loop."""
+    loop = asyncio.get_running_loop()
+    key = (id(loop), session_key)
+
+    with _registry_thread_lock:
+        if key not in _lock_registry:
+            _lock_registry[key] = asyncio.Lock()
+        return _lock_registry[key]
+
+
+class SessionLock:
+    """Async lock for session access."""
+
+    def __init__(self, session_key: str, timeout: float = 30.0) -> None:
+        self.session_key = session_key
+        self.timeout = timeout
+        self._acquired: bool = False
+
+    async def acquire(self) -> bool:
+        """Acquire the lock. Returns True if acquired, raises LockTimeoutError on timeout."""
+        lock = _get_lock_for_session(self.session_key)
+
+        try:
+            await asyncio.wait_for(lock.acquire(), timeout=self.timeout)
+            self._acquired = True
+            return True
+        except asyncio.TimeoutError:
+            raise LockTimeoutError(self.session_key, self.timeout)
+
+    async def release(self) -> None:
+        """Release the lock."""
+        if self._acquired:
+            lock = _get_lock_for_session(self.session_key)
+            try:
+                lock.release()
+            except RuntimeError:
+                # Lock was not held
+                pass
+            self._acquired = False
+
+    async def __aenter__(self) -> "SessionLock":
+        """Async context manager entry."""
+        await self.acquire()
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Async context manager exit."""
+        await self.release()
+
+    @property
+    def is_locked(self) -> bool:
+        """Check if lock is currently held by this instance."""
+        return self._acquired
+
+
+class SessionLockManager:
+    """Manages locks for multiple sessions."""
+
+    def __init__(self, timeout: float = 30.0) -> None:
+        self._locks: dict[str, SessionLock] = {}
+        self._timeout = timeout
+
+    def get_lock(self, session_key: str) -> SessionLock:
+        """Get or create a lock for a session."""
+        if session_key not in self._locks:
+            self._locks[session_key] = SessionLock(session_key, self._timeout)
+        return self._locks[session_key]
+
+    async def acquire(self, session_key: str) -> SessionLock:
+        """Acquire lock for a session."""
+        lock = self.get_lock(session_key)
+        await lock.acquire()
+        return lock
+
+    async def release(self, session_key: str) -> None:
+        """Release lock for a session."""
+        if session_key in self._locks:
+            await self._locks[session_key].release()
+