hexdag 0.5.0.dev1__py3-none-any.whl

hexdag/builtin/adapters/memory/__init__.py

@@ -0,0 +1,47 @@
+"""Memory adapter implementations for HexDAG.
+
+This module provides various Memory port implementations:
+- InMemoryMemory: Fast in-memory dictionary storage
+- FileMemoryAdapter: File-based persistent storage
+- SQLiteMemoryAdapter: SQLite-backed persistent storage
+
+And memory plugins with structure-specific operations:
+- SessionMemoryPlugin: Conversation history and context
+- StateMemoryPlugin: Structured entities, relationships, and beliefs
+
+Schemas for memory plugins:
+- ConversationHistory: Session conversation schema
+- EntityState: Entity and relationship schema
+- BeliefState: Hinton-style belief probability schema
+- EventLog: Reasoning trace schema
+- ReasoningStep: Individual reasoning step schema
+"""
+
+from .file_memory_adapter import FileMemoryAdapter
+from .in_memory_memory import InMemoryMemory
+from .schemas import (
+    BeliefState,
+    ConversationHistory,
+    EntityState,
+    EventLog,
+    ReasoningStep,
+)
+from .session_memory import SessionMemoryPlugin
+from .sqlite_memory_adapter import SQLiteMemoryAdapter
+from .state_memory import StateMemoryPlugin
+
+__all__ = [
+    # Base memory adapters
+    "InMemoryMemory",
+    "FileMemoryAdapter",
+    "SQLiteMemoryAdapter",
+    # Memory plugins
+    "SessionMemoryPlugin",
+    "StateMemoryPlugin",
+    # Schemas
+    "BeliefState",
+    "ConversationHistory",
+    "EntityState",
+    "EventLog",
+    "ReasoningStep",
+]

hexdag/builtin/adapters/memory/file_memory_adapter.py

@@ -0,0 +1,297 @@
+"""File-based Memory adapter for JSON/YAML/pickle storage.
+
+This adapter provides file-based key-value storage, allowing Memory Port
+to work with various file formats.
+
+SECURITY WARNING: The pickle format can execute arbitrary code during
+deserialization. Only use pickle with trusted data sources. For untrusted
+data, use JSON or TEXT formats instead.
+"""
+
+import json
+import pickle  # nosec B403 - Pickle usage documented, user must choose format
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+
+from hexdag.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class FileFormat(StrEnum):
+    """Supported file formats for storage.
+
+    Security Notes
+    --------------
+    - JSON: Safe for untrusted data, human-readable
+    - TEXT: Safe for untrusted data, stores as plain text
+    - PICKLE: **UNSAFE for untrusted data** - can execute arbitrary code
+      Only use pickle with data you control or trust completely.
+    """
+
+    JSON = "json"
+    PICKLE = "pickle"
+    TEXT = "text"
+
+
+class FileMemoryAdapter:
+    """Memory adapter backed by file system.
+
+    Provides persistent key-value storage using files, with support for
+    multiple formats (JSON, pickle, text). Each key is stored as a separate
+    file in the specified directory.
+
+    This adapter is ideal for:
+    - Configuration files
+    - Pipeline definitions
+    - Human-readable checkpoints
+    - Data serialization
+
+    Parameters
+    ----------
+    base_path : str | Path
+        Base directory for file storage
+    format : FileFormat, default=FileFormat.JSON
+        File format to use (json, pickle, text)
+    create_dirs : bool, default=True
+        Automatically create directory structure
+    extension : str | None
+        Custom file extension (defaults to format)
+
+    Examples
+    --------
+    Example usage::
+
+        memory = FileMemoryAdapter(base_path="./data", format="json")
+        memory = FileMemoryAdapter(base_path="./cache", format="pickle")
+    """
+
+    def __init__(
+        self,
+        base_path: str | Path = "./memory_store",
+        format: FileFormat | str = FileFormat.JSON,
+        create_dirs: bool = True,
+        extension: str | None = None,
+    ) -> None:
+        """Initialize file memory adapter.
+
+        Parameters
+        ----------
+        base_path : str | Path
+            Base directory for file storage
+        format : FileFormat | str
+            File format (json, pickle, text)
+        create_dirs : bool
+            Automatically create directory if it doesn't exist
+        extension : str | None
+            Custom file extension (defaults to format name)
+        """
+        self.base_path = Path(base_path)
+        self.format = FileFormat(format) if isinstance(format, str) else format
+        self.create_dirs = create_dirs
+        self.extension = extension or self.format.value
+
+        if self.create_dirs:
+            self.base_path.mkdir(parents=True, exist_ok=True)
+            logger.debug(f"Initialized file storage at '{self.base_path}'")
+
+    def _get_file_path(self, key: str) -> Path:
+        """Get file path for a key.
+
+        Parameters
+        ----------
+        key : str
+            Storage key
+
+        Returns
+        -------
+        Path
+            Full file path for the key
+        """
+        # Sanitize key to be filesystem-safe
+        safe_key = key.replace("/", "_").replace(":", "_")
+        return self.base_path / f"{safe_key}.{self.extension}"
+
+    def _serialize(self, value: Any) -> bytes | str:
+        """Serialize value based on format.
+
+        Parameters
+        ----------
+        value : Any
+            Value to serialize
+
+        Returns
+        -------
+        bytes | str
+            Serialized value
+        """
+        if self.format == FileFormat.JSON:
+            return json.dumps(value, indent=2)
+        if self.format == FileFormat.PICKLE:
+            return pickle.dumps(value)
+        # TEXT
+        return str(value)
+
+    def _deserialize(self, data: bytes | str) -> Any:
+        """Deserialize value based on format.
+
+        Parameters
+        ----------
+        data : bytes | str
+            Serialized data
+
+        Returns
+        -------
+        Any
+            Deserialized value
+        """
+        if self.format == FileFormat.JSON:
+            # JSON only accepts str, not bytes
+            if isinstance(data, bytes):
+                data = data.decode("utf-8")
+            return json.loads(data)
+        if self.format == FileFormat.PICKLE:
+            # Pickle only accepts bytes
+            if isinstance(data, str):
+                data = data.encode("utf-8")
+            # WARNING: Pickle can execute arbitrary code - only use with trusted data
+            return pickle.loads(data)  # nosec B301 - User explicitly chose pickle format
+        # TEXT
+        return data
+
+    async def aget(self, key: str) -> Any:
+        """Retrieve a value from file storage.
+
+        Parameters
+        ----------
+        key : str
+            The key to retrieve
+
+        Returns
+        -------
+        Any
+            The stored value, or None if key doesn't exist
+        """
+        file_path = self._get_file_path(key)
+
+        if not file_path.exists():
+            return None
+
+        try:
+            data: bytes | str
+            if self.format == FileFormat.PICKLE:
+                data = file_path.read_bytes()
+            else:
+                data = file_path.read_text(encoding="utf-8")
+
+            return self._deserialize(data)
+        except Exception as e:
+            logger.error(f"Failed to read key '{key}' from {file_path}: {e}")
+            return None
+
+    async def aset(self, key: str, value: Any) -> None:
+        """Store a value in file storage.
+
+        Parameters
+        ----------
+        key : str
+            The key to store under
+        value : Any
+            The value to store
+        """
+        file_path = self._get_file_path(key)
+
+        # Ensure parent directory exists
+        if self.create_dirs:
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            serialized = self._serialize(value)
+
+            if self.format == FileFormat.PICKLE:
+                # Pickle always returns bytes
+                if not isinstance(serialized, bytes):
+                    raise TypeError(f"Expected bytes for pickle format, got {type(serialized)}")
+                file_path.write_bytes(serialized)
+            else:
+                # JSON and TEXT always return str
+                if not isinstance(serialized, str):
+                    raise TypeError(
+                        f"Expected str for {self.format} format, got {type(serialized)}"
+                    )
+                file_path.write_text(serialized, encoding="utf-8")
+
+            logger.debug(f"Stored key '{key}' at {file_path}")
+        except Exception as e:
+            logger.error(f"Failed to write key '{key}' to {file_path}: {e}")
+            raise
+
+    async def adelete(self, key: str) -> bool:
+        """Delete a key from file storage.
+
+        Parameters
+        ----------
+        key : str
+            The key to delete
+
+        Returns
+        -------
+        bool
+            True if key existed and was deleted, False otherwise
+        """
+        file_path = self._get_file_path(key)
+
+        if not file_path.exists():
+            return False
+
+        try:
+            file_path.unlink()
+            logger.debug(f"Deleted key '{key}' from {file_path}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete key '{key}' at {file_path}: {e}")
+            return False
+
+    async def alist_keys(self, prefix: str | None = None) -> list[str]:
+        """List all keys in storage, optionally filtered by prefix.
+
+        Parameters
+        ----------
+        prefix : str | None
+            Optional prefix to filter keys
+
+        Returns
+        -------
+        list[str]
+            List of matching keys
+        """
+        if not self.base_path.exists():
+            return []
+
+        keys = []
+        pattern = f"*.{self.extension}"
+
+        for file_path in self.base_path.glob(pattern):
+            # Remove extension and reverse sanitization
+            key = file_path.stem
+
+            if prefix is None or key.startswith(prefix):
+                keys.append(key)
+
+        return sorted(keys)
+
+    async def aclear(self) -> None:
+        """Clear all keys from storage."""
+        if not self.base_path.exists():
+            return
+
+        pattern = f"*.{self.extension}"
+        for file_path in self.base_path.glob(pattern):
+            file_path.unlink()
+
+        logger.info(f"Cleared all files from {self.base_path}")
+
+    def __repr__(self) -> str:
+        """Return string representation."""
+        return f"FileMemoryAdapter(path='{self.base_path}', format='{self.format.value}')"

hexdag/builtin/adapters/memory/in_memory_memory.py

@@ -0,0 +1,216 @@
+"""In-memory implementation of Memory for testing purposes."""
+
+import asyncio
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from hexdag.core.ports.healthcheck import HealthStatus
+
+
+from hexdag.core.ports.memory import Memory
+
+__all__ = ["InMemoryMemory"]
+
+
+class InMemoryMemory(Memory):
+    """In-memory implementation of Memory for testing.
+
+    Features:
+    - Key-value storage in memory
+    - Access history tracking
+    - Delay simulation
+    - Size limits
+    - Reset functionality
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize the in-memory storage.
+
+        Parameters
+        ----------
+        **kwargs : Any
+            Configuration options (max_size, delay_seconds)
+        """
+        self.max_size = kwargs.get("max_size", 1000)
+        self.delay_seconds = kwargs.get("delay_seconds", 0.0)
+
+        # State (not from config)
+        self.storage: dict[str, Any] = {}
+        self.access_history: list[dict[str, Any]] = []
+
+    async def aget(self, key: str) -> Any:
+        """Retrieve a value from memory.
+
+        Args
+        ----
+            key: The key to retrieve
+
+        Returns
+        -------
+            The stored value, or None if key doesn't exist
+        """
+        # Simulate access delay
+        if self.delay_seconds > 0:
+            await asyncio.sleep(self.delay_seconds)
+
+        result = self.storage.get(key)
+
+        # Log the access
+        self.access_history.append({
+            "operation": "get",
+            "key": key,
+            "found": key in self.storage,
+            "timestamp": asyncio.get_event_loop().time(),
+        })
+
+        return result
+
+    async def aset(self, key: str, value: Any) -> None:
+        """Store a value in memory.
+
+        Args
+        ----
+            key: The key to store under
+            value: The value to store
+
+        Raises
+        ------
+        MemoryError
+            If max_size is exceeded
+        """
+        # Simulate access delay
+        if self.delay_seconds > 0:
+            await asyncio.sleep(self.delay_seconds)
+
+        # Check size limit
+        if (
+            self.max_size is not None
+            and key not in self.storage
+            and len(self.storage) >= self.max_size
+        ):
+            raise MemoryError(f"Memory limit of {self.max_size} items exceeded")
+
+        self.storage[key] = value
+
+        # Log the access
+        self.access_history.append({
+            "operation": "set",
+            "key": key,
+            "value_type": type(value).__name__,
+            "timestamp": asyncio.get_event_loop().time(),
+        })
+
+    def clear(self) -> None:
+        """Clear all stored data."""
+        self.storage.clear()
+
+    def reset(self) -> None:
+        """Reset the memory to initial state, clearing both data and history."""
+        self.storage.clear()
+        self.access_history.clear()
+
+    def get_access_history(self) -> list[dict[str, Any]]:
+        """Get the history of all memory access operations.
+
+        Returns
+        -------
+        list[dict[str, Any]]
+            List of access history records with operation, key, and timestamp
+        """
+        return self.access_history.copy()
+
+    def get_stored_keys(self) -> list[str]:
+        """Get list of all stored keys.
+
+        Returns
+        -------
+        list[str]
+            List of all keys currently stored in memory
+        """
+        return list(self.storage.keys())
+
+    def has_key(self, key: str) -> bool:
+        """Check if a key exists in storage.
+
+        Parameters
+        ----------
+        key : str
+            The key to check for existence
+
+        Returns
+        -------
+        bool
+            True if the key exists, False otherwise
+        """
+        return key in self.storage
+
+    def size(self) -> int:
+        """Get the number of stored items.
+
+        Returns
+        -------
+        int
+            Number of items currently stored in memory
+        """
+        return len(self.storage)
+
+    async def ahealth_check(self) -> "HealthStatus":
+        """Check health of in-memory storage.
+
+        Returns
+        -------
+        HealthStatus
+            Health status indicating storage is operational
+
+        Examples
+        --------
+        Example usage::
+
+            memory = InMemoryMemory()
+            status = await memory.ahealth_check()
+        """
+        from hexdag.core.ports.healthcheck import HealthStatus
+
+        try:
+            # Test basic operations
+            test_key = "__health_check__"
+            await self.aset(test_key, "test")
+            value = await self.aget(test_key)
+
+            # Clean up test key
+            if test_key in self.storage:
+                del self.storage[test_key]
+
+            if value != "test":
+                return HealthStatus(
+                    status="unhealthy",
+                    adapter_name="InMemoryMemory",
+                    error=Exception("Storage read/write verification failed"),
+                )
+
+            details: dict[str, Any] = {"size": len(self.storage)}
+            if self.max_size is not None:
+                usage_percent = (len(self.storage) / self.max_size) * 100
+                details["max_size"] = self.max_size
+                details["usage_percent"] = round(usage_percent, 1)
+
+                if usage_percent > 90:
+                    return HealthStatus(
+                        status="degraded",
+                        adapter_name="InMemoryMemory",
+                        details=details,
+                    )
+
+            return HealthStatus(
+                status="healthy",
+                adapter_name="InMemoryMemory",
+                details=details,
+                latency_ms=0.1,  # In-memory is always fast
+            )
+
+        except Exception as e:
+            return HealthStatus(
+                status="unhealthy",
+                adapter_name="InMemoryMemory",
+                error=e,
+            )

hexdag/builtin/adapters/memory/schemas.py

@@ -0,0 +1,57 @@
+"""Pydantic schemas for memory plugins."""
+
+import time
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class ConversationHistory(BaseModel):
+    """Schema for conversation history in SessionMemoryPlugin."""
+
+    session_id: str
+    messages: list[dict[str, str]] = Field(default_factory=list)
+    timestamps: list[float] = Field(default_factory=list)
+    token_count: int = 0
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class EntityState(BaseModel):
+    """Schema for entity state in StateMemoryPlugin."""
+
+    entities: dict[str, dict[str, Any]] = Field(default_factory=dict)
+    relationships: list[tuple[str, str, str]] = Field(default_factory=list)
+    updated_at: float = Field(default_factory=time.time)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class BeliefState(BaseModel):
+    """Hinton-style belief state for StateMemoryPlugin.
+
+    Represents probability distribution over hypotheses with confidence scores.
+    """
+
+    beliefs: dict[str, float] = Field(default_factory=dict)
+    confidence: float = 0.0
+    evidence: list[str] = Field(default_factory=list)
+    updated_at: float = Field(default_factory=time.time)
+
+
+class ReasoningStep(BaseModel):
+    """Schema for reasoning step in EventMemoryPlugin."""
+
+    step_num: int
+    thought: str
+    tool_used: str | None = None
+    tool_result: Any = None
+    timestamp: float = Field(default_factory=time.time)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class EventLog(BaseModel):
+    """Schema for event log in EventMemoryPlugin."""
+
+    agent_id: str
+    events: list[ReasoningStep] = Field(default_factory=list)
+    created_at: float = Field(default_factory=time.time)
+    updated_at: float = Field(default_factory=time.time)