alma-memory 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

alma/storage/constants.py

@@ -0,0 +1,103 @@
+"""
+ALMA Storage Constants.
+
+Canonical naming conventions for memory types across all storage backends.
+This ensures consistency for:
+- Data migration between backends
+- Backend-agnostic code
+- Documentation consistency
+"""
+
+from typing import Dict
+
+
+class MemoryType:
+    """
+    Canonical memory type identifiers.
+
+    These are the internal names used consistently across all backends.
+    Each backend may add a prefix or transform these for their specific
+    storage format, but the canonical names remain constant.
+    """
+
+    HEURISTICS = "heuristics"
+    OUTCOMES = "outcomes"
+    PREFERENCES = "preferences"
+    DOMAIN_KNOWLEDGE = "domain_knowledge"
+    ANTI_PATTERNS = "anti_patterns"
+
+    # All memory types as a tuple for iteration
+    ALL = (
+        HEURISTICS,
+        OUTCOMES,
+        PREFERENCES,
+        DOMAIN_KNOWLEDGE,
+        ANTI_PATTERNS,
+    )
+
+    # Memory types that support embeddings/vector search
+    VECTOR_ENABLED = (
+        HEURISTICS,
+        OUTCOMES,
+        DOMAIN_KNOWLEDGE,
+        ANTI_PATTERNS,
+    )
+
+
+def get_table_name(memory_type: str, prefix: str = "") -> str:
+    """
+    Get the table/container name for a memory type.
+
+    Args:
+        memory_type: One of the MemoryType constants
+        prefix: Optional prefix to add (e.g., "alma_" for PostgreSQL)
+
+    Returns:
+        The formatted table/container name
+
+    Example:
+        >>> get_table_name(MemoryType.HEURISTICS, "alma_")
+        'alma_heuristics'
+        >>> get_table_name(MemoryType.DOMAIN_KNOWLEDGE)
+        'domain_knowledge'
+    """
+    if memory_type not in MemoryType.ALL:
+        raise ValueError(f"Unknown memory type: {memory_type}")
+    return f"{prefix}{memory_type}"
+
+
+def get_table_names(prefix: str = "") -> Dict[str, str]:
+    """
+    Get all table/container names with an optional prefix.
+
+    Args:
+        prefix: Optional prefix to add (e.g., "alma_" for PostgreSQL)
+
+    Returns:
+        Dict mapping canonical memory type to table/container name
+
+    Example:
+        >>> get_table_names("alma_")
+        {
+            'heuristics': 'alma_heuristics',
+            'outcomes': 'alma_outcomes',
+            'preferences': 'alma_preferences',
+            'domain_knowledge': 'alma_domain_knowledge',
+            'anti_patterns': 'alma_anti_patterns',
+        }
+    """
+    return {mt: get_table_name(mt, prefix) for mt in MemoryType.ALL}
+
+
+# Pre-computed table name mappings for each backend
+# These are the canonical mappings that should be used
+
+# PostgreSQL uses alma_ prefix with underscores
+POSTGRESQL_TABLE_NAMES = get_table_names("alma_")
+
+# SQLite uses no prefix (local file-based, no collision risk)
+SQLITE_TABLE_NAMES = get_table_names("")
+
+# Azure Cosmos uses alma_ prefix with underscores (standardized)
+# Note: Previously used hyphens, now standardized to underscores
+AZURE_COSMOS_CONTAINER_NAMES = get_table_names("alma_")

alma/storage/file_based.py

@@ -6,21 +6,20 @@ No vector search - uses basic text matching for retrieval.
 """
 
 import json
-import uuid
 import logging
-from pathlib import Path
 from datetime import datetime, timezone
-from typing import Optional, List, Dict, Any
-from dataclasses import asdict
+from pathlib import Path
+from typing import Any, Dict, List, Optional
 
+from alma.storage.base import StorageBackend
+from alma.storage.constants import MemoryType
 from alma.types import (
+    AntiPattern,
+    DomainKnowledge,
     Heuristic,
     Outcome,
     UserPreference,
-    DomainKnowledge,
-    AntiPattern,
 )
-from alma.storage.base import StorageBackend
 
 logger = logging.getLogger(__name__)
 
@@ -51,14 +50,8 @@ class FileBasedStorage(StorageBackend):
         self.storage_dir = Path(storage_dir)
         self.storage_dir.mkdir(parents=True, exist_ok=True)
 
-        # File paths
-        self._files = {
-            "heuristics": self.storage_dir / "heuristics.json",
-            "outcomes": self.storage_dir / "outcomes.json",
-            "preferences": self.storage_dir / "preferences.json",
-            "domain_knowledge": self.storage_dir / "domain_knowledge.json",
-            "anti_patterns": self.storage_dir / "anti_patterns.json",
-        }
+        # File paths (using canonical memory type names)
+        self._files = {mt: self.storage_dir / f"{mt}.json" for mt in MemoryType.ALL}
 
         # Initialize empty files if they don't exist
         for file_path in self._files.values():
@@ -74,46 +67,86 @@ class FileBasedStorage(StorageBackend):
     # ==================== WRITE OPERATIONS ====================
 
     def save_heuristic(self, heuristic: Heuristic) -> str:
-        """Save a heuristic."""
+        """Save a heuristic (UPSERT - update if exists, insert if new)."""
         data = self._read_json(self._files["heuristics"])
         record = self._to_dict(heuristic)
-        data.append(record)
+        # Find and replace existing, or append new
+        found = False
+        for i, existing in enumerate(data):
+            if existing.get("id") == record["id"]:
+                data[i] = record
+                found = True
+                break
+        if not found:
+            data.append(record)
         self._write_json(self._files["heuristics"], data)
         logger.debug(f"Saved heuristic: {heuristic.id}")
         return heuristic.id
 
     def save_outcome(self, outcome: Outcome) -> str:
-        """Save an outcome."""
+        """Save an outcome (UPSERT - update if exists, insert if new)."""
         data = self._read_json(self._files["outcomes"])
         record = self._to_dict(outcome)
-        data.append(record)
+        # Find and replace existing, or append new
+        found = False
+        for i, existing in enumerate(data):
+            if existing.get("id") == record["id"]:
+                data[i] = record
+                found = True
+                break
+        if not found:
+            data.append(record)
         self._write_json(self._files["outcomes"], data)
         logger.debug(f"Saved outcome: {outcome.id}")
         return outcome.id
 
     def save_user_preference(self, preference: UserPreference) -> str:
-        """Save a user preference."""
+        """Save a user preference (UPSERT - update if exists, insert if new)."""
         data = self._read_json(self._files["preferences"])
         record = self._to_dict(preference)
-        data.append(record)
+        # Find and replace existing, or append new
+        found = False
+        for i, existing in enumerate(data):
+            if existing.get("id") == record["id"]:
+                data[i] = record
+                found = True
+                break
+        if not found:
+            data.append(record)
         self._write_json(self._files["preferences"], data)
         logger.debug(f"Saved preference: {preference.id}")
         return preference.id
 
     def save_domain_knowledge(self, knowledge: DomainKnowledge) -> str:
-        """Save domain knowledge."""
+        """Save domain knowledge (UPSERT - update if exists, insert if new)."""
         data = self._read_json(self._files["domain_knowledge"])
         record = self._to_dict(knowledge)
-        data.append(record)
+        # Find and replace existing, or append new
+        found = False
+        for i, existing in enumerate(data):
+            if existing.get("id") == record["id"]:
+                data[i] = record
+                found = True
+                break
+        if not found:
+            data.append(record)
         self._write_json(self._files["domain_knowledge"], data)
         logger.debug(f"Saved domain knowledge: {knowledge.id}")
         return knowledge.id
 
     def save_anti_pattern(self, anti_pattern: AntiPattern) -> str:
-        """Save an anti-pattern."""
+        """Save an anti-pattern (UPSERT - update if exists, insert if new)."""
         data = self._read_json(self._files["anti_patterns"])
         record = self._to_dict(anti_pattern)
-        data.append(record)
+        # Find and replace existing, or append new
+        found = False
+        for i, existing in enumerate(data):
+            if existing.get("id") == record["id"]:
+                data[i] = record
+                found = True
+                break
+        if not found:
+            data.append(record)
         self._write_json(self._files["anti_patterns"], data)
         logger.debug(f"Saved anti-pattern: {anti_pattern.id}")
         return anti_pattern.id
@@ -451,9 +484,7 @@ class FileBasedStorage(StorageBackend):
                         count += 1
             stats[f"{name}_count"] = count
 
-        stats["total_count"] = sum(
-            stats[k] for k in stats if k.endswith("_count")
-        )
+        stats["total_count"] = sum(stats[k] for k in stats if k.endswith("_count"))
 
         return stats
 

alma/storage/migrations/__init__.py

@@ -0,0 +1,21 @@
+"""
+ALMA Schema Migration Framework.
+
+Provides version tracking and migration capabilities for storage backends.
+"""
+
+from alma.storage.migrations.base import (
+    Migration,
+    MigrationError,
+    MigrationRegistry,
+    SchemaVersion,
+)
+from alma.storage.migrations.runner import MigrationRunner
+
+__all__ = [
+    "Migration",
+    "MigrationError",
+    "MigrationRegistry",
+    "MigrationRunner",
+    "SchemaVersion",
+]

alma/storage/migrations/base.py

@@ -0,0 +1,321 @@
+"""
+ALMA Migration Framework - Base Classes.
+
+Provides abstract migration classes and version tracking utilities.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, List, Optional, Type
+
+logger = logging.getLogger(__name__)
+
+
+class MigrationError(Exception):
+    """Exception raised when a migration fails."""
+
+    def __init__(
+        self,
+        message: str,
+        version: Optional[str] = None,
+        cause: Optional[Exception] = None,
+    ):
+        self.version = version
+        self.cause = cause
+        super().__init__(message)
+
+
+@dataclass
+class SchemaVersion:
+    """
+    Represents a schema version record.
+
+    Attributes:
+        version: Semantic version string (e.g., "1.0.0")
+        applied_at: When the migration was applied
+        description: Human-readable description of changes
+        checksum: Optional hash for integrity verification
+    """
+
+    version: str
+    applied_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    description: str = ""
+    checksum: Optional[str] = None
+
+    def __lt__(self, other: "SchemaVersion") -> bool:
+        """Compare versions for sorting."""
+        return self._parse_version(self.version) < self._parse_version(other.version)
+
+    @staticmethod
+    def _parse_version(version: str) -> tuple:
+        """Parse version string into comparable tuple."""
+        parts = version.split(".")
+        result = []
+        for part in parts:
+            try:
+                result.append(int(part))
+            except ValueError:
+                result.append(part)
+        return tuple(result)
+
+
+class Migration(ABC):
+    """
+    Abstract base class for schema migrations.
+
+    Subclasses must implement upgrade() and optionally downgrade().
+
+    Example:
+        class AddTagsColumn(Migration):
+            version = "1.1.0"
+            description = "Add tags column to heuristics table"
+
+            def upgrade(self, connection):
+                connection.execute(
+                    "ALTER TABLE heuristics ADD COLUMN tags TEXT"
+                )
+
+            def downgrade(self, connection):
+                connection.execute(
+                    "ALTER TABLE heuristics DROP COLUMN tags"
+                )
+    """
+
+    # These must be set by subclasses
+    version: str = ""
+    description: str = ""
+    # Optional: previous version this migration depends on
+    depends_on: Optional[str] = None
+
+    @abstractmethod
+    def upgrade(self, connection: Any) -> None:
+        """
+        Apply the migration.
+
+        Args:
+            connection: Database connection or storage instance
+        """
+        pass
+
+    def downgrade(self, connection: Any) -> None:
+        """
+        Revert the migration (optional).
+
+        Args:
+            connection: Database connection or storage instance
+
+        Raises:
+            NotImplementedError: If downgrade is not supported
+        """
+        raise NotImplementedError(
+            f"Downgrade not implemented for migration {self.version}"
+        )
+
+    def pre_check(self, connection: Any) -> bool:
+        """
+        Optional pre-migration check.
+
+        Override to verify prerequisites before migration.
+
+        Args:
+            connection: Database connection
+
+        Returns:
+            True if migration can proceed, False otherwise
+        """
+        return True
+
+    def post_check(self, connection: Any) -> bool:
+        """
+        Optional post-migration verification.
+
+        Override to verify migration was successful.
+
+        Args:
+            connection: Database connection
+
+        Returns:
+            True if migration was successful
+        """
+        return True
+
+
+class MigrationRegistry:
+    """
+    Registry for available migrations.
+
+    Manages migration discovery, ordering, and execution planning.
+    """
+
+    def __init__(self) -> None:
+        self._migrations: Dict[str, Type[Migration]] = {}
+        self._backend_migrations: Dict[str, Dict[str, Type[Migration]]] = {}
+
+    def register(
+        self, migration_class: Type[Migration], backend: Optional[str] = None
+    ) -> Type[Migration]:
+        """
+        Register a migration class.
+
+        Args:
+            migration_class: The migration class to register
+            backend: Optional backend name (e.g., "sqlite", "postgresql")
+
+        Returns:
+            The migration class (for use as decorator)
+        """
+        version = migration_class.version
+        if not version:
+            raise ValueError(
+                f"Migration {migration_class.__name__} must have a version"
+            )
+
+        if backend:
+            if backend not in self._backend_migrations:
+                self._backend_migrations[backend] = {}
+            self._backend_migrations[backend][version] = migration_class
+            logger.debug(f"Registered migration {version} for backend {backend}")
+        else:
+            self._migrations[version] = migration_class
+            logger.debug(f"Registered global migration {version}")
+
+        return migration_class
+
+    def get_migration(
+        self, version: str, backend: Optional[str] = None
+    ) -> Optional[Type[Migration]]:
+        """
+        Get a migration class by version.
+
+        Args:
+            version: Version string to look up
+            backend: Optional backend name
+
+        Returns:
+            Migration class or None if not found
+        """
+        if backend and backend in self._backend_migrations:
+            migration = self._backend_migrations[backend].get(version)
+            if migration:
+                return migration
+        return self._migrations.get(version)
+
+    def get_all_migrations(
+        self, backend: Optional[str] = None
+    ) -> List[Type[Migration]]:
+        """
+        Get all migrations in version order.
+
+        Args:
+            backend: Optional backend name to filter migrations
+
+        Returns:
+            List of migration classes sorted by version
+        """
+        migrations = dict(self._migrations)
+        if backend and backend in self._backend_migrations:
+            migrations.update(self._backend_migrations[backend])
+
+        return [
+            cls
+            for _, cls in sorted(
+                migrations.items(),
+                key=lambda x: SchemaVersion._parse_version(x[0]),
+            )
+        ]
+
+    def get_pending_migrations(
+        self,
+        current_version: Optional[str],
+        backend: Optional[str] = None,
+    ) -> List[Type[Migration]]:
+        """
+        Get migrations that need to be applied.
+
+        Args:
+            current_version: Current schema version (None if fresh install)
+            backend: Optional backend name
+
+        Returns:
+            List of migration classes that need to be applied
+        """
+        all_migrations = self.get_all_migrations(backend)
+
+        if current_version is None:
+            return all_migrations
+
+        current = SchemaVersion._parse_version(current_version)
+        return [
+            m
+            for m in all_migrations
+            if SchemaVersion._parse_version(m.version) > current
+        ]
+
+    def get_rollback_migrations(
+        self,
+        current_version: str,
+        target_version: str,
+        backend: Optional[str] = None,
+    ) -> List[Type[Migration]]:
+        """
+        Get migrations that need to be rolled back.
+
+        Args:
+            current_version: Current schema version
+            target_version: Target version to roll back to
+            backend: Optional backend name
+
+        Returns:
+            List of migration classes to roll back (in reverse order)
+        """
+        all_migrations = self.get_all_migrations(backend)
+
+        current = SchemaVersion._parse_version(current_version)
+        target = SchemaVersion._parse_version(target_version)
+
+        rollback = [
+            m
+            for m in all_migrations
+            if target < SchemaVersion._parse_version(m.version) <= current
+        ]
+
+        # Return in reverse order for rollback
+        return list(reversed(rollback))
+
+
+# Global registry instance
+_global_registry = MigrationRegistry()
+
+
+def register_migration(
+    backend: Optional[str] = None,
+) -> Callable[[Type[Migration]], Type[Migration]]:
+    """
+    Decorator to register a migration class.
+
+    Args:
+        backend: Optional backend name
+
+    Example:
+        @register_migration()
+        class MyMigration(Migration):
+            version = "1.0.0"
+            ...
+
+        @register_migration(backend="postgresql")
+        class PostgresSpecificMigration(Migration):
+            version = "1.0.1"
+            ...
+    """
+
+    def decorator(cls: Type[Migration]) -> Type[Migration]:
+        return _global_registry.register(cls, backend)
+
+    return decorator
+
+
+def get_registry() -> MigrationRegistry:
+    """Get the global migration registry."""
+    return _global_registry