tribalmemory 0.1.0__py3-none-any.whl

tribalmemory/a21/providers/base.py

@@ -0,0 +1,241 @@
+"""Abstract base classes for all providers.
+
+These define the contracts that provider implementations must satisfy.
+Designed for extensibility and forward compatibility.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional, Any, TypeVar, Generic
+from enum import Enum
+
+from ...interfaces import MemoryEntry, RecallResult, StoreResult, MemorySource
+
+
+# Type variable for provider-specific configuration
+TConfig = TypeVar('TConfig')
+
+
+class ProviderStatus(Enum):
+    """Provider health status."""
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNAVAILABLE = "unavailable"
+    INITIALIZING = "initializing"
+
+
+@dataclass
+class ProviderHealth:
+    """Health check result for a provider."""
+    status: ProviderStatus
+    latency_ms: Optional[float] = None
+    message: Optional[str] = None
+    last_check: datetime = None
+    
+    def __post_init__(self):
+        if self.last_check is None:
+            self.last_check = datetime.utcnow()
+
+
+class Provider(ABC, Generic[TConfig]):
+    """Base class for all providers.
+    
+    Provides common functionality:
+    - Configuration management
+    - Health checking
+    - Lifecycle management (init/shutdown)
+    - Metrics collection hooks
+    """
+    
+    def __init__(self, config: TConfig):
+        self.config = config
+        self._initialized = False
+    
+    @abstractmethod
+    async def initialize(self) -> None:
+        """Initialize the provider. Called once before first use."""
+        pass
+    
+    @abstractmethod
+    async def shutdown(self) -> None:
+        """Gracefully shutdown the provider."""
+        pass
+    
+    @abstractmethod
+    async def health_check(self) -> ProviderHealth:
+        """Check provider health and connectivity."""
+        pass
+    
+    @property
+    def is_initialized(self) -> bool:
+        return self._initialized
+    
+    async def __aenter__(self):
+        if not self._initialized:
+            await self.initialize()
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.shutdown()
+
+
+class EmbeddingProvider(Provider[TConfig]):
+    """Abstract embedding provider.
+    
+    Responsible for converting text to vector embeddings.
+    Implementations may use OpenAI, local models, or other services.
+    """
+    
+    @property
+    @abstractmethod
+    def dimensions(self) -> int:
+        """Return the embedding dimension size."""
+        pass
+    
+    @property
+    @abstractmethod
+    def model_name(self) -> str:
+        """Return the model identifier."""
+        pass
+    
+    @abstractmethod
+    async def embed(self, text: str) -> list[float]:
+        """Generate embedding for a single text."""
+        pass
+    
+    @abstractmethod
+    async def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """Generate embeddings for multiple texts efficiently."""
+        pass
+    
+    def similarity(self, a: list[float], b: list[float]) -> float:
+        """Calculate cosine similarity between two embeddings.
+        
+        Uses the formula: cos(θ) = (a · b) / (||a|| * ||b||)
+        
+        Args:
+            a: First embedding vector
+            b: Second embedding vector
+            
+        Returns:
+            Cosine similarity score between -1.0 and 1.0
+        """
+        import math
+        dot = sum(x * y for x, y in zip(a, b))
+        norm_a = math.sqrt(sum(x * x for x in a))
+        norm_b = math.sqrt(sum(x * x for x in b))
+        if norm_a == 0 or norm_b == 0:
+            return 0.0
+        return dot / (norm_a * norm_b)
+
+
+class StorageProvider(Provider[TConfig]):
+    """Abstract storage provider.
+    
+    Responsible for persisting and retrieving memory entries.
+    Implementations may use LanceDB, Pinecone, Postgres+pgvector, etc.
+    """
+    
+    @abstractmethod
+    async def store(self, entry: MemoryEntry) -> StoreResult:
+        """Store a memory entry."""
+        pass
+    
+    @abstractmethod
+    async def recall(
+        self,
+        query_embedding: list[float],
+        limit: int = 10,
+        min_similarity: float = 0.7,
+        filters: Optional[dict[str, Any]] = None,
+    ) -> list[RecallResult]:
+        """Recall memories similar to query embedding.
+        
+        Args:
+            query_embedding: Vector to search for
+            limit: Maximum results
+            min_similarity: Minimum similarity threshold
+            filters: Optional metadata filters (e.g., tags, source_instance)
+        """
+        pass
+    
+    @abstractmethod
+    async def get(self, memory_id: str) -> Optional[MemoryEntry]:
+        """Get a specific memory by ID."""
+        pass
+    
+    @abstractmethod
+    async def delete(self, memory_id: str) -> bool:
+        """Soft delete a memory."""
+        pass
+    
+    @abstractmethod
+    async def list(
+        self,
+        limit: int = 100,
+        offset: int = 0,
+        filters: Optional[dict[str, Any]] = None,
+    ) -> list[MemoryEntry]:
+        """List memories with pagination and filtering."""
+        pass
+    
+    @abstractmethod
+    async def count(self, filters: Optional[dict[str, Any]] = None) -> int:
+        """Count memories matching filters."""
+        pass
+
+
+class TimestampProvider(Provider[TConfig]):
+    """Abstract timestamp provider.
+    
+    Responsible for generating cryptographic timestamps (RFC 3161).
+    Used for provenance verification.
+    """
+    
+    @abstractmethod
+    async def timestamp(self, data: bytes) -> bytes:
+        """Generate a timestamp token for data."""
+        pass
+    
+    @abstractmethod
+    async def verify(self, data: bytes, token: bytes) -> tuple[bool, Optional[datetime]]:
+        """Verify a timestamp token."""
+        pass
+
+
+class DeduplicationProvider(Provider[TConfig]):
+    """Abstract deduplication provider.
+    
+    Responsible for detecting duplicate or near-duplicate memories.
+    May use embedding similarity, hashing, or hybrid approaches.
+    """
+    
+    @abstractmethod
+    async def is_duplicate(
+        self,
+        content: str,
+        embedding: list[float],
+    ) -> tuple[bool, Optional[str]]:
+        """Check if content is duplicate.
+        
+        Returns:
+            Tuple of (is_duplicate, duplicate_id).
+            Use find_similar() or get_duplicate_report() if similarity score needed.
+        """
+        pass
+    
+    @abstractmethod
+    async def find_similar(
+        self,
+        content: str,
+        embedding: list[float],
+        threshold: float = 0.85,
+        limit: int = 10,
+    ) -> list[tuple[str, float]]:
+        """Find similar memories.
+        
+        Returns:
+            List of (memory_id, similarity_score)
+        """
+        pass

tribalmemory/a21/providers/deduplication.py

@@ -0,0 +1,99 @@
+"""Deduplication provider."""
+
+from typing import Optional
+
+from .base import DeduplicationProvider, StorageProvider, EmbeddingProvider, ProviderHealth, ProviderStatus
+from ..config.providers import DeduplicationConfig
+
+
+class EmbeddingDeduplicationProvider(DeduplicationProvider[DeduplicationConfig]):
+    """Deduplication using embedding similarity.
+    
+    Detects duplicates by comparing embeddings against stored memories.
+    Uses configurable thresholds for exact and near-duplicate detection.
+    """
+    
+    def __init__(
+        self,
+        config: DeduplicationConfig,
+        storage_provider: StorageProvider,
+        embedding_provider: EmbeddingProvider,
+    ):
+        """Initialize deduplication provider.
+        
+        Args:
+            config: Deduplication configuration
+            storage_provider: Initialized storage provider for recall queries
+            embedding_provider: Initialized embedding provider for similarity calculations
+        """
+        super().__init__(config)
+        self._storage = storage_provider
+        self._embedding = embedding_provider
+    
+    async def initialize(self) -> None:
+        """Initialize provider. Storage and embedding must already be initialized."""
+        self._initialized = True
+    
+    async def shutdown(self) -> None:
+        """Shutdown provider."""
+        self._initialized = False
+    
+    async def health_check(self) -> ProviderHealth:
+        """Check provider health."""
+        return ProviderHealth(
+            status=ProviderStatus.HEALTHY,
+            message="Deduplication ready"
+        )
+    
+    async def is_duplicate(
+        self,
+        content: str,
+        embedding: list[float],
+    ) -> tuple[bool, Optional[str]]:
+        """Check if content is a duplicate.
+        
+        Args:
+            content: Text content to check
+            embedding: Pre-computed embedding for the content
+            
+        Returns:
+            Tuple of (is_duplicate, duplicate_id)
+        """
+        results = await self._storage.recall(
+            embedding,
+            limit=1,
+            min_similarity=self.config.exact_threshold
+        )
+        
+        if results and results[0].similarity_score >= self.config.exact_threshold:
+            return True, results[0].memory.id
+        
+        return False, None
+    
+    async def find_similar(
+        self,
+        content: str,
+        embedding: list[float],
+        threshold: float = None,
+        limit: int = 10,
+    ) -> list[tuple[str, float]]:
+        """Find similar memories.
+        
+        Args:
+            content: Text content to search for
+            embedding: Pre-computed embedding for the content
+            threshold: Minimum similarity (defaults to config.near_threshold)
+            limit: Maximum results
+            
+        Returns:
+            List of (memory_id, similarity_score) tuples
+        """
+        threshold = threshold or self.config.near_threshold
+        
+        results = await self._storage.recall(
+            embedding,
+            limit=limit,
+            min_similarity=threshold
+        )
+        
+        return [(r.memory.id, r.similarity_score) for r in results]

tribalmemory/a21/providers/lancedb.py

@@ -0,0 +1,232 @@
+"""LanceDB storage provider."""
+
+import json
+import logging
+import re
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Any
+
+from .base import StorageProvider, EmbeddingProvider, ProviderHealth, ProviderStatus
+from ..config.providers import StorageConfig
+from ...interfaces import MemoryEntry, MemorySource, RecallResult, StoreResult
+
+logger = logging.getLogger(__name__)
+
+
+class LanceDBStorageProvider(StorageProvider[StorageConfig]):
+    """LanceDB-backed storage provider."""
+    
+    def __init__(
+        self,
+        config: StorageConfig,
+        embedding_provider: EmbeddingProvider,
+    ):
+        super().__init__(config)
+        self._embedding = embedding_provider
+        self._db = None
+        self._table = None
+    
+    async def initialize(self) -> None:
+        try:
+            import lancedb
+        except ImportError:
+            raise ImportError("LanceDB not installed. Run: pip install lancedb")
+        
+        if self.config.uri:
+            self._db = lancedb.connect(self.config.uri, api_key=self.config.api_key)
+        elif self.config.path:
+            Path(self.config.path).mkdir(parents=True, exist_ok=True)
+            self._db = lancedb.connect(self.config.path)
+        else:
+            raise ValueError("LanceDB requires path or uri")
+        
+        if self.config.table_name in self._db.table_names():
+            self._table = self._db.open_table(self.config.table_name)
+        else:
+            self._table = self._create_table()
+        
+        self._initialized = True
+    
+    async def shutdown(self) -> None:
+        self._db = None
+        self._table = None
+        self._initialized = False
+    
+    async def health_check(self) -> ProviderHealth:
+        if not self._table:
+            return ProviderHealth(
+                status=ProviderStatus.UNAVAILABLE,
+                message="Table not initialized"
+            )
+        
+        try:
+            start = datetime.utcnow()
+            count = await self.count()
+            latency = (datetime.utcnow() - start).total_seconds() * 1000
+            return ProviderHealth(
+                status=ProviderStatus.HEALTHY,
+                latency_ms=latency,
+                message=f"LanceDB with {count} entries"
+            )
+        except Exception as e:
+            return ProviderHealth(
+                status=ProviderStatus.DEGRADED,
+                message=str(e)
+            )
+    
+    def _create_table(self):
+        import pyarrow as pa
+        
+        schema = pa.schema([
+            pa.field("id", pa.string()),
+            pa.field("content", pa.string()),
+            pa.field("vector", pa.list_(pa.float32(), self.config.embedding_dimensions)),
+            pa.field("source_instance", pa.string()),
+            pa.field("source_type", pa.string()),
+            pa.field("created_at", pa.string()),
+            pa.field("updated_at", pa.string()),
+            pa.field("tags", pa.string()),
+            pa.field("context", pa.string()),
+            pa.field("confidence", pa.float32()),
+            pa.field("supersedes", pa.string()),
+            pa.field("related_to", pa.string()),
+            pa.field("deleted", pa.bool_()),
+        ])
+        
+        return self._db.create_table(self.config.table_name, schema=schema)
+    
+    async def store(self, entry: MemoryEntry) -> StoreResult:
+        if entry.embedding is None:
+            entry.embedding = await self._embedding.embed(entry.content)
+        
+        # Validate dimensions
+        if len(entry.embedding) != self.config.embedding_dimensions:
+            return StoreResult(
+                success=False,
+                error=f"Invalid embedding dimension: {len(entry.embedding)}"
+            )
+        
+        row = {
+            "id": entry.id,
+            "content": entry.content,
+            "vector": entry.embedding,
+            "source_instance": entry.source_instance,
+            "source_type": entry.source_type.value,
+            "created_at": entry.created_at.isoformat(),
+            "updated_at": entry.updated_at.isoformat(),
+            "tags": json.dumps(entry.tags),
+            "context": entry.context or "",
+            "confidence": entry.confidence,
+            "supersedes": entry.supersedes or "",
+            "related_to": json.dumps(entry.related_to),
+            "deleted": False,
+        }
+        
+        try:
+            self._table.add([row])
+            return StoreResult(success=True, memory_id=entry.id)
+        except Exception as e:
+            logger.error(f"Failed to store memory {entry.id}: {e}")
+            return StoreResult(success=False, error=str(e))
+    
+    async def recall(
+        self,
+        query_embedding: list[float],
+        limit: int = 10,
+        min_similarity: float = 0.7,
+        filters: Optional[dict[str, Any]] = None,
+    ) -> list[RecallResult]:
+        start = time.perf_counter()
+        
+        query = self._table.search(query_embedding).where("deleted = false")
+        
+        if filters:
+            for key, value in filters.items():
+                if key == "source_instance":
+                    safe_val = self._sanitize(value)
+                    query = query.where(f"source_instance = '{safe_val}'")
+        
+        results = query.limit(limit * 2).to_list()
+        elapsed_ms = (time.perf_counter() - start) * 1000
+        
+        recall_results = []
+        for row in results:
+            distance = row.get("_distance", 0)
+            similarity = max(0, 1 - (distance * distance / 2))
+            
+            if similarity < min_similarity:
+                continue
+            
+            entry = self._row_to_entry(row)
+            recall_results.append(RecallResult(
+                memory=entry,
+                similarity_score=similarity,
+                retrieval_time_ms=elapsed_ms
+            ))
+        
+        recall_results.sort(key=lambda x: x.similarity_score, reverse=True)
+        return recall_results[:limit]
+    
+    async def get(self, memory_id: str) -> Optional[MemoryEntry]:
+        safe_id = self._sanitize(memory_id)
+        
+        results = (
+            self._table.search()
+            .where(f"id = '{safe_id}' AND deleted = false")
+            .limit(1)
+            .to_list()
+        )
+        
+        if not results:
+            return None
+        return self._row_to_entry(results[0])
+    
+    async def delete(self, memory_id: str) -> bool:
+        safe_id = self._sanitize(memory_id)
+        
+        try:
+            self._table.update(
+                where=f"id = '{safe_id}'",
+                values={"deleted": True, "updated_at": datetime.utcnow().isoformat()}
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete memory {memory_id}: {e}")
+            return False
+    
+    async def list(
+        self,
+        limit: int = 100,
+        offset: int = 0,
+        filters: Optional[dict[str, Any]] = None,
+    ) -> list[MemoryEntry]:
+        query = self._table.search().where("deleted = false")
+        results = query.limit(limit + offset).to_list()
+        return [self._row_to_entry(r) for r in results[offset:offset + limit]]
+    
+    async def count(self, filters: Optional[dict[str, Any]] = None) -> int:
+        results = self._table.search().where("deleted = false").to_list()
+        return len(results)
+    
+    def _sanitize(self, value: str) -> str:
+        if not re.match(r'^[a-zA-Z0-9\-_]+$', value):
+            raise ValueError(f"Invalid value format: {value[:20]}...")
+        return value
+    
+    def _row_to_entry(self, row: dict) -> MemoryEntry:
+        return MemoryEntry(
+            id=row["id"],
+            content=row["content"],
+            embedding=row.get("vector"),
+            source_instance=row.get("source_instance", "unknown"),
+            source_type=MemorySource(row.get("source_type", "unknown")),
+            created_at=datetime.fromisoformat(row["created_at"]) if row.get("created_at") else datetime.utcnow(),
+            updated_at=datetime.fromisoformat(row["updated_at"]) if row.get("updated_at") else datetime.utcnow(),
+            tags=json.loads(row.get("tags", "[]")),
+            context=row.get("context") or None,
+            confidence=row.get("confidence", 1.0),
+            supersedes=row.get("supersedes") or None,
+            related_to=json.loads(row.get("related_to", "[]")),
+        )

tribalmemory/a21/providers/memory.py

@@ -0,0 +1,128 @@
+"""In-memory storage provider."""
+
+import time
+from datetime import datetime
+from typing import Optional, Any, Callable
+
+from .base import StorageProvider, EmbeddingProvider, ProviderHealth, ProviderStatus
+from ..config.providers import StorageConfig
+from ...interfaces import MemoryEntry, RecallResult, StoreResult
+
+
+class InMemoryStorageProvider(StorageProvider[StorageConfig]):
+    """In-memory storage for testing and development."""
+    
+    def __init__(
+        self,
+        config: StorageConfig,
+        embedding_provider: EmbeddingProvider,
+    ):
+        super().__init__(config)
+        self._embedding = embedding_provider
+        self._store: dict[str, MemoryEntry] = {}
+        self._deleted: set[str] = set()
+    
+    async def initialize(self) -> None:
+        self._initialized = True
+    
+    async def shutdown(self) -> None:
+        self._store.clear()
+        self._deleted.clear()
+        self._initialized = False
+    
+    async def health_check(self) -> ProviderHealth:
+        return ProviderHealth(
+            status=ProviderStatus.HEALTHY,
+            latency_ms=0.1,
+            message=f"In-memory store with {len(self._store)} entries"
+        )
+    
+    async def store(self, entry: MemoryEntry) -> StoreResult:
+        if entry.embedding is None:
+            entry.embedding = await self._embedding.embed(entry.content)
+        
+        # Validate embedding dimensions
+        if len(entry.embedding) != self.config.embedding_dimensions:
+            return StoreResult(
+                success=False,
+                error=f"Invalid embedding dimension: expected {self.config.embedding_dimensions}, got {len(entry.embedding)}"
+            )
+        
+        self._store[entry.id] = entry
+        return StoreResult(success=True, memory_id=entry.id)
+    
+    async def recall(
+        self,
+        query_embedding: list[float],
+        limit: int = 10,
+        min_similarity: float = 0.7,
+        filters: Optional[dict[str, Any]] = None,
+    ) -> list[RecallResult]:
+        start = time.perf_counter()
+        
+        results = []
+        for entry in self._store.values():
+            if entry.id in self._deleted:
+                continue
+            if entry.embedding is None:
+                continue
+            
+            # Apply filters
+            if filters and not self._matches_filters(entry, filters):
+                continue
+            
+            sim = self._embedding.similarity(query_embedding, entry.embedding)
+            if sim >= min_similarity:
+                results.append((entry, sim))
+        
+        results.sort(key=lambda x: x[1], reverse=True)
+        elapsed_ms = (time.perf_counter() - start) * 1000
+        
+        return [
+            RecallResult(memory=e, similarity_score=s, retrieval_time_ms=elapsed_ms)
+            for e, s in results[:limit]
+        ]
+    
+    async def get(self, memory_id: str) -> Optional[MemoryEntry]:
+        if memory_id in self._deleted:
+            return None
+        return self._store.get(memory_id)
+    
+    async def delete(self, memory_id: str) -> bool:
+        if memory_id in self._store:
+            self._deleted.add(memory_id)
+            return True
+        return False
+    
+    async def list(
+        self,
+        limit: int = 100,
+        offset: int = 0,
+        filters: Optional[dict[str, Any]] = None,
+    ) -> list[MemoryEntry]:
+        entries = [
+            e for e in self._store.values()
+            if e.id not in self._deleted
+        ]
+        
+        if filters:
+            entries = [e for e in entries if self._matches_filters(e, filters)]
+        
+        return entries[offset:offset + limit]
+    
+    async def count(self, filters: Optional[dict[str, Any]] = None) -> int:
+        entries = await self.list(limit=100000, filters=filters)
+        return len(entries)
+    
+    def _matches_filters(self, entry: MemoryEntry, filters: dict[str, Any]) -> bool:
+        for key, value in filters.items():
+            if key == "tags":
+                if not any(t in entry.tags for t in value):
+                    return False
+            elif key == "source_instance":
+                if entry.source_instance != value:
+                    return False
+            elif key == "source_type":
+                if entry.source_type.value != value:
+                    return False
+        return True