tribalmemory 0.1.0__py3-none-any.whl

tribalmemory/__init__.py

@@ -0,0 +1,3 @@
+"""Tribal Memory - Shared memory infrastructure for multi-instance AI agents."""
+
+__version__ = "0.1.0"

tribalmemory/a21/__init__.py

@@ -0,0 +1,38 @@
+"""A2.1 Interface-First Implementation.
+
+This module provides a highly abstracted, plugin-based architecture for
+tribal memory. Key design principles:
+
+1. **Provider Pattern**: All backends (embeddings, storage, timestamps) are
+   swappable providers that implement standard interfaces.
+
+2. **Dependency Injection**: Components receive their dependencies through
+   a central container, making testing and configuration easier.
+
+3. **Configuration-Driven**: Setup is driven by configuration objects,
+   not hardcoded values.
+
+4. **Forward Compatible**: Interfaces are designed to accommodate future
+   features (multi-tenancy, sharding, replication) without breaking changes.
+
+Usage:
+    from tribalmemory.a21 import MemorySystem, SystemConfig
+    
+    config = SystemConfig.from_env()
+    system = MemorySystem(config)
+    
+    await system.remember("Joe prefers TypeScript")
+    results = await system.recall("What language?")
+"""
+
+from .system import MemorySystem
+from .config import SystemConfig, EmbeddingConfig, StorageConfig
+from .container import Container
+
+__all__ = [
+    "MemorySystem",
+    "SystemConfig",
+    "EmbeddingConfig",
+    "StorageConfig",
+    "Container",
+]

tribalmemory/a21/config/__init__.py

@@ -0,0 +1,20 @@
+"""Configuration system for A2.1.
+
+Provides strongly-typed configuration objects that can be loaded from:
+- Environment variables
+- YAML/JSON files
+- Programmatic construction
+
+All configuration is validated at load time.
+"""
+
+from .system import SystemConfig
+from .providers import EmbeddingConfig, StorageConfig, TimestampConfig, DeduplicationConfig
+
+__all__ = [
+    "SystemConfig",
+    "EmbeddingConfig",
+    "StorageConfig",
+    "TimestampConfig",
+    "DeduplicationConfig",
+]

tribalmemory/a21/config/providers.py

@@ -0,0 +1,104 @@
+"""Provider-specific configuration classes."""
+
+from dataclasses import dataclass, field
+from typing import Optional, Literal
+from enum import Enum
+
+
+class EmbeddingProviderType(Enum):
+    """Available embedding providers."""
+    OPENAI = "openai"
+    LOCAL = "local"  # Future: local model support
+    MOCK = "mock"
+
+
+class StorageProviderType(Enum):
+    """Available storage providers."""
+    LANCEDB = "lancedb"
+    MEMORY = "memory"
+    # Future: PINECONE = "pinecone", POSTGRES = "postgres"
+
+
+class TimestampProviderType(Enum):
+    """Available timestamp providers."""
+    RFC3161 = "rfc3161"
+    MOCK = "mock"
+    NONE = "none"
+
+
+@dataclass
+class EmbeddingConfig:
+    """Configuration for embedding provider.
+    
+    Attributes:
+        provider: Which embedding provider to use
+        model: Model name (e.g., "text-embedding-3-small")
+        dimensions: Embedding dimension size
+        api_key: API key (for cloud providers)
+        api_base: Custom API base URL
+        max_retries: Max retry attempts
+        timeout_seconds: Request timeout
+        backoff_base: Exponential backoff base
+        backoff_max: Maximum backoff delay
+        batch_size: Max texts per batch request
+    """
+    provider: EmbeddingProviderType = EmbeddingProviderType.OPENAI
+    model: str = "text-embedding-3-small"
+    dimensions: int = 1536
+    api_key: Optional[str] = None
+    api_base: Optional[str] = None
+    max_retries: int = 3
+    timeout_seconds: float = 30.0
+    backoff_base: float = 2.0
+    backoff_max: float = 60.0
+    batch_size: int = 100
+
+
+@dataclass
+class StorageConfig:
+    """Configuration for storage provider.
+    
+    Attributes:
+        provider: Which storage provider to use
+        path: Local database path (for LanceDB local)
+        uri: Cloud database URI (for LanceDB Cloud)
+        api_key: API key (for cloud storage)
+        table_name: Name of the memories table
+        embedding_dimensions: Expected embedding size (for validation)
+    """
+    provider: StorageProviderType = StorageProviderType.MEMORY
+    path: Optional[str] = None
+    uri: Optional[str] = None
+    api_key: Optional[str] = None
+    table_name: str = "memories"
+    embedding_dimensions: int = 1536
+
+
+@dataclass
+class TimestampConfig:
+    """Configuration for timestamp provider.
+    
+    Attributes:
+        provider: Which timestamp provider to use
+        tsa_url: RFC 3161 Time Stamp Authority URL
+        tsa_cert_path: Path to TSA certificate for verification
+    """
+    provider: TimestampProviderType = TimestampProviderType.NONE
+    tsa_url: Optional[str] = None
+    tsa_cert_path: Optional[str] = None
+
+
+@dataclass
+class DeduplicationConfig:
+    """Configuration for deduplication.
+    
+    Attributes:
+        enabled: Whether to check for duplicates
+        exact_threshold: Similarity threshold for exact duplicates (reject)
+        near_threshold: Similarity threshold for near-duplicates (warn)
+        strategy: Deduplication strategy
+    """
+    enabled: bool = True
+    exact_threshold: float = 0.98
+    near_threshold: float = 0.90
+    strategy: Literal["embedding", "hash", "hybrid"] = "embedding"

tribalmemory/a21/config/system.py

@@ -0,0 +1,184 @@
+"""System-wide configuration."""
+
+import os
+from dataclasses import dataclass, field
+from typing import Optional
+from pathlib import Path
+
+from .providers import (
+    EmbeddingConfig,
+    StorageConfig,
+    TimestampConfig,
+    DeduplicationConfig,
+    EmbeddingProviderType,
+    StorageProviderType,
+)
+
+
+@dataclass
+class SystemConfig:
+    """Complete system configuration.
+    
+    Combines all provider configurations into a single object.
+    Can be loaded from environment variables or constructed programmatically.
+    
+    Attributes:
+        instance_id: Unique identifier for this agent instance
+        embedding: Embedding provider configuration
+        storage: Storage provider configuration
+        timestamp: Timestamp provider configuration
+        deduplication: Deduplication configuration
+        debug: Enable debug logging
+    """
+    instance_id: str = "default"
+    embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
+    storage: StorageConfig = field(default_factory=StorageConfig)
+    timestamp: TimestampConfig = field(default_factory=TimestampConfig)
+    deduplication: DeduplicationConfig = field(default_factory=DeduplicationConfig)
+    debug: bool = False
+    
+    @classmethod
+    def from_env(cls, prefix: str = "TRIBAL_MEMORY") -> "SystemConfig":
+        """Load configuration from environment variables.
+        
+        Environment variables:
+            {prefix}_INSTANCE_ID: Instance identifier
+            {prefix}_DEBUG: Enable debug mode
+            
+            {prefix}_EMBEDDING_PROVIDER: openai|local|mock
+            {prefix}_EMBEDDING_MODEL: Model name
+            {prefix}_EMBEDDING_API_KEY: API key (or OPENAI_API_KEY)
+            
+            {prefix}_STORAGE_PROVIDER: lancedb|memory
+            {prefix}_STORAGE_PATH: Local database path
+            {prefix}_STORAGE_URI: Cloud database URI
+            
+            {prefix}_DEDUP_ENABLED: true|false
+            {prefix}_DEDUP_EXACT_THRESHOLD: Float (0-1)
+            {prefix}_DEDUP_NEAR_THRESHOLD: Float (0-1)
+        """
+        def get(key: str, default: str = None) -> Optional[str]:
+            return os.environ.get(f"{prefix}_{key}", default)
+        
+        def get_bool(key: str, default: bool = False) -> bool:
+            val = get(key)
+            if val is None:
+                return default
+            return val.lower() in ("true", "1", "yes")
+        
+        def get_float(key: str, default: float) -> float:
+            val = get(key)
+            return float(val) if val else default
+        
+        # Embedding config
+        embedding = EmbeddingConfig(
+            provider=EmbeddingProviderType(get("EMBEDDING_PROVIDER", "openai")),
+            model=get("EMBEDDING_MODEL", "text-embedding-3-small"),
+            api_key=get("EMBEDDING_API_KEY") or os.environ.get("OPENAI_API_KEY"),
+        )
+        
+        # Storage config
+        storage = StorageConfig(
+            provider=StorageProviderType(get("STORAGE_PROVIDER", "memory")),
+            path=get("STORAGE_PATH"),
+            uri=get("STORAGE_URI"),
+            api_key=get("STORAGE_API_KEY") or os.environ.get("LANCEDB_API_KEY"),
+        )
+        
+        # Deduplication config
+        deduplication = DeduplicationConfig(
+            enabled=get_bool("DEDUP_ENABLED", True),
+            exact_threshold=get_float("DEDUP_EXACT_THRESHOLD", 0.98),
+            near_threshold=get_float("DEDUP_NEAR_THRESHOLD", 0.90),
+        )
+        
+        return cls(
+            instance_id=get("INSTANCE_ID", "default"),
+            embedding=embedding,
+            storage=storage,
+            deduplication=deduplication,
+            debug=get_bool("DEBUG", False),
+        )
+    
+    @classmethod
+    def for_testing(cls, instance_id: str = "test") -> "SystemConfig":
+        """Create a configuration suitable for testing.
+        
+        Uses mock/in-memory providers to avoid external dependencies.
+        """
+        return cls(
+            instance_id=instance_id,
+            embedding=EmbeddingConfig(
+                provider=EmbeddingProviderType.MOCK,
+            ),
+            storage=StorageConfig(
+                provider=StorageProviderType.MEMORY,
+            ),
+            deduplication=DeduplicationConfig(
+                enabled=True,
+                exact_threshold=0.90,  # Lower for deterministic mock embeddings
+            ),
+            debug=True,
+        )
+    
+    def validate(self) -> list[str]:
+        """Validate configuration and return list of errors.
+        
+        Checks:
+        - Required API keys for providers
+        - Storage path/uri requirements
+        - Dimension consistency
+        - Timeout and threshold bounds
+        - Batch size validity
+        - Instance ID format
+        """
+        errors = []
+        
+        # Instance ID validation
+        if not self.instance_id or not self.instance_id.strip():
+            errors.append("instance_id cannot be empty")
+        
+        # Embedding validation
+        if self.embedding.provider == EmbeddingProviderType.OPENAI:
+            if not self.embedding.api_key:
+                errors.append("OpenAI embedding requires API key")
+        
+        # Embedding config bounds
+        if self.embedding.timeout_seconds <= 0:
+            errors.append(f"embedding.timeout_seconds must be positive, got {self.embedding.timeout_seconds}")
+        if self.embedding.batch_size <= 0:
+            errors.append(f"embedding.batch_size must be positive, got {self.embedding.batch_size}")
+        if self.embedding.dimensions <= 0:
+            errors.append(f"embedding.dimensions must be positive, got {self.embedding.dimensions}")
+        
+        # Storage validation
+        if self.storage.provider == StorageProviderType.LANCEDB:
+            if not self.storage.path and not self.storage.uri:
+                errors.append("LanceDB storage requires path or uri")
+        
+        # Dimension consistency
+        if self.embedding.dimensions != self.storage.embedding_dimensions:
+            errors.append(
+                f"Embedding dimensions mismatch: "
+                f"embedding={self.embedding.dimensions}, storage={self.storage.embedding_dimensions}"
+            )
+        
+        # Deduplication threshold validation
+        if self.deduplication.enabled:
+            if not (0.0 <= self.deduplication.exact_threshold <= 1.0):
+                errors.append(
+                    f"deduplication.exact_threshold must be between 0 and 1, "
+                    f"got {self.deduplication.exact_threshold}"
+                )
+            if not (0.0 <= self.deduplication.near_threshold <= 1.0):
+                errors.append(
+                    f"deduplication.near_threshold must be between 0 and 1, "
+                    f"got {self.deduplication.near_threshold}"
+                )
+            if self.deduplication.near_threshold > self.deduplication.exact_threshold:
+                errors.append(
+                    f"deduplication.near_threshold ({self.deduplication.near_threshold}) "
+                    f"should not exceed exact_threshold ({self.deduplication.exact_threshold})"
+                )
+        
+        return errors

tribalmemory/a21/container/__init__.py

@@ -0,0 +1,8 @@
+"""Dependency Injection Container.
+
+Manages provider lifecycle and dependency resolution.
+"""
+
+from .container import Container
+
+__all__ = ["Container"]

tribalmemory/a21/container/container.py

@@ -0,0 +1,212 @@
+"""Dependency injection container for A2.1."""
+
+from typing import Optional, TypeVar, Type
+import logging
+
+from ..config import SystemConfig
+from ..providers.base import (
+    EmbeddingProvider,
+    StorageProvider,
+    TimestampProvider,
+    DeduplicationProvider,
+    ProviderHealth,
+    ProviderStatus,
+)
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar('T')
+
+
+class Container:
+    """Dependency injection container.
+    
+    Manages the lifecycle of all providers and provides them to consumers.
+    Supports lazy initialization and graceful shutdown.
+    
+    Usage:
+        container = Container(config)
+        await container.initialize()
+        
+        embedding = container.embedding
+        storage = container.storage
+        
+        await container.shutdown()
+    """
+    
+    def __init__(self, config: SystemConfig):
+        self.config = config
+        self._embedding: Optional[EmbeddingProvider] = None
+        self._storage: Optional[StorageProvider] = None
+        self._timestamp: Optional[TimestampProvider] = None
+        self._deduplication: Optional[DeduplicationProvider] = None
+        self._initialized = False
+    
+    async def initialize(self) -> None:
+        """Initialize all providers.
+        
+        Providers are created and initialized in dependency order:
+        1. Embedding (no dependencies)
+        2. Storage (depends on embedding dimensions)
+        3. Deduplication (depends on both storage and embedding)
+        4. Timestamp (no dependencies)
+        """
+        if self._initialized:
+            return
+        
+        logger.info(f"Initializing container for instance: {self.config.instance_id}")
+        
+        # Create and initialize providers in dependency order
+        self._embedding = self._create_embedding_provider()
+        await self._embedding.initialize()
+        
+        self._storage = self._create_storage_provider()
+        await self._storage.initialize()
+        
+        # Deduplication depends on both storage and embedding - create after they're initialized
+        self._deduplication = self._create_deduplication_provider(
+            storage=self._storage,
+            embedding=self._embedding,
+        )
+        if self._deduplication:
+            await self._deduplication.initialize()
+        
+        if self.config.timestamp.provider.value != "none":
+            self._timestamp = self._create_timestamp_provider()
+            await self._timestamp.initialize()
+        
+        self._initialized = True
+        logger.info("Container initialized successfully")
+    
+    async def shutdown(self) -> None:
+        """Shutdown all providers gracefully."""
+        if not self._initialized:
+            return
+        
+        logger.info("Shutting down container")
+        
+        # Shutdown in reverse order
+        if self._timestamp:
+            await self._timestamp.shutdown()
+        if self._deduplication:
+            await self._deduplication.shutdown()
+        await self._storage.shutdown()
+        await self._embedding.shutdown()
+        
+        self._initialized = False
+        logger.info("Container shutdown complete")
+    
+    async def health_check(self) -> dict[str, ProviderHealth]:
+        """Check health of all providers."""
+        results = {}
+        
+        if self._embedding:
+            results["embedding"] = await self._embedding.health_check()
+        if self._storage:
+            results["storage"] = await self._storage.health_check()
+        if self._timestamp:
+            results["timestamp"] = await self._timestamp.health_check()
+        if self._deduplication:
+            results["deduplication"] = await self._deduplication.health_check()
+        
+        return results
+    
+    @property
+    def embedding(self) -> EmbeddingProvider:
+        """Get the embedding provider."""
+        if not self._embedding:
+            raise RuntimeError("Container not initialized. Call initialize() first.")
+        return self._embedding
+    
+    @property
+    def storage(self) -> StorageProvider:
+        """Get the storage provider."""
+        if not self._storage:
+            raise RuntimeError("Container not initialized. Call initialize() first.")
+        return self._storage
+    
+    @property
+    def timestamp(self) -> Optional[TimestampProvider]:
+        """Get the timestamp provider (may be None if disabled)."""
+        return self._timestamp
+    
+    @property
+    def deduplication(self) -> Optional[DeduplicationProvider]:
+        """Get the deduplication provider."""
+        return self._deduplication
+    
+    def _create_embedding_provider(self) -> EmbeddingProvider:
+        """Create embedding provider based on config."""
+        from ..config.providers import EmbeddingProviderType
+        from ..providers.openai import OpenAIEmbeddingProvider
+        from ..providers.mock import MockEmbeddingProvider
+        
+        cfg = self.config.embedding
+        
+        if cfg.provider == EmbeddingProviderType.OPENAI:
+            return OpenAIEmbeddingProvider(cfg)
+        elif cfg.provider == EmbeddingProviderType.MOCK:
+            return MockEmbeddingProvider(cfg)
+        else:
+            raise ValueError(f"Unknown embedding provider: {cfg.provider}")
+    
+    def _create_storage_provider(self) -> StorageProvider:
+        """Create storage provider based on config."""
+        from ..config.providers import StorageProviderType
+        from ..providers.lancedb import LanceDBStorageProvider
+        from ..providers.memory import InMemoryStorageProvider
+        
+        cfg = self.config.storage
+        
+        if cfg.provider == StorageProviderType.LANCEDB:
+            return LanceDBStorageProvider(cfg, self._embedding)
+        elif cfg.provider == StorageProviderType.MEMORY:
+            return InMemoryStorageProvider(cfg, self._embedding)
+        else:
+            raise ValueError(f"Unknown storage provider: {cfg.provider}")
+    
+    def _create_timestamp_provider(self) -> TimestampProvider:
+        """Create timestamp provider based on config."""
+        from ..config.providers import TimestampProviderType
+        from ..providers.timestamp import RFC3161TimestampProvider, MockTimestampProvider
+        
+        cfg = self.config.timestamp
+        
+        if cfg.provider == TimestampProviderType.RFC3161:
+            return RFC3161TimestampProvider(cfg)
+        elif cfg.provider == TimestampProviderType.MOCK:
+            return MockTimestampProvider(cfg)
+        else:
+            raise ValueError(f"Unknown timestamp provider: {cfg.provider}")
+    
+    def _create_deduplication_provider(
+        self,
+        storage: StorageProvider,
+        embedding: EmbeddingProvider,
+    ) -> Optional[DeduplicationProvider]:
+        """Create deduplication provider based on config.
+        
+        Args:
+            storage: Initialized storage provider
+            embedding: Initialized embedding provider
+            
+        Returns:
+            DeduplicationProvider or None if disabled
+        """
+        if not self.config.deduplication.enabled:
+            return None
+        
+        from ..providers.deduplication import EmbeddingDeduplicationProvider
+        
+        return EmbeddingDeduplicationProvider(
+            self.config.deduplication,
+            storage_provider=storage,
+            embedding_provider=embedding,
+        )
+    
+    async def __aenter__(self):
+        await self.initialize()
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.shutdown()

tribalmemory/a21/providers/__init__.py

@@ -0,0 +1,32 @@
+"""Provider interfaces and implementations.
+
+Providers are swappable backends that implement standard interfaces.
+Each provider type has an abstract base and concrete implementations.
+"""
+
+from .base import (
+    EmbeddingProvider,
+    StorageProvider,
+    TimestampProvider,
+    DeduplicationProvider,
+)
+from .openai import OpenAIEmbeddingProvider
+from .lancedb import LanceDBStorageProvider
+from .memory import InMemoryStorageProvider
+from .timestamp import RFC3161TimestampProvider, MockTimestampProvider
+
+__all__ = [
+    # Base interfaces
+    "EmbeddingProvider",
+    "StorageProvider",
+    "TimestampProvider",
+    "DeduplicationProvider",
+    # Embedding providers
+    "OpenAIEmbeddingProvider",
+    # Storage providers
+    "LanceDBStorageProvider",
+    "InMemoryStorageProvider",
+    # Timestamp providers
+    "RFC3161TimestampProvider",
+    "MockTimestampProvider",
+]