slide-narrator 5.5.0__py3-none-any.whl

narrator/database/thread_store.py

@@ -0,0 +1,282 @@
+"""Thread storage implementation."""
+from typing import Optional, Dict, Any, List
+from ..models.thread import Thread
+from ..models.message import Message
+from ..utils.logging import get_logger
+from .storage_backend import MemoryBackend, SQLBackend
+
+logger = get_logger(__name__)
+
+class ThreadStore:
+    """
+    Thread storage implementation with pluggable backends.
+    Supports both in-memory and SQL (SQLite/PostgreSQL) storage.
+    
+    Key characteristics:
+    - Unified interface for all storage types
+    - Memory backend for development/testing (default)
+    - SQLite for local persistence
+    - PostgreSQL for production
+    - Built-in connection pooling for SQLBackend
+    
+    Usage:
+        # RECOMMENDED: Factory pattern for immediate connection validation
+        from narrator import ThreadStore
+        store = await ThreadStore.create("postgresql+asyncpg://user:pass@localhost/dbname")
+        
+        # Or for in-memory storage:
+        store = await ThreadStore.create()  # Uses memory backend
+        
+        # Direct constructor (connects on first operation):
+        store = ThreadStore("postgresql+asyncpg://user:pass@localhost/dbname")
+        
+    Connection pooling settings can be configured via environment variables:
+            - NARRATOR_DB_POOL_SIZE: Max number of connections to keep open (default: 5)
+    - NARRATOR_DB_MAX_OVERFLOW: Max number of connections to create above pool_size (default: 10)
+    - NARRATOR_DB_POOL_TIMEOUT: Seconds to wait for a connection from pool (default: 30)
+    - NARRATOR_DB_POOL_RECYCLE: Seconds after which a connection is recycled (default: 300)
+    """
+    
+    def __init__(self, database_url = None):
+        """
+        Initialize thread store with optional database URL.
+        If no URL is provided, uses in-memory storage by default.
+        This constructor doesn't establish database connections - they happen on first use.
+        
+        For immediate connection validation, use the async factory method:
+        `store = await ThreadStore.create(database_url)`
+        
+        Args:
+            database_url: SQLAlchemy async database URL. Examples:
+                - "postgresql+asyncpg://user:pass@localhost/dbname"
+                - "sqlite+aiosqlite:///path/to/db.sqlite"
+                - None for in-memory storage
+        """
+        if database_url is None:
+            # Default to in-memory storage
+            logger.info("No database URL provided. Using in-memory storage.")
+            self._backend = MemoryBackend()
+        else:
+            # Use SQLBackend with the provided URL
+            logger.info(f"Using database URL: {database_url}")
+            self._backend = SQLBackend(database_url)
+        
+        # Add initialization flag
+        self._initialized = False
+    
+    @classmethod
+    async def create(cls, database_url = None):
+        """
+        Factory method to create and initialize a ThreadStore.
+        This method connects to the database immediately, allowing early validation
+        of connection parameters.
+        
+        Args:
+            database_url: SQLAlchemy async database URL. Examples:
+                - "postgresql+asyncpg://user:pass@localhost/dbname"
+                - "sqlite+aiosqlite:///path/to/db.sqlite"
+                - None for in-memory storage
+                
+        Returns:
+            Initialized ThreadStore instance
+            
+        Raises:
+            Exception: If database connection fails
+        """
+        # Create instance
+        store = cls(database_url)
+        
+        # Initialize immediately
+        try:
+            await store.initialize()
+        except Exception as e:
+            # If a database URL was provided but initialization failed, we should raise the error
+            # instead of silently falling back to memory storage
+            if database_url is not None:
+                raise RuntimeError(f"Failed to initialize database with URL {database_url}: {str(e)}") from e
+            raise
+        
+        return store
+    
+    async def _ensure_initialized(self) -> None:
+        """Ensure the storage backend is initialized."""
+        if not self._initialized:
+            await self.initialize()
+            self._initialized = True
+    
+    async def initialize(self) -> None:
+        """Initialize the storage backend."""
+        await self._backend.initialize()
+        self._initialized = True
+    
+    async def save(self, thread: Thread) -> Thread:
+        """
+        Save a thread to storage, filtering out system messages.
+        
+        System messages are not persisted to storage by design, but are kept
+        in the original Thread object in memory.
+        
+        Args:
+            thread: The Thread object to save
+            
+        Returns:
+            The original Thread object (with system messages intact)
+        """
+        await self._ensure_initialized()
+        
+        # Create a filtered copy of the thread without system messages
+        filtered_thread = Thread(
+            id=thread.id,
+            title=thread.title,
+            created_at=thread.created_at,
+            updated_at=thread.updated_at,
+            attributes=thread.attributes.copy() if thread.attributes else {},
+            platforms=thread.platforms.copy() if thread.platforms else {}
+        )
+        
+        # Only copy non-system messages to the filtered thread
+        for message in thread.messages:
+            if message.role != "system":
+                # We create a shallow copy of the message to preserve the original
+                filtered_thread.messages.append(message)
+        
+        # Save the filtered thread to storage
+        await self._backend.save(filtered_thread)
+        
+        # Return the original thread (with system messages intact)
+        return thread
+    
+    async def get(self, thread_id: str) -> Optional[Thread]:
+        """Get a thread by ID."""
+        await self._ensure_initialized()
+        return await self._backend.get(thread_id)
+    
+    async def delete(self, thread_id: str) -> bool:
+        """Delete a thread by ID."""
+        await self._ensure_initialized()
+        return await self._backend.delete(thread_id)
+    
+    async def list(self, limit: int = 100, offset: int = 0) -> List[Thread]:
+        """List threads with pagination."""
+        await self._ensure_initialized()
+        return await self._backend.list(limit, offset)
+    
+    async def find_by_attributes(self, attributes: Dict[str, Any]) -> List[Thread]:
+        """Find threads by matching attributes."""
+        await self._ensure_initialized()
+        return await self._backend.find_by_attributes(attributes)
+    
+    async def find_by_platform(self, platform_name: str, properties: Dict[str, Any]) -> List[Thread]:
+        """Find threads by platform name and properties."""
+        await self._ensure_initialized()
+        return await self._backend.find_by_platform(platform_name, properties)
+    
+    async def list_recent(self, limit: Optional[int] = None) -> List[Thread]:
+        """List recent threads."""
+        await self._ensure_initialized()
+        return await self._backend.list_recent(limit)
+        
+    async def find_messages_by_attribute(self, path: str, value: Any) -> List[Message]:
+        """
+        Find messages with a specific attribute at a given JSON path.
+        This is useful for finding messages with specific metadata (like a Slack ts).
+        
+        Args:
+            path: Dot-notation path to the attribute (e.g., "platforms.slack.ts")
+            value: The value to search for
+            
+        Returns:
+            List of Message objects that match the criteria (possibly empty)
+        """
+        await self._ensure_initialized()
+        if hasattr(self._backend, 'find_messages_by_attribute'):
+            results = await self._backend.find_messages_by_attribute(path, value)
+            
+            # Handle different return types from different backends
+            messages = []
+            for item in results:
+                if hasattr(item, 'model_dump'):  # It's a Message object (from MemoryBackend)
+                    messages.append(item)
+                elif hasattr(self._backend, '_create_message_from_record'):  # It's a MessageRecord (from SQLBackend)
+                    message = self._backend._create_message_from_record(item)
+                    messages.append(message)
+            
+            return messages
+        else:
+            # Fallback implementation for backends that don't support this method
+            # This is less efficient but provides compatibility
+            messages = []
+            threads = await self._backend.list_recent(100)  # Get recent threads
+            
+            # Check each thread's messages
+            for thread in threads:
+                for message in thread.messages:
+                    # Navigate the path to get the value
+                    current = message
+                    parts = path.split('.')
+                    
+                    for part in parts:
+                        if isinstance(current, dict) and part in current:
+                            current = current[part]
+                        elif hasattr(current, part):
+                            current = getattr(current, part)
+                        else:
+                            current = None
+                            break
+                    
+                    # Check if we found a match
+                    if current == value:
+                        messages.append(message)
+            
+            return messages
+
+    # Add properties to expose backend attributes
+    @property
+    def database_url(self):
+        return getattr(self._backend, "database_url", None)
+
+    @property
+    def engine(self):
+        return getattr(self._backend, "engine", None)
+
+    async def get_thread_by_message_id(self, message_id: str) -> Optional[Thread]:
+        """
+        Find a thread containing a specific message ID.
+        
+        Args:
+            message_id: The ID of the message to find
+            
+        Returns:
+            The Thread containing the message, or None if not found
+        """
+        await self._ensure_initialized()
+        
+        # Check if backend has native implementation
+        if hasattr(self._backend, 'get_thread_by_message_id'):
+            return await self._backend.get_thread_by_message_id(message_id)
+        
+        # Fallback implementation for backends that don't support this method
+        threads = await self._backend.list_recent(500)  # Get recent threads
+        
+        # Check each thread's messages for the message ID
+        for thread in threads:
+            for message in thread.messages:
+                if message.id == message_id:
+                    return thread
+        
+        return None
+
+# Optional PostgreSQL-specific implementation
+try:
+    import asyncpg
+    
+    class SQLAlchemyThreadStore(ThreadStore):
+        """PostgreSQL-based thread storage for production use."""
+        
+        def __init__(self, database_url):
+            if not database_url.startswith('postgresql+asyncpg://'):
+                database_url = database_url.replace('postgresql://', 'postgresql+asyncpg://')
+            super().__init__(database_url)
+        
+except ImportError:
+    pass 

narrator/models/__init__.py

@@ -0,0 +1,9 @@
+"""
+Models package for Tyler Stores
+"""
+
+from .thread import Thread
+from .message import Message
+from .attachment import Attachment
+
+__all__ = ["Thread", "Message", "Attachment"] 

narrator/models/attachment.py

@@ -0,0 +1,386 @@
+from typing import Dict, Optional, Any, Union, Literal
+from pydantic import BaseModel, computed_field
+import base64
+import io
+import filetype
+import mimetypes
+import logging
+from pathlib import Path
+from ..storage.file_store import FileStore
+import hashlib
+
+class Attachment(BaseModel):
+    """Represents a file attached to a message"""
+    filename: str
+    content: Optional[Union[bytes, str]] = None  # Can be either bytes or base64 string
+    mime_type: Optional[str] = None
+    attributes: Optional[Dict[str, Any]] = None  # Renamed from processed_content
+    file_id: Optional[str] = None  # Reference to stored file
+    storage_path: Optional[str] = None  # Path in storage backend
+    storage_backend: Optional[str] = None  # Storage backend type
+    status: Literal["pending", "stored", "failed"] = "pending"
+
+    @computed_field
+    @property
+    def id(self) -> str:
+        """Generate a unique ID based on content hash"""
+        if self.content is None:
+            # If no content, use filename and other attributes
+            hash_input = f"{self.filename}{self.mime_type or ''}"
+            return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
+        
+        # Get content as bytes for hashing
+        if isinstance(self.content, bytes):
+            content_bytes = self.content
+        elif isinstance(self.content, str):
+            # Try to decode as base64 first
+            try:
+                content_bytes = base64.b64decode(self.content)
+            except:
+                # If not base64, encode as UTF-8
+                content_bytes = self.content.encode('utf-8')
+        else:
+            # Fallback to filename hash
+            return hashlib.sha256(self.filename.encode()).hexdigest()[:16]
+            
+        # Create hash of filename + content
+        hash_input = self.filename.encode() + content_bytes
+        return hashlib.sha256(hash_input).hexdigest()[:16]
+
+    @classmethod
+    def from_file_path(cls, file_path: Union[str, Path]) -> 'Attachment':
+        """Create an attachment from a file path"""
+        file_path = Path(file_path)
+        
+        if not file_path.exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+        
+        # Read file content
+        content = file_path.read_bytes()
+        
+        # Detect MIME type
+        mime_type = filetype.guess_mime(content)
+        
+        if not mime_type:
+            # Fallback: extension-based detection
+            mime_type, _ = mimetypes.guess_type(str(file_path))
+        
+        if not mime_type:
+            # Default: binary
+            mime_type = 'application/octet-stream'
+        
+        return cls(
+            filename=file_path.name,
+            content=content,
+            mime_type=mime_type
+        )
+
+    def detect_mime_type(self) -> None:
+        """Detect and set MIME type from content"""
+        if self.content is None:
+            logging.getLogger(__name__).warning(f"Cannot detect MIME type for {self.filename}: no content")
+            return
+        
+        # Get content as bytes
+        if isinstance(self.content, bytes):
+            content_bytes = self.content
+        elif isinstance(self.content, str):
+            try:
+                content_bytes = base64.b64decode(self.content)
+            except:
+                content_bytes = self.content.encode('utf-8')
+        else:
+            logging.getLogger(__name__).warning(f"Cannot detect MIME type for {self.filename}: invalid content type")
+            return
+        
+        # Detect MIME type
+        detected_mime_type = filetype.guess_mime(content_bytes)
+        
+        if not detected_mime_type:
+            # Fallback: extension-based detection
+            detected_mime_type, _ = mimetypes.guess_type(self.filename)
+        
+        if not detected_mime_type:
+            # Default: binary
+            detected_mime_type = 'application/octet-stream'
+        
+        if not self.mime_type:
+            self.mime_type = detected_mime_type
+            logging.getLogger(__name__).debug(f"Detected MIME type for {self.filename}: {self.mime_type}")
+        else:
+            logging.getLogger(__name__).debug(f"MIME type already set for {self.filename}: {self.mime_type}")
+
+    def model_dump(self, mode: str = "json") -> Dict[str, Any]:
+        """Convert attachment to a dictionary suitable for JSON serialization
+        
+        Args:
+            mode: Serialization mode, either "json" or "python". 
+                 "json" converts datetimes to ISO strings (default).
+                 "python" keeps datetimes as datetime objects.
+        """
+        data = {
+            "filename": self.filename,
+            "mime_type": self.mime_type,
+            "attributes": self.attributes,
+            "file_id": self.file_id,
+            "storage_path": self.storage_path,
+            "storage_backend": self.storage_backend,
+            "status": self.status
+        }
+        
+        return data
+        
+    async def get_content_bytes(self, file_store: Optional[FileStore] = None) -> bytes:
+        """Get the content as bytes, converting from base64 if necessary
+        
+        If file_id is present, retrieves content from file storage.
+        Otherwise falls back to content field.
+        
+        Args:
+            file_store: FileStore instance to use for retrieving file content.
+                       Required when file_id is present.
+        """
+        logging.getLogger(__name__).debug(f"Getting content bytes for {self.filename}")
+        
+        if self.file_id:
+            logging.getLogger(__name__).debug(f"Retrieving content from file store for file_id: {self.file_id}")
+            if file_store is None:
+                raise ValueError("FileStore instance required to retrieve content for file_id")
+            if self.storage_path is None:
+                raise ValueError("storage_path required to retrieve content for file_id")
+            return await file_store.get(self.file_id, self.storage_path)
+            
+        if isinstance(self.content, bytes):
+            logging.getLogger(__name__).debug(f"Content is already in bytes format for {self.filename}")
+            return self.content
+        elif isinstance(self.content, str):
+            logging.getLogger(__name__).debug(f"Converting string content for {self.filename}")
+            if self.content.startswith('data:'):
+                # Handle data URLs
+                logging.getLogger(__name__).debug("Detected data URL format")
+                header, encoded = self.content.split(",", 1)
+                logging.getLogger(__name__).debug(f"Data URL header: {header}")
+                try:
+                    decoded = base64.b64decode(encoded)
+                    logging.getLogger(__name__).debug(f"Successfully decoded data URL content, size: {len(decoded)} bytes")
+                    return decoded
+                except Exception as e:
+                    logging.getLogger(__name__).error(f"Failed to decode data URL content: {e}")
+                    raise
+            else:
+                try:
+                    # Try base64 decode
+                    logging.getLogger(__name__).debug("Attempting base64 decode")
+                    decoded = base64.b64decode(self.content)
+                    logging.getLogger(__name__).debug(f"Successfully decoded base64 content, size: {len(decoded)} bytes")
+                    return decoded
+                except:
+                    logging.getLogger(__name__).debug("Not base64, treating as UTF-8 text")
+                    # If not base64, encode as UTF-8
+                    return self.content.encode('utf-8')
+                
+        raise ValueError("No content available - attachment has neither file_id nor content")
+
+    def update_attributes_with_url(self) -> None:
+        """Update attributes with URL after storage_path is set."""
+        if self.storage_path:
+            if not self.attributes:
+                self.attributes = {}
+            
+            try:
+                # Get the file URL from FileStore
+                self.attributes["url"] = FileStore.get_file_url(self.storage_path)
+                logging.getLogger(__name__).debug(f"Updated attributes with URL: {self.attributes['url']}")
+            except Exception as e:
+                # Log the error but don't fail - the URL will be missing but that's better than crashing
+                logging.getLogger(__name__).error(f"Failed to construct URL for attachment: {e}")
+                self.attributes["error"] = f"Failed to construct URL: {str(e)}"
+
+    async def process_and_store(self, file_store: FileStore, force: bool = False) -> None:
+        """Process the attachment content and store it in the file store.
+        
+        Args:
+            file_store: FileStore instance to use for storing files
+            force: Whether to force processing even if already stored
+        """
+        logging.getLogger(__name__).debug(f"Starting process_and_store for {self.filename} (force={force})")
+        logging.getLogger(__name__).debug(f"Initial state - mime_type: {self.mime_type}, status: {self.status}, content type: {type(self.content)}")
+        
+        if not force and self.status == "stored":
+            logging.getLogger(__name__).info(f"Skipping process_and_store for {self.filename} - already stored")
+            return
+
+        if self.content is None:
+            logging.getLogger(__name__).error(f"Cannot process attachment {self.filename}: no content provided")
+            self.status = "failed"
+            raise RuntimeError(f"Cannot process attachment {self.filename}: no content provided")
+
+        try:
+            # Get content as bytes first
+            logging.getLogger(__name__).debug("Converting content to bytes")
+            content_bytes = await self.get_content_bytes(file_store=file_store)
+            logging.getLogger(__name__).debug(f"Successfully converted content to bytes, size: {len(content_bytes)} bytes")
+
+            # Detect/verify MIME type
+            logging.getLogger(__name__).debug("Detecting MIME type")
+            detected_mime_type = filetype.guess_mime(content_bytes)
+            
+            if not detected_mime_type:
+                # Fallback: extension-based detection
+                detected_mime_type, _ = mimetypes.guess_type(self.filename)
+            
+            if not detected_mime_type:
+                # Default: binary
+                detected_mime_type = 'application/octet-stream'
+            
+            logging.getLogger(__name__).debug(f"Detected MIME type: {detected_mime_type}")
+            
+            if not self.mime_type:
+                self.mime_type = detected_mime_type
+                logging.getLogger(__name__).debug(f"Set MIME type to detected type: {self.mime_type}")
+            elif self.mime_type != detected_mime_type:
+                logging.getLogger(__name__).warning(f"Provided MIME type {self.mime_type} doesn't match detected type {detected_mime_type}")
+
+            # Initialize attributes
+            if not self.attributes:
+                self.attributes = {}
+
+            # Process content based on MIME type
+            logging.getLogger(__name__).debug(f"Processing content based on MIME type: {self.mime_type}")
+            
+            if self.mime_type.startswith('image/'):
+                logging.getLogger(__name__).debug("Processing as image")
+                self.attributes.update({
+                    "type": "image",
+                    "description": f"Image file {self.filename}",
+                    "mime_type": self.mime_type
+                })
+
+            elif self.mime_type.startswith('audio/'):
+                logging.getLogger(__name__).debug("Processing as audio")
+                self.attributes.update({
+                    "type": "audio",
+                    "description": f"Audio file {self.filename}",
+                    "mime_type": self.mime_type
+                })
+
+            elif self.mime_type == 'application/pdf':
+                logging.getLogger(__name__).debug("Processing as PDF")
+                try:
+                    from pypdf import PdfReader
+                    reader = PdfReader(io.BytesIO(content_bytes))
+                    text = ""
+                    for page in reader.pages:
+                        try:
+                            extracted = page.extract_text()
+                            if extracted:
+                                text += extracted + "\n"
+                        except Exception as e:
+                            logging.getLogger(__name__).warning(f"Error extracting text from PDF page: {e}")
+                            continue
+                    self.attributes.update({
+                        "type": "document",
+                        "text": text.strip(),
+                        "overview": f"Extracted text from {self.filename}",
+                        "mime_type": self.mime_type
+                    })
+                except ImportError:
+                    logging.getLogger(__name__).warning("pypdf not available, skipping PDF text extraction")
+                    self.attributes.update({
+                        "type": "document",
+                        "description": f"PDF document {self.filename}",
+                        "mime_type": self.mime_type
+                    })
+
+            elif self.mime_type.startswith('text/'):
+                logging.getLogger(__name__).debug("Processing as text")
+                try:
+                    text = content_bytes.decode('utf-8')
+                    self.attributes.update({
+                        "type": "text",
+                        "text": text,
+                        "mime_type": self.mime_type
+                    })
+                except UnicodeDecodeError:
+                    logging.getLogger(__name__).warning("UTF-8 decode failed, trying alternative encodings")
+                    # Try alternative encodings
+                    for encoding in ['latin-1', 'cp1252', 'iso-8859-1']:
+                        try:
+                            text = content_bytes.decode(encoding)
+                            self.attributes.update({
+                                "type": "text",
+                                "text": text,
+                                "encoding": encoding,
+                                "mime_type": self.mime_type
+                            })
+                            logging.getLogger(__name__).debug(f"Successfully decoded text using {encoding}")
+                            break
+                        except UnicodeDecodeError:
+                            continue
+
+            elif self.mime_type == 'application/json':
+                logging.getLogger(__name__).debug("Processing as JSON")
+                import json
+                try:
+                    json_text = content_bytes.decode('utf-8')
+                    json_data = json.loads(json_text)
+                    self.attributes.update({
+                        "type": "json",
+                        "overview": "JSON data structure",
+                        "parsed_content": json_data,
+                        "mime_type": self.mime_type
+                    })
+                except Exception as e:
+                    logging.getLogger(__name__).warning(f"Error parsing JSON content: {e}")
+                    self.attributes.update({
+                        "type": "json",
+                        "error": f"Failed to parse JSON: {str(e)}",
+                        "mime_type": self.mime_type
+                    })
+
+            else:
+                logging.getLogger(__name__).debug(f"Processing as binary file with MIME type: {self.mime_type}")
+                self.attributes.update({
+                    "type": "binary",
+                    "description": f"Binary file {self.filename}",
+                    "mime_type": self.mime_type
+                })
+
+            # Store the file
+            logging.getLogger(__name__).debug("Storing file in FileStore")
+            
+            try:
+                logging.getLogger(__name__).debug(f"Saving file to storage, content size: {len(content_bytes)} bytes")
+                result = await file_store.save(content_bytes, self.filename, self.mime_type)
+                logging.getLogger(__name__).debug(f"Successfully saved file. Result: {result}")
+                
+                self.file_id = result['id']
+                self.storage_backend = result['storage_backend']
+                self.storage_path = result['storage_path']
+                self.status = "stored"
+                
+                # Update filename to match the one created by the file store
+                # Extract the actual filename from the storage path
+                new_filename = Path(self.storage_path).name
+                logging.getLogger(__name__).debug(f"Updating attachment filename from {self.filename} to {new_filename}")
+                self.filename = new_filename
+                
+                # Add storage info to attributes
+                self.attributes["storage_path"] = self.storage_path
+                self.update_attributes_with_url()
+                
+                # Clear content after successful storage
+                self.content = None
+                logging.getLogger(__name__).debug(f"Cleared content after successful storage for {self.filename}")
+                
+                logging.getLogger(__name__).debug(f"Successfully processed and stored attachment {self.filename}")
+                
+            except Exception as e:
+                logging.getLogger(__name__).error(f"Error processing attachment {self.filename}: {e}")
+                self.status = "failed"
+                raise
+
+        except Exception as e:
+            logging.getLogger(__name__).error(f"Failed to process attachment {self.filename}: {str(e)}")
+            self.status = "failed"
+            raise RuntimeError(f"Failed to process attachment {self.filename}: {str(e)}") from e