keep-skill 0.1.0__py3-none-any.whl

keep/providers/base.py

@@ -0,0 +1,416 @@
+"""
+Base provider protocols.
+
+These define the interfaces that concrete providers must implement.
+Using Protocol for structural subtyping - no explicit inheritance required.
+"""
+
+from dataclasses import dataclass
+from typing import Any, Protocol, runtime_checkable
+
+
+# -----------------------------------------------------------------------------
+# Document Fetching
+# -----------------------------------------------------------------------------
+
+@dataclass
+class Document:
+    """
+    A fetched document ready for processing.
+    
+    Attributes:
+        uri: Original URI that was fetched
+        content: Text content of the document
+        content_type: MIME type if known (e.g., "text/markdown", "text/plain")
+        metadata: Additional metadata from the source (headers, file stats, etc.)
+    """
+    uri: str
+    content: str
+    content_type: str | None = None
+    metadata: dict[str, Any] | None = None
+
+
+@runtime_checkable
+class DocumentProvider(Protocol):
+    """
+    Fetches document content from a URI.
+    
+    Implementations handle specific URI schemes (file://, https://, s3://, etc.)
+    and convert the content to text.
+    
+    Example implementation:
+        class FileDocumentProvider:
+            def supports(self, uri: str) -> bool:
+                return uri.startswith("file://")
+            
+            def fetch(self, uri: str) -> Document:
+                path = uri.removeprefix("file://")
+                content = Path(path).read_text()
+                return Document(uri=uri, content=content, content_type="text/plain")
+    """
+    
+    def supports(self, uri: str) -> bool:
+        """
+        Check if this provider can handle the given URI.
+        
+        Args:
+            uri: The URI to check
+            
+        Returns:
+            True if this provider can fetch the URI
+        """
+        ...
+    
+    def fetch(self, uri: str) -> Document:
+        """
+        Fetch and return the document content.
+        
+        Args:
+            uri: The URI to fetch
+            
+        Returns:
+            Document with text content
+            
+        Raises:
+            ValueError: If URI is malformed
+            IOError: If document cannot be fetched
+        """
+        ...
+
+
+# -----------------------------------------------------------------------------
+# Embedding Generation
+# -----------------------------------------------------------------------------
+
+@runtime_checkable
+class EmbeddingProvider(Protocol):
+    """
+    Generates vector embeddings from text.
+    
+    Embeddings enable semantic similarity search. The same provider instance
+    must be used for both indexing and querying to ensure consistent vectors.
+    
+    Example implementation:
+        class SentenceTransformerEmbedding:
+            def __init__(self, model_name: str = "all-MiniLM-L6-v2"):
+                from sentence_transformers import SentenceTransformer
+                self.model = SentenceTransformer(model_name)
+            
+            @property
+            def dimension(self) -> int:
+                return self.model.get_sentence_embedding_dimension()
+            
+            def embed(self, text: str) -> list[float]:
+                return self.model.encode(text).tolist()
+            
+            def embed_batch(self, texts: list[str]) -> list[list[float]]:
+                return self.model.encode(texts).tolist()
+    """
+    
+    @property
+    def dimension(self) -> int:
+        """
+        The dimensionality of the embedding vectors.
+        
+        This must be consistent across all calls. ChromaDb and other vector
+        stores need this to configure the index.
+        """
+        ...
+    
+    def embed(self, text: str) -> list[float]:
+        """
+        Generate an embedding vector for the given text.
+        
+        Args:
+            text: The text to embed
+            
+        Returns:
+            A list of floats representing the embedding vector
+        """
+        ...
+    
+    def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """
+        Generate embeddings for multiple texts.
+        
+        Batch processing is often more efficient than individual calls.
+        
+        Args:
+            texts: List of texts to embed
+            
+        Returns:
+            List of embedding vectors, one per input text
+        """
+        ...
+
+
+# -----------------------------------------------------------------------------
+# Summarization
+# -----------------------------------------------------------------------------
+
+@runtime_checkable
+class SummarizationProvider(Protocol):
+    """
+    Generates concise summaries of document content.
+    
+    Summaries are stored alongside items and enable quick recall without
+    fetching the original document. They're also used for full-text search.
+    
+    Example implementation:
+        class OpenAISummarization:
+            def __init__(self, model: str = "gpt-4o-mini"):
+                self.client = OpenAI()
+                self.model = model
+            
+            def summarize(self, content: str, max_length: int = 500) -> str:
+                response = self.client.chat.completions.create(
+                    model=self.model,
+                    messages=[
+                        {"role": "system", "content": "Summarize concisely."},
+                        {"role": "user", "content": content}
+                    ],
+                    max_tokens=max_length
+                )
+                return response.choices[0].message.content
+    """
+    
+    def summarize(self, content: str, *, max_length: int = 500) -> str:
+        """
+        Generate a summary of the content.
+        
+        Args:
+            content: The full document content
+            max_length: Approximate maximum length in characters
+            
+        Returns:
+            A concise summary of the content
+        """
+        ...
+
+
+# -----------------------------------------------------------------------------
+# Tagging
+# -----------------------------------------------------------------------------
+
+@runtime_checkable
+class TaggingProvider(Protocol):
+    """
+    Generates structured tags from document content.
+    
+    Tags enable traditional navigation and filtering. The provider analyzes
+    content and returns relevant key-value pairs.
+    
+    Example implementation:
+        class OpenAITagging:
+            def __init__(self, model: str = "gpt-4o-mini"):
+                self.client = OpenAI()
+                self.model = model
+            
+            def tag(self, content: str) -> dict[str, str]:
+                response = self.client.chat.completions.create(
+                    model=self.model,
+                    messages=[...],
+                    response_format={"type": "json_object"}
+                )
+                return json.loads(response.choices[0].message.content)
+    """
+    
+    def tag(self, content: str) -> dict[str, str]:
+        """
+        Generate tags for the content.
+        
+        Args:
+            content: The full document content
+            
+        Returns:
+            Dictionary of tag key-value pairs
+            
+        Note:
+            Keys should be lowercase with underscores (e.g., "content_type").
+            Values should be simple strings.
+            System tags (keys starting with "_") should not be generated here.
+        """
+        ...
+
+
+# -----------------------------------------------------------------------------
+# Provider Registry
+# -----------------------------------------------------------------------------
+
+class ProviderRegistry:
+    """
+    Registry for discovering and instantiating providers.
+
+    Providers are registered by name and can be instantiated from configuration.
+    This allows the store configuration (TOML) to specify providers by name
+    rather than requiring code changes.
+
+    Example:
+        registry = ProviderRegistry()
+        registry.register_embedding("sentence-transformers", SentenceTransformerEmbedding)
+        registry.register_embedding("openai", OpenAIEmbedding)
+
+        # Later, from config:
+        provider = registry.create_embedding("sentence-transformers", {"model": "all-MiniLM-L6-v2"})
+    """
+
+    def __init__(self):
+        self._embedding_providers: dict[str, type] = {}
+        self._summarization_providers: dict[str, type] = {}
+        self._tagging_providers: dict[str, type] = {}
+        self._document_providers: dict[str, type] = {}
+        self._lazy_loaded = False
+    
+    def _ensure_providers_loaded(self) -> None:
+        """Lazily load all provider modules."""
+        if self._lazy_loaded:
+            return
+
+        self._lazy_loaded = True
+
+        # Import provider modules to trigger registration
+        # These imports are safe - they only register classes, don't instantiate
+        try:
+            from . import documents
+        except ImportError:
+            pass  # Document provider might not be available
+
+        try:
+            from . import embeddings
+        except ImportError:
+            pass  # Embedding providers might not be available
+
+        try:
+            from . import summarization
+        except ImportError:
+            pass  # Summarization providers might not be available
+
+        try:
+            from . import llm
+        except ImportError:
+            pass  # LLM providers might not be available
+
+        try:
+            from . import mlx
+        except ImportError:
+            pass  # MLX providers might not be available
+
+    # Registration methods
+
+    def register_embedding(self, name: str, provider_class: type) -> None:
+        """Register an embedding provider class."""
+        self._embedding_providers[name] = provider_class
+    
+    def register_summarization(self, name: str, provider_class: type) -> None:
+        """Register a summarization provider class."""
+        self._summarization_providers[name] = provider_class
+    
+    def register_tagging(self, name: str, provider_class: type) -> None:
+        """Register a tagging provider class."""
+        self._tagging_providers[name] = provider_class
+    
+    def register_document(self, name: str, provider_class: type) -> None:
+        """Register a document provider class."""
+        self._document_providers[name] = provider_class
+    
+    # Factory methods
+
+    def create_embedding(self, name: str, params: dict | None = None) -> EmbeddingProvider:
+        """Create an embedding provider instance."""
+        self._ensure_providers_loaded()
+        if name not in self._embedding_providers:
+            available = ", ".join(self._embedding_providers.keys()) or "none"
+            raise ValueError(
+                f"Unknown embedding provider: '{name}'. "
+                f"Available providers: {available}. "
+                f"Install missing dependencies or check provider name."
+            )
+        try:
+            return self._embedding_providers[name](**(params or {}))
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to create embedding provider '{name}': {e}\n"
+                f"Make sure required dependencies are installed."
+            ) from e
+    
+    def create_summarization(self, name: str, params: dict | None = None) -> SummarizationProvider:
+        """Create a summarization provider instance."""
+        self._ensure_providers_loaded()
+        if name not in self._summarization_providers:
+            available = ", ".join(self._summarization_providers.keys()) or "none"
+            raise ValueError(
+                f"Unknown summarization provider: '{name}'. "
+                f"Available providers: {available}. "
+                f"Install missing dependencies or check provider name."
+            )
+        try:
+            return self._summarization_providers[name](**(params or {}))
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to create summarization provider '{name}': {e}\n"
+                f"Make sure required dependencies are installed."
+            ) from e
+    
+    def create_tagging(self, name: str, params: dict | None = None) -> TaggingProvider:
+        """Create a tagging provider instance."""
+        self._ensure_providers_loaded()
+        if name not in self._tagging_providers:
+            available = ", ".join(self._tagging_providers.keys()) or "none"
+            raise ValueError(
+                f"Unknown tagging provider: '{name}'. "
+                f"Available providers: {available}. "
+                f"Install missing dependencies or check provider name."
+            )
+        try:
+            return self._tagging_providers[name](**(params or {}))
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to create tagging provider '{name}': {e}\n"
+                f"Make sure required dependencies are installed."
+            ) from e
+    
+    def create_document(self, name: str, params: dict | None = None) -> DocumentProvider:
+        """Create a document provider instance."""
+        self._ensure_providers_loaded()
+        if name not in self._document_providers:
+            available = ", ".join(self._document_providers.keys()) or "none"
+            raise ValueError(
+                f"Unknown document provider: '{name}'. "
+                f"Available providers: {available}. "
+                f"Install missing dependencies or check provider name."
+            )
+        try:
+            return self._document_providers[name](**(params or {}))
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to create document provider '{name}': {e}\n"
+                f"Make sure required dependencies are installed."
+            ) from e
+    
+    # Introspection
+    
+    def list_embedding_providers(self) -> list[str]:
+        """List registered embedding provider names."""
+        return list(self._embedding_providers.keys())
+    
+    def list_summarization_providers(self) -> list[str]:
+        """List registered summarization provider names."""
+        return list(self._summarization_providers.keys())
+    
+    def list_tagging_providers(self) -> list[str]:
+        """List registered tagging provider names."""
+        return list(self._tagging_providers.keys())
+    
+    def list_document_providers(self) -> list[str]:
+        """List registered document provider names."""
+        return list(self._document_providers.keys())
+
+
+# Global registry instance
+# Concrete providers register themselves on import
+_registry = ProviderRegistry()
+
+
+def get_registry() -> ProviderRegistry:
+    """Get the global provider registry."""
+    return _registry

keep/providers/documents.py

@@ -0,0 +1,250 @@
+"""
+Document providers for fetching content from various URI schemes.
+"""
+
+from pathlib import Path
+from urllib.parse import urlparse
+
+from .base import Document, DocumentProvider, get_registry
+
+
+class FileDocumentProvider:
+    """
+    Fetches documents from the local filesystem.
+
+    Supports file:// URIs and attempts to detect content type from extension.
+    Performs text extraction for PDF and HTML files.
+    """
+
+    EXTENSION_TYPES = {
+        ".md": "text/markdown",
+        ".markdown": "text/markdown",
+        ".txt": "text/plain",
+        ".py": "text/x-python",
+        ".js": "text/javascript",
+        ".ts": "text/typescript",
+        ".json": "application/json",
+        ".yaml": "text/yaml",
+        ".yml": "text/yaml",
+        ".html": "text/html",
+        ".htm": "text/html",
+        ".css": "text/css",
+        ".xml": "application/xml",
+        ".rst": "text/x-rst",
+        ".pdf": "application/pdf",
+    }
+    
+    def supports(self, uri: str) -> bool:
+        """Check if this is a file:// URI or bare path."""
+        return uri.startswith("file://") or uri.startswith("/")
+    
+    def fetch(self, uri: str) -> Document:
+        """Read file content from the filesystem with text extraction for PDF/HTML."""
+        # Normalize to path
+        if uri.startswith("file://"):
+            path_str = uri.removeprefix("file://")
+        else:
+            path_str = uri
+
+        path = Path(path_str)
+
+        if not path.exists():
+            raise IOError(f"File not found: {path}")
+
+        if not path.is_file():
+            raise IOError(f"Not a file: {path}")
+
+        # Detect content type
+        suffix = path.suffix.lower()
+        content_type = self.EXTENSION_TYPES.get(suffix, "text/plain")
+
+        # Extract text based on file type
+        if suffix == ".pdf":
+            content = self._extract_pdf_text(path)
+        elif suffix in (".html", ".htm"):
+            content = self._extract_html_text(path)
+        else:
+            # Read as plain text
+            try:
+                content = path.read_text(encoding="utf-8")
+            except UnicodeDecodeError:
+                raise IOError(f"Cannot read file as text: {path}")
+
+        # Gather metadata
+        stat = path.stat()
+        metadata = {
+            "size": stat.st_size,
+            "modified": stat.st_mtime,
+            "name": path.name,
+        }
+
+        return Document(
+            uri=f"file://{path.resolve()}",  # Normalize to absolute
+            content=content,
+            content_type=content_type,
+            metadata=metadata,
+        )
+
+    def _extract_pdf_text(self, path: Path) -> str:
+        """Extract text from PDF file."""
+        try:
+            from pypdf import PdfReader
+        except ImportError:
+            raise IOError(
+                f"PDF support requires 'pypdf' library. "
+                f"Install with: pip install pypdf\n"
+                f"Cannot read PDF: {path}"
+            )
+
+        try:
+            reader = PdfReader(path)
+            text_parts = []
+            for page in reader.pages:
+                text = page.extract_text()
+                if text:
+                    text_parts.append(text)
+
+            if not text_parts:
+                raise IOError(f"No text extracted from PDF: {path}")
+
+            return "\n\n".join(text_parts)
+        except Exception as e:
+            raise IOError(f"Failed to extract text from PDF {path}: {e}")
+
+    def _extract_html_text(self, path: Path) -> str:
+        """Extract text from HTML file."""
+        try:
+            from bs4 import BeautifulSoup
+        except ImportError:
+            raise IOError(
+                f"HTML text extraction requires 'beautifulsoup4' library. "
+                f"Install with: pip install beautifulsoup4\n"
+                f"Cannot extract text from HTML: {path}"
+            )
+
+        try:
+            html_content = path.read_text(encoding="utf-8")
+            soup = BeautifulSoup(html_content, "html.parser")
+
+            # Remove script and style elements
+            for script in soup(["script", "style"]):
+                script.decompose()
+
+            # Get text
+            text = soup.get_text()
+
+            # Clean up whitespace
+            lines = (line.strip() for line in text.splitlines())
+            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+            text = '\n'.join(chunk for chunk in chunks if chunk)
+
+            return text
+        except Exception as e:
+            raise IOError(f"Failed to extract text from HTML {path}: {e}")
+
+
+class HttpDocumentProvider:
+    """
+    Fetches documents from HTTP/HTTPS URLs.
+    
+    Requires the `requests` library (optional dependency).
+    """
+    
+    def __init__(self, timeout: int = 30, max_size: int = 10_000_000):
+        """
+        Args:
+            timeout: Request timeout in seconds
+            max_size: Maximum content size in bytes
+        """
+        self.timeout = timeout
+        self.max_size = max_size
+    
+    def supports(self, uri: str) -> bool:
+        """Check if this is an HTTP(S) URL."""
+        return uri.startswith("http://") or uri.startswith("https://")
+    
+    def fetch(self, uri: str) -> Document:
+        """Fetch content from HTTP URL."""
+        try:
+            import requests
+        except ImportError:
+            raise RuntimeError("HTTP document fetching requires 'requests' library")
+
+        try:
+            # Use context manager to ensure connection is closed
+            with requests.get(
+                uri,
+                timeout=self.timeout,
+                headers={"User-Agent": "keep/0.1"},
+                stream=True,
+            ) as response:
+                response.raise_for_status()
+
+                # Check size
+                content_length = response.headers.get("content-length")
+                if content_length and int(content_length) > self.max_size:
+                    raise IOError(f"Content too large: {content_length} bytes")
+
+                # Read content with size limit
+                content = response.text[:self.max_size]
+
+                # Get content type
+                content_type = response.headers.get("content-type", "text/plain")
+                if ";" in content_type:
+                    content_type = content_type.split(";")[0].strip()
+
+                return Document(
+                    uri=uri,
+                    content=content,
+                    content_type=content_type,
+                    metadata={
+                        "status_code": response.status_code,
+                        "headers": dict(response.headers),
+                    },
+                )
+        except requests.RequestException as e:
+            raise IOError(f"Failed to fetch {uri}: {e}")
+
+
+class CompositeDocumentProvider:
+    """
+    Combines multiple document providers, delegating to the appropriate one.
+    
+    This is the default provider used by Keeper.
+    """
+    
+    def __init__(self, providers: list[DocumentProvider] | None = None):
+        """
+        Args:
+            providers: List of providers to try. If None, uses defaults.
+        """
+        if providers is None:
+            self._providers = [
+                FileDocumentProvider(),
+                HttpDocumentProvider(),
+            ]
+        else:
+            self._providers = list(providers)
+    
+    def supports(self, uri: str) -> bool:
+        """Check if any provider supports this URI."""
+        return any(p.supports(uri) for p in self._providers)
+    
+    def fetch(self, uri: str) -> Document:
+        """Fetch using the first provider that supports this URI."""
+        for provider in self._providers:
+            if provider.supports(uri):
+                return provider.fetch(uri)
+        
+        raise ValueError(f"No provider supports URI: {uri}")
+    
+    def add_provider(self, provider: DocumentProvider) -> None:
+        """Add a provider to the list (checked first)."""
+        self._providers.insert(0, provider)
+
+
+# Register providers
+_registry = get_registry()
+_registry.register_document("file", FileDocumentProvider)
+_registry.register_document("http", HttpDocumentProvider)
+_registry.register_document("composite", CompositeDocumentProvider)