stele-context 0.7.0__py3-none-any.whl

stele/__init__.py

@@ -0,0 +1,36 @@
+"""
+Stele — Local context cache for LLM agents with semantic chunking
+and vector search.
+
+Smart context cache that avoids re-reading unchanged files by caching
+chunk data with semantic search. Routes documents through modality-specific
+chunkers, stores chunk content with HNSW vector indexing, and provides
+fast retrieval via semantic search.
+
+Key Features:
+- Dynamic semantic chunking with modality-specific chunkers
+- HNSW vector index for O(log n) similarity search
+- Chunk content persistence for instant retrieval
+- Change detection with hash + semantic comparison
+- Session management with rollback support
+- Built-in MCP server for agent integration
+
+All operations are 100% offline and local-only. No internet access required.
+"""
+
+__version__ = "0.9.0"
+__author__ = "Stele Contributors"
+__license__ = "MIT"
+
+from stele.engine import Stele
+from stele.storage import StorageBackend
+from stele.session import SessionManager
+from stele.mcp_server import MCPServer
+
+__all__ = [
+    "Stele",
+    "StorageBackend",
+    "SessionManager",
+    "MCPServer",
+    "__version__",
+]

stele/bm25.py

@@ -0,0 +1,125 @@
+"""
+BM25 keyword index for Stele.
+
+Provides term-frequency based scoring to complement HNSW vector search
+for hybrid retrieval. Pure Python implementation with zero dependencies.
+"""
+
+import math
+import re
+from collections import Counter
+from typing import Dict, List
+
+_WORD_RE = re.compile(r"\b[a-zA-Z_][a-zA-Z0-9_]*\b")
+
+
+class BM25Index:
+    """
+    Okapi BM25 keyword index for hybrid search.
+
+    Maintains term frequencies and inverse document frequencies
+    for fast keyword scoring. Designed to run alongside HNSW
+    vector search — HNSW finds semantic neighbours, BM25 boosts
+    exact keyword matches.
+    """
+
+    def __init__(self, k1: float = 1.5, b: float = 0.75):
+        self.k1 = k1
+        self.b = b
+        self.doc_freqs: Counter = Counter()
+        self.doc_lengths: Dict[str, int] = {}
+        self.term_freqs: Dict[str, Counter] = {}
+        self.avg_dl: float = 0.0
+        self.n_docs: int = 0
+
+    def add_document(self, doc_id: str, text: str) -> None:
+        """Add or replace a document in the index."""
+        if doc_id in self.term_freqs:
+            self.remove_document(doc_id)
+
+        terms = self._tokenize(text)
+        self.term_freqs[doc_id] = Counter(terms)
+        self.doc_lengths[doc_id] = len(terms)
+        for term in set(terms):
+            self.doc_freqs[term] += 1
+        self.n_docs += 1
+        self._update_avg_dl()
+
+    def remove_document(self, doc_id: str) -> None:
+        """Remove a document from the index."""
+        if doc_id not in self.term_freqs:
+            return
+        for term in self.term_freqs[doc_id]:
+            self.doc_freqs[term] -= 1
+            if self.doc_freqs[term] <= 0:
+                del self.doc_freqs[term]
+        del self.term_freqs[doc_id]
+        del self.doc_lengths[doc_id]
+        self.n_docs -= 1
+        self._update_avg_dl()
+
+    def score(self, query: str, doc_id: str) -> float:
+        """Compute BM25 score for a query against a single document."""
+        return self._score_terms(self._tokenize(query), doc_id)
+
+    def _score_terms(self, query_terms: List[str], doc_id: str) -> float:
+        """Compute BM25 score from pre-tokenized query terms."""
+        if doc_id not in self.term_freqs or self.avg_dl == 0:
+            return 0.0
+
+        total = 0.0
+        dl = self.doc_lengths[doc_id]
+        tf_map = self.term_freqs[doc_id]
+
+        for term in query_terms:
+            n = self.doc_freqs.get(term, 0)
+            if n == 0:
+                continue
+            idf = math.log((self.n_docs - n + 0.5) / (n + 0.5) + 1.0)
+            tf = tf_map.get(term, 0)
+            numerator = tf * (self.k1 + 1.0)
+            denominator = tf + self.k1 * (1.0 - self.b + self.b * dl / self.avg_dl)
+            total += idf * numerator / denominator
+
+        return total
+
+    def score_batch(self, query: str, doc_ids: List[str]) -> Dict[str, float]:
+        """Score multiple documents against a query (tokenizes once)."""
+        terms = self._tokenize(query)
+        return {doc_id: self._score_terms(terms, doc_id) for doc_id in doc_ids}
+
+    def _tokenize(self, text: str) -> List[str]:
+        """Tokenize text into lowercase word terms (len > 1)."""
+        return [w.lower() for w in _WORD_RE.findall(text) if len(w) > 1]
+
+    def _update_avg_dl(self) -> None:
+        """Recompute average document length."""
+        if self.n_docs > 0:
+            self.avg_dl = sum(self.doc_lengths.values()) / self.n_docs
+        else:
+            self.avg_dl = 0.0
+
+    def to_dict(self) -> Dict:
+        """Serialize to a plain dict for persistence."""
+        return {
+            "k1": self.k1,
+            "b": self.b,
+            "doc_freqs": dict(self.doc_freqs),
+            "doc_lengths": self.doc_lengths,
+            "term_freqs": {doc_id: dict(tf) for doc_id, tf in self.term_freqs.items()},
+            "avg_dl": self.avg_dl,
+            "n_docs": self.n_docs,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict) -> "BM25Index":
+        """Reconstruct from serialized dict."""
+        idx = cls(k1=data["k1"], b=data["b"])
+        idx.doc_freqs = Counter(data["doc_freqs"])
+        idx.doc_lengths = data["doc_lengths"]
+        idx.term_freqs = {
+            doc_id: Counter(tf) for doc_id, tf in data["term_freqs"].items()
+        }
+        idx.avg_dl = data["avg_dl"]
+        idx.n_docs = data["n_docs"]
+        return idx

stele/chunkers/__init__.py

@@ -0,0 +1,67 @@
+"""
+Stele chunkers module.
+
+Provides modality-specific chunkers for different file types:
+- TextChunker: Plain text files (zero dependencies)
+- CodeChunker: Code files with AST awareness (zero dependencies)
+- ImageChunker: Image files (requires Pillow)
+- PDFChunker: PDF files (requires pymupdf)
+- AudioChunker: Audio files (requires librosa)
+- VideoChunker: Video files (requires opencv)
+
+All chunkers follow the same interface and can be registered with Stele.
+"""
+
+from stele.chunkers.base import BaseChunker, Chunk
+from stele.chunkers.text import TextChunker
+from stele.chunkers.code import CodeChunker
+
+# Optional chunkers (require additional dependencies)
+# Each chunker module imports successfully even without its optional dependency,
+# but the constructor raises ImportError. Check the inner availability flag.
+try:
+    from stele.chunkers.image import ImageChunker, HAS_PIL
+
+    HAS_IMAGE_CHUNKER = HAS_PIL
+except ImportError:
+    HAS_IMAGE_CHUNKER = False
+    ImageChunker = None  # type: ignore
+
+try:
+    from stele.chunkers.pdf import PDFChunker, HAS_PYMUPDF
+
+    HAS_PDF_CHUNKER = HAS_PYMUPDF
+except ImportError:
+    HAS_PDF_CHUNKER = False
+    PDFChunker = None  # type: ignore
+
+try:
+    from stele.chunkers.audio import AudioChunker, HAS_LIBROSA
+
+    HAS_AUDIO_CHUNKER = HAS_LIBROSA
+except ImportError:
+    HAS_AUDIO_CHUNKER = False
+    AudioChunker = None  # type: ignore
+
+try:
+    from stele.chunkers.video import VideoChunker, HAS_OPENCV
+
+    HAS_VIDEO_CHUNKER = HAS_OPENCV
+except ImportError:
+    HAS_VIDEO_CHUNKER = False
+    VideoChunker = None  # type: ignore
+
+__all__ = [
+    "BaseChunker",
+    "Chunk",
+    "TextChunker",
+    "CodeChunker",
+    "ImageChunker",
+    "PDFChunker",
+    "AudioChunker",
+    "VideoChunker",
+    "HAS_IMAGE_CHUNKER",
+    "HAS_PDF_CHUNKER",
+    "HAS_AUDIO_CHUNKER",
+    "HAS_VIDEO_CHUNKER",
+]

stele/chunkers/audio.py

@@ -0,0 +1,198 @@
+"""
+Audio chunker for Stele.
+
+Splits audio files into time-based segments with MFCC features.
+Requires librosa for audio processing.
+
+Install: pip install stele[audio]
+"""
+
+from typing import Any, Dict, List
+
+from stele.chunkers.base import BaseChunker, Chunk
+
+# Check for librosa
+try:
+    import librosa
+    import numpy as np
+
+    HAS_LIBROSA = True
+except ImportError:
+    HAS_LIBROSA = False
+    librosa = None  # type: ignore
+    np = None  # type: ignore
+
+
+class AudioChunker(BaseChunker):
+    """
+    Chunker for audio files.
+
+    Supports:
+    - Time-based segmentation
+    - MFCC feature extraction
+    - Spectral features
+
+    Requires: librosa (pip install stele[audio])
+    """
+
+    def __init__(
+        self,
+        segment_duration: float = 30.0,  # seconds
+        sample_rate: int = 22050,
+        n_mfcc: int = 13,
+    ):
+        """
+        Initialize audio chunker.
+
+        Args:
+            segment_duration: Duration of each segment in seconds
+            sample_rate: Sample rate for processing
+            n_mfcc: Number of MFCC coefficients
+        """
+        if not HAS_LIBROSA:
+            raise ImportError(
+                "librosa is required for audio support. "
+                "Install with: pip install stele[audio]"
+            )
+
+        self.segment_duration = segment_duration
+        self.sample_rate = sample_rate
+        self.n_mfcc = n_mfcc
+
+    def supported_extensions(self) -> List[str]:
+        """Return supported audio file extensions."""
+        return [
+            ".mp3",
+            ".wav",
+            ".ogg",
+            ".flac",
+            ".m4a",
+            ".aac",
+            ".wma",
+        ]
+
+    def chunk(
+        self,
+        content: Any,
+        document_path: str,
+        **kwargs: Any,
+    ) -> List[Chunk]:
+        """
+        Split audio into chunks.
+
+        Args:
+            content: Audio content (bytes or file path)
+            document_path: Path to source document
+            **kwargs: Additional options
+
+        Returns:
+            List of Chunk objects
+        """
+        # Load audio
+        if isinstance(content, bytes):
+            # Save to temp file for librosa
+            import tempfile
+
+            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+                f.write(content)
+                temp_path = f.name
+
+            try:
+                y, sr = librosa.load(temp_path, sr=self.sample_rate)
+            finally:
+                import os
+
+                os.unlink(temp_path)
+        elif isinstance(content, str):
+            y, sr = librosa.load(content, sr=self.sample_rate)
+        else:
+            raise ValueError(f"Unsupported content type: {type(content)}")
+
+        # Compute segment samples
+        segment_samples = int(self.segment_duration * sr)
+
+        # Create chunks
+        chunks: List[Chunk] = []
+        chunk_index = 0
+
+        for start_sample in range(0, len(y), segment_samples):
+            end_sample = min(start_sample + segment_samples, len(y))
+            segment = y[start_sample:end_sample]
+
+            # Skip very short segments
+            if len(segment) < sr * 0.1:  # Less than 0.1 seconds
+                continue
+
+            # Compute features
+            mfcc = self._compute_mfcc(segment, sr)
+            spectral_features = self._compute_spectral_features(segment, sr)
+
+            # Time range
+            start_time = start_sample / sr
+            end_time = end_sample / sr
+
+            chunk = Chunk(
+                content=segment.tobytes(),
+                modality="audio",
+                start_pos=int(start_time * 1000),  # milliseconds
+                end_pos=int(end_time * 1000),
+                document_path=document_path,
+                chunk_index=chunk_index,
+                metadata={
+                    "start_time": start_time,
+                    "end_time": end_time,
+                    "duration": end_time - start_time,
+                    "sample_rate": sr,
+                    "mfcc_mean": mfcc.tolist(),
+                    "spectral_features": spectral_features,
+                },
+            )
+            chunks.append(chunk)
+            chunk_index += 1
+
+        # Handle empty audio
+        if not chunks:
+            chunks.append(
+                Chunk(
+                    content=b"",
+                    modality="audio",
+                    start_pos=0,
+                    end_pos=0,
+                    document_path=document_path,
+                    chunk_index=0,
+                    metadata={"sample_rate": sr},
+                )
+            )
+
+        return chunks
+
+    def _compute_mfcc(self, y: Any, sr: int) -> Any:
+        """Compute MFCC features."""
+        mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=self.n_mfcc)
+        # Return mean across time
+        return mfcc.mean(axis=1)
+
+    def _compute_spectral_features(self, y: Any, sr: int) -> Dict[str, float]:
+        """Compute spectral features."""
+        # Spectral centroid
+        spectral_centroid = librosa.feature.spectral_centroid(y=y, sr=sr).mean()
+
+        # Spectral bandwidth
+        spectral_bandwidth = librosa.feature.spectral_bandwidth(y=y, sr=sr).mean()
+
+        # Spectral rolloff
+        spectral_rolloff = librosa.feature.spectral_rolloff(y=y, sr=sr).mean()
+
+        # Zero crossing rate
+        zero_crossing_rate = librosa.feature.zero_crossing_rate(y).mean()
+
+        # RMS energy
+        rms = librosa.feature.rms(y=y).mean()
+
+        return {
+            "spectral_centroid": float(spectral_centroid),
+            "spectral_bandwidth": float(spectral_bandwidth),
+            "spectral_rolloff": float(spectral_rolloff),
+            "zero_crossing_rate": float(zero_crossing_rate),
+            "rms": float(rms),
+        }