coffloader 0.1.0__py3-none-any.whl

coffloader/__init__.py

@@ -0,0 +1,17 @@
+"""coffloader — External memory for AI agents."""
+
+from .backends import CompositeBackend, LocalBackend, MemoryBackend
+from .store import Coffloader
+from .toc import InspectResult, TocEntry, WriteResult
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Coffloader",
+    "TocEntry",
+    "WriteResult",
+    "InspectResult",
+    "CompositeBackend",
+    "LocalBackend",
+    "MemoryBackend",
+]

coffloader/backends/__init__.py

@@ -0,0 +1,8 @@
+"""Storage backends for coffloader."""
+
+from .base import BackendProtocol
+from .composite import CompositeBackend
+from .local import LocalBackend
+from .memory import MemoryBackend
+
+__all__ = ["BackendProtocol", "CompositeBackend", "LocalBackend", "MemoryBackend"]

coffloader/backends/base.py

@@ -0,0 +1,23 @@
+"""Backend protocol for VFS storage."""
+
+from typing import Protocol
+
+
+class BackendProtocol(Protocol):
+    """Interface for storage backends."""
+
+    def write(self, path: str, data: bytes) -> None:
+        """Store data at the given path."""
+        ...
+
+    def read(self, path: str) -> bytes:
+        """Read data from the given path. Raises KeyError if not found."""
+        ...
+
+    def delete(self, path: str) -> bool:
+        """Delete data at the given path. Returns True if deleted, False if not found."""
+        ...
+
+    def exists(self, path: str) -> bool:
+        """Check if path exists."""
+        ...

coffloader/backends/composite.py

@@ -0,0 +1,41 @@
+"""Composite backend that routes by path prefix."""
+
+from __future__ import annotations
+
+from .base import BackendProtocol
+from .memory import MemoryBackend
+
+
+class CompositeBackend:
+    """Route paths to different backends based on prefix.
+
+    Longest matching prefix wins. Unmatched paths go to the default backend.
+    """
+
+    def __init__(
+        self,
+        default: BackendProtocol | None = None,
+        routes: dict[str, BackendProtocol] | None = None,
+    ) -> None:
+        self._default: BackendProtocol = default or MemoryBackend()
+        self._routes = routes or {}
+        # Sort routes by length descending for longest-prefix matching
+        self._sorted_prefixes = sorted(self._routes.keys(), key=len, reverse=True)
+
+    def _get_backend(self, path: str) -> BackendProtocol:
+        for prefix in self._sorted_prefixes:
+            if path.startswith(prefix):
+                return self._routes[prefix]
+        return self._default
+
+    def write(self, path: str, data: bytes) -> None:
+        self._get_backend(path).write(path, data)
+
+    def read(self, path: str) -> bytes:
+        return self._get_backend(path).read(path)
+
+    def delete(self, path: str) -> bool:
+        return self._get_backend(path).delete(path)
+
+    def exists(self, path: str) -> bool:
+        return self._get_backend(path).exists(path)

coffloader/backends/local.py

@@ -0,0 +1,37 @@
+"""Local filesystem storage backend."""
+
+from pathlib import Path
+
+
+class LocalBackend:
+    """Store blobs on local disk under a root directory."""
+
+    def __init__(self, root: str | Path) -> None:
+        self._root = Path(root).resolve()
+        self._root.mkdir(parents=True, exist_ok=True)
+
+    def _resolve(self, path: str) -> Path:
+        # Strip leading slash for joining
+        relative = path.lstrip("/")
+        return self._root / relative
+
+    def write(self, path: str, data: bytes) -> None:
+        file_path = self._resolve(path)
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        file_path.write_bytes(data)
+
+    def read(self, path: str) -> bytes:
+        file_path = self._resolve(path)
+        if not file_path.exists():
+            raise KeyError(f"Path not found: {path}")
+        return file_path.read_bytes()
+
+    def delete(self, path: str) -> bool:
+        file_path = self._resolve(path)
+        if file_path.exists():
+            file_path.unlink()
+            return True
+        return False
+
+    def exists(self, path: str) -> bool:
+        return self._resolve(path).exists()

coffloader/backends/memory.py

@@ -0,0 +1,25 @@
+"""In-memory storage backend."""
+
+
+class MemoryBackend:
+    """Store blobs in a Python dict. Data lost on process exit."""
+
+    def __init__(self) -> None:
+        self._store: dict[str, bytes] = {}
+
+    def write(self, path: str, data: bytes) -> None:
+        self._store[path] = data
+
+    def read(self, path: str) -> bytes:
+        if path not in self._store:
+            raise KeyError(f"Path not found: {path}")
+        return self._store[path]
+
+    def delete(self, path: str) -> bool:
+        if path in self._store:
+            del self._store[path]
+            return True
+        return False
+
+    def exists(self, path: str) -> bool:
+        return path in self._store

coffloader/index/__init__.py

@@ -0,0 +1,15 @@
+"""Index implementations for TOC search."""
+
+from .fts import FTSIndex
+
+# Optional imports for embedding-based search
+try:
+    from .embeddings import EmbeddingIndex
+    from .hybrid import HybridIndex
+    EMBEDDINGS_AVAILABLE = True
+except ImportError:
+    EMBEDDINGS_AVAILABLE = False
+    EmbeddingIndex = None  # type: ignore
+    HybridIndex = None  # type: ignore
+
+__all__ = ["FTSIndex", "EmbeddingIndex", "HybridIndex", "EMBEDDINGS_AVAILABLE"]

coffloader/index/embeddings.py

@@ -0,0 +1,181 @@
+"""Embedding-based semantic search using sentence-transformers."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from typing import Any
+
+import numpy as np
+
+from ..toc import TocEntry
+
+# Lazy load to avoid import cost when not using embeddings
+_model = None
+
+
+def _get_model():
+    """Lazy load the embedding model."""
+    global _model
+    if _model is None:
+        from sentence_transformers import SentenceTransformer
+        _model = SentenceTransformer("all-MiniLM-L6-v2")
+    return _model
+
+
+def embed_text(text: str) -> np.ndarray:
+    """Embed text into a vector (float32)."""
+    model = _get_model()
+    embedding = model.encode(text, convert_to_numpy=True)
+    return embedding.astype(np.float32)
+
+
+def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
+    """Compute cosine similarity between two vectors."""
+    return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))
+
+
+class EmbeddingIndex:
+    """Semantic search index using embeddings + cosine similarity."""
+
+    def __init__(self, db_path: str = ":memory:", min_similarity: float = 0.3) -> None:
+        self._conn = sqlite3.connect(db_path, check_same_thread=False)
+        self._conn.row_factory = sqlite3.Row
+        self._min_similarity = min_similarity
+        self._init_schema()
+
+    def _init_schema(self) -> None:
+        cur = self._conn.cursor()
+        cur.execute("""
+            CREATE TABLE IF NOT EXISTS toc_embeddings (
+                id TEXT PRIMARY KEY,
+                address TEXT NOT NULL,
+                summary TEXT NOT NULL,
+                metadata TEXT NOT NULL,
+                content_hash TEXT,
+                byte_count INTEGER,
+                created_at TEXT,
+                excluded INTEGER DEFAULT 0,
+                embedding BLOB
+            )
+        """)
+        self._conn.commit()
+
+    def add(self, entry: TocEntry) -> None:
+        """Add or update a TOC entry with its embedding."""
+        cur = self._conn.cursor()
+        metadata_json = json.dumps(entry.metadata)
+        
+        # Embed the summary
+        embedding = embed_text(entry.summary)
+        embedding_blob = embedding.tobytes()
+
+        cur.execute(
+            """
+            INSERT OR REPLACE INTO toc_embeddings 
+            (id, address, summary, metadata, content_hash, byte_count, created_at, excluded, embedding)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                entry.id,
+                entry.address,
+                entry.summary,
+                metadata_json,
+                entry.content_hash,
+                entry.byte_count,
+                entry.created_at,
+                1 if entry.excluded else 0,
+                embedding_blob,
+            ),
+        )
+        self._conn.commit()
+
+    def remove(self, entry_id: str) -> bool:
+        """Remove a TOC entry by ID."""
+        cur = self._conn.cursor()
+        cur.execute("DELETE FROM toc_embeddings WHERE id = ?", (entry_id,))
+        self._conn.commit()
+        return cur.rowcount > 0
+
+    def get(self, entry_id: str) -> TocEntry | None:
+        """Get a TOC entry by ID."""
+        cur = self._conn.cursor()
+        cur.execute("SELECT * FROM toc_embeddings WHERE id = ?", (entry_id,))
+        row = cur.fetchone()
+        if row is None:
+            return None
+        return self._row_to_entry(row)
+
+    def get_by_address(self, address: str) -> TocEntry | None:
+        """Get a TOC entry by address."""
+        cur = self._conn.cursor()
+        cur.execute("SELECT * FROM toc_embeddings WHERE address = ?", (address,))
+        row = cur.fetchone()
+        if row is None:
+            return None
+        return self._row_to_entry(row)
+
+    def search(
+        self,
+        query: str,
+        k: int = 5,
+        filters: dict[str, Any] | None = None,
+        namespace: str | None = None,
+    ) -> list[TocEntry]:
+        """Search using cosine similarity on embeddings."""
+        cur = self._conn.cursor()
+
+        # Build filter SQL
+        where_clauses = []
+        params: list[Any] = []
+
+        if namespace:
+            where_clauses.append("address LIKE ?")
+            params.append(f"{namespace}%")
+
+        if filters:
+            for key, value in filters.items():
+                where_clauses.append(f"json_extract(metadata, '$.{key}') = ?")
+                params.append(value)
+
+        where_sql = " AND ".join(where_clauses) if where_clauses else "1=1"
+
+        # Get all matching entries
+        sql = f"SELECT * FROM toc_embeddings WHERE {where_sql}"
+        cur.execute(sql, params)
+        rows = cur.fetchall()
+
+        if not rows:
+            return []
+
+        # Embed query and compute similarities
+        query_embedding = embed_text(query)
+        
+        scored = []
+        for row in rows:
+            embedding = np.frombuffer(row["embedding"], dtype=np.float32)
+            score = cosine_similarity(query_embedding, embedding)
+            # Filter out low-similarity results
+            if score >= self._min_similarity:
+                scored.append((score, row))
+
+        # Sort by score descending, take top k
+        scored.sort(key=lambda x: x[0], reverse=True)
+        
+        return [self._row_to_entry(row) for _, row in scored[:k]]
+
+    def _row_to_entry(self, row: sqlite3.Row) -> TocEntry:
+        return TocEntry(
+            id=row["id"],
+            address=row["address"],
+            summary=row["summary"],
+            metadata=json.loads(row["metadata"]),
+            content_hash=row["content_hash"] or "",
+            byte_count=row["byte_count"] or 0,
+            created_at=row["created_at"] or "",
+            excluded=bool(row["excluded"]),
+        )
+
+    def close(self) -> None:
+        """Close the database connection."""
+        self._conn.close()

coffloader/index/fts.py

@@ -0,0 +1,218 @@
+"""SQLite FTS5 index for TOC search."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from typing import Any
+
+from ..toc import TocEntry
+
+# FTS5 special characters that need escaping or removal
+FTS5_SPECIAL = set('"*?:^(){}[]\'')
+
+
+def _sanitize_fts_query(query: str) -> str:
+    """Remove FTS5 special characters from query."""
+    return "".join(c if c not in FTS5_SPECIAL else " " for c in query)
+
+
+class FTSIndex:
+    """Full-text search index using SQLite FTS5.
+
+    Indexes summary and serialized metadata for BM25 ranking.
+    """
+
+    def __init__(self, db_path: str = ":memory:") -> None:
+        self._conn = sqlite3.connect(db_path, check_same_thread=False)
+        self._conn.row_factory = sqlite3.Row
+        self._init_schema()
+
+    def _init_schema(self) -> None:
+        cur = self._conn.cursor()
+        # Main table for TOC entries
+        cur.execute("""
+            CREATE TABLE IF NOT EXISTS toc (
+                id TEXT PRIMARY KEY,
+                address TEXT NOT NULL,
+                summary TEXT NOT NULL,
+                metadata TEXT NOT NULL,
+                content_hash TEXT,
+                byte_count INTEGER,
+                created_at TEXT,
+                excluded INTEGER DEFAULT 0
+            )
+        """)
+        # Standalone FTS5 table (not external content)
+        cur.execute("""
+            CREATE VIRTUAL TABLE IF NOT EXISTS toc_fts USING fts5(
+                id,
+                summary,
+                metadata_text
+            )
+        """)
+        self._conn.commit()
+
+    def add(self, entry: TocEntry) -> None:
+        """Add or update a TOC entry in the index."""
+        cur = self._conn.cursor()
+        metadata_json = json.dumps(entry.metadata)
+
+        # Check if exists for update vs insert
+        cur.execute("SELECT id FROM toc WHERE id = ?", (entry.id,))
+        exists = cur.fetchone() is not None
+
+        if exists:
+            # Update toc
+            cur.execute(
+                """
+                UPDATE toc SET address=?, summary=?, metadata=?, content_hash=?, 
+                               byte_count=?, created_at=?, excluded=?
+                WHERE id=?
+                """,
+                (
+                    entry.address,
+                    entry.summary,
+                    metadata_json,
+                    entry.content_hash,
+                    entry.byte_count,
+                    entry.created_at,
+                    1 if entry.excluded else 0,
+                    entry.id,
+                ),
+            )
+            # Update FTS
+            cur.execute("DELETE FROM toc_fts WHERE id = ?", (entry.id,))
+            cur.execute(
+                "INSERT INTO toc_fts (id, summary, metadata_text) VALUES (?, ?, ?)",
+                (entry.id, entry.summary, metadata_json),
+            )
+        else:
+            # Insert into toc
+            cur.execute(
+                """
+                INSERT INTO toc 
+                (id, address, summary, metadata, content_hash, byte_count, created_at, excluded)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    entry.id,
+                    entry.address,
+                    entry.summary,
+                    metadata_json,
+                    entry.content_hash,
+                    entry.byte_count,
+                    entry.created_at,
+                    1 if entry.excluded else 0,
+                ),
+            )
+            # Insert into FTS
+            cur.execute(
+                "INSERT INTO toc_fts (id, summary, metadata_text) VALUES (?, ?, ?)",
+                (entry.id, entry.summary, metadata_json),
+            )
+
+        self._conn.commit()
+
+    def remove(self, entry_id: str) -> bool:
+        """Remove a TOC entry by ID. Returns True if deleted."""
+        cur = self._conn.cursor()
+        cur.execute("DELETE FROM toc WHERE id = ?", (entry_id,))
+        cur.execute("DELETE FROM toc_fts WHERE id = ?", (entry_id,))
+        self._conn.commit()
+        return cur.rowcount > 0
+
+    def get(self, entry_id: str) -> TocEntry | None:
+        """Get a TOC entry by ID."""
+        cur = self._conn.cursor()
+        cur.execute("SELECT * FROM toc WHERE id = ?", (entry_id,))
+        row = cur.fetchone()
+        if row is None:
+            return None
+        return self._row_to_entry(row)
+
+    def get_by_address(self, address: str) -> TocEntry | None:
+        """Get a TOC entry by address."""
+        cur = self._conn.cursor()
+        cur.execute("SELECT * FROM toc WHERE address = ?", (address,))
+        row = cur.fetchone()
+        if row is None:
+            return None
+        return self._row_to_entry(row)
+
+    def search(
+        self,
+        query: str,
+        k: int = 5,
+        filters: dict[str, Any] | None = None,
+        namespace: str | None = None,
+    ) -> list[TocEntry]:
+        """Search TOC entries using FTS5 BM25 ranking.
+
+        Args:
+            query: Search query string
+            k: Max results to return
+            filters: Metadata field filters (exact match)
+            namespace: Filter by address prefix
+        """
+        cur = self._conn.cursor()
+        
+        # Sanitize query for FTS5
+        safe_query = _sanitize_fts_query(query.strip())
+
+        if safe_query:
+            # FTS search with BM25 ranking
+            base_sql = """
+                SELECT toc.* FROM toc
+                INNER JOIN toc_fts ON toc.id = toc_fts.id
+                WHERE toc_fts MATCH ?
+            """
+            params: list[Any] = [safe_query]
+
+            if namespace:
+                base_sql += " AND toc.address LIKE ?"
+                params.append(f"{namespace}%")
+
+            if filters:
+                for key, value in filters.items():
+                    base_sql += f" AND json_extract(toc.metadata, '$.{key}') = ?"
+                    params.append(value)
+
+            base_sql += " ORDER BY bm25(toc_fts) LIMIT ?"
+            params.append(k)
+
+        else:
+            # No query, just filter and sort by recency
+            base_sql = "SELECT * FROM toc WHERE 1=1"
+            params = []
+
+            if namespace:
+                base_sql += " AND address LIKE ?"
+                params.append(f"{namespace}%")
+
+            if filters:
+                for key, value in filters.items():
+                    base_sql += f" AND json_extract(metadata, '$.{key}') = ?"
+                    params.append(value)
+
+            base_sql += " ORDER BY created_at DESC LIMIT ?"
+            params.append(k)
+
+        cur.execute(base_sql, params)
+        return [self._row_to_entry(row) for row in cur.fetchall()]
+
+    def _row_to_entry(self, row: sqlite3.Row) -> TocEntry:
+        return TocEntry(
+            id=row["id"],
+            address=row["address"],
+            summary=row["summary"],
+            metadata=json.loads(row["metadata"]),
+            content_hash=row["content_hash"] or "",
+            byte_count=row["byte_count"] or 0,
+            created_at=row["created_at"] or "",
+            excluded=bool(row["excluded"]),
+        )
+
+    def close(self) -> None:
+        """Close the database connection."""
+        self._conn.close()

coffloader/index/hybrid.py

@@ -0,0 +1,80 @@
+"""Hybrid search combining BM25 and embeddings via Reciprocal Rank Fusion."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..toc import TocEntry
+from .fts import FTSIndex
+from .embeddings import EmbeddingIndex
+
+
+class HybridIndex:
+    """Combines FTS5/BM25 and embedding search using RRF.
+    
+    Reciprocal Rank Fusion merges ranked lists without needing
+    normalized scores. Formula: RRF(d) = Σ 1/(k + rank(d))
+    """
+
+    def __init__(self, db_path: str = ":memory:", rrf_k: int = 60, min_similarity: float = 0.3) -> None:
+        self._fts = FTSIndex(db_path=db_path)
+        self._embed = EmbeddingIndex(db_path=db_path, min_similarity=min_similarity)
+        self._rrf_k = rrf_k  # RRF constant (typically 60)
+
+    def add(self, entry: TocEntry) -> None:
+        """Add entry to both indexes."""
+        self._fts.add(entry)
+        self._embed.add(entry)
+
+    def remove(self, entry_id: str) -> bool:
+        """Remove entry from both indexes."""
+        fts_removed = self._fts.remove(entry_id)
+        embed_removed = self._embed.remove(entry_id)
+        return fts_removed or embed_removed
+
+    def get(self, entry_id: str) -> TocEntry | None:
+        """Get entry by ID."""
+        return self._fts.get(entry_id)
+
+    def get_by_address(self, address: str) -> TocEntry | None:
+        """Get entry by address."""
+        return self._fts.get_by_address(address)
+
+    def search(
+        self,
+        query: str,
+        k: int = 5,
+        filters: dict[str, Any] | None = None,
+        namespace: str | None = None,
+    ) -> list[TocEntry]:
+        """Search using RRF fusion of BM25 and embedding results."""
+        # Get more candidates from each to allow good fusion
+        n_candidates = k * 3
+
+        # BM25 results
+        fts_results = self._fts.search(query, k=n_candidates, filters=filters, namespace=namespace)
+        
+        # Embedding results
+        embed_results = self._embed.search(query, k=n_candidates, filters=filters, namespace=namespace)
+
+        # Compute RRF scores
+        rrf_scores: dict[str, float] = {}
+        entry_map: dict[str, TocEntry] = {}
+
+        for rank, entry in enumerate(fts_results):
+            rrf_scores[entry.id] = rrf_scores.get(entry.id, 0) + 1 / (self._rrf_k + rank + 1)
+            entry_map[entry.id] = entry
+
+        for rank, entry in enumerate(embed_results):
+            rrf_scores[entry.id] = rrf_scores.get(entry.id, 0) + 1 / (self._rrf_k + rank + 1)
+            entry_map[entry.id] = entry
+
+        # Sort by RRF score descending
+        sorted_ids = sorted(rrf_scores.keys(), key=lambda x: rrf_scores[x], reverse=True)
+
+        return [entry_map[id] for id in sorted_ids[:k]]
+
+    def close(self) -> None:
+        """Close both indexes."""
+        self._fts.close()
+        self._embed.close()

coffloader/store.py

@@ -0,0 +1,279 @@
+"""Main Coffloader store."""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from typing import Any
+
+from .backends.base import BackendProtocol
+from .backends.memory import MemoryBackend
+from .index import EMBEDDINGS_AVAILABLE, FTSIndex
+from .toc import InspectResult, TocEntry, WriteResult
+
+
+class Coffloader:
+    """External memory for AI agents.
+
+    Offloads content to a VFS backend, indexes caller-provided summaries,
+    and retrieves via search.
+    """
+
+    DEFAULT_MAX_BYTES = 512_000
+    DEFAULT_MIN_SIMILARITY = 0.3
+
+    def __init__(
+        self,
+        backend: BackendProtocol | None = None,
+        index_path: str | None = None,
+        max_bytes: int = DEFAULT_MAX_BYTES,
+        max_estimated_tokens: int = 128_000,
+        on_oversize: str = "reject",
+        summary_max_chars: int = 1000,
+        hybrid: bool = True,
+        min_similarity: float = DEFAULT_MIN_SIMILARITY,
+    ) -> None:
+        """Initialize coffloader.
+
+        Args:
+            backend: Storage backend (default: in-memory)
+            index_path: SQLite index path (default: in-memory)
+            max_bytes: Max content size in bytes (default: 512_000)
+            max_estimated_tokens: Max estimated tokens (bytes / 4)
+            on_oversize: "reject" or "metadata_only"
+            summary_max_chars: Max summary length to index
+            hybrid: Use hybrid BM25+embeddings search if available (default: True)
+            min_similarity: Minimum cosine similarity for embedding results (default: 0.3).
+                           Lower = more results, possibly less relevant.
+                           Higher = fewer results, more relevant.
+                           Set to 0.0 to disable filtering.
+        """
+        self._backend = backend or MemoryBackend()
+        
+        # Use hybrid index if requested and embeddings are available
+        if hybrid and EMBEDDINGS_AVAILABLE:
+            from .index import HybridIndex
+            self._index = HybridIndex(db_path=index_path or ":memory:", min_similarity=min_similarity)
+            self._hybrid = True
+        else:
+            self._index = FTSIndex(db_path=index_path or ":memory:")
+            self._hybrid = False
+        
+        self._max_bytes = max_bytes
+        self._max_estimated_tokens = max_estimated_tokens
+        self._on_oversize = on_oversize
+        self._summary_max_chars = summary_max_chars
+        self._min_similarity = min_similarity
+    
+    @property
+    def is_hybrid(self) -> bool:
+        """Return True if using hybrid BM25+embeddings search."""
+        return self._hybrid
+
+    def inspect(self, content: str | bytes) -> InspectResult:
+        """Check if content is within size limits without storing.
+
+        Args:
+            content: Content to check
+
+        Returns:
+            InspectResult with acceptability and size stats
+        """
+        data = content.encode("utf-8") if isinstance(content, str) else content
+        byte_count = len(data)
+        estimated_tokens = byte_count // 4
+
+        if byte_count > self._max_bytes:
+            return InspectResult(
+                acceptable=False,
+                byte_count=byte_count,
+                estimated_tokens=estimated_tokens,
+                reason=f"content_exceeds_limit: {byte_count} bytes > {self._max_bytes}",
+            )
+
+        if estimated_tokens > self._max_estimated_tokens:
+            return InspectResult(
+                acceptable=False,
+                byte_count=byte_count,
+                estimated_tokens=estimated_tokens,
+                reason=f"content_exceeds_limit: ~{estimated_tokens} tokens > {self._max_estimated_tokens}",
+            )
+
+        return InspectResult(
+            acceptable=True,
+            byte_count=byte_count,
+            estimated_tokens=estimated_tokens,
+        )
+
+    def write(
+        self,
+        content: str | bytes,
+        summary: str,
+        *,
+        metadata: dict[str, Any] | None = None,
+        path: str | None = None,
+    ) -> WriteResult:
+        """Store content and create a TOC entry.
+
+        Args:
+            content: Full content to store
+            summary: Required summary for indexing (caller-provided)
+            metadata: Optional metadata fields for filtering
+            path: Optional VFS path (auto-generated if omitted)
+
+        Returns:
+            WriteResult with ok status, id, address, or rejection reason
+        """
+        if not summary or not summary.strip():
+            return WriteResult(
+                ok=False,
+                reason="summary_required",
+                stats={"message": "summary field is required and cannot be empty"},
+            )
+
+        data = content.encode("utf-8") if isinstance(content, str) else content
+        byte_count = len(data)
+        estimated_tokens = byte_count // 4
+
+        # Check size limits
+        check = self.inspect(content)
+        if not check.acceptable:
+            if self._on_oversize == "reject":
+                return WriteResult(
+                    ok=False,
+                    reason="content_exceeds_limit",
+                    stats={
+                        "byte_count": byte_count,
+                        "estimated_tokens": estimated_tokens,
+                        "max_bytes": self._max_bytes,
+                        "max_estimated_tokens": self._max_estimated_tokens,
+                    },
+                )
+            # metadata_only mode: create TOC entry but don't store blob
+            entry_id = str(uuid.uuid4())
+            address = path or f"/excluded/{entry_id}"
+            content_hash = f"sha256:{hashlib.sha256(data).hexdigest()}"
+
+            entry = TocEntry(
+                id=entry_id,
+                address=address,
+                summary=summary[: self._summary_max_chars],
+                metadata=metadata or {},
+                content_hash=content_hash,
+                byte_count=byte_count,
+                excluded=True,
+            )
+            self._index.add(entry)
+
+            return WriteResult(
+                ok=True,
+                id=entry_id,
+                address=address,
+                stats={
+                    "byte_count": byte_count,
+                    "estimated_tokens": estimated_tokens,
+                    "excluded": True,
+                },
+            )
+
+        # Normal write
+        entry_id = str(uuid.uuid4())
+        address = path or f"/content/{entry_id}"
+        content_hash = f"sha256:{hashlib.sha256(data).hexdigest()}"
+
+        # Store blob
+        self._backend.write(address, data)
+
+        # Index TOC entry
+        entry = TocEntry(
+            id=entry_id,
+            address=address,
+            summary=summary[: self._summary_max_chars],
+            metadata=metadata or {},
+            content_hash=content_hash,
+            byte_count=byte_count,
+            excluded=False,
+        )
+        self._index.add(entry)
+
+        return WriteResult(
+            ok=True,
+            id=entry_id,
+            address=address,
+            stats={
+                "byte_count": byte_count,
+                "estimated_tokens": estimated_tokens,
+            },
+        )
+
+    def search(
+        self,
+        query: str,
+        k: int = 5,
+        *,
+        filters: dict[str, Any] | None = None,
+        namespace: str | None = None,
+    ) -> list[TocEntry]:
+        """Search the TOC index.
+
+        Args:
+            query: Search query
+            k: Max results to return
+            filters: Metadata field filters (exact match)
+            namespace: Filter by address prefix
+
+        Returns:
+            List of matching TocEntry objects (summary + address, not full content)
+        """
+        return self._index.search(query, k=k, filters=filters, namespace=namespace)
+
+    def read(self, address: str) -> bytes:
+        """Read full content from a VFS address.
+
+        Args:
+            address: VFS path returned by write() or search()
+
+        Returns:
+            Stored content as bytes
+
+        Raises:
+            KeyError: If address not found
+        """
+        return self._backend.read(address)
+
+    def read_text(self, address: str, encoding: str = "utf-8") -> str:
+        """Read full content as text.
+
+        Args:
+            address: VFS path
+            encoding: Text encoding (default: utf-8)
+
+        Returns:
+            Stored content as string
+        """
+        return self._backend.read(address).decode(encoding)
+
+    def delete(self, address: str) -> bool:
+        """Delete content and its TOC entry.
+
+        Args:
+            address: VFS path to delete
+
+        Returns:
+            True if deleted, False if not found
+        """
+        entry = self._index.get_by_address(address)
+        if entry:
+            self._index.remove(entry.id)
+        return self._backend.delete(address)
+
+    def get_entry(self, address: str) -> TocEntry | None:
+        """Get TOC entry by address without loading content.
+
+        Args:
+            address: VFS path
+
+        Returns:
+            TocEntry or None if not found
+        """
+        return self._index.get_by_address(address)

coffloader/toc.py

@@ -0,0 +1,67 @@
+"""Table of Context (TOC) entry and result models."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+
+@dataclass
+class TocEntry:
+    """A single entry in the Table of Context."""
+
+    id: str
+    address: str
+    summary: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+    content_hash: str = ""
+    byte_count: int = 0
+    created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    excluded: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "address": self.address,
+            "summary": self.summary,
+            "metadata": self.metadata,
+            "content_hash": self.content_hash,
+            "byte_count": self.byte_count,
+            "created_at": self.created_at,
+            "excluded": self.excluded,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "TocEntry":
+        return cls(
+            id=data["id"],
+            address=data["address"],
+            summary=data["summary"],
+            metadata=data.get("metadata", {}),
+            content_hash=data.get("content_hash", ""),
+            byte_count=data.get("byte_count", 0),
+            created_at=data.get("created_at", ""),
+            excluded=data.get("excluded", False),
+        )
+
+
+@dataclass
+class WriteResult:
+    """Result of a write operation."""
+
+    ok: bool
+    id: str = ""
+    address: str = ""
+    reason: str = ""
+    stats: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class InspectResult:
+    """Result of an inspect (pre-check) operation."""
+
+    acceptable: bool
+    byte_count: int
+    estimated_tokens: int
+    reason: str = ""

coffloader-0.1.0.dist-info/METADATA

@@ -0,0 +1,201 @@
+Metadata-Version: 2.4
+Name: coffloader
+Version: 0.1.0
+Summary: External memory for AI agents — offload context to a VFS, index summaries, retrieve on demand.
+Project-URL: Homepage, https://github.com/mingyk/coffloader
+Project-URL: Repository, https://github.com/mingyk/coffloader
+Author: coffloader contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent,context,llm,memory,rag,vfs
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Provides-Extra: embed
+Requires-Dist: numpy>=1.21; extra == 'embed'
+Requires-Dist: sentence-transformers>=2.2; extra == 'embed'
+Description-Content-Type: text/markdown
+
+# coffloader
+
+**External memory for AI agents** — offload context to a VFS, index caller-provided summaries, retrieve on demand.
+
+[![Python](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/downloads/)
+[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+[![Status](https://img.shields.io/badge/status-pre--alpha-orange.svg)](#status)
+
+```bash
+pip install coffloader              # core (BM25 search)
+pip install coffloader[embed]       # + semantic search (sentence-transformers)
+```
+
+---
+
+## What it does
+
+Agents accumulate context faster than any window allows. coffloader offloads content to storage, keeps a searchable index of summaries, and retrieves full content on demand.
+
+```
+write(content, summary) → store blob + index summary
+search(query)           → top-k summaries + addresses
+read(address)           → full content
+```
+
+**Key constraints:**
+- `summary` is **required** on write — your agent/LLM provides it, not coffloader
+- No LLM calls inside the library — pure storage and retrieval
+- Caller handles contradiction detection, dedup, and reasoning
+
+---
+
+## Quick start
+
+```python
+from coffloader import Coffloader
+
+store = Coffloader()
+
+# 1. Offload a conversation segment (summary comes from your agent)
+store.write(
+    content="[Turn 1] User: I was charged twice for order #9910...",
+    summary="Customer reports duplicate charge on order #9910",
+    metadata={"session_id": "ticket_8842", "segment": 1},
+    path="/sessions/ticket_8842/seg_001.txt",
+)
+
+# 2. Later: search when user asks about earlier context
+hits = store.search("order number", namespace="/sessions/ticket_8842/")
+
+# 3. Load full content and inject into your LLM
+text = store.read_text(hits[0].address)
+```
+
+**The loop:** offload cold context → search when needed → read and inject.
+
+---
+
+## API
+
+```python
+store = Coffloader(
+    backend=None,           # default: in-memory VFS
+    max_bytes=512_000,      # default: 512 KB — reject oversized payloads
+    on_oversize="reject",   # "reject" or "metadata_only"
+    hybrid=True,            # default: True — use BM25 + embeddings if available
+    min_similarity=0.3,     # default: 0.3 — filter out weak embedding matches
+                            # lower = more results, less relevant
+                            # higher = fewer results, more relevant  
+                            # set to 0.0 to disable filtering
+)
+
+# Store content with a caller-provided summary
+result = store.write(content, summary, metadata={}, path=None)
+
+# Search indexed summaries (returns TocEntry list, not full content)
+hits = store.search(query, k=5, filters={}, namespace=None)
+#                         ^^^ number of results to return
+
+# Load full content
+data = store.read(address)          # bytes
+text = store.read_text(address)     # str
+
+# Check size before writing
+check = store.inspect(content)      # .acceptable, .byte_count
+
+# Delete
+store.delete(address)
+```
+
+**Defaults are exposed as class attributes:**
+```python
+Coffloader.DEFAULT_MAX_BYTES       # 512_000
+Coffloader.DEFAULT_MIN_SIMILARITY  # 0.3
+```
+
+---
+
+## Composite backends
+
+Route paths to different storage:
+
+```python
+from coffloader import Coffloader, CompositeBackend, LocalBackend, MemoryBackend
+
+store = Coffloader(
+    backend=CompositeBackend(
+        default=MemoryBackend(),
+        routes={"/archive/": LocalBackend(root="./data")},
+    )
+)
+```
+
+---
+
+## Patterns
+
+**Long session (segmented):** Offload every ~15 turns. Search returns precise segments, not the whole transcript.
+
+```python
+store.write(content=turns_1_15, summary="...", path="/sessions/abc/seg_001.txt")
+store.write(content=turns_16_30, summary="...", path="/sessions/abc/seg_002.txt")
+```
+
+**Tool output:** Offload large grep/API results with a structural summary (no LLM needed).
+
+```python
+store.write(
+    content=grep_output,
+    summary=f"grep error src/ → {n} matches",
+    path=f"/active/{session}/tool_001.txt",
+)
+```
+
+**Multi-agent:** Use namespaces for isolation (`/agent/{id}/`) or sharing (`/shared/`).
+
+---
+
+## Limits
+
+- Max payload: 512 KB by default (configurable)
+- Oversized content is rejected or recorded as metadata-only
+- No silent truncation
+
+---
+
+## Status
+
+Pre-alpha. Core API is stable: `write`, `search`, `read`, `inspect`, `delete`.
+
+**Working:**
+- BM25 (keyword) search via SQLite FTS5
+- Semantic search via `[embed]` optional extra
+- Hybrid search (BM25 + embeddings) with Reciprocal Rank Fusion
+
+**Not yet implemented:**
+- Persistent index to disk
+- Sharded TOC for large corpora
+
+---
+
+## Non-goals
+
+- LLM calls from the library
+- Automatic dedup, contradiction detection, or memory merge
+- Knowledge graphs or hierarchical rollups
+
+---
+
+## License
+
+MIT

coffloader-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+coffloader/__init__.py,sha256=hud-ptVjgFDi7kx5cOq_R1JlBwfNKXW897a2tvuPMQs,382
+coffloader/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+coffloader/store.py,sha256=bn9CcSCN5Xx7Z--Gjz2I0LUQwgoTiqBXELQOpwWYLnI,9082
+coffloader/toc.py,sha256=ucEOIElFI1uvBkr6mxKBrWuqBKfc8U7VDrnxuBjX79U,1771
+coffloader/backends/__init__.py,sha256=6VV_H42MNyUXIA8LJktE7HpfE59IhWyBhc81Tm9nnxg,264
+coffloader/backends/base.py,sha256=Tr_qtQbd4VCDOjmB7Hasvj-n5TAx-VH1j8nTNpJVJi0,618
+coffloader/backends/composite.py,sha256=slKjVwSTfapaetkdarlgfpwLqCNL6dQ5T0lOkbYv3bE,1339
+coffloader/backends/local.py,sha256=rC-CGLXK86waQtQshddn6O72mcXvQLw7FFF2lS9W5sc,1127
+coffloader/backends/memory.py,sha256=mfpswR9QslgDzl0tpBPr8KV4OXtBWExCYw7CZCo9_Y8,680
+coffloader/index/__init__.py,sha256=4INnoyRhX2cnGoRAzV2tihUoJ-0vk8_rBQmrtkAY93o,449
+coffloader/index/embeddings.py,sha256=9Rc-fYPFkYy_S4axOCm89LJ5XotIUfQxuL2Q3A84kZo,5771
+coffloader/index/fts.py,sha256=GItaVHJ4HhzEsbV-7lr7mpkVqXh0CgbEB4gFNjXpC1k,7140
+coffloader/index/hybrid.py,sha256=JjXkd0Mxi-F53hDVVsg8Uz6fQVfvqdcEuD2Ax7sXihc,2799
+coffloader-0.1.0.dist-info/METADATA,sha256=7eAtz4iQIxlI0HuXI5jHBkX_jSIPTCdMRO6MY6FOdpk,5934
+coffloader-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+coffloader-0.1.0.dist-info/licenses/LICENSE,sha256=y5U7o9B9iqrJ8ZNu6APNngmk0-SB5602vo4jQGETgyo,1080
+coffloader-0.1.0.dist-info/RECORD,,

coffloader-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

coffloader-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 coffloader contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.