tribalmemory 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

tribalmemory/server/app.py

@@ -1,5 +1,6 @@
 """FastAPI application for tribal-memory service."""
 
+import asyncio
 import logging
 from contextlib import asynccontextmanager
 from pathlib import Path
@@ -10,11 +11,13 @@ from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
 from ..services import create_memory_service, TribalMemoryService
+from ..services.session_store import SessionStore
 from .config import TribalMemoryConfig
 from .routes import router
 
 # Global service instance (set during lifespan)
 _memory_service: Optional[TribalMemoryService] = None
+_session_store: Optional[SessionStore] = None
 _instance_id: Optional[str] = None
 
 logger = logging.getLogger("tribalmemory.server")
@@ -23,7 +26,7 @@ logger = logging.getLogger("tribalmemory.server")
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Application lifespan manager."""
-    global _memory_service, _instance_id
+    global _memory_service, _session_store, _instance_id
 
     config: TribalMemoryConfig = app.state.config
 
@@ -43,18 +46,66 @@ async def lifespan(app: FastAPI):
         api_base=config.embedding.api_base,
         embedding_model=config.embedding.model,
         embedding_dimensions=config.embedding.dimensions,
+        hybrid_search=config.search.hybrid_enabled,
+        hybrid_vector_weight=config.search.vector_weight,
+        hybrid_text_weight=config.search.text_weight,
+        hybrid_candidate_multiplier=config.search.candidate_multiplier,
     )
 
-    logger.info(f"Memory service initialized (db: {config.db.path})")
+    # Create session store (shares embedding service and vector store)
+    _session_store = SessionStore(
+        instance_id=config.instance_id,
+        embedding_service=_memory_service.embedding_service,
+        vector_store=_memory_service.vector_store,
+    )
+
+    search_mode = "hybrid (vector + BM25)" if config.search.hybrid_enabled else "vector-only"
+    logger.info(f"Memory service initialized (db: {config.db.path}, search: {search_mode})")
+    logger.info(f"Session store initialized (retention: {config.server.session_retention_days} days)")
+
+    # Start background session cleanup task
+    cleanup_task = asyncio.create_task(
+        _session_cleanup_loop(
+            _session_store,
+            config.server.session_retention_days,
+        )
+    )
 
     yield
 
     # Cleanup
+    cleanup_task.cancel()
+    try:
+        await cleanup_task
+    except asyncio.CancelledError:
+        pass
     logger.info("Shutting down tribal-memory service")
     _memory_service = None
+    _session_store = None
     _instance_id = None
 
 
+async def _session_cleanup_loop(
+    session_store: SessionStore,
+    retention_days: int,
+) -> None:
+    """Background task that periodically cleans up expired session chunks.
+    
+    Runs every 6 hours. Deletes session chunks older than retention_days.
+    """
+    cleanup_interval = 6 * 60 * 60  # 6 hours in seconds
+    while True:
+        try:
+            await asyncio.sleep(cleanup_interval)
+            deleted = await session_store.cleanup(retention_days=retention_days)
+            if deleted > 0:
+                logger.info(f"Session cleanup: deleted {deleted} expired chunks (retention: {retention_days} days)")
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            logger.exception("Session cleanup failed")
+
+
 def create_app(config: Optional[TribalMemoryConfig] = None) -> FastAPI:
     """Create FastAPI application.
     

tribalmemory/server/config.py

@@ -51,6 +51,44 @@ class ServerConfig:
     """HTTP server configuration."""
     host: str = "127.0.0.1"
     port: int = 18790
+    session_retention_days: int = 30  # Days to retain session chunks
+
+
+@dataclass
+class SearchConfig:
+    """Search configuration for hybrid BM25 + vector search."""
+    hybrid_enabled: bool = True
+    vector_weight: float = 0.7
+    text_weight: float = 0.3
+    candidate_multiplier: int = 4
+    # Reranking configuration
+    reranking: str = "heuristic"  # "auto" | "cross-encoder" | "heuristic" | "none"
+    recency_decay_days: float = 30.0  # Half-life for recency boost
+    tag_boost_weight: float = 0.1  # Weight for tag match boost
+    rerank_pool_multiplier: int = 2  # How many candidates to give reranker (N * limit)
+
+    def __post_init__(self):
+        if self.vector_weight < 0:
+            raise ValueError("vector_weight must be non-negative")
+        if self.text_weight < 0:
+            raise ValueError("text_weight must be non-negative")
+        if self.vector_weight == 0 and self.text_weight == 0:
+            raise ValueError(
+                "At least one of vector_weight or text_weight must be > 0"
+            )
+        if self.candidate_multiplier < 1:
+            raise ValueError("candidate_multiplier must be >= 1")
+        if self.reranking not in ("auto", "cross-encoder", "heuristic", "none"):
+            raise ValueError(
+                f"Invalid reranking mode: {self.reranking}. "
+                f"Valid options: 'auto', 'cross-encoder', 'heuristic', 'none'"
+            )
+        if self.recency_decay_days <= 0:
+            raise ValueError("recency_decay_days must be positive")
+        if self.tag_boost_weight < 0:
+            raise ValueError("tag_boost_weight must be non-negative")
+        if self.rerank_pool_multiplier < 1:
+            raise ValueError("rerank_pool_multiplier must be >= 1")
 
 
 @dataclass
@@ -60,6 +98,7 @@ class TribalMemoryConfig:
     db: DatabaseConfig = field(default_factory=DatabaseConfig)
     embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
     server: ServerConfig = field(default_factory=ServerConfig)
+    search: SearchConfig = field(default_factory=SearchConfig)
 
     @classmethod
     def from_file(cls, path: str | Path) -> "TribalMemoryConfig":
@@ -79,12 +118,14 @@ class TribalMemoryConfig:
         db_data = data.get("db", {})
         embedding_data = data.get("embedding", {})
         server_data = data.get("server", {})
+        search_data = data.get("search", {})
 
         return cls(
             instance_id=data.get("instance_id", "default"),
             db=DatabaseConfig(**db_data) if db_data else DatabaseConfig(),
             embedding=EmbeddingConfig(**embedding_data) if embedding_data else EmbeddingConfig(),
             server=ServerConfig(**server_data) if server_data else ServerConfig(),
+            search=SearchConfig(**search_data) if search_data else SearchConfig(),
         )
 
     @classmethod

tribalmemory/server/models.py

@@ -204,3 +204,68 @@ class ImportResponse(BaseModel):
     duration_ms: float = 0.0
     error_details: list[str] = Field(default_factory=list)
     error: Optional[str] = None
+
+# =============================================================================
+# Session Indexing Models (Issue #38)
+# =============================================================================
+
+class SessionMessageRequest(BaseModel):
+    """A single message in a session transcript."""
+    role: str = Field(..., description="Message role (user, assistant, system)")
+    content: str = Field(..., description="Message content")
+    timestamp: datetime = Field(..., description="When the message was sent")
+
+
+class SessionIngestRequest(BaseModel):
+    """Request to ingest session transcript."""
+    session_id: str = Field(..., description="Unique session identifier")
+    messages: list[SessionMessageRequest] = Field(
+        ..., description="Conversation messages to index"
+    )
+    instance_id: Optional[str] = Field(
+        default=None,
+        description="Override instance ID (defaults to server's instance_id)"
+    )
+
+
+class SessionIngestResponse(BaseModel):
+    """Response from session ingestion."""
+    success: bool
+    chunks_created: int = 0
+    messages_processed: int = 0
+    error: Optional[str] = None
+
+
+class SessionSearchRequest(BaseModel):
+    """Request to search session transcripts."""
+    query: str = Field(..., description="Natural language search query")
+    session_id: Optional[str] = Field(
+        default=None,
+        description="Filter to specific session (optional)"
+    )
+    limit: int = Field(default=5, ge=1, le=50, description="Maximum results")
+    min_relevance: float = Field(
+        default=0.0,
+        ge=0.0,
+        le=1.0,
+        description="Minimum similarity score"
+    )
+
+
+class SessionChunkResponse(BaseModel):
+    """A session transcript chunk result."""
+    chunk_id: str
+    session_id: str
+    instance_id: str
+    content: str
+    similarity_score: float
+    start_time: datetime
+    end_time: datetime
+    chunk_index: int
+
+
+class SessionSearchResponse(BaseModel):
+    """Response from session search."""
+    results: list[SessionChunkResponse]
+    query: str
+    error: Optional[str] = None

tribalmemory/server/routes.py

@@ -7,6 +7,7 @@ from fastapi import APIRouter, HTTPException, Depends
 
 from ..interfaces import MemorySource, MemoryEntry
 from ..services import TribalMemoryService
+from ..services.session_store import SessionStore, SessionMessage
 from .models import (
     RememberRequest,
     RecallRequest,
@@ -24,6 +25,11 @@ from .models import (
     ExportResponse,
     ImportRequest,
     ImportResponse,
+    SessionIngestRequest,
+    SessionIngestResponse,
+    SessionSearchRequest,
+    SessionSearchResponse,
+    SessionChunkResponse,
 )
 
 router = APIRouter(prefix="/v1", tags=["memory"])
@@ -40,6 +46,17 @@ def get_memory_service() -> TribalMemoryService:
     return _memory_service
 
 
+def get_session_store() -> SessionStore:
+    """Dependency injection for session store.
+    
+    This is set by the app during startup.
+    """
+    from .app import _session_store
+    if _session_store is None:
+        raise HTTPException(status_code=503, detail="Session store not initialized")
+    return _session_store
+
+
 def get_instance_id() -> str:
     """Get the current instance ID."""
     from .app import _instance_id
@@ -376,3 +393,54 @@ async def shutdown() -> ShutdownResponse:
         0.5, lambda: os.kill(os.getpid(), signal.SIGTERM)
     )
     return ShutdownResponse(status="shutting_down")
+
+# =============================================================================
+# Session Indexing Routes (Issue #38)
+# =============================================================================
+
+@router.post("/sessions/ingest", response_model=SessionIngestResponse)
+async def ingest_session(
+    request: SessionIngestRequest,
+    store: SessionStore = Depends(get_session_store),
+    instance_id: str = Depends(get_instance_id),
+) -> SessionIngestResponse:
+    """Ingest a session transcript for indexing."""
+    try:
+        messages = [
+            SessionMessage(role=msg.role, content=msg.content, timestamp=msg.timestamp)
+            for msg in request.messages
+        ]
+        
+        result = await store.ingest(
+            session_id=request.session_id,
+            messages=messages,
+            instance_id=request.instance_id or instance_id,
+        )
+        
+        return SessionIngestResponse(
+            success=result.get("success", False),
+            chunks_created=result.get("chunks_created", 0),
+            messages_processed=result.get("messages_processed", 0),
+            error=result.get("error"),
+        )
+    except Exception as e:
+        return SessionIngestResponse(success=False, error=str(e))
+
+
+@router.get("/sessions/search", response_model=SessionSearchResponse)
+async def search_sessions(
+    query: str,
+    session_id: Optional[str] = None,
+    limit: int = 5,
+    min_relevance: float = 0.0,
+    store: SessionStore = Depends(get_session_store),
+) -> SessionSearchResponse:
+    """Search session transcripts by semantic similarity."""
+    try:
+        results = await store.search(query, session_id, limit, min_relevance)
+        return SessionSearchResponse(
+            results=[SessionChunkResponse(**r) for r in results],
+            query=query,
+        )
+    except Exception as e:
+        return SessionSearchResponse(results=[], query=query, error=str(e))

tribalmemory/services/fts_store.py

@@ -0,0 +1,255 @@
+"""SQLite FTS5 full-text search store for BM25 hybrid search.
+
+Provides keyword-based BM25 search alongside LanceDB vector search.
+FTS5 excels at exact-token queries (error strings, config names, IDs)
+while vector search handles semantic/fuzzy queries.
+
+The two are combined via hybrid scoring:
+    finalScore = vectorWeight * vectorScore + textWeight * bm25Score
+"""
+
+import logging
+import sqlite3
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class FTSStore:
+    """SQLite FTS5 store for keyword search over memories.
+
+    Creates a FTS5 virtual table alongside the main vector store.
+    Supports index, search, delete, and update operations.
+
+    Note: All methods are synchronous. SQLite operations are typically
+    sub-millisecond for the document counts we handle (<100k). If latency
+    becomes an issue on slow storage, wrap calls in asyncio.to_thread().
+    """
+
+    def __init__(self, db_path: str):
+        """Initialize FTS store.
+
+        Args:
+            db_path: Path to the SQLite database file. Created if missing.
+        """
+        self.db_path = db_path
+        Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+        self._conn: Optional[sqlite3.Connection] = None
+        self._fts_available: Optional[bool] = None
+        self._ensure_initialized()
+
+    def _get_conn(self) -> sqlite3.Connection:
+        if self._conn is None:
+            self._conn = sqlite3.connect(self.db_path)
+            self._conn.row_factory = sqlite3.Row
+        return self._conn
+
+    def _ensure_initialized(self) -> None:
+        """Create FTS5 virtual table if it doesn't exist."""
+        conn = self._get_conn()
+        if not self.is_available():
+            logger.warning("FTS5 not available in this SQLite build")
+            return
+        conn.execute("""
+            CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts
+            USING fts5(id, content, tags, tokenize='porter')
+        """)
+        # Mapping table to track which IDs are indexed (for upsert/delete)
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS fts_ids (
+                id TEXT PRIMARY KEY
+            )
+        """)
+        conn.commit()
+
+    def is_available(self) -> bool:
+        """Check if FTS5 is available in the current SQLite build."""
+        if self._fts_available is not None:
+            return self._fts_available
+        try:
+            conn = self._get_conn()
+            conn.execute(
+                "CREATE VIRTUAL TABLE IF NOT EXISTS _fts5_test "
+                "USING fts5(test_col)"
+            )
+            conn.execute("DROP TABLE IF EXISTS _fts5_test")
+            conn.commit()
+            self._fts_available = True
+        except sqlite3.OperationalError:
+            self._fts_available = False
+        return self._fts_available
+
+    def index(self, memory_id: str, content: str, tags: list[str]) -> None:
+        """Index a memory for full-text search.
+
+        If the memory_id already exists, it is replaced (upsert).
+        """
+        if not self.is_available():
+            return
+        conn = self._get_conn()
+        tags_text = " ".join(tags)
+
+        # Check if exists — delete first for upsert
+        existing = conn.execute(
+            "SELECT id FROM fts_ids WHERE id = ?", (memory_id,)
+        ).fetchone()
+        if existing:
+            conn.execute(
+                "DELETE FROM memories_fts WHERE id = ?", (memory_id,)
+            )
+
+        conn.execute(
+            "INSERT INTO memories_fts (id, content, tags) VALUES (?, ?, ?)",
+            (memory_id, content, tags_text),
+        )
+        conn.execute(
+            "INSERT OR REPLACE INTO fts_ids (id) VALUES (?)",
+            (memory_id,),
+        )
+        conn.commit()
+
+    def search(
+        self, query: str, limit: int = 10
+    ) -> list[dict]:
+        """Search memories using BM25.
+
+        Returns list of dicts with 'id' and 'rank' keys.
+        BM25 rank is negative; more negative = better match.
+        """
+        if not self.is_available():
+            return []
+        conn = self._get_conn()
+        # Use bm25() for ranking. FTS5 bm25() returns negative values
+        # where more negative = better match.
+        try:
+            rows = conn.execute(
+                """
+                SELECT id, rank
+                FROM memories_fts
+                WHERE memories_fts MATCH ?
+                ORDER BY rank
+                LIMIT ?
+                """,
+                (query, limit),
+            ).fetchall()
+            return [{"id": row["id"], "rank": row["rank"]} for row in rows]
+        except sqlite3.OperationalError as e:
+            # Malformed FTS query (unbalanced quotes, etc.)
+            logger.warning(f"FTS5 search error: {e}")
+            return []
+
+    def delete(self, memory_id: str) -> None:
+        """Remove a memory from the FTS index."""
+        if not self.is_available():
+            return
+        conn = self._get_conn()
+        conn.execute(
+            "DELETE FROM memories_fts WHERE id = ?", (memory_id,)
+        )
+        conn.execute("DELETE FROM fts_ids WHERE id = ?", (memory_id,))
+        conn.commit()
+
+    def count(self) -> int:
+        """Return number of indexed documents."""
+        if not self.is_available():
+            return 0
+        conn = self._get_conn()
+        row = conn.execute("SELECT COUNT(*) FROM fts_ids").fetchone()
+        return row[0]
+
+    def close(self) -> None:
+        """Close the database connection."""
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+
+
+def bm25_rank_to_score(rank: float) -> float:
+    """Convert BM25 rank to a 0..1 score.
+
+    FTS5 bm25() returns negative values where more negative = better.
+    We use: score = 1 / (1 + abs(rank))
+    """
+    return 1.0 / (1.0 + abs(rank))
+
+
+def hybrid_merge(
+    vector_results: list[dict],
+    bm25_results: list[dict],
+    vector_weight: float = 0.7,
+    text_weight: float = 0.3,
+) -> list[dict]:
+    """Merge vector similarity and BM25 results with weighted scoring.
+
+    BM25 ranks are min-max normalized to 0..1 so they're comparable
+    with vector similarity scores (also 0..1). The best BM25 hit gets
+    score 1.0, the worst gets a proportional score.
+
+    Args:
+        vector_results: List of {"id": str, "score": float} (0..1 cosine sim)
+        bm25_results: List of {"id": str, "rank": float} (negative BM25 rank)
+        vector_weight: Weight for vector similarity score
+        text_weight: Weight for BM25 text score
+
+    Returns:
+        Merged list sorted by final_score descending.
+        Each dict has: id, vector_score, text_score, final_score.
+    """
+    # Normalize weights
+    total = vector_weight + text_weight
+    if total > 0:
+        vector_weight /= total
+        text_weight /= total
+
+    # Min-max normalize BM25 ranks to 0..1
+    # BM25 ranks are negative; more negative = better match.
+    # When empty, skip normalization entirely — no BM25 contribution.
+    bm25_normalized: dict[str, float] = {}
+    if bm25_results:
+        abs_ranks = [abs(br["rank"]) for br in bm25_results]
+        max_rank = max(abs_ranks)
+        min_rank = min(abs_ranks)
+        rank_range = max_rank - min_rank
+
+        for br in bm25_results:
+            if rank_range > 0:
+                # Normalize: best rank (highest abs) → 1.0, worst → ~0
+                score = (abs(br["rank"]) - min_rank) / rank_range
+            else:
+                # All same rank → all get 1.0
+                score = 1.0
+            bm25_normalized[br["id"]] = score
+
+    # Build candidate map
+    candidates: dict[str, dict] = {}
+
+    for vr in vector_results:
+        mid = vr["id"]
+        candidates[mid] = {
+            "id": mid,
+            "vector_score": vr["score"],
+            "text_score": 0.0,
+        }
+
+    for mid, text_score in bm25_normalized.items():
+        if mid in candidates:
+            candidates[mid]["text_score"] = text_score
+        else:
+            candidates[mid] = {
+                "id": mid,
+                "vector_score": 0.0,
+                "text_score": text_score,
+            }
+
+    # Compute final scores
+    for c in candidates.values():
+        c["final_score"] = (
+            vector_weight * c["vector_score"]
+            + text_weight * c["text_score"]
+        )
+
+    # Sort by final score descending
+    return sorted(
+        candidates.values(), key=lambda x: x["final_score"], reverse=True
+    )