tribalmemory 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

tribalmemory/server/models.py

@@ -204,3 +204,68 @@ class ImportResponse(BaseModel):
     duration_ms: float = 0.0
     error_details: list[str] = Field(default_factory=list)
     error: Optional[str] = None
+
+# =============================================================================
+# Session Indexing Models (Issue #38)
+# =============================================================================
+
+class SessionMessageRequest(BaseModel):
+    """A single message in a session transcript."""
+    role: str = Field(..., description="Message role (user, assistant, system)")
+    content: str = Field(..., description="Message content")
+    timestamp: datetime = Field(..., description="When the message was sent")
+
+
+class SessionIngestRequest(BaseModel):
+    """Request to ingest session transcript."""
+    session_id: str = Field(..., description="Unique session identifier")
+    messages: list[SessionMessageRequest] = Field(
+        ..., description="Conversation messages to index"
+    )
+    instance_id: Optional[str] = Field(
+        default=None,
+        description="Override instance ID (defaults to server's instance_id)"
+    )
+
+
+class SessionIngestResponse(BaseModel):
+    """Response from session ingestion."""
+    success: bool
+    chunks_created: int = 0
+    messages_processed: int = 0
+    error: Optional[str] = None
+
+
+class SessionSearchRequest(BaseModel):
+    """Request to search session transcripts."""
+    query: str = Field(..., description="Natural language search query")
+    session_id: Optional[str] = Field(
+        default=None,
+        description="Filter to specific session (optional)"
+    )
+    limit: int = Field(default=5, ge=1, le=50, description="Maximum results")
+    min_relevance: float = Field(
+        default=0.0,
+        ge=0.0,
+        le=1.0,
+        description="Minimum similarity score"
+    )
+
+
+class SessionChunkResponse(BaseModel):
+    """A session transcript chunk result."""
+    chunk_id: str
+    session_id: str
+    instance_id: str
+    content: str
+    similarity_score: float
+    start_time: datetime
+    end_time: datetime
+    chunk_index: int
+
+
+class SessionSearchResponse(BaseModel):
+    """Response from session search."""
+    results: list[SessionChunkResponse]
+    query: str
+    error: Optional[str] = None

tribalmemory/server/routes.py

@@ -7,6 +7,7 @@ from fastapi import APIRouter, HTTPException, Depends
 
 from ..interfaces import MemorySource, MemoryEntry
 from ..services import TribalMemoryService
+from ..services.session_store import SessionStore, SessionMessage
 from .models import (
     RememberRequest,
     RecallRequest,
@@ -24,6 +25,11 @@ from .models import (
     ExportResponse,
     ImportRequest,
     ImportResponse,
+    SessionIngestRequest,
+    SessionIngestResponse,
+    SessionSearchRequest,
+    SessionSearchResponse,
+    SessionChunkResponse,
 )
 
 router = APIRouter(prefix="/v1", tags=["memory"])
@@ -40,6 +46,17 @@ def get_memory_service() -> TribalMemoryService:
     return _memory_service
 
 
+def get_session_store() -> SessionStore:
+    """Dependency injection for session store.
+    
+    This is set by the app during startup.
+    """
+    from .app import _session_store
+    if _session_store is None:
+        raise HTTPException(status_code=503, detail="Session store not initialized")
+    return _session_store
+
+
 def get_instance_id() -> str:
     """Get the current instance ID."""
     from .app import _instance_id
@@ -376,3 +393,54 @@ async def shutdown() -> ShutdownResponse:
         0.5, lambda: os.kill(os.getpid(), signal.SIGTERM)
     )
     return ShutdownResponse(status="shutting_down")
+
+# =============================================================================
+# Session Indexing Routes (Issue #38)
+# =============================================================================
+
+@router.post("/sessions/ingest", response_model=SessionIngestResponse)
+async def ingest_session(
+    request: SessionIngestRequest,
+    store: SessionStore = Depends(get_session_store),
+    instance_id: str = Depends(get_instance_id),
+) -> SessionIngestResponse:
+    """Ingest a session transcript for indexing."""
+    try:
+        messages = [
+            SessionMessage(role=msg.role, content=msg.content, timestamp=msg.timestamp)
+            for msg in request.messages
+        ]
+        
+        result = await store.ingest(
+            session_id=request.session_id,
+            messages=messages,
+            instance_id=request.instance_id or instance_id,
+        )
+        
+        return SessionIngestResponse(
+            success=result.get("success", False),
+            chunks_created=result.get("chunks_created", 0),
+            messages_processed=result.get("messages_processed", 0),
+            error=result.get("error"),
+        )
+    except Exception as e:
+        return SessionIngestResponse(success=False, error=str(e))
+
+
+@router.get("/sessions/search", response_model=SessionSearchResponse)
+async def search_sessions(
+    query: str,
+    session_id: Optional[str] = None,
+    limit: int = 5,
+    min_relevance: float = 0.0,
+    store: SessionStore = Depends(get_session_store),
+) -> SessionSearchResponse:
+    """Search session transcripts by semantic similarity."""
+    try:
+        results = await store.search(query, session_id, limit, min_relevance)
+        return SessionSearchResponse(
+            results=[SessionChunkResponse(**r) for r in results],
+            query=query,
+        )
+    except Exception as e:
+        return SessionSearchResponse(results=[], query=query, error=str(e))

tribalmemory/services/fts_store.py

@@ -0,0 +1,255 @@
+"""SQLite FTS5 full-text search store for BM25 hybrid search.
+
+Provides keyword-based BM25 search alongside LanceDB vector search.
+FTS5 excels at exact-token queries (error strings, config names, IDs)
+while vector search handles semantic/fuzzy queries.
+
+The two are combined via hybrid scoring:
+    finalScore = vectorWeight * vectorScore + textWeight * bm25Score
+"""
+
+import logging
+import sqlite3
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class FTSStore:
+    """SQLite FTS5 store for keyword search over memories.
+
+    Creates a FTS5 virtual table alongside the main vector store.
+    Supports index, search, delete, and update operations.
+
+    Note: All methods are synchronous. SQLite operations are typically
+    sub-millisecond for the document counts we handle (<100k). If latency
+    becomes an issue on slow storage, wrap calls in asyncio.to_thread().
+    """
+
+    def __init__(self, db_path: str):
+        """Initialize FTS store.
+
+        Args:
+            db_path: Path to the SQLite database file. Created if missing.
+        """
+        self.db_path = db_path
+        Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+        self._conn: Optional[sqlite3.Connection] = None
+        self._fts_available: Optional[bool] = None
+        self._ensure_initialized()
+
+    def _get_conn(self) -> sqlite3.Connection:
+        if self._conn is None:
+            self._conn = sqlite3.connect(self.db_path)
+            self._conn.row_factory = sqlite3.Row
+        return self._conn
+
+    def _ensure_initialized(self) -> None:
+        """Create FTS5 virtual table if it doesn't exist."""
+        conn = self._get_conn()
+        if not self.is_available():
+            logger.warning("FTS5 not available in this SQLite build")
+            return
+        conn.execute("""
+            CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts
+            USING fts5(id, content, tags, tokenize='porter')
+        """)
+        # Mapping table to track which IDs are indexed (for upsert/delete)
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS fts_ids (
+                id TEXT PRIMARY KEY
+            )
+        """)
+        conn.commit()
+
+    def is_available(self) -> bool:
+        """Check if FTS5 is available in the current SQLite build."""
+        if self._fts_available is not None:
+            return self._fts_available
+        try:
+            conn = self._get_conn()
+            conn.execute(
+                "CREATE VIRTUAL TABLE IF NOT EXISTS _fts5_test "
+                "USING fts5(test_col)"
+            )
+            conn.execute("DROP TABLE IF EXISTS _fts5_test")
+            conn.commit()
+            self._fts_available = True
+        except sqlite3.OperationalError:
+            self._fts_available = False
+        return self._fts_available
+
+    def index(self, memory_id: str, content: str, tags: list[str]) -> None:
+        """Index a memory for full-text search.
+
+        If the memory_id already exists, it is replaced (upsert).
+        """
+        if not self.is_available():
+            return
+        conn = self._get_conn()
+        tags_text = " ".join(tags)
+
+        # Check if exists — delete first for upsert
+        existing = conn.execute(
+            "SELECT id FROM fts_ids WHERE id = ?", (memory_id,)
+        ).fetchone()
+        if existing:
+            conn.execute(
+                "DELETE FROM memories_fts WHERE id = ?", (memory_id,)
+            )
+
+        conn.execute(
+            "INSERT INTO memories_fts (id, content, tags) VALUES (?, ?, ?)",
+            (memory_id, content, tags_text),
+        )
+        conn.execute(
+            "INSERT OR REPLACE INTO fts_ids (id) VALUES (?)",
+            (memory_id,),
+        )
+        conn.commit()
+
+    def search(
+        self, query: str, limit: int = 10
+    ) -> list[dict]:
+        """Search memories using BM25.
+
+        Returns list of dicts with 'id' and 'rank' keys.
+        BM25 rank is negative; more negative = better match.
+        """
+        if not self.is_available():
+            return []
+        conn = self._get_conn()
+        # Use bm25() for ranking. FTS5 bm25() returns negative values
+        # where more negative = better match.
+        try:
+            rows = conn.execute(
+                """
+                SELECT id, rank
+                FROM memories_fts
+                WHERE memories_fts MATCH ?
+                ORDER BY rank
+                LIMIT ?
+                """,
+                (query, limit),
+            ).fetchall()
+            return [{"id": row["id"], "rank": row["rank"]} for row in rows]
+        except sqlite3.OperationalError as e:
+            # Malformed FTS query (unbalanced quotes, etc.)
+            logger.warning(f"FTS5 search error: {e}")
+            return []
+
+    def delete(self, memory_id: str) -> None:
+        """Remove a memory from the FTS index."""
+        if not self.is_available():
+            return
+        conn = self._get_conn()
+        conn.execute(
+            "DELETE FROM memories_fts WHERE id = ?", (memory_id,)
+        )
+        conn.execute("DELETE FROM fts_ids WHERE id = ?", (memory_id,))
+        conn.commit()
+
+    def count(self) -> int:
+        """Return number of indexed documents."""
+        if not self.is_available():
+            return 0
+        conn = self._get_conn()
+        row = conn.execute("SELECT COUNT(*) FROM fts_ids").fetchone()
+        return row[0]
+
+    def close(self) -> None:
+        """Close the database connection."""
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+
+
+def bm25_rank_to_score(rank: float) -> float:
+    """Convert BM25 rank to a 0..1 score.
+
+    FTS5 bm25() returns negative values where more negative = better.
+    We use: score = 1 / (1 + abs(rank))
+    """
+    return 1.0 / (1.0 + abs(rank))
+
+
+def hybrid_merge(
+    vector_results: list[dict],
+    bm25_results: list[dict],
+    vector_weight: float = 0.7,
+    text_weight: float = 0.3,
+) -> list[dict]:
+    """Merge vector similarity and BM25 results with weighted scoring.
+
+    BM25 ranks are min-max normalized to 0..1 so they're comparable
+    with vector similarity scores (also 0..1). The best BM25 hit gets
+    score 1.0, the worst gets a proportional score.
+
+    Args:
+        vector_results: List of {"id": str, "score": float} (0..1 cosine sim)
+        bm25_results: List of {"id": str, "rank": float} (negative BM25 rank)
+        vector_weight: Weight for vector similarity score
+        text_weight: Weight for BM25 text score
+
+    Returns:
+        Merged list sorted by final_score descending.
+        Each dict has: id, vector_score, text_score, final_score.
+    """
+    # Normalize weights
+    total = vector_weight + text_weight
+    if total > 0:
+        vector_weight /= total
+        text_weight /= total
+
+    # Min-max normalize BM25 ranks to 0..1
+    # BM25 ranks are negative; more negative = better match.
+    # When empty, skip normalization entirely — no BM25 contribution.
+    bm25_normalized: dict[str, float] = {}
+    if bm25_results:
+        abs_ranks = [abs(br["rank"]) for br in bm25_results]
+        max_rank = max(abs_ranks)
+        min_rank = min(abs_ranks)
+        rank_range = max_rank - min_rank
+
+        for br in bm25_results:
+            if rank_range > 0:
+                # Normalize: best rank (highest abs) → 1.0, worst → ~0
+                score = (abs(br["rank"]) - min_rank) / rank_range
+            else:
+                # All same rank → all get 1.0
+                score = 1.0
+            bm25_normalized[br["id"]] = score
+
+    # Build candidate map
+    candidates: dict[str, dict] = {}
+
+    for vr in vector_results:
+        mid = vr["id"]
+        candidates[mid] = {
+            "id": mid,
+            "vector_score": vr["score"],
+            "text_score": 0.0,
+        }
+
+    for mid, text_score in bm25_normalized.items():
+        if mid in candidates:
+            candidates[mid]["text_score"] = text_score
+        else:
+            candidates[mid] = {
+                "id": mid,
+                "vector_score": 0.0,
+                "text_score": text_score,
+            }
+
+    # Compute final scores
+    for c in candidates.values():
+        c["final_score"] = (
+            vector_weight * c["vector_score"]
+            + text_weight * c["text_score"]
+        )
+
+    # Sort by final score descending
+    return sorted(
+        candidates.values(), key=lambda x: x["final_score"], reverse=True
+    )