omni-cortex 1.17.1__py3-none-any.whl → 1.17.2__py3-none-any.whl

omni_cortex/search/keyword.py

@@ -0,0 +1,204 @@
+"""Keyword search using SQLite FTS5."""
+
+import sqlite3
+from typing import Optional
+
+from ..models.memory import Memory, _row_to_memory
+
+
+def keyword_search(
+    conn: sqlite3.Connection,
+    query: str,
+    type_filter: Optional[str] = None,
+    tags_filter: Optional[list[str]] = None,
+    status_filter: Optional[str] = None,
+    min_importance: Optional[int] = None,
+    include_archived: bool = False,
+    limit: int = 10,
+) -> list[tuple[Memory, float]]:
+    """Search memories using FTS5 keyword search.
+
+    Args:
+        conn: Database connection
+        query: Search query string
+        type_filter: Filter by memory type
+        tags_filter: Filter by tags
+        status_filter: Filter by status
+        min_importance: Minimum importance score
+        include_archived: Include archived memories
+        limit: Maximum results
+
+    Returns:
+        List of (Memory, score) tuples
+    """
+    # Build the FTS query
+    # Escape special FTS5 characters
+    fts_query = _escape_fts_query(query)
+
+    # Build WHERE conditions for the join
+    where_conditions = []
+    params: list = [fts_query]
+
+    if type_filter:
+        where_conditions.append("m.type = ?")
+        params.append(type_filter)
+
+    if status_filter:
+        where_conditions.append("m.status = ?")
+        params.append(status_filter)
+    elif not include_archived:
+        where_conditions.append("m.status != 'archived'")
+
+    if min_importance is not None:
+        where_conditions.append("m.importance_score >= ?")
+        params.append(min_importance)
+
+    if tags_filter:
+        tag_conditions = []
+        for tag in tags_filter:
+            tag_conditions.append("m.tags LIKE ?")
+            params.append(f'%"{tag}"%')
+        where_conditions.append(f"({' OR '.join(tag_conditions)})")
+
+    where_sql = ""
+    if where_conditions:
+        where_sql = "AND " + " AND ".join(where_conditions)
+
+    params.append(limit)
+
+    cursor = conn.cursor()
+
+    # Use FTS5 with bm25 ranking
+    try:
+        cursor.execute(
+            f"""
+            SELECT m.*, bm25(memories_fts) as score
+            FROM memories_fts fts
+            JOIN memories m ON fts.rowid = m.rowid
+            WHERE memories_fts MATCH ?
+            {where_sql}
+            ORDER BY score
+            LIMIT ?
+            """,
+            params,
+        )
+
+        results = []
+        for row in cursor.fetchall():
+            # bm25 returns negative scores (more negative = better match)
+            # Convert to positive scores for our ranking
+            score = -row["score"]
+            memory = _row_to_memory(row)
+            results.append((memory, score))
+
+        return results
+
+    except sqlite3.OperationalError as e:
+        # If FTS query fails, fall back to LIKE search
+        if "fts5" in str(e).lower() or "match" in str(e).lower():
+            return _fallback_like_search(
+                conn, query, type_filter, tags_filter, status_filter,
+                min_importance, include_archived, limit
+            )
+        raise
+
+
+def _escape_fts_query(query: str) -> str:
+    """Escape special characters for FTS5 query.
+
+    Args:
+        query: Raw search query
+
+    Returns:
+        Escaped FTS5 query
+    """
+    # Remove FTS5 special characters that could cause syntax errors
+    special_chars = ['"', "'", "(", ")", "*", ":", "^", "-", "+"]
+    escaped = query
+    for char in special_chars:
+        escaped = escaped.replace(char, " ")
+
+    # Clean up whitespace
+    words = escaped.split()
+
+    # Handle empty query
+    if not words:
+        return '""'
+
+    # For simple queries, just use OR matching
+    if len(words) == 1:
+        return f'"{words[0]}"'
+
+    # For multi-word queries, match any word
+    return " OR ".join(f'"{word}"' for word in words)
+
+
+def _fallback_like_search(
+    conn: sqlite3.Connection,
+    query: str,
+    type_filter: Optional[str],
+    tags_filter: Optional[list[str]],
+    status_filter: Optional[str],
+    min_importance: Optional[int],
+    include_archived: bool,
+    limit: int,
+) -> list[tuple[Memory, float]]:
+    """Fallback to LIKE search if FTS5 fails."""
+    words = query.lower().split()
+    if not words:
+        return []
+
+    where_conditions = []
+    params: list = []
+
+    # Match any word in content or context
+    word_conditions = []
+    for word in words:
+        word_conditions.append("(LOWER(content) LIKE ? OR LOWER(context) LIKE ?)")
+        params.extend([f"%{word}%", f"%{word}%"])
+    where_conditions.append(f"({' OR '.join(word_conditions)})")
+
+    if type_filter:
+        where_conditions.append("type = ?")
+        params.append(type_filter)
+
+    if status_filter:
+        where_conditions.append("status = ?")
+        params.append(status_filter)
+    elif not include_archived:
+        where_conditions.append("status != 'archived'")
+
+    if min_importance is not None:
+        where_conditions.append("importance_score >= ?")
+        params.append(min_importance)
+
+    if tags_filter:
+        tag_conds = []
+        for tag in tags_filter:
+            tag_conds.append("tags LIKE ?")
+            params.append(f'%"{tag}"%')
+        where_conditions.append(f"({' OR '.join(tag_conds)})")
+
+    params.append(limit)
+
+    cursor = conn.cursor()
+    cursor.execute(
+        f"""
+        SELECT *
+        FROM memories
+        WHERE {' AND '.join(where_conditions)}
+        ORDER BY importance_score DESC, last_accessed DESC
+        LIMIT ?
+        """,
+        params,
+    )
+
+    results = []
+    for row in cursor.fetchall():
+        memory = _row_to_memory(row)
+        # Calculate a simple score based on word matches
+        content = (memory.content + " " + (memory.context or "")).lower()
+        score = sum(1 for word in words if word in content)
+        results.append((memory, float(score)))
+
+    return results

omni_cortex/search/ranking.py

@@ -0,0 +1,127 @@
+"""Multi-factor relevance ranking for search results."""
+
+import math
+from datetime import datetime, timezone
+from typing import Optional
+
+from ..models.memory import Memory
+from ..utils.timestamps import parse_iso
+
+
+def calculate_relevance_score(
+    memory: Memory,
+    keyword_score: float = 0.0,
+    semantic_score: float = 0.0,
+    query: Optional[str] = None,
+) -> float:
+    """Calculate multi-factor relevance score for a memory.
+
+    Scoring factors:
+    - Keyword match score (40%)
+    - Semantic similarity score (40%)
+    - Access frequency (log scale, max +20)
+    - Recency (exponential decay, max +15)
+    - Freshness status bonus/penalty
+    - Importance score (0-15)
+
+    Args:
+        memory: Memory object to score
+        keyword_score: Score from keyword search (0-1 normalized)
+        semantic_score: Score from semantic search (0-1 normalized)
+        query: Optional query string for additional matching
+
+    Returns:
+        Combined relevance score (higher = more relevant)
+    """
+    score = 0.0
+
+    # Base scores from search (40% each, max 80 combined)
+    # Normalize to 0-40 range
+    score += min(40.0, keyword_score * 40.0)
+    score += min(40.0, semantic_score * 40.0)
+
+    # Access frequency bonus (log scale, max +20)
+    # More frequently accessed memories are likely more useful
+    access_count = memory.access_count or 0
+    access_bonus = min(20.0, math.log1p(access_count) * 5.0)
+    score += access_bonus
+
+    # Recency bonus (exponential decay over 30 days, max +15)
+    # Recently accessed memories are more relevant
+    last_accessed = parse_iso(memory.last_accessed)
+    now = datetime.now(timezone.utc)
+    days_since_access = (now - last_accessed).days
+
+    recency_bonus = max(0.0, 15.0 * math.exp(-days_since_access / 30.0))
+    score += recency_bonus
+
+    # Freshness status bonus/penalty
+    freshness_bonus = {
+        "fresh": 10.0,
+        "needs_review": 0.0,
+        "outdated": -10.0,
+        "archived": -30.0,
+    }
+    score += freshness_bonus.get(memory.status, 0.0)
+
+    # Importance score contribution (0-100 scaled to 0-15)
+    importance = memory.importance_score or 50.0
+    score += importance * 0.15
+
+    # Exact phrase match bonus (if query provided)
+    if query and query.lower() in memory.content.lower():
+        score += 10.0
+
+    return score
+
+
+def rank_memories(
+    memories_with_scores: list[tuple[Memory, float, float]],
+    query: Optional[str] = None,
+) -> list[tuple[Memory, float]]:
+    """Rank memories by combined relevance score.
+
+    Args:
+        memories_with_scores: List of (Memory, keyword_score, semantic_score) tuples
+        query: Optional query string
+
+    Returns:
+        List of (Memory, final_score) tuples, sorted by score descending
+    """
+    results = []
+
+    for memory, keyword_score, semantic_score in memories_with_scores:
+        final_score = calculate_relevance_score(
+            memory,
+            keyword_score=keyword_score,
+            semantic_score=semantic_score,
+            query=query,
+        )
+        results.append((memory, final_score))
+
+    # Sort by score descending
+    results.sort(key=lambda x: x[1], reverse=True)
+
+    return results
+
+
+def normalize_scores(scores: list[float]) -> list[float]:
+    """Normalize a list of scores to 0-1 range.
+
+    Args:
+        scores: List of raw scores
+
+    Returns:
+        List of normalized scores
+    """
+    if not scores:
+        return []
+
+    min_score = min(scores)
+    max_score = max(scores)
+    range_score = max_score - min_score
+
+    if range_score == 0:
+        return [1.0] * len(scores)
+
+    return [(s - min_score) / range_score for s in scores]

omni_cortex/search/semantic.py

@@ -0,0 +1,232 @@
+"""Semantic search using vector embeddings."""
+
+import logging
+import sqlite3
+from typing import Optional
+
+import numpy as np
+
+from ..models.memory import Memory, _row_to_memory
+from ..embeddings.local import (
+    generate_embedding,
+    blob_to_vector,
+    DEFAULT_MODEL_NAME,
+)
+from ..config import load_config
+
+logger = logging.getLogger(__name__)
+
+
+def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
+    """Calculate cosine similarity between two vectors.
+
+    Args:
+        a: First vector
+        b: Second vector
+
+    Returns:
+        Similarity score between -1 and 1
+    """
+    dot_product = np.dot(a, b)
+    norm_a = np.linalg.norm(a)
+    norm_b = np.linalg.norm(b)
+
+    if norm_a == 0 or norm_b == 0:
+        return 0.0
+
+    return float(dot_product / (norm_a * norm_b))
+
+
+def semantic_search(
+    conn: sqlite3.Connection,
+    query: str,
+    type_filter: Optional[str] = None,
+    tags_filter: Optional[list[str]] = None,
+    status_filter: Optional[str] = None,
+    min_importance: Optional[int] = None,
+    include_archived: bool = False,
+    limit: int = 10,
+    similarity_threshold: float = 0.3,
+    model_name: str = DEFAULT_MODEL_NAME,
+) -> list[tuple[Memory, float]]:
+    """Search memories using semantic similarity.
+
+    Args:
+        conn: Database connection
+        query: Search query string
+        type_filter: Filter by memory type
+        tags_filter: Filter by tags
+        status_filter: Filter by status
+        min_importance: Minimum importance score
+        include_archived: Include archived memories
+        limit: Maximum results
+        similarity_threshold: Minimum similarity score
+        model_name: Embedding model to use
+
+    Returns:
+        List of (Memory, similarity_score) tuples
+    """
+    # Check if embeddings are enabled - skip semantic search if disabled
+    config = load_config()
+    if not config.embedding_enabled:
+        logger.debug("Embeddings disabled, skipping semantic search")
+        return []
+
+    # Generate embedding for query
+    try:
+        query_embedding = generate_embedding(query, model_name)
+    except ImportError:
+        logger.warning("sentence-transformers not available, cannot perform semantic search")
+        return []
+    except Exception as e:
+        logger.error(f"Failed to generate query embedding: {e}")
+        return []
+
+    # Build WHERE conditions for filtering
+    where_conditions = []
+    params: list = []
+
+    # Only search memories with embeddings
+    where_conditions.append("m.has_embedding = 1")
+
+    if type_filter:
+        where_conditions.append("m.type = ?")
+        params.append(type_filter)
+
+    if status_filter:
+        where_conditions.append("m.status = ?")
+        params.append(status_filter)
+    elif not include_archived:
+        where_conditions.append("m.status != 'archived'")
+
+    if min_importance is not None:
+        where_conditions.append("m.importance_score >= ?")
+        params.append(min_importance)
+
+    if tags_filter:
+        tag_conditions = []
+        for tag in tags_filter:
+            tag_conditions.append("m.tags LIKE ?")
+            params.append(f'%"{tag}"%')
+        where_conditions.append(f"({' OR '.join(tag_conditions)})")
+
+    where_sql = "WHERE " + " AND ".join(where_conditions)
+
+    # Get all matching memories with their embeddings
+    cursor = conn.cursor()
+    cursor.execute(
+        f"""
+        SELECT m.*, e.vector
+        FROM memories m
+        JOIN embeddings e ON m.id = e.memory_id
+        {where_sql}
+        """,
+        params,
+    )
+
+    # Calculate similarity scores
+    results = []
+    for row in cursor.fetchall():
+        memory_embedding = blob_to_vector(row["vector"])
+        similarity = cosine_similarity(query_embedding, memory_embedding)
+
+        # Apply threshold
+        if similarity >= similarity_threshold:
+            memory = _row_to_memory(row)
+            results.append((memory, similarity))
+
+    # Sort by similarity (highest first)
+    results.sort(key=lambda x: x[1], reverse=True)
+
+    # Limit results
+    return results[:limit]
+
+
+def find_similar_memories(
+    conn: sqlite3.Connection,
+    memory_id: str,
+    limit: int = 10,
+    similarity_threshold: float = 0.5,
+    exclude_ids: Optional[list[str]] = None,
+) -> list[tuple[Memory, float]]:
+    """Find memories similar to a given memory.
+
+    Args:
+        conn: Database connection
+        memory_id: ID of the source memory
+        limit: Maximum results
+        similarity_threshold: Minimum similarity score
+        exclude_ids: Memory IDs to exclude
+
+    Returns:
+        List of (Memory, similarity_score) tuples
+    """
+    # Get the source memory's embedding
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT vector FROM embeddings WHERE memory_id = ?",
+        (memory_id,),
+    )
+    row = cursor.fetchone()
+
+    if not row:
+        logger.warning(f"No embedding found for memory {memory_id}")
+        return []
+
+    source_embedding = blob_to_vector(row["vector"])
+
+    # Build exclusion list
+    exclude_set = set(exclude_ids or [])
+    exclude_set.add(memory_id)  # Always exclude the source
+
+    # Get all other embeddings
+    cursor.execute(
+        """
+        SELECT m.*, e.vector
+        FROM memories m
+        JOIN embeddings e ON m.id = e.memory_id
+        WHERE m.has_embedding = 1 AND m.status != 'archived'
+        """
+    )
+
+    results = []
+    for row in cursor.fetchall():
+        if row["id"] in exclude_set:
+            continue
+
+        memory_embedding = blob_to_vector(row["vector"])
+        similarity = cosine_similarity(source_embedding, memory_embedding)
+
+        if similarity >= similarity_threshold:
+            memory = _row_to_memory(row)
+            results.append((memory, similarity))
+
+    # Sort by similarity
+    results.sort(key=lambda x: x[1], reverse=True)
+
+    return results[:limit]
+
+
+def get_embedding_coverage(conn: sqlite3.Connection) -> dict:
+    """Get statistics about embedding coverage.
+
+    Returns:
+        Dict with total_memories, with_embeddings, without_embeddings, coverage_pct
+    """
+    cursor = conn.cursor()
+
+    cursor.execute("SELECT COUNT(*) FROM memories")
+    total = cursor.fetchone()[0]
+
+    cursor.execute("SELECT COUNT(*) FROM memories WHERE has_embedding = 1")
+    with_embeddings = cursor.fetchone()[0]
+
+    without_embeddings = total - with_embeddings
+    coverage_pct = (with_embeddings / total * 100) if total > 0 else 0.0
+
+    return {
+        "total_memories": total,
+        "with_embeddings": with_embeddings,
+        "without_embeddings": without_embeddings,
+        "coverage_pct": round(coverage_pct, 1),
+    }