icdev 0.0.3__py3-none-any.whl

tools/memory/hybrid_search.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Hybrid search: combines BM25 keyword search + semantic vector search.
+
+Supports user-scoped queries (D180), time-decay ranking (D147), and JSON output.
+"""
+
+import argparse
+import json
+import sqlite3
+import struct
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+DB_PATH = BASE_DIR / "data" / "memory.db"
+
+
+def get_all_entries(user_id=None, tenant_id=None):
+    conn = sqlite3.connect(str(DB_PATH))
+    c = conn.cursor()
+
+    sql = "SELECT id, content, type, importance, embedding, created_at FROM memory_entries WHERE 1=1"
+    params = []
+
+    if user_id:
+        sql += " AND (user_id = ? OR user_id IS NULL)"
+        params.append(user_id)
+    if tenant_id:
+        sql += " AND (tenant_id = ? OR tenant_id IS NULL)"
+        params.append(tenant_id)
+
+    c.execute(sql, params)
+    rows = c.fetchall()
+    conn.close()
+    return rows
+
+
+def bm25_search(query, entries):
+    """BM25 keyword ranking with fallback to simple term frequency."""
+    documents = [entry[1] for entry in entries]
+
+    try:
+        from rank_bm25 import BM25Okapi
+        tokenized = [doc.lower().split() for doc in documents]
+        bm25 = BM25Okapi(tokenized)
+        scores = bm25.get_scores(query.lower().split())
+    except ImportError:
+        # Fallback: simple term frequency
+        query_terms = query.lower().split()
+        scores = []
+        for doc in documents:
+            doc_lower = doc.lower()
+            score = sum(doc_lower.count(term) for term in query_terms)
+            scores.append(float(score))
+
+    # Normalize scores to 0-1
+    max_score = max(scores) if scores and max(scores) > 0 else 1.0
+    return [s / max_score for s in scores]
+
+
+def semantic_search(query, entries):
+    """Semantic similarity using embeddings (vendor-agnostic via LLM provider)."""
+    # Try LLM provider system first (supports OpenAI, Ollama, Bedrock Titan)
+    query_emb = None
+    try:
+        from tools.llm import get_embedding_provider
+        provider = get_embedding_provider()
+        query_emb = provider.embed(query)
+    except Exception:
+        pass
+
+    # Fallback to direct OpenAI
+    if query_emb is None:
+        try:
+            from dotenv import load_dotenv
+            load_dotenv(BASE_DIR / ".env")
+        except ImportError:
+            pass
+
+        import os
+        api_key = os.environ.get("OPENAI_API_KEY")
+        if not api_key:
+            return None
+
+        try:
+            import openai
+            client = openai.OpenAI(api_key=api_key)
+            response = client.embeddings.create(input=query, model="text-embedding-3-small")
+            query_emb = response.data[0].embedding
+        except (ImportError, Exception):
+            return None
+
+    if query_emb is None:
+        return None
+
+    scores = []
+    for entry in entries:
+        emb_blob = entry[4]
+        if emb_blob is None:
+            scores.append(0.0)
+            continue
+        n = len(emb_blob) // 4
+        stored_emb = list(struct.unpack(f"{n}f", emb_blob))
+
+        # Cosine similarity
+        try:
+            import numpy as np
+            a, b = np.array(query_emb), np.array(stored_emb)
+            score = float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))
+        except ImportError:
+            dot = sum(x * y for x, y in zip(query_emb, stored_emb))
+            norm_a = sum(x * x for x in query_emb) ** 0.5
+            norm_b = sum(x * x for x in stored_emb) ** 0.5
+            score = dot / (norm_a * norm_b) if norm_a and norm_b else 0.0
+
+        scores.append(score)
+
+    # Normalize to 0-1
+    max_score = max(scores) if scores and max(scores) > 0 else 1.0
+    return [s / max_score for s in scores]
+
+
+def hybrid_rank(entries, bm25_scores, semantic_scores, bm25_weight, semantic_weight,
+                time_decay_enabled=False, decay_config=None):
+    """Combine BM25 and semantic scores, with optional time-decay (D147)."""
+    results = []
+    for i, entry in enumerate(entries):
+        bm25_s = bm25_scores[i] if bm25_scores else 0.0
+        sem_s = semantic_scores[i] if semantic_scores else 0.0
+
+        if semantic_scores is None:
+            # No semantic available, use BM25 only
+            combined = bm25_s
+        else:
+            combined = (bm25_weight * bm25_s) + (semantic_weight * sem_s)
+
+        id_, content, type_, importance, _, created_at = entry
+
+        # D147: Apply time-decay reranking when enabled
+        if time_decay_enabled:
+            try:
+                from tools.memory.time_decay import compute_time_aware_score
+                combined = compute_time_aware_score(
+                    base_score=combined,
+                    created_at=created_at or "",
+                    memory_type=type_ or "event",
+                    importance=importance or 5,
+                    config=decay_config,
+                )
+            except (ImportError, Exception):
+                pass  # Fall through to original combined score
+
+        results.append((combined, id_, content, type_, importance, created_at))
+
+    results.sort(reverse=True, key=lambda x: x[0])
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Hybrid search (BM25 + semantic)")
+    parser.add_argument("--query", required=True, help="Search query")
+    parser.add_argument("--limit", type=int, default=10, help="Max results")
+    parser.add_argument("--bm25-weight", type=float, default=0.7, help="BM25 weight (default 0.7)")
+    parser.add_argument("--semantic-weight", type=float, default=0.3, help="Semantic weight (default 0.3)")
+    parser.add_argument("--time-decay", action="store_true", help="Enable time-decay scoring (D147)")
+    parser.add_argument("--user-id", help="Filter by user ID (D180)")
+    parser.add_argument("--tenant-id", help="Filter by tenant ID (D180)")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    args = parser.parse_args()
+
+    entries = get_all_entries(user_id=args.user_id, tenant_id=args.tenant_id)
+    if not entries:
+        if args.json:
+            print(json.dumps({"classification": "CUI // SP-CTI", "count": 0, "entries": []}))
+        else:
+            print("No memory entries found.")
+        return
+
+    bm25_scores = bm25_search(args.query, entries)
+    semantic_scores = semantic_search(args.query, entries)
+
+    if semantic_scores is None and not args.json:
+        print("(Semantic search unavailable — using keyword search only)")
+
+    # D147: Load time-decay config if enabled
+    decay_config = None
+    if args.time_decay:
+        try:
+            from tools.memory.time_decay import load_decay_config
+            decay_config = load_decay_config()
+        except (ImportError, Exception):
+            if not args.json:
+                print("(Time-decay module unavailable — using standard ranking)")
+
+    results = hybrid_rank(entries, bm25_scores, semantic_scores, args.bm25_weight, args.semantic_weight,
+                          time_decay_enabled=args.time_decay, decay_config=decay_config)
+
+    # Log access
+    search_type = "hybrid_time_decay" if args.time_decay else "hybrid"
+    conn = sqlite3.connect(str(DB_PATH))
+    c = conn.cursor()
+    c.execute(
+        "INSERT INTO memory_access_log (query, results_count, search_type) VALUES (?, ?, ?)",
+        (args.query, min(args.limit, len(results)), search_type),
+    )
+    conn.commit()
+    conn.close()
+
+    if args.json:
+        output_entries = []
+        for score, id_, content, type_, importance, created_at in results[: args.limit]:
+            if score > 0:
+                output_entries.append({
+                    "id": id_,
+                    "score": round(score, 4),
+                    "content": content,
+                    "type": type_,
+                    "importance": importance,
+                    "created_at": created_at,
+                })
+        print(json.dumps({
+            "classification": "CUI // SP-CTI",
+            "count": len(output_entries),
+            "search_type": search_type,
+            "semantic_available": semantic_scores is not None,
+            "entries": output_entries,
+        }, indent=2))
+    else:
+        for score, id_, content, type_, importance, created_at in results[: args.limit]:
+            if score > 0:
+                print(f"[#{id_}] (score:{score:.3f}, {type_}, importance:{importance}) {content}  — {created_at}")
+
+
+if __name__ == "__main__":
+    main()

tools/memory/maintenance_cron.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Memory maintenance orchestrator — flush, embed, backup, prune (D179-D182).
+
+Runs a 4-step maintenance pipeline:
+  1. Flush auto-capture buffer to memory_entries
+  2. Generate embeddings for unembedded entries (D72 compliant)
+  3. Prune stale low-importance entries
+  4. Backup memory.db
+
+Usage:
+    python tools/memory/maintenance_cron.py --all --json
+    python tools/memory/maintenance_cron.py --flush-buffer --json
+    python tools/memory/maintenance_cron.py --embed-unembedded --json
+    python tools/memory/maintenance_cron.py --prune-stale --days 180 --json
+    python tools/memory/maintenance_cron.py --backup --json
+"""
+
+import argparse
+import json
+import sqlite3
+import struct
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+DB_PATH = BASE_DIR / "data" / "memory.db"
+
+sys.path.insert(0, str(BASE_DIR))
+
+
+def _load_config():
+    """Load maintenance settings from memory_config.yaml."""
+    config_path = BASE_DIR / "args" / "memory_config.yaml"
+    defaults = {
+        "prune_stale_days": 180,
+        "prune_min_importance": 3,
+        "prune_types": ["event", "thinking"],
+        "embed_batch_size": 20,
+        "backup_before_prune": True,
+    }
+    try:
+        import yaml
+        if config_path.exists():
+            with open(config_path, encoding="utf-8") as f:
+                data = yaml.safe_load(f) or {}
+            maint = data.get("maintenance", {})
+            for key in defaults:
+                if key in maint:
+                    defaults[key] = maint[key]
+    except (ImportError, Exception):
+        pass
+    return defaults
+
+
+def flush_buffer(db_path=None):
+    """Flush memory buffer to memory_entries."""
+    try:
+        from tools.memory.auto_capture import flush_buffer as _flush
+        return _flush(db_path=db_path)
+    except (ImportError, Exception) as exc:
+        return {"flushed": 0, "error": str(exc)}
+
+
+def embed_unembedded(db_path=None):
+    """Generate embeddings for entries missing them (D72 compliant)."""
+    path = db_path or DB_PATH
+    conn = sqlite3.connect(str(path))
+    c = conn.cursor()
+    c.execute("SELECT id, content FROM memory_entries WHERE embedding IS NULL")
+    rows = c.fetchall()
+
+    if not rows:
+        conn.close()
+        return {"embedded": 0, "status": "all_embedded"}
+
+    # Use LLM provider abstraction (D72)
+    provider = None
+    provider_name = "none"
+    try:
+        from tools.llm import get_embedding_provider
+        provider = get_embedding_provider()
+        provider_name = "llm_provider"
+    except Exception:
+        pass
+
+    # Fallback to direct OpenAI
+    if provider is None:
+        try:
+            from dotenv import load_dotenv
+            load_dotenv(BASE_DIR / ".env")
+        except ImportError:
+            pass
+
+        import os
+        api_key = os.environ.get("OPENAI_API_KEY")
+        if api_key:
+            try:
+                import openai
+                provider = openai.OpenAI(api_key=api_key)
+                provider_name = "openai_direct"
+            except ImportError:
+                pass
+
+    if provider is None:
+        conn.close()
+        return {"embedded": 0, "status": "no_provider", "total_unembedded": len(rows)}
+
+    cfg = _load_config()
+    batch_size = cfg.get("embed_batch_size", 20)
+    embedded = 0
+    errors = 0
+
+    for i in range(0, len(rows), batch_size):
+        batch = rows[i:i + batch_size]
+        texts = [row[1] for row in batch]
+        ids = [row[0] for row in batch]
+
+        try:
+            if hasattr(provider, "embed"):
+                # LLM provider interface
+                for j, text in enumerate(texts):
+                    emb = provider.embed(text)
+                    blob = struct.pack(f"{len(emb)}f", *emb)
+                    c.execute(
+                        "UPDATE memory_entries SET embedding = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
+                        (blob, ids[j]),
+                    )
+                    embedded += 1
+            else:
+                # Direct OpenAI client
+                response = provider.embeddings.create(
+                    input=texts, model="text-embedding-3-small"
+                )
+                for j, emb_data in enumerate(response.data):
+                    blob = struct.pack(f"{len(emb_data.embedding)}f", *emb_data.embedding)
+                    c.execute(
+                        "UPDATE memory_entries SET embedding = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
+                        (blob, ids[j]),
+                    )
+                    embedded += 1
+            conn.commit()
+        except Exception:
+            errors += 1
+
+    conn.close()
+    return {
+        "embedded": embedded,
+        "errors": errors,
+        "total_unembedded": len(rows),
+        "provider": provider_name,
+    }
+
+
+def prune_stale(days=None, db_path=None):
+    """Remove low-importance entries older than threshold.
+
+    Only prunes entries with importance <= configured threshold and
+    type in configured prune_types list.
+    """
+    cfg = _load_config()
+    prune_days = days or cfg.get("prune_stale_days", 180)
+    min_importance = cfg.get("prune_min_importance", 3)
+    prune_types = cfg.get("prune_types", ["event", "thinking"])
+
+    path = db_path or DB_PATH
+    conn = sqlite3.connect(str(path))
+    c = conn.cursor()
+
+    placeholders = ",".join("?" * len(prune_types))
+    c.execute(
+        f"""DELETE FROM memory_entries
+           WHERE importance <= ?
+             AND type IN ({placeholders})
+             AND created_at < datetime('now', ? || ' days')""",
+        [min_importance] + prune_types + [str(-prune_days)],
+    )
+    pruned = c.rowcount
+    conn.commit()
+    conn.close()
+    return {
+        "pruned": pruned,
+        "threshold_days": prune_days,
+        "min_importance": min_importance,
+        "prune_types": prune_types,
+    }
+
+
+def backup_memory(db_path=None):
+    """Backup memory.db using the backup manager."""
+    try:
+        from tools.db.backup_manager import BackupManager
+        mgr = BackupManager()
+        result = mgr.backup_sqlite(db_path or DB_PATH)
+        return {"status": "ok", "backup_path": str(result.get("backup_path", ""))}
+    except Exception as exc:
+        return {"status": "error", "error": str(exc)}
+
+
+def run_all(db_path=None, prune_days=None):
+    """Run full maintenance pipeline: flush → embed → prune → backup."""
+    cfg = _load_config()
+    results = {
+        "classification": "CUI // SP-CTI",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    }
+
+    start = time.monotonic()
+
+    # Step 1: Flush buffer
+    results["flush"] = flush_buffer(db_path)
+
+    # Step 2: Embed unembedded entries
+    results["embed"] = embed_unembedded(db_path)
+
+    # Step 3: Backup before prune (if configured)
+    if cfg.get("backup_before_prune", True):
+        results["backup"] = backup_memory(db_path)
+
+    # Step 4: Prune stale entries
+    results["prune"] = prune_stale(prune_days, db_path)
+
+    # Step 5: Backup after prune (if not done before)
+    if not cfg.get("backup_before_prune", True):
+        results["backup"] = backup_memory(db_path)
+
+    results["duration_ms"] = int((time.monotonic() - start) * 1000)
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Memory maintenance orchestrator (D179-D182)"
+    )
+    parser.add_argument("--all", action="store_true",
+                        help="Run full maintenance pipeline")
+    parser.add_argument("--flush-buffer", action="store_true",
+                        help="Flush auto-capture buffer only")
+    parser.add_argument("--embed-unembedded", action="store_true",
+                        help="Generate embeddings for unembedded entries")
+    parser.add_argument("--prune-stale", action="store_true",
+                        help="Prune stale low-importance entries")
+    parser.add_argument("--backup", action="store_true",
+                        help="Backup memory.db")
+    parser.add_argument("--days", type=int,
+                        help="Override prune threshold days")
+    parser.add_argument("--json", action="store_true",
+                        help="JSON output")
+    args = parser.parse_args()
+
+    if args.all:
+        result = run_all(prune_days=args.days)
+    elif args.flush_buffer:
+        result = flush_buffer()
+    elif args.embed_unembedded:
+        result = embed_unembedded()
+    elif args.prune_stale:
+        result = prune_stale(days=args.days)
+    elif args.backup:
+        result = backup_memory()
+    else:
+        parser.print_help()
+        return
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        if args.all:
+            fl = result.get("flush", {})
+            em = result.get("embed", {})
+            pr = result.get("prune", {})
+            bk = result.get("backup", {})
+            print(f"Maintenance complete ({result.get('duration_ms', 0)}ms):")
+            print(f"  Flush:  {fl.get('flushed', 0)} entries flushed")
+            print(f"  Embed:  {em.get('embedded', 0)} entries embedded "
+                  f"(provider: {em.get('provider', 'n/a')})")
+            print(f"  Prune:  {pr.get('pruned', 0)} entries pruned "
+                  f"(>{pr.get('threshold_days', '?')} days)")
+            print(f"  Backup: {bk.get('status', 'skipped')}")
+        else:
+            print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()