icdev 1.0.0__py3-none-any.whl

icdev/tools/memory/history_compressor.py

@@ -0,0 +1,334 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""3-Tier history compression for chat contexts (Phase 44 — D271-D274).
+
+Opt-in per context. Budget: Current Topic 50%, Historical Topics 30%, Bulk 20%.
+Topic boundary detection via time gap >30min OR keyword shift >60%.
+Summarization via LLMRouter (falls back to truncation).
+Originals preserved in DB (is_compressed=0).
+
+Usage:
+    from icdev.tools.memory.history_compressor import HistoryCompressor
+
+    compressor = HistoryCompressor()
+    compressed = compressor.compress(messages, budget_tokens=4000)
+"""
+
+import logging
+import re
+import string
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger("icdev.history_compressor")
+
+# Stopwords for keyword extraction (small set, air-gap safe)
+_STOPWORDS = frozenset({
+    "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+    "have", "has", "had", "do", "does", "did", "will", "would", "could",
+    "should", "may", "might", "can", "shall", "must", "need", "this",
+    "that", "these", "those", "it", "its", "i", "you", "he", "she", "we",
+    "they", "me", "him", "her", "us", "them", "my", "your", "his", "our",
+    "their", "what", "which", "who", "whom", "when", "where", "why", "how",
+    "not", "no", "but", "or", "and", "if", "then", "else", "for", "in",
+    "on", "at", "to", "of", "by", "with", "from", "as", "into", "about",
+    "so", "up", "out", "just", "also", "very", "too", "all", "any", "each",
+    "some", "more", "most", "other", "than", "only", "own", "same", "such",
+})
+
+# Default budget allocation
+DEFAULT_BUDGET = {
+    "current_topic": 0.50,
+    "historical_topics": 0.30,
+    "bulk": 0.20,
+}
+
+# Topic boundary thresholds
+DEFAULT_TIME_GAP_MINUTES = 30
+DEFAULT_KEYWORD_SHIFT_THRESHOLD = 0.60
+
+
+@dataclass
+class TopicBoundary:
+    """Represents a detected topic segment in conversation history."""
+
+    start_turn: int
+    end_turn: int
+    keywords: List[str] = field(default_factory=list)
+    time_span_minutes: float = 0.0
+    message_count: int = 0
+    summary: str = ""
+
+
+class HistoryCompressor:
+    """Compress conversation history using 3-tier proportional budget.
+
+    Tiers:
+    1. Current Topic (50%) — recent messages in the active topic
+    2. Historical Topics (30%) — summaries of prior topic segments
+    3. Bulk (20%) — merged mega-summary of oldest topics
+    """
+
+    def __init__(
+        self,
+        budget_allocation: Optional[Dict[str, float]] = None,
+        time_gap_minutes: int = DEFAULT_TIME_GAP_MINUTES,
+        keyword_shift_threshold: float = DEFAULT_KEYWORD_SHIFT_THRESHOLD,
+    ):
+        self._budget = budget_allocation or dict(DEFAULT_BUDGET)
+        self._time_gap_minutes = time_gap_minutes
+        self._keyword_shift_threshold = keyword_shift_threshold
+
+    def compress(
+        self,
+        messages: List[dict],
+        budget_tokens: int = 4000,
+    ) -> List[dict]:
+        """Compress messages to fit within budget_tokens.
+
+        Args:
+            messages: List of message dicts with 'role', 'content', 'turn_number', 'created_at'.
+            budget_tokens: Maximum token budget for compressed output.
+
+        Returns:
+            List of (possibly summarized) message dicts.
+        """
+        if not messages:
+            return []
+
+        total_tokens = sum(self._estimate_tokens(m.get("content", "")) for m in messages)
+        if total_tokens <= budget_tokens:
+            return messages  # Already within budget
+
+        # Detect topic boundaries
+        boundaries = self.detect_topic_boundaries(messages)
+
+        if not boundaries:
+            # No boundaries detected — truncate to budget
+            return self._truncate_to_budget(messages, budget_tokens)
+
+        # Allocate budget across tiers
+        current_budget = int(budget_tokens * self._budget["current_topic"])
+        historical_budget = int(budget_tokens * self._budget["historical_topics"])
+        bulk_budget = int(budget_tokens * self._budget["bulk"])
+
+        result = []
+
+        # Tier 3: Bulk — merge oldest topics into mega-summary
+        if len(boundaries) > 2:
+            bulk_topics = boundaries[:-2]
+            bulk_messages = []
+            for b in bulk_topics:
+                bulk_messages.extend(
+                    m for m in messages
+                    if b.start_turn <= m.get("turn_number", 0) <= b.end_turn
+                )
+            if bulk_messages:
+                bulk_summary = self._merge_summaries(
+                    [self._summarize_topic(
+                        [m for m in messages if b.start_turn <= m.get("turn_number", 0) <= b.end_turn],
+                        max_tokens=bulk_budget // max(len(bulk_topics), 1),
+                    ) for b in bulk_topics],
+                    max_tokens=bulk_budget,
+                )
+                if bulk_summary:
+                    result.append({
+                        "role": "system",
+                        "content": f"[Compressed history — {len(bulk_topics)} topics]\n{bulk_summary}",
+                        "content_type": "summary",
+                        "is_compressed": True,
+                        "compression_tier": "bulk",
+                    })
+
+        # Tier 2: Historical — summaries of middle topics
+        if len(boundaries) > 1:
+            hist_topics = boundaries[-2:-1] if len(boundaries) > 2 else boundaries[:-1]
+            per_topic_budget = historical_budget // max(len(hist_topics), 1)
+            for b in hist_topics:
+                topic_msgs = [
+                    m for m in messages
+                    if b.start_turn <= m.get("turn_number", 0) <= b.end_turn
+                ]
+                summary = self._summarize_topic(topic_msgs, max_tokens=per_topic_budget)
+                if summary:
+                    result.append({
+                        "role": "system",
+                        "content": f"[Topic summary — turns {b.start_turn}-{b.end_turn}]\n{summary}",
+                        "content_type": "summary",
+                        "is_compressed": True,
+                        "compression_tier": "historical",
+                    })
+
+        # Tier 1: Current — keep most recent topic messages within budget
+        current_boundary = boundaries[-1]
+        current_msgs = [
+            m for m in messages
+            if m.get("turn_number", 0) >= current_boundary.start_turn
+        ]
+        result.extend(self._truncate_to_budget(current_msgs, current_budget))
+
+        return result
+
+    def detect_topic_boundaries(
+        self,
+        messages: List[dict],
+    ) -> List[TopicBoundary]:
+        """Detect topic boundaries via time gap and keyword shift heuristics.
+
+        Air-gap safe — no LLM required.
+        """
+        if not messages:
+            return []
+
+        boundaries = []
+        current_start = 0
+
+        for i in range(1, len(messages)):
+            prev = messages[i - 1]
+            curr = messages[i]
+
+            is_boundary = False
+
+            # Check time gap
+            time_gap = self._time_gap_minutes_between(prev, curr)
+            if time_gap >= self._time_gap_minutes:
+                is_boundary = True
+
+            # Check keyword shift
+            if not is_boundary and i >= 3:
+                # Compare keywords of last 3 messages vs current 3 messages
+                prev_window = messages[max(0, i - 3):i]
+                curr_window = messages[i:min(len(messages), i + 3)]
+                prev_kw = self._extract_keywords_from_messages(prev_window)
+                curr_kw = self._extract_keywords_from_messages(curr_window)
+
+                if prev_kw and curr_kw:
+                    overlap = len(prev_kw & curr_kw)
+                    total = len(prev_kw | curr_kw)
+                    similarity = overlap / total if total > 0 else 1.0
+                    if (1.0 - similarity) >= self._keyword_shift_threshold:
+                        is_boundary = True
+
+            if is_boundary:
+                segment_msgs = messages[current_start:i]
+                keywords = list(self._extract_keywords_from_messages(segment_msgs))[:10]
+                boundaries.append(TopicBoundary(
+                    start_turn=messages[current_start].get("turn_number", current_start),
+                    end_turn=messages[i - 1].get("turn_number", i - 1),
+                    keywords=keywords,
+                    message_count=len(segment_msgs),
+                ))
+                current_start = i
+
+        # Final segment
+        final_msgs = messages[current_start:]
+        if final_msgs:
+            keywords = list(self._extract_keywords_from_messages(final_msgs))[:10]
+            boundaries.append(TopicBoundary(
+                start_turn=messages[current_start].get("turn_number", current_start),
+                end_turn=messages[-1].get("turn_number", len(messages) - 1),
+                keywords=keywords,
+                message_count=len(final_msgs),
+            ))
+
+        return boundaries
+
+    def _summarize_topic(
+        self,
+        messages: List[dict],
+        max_tokens: int = 200,
+    ) -> str:
+        """Summarize a topic segment. Uses LLM if available, falls back to truncation."""
+        if not messages:
+            return ""
+
+        combined = "\n".join(
+            f"{m.get('role', 'user')}: {m.get('content', '')}"
+            for m in messages
+        )
+
+        # Try LLM summarization
+        try:
+            from icdev.tools.llm.router import LLMRouter
+            router = LLMRouter()
+            response = router.generate(
+                function_name="history_summarize",
+                messages=[{
+                    "role": "user",
+                    "content": f"Summarize this conversation segment in {max_tokens // 4} words or fewer:\n\n{combined[:3000]}",
+                }],
+            )
+            summary = response.get("content", str(response)) if isinstance(response, dict) else str(response)
+            return summary[:max_tokens * 4]  # Rough token-to-char ratio
+        except (ImportError, Exception) as exc:
+            logger.debug("LLM summarization unavailable: %s — using truncation", exc)
+
+        # Fallback: truncation
+        max_chars = max_tokens * 4
+        if len(combined) <= max_chars:
+            return combined
+        return combined[:max_chars - 3] + "..."
+
+    def _merge_summaries(
+        self,
+        summaries: List[str],
+        max_tokens: int = 500,
+    ) -> str:
+        """Merge multiple topic summaries into a bulk summary."""
+        combined = "\n---\n".join(s for s in summaries if s)
+        max_chars = max_tokens * 4
+        if len(combined) <= max_chars:
+            return combined
+        return combined[:max_chars - 3] + "..."
+
+    def _truncate_to_budget(
+        self,
+        messages: List[dict],
+        budget_tokens: int,
+    ) -> List[dict]:
+        """Keep most recent messages that fit within budget."""
+        result = []
+        used = 0
+        for msg in reversed(messages):
+            tokens = self._estimate_tokens(msg.get("content", ""))
+            if used + tokens > budget_tokens:
+                break
+            result.insert(0, msg)
+            used += tokens
+        return result
+
+    @staticmethod
+    def _estimate_tokens(text: str) -> int:
+        """Estimate token count. Rough: len(text) // 4."""
+        return max(1, len(text) // 4)
+
+    @staticmethod
+    def _extract_keywords(text: str) -> set:
+        """Extract keywords from text using stdlib word extraction + stopword filter."""
+        words = re.findall(r'\b[a-zA-Z]{3,}\b', text.lower())
+        return {w for w in words if w not in _STOPWORDS}
+
+    def _extract_keywords_from_messages(self, messages: List[dict]) -> set:
+        """Extract keywords from a list of messages."""
+        keywords = set()
+        for m in messages:
+            keywords |= self._extract_keywords(m.get("content", ""))
+        return keywords
+
+    @staticmethod
+    def _time_gap_minutes_between(msg1: dict, msg2: dict) -> float:
+        """Calculate time gap in minutes between two messages."""
+        try:
+            t1 = msg1.get("created_at", "")
+            t2 = msg2.get("created_at", "")
+            if not t1 or not t2:
+                return 0.0
+
+            # Parse ISO timestamps
+            dt1 = datetime.fromisoformat(t1.replace("Z", "+00:00"))
+            dt2 = datetime.fromisoformat(t2.replace("Z", "+00:00"))
+            delta = abs((dt2 - dt1).total_seconds())
+            return delta / 60.0
+        except (ValueError, TypeError):
+            return 0.0

icdev/tools/memory/hybrid_search.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Hybrid search: combines BM25 keyword search + semantic vector search.
+
+Supports user-scoped queries (D180), time-decay ranking (D147), and JSON output.
+"""
+
+import argparse
+import json
+import sqlite3
+import struct
+from pathlib import Path
+from icdev._paths import get_project_root
+
+BASE_DIR = get_project_root()
+DB_PATH = BASE_DIR / "data" / "memory.db"
+
+
+def get_all_entries(user_id=None, tenant_id=None):
+    conn = sqlite3.connect(str(DB_PATH))
+    c = conn.cursor()
+
+    sql = "SELECT id, content, type, importance, embedding, created_at FROM memory_entries WHERE 1=1"
+    params = []
+
+    if user_id:
+        sql += " AND (user_id = ? OR user_id IS NULL)"
+        params.append(user_id)
+    if tenant_id:
+        sql += " AND (tenant_id = ? OR tenant_id IS NULL)"
+        params.append(tenant_id)
+
+    c.execute(sql, params)
+    rows = c.fetchall()
+    conn.close()
+    return rows
+
+
+def bm25_search(query, entries):
+    """BM25 keyword ranking with fallback to simple term frequency."""
+    documents = [entry[1] for entry in entries]
+
+    try:
+        from rank_bm25 import BM25Okapi
+        tokenized = [doc.lower().split() for doc in documents]
+        bm25 = BM25Okapi(tokenized)
+        scores = bm25.get_scores(query.lower().split())
+    except ImportError:
+        # Fallback: simple term frequency
+        query_terms = query.lower().split()
+        scores = []
+        for doc in documents:
+            doc_lower = doc.lower()
+            score = sum(doc_lower.count(term) for term in query_terms)
+            scores.append(float(score))
+
+    # Normalize scores to 0-1
+    max_score = max(scores) if scores and max(scores) > 0 else 1.0
+    return [s / max_score for s in scores]
+
+
+def semantic_search(query, entries):
+    """Semantic similarity using embeddings (vendor-agnostic via LLM provider)."""
+    # Try LLM provider system first (supports OpenAI, Ollama, Bedrock Titan)
+    query_emb = None
+    try:
+        from icdev.tools.llm import get_embedding_provider
+        provider = get_embedding_provider()
+        query_emb = provider.embed(query)
+    except Exception:
+        pass
+
+    # Fallback to direct OpenAI
+    if query_emb is None:
+        try:
+            from dotenv import load_dotenv
+            load_dotenv(BASE_DIR / ".env")
+        except ImportError:
+            pass
+
+        import os
+        api_key = os.environ.get("OPENAI_API_KEY")
+        if not api_key:
+            return None
+
+        try:
+            import openai
+            client = openai.OpenAI(api_key=api_key)
+            response = client.embeddings.create(input=query, model="text-embedding-3-small")
+            query_emb = response.data[0].embedding
+        except (ImportError, Exception):
+            return None
+
+    if query_emb is None:
+        return None
+
+    scores = []
+    for entry in entries:
+        emb_blob = entry[4]
+        if emb_blob is None:
+            scores.append(0.0)
+            continue
+        n = len(emb_blob) // 4
+        stored_emb = list(struct.unpack(f"{n}f", emb_blob))
+
+        # Cosine similarity
+        try:
+            import numpy as np
+            a, b = np.array(query_emb), np.array(stored_emb)
+            score = float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))
+        except ImportError:
+            dot = sum(x * y for x, y in zip(query_emb, stored_emb))
+            norm_a = sum(x * x for x in query_emb) ** 0.5
+            norm_b = sum(x * x for x in stored_emb) ** 0.5
+            score = dot / (norm_a * norm_b) if norm_a and norm_b else 0.0
+
+        scores.append(score)
+
+    # Normalize to 0-1
+    max_score = max(scores) if scores and max(scores) > 0 else 1.0
+    return [s / max_score for s in scores]
+
+
+def hybrid_rank(entries, bm25_scores, semantic_scores, bm25_weight, semantic_weight,
+                time_decay_enabled=False, decay_config=None):
+    """Combine BM25 and semantic scores, with optional time-decay (D147)."""
+    results = []
+    for i, entry in enumerate(entries):
+        bm25_s = bm25_scores[i] if bm25_scores else 0.0
+        sem_s = semantic_scores[i] if semantic_scores else 0.0
+
+        if semantic_scores is None:
+            # No semantic available, use BM25 only
+            combined = bm25_s
+        else:
+            combined = (bm25_weight * bm25_s) + (semantic_weight * sem_s)
+
+        id_, content, type_, importance, _, created_at = entry
+
+        # D147: Apply time-decay reranking when enabled
+        if time_decay_enabled:
+            try:
+                from icdev.tools.memory.time_decay import compute_time_aware_score
+                combined = compute_time_aware_score(
+                    base_score=combined,
+                    created_at=created_at or "",
+                    memory_type=type_ or "event",
+                    importance=importance or 5,
+                    config=decay_config,
+                )
+            except (ImportError, Exception):
+                pass  # Fall through to original combined score
+
+        results.append((combined, id_, content, type_, importance, created_at))
+
+    results.sort(reverse=True, key=lambda x: x[0])
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Hybrid search (BM25 + semantic)")
+    parser.add_argument("--query", required=True, help="Search query")
+    parser.add_argument("--limit", type=int, default=10, help="Max results")
+    parser.add_argument("--bm25-weight", type=float, default=0.7, help="BM25 weight (default 0.7)")
+    parser.add_argument("--semantic-weight", type=float, default=0.3, help="Semantic weight (default 0.3)")
+    parser.add_argument("--time-decay", action="store_true", help="Enable time-decay scoring (D147)")
+    parser.add_argument("--user-id", help="Filter by user ID (D180)")
+    parser.add_argument("--tenant-id", help="Filter by tenant ID (D180)")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    args = parser.parse_args()
+
+    entries = get_all_entries(user_id=args.user_id, tenant_id=args.tenant_id)
+    if not entries:
+        if args.json:
+            print(json.dumps({"classification": "CUI // SP-CTI", "count": 0, "entries": []}))
+        else:
+            print("No memory entries found.")
+        return
+
+    bm25_scores = bm25_search(args.query, entries)
+    semantic_scores = semantic_search(args.query, entries)
+
+    if semantic_scores is None and not args.json:
+        print("(Semantic search unavailable — using keyword search only)")
+
+    # D147: Load time-decay config if enabled
+    decay_config = None
+    if args.time_decay:
+        try:
+            from icdev.tools.memory.time_decay import load_decay_config
+            decay_config = load_decay_config()
+        except (ImportError, Exception):
+            if not args.json:
+                print("(Time-decay module unavailable — using standard ranking)")
+
+    results = hybrid_rank(entries, bm25_scores, semantic_scores, args.bm25_weight, args.semantic_weight,
+                          time_decay_enabled=args.time_decay, decay_config=decay_config)
+
+    # Log access
+    search_type = "hybrid_time_decay" if args.time_decay else "hybrid"
+    conn = sqlite3.connect(str(DB_PATH))
+    c = conn.cursor()
+    c.execute(
+        "INSERT INTO memory_access_log (query, results_count, search_type) VALUES (?, ?, ?)",
+        (args.query, min(args.limit, len(results)), search_type),
+    )
+    conn.commit()
+    conn.close()
+
+    if args.json:
+        output_entries = []
+        for score, id_, content, type_, importance, created_at in results[: args.limit]:
+            if score > 0:
+                output_entries.append({
+                    "id": id_,
+                    "score": round(score, 4),
+                    "content": content,
+                    "type": type_,
+                    "importance": importance,
+                    "created_at": created_at,
+                })
+        print(json.dumps({
+            "classification": "CUI // SP-CTI",
+            "count": len(output_entries),
+            "search_type": search_type,
+            "semantic_available": semantic_scores is not None,
+            "entries": output_entries,
+        }, indent=2))
+    else:
+        for score, id_, content, type_, importance, created_at in results[: args.limit]:
+            if score > 0:
+                print(f"[#{id_}] (score:{score:.3f}, {type_}, importance:{importance}) {content}  — {created_at}")
+
+
+if __name__ == "__main__":
+    main()