empathy-framework 5.3.0__py3-none-any.whl → 5.4.0__py3-none-any.whl

empathy_os/memory/summary_index.py

@@ -1,583 +0,0 @@
-"""Conversation Summary Index for Empathy Framework
-
-Redis-backed conversation summary with topic indexing for:
-- Efficient agent context handoff (80% token savings)
-- Cross-session decision tracking
-- Intelligent context filtering by topic
-- 8x faster sub-agent startup
-
-Architecture:
-- HSET for summary hash (topics, decisions, files, working_on)
-- ZADD for timeline sorted set (timestamp-ordered events)
-- SADD for topic index (find sessions by topic)
-
-Copyright 2025 Smart AI Memory, LLC
-Licensed under Fair Source 0.9
-"""
-
-import json
-import time
-from dataclasses import dataclass, field
-from datetime import datetime
-from typing import Any
-
-from .short_term import RedisShortTermMemory
-
-
-@dataclass
-class AgentContext:
-    """Compact context package for sub-agent handoff.
-
-    Token-efficient representation of conversation state:
-    ~2,000 tokens vs 50,000+ for full history = 96% savings
-    """
-
-    working_on: str
-    decisions: list[str] = field(default_factory=list)
-    relevant_files: list[str] = field(default_factory=list)
-    recent_events: list[dict] = field(default_factory=list)
-    open_questions: list[str] = field(default_factory=list)
-    topics: list[str] = field(default_factory=list)
-    session_id: str = ""
-    updated_at: str = ""
-
-    def to_prompt(self) -> str:
-        """Convert to markdown prompt for agent injection.
-
-        Returns:
-            Formatted markdown (~2,000 tokens)
-
-        """
-        lines = ["## Session Context (from memory)", ""]
-
-        if self.working_on:
-            lines.append(f"**Working on:** {self.working_on}")
-            lines.append("")
-
-        if self.decisions:
-            lines.append("**Key decisions made:**")
-            for decision in self.decisions[-5:]:  # Last 5 decisions
-                lines.append(f"- {decision}")
-            lines.append("")
-
-        if self.open_questions:
-            lines.append("**Open questions:**")
-            for question in self.open_questions[-3:]:  # Last 3 questions
-                lines.append(f"- {question}")
-            lines.append("")
-
-        if self.recent_events:
-            lines.append("**Recent timeline:**")
-            for i, event in enumerate(self.recent_events[-5:], 1):  # Last 5 events
-                event_type = event.get("type", "event")
-                content = event.get("content", str(event))
-                lines.append(f"{i}. {event_type.capitalize()}: {content}")
-            lines.append("")
-
-        if self.relevant_files:
-            files_str = ", ".join(self.relevant_files[-10:])  # Last 10 files
-            lines.append(f"**Key files:** {files_str}")
-
-        return "\n".join(lines)
-
-    def to_dict(self) -> dict[str, Any]:
-        """Convert to dictionary for serialization."""
-        return {
-            "working_on": self.working_on,
-            "decisions": self.decisions,
-            "relevant_files": self.relevant_files,
-            "recent_events": self.recent_events,
-            "open_questions": self.open_questions,
-            "topics": self.topics,
-            "session_id": self.session_id,
-            "updated_at": self.updated_at,
-        }
-
-
-class ConversationSummaryIndex:
-    """Redis-backed conversation summary with topic indexing.
-
-    Dramatically reduces token usage for agent handoffs:
-    - Full history: 50,000+ tokens
-    - Summary + timeline: 2,000-5,000 tokens
-    - Savings: ~80%+ per sub-agent
-
-    Example:
-        >>> index = ConversationSummaryIndex(redis_memory)
-        >>>
-        >>> # Update summary incrementally
-        >>> await index.update_summary("session123", {
-        ...     "type": "decision",
-        ...     "content": "Use JWT for authentication"
-        ... })
-        >>>
-        >>> # Get context for sub-agent (filtered by topics)
-        >>> context = await index.get_context_for_agent(
-        ...     "session123",
-        ...     focus_topics=["auth", "security"]
-        ... )
-        >>>
-        >>> # Cross-session decision recall
-        >>> decisions = await index.recall_decisions("authentication", days_back=7)
-
-    """
-
-    PREFIX_SUMMARY = "empathy:summary:"
-    PREFIX_TIMELINE = "empathy:timeline:"
-    PREFIX_TOPIC = "empathy:topic:"
-
-    # TTL: 24 hours (configurable)
-    DEFAULT_TTL = 86400
-
-    def __init__(
-        self,
-        redis_memory: RedisShortTermMemory,
-        ttl: int = DEFAULT_TTL,
-    ):
-        """Initialize summary index.
-
-        Args:
-            redis_memory: Redis connection from short-term memory
-            ttl: Time-to-live in seconds (default 24h)
-
-        """
-        self._memory = redis_memory
-        self._ttl = ttl
-
-    # === Low-level Redis operations ===
-
-    def _hset(self, key: str, mapping: dict[str, str]) -> bool:
-        """Set multiple hash fields."""
-        if self._memory.use_mock:
-            # Mock: store as dict
-            if key not in self._memory._mock_storage:
-                self._memory._mock_storage[key] = ({}, None)
-            existing, expires = self._memory._mock_storage[key]
-            if isinstance(existing, dict):
-                existing.update(mapping)
-            else:
-                existing = dict(mapping)
-            expires = time.time() + self._ttl
-            self._memory._mock_storage[key] = (existing, expires)
-            return True
-        if self._memory._client is None:
-            return False
-        return bool(self._memory._client.hset(key, mapping=mapping))
-
-    def _hget(self, key: str, field: str) -> str | None:
-        """Get single hash field."""
-        if self._memory.use_mock:
-            if key in self._memory._mock_storage:
-                data, expires = self._memory._mock_storage[key]
-                if expires and time.time() > expires:
-                    del self._memory._mock_storage[key]
-                    return None
-                if isinstance(data, dict):
-                    return data.get(field)
-            return None
-        if self._memory._client is None:
-            return None
-        result = self._memory._client.hget(key, field)
-        return str(result) if result else None
-
-    def _hgetall(self, key: str) -> dict[str, str]:
-        """Get all hash fields."""
-        if self._memory.use_mock:
-            if key in self._memory._mock_storage:
-                data, expires = self._memory._mock_storage[key]
-                if expires and time.time() > expires:
-                    del self._memory._mock_storage[key]
-                    return {}
-                if isinstance(data, dict):
-                    return data
-            return {}
-        if self._memory._client is None:
-            return {}
-        result = self._memory._client.hgetall(key)
-        return dict(result) if result else {}
-
-    def _zadd(self, key: str, score: float, member: str) -> bool:
-        """Add to sorted set with score."""
-        if self._memory.use_mock:
-            if key not in self._memory._mock_storage:
-                self._memory._mock_storage[key] = ([], None)
-            data, _ = self._memory._mock_storage[key]
-            if not isinstance(data, list):
-                data = []
-            # Remove existing entry with same member
-            data = [(s, m) for s, m in data if m != member]
-            data.append((score, member))
-            data.sort(key=lambda x: x[0], reverse=True)  # Highest score first
-            expires = time.time() + self._ttl
-            self._memory._mock_storage[key] = (data, expires)
-            return True
-        if self._memory._client is None:
-            return False
-        return bool(self._memory._client.zadd(key, {member: score}))
-
-    def _zrevrange(self, key: str, start: int, stop: int) -> list[str]:
-        """Get range from sorted set (highest scores first)."""
-        if self._memory.use_mock:
-            if key in self._memory._mock_storage:
-                data, expires = self._memory._mock_storage[key]
-                if expires and time.time() > expires:
-                    del self._memory._mock_storage[key]
-                    return []
-                if isinstance(data, list):
-                    # Data is list of (score, member) tuples
-                    members = [m for _, m in data[start : stop + 1]]
-                    return members
-            return []
-        if self._memory._client is None:
-            return []
-        result = self._memory._client.zrevrange(key, start, stop)
-        return list(result) if result else []
-
-    def _sadd(self, key: str, *members: str) -> int:
-        """Add members to set."""
-        if self._memory.use_mock:
-            if key not in self._memory._mock_storage:
-                self._memory._mock_storage[key] = (set(), None)
-            data, _ = self._memory._mock_storage[key]
-            if not isinstance(data, set):
-                data = set()
-            added = 0
-            for member in members:
-                if member not in data:
-                    data.add(member)
-                    added += 1
-            expires = time.time() + self._ttl
-            self._memory._mock_storage[key] = (data, expires)
-            return added
-        if self._memory._client is None:
-            return 0
-        return int(self._memory._client.sadd(key, *members))
-
-    def _smembers(self, key: str) -> set[str]:
-        """Get all members of set."""
-        if self._memory.use_mock:
-            if key in self._memory._mock_storage:
-                data, expires = self._memory._mock_storage[key]
-                if expires and time.time() > expires:
-                    del self._memory._mock_storage[key]
-                    return set()
-                if isinstance(data, set):
-                    return data
-            return set()
-        if self._memory._client is None:
-            return set()
-        result = self._memory._client.smembers(key)
-        return set(result) if result else set()
-
-    def _expire(self, key: str, seconds: int) -> bool:
-        """Set key expiration."""
-        if self._memory.use_mock:
-            if key in self._memory._mock_storage:
-                data, _ = self._memory._mock_storage[key]
-                self._memory._mock_storage[key] = (data, time.time() + seconds)
-                return True
-            return False
-        if self._memory._client is None:
-            return False
-        return bool(self._memory._client.expire(key, seconds))
-
-    # === Public API ===
-
-    def update_summary(
-        self,
-        session_id: str,
-        event: dict[str, Any],
-    ) -> bool:
-        """Incrementally update session summary with new event.
-
-        Args:
-            session_id: Unique session identifier
-            event: Event dict with 'type' and 'content' keys
-                Types: 'decision', 'completed', 'started', 'blocked',
-                       'question', 'file_modified'
-
-        Returns:
-            True if successful
-
-        Example:
-            >>> index.update_summary("session123", {
-            ...     "type": "decision",
-            ...     "content": "Use JWT for authentication"
-            ... })
-
-        """
-        summary_key = f"{self.PREFIX_SUMMARY}{session_id}"
-        timeline_key = f"{self.PREFIX_TIMELINE}{session_id}"
-
-        event_type = event.get("type", "event")
-        content = event.get("content", "")
-        timestamp = time.time()
-
-        # Add to timeline (sorted by timestamp)
-        timeline_entry = json.dumps(
-            {
-                "type": event_type,
-                "content": content,
-                "timestamp": datetime.now().isoformat(),
-            },
-        )
-        self._zadd(timeline_key, timestamp, timeline_entry)
-        self._expire(timeline_key, self._ttl)
-
-        # Update summary based on event type
-        summary = self._hgetall(summary_key)
-
-        # Update 'working_on' for started/in_progress events
-        if event_type in ("started", "in_progress"):
-            summary["working_on"] = content
-
-        # Append to decisions list
-        if event_type == "decision":
-            decisions = json.loads(summary.get("decisions", "[]"))
-            decisions.append(content)
-            summary["decisions"] = json.dumps(decisions[-20:])  # Keep last 20
-
-        # Append to open questions
-        if event_type == "question":
-            questions = json.loads(summary.get("open_questions", "[]"))
-            questions.append(content)
-            summary["open_questions"] = json.dumps(questions[-10:])
-
-        # Clear question if answered
-        if event_type == "answered":
-            questions = json.loads(summary.get("open_questions", "[]"))
-            questions = [q for q in questions if content.lower() not in q.lower()]
-            summary["open_questions"] = json.dumps(questions)
-
-        # Update files list
-        if event_type == "file_modified":
-            files = json.loads(summary.get("key_files", "[]"))
-            if content not in files:
-                files.append(content)
-            summary["key_files"] = json.dumps(files[-20:])
-
-        # Extract and index topics
-        topics = self._extract_topics(content)
-        if topics:
-            existing_topics = json.loads(summary.get("topics", "[]"))
-            for topic in topics:
-                if topic not in existing_topics:
-                    existing_topics.append(topic)
-                # Add session to topic index
-                topic_key = f"{self.PREFIX_TOPIC}{topic}"
-                self._sadd(topic_key, session_id)
-                self._expire(topic_key, self._ttl)
-            summary["topics"] = json.dumps(existing_topics[-30:])
-
-        # Update timestamp
-        summary["updated_at"] = datetime.now().isoformat()
-
-        # Store summary
-        self._hset(summary_key, summary)
-        self._expire(summary_key, self._ttl)
-
-        return True
-
-    def get_context_for_agent(
-        self,
-        session_id: str,
-        focus_topics: list[str] | None = None,
-        max_timeline_entries: int = 10,
-    ) -> AgentContext:
-        """Build compact context for sub-agent handoff.
-
-        Args:
-            session_id: Session to get context from
-            focus_topics: Optional list of topics to filter by
-            max_timeline_entries: Max timeline events to include
-
-        Returns:
-            AgentContext ready for prompt injection
-
-        Example:
-            >>> context = index.get_context_for_agent(
-            ...     "session123",
-            ...     focus_topics=["auth", "security"]
-            ... )
-            >>> prompt = f"Previous context:\\n{context.to_prompt()}\\n\\nTask: ..."
-
-        """
-        summary_key = f"{self.PREFIX_SUMMARY}{session_id}"
-        timeline_key = f"{self.PREFIX_TIMELINE}{session_id}"
-
-        # Get summary
-        summary = self._hgetall(summary_key)
-
-        # Get timeline
-        raw_timeline = self._zrevrange(timeline_key, 0, max_timeline_entries - 1)
-        timeline = []
-        for entry in raw_timeline:
-            try:
-                timeline.append(json.loads(entry))
-            except json.JSONDecodeError:
-                pass
-
-        # Parse summary fields
-        working_on = summary.get("working_on", "")
-        decisions = json.loads(summary.get("decisions", "[]"))
-        open_questions = json.loads(summary.get("open_questions", "[]"))
-        key_files = json.loads(summary.get("key_files", "[]"))
-        topics = json.loads(summary.get("topics", "[]"))
-        updated_at = summary.get("updated_at", "")
-
-        # Filter by focus topics if provided
-        if focus_topics:
-            focus_set = {t.lower() for t in focus_topics}
-
-            # Filter decisions
-            decisions = [d for d in decisions if any(t in d.lower() for t in focus_set)]
-
-            # Filter timeline
-            timeline = [e for e in timeline if any(t in str(e).lower() for t in focus_set)]
-
-            # Filter files (keep all - they're relevant context)
-
-        return AgentContext(
-            working_on=working_on,
-            decisions=decisions,
-            relevant_files=key_files,
-            recent_events=timeline,
-            open_questions=open_questions,
-            topics=topics,
-            session_id=session_id,
-            updated_at=updated_at,
-        )
-
-    def recall_decisions(
-        self,
-        topic: str,
-        days_back: int = 7,
-    ) -> list[dict[str, Any]]:
-        """Search for decisions across sessions by topic.
-
-        Args:
-            topic: Topic to search for
-            days_back: How many days of history to search
-
-        Returns:
-            List of decisions with session context
-
-        Example:
-            >>> decisions = index.recall_decisions("authentication")
-            >>> for d in decisions:
-            ...     print(f"{d['session']}: {d['decision']}")
-
-        """
-        topic_key = f"{self.PREFIX_TOPIC}{topic.lower()}"
-        session_ids = self._smembers(topic_key)
-
-        results = []
-        cutoff = time.time() - (days_back * 86400)
-
-        for session_id in session_ids:
-            summary_key = f"{self.PREFIX_SUMMARY}{session_id}"
-            summary = self._hgetall(summary_key)
-
-            if not summary:
-                continue
-
-            # Check if within time range
-            updated_at = summary.get("updated_at", "")
-            if updated_at:
-                try:
-                    updated_ts = datetime.fromisoformat(updated_at).timestamp()
-                    if updated_ts < cutoff:
-                        continue
-                except (ValueError, TypeError):
-                    pass
-
-            # Get decisions
-            decisions = json.loads(summary.get("decisions", "[]"))
-
-            for decision in decisions:
-                if topic.lower() in decision.lower():
-                    results.append(
-                        {
-                            "session": session_id,
-                            "decision": decision,
-                            "date": updated_at,
-                            "topics": json.loads(summary.get("topics", "[]")),
-                        },
-                    )
-
-        return results
-
-    def get_sessions_by_topic(self, topic: str) -> set[str]:
-        """Find all sessions that discussed a topic.
-
-        Args:
-            topic: Topic to search for
-
-        Returns:
-            Set of session IDs
-
-        """
-        topic_key = f"{self.PREFIX_TOPIC}{topic.lower()}"
-        return self._smembers(topic_key)
-
-    def clear_session(self, session_id: str) -> bool:
-        """Clear all data for a session.
-
-        Args:
-            session_id: Session to clear
-
-        Returns:
-            True if successful
-
-        """
-        summary_key = f"{self.PREFIX_SUMMARY}{session_id}"
-        timeline_key = f"{self.PREFIX_TIMELINE}{session_id}"
-
-        # Get topics to clean up topic indexes
-        summary = self._hgetall(summary_key)
-        topics = json.loads(summary.get("topics", "[]"))
-
-        # Remove from topic indexes
-        for topic in topics:
-            topic_key = f"{self.PREFIX_TOPIC}{topic}"
-            if self._memory.use_mock:
-                if topic_key in self._memory._mock_storage:
-                    data, expires = self._memory._mock_storage[topic_key]
-                    if isinstance(data, set):
-                        data.discard(session_id)
-            elif self._memory._client is not None:
-                self._memory._client.srem(topic_key, session_id)
-
-        # Delete summary and timeline
-        self._memory._delete(summary_key)
-        self._memory._delete(timeline_key)
-
-        return True
-
-    def _extract_topics(self, content: str) -> list[str]:
-        """Extract topics from content.
-
-        Simple keyword extraction for now - can be enhanced with NLP.
-        """
-        # Common development topics
-        topic_keywords = {
-            "auth": ["auth", "authentication", "login", "jwt", "oauth", "session"],
-            "security": ["security", "vulnerable", "xss", "injection", "csrf"],
-            "testing": ["test", "pytest", "unittest", "mock", "coverage"],
-            "database": ["database", "sql", "postgres", "redis", "mongodb"],
-            "api": ["api", "endpoint", "rest", "graphql", "request"],
-            "performance": ["performance", "slow", "optimize", "cache", "latency"],
-            "deploy": ["deploy", "docker", "kubernetes", "ci/cd", "production"],
-            "error": ["error", "exception", "bug", "fix", "debug"],
-            "refactor": ["refactor", "cleanup", "reorganize", "restructure"],
-            "documentation": ["docs", "readme", "docstring", "comment"],
-        }
-
-        content_lower = content.lower()
-        found_topics = []
-
-        for topic, keywords in topic_keywords.items():
-            if any(kw in content_lower for kw in keywords):
-                found_topics.append(topic)
-
-        return found_topics