opencode-semantic-memory 0.1.0__py3-none-any.whl

opencode_memory/ingestion/opencode_db.py

@@ -0,0 +1,448 @@
+"""OpenCode database observer for automatic conversation tracking."""
+
+import asyncio
+import json
+import logging
+import re
+import sqlite3
+from collections import Counter
+from collections.abc import Callable
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from opencode_memory.ingestion.extractors import extract_all_insights
+from opencode_memory.models import Memory, MemoryCategory
+from opencode_memory.project import detect_project_from_path, detect_project_from_git
+
+logger = logging.getLogger(__name__)
+
+CONVERSATION_ENTITY_PATTERNS = [
+    (r"!(\d+)", "mr"),
+    (r"#(\d+)", "issue"),
+    (r"&(\d+)", "epic"),
+    (r"@([\w\.-]+)", "person"),
+]
+
+
+class OpenCodeDBObserver:
+    """Observe the OpenCode SQLite database for new sessions and messages."""
+
+    def __init__(self, db_path: Path):
+        self.db_path = db_path
+        self.last_processed_time: datetime | None = None
+        self.last_message_id: str | None = None
+        self.tool_usage_counts: Counter[str] = Counter()
+        self._polling_task: asyncio.Task | None = None
+        self._stop_polling = False
+        self._on_memories_callback: Callable[[list[Memory]], None] | None = None
+
+    def db_exists(self) -> bool:
+        """Check if the OpenCode database file exists."""
+        return self.db_path.exists()
+
+    def _get_conn(self) -> sqlite3.Connection:
+        """Get a read-only connection to the OpenCode database."""
+        if not self.db_exists():
+            raise FileNotFoundError(f"OpenCode database not found: {self.db_path}")
+        conn = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def get_new_sessions(self, since: datetime | None = None) -> list[dict[str, Any]]:
+        """Get sessions created since the given time."""
+        with self._get_conn() as conn:
+            if since:
+                since_ts = int(since.timestamp() * 1000)
+                cursor = conn.execute(
+                    """
+                    SELECT * FROM session 
+                    WHERE time_created > ?
+                    ORDER BY time_created ASC
+                    """,
+                    (since_ts,),
+                )
+            else:
+                cursor = conn.execute("SELECT * FROM session ORDER BY time_created ASC")
+
+            sessions = []
+            for row in cursor.fetchall():
+                sessions.append(dict(row))
+            return sessions
+
+    def get_session_messages(
+        self, session_id: str, since_message_id: str | None = None
+    ) -> list[dict[str, Any]]:
+        """Get messages for a session."""
+        with self._get_conn() as conn:
+            if since_message_id:
+                cursor = conn.execute(
+                    """
+                    SELECT * FROM message 
+                    WHERE session_id = ? AND id > ?
+                    ORDER BY time_created ASC
+                    """,
+                    (session_id, since_message_id),
+                )
+            else:
+                cursor = conn.execute(
+                    """
+                    SELECT * FROM message 
+                    WHERE session_id = ?
+                    ORDER BY time_created ASC
+                    """,
+                    (session_id,),
+                )
+
+            messages = []
+            for row in cursor.fetchall():
+                msg = dict(row)
+                if msg.get("data"):
+                    try:
+                        msg["data"] = json.loads(msg["data"])
+                    except json.JSONDecodeError:
+                        pass
+                messages.append(msg)
+            return messages
+
+    def get_message_parts(self, message_id: str) -> list[dict[str, Any]]:
+        """Get parts (content) for a message."""
+        with self._get_conn() as conn:
+            cursor = conn.execute(
+                "SELECT * FROM part WHERE message_id = ? ORDER BY time_created ASC",
+                (message_id,),
+            )
+
+            parts = []
+            for row in cursor.fetchall():
+                part = dict(row)
+                if part.get("data"):
+                    try:
+                        part["data"] = json.loads(part["data"])
+                    except json.JSONDecodeError:
+                        pass
+                parts.append(part)
+            return parts
+
+    def extract_session_memories(
+        self, session: dict[str, Any]
+    ) -> tuple[Memory | None, Memory | None]:
+        """Extract full conversation and summary memories from a session.
+
+        Returns:
+            Tuple of (full_conversation, summary) memories. Either may be None.
+        """
+        title = session.get("title", "Untitled session")
+        session_id = session["id"]
+
+        messages = self.get_session_messages(session_id)
+        if not messages:
+            return None, None
+
+        user_messages = []
+        assistant_messages = []
+        tool_calls = []
+        all_text = []
+        conversation_parts = []  # Full conversation with roles
+
+        for msg in messages:
+            data = msg.get("data", {})
+            role = data.get("role")
+
+            parts = self.get_message_parts(msg["id"])
+            for part in parts:
+                part_data = part.get("data", {})
+                if part_data.get("type") == "text":
+                    text = part_data.get("text", "")
+                    all_text.append(text)
+                    if role == "user":
+                        user_messages.append(text)
+                        conversation_parts.append(f"USER: {text}")
+                    elif role == "assistant":
+                        assistant_messages.append(text)
+                        conversation_parts.append(f"ASSISTANT: {text}")
+                elif part_data.get("type") == "tool":
+                    tool_name = part_data.get("tool", "unknown")
+                    tool_calls.append(tool_name)
+                    self.tool_usage_counts[tool_name] += 1
+
+        if not user_messages:
+            return None, None
+
+        combined_text = "\n".join(all_text)
+        entities = self.extract_entities_from_text(combined_text)
+
+        entity_summary = self._summarize_entities(entities)
+        topics = self._extract_topics(user_messages)
+        outcome = self._detect_outcome(assistant_messages, tool_calls)
+
+        # Build summary
+        summary_parts = [f"Session: {title}"]
+        summary_parts.append(
+            f"Messages: {len(user_messages)} user, {len(assistant_messages)} assistant"
+        )
+
+        if tool_calls:
+            tool_counts: dict[str, int] = {}
+            for tool in tool_calls:
+                tool_counts[tool] = tool_counts.get(tool, 0) + 1
+            top_tools = sorted(tool_counts.items(), key=lambda x: -x[1])[:5]
+            summary_parts.append(f"Tools: {', '.join(f'{t[0]}({t[1]})' for t in top_tools)}")
+
+        if entity_summary:
+            summary_parts.append(f"Entities: {entity_summary}")
+
+        if topics:
+            summary_parts.append(f"Topics: {', '.join(topics[:5])}")
+
+        if outcome:
+            summary_parts.append(f"Outcome: {outcome}")
+
+        first_query = user_messages[0][:150] if user_messages else ""
+        summary_parts.append(f"Started with: {first_query}")
+
+        directory = session.get("directory")
+        project = None
+        if directory:
+            project = detect_project_from_path(directory) or detect_project_from_git(directory)
+
+        created_at = datetime.fromtimestamp(session["time_created"] / 1000)
+        entity_refs = [ref for _, ref in entities[:10]]
+
+        # Full conversation memory
+        full_content = f"Session: {title}\n\n" + "\n\n".join(conversation_parts)
+        full_memory = Memory(
+            source_file=f"opencode:session:{session_id}",
+            project=project,
+            category=MemoryCategory.CONVERSATION,
+            content=full_content,
+            what=title,
+            entities=entity_refs,
+            created_at=created_at,
+        )
+
+        # Summary memory
+        summary_memory = Memory(
+            source_file=f"opencode:session:{session_id}:summary",
+            project=project,
+            category=MemoryCategory.CONVERSATION_SUMMARY,
+            content="\n".join(summary_parts),
+            what=title,
+            entities=entity_refs,
+            created_at=created_at,
+        )
+
+        return full_memory, summary_memory
+
+    def extract_session_summary(self, session: dict[str, Any]) -> Memory | None:
+        """Extract a summary memory from a session (legacy compatibility)."""
+        _, summary = self.extract_session_memories(session)
+        return summary
+
+    def _summarize_entities(self, entities: list[tuple[str, str]]) -> str:
+        """Summarize entities by type."""
+        by_type: dict[str, list[str]] = {}
+        for entity_type, ref in entities:
+            by_type.setdefault(entity_type, []).append(ref)
+
+        parts = []
+        for etype in ["mr", "issue", "epic", "person"]:
+            if etype in by_type:
+                refs = by_type[etype][:3]
+                if len(by_type[etype]) > 3:
+                    parts.append(f"{', '.join(refs)} +{len(by_type[etype]) - 3} more {etype}s")
+                else:
+                    parts.append(", ".join(refs))
+
+        return "; ".join(parts)
+
+    def _extract_topics(self, user_messages: list[str]) -> list[str]:
+        """Extract key topics from user messages."""
+        topic_keywords = [
+            "review",
+            "fix",
+            "bug",
+            "feature",
+            "test",
+            "deploy",
+            "merge",
+            "pipeline",
+            "migration",
+            "database",
+            "api",
+            "security",
+            "performance",
+            "refactor",
+            "documentation",
+            "config",
+            "error",
+            "issue",
+            "MR",
+        ]
+
+        combined = " ".join(user_messages).lower()
+        found_topics = []
+        for keyword in topic_keywords:
+            if keyword.lower() in combined:
+                found_topics.append(keyword)
+
+        return found_topics
+
+    def _detect_outcome(self, assistant_messages: list[str], tool_calls: list[str]) -> str:
+        """Detect likely session outcome from assistant responses and tools used."""
+        if not assistant_messages:
+            return ""
+
+        last_messages = " ".join(assistant_messages[-3:]).lower()
+
+        if any(word in last_messages for word in ["committed", "pushed", "merged"]):
+            return "Code changes committed"
+        if any(
+            word in last_messages for word in ["created mr", "opened mr", "merge request created"]
+        ):
+            return "MR created"
+        if any(word in last_messages for word in ["test", "passed", "all tests"]):
+            return "Tests run"
+        if any(word in last_messages for word in ["fixed", "resolved", "solved"]):
+            return "Issue resolved"
+        if any(word in last_messages for word in ["reviewed", "lgtm", "approved"]):
+            return "Review completed"
+
+        if "gitlab_create_merge_request" in tool_calls:
+            return "MR created"
+        if "gitlab_approve_merge_request" in tool_calls:
+            return "MR approved"
+        if any("commit" in t for t in tool_calls):
+            return "Code committed"
+
+        return ""
+
+    def extract_session_insights(self, session: dict[str, Any]) -> list[Memory]:
+        """Extract decisions, blockers, and learnings from session messages."""
+        session_id = session["id"]
+        created_at = datetime.fromtimestamp(session["time_created"] / 1000)
+
+        directory = session.get("directory")
+        project = None
+        if directory:
+            project = detect_project_from_path(directory) or detect_project_from_git(directory)
+
+        messages = self.get_session_messages(session_id)
+        if not messages:
+            return []
+
+        all_text = []
+        for msg in messages:
+            parts = self.get_message_parts(msg["id"])
+            for part in parts:
+                part_data = part.get("data", {})
+                if part_data.get("type") == "text":
+                    all_text.append(part_data.get("text", ""))
+
+        combined_text = "\n".join(all_text)
+        insights = extract_all_insights(combined_text)
+
+        memories = []
+        seen_contents: set[str] = set()
+
+        for insight in insights:
+            content_key = insight.content[:100].lower()
+            if content_key in seen_contents:
+                continue
+            seen_contents.add(content_key)
+
+            category = {
+                "decision": MemoryCategory.DECISION,
+                "blocker": MemoryCategory.BLOCKER,
+                "fact": MemoryCategory.FACT,
+            }.get(insight.category, MemoryCategory.FACT)
+
+            memories.append(
+                Memory(
+                    source_file=f"opencode:session:{session_id}",
+                    project=project,
+                    category=category,
+                    content=insight.content,
+                    what=f"Extracted from session: {insight.category}",
+                    created_at=created_at,
+                )
+            )
+
+        return memories
+
+    def poll_new_content(self, since: datetime | None = None) -> list[Memory]:
+        """Poll for new content and return memories."""
+        if not self.db_exists():
+            logger.debug(f"OpenCode database not found at {self.db_path}, skipping poll")
+            return []
+
+        memories = []
+
+        sessions = self.get_new_sessions(since)
+        for session in sessions:
+            memory = self.extract_session_summary(session)
+            if memory:
+                memories.append(memory)
+
+        return memories
+
+    def extract_entities_from_text(self, text: str) -> list[tuple[str, str]]:
+        """Extract entity references (!123, #456, @user) from text."""
+        entities: list[tuple[str, str]] = []
+        seen: set[str] = set()
+
+        for pattern, entity_type in CONVERSATION_ENTITY_PATTERNS:
+            for match in re.finditer(pattern, text):
+                ref = match.group(0)
+                if ref not in seen:
+                    seen.add(ref)
+                    entities.append((entity_type, ref))
+
+        return entities
+
+    def get_tool_usage_stats(self) -> dict[str, int]:
+        """Return tool usage counts from tracked sessions."""
+        return dict(self.tool_usage_counts.most_common())
+
+    async def start_polling(
+        self,
+        interval_seconds: float = 30.0,
+        on_memories: Callable[[list[Memory]], None] | None = None,
+    ) -> None:
+        """Start background polling for new content.
+
+        Args:
+            interval_seconds: How often to poll (default 30s)
+            on_memories: Optional callback invoked with new memories
+        """
+        self._on_memories_callback = on_memories
+        self._stop_polling = False
+
+        logger.info(f"Starting OpenCode DB polling every {interval_seconds}s")
+
+        while not self._stop_polling:
+            try:
+                memories = self.poll_new_content(self.last_processed_time)
+
+                for memory in memories:
+                    entities = self.extract_entities_from_text(memory.content)
+                    for _, ref in entities:
+                        if ref not in memory.entities:
+                            memory.entities.append(ref)
+
+                if memories:
+                    self.last_processed_time = datetime.now(UTC)
+                    logger.debug(f"Polled {len(memories)} new memories")
+
+                    if self._on_memories_callback:
+                        self._on_memories_callback(memories)
+
+            except Exception as e:
+                logger.error(f"Error polling OpenCode DB: {e}")
+
+            await asyncio.sleep(interval_seconds)
+
+    def stop_polling(self) -> None:
+        """Stop the background polling."""
+        self._stop_polling = True
+        logger.info("Stopping OpenCode DB polling")