opencode-semantic-memory 0.1.0__py3-none-any.whl

opencode_memory/enrichment/gitlab.py

@@ -0,0 +1,237 @@
+"""GitLab API client for entity enrichment."""
+
+import asyncio
+import logging
+import os
+from datetime import UTC, datetime
+
+import httpx
+
+from opencode_memory.models import Entity, EntityType
+
+logger = logging.getLogger(__name__)
+
+GITLAB_API_URL = "https://gitlab.com/api/v4"
+DEFAULT_PROJECT = "gitlab-org/gitlab"
+
+# Rate limiting: GitLab allows 2000 requests/minute for authenticated users
+# We use a conservative limit to leave headroom for other tools
+RATE_LIMIT_REQUESTS = 30  # requests per window
+RATE_LIMIT_WINDOW = 60.0  # seconds
+
+
+class GitLabEnricher:
+    """Fetch entity metadata from GitLab API."""
+
+    def __init__(
+        self,
+        token: str | None = None,
+        base_url: str = GITLAB_API_URL,
+        default_project: str = DEFAULT_PROJECT,
+        rate_limit: int = RATE_LIMIT_REQUESTS,
+        rate_window: float = RATE_LIMIT_WINDOW,
+    ):
+        self.token = token or os.environ.get("GITLAB_TOKEN")
+        self.base_url = base_url
+        self.default_project = default_project
+        self._client: httpx.AsyncClient | None = None
+
+        # Rate limiting state
+        self._rate_limit = rate_limit
+        self._rate_window = rate_window
+        self._request_times: list[float] = []
+        self._rate_lock = asyncio.Lock()
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            headers = {}
+            if self.token:
+                headers["PRIVATE-TOKEN"] = self.token
+            self._client = httpx.AsyncClient(
+                base_url=self.base_url,
+                headers=headers,
+                timeout=30.0,
+            )
+        return self._client
+
+    async def close(self) -> None:
+        if self._client:
+            await self._client.aclose()
+            self._client = None
+
+    async def _wait_for_rate_limit(self) -> None:
+        """Wait if necessary to stay within rate limits."""
+        async with self._rate_lock:
+            now = asyncio.get_event_loop().time()
+
+            # Remove requests outside the window
+            cutoff = now - self._rate_window
+            self._request_times = [t for t in self._request_times if t > cutoff]
+
+            # If at limit, wait for oldest request to expire
+            if len(self._request_times) >= self._rate_limit:
+                oldest = self._request_times[0]
+                wait_time = oldest + self._rate_window - now
+                if wait_time > 0:
+                    logger.debug(f"Rate limit reached, waiting {wait_time:.1f}s")
+                    await asyncio.sleep(wait_time)
+                    # Re-filter after sleeping
+                    now = asyncio.get_event_loop().time()
+                    cutoff = now - self._rate_window
+                    self._request_times = [t for t in self._request_times if t > cutoff]
+
+            # Record this request
+            self._request_times.append(now)
+
+    async def enrich_entity(self, entity: Entity) -> Entity:
+        """Fetch metadata for an entity and update it."""
+        if not self.token:
+            logger.debug("No GitLab token available, skipping enrichment")
+            return entity
+
+        try:
+            if entity.type == EntityType.MR:
+                return await self._enrich_mr(entity)
+            elif entity.type == EntityType.ISSUE:
+                return await self._enrich_issue(entity)
+            elif entity.type == EntityType.EPIC:
+                return await self._enrich_epic(entity)
+            elif entity.type == EntityType.PERSON:
+                return await self._enrich_user(entity)
+        except Exception as e:
+            logger.warning(f"Failed to enrich {entity.type.value} {entity.ref}: {e}")
+
+        return entity
+
+    def _extract_iid(self, ref: str) -> int | None:
+        """Extract IID from ref like !123 or #456."""
+        if ref and len(ref) > 1:
+            try:
+                return int(ref[1:])
+            except ValueError:
+                return None
+        return None
+
+    def _get_project_path(self, entity: Entity) -> str:
+        """Get project path, URL-encoded."""
+        project = entity.project or self.default_project
+        return project.replace("/", "%2F")
+
+    async def _enrich_mr(self, entity: Entity) -> Entity:
+        """Fetch MR metadata."""
+        iid = self._extract_iid(entity.ref)
+        if not iid:
+            return entity
+
+        await self._wait_for_rate_limit()
+        client = await self._get_client()
+        project = self._get_project_path(entity)
+        response = await client.get(f"/projects/{project}/merge_requests/{iid}")
+
+        if response.status_code == 200:
+            data = response.json()
+            entity.title = data.get("title")
+            entity.metadata = {
+                "state": data.get("state"),
+                "author": data.get("author", {}).get("username"),
+                "web_url": data.get("web_url"),
+                "labels": data.get("labels", []),
+                "draft": data.get("draft", False),
+                "merged_at": data.get("merged_at"),
+            }
+            entity.updated_at = datetime.now(UTC)
+            logger.debug(f"Enriched MR {entity.ref}: {entity.title}")
+
+        return entity
+
+    async def _enrich_issue(self, entity: Entity) -> Entity:
+        """Fetch issue metadata."""
+        iid = self._extract_iid(entity.ref)
+        if not iid:
+            return entity
+
+        await self._wait_for_rate_limit()
+        client = await self._get_client()
+        project = self._get_project_path(entity)
+        response = await client.get(f"/projects/{project}/issues/{iid}")
+
+        if response.status_code == 200:
+            data = response.json()
+            entity.title = data.get("title")
+            entity.metadata = {
+                "state": data.get("state"),
+                "author": data.get("author", {}).get("username"),
+                "web_url": data.get("web_url"),
+                "labels": data.get("labels", []),
+                "assignees": [a.get("username") for a in data.get("assignees", [])],
+                "milestone": data.get("milestone", {}).get("title")
+                if data.get("milestone")
+                else None,
+            }
+            entity.updated_at = datetime.now(UTC)
+            logger.debug(f"Enriched issue {entity.ref}: {entity.title}")
+
+        return entity
+
+    async def _enrich_epic(self, entity: Entity) -> Entity:
+        """Fetch epic metadata."""
+        iid = self._extract_iid(entity.ref)
+        if not iid:
+            return entity
+
+        await self._wait_for_rate_limit()
+        client = await self._get_client()
+        # Extract group from project path if available, fallback to gitlab-org
+        group = (
+            entity.project.split("/")[0]
+            if entity.project and "/" in entity.project
+            else "gitlab-org"
+        )
+        response = await client.get(f"/groups/{group}/epics/{iid}")
+
+        if response.status_code == 200:
+            data = response.json()
+            entity.title = data.get("title")
+            entity.metadata = {
+                "state": data.get("state"),
+                "author": data.get("author", {}).get("username"),
+                "web_url": data.get("web_url"),
+                "labels": data.get("labels", []),
+            }
+            entity.updated_at = datetime.now(UTC)
+            logger.debug(f"Enriched epic {entity.ref}: {entity.title}")
+
+        return entity
+
+    async def _enrich_user(self, entity: Entity) -> Entity:
+        """Fetch user metadata."""
+        username = entity.ref.lstrip("@") if entity.ref else None
+        if not username:
+            return entity
+
+        await self._wait_for_rate_limit()
+        client = await self._get_client()
+        response = await client.get("/users", params={"username": username})
+
+        if response.status_code == 200:
+            users = response.json()
+            if users:
+                data = users[0]
+                entity.title = data.get("name")
+                entity.metadata = {
+                    "username": data.get("username"),
+                    "web_url": data.get("web_url"),
+                    "state": data.get("state"),
+                }
+                entity.updated_at = datetime.now(UTC)
+                logger.debug(f"Enriched user {entity.ref}: {entity.title}")
+
+        return entity
+
+    async def enrich_entities(self, entities: list[Entity]) -> list[Entity]:
+        """Enrich multiple entities, with rate limiting."""
+        results = []
+        for entity in entities:
+            enriched = await self.enrich_entity(entity)
+            results.append(enriched)
+        return results

opencode_memory/extraction.py

@@ -0,0 +1,225 @@
+"""LLM-based knowledge extraction from conversations."""
+
+import asyncio
+import json
+import logging
+import shutil
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+from opencode_memory.ingestion.embeddings import EmbeddingEngine
+from opencode_memory.models import LinkType, Memory, MemoryCategory, MemoryLink
+from opencode_memory.storage.sqlite import SQLiteStorage
+from opencode_memory.storage.vectors import VectorStorage
+
+logger = logging.getLogger(__name__)
+
+EXTRACTION_PROMPT = """Analyze this conversation and extract valuable knowledge that should be remembered for future sessions.
+
+Extract ONLY high-value, reusable knowledge. Be selective - not every conversation has extractable knowledge.
+
+For each piece of knowledge, output a JSON object on its own line with these fields:
+- category: one of "procedure", "directive", "decision", "fact"
+- content: the knowledge itself (clear, actionable, standalone)
+- what: brief summary (5-10 words)
+- why: why this matters or context
+- learned: key takeaway for future
+
+Categories:
+- procedure: How to do something (steps, commands, workflows)
+- directive: Always/never rules, standing instructions  
+- decision: Architectural or design choices with reasoning
+- fact: Project-specific information worth remembering
+
+Output ONLY valid JSON lines, one per extracted item. If nothing worth extracting, output nothing.
+
+CONVERSATION:
+"""
+
+
+def _find_opencode() -> str | None:
+    """Find opencode binary."""
+    opencode_path = shutil.which("opencode")
+    if opencode_path:
+        return opencode_path
+
+    for path in [
+        Path.home() / ".opencode/bin/opencode",
+        Path("/usr/local/bin/opencode"),
+    ]:
+        if path.exists():
+            return str(path)
+
+    return None
+
+
+async def call_opencode(prompt: str, working_directory: str | None = None) -> str:
+    """Call opencode CLI to process prompt."""
+    opencode_path = _find_opencode()
+    if not opencode_path:
+        raise FileNotFoundError("opencode not found in PATH or ~/.opencode/bin/")
+
+    cwd = working_directory or str(Path.home() / "gitlab_projects")
+
+    proc = await asyncio.create_subprocess_exec(
+        opencode_path,
+        "run",
+        "--dangerously-skip-permissions",
+        prompt,
+        cwd=cwd,
+        stdin=asyncio.subprocess.DEVNULL,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await asyncio.wait_for(
+        proc.communicate(),
+        timeout=120.0,
+    )
+    return stdout.decode() if stdout else ""
+
+
+async def extract_knowledge_from_conversation(
+    conv_id: int,
+    content: str,
+    project: str | None,
+    source_file: str | None,
+    sqlite: SQLiteStorage,
+    embeddings: EmbeddingEngine,
+    vectors: VectorStorage,
+    working_directory: str | None = None,
+) -> int:
+    """Extract knowledge from a single conversation and store it.
+
+    Returns number of items extracted and stored.
+    """
+    # Truncate very long conversations
+    if len(content) > 15000:
+        content = content[:15000] + "\n\n[... truncated ...]"
+
+    full_prompt = EXTRACTION_PROMPT + content
+
+    try:
+        response = await call_opencode(full_prompt, working_directory=working_directory)
+    except asyncio.TimeoutError:
+        logger.warning(f"Timeout extracting from conversation {conv_id}")
+        return 0
+    except FileNotFoundError as e:
+        logger.error(f"opencode not found: {e}")
+        return 0
+    except Exception as e:
+        logger.error(f"Error calling opencode for conversation {conv_id}: {e}")
+        return 0
+
+    # Parse JSON lines from response
+    extracted = []
+    for line in response.split("\n"):
+        line = line.strip()
+        if not line or not line.startswith("{"):
+            continue
+        try:
+            item = json.loads(line)
+            if "category" in item and "content" in item:
+                extracted.append(item)
+        except json.JSONDecodeError:
+            continue
+
+    if not extracted:
+        return 0
+
+    count = 0
+    category_map = {
+        "procedure": MemoryCategory.PROCEDURE,
+        "directive": MemoryCategory.DIRECTIVE,
+        "decision": MemoryCategory.DECISION,
+        "fact": MemoryCategory.FACT,
+    }
+
+    for item in extracted:
+        category_str = item.get("category", "fact")
+        category = category_map.get(category_str, MemoryCategory.FACT)
+
+        memory = Memory(
+            category=category,
+            content=item.get("content", ""),
+            what=item.get("what"),
+            why=item.get("why"),
+            learned=item.get("learned"),
+            project=project,
+            source_file=source_file,
+        )
+
+        memory_id = sqlite.insert_memory(memory)
+
+        # Embed and store vector
+        embedding = embeddings.embed(memory.embedding_content())
+        vectors.add(f"mem_{memory_id}", memory_id, memory.embedding_content(), embedding)
+
+        # Link back to source conversation
+        link = MemoryLink(
+            source_memory_id=conv_id,
+            target_memory_id=memory_id,
+            link_type=LinkType.EXTENDS,
+            strength=0.9,
+            reason="Knowledge extracted from conversation via LLM",
+        )
+        sqlite.insert_link(link)
+
+        count += 1
+        logger.info(f"Extracted [{category_str}]: {item.get('what', 'No summary')[:50]}")
+
+    return count
+
+
+def get_unprocessed_conversations(
+    sqlite: SQLiteStorage,
+    since_days: int | None = None,
+    limit: int = 100,
+) -> list[dict]:
+    """Get conversations that haven't been processed for knowledge extraction.
+
+    Args:
+        since_days: Only look at conversations from last N days. None = all time.
+        limit: Maximum conversations to return.
+    """
+    with sqlite._get_conn() as conn:
+        if since_days is not None:
+            cutoff = datetime.now(UTC) - timedelta(days=since_days)
+            cursor = conn.execute(
+                """
+                SELECT m.id, m.content, m.what, m.project, m.source_file, m.created_at
+                FROM memories m
+                WHERE m.category = 'conversation'
+                  AND m.resolved_at IS NULL
+                  AND m.created_at > ?
+                  AND NOT EXISTS (
+                      SELECT 1 FROM memory_links ml
+                      JOIN memories m2 ON ml.target_memory_id = m2.id
+                      WHERE ml.source_memory_id = m.id
+                        AND m2.category IN ('procedure', 'directive', 'decision')
+                        AND ml.reason LIKE '%extracted from conversation%'
+                  )
+                ORDER BY m.created_at DESC
+                LIMIT ?
+                """,
+                (cutoff.isoformat(), limit),
+            )
+        else:
+            cursor = conn.execute(
+                """
+                SELECT m.id, m.content, m.what, m.project, m.source_file, m.created_at
+                FROM memories m
+                WHERE m.category = 'conversation'
+                  AND m.resolved_at IS NULL
+                  AND NOT EXISTS (
+                      SELECT 1 FROM memory_links ml
+                      JOIN memories m2 ON ml.target_memory_id = m2.id
+                      WHERE ml.source_memory_id = m.id
+                        AND m2.category IN ('procedure', 'directive', 'decision')
+                        AND ml.reason LIKE '%extracted from conversation%'
+                  )
+                ORDER BY m.created_at DESC
+                LIMIT ?
+                """,
+                (limit,),
+            )
+        return [dict(row) for row in cursor.fetchall()]

opencode_memory/historical_ingest.py

@@ -0,0 +1,142 @@
+"""Historical ingest of OpenCode database sessions."""
+
+import logging
+import sqlite3
+from datetime import UTC, datetime
+from pathlib import Path
+
+from opencode_memory.config import Config
+from opencode_memory.ingestion.embeddings import EmbeddingEngine
+from opencode_memory.ingestion.opencode_db import OpenCodeDBObserver
+from opencode_memory.storage.sqlite import SQLiteStorage
+from opencode_memory.storage.vectors import VectorStorage
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def run_historical_ingest(
+    opencode_db_path: Path | None = None,
+    batch_size: int = 100,
+    max_sessions: int | None = None,
+) -> None:
+    """Run historical ingest of OpenCode sessions.
+
+    Args:
+        opencode_db_path: Path to OpenCode database. Defaults to ~/.local/share/opencode/opencode.db
+        batch_size: Number of sessions to process before reporting progress
+        max_sessions: Maximum sessions to process (None for all)
+    """
+    if opencode_db_path is None:
+        opencode_db_path = Path.home() / ".local/share/opencode/opencode.db"
+
+    if not opencode_db_path.exists():
+        logger.error(f"OpenCode database not found: {opencode_db_path}")
+        return
+
+    config = Config.load()
+    sqlite = SQLiteStorage(config.db_path)
+    embeddings = EmbeddingEngine()
+    vectors = VectorStorage(config.vectors_path, embeddings.dimension)
+    observer = OpenCodeDBObserver(opencode_db_path)
+
+    ingest_state = sqlite.get_ingest_state("opencode_db")
+    last_session_id = ingest_state.get("last_id") if ingest_state else None
+
+    logger.info("Starting historical ingest of OpenCode database")
+    logger.info(f"Source: {opencode_db_path}")
+    if last_session_id:
+        logger.info(f"Resuming from session: {last_session_id}")
+
+    with sqlite3.connect(f"file:{opencode_db_path}?mode=ro", uri=True) as conn:
+        conn.row_factory = sqlite3.Row
+
+        if last_session_id:
+            cursor = conn.execute("SELECT COUNT(*) FROM session WHERE id > ?", (last_session_id,))
+        else:
+            cursor = conn.execute("SELECT COUNT(*) FROM session")
+
+        total_sessions = cursor.fetchone()[0]
+        if max_sessions:
+            total_sessions = min(total_sessions, max_sessions)
+
+        logger.info(f"Sessions to process: {total_sessions}")
+
+        if last_session_id:
+            cursor = conn.execute(
+                "SELECT * FROM session WHERE id > ? ORDER BY time_created ASC", (last_session_id,)
+            )
+        else:
+            cursor = conn.execute("SELECT * FROM session ORDER BY time_created ASC")
+
+        processed = 0
+        memories_created = 0
+
+        for row in cursor:
+            if max_sessions and processed >= max_sessions:
+                break
+
+            session = dict(row)
+            session_id = session["id"]
+
+            try:
+                memory = observer.extract_session_summary(session)
+                if memory:
+                    memory_id = sqlite.insert_memory(memory)
+                    embedding = embeddings.embed(memory.embedding_content())
+                    vectors.add(
+                        f"mem_{memory_id}", memory_id, memory.embedding_content(), embedding
+                    )
+                    memories_created += 1
+            except Exception as e:
+                logger.warning(f"Error processing session {session_id}: {e}")
+
+            processed += 1
+
+            if processed % batch_size == 0:
+                sqlite.set_ingest_state("opencode_db", datetime.now(UTC).isoformat(), session_id)
+                logger.info(
+                    f"Progress: {processed}/{total_sessions} sessions, {memories_created} memories"
+                )
+
+        sqlite.set_ingest_state(
+            "opencode_db",
+            datetime.now(UTC).isoformat(),
+            session_id if processed > 0 else last_session_id,
+        )
+
+        logger.info(
+            f"Complete: {processed} sessions processed, {memories_created} memories created"
+        )
+
+
+def main() -> None:
+    """CLI entry point for historical ingest."""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Historical ingest of OpenCode sessions")
+    parser.add_argument(
+        "--db-path",
+        type=Path,
+        default=None,
+        help="Path to OpenCode database",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=100,
+        help="Sessions per progress report",
+    )
+    parser.add_argument(
+        "--max-sessions",
+        type=int,
+        default=None,
+        help="Maximum sessions to process",
+    )
+
+    args = parser.parse_args()
+    run_historical_ingest(args.db_path, args.batch_size, args.max_sessions)
+
+
+if __name__ == "__main__":
+    main()