opencode-semantic-memory 0.1.0__py3-none-any.whl

opencode_memory/daemon.py

@@ -0,0 +1,879 @@
+"""Background ingestion daemon for automatic file and database monitoring."""
+
+import asyncio
+import functools
+import logging
+import os
+import sqlite3
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any, Callable, TypeVar
+
+from opencode_memory.config import Config
+from opencode_memory.enrichment.gitlab import GitLabEnricher
+from opencode_memory.ingestion.embeddings import EmbeddingEngine
+from opencode_memory.ingestion.opencode_db import OpenCodeDBObserver
+from opencode_memory.ingestion.parser import MarkdownParser
+from opencode_memory.ingestion.watcher import FileWatcher
+from opencode_memory.linking.linker import MemoryLinker
+from opencode_memory.models import Entity, LinkType, MemoryLink
+from opencode_memory.storage.sqlite import SQLiteStorage
+from opencode_memory.storage.vectors import VectorStorage
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+_db_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="db_")
+
+
+async def run_in_thread(func: Callable[..., T], *args: Any, **kwargs: Any) -> T:
+    """Run a blocking function in a thread pool to avoid blocking the event loop."""
+    loop = asyncio.get_event_loop()
+    return await loop.run_in_executor(_db_executor, functools.partial(func, *args, **kwargs))
+
+
+class BackgroundThrottle:
+    """Throttle background work when MCP requests are active.
+
+    MCP request handlers call `priority_context()` to signal they need CPU.
+    Background tasks call `yield_to_priority()` between work units to pause
+    if priority work is waiting.
+    """
+
+    def __init__(self):
+        self._priority_count = 0
+        self._lock = asyncio.Lock()
+        self._idle = asyncio.Event()
+        self._idle.set()  # Start idle
+
+    @asynccontextmanager
+    async def priority_context(self):
+        """Context manager for priority (MCP) work."""
+        async with self._lock:
+            self._priority_count += 1
+            self._idle.clear()
+        try:
+            yield
+        finally:
+            async with self._lock:
+                self._priority_count -= 1
+                if self._priority_count == 0:
+                    self._idle.set()
+
+    async def yield_to_priority(self, min_delay: float = 0.1):
+        """Yield CPU to priority work if any is waiting."""
+        # Always yield briefly to let event loop process other tasks
+        await asyncio.sleep(min_delay)
+        # If priority work is active, wait for it to finish
+        if not self._idle.is_set():
+            logger.debug("Background work pausing for priority requests")
+            await self._idle.wait()
+            await asyncio.sleep(min_delay)  # Brief pause after priority work
+
+
+# Global throttle instance - used by daemon and can be imported by server
+background_throttle = BackgroundThrottle()
+
+
+CATCHUP_BATCH_SIZE = 50
+CATCHUP_DELAY_SECONDS = 2.0
+EMBEDDING_BATCH_SIZE = 16  # Max texts to embed at once (keeps CPU bursts short)
+RESCAN_RECENT_HOURS = 6
+RESCAN_INTERVAL_MINUTES = 10
+ENRICHMENT_INTERVAL_MINUTES = 5
+ENRICHMENT_BATCH_SIZE = 10
+ENRICHMENT_STALE_HOURS = 24
+CLEANUP_INTERVAL_HOURS = 1  # Hourly
+CLEANUP_RESOLVED_BLOCKERS_DAYS = 90
+CLEANUP_CONVERSATIONS_DAYS = 180
+VECTOR_KEEP_VERSIONS = 10  # Keep only last 10 LanceDB versions
+LINKING_INTERVAL_MINUTES = 5  # Run linking every 5 minutes (was 15)
+LINKING_BATCH_SIZE = 100  # Process up to 100 memories per pass (was 30)
+EXTRACTION_INTERVAL_HOURS = 6  # Run knowledge extraction every 6 hours
+EXTRACTION_SINCE_DAYS = 30  # Look at conversations from last 30 days
+
+
+class IngestionDaemon:
+    """Background daemon for automatic ingestion from files and OpenCode database."""
+
+    def __init__(
+        self,
+        config: Config,
+        sqlite: SQLiteStorage,
+        vectors: VectorStorage,
+        embeddings: EmbeddingEngine,
+    ):
+        self.config = config
+        self.sqlite = sqlite
+        self.vectors = vectors
+        self.embeddings = embeddings
+        self.parser = MarkdownParser()
+
+        self._file_watcher: FileWatcher | None = None
+        self._db_observer: OpenCodeDBObserver | None = None
+        self._enricher: GitLabEnricher | None = None
+        self._linker: MemoryLinker | None = None
+        self._poll_task: asyncio.Task | None = None
+        self._catchup_task: asyncio.Task | None = None
+        self._rescan_task: asyncio.Task | None = None
+        self._enrichment_task: asyncio.Task | None = None
+        self._cleanup_task: asyncio.Task | None = None
+        self._linking_task: asyncio.Task | None = None
+        self._extraction_task: asyncio.Task | None = None
+        self._running = False
+        self._loop: asyncio.AbstractEventLoop | None = None
+
+    async def _embed_batch_throttled(self, texts: list[str]) -> list[list[float]]:
+        """Embed texts one at a time, yielding to priority between each."""
+        all_embeddings: list[list[float]] = []
+        for text in texts:
+            # Check priority BEFORE starting embedding (which blocks GIL)
+            await background_throttle.yield_to_priority(min_delay=0.05)
+            embedding = await self.embeddings.embed_async(text)
+            all_embeddings.append(embedding)
+            # Also yield after to let event loop process responses
+            await asyncio.sleep(0)
+        return all_embeddings
+
+    def _queue_file_ingest(self, path: Path) -> None:
+        """Queue a file for async ingestion (called by watcher from separate thread)."""
+        if self._loop is None:
+            logger.warning(f"Event loop not set, cannot queue file: {path}")
+            return
+        asyncio.run_coroutine_threadsafe(self._ingest_file_async(path), self._loop)
+
+    async def _ingest_file_async(self, path: Path) -> None:
+        """Ingest a single file asynchronously."""
+        if not path.exists() or not path.is_file():
+            logger.warning(f"File not found or not a file: {path}")
+            return
+
+        try:
+            logger.info(f"Ingesting file: {path}")
+            # Run file parsing in thread to avoid blocking event loop
+            doc = await run_in_thread(self.parser.parse_file, path)
+
+            entity_ids = []
+            for entity_type, ref in doc.entities:
+                entity = Entity(type=entity_type, ref=ref)
+                entity_id = self.sqlite.upsert_entity(entity)
+                entity_ids.append(entity_id)
+
+            for memory in doc.memories:
+                memory_id = self.sqlite.insert_memory(memory, entity_ids)
+                embedding = await self.embeddings.embed_async(memory.embedding_content())
+                self.vectors.add(
+                    f"mem_{memory_id}", memory_id, memory.embedding_content(), embedding
+                )
+
+            logger.info(
+                f"Ingested {path}: {len(doc.entities)} entities, {len(doc.memories)} memories"
+            )
+        except Exception as e:
+            logger.exception(f"Error ingesting file {path}: {e}")
+
+    async def poll_opencode_db(self) -> None:
+        """Poll OpenCode DB for new sessions (called periodically)."""
+        if self._db_observer is None:
+            return
+
+        # Skip if OpenCode database doesn't exist yet (normal for fresh installs)
+        if not self._db_observer.db_exists():
+            return
+
+        try:
+            state = self.sqlite.get_ingest_state("opencode_db")
+            since = None
+            if state and state.get("last_processed"):
+                since = datetime.fromisoformat(state["last_processed"])
+
+            # Run blocking poll in thread pool
+            memories = await run_in_thread(self._db_observer.poll_new_content, since)
+
+            if memories:
+                await background_throttle.yield_to_priority()
+
+                # Batch embed all memories
+                texts = [m.embedding_content() for m in memories]
+                embeddings = await self._embed_batch_throttled(texts)
+
+                for memory, embedding in zip(memories, embeddings):
+                    memory_id = self.sqlite.insert_memory(memory)
+                    self.vectors.add(
+                        f"mem_{memory_id}", memory_id, memory.embedding_content(), embedding
+                    )
+
+                logger.info(f"Ingested {len(memories)} sessions from OpenCode database")
+                self.sqlite.set_ingest_state("opencode_db", datetime.now(UTC).isoformat())
+        except FileNotFoundError:
+            # Database doesn't exist yet - this is fine, skip silently
+            pass
+        except Exception as e:
+            logger.exception(f"Error polling OpenCode database: {e}")
+
+    async def _poll_loop(self) -> None:
+        """Background loop for DB polling."""
+        await asyncio.sleep(15)  # Wait 15s before first poll (ProcessPool avoids GIL)
+        interval = self.config.ingestion.db_poll_interval
+        while self._running:
+            await background_throttle.yield_to_priority()
+            await self.poll_opencode_db()
+            await asyncio.sleep(interval)
+
+    async def _enrichment_loop(self) -> None:
+        """Periodically enrich entities with GitLab metadata using tiered scheduling.
+
+        Entities are refreshed based on activity level:
+        - Hot (mentioned in last 24h): refresh hourly
+        - Warm (mentioned in last 7 days): refresh every 6 hours
+        - Cool (mentioned in last 30 days): refresh daily
+        - Cold (older): refresh weekly
+        """
+        if not self._enricher:
+            return
+
+        await asyncio.sleep(30)
+
+        while self._running:
+            try:
+                await background_throttle.yield_to_priority()
+
+                # Get entities due for refresh based on tiered scheduling
+                entities_to_refresh = self.sqlite.get_entities_for_refresh(
+                    limit=ENRICHMENT_BATCH_SIZE,
+                )
+
+                if entities_to_refresh:
+                    logger.debug(f"Enriching {len(entities_to_refresh)} entities (tiered refresh)")
+                    enriched_count = 0
+
+                    for entity in entities_to_refresh:
+                        if not self._running:
+                            break
+
+                        try:
+                            old_state = entity.metadata.get("state") if entity.metadata else None
+                            enriched = await self._enricher.enrich_entity(entity)
+
+                            if enriched.title or enriched.metadata:
+                                self.sqlite.upsert_entity(enriched)
+                                # Mark as enriched to update last_enriched_at
+                                if entity.id:
+                                    self.sqlite.update_entity_enriched(entity.id)
+                                enriched_count += 1
+
+                                if enriched.metadata:
+                                    new_state = enriched.metadata.get("state")
+
+                                    # State changed - bump to hot for continued monitoring
+                                    if old_state != new_state and entity.id:
+                                        self.sqlite.bump_entity_mention(entity.id)
+                                        logger.debug(
+                                            f"Entity {entity.ref} state changed: "
+                                            f"{old_state} -> {new_state}"
+                                        )
+
+                                    # Auto-resolve blockers for closed entities
+                                    if new_state in ("merged", "closed"):
+                                        resolved = self.sqlite.resolve_blockers_for_entity(
+                                            entity.ref
+                                        )
+                                        if resolved > 0:
+                                            logger.info(
+                                                f"Auto-resolved {resolved} blockers for "
+                                                f"{new_state} entity {entity.ref}"
+                                            )
+                        except Exception as e:
+                            logger.debug(f"Failed to enrich {entity.ref}: {e}")
+
+                        await asyncio.sleep(0.5)
+
+                    if enriched_count > 0:
+                        logger.info(f"Enriched {enriched_count} entities with GitLab metadata")
+
+            except Exception as e:
+                logger.exception(f"Error in enrichment loop: {e}")
+
+            # Run more frequently since tiered scheduling handles the intervals
+            await asyncio.sleep(ENRICHMENT_INTERVAL_MINUTES * 60)
+
+    async def _cleanup_loop(self) -> None:
+        """Periodically archive old/expired memories and cleanup vector storage."""
+        # Initial delay before first cleanup (5 min)
+        await asyncio.sleep(300)
+
+        while self._running:
+            try:
+                await background_throttle.yield_to_priority()
+
+                # Memory cleanup - yield between operations to allow MCP requests through
+                archived_expired = self.sqlite.archive_expired_memories()
+                if archived_expired > 0:
+                    logger.info(f"Archived {archived_expired} expired memories")
+
+                await background_throttle.yield_to_priority(min_delay=0.05)
+
+                archived_blockers = self.sqlite.archive_old_resolved_blockers(
+                    days_old=CLEANUP_RESOLVED_BLOCKERS_DAYS
+                )
+                if archived_blockers > 0:
+                    logger.info(f"Archived {archived_blockers} old resolved blockers")
+
+                await background_throttle.yield_to_priority(min_delay=0.05)
+
+                archived_conversations = self.sqlite.archive_old_conversations(
+                    days_old=CLEANUP_CONVERSATIONS_DAYS
+                )
+                if archived_conversations > 0:
+                    logger.info(f"Archived {archived_conversations} old conversations")
+
+                total = archived_expired + archived_blockers + archived_conversations
+                if total > 0:
+                    logger.info(f"Cleanup complete: archived {total} memories total")
+
+                await background_throttle.yield_to_priority(min_delay=0.05)
+
+                # Vector storage cleanup - keep only recent versions
+                vector_stats = await run_in_thread(
+                    self.vectors.cleanup_old_versions, VECTOR_KEEP_VERSIONS
+                )
+                if vector_stats.get("status") == "success":
+                    removed = vector_stats.get("versions_removed", 0)
+                    if removed > 0:
+                        logger.info(
+                            f"Vector cleanup: removed {removed} old versions, "
+                            f"freed {vector_stats.get('bytes_freed', 0) / 1024 / 1024:.1f} MB"
+                        )
+                elif vector_stats.get("status") == "error":
+                    logger.warning(f"Vector cleanup failed: {vector_stats.get('error')}")
+
+            except Exception as e:
+                logger.exception(f"Error in cleanup loop: {e}")
+
+            await asyncio.sleep(CLEANUP_INTERVAL_HOURS * 3600)
+
+    async def _linking_loop(self) -> None:
+        """Periodically discover and create links between memories."""
+        # Initial delay before starting
+        await asyncio.sleep(120)
+
+        while self._running:
+            try:
+                if self._linker:
+                    await background_throttle.yield_to_priority()
+
+                    stats = await self._linker.run_linking_pass(limit=LINKING_BATCH_SIZE)
+
+                    if stats.get("links_created", 0) > 0:
+                        logger.info(
+                            f"Memory linking: processed {stats['processed']} memories, "
+                            f"created {stats['links_created']} links"
+                        )
+
+            except Exception as e:
+                logger.exception(f"Error in linking loop: {e}")
+
+            await asyncio.sleep(LINKING_INTERVAL_MINUTES * 60)
+
+    async def _extraction_loop(self) -> None:
+        """Periodically extract knowledge from conversations using LLM.
+
+        Processes one conversation at a time, commits immediately,
+        to avoid memory runaway with large batches.
+        """
+        from opencode_memory.extraction import (
+            extract_knowledge_from_conversation,
+            get_unprocessed_conversations,
+            _find_opencode,
+        )
+
+        # Initial delay - let other tasks settle first
+        await asyncio.sleep(600)  # 10 minutes
+
+        # Check if opencode is available
+        if not _find_opencode():
+            logger.warning("Knowledge extraction disabled: opencode not found")
+            return
+
+        logger.info("Knowledge extraction task started (runs every 6 hours)")
+
+        while self._running:
+            try:
+                await background_throttle.yield_to_priority()
+
+                # Get unprocessed conversations
+                conversations = get_unprocessed_conversations(
+                    self.sqlite,
+                    since_days=EXTRACTION_SINCE_DAYS,
+                    limit=50,  # Process up to 50 per cycle
+                )
+
+                if not conversations:
+                    logger.debug("No conversations to extract knowledge from")
+                else:
+                    logger.info(
+                        f"Knowledge extraction: {len(conversations)} conversations to process"
+                    )
+                    total_extracted = 0
+
+                    for conv in conversations:
+                        if not self._running:
+                            break
+
+                        await background_throttle.yield_to_priority()
+
+                        conv_id = conv["id"]
+                        content = conv["content"]
+                        title = conv.get("what", "Untitled")[:50]
+
+                        logger.info(f"Extracting from: {title}...")
+
+                        count = await extract_knowledge_from_conversation(
+                            conv_id=conv_id,
+                            content=content,
+                            project=conv.get("project"),
+                            source_file=conv.get("source_file"),
+                            sqlite=self.sqlite,
+                            embeddings=self.embeddings,
+                            vectors=self.vectors,
+                            working_directory=self.config.ingestion.working_directory,
+                        )
+
+                        total_extracted += count
+
+                        # Small delay between conversations to avoid overwhelming
+                        await asyncio.sleep(5)
+
+                    if total_extracted > 0:
+                        logger.info(
+                            f"Knowledge extraction complete: {total_extracted} items extracted"
+                        )
+
+            except Exception as e:
+                logger.exception(f"Error in extraction loop: {e}")
+
+            await asyncio.sleep(EXTRACTION_INTERVAL_HOURS * 3600)
+
+    async def _rescan_recent_sessions(self, opencode_db_path: Path) -> None:
+        """Periodically re-scan recent sessions to catch ongoing conversations."""
+        await asyncio.sleep(60)
+        observer = OpenCodeDBObserver(opencode_db_path)
+
+        def _fetch_recent_sessions(cutoff_ms: int) -> list[dict]:
+            """Fetch recent sessions from OpenCode DB (runs in thread)."""
+            if not opencode_db_path.exists():
+                logger.debug(f"OpenCode database not found at {opencode_db_path}, skipping rescan")
+                return []
+            with sqlite3.connect(f"file:{opencode_db_path}?mode=ro", uri=True) as conn:
+                conn.row_factory = sqlite3.Row
+                cursor = conn.execute(
+                    """
+                    SELECT s.id, s.title, s.time_created,
+                           MAX(m.time_created) as last_message_time,
+                           COUNT(m.id) as message_count
+                    FROM session s
+                    LEFT JOIN message m ON m.session_id = s.id
+                    WHERE s.time_created > ? OR m.time_created > ?
+                    GROUP BY s.id
+                    ORDER BY COALESCE(MAX(m.time_created), s.time_created) DESC
+                    """,
+                    (cutoff_ms, cutoff_ms),
+                )
+                return [dict(row) for row in cursor.fetchall()]
+
+        def _fetch_session_by_id(session_id: str) -> dict | None:
+            """Fetch a single session by ID (runs in thread)."""
+            if not opencode_db_path.exists():
+                logger.debug(f"OpenCode database not found at {opencode_db_path}")
+                return None
+            with sqlite3.connect(f"file:{opencode_db_path}?mode=ro", uri=True) as conn:
+                conn.row_factory = sqlite3.Row
+                cursor = conn.execute("SELECT * FROM session WHERE id = ?", (session_id,))
+                row = cursor.fetchone()
+                return dict(row) if row else None
+
+        while self._running:
+            try:
+                cutoff_ms = int(
+                    (datetime.now(UTC) - timedelta(hours=RESCAN_RECENT_HOURS)).timestamp() * 1000
+                )
+
+                recent_sessions = await run_in_thread(_fetch_recent_sessions, cutoff_ms)
+
+                sessions_to_rescan = []
+                for row in recent_sessions:
+                    session_id = row["id"]
+                    message_count = row["message_count"] or 0
+                    current_memory_count = self.sqlite.get_session_memory_count(session_id)
+                    expected_min_memories = 1 + (message_count // 50)
+
+                    if current_memory_count < expected_min_memories or message_count > 10:
+                        session = await run_in_thread(_fetch_session_by_id, session_id)
+                        if session:
+                            sessions_to_rescan.append(session)
+
+                if not sessions_to_rescan:
+                    continue
+
+                await background_throttle.yield_to_priority()
+
+                # Delete old memories for sessions being rescanned
+                for session in sessions_to_rescan:
+                    session_id = session["id"]
+                    self.sqlite.delete_memories_for_session(session_id)
+                    self.vectors.delete_by_prefix(f"opencode:session:{session_id}")
+
+                # Extract all memories in batch
+                def _extract_batch() -> list[tuple[dict, Any, list]]:
+                    results = []
+                    for session in sessions_to_rescan:
+                        memory = observer.extract_session_summary(session)
+                        insights = observer.extract_session_insights(session)
+                        results.append((session, memory, insights))
+                    return results
+
+                extractions = await run_in_thread(_extract_batch)
+
+                # Collect all texts for batch embedding
+                memories_to_embed: list[tuple[Any, str]] = []
+                for session, memory, insights in extractions:
+                    if memory:
+                        memories_to_embed.append((memory, memory.embedding_content()))
+                    for insight in insights:
+                        memories_to_embed.append((insight, insight.embedding_content()))
+
+                if memories_to_embed:
+                    await background_throttle.yield_to_priority()
+                    texts = [content for _, content in memories_to_embed]
+                    embeddings = await self._embed_batch_throttled(texts)
+
+                    for (memory, content), embedding in zip(memories_to_embed, embeddings):
+                        memory_id = self.sqlite.insert_memory(memory)
+                        self.vectors.add(f"mem_{memory_id}", memory_id, content, embedding)
+
+                if sessions_to_rescan:
+                    logger.info(
+                        f"Rescanned {len(sessions_to_rescan)} recent sessions "
+                        f"({len(memories_to_embed)} memories)"
+                    )
+
+            except Exception as e:
+                logger.exception(f"Error rescanning recent sessions: {e}")
+
+            await asyncio.sleep(RESCAN_INTERVAL_MINUTES * 60)
+
+    async def _catchup_historical_sessions(self, opencode_db_path: Path) -> None:
+        """Ingest any sessions not yet in memory (runs deprioritized in background)."""
+        await asyncio.sleep(30)  # Wait 30s before heavy work (ProcessPool avoids GIL)
+
+        # Skip if OpenCode database doesn't exist yet
+        if not opencode_db_path.exists():
+            logger.debug("OpenCode database not found, skipping historical catchup")
+            return
+
+        def _fetch_all_session_ids() -> set[str]:
+            """Fetch all session IDs from OpenCode DB (runs in thread)."""
+            if not opencode_db_path.exists():
+                logger.debug(f"OpenCode database not found at {opencode_db_path}, skipping catchup")
+                return set()
+            with sqlite3.connect(f"file:{opencode_db_path}?mode=ro", uri=True) as conn:
+                conn.row_factory = sqlite3.Row
+                cursor = conn.execute("SELECT id FROM session ORDER BY time_created ASC")
+                return {row["id"] for row in cursor.fetchall()}
+
+        def _fetch_sessions_batch(session_ids: list[str]) -> list[dict]:
+            """Fetch multiple sessions by ID (runs in thread)."""
+            if not opencode_db_path.exists():
+                logger.debug(f"OpenCode database not found at {opencode_db_path}")
+                return []
+            with sqlite3.connect(f"file:{opencode_db_path}?mode=ro", uri=True) as conn:
+                conn.row_factory = sqlite3.Row
+                results = []
+                for sid in session_ids:
+                    cursor = conn.execute("SELECT * FROM session WHERE id = ?", (sid,))
+                    row = cursor.fetchone()
+                    if row:
+                        results.append(dict(row))
+                return results
+
+        def _extract_memories_batch(
+            observer: OpenCodeDBObserver, sessions: list[dict]
+        ) -> list[tuple[Any, Any, list]]:
+            """Extract full conversation, summary, and insights for a batch of sessions.
+
+            Returns list of (full_memory, summary_memory, insights) tuples.
+            """
+            results = []
+            for session in sessions:
+                full_memory, summary_memory = observer.extract_session_memories(session)
+                insights = observer.extract_session_insights(session)
+                results.append((full_memory, summary_memory, insights))
+            return results
+
+        try:
+            all_session_ids = await run_in_thread(_fetch_all_session_ids)
+
+            ingested_sources = self.sqlite.get_ingested_session_ids()
+            ingested_ids = {
+                src.replace("opencode:session:", "")
+                for src in ingested_sources
+                if src.startswith("opencode:session:")
+            }
+
+            missing_ids = list(all_session_ids - ingested_ids)
+            if not missing_ids:
+                logger.info("Historical catchup: all sessions already ingested")
+                return
+
+            logger.info(f"Historical catchup: {len(missing_ids)} sessions to ingest")
+
+            observer = OpenCodeDBObserver(opencode_db_path)
+            processed = 0
+
+            # Process in batches for efficient embedding
+            for batch_start in range(0, len(missing_ids), CATCHUP_BATCH_SIZE):
+                if not self._running:
+                    logger.info("Historical catchup: stopped early (daemon shutting down)")
+                    return
+
+                await background_throttle.yield_to_priority()
+
+                batch_ids = missing_ids[batch_start : batch_start + CATCHUP_BATCH_SIZE]
+
+                # Fetch sessions in batch
+                sessions = await run_in_thread(_fetch_sessions_batch, batch_ids)
+                if not sessions:
+                    continue
+
+                # Extract memories in batch (blocking but in thread)
+                extractions = await run_in_thread(_extract_memories_batch, observer, sessions)
+
+                # Collect all texts for batch embedding
+                # Track pairs of (full, summary) for linking after storage
+                memories_to_embed: list[tuple[Any, str]] = []  # (memory, content)
+                link_pairs: list[
+                    tuple[int, int]
+                ] = []  # (full_idx, summary_idx) in memories_to_embed
+
+                for full_memory, summary_memory, insights in extractions:
+                    full_idx = None
+                    summary_idx = None
+
+                    if full_memory:
+                        full_idx = len(memories_to_embed)
+                        memories_to_embed.append((full_memory, full_memory.embedding_content()))
+                    if summary_memory:
+                        summary_idx = len(memories_to_embed)
+                        memories_to_embed.append(
+                            (summary_memory, summary_memory.embedding_content())
+                        )
+
+                    if full_idx is not None and summary_idx is not None:
+                        link_pairs.append((full_idx, summary_idx))
+
+                    for insight in insights:
+                        memories_to_embed.append((insight, insight.embedding_content()))
+
+                if not memories_to_embed:
+                    processed += len(sessions)
+                    continue
+
+                # Batch embed all at once
+                await background_throttle.yield_to_priority()
+                texts = [content for _, content in memories_to_embed]
+                embeddings = await self._embed_batch_throttled(texts)
+
+                # Store with embeddings, track IDs for linking
+                stored_ids: list[int] = []
+                for (memory, content), embedding in zip(memories_to_embed, embeddings):
+                    memory_id = self.sqlite.insert_memory(memory)
+                    self.vectors.add(f"mem_{memory_id}", memory_id, content, embedding)
+                    stored_ids.append(memory_id)
+
+                # Create links between full conversations and their summaries
+                for full_idx, summary_idx in link_pairs:
+                    full_id = stored_ids[full_idx]
+                    summary_id = stored_ids[summary_idx]
+                    link = MemoryLink(
+                        source_memory_id=full_id,
+                        target_memory_id=summary_id,
+                        link_type=LinkType.EXTENDS,
+                        strength=1.0,
+                        reason="Summary of full conversation",
+                    )
+                    self.sqlite.insert_link(link)
+
+                processed += len(sessions)
+                logger.info(
+                    f"Historical catchup: {processed}/{len(missing_ids)} sessions "
+                    f"({len(memories_to_embed)} memories)"
+                )
+                await background_throttle.yield_to_priority(min_delay=CATCHUP_DELAY_SECONDS)
+
+            logger.info(f"Historical catchup complete: {processed} sessions ingested")
+
+        except Exception as e:
+            logger.exception(f"Error during historical catchup: {e}")
+
+    async def start(self) -> None:
+        """Start file watcher and DB polling in background."""
+        if self._running:
+            logger.warning("Daemon already running")
+            return
+
+        self._running = True
+        logger.info("Starting ingestion daemon")
+
+        # Store the event loop for queuing from the file watcher thread
+        self._loop = asyncio.get_event_loop()
+
+        # File watcher queues to the event loop to avoid blocking
+        self._file_watcher = FileWatcher(self._queue_file_ingest)
+        opencode_dir = Path.home() / ".opencode"
+        if opencode_dir.exists():
+            self._file_watcher.add_watch(opencode_dir)
+        else:
+            logger.warning(f"OpenCode directory not found: {opencode_dir}")
+
+        # Find OpenCode database from watch_paths config
+        opencode_db_path: Path | None = None
+        for watch_path in self.config.ingestion.watch_paths:
+            path = Path(watch_path).expanduser()
+            if path.exists() and path.is_dir():
+                self._file_watcher.add_watch(path)
+            elif path.suffix == ".db":
+                # This is the OpenCode database path
+                opencode_db_path = path
+                logger.info(f"OpenCode database configured at: {opencode_db_path}")
+
+        self._file_watcher.start()
+        logger.info("File watcher started")
+
+        # Start OpenCode DB observer if configured
+        if opencode_db_path is not None:
+            self._db_observer = OpenCodeDBObserver(opencode_db_path)
+            self._poll_task = asyncio.create_task(self._poll_loop())
+            # Delay heavy background tasks to avoid blocking during startup
+            # These run in background with low priority and check if DB exists
+            self._catchup_task = asyncio.create_task(
+                self._catchup_historical_sessions(opencode_db_path)
+            )
+            self._rescan_task = asyncio.create_task(self._rescan_recent_sessions(opencode_db_path))
+            if opencode_db_path.exists():
+                logger.info(f"OpenCode DB observer started: {opencode_db_path}")
+            else:
+                logger.info(
+                    f"OpenCode database not found yet at {opencode_db_path}, will poll when it appears"
+                )
+        else:
+            logger.info("No OpenCode database configured in watch_paths")
+
+        if os.environ.get("GITLAB_TOKEN"):
+            self._enricher = GitLabEnricher()
+            self._enrichment_task = asyncio.create_task(self._enrichment_loop())
+            logger.info("GitLab entity enrichment enabled")
+        else:
+            logger.info("No GITLAB_TOKEN found, entity enrichment disabled")
+
+        self._cleanup_task = asyncio.create_task(self._cleanup_loop())
+        logger.info("Memory cleanup task started (runs hourly)")
+
+        self._linker = MemoryLinker(self.sqlite, self.vectors, self.embeddings)
+        self._linking_task = asyncio.create_task(self._linking_loop())
+        logger.info("Memory linking task started (runs every 15 min)")
+
+        # Knowledge extraction runs in background with long delays
+        # Only enable if explicitly configured (uses LLM API, costs money)
+        if self.config.ingestion.llm_extraction:
+            self._extraction_task = asyncio.create_task(self._extraction_loop())
+            logger.warning(
+                "LLM knowledge extraction enabled - this will use your LLM API and incur costs"
+            )
+        else:
+            logger.info(
+                "LLM knowledge extraction disabled (set llm_extraction=true in config to enable)"
+            )
+
+    async def stop(self) -> None:
+        """Stop all background tasks."""
+        if not self._running:
+            return
+
+        logger.info("Stopping ingestion daemon")
+        self._running = False
+
+        if self._extraction_task:
+            self._extraction_task.cancel()
+            try:
+                await self._extraction_task
+            except asyncio.CancelledError:
+                pass
+            self._extraction_task = None
+
+        if self._linking_task:
+            self._linking_task.cancel()
+            try:
+                await self._linking_task
+            except asyncio.CancelledError:
+                pass
+            self._linking_task = None
+            self._linker = None
+
+        if self._cleanup_task:
+            self._cleanup_task.cancel()
+            try:
+                await self._cleanup_task
+            except asyncio.CancelledError:
+                pass
+            self._cleanup_task = None
+
+        if self._enrichment_task:
+            self._enrichment_task.cancel()
+            try:
+                await self._enrichment_task
+            except asyncio.CancelledError:
+                pass
+            self._enrichment_task = None
+
+        if self._enricher:
+            await self._enricher.close()
+            self._enricher = None
+
+        if self._rescan_task:
+            self._rescan_task.cancel()
+            try:
+                await self._rescan_task
+            except asyncio.CancelledError:
+                pass
+            self._rescan_task = None
+
+        if self._catchup_task:
+            self._catchup_task.cancel()
+            try:
+                await self._catchup_task
+            except asyncio.CancelledError:
+                pass
+            self._catchup_task = None
+
+        if self._poll_task:
+            self._poll_task.cancel()
+            try:
+                await self._poll_task
+            except asyncio.CancelledError:
+                pass
+            self._poll_task = None
+
+        if self._file_watcher:
+            self._file_watcher.stop()
+            self._file_watcher = None
+
+        self._db_observer = None
+        logger.info("Ingestion daemon stopped")
+
+    @property
+    def is_running(self) -> bool:
+        """Check if daemon is running."""
+        return self._running