deja-cli 0.1.0__py3-none-any.whl

deja/ingest/watchers/base.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import asyncio
+import sys
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Callable, Awaitable, Optional
+
+from deja.core.store import MemoryStore
+from deja.llm.base import LLMAdapter
+from deja.llm.embedding import EmbeddingAdapter
+
+
+class BaseWatcher(ABC):
+    def __init__(
+        self,
+        store: MemoryStore,
+        extractor_fn: Callable[..., Awaitable[list[dict]]],
+        adapter: Optional[LLMAdapter],
+        debounce_seconds: int = 30,
+        embedding_adapter: Optional[EmbeddingAdapter] = None,
+    ) -> None:
+        self.store = store
+        self.extractor_fn = extractor_fn
+        self.adapter = adapter
+        self.debounce_seconds = debounce_seconds
+        self.embedding_adapter = embedding_adapter
+        # path -> last mtime we processed
+        self._processed: dict[Path, float] = {}
+        # path -> pending debounce task
+        self._pending: dict[Path, asyncio.TimerHandle] = {}
+
+    @abstractmethod
+    def get_watch_paths(self) -> list[Path]: ...
+
+    @abstractmethod
+    def should_process(self, path: Path) -> bool: ...
+
+    @abstractmethod
+    def get_project_name(self, path: Path) -> str: ...
+
+    def parse_transcript(self, content: str) -> str:
+        """Convert raw file content to a plain-text transcript for extraction.
+
+        Default implementation returns content unchanged (suitable for plain text
+        files like summary.md). Subclasses override this to parse JSON/JSONL formats.
+        """
+        return content
+
+    def handle_file_event(self, path: Path) -> None:
+        """Called from watchdog thread. Schedules debounced async processing."""
+        if not self.should_process(path):
+            return
+
+        loop = asyncio.get_event_loop()
+
+        # Cancel any existing pending timer for this path
+        if path in self._pending:
+            self._pending[path].cancel()
+
+        handle = loop.call_later(
+            self.debounce_seconds,
+            lambda: asyncio.ensure_future(self._process(path)),
+        )
+        self._pending[path] = handle
+
+    async def _process(self, path: Path) -> None:
+        """Read file and run extraction pipeline."""
+        self._pending.pop(path, None)
+
+        if not path.exists():
+            return
+
+        try:
+            mtime = path.stat().st_mtime
+        except OSError:
+            return
+
+        # Skip if we already processed this exact version
+        if self._processed.get(path) == mtime:
+            return
+
+        try:
+            content = path.read_text(encoding="utf-8", errors="replace")
+        except OSError as e:
+            print(f"[deja] Failed to read {path}: {e}", file=sys.stderr)
+            return
+
+        project = self.get_project_name(path)
+        source = self.__class__.__name__.lower().replace("watcher", "")
+
+        content = self.parse_transcript(content)
+
+        if self.adapter is None:
+            # No LLM configured — save the raw summary as a single progress memory.
+            # The agent-driven path (CLAUDE.md skill prompt) handles structured extraction.
+            raw_memory = {
+                "type": "progress",
+                "category": "agent",
+                "content": content[:2000].strip(),  # truncate to avoid huge entries
+                "scope": f"project:{project}",
+                "project": project,
+                "source": source,
+                "confidence": 0.5,
+            }
+            try:
+                await self.store.save(raw_memory)
+                print(
+                    f"[deja] Saved raw summary from {path} (no LLM configured)",
+                    file=sys.stderr,
+                )
+            except Exception as e:
+                print(f"[deja] Failed to save raw summary: {e}", file=sys.stderr)
+            self._processed[path] = mtime
+            return
+
+        try:
+            memories = await self.extractor_fn(content, project, source, self.adapter)
+        except Exception as e:
+            print(f"[deja] Extraction failed for {path}: {e}", file=sys.stderr)
+            return
+
+        saved = 0
+        for memory in memories:
+            try:
+                emb_bytes = None
+                if self.embedding_adapter is not None:
+                    try:
+                        emb = await self.embedding_adapter.embed(memory["content"])
+                        emb_bytes = EmbeddingAdapter.to_bytes(emb)
+                    except Exception as emb_e:
+                        print(f"[deja] Embedding failed: {emb_e}", file=sys.stderr)
+                await self.store.save(memory, emb_bytes)
+                saved += 1
+            except Exception as e:
+                print(f"[deja] Failed to save memory: {e}", file=sys.stderr)
+
+        self._processed[path] = mtime
+        if saved:
+            print(
+                f"[deja] Saved {saved} memories from {path} (project: {project})",
+                file=sys.stderr,
+            )

deja/ingest/watchers/claude_code.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+from deja.ingest.watchers.base import BaseWatcher
+
+
+class ClaudeCodeWatcher(BaseWatcher):
+    """Watches ~/.claude/projects/**/session-memory/summary.md for new session summaries."""
+
+    def get_watch_paths(self) -> list[Path]:
+        return [Path.home() / ".claude" / "projects"]
+
+    def should_process(self, path: Path) -> bool:
+        """Process summary.md files inside session-memory/ directories."""
+        return (
+            path.name == "summary.md"
+            and path.parent.name == "session-memory"
+        )
+
+    def get_project_name(self, path: Path) -> str:
+        """Extract project name from path.
+
+        Path structure: ~/.claude/projects/<hash>/session-memory/summary.md
+        First try reading ~/.claude/projects/<hash>/.project (if it exists).
+        Fall back to the hash directory name.
+        """
+        # path.parents: [session-memory/, <hash>/, projects/, .claude/, ~/, ...]
+        try:
+            hash_dir = path.parent.parent  # <hash> dir
+            project_file = hash_dir / ".project"
+            if project_file.exists():
+                content = project_file.read_text(encoding="utf-8").strip()
+                # Could be JSON with a name field or plain text
+                try:
+                    data = json.loads(content)
+                    if isinstance(data, dict) and "name" in data:
+                        return str(data["name"])
+                    if isinstance(data, dict) and "project_name" in data:
+                        return str(data["project_name"])
+                except (json.JSONDecodeError, KeyError):
+                    if content:
+                        return content
+            # Claude Code's hash dir name is the absolute project path with '/' → '-'.
+            # e.g. "-Users-tree-Desktop-deja" encodes /Users/tree/Desktop/deja
+            # Reconstruct the path and check if it actually exists on disk —
+            # if it does, Path.name gives the correct project name regardless of hyphens.
+            raw = hash_dir.name
+            reconstructed = Path("/" + raw.lstrip("-").replace("-", "/"))
+            if reconstructed.exists():
+                return reconstructed.name
+            # Reconstructed path doesn't exist (project path contains hyphens, so
+            # the round-trip is ambiguous). Fall back to last dash-separated token.
+            return raw.split("-")[-1] or raw
+        except Exception as e:
+            print(
+                f"[deja] Could not determine project name for {path}: {e}",
+                file=sys.stderr,
+            )
+            return "unknown"

deja/ingest/watchers/codex_cli.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+from deja.ingest.watchers.base import BaseWatcher
+
+
+class CodexCLIWatcher(BaseWatcher):
+    """Watches ~/.codex/sessions/**/rollout-*.jsonl for Codex CLI session files.
+
+    File structure:
+      ~/.codex/sessions/<year>/<month>/<day>/rollout-<timestamp>-<uuid>.jsonl
+
+    Each line is a JSON object with a "type" field:
+      - session_meta: payload.cwd gives the project directory
+      - response_item: payload.role in (user, assistant, developer)
+          user:      payload.content[].type == "input_text"
+          assistant: payload.content[].type == "output_text"
+          developer: system context — skipped
+      - event_msg, turn_context: metadata — skipped
+
+    Project name is derived from cwd in the session_meta line.
+    """
+
+    def get_watch_paths(self) -> list[Path]:
+        return [Path.home() / ".codex" / "sessions"]
+
+    def should_process(self, path: Path) -> bool:
+        """Process rollout-*.jsonl session files."""
+        return path.suffix == ".jsonl" and path.stem.startswith("rollout-")
+
+    def get_project_name(self, path: Path) -> str:
+        """Extract project name from cwd in the session_meta line."""
+        try:
+            for raw_line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+                try:
+                    d = json.loads(raw_line)
+                    if d.get("type") == "session_meta":
+                        cwd = d.get("payload", {}).get("cwd", "")
+                        if cwd:
+                            return Path(cwd).name
+                except (json.JSONDecodeError, KeyError):
+                    continue
+        except OSError as e:
+            print(
+                f"[deja] Could not determine project name for {path}: {e}",
+                file=sys.stderr,
+            )
+        # Fallback: stem of the filename (rollout-<timestamp>-<uuid>)
+        return path.stem
+
+    def parse_transcript(self, content: str) -> str:
+        """Parse Codex JSONL into a plain-text conversation transcript.
+
+        Extracts user and assistant turns from response_item lines.
+        Skips developer (system context), event_msg, and turn_context lines.
+        """
+        turns = []
+
+        for raw_line in content.splitlines():
+            try:
+                d = json.loads(raw_line)
+            except (json.JSONDecodeError, ValueError):
+                continue
+
+            if d.get("type") != "response_item":
+                continue
+
+            payload = d.get("payload", {})
+            role = payload.get("role", "")
+            if role not in ("user", "assistant"):
+                continue
+
+            content_items = payload.get("content", [])
+            if not isinstance(content_items, list):
+                continue
+
+            texts = []
+            for item in content_items:
+                if not isinstance(item, dict):
+                    continue
+                item_type = item.get("type", "")
+                # user turns use input_text; assistant turns use output_text
+                if item_type in ("input_text", "output_text"):
+                    text = item.get("text", "").strip()
+                    if text:
+                        texts.append(text)
+
+            if texts:
+                label = "User" if role == "user" else "Codex"
+                turns.append(f"{label}: {chr(10).join(texts)}")
+
+        return "\n\n".join(turns)

deja/ingest/watchers/gemini_cli.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+from deja.ingest.watchers.base import BaseWatcher
+
+
+class GeminiCLIWatcher(BaseWatcher):
+    """Watches ~/.gemini/tmp/**/chats/session-*.json for Gemini CLI session files.
+
+    File structure:
+      ~/.gemini/tmp/<project-dir>/chats/session-<timestamp>.json
+
+    Each file is a JSON object:
+      {
+        "sessionId": "...",
+        "projectHash": "...",
+        "startTime": "...",
+        "lastUpdated": "...",
+        "messages": [
+          {"type": "user",   "content": [{"text": "..."}]},
+          {"type": "gemini", "content": "..."},
+          {"type": "info",   ...},   # skip
+          {"type": "error",  ...},   # skip
+        ]
+      }
+
+    Project name is read from .project_root in the project directory, which
+    contains the absolute path to the project on disk.
+    """
+
+    def get_watch_paths(self) -> list[Path]:
+        return [Path.home() / ".gemini" / "tmp"]
+
+    def should_process(self, path: Path) -> bool:
+        """Process session-*.json files inside chats/ directories."""
+        return (
+            path.suffix == ".json"
+            and path.stem.startswith("session-")
+            and path.parent.name == "chats"
+        )
+
+    def get_project_name(self, path: Path) -> str:
+        """Extract project name from .project_root file.
+
+        Path structure: ~/.gemini/tmp/<project-dir>/chats/session-*.json
+        The .project_root file in <project-dir> contains the absolute project path.
+        """
+        try:
+            project_dir = path.parent.parent  # <project-dir>
+            project_root_file = project_dir / ".project_root"
+            if project_root_file.exists():
+                root_path = project_root_file.read_text(encoding="utf-8").strip()
+                if root_path:
+                    return Path(root_path).name
+            # Fallback: use the project dir name as-is (may be a hash or human name)
+            return project_dir.name
+        except Exception as e:
+            print(
+                f"[deja] Could not determine project name for {path}: {e}",
+                file=sys.stderr,
+            )
+            return "unknown"
+
+    def parse_transcript(self, content: str) -> str:
+        """Parse Gemini session JSON into a plain-text conversation transcript."""
+        try:
+            data = json.loads(content)
+        except (json.JSONDecodeError, ValueError):
+            return content  # not valid JSON — pass raw, extractor will handle it
+
+        turns = []
+        for msg in data.get("messages", []):
+            msg_type = msg.get("type", "")
+
+            if msg_type == "user":
+                raw_content = msg.get("content", [])
+                if isinstance(raw_content, list):
+                    text = " ".join(
+                        c.get("text", "")
+                        for c in raw_content
+                        if isinstance(c, dict)
+                    ).strip()
+                else:
+                    text = str(raw_content).strip()
+                if text:
+                    turns.append(f"User: {text}")
+
+            elif msg_type == "gemini":
+                text = msg.get("content", "")
+                if isinstance(text, str) and text.strip():
+                    turns.append(f"Gemini: {text.strip()}")
+
+        return "\n\n".join(turns)