lorekeep 0.4.0__tar.gz → 0.5.0__tar.gz

lorekeep-0.5.0/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "0.5.0"
+}

{lorekeep-0.4.0 → lorekeep-0.5.0}/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.5.0](https://github.com/manhhailua/lorekeep/compare/v0.4.0...v0.5.0) (2026-06-28)
+
+
+### Features
+
+* add import --from codex and --from opencode + hooks for all 4 agents ([#66](https://github.com/manhhailua/lorekeep/issues/66)) ([8a1e364](https://github.com/manhhailua/lorekeep/commit/8a1e364e19cf1b45944cea2ed89d9d7468fec9c6))
+
 ## [0.4.0](https://github.com/manhhailua/lorekeep/compare/v0.3.1...v0.4.0) (2026-06-28)
 
 

{lorekeep-0.4.0 → lorekeep-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lorekeep
-Version: 0.4.0
+Version: 0.5.0
 Summary: Temporal knowledge graph for AI agents via MCP
 Project-URL: Homepage, https://github.com/manhhailua/lorekeep
 Project-URL: Repository, https://github.com/manhhailua/lorekeep

{lorekeep-0.4.0 → lorekeep-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lorekeep"
-version = "0.4.0"
+version = "0.5.0"
 description = "Temporal knowledge graph for AI agents via MCP"
 readme = "README.md"
 license = {text = "MIT"}

{lorekeep-0.4.0 → lorekeep-0.5.0}/src/lorekeep/__init__.py

@@ -1,3 +1,3 @@
 """Lorekeep — temporal knowledge graph for AI agents via MCP."""
 
-__version__ = "0.4.0"
+__version__ = "0.5.0"

{lorekeep-0.4.0 → lorekeep-0.5.0}/src/lorekeep/cli.py

@@ -65,44 +65,35 @@ def version() -> None:
 
 @app.command(hidden=True)
 def hook() -> None:
-    """SessionEnd hook: quick-import Claude memory files to raw/.
+    """Session lifecycle hook: quick-import memories from all agents.
 
-    Designed to be registered as a Claude Code SessionEnd hook.
-    Reads hook JSON from stdin (transcript_path), imports memory files,
-    and lets the daemon pick up the raw/ change for compile.
+    Agent-agnostic — tries Claude and Codex memory imports (idempotent
+    via manifest). Each is a no-op if nothing changed. Daemon picks up
+    raw/ changes for compile.
     """
-    import sys
+    p = resolve_paths()
+    total = 0
 
-    session_dir = None
     try:
-        raw_stdin = sys.stdin.read()
-        if raw_stdin.strip():
-            data = json.loads(raw_stdin)
-            transcript_path = data.get("transcript_path", "")
-            if transcript_path:
-                session_dir = Path(transcript_path).expanduser().parent
+        from lorekeep.importer.claude import find_current_session as find_claude
+        from lorekeep.importer.claude import import_memories as import_claude_mem
+        session_dir = find_claude()
+        if session_dir and (session_dir / "memory").is_dir():
+            written = import_claude_mem(session_dir, p["raw"], "claude-memory")
+            total += len(written)
     except Exception:
         pass
 
-    if session_dir is None:
-        try:
-            from lorekeep.importer.claude import find_current_session
-            session_dir = find_current_session()
-        except Exception:
-            return
-
-    if session_dir is None or not (session_dir / "memory").is_dir():
-        return
-
-    p = resolve_paths()
     try:
-        from lorekeep.importer.claude import import_memories
-        written = import_memories(session_dir, p["raw"], "claude-memory")
-        if written:
-            typer.echo(f"lorekeep: imported {len(written)} memory file(s)")
+        from lorekeep.importer.codex import import_memories as import_codex_mem
+        written = import_codex_mem(p["raw"], "codex-memory")
+        total += len(written)
     except Exception:
         pass
 
+    if total:
+        typer.echo(f"lorekeep: imported {total} memory file(s)")
+
 
 @app.command()
 def compile() -> None:
@@ -289,7 +280,7 @@ def mcp_add(
         raise typer.Exit(code=1)
     written = writers[agent].write_config(target, command, args, ns)
     typer.echo(f"wrote {agent} config -> {written}")
-    if agent in ("claude", "cursor") and hasattr(writers[agent], "write_hook"):
+    if hasattr(writers[agent], "write_hook"):
         hook_path = writers[agent].write_hook(target, hook_cmd, hook_args)
         typer.echo(f"wrote session-end hook -> {hook_path}")
     typer.echo("\n" + agent_memory_snippet())
@@ -510,7 +501,7 @@ def _auto_wire_agents(p: dict, ns: str) -> None:
         try:
             written = writer.write_config(target, command, args, ns)
             typer.echo(f"  wired {agent_name} -> {written}")
-            if agent_name in ("claude", "cursor") and hasattr(writer, "write_hook"):
+            if hasattr(writer, "write_hook"):
                 hook_path = writer.write_hook(target, hook_cmd, hook_args)
                 typer.echo(f"  hooked {agent_name} session-end -> {hook_path}")
         except Exception as exc:
@@ -663,18 +654,80 @@ def import_cmd(
     """Import knowledge from an agent's sessions into raw/.
 
     Sources:
-      claude   Claude Code sessions. --quick copies memory/*.md only (no LLM);
-               default (deep) adds LLM-summarized transcript analysis.
-      cursor   Cursor composer conversations (GLOBAL state.vscdb). Deep-only --
-               --quick is rejected (Cursor has no curated memory files).
+      claude    Claude Code sessions. --quick copies memory/*.md only (no LLM);
+                default (deep) adds LLM-summarized transcript analysis.
+      cursor    Cursor composer conversations (GLOBAL state.vscdb). Deep-only.
+      codex     Codex CLI rollout transcripts ($CODEX_HOME/sessions/).
+                --quick copies memories/*.md only; default (deep) summarizes.
+      opencode  opencode sessions (SQLite DB). Deep-only — no memory dir.
     """
-    if from_source not in ("claude", "cursor"):
-        typer.echo(f"unknown source: {from_source} (claude | cursor)")
+    if from_source not in ("claude", "cursor", "codex", "opencode"):
+        typer.echo(f"unknown source: {from_source} (claude | cursor | codex | opencode)")
         raise typer.Exit(code=1)
 
     p = resolve_paths()
     config = load_config(p["config"])
 
+    # --- Codex: rollout JSONL transcripts, quick + deep --------------------
+    if from_source == "codex":
+        from lorekeep.importer.codex import find_current_session as find_codex, import_codex
+
+        rollout_path = Path(session_path).expanduser() if session_path else None
+        if rollout_path is None:
+            rollout_path = find_codex()
+        if rollout_path is None and not quick:
+            typer.echo("error: no Codex session found for this project. "
+                       "Run Codex CLI here first, or pass --session-path.")
+            raise typer.Exit(code=1)
+
+        result = import_codex(
+            raw_root=p["raw"],
+            rollout_path=rollout_path,
+            quick=quick,
+            memory_ns=memory_ns,
+            session_ns=session_ns or "codex-session",
+            provider=None if quick else _make_import_provider(config),
+            dry_run=dry_run,
+        )
+        mem_count = len(result.get("memory", []))
+        ses_count = len(result.get("session", []))
+        if dry_run:
+            typer.echo(f"dry-run: would import {mem_count} memories, {ses_count} session files")
+        else:
+            typer.echo(f"imported: {mem_count} memories -> raw/{memory_ns}/, "
+                       f"{ses_count} session files -> raw/{session_ns or 'codex-session'}/")
+        return
+
+    # --- opencode: SQLite DB, deep-only ------------------------------------
+    if from_source == "opencode":
+        if quick:
+            typer.echo("error: opencode import is deep-only (--quick not supported)")
+            raise typer.Exit(code=1)
+
+        from lorekeep.importer.opencode import find_current_session as find_oc, import_opencode
+
+        sid = session_path or find_oc()
+        if sid is None:
+            typer.echo("error: no opencode session found for this project. "
+                       "Run opencode here first, or pass --session-path <session-id>.")
+            raise typer.Exit(code=1)
+
+        ns = session_ns or "opencode-session"
+        result = import_opencode(
+            raw_root=p["raw"],
+            session_id=sid,
+            session_ns=ns,
+            provider=_make_import_provider(config),
+            dry_run=dry_run,
+        )
+        ses_count = len(result.get("session", []))
+        if dry_run:
+            typer.echo(f"dry-run: would import {ses_count} opencode session files")
+        else:
+            typer.echo(f"imported: {ses_count} session files -> raw/{ns}/")
+            typer.echo("next: lorekeep compile")
+        return
+
     # --- Cursor: global composer conversations, deep-only ------------------
     if from_source == "cursor":
         if quick:

lorekeep-0.5.0/src/lorekeep/importer/codex.py

@@ -0,0 +1,287 @@
+"""Import knowledge from Codex CLI sessions into lorekeep raw/ tree.
+
+Codex stores sessions as JSONL rollout files at:
+  $CODEX_HOME/sessions/YYYY/MM/DD/rollout-<ts>-<uuid>.jsonl
+
+Each line = {"timestamp": "...", "type": "...", "payload": {...}}
+The first line is always "session_meta" with cwd, session_id, model.
+Conversation turns are in "response_item" lines with payload.type == "message".
+
+Memory files live at $CODEX_HOME/memories/*.md (optional, may not exist).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from pathlib import Path
+
+from lorekeep.compile.providers import LLMProvider
+from lorekeep.importer.claude import (
+    ConversationTurn,
+    chunk_turns,
+    summarize_batch,
+    load_import_manifest,
+    save_import_manifest,
+)
+
+_CODEX_PREFIX_RE = re.compile(r"^## My request for Codex:\n", re.DOTALL)
+
+
+def _codex_home() -> Path:
+    return Path(os.environ.get("CODEX_HOME", Path.home() / ".codex"))
+
+
+# ---------------------------------------------------------------------------
+# Session discovery
+# ---------------------------------------------------------------------------
+
+
+def find_current_session(cwd: Path | None = None) -> Path | None:
+    """Find the most recent Codex rollout file for the given cwd.
+
+    Codex shards sessions by date, not by project. The cwd is stored
+    inside the first line (session_meta) of each rollout file.
+    """
+    cwd = str((cwd or Path.cwd()).resolve())
+    sessions_dir = _codex_home() / "sessions"
+    if not sessions_dir.is_dir():
+        return None
+
+    rollouts = sorted(sessions_dir.rglob("rollout-*.jsonl"), key=lambda p: p.stat().st_mtime, reverse=True)
+    for rp in rollouts:
+        try:
+            first_line = rp.read_text(encoding="utf-8").splitlines()[0].strip()
+            if not first_line:
+                continue
+            rec = json.loads(first_line)
+            if rec.get("type") == "session_meta":
+                session_cwd = rec.get("payload", {}).get("cwd", "")
+                if session_cwd and Path(session_cwd).resolve() == Path(cwd):
+                    return rp
+        except (json.JSONDecodeError, IndexError, OSError):
+            continue
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Transcript parsing
+# ---------------------------------------------------------------------------
+
+
+def _extract_message_text(content: list[dict] | str) -> str:
+    """Extract text from a Codex message content array."""
+    if isinstance(content, str):
+        return content
+    parts = []
+    for block in content:
+        if isinstance(block, dict):
+            text = block.get("text", "")
+            if text:
+                parts.append(text)
+    return "\n".join(parts)
+
+
+def _strip_codex_prefix(text: str) -> str:
+    """Strip Codex's '## My request for Codex:' prefix from user messages."""
+    return _CODEX_PREFIX_RE.sub("", text)
+
+
+def parse_rollout(rollout_path: Path) -> list[ConversationTurn]:
+    """Parse a Codex rollout JSONL file into structured conversation turns."""
+    turns: list[ConversationTurn] = []
+    current_user: str | None = None
+    current_assistant: list[str] = []
+    current_tools: list[str] = []
+
+    for line in rollout_path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            rec = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+
+        if rec.get("type") != "response_item":
+            continue
+
+        payload = rec.get("payload", {})
+        ptype = payload.get("type", "")
+
+        if ptype == "message":
+            role = payload.get("role", "")
+            text = _extract_message_text(payload.get("content", []))
+
+            if role == "user":
+                if current_user is not None:
+                    turns.append(ConversationTurn(
+                        user_content=current_user,
+                        assistant_text="\n\n".join(current_assistant),
+                        tool_calls=list(current_tools),
+                    ))
+                current_user = _strip_codex_prefix(text)
+                current_assistant = []
+                current_tools = []
+            elif role == "assistant":
+                current_assistant.append(text)
+
+        elif ptype in ("function_call", "custom_tool_call"):
+            name = payload.get("name", "unknown")
+            if name not in current_tools:
+                current_tools.append(name)
+
+    if current_user is not None:
+        turns.append(ConversationTurn(
+            user_content=current_user,
+            assistant_text="\n\n".join(current_assistant),
+            tool_calls=list(current_tools),
+        ))
+
+    return turns
+
+
+# ---------------------------------------------------------------------------
+# Memory import (quick mode)
+# ---------------------------------------------------------------------------
+
+
+def import_memories(
+    raw_root: Path,
+    namespace: str = "codex-memory",
+    *,
+    dry_run: bool = False,
+) -> list[Path]:
+    """Copy memories/*.md files from $CODEX_HOME into raw/<namespace>/.
+
+    Returns list of written paths. Idempotent via SHA-256 manifest.
+    """
+    mem_dir = _codex_home() / "memories"
+    if not mem_dir.is_dir():
+        return []
+
+    dest_dir = raw_root / namespace
+    manifest = load_import_manifest(raw_root, namespace)
+    written: list[Path] = []
+
+    for src in sorted(mem_dir.glob("*.md")):
+        content = src.read_text(encoding="utf-8")
+        h = hashlib.sha256(content.encode()).hexdigest()
+        if manifest.get(str(src)) == h:
+            continue
+
+        if dry_run:
+            written.append(dest_dir / src.name)
+            continue
+
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        dest = dest_dir / src.name
+        dest.write_text(content, encoding="utf-8")
+        manifest[str(src)] = h
+        written.append(dest)
+
+    if not dry_run and written:
+        save_import_manifest(raw_root, namespace, manifest)
+
+    return written
+
+
+# ---------------------------------------------------------------------------
+# Deep session import
+# ---------------------------------------------------------------------------
+
+
+def import_session_deep(
+    rollout_path: Path,
+    raw_root: Path,
+    namespace: str = "codex-session",
+    provider: LLMProvider | None = None,
+    *,
+    batch_max_chars: int = 20_000,
+    dry_run: bool = False,
+) -> list[Path]:
+    """Deep-import a Codex session: parse, chunk, summarize each batch via LLM."""
+    session_id = rollout_path.stem
+
+    transcript_hash = hashlib.sha256(rollout_path.read_bytes()).hexdigest()
+    manifest = load_import_manifest(raw_root, namespace)
+    manifest_key = str(rollout_path)
+
+    if not dry_run and manifest.get(manifest_key) == transcript_hash:
+        return []
+
+    turns = parse_rollout(rollout_path)
+    if not turns:
+        return []
+
+    batches = chunk_turns(turns, max_chars=batch_max_chars)
+    dest_dir = raw_root / namespace
+    written: list[Path] = []
+
+    previous_summary = ""
+    for i, batch in enumerate(batches):
+        if dry_run:
+            dest = dest_dir / f"session-{session_id}-batch-{i + 1:02d}.md"
+            written.append(dest)
+            continue
+
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        dest = dest_dir / f"session-{session_id}-batch-{i + 1:02d}.md"
+
+        try:
+            if provider is None:
+                raise RuntimeError("deep mode requires a provider")
+            md = summarize_batch(
+                batch, i, len(batches), namespace, session_id,
+                provider, previous_summary,
+            )
+        except Exception as exc:
+            md = f"# Error summarizing batch {i + 1}\n\nLLM call failed: {exc}\n"
+
+        dest.write_text(md, encoding="utf-8")
+        previous_summary = md[:1000]
+        written.append(dest)
+
+    if not dry_run and written:
+        manifest[manifest_key] = transcript_hash
+        save_import_manifest(raw_root, namespace, manifest)
+
+    return written
+
+
+# ---------------------------------------------------------------------------
+# Orchestrator
+# ---------------------------------------------------------------------------
+
+
+def import_codex(
+    raw_root: Path,
+    rollout_path: Path | None = None,
+    *,
+    quick: bool = False,
+    memory_ns: str = "codex-memory",
+    session_ns: str = "codex-session",
+    provider: LLMProvider | None = None,
+    batch_max_chars: int = 20_000,
+    dry_run: bool = False,
+) -> dict[str, list[Path]]:
+    """Import a Codex session into the lorekeep raw/ tree.
+
+    Returns ``{"memory": [...], "session": [...]}`` with written file paths.
+    """
+    result: dict[str, list[Path]] = {"memory": [], "session": []}
+
+    result["memory"] = import_memories(raw_root, namespace=memory_ns, dry_run=dry_run)
+
+    if not quick:
+        if rollout_path is None:
+            rollout_path = find_current_session()
+        if rollout_path is not None:
+            result["session"] = import_session_deep(
+                rollout_path, raw_root, namespace=session_ns,
+                provider=provider, batch_max_chars=batch_max_chars,
+                dry_run=dry_run,
+            )
+
+    return result