agents-memory 0.42__py3-none-any.whl

agent_memory/ingest_catalog.py

@@ -0,0 +1,346 @@
+"""Catalog phase: chats-index.md + entity reference cards (bodies stay on disk)."""
+from __future__ import annotations
+
+import json
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List, Tuple
+
+from .ingest_common import chats_index_path, clip, record_phase, write_entity_card
+from .ingest_config import (
+    discover_openai_exports,
+    get_source,
+    list_sources,
+    load_ingest,
+    resolve_source_roots,
+)
+from .store import USER_MEMORY, ensure_memory_layout, _write
+
+Row = Tuple[str, str, str]
+
+
+def _ts(value) -> str:
+    if value is None:
+        return ""
+    try:
+        if isinstance(value, (int, float)):
+            if value > 1e12:
+                value = value / 1000.0
+            return datetime.fromtimestamp(value, tz=timezone.utc).strftime("%Y-%m-%d")
+        return str(value)[:10]
+    except (OSError, OverflowError, ValueError, TypeError):
+        return ""
+
+
+def _md_cell(text: str) -> str:
+    return clip(text).replace("|", "\\|")
+
+
+def openai_rows(root: Path) -> List[Row]:
+    rows: List[Row] = []
+    if not root.is_dir():
+        return rows
+    for shard in sorted(root.glob("conversations-*.json")):
+        data = json.loads(shard.read_text(encoding="utf-8"))
+        for conv in data if isinstance(data, list) else []:
+            if not isinstance(conv, dict):
+                continue
+            title = conv.get("title") or "(untitled)"
+            created = _ts(conv.get("create_time"))
+            rows.append((created, title, shard.name))
+    rows.sort(key=lambda r: r[0], reverse=True)
+    return rows
+
+
+def agent_transcript_rows(root: Path) -> List[Row]:
+    rows: List[Row] = []
+    if not root.is_dir():
+        return rows
+    tdir = root / "agent-transcripts" if (root / "agent-transcripts").is_dir() else root
+    workspace = root.name
+    if workspace.startswith("c-Users-"):
+        parts = workspace.split("-")
+        if len(parts) > 3:
+            workspace = "/".join(parts[3:])
+    for path in sorted(tdir.glob("*/*.jsonl")):
+        if "subagents" in path.parts:
+            continue
+        title = path.parent.name[:8]
+        try:
+            with path.open(encoding="utf-8", errors="replace") as fh:
+                for line in fh:
+                    obj = json.loads(line)
+                    if obj.get("role") != "user":
+                        continue
+                    msg = obj.get("message") or {}
+                    content = msg.get("content") or []
+                    texts = []
+                    for part in content:
+                        if isinstance(part, dict) and part.get("type") == "text":
+                            texts.append(part.get("text") or "")
+                    blob = "\n".join(texts)
+                    m = re.search(r"<user_query>\s*(.*?)\s*</user_query>", blob, re.S)
+                    if m:
+                        title = clip(m.group(1), 90)
+                    elif blob.strip():
+                        title = clip(blob, 90)
+                    break
+        except (OSError, json.JSONDecodeError):
+            pass
+        rows.append((workspace, title, str(path)))
+    return rows
+
+
+def copilot_rows(root: Path) -> List[Row]:
+    rows: List[Row] = []
+    if not root.is_dir():
+        return rows
+    if root.name == "chatSessions":
+        sessions = sorted(root.glob("*.jsonl"))
+    elif (root / "chatSessions").is_dir():
+        sessions = sorted((root / "chatSessions").glob("*.jsonl"))
+    else:
+        sessions = sorted(root.glob("*/chatSessions/*.jsonl"))
+        if not sessions:
+            sessions = sorted(root.rglob("*.jsonl"))
+
+    for session in sessions:
+        folder = "?"
+        for parent in (session.parent, session.parent.parent, session.parent.parent.parent):
+            wj = parent / "workspace.json"
+            if wj.exists():
+                try:
+                    raw = json.loads(wj.read_text(encoding="utf-8")).get("folder") or "?"
+                    folder = raw.replace("file:///", "").replace("%3A", ":")
+                    break
+                except (json.JSONDecodeError, OSError):
+                    pass
+        title = session.stem[:8]
+        try:
+            with session.open(encoding="utf-8", errors="replace") as fh:
+                for line in fh:
+                    if not line.strip():
+                        continue
+                    obj = json.loads(line)
+                    if obj.get("kind") == 1 and obj.get("k") == ["customTitle"] and obj.get("v"):
+                        title = clip(str(obj["v"]), 90)
+                        break
+                    v = obj.get("v") or {}
+                    if isinstance(v, dict):
+                        input_text = v.get("inputText") or ""
+                        if input_text.strip():
+                            title = clip(input_text, 90)
+        except (OSError, json.JSONDecodeError):
+            pass
+        rows.append((folder, title, str(session)))
+    return rows
+
+
+def claude_rows(root: Path) -> List[Row]:
+    rows: List[Row] = []
+    if not root.is_dir():
+        return rows
+    for path in sorted(root.rglob("*.jsonl")):
+        project = path.parent.name
+        title = path.stem[:12]
+        try:
+            with path.open(encoding="utf-8", errors="replace") as fh:
+                for line in fh:
+                    obj = json.loads(line)
+                    role = obj.get("role") or (obj.get("message") or {}).get("role")
+                    if role != "user":
+                        continue
+                    msg = obj.get("message") or obj
+                    content = msg.get("content")
+                    blob = ""
+                    if isinstance(content, str):
+                        blob = content
+                    elif isinstance(content, list):
+                        texts = [
+                            p.get("text") or ""
+                            for p in content
+                            if isinstance(p, dict) and p.get("type") == "text"
+                        ]
+                        blob = " ".join(texts)
+                    if blob.strip():
+                        title = clip(blob, 90)
+                        break
+        except (OSError, json.JSONDecodeError):
+            pass
+        rows.append((project, title, str(path)))
+    return rows
+
+
+def brain_rows(label: str, root: Path) -> List[Row]:
+    rows: List[Row] = []
+    if not root.is_dir():
+        return rows
+    for brain in sorted(root.iterdir()):
+        if not brain.is_dir() or brain.name.startswith("."):
+            continue
+        title = brain.name[:8]
+        for name in ("task.md", "walkthrough.md", "implementation_plan.md"):
+            path = brain / name
+            if path.exists():
+                for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+                    line = line.strip()
+                    if line.startswith("# "):
+                        title = clip(line[2:], 90)
+                        break
+                break
+        rows.append((label, title, str(brain)))
+    return rows
+
+
+def pi_rows(root: Path) -> List[Row]:
+    rows: List[Row] = []
+    for path in sorted(root.rglob("*.jsonl")):
+        cwd = ""
+        title = path.stem[:19]
+        try:
+            with path.open(encoding="utf-8", errors="replace") as fh:
+                for line in fh:
+                    obj = json.loads(line)
+                    if obj.get("type") == "session":
+                        cwd = obj.get("cwd") or ""
+                    if obj.get("type") == "message":
+                        msg = obj.get("message") or {}
+                        if msg.get("role") != "user":
+                            continue
+                        parts = msg.get("content") or []
+                        texts = [
+                            p.get("text") or ""
+                            for p in parts
+                            if isinstance(p, dict) and p.get("type") == "text"
+                        ]
+                        blob = " ".join(texts).strip()
+                        if blob:
+                            title = clip(blob, 90)
+                            break
+        except (OSError, json.JSONDecodeError):
+            pass
+        rows.append((cwd, title, str(path)))
+    return rows
+
+
+def collect_source_rows(src: dict) -> List[Row]:
+    kind = src.get("kind") or ""
+    label = str(src.get("label") or src.get("id") or kind)
+    rows: List[Row] = []
+    for path in resolve_source_roots(src):
+        if kind == "agent-jsonl":
+            rows.extend(agent_transcript_rows(path))
+        elif kind == "copilot-jsonl":
+            rows.extend(copilot_rows(path))
+        elif kind == "claude-jsonl":
+            rows.extend(claude_rows(path))
+        elif kind == "antigravity-brain":
+            rows.extend(brain_rows(label, path))
+        elif kind == "pi-jsonl":
+            rows.extend(pi_rows(path))
+    return rows
+
+
+def render_table(headers: List[str], rows: List[Tuple]) -> str:
+    lines = [
+        "| " + " | ".join(headers) + " |",
+        "| " + " | ".join("---" for _ in headers) + " |",
+    ]
+    for row in rows:
+        lines.append("| " + " | ".join(_md_cell(str(c)) for c in row) + " |")
+    return "\n".join(lines)
+
+
+def catalog_source(src: dict) -> int:
+    sid = str(src["id"])
+    if not src.get("catalog", True):
+        return 0
+    roots = resolve_source_roots(src)
+    if src.get("kind") == "openai-export":
+        count = sum(len(openai_rows(r)) for r in roots)
+    else:
+        count = len(collect_source_rows(src))
+    write_entity_card(src, roots, count)
+    record_phase(sid, "catalog", catalog_count=count, paths=[str(p) for p in roots[:8]])
+    return count
+
+
+def build_index(cfg: dict | None = None) -> Path:
+    cfg = cfg or load_ingest()
+    exports = discover_openai_exports(cfg)
+    openai: List[Row] = []
+    for export_dir in exports:
+        openai.extend(openai_rows(export_dir))
+
+    source_tables: list[tuple[str, List[Row]]] = []
+    map_rows: list[tuple[str, str, str, str]] = []
+
+    openai_src = get_source("openai-export", cfg)
+    if openai_src and openai_src.get("catalog", True):
+        catalog_source(openai_src)
+        if len(openai) > 0:
+            map_rows.append(
+                (
+                    str(openai_src.get("label") or "Open AI — GDPR export"),
+                    str(exports[0]) if exports else "",
+                    str(len(openai)),
+                    "openai-export",
+                )
+            )
+
+    for src in list_sources(cfg):
+        if (src.get("kind") or "") == "openai-export":
+            continue
+        if not src.get("catalog", True):
+            continue
+        rows = collect_source_rows(src)
+        catalog_source(src)
+        if rows:
+            source_tables.append((str(src.get("label") or src.get("id")), rows))
+            roots = resolve_source_roots(src)
+            map_rows.append(
+                (
+                    str(src.get("label") or src.get("id")),
+                    ", ".join(str(p) for p in roots[:2]),
+                    str(len(rows)),
+                    str(src.get("kind") or ""),
+                )
+            )
+
+    map_section = (
+        render_table(["source", "path", "count", "kind"], map_rows)
+        if map_rows
+        else "*(no chat stores found on disk yet; configure sources in ingest.json)*"
+    )
+
+    parts = [
+        "# Chat index",
+        "",
+        "Catalog of **where chats already live**. Not a transcript dump.",
+        "Configure sources in `~/.agents/memory/ingest.json`. "
+        "Run `python -m agent_memory ingest catalog` to refresh.",
+        "",
+        "## Store map",
+        "",
+        map_section,
+        "",
+    ]
+    if openai:
+        parts.extend(["## Open AI — GDPR export", "", render_table(["date", "title", "shard"], openai), ""])
+    for title, rows in source_tables:
+        if not rows:
+            continue
+        parts.extend([f"## {title}", "", render_table(["workspace", "title hint", "path"], rows), ""])
+
+    index = chats_index_path()
+    USER_MEMORY.mkdir(parents=True, exist_ok=True)
+    _write(index, "\n".join(parts) + "\n")
+    return index
+
+
+def run_catalog(cfg: dict | None = None) -> dict:
+    ensure_memory_layout()
+    cfg = cfg or load_ingest()
+    path = build_index(cfg)
+    return {"chats_index": str(path), "sources": len(list_sources(cfg))}

agent_memory/ingest_chats.py

@@ -0,0 +1,14 @@
+"""Build ~/.agents/memory/chats-index.md — thin wrapper around ingest catalog phase."""
+from __future__ import annotations
+
+from .ingest_catalog import run_catalog
+
+
+def main() -> int:
+    result = run_catalog()
+    print(f"wrote {result['chats_index']}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agent_memory/ingest_common.py

@@ -0,0 +1,170 @@
+"""Shared ingest filters, staging paths, and reversible state."""
+from __future__ import annotations
+
+import json
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List
+
+from .store import USER_MEMORY, _append_bullet, _read, _write
+
+HOW_TO = re.compile(
+    r"^(how (can|do|to|would)|write (a|me|the)|fix |create (a|an) |"
+    r"implement |help me|can you|could you|please |instead of |"
+    r"why (is|does|do|won't)|what is the (best|correct) )",
+    re.I,
+)
+PII = re.compile(
+    r"("
+    r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
+    r"|(\+?\d[\d\s().-]{8,}\d)"
+    r"|sk-[A-Za-z0-9]{10,}"
+    r"|github_pat_[A-Za-z0-9_]+"
+    r"|ghp_[A-Za-z0-9]+"
+    r"|xox[baprs]-[A-Za-z0-9-]+"
+    r")",
+    re.I,
+)
+CODEISH = re.compile(
+    r"^\s*(def |class |import |from |function |const |let |var |#include )",
+    re.M,
+)
+
+
+def staging_path(source_id: str) -> Path:
+    return USER_MEMORY / "staging" / "ingest" / source_id / "captured.md"
+
+
+def ingest_state_path() -> Path:
+    return USER_MEMORY / "ingest" / "state.json"
+
+
+def chats_index_path() -> Path:
+    return USER_MEMORY / "chats-index.md"
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def scrub(text: str) -> str:
+    text = re.sub(r"```.*?```", " ", text, flags=re.S)
+    text = PII.sub("[redacted]", text)
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+
+def keep_user_line(title: str, text: str) -> bool:
+    if not text or len(text) < 12:
+        return False
+    if text.count("[redacted]") and len(text) < 40:
+        return False
+    if len(text) > 600:
+        return False
+    if CODEISH.search(text) and len(text) > 120:
+        return False
+    if HOW_TO.search(text):
+        return False
+    return True
+
+
+def clip(text: str, n: int = 120) -> str:
+    text = re.sub(r"\s+", " ", text or "").strip()
+    return text[: n - 1] + "…" if len(text) > n else text
+
+
+def entity_path(source_id: str) -> Path:
+    return USER_MEMORY / "entities" / f"chat-source-{source_id}.md"
+
+
+def staging_header(source_id: str, label: str) -> str:
+    return (
+        f"# Staging (ingest: {source_id})\n\n"
+        f"Source: {label}. Not memory. "
+        "Distill each bullet with MCP `add_memory(kind=..., name=...)`, then delete here.\n\n"
+        "## Captured\n\n"
+    )
+
+
+def load_state() -> dict:
+    path = ingest_state_path()
+    if not path.is_file():
+        return {"sources": {}}
+    try:
+        data = json.loads(_read(path))
+        if isinstance(data, dict):
+            return data
+    except json.JSONDecodeError:
+        pass
+    return {"sources": {}}
+
+
+def save_state(state: dict) -> None:
+    path = ingest_state_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    _write(path, json.dumps(state, indent=2, ensure_ascii=False))
+
+
+def record_phase(source_id: str, phase: str, **fields: Any) -> None:
+    state = load_state()
+    entry = state.setdefault("sources", {}).setdefault(source_id, {})
+    entry[f"last_{phase}"] = utc_now()
+    entry.update(fields)
+    save_state(state)
+
+
+def write_staging(source_id: str, label: str, lines: List[str]) -> Path:
+    path = staging_path(source_id)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    header = staging_header(source_id, label)
+    body = header
+    for line in lines:
+        body += f"- {line}\n"
+    if body == header:
+        body += "- (none yet)\n"
+    _write(path, body)
+    return path
+
+
+def append_staging_bullets(source_id: str, label: str, lines: List[str]) -> Path:
+    path = staging_path(source_id)
+    if not path.exists():
+        return write_staging(source_id, label, lines)
+    for line in lines:
+        if line.strip():
+            _append_bullet(path, line.strip())
+    return path
+
+
+def write_entity_card(source: dict, paths: List[Path], catalog_count: int) -> Path:
+    sid = str(source["id"])
+    label = str(source.get("label") or sid)
+    kind = str(source.get("kind") or "")
+    path_lines = "\n".join(f"- `{p}`" for p in paths[:8]) or "- (not found on disk)"
+    body = (
+        f"---\n"
+        f"id: {sid}\n"
+        f"kind: {kind}\n"
+        f"role: chat-source\n"
+        f"---\n\n"
+        f"# {label}\n\n"
+        f"**Kind:** `{kind}`  \n"
+        f"**Catalog count:** {catalog_count}  \n"
+        f"**Paths:**\n{path_lines}\n\n"
+        f"Bodies stay on disk. Titles in `chats-index.md`. "
+        f"Extract: `python -m agent_memory ingest extract --source {sid}` -> "
+        f"`staging/ingest/{sid}/captured.md`. "
+        f"Distill with MCP `add_memory`, then clear staging.\n"
+    )
+    dest = entity_path(sid)
+    _write(dest, body)
+    return dest
+
+
+def format_bullet(title: str, text: str, origin: str = "") -> str:
+    title = clip(title, 80)
+    text = clip(text, 280)
+    if origin:
+        return f"[{title} @ {origin}] {text}"
+    return f"[{title}] {text}"