fridai 0.1.0__py3-none-any.whl

fridai/__init__.py

@@ -0,0 +1 @@
+# fridai package

fridai/cli.py

@@ -0,0 +1,98 @@
+"""fridai CLI (stdlib argparse) — just two things: indexing + running the MCP server.
+
+  fridai index [--source all|code|commits|agent] [--path DIR]
+  fridai mcp        # MCP server (agents recall via the recall tool)
+  fridai stats
+No local LLM / answer generation (search & recall only). Embedding via fastembed.
+"""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from .core import config, embeddings
+from .core.store import Store
+from .core.sources import agent_recall, code, commits
+
+
+def _open_store(redact: bool = True) -> Store:
+    config.ensure_home()
+    fresh = not Path(config.DB_PATH).exists()
+    store = Store(config.DB_PATH, redact=redact)
+    if fresh:
+        print(f"👋 fridai first run — creating the index ({config.HOME})")
+        print("   embedding (fastembed):",
+              "ON (semantic)" if embeddings.get_embedder() else "OFF (lexical only — `pip install fastembed`)")
+    return store
+
+
+def cmd_index(args) -> None:
+    valid = ("agent", "code", "commits", "all")
+    if args.source not in valid:
+        raise SystemExit(f"--source must be one of {valid}")
+    store = _open_store(redact=not args.no_redact)
+    print("secret redaction:", "OFF (--no-redact)" if args.no_redact else "ON")
+    embedder = None if args.no_embed else embeddings.get_embedder()
+    print("semantic embedding:", "ON (fastembed)" if embedder else "OFF (lexical only)")
+
+    if args.source in ("agent", "all"):
+        r = agent_recall.index_all(store, embedder=embedder, reindex=args.reindex)
+        print(f"  conversations (Claude+Codex+Gemini): {r['turns']} turns / {r['files']} sessions (skipped {r['skipped']})")
+
+    path = args.path or "."
+    if args.source in ("code", "all"):
+        r = code.index_code(path, store, reindex=args.reindex, embedder=embedder,
+                            prune=not args.no_prune)
+        print(f"  code: {r['chunks']} chunks / {r['files']} files (skipped {r['skipped']}, pruned {r['pruned']})")
+    if args.source in ("commits", "all"):
+        r = commits.index_commits(path, store, reindex=args.reindex, embedder=embedder)
+        print(f"  commits: {r['commits']}")
+
+    store.close()
+    print(f"Done → {config.DB_PATH}")
+
+
+def cmd_mcp(args) -> None:
+    from . import server
+    server.serve()
+
+
+def cmd_stats(args) -> None:
+    store = _open_store()
+    st = store.stats()
+    store.close()
+    print(f"\nTotal documents: {st['total']}")
+    if st["by_type"]:
+        print("By source:", ", ".join(f"{k}={v}" for k, v in st["by_type"].items()))
+    print("Embedding (semantic):", "ON" if embeddings.get_embedder() else "OFF (lexical search)")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(prog="fridai",
+                                 description="fridai MCP server — recall past code/commits/AI conversations (search-only)")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    ix = sub.add_parser("index", help="index sources (code/commits/AI conversations)")
+    ix.add_argument("--source", default="all", help="agent | code | commits | all (default: all)")
+    ix.add_argument("--path", help="target repo path for code/commits (default: current dir)")
+    ix.add_argument("--reindex", action="store_true", help="ignore incremental state and reindex everything")
+    ix.add_argument("--no-embed", action="store_true", help="skip embeddings (lexical only)")
+    ix.add_argument("--no-prune", action="store_true", help="skip pruning deleted files (code)")
+    ix.add_argument("--no-redact", action="store_true", help="turn off secret redaction (default ON)")
+    ix.set_defaults(func=cmd_index)
+
+    mp = sub.add_parser("mcp", help="run the MCP server — agents recall past memory via `recall`")
+    mp.set_defaults(func=cmd_mcp)
+
+    st = sub.add_parser("stats", help="index overview")
+    st.set_defaults(func=cmd_stats)
+    return ap
+
+
+def main(argv=None) -> None:
+    args = build_parser().parse_args(argv)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

fridai/core/config.py

@@ -0,0 +1,37 @@
+"""fridai global settings — paths/constants. No external dependencies."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+# Data home (kept separate from the working repo). Override with FRIDAI_HOME (for tests).
+HOME = Path(os.environ.get("FRIDAI_HOME", Path.home() / ".fridai"))
+DB_PATH = HOME / "index.db"
+
+# F4 sources: coding-agent conversation locations. Silently skipped if absent.
+CLAUDE_PROJECTS = Path(
+    os.environ.get("FRIDAI_CLAUDE_PROJECTS", Path.home() / ".claude" / "projects")
+)
+CODEX_SESSIONS = Path(
+    os.environ.get("FRIDAI_CODEX_SESSIONS", Path.home() / ".codex" / "sessions")
+)
+GEMINI_SESSIONS = Path(
+    os.environ.get("FRIDAI_GEMINI_SESSIONS", Path.home() / ".gemini" / "tmp")
+)
+
+# High-entropy secret heuristic. Off by default (precise rules only) — long camelCase
+# identifiers etc. produce too many false positives.
+REDACT_ENTROPY = os.environ.get("FRIDAI_REDACT_ENTROPY", "").lower() in ("1", "true", "yes")
+
+# How far to demote agent_turns with no work signal (pure question/recall turns) in ranking.
+# Larger = solution artifacts (code/commits/work turns) rank first. 0 disables the rerank.
+WORK_PENALTY = int(os.environ.get("FRIDAI_WORK_PENALTY", "8"))
+
+# Time window (minutes) for matching a question to its resulting commit.
+COMMIT_WINDOW_MIN = int(os.environ.get("FRIDAI_COMMIT_WINDOW_MIN", "180"))
+
+
+def ensure_home() -> Path:
+    """Ensure the data-home directory exists and return it."""
+    HOME.mkdir(parents=True, exist_ok=True)
+    return HOME

fridai/core/embeddings.py

@@ -0,0 +1,65 @@
+"""Embedding provider — fastembed (onnx) only + fallback.
+
+fridai uses only fastembed so it runs 100% offline (no local LLM / remote API).
+`get_embedder()`: FastEmbedEmbedder if fastembed is installed, else None (lexical fallback).
+Disable with `FRIDAI_EMBED_BACKEND=none`. `.model_id` identifies index-query consistency.
+"""
+from __future__ import annotations
+
+import os
+
+# fastembed (onnxruntime, no torch). If absent -> None -> lexical (FTS) fallback.
+try:
+    from fastembed import TextEmbedding as _FastEmbed
+except Exception:
+    _FastEmbed = None
+
+_FASTEMBED_MODEL = os.environ.get("FRIDAI_FASTEMBED_MODEL", "nomic-ai/nomic-embed-text-v1.5")
+
+
+class FastEmbedEmbedder:
+    """onnx-based local embedder (no external server). Model is lazy-loaded once."""
+    _model = None
+
+    def __init__(self, model: str | None = None):
+        self.model = model or _FASTEMBED_MODEL
+
+    @property
+    def model_id(self) -> str:
+        return f"fastembed:{self.model}"
+
+    def available(self) -> bool:
+        return _FastEmbed is not None
+
+    def embed(self, text: str) -> list[float] | None:
+        if _FastEmbed is None:
+            return None
+        try:
+            if FastEmbedEmbedder._model is None:
+                FastEmbedEmbedder._model = _FastEmbed(model_name=self.model)
+            vecs = list(FastEmbedEmbedder._model.embed([text[:2000]]))
+            return [float(x) for x in vecs[0]] if vecs else None
+        except Exception:
+            return None
+
+
+def get_embedder(model: str | None = None):
+    """Pick the available embedder. fastembed -> None. Disable with FRIDAI_EMBED_BACKEND=none."""
+    if os.environ.get("FRIDAI_EMBED_BACKEND", "").lower() == "none":
+        return None
+    fe = FastEmbedEmbedder(model)
+    return fe if fe.available() else None
+
+
+def _embed_input(doc) -> str:
+    """Embedding input. If an enrichment summary exists, prepend it so the 'why' is in the vector too."""
+    summary = (doc.meta or {}).get("summary")
+    return f"{summary}\n{doc.text}" if summary else doc.text
+
+
+def embed_documents(docs, embedder) -> list:
+    """Fill in embeddings for docs (skip those already set). Leaves None if the embedder fails."""
+    for d in docs:
+        if d.embedding is None:
+            d.embedding = embedder.embed(_embed_input(d))
+    return docs

fridai/core/models.py

@@ -0,0 +1,36 @@
+"""Unified data model. Normalizes every source (code / commit / note / AI conversation) into a Document."""
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from datetime import datetime
+
+
+@dataclass
+class Document:
+    """Basic unit of indexing and search."""
+    id: str
+    source_type: str            # "code" | "commit" | "note" | "agent_turn"
+    repo: str
+    path: str                   # file path / commit hash / session id
+    title: str
+    text: str
+    timestamp: datetime | None = None
+    meta: dict = field(default_factory=dict)
+    embedding: list[float] | None = None
+
+
+@dataclass
+class SearchHit:
+    document: Document
+    score: float                # higher = more relevant
+
+
+def make_id(source_type: str, *parts: str) -> str:
+    """Stable document id from source/path/location (identical across reindexing)."""
+    h = hashlib.sha1()
+    h.update(source_type.encode())
+    for p in parts:
+        h.update(b"\x00")
+        h.update(str(p).encode())
+    return f"{source_type}:{h.hexdigest()[:16]}"

fridai/core/redact.py

@@ -0,0 +1,87 @@
+"""Secret redaction. Masks secrets as an indexing pre-step.
+
+Core to user trust — even locally, secrets shouldn't leak into a plaintext index.
+A precise regex ruleset + a conservative high-entropy heuristic. Disable with `--no-redact`.
+"""
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+
+from . import config
+
+_MASK = "«REDACTED:{}»"
+
+# Precise patterns (almost no false positives)
+_RULES: list[tuple[re.Pattern, str]] = [
+    (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----",
+                re.DOTALL), "PRIVATE_KEY"),
+    (re.compile(r"AKIA[0-9A-Z]{16}"), "AWS_ACCESS_KEY"),
+    (re.compile(r"ghp_[0-9A-Za-z]{36}"), "GITHUB_TOKEN"),
+    (re.compile(r"gh[oprsu]_[0-9A-Za-z]{36,}"), "GITHUB_TOKEN"),
+    (re.compile(r"xox[baprs]-[0-9A-Za-z-]{10,}"), "SLACK_TOKEN"),
+    (re.compile(r"sk-[A-Za-z0-9]{20,}"), "API_KEY"),
+    (re.compile(r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"), "JWT"),
+    # key=value style secrets (mask the value only)
+    # excludes the mask char («) from the value charset so already-masked tokens aren't re-matched
+    (re.compile(r"(?i)\b(password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|"
+                r"client[_-]?secret)\b(\s*[=:]\s*)([\"']?)([^\s\"'«]{6,})\3"),
+     "SECRET_KV"),
+]
+
+# High-entropy candidate token: 32+ chars, mixed upper/lower/digits (excludes git SHA (lowercase hex) & snake_case)
+_HE_TOKEN = re.compile(r"[A-Za-z0-9+/=_-]{32,}")
+
+
+def _shannon(s: str) -> float:
+    if not s:
+        return 0.0
+    counts = Counter(s)
+    n = len(s)
+    return -sum((c / n) * math.log2(c / n) for c in counts.values())
+
+
+def _looks_secret(tok: str) -> bool:
+    return (any(c.isupper() for c in tok) and any(c.islower() for c in tok)
+            and any(c.isdigit() for c in tok) and _shannon(tok) > 4.0)
+
+
+def redact_text(text: str, entropy: bool | None = None) -> tuple[str, int]:
+    """Returns (masked text, mask count). entropy=None follows config.REDACT_ENTROPY (default OFF)."""
+    if not text:
+        return text, 0
+    if entropy is None:
+        entropy = config.REDACT_ENTROPY
+    n = 0
+    for rx, label in _RULES:
+        if label == "SECRET_KV":
+            text, c = rx.subn(lambda m: f"{m.group(1)}{m.group(2)}{_MASK.format('SECRET')}", text)
+        else:
+            text, c = rx.subn(_MASK.format(label), text)
+        n += c
+
+    if entropy:                     # off by default (many false positives); only when explicitly enabled
+        def _he(m):
+            nonlocal n
+            if _looks_secret(m.group(0)):
+                n += 1
+                return _MASK.format("HIGH_ENTROPY")
+            return m.group(0)
+        text = _HE_TOKEN.sub(_he, text)
+    return text, n
+
+
+_META_KEYS = ("question", "answer", "answer_summary", "summary")
+
+
+def redact_document(doc) -> int:
+    """Mask the document's text/title and key meta strings in place. Returns total mask count."""
+    total = 0
+    doc.text, c = redact_text(doc.text); total += c
+    doc.title, c = redact_text(doc.title); total += c
+    for k in _META_KEYS:
+        v = doc.meta.get(k)
+        if isinstance(v, str):
+            doc.meta[k], c = redact_text(v); total += c
+    return total

fridai/core/search.py

@@ -0,0 +1,136 @@
+"""Pure retrieval (search). No LLM dependency.
+
+Lexical (BM25) / vector (cosine) / hybrid (RRF) retrieval + source citation + context serialization.
+"""
+from __future__ import annotations
+
+from . import config
+from .models import Document, SearchHit
+from .store import Store
+
+
+def citation(doc: Document) -> str:
+    """Human-readable source string per document type."""
+    when = doc.timestamp.astimezone().strftime("%Y-%m-%d") if doc.timestamp else "?"
+    if doc.source_type == "code":
+        return f"{doc.repo}/{doc.meta.get('path', doc.path)}:" \
+               f"{doc.meta.get('start_line')}-{doc.meta.get('end_line')}"
+    if doc.source_type == "commit":
+        return f"{doc.repo}@{doc.meta.get('sha', doc.path)} \"{doc.title}\""
+    if doc.source_type == "agent_turn":
+        agent = doc.meta.get("agent")
+        tag = f"[{agent}] " if agent and agent != "claude" else ""   # mark non-Claude agents
+        return f"{doc.repo} {when} {tag}session:{doc.meta.get('session_title') or doc.title}"
+    if doc.source_type == "note":
+        return f"note {when}"
+    return doc.path or doc.id
+
+
+RRF_C = 60   # RRF constant (larger softens rank-gap impact). Conventional value 60.
+
+
+def rrf_fuse(ranked_lists: list[list[SearchHit]], k: int, c: int = RRF_C) -> list[SearchHit]:
+    """Reciprocal Rank Fusion: fuse multiple ranked results by rank.
+
+    A document at rank r (0-based) in each list gets 1/(c+r+1) added to its score.
+    Robust even when score scales differ (BM25 vs cosine).
+    """
+    scores: dict[str, float] = {}
+    docs: dict[str, SearchHit] = {}
+    for hits in ranked_lists:
+        for rank, h in enumerate(hits):
+            did = h.document.id
+            scores[did] = scores.get(did, 0.0) + 1.0 / (c + rank + 1)
+            docs.setdefault(did, h)
+    fused = [SearchHit(docs[i].document, s) for i, s in scores.items()]
+    fused.sort(key=lambda x: -x.score)
+    return fused[:k]
+
+
+def hybrid_retrieve(store: Store, query: str, k: int = 5, *, embedder,
+                    repo=None, source_type=None, since=None) -> list[SearchHit]:
+    """Fuse lexical (BM25) and vector (cosine) via RRF. Returns lexical only if no vectors."""
+    pool = max(k * 2, 10)
+    lex = store.search_lexical(query, k=pool, repo=repo, source_type=source_type, since=since)
+    qv = embedder.embed(query)
+    vec = store.search_vector(qv, k=pool, repo=repo, source_type=source_type,
+                              since=since) if qv else []
+    if not vec:
+        return lex[:k]
+    return rrf_fuse([lex, vec], k)
+
+
+def work_signal(doc: Document) -> bool:
+    """Is this document 'actual work output'? code/commit/note are inherently work output.
+    An agent_turn counts as a work turn only if it has file edits (files) or a **file-matched
+    resulting commit**. (Time-proximity 'guessed' commits also attach to question turns, so
+    they don't count as a work signal.)"""
+    if doc.source_type != "agent_turn":
+        return True
+    m = doc.meta
+    if m.get("files"):
+        return True
+    return any(len(c) > 2 and c[2] == "file" for c in (m.get("commits") or []))
+
+
+def rerank_work_signal(hits: list[SearchHit], penalty: int | None = None) -> list[SearchHit]:
+    """Demote agent_turns with no work signal (pure question/recall turns) by `penalty` positions.
+
+    Mitigates recursive self-pollution: 'how did I do this?' question turns pushing out the
+    actual solution turns/commits. Position-based penalty (ties preserve original order).
+    penalty=None uses config.WORK_PENALTY.
+    """
+    if penalty is None:
+        penalty = config.WORK_PENALTY
+    keyed = [(i + (0 if work_signal(h.document) else penalty), i, h)
+             for i, h in enumerate(hits)]
+    keyed.sort(key=lambda x: (x[0], x[1]))
+    return [h for _, _, h in keyed]
+
+
+import re as _re
+
+DEDUP_JACCARD = 0.8   # question-token Jaccard at/above this = treated as the same question
+
+
+def _q_tokens(doc: Document) -> set:
+    text = doc.meta.get("question") or doc.title or doc.text or ""
+    return set(_re.findall(r"\w+", text.lower()))
+
+
+def dedup_results(hits: list[SearchHit], thr: float = DEDUP_JACCARD) -> list[SearchHit]:
+    """Collapse near-duplicate agent_turns (repeated same/similar questions) to one representative.
+
+    Preserves post-rerank order, keeping the first (= top-ranked / work-signal) item as the
+    representative. code/commit/note are not affected.
+    """
+    seen: list[set] = []
+    out = []
+    for h in hits:
+        if h.document.source_type == "agent_turn":
+            toks = _q_tokens(h.document)
+            if toks and any(len(toks & s) / len(toks | s) >= thr for s in seen):
+                continue           # near-identical to a question already seen -> skip
+            seen.append(toks)
+        out.append(h)
+    return out
+
+
+def retrieve(store: Store, query: str, k: int = 5, *, embedder=None,
+             repo=None, source_type=None, since=None) -> list[SearchHit]:
+    """Hybrid (BM25+vector RRF) if embedder given, else lexical. Work-signal rerank + dedup, then top-k."""
+    pool = max(k * 3, 15)           # retrieve generously, then trim after rerank/dedup
+    if embedder is not None:
+        hits = hybrid_retrieve(store, query, pool, embedder=embedder,
+                               repo=repo, source_type=source_type, since=since)
+    else:
+        hits = store.search_lexical(query, k=pool, repo=repo, source_type=source_type, since=since)
+    return dedup_results(rerank_work_signal(hits))[:k]
+
+
+def build_context(hits: list[SearchHit]) -> str:
+    """Context for an LLM prompt. Numbers each block and attaches its source."""
+    return "\n\n".join(
+        f"[{i}] ({citation(h.document)})\n{h.document.text}"
+        for i, h in enumerate(hits, 1)
+    )

fridai/core/sources/__init__.py

@@ -0,0 +1 @@
+"""Indexing sources (code / commits / agent recall: claude·codex·gemini)."""

fridai/core/sources/agent_recall.py

@@ -0,0 +1,176 @@
+"""F4 shared base — common data structures & engine for coding-agent conversations
+(agent-independent).
+
+Per-agent parsers live in sibling modules: claude_recall / codex_recall / gemini_recall.
+Each parser turns session files into a list of `Turn` and reuses the shared machinery here:
+  - `Turn`             : normalized unit of question/answer/context (cwd·branch·files·commits)
+  - `link_commits`     : match a question to its resulting commit (time window ∩ touched files)
+  - `turn_to_document` : Turn -> indexed Document (records source agent in meta['agent'])
+  - `index_sessions`   : mtime-incremental indexing engine over session files (parser callback)
+  - `index_all`        : sum indexing across all registered agents
+"""
+from __future__ import annotations
+
+import re
+import subprocess
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from .. import config, embeddings
+from ..models import Document, make_id
+
+
+@dataclass
+class Turn:
+    question: str
+    answer: str = ""
+    when: datetime | None = None
+    cwd: str = ""
+    repo: str = ""
+    branch: str = ""
+    session_id: str = ""
+    session_title: str = ""
+    tools: list[str] = field(default_factory=list)
+    files: list[str] = field(default_factory=list)
+    commits: list[tuple[str, str, str]] = field(default_factory=list)  # (hash, subject, kind)
+
+
+def _clean(text: str) -> str:
+    """Strip harness-injected blocks (system reminders / background task notifications). Shared by parsers."""
+    text = re.sub(r"<system-reminder>.*?</system-reminder>", "", text, flags=re.DOTALL)
+    text = re.sub(r"<task-notification>.*?</task-notification>", "", text, flags=re.DOTALL)
+    return text.strip()
+
+
+# ── question -> resulting-commit matching ──
+_commit_cache: dict[str, list[tuple[datetime, str, str, set]]] = {}
+
+
+def _repo_commits(cwd: str):
+    if not cwd or cwd in _commit_cache:
+        return _commit_cache.get(cwd, [])
+    out, cur = [], None
+    if (Path(cwd) / ".git").exists():
+        try:
+            raw = subprocess.run(
+                ["git", "-C", cwd, "log", "--all", "--name-only",
+                 "--pretty=format:@@@%cI%x09%h%x09%s"],
+                capture_output=True, text=True, timeout=20,
+            ).stdout
+            for ln in raw.splitlines():
+                if ln.startswith("@@@"):
+                    if cur:
+                        out.append(cur)
+                    parts = ln[3:].split("\t", 2)
+                    cur = None
+                    if len(parts) == 3:
+                        try:
+                            # the 'Z' suffix is only accepted by fromisoformat on 3.11+ -> normalize
+                            cur = (datetime.fromisoformat(parts[0].replace("Z", "+00:00")),
+                                   parts[1], parts[2], set())
+                        except ValueError:
+                            cur = None
+                elif ln.strip() and cur:
+                    cur[3].add(Path(ln.strip()).name)
+            if cur:
+                out.append(cur)
+        except Exception:
+            pass
+    _commit_cache[cwd] = out
+    return out
+
+
+def link_commits(turn: Turn, window_min: int | None = None) -> None:
+    window_min = config.COMMIT_WINDOW_MIN if window_min is None else window_min
+    if not turn.when:
+        return
+    lo, hi = turn.when, turn.when + timedelta(minutes=window_min)
+    cands = [c for c in _repo_commits(turn.cwd) if lo <= c[0] <= hi]
+    if not cands:
+        turn.commits = []
+        return
+    turn_files = {Path(f).name for f in turn.files}
+    strong = [(len(turn_files & c[3]), c[0] - turn.when, c) for c in cands if turn_files & c[3]]
+    if strong:
+        strong.sort(key=lambda x: (-x[0], x[1]))
+        turn.commits = [(c[1], c[2], "file") for _, _, c in strong[:3]]
+        return
+    nonmerge = [c for c in cands if c[3]] or cands
+    nonmerge.sort(key=lambda c: c[0] - turn.when)
+    turn.commits = [(nonmerge[0][1], nonmerge[0][2], "time")]
+
+
+# ── summarize / Document conversion ──
+def summarize(answer: str, n: int = 130) -> str:
+    if not answer:
+        return ""
+    text = " ".join(answer.split())
+    parts = re.split(r"(?<=[.!?。])\s+|(?<=[다요죠음])\s+", text)
+    s = next((p for p in parts if len(p.strip()) > 8), text)
+    return s if len(s) <= n else s[: n - 1] + "…"
+
+
+def turn_to_document(turn: Turn, agent: str = "claude") -> Document:
+    title = turn.session_title or turn.question.splitlines()[0][:80]
+    return Document(
+        id=make_id("agent_turn", turn.session_id, str(turn.when), turn.question[:120]),
+        source_type="agent_turn", repo=turn.repo, path=turn.session_id,
+        title=title, text=f"{turn.question}\n\n{turn.answer}".strip(),
+        timestamp=turn.when,
+        meta={
+            "question": turn.question, "answer": turn.answer,
+            "answer_summary": summarize(turn.answer),
+            "branch": turn.branch, "cwd": turn.cwd,
+            "tools": turn.tools, "files": turn.files,
+            "session_title": turn.session_title, "session_id": turn.session_id,
+            "commits": turn.commits, "agent": agent,   # which agent this record came from
+        },
+    )
+
+
+# ── shared incremental indexing engine ──
+def index_sessions(store, files, parse, *, embedder=None, reindex: bool = False,
+                   agent: str = "claude", state_prefix: str = "agent") -> dict:
+    """Agent-independent indexing engine — index session files incrementally by mtime.
+
+    `files`: session file paths. `parse`: (Path)->list[Turn], the per-adapter parser.
+    Each file's mtime is stored in index_state (`{state_prefix}:<path>`) so unchanged files skip.
+    Turn ids are stable so re-upsert is idempotent (no delete handling). Returns {turns, files, skipped}.
+    """
+    turns_total = files_done = skipped = 0
+    for path in sorted(files):
+        key = f"{state_prefix}:{path}"
+        try:
+            mtime = str(path.stat().st_mtime)
+        except OSError:
+            continue
+        if not reindex and store.get_state(key) == mtime:
+            skipped += 1
+            continue
+        docs = []
+        for t in parse(path):
+            link_commits(t)
+            docs.append(turn_to_document(t, agent=agent))
+        if embedder and docs:
+            embeddings.embed_documents(docs, embedder)
+        if docs:
+            store.upsert(docs)
+        store.set_state(key, mtime)
+        files_done += 1
+        turns_total += len(docs)
+    return {"turns": turns_total, "files": files_done, "skipped": skipped}
+
+
+def index_all(store, projects_dir: Path | None = None, codex_dir: Path | None = None,
+              gemini_dir: Path | None = None, *, embedder=None, reindex: bool = False) -> dict:
+    """Index and sum conversations across all registered agents (Claude Code + Codex CLI + Gemini CLI).
+    Each source yields 0 if its directory is absent. Called by `index --source agent`."""
+    from . import claude_recall, codex_recall, gemini_recall
+    total = {"turns": 0, "files": 0, "skipped": 0}
+    for r in (claude_recall.index_claude(store, projects_dir, embedder=embedder, reindex=reindex),
+              codex_recall.index_codex(store, codex_dir, embedder=embedder, reindex=reindex),
+              gemini_recall.index_gemini(store, gemini_dir, embedder=embedder, reindex=reindex)):
+        for k in total:
+            total[k] += r[k]
+    return total