devin-search 0.1.0__py3-none-any.whl

devin_search/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

devin_search/cli.py

@@ -0,0 +1,221 @@
+"""Thin CLI wrapper — all logic lives in the library modules."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from devin_internals import SchemaError
+
+from devin_search import __version__
+from devin_search.history import attach_history_notes
+from devin_search.fmt import (
+    hits_table,
+    hits_to_dicts,
+    stats_line,
+    stats_to_dict,
+)
+from devin_search.index import build_index
+from devin_search.paths import (
+    default_acp_dir,
+    default_index_path,
+    default_sessions_db,
+)
+from devin_search.query import parse_since, search
+
+
+def _resolve_index(arg: str | None) -> Path:
+    return Path(arg).expanduser() if arg else default_index_path()
+
+
+def _cmd_index(args: argparse.Namespace) -> int:
+    sessions_db = (
+        Path(args.sessions_db).expanduser()
+        if args.sessions_db
+        else default_sessions_db()
+    )
+    acp_dir = (
+        None
+        if args.no_acp
+        else Path(args.acp_dir).expanduser()
+        if args.acp_dir
+        else default_acp_dir()
+    )
+    if sessions_db is None and acp_dir is None:
+        print(
+            "devin-search: no Devin stores found in the default locations; "
+            "pass --sessions-db and/or --acp-dir",
+            file=sys.stderr,
+        )
+        return 2
+    if sessions_db is not None and not sessions_db.exists():
+        print(
+            f"devin-search: {sessions_db}: no such file", file=sys.stderr
+        )
+        return 2
+    try:
+        stats = build_index(
+            _resolve_index(args.index),
+            sessions_db=sessions_db,
+            acp_dir=acp_dir,
+            rebuild=args.rebuild,
+        )
+    except SchemaError as exc:
+        print(f"devin-search: {exc}", file=sys.stderr)
+        return 2
+    if args.json:
+        print(json.dumps(stats_to_dict(stats), ensure_ascii=False, indent=2))
+    else:
+        print(stats_line(stats))
+    return 0
+
+
+def _cmd_query(args: argparse.Namespace) -> int:
+    try:
+        since = parse_since(args.since) if args.since else None
+    except ValueError:
+        print(
+            f"devin-search: --since {args.since!r} is not a date "
+            "(YYYY-MM-DD) or epoch ms",
+            file=sys.stderr,
+        )
+        return 2
+    try:
+        hits = search(
+            _resolve_index(args.index),
+            args.term,
+            role=args.role,
+            project=args.project,
+            since=since,
+            limit=args.limit,
+        )
+    except FileNotFoundError as exc:
+        print(f"devin-search: {exc}", file=sys.stderr)
+        return 2
+    if not args.no_log:
+        from devin_search.misslog import record
+        record(_resolve_index(args.index), args.term, len(hits),
+               role=args.role, project=args.project, since=args.since)
+    if args.history_dir:
+        hits = attach_history_notes(hits, args.history_dir)
+    if args.json:
+        print(
+            json.dumps(
+                {"term": args.term, "hits": hits_to_dicts(hits)},
+                ensure_ascii=False,
+                indent=2,
+            )
+        )
+    else:
+        print(hits_table(hits))
+    return 0 if hits else 1
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="devin-search",
+        description="Full-text search across all Devin sessions "
+        "(sessions.db + acp-messages → local FTS5 index).",
+    )
+    p.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}"
+    )
+    sub = p.add_subparsers(dest="command", required=True)
+
+    ix = sub.add_parser(
+        "index", help="build/update the local search index (incremental)"
+    )
+    ix.add_argument(
+        "--sessions-db",
+        help="path to Devin's sessions.db (default: auto-detect)",
+    )
+    ix.add_argument(
+        "--acp-dir",
+        help="path to Devin's User/acp-messages dir (default: auto-detect)",
+    )
+    ix.add_argument(
+        "--no-acp",
+        action="store_true",
+        help="index sessions.db only — never auto-detect the real "
+        "acp-messages dir (also the deterministic choice for tests)",
+    )
+    ix.add_argument("--index", help="index file path (default: app dir)")
+    ix.add_argument(
+        "--rebuild",
+        action="store_true",
+        help="drop and rebuild the index from scratch",
+    )
+    ix.add_argument(
+        "--json", action="store_true", help="machine-readable JSON output"
+    )
+    ix.set_defaults(func=_cmd_index)
+
+    q = sub.add_parser("query", help="search the index")
+    q.add_argument("term", help="free-text search (tokens ANDed)")
+    q.add_argument("--role", help="filter: user/assistant/tool/system/shell")
+    q.add_argument(
+        "--project", help="substring filter on the session working directory"
+    )
+    q.add_argument(
+        "--since", help="only hits at/after this date (YYYY-MM-DD or epoch ms)"
+    )
+    q.add_argument("--limit", type=int, default=20)
+    q.add_argument("--index", help="index file path (default: app dir)")
+    q.add_argument(
+        "--history-dir",
+        help="devin-history export dir — hits link to their "
+        "<YYYY-MM-DD>_<session-id>.md notes when present",
+    )
+    q.add_argument(
+        "--json", action="store_true", help="machine-readable JSON output"
+    )
+    q.add_argument(
+        "--no-log", action="store_true",
+        help="skip the local query log (<index>.queries.jsonl)",
+    )
+    q.set_defaults(func=_cmd_query)
+
+    m = sub.add_parser(
+        "misses",
+        help="zero-hit query stats from the local query log "
+        "(the objective trigger for semantic search)",
+    )
+    m.add_argument("--index", help="index file path (default: app dir)")
+    m.add_argument("--json", action="store_true")
+    m.set_defaults(func=_cmd_misses)
+    return p
+
+
+def _cmd_misses(args: argparse.Namespace) -> int:
+    from devin_search.misslog import summarize
+    s = summarize(_resolve_index(args.index))
+    if args.json:
+        print(json.dumps(s, ensure_ascii=False, indent=2))
+        return 0
+    print(f"query log: {s['log']}")
+    print(f"  {s['total_queries']} queries · "
+          f"{s['zero_hit']} zero-hit ({s['zero_hit_rate']:.0%})")
+    for month, b in sorted(s["per_month"].items()):
+        print(f"  {month}: {b['queries']} queries, "
+              f"{b['zero_hit']} zero-hit")
+    if s["top_missed_terms"]:
+        print("  top missed terms:")
+        for term, n in s["top_missed_terms"]:
+            print(f"    {n:>3}× {term}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")
+        except (AttributeError, ValueError):
+            pass
+    args = build_parser().parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

devin_search/extract.py

@@ -0,0 +1,112 @@
+"""Tolerant extraction of searchable text and roles from Devin payloads.
+
+SCHEMA.md (devin-internals-spec) marks every payload column as *unstable*:
+``chat_message`` has been seen as ``{"role", "content"}`` and
+``{"role", "text"}`` (with ``content`` sometimes a list of typed blocks),
+``tool_call_*_json`` and ACP ``payload`` shapes are undocumented. Extraction
+therefore never assumes a shape: known roles are normalized, text is pulled
+from known keys first and from a recursive string walk as fallback, and
+anything undecodable yields ``None`` rather than a crash.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+_ROLE_ALIASES = {
+    "user": "user",
+    "assistant": "assistant",
+    "agent": "assistant",
+    "thought": "assistant",
+    "tool": "tool",
+    "tool_call": "tool",
+    "tool_result": "tool",
+    "system": "system",
+}
+
+
+def normalize_role(raw: Any) -> str:
+    """Map a raw role/kind string to a normalized role."""
+    if not isinstance(raw, str) or not raw:
+        return "unknown"
+    # acp kinds may be namespaced ("fixture.user"); take the last segment.
+    short = raw.rsplit(".", 1)[-1].lower()
+    return _ROLE_ALIASES.get(short, short)
+
+
+def _block_text(block: Any) -> str:
+    if isinstance(block, str):
+        return block
+    if isinstance(block, dict):
+        inner = block.get("content")
+        if isinstance(inner, dict):
+            inner = inner.get("text")
+        if isinstance(inner, str):
+            return inner
+        text = block.get("text")
+        if isinstance(text, str):
+            return text
+    return ""
+
+
+def _walk_strings(value: Any, out: list[str], *, _depth: int = 0) -> None:
+    """Collect string leaves from arbitrary JSON — the fallback path."""
+    if _depth > 8:
+        return
+    if isinstance(value, str):
+        if value.strip():
+            out.append(value)
+    elif isinstance(value, dict):
+        for v in value.values():
+            _walk_strings(v, out, _depth=_depth + 1)
+    elif isinstance(value, (list, tuple)):
+        for v in value:
+            _walk_strings(v, out, _depth=_depth + 1)
+
+
+def extract_text(d: dict[str, Any]) -> str:
+    """Pull human-readable text out of a decoded payload dict."""
+    for key in ("content", "text"):
+        value = d.get(key)
+        if isinstance(value, str):
+            return value
+        if isinstance(value, list):
+            parts = [_block_text(b) for b in value]
+            text = "\n".join(p for p in parts if p)
+            if text:
+                return text
+    parts: list[str] = []
+    _walk_strings(d, parts)
+    return "\n".join(parts)
+
+
+def extract_chat_message(raw: str) -> tuple[str, str] | None:
+    """``chat_message`` blob → ``(role, text)``; ``None`` when undecodable."""
+    try:
+        d = json.loads(raw)
+    except (json.JSONDecodeError, TypeError):
+        return None
+    if not isinstance(d, dict):
+        return None
+    return normalize_role(d.get("role")), extract_text(d)
+
+
+def extract_payload(raw: str | None, kind: Any = None) -> tuple[str, str] | None:
+    """Opaque JSON payload (tool calls, acp messages) → ``(role, text)``.
+
+    ``kind`` supplies the role when the payload itself has none.
+    """
+    if raw is None:
+        return None
+    try:
+        d = json.loads(raw)
+    except (json.JSONDecodeError, TypeError):
+        # Not JSON — still searchable as plain text.
+        return normalize_role(kind), raw
+    if isinstance(d, dict):
+        role = normalize_role(d.get("role") or d.get("kind") or kind)
+        return role, extract_text(d)
+    if isinstance(d, str):
+        return normalize_role(kind), d
+    return normalize_role(kind), json.dumps(d)

devin_search/fmt.py

@@ -0,0 +1,96 @@
+"""Pure emitters — terminal table and JSON shapes. No logic, no I/O."""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from typing import Iterable
+
+from devin_search.index import IndexStats
+from devin_search.query import Hit
+
+
+def fmt_ts(ts_ms: int) -> str:
+    if not ts_ms:
+        return "-"
+    return datetime.fromtimestamp(ts_ms / 1000, tz=timezone.utc).strftime(
+        "%Y-%m-%d %H:%M"
+    )
+
+
+def _flatten(text: str, width: int) -> str:
+    flat = re.sub(r"\s+", " ", text).strip()
+    if len(flat) <= width:
+        return flat
+    return flat[: width - 1] + "…"
+
+
+def _project_label(project: str) -> str:
+    tail = project.rstrip("/\\").rsplit("/", 1)[-1].rsplit("\\", 1)[-1]
+    return tail or project or "-"
+
+
+def hits_table(hits: Iterable[Hit]) -> str:
+    """Fixed-width table: when · role · project · session · snippet."""
+    hits = list(hits)
+    if not hits:
+        return "no hits"
+    header = (
+        f"{'WHEN':<16} {'ROLE':<10} {'PROJECT':<16} {'SESSION':<10} SNIPPET"
+    )
+    lines = [header, "-" * len(header) + "-----"]
+    for h in hits:
+        lines.append(
+            f"{fmt_ts(h.ts):<16} {h.role:<10} {_project_label(h.project):<16}"
+            f" {h.session_id[:8]:<10} {_flatten(h.snippet, 60)}"
+        )
+        if h.history_note:
+            lines.append(
+                f"{'':<16} {'':<10} {'':<16} {'':<10} note: {h.history_note}"
+            )
+    return "\n".join(lines)
+
+
+def hit_to_dict(h: Hit) -> dict:
+    return {
+        "session_id": h.session_id,
+        "role": h.role,
+        "ts": h.ts,
+        "project": h.project,
+        "session_title": h.session_title,
+        "source": h.source,
+        "ref": h.ref,
+        "snippet": h.snippet,
+        "rank": h.rank,
+        "history_note": h.history_note,
+    }
+
+
+def hits_to_dicts(hits: Iterable[Hit]) -> list[dict]:
+    return [hit_to_dict(h) for h in hits]
+
+
+def stats_to_dict(stats: IndexStats) -> dict:
+    return {
+        "index": str(stats.index_path),
+        "rebuilt": stats.rebuilt,
+        "indexed": stats.indexed,
+        "removed": stats.removed,
+        "sources": stats.sources,
+        "by_source": {
+            "message_nodes": stats.message_nodes,
+            "prompt_history": stats.prompt_history,
+            "tool_calls": stats.tool_calls,
+            "acp_messages": stats.acp_messages,
+        },
+    }
+
+
+def stats_line(stats: IndexStats) -> str:
+    verb = "rebuilt" if stats.rebuilt else "indexed"
+    return (
+        f"{verb}: +{stats.indexed} docs"
+        f" (nodes {stats.message_nodes} · prompts {stats.prompt_history}"
+        f" · tools {stats.tool_calls} · acp {stats.acp_messages})"
+        f" · removed {stats.removed} → {stats.index_path}"
+    )

devin_search/history.py

@@ -0,0 +1,52 @@
+"""Optional links from search hits to devin-history export notes.
+
+``devin-history export`` writes one note per session named
+``<YYYY-MM-DD>_<session-id>.md`` (or ``.json``) in its export directory.
+The index does not store the session's ``created_at``, so matching is by
+the ``_<session-id>.<ext>`` filename suffix. Everything here is
+best-effort: a missing directory or missing note produces no link and no
+error.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from pathlib import Path
+from typing import Iterable
+
+from devin_search.query import Hit
+
+# Preferred note formats first — an export dir may hold both.
+_NOTE_SUFFIXES = (".md", ".json")
+
+
+def history_note(history_dir: str | Path, session_id: str) -> Path | None:
+    """The exported note for ``session_id`` in ``history_dir``, or ``None``."""
+    if not session_id:
+        return None
+    d = Path(history_dir).expanduser()
+    if not d.is_dir():
+        return None
+    names = sorted(p.name for p in d.iterdir() if p.is_file())
+    for ext in _NOTE_SUFFIXES:
+        for name in names:
+            if name.endswith(f"_{session_id}{ext}"):
+                return d / name
+    return None
+
+
+def attach_history_notes(
+    hits: Iterable[Hit], history_dir: str | Path
+) -> list[Hit]:
+    """Return hits with ``history_note`` set when the note file exists."""
+    d = Path(history_dir).expanduser()
+    if not d.is_dir():
+        return list(hits)
+    cache: dict[str, Path | None] = {}
+    out: list[Hit] = []
+    for h in hits:
+        if h.session_id not in cache:
+            cache[h.session_id] = history_note(d, h.session_id)
+        note = cache[h.session_id]
+        out.append(h if note is None else replace(h, history_note=str(note)))
+    return out