agrep 0.1.0__py3-none-win_amd64.whl

agrep/__init__.py

@@ -0,0 +1,9 @@
+"""agrep — grep your cross-agent chat history, one command.
+
+This package is a thin wrapper that ships the existing flat agrep codebase (cli.py,
+reindex.py, py/, web/) plus the prebuilt rust ingest binary, so `uvx agrep` /
+`pipx run agrep` / `pip install agrep` give you the explorer with no clone and no
+cargo. The real logic lives in the bundled modules; see __main__.py.
+"""
+
+__version__ = "0.1.0"

agrep/__main__.py

@@ -0,0 +1,37 @@
+"""Console entry for `agrep` (and `python -m agrep`).
+
+The wheel installs the flat tilt tree as package data under this package:
+
+    site-packages/agrep/
+        cli.py  reindex.py  py/*.py  web/app.html  _bin/agrep-rs[.exe]
+
+We point the code at the bundled rust binary (AGREP_RS_BIN), put the bundled dirs on
+sys.path so the existing flat imports (`import common`, `import cli`) resolve, then
+hand off to cli.py. Bare `agrep` prints status + usage and exits (the explorer
+is `agrep ui`), same as `python cli.py`. Data and the smart-tier venv go to a per-user
+dir (see common.py), since site-packages is read-only.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+PKG = Path(__file__).resolve().parent
+
+
+def main() -> int:
+    exe = "agrep-rs.exe" if sys.platform == "win32" else "agrep-rs"
+    bundled = PKG / "_bin" / exe
+    if bundled.exists():
+        os.environ.setdefault("AGREP_RS_BIN", str(bundled))
+    # py/ first so flat `import common` wins; PKG so `import cli` finds cli.py
+    sys.path.insert(0, str(PKG / "py"))
+    sys.path.insert(0, str(PKG))
+    import cli  # noqa: PLC0415 -- bundled module, resolvable only after the path setup
+    return cli.main()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agrep/cli.py

@@ -0,0 +1,315 @@
+#!/usr/bin/env python
+"""agrep — grep and explore your cross-agent chat history.
+
+  agrep "race condition"    grep your whole agent history; print matches  (the namesake)
+  agrep around <id> <turn>  show the conversation around a search hit, tools inline
+  agrep resume <id>         jump back into a past session in its agent, cd'd there
+  agrep ui                  the explorer (tilt): index, serve, and open the app
+  agrep doctor              check what's installed and what each tier needs
+  agrep index               just (re)build the index from your agent stores
+  agrep reindex             full pipeline: index + embeddings + affect + topics + arcs
+  agrep serve               just run the server (it auto-indexes in the background)
+  agrep tail                follow live agent events as JSON lines
+
+A bare first argument that isn't a command is treated as a search, so `agrep deadlock`
+greps. Bare `agrep` prints status + usage (it never starts a server). In a dev checkout
+the same commands run as `python cli.py <cmd>`. Run `agrep <command> --help` for a
+command's own options.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import socket
+import subprocess
+import sys
+import time
+import webbrowser
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+WIN = sys.platform == "win32"
+sys.path.insert(0, str(ROOT / "py"))
+import common  # noqa: E402  -- single source for binary / venv / data paths
+
+INGEST_BIN = common.ingest_bin()
+
+
+def _version() -> str:
+    """Single-sourced from agrep/__init__.py (pyproject reads the same file).
+    Resolves both installed (site-packages/agrep) and dev (repo root/agrep)."""
+    try:
+        from agrep import __version__
+        return __version__
+    except ImportError:
+        return "dev"
+
+
+def _server_python() -> str:
+    """The python the SERVER runs under. Semantic search needs the smart tier's
+    deps (numpy/torch/...), which live in the venv — launching the server with
+    whatever python invoked us silently downgrades every 'meaning' search
+    to keyword when that python lacks them (it logs one line and carries on).
+    Prefer the venv whenever it exists; plain interpreters still run the core."""
+    return common.venv_python()
+
+
+def _ensure_binary() -> bool:
+    if INGEST_BIN.exists():
+        return True
+    import shutil
+    if not shutil.which("cargo"):
+        print("  ! no ingest binary and no cargo on PATH.")
+        print(f"    install Rust (https://rustup.rs) and re-run, or `{common.cli_name()} serve` "
+              "to view an existing index.")
+        return False
+    print("=== first run: building the ingest binary (cargo build --release) ===", flush=True)
+    return subprocess.run(["cargo", "build", "--release"], cwd=str(ROOT)).returncode == 0 \
+        and INGEST_BIN.exists()
+
+
+def _index() -> bool:
+    # the ingest invocation + derived-db refresh live in common.build_index() (shared
+    # with the auto-index-on-first-search path); here we just wrap it with progress.
+    print("=== indexing transcripts ===", flush=True)
+    t = time.perf_counter()
+    ok = common.build_index()
+    if ok:
+        print(f"  ({time.perf_counter() - t:.1f}s)", flush=True)
+    return ok
+
+
+def _wait_for(port: int, timeout: float = 30.0) -> bool:
+    t0 = time.time()
+    while time.time() - t0 < timeout:
+        try:
+            with socket.create_connection(("127.0.0.1", port), timeout=0.5):
+                return True
+        except OSError:
+            time.sleep(0.25)
+    return False
+
+
+# --- status (bare `agrep`) ------------------------------------------------
+
+def _fmt_age(seconds: float) -> str:
+    """Compact human age: '3s', '12m', '5h', '2d' ago. Coarse on purpose — the
+    status line wants a glance, not a stopwatch."""
+    s = int(max(0, seconds))
+    if s < 60:
+        return f"{s}s"
+    if s < 3600:
+        return f"{s // 60}m"
+    if s < 86400:
+        return f"{s // 3600}h"
+    return f"{s // 86400}d"
+
+
+def _status_lines(cli: str) -> list[str]:
+    """Cheap index summary for the bare-`agrep` banner. Reads sessions.jsonl (one
+    line per chat, with the per-chat message count `n` and `agent`) — never parses
+    messages.jsonl, which is ~50 MB. mtime of messages.jsonl dates the last index;
+    corpus.db's presence + size says whether keyword search is ready; the smart-tier
+    venv's existence says whether meaning search is installed."""
+    sessions = common.DATA_DIR / "sessions.jsonl"
+    out: list[str] = []
+    if not sessions.exists():
+        out.append(f"  no index yet — any search will build it on first run, "
+                   f"or run `{cli} index`.")
+        return out
+
+    n_sessions = n_messages = 0
+    agents: set[str] = set()
+    with sessions.open(encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                o = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            n_sessions += 1
+            n_messages += int(o.get("n", 0))
+            a = o.get("agent")
+            if a:
+                agents.add(a)
+
+    ag = ", ".join(sorted(agents)) if agents else "—"
+    out.append(f"  {n_messages:,} messages · {n_sessions:,} sessions · "
+               f"{len(agents)} agent{'s' if len(agents) != 1 else ''} ({ag})")
+
+    msgs = common.MESSAGES_PATH
+    if msgs.exists():
+        out.append(f"  last indexed {_fmt_age(time.time() - msgs.stat().st_mtime)} ago")
+
+    db = common.DATA_DIR / "corpus.db"
+    ready = db.exists() and db.stat().st_size > 0
+    out.append(f"  search index: {'ready' if ready else 'missing (builds on first search)'}")
+
+    smart = "installed" if Path(common.venv_python()) != Path(sys.executable) else \
+        f"not installed (keyword only; `{cli} doctor` to add meaning search)"
+    out.append(f"  smart tier: {smart}")
+    return out
+
+
+def cmd_status(a) -> int:
+    """Bare `agrep`: print where the index stands and how to drive the tool, then exit.
+    Deliberately does NOT start a server — the explorer is `agrep ui`."""
+    # the banner carries em-dashes / middots; windows consoles default to cp1252.
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")
+        except Exception:  # noqa: BLE001 -- not a reconfigurable stream (piped oddly)
+            pass
+    cli = common.cli_name()  # `python cli.py` in a dev checkout, `agrep` once installed
+    print("agrep — grep and explore your cross-agent chat history\n")
+    for line in _status_lines(cli):
+        print(line)
+    print("\ntry:")
+    print(f'  {cli} "race condition"        grep every agent for a phrase')
+    print(f"  {cli} deadlock --agent codex  filter to one agent")
+    print(f"  {cli} -E 'TODO|FIXME'         regex search")
+    print(f"  {cli} -l auth                 which chats mention it")
+    print(f"  {cli} around <id> <turn>      the conversation around a hit")
+    print(f"  {cli} resume <id>             reopen a past session in its agent")
+    print(f"  {cli} ui                      the explorer: index, serve, open the app")
+    print(f"\n{cli} <command> -h for a command's own options.")
+    return 0
+
+
+# --- subcommands ----------------------------------------------------------
+
+def cmd_up(a) -> int:
+    if not a.no_index and _ensure_binary():
+        _index()
+    url = f"http://127.0.0.1:{a.port}"
+    print(f"=== serving {url} ===", flush=True)
+    srv = subprocess.Popen([_server_python(), str(ROOT / "py" / "server.py"),
+                            "--port", str(a.port)], cwd=str(ROOT))
+    try:
+        if not _wait_for(a.port):
+            print("  ! server didn't come up within 30s; see the output above.")
+            srv.terminate()
+            return 1
+        if not a.no_open:
+            webbrowser.open(url)
+        print("  ctrl-c stops the server.", flush=True)
+        return srv.wait()
+    except KeyboardInterrupt:
+        srv.terminate()
+        return 0
+
+
+def cmd_index(a) -> int:
+    if not _ensure_binary():
+        return 1
+    return 0 if _index() else 1
+
+
+def cmd_reindex(a) -> int:
+    return subprocess.run([sys.executable, str(ROOT / "reindex.py"), *a.rest],
+                          cwd=str(ROOT)).returncode
+
+
+def cmd_serve(a) -> int:
+    return subprocess.run([_server_python(), str(ROOT / "py" / "server.py"), *a.rest],
+                          cwd=str(ROOT)).returncode
+
+
+def cmd_doctor(a) -> int:
+    return subprocess.run([sys.executable, str(ROOT / "py" / "doctor.py"), *a.rest],
+                          cwd=str(ROOT)).returncode
+
+
+def cmd_tail(a) -> int:
+    return subprocess.run([sys.executable, str(ROOT / "py" / "tail.py"), *a.rest],
+                          cwd=str(ROOT)).returncode
+
+
+def cmd_search(a) -> int:
+    # in-process (stdlib-only, like resume): spawning a second interpreter doubled
+    # the cold-start cost of the single hottest command. --semantic just queries a
+    # running server, so no torch in this process either way.
+    import search
+    return search.main(a.rest)
+
+
+def cmd_around(a) -> int:
+    # core-tier like search: stdlib over the materialized index, runs in-process.
+    import around
+    return around.main(a.rest)
+
+
+def cmd_resume(a) -> int:
+    # imported and called in-process (not a subprocess) so the resumed agent is a direct
+    # child of this process and cleanly inherits the terminal.
+    import resume
+    return resume.main(a.rest)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(
+        prog="agrep", description="grep and explore your cross-agent chat history")
+    p.add_argument("-V", "--version", action="version", version=f"agrep {_version()}")
+    sub = p.add_subparsers(dest="cmd")
+
+    # `ui` is the explorer (tilt): index + serve + open. `up` is a kept-working alias
+    # (it lives in scripts / muscle memory) but only `ui` is advertised — a subparser
+    # added without help= is omitted from the command listing, so `up` stays hidden.
+    for name in ("ui", "up"):
+        kw = {"help": "index, serve, and open the explorer (tilt)"} if name == "ui" else {}
+        u = sub.add_parser(name, **kw)
+        u.add_argument("--port", type=int, default=8732)
+        u.add_argument("--no-open", action="store_true", help="don't open the browser")
+        u.add_argument("--no-index", action="store_true", help="serve the existing index as-is")
+        u.set_defaults(fn=cmd_up)
+
+    di = sub.add_parser("doctor", help="check installed tiers; --fix does safe setup")
+    di.set_defaults(fn=cmd_doctor)
+
+    ix = sub.add_parser("index", help="rebuild the base index from your agent stores")
+    ix.set_defaults(fn=cmd_index)
+
+    rx = sub.add_parser("reindex", help="full pipeline (embeddings/affect/topics/arcs)")
+    rx.set_defaults(fn=cmd_reindex)
+
+    sv = sub.add_parser("serve", help="run the server only (auto-indexes in background)")
+    sv.set_defaults(fn=cmd_serve)
+
+    ta = sub.add_parser("tail", help="follow live agent events as JSON lines (turn ends by default)")
+    ta.set_defaults(fn=cmd_tail)
+
+    se = sub.add_parser("search", help="grep your chat history (keyword; --semantic for meaning)")
+    se.set_defaults(fn=cmd_search)
+
+    ar = sub.add_parser("around", help="show the conversation around one turn of a chat")
+    ar.set_defaults(fn=cmd_around)
+
+    rs = sub.add_parser("resume", help="resume a past session in its own agent, cd'd there")
+    rs.set_defaults(fn=cmd_resume)
+
+    # The agrep promise: a bare pattern greps. If the first arg isn't a known verb
+    # (and isn't a global flag), treat the whole invocation as a search — so
+    # `agrep "rust simd"` works, while `agrep ui` / `agrep serve --port N` still
+    # dispatch. `agrep` alone prints status + usage; `agrep -h` shows top-level help.
+    raw = sys.argv[1:]
+    verbs = set(sub.choices)
+    if raw and raw[0] not in verbs and raw[0] not in ("-h", "--help", "-V", "--version"):
+        return cmd_search(argparse.Namespace(rest=raw))
+
+    # parse_known_args instead of REMAINDER positionals: REMAINDER errors on
+    # leading optionals (`agrep serve --port N` never reached the server), and
+    # mixing it with parse_known_args scrambles token order. Unknown args pass
+    # through to the subcommand verbatim.
+    args, unknown = p.parse_known_args()
+    args.rest = unknown
+    if not getattr(args, "fn", None):
+        # bare `agrep` prints status + usage and exits — the explorer is `agrep ui`.
+        return cmd_status(args)
+    return args.fn(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agrep/py/README.md

@@ -0,0 +1,150 @@
+# tilt — Python semantic sidecar
+
+This directory is the Python side of tilt: it turns the developer-chat messages
+that `tilt scan` produces into (a) semantic embeddings for similarity search and
+(b) a graded affect read per message. The Rust core (`crates/agrep-core`) reads
+the files written here.
+
+A venv is expected at `py/.venv` (see the repo-root `requirements.txt`). Use its
+interpreter for the ML stages:
+
+```
+# Windows
+py/.venv/Scripts/python embed.py
+# macOS / Linux
+py/.venv/bin/python embed.py
+```
+
+Requirements: `transformers>=4.51.0` (needed for Qwen3), `sentence-transformers`,
+`torch`, `numpy`, `scikit-learn`. A CUDA GPU speeds embeddings/affect up; everything
+also runs on CPU (slower), and the code auto-detects which. None of this is needed for
+the core explorer — see the repo-root README's tier table.
+
+---
+
+## The data contract (Python writes, Rust reads — must agree exactly)
+
+| File | Format |
+|---|---|
+| `data/messages.jsonl` | one JSON per line: `{id, agent, project, session, ts, turn, text}`. `id == "agent:session:turn"`. Produced by `tilt scan`. **Input** to every script here. |
+| `data/embeddings.f32` | raw little-endian `f32`, row-major, `N` rows × `D` cols. **Each row is L2-normalized.** `D = 256` (Matryoshka truncation of the model's native dim, then renormalized). |
+| `data/embeddings.ids` | UTF-8, one message id per line. Row `r` of `embeddings.f32` ↔ line `r` here. **The ids file is the authority for row order** — it need not match `messages.jsonl` order. |
+| `data/query.f32` | a single `D=256`-dim L2-normalized `f32` vector (the current search query). |
+| `data/emotions.jsonl` | one JSON per line: `{id, rage_raw, hype_raw, top:[...], routed_to_judge}`. |
+
+Because every stored row is L2-normalized, **cosine similarity == dot product** —
+exactly what the Rust AVX2 brute-force kernel computes.
+
+---
+
+## How to run
+
+### 1. Embed all messages
+
+```
+.venv/Scripts/python.exe embed.py
+```
+
+Reads `data/messages.jsonl`, embeds each message as a **passage**, truncates to
+256-d, L2-normalizes, and writes `data/embeddings.f32` + `data/embeddings.ids`.
+Prints model used, device, count, dim, elapsed, and approx VRAM (to stderr).
+
+Quick check without embedding everything:
+
+```
+.venv/Scripts/python.exe embed.py --smoke 8
+```
+
+### 2. Embed a search query
+
+```
+.venv/Scripts/python.exe embed_query.py "why is the build still broken"
+```
+
+Applies the correct **query-side** prefix for whichever model is configured,
+truncates to 256-d, normalizes, and writes `data/query.f32`. The Rust side then
+dots `query.f32` against `embeddings.f32` to rank messages.
+
+### 3. Affect gate
+
+```
+.venv/Scripts/python.exe emotion.py
+.venv/Scripts/python.exe emotion.py --smoke 16
+.venv/Scripts/python.exe emotion.py --profanity-ids data/profanity.ids
+```
+
+Reads `data/messages.jsonl`, runs a GoEmotions classifier, and writes
+`data/emotions.jsonl` with per-message `rage_raw`, `hype_raw`, top-3 labels, and
+a `routed_to_judge` flag.
+
+Typical query flow end to end:
+
+```
+tilt scan                                  # (Rust) writes data/messages.jsonl
+.venv/Scripts/python.exe embed.py          # writes embeddings.f32 + .ids
+.venv/Scripts/python.exe embed_query.py "…" # writes query.f32
+tilt search …                              # (Rust) AVX2 dot-product over the matrix
+```
+
+`embed.py` and `emotion.py` are independent — run them in either order.
+
+---
+
+## resolve-or-fallback model loading
+
+Each script tries a **primary** model and, on any load failure (404, gated repo,
+out-of-memory, load error), falls through to the next candidate, logging which
+id was used. So a missing or gated Qwen3 repo degrades gracefully to a smaller
+permissive model rather than crashing.
+
+| Script | Primary | Fallbacks |
+|---|---|---|
+| `embed.py` / `embed_query.py` | `Qwen/Qwen3-Embedding-0.6B` (1024-d) | `BAAI/bge-base-en-v1.5` (768-d), `sentence-transformers/all-MiniLM-L6-v2` (384-d) |
+| `emotion.py` | `cirimus/modernbert-base-go-emotions` | `SamLowe/roberta-base-go_emotions` |
+
+**Model-specific embedding handling** (branched on which candidate loaded):
+
+- **Qwen3-Embedding** — last-token pooling, **left padding**, and an asymmetric
+  instruction: **passages get NO prefix**; **queries** get
+  `Instruct: Retrieve developer-chat messages relevant to the query\nQuery: {q}`.
+- **BGE** — mean pooling; passages no prefix; queries get
+  `Represent this sentence for searching relevant passages: {q}`.
+- **MiniLM** — mean pooling; no prefix on either side.
+
+`embed_query.py` re-runs the same resolve list so the query is embedded by the
+same model that produced the matrix, and applies the matching query prefix.
+
+> Whichever model `embed.py` uses, `embed_query.py` must use the same one —
+> cosine is only meaningful within a single embedding space. If you pin or
+> change the candidate list, change it in `embed.py` (the single source of
+> truth `embed_query.py` imports from).
+
+---
+
+## Affect gate details (`emotion.py`)
+
+Multi-label GoEmotions (sigmoid per label, **graded** — not argmax). Per message:
+
+- `rage_raw = anger + annoyance + disapproval + disgust + disappointment`
+- `hype_raw = excitement + admiration + joy + approval + amusement`
+- `top` = the 3 highest-scoring emotion labels
+- `routed_to_judge = profanity-present AND ambiguous`, where *ambiguous* means
+  rage and hype are within a small margin of each other, or both are weak.
+  Profanity comes from `--profanity-ids` (an upstream set) when provided, else
+  is recomputed from a small built-in lexicon.
+
+**The LLM judge is a separate, later stage.** Messages flagged
+`routed_to_judge=true` are intended to be handed to an LLM (Qwen3.5-4B) for a
+final affect verdict. That judge is **not** implemented here — `emotion.py` only
+scores and routes.
+
+---
+
+## File map
+
+| File | Role |
+|---|---|
+| `common.py` | message loading; embeddings/ids/query writers (little-endian f32, L2-normalized rows); Matryoshka truncation; resolve-or-fallback model loader; device/VRAM helpers. |
+| `embed.py` | embed every message (passages) → `embeddings.f32` + `.ids`. `--smoke N`. |
+| `embed_query.py` | embed one query string (query prefix) → `query.f32`. |
+| `emotion.py` | affect gate → `emotions.jsonl`. `--smoke N`, `--profanity-ids`. |