trovex 0.11.0__py3-none-any.whl

trovex/__init__.py

@@ -0,0 +1,3 @@
+"""trovex — token-efficient routing for agent-generated .md docs."""
+
+__version__ = "0.11.0"

trovex/backup.py

@@ -0,0 +1,58 @@
+"""Online SQLite backups for the trovex store (now the sole corpus).
+
+Uses the sqlite backup API for a consistent snapshot even while the server is
+writing, after a WAL checkpoint. Keeps the last N, prunes the rest.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import time
+from pathlib import Path
+
+KEEP = 7  # the db is ~340MB (chunk vectors) — 7 daily = ~2.4GB
+
+
+def backup_dir(data_dir: Path) -> Path:
+    return data_dir / "backups"
+
+
+def make_backup(db_path: Path, data_dir: Path) -> Path:
+    bdir = backup_dir(data_dir)
+    bdir.mkdir(parents=True, exist_ok=True)
+    # Flush WAL into the main db so the snapshot is complete.
+    flush = sqlite3.connect(str(db_path))
+    try:
+        flush.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+    finally:
+        flush.close()
+    dest = bdir / f"trovex-{time.strftime('%Y%m%d-%H%M%S')}.db"
+    src = sqlite3.connect(str(db_path))
+    dst = sqlite3.connect(str(dest))
+    try:
+        src.backup(dst)  # consistent online copy
+    finally:
+        dst.close()
+        src.close()
+    prune(data_dir)
+    return dest
+
+
+def list_backups(data_dir: Path) -> list[dict]:
+    bdir = backup_dir(data_dir)
+    if not bdir.exists():
+        return []
+    out = []
+    for p in sorted(bdir.glob("trovex-*.db"), reverse=True):
+        st = p.stat()
+        out.append({"name": p.name, "size_bytes": st.st_size, "mtime": st.st_mtime})
+    return out
+
+
+def prune(data_dir: Path, keep: int = KEEP) -> int:
+    backups = sorted(backup_dir(data_dir).glob("trovex-*.db"), reverse=True)
+    removed = 0
+    for p in backups[keep:]:
+        p.unlink()
+        removed += 1
+    return removed

trovex/boot.py

@@ -0,0 +1,51 @@
+"""Active-memory boot recall (RFC 330e7d43, step 2).
+
+Serves an agent its OWN recent records as a token-light pointer pack, scoped
+server-side: owner/<agent> + kind=record. Scope first, score second — global
+vector + an absolute threshold cross-injects; owner-scope yields precision≈1
+by construction. The pack is ~80 tokens (titles + ids, not bodies); the agent
+pulls a full record on demand via trovex_read(doc_id).
+"""
+
+from __future__ import annotations
+
+from .search import Searcher
+
+BOOT_QUERY = "current state resume open work in flight next steps gotchas"
+
+
+def boot_pointers(
+    searcher: Searcher,
+    agent: str,
+    *,
+    k: int = 5,
+    floor: float = 0.62,
+    q: str | None = None,
+) -> dict:
+    """The agent's own records as a pointer pack. Empty (zero cost) when nothing
+    clears scope + floor — a session for an unknown agent injects nothing."""
+    results = searcher.search(
+        q or BOOT_QUERY,
+        limit=k,
+        source_ids=["trovex"],
+        kind="record",
+        tags=[f"owner/{agent}"],
+    )
+    results = [r for r in results if r.score >= floor]
+    if not results:
+        return {"agent": agent, "pointers": [], "render": "", "tokens_est": 0}
+
+    pointers = [
+        {"id": r.path, "title": r.title, "score": round(r.score, 3)}
+        for r in results
+    ]
+    lines = [f"## Resume — {agent} (trovex active memory)"]
+    lines += [f"- {p['title']}  (trovex:{p['id']})" for p in pointers]
+    lines.append("Pull any with trovex_read(doc_id) for the full record.")
+    render = "\n".join(lines)
+    return {
+        "agent": agent,
+        "pointers": pointers,
+        "render": render,
+        "tokens_est": max(1, len(render) // 4),
+    }

trovex/cache.py

@@ -0,0 +1,75 @@
+"""Exact-match query cache for the trovex() tool.
+
+A repeat of the same query against an unchanged corpus skips the candidate
+search + the LLM reranker (the cost driver). Keyed on (normalized query,
+summary) + a corpus version derived from the docs table, so any trovex_write /
+delete invalidates stale entries automatically — no write-path hook needed.
+
+Exact match only: zero false-hit risk. Measured ~24% hit-rate on real traffic;
+a semantic layer (cosine ≥ τ over cached query embeddings) can sit on top later
+without changing this contract.
+"""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+import time
+
+_WS = re.compile(r"\s+")
+
+
+def _norm(q: str) -> str:
+    return _WS.sub(" ", q.strip().lower())
+
+
+def _key(q: str, summary: bool) -> str:
+    return f"{_norm(q)}|{int(summary)}"
+
+
+def _ensure(db: sqlite3.Connection) -> None:
+    db.execute(
+        """CREATE TABLE IF NOT EXISTS query_cache (
+            key            TEXT PRIMARY KEY,
+            corpus_version TEXT    NOT NULL,
+            output         TEXT    NOT NULL,
+            n_results      INTEGER NOT NULL,
+            whr            INTEGER NOT NULL,
+            top_tokens     INTEGER NOT NULL,
+            resp_tokens    INTEGER NOT NULL,
+            created_at     REAL    NOT NULL
+        )"""
+    )
+
+
+def corpus_version(db: sqlite3.Connection) -> str:
+    """Cheap version string; changes on any write (mtime bumps) or delete (count)."""
+    r = db.execute("SELECT COUNT(*) AS c, COALESCE(MAX(mtime), 0) AS m FROM docs").fetchone()
+    return f"{r['c']}:{r['m']}"
+
+
+def get(db: sqlite3.Connection, q: str, summary: bool, version: str) -> dict | None:
+    _ensure(db)
+    row = db.execute(
+        """SELECT output, n_results, whr, top_tokens, resp_tokens
+           FROM query_cache WHERE key = ? AND corpus_version = ?""",
+        (_key(q, summary), version),
+    ).fetchone()
+    return dict(row) if row else None
+
+
+def put(db: sqlite3.Connection, q: str, summary: bool, version: str, output: str,
+        n_results: int, whr: int, top_tokens: int, resp_tokens: int) -> None:
+    _ensure(db)
+    db.execute(
+        """INSERT INTO query_cache
+             (key, corpus_version, output, n_results, whr, top_tokens, resp_tokens, created_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+           ON CONFLICT(key) DO UPDATE SET
+             corpus_version = excluded.corpus_version, output = excluded.output,
+             n_results = excluded.n_results, whr = excluded.whr,
+             top_tokens = excluded.top_tokens, resp_tokens = excluded.resp_tokens,
+             created_at = excluded.created_at""",
+        (_key(q, summary), version, output, n_results, whr, top_tokens, resp_tokens, time.time()),
+    )
+    db.commit()

trovex/capture.py

@@ -0,0 +1,105 @@
+"""Active-memory capture (RFC 330e7d43, steps 3-4).
+
+Writes an agent's current-state record so the next /api/boot recalls FRESH state.
+Two paths, in increasing risk:
+
+- **free-summary** (step 3, frequent): PostCompact already distilled the
+  conversation — store that summary verbatim, NO LLM.
+- **transcript distil** (step 4, fallback for sessions with no compaction): an
+  LLM compresses the transcript. BYOK + best-effort (no key / error → no
+  capture, never raises). MERGES with the agent's prior state so a truncated
+  window doesn't lose earlier work (RFC residual bet #2: 24k → merge).
+
+Both upsert the deterministic doc ``owner-<agent>-current-state`` (owner/<agent>
++ kind=record + type/current-state) — stable id ⇒ in-place overwrite, one
+canonical record, no dup pile.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+from openai import OpenAI
+
+from .store import SqliteStore
+from .usage import current_openai_key, current_rerank_model
+
+log = logging.getLogger(__name__)
+
+DISTIL_MODEL = os.environ.get("TROVEX_DISTIL_MODEL", "gpt-5.4-mini")
+DISTIL_TIMEOUT_SEC = 20.0
+MAX_TRANSCRIPT_CHARS = 24000
+
+DISTIL_SYSTEM = (
+    "You compress one coding-agent session into a durable current-state record. "
+    "You are given the agent's PRIOR state (may be empty) and the RECENT session "
+    "transcript. Produce the UPDATED state as markdown with ONLY these sections, "
+    "omitting any that are empty:\n"
+    "### Done this session\n### In flight (verify/continue)\n### Gotchas (don't repeat)\n"
+    "### Next\n### Pointers (trovex ids / files)\n"
+    "Merge: carry forward still-relevant prior items, add new ones, drop done/stale. "
+    "Terse, facts only, no narration. If nothing durable, output exactly NO-SIGNAL."
+)
+
+
+def distil_summary(transcript: str, *, prior: str = "") -> str | None:
+    """LLM-distil a transcript into a current-state summary, merged with prior
+    state. BYOK + best-effort: no key, short input, or any error → None (caller
+    falls back). Never raises into the caller."""
+    key = current_openai_key.get()
+    transcript = (transcript or "").strip()
+    if not key or len(transcript) < 40:
+        return None
+    model = current_rerank_model.get() or DISTIL_MODEL
+    window = transcript[-MAX_TRANSCRIPT_CHARS:]
+    user = f"PRIOR STATE:\n{prior or '(none)'}\n\nRECENT TRANSCRIPT:\n{window}"
+    params: dict = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": DISTIL_SYSTEM},
+            {"role": "user", "content": user},
+        ],
+    }
+    if model.startswith(("gpt-5", "o1", "o3", "o4")):
+        params["max_completion_tokens"] = 2048
+    else:
+        params["max_tokens"] = 1024
+        params["temperature"] = 0
+    try:
+        client = OpenAI(api_key=key, timeout=DISTIL_TIMEOUT_SEC)
+        resp = client.chat.completions.create(**params)
+    except Exception:  # best-effort, never block the agent
+        log.warning("distil failed")
+        return None
+    md = (resp.choices[0].message.content or "").strip()
+    if md == "NO-SIGNAL" or len(md) < 40:
+        return None
+    return md
+
+
+def capture_state(
+    store: SqliteStore,
+    agent: str,
+    summary: str = "",
+    *,
+    transcript: str = "",
+    reason: str = "postcompact",
+) -> dict:
+    summary = (summary or "").strip()
+    # Free path takes the summary verbatim; transcript path distils (merging the
+    # existing record forward so truncation doesn't lose earlier state).
+    if not summary and transcript:
+        existing = store.get(f"owner-{agent}-current-state")
+        summary = distil_summary(transcript, prior=existing.content if existing else "") or ""
+    if len(summary) < 20:
+        return {"captured": False, "reason": "no durable signal"}
+    doc_id = f"owner-{agent}-current-state"
+    content = f"# {agent} — current state ({reason})\n\n{summary}"
+    store.put(
+        content,
+        kind="record",
+        ext_id=doc_id,
+        tags=[f"owner/{agent}", "type/current-state", f"capture/{reason}"],
+    )
+    return {"captured": True, "doc_id": doc_id, "tokens": max(1, len(content) // 4)}

trovex/chunking.py

@@ -0,0 +1,109 @@
+"""Structure-aware markdown chunking for chunk-level retrieval.
+
+The literature consensus (arXiv:2603.24556, 2606.00881) is that structure-aware
+chunking beats semantic/sliding-window at lower cost — and markdown gives us the
+structure for free. We split on headings, keep a heading breadcrumb per chunk,
+and resplit oversized sections by paragraph windows.
+
+Each chunk's embed text is *prefix-fused* with its breadcrumb ("title > h1 > h2")
+— the single biggest retrieval gain per arXiv:2510.24402, kept small ("seasoning",
+not a metadata dump) per the same line of work.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+FRONTMATTER_RE = re.compile(r"^---\s*\n.*?\n---\s*\n", re.DOTALL)
+HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*#*$")
+FENCE_RE = re.compile(r"^\s*```")
+
+DEFAULT_MAX_TOKENS = 450
+
+
+@dataclass
+class Chunk:
+    index: int
+    heading_path: list[str] = field(default_factory=list)
+    text: str = ""
+    tokens_est: int = 0
+
+    def breadcrumb(self, title: str = "") -> str:
+        parts = ([title] if title else []) + self.heading_path
+        return " > ".join(p for p in parts if p)
+
+    def embed_text(self, title: str = "") -> str:
+        """Prefix-fusion: breadcrumb + body — the text we actually embed."""
+        bc = self.breadcrumb(title)
+        return f"{bc}\n\n{self.text}" if bc else self.text
+
+
+def _est_tokens(text: str) -> int:
+    return len(text) // 4
+
+
+def _split_to_size(text: str, max_tokens: int) -> list[str]:
+    if _est_tokens(text) <= max_tokens:
+        return [text]
+    paras = re.split(r"\n\s*\n", text)
+    out: list[str] = []
+    cur: list[str] = []
+    cur_tok = 0
+    for p in paras:
+        pt = _est_tokens(p)
+        if cur and cur_tok + pt > max_tokens:
+            out.append("\n\n".join(cur))
+            cur, cur_tok = [], 0
+        cur.append(p)
+        cur_tok += pt
+        if cur_tok > max_tokens and len(cur) == 1:  # lone oversized paragraph
+            out.append("\n\n".join(cur))
+            cur, cur_tok = [], 0
+    if cur:
+        out.append("\n\n".join(cur))
+    return [o for o in (s.strip() for s in out) if o]
+
+
+def chunk_markdown(content: str, max_tokens: int = DEFAULT_MAX_TOKENS) -> list[Chunk]:
+    """Split markdown into structure-aware chunks with heading breadcrumbs."""
+    content = FRONTMATTER_RE.sub("", content)
+    lines = content.splitlines()
+
+    sections: list[tuple[list[str], str]] = []
+    stack: list[tuple[int, str]] = []  # (level, heading text)
+    path: list[str] = []
+    body: list[str] = []
+    in_fence = False
+
+    def flush() -> None:
+        text = "\n".join(body).strip()
+        if text:
+            sections.append((list(path), text))
+
+    for line in lines:
+        if FENCE_RE.match(line):
+            in_fence = not in_fence
+            body.append(line)
+            continue
+        m = None if in_fence else HEADING_RE.match(line)
+        if m:
+            flush()
+            body = []
+            level, htext = len(m.group(1)), m.group(2).strip()
+            while stack and stack[-1][0] >= level:
+                stack.pop()
+            stack.append((level, htext))
+            path = [t for _, t in stack]
+        else:
+            body.append(line)
+    flush()
+
+    chunks: list[Chunk] = []
+    for sec_path, text in sections:
+        for piece in _split_to_size(text, max_tokens):
+            chunks.append(Chunk(
+                index=len(chunks), heading_path=sec_path,
+                text=piece, tokens_est=_est_tokens(piece),
+            ))
+    return chunks