drydock-cli 3.0.136__tar.gz → 3.0.138__tar.gz

{drydock_cli-3.0.136/drydock_cli.egg-info → drydock_cli-3.0.138}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: drydock-cli
-Version: 3.0.136
+Version: 3.0.138
 Summary: Drydock — a local, provider-agnostic terminal coding agent for local LLMs
 Author: Frank Bobe III
 License-Expression: Apache-2.0

{drydock_cli-3.0.136 → drydock_cli-3.0.138}/drydock/agent.py

@@ -383,6 +383,9 @@ def run(
                 state.task.phase = "complete"
             _emit(state, "done", phase=state.task.phase, edited=session_has_edited,
                   verified=bool(last_verification and last_verification.status == "pass"))
+            if config.get("trajectory_file"):
+                from drydock import trajectory
+                trajectory.record(system_prompt, state, config)
             break
 
         # Execute each tool call

{drydock_cli-3.0.136 → drydock_cli-3.0.138}/drydock/config.py

@@ -55,6 +55,10 @@ DEFAULTS: dict[str, object] = {
     # Durable per-session execution trace (events.py) written as JSONL to
     # ~/.drydock/events/ — inspect/diagnose/replay a task. Set false to disable.
     "event_log": True,
+    # Verified-trajectory export (RSI training-data collection). When set to a path,
+    # drydock writes the full task trajectory (system prompt + transcript) there at
+    # task end; a benchmark harness keeps only verifier-passing ones. Empty = off.
+    "trajectory_file": "",
     "theme": "harbor",
     # Optional SECOND model ("advisor") for a stronger second opinion — e.g. a
     # Gemini OpenAI-compatible endpoint on another box. Empty = disabled. The

{drydock_cli-3.0.136 → drydock_cli-3.0.138}/drydock/graphrag.py

@@ -270,16 +270,16 @@ def _iter_text_files(paths: list[str]):
                         yield fp
 
 
-def _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip_sources):
-    """Chunk + extract + graph every text file under paths into the (mutable)
-    accumulators, skipping any source already present. Returns files added."""
-    added = 0
+def _iter_file_chunks(paths, cwd, skip_sources, progress=None):
+    """Yield (rel_source, [(body, entities), …]) for each ingestible file, one file
+    at a time (so callers never hold every file's text at once). Per-file isolation:
+    one unreadable/odd file (a locked Word doc, weird encoding, PDF quirk) is skipped,
+    not fatal. Calls progress(files_done, rel) after each file so a long build can
+    report instead of appearing frozen."""
     cleaned = [_unquote(p) for p in paths]
+    files_done = 0
     for fp in _iter_text_files([str(Path(cwd) / p) if not os.path.isabs(p) else p
                                 for p in cleaned]):
-        # Per-file isolation: ONE unreadable/odd file (a locked Word doc, a weird
-        # encoding, a PDF backend quirk) must never crash a whole-folder build —
-        # skip it and keep indexing the rest.
         try:
             rel = os.path.relpath(str(fp), cwd)
             if rel in skip_sources:
@@ -289,14 +289,25 @@ def _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip_sources):
             else:
                 text = fp.read_text("utf-8", "ignore")
             if not text or not text.strip():
-                continue  # unreadable / empty — skip cleanly
-            # Build this file's chunks locally; commit only if it ALL succeeds so a
-            # mid-file failure leaves no partial state and isn't miscounted.
+                continue
             local = [(body, extract_entities(body)) for body in _chunk_text(text)]
         except Exception:  # noqa: BLE001 — isolate a bad file, never abort the build
             continue
+        skip_sources.add(rel)
+        files_done += 1
+        if progress:
+            try:
+                progress(files_done, rel)
+            except Exception:  # noqa: BLE001 — a progress callback must never break a build
+                pass
+        yield rel, local
+
+
+def _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip_sources, progress=None):
+    """In-memory ingest into the accumulators (legacy JSON path — small indexes)."""
+    added = 0
+    for rel, local in _iter_file_chunks(paths, cwd, skip_sources, progress):
         added += 1
-        skip_sources.add(rel)  # don't double-ingest the same file in one call
         for body, ents in local:
             cid = len(chunks)
             chunks.append({"id": cid, "source": rel, "text": body, "entities": ents})
@@ -309,6 +320,56 @@ def _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip_sources):
     return added
 
 
+def _build_sqlite_stream(paths, store_path, cwd, skip_sources, progress=None, append=False):
+    """Streaming SQLite build: chunk text is inserted per-file (never held for the
+    whole corpus, so memory stays bounded even for a huge folder). Entities/edges
+    aggregate in memory (far smaller than the text) and are written at the end."""
+    sp = Path(store_path)
+    sp.parent.mkdir(parents=True, exist_ok=True)
+    if not append and sp.exists():
+        sp.unlink()
+    con = sqlite3.connect(str(sp))
+    entity_chunks: dict[str, set[int]] = defaultdict(set)
+    edges: dict[str, Counter] = defaultdict(Counter)
+    added = 0
+    try:
+        con.executescript(_SCHEMA)
+        # continue chunk ids after any existing rows (append mode)
+        row = con.execute("SELECT COALESCE(MAX(id), -1) FROM chunks").fetchone()
+        next_id = (row[0] if row else -1) + 1
+        if append:  # seed the in-memory graph with what's already stored
+            for r in con.execute("SELECT entity, chunk_id FROM entities"):
+                entity_chunks[r[0]].add(r[1])
+            for r in con.execute("SELECT a, b, weight FROM edges"):
+                edges[r[0]][r[1]] = r[2]
+            con.execute("DELETE FROM entities"); con.execute("DELETE FROM edges")
+        for rel, local in _iter_file_chunks(paths, cwd, skip_sources, progress):
+            added += 1
+            rows = []
+            for body, ents in local:
+                cid = next_id; next_id += 1
+                con.execute("INSERT INTO chunks(id,source,text) VALUES(?,?,?)", (cid, rel, body))
+                con.execute("INSERT INTO chunks_fts(rowid,text) VALUES(?,?)", (cid, body))
+                for e in ents:
+                    entity_chunks[e].add(cid)
+                for i, a in enumerate(ents):
+                    for b in ents[i + 1:]:
+                        edges[a][b] += 1; edges[b][a] += 1
+                rows.append(cid)
+            if added % 200 == 0:
+                con.commit()
+        con.executemany("INSERT INTO entities(entity,chunk_id) VALUES(?,?)",
+                        ((e, cid) for e, cids in entity_chunks.items() for cid in cids))
+        con.executemany("INSERT INTO edges(a,b,weight) VALUES(?,?,?)",
+                        ((a, b, w) for a, nbrs in edges.items() for b, w in nbrs.items()))
+        con.execute("INSERT OR REPLACE INTO meta(key,value) VALUES('version','2')")
+        con.commit()
+        return {"files": added, "chunks": next_id, "entities": len(entity_chunks),
+                "edges": sum(len(n) for n in edges.values()) // 2}
+    finally:
+        con.close()
+
+
 def _save(store_path, chunks, entity_chunks, edges, files_added):
     # SQLite by default (scalable). A caller can still target a .json path
     # explicitly (small/portable indexes, tests) and get the legacy format.
@@ -334,13 +395,17 @@ def _save(store_path, chunks, entity_chunks, edges, files_added):
     }
 
 
-def build_index(paths: list[str], store_path: str | Path, *, cwd: str = ".") -> dict:
+def build_index(paths: list[str], store_path: str | Path, *, cwd: str = ".",
+                progress=None) -> dict:
     """Build (or REBUILD from scratch) the knowledge graph from paths; persist to
-    store_path. Returns {files, chunks, entities, edges}."""
+    store_path. Returns {files, chunks, entities, edges}. `progress(files_done, src)`
+    is called per file (for a UI). SQLite stores stream to disk (bounded memory)."""
+    if _is_sqlite(store_path):
+        return _build_sqlite_stream(paths, store_path, cwd, set(), progress, append=False)
     chunks: list[dict] = []
     entity_chunks: dict[str, set[int]] = defaultdict(set)
     edges: dict[str, Counter] = defaultdict(Counter)
-    added = _ingest_files(paths, cwd, chunks, entity_chunks, edges, set())
+    added = _ingest_files(paths, cwd, chunks, entity_chunks, edges, set(), progress)
     return _save(store_path, chunks, entity_chunks, edges, added)
 
 
@@ -366,13 +431,17 @@ def _read_all(store_path) -> dict:
     return json.loads(Path(sp).read_text("utf-8"))
 
 
-def add_to_index(paths: list[str], store_path: str | Path, *, cwd: str = ".") -> dict:
+def add_to_index(paths: list[str], store_path: str | Path, *, cwd: str = ".",
+                 progress=None) -> dict:
     """Incrementally ADD documents to an existing index (build it if none yet).
     Files already indexed (by relative path) are skipped — clear+build to refresh
     changed files. Returns {files (added), chunks, entities, edges} totals."""
     sp = _resolve_store(store_path)
     if not sp.exists():
-        return build_index(paths, store_path, cwd=cwd)
+        return build_index(paths, store_path, cwd=cwd, progress=progress)
+    if _is_sqlite(sp):
+        skip = {r[0] for r in sqlite3.connect(str(sp)).execute("SELECT DISTINCT source FROM chunks")}
+        return _build_sqlite_stream(paths, sp, cwd, skip, progress, append=True)
     existing = _read_all(sp)
     chunks: list[dict] = list(existing.get("chunks", []))
     entity_chunks: dict[str, set[int]] = defaultdict(set)
@@ -382,7 +451,7 @@ def add_to_index(paths: list[str], store_path: str | Path, *, cwd: str = ".") ->
     for a, nbrs in existing.get("edges", {}).items():
         edges[a] = Counter(nbrs)
     skip = {c["source"] for c in chunks}
-    added = _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip)
+    added = _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip, progress)
     return _save(sp, chunks, entity_chunks, edges, added)
 
 

drydock_cli-3.0.138/drydock/trajectory.py

@@ -0,0 +1,75 @@
+"""Verified-trajectory export — the training-data side of the RSI loop.
+
+When `trajectory_file` is set (a benchmark harness sets it per task), drydock writes
+the FULL task trajectory — system prompt + the complete message transcript (every
+assistant turn with its tool calls, every tool result) — to that file at task end.
+The harness then keeps ONLY the trajectories the deterministic verifier confirmed
+correct, so the resulting corpus is solutions that provably work (no self-grading).
+
+That corpus is the seed for self-distillation and the same format a stronger teacher
+model feeds into. Export is opt-in and fully defensive — it must never affect a run.
+All logic original to Drydock.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+_MAX_FIELD = 200_000  # cap any single message body so one huge tool dump can't bloat the record
+
+
+def _clip(v):
+    if isinstance(v, str) and len(v) > _MAX_FIELD:
+        return v[:_MAX_FIELD] + f"\n[… {len(v) - _MAX_FIELD} chars truncated]"
+    return v
+
+
+def _sanitize(messages: list) -> list:
+    """A JSON-safe copy of the transcript with oversized bodies clipped."""
+    out = []
+    for m in messages:
+        if not isinstance(m, dict):
+            continue
+        mm = {k: _clip(v) for k, v in m.items()}
+        out.append(mm)
+    return out
+
+
+def build_record(system_prompt: str, state, config: dict, meta: dict | None = None) -> dict:
+    """Assemble the trajectory record (does not write). Verified/reward are left None
+    for the harness to stamp after the verifier runs."""
+    task = getattr(state, "task", None)
+    tools = sorted({m.get("name", "") for m in state.messages
+                    if isinstance(m, dict) and m.get("role") == "tool" and m.get("name")})
+    return {
+        "task": (meta or {}).get("task", ""),
+        "model": config.get("model", ""),
+        "objective": getattr(task, "objective", "") if task else "",
+        "acceptance_criteria": list(getattr(task, "acceptance_criteria", []) or []) if task else [],
+        "phase": getattr(task, "phase", "") if task else "",
+        "system": system_prompt,
+        "messages": _sanitize(state.messages),
+        "n_messages": len(state.messages),
+        "tools": tools,
+        "in_tokens": getattr(state, "total_input_tokens", 0),
+        "out_tokens": getattr(state, "total_output_tokens", 0),
+        "verified": None,   # stamped True by the collector iff the verifier passed
+        "reward": None,
+    }
+
+
+def record(system_prompt: str, state, config: dict, meta: dict | None = None):
+    """Write the current full trajectory to config['trajectory_file'] (overwrite, so
+    it always holds the latest complete transcript). No-op if unset. Never raises."""
+    path = config.get("trajectory_file")
+    if not path:
+        return None
+    try:
+        p = Path(path)
+        if p.parent and not p.parent.exists():
+            p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_text(json.dumps(build_record(system_prompt, state, config, meta), default=str),
+                     encoding="utf-8")
+        return path
+    except Exception:  # noqa: BLE001 — export must never break a run
+        return None

{drydock_cli-3.0.136 → drydock_cli-3.0.138}/drydock/tui/app.py

@@ -1069,6 +1069,44 @@ class DrydockApp(App):
                 "(recent turns are always preserved)."
             )
 
+    def _graphrag_build(self, sub: str, rest: str, store, cwd: str) -> None:
+        """Worker: build/add a knowledge base off the UI thread, streaming a
+        throttled progress line so a huge folder never looks frozen."""
+        from drydock import graphrag
+        import time as _t
+
+        last = [0.0]
+
+        def progress(files_done: int, src: str) -> None:
+            now = _t.monotonic()
+            if now - last[0] >= 1.0:          # throttle UI updates to ~1/s
+                last[0] = now
+                self.call_from_thread(
+                    self.query_one("#working", Static).update,
+                    f"  ⚓ indexing… {files_done} files ({src[-48:]})")
+
+        try:
+            fn = graphrag.build_index if sub == "build" else graphrag.add_to_index
+            stats = fn([rest], store, cwd=cwd, progress=progress)
+        except Exception as e:  # noqa: BLE001 — surface, never crash the TUI
+            self.call_from_thread(self.query_one("#working", Static).update, "")
+            self.call_from_thread(self._mount, ErrorMessage(f"graphrag {sub} failed: {e}"))
+            return
+        self.call_from_thread(self.query_one("#working", Static).update, "")
+        if sub == "add" and stats["files"] == 0:
+            self.call_from_thread(
+                self._info, f"No new documents under {rest} (already indexed, or no text found).")
+            return
+        if not stats["chunks"]:
+            self.call_from_thread(self._info, f"No text found under {rest}. Nothing was indexed.")
+            return
+        verb2 = "built" if sub == "build" else f"updated (+{stats['files']} new files)"
+        self.call_from_thread(
+            self._info,
+            f"✓ Knowledge base {verb2}: {stats['chunks']} chunks · "
+            f"{stats['entities']} entities · {stats['edges']} edges.\n"
+            f"Stored at {store}. The agent draws on it via the Knowledge tool.")
+
     def _cmd_graphrag(self, arg: str) -> None:
         """Build / inspect / clear the project's GraphRAG knowledge base. Once
         built, the agent retrieves from it via the read-only Knowledge tool."""
@@ -1085,25 +1123,11 @@ class DrydockApp(App):
                 self._info(f"usage: /graphrag {sub} <path>   (a file or directory of docs/code)")
                 return
             verb = "Rebuilding" if sub == "build" else "Ingesting into"
-            self._info(f"{verb} knowledge base from {rest} …")
-            try:
-                fn = graphrag.build_index if sub == "build" else graphrag.add_to_index
-                stats = fn([rest], store, cwd=cwd)
-            except Exception as e:  # noqa: BLE001 — surface, never crash the TUI
-                self._mount(ErrorMessage(f"graphrag {sub} failed: {e}"))
-                return
-            if sub == "add" and stats["files"] == 0:
-                self._info(f"No new documents under {rest} (already indexed, or no text found).")
-                return
-            if not stats["chunks"]:
-                self._info(f"No text found under {rest}. Nothing was indexed.")
-                return
-            verb2 = "built" if sub == "build" else f"updated (+{stats['files']} new files)"
-            self._info(
-                f"✓ Knowledge base {verb2}: {stats['chunks']} chunks · "
-                f"{stats['entities']} entities · {stats['edges']} edges.\n"
-                f"Stored at {store}. The agent draws on it via the Knowledge tool."
-            )
+            self._info(f"{verb} knowledge base from {rest} … (runs in the background; "
+                       "progress below — the TUI stays responsive)")
+            # Run OFF the UI thread so a large folder can't freeze the interface,
+            # and stream progress (files indexed) back so it never looks locked up.
+            self.run_worker(lambda: self._graphrag_build(sub, rest, store, cwd), thread=True)
         elif sub == "query":
             if not rest:
                 self._info("usage: /graphrag query <question>   (test what the KB returns)")

{drydock_cli-3.0.136 → drydock_cli-3.0.138/drydock_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: drydock-cli
-Version: 3.0.136
+Version: 3.0.138
 Summary: Drydock — a local, provider-agnostic terminal coding agent for local LLMs
 Author: Frank Bobe III
 License-Expression: Apache-2.0

{drydock_cli-3.0.136 → drydock_cli-3.0.138}/drydock_cli.egg-info/SOURCES.txt

@@ -30,6 +30,7 @@ drydock/suggest.py
 drydock/task_state.py
 drydock/tool_registry.py
 drydock/tool_result.py
+drydock/trajectory.py
 drydock/tuning.py
 drydock/verification.py
 drydock/web.py
@@ -133,6 +134,7 @@ tests/test_tool_arg_coercion_more.py
 tests/test_tool_arg_parsing.py
 tests/test_tool_result.py
 tests/test_tools_undo.py
+tests/test_trajectory.py
 tests/test_tui.py
 tests/test_tuning.py
 tests/test_verification_gate.py

{drydock_cli-3.0.136 → drydock_cli-3.0.138}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 # PyPI distribution name is drydock-cli (the established install name, continued
 # from the retired v2 fork); the import package + CLI command stay `drydock`.
 name = "drydock-cli"
-version = "3.0.136"
+version = "3.0.138"
 description = "Drydock — a local, provider-agnostic terminal coding agent for local LLMs"
 readme = "README.md"
 requires-python = ">=3.11"

{drydock_cli-3.0.136 → drydock_cli-3.0.138}/tests/test_graphrag_sqlite.py

@@ -68,3 +68,34 @@ def test_resolve_prefers_db_then_json():
 
 def test_missing_store_returns_none():
     assert g.load_index("/no/such/graphrag.db") is None
+
+
+def test_streaming_build_reports_progress():
+    """SQLite build streams to disk and calls progress per file (so the UI can show
+    it and never looks frozen)."""
+    d, docs = _docs()
+    open(os.path.join(docs, "c.md"), "w").write("# More\nExtra content about billing.")
+    seen = []
+    store = os.path.join(d, ".drydock", "graphrag.db")
+    stats = g.build_index([docs], store, cwd=d, progress=lambda n, s: seen.append((n, s)))
+    assert stats["files"] == 3 and len(seen) == 3
+    assert seen[-1][0] == 3                    # last progress reports 3 files done
+    # index is queryable
+    assert g.query_index(g.load_index(store), "billing", k=2)["chunks"]
+
+
+def test_streaming_append_skips_existing_and_reports():
+    d, docs = _docs()
+    store = os.path.join(d, ".drydock", "graphrag.db")
+    g.build_index([docs], store, cwd=d)
+    open(os.path.join(docs, "n.md"), "w").write("# New\nThe RefundBot processes claims.")
+    seen = []
+    s2 = g.add_to_index([docs], store, cwd=d, progress=lambda n, s: seen.append(s))
+    assert s2["files"] == 1 and seen == ["docs/n.md"]   # only the new file ingested
+    assert g.query_index(g.load_index(store), "refundbot claims", k=2)["chunks"]
+
+
+def test_build_without_progress_is_backward_compatible():
+    d, docs = _docs()
+    store = os.path.join(d, ".drydock", "graphrag.db")
+    assert g.build_index([docs], store, cwd=d)["files"] == 2   # no progress arg → still works

drydock_cli-3.0.138/tests/test_trajectory.py

@@ -0,0 +1,58 @@
+"""Verified-trajectory export (RSI training-data collection): drydock writes the
+full task transcript when trajectory_file is set, and does nothing otherwise."""
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+
+from drydock import agent, trajectory
+from drydock.providers import AssistantTurn
+
+
+def _run(cfg, msg="Do the task.\n1. a criterion"):
+    turns = [
+        AssistantTurn("", [{"id": "1", "name": "Write",
+                            "input": {"file_path": "/tmp/t.py", "content": "x"}}], 10, 5),
+        AssistantTurn("", [{"id": "2", "name": "Bash", "input": {"command": "python /tmp/t.py"}}], 8, 4),
+        AssistantTurn("done, verified", [], 6, 3),
+    ]
+    i = {"n": 0}
+    def fake(model, system, messages, tool_schemas, config):
+        t = turns[min(i["n"], 2)]; i["n"] += 1; yield t
+    orig = agent.stream; agent.stream = fake
+    try:
+        st = agent.AgentState()
+        list(agent.run(msg, st, cfg, "SYSTEM"))
+        return st
+    finally:
+        agent.stream = orig
+
+
+def test_exports_full_trajectory_when_enabled():
+    tf = os.path.join(tempfile.mkdtemp(), "traj.json")
+    cfg = {"model": "gemma4", "context_limit": 65536, "max_tokens": 8192, "trajectory_file": tf}
+    _run(cfg)
+    rec = json.load(open(tf))
+    assert rec["system"] == "SYSTEM" or rec["system"].startswith("SYSTEM")
+    assert rec["objective"].startswith("Do the task")
+    assert rec["n_messages"] >= 4 and set(rec["tools"]) == {"Bash", "Write"}
+    assert rec["verified"] is None and rec["reward"] is None   # collector stamps these
+    assert [m["role"] for m in rec["messages"]][0] == "user"
+
+
+def test_off_by_default_no_file_no_crash():
+    st = _run({"model": "gemma4", "context_limit": 65536, "max_tokens": 8192})
+    assert st.messages   # ran fine; nothing exported
+
+
+def test_record_never_raises_on_bad_path():
+    st = agent.AgentState(); st.messages = [{"role": "user", "content": "x"}]
+    assert trajectory.record("s", st, {"trajectory_file": "/root/no/perm/x.json"}) is None
+
+
+def test_build_record_clips_huge_bodies():
+    st = agent.AgentState()
+    st.messages = [{"role": "tool", "name": "Bash", "content": "A" * 500_000}]
+    rec = trajectory.build_record("s", st, {"model": "m"})
+    assert len(rec["messages"][0]["content"]) < 250_000 and "truncated" in rec["messages"][0]["content"]