drydock-cli 3.0.136__tar.gz → 3.0.137__tar.gz

{drydock_cli-3.0.136/drydock_cli.egg-info → drydock_cli-3.0.137}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: drydock-cli
-Version: 3.0.136
+Version: 3.0.137
 Summary: Drydock — a local, provider-agnostic terminal coding agent for local LLMs
 Author: Frank Bobe III
 License-Expression: Apache-2.0

{drydock_cli-3.0.136 → drydock_cli-3.0.137}/drydock/graphrag.py

@@ -270,16 +270,16 @@ def _iter_text_files(paths: list[str]):
                         yield fp
 
 
-def _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip_sources):
-    """Chunk + extract + graph every text file under paths into the (mutable)
-    accumulators, skipping any source already present. Returns files added."""
-    added = 0
+def _iter_file_chunks(paths, cwd, skip_sources, progress=None):
+    """Yield (rel_source, [(body, entities), …]) for each ingestible file, one file
+    at a time (so callers never hold every file's text at once). Per-file isolation:
+    one unreadable/odd file (a locked Word doc, weird encoding, PDF quirk) is skipped,
+    not fatal. Calls progress(files_done, rel) after each file so a long build can
+    report instead of appearing frozen."""
     cleaned = [_unquote(p) for p in paths]
+    files_done = 0
     for fp in _iter_text_files([str(Path(cwd) / p) if not os.path.isabs(p) else p
                                 for p in cleaned]):
-        # Per-file isolation: ONE unreadable/odd file (a locked Word doc, a weird
-        # encoding, a PDF backend quirk) must never crash a whole-folder build —
-        # skip it and keep indexing the rest.
         try:
             rel = os.path.relpath(str(fp), cwd)
             if rel in skip_sources:
@@ -289,14 +289,25 @@ def _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip_sources):
             else:
                 text = fp.read_text("utf-8", "ignore")
             if not text or not text.strip():
-                continue  # unreadable / empty — skip cleanly
-            # Build this file's chunks locally; commit only if it ALL succeeds so a
-            # mid-file failure leaves no partial state and isn't miscounted.
+                continue
             local = [(body, extract_entities(body)) for body in _chunk_text(text)]
         except Exception:  # noqa: BLE001 — isolate a bad file, never abort the build
             continue
+        skip_sources.add(rel)
+        files_done += 1
+        if progress:
+            try:
+                progress(files_done, rel)
+            except Exception:  # noqa: BLE001 — a progress callback must never break a build
+                pass
+        yield rel, local
+
+
+def _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip_sources, progress=None):
+    """In-memory ingest into the accumulators (legacy JSON path — small indexes)."""
+    added = 0
+    for rel, local in _iter_file_chunks(paths, cwd, skip_sources, progress):
         added += 1
-        skip_sources.add(rel)  # don't double-ingest the same file in one call
         for body, ents in local:
             cid = len(chunks)
             chunks.append({"id": cid, "source": rel, "text": body, "entities": ents})
@@ -309,6 +320,56 @@ def _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip_sources):
     return added
 
 
+def _build_sqlite_stream(paths, store_path, cwd, skip_sources, progress=None, append=False):
+    """Streaming SQLite build: chunk text is inserted per-file (never held for the
+    whole corpus, so memory stays bounded even for a huge folder). Entities/edges
+    aggregate in memory (far smaller than the text) and are written at the end."""
+    sp = Path(store_path)
+    sp.parent.mkdir(parents=True, exist_ok=True)
+    if not append and sp.exists():
+        sp.unlink()
+    con = sqlite3.connect(str(sp))
+    entity_chunks: dict[str, set[int]] = defaultdict(set)
+    edges: dict[str, Counter] = defaultdict(Counter)
+    added = 0
+    try:
+        con.executescript(_SCHEMA)
+        # continue chunk ids after any existing rows (append mode)
+        row = con.execute("SELECT COALESCE(MAX(id), -1) FROM chunks").fetchone()
+        next_id = (row[0] if row else -1) + 1
+        if append:  # seed the in-memory graph with what's already stored
+            for r in con.execute("SELECT entity, chunk_id FROM entities"):
+                entity_chunks[r[0]].add(r[1])
+            for r in con.execute("SELECT a, b, weight FROM edges"):
+                edges[r[0]][r[1]] = r[2]
+            con.execute("DELETE FROM entities"); con.execute("DELETE FROM edges")
+        for rel, local in _iter_file_chunks(paths, cwd, skip_sources, progress):
+            added += 1
+            rows = []
+            for body, ents in local:
+                cid = next_id; next_id += 1
+                con.execute("INSERT INTO chunks(id,source,text) VALUES(?,?,?)", (cid, rel, body))
+                con.execute("INSERT INTO chunks_fts(rowid,text) VALUES(?,?)", (cid, body))
+                for e in ents:
+                    entity_chunks[e].add(cid)
+                for i, a in enumerate(ents):
+                    for b in ents[i + 1:]:
+                        edges[a][b] += 1; edges[b][a] += 1
+                rows.append(cid)
+            if added % 200 == 0:
+                con.commit()
+        con.executemany("INSERT INTO entities(entity,chunk_id) VALUES(?,?)",
+                        ((e, cid) for e, cids in entity_chunks.items() for cid in cids))
+        con.executemany("INSERT INTO edges(a,b,weight) VALUES(?,?,?)",
+                        ((a, b, w) for a, nbrs in edges.items() for b, w in nbrs.items()))
+        con.execute("INSERT OR REPLACE INTO meta(key,value) VALUES('version','2')")
+        con.commit()
+        return {"files": added, "chunks": next_id, "entities": len(entity_chunks),
+                "edges": sum(len(n) for n in edges.values()) // 2}
+    finally:
+        con.close()
+
+
 def _save(store_path, chunks, entity_chunks, edges, files_added):
     # SQLite by default (scalable). A caller can still target a .json path
     # explicitly (small/portable indexes, tests) and get the legacy format.
@@ -334,13 +395,17 @@ def _save(store_path, chunks, entity_chunks, edges, files_added):
     }
 
 
-def build_index(paths: list[str], store_path: str | Path, *, cwd: str = ".") -> dict:
+def build_index(paths: list[str], store_path: str | Path, *, cwd: str = ".",
+                progress=None) -> dict:
     """Build (or REBUILD from scratch) the knowledge graph from paths; persist to
-    store_path. Returns {files, chunks, entities, edges}."""
+    store_path. Returns {files, chunks, entities, edges}. `progress(files_done, src)`
+    is called per file (for a UI). SQLite stores stream to disk (bounded memory)."""
+    if _is_sqlite(store_path):
+        return _build_sqlite_stream(paths, store_path, cwd, set(), progress, append=False)
     chunks: list[dict] = []
     entity_chunks: dict[str, set[int]] = defaultdict(set)
     edges: dict[str, Counter] = defaultdict(Counter)
-    added = _ingest_files(paths, cwd, chunks, entity_chunks, edges, set())
+    added = _ingest_files(paths, cwd, chunks, entity_chunks, edges, set(), progress)
     return _save(store_path, chunks, entity_chunks, edges, added)
 
 
@@ -366,13 +431,17 @@ def _read_all(store_path) -> dict:
     return json.loads(Path(sp).read_text("utf-8"))
 
 
-def add_to_index(paths: list[str], store_path: str | Path, *, cwd: str = ".") -> dict:
+def add_to_index(paths: list[str], store_path: str | Path, *, cwd: str = ".",
+                 progress=None) -> dict:
     """Incrementally ADD documents to an existing index (build it if none yet).
     Files already indexed (by relative path) are skipped — clear+build to refresh
     changed files. Returns {files (added), chunks, entities, edges} totals."""
     sp = _resolve_store(store_path)
     if not sp.exists():
-        return build_index(paths, store_path, cwd=cwd)
+        return build_index(paths, store_path, cwd=cwd, progress=progress)
+    if _is_sqlite(sp):
+        skip = {r[0] for r in sqlite3.connect(str(sp)).execute("SELECT DISTINCT source FROM chunks")}
+        return _build_sqlite_stream(paths, sp, cwd, skip, progress, append=True)
     existing = _read_all(sp)
     chunks: list[dict] = list(existing.get("chunks", []))
     entity_chunks: dict[str, set[int]] = defaultdict(set)
@@ -382,7 +451,7 @@ def add_to_index(paths: list[str], store_path: str | Path, *, cwd: str = ".") ->
     for a, nbrs in existing.get("edges", {}).items():
         edges[a] = Counter(nbrs)
     skip = {c["source"] for c in chunks}
-    added = _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip)
+    added = _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip, progress)
     return _save(sp, chunks, entity_chunks, edges, added)
 
 

{drydock_cli-3.0.136 → drydock_cli-3.0.137}/drydock/tui/app.py

@@ -1069,6 +1069,44 @@ class DrydockApp(App):
                 "(recent turns are always preserved)."
             )
 
+    def _graphrag_build(self, sub: str, rest: str, store, cwd: str) -> None:
+        """Worker: build/add a knowledge base off the UI thread, streaming a
+        throttled progress line so a huge folder never looks frozen."""
+        from drydock import graphrag
+        import time as _t
+
+        last = [0.0]
+
+        def progress(files_done: int, src: str) -> None:
+            now = _t.monotonic()
+            if now - last[0] >= 1.0:          # throttle UI updates to ~1/s
+                last[0] = now
+                self.call_from_thread(
+                    self.query_one("#working", Static).update,
+                    f"  ⚓ indexing… {files_done} files ({src[-48:]})")
+
+        try:
+            fn = graphrag.build_index if sub == "build" else graphrag.add_to_index
+            stats = fn([rest], store, cwd=cwd, progress=progress)
+        except Exception as e:  # noqa: BLE001 — surface, never crash the TUI
+            self.call_from_thread(self.query_one("#working", Static).update, "")
+            self.call_from_thread(self._mount, ErrorMessage(f"graphrag {sub} failed: {e}"))
+            return
+        self.call_from_thread(self.query_one("#working", Static).update, "")
+        if sub == "add" and stats["files"] == 0:
+            self.call_from_thread(
+                self._info, f"No new documents under {rest} (already indexed, or no text found).")
+            return
+        if not stats["chunks"]:
+            self.call_from_thread(self._info, f"No text found under {rest}. Nothing was indexed.")
+            return
+        verb2 = "built" if sub == "build" else f"updated (+{stats['files']} new files)"
+        self.call_from_thread(
+            self._info,
+            f"✓ Knowledge base {verb2}: {stats['chunks']} chunks · "
+            f"{stats['entities']} entities · {stats['edges']} edges.\n"
+            f"Stored at {store}. The agent draws on it via the Knowledge tool.")
+
     def _cmd_graphrag(self, arg: str) -> None:
         """Build / inspect / clear the project's GraphRAG knowledge base. Once
         built, the agent retrieves from it via the read-only Knowledge tool."""
@@ -1085,25 +1123,11 @@ class DrydockApp(App):
                 self._info(f"usage: /graphrag {sub} <path>   (a file or directory of docs/code)")
                 return
             verb = "Rebuilding" if sub == "build" else "Ingesting into"
-            self._info(f"{verb} knowledge base from {rest} …")
-            try:
-                fn = graphrag.build_index if sub == "build" else graphrag.add_to_index
-                stats = fn([rest], store, cwd=cwd)
-            except Exception as e:  # noqa: BLE001 — surface, never crash the TUI
-                self._mount(ErrorMessage(f"graphrag {sub} failed: {e}"))
-                return
-            if sub == "add" and stats["files"] == 0:
-                self._info(f"No new documents under {rest} (already indexed, or no text found).")
-                return
-            if not stats["chunks"]:
-                self._info(f"No text found under {rest}. Nothing was indexed.")
-                return
-            verb2 = "built" if sub == "build" else f"updated (+{stats['files']} new files)"
-            self._info(
-                f"✓ Knowledge base {verb2}: {stats['chunks']} chunks · "
-                f"{stats['entities']} entities · {stats['edges']} edges.\n"
-                f"Stored at {store}. The agent draws on it via the Knowledge tool."
-            )
+            self._info(f"{verb} knowledge base from {rest} … (runs in the background; "
+                       "progress below — the TUI stays responsive)")
+            # Run OFF the UI thread so a large folder can't freeze the interface,
+            # and stream progress (files indexed) back so it never looks locked up.
+            self.run_worker(lambda: self._graphrag_build(sub, rest, store, cwd), thread=True)
         elif sub == "query":
             if not rest:
                 self._info("usage: /graphrag query <question>   (test what the KB returns)")

{drydock_cli-3.0.136 → drydock_cli-3.0.137/drydock_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: drydock-cli
-Version: 3.0.136
+Version: 3.0.137
 Summary: Drydock — a local, provider-agnostic terminal coding agent for local LLMs
 Author: Frank Bobe III
 License-Expression: Apache-2.0

{drydock_cli-3.0.136 → drydock_cli-3.0.137}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 # PyPI distribution name is drydock-cli (the established install name, continued
 # from the retired v2 fork); the import package + CLI command stay `drydock`.
 name = "drydock-cli"
-version = "3.0.136"
+version = "3.0.137"
 description = "Drydock — a local, provider-agnostic terminal coding agent for local LLMs"
 readme = "README.md"
 requires-python = ">=3.11"

{drydock_cli-3.0.136 → drydock_cli-3.0.137}/tests/test_graphrag_sqlite.py

@@ -68,3 +68,34 @@ def test_resolve_prefers_db_then_json():
 
 def test_missing_store_returns_none():
     assert g.load_index("/no/such/graphrag.db") is None
+
+
+def test_streaming_build_reports_progress():
+    """SQLite build streams to disk and calls progress per file (so the UI can show
+    it and never looks frozen)."""
+    d, docs = _docs()
+    open(os.path.join(docs, "c.md"), "w").write("# More\nExtra content about billing.")
+    seen = []
+    store = os.path.join(d, ".drydock", "graphrag.db")
+    stats = g.build_index([docs], store, cwd=d, progress=lambda n, s: seen.append((n, s)))
+    assert stats["files"] == 3 and len(seen) == 3
+    assert seen[-1][0] == 3                    # last progress reports 3 files done
+    # index is queryable
+    assert g.query_index(g.load_index(store), "billing", k=2)["chunks"]
+
+
+def test_streaming_append_skips_existing_and_reports():
+    d, docs = _docs()
+    store = os.path.join(d, ".drydock", "graphrag.db")
+    g.build_index([docs], store, cwd=d)
+    open(os.path.join(docs, "n.md"), "w").write("# New\nThe RefundBot processes claims.")
+    seen = []
+    s2 = g.add_to_index([docs], store, cwd=d, progress=lambda n, s: seen.append(s))
+    assert s2["files"] == 1 and seen == ["docs/n.md"]   # only the new file ingested
+    assert g.query_index(g.load_index(store), "refundbot claims", k=2)["chunks"]
+
+
+def test_build_without_progress_is_backward_compatible():
+    d, docs = _docs()
+    store = os.path.join(d, ".drydock", "graphrag.db")
+    assert g.build_index([docs], store, cwd=d)["files"] == 2   # no progress arg → still works