drydock-cli 3.0.61__tar.gz → 3.0.62__tar.gz

{drydock_cli-3.0.61/drydock_cli.egg-info → drydock_cli-3.0.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: drydock-cli
-Version: 3.0.61
+Version: 3.0.62
 Summary: Drydock — a local, provider-agnostic terminal coding agent for local LLMs
 Author: Frank Bobe III
 License-Expression: Apache-2.0
@@ -18,6 +18,8 @@ Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-timeout; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pyright; extra == "dev"
+Provides-Extra: pdf
+Requires-Dist: pypdf>=4; extra == "pdf"
 Dynamic: license-file
 
 # ⚓ Drydock
@@ -103,9 +105,10 @@ Typed into the prompt. The agent also knows these, so you can just **ask it**
 
 Once built, the agent **automatically** retrieves from it (read-only `Knowledge`
 tool) when a question touches your material. Ingests text formats
-(`.md .txt .py .js .json .yaml .sql …`); convert PDFs/Word to text first
-(e.g. `pdftotext file.pdf file.txt`). The index is a single JSON at
-`<project>/.drydock/graphrag.json` — clean-room, stdlib-only, no embeddings.
+(`.md .txt .py .js .json .yaml .sql …`) **plus PDF and Word (`.docx`)**. `.docx`
+needs nothing extra; PDF uses the `pdftotext` binary (poppler) if present, else
+`pip install drydock-cli[pdf]` (pypdf). The index is a single JSON at
+`<project>/.drydock/graphrag.json` — clean-room, no embeddings.
 
 ### Custom skills
 

{drydock_cli-3.0.61 → drydock_cli-3.0.62}/README.md

@@ -81,9 +81,10 @@ Typed into the prompt. The agent also knows these, so you can just **ask it**
 
 Once built, the agent **automatically** retrieves from it (read-only `Knowledge`
 tool) when a question touches your material. Ingests text formats
-(`.md .txt .py .js .json .yaml .sql …`); convert PDFs/Word to text first
-(e.g. `pdftotext file.pdf file.txt`). The index is a single JSON at
-`<project>/.drydock/graphrag.json` — clean-room, stdlib-only, no embeddings.
+(`.md .txt .py .js .json .yaml .sql …`) **plus PDF and Word (`.docx`)**. `.docx`
+needs nothing extra; PDF uses the `pdftotext` binary (poppler) if present, else
+`pip install drydock-cli[pdf]` (pypdf). The index is a single JSON at
+`<project>/.drydock/graphrag.json` — clean-room, no embeddings.
 
 ### Custom skills
 

drydock_cli-3.0.62/drydock/extract.py

@@ -0,0 +1,94 @@
+"""Document text extraction for GraphRAG ingestion — PDF + Word (.docx).
+
+Kept clean-room and dependency-light:
+  • .docx is a zip of XML, so we read it with STDLIB only (zipfile + ElementTree)
+    — no dependency.
+  • .pdf uses whatever the system already has: the `pdftotext` binary (poppler)
+    if present, else the optional `pypdf` package (`pip install drydock-cli[pdf]`).
+
+Returns None when a document can't be read (unsupported, corrupt, or no PDF
+backend available) so ingestion skips it cleanly instead of failing.
+
+All logic original to Drydock.
+"""
+from __future__ import annotations
+
+import subprocess
+import xml.etree.ElementTree as ET
+import zipfile
+from pathlib import Path
+
+# Extensions we can pull text out of (beyond the plain-text formats GraphRAG
+# already reads directly).
+EXTRACTABLE_EXT = {".pdf", ".docx"}
+
+_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+
+
+def _extract_docx(path: str | Path) -> str | None:
+    """Pull paragraph text from a .docx (stdlib only). Each <w:p> is a paragraph;
+    its text lives in <w:t> runs."""
+    try:
+        with zipfile.ZipFile(path) as z:
+            xml = z.read("word/document.xml")
+    except (zipfile.BadZipFile, KeyError, OSError):
+        return None
+    try:
+        root = ET.fromstring(xml)
+    except ET.ParseError:
+        return None
+    paras: list[str] = []
+    for p in root.iter(f"{_W}p"):
+        runs = [t.text for t in p.iter(f"{_W}t") if t.text]
+        if runs:
+            paras.append("".join(runs))
+    text = "\n\n".join(paras).strip()
+    return text or None
+
+
+def _extract_pdf(path: str | Path) -> str | None:
+    """Extract text from a PDF: try the `pdftotext` binary first, then `pypdf`.
+    Returns None if neither is available or the PDF has no extractable text."""
+    # 1) pdftotext (poppler) — fast and common on Linux/macOS.
+    try:
+        r = subprocess.run(
+            ["pdftotext", "-q", str(path), "-"],
+            capture_output=True, text=True, timeout=120,
+        )
+        if r.returncode == 0 and r.stdout.strip():
+            return r.stdout
+    except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
+        pass
+    # 2) pypdf (optional dependency: drydock-cli[pdf]).
+    try:
+        import pypdf
+
+        reader = pypdf.PdfReader(str(path))
+        text = "\n\n".join((pg.extract_text() or "") for pg in reader.pages).strip()
+        return text or None
+    except Exception:  # noqa: BLE001 — optional dep missing / unreadable PDF
+        return None
+
+
+def extract_document(path: str | Path) -> str | None:
+    """Extract text from a supported document (.pdf/.docx), or None."""
+    ext = Path(path).suffix.lower()
+    if ext == ".docx":
+        return _extract_docx(path)
+    if ext == ".pdf":
+        return _extract_pdf(path)
+    return None
+
+
+def pdf_backend_available() -> bool:
+    """Whether a PDF can be extracted at all (pdftotext binary or pypdf)."""
+    from shutil import which
+
+    if which("pdftotext"):
+        return True
+    try:
+        import pypdf  # noqa: F401
+
+        return True
+    except ImportError:
+        return False

{drydock_cli-3.0.61 → drydock_cli-3.0.62}/drydock/graphrag.py

@@ -26,6 +26,8 @@ import json
 import os
 import re
 from collections import Counter, defaultdict
+
+from drydock import extract
 from pathlib import Path
 
 # Files we ingest as text. Everything else (binaries, images) is skipped.
@@ -106,7 +108,8 @@ def _iter_text_files(paths: list[str]):
                 dirs[:] = [d for d in dirs if d not in _SKIP_DIRS]
                 for f in files:
                     fp = Path(root) / f
-                    if fp.suffix.lower() in _TEXT_EXT or fp.suffix == "":
+                    ext = fp.suffix.lower()
+                    if ext in _TEXT_EXT or ext in extract.EXTRACTABLE_EXT or fp.suffix == "":
                         yield fp
 
 
@@ -119,10 +122,15 @@ def _ingest_files(paths, cwd, chunks, entity_chunks, edges, skip_sources):
         rel = os.path.relpath(str(fp), cwd)
         if rel in skip_sources:
             continue
-        try:
-            text = fp.read_text("utf-8", "ignore")
-        except OSError:
-            continue
+        if fp.suffix.lower() in extract.EXTRACTABLE_EXT:
+            text = extract.extract_document(fp)  # PDF/Word → text (or None)
+            if not text:
+                continue  # unreadable / no PDF backend — skip cleanly
+        else:
+            try:
+                text = fp.read_text("utf-8", "ignore")
+            except OSError:
+                continue
         if not text.strip():
             continue
         added += 1

{drydock_cli-3.0.61 → drydock_cli-3.0.62}/drydock/tuning.py

@@ -217,7 +217,7 @@ _DRYDOCK_COMMANDS_HELP = (
     "folder), `/graphrag add <path>` to add more, `/graphrag query <q>` to test "
     "it, `/graphrag status`, `/graphrag clear`. Once built, you automatically use "
     "the `Knowledge` tool to draw on it. It ingests text formats (md/txt/code/"
-    "json/yaml/…); PDFs/Word must be converted to text first.\n"
+    "json/yaml/…) plus PDF and Word (.docx) directly.\n"
     "- Custom skills (reusable `/<name>` prompts): `/skills new <name> <prompt "
     "text>` creates one (use $ARGS in the prompt for trailing input); `/skills` "
     "lists them; then they run it as `/<name>`.\n"

{drydock_cli-3.0.61 → drydock_cli-3.0.62/drydock_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: drydock-cli
-Version: 3.0.61
+Version: 3.0.62
 Summary: Drydock — a local, provider-agnostic terminal coding agent for local LLMs
 Author: Frank Bobe III
 License-Expression: Apache-2.0
@@ -18,6 +18,8 @@ Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-timeout; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pyright; extra == "dev"
+Provides-Extra: pdf
+Requires-Dist: pypdf>=4; extra == "pdf"
 Dynamic: license-file
 
 # ⚓ Drydock
@@ -103,9 +105,10 @@ Typed into the prompt. The agent also knows these, so you can just **ask it**
 
 Once built, the agent **automatically** retrieves from it (read-only `Knowledge`
 tool) when a question touches your material. Ingests text formats
-(`.md .txt .py .js .json .yaml .sql …`); convert PDFs/Word to text first
-(e.g. `pdftotext file.pdf file.txt`). The index is a single JSON at
-`<project>/.drydock/graphrag.json` — clean-room, stdlib-only, no embeddings.
+(`.md .txt .py .js .json .yaml .sql …`) **plus PDF and Word (`.docx`)**. `.docx`
+needs nothing extra; PDF uses the `pdftotext` binary (poppler) if present, else
+`pip install drydock-cli[pdf]` (pypdf). The index is a single JSON at
+`<project>/.drydock/graphrag.json` — clean-room, no embeddings.
 
 ### Custom skills
 

{drydock_cli-3.0.61 → drydock_cli-3.0.62}/drydock_cli.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ drydock/cli.py
 drydock/compaction.py
 drydock/config.py
 drydock/detect.py
+drydock/extract.py
 drydock/gittools.py
 drydock/graphrag.py
 drydock/guards.py
@@ -47,6 +48,7 @@ tests/test_context_limit_config.py
 tests/test_detect.py
 tests/test_dispatch.py
 tests/test_empty_response.py
+tests/test_extract.py
 tests/test_failure_loop.py
 tests/test_first_run_setup.py
 tests/test_gittools.py

{drydock_cli-3.0.61 → drydock_cli-3.0.62}/drydock_cli.egg-info/requires.txt

@@ -6,3 +6,6 @@ pytest
 pytest-timeout
 ruff
 pyright
+
+[pdf]
+pypdf>=4

{drydock_cli-3.0.61 → drydock_cli-3.0.62}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 # PyPI distribution name is drydock-cli (the established install name, continued
 # from the retired v2 fork); the import package + CLI command stay `drydock`.
 name = "drydock-cli"
-version = "3.0.61"
+version = "3.0.62"
 description = "Drydock — a local, provider-agnostic terminal coding agent for local LLMs"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -29,6 +29,9 @@ drydock = "drydock.cli:main"
 
 [project.optional-dependencies]
 dev = ["pytest", "pytest-timeout", "ruff", "pyright"]
+# PDF ingestion fallback when the `pdftotext` binary (poppler) isn't installed.
+# .docx needs nothing extra (stdlib zip/xml). `pip install drydock-cli[pdf]`.
+pdf = ["pypdf>=4"]
 
 [tool.ruff]
 target-version = "py312"

drydock_cli-3.0.62/tests/test_extract.py

@@ -0,0 +1,71 @@
+"""PDF + Word (.docx) document extraction for GraphRAG ingestion."""
+from __future__ import annotations
+
+import zipfile
+
+import pytest
+
+from drydock import extract, graphrag
+
+
+def _make_docx(path, text):
+    doc = ('<?xml version="1.0"?><w:document xmlns:w='
+           '"http://schemas.openxmlformats.org/wordprocessingml/2006/main">'
+           f'<w:body><w:p><w:r><w:t>{text}</w:t></w:r></w:p></w:body></w:document>')
+    with zipfile.ZipFile(path, "w") as z:
+        z.writestr("word/document.xml", doc)
+
+
+def _make_pdf(path, text):
+    objs = [b"<< /Type /Catalog /Pages 2 0 R >>",
+            b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+            b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R "
+            b"/Resources << /Font << /F1 5 0 R >> >> >>"]
+    stream = b"BT /F1 18 Tf 72 700 Td (" + text.encode() + b") Tj ET"
+    objs.append(b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream")
+    objs.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
+    out = b"%PDF-1.4\n"; offs = []
+    for i, o in enumerate(objs, 1):
+        offs.append(len(out)); out += str(i).encode() + b" 0 obj\n" + o + b"\nendobj\n"
+    x = len(out); out += b"xref\n0 " + str(len(objs) + 1).encode() + b"\n0000000000 65535 f \n"
+    for o in offs:
+        out += ("%010d 00000 n \n" % o).encode()
+    out += (b"trailer\n<< /Size " + str(len(objs) + 1).encode() + b" /Root 1 0 R >>\n"
+            b"startxref\n" + str(x).encode() + b"\n%%EOF")
+    path.write_bytes(out)
+
+
+def test_docx_extraction(tmp_path):
+    p = tmp_path / "d.docx"
+    _make_docx(p, "The Nimbus service uses the Falcon-Token header.")
+    assert extract.extract_document(p) == "The Nimbus service uses the Falcon-Token header."
+
+
+def test_pdf_extraction(tmp_path):
+    if not extract.pdf_backend_available():
+        pytest.skip("no PDF backend (pdftotext/pypdf) available")
+    p = tmp_path / "m.pdf"
+    _make_pdf(p, "Refunds via the Zephyr gateway.")
+    out = extract.extract_document(p)
+    assert out and "Zephyr gateway" in out
+
+
+def test_unsupported_and_corrupt_return_none(tmp_path):
+    assert extract.extract_document(tmp_path / "x.png") is None
+    bad = tmp_path / "bad.docx"; bad.write_text("not a zip")
+    assert extract.extract_document(bad) is None
+
+
+def test_graphrag_ingests_docx_and_pdf(tmp_path):
+    _make_docx(tmp_path / "spec.docx", "The Nimbus API uses the Falcon-Token header for auth.")
+    if extract.pdf_backend_available():
+        _make_pdf(tmp_path / "manual.pdf", "Refunds are issued via the Zephyr gateway.")
+    store = tmp_path / ".drydock" / "graphrag.json"
+    graphrag.build_index(["."], store, cwd=str(tmp_path))
+    idx = graphrag.load_index(store)
+    srcs = graphrag.sources(idx)
+    assert "spec.docx" in srcs
+    assert graphrag.query_index(idx, "Falcon-Token header")["chunks"][0]["source"] == "spec.docx"
+    if extract.pdf_backend_available():
+        assert "manual.pdf" in srcs
+        assert graphrag.query_index(idx, "refunds Zephyr gateway")["chunks"][0]["source"] == "manual.pdf"