rc-docparser 0.2.0__py3-none-any.whl

docparser/__init__.py

@@ -0,0 +1,87 @@
+"""docparser - convert research literature into structured Markdown + JSON corpora.
+
+Public API
+----------
+``WorkspaceLayout``
+    Dataclass describing where raw / parsed / asset / cache files live. All
+    parsers accept a layout instead of relying on global paths, so the library
+    can be embedded in any project.
+``parse_docx``, ``parse_xlsx``, ``parse_pdf``, ``parse_html``
+    Per-format parsers. Each returns the structured payload and (by default)
+    writes ``document.md`` + ``document.json`` plus extracted assets into the
+    layout's ``parsed_dir_for(source)`` and ``assets_dir_for(source)``.
+``parse_path``
+    Dispatcher that picks a parser by file extension.
+``run_all``
+    Walk a directory and parse every supported file under it.
+``caption_image``
+    OpenRouter VLM helper, used by all parsers when a captioner is supplied.
+"""
+from __future__ import annotations
+
+__version__ = "0.2.0"
+
+from .common import (
+    WorkspaceLayout,
+    bytes_sha1,
+    file_sha1,
+    slugify,
+    truncate,
+    utc_now_iso,
+    write_json,
+    write_text,
+)
+from .csvtab import parse_csv
+from .docx import parse_docx
+from .orchestrator import SUPPORTED_EXTENSIONS, parse_path, run_all
+from .text import parse_text
+from .xlsx import parse_xlsx
+
+__all__ = [
+    "SUPPORTED_EXTENSIONS",
+    "WorkspaceLayout",
+    "__version__",
+    "bytes_sha1",
+    "caption_image",
+    "file_sha1",
+    "parse_csv",
+    "parse_docx",
+    "parse_epub",
+    "parse_html",
+    "parse_path",
+    "parse_pdf",
+    "parse_pptx",
+    "parse_text",
+    "parse_xlsx",
+    "run_all",
+    "slugify",
+    "truncate",
+    "utc_now_iso",
+    "write_json",
+    "write_text",
+]
+
+
+def __getattr__(name: str):
+    """Lazy-import optional-extra modules so missing deps surface a clear error."""
+    if name == "caption_image":
+        from .image import caption_image
+
+        return caption_image
+    if name == "parse_pdf":
+        from .pdf import parse_pdf
+
+        return parse_pdf
+    if name == "parse_html":
+        from .html import parse_html
+
+        return parse_html
+    if name == "parse_pptx":
+        from .pptx import parse_pptx
+
+        return parse_pptx
+    if name == "parse_epub":
+        from .epub import parse_epub
+
+        return parse_epub
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

docparser/cli.py

@@ -0,0 +1,209 @@
+"""docparser command-line interface.
+
+Subcommands
+-----------
+- ``docparser parse <FILE>``       parse a single file
+- ``docparser parse-all <DIR>``    walk a directory and parse everything
+- ``docparser version``            print package version
+
+The CLI uses argparse only (no extra deps) so it works whether or not the
+optional extras are installed.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__
+from .common import WorkspaceLayout
+from .orchestrator import SUPPORTED_EXTENSIONS, parse_path, run_all
+
+
+def _layout_from_args(args: argparse.Namespace) -> WorkspaceLayout:
+    if args.workspace:
+        layout = WorkspaceLayout.under(args.workspace)
+    else:
+        layout = WorkspaceLayout()
+    if getattr(args, "raw_dir", None):
+        layout.raw_dir = Path(args.raw_dir)
+    if getattr(args, "parsed_dir", None):
+        layout.parsed_dir = Path(args.parsed_dir)
+    if getattr(args, "assets_dir", None):
+        layout.assets_dir = Path(args.assets_dir)
+    if getattr(args, "cache_dir", None):
+        layout.cache_dir = Path(args.cache_dir)
+    return layout
+
+
+def _add_layout_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--workspace", help="Convenience: rooted layout under this dir.")
+    p.add_argument("--raw-dir", help="Override raw_dir (defaults to data/raw).")
+    p.add_argument("--parsed-dir", help="Override parsed_dir (defaults to data/parsed).")
+    p.add_argument("--assets-dir", help="Override assets_dir (defaults to data/assets).")
+    p.add_argument("--cache-dir", help="Override cache_dir (defaults to .cache).")
+
+
+def _add_vlm_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument(
+        "--no-vlm",
+        action="store_true",
+        help="Skip vision-language captioning of extracted images.",
+    )
+    p.add_argument(
+        "--max-images",
+        type=int,
+        default=None,
+        help="Cap total VLM calls per run (cached calls don't count).",
+    )
+    p.add_argument(
+        "--vlm-provider",
+        default=None,
+        help=(
+            "Captioning backend: openrouter (default), openai, gemini, local, "
+            "or transformers (fully local)."
+        ),
+    )
+    p.add_argument(
+        "--vlm-model",
+        default=None,
+        help="Override the captioning model for the chosen provider.",
+    )
+
+
+def _add_pdf_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument(
+        "--pdf-backend",
+        default="builtin",
+        choices=["builtin", "pymupdf4llm", "docling", "marker"],
+        help="High-fidelity PDF conversion backend (default: builtin).",
+    )
+    p.add_argument(
+        "--ocr",
+        default="off",
+        choices=["off", "auto", "force"],
+        help="OCR scanned/low-text PDF pages (requires the [ocr] extra).",
+    )
+    p.add_argument(
+        "--pdf-tables",
+        action="store_true",
+        help="Extract PDF tables with pdfplumber (requires the [tables] extra).",
+    )
+
+
+def cmd_parse(args: argparse.Namespace) -> int:
+    src = Path(args.source)
+    if not src.exists():
+        print(f"error: {src} does not exist", file=sys.stderr)
+        return 2
+    suffix = src.suffix.lower()
+    if suffix not in SUPPORTED_EXTENSIONS:
+        print(
+            f"error: unsupported extension {suffix!r}; supported: "
+            f"{sorted(SUPPORTED_EXTENSIONS)}",
+            file=sys.stderr,
+        )
+        return 2
+    layout = _layout_from_args(args)
+    layout.ensure()
+
+    captioner = None
+    if not args.no_vlm:
+        from .orchestrator import make_captioner
+
+        captioner = make_captioner(
+            enabled=True,
+            layout=layout,
+            max_calls=args.max_images,
+            provider=args.vlm_provider,
+            model=args.vlm_model,
+        )
+    pdf_kwargs: dict = {}
+    if suffix == ".pdf":
+        pdf_kwargs = {
+            "backend": args.pdf_backend,
+            "ocr": args.ocr,
+            "extract_tables": args.pdf_tables,
+        }
+    payload = parse_path(src, layout, captioner=captioner, **pdf_kwargs)
+    print(f"[docparser] {src.name}  stats={payload.get('stats', {})}")
+    print(f"[docparser] wrote {layout.parsed_dir_for(src) / 'document.md'}")
+    return 0
+
+
+def cmd_parse_all(args: argparse.Namespace) -> int:
+    layout = _layout_from_args(args)
+    if args.directory:
+        layout.raw_dir = Path(args.directory)
+    if not layout.raw_dir.exists():
+        print(f"error: {layout.raw_dir} does not exist", file=sys.stderr)
+        return 2
+    runs = run_all(
+        layout,
+        use_vlm=not args.no_vlm,
+        only=args.only,
+        max_images=args.max_images,
+        continue_on_error=args.continue_on_error,
+        vlm_provider=args.vlm_provider,
+        vlm_model=args.vlm_model,
+        pdf_backend=args.pdf_backend,
+        ocr=args.ocr,
+        extract_tables=args.pdf_tables,
+    )
+    print(f"[docparser] parsed {len(runs)} sources")
+    return 0
+
+
+def cmd_version(_args: argparse.Namespace) -> int:
+    print(__version__)
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="docparser",
+        description=(
+            "Convert .docx/.xlsx/.pdf/.html research literature into "
+            "Markdown + JSON corpora."
+        ),
+    )
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    p_parse = sub.add_parser("parse", help="Parse a single file.")
+    p_parse.add_argument(
+        "source",
+        help="Path to a supported file (.docx/.xlsx/.pdf/.html/.pptx/.epub/.txt/.md/.csv/.tsv).",
+    )
+    _add_layout_args(p_parse)
+    _add_vlm_args(p_parse)
+    _add_pdf_args(p_parse)
+    p_parse.set_defaults(func=cmd_parse)
+
+    p_all = sub.add_parser("parse-all", help="Walk a directory and parse every supported file.")
+    p_all.add_argument(
+        "directory",
+        nargs="?",
+        default=None,
+        help="Directory to walk. Defaults to the layout's raw_dir.",
+    )
+    p_all.add_argument("--only", default=None, help="Substring filter on filename.")
+    p_all.add_argument("--continue-on-error", action="store_true")
+    _add_layout_args(p_all)
+    _add_vlm_args(p_all)
+    _add_pdf_args(p_all)
+    p_all.set_defaults(func=cmd_parse_all)
+
+    p_ver = sub.add_parser("version", help="Print package version.")
+    p_ver.set_defaults(func=cmd_version)
+
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

docparser/common.py

@@ -0,0 +1,163 @@
+"""Shared utilities and the ``WorkspaceLayout`` dataclass.
+
+The original codebase pinned paths to ``REPO_ROOT``. For a library that's the
+wrong shape: callers want to choose where parsed output lands, where extracted
+assets go, and where the VLM cache lives. ``WorkspaceLayout`` captures those
+four directories explicitly and every parser accepts one.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import hashlib
+import json
+import os
+import re
+import unicodedata
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# slug / hash helpers (pure)
+
+
+_slug_re = re.compile(r"[^a-z0-9]+")
+
+
+def slugify(text: str, *, max_len: int = 80) -> str:
+    """ASCII slug suitable for filenames and IDs."""
+    if text is None:
+        return ""
+    norm = unicodedata.normalize("NFKD", str(text))
+    norm = norm.encode("ascii", "ignore").decode("ascii").lower()
+    norm = _slug_re.sub("-", norm).strip("-")
+    return norm[:max_len] or "untitled"
+
+
+def file_sha1(path: Path, *, chunk: int = 1 << 20) -> str:
+    h = hashlib.sha1()
+    with Path(path).open("rb") as fh:
+        while True:
+            buf = fh.read(chunk)
+            if not buf:
+                break
+            h.update(buf)
+    return h.hexdigest()
+
+
+def bytes_sha1(data: bytes) -> str:
+    return hashlib.sha1(data).hexdigest()
+
+
+def utc_now_iso() -> str:
+    return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _json_default(obj: Any) -> Any:
+    if isinstance(obj, (_dt.datetime, _dt.date, _dt.time)):
+        return obj.isoformat()
+    if isinstance(obj, bytes):
+        return obj.decode("utf-8", "replace")
+    if isinstance(obj, set):
+        return sorted(obj)
+    if hasattr(obj, "isoformat"):
+        return obj.isoformat()
+    return str(obj)
+
+
+def write_json(path: Path, payload: Any) -> None:
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with Path(path).open("w", encoding="utf-8") as fh:
+        json.dump(payload, fh, indent=2, ensure_ascii=False, default=_json_default)
+        fh.write("\n")
+
+
+def write_text(path: Path, text: str) -> None:
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with Path(path).open("w", encoding="utf-8") as fh:
+        fh.write(text)
+
+
+def chunked(seq: Iterable[Any], n: int) -> Iterable[list[Any]]:
+    buf: list[Any] = []
+    for item in seq:
+        buf.append(item)
+        if len(buf) >= n:
+            yield buf
+            buf = []
+    if buf:
+        yield buf
+
+
+def truncate(text: str, max_chars: int = 240) -> str:
+    if not text:
+        return ""
+    text = " ".join(text.split())
+    return text if len(text) <= max_chars else text[: max_chars - 1] + "\u2026"
+
+
+# ---------------------------------------------------------------------------
+# WorkspaceLayout
+
+
+@dataclass
+class WorkspaceLayout:
+    """Directory plan for parser output.
+
+    Parameters
+    ----------
+    raw_dir : Path
+        Where original source files live (or symlinks to them). Walked by
+        :func:`docparser.run_all`.
+    parsed_dir : Path
+        Where ``document.md`` and ``document.json`` are written (one folder
+        per source).
+    assets_dir : Path
+        Where extracted images go (one folder per source).
+    cache_dir : Path
+        On-disk cache for VLM responses (keyed by image SHA-1) and any other
+        caches the parsers want to reuse across runs.
+    """
+
+    raw_dir: Path = field(default_factory=lambda: Path("data/raw"))
+    parsed_dir: Path = field(default_factory=lambda: Path("data/parsed"))
+    assets_dir: Path = field(default_factory=lambda: Path("data/assets"))
+    cache_dir: Path = field(default_factory=lambda: Path(".cache"))
+
+    def __post_init__(self) -> None:
+        self.raw_dir = Path(self.raw_dir)
+        self.parsed_dir = Path(self.parsed_dir)
+        self.assets_dir = Path(self.assets_dir)
+        self.cache_dir = Path(self.cache_dir)
+
+    # convenience -----------------------------------------------------------
+
+    @classmethod
+    def under(cls, root: Path | str) -> WorkspaceLayout:
+        """Build a default layout rooted at ``root``."""
+        root = Path(root)
+        return cls(
+            raw_dir=root / "data" / "raw",
+            parsed_dir=root / "data" / "parsed",
+            assets_dir=root / "data" / "assets",
+            cache_dir=root / ".cache",
+        )
+
+    def ensure(self) -> None:
+        for d in (self.raw_dir, self.parsed_dir, self.assets_dir, self.cache_dir):
+            d.mkdir(parents=True, exist_ok=True)
+
+    def parsed_dir_for(self, source: Path | str) -> Path:
+        return self.parsed_dir / slugify(Path(source).stem)
+
+    def assets_dir_for(self, source: Path | str) -> Path:
+        return self.assets_dir / slugify(Path(source).stem)
+
+    def relpath_from_parsed(self, target: Path | str, source: Path | str) -> str:
+        """Markdown-friendly relative path from a source's parsed dir to ``target``."""
+        base = self.parsed_dir_for(source)
+        try:
+            return str(Path(target).resolve().relative_to(base.resolve()))
+        except ValueError:
+            return os.path.relpath(target, base)

docparser/csvtab.py

@@ -0,0 +1,131 @@
+"""CSV / TSV parser (core, stdlib ``csv``).
+
+Sniffs the delimiter, renders a Markdown table, and stores every row as JSON.
+The first row is treated as a header when it looks like one (all non-empty and
+not purely numeric); otherwise synthetic ``col1..colN`` headers are used.
+"""
+from __future__ import annotations
+
+import csv
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from .common import (
+    WorkspaceLayout,
+    file_sha1,
+    truncate,
+    utc_now_iso,
+    write_json,
+    write_text,
+)
+
+
+def _sniff_delimiter(sample: str, suffix: str) -> str:
+    if suffix == ".tsv":
+        return "\t"
+    try:
+        dialect = csv.Sniffer().sniff(sample, delimiters=",\t;|")
+        return dialect.delimiter
+    except csv.Error:
+        return "\t" if suffix == ".tsv" else ","
+
+
+def _looks_like_header(row: list[str]) -> bool:
+    if not row or any(not c.strip() for c in row):
+        return False
+    for c in row:
+        try:
+            float(c.replace(",", ""))
+            return False
+        except ValueError:
+            continue
+    return True
+
+
+def _md_cell(v: str) -> str:
+    return truncate(str(v).replace("|", "\\|").replace("\r", " ").replace("\n", " <br> "), 400)
+
+
+def parse_csv(
+    source: Path | str,
+    layout: WorkspaceLayout | None = None,
+    *,
+    captioner: Callable[..., dict[str, Any]] | None = None,
+    write_outputs: bool = True,
+) -> dict[str, Any]:
+    """Parse a ``.csv`` / ``.tsv`` file into Markdown + JSON."""
+    _ = captioner
+    source = Path(source)
+    layout = layout or WorkspaceLayout()
+    real_source = source.resolve()
+    suffix = source.suffix.lower()
+
+    text = real_source.read_text(encoding="utf-8", errors="replace")
+    delimiter = _sniff_delimiter(text[:4096], suffix)
+    reader = csv.reader(text.splitlines(), delimiter=delimiter)
+    rows: list[list[str]] = [list(r) for r in reader]
+
+    out_dir = layout.parsed_dir_for(source)
+    if write_outputs:
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+    ncols = max((len(r) for r in rows), default=0)
+    has_header = bool(rows) and _looks_like_header(rows[0])
+    if has_header:
+        header = [c.strip() for c in rows[0]] + [""] * (ncols - len(rows[0]))
+        body = rows[1:]
+    else:
+        header = [f"col{i + 1}" for i in range(ncols)]
+        body = rows
+
+    records: list[dict[str, str]] = []
+    for r in body:
+        padded = list(r) + [""] * (ncols - len(r))
+        records.append({header[i] or f"col{i + 1}": padded[i] for i in range(ncols)})
+
+    md_lines = [
+        f"# {source.stem}",
+        "",
+        f"> Source: `{source.name}` \u00b7 sha1 `{file_sha1(real_source)[:12]}` "
+        f"\u00b7 parsed `{utc_now_iso()}` \u00b7 {len(rows)} rows \u00d7 {ncols} cols",
+        "",
+    ]
+    if ncols:
+        md_lines.append("| " + " | ".join(_md_cell(h) for h in header) + " |")
+        md_lines.append("| " + " | ".join(["---"] * ncols) + " |")
+        for r in body:
+            padded = list(r) + [""] * (ncols - len(r))
+            md_lines.append("| " + " | ".join(_md_cell(c) for c in padded) + " |")
+        md_lines.append("")
+    else:
+        md_lines.append("_(empty file)_")
+        md_lines.append("")
+
+    md_text = "\n".join(md_lines).rstrip() + "\n"
+
+    json_payload = {
+        "source": {
+            "filename": source.name,
+            "absolute_path": str(real_source),
+            "sha1": file_sha1(real_source),
+            "size_bytes": real_source.stat().st_size,
+            "kind": "csv",
+        },
+        "parsed_at": utc_now_iso(),
+        "delimiter": delimiter,
+        "has_header": has_header,
+        "header": header,
+        "rows": records,
+        "stats": {
+            "n_rows": len(body),
+            "n_cols": ncols,
+            "has_header": has_header,
+        },
+    }
+
+    if write_outputs:
+        write_text(out_dir / "document.md", md_text)
+        write_json(out_dir / "document.json", json_payload)
+
+    return json_payload