rc-docparser 0.2.0__py3-none-any.whl

docparser/xlsx.py

@@ -0,0 +1,319 @@
+"""XLSX parser: emits Markdown + JSON for every sheet/row/column.
+
+Preserves cell value, openpyxl ``data_type``, formula (separate pass with
+``data_only=False``), hyperlinks, comments, merged ranges, frozen panes, and
+embedded images.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import io
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from openpyxl import load_workbook
+from openpyxl.cell.cell import Cell
+from openpyxl.utils import get_column_letter
+from openpyxl.worksheet.worksheet import Worksheet
+
+from .common import (
+    WorkspaceLayout,
+    bytes_sha1,
+    file_sha1,
+    truncate,
+    utc_now_iso,
+    write_json,
+    write_text,
+)
+
+
+def _value_to_jsonable(v: Any) -> Any:
+    if v is None:
+        return None
+    if isinstance(v, bool):
+        return v
+    if isinstance(v, (int, float, str)):
+        return v
+    if isinstance(v, (_dt.datetime, _dt.date, _dt.time)):
+        return v.isoformat()
+    if isinstance(v, _dt.timedelta):
+        return v.total_seconds()
+    return str(v)
+
+
+def _cell_record(cell: Cell, formulas_ws: Worksheet | None) -> dict[str, Any]:
+    rec: dict[str, Any] = {
+        "addr": cell.coordinate,
+        "row": cell.row,
+        "col": cell.column,
+        "col_letter": get_column_letter(cell.column),
+        "value": _value_to_jsonable(cell.value),
+        "data_type": cell.data_type,
+    }
+    if cell.number_format and cell.number_format != "General":
+        rec["number_format"] = cell.number_format
+    if cell.hyperlink is not None:
+        target = getattr(cell.hyperlink, "target", None)
+        if target:
+            rec["hyperlink"] = target
+    if formulas_ws is not None:
+        try:
+            f_cell = formulas_ws[cell.coordinate]
+            f_val = f_cell.value
+            if isinstance(f_val, str) and f_val.startswith("="):
+                rec["formula"] = f_val
+        except Exception:
+            pass
+    if cell.comment is not None:
+        try:
+            rec["comment"] = str(cell.comment.text)
+        except Exception:
+            pass
+    return rec
+
+
+def _extract_images(
+    ws: Worksheet, asset_dir: Path, layout: WorkspaceLayout, source: Path, start_seq: int, write_outputs: bool
+) -> list[dict[str, Any]]:
+    images: list[dict[str, Any]] = []
+    seq = start_seq
+    for img in list(getattr(ws, "_images", []) or []):
+        try:
+            ref = img.ref
+            data: bytes | None = None
+            if hasattr(ref, "read"):
+                data = ref.read()
+                try:
+                    ref.seek(0)
+                except Exception:
+                    pass
+            elif isinstance(ref, (bytes, bytearray)):
+                data = bytes(ref)
+            else:
+                pil_img = getattr(img, "image", None)
+                if pil_img is not None:
+                    buf = io.BytesIO()
+                    pil_img.save(buf, format=pil_img.format or "PNG")
+                    data = buf.getvalue()
+            if not data:
+                continue
+            seq += 1
+            sha = bytes_sha1(data)
+            ext = (Path(getattr(img, "path", "") or "").suffix.lstrip(".") or "png").lower()
+            asset_name = f"img-{ws.title.replace('/', '_')}-{seq:03d}-{sha[:10]}.{ext}"
+            asset_path = asset_dir / asset_name
+            if write_outputs and not asset_path.exists():
+                asset_path.write_bytes(data)
+            anchor = getattr(img, "anchor", None)
+            anchor_cell = None
+            try:
+                if anchor and hasattr(anchor, "_from"):
+                    anchor_cell = (
+                        f"{get_column_letter(anchor._from.col + 1)}{anchor._from.row + 1}"
+                    )
+            except Exception:
+                anchor_cell = None
+            images.append(
+                {
+                    "seq": seq,
+                    "sha1": sha,
+                    "ext": ext,
+                    "asset_path": layout.relpath_from_parsed(asset_path, source),
+                    "abs_asset_path": str(asset_path),
+                    "sheet": ws.title,
+                    "anchor_cell": anchor_cell,
+                    "blob": data,
+                }
+            )
+        except Exception:
+            continue
+    return images
+
+
+def _md_cell(v: Any) -> str:
+    if v is None:
+        return ""
+    if isinstance(v, bool):
+        return "TRUE" if v else "FALSE"
+    s = str(v)
+    s = s.replace("|", "\\|").replace("\r", " ").replace("\n", " <br> ")
+    return truncate(s, 400)
+
+
+def parse_xlsx(
+    source: Path | str,
+    layout: WorkspaceLayout | None = None,
+    *,
+    captioner: Callable[..., dict[str, Any]] | None = None,
+    write_outputs: bool = True,
+) -> dict[str, Any]:
+    """Parse an XLSX workbook into Markdown + JSON.
+
+    See :func:`docparser.parse_docx` for parameter conventions.
+    """
+    source = Path(source)
+    layout = layout or WorkspaceLayout()
+    real_source = source.resolve()
+
+    wb_values = load_workbook(filename=str(real_source), data_only=True, read_only=False)
+    wb_formulas = load_workbook(filename=str(real_source), data_only=False, read_only=False)
+
+    out_dir = layout.parsed_dir_for(source)
+    asset_dir = layout.assets_dir_for(source)
+    if write_outputs:
+        out_dir.mkdir(parents=True, exist_ok=True)
+        asset_dir.mkdir(parents=True, exist_ok=True)
+
+    sheets_payload: list[dict[str, Any]] = []
+    md_lines: list[str] = [f"# {source.stem}", ""]
+    md_lines.append(
+        f"> Source: `{source.name}` \u00b7 sha1 `{file_sha1(real_source)[:12]}` "
+        f"\u00b7 parsed `{utc_now_iso()}` \u00b7 sheets: {len(wb_values.sheetnames)}"
+    )
+    md_lines.append("")
+    md_lines.append("## Sheets")
+    md_lines.append("")
+    for s in wb_values.sheetnames:
+        md_lines.append(f"- [{s}](#sheet-{s.lower().replace(' ', '-')})")
+    md_lines.append("")
+
+    image_seq = 0
+    for sheet_name in wb_values.sheetnames:
+        ws = wb_values[sheet_name]
+        ws_f = wb_formulas[sheet_name] if sheet_name in wb_formulas.sheetnames else None
+        max_row = ws.max_row or 0
+        max_col = ws.max_column or 0
+
+        rows_records: list[list[dict[str, Any]]] = []
+        any_value = False
+        for row in ws.iter_rows(min_row=1, max_row=max_row, max_col=max_col):
+            row_recs = []
+            for cell in row:
+                rec = _cell_record(cell, ws_f)
+                row_recs.append(rec)
+                if rec["value"] not in (None, "") or rec.get("formula"):
+                    any_value = True
+            rows_records.append(row_recs)
+
+        merged = [str(r) for r in (ws.merged_cells.ranges or [])]
+        frozen = ws.freeze_panes
+        sheet_images = _extract_images(ws, asset_dir, layout, source, image_seq, write_outputs)
+        image_seq = max([image_seq] + [im["seq"] for im in sheet_images])
+
+        sheet_payload = {
+            "name": sheet_name,
+            "max_row": max_row,
+            "max_col": max_col,
+            "frozen_panes": frozen,
+            "merged_ranges": merged,
+            "n_nonempty_cells": sum(
+                1 for row in rows_records for c in row if c["value"] not in (None, "")
+            ),
+            "rows": rows_records,
+            "images": [{k: v for k, v in im.items() if k != "blob"} for im in sheet_images],
+        }
+        sheets_payload.append(sheet_payload)
+
+        md_lines.append(f"## Sheet: {sheet_name}")
+        md_lines.append("")
+        md_lines.append(
+            f"_dimensions:_ {max_row} rows \u00d7 {max_col} cols  "
+            f"\u00b7 _frozen:_ `{frozen or 'none'}`  "
+            f"\u00b7 _merged ranges:_ {len(merged)}  "
+            f"\u00b7 _images:_ {len(sheet_images)}"
+        )
+        md_lines.append("")
+        if not any_value and not sheet_images:
+            md_lines.append("_(empty sheet)_")
+            md_lines.append("")
+            continue
+
+        header_idx = 0
+        for i, row in enumerate(rows_records):
+            if any(c["value"] not in (None, "") for c in row):
+                header_idx = i
+                break
+        header = rows_records[header_idx] if rows_records else []
+        body = rows_records[header_idx + 1 :]
+
+        ncols = max_col
+        header_cells = [
+            _md_cell(header[c]["value"] if c < len(header) else "") for c in range(ncols)
+        ]
+        if not any(h.strip() for h in header_cells):
+            header_cells = [get_column_letter(c + 1) for c in range(ncols)]
+        md_lines.append("| " + " | ".join(header_cells) + " |")
+        md_lines.append("| " + " | ".join(["---"] * ncols) + " |")
+        for row in body:
+            cells = []
+            for c in range(ncols):
+                v = row[c]["value"] if c < len(row) else ""
+                f = row[c].get("formula") if c < len(row) else None
+                cells.append(_md_cell(f if f and (v is None or v == "") else v))
+            md_lines.append("| " + " | ".join(cells) + " |")
+        md_lines.append("")
+
+        if sheet_images:
+            md_lines.append(f"### Images in `{sheet_name}`")
+            md_lines.append("")
+            for im in sheet_images:
+                rel = im["asset_path"]
+                cap = None
+                if captioner is not None:
+                    try:
+                        cap = captioner(
+                            image_bytes=im["blob"],
+                            mime="image/" + (im["ext"] if im["ext"] != "jpg" else "jpeg"),
+                            doc_name=f"{source.name} :: sheet {sheet_name}",
+                            nearby_caption=f"Anchor cell: {im.get('anchor_cell') or 'unknown'}",
+                            context=f"Workbook image inside sheet '{sheet_name}'.",
+                        )
+                    except Exception as exc:
+                        cap = {"error": str(exc)}
+                im["semantic"] = cap
+                alt = (cap or {}).get("caption") or f"image-{im['seq']}"
+                md_lines.append(f"![{alt}]({rel})")
+                if cap and cap.get("description"):
+                    md_lines.append("")
+                    md_lines.append(f"> **VLM caption.** {cap.get('caption','')}")
+                    md_lines.append(">")
+                    md_lines.append(f"> {cap.get('description','')}")
+                    if cap.get("visible_text"):
+                        md_lines.append(">")
+                        vt = cap["visible_text"].replace("\n", "\n> ")
+                        md_lines.append(f"> *Visible text:* {vt}")
+                    if cap.get("tags"):
+                        md_lines.append(">")
+                        md_lines.append("> *Tags:* " + ", ".join(cap["tags"]))
+                md_lines.append("")
+            sheet_payload["images"] = [
+                {k: v for k, v in im.items() if k != "blob"} for im in sheet_images
+            ]
+
+    md_text = "\n".join(md_lines).rstrip() + "\n"
+
+    json_payload = {
+        "source": {
+            "filename": source.name,
+            "absolute_path": str(real_source),
+            "sha1": file_sha1(real_source),
+            "size_bytes": real_source.stat().st_size,
+            "kind": "xlsx",
+        },
+        "parsed_at": utc_now_iso(),
+        "sheet_names": list(wb_values.sheetnames),
+        "sheets": sheets_payload,
+        "stats": {
+            "n_sheets": len(sheets_payload),
+            "n_rows_total": sum(s["max_row"] for s in sheets_payload),
+            "n_nonempty_cells_total": sum(s["n_nonempty_cells"] for s in sheets_payload),
+            "n_images_total": sum(len(s["images"]) for s in sheets_payload),
+        },
+    }
+
+    if write_outputs:
+        write_text(out_dir / "document.md", md_text)
+        write_json(out_dir / "document.json", json_payload)
+
+    return json_payload

rc_docparser-0.2.0.dist-info/METADATA

@@ -0,0 +1,344 @@
+Metadata-Version: 2.4
+Name: rc-docparser
+Version: 0.2.0
+Summary: Convert research literature (.docx, .xlsx, .pdf, .html, .pptx, .epub, .txt, .md, .csv) into structured Markdown + JSON corpora, with optional VLM image semantic captioning.
+Project-URL: Homepage, https://github.com/Research-Commons/docparser
+Project-URL: Repository, https://github.com/Research-Commons/docparser
+Project-URL: Issues, https://github.com/Research-Commons/docparser/issues
+Project-URL: Changelog, https://github.com/Research-Commons/docparser/blob/main/CHANGELOG.md
+Author-email: Research Commons <shubhankitsingh@researchcommons.ai>
+License: MIT License
+        
+        Copyright (c) 2026 Research Commons
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: corpus,csv,docx,epub,html,literature,markdown,ocr,parser,pdf,pptx,rag,vlm,xlsx
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Text Processing :: Markup
+Requires-Python: >=3.10
+Requires-Dist: lxml>=5.3.0
+Requires-Dist: openpyxl>=3.1.5
+Requires-Dist: pillow>=10.4.0
+Requires-Dist: python-docx>=1.1.2
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: tqdm>=4.66.5
+Provides-Extra: all
+Requires-Dist: beautifulsoup4>=4.12.0; extra == 'all'
+Requires-Dist: ebooklib>=0.18; extra == 'all'
+Requires-Dist: numpy>=1.24.0; extra == 'all'
+Requires-Dist: pdfplumber>=0.11.0; extra == 'all'
+Requires-Dist: pymupdf>=1.24.0; extra == 'all'
+Requires-Dist: python-pptx>=1.0.0; extra == 'all'
+Requires-Dist: rapidocr-onnxruntime>=1.3.0; extra == 'all'
+Requires-Dist: requests>=2.32.3; extra == 'all'
+Requires-Dist: trafilatura>=1.12.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: build>=1.2.0; extra == 'dev'
+Requires-Dist: ebooklib>=0.18; extra == 'dev'
+Requires-Dist: mypy>=1.10.0; extra == 'dev'
+Requires-Dist: pandas>=2.2.0; extra == 'dev'
+Requires-Dist: pdfplumber>=0.11.0; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: python-pptx>=1.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.6.0; extra == 'dev'
+Requires-Dist: twine>=5.1.0; extra == 'dev'
+Provides-Extra: docling
+Requires-Dist: docling>=2.0.0; extra == 'docling'
+Provides-Extra: epub
+Requires-Dist: beautifulsoup4>=4.12.0; extra == 'epub'
+Requires-Dist: ebooklib>=0.18; extra == 'epub'
+Provides-Extra: html
+Requires-Dist: beautifulsoup4>=4.12.0; extra == 'html'
+Requires-Dist: trafilatura>=1.12.0; extra == 'html'
+Provides-Extra: localvlm
+Requires-Dist: pillow>=10.4.0; extra == 'localvlm'
+Requires-Dist: torch>=2.2.0; extra == 'localvlm'
+Requires-Dist: transformers>=4.40.0; extra == 'localvlm'
+Provides-Extra: marker
+Requires-Dist: marker-pdf>=1.0.0; extra == 'marker'
+Provides-Extra: ocr
+Requires-Dist: numpy>=1.24.0; extra == 'ocr'
+Requires-Dist: rapidocr-onnxruntime>=1.3.0; extra == 'ocr'
+Provides-Extra: pdf
+Requires-Dist: pymupdf>=1.24.0; extra == 'pdf'
+Provides-Extra: pptx
+Requires-Dist: python-pptx>=1.0.0; extra == 'pptx'
+Provides-Extra: pymupdf4llm
+Requires-Dist: pymupdf4llm>=0.0.17; extra == 'pymupdf4llm'
+Provides-Extra: tables
+Requires-Dist: pdfplumber>=0.11.0; extra == 'tables'
+Provides-Extra: vlm
+Requires-Dist: requests>=2.32.3; extra == 'vlm'
+Description-Content-Type: text/markdown
+
+# docparser
+
+Convert research literature (`.docx`, `.xlsx`, `.pdf`, `.html`, `.pptx`,
+`.epub`, `.txt`, `.md`, `.csv`) into a clean, reproducible **Markdown + JSON**
+corpus, with optional **vision-language captioning** of every embedded figure
+via OpenRouter, OpenAI, Gemini, a local server, or a fully-local model.
+
+```text
+                              ┌────────────────┐
+  data/raw/*.docx             │   docparser    │       data/parsed/<slug>/
+  data/raw/*.xlsx             │  - parse_docx  │       document.md
+  data/raw/*.pdf      ─────►  │  - parse_xlsx  │ ────► document.json
+  data/raw/*.html             │  - parse_pdf   │
+  data/raw/*.pptx             │  - parse_html  │       data/assets/<slug>/
+  data/raw/*.epub             │  - parse_pptx  │       img-*.png
+  data/raw/*.txt|md|csv       │  - parse_epub  │
+                              │  - VLM caption │
+                              └────────────────┘
+```
+
+## Install
+
+```bash
+pip install docparser              # core: docx + xlsx + txt/md + csv/tsv
+pip install 'docparser[pdf]'       # + PyMuPDF for PDFs
+pip install 'docparser[html]'      # + trafilatura + bs4 for HTML
+pip install 'docparser[pptx]'      # + python-pptx for PowerPoint
+pip install 'docparser[epub]'      # + EbookLib + bs4 for EPUB
+pip install 'docparser[vlm]'       # + requests for API VLM captions
+pip install 'docparser[all]'       # everything above (recommended)
+```
+
+Higher-fidelity / heavier features are separate opt-in extras (so the core
+install stays small and MIT):
+
+```bash
+pip install 'docparser[tables]'       # + pdfplumber for PDF table extraction
+pip install 'docparser[ocr]'          # + rapidocr-onnxruntime for scanned PDFs
+pip install 'docparser[pymupdf4llm]'  # PyMuPDF4LLM PDF backend (AGPL/commercial)
+pip install 'docparser[docling]'      # IBM Docling PDF backend (MIT)
+pip install 'docparser[marker]'       # Datalab Marker PDF backend (GPL-3.0)
+pip install 'docparser[localvlm]'     # transformers/torch local captioning
+```
+
+`docparser` requires Python 3.10+.
+
+## Quick start (library)
+
+```python
+from docparser import WorkspaceLayout, run_all
+
+layout = WorkspaceLayout.under("./project")   # data/raw, data/parsed, data/assets, .cache
+layout.ensure()
+
+run_all(layout, use_vlm=False)                # parse everything in data/raw
+```
+
+For a single file:
+
+```python
+from docparser import parse_path, WorkspaceLayout
+
+layout = WorkspaceLayout.under(".")
+payload = parse_path("paper.pdf", layout)
+print(payload["stats"])
+```
+
+## Quick start (CLI)
+
+```bash
+# parse a single file
+docparser parse paper.pdf --workspace ./out --no-vlm
+
+# walk a whole directory
+docparser parse-all --workspace ./project --no-vlm
+
+# enable VLM captioning (requires OPENROUTER_API_KEY in env or .env)
+export OPENROUTER_API_KEY=sk-or-v1-...
+docparser parse-all --workspace ./project --max-images 50
+
+# higher-fidelity PDF: pick a backend, OCR scanned pages, extract tables
+docparser parse paper.pdf --pdf-backend docling --ocr auto --pdf-tables --no-vlm
+
+# caption with a different provider
+docparser parse-all --workspace ./project --vlm-provider openai --vlm-model gpt-4o-mini
+
+docparser version
+```
+
+## What gets captured
+
+### `.docx`
+- Walks the document body in **document order** (paragraphs + tables + drawings).
+- Preserves heading hierarchy (`section_path`) on every block.
+- Extracts every embedded image to `data/assets/<slug>/` with a stable
+  `img-<seq>-<sha10>.<ext>` name.
+- Detects figure/table captions (style `Caption` or text matching
+  `Figure 1: …` / `Fig. 1.` / `Table 1.`) and associates the caption with the
+  preceding image.
+- Captures `context_before` and `context_after` for every image so the VLM has
+  document-grounded context.
+
+### `.xlsx`
+- Iterates **every sheet, every row, every column**.
+- For each cell stores: address, row/col indices, value, openpyxl `data_type`,
+  `number_format`, `hyperlink`, `comment`, and the **formula** (from a second
+  pass with `data_only=False`).
+- Stores `merged_ranges`, `frozen_panes`, and any embedded images.
+- Markdown rendering uses the first non-empty row as a header heuristic and
+  preserves multi-line cells with `<br>`.
+
+### `.pdf`
+- Page-by-page text extraction in reading order via PyMuPDF's blocks API.
+- Best-effort heading detection from font size (≥120% of the body-text median
+  promotes a line to a heading; bold flag tracked).
+- Embedded raster images extracted via `doc.extract_image(xref)`.
+- **Pluggable backends** (`backend="pymupdf4llm" | "docling" | "marker"`) route
+  conversion to a higher-fidelity engine; their Markdown is normalized into the
+  same block schema. Images are still extracted via PyMuPDF.
+- **OCR** (`ocr="auto" | "force"`, `[ocr]` extra) recognizes text on scanned /
+  low-text pages; OCR'd blocks carry `"ocr": true`.
+- **Tables** (`extract_tables=True`, `[tables]` extra) emit real `table` blocks
+  via `pdfplumber`.
+
+### `.html`
+- Article-grade body extraction via `trafilatura`.
+- Plus a **structural** BeautifulSoup walk that emits typed blocks
+  (`heading` / `paragraph` / `list` / `table` / `image`) so downstream RAG
+  layers can rely on the JSON.
+
+### `.pptx`
+- Walks slides in presentation order; each slide becomes a section.
+- Emits per-slide headings (slide title), bulleted text frames (with list
+  level), tables, pictures, and **speaker notes**.
+- Embedded pictures extracted and optionally captioned.
+
+### `.epub`
+- Walks the spine in reading order; per-chapter BeautifulSoup structural walk.
+- Captures metadata (title/author/language), headings, paragraphs, lists,
+  tables, and embedded images (resolved from the EPUB image manifest).
+
+### `.txt` / `.md` and `.csv` / `.tsv` (core, no extras)
+- Plain text is split into paragraph blocks; Markdown is passed through and
+  also decomposed into heading / list / code / paragraph blocks.
+- CSV/TSV: delimiter sniffing, header detection, a Markdown table, and one JSON
+  record per row.
+
+### Images (`[vlm]` extra)
+Each image is sent to a vision-language model (default provider OpenRouter,
+model `anthropic/claude-sonnet-4`) along with its surrounding caption +
+context. Any OpenAI-compatible provider works via `--vlm-provider`
+(`openrouter` / `openai` / `gemini` / `local`), or use a fully-local
+`transformers` model with `--vlm-provider transformers` (`[localvlm]` extra).
+The model returns a strict JSON object:
+
+```json
+{
+  "caption":           "one-sentence figure caption",
+  "description":       "2–5 sentence paragraph",
+  "visible_text":      "OCR-style transcription",
+  "tags":              ["world-model", "diagram", "..."],
+  "image_kind":        "diagram | plot | screenshot | photo | equation | table | ...",
+  "domain_relevance":  "how this relates to the document's topic"
+}
+```
+
+Results are cached on disk at `<cache_dir>/vlm/<model>/<sha1>.json`, keyed by
+**SHA-1 of the image bytes × model**, so re-runs are free until the source
+image bytes change.
+
+## Configuration (`.env`)
+
+| Var | Default | Purpose |
+| --- | --- | --- |
+| `DOCPARSER_VLM_PROVIDER` | `openrouter` | `openrouter` / `openai` / `gemini` / `local` |
+| `OPENROUTER_API_KEY` | _required for OpenRouter_ | OpenRouter key (`sk-or-...`) |
+| `OPENROUTER_VLM_MODEL` | `anthropic/claude-sonnet-4` | any vision-capable OpenRouter model |
+| `OPENROUTER_BASE_URL` | `https://openrouter.ai/api/v1` | override for a proxy |
+| `OPENROUTER_REFERER` / `OPENROUTER_TITLE` | repo URL / `docparser` | OpenRouter attribution headers |
+| `OPENAI_API_KEY` / `OPENAI_VLM_MODEL` | _required for OpenAI_ / `gpt-4o-mini` | OpenAI provider |
+| `GEMINI_API_KEY` / `GEMINI_VLM_MODEL` | _required for Gemini_ / `gemini-1.5-flash` | Gemini provider |
+| `DOCPARSER_VLM_BASE_URL` | `http://localhost:11434/v1` | base URL for the `local` provider |
+| `DOCPARSER_VLM_API_KEY` / `DOCPARSER_VLM_MODEL` | — / `llava` | key + model for the `local` provider |
+| `DOCPARSER_LOCAL_VLM_MODEL` | `Salesforce/blip-image-captioning-large` | model for the `transformers` backend |
+
+## API reference (highlights)
+
+- `WorkspaceLayout(raw_dir, parsed_dir, assets_dir, cache_dir)` —
+  dataclass describing where parser output lives. Use `.under(root)` for the
+  default `data/raw + data/parsed + data/assets + .cache` layout under a root.
+- `parse_docx(source, layout=None, *, captioner=None, write_outputs=True)` →
+  payload dict.
+- `parse_xlsx(source, layout=None, *, captioner=None, write_outputs=True)` →
+  payload dict.
+- `parse_pdf(source, layout=None, *, captioner=None, write_outputs=True, extract_images=True, backend="builtin", ocr="off", extract_tables=False)` →
+  payload dict. (requires `[pdf]`; backends/OCR/tables require their extras)
+- `parse_html(source, layout=None, *, captioner=None, write_outputs=True, use_trafilatura=True)` →
+  payload dict. `source` may be a path or `http(s)://` URL. (requires `[html]`)
+- `parse_pptx(source, layout=None, *, captioner=None, write_outputs=True)` →
+  payload dict. (requires `[pptx]`)
+- `parse_epub(source, layout=None, *, captioner=None, write_outputs=True)` →
+  payload dict. (requires `[epub]`)
+- `parse_text(source, layout=None, ...)` / `parse_csv(source, layout=None, ...)` —
+  core parsers for `.txt`/`.md` and `.csv`/`.tsv`.
+- `parse_path(source, layout=None, **kwargs)` — dispatches by extension;
+  PDF-only kwargs (`backend`, `ocr`, `extract_tables`) are forwarded to PDFs.
+- `run_all(layout, *, use_vlm=True, only=None, max_images=None, continue_on_error=False, vlm_provider=None, vlm_model=None, pdf_backend="builtin", ocr="off", extract_tables=False)` —
+  walks `layout.raw_dir`, parses everything supported, writes a top-level
+  `CORPUS.md` and `data/parsed/corpus.json`.
+- `caption_image(image_bytes, *, mime, doc_name, nearby_caption, context, provider=None, model=None, layout=None, ...)` →
+  `VLMResult`. (requires `[vlm]`)
+
+## Development
+
+```bash
+git clone https://github.com/Research-Commons/docparser
+cd docparser
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[all,dev]"
+pytest -ra
+ruff check src tests
+mypy
+python -m build           # produces dist/*.whl + *.tar.gz
+twine check dist/*
+```
+
+### Publishing
+
+CI runs lint + mypy + tests on Python 3.10-3.12 and builds the distribution on
+every push/PR (`.github/workflows/ci.yml`). Pushing a version tag (e.g.
+`v0.2.0`) triggers `.github/workflows/publish.yml`, which builds and uploads to
+PyPI via **Trusted Publishing** (OIDC, no stored token) — configure a trusted
+publisher for the project on PyPI first. To publish manually instead:
+
+```bash
+python -m build
+twine check dist/*
+twine upload dist/*
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).

rc_docparser-0.2.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+docparser/__init__.py,sha256=RII8KjcE7EzeBPthWp7-8cYiih9v782lGmctSc12zlU,2307
+docparser/cli.py,sha256=eDyGm1loOaut-JZh-85PU171_hKlb6XFc_UBnYu1030,6531
+docparser/common.py,sha256=o7tt8OQWAGQnhFqqkq2D0wNx5eV_cNeN7WMUNVQkDbU,5231
+docparser/csvtab.py,sha256=nmzWlNNqGYBrTwgJHk5Kcx0U55Fd3_EuVD-PjBt7Ecs,3960
+docparser/docx.py,sha256=oSkwdp4XNY2ZIEaMOilDhg0fiYrpAto5FTJP_FIlclY,17088
+docparser/epub.py,sha256=rIBiIBCMIgyo__uR2D0bMn2QEuBayoMYK4eheI2dsZY,12608
+docparser/html.py,sha256=SV_zc2wdXjPkKhSdz4mBKg-c8AAyf7W6SEV2o0Zhh0w,11376
+docparser/image.py,sha256=ZZk5UchqX3GtcB2U0rPGCnyL2NSx6Jmf-5k0UFtqROw,12392
+docparser/localvlm.py,sha256=LAcD79mEqDBBoyc7v0Ic-W5SexqZlN4ulxMv7f9Rphg,3757
+docparser/ocr.py,sha256=mWMPOx0eKm3VrDLDhZ7hQcNzj5RotxtamKVbxi9WpoA,2103
+docparser/orchestrator.py,sha256=5Or11Ye3yVE4rOGA9MEviWT2QOIWXvWg-1RhkwdvbhA,9813
+docparser/pdf.py,sha256=d3Zyj-zpY7EFs1sWvzDXQshQXAJGkjCIIBKkf1yCuRs,16698
+docparser/pdf_backends.py,sha256=5P1PrEAGb8UZpYWfyi7D0WYHo-oWed7S3By1CTe1n2Q,3348
+docparser/pptx.py,sha256=8c-aLyqnoYhJj4b_ZvA8NahvwRWPicwV5_VMlrCMNQI,12020
+docparser/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docparser/text.py,sha256=r3u-bk09jAfpvYq7yuM7yYOLRjY1ChA0sDi619pzZiE,5930
+docparser/xlsx.py,sha256=6s-dtbyN92qFRLT3dKi_1HVuUGzCa8IyZ1RJLwwNDP8,11758
+rc_docparser-0.2.0.dist-info/METADATA,sha256=IFGuaKCRvT8VZoYfo1UPWrSss_JmwFdAd0sMFfNCSCA,15512
+rc_docparser-0.2.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+rc_docparser-0.2.0.dist-info/entry_points.txt,sha256=VFP7dbtsKVgtvwjo63QThpCN65nmXpp3OIvvokVz39s,49
+rc_docparser-0.2.0.dist-info/licenses/LICENSE,sha256=509nFfR8HUUnDqGdHdYzy9CHL_7cCxKsR7zqZJko0N0,1073
+rc_docparser-0.2.0.dist-info/RECORD,,