docpluck 1.5.0__py3-none-any.whl

docpluck/__init__.py

@@ -0,0 +1,89 @@
+"""
+docpluck — PDF, DOCX, and HTML text extraction and normalization for academic papers
+====================================================================================
+
+A Python library for extracting and normalizing text from academic documents.
+Built from cross-project lessons across 8,000+ PDFs from psychology, medicine,
+economics, physics, and biology.
+
+Supports:
+- **PDF** via pdftotext (default mode, with pdfplumber SMP fallback)
+- **DOCX** via mammoth (DOCX → HTML → text, preserves soft breaks)
+- **HTML** via beautifulsoup4 + lxml (custom block/inline-aware tree-walk)
+
+Quick start::
+
+    from docpluck import extract_pdf, extract_docx, extract_html
+    from docpluck import normalize_text, NormalizationLevel, compute_quality_score
+
+    # PDF
+    with open("paper.pdf", "rb") as f:
+        text, method = extract_pdf(f.read())
+
+    # DOCX (requires: pip install docpluck[docx])
+    with open("paper.docx", "rb") as f:
+        text, method = extract_docx(f.read())
+
+    # HTML (requires: pip install docpluck[html])
+    with open("paper.html", "rb") as f:
+        text, method = extract_html(f.read())
+
+    # Normalization and quality scoring work on text from any source
+    normalized, report = normalize_text(text, NormalizationLevel.academic)
+    quality = compute_quality_score(normalized)
+
+    print(f"Method: {method}")
+    print(f"Quality: {quality['score']}/100 ({quality['confidence']})")
+    print(f"Steps applied: {report.steps_applied}")
+
+Installation::
+
+    pip install docpluck             # PDF only (pdfplumber)
+    pip install docpluck[docx]       # + mammoth
+    pip install docpluck[html]       # + beautifulsoup4 + lxml
+    pip install docpluck[all]        # everything
+
+    # extract_pdf() also requires poppler-utils:
+    #   Linux/WSL: apt-get install poppler-utils
+    #   macOS:     brew install poppler
+    #   Windows:   https://github.com/oschwartz10612/poppler-windows/releases
+
+See Also:
+    - docs/README.md — Full usage guide and API reference
+    - docs/DESIGN.md — Implementation decisions and rationale
+    - docs/BENCHMARKS.md — Benchmark results across all supported formats
+    - docs/NORMALIZATION.md — All 15 pipeline steps documented
+"""
+
+from .extract import extract_pdf, extract_pdf_file, count_pages
+from .extract_docx import extract_docx
+from .extract_html import extract_html, html_to_text
+from .normalize import normalize_text, NormalizationLevel, NormalizationReport
+from .quality import compute_quality_score
+from .batch import ExtractionReport, extract_to_dir
+from .version import get_version_info
+
+__version__ = "1.5.0"
+__author__ = "Gilad Feldman"
+__license__ = "MIT"
+
+__all__ = [
+    # Extraction
+    "extract_pdf",
+    "extract_pdf_file",
+    "extract_docx",
+    "extract_html",
+    "html_to_text",
+    "count_pages",
+    # Normalization
+    "normalize_text",
+    "NormalizationLevel",
+    "NormalizationReport",
+    # Quality
+    "compute_quality_score",
+    # Batch
+    "ExtractionReport",
+    "extract_to_dir",
+    # Version
+    "get_version_info",
+]

docpluck/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

docpluck/batch.py

@@ -0,0 +1,183 @@
+"""
+Batch extraction helper for directory-level runs.
+
+MetaESCI, Scimeto, and ESCImate all want the same "walk a list of PDFs,
+normalize them, drop a sidecar, and give me a receipt" pattern. Instead of
+each downstream re-implementing it, :func:`extract_to_dir` lives here and
+returns an :class:`ExtractionReport` that doubles as a reproducibility
+receipt (``docpluck_version``, ``normalize_version``, ``git_sha``, per-file
+status).
+
+Example::
+
+    from docpluck import extract_to_dir, NormalizationLevel
+
+    report = extract_to_dir(
+        pdf_paths=list(Path("pdfs").glob("*.pdf")),
+        out_dir="normalized_text",
+        level=NormalizationLevel.academic,
+    )
+    print(f"{report.n_ok}/{report.n_total} ok, {report.elapsed_seconds:.1f}s")
+    report.write_receipt("normalized_text/_docpluck_receipt.json")
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Iterable, Optional, Union
+
+from .extract import extract_pdf_file
+from .normalize import NormalizationLevel, normalize_text
+from .version import get_version_info
+
+
+@dataclass
+class ExtractionFileResult:
+    path: str
+    ok: bool
+    method: Optional[str] = None
+    n_chars_raw: int = 0
+    n_chars_normalized: int = 0
+    normalize_steps_changed: list[str] = field(default_factory=list)
+    error: Optional[str] = None
+    elapsed_seconds: float = 0.0
+
+
+@dataclass
+class ExtractionReport:
+    """Machine-readable receipt for a batch extraction run.
+
+    Contains the docpluck version metadata, per-file results, and aggregate
+    counts. Serializable to JSON via :meth:`to_dict` / :meth:`write_receipt`
+    so downstream pipelines can pin reproducibility against a fixed run.
+    """
+
+    docpluck_version: str
+    normalize_version: str
+    git_sha: str
+    level: str
+    out_dir: str
+    n_total: int = 0
+    n_ok: int = 0
+    n_failed: int = 0
+    elapsed_seconds: float = 0.0
+    results: list[ExtractionFileResult] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "docpluck_version": self.docpluck_version,
+            "normalize_version": self.normalize_version,
+            "git_sha": self.git_sha,
+            "level": self.level,
+            "out_dir": self.out_dir,
+            "n_total": self.n_total,
+            "n_ok": self.n_ok,
+            "n_failed": self.n_failed,
+            "elapsed_seconds": round(self.elapsed_seconds, 3),
+            "results": [asdict(r) for r in self.results],
+        }
+
+    def write_receipt(self, path: Union[str, Path]) -> Path:
+        out = Path(path)
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(json.dumps(self.to_dict(), indent=2), encoding="utf-8")
+        return out
+
+
+def extract_to_dir(
+    pdf_paths: Iterable[Union[str, Path]],
+    out_dir: Union[str, Path],
+    level: NormalizationLevel = NormalizationLevel.academic,
+    write_sidecar: bool = True,
+) -> ExtractionReport:
+    """Extract and normalize a collection of PDFs into a directory.
+
+    For each input PDF, writes ``<stem>.txt`` containing normalized text.
+    When ``write_sidecar`` is true (default), also writes ``<stem>.json``
+    with per-file metadata (method, normalize steps, timings, errors).
+
+    Missing files are recorded as failures on the report — this function
+    does not raise on individual file errors, only on argument errors.
+
+    Args:
+        pdf_paths: Iterable of PDF paths. Each path must point to a file.
+        out_dir: Directory that will receive ``<stem>.txt`` (and sidecars).
+            Created if it does not exist.
+        level: Normalization level. Defaults to ``academic``.
+        write_sidecar: Whether to emit the per-file ``.json`` sidecar.
+
+    Returns:
+        :class:`ExtractionReport` with aggregate counts and per-file results.
+    """
+    info = get_version_info()
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+
+    report = ExtractionReport(
+        docpluck_version=info["version"],
+        normalize_version=info["normalize_version"],
+        git_sha=info["git_sha"],
+        level=level.value if isinstance(level, NormalizationLevel) else str(level),
+        out_dir=str(out),
+    )
+
+    batch_start = time.monotonic()
+    for p in pdf_paths:
+        p = Path(p)
+        report.n_total += 1
+        file_start = time.monotonic()
+        result = ExtractionFileResult(path=str(p), ok=False)
+
+        try:
+            raw_text, method = extract_pdf_file(p)
+            result.method = method
+            result.n_chars_raw = len(raw_text)
+
+            if raw_text.startswith("ERROR:"):
+                result.error = raw_text
+            else:
+                normalized, norm_report = normalize_text(raw_text, level)
+                result.n_chars_normalized = len(normalized)
+                result.normalize_steps_changed = list(
+                    getattr(norm_report, "steps_changed", norm_report.steps_applied)
+                )
+
+                text_path = out / f"{p.stem}.txt"
+                text_path.write_text(normalized, encoding="utf-8")
+
+                if write_sidecar:
+                    sidecar = {
+                        "source": str(p),
+                        "method": method,
+                        "level": level.value if isinstance(level, NormalizationLevel) else str(level),
+                        "normalize_version": info["normalize_version"],
+                        "docpluck_version": info["version"],
+                        "git_sha": info["git_sha"],
+                        "n_chars_raw": result.n_chars_raw,
+                        "n_chars_normalized": result.n_chars_normalized,
+                        "steps_changed": result.normalize_steps_changed,
+                        "changes_made": dict(norm_report.changes_made),
+                    }
+                    sidecar_path = out / f"{p.stem}.json"
+                    sidecar_path.write_text(
+                        json.dumps(sidecar, indent=2), encoding="utf-8"
+                    )
+
+                result.ok = True
+        except FileNotFoundError as e:
+            result.error = f"FileNotFoundError: {e}"
+        except Exception as e:  # noqa: BLE001 — batch runner must never raise
+            result.error = f"{type(e).__name__}: {e}"
+
+        result.elapsed_seconds = round(time.monotonic() - file_start, 3)
+        report.results.append(result)
+        if result.ok:
+            report.n_ok += 1
+        else:
+            report.n_failed += 1
+
+    report.elapsed_seconds = time.monotonic() - batch_start
+    return report

docpluck/cli.py

@@ -0,0 +1,35 @@
+"""
+Minimal docpluck CLI.
+
+Currently supports a single flag::
+
+    docpluck --version
+
+which prints ``{version, normalize_version, git_sha}`` as JSON. Downstream
+batch runners call this once per run and write the output next to their
+results as a reproducibility receipt (see MetaESCI request D3).
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+
+from .version import get_version_info
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = list(sys.argv[1:] if argv is None else argv)
+    if not args or args[0] in ("-V", "--version", "version"):
+        print(json.dumps(get_version_info()))
+        return 0
+    if args[0] in ("-h", "--help", "help"):
+        print("usage: docpluck [--version]")
+        return 0
+    print(f"docpluck: unknown argument: {args[0]}", file=sys.stderr)
+    print("usage: docpluck [--version]", file=sys.stderr)
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

docpluck/extract.py

@@ -0,0 +1,191 @@
+"""
+PDF Text Extraction
+====================
+Primary engine: pdftotext default mode (no -layout flag)
+Fallback: pdfplumber for PDFs with SMP Unicode (Mathematical Italic fonts)
+
+Requires poppler-utils installed on the system:
+  - Linux/WSL: apt-get install poppler-utils
+  - macOS: brew install poppler
+  - Windows: https://github.com/oschwartz10612/poppler-windows/releases
+
+Key design decision: pdftotext default mode (NO -layout flag).
+The -layout flag preserves physical column layout, causing column interleaving
+that breaks statistical pattern matching. Default mode correctly reconstructs
+reading order. Verified on 50 PDFs across 8 citation styles — see BENCHMARKS.md.
+"""
+
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Optional, Union
+
+
+def extract_pdf(pdf_bytes: bytes) -> tuple[str, str]:
+    """Extract text from PDF bytes.
+
+    Uses pdftotext as the primary engine. Automatically falls back to
+    pdfplumber if the PDF contains SMP Unicode characters (e.g. Mathematical
+    Italic fonts used by Nature/Cell journals) that Xpdf cannot handle.
+
+    Args:
+        pdf_bytes: Raw PDF file content as bytes.
+
+    Returns:
+        A tuple of (text, method) where:
+          - text: Extracted plain text. May start with "ERROR: ..." if extraction
+            failed — check with text.startswith("ERROR:").
+          - method: Engine used. One of:
+              "pdftotext_default"                   — normal extraction
+              "pdftotext_default+pdfplumber_recovery" — SMP fallback triggered
+
+    Requires:
+        pdftotext binary (from poppler-utils) on PATH.
+
+    Example:
+        with open("paper.pdf", "rb") as f:
+            text, method = extract_pdf(f.read())
+        print(f"Extracted {len(text)} chars via {method}")
+    """
+    with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
+        tmp.write(pdf_bytes)
+        tmp_path = tmp.name
+
+    try:
+        # Primary: pdftotext default mode (no -layout flag — critical)
+        result = subprocess.run(
+            ["pdftotext", "-enc", "UTF-8", tmp_path, "-"],
+            capture_output=True,
+            timeout=120,
+            encoding="utf-8",
+            errors="replace",
+        )
+
+        if result.returncode != 0:
+            return f"ERROR: pdftotext failed with code {result.returncode}", "error"
+
+        text = result.stdout
+        method = "pdftotext_default"
+
+        # SMP recovery: Xpdf replaces U+FFFF+ characters with U+FFFD (replacement char).
+        # pdfplumber handles these correctly and remaps them to ASCII equivalents.
+        if text.count("\ufffd") > 0:
+            recovered = _recover_with_pdfplumber(tmp_path)
+            if recovered:
+                text = recovered
+                method = "pdftotext_default+pdfplumber_recovery"
+
+        return text, method
+
+    finally:
+        os.unlink(tmp_path)
+
+
+def extract_pdf_file(path: Union[str, Path]) -> tuple[str, str]:
+    """Extract text from a PDF file on disk.
+
+    Thin convenience wrapper around ``extract_pdf`` that reads ``path`` and
+    raises a clean ``FileNotFoundError`` when the file does not exist, instead
+    of the generic exception pdftotext emits on a missing input. Useful for
+    batch runners that walk directories and want actionable errors.
+
+    Args:
+        path: Path to the PDF file on disk (str or pathlib.Path).
+
+    Returns:
+        Same tuple as ``extract_pdf``: ``(text, method)``.
+
+    Raises:
+        FileNotFoundError: If ``path`` does not exist or is not a regular file.
+
+    Example:
+        text, method = extract_pdf_file("paper.pdf")
+    """
+    p = Path(path)
+    if not p.is_file():
+        # is_file() returns False for both missing paths and non-file entries
+        # (directories, broken symlinks). Distinguish for a clearer error.
+        if p.exists():
+            raise FileNotFoundError(f"Path is not a regular file: {p}")
+        raise FileNotFoundError(f"PDF file not found: {p}")
+    return extract_pdf(p.read_bytes())
+
+
+def count_pages(pdf_bytes: bytes) -> int:
+    """Count the number of pages in a PDF.
+
+    Uses byte pattern matching (no external binary required). Fast and
+    reliable for well-formed PDFs. Returns 1 as a minimum.
+
+    Args:
+        pdf_bytes: Raw PDF file content as bytes.
+
+    Returns:
+        Page count (integer, minimum 1).
+
+    Example:
+        with open("paper.pdf", "rb") as f:
+            n = count_pages(f.read())
+        print(f"{n} pages")
+    """
+    try:
+        count = pdf_bytes.count(b"/Type /Page") - pdf_bytes.count(b"/Type /Pages")
+        return max(count, 1)
+    except Exception:
+        return 0
+
+
+def _recover_with_pdfplumber(pdf_path: str) -> Optional[str]:
+    """Recover text using pdfplumber when pdftotext produces garbled output.
+
+    Triggered when U+FFFD (replacement character) appears in pdftotext output,
+    which indicates SMP Mathematical Italic fonts (U+1D434-U+1D467) that
+    Xpdf/poppler cannot decode. pdfplumber (using pdfminer) handles these
+    correctly. Maps the recovered SMP characters to ASCII equivalents so
+    downstream regex patterns work normally.
+
+    Args:
+        pdf_path: Path to the PDF file on disk.
+
+    Returns:
+        Recovered text string, or None if recovery failed.
+    """
+    try:
+        import pdfplumber
+
+        smp_to_ascii: dict[str, str] = {}
+        # Math italic capitals A-Z: U+1D434–U+1D44D
+        for i, letter in enumerate("ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
+            smp_to_ascii[chr(0x1D434 + i)] = letter
+        # Math italic small a-z: U+1D44E–U+1D467
+        for i, letter in enumerate("abcdefghijklmnopqrstuvwxyz"):
+            smp_to_ascii[chr(0x1D44E + i)] = letter
+        # Math italic Greek (common in physics/biology papers)
+        greek = {
+            0x1D6E2: "A", 0x1D6E4: "G", 0x1D6E5: "D", 0x1D6F4: "S",
+            0x1D6F7: "Ph", 0x1D6F8: "Ch", 0x1D6F9: "Ps", 0x1D6FA: "O",
+            0x1D6FC: "a", 0x1D6FD: "b", 0x1D6FE: "g", 0x1D6FF: "d",
+            0x1D700: "e", 0x1D701: "z", 0x1D702: "n", 0x1D703: "th",
+            0x1D707: "m", 0x1D70B: "pi", 0x1D70C: "r", 0x1D70E: "s",
+            0x1D711: "ph", 0x1D712: "ch", 0x1D713: "ps",
+        }
+        for cp, repl in greek.items():
+            smp_to_ascii[chr(cp)] = repl
+
+        pages_text = []
+        with pdfplumber.open(pdf_path) as pdf:
+            for page in pdf.pages:
+                page_text = page.extract_text()
+                if page_text:
+                    pages_text.append(page_text)
+
+        full_text = "\n\n".join(pages_text)
+
+        for smp_char, ascii_equiv in smp_to_ascii.items():
+            full_text = full_text.replace(smp_char, ascii_equiv)
+
+        return full_text
+
+    except Exception:
+        return None

docpluck/extract_docx.py

@@ -0,0 +1,64 @@
+"""
+DOCX Text Extraction
+=====================
+Primary engine: mammoth (DOCX → HTML → text via html_to_text)
+
+Why mammoth:
+- `mammoth.convert_to_html()` preserves Shift+Enter soft breaks as <br> tags
+  (critical for academic documents with poetry, equations, addresses, etc.)
+- `mammoth.extract_raw_text()` loses intra-paragraph line breaks — do NOT use it
+- python-docx only provides paragraph-level access, not enough structure
+- docx2txt is effectively abandoned
+- pypandoc requires a binary (pandoc) that's hard to deploy
+- BSD-2 license, available in both Python (mammoth) and Node.js (mammoth.js)
+- Battle-tested in Scimeto production since Dec 2025
+
+Known limitations:
+- OMML equations (Office Math) are silently dropped. Papers with inline
+  stats inside equation objects will lose those values. In practice this is
+  rare in social science papers where stats are written as plain text.
+- Tracked changes: only deleted paragraphs are handled minimally.
+- Memory: peak usage is ~3-5x file size. Not a concern for single-file
+  processing but worth noting for very large documents.
+
+Requires the `docx` optional dependency:
+  pip install docpluck[docx]
+"""
+import io
+
+from .extract_html import html_to_text
+
+
+def extract_docx(docx_bytes: bytes) -> tuple[str, str]:
+    """Extract text from DOCX file bytes.
+
+    Converts the DOCX to HTML via mammoth (preserving soft breaks and block
+    structure), then runs it through html_to_text() for the final plain-text
+    output. This two-step pipeline is what makes soft-break preservation work.
+
+    Args:
+        docx_bytes: Raw DOCX file content as bytes.
+
+    Returns:
+        A tuple of (text, method) where:
+          - text: Extracted plain text with block/inline-aware formatting.
+          - method: Always "mammoth".
+
+    Raises:
+        ValueError: If the DOCX is malformed (mammoth raises — we re-raise).
+        ImportError: If mammoth is not installed.
+
+    Requires:
+        mammoth (install with `pip install docpluck[docx]`).
+
+    Example:
+        with open("paper.docx", "rb") as f:
+            text, method = extract_docx(f.read())
+    """
+    # Lazy import so the core library works without mammoth installed
+    import mammoth
+
+    result = mammoth.convert_to_html(io.BytesIO(docx_bytes))
+    html = result.value
+    text = html_to_text(html)
+    return text, "mammoth"