parseoff 0.2.1__py3-none-any.whl

parseoff/registry.py

@@ -0,0 +1,149 @@
+"""Known parsers: licence, surveyed install size, warnings and detection helpers.
+
+The surveyed numbers (licence, size) come from a one-off survey and are labelled as such wherever
+they are shown. Installed versions are always read at runtime from the installed distribution.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+from dataclasses import dataclass
+from importlib import metadata
+from pathlib import Path
+
+from .i18n import MESSAGES, t
+
+SURVEY_NOTE = "surveyed 2026-11-10, macOS arm64, Python 3.12, isolated install"
+HEAVY_INSTALL_MB = 100.0
+
+
+@dataclass(frozen=True)
+class ParserInfo:
+    name: str
+    dists: tuple[str, ...]  # distribution names tried for the installed version
+    module: str  # top-level module used to detect an installation
+    license: str
+    size_mb: float | None  # surveyed isolated install size; None = not measured
+    size_text: str
+    auto: bool  # run automatically when installed (False = only via --parsers)
+    core: bool  # installed together with parseoff
+    copyleft: bool = False
+    weights_restricted: bool = False
+    model_download: bool = False
+    install_hint: str = ""
+    note: str = ""
+
+
+REGISTRY: dict[str, ParserInfo] = {
+    p.name: p
+    for p in (
+        ParserInfo(
+            "pypdf", ("pypdf",), "pypdf", "BSD-3-Clause", 1.8, "1.8 MB", True, True,
+            note="pure Python; text in content-stream order, no structure",
+        ),
+        ParserInfo(
+            "pdfplumber", ("pdfplumber",), "pdfplumber", "MIT", 41.0, "41 MB", True, True,
+            note="text by position + tables as GFM (includes pypdfium2, pdfminer.six)",
+        ),
+        ParserInfo(
+            "pdfminer", ("pdfminer.six",), "pdfminer", "MIT", None, "part of pdfplumber", True, True,
+            note="layout-box text; markitdown's PDF path is pdfminer text extraction",
+        ),
+        ParserInfo(
+            "markitdown", ("markitdown",), "markitdown", "MIT", 147.0, "147 MB", True, False,
+            install_hint='uv sync --extra markitdown',
+            note="most of the size is onnxruntime; page-split (one conversion per page)",
+        ),
+        ParserInfo(
+            "pymupdf4llm", ("pymupdf4llm", "pymupdf"), "pymupdf4llm", "AGPL-3.0 or commercial",
+            214.0, "214 MB", True, False, copyleft=True,
+            install_hint="uv pip install pymupdf4llm; AGPL-3.0, check its terms",
+            note="detected only if you installed it; not an extra of parseoff",
+        ),
+        ParserInfo(
+            "docling", ("docling",), "docling", "MIT", None, "not measured (large, torch)",
+            False, False, model_download=True,
+            install_hint='uv sync --extra docling',
+            note="experimental",
+        ),
+    )
+}
+BUILTIN_NAMES = tuple(REGISTRY)
+
+
+def is_importable(module: str) -> bool:
+    """True if `module` can be imported (without importing it). Tolerates fake modules in tests."""
+    if module in sys.modules:
+        return True
+    try:
+        return importlib.util.find_spec(module) is not None
+    except (ImportError, ValueError):
+        return False
+
+
+def installed_version(name: str) -> str | None:
+    """Installed distribution version for a known parser, or None if not installed."""
+    info = REGISTRY[name]
+    for dist in info.dists:
+        try:
+            return metadata.version(dist)
+        except metadata.PackageNotFoundError:
+            continue
+    return None
+
+
+def is_installed(name: str) -> bool:
+    return is_importable(REGISTRY[name].module)
+
+
+def docling_models_present() -> bool:
+    """Existence check only: DOCLING_ARTIFACTS_PATH or docling's default cache directory."""
+    env = os.environ.get("DOCLING_ARTIFACTS_PATH")
+    if env and Path(env).is_dir():
+        return True
+    return (Path.home() / ".cache" / "docling" / "models").is_dir()
+
+
+def disk_size_mb(name: str) -> float | None:
+    """Size of the installed distribution's own files (not its dependencies), or None."""
+    info = REGISTRY[name]
+    for dist in info.dists:
+        try:
+            files = metadata.distribution(dist).files
+        except metadata.PackageNotFoundError:
+            continue
+        if not files:
+            return None
+        total = 0
+        for f in files:
+            try:
+                total += f.locate().stat().st_size
+            except OSError:
+                continue
+        return total / 1_000_000
+    return None
+
+
+# --- screen text for registry facts (English values stay in REGISTRY, Korean ones live in i18n) ----
+
+
+def size_text(name: str, lang: str = "en") -> str:
+    """Surveyed install size as text; the two non-numeric ones ('part of ...', 'not measured') translate."""
+    return _facts_text(f"size.{name}", REGISTRY[name].size_text, lang)
+
+
+def install_hint(name: str, lang: str = "en") -> str:
+    """How to install a parser. Commands stay as they are; only the explanation is translated."""
+    return _facts_text(f"hint.{name}", REGISTRY[name].install_hint, lang)
+
+
+def survey_note(lang: str = "en") -> str:
+    return SURVEY_NOTE if lang == "en" else t("survey", lang)
+
+
+def _facts_text(key: str, default: str, lang: str) -> str:
+    if lang != "en" and key in MESSAGES.get(lang, {}):
+        return t(key, lang)
+    return default

parseoff/report_html.py

@@ -0,0 +1,173 @@
+"""Static HTML report: index.html plus one doc-<slug>.html per PDF.
+
+Self-contained: inline CSS, no scripts, no external resources. Every piece of text that comes from a
+document or a parser goes through html.escape. Page images are relative links to img/*.png.
+The screen language is fixed when the report is written (``<html lang>`` follows it).
+"""
+
+from __future__ import annotations
+
+from html import escape
+
+from .findings import message_for, skip_reason
+from .i18n import DEFAULT_LANG, t
+from .model import DocReport, Finding, RunReport
+from .registry import REGISTRY, installed_version, size_text, survey_note
+
+CSS = """
+body{font-family:system-ui,sans-serif;margin:24px;color:#1b1f24;background:#fff}
+h1,h2{margin:.6em 0 .3em} table{border-collapse:collapse;margin:8px 0}
+th,td{border:1px solid #c9d1d9;padding:4px 8px;text-align:left;font-size:14px;vertical-align:top}
+th{background:#f3f4f6} .notice{background:#fff8c5;border:1px solid #d4a72c;padding:8px 12px;margin:8px 0}
+.badge{display:inline-block;border-radius:4px;padding:0 6px;font-size:12px;font-weight:600;margin-right:4px}
+.FAIL{background:#ffd7d5;color:#82071e}.WARN{background:#fff1c2;color:#6a4b00}.INFO{background:#ddf4ff;color:#0a3069}
+.page{border-top:2px solid #8c959f;margin-top:18px;padding-top:6px}
+.grid{display:flex;gap:12px;align-items:flex-start;overflow-x:auto}
+.col{min-width:260px;flex:1 1 260px;max-width:520px}.img img{max-width:340px;border:1px solid #c9d1d9}
+pre{background:#f6f8fa;border:1px solid #d0d7de;padding:8px;white-space:pre-wrap;word-break:break-word;font-size:12px}
+.muted{color:#57606a;font-size:13px} details{margin:4px 0}
+"""
+
+
+def _e(value: object) -> str:
+    return escape(str(value), quote=True)
+
+
+def _pct(value: float | None) -> str:
+    return "-" if value is None else f"{value:.0%}"
+
+
+def _badge(f: Finding, lang: str) -> str:
+    return f'<span class="badge {f.level}" title="{_e(message_for(f, lang))}">{_e(f.code)}</span>'
+
+
+def _page(title: str, body: str, lang: str) -> str:
+    return (
+        f'<!doctype html>\n<html lang="{_e(lang)}"><head><meta charset="utf-8">'
+        f"<title>{_e(title)}</title><style>{CSS}</style></head><body>\n{body}\n</body></html>\n"
+    )
+
+
+def _notice(report: RunReport, lang: str) -> str:
+    return f'<div class="notice">{_e(t("notice.demo" if report.demo else "notice", lang))}</div>'
+
+
+def _finding_li(f: Finding, slug_of: dict[str, str], lang: str) -> str:
+    where = []
+    if f.doc:
+        label = f.doc + (t("where.page", lang, page=f.page) if f.page else "")
+        link = f"doc-{slug_of[f.doc]}.html" + (f"#p{f.page}" if f.page else "")
+        where.append(f'<a href="{_e(link)}">{_e(label)}</a>')
+    if f.parser:
+        where.append(_e(f.parser))
+    return f"<li>{_badge(f, lang)} {' / '.join(where)} {_e(message_for(f, lang))}</li>"
+
+
+def render_index(report: RunReport, lang: str = DEFAULT_LANG) -> str:
+    c = report.counts()
+    slug_of = {d.doc_id: d.slug for d in report.docs}
+    parts = [f"<h1>{_e(t('html.title', lang))}</h1>", _notice(report, lang)]
+    parts.append(
+        f'<p><span class="badge FAIL">FAIL {c["FAIL"]}</span><span class="badge WARN">WARN {c["WARN"]}</span>'
+        f'<span class="badge INFO">INFO {c["INFO"]}</span></p>'
+    )
+
+    th = lambda key: f"<th>{_e(t(key, lang))}</th>"  # noqa: E731 - tiny local helper
+    parts.append(f"<h2>{_e(t('html.parsers', lang))}</h2><table><tr>{th('tbl.parser')}{th('tbl.version')}{th('tbl.license')}"
+                 f"<th>{_e(t('html.size_head', lang, survey=survey_note(lang)))}</th>{th('tbl.status')}</tr>")
+    for name in report.parsers:
+        info = REGISTRY.get(name)
+        parts.append(
+            f"<tr><td>{_e(name)}</td><td>{_e(installed_version(name) if info else 'custom')}</td>"
+            f"<td>{_e(info.license if info else t('license.custom', lang))}</td>"
+            f"<td>{_e(size_text(name, lang) if info else '-')}</td><td>{_e(t('html.ran', lang))}</td></tr>"
+        )
+    for s in report.skipped:
+        parts.append(f"<tr><td>{_e(s.name)}</td><td>-</td><td>{_e(REGISTRY[s.name].license)}</td>"
+                     f"<td>{_e(size_text(s.name, lang))}</td>"
+                     f"<td>{_e(t('html.skipped', lang, reason=skip_reason(s, lang)))}</td></tr>")
+    parts.append("</table>")
+
+    parts.append(f"<h2>{_e(t('html.docs', lang))}</h2><table><tr>"
+                 + "".join(th(f"tbl.{k}") for k in ("document", "parser", "status", "seconds", "coverage", "tables", "headings", "tokens"))
+                 + "</tr>")
+    for doc in report.docs:
+        link = f'<a href="doc-{_e(doc.slug)}.html">{_e(doc.doc_id)}</a>'
+        if doc.open_error:
+            parts.append(f"<tr><td>{link}</td><td colspan=\"7\">{_e(t('html.cannot_open_row', lang, error=doc.open_error))}</td></tr>")
+        for name, out in doc.outputs.items():
+            pm = doc.metrics.parsers.get(name) if doc.metrics else None
+            cells = ("-", "-", "-", "-")
+            if pm:
+                cells = (_pct(pm.mean_coverage), str(sum(m.tables for m in pm.pages)),
+                         str(sum(m.headings for m in pm.pages)), str(sum(m.tokens_est for m in pm.pages)))
+            parts.append(
+                f"<tr><td>{link}</td><td>{_e(name)}</td><td>{_e(out.status)}</td><td>{out.seconds:.2f}</td>"
+                + "".join(f"<td>{_e(x)}</td>" for x in cells) + "</tr>"
+            )
+    parts.append(f"</table><p class=\"muted\">{_e(t('html.tokens_note', lang))}</p>")
+
+    parts.append(f"<h2>{_e(t('html.agreement', lang))}</h2>")
+    for doc in report.docs:
+        if not doc.metrics or not doc.metrics.mean_agreement:
+            continue
+        names = list(doc.metrics.parsers)
+        parts.append(f"<h3>{_e(doc.doc_id)}</h3><table><tr><th></th>" + "".join(f"<th>{_e(n)}</th>" for n in names) + "</tr>")
+        for a in names:
+            row = []
+            for b in names:
+                if a == b:
+                    row.append("1.00")
+                else:
+                    v = doc.metrics.mean_agreement.get((a, b), doc.metrics.mean_agreement.get((b, a)))
+                    row.append("-" if v is None else f"{v:.2f}")
+            parts.append(f"<tr><th>{_e(a)}</th>" + "".join(f"<td>{r}</td>" for r in row) + "</tr>")
+        parts.append("</table>")
+
+    parts.append(f"<h2>{_e(t('html.findings', lang))}</h2>")
+    parts.append("<ul>" + "".join(_finding_li(f, slug_of, lang) for f in report.findings) + "</ul>" if report.findings
+                 else f"<p>{_e(t('html.no_findings', lang))}</p>")
+    return _page(t("html.title", lang), "\n".join(parts), lang)
+
+
+def render_doc(report: RunReport, doc: DocReport, lang: str = DEFAULT_LANG) -> str:
+    parts = [f"<h1>{_e(doc.doc_id)}</h1>", f'<p><a href="index.html">{_e(t("html.back", lang))}</a></p>', _notice(report, lang)]
+    if doc.open_error:
+        parts.append(f"<p>{_e(t('html.cannot_open', lang, error=doc.open_error))}</p>")
+        return _page(doc.doc_id, "\n".join(parts), lang)
+    doc_findings = [f for f in report.findings if f.doc == doc.doc_id]
+    parts.append(f'<p class="muted">{_e(t("html.docinfo", lang, sha=doc.sha256[:16], total=doc.total_pages, analysed=len(doc.pages)))}</p>')
+    parts.append("<p>" + " ".join(_badge(f, lang) for f in doc_findings if f.page is None) + "</p>")
+    for name, out in doc.outputs.items():
+        if out.status != "ok" or out.stderr:
+            shown = t(out.error_key, lang, **out.error_params) if out.error_key else out.error
+            parts.append(
+                f"<details><summary>{_e(name)}: {_e(out.status)} {_e(shown)}</summary>"
+                f"<pre>{_e(out.stderr or t('html.no_stderr', lang))}</pre></details>"
+            )
+    for idx, page in enumerate(doc.pages):
+        page_findings = [f for f in doc_findings if f.page == page]
+        parts.append(f'<div class="page" id="p{page}"><h2>{_e(t("html.page", lang, page=page))}</h2>')
+        parts.append("<p>" + " ".join(_badge(f, lang) for f in page_findings)
+                     + f" <span class=\"muted\">{_e(t('html.layer', lang, chars=doc.layer_chars[idx]))}</span></p>")
+        if doc.metrics and doc.metrics.agreement.get(page):
+            pairs = ", ".join(
+                f"{a}/{b} {v:.2f}" for (a, b), v in doc.metrics.agreement[page].items() if v is not None
+            )
+            if pairs:
+                parts.append(f'<p class="muted">{_e(t("html.agreement_row", lang, pairs=pairs))}</p>')
+        parts.append('<div class="grid">')
+        if doc.rendered:
+            parts.append(f'<div class="col img"><b>{_e(t("html.original", lang))}</b><br><img src="img/{_e(doc.slug)}-p{page}.png" alt="{_e(t("html.page_alt", lang, page=page))}"></div>')
+        for name, out in doc.outputs.items():
+            pm = doc.metrics.parsers.get(name) if doc.metrics else None
+            meta = ""
+            if pm:
+                m = pm.pages[idx]
+                meta = ("<div class=\"muted\">"
+                        + _e(t("html.page_metrics", lang, chars=m.chars, coverage=_pct(m.coverage), tables=m.tables, headings=m.headings))
+                        + "</div>")
+            text = out.pages[idx] if out.status == "ok" and idx < len(out.pages) else f"({out.status})"
+            parts.append(f'<div class="col"><b>{_e(name)}</b>{meta}<pre>{_e(text)}</pre></div>')
+        parts.append("</div></div>")
+    return _page(doc.doc_id, "\n".join(parts), lang)

parseoff/report_text.py

@@ -0,0 +1,139 @@
+"""summary.json and summary.md. No timestamps, no absolute paths, sorted keys: diff-friendly."""
+
+from __future__ import annotations
+
+import json
+
+from . import __version__
+from .findings import message_for, skip_reason
+from .i18n import DEFAULT_LANG, t
+from .model import RunReport
+from .registry import REGISTRY, SURVEY_NOTE, installed_version, size_text, survey_note
+
+NOTICE = "This output contains text from your documents. parseoff sends nothing over the network."
+DEMO_NOTICE = "Practice (mock) mode: synthetic sample PDFs, but the parsers are real."
+
+
+def _r(value: float | None) -> float | None:
+    return None if value is None else round(value, 4)
+
+
+def _pair(pair: tuple[str, str]) -> str:
+    return f"{pair[0]} | {pair[1]}"
+
+
+def build_summary(report: RunReport) -> dict:
+    parsers = []
+    for name in report.parsers:
+        info = REGISTRY.get(name)
+        parsers.append({
+            "name": name, "status": "ran",
+            "version": installed_version(name) if info else "custom",
+            "license": info.license if info else "unknown (custom)",
+            "install_size_surveyed": info.size_text if info else None,
+        })
+    parsers += [{"name": s.name, "status": "skipped", "reason": s.reason} for s in report.skipped]
+
+    docs = []
+    for doc in report.docs:
+        entry: dict = {
+            "id": doc.doc_id, "sha256": doc.sha256, "total_pages": doc.total_pages,
+            "analyzed_pages": doc.pages, "text_layer_chars": doc.layer_chars,
+        }
+        if doc.open_error:
+            entry["error"] = doc.open_error
+        per_parser = {}
+        for name, out in doc.outputs.items():
+            item: dict = {"status": out.status, "version": out.version, "seconds": round(out.seconds, 3)}
+            if out.error:
+                item["error"] = out.error
+            pm = doc.metrics.parsers.get(name) if doc.metrics else None
+            if pm:
+                item.update({
+                    "mean_coverage": _r(pm.mean_coverage),
+                    "boilerplate_ratio": _r(pm.boilerplate_ratio),
+                    "pages": [
+                        {"page": p, "chars": m.chars, "tokens_est": m.tokens_est,
+                         "coverage": _r(m.coverage), "tables": m.tables, "headings": m.headings}
+                        for p, m in zip(doc.pages, pm.pages)
+                    ],
+                })
+            per_parser[name] = item
+        entry["parsers"] = per_parser
+        if doc.metrics:
+            entry["agreement"] = {
+                _pair(pair): {
+                    "mean": _r(mean),
+                    "pages": {str(p): _r(doc.metrics.agreement[p][pair]) for p in doc.pages},
+                }
+                for pair, mean in doc.metrics.mean_agreement.items()
+            }
+        docs.append(entry)
+
+    return {
+        "parseoff_version": __version__,
+        "mode": "demo" if report.demo else "run",
+        "notes": {"tokens_est": "characters / 4, an estimate", "install_size": SURVEY_NOTE},
+        "thresholds": {"min_coverage": report.thresholds.min_coverage, "min_agreement": report.thresholds.min_agreement},
+        "counts": report.counts(),
+        "parsers": parsers,
+        "documents": docs,
+        "findings": [
+            {"code": f.code, "level": f.level, "doc": f.doc, "page": f.page, "parser": f.parser, "message": f.message}
+            for f in report.findings
+        ],
+    }
+
+
+def summary_json(report: RunReport) -> str:
+    return json.dumps(build_summary(report), indent=2, sort_keys=True, ensure_ascii=False) + "\n"
+
+
+def _md(value: object) -> str:
+    """One-line, pipe-safe text for a Markdown table cell."""
+    return " ".join(str(value).split()).replace("|", "\\|")
+
+
+def _cell(value: float | None, pct: bool = False) -> str:
+    if value is None:
+        return "-"
+    return f"{value:.0%}" if pct else f"{value:.2f}"
+
+
+def summary_markdown(report: RunReport, lang: str = DEFAULT_LANG) -> str:
+    """summary.md in the screen language. Numbers, parser names and finding codes never change."""
+    c = report.counts()
+    lines = [f"# {t('md.title', lang)}", ""]
+    lines.append(f"> {t('notice.demo' if report.demo else 'notice', lang)}")
+    lines += ["", f"FAIL {c['FAIL']}, WARN {c['WARN']}, INFO {c['INFO']}", "", f"## {t('md.parsers', lang)}", ""]
+    heads = [t("tbl.parser", lang), t("tbl.version", lang), t("tbl.license", lang), t("md.size_head", lang), t("tbl.status", lang)]
+    lines += ["| " + " | ".join(heads) + " |", "|---|---|---|---|---|"]
+    skipped = {s.name: s for s in report.skipped}
+    for p in build_summary(report)["parsers"]:
+        if p["status"] == "ran":
+            status = t("md.ran", lang)
+        else:
+            status = t("md.skipped", lang, reason=skip_reason(skipped[p["name"]], lang))
+        info = REGISTRY.get(p["name"])
+        license_ = p.get("license", "-") if info or p["status"] != "ran" else t("license.custom", lang)
+        size = (size_text(p["name"], lang) if info and p["status"] == "ran" else None) or "-"
+        lines.append(f"| {_md(p['name'])} | {p.get('version') or '-'} | {_md(license_)} | {_md(size)} | {_md(status)} |")
+    lines += ["", f"## {t('md.documents', lang)}", ""]
+    heads = [t(f"tbl.{k}", lang) for k in ("document", "parser", "status", "seconds", "coverage", "tables", "headings", "tokens")]
+    lines += ["| " + " | ".join(heads) + " |", "|---|---|---|---|---|---|---|---|"]
+    for doc in report.docs:
+        for name, out in doc.outputs.items():
+            pm = doc.metrics.parsers.get(name) if doc.metrics else None
+            tables = sum(m.tables for m in pm.pages) if pm else "-"
+            heads_n = sum(m.headings for m in pm.pages) if pm else "-"
+            toks = sum(m.tokens_est for m in pm.pages) if pm else "-"
+            cov = _cell(pm.mean_coverage, pct=True) if pm else "-"
+            lines.append(f"| {_md(doc.doc_id)} | {_md(name)} | {out.status} | {out.seconds:.2f} | {cov} | {tables} | {heads_n} | {toks} |")
+    lines += ["", f"## {t('md.findings', lang)}", ""]
+    if not report.findings:
+        lines.append(t("md.no_findings", lang))
+    for f in report.findings:
+        where = " ".join(x for x in (f.doc, f"p{f.page}" if f.page else None, f.parser) if x)
+        lines.append(f"- **{f.level}** `{f.code}` {_md(where)}: {_md(message_for(f, lang))}")
+    lines += ["", t("md.foot", lang, survey=survey_note(lang)), ""]
+    return "\n".join(lines)