parsearena 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,148 @@
1
+ """Shared harness for parser runners.
2
+
3
+ Runners execute inside each parser's own isolated virtual environment, so this
4
+ file must only use the standard library. It is copied next to every runner.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import os
11
+ import signal
12
+ import subprocess
13
+ import sys
14
+ import time
15
+ import traceback
16
+
17
+
18
+ def pick_device(requested: str) -> str:
19
+ if requested != "auto":
20
+ return requested
21
+ try:
22
+ import torch
23
+
24
+ if torch.cuda.is_available():
25
+ return "cuda"
26
+ mps = getattr(torch.backends, "mps", None)
27
+ if mps is not None and mps.is_available():
28
+ return "mps"
29
+ except Exception:
30
+ pass
31
+ return "cpu"
32
+
33
+
34
+ def log(msg: str) -> None:
35
+ print(msg, file=sys.stderr, flush=True)
36
+
37
+
38
+ def descendants(pid: int) -> list[int]:
39
+ """All processes below ``pid``, found through ``ps`` (no psutil in parser envs)."""
40
+ try:
41
+ out = subprocess.run(
42
+ ["ps", "-A", "-o", "pid=,ppid="], capture_output=True, text=True, timeout=10
43
+ ).stdout
44
+ except (OSError, subprocess.SubprocessError):
45
+ return []
46
+ children: dict[int, list[int]] = {}
47
+ for line in out.splitlines():
48
+ parts = line.split()
49
+ if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit():
50
+ children.setdefault(int(parts[1]), []).append(int(parts[0]))
51
+ found, stack = [], [pid]
52
+ while stack:
53
+ for child in children.get(stack.pop(), []):
54
+ found.append(child)
55
+ stack.append(child)
56
+ return found
57
+
58
+
59
+ def kill_descendants(grace: float = 3.0) -> None:
60
+ """Stop every process this runner started, including helper servers that a parser
61
+ spawned in their own session (Marker's Surya servers do), so none outlive the run."""
62
+ pids = descendants(os.getpid())
63
+ for sig in (signal.SIGTERM, signal.SIGKILL):
64
+ alive = []
65
+ for p in pids:
66
+ try:
67
+ os.kill(p, sig)
68
+ alive.append(p)
69
+ except (ProcessLookupError, PermissionError):
70
+ pass
71
+ if not alive or sig == signal.SIGKILL:
72
+ break
73
+ deadline = time.time() + grace
74
+ while time.time() < deadline and any(_alive(p) for p in alive):
75
+ time.sleep(0.1)
76
+ pids = [p for p in alive if _alive(p)]
77
+
78
+
79
+ def _alive(pid: int) -> bool:
80
+ try:
81
+ os.kill(pid, 0)
82
+ except (ProcessLookupError, PermissionError):
83
+ return False
84
+ try: # a zombie child still answers kill(0); reap it if it is ours
85
+ done, _ = os.waitpid(pid, os.WNOHANG)
86
+ return done == 0
87
+ except ChildProcessError:
88
+ return True
89
+
90
+
91
+ def run(setup, convert, version: str, device: str | None = None, teardown=None) -> None:
92
+ """Run the parser, then always shut its helpers down (see kill_descendants)."""
93
+ try:
94
+ _run(setup, convert, version, device)
95
+ finally:
96
+ if teardown is not None:
97
+ try:
98
+ teardown()
99
+ except Exception as exc: # noqa: BLE001
100
+ log(f"teardown failed: {type(exc).__name__}: {exc}")
101
+ kill_descendants()
102
+
103
+
104
+ def _run(setup, convert, version: str, device: str | None = None) -> None:
105
+ """Load the manifest, set the parser up once, convert every document.
106
+
107
+ setup(device, options) -> state
108
+ convert(state, pdf_path, num_pages) -> list[str] (one markdown/text string per page)
109
+ """
110
+ with open(sys.argv[1], encoding="utf-8") as fh:
111
+ manifest = json.load(fh)
112
+ # A runner that does not use torch devices passes its own device label.
113
+ device = device or pick_device(manifest.get("device", "auto"))
114
+ log(f"device={device}")
115
+ t0 = time.perf_counter()
116
+ state = setup(device, manifest.get("options", {}))
117
+ docs = manifest["docs"]
118
+ warmup = None
119
+ wdoc = manifest.get("warmup_doc")
120
+ if wdoc:
121
+ # Most parsers load models lazily (layout, tables, OCR). Convert a small
122
+ # warm-up PDF (one text page, one image-only page) untimed, so per-page
123
+ # timings measure parsing and not model loading.
124
+ try:
125
+ convert(state, wdoc["path"], wdoc["num_pages"])
126
+ except Exception as exc: # noqa: BLE001
127
+ log(f"warmup failed: {type(exc).__name__}: {exc}")
128
+ warmup = time.perf_counter() - t0
129
+ log(f"warmup {warmup:.1f}s")
130
+ for doc in docs:
131
+ res: dict = {"device": device, "version": version, "warmup_seconds": warmup}
132
+ t = time.perf_counter()
133
+ try:
134
+ pages = convert(state, doc["path"], doc["num_pages"])
135
+ pages = [p or "" for p in pages][: doc["num_pages"]]
136
+ pages += [""] * (doc["num_pages"] - len(pages))
137
+ res.update(pages=pages, error=None)
138
+ except Exception as exc: # noqa: BLE001 - every failure is recorded, not raised
139
+ res.update(
140
+ pages=[],
141
+ error=f"{type(exc).__name__}: {exc}"[:1000],
142
+ traceback=traceback.format_exc()[-4000:],
143
+ )
144
+ res["seconds"] = time.perf_counter() - t
145
+ with open(doc["out"], "w", encoding="utf-8") as fh:
146
+ json.dump(res, fh, ensure_ascii=False)
147
+ status = "failed" if res["error"] else "ok"
148
+ log(f"{status} {doc['id']} {res['seconds']:.1f}s")
@@ -0,0 +1,25 @@
1
+ """Docling with its default PDF pipeline (layout, table structure, automatic OCR)."""
2
+
3
+ from importlib.metadata import version
4
+
5
+ from _common import run
6
+
7
+
8
+ def setup(device, options):
9
+ from docling.datamodel.accelerator_options import AcceleratorOptions
10
+ from docling.datamodel.base_models import InputFormat
11
+ from docling.datamodel.pipeline_options import PdfPipelineOptions
12
+ from docling.document_converter import DocumentConverter, PdfFormatOption
13
+
14
+ opts = PdfPipelineOptions()
15
+ opts.accelerator_options = AcceleratorOptions(device=device)
16
+ return DocumentConverter(format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=opts)})
17
+
18
+
19
+ def convert(converter, path, num_pages):
20
+ doc = converter.convert(path).document
21
+ return [doc.export_to_markdown(page_no=i) for i in range(1, num_pages + 1)]
22
+
23
+
24
+ if __name__ == "__main__":
25
+ run(setup, convert, version("docling"))
@@ -0,0 +1,54 @@
1
+ """Marker with its default models, markdown output, paginated so pages can be aligned."""
2
+
3
+ import re
4
+ from importlib.metadata import version
5
+
6
+ from _common import run
7
+
8
+ SEP = re.compile(r"\n*\{(\d+)\}-{10,}\n*")
9
+
10
+
11
+ def setup(device, options):
12
+ import os
13
+
14
+ if device != "auto":
15
+ os.environ.setdefault("TORCH_DEVICE", device)
16
+ from marker.config.parser import ConfigParser
17
+ from marker.converters.pdf import PdfConverter
18
+ from marker.models import create_model_dict
19
+
20
+ config = ConfigParser(
21
+ {"output_format": "markdown", "paginate_output": True, "disable_image_extraction": True}
22
+ )
23
+ return PdfConverter(
24
+ artifact_dict=create_model_dict(),
25
+ processor_list=config.get_processors(),
26
+ renderer=config.get_renderer(),
27
+ config=config.generate_config_dict(),
28
+ )
29
+
30
+
31
+ def convert(converter, path, num_pages):
32
+ text = converter(path).markdown
33
+ pages = [""] * num_pages
34
+ parts = SEP.split(text)
35
+ # parts = [before, id0, text0, id1, text1, ...]
36
+ for i in range(1, len(parts) - 1, 2):
37
+ idx = int(parts[i])
38
+ if 0 <= idx < num_pages:
39
+ pages[idx] = parts[i + 1].strip()
40
+ if len(parts) == 1:
41
+ pages[0] = text
42
+ return pages
43
+
44
+
45
+ if __name__ == "__main__":
46
+
47
+ def teardown():
48
+ # Marker 2.x starts Surya inference servers (layout, OCR error, llama.cpp);
49
+ # stop them before the runner exits.
50
+ from marker.models import shutdown_models
51
+
52
+ shutdown_models()
53
+
54
+ run(setup, convert, version("marker-pdf"), teardown=teardown)
@@ -0,0 +1,59 @@
1
+ """MinerU local analysis at effort "medium" (layout, OCR, tables, formulas; no VLM).
2
+
3
+ MinerU's "high"/"xhigh" efforts route pages through a vision-language model,
4
+ which is out of scope for parsearena v0.1.
5
+ """
6
+
7
+ import os
8
+ from importlib.metadata import version
9
+
10
+ from _common import run
11
+
12
+ os.environ.setdefault("MINERU_TELEMETRY_DISABLED", "1")
13
+
14
+
15
+ def setup(device, options):
16
+ if device != "auto":
17
+ os.environ.setdefault("MINERU_DEVICE_MODE", device)
18
+ from mineru.backend.analyze import doc_analyze
19
+ from mineru.render.content_list import render_content_list
20
+
21
+ return doc_analyze, render_content_list
22
+
23
+
24
+ def _render(item) -> str:
25
+ kind = item.get("type")
26
+ if kind == "table":
27
+ parts = [*item.get("table_caption", []), item.get("table_body", ""), *item.get("table_footnote", [])]
28
+ return "\n\n".join(p for p in parts if p)
29
+ if kind in ("image", "chart"):
30
+ parts = [
31
+ *item.get("image_caption", []),
32
+ *item.get("chart_caption", []),
33
+ *item.get("image_footnote", []),
34
+ ]
35
+ return "\n\n".join(p for p in parts if p)
36
+ if kind == "list":
37
+ return "\n".join(f"- {x}" for x in item.get("list_items", []))
38
+ text = item.get("text", "") or ""
39
+ level = item.get("text_level")
40
+ if kind == "text" and level:
41
+ return "#" * min(int(level) + 1, 6) + " " + text
42
+ return text
43
+
44
+
45
+ def convert(state, path, num_pages):
46
+ doc_analyze, render_content_list = state
47
+ with open(path, "rb") as fh:
48
+ middle_json, _ = doc_analyze(fh.read(), effort="medium")
49
+ pages = [[] for _ in range(num_pages)]
50
+ for item in render_content_list(middle_json):
51
+ idx = item.get("page_idx", 0)
52
+ text = _render(item).strip()
53
+ if text and 0 <= idx < num_pages:
54
+ pages[idx].append(text)
55
+ return ["\n\n".join(p) for p in pages]
56
+
57
+
58
+ if __name__ == "__main__":
59
+ run(setup, convert, version("mineru"))
@@ -0,0 +1,22 @@
1
+ """pypdf baseline: the text layer of each page, nothing else."""
2
+
3
+ import logging
4
+ from importlib.metadata import version
5
+
6
+ from _common import run
7
+
8
+ logging.disable(logging.WARNING)
9
+
10
+
11
+ def setup(device, options):
12
+ return None
13
+
14
+
15
+ def convert(state, path, num_pages):
16
+ from pypdf import PdfReader
17
+
18
+ return [page.extract_text() or "" for page in PdfReader(path).pages]
19
+
20
+
21
+ if __name__ == "__main__":
22
+ run(setup, convert, version("pypdf"), device="cpu")
@@ -0,0 +1,38 @@
1
+ """Unstructured partition_pdf with the hi_res strategy and table structure inference."""
2
+
3
+ from importlib.metadata import version
4
+
5
+ from _common import run
6
+
7
+
8
+ def setup(device, options):
9
+ from unstructured.partition.pdf import partition_pdf
10
+
11
+ return partition_pdf
12
+
13
+
14
+ def _render(el) -> str:
15
+ category = getattr(el, "category", "")
16
+ if category == "Table":
17
+ html = getattr(el.metadata, "text_as_html", None)
18
+ return html or el.text
19
+ if category == "Title":
20
+ return f"## {el.text}"
21
+ if category == "ListItem":
22
+ return f"- {el.text}"
23
+ return el.text
24
+
25
+
26
+ def convert(partition_pdf, path, num_pages):
27
+ elements = partition_pdf(filename=path, strategy="hi_res", infer_table_structure=True)
28
+ pages = [[] for _ in range(num_pages)]
29
+ for el in elements:
30
+ page = (getattr(el.metadata, "page_number", None) or 1) - 1
31
+ if 0 <= page < num_pages and el.text and el.text.strip():
32
+ pages[page].append(_render(el))
33
+ return ["\n\n".join(p) for p in pages]
34
+
35
+
36
+ if __name__ == "__main__":
37
+ # hi_res runs its layout model with onnxruntime on the CPU.
38
+ run(setup, convert, version("unstructured"), device="cpu")
parsearena/text.py ADDED
@@ -0,0 +1,90 @@
1
+ """Normalization and the one chunker every parser output goes through."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import html
6
+ import re
7
+ import unicodedata
8
+ from dataclasses import dataclass
9
+
10
+ _TAG = re.compile(r"<[^>]{1,200}>")
11
+ _MD_MARKUP = re.compile(r"[*_`#|>~$\\]")
12
+ _HYPHEN_BREAK = re.compile(r"(\w)-\s*\n\s*(\w)")
13
+ _WS = re.compile(r"\s+")
14
+ _WORD = re.compile(r"\w+", re.UNICODE)
15
+ _SPACES = re.compile(r"[ \t\u00a0]{2,}")
16
+ _TABLE_RULE = re.compile(r"^\s*\|?[\s:|-]*-{3,}[\s:|-]*\|?\s*$", re.M)
17
+
18
+
19
+ def normalize(text: str) -> str:
20
+ """Lowercase, strip markdown/HTML markup and collapse whitespace.
21
+
22
+ Used to decide whether an answer string survives parsing, so it removes
23
+ formatting differences between parsers (tables as pipes or HTML, bold,
24
+ LaTeX dollars) but keeps the characters that carry meaning.
25
+ """
26
+ text = unicodedata.normalize("NFKC", text or "")
27
+ text = _HYPHEN_BREAK.sub(r"\1\2", text)
28
+ text = html.unescape(_TAG.sub(" ", text))
29
+ text = _MD_MARKUP.sub(" ", text)
30
+ text = text.replace("−", "-").replace("–", "-").replace("—", "-")
31
+ return _WS.sub(" ", text).strip().lower()
32
+
33
+
34
+ def contains(haystack_norm: str, needle: str) -> bool:
35
+ n = normalize(needle)
36
+ return bool(n) and n in haystack_norm
37
+
38
+
39
+ def tokenize(text: str) -> list[str]:
40
+ return _WORD.findall(normalize(text))
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class Chunk:
45
+ doc: str
46
+ index: int
47
+ text: str
48
+
49
+
50
+ def _split_long(block: str, max_chars: int) -> list[str]:
51
+ if len(block) <= max_chars:
52
+ return [block]
53
+ parts: list[str] = []
54
+ for line in block.split("\n"):
55
+ while len(line) > max_chars:
56
+ cut = line.rfind(" ", 0, max_chars)
57
+ cut = cut if cut > max_chars // 2 else max_chars
58
+ parts.append(line[:cut])
59
+ line = line[cut:].lstrip()
60
+ parts.append(line)
61
+ return [p for p in parts if p.strip()]
62
+
63
+
64
+ def chunk_text(doc: str, text: str, max_chars: int = 1200) -> list[Chunk]:
65
+ """Pack paragraphs greedily into chunks of at most ``max_chars``.
66
+
67
+ Paragraphs are separated by blank lines. A paragraph that is too long (a big
68
+ table) is split by lines, and a line that is too long by words. The same
69
+ function is applied to every parser, so differences in retrieval come from
70
+ the parser output and not from the chunking.
71
+ """
72
+ # Padding spaces and markdown table rules carry no content; drop them so
73
+ # chunk size measures text, not layout.
74
+ text = _TABLE_RULE.sub("", _SPACES.sub(" ", text or ""))
75
+ pieces: list[str] = []
76
+ for block in re.split(r"\n\s*\n", text):
77
+ block = block.strip()
78
+ if block:
79
+ pieces.extend(_split_long(block, max_chars))
80
+ chunks: list[str] = []
81
+ current = ""
82
+ for piece in pieces:
83
+ if current and len(current) + 2 + len(piece) > max_chars:
84
+ chunks.append(current)
85
+ current = piece
86
+ else:
87
+ current = f"{current}\n\n{piece}" if current else piece
88
+ if current:
89
+ chunks.append(current)
90
+ return [Chunk(doc=doc, index=i, text=c) for i, c in enumerate(chunks)]