parsearena 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsearena/__init__.py +3 -0
- parsearena/cli.py +288 -0
- parsearena/demo/pdfs/NOTICE.md +36 -0
- parsearena/demo/pdfs/census-income-2023.pdf +0 -0
- parsearena/demo/pdfs/dpo-arxiv-2305.18290.pdf +0 -0
- parsearena/demo/pdfs/naca-report-1135-scan.pdf +0 -0
- parsearena/demo/questions.yaml +104 -0
- parsearena/demo/recorded.json +304 -0
- parsearena/errors.py +14 -0
- parsearena/evaluate.py +130 -0
- parsearena/llm.py +154 -0
- parsearena/parsers.py +197 -0
- parsearena/pipeline.py +276 -0
- parsearena/questions.py +125 -0
- parsearena/render.py +40 -0
- parsearena/report.py +57 -0
- parsearena/report_template.html +450 -0
- parsearena/retrieval.py +197 -0
- parsearena/runners/_common.py +148 -0
- parsearena/runners/run_docling.py +25 -0
- parsearena/runners/run_marker.py +54 -0
- parsearena/runners/run_mineru.py +59 -0
- parsearena/runners/run_pypdf.py +22 -0
- parsearena/runners/run_unstructured.py +38 -0
- parsearena/text.py +90 -0
- parsearena-0.1.3.dist-info/METADATA +267 -0
- parsearena-0.1.3.dist-info/RECORD +30 -0
- parsearena-0.1.3.dist-info/WHEEL +4 -0
- parsearena-0.1.3.dist-info/entry_points.txt +2 -0
- parsearena-0.1.3.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Shared harness for parser runners.
|
|
2
|
+
|
|
3
|
+
Runners execute inside each parser's own isolated virtual environment, so this
|
|
4
|
+
file must only use the standard library. It is copied next to every runner.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import signal
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
import traceback
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def pick_device(requested: str) -> str:
|
|
19
|
+
if requested != "auto":
|
|
20
|
+
return requested
|
|
21
|
+
try:
|
|
22
|
+
import torch
|
|
23
|
+
|
|
24
|
+
if torch.cuda.is_available():
|
|
25
|
+
return "cuda"
|
|
26
|
+
mps = getattr(torch.backends, "mps", None)
|
|
27
|
+
if mps is not None and mps.is_available():
|
|
28
|
+
return "mps"
|
|
29
|
+
except Exception:
|
|
30
|
+
pass
|
|
31
|
+
return "cpu"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def log(msg: str) -> None:
|
|
35
|
+
print(msg, file=sys.stderr, flush=True)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def descendants(pid: int) -> list[int]:
|
|
39
|
+
"""All processes below ``pid``, found through ``ps`` (no psutil in parser envs)."""
|
|
40
|
+
try:
|
|
41
|
+
out = subprocess.run(
|
|
42
|
+
["ps", "-A", "-o", "pid=,ppid="], capture_output=True, text=True, timeout=10
|
|
43
|
+
).stdout
|
|
44
|
+
except (OSError, subprocess.SubprocessError):
|
|
45
|
+
return []
|
|
46
|
+
children: dict[int, list[int]] = {}
|
|
47
|
+
for line in out.splitlines():
|
|
48
|
+
parts = line.split()
|
|
49
|
+
if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit():
|
|
50
|
+
children.setdefault(int(parts[1]), []).append(int(parts[0]))
|
|
51
|
+
found, stack = [], [pid]
|
|
52
|
+
while stack:
|
|
53
|
+
for child in children.get(stack.pop(), []):
|
|
54
|
+
found.append(child)
|
|
55
|
+
stack.append(child)
|
|
56
|
+
return found
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def kill_descendants(grace: float = 3.0) -> None:
|
|
60
|
+
"""Stop every process this runner started, including helper servers that a parser
|
|
61
|
+
spawned in their own session (Marker's Surya servers do), so none outlive the run."""
|
|
62
|
+
pids = descendants(os.getpid())
|
|
63
|
+
for sig in (signal.SIGTERM, signal.SIGKILL):
|
|
64
|
+
alive = []
|
|
65
|
+
for p in pids:
|
|
66
|
+
try:
|
|
67
|
+
os.kill(p, sig)
|
|
68
|
+
alive.append(p)
|
|
69
|
+
except (ProcessLookupError, PermissionError):
|
|
70
|
+
pass
|
|
71
|
+
if not alive or sig == signal.SIGKILL:
|
|
72
|
+
break
|
|
73
|
+
deadline = time.time() + grace
|
|
74
|
+
while time.time() < deadline and any(_alive(p) for p in alive):
|
|
75
|
+
time.sleep(0.1)
|
|
76
|
+
pids = [p for p in alive if _alive(p)]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _alive(pid: int) -> bool:
|
|
80
|
+
try:
|
|
81
|
+
os.kill(pid, 0)
|
|
82
|
+
except (ProcessLookupError, PermissionError):
|
|
83
|
+
return False
|
|
84
|
+
try: # a zombie child still answers kill(0); reap it if it is ours
|
|
85
|
+
done, _ = os.waitpid(pid, os.WNOHANG)
|
|
86
|
+
return done == 0
|
|
87
|
+
except ChildProcessError:
|
|
88
|
+
return True
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def run(setup, convert, version: str, device: str | None = None, teardown=None) -> None:
|
|
92
|
+
"""Run the parser, then always shut its helpers down (see kill_descendants)."""
|
|
93
|
+
try:
|
|
94
|
+
_run(setup, convert, version, device)
|
|
95
|
+
finally:
|
|
96
|
+
if teardown is not None:
|
|
97
|
+
try:
|
|
98
|
+
teardown()
|
|
99
|
+
except Exception as exc: # noqa: BLE001
|
|
100
|
+
log(f"teardown failed: {type(exc).__name__}: {exc}")
|
|
101
|
+
kill_descendants()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _run(setup, convert, version: str, device: str | None = None) -> None:
|
|
105
|
+
"""Load the manifest, set the parser up once, convert every document.
|
|
106
|
+
|
|
107
|
+
setup(device, options) -> state
|
|
108
|
+
convert(state, pdf_path, num_pages) -> list[str] (one markdown/text string per page)
|
|
109
|
+
"""
|
|
110
|
+
with open(sys.argv[1], encoding="utf-8") as fh:
|
|
111
|
+
manifest = json.load(fh)
|
|
112
|
+
# A runner that does not use torch devices passes its own device label.
|
|
113
|
+
device = device or pick_device(manifest.get("device", "auto"))
|
|
114
|
+
log(f"device={device}")
|
|
115
|
+
t0 = time.perf_counter()
|
|
116
|
+
state = setup(device, manifest.get("options", {}))
|
|
117
|
+
docs = manifest["docs"]
|
|
118
|
+
warmup = None
|
|
119
|
+
wdoc = manifest.get("warmup_doc")
|
|
120
|
+
if wdoc:
|
|
121
|
+
# Most parsers load models lazily (layout, tables, OCR). Convert a small
|
|
122
|
+
# warm-up PDF (one text page, one image-only page) untimed, so per-page
|
|
123
|
+
# timings measure parsing and not model loading.
|
|
124
|
+
try:
|
|
125
|
+
convert(state, wdoc["path"], wdoc["num_pages"])
|
|
126
|
+
except Exception as exc: # noqa: BLE001
|
|
127
|
+
log(f"warmup failed: {type(exc).__name__}: {exc}")
|
|
128
|
+
warmup = time.perf_counter() - t0
|
|
129
|
+
log(f"warmup {warmup:.1f}s")
|
|
130
|
+
for doc in docs:
|
|
131
|
+
res: dict = {"device": device, "version": version, "warmup_seconds": warmup}
|
|
132
|
+
t = time.perf_counter()
|
|
133
|
+
try:
|
|
134
|
+
pages = convert(state, doc["path"], doc["num_pages"])
|
|
135
|
+
pages = [p or "" for p in pages][: doc["num_pages"]]
|
|
136
|
+
pages += [""] * (doc["num_pages"] - len(pages))
|
|
137
|
+
res.update(pages=pages, error=None)
|
|
138
|
+
except Exception as exc: # noqa: BLE001 - every failure is recorded, not raised
|
|
139
|
+
res.update(
|
|
140
|
+
pages=[],
|
|
141
|
+
error=f"{type(exc).__name__}: {exc}"[:1000],
|
|
142
|
+
traceback=traceback.format_exc()[-4000:],
|
|
143
|
+
)
|
|
144
|
+
res["seconds"] = time.perf_counter() - t
|
|
145
|
+
with open(doc["out"], "w", encoding="utf-8") as fh:
|
|
146
|
+
json.dump(res, fh, ensure_ascii=False)
|
|
147
|
+
status = "failed" if res["error"] else "ok"
|
|
148
|
+
log(f"{status} {doc['id']} {res['seconds']:.1f}s")
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Docling with its default PDF pipeline (layout, table structure, automatic OCR)."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import version
|
|
4
|
+
|
|
5
|
+
from _common import run
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def setup(device, options):
|
|
9
|
+
from docling.datamodel.accelerator_options import AcceleratorOptions
|
|
10
|
+
from docling.datamodel.base_models import InputFormat
|
|
11
|
+
from docling.datamodel.pipeline_options import PdfPipelineOptions
|
|
12
|
+
from docling.document_converter import DocumentConverter, PdfFormatOption
|
|
13
|
+
|
|
14
|
+
opts = PdfPipelineOptions()
|
|
15
|
+
opts.accelerator_options = AcceleratorOptions(device=device)
|
|
16
|
+
return DocumentConverter(format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=opts)})
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def convert(converter, path, num_pages):
|
|
20
|
+
doc = converter.convert(path).document
|
|
21
|
+
return [doc.export_to_markdown(page_no=i) for i in range(1, num_pages + 1)]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
if __name__ == "__main__":
|
|
25
|
+
run(setup, convert, version("docling"))
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Marker with its default models, markdown output, paginated so pages can be aligned."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from importlib.metadata import version
|
|
5
|
+
|
|
6
|
+
from _common import run
|
|
7
|
+
|
|
8
|
+
SEP = re.compile(r"\n*\{(\d+)\}-{10,}\n*")
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def setup(device, options):
|
|
12
|
+
import os
|
|
13
|
+
|
|
14
|
+
if device != "auto":
|
|
15
|
+
os.environ.setdefault("TORCH_DEVICE", device)
|
|
16
|
+
from marker.config.parser import ConfigParser
|
|
17
|
+
from marker.converters.pdf import PdfConverter
|
|
18
|
+
from marker.models import create_model_dict
|
|
19
|
+
|
|
20
|
+
config = ConfigParser(
|
|
21
|
+
{"output_format": "markdown", "paginate_output": True, "disable_image_extraction": True}
|
|
22
|
+
)
|
|
23
|
+
return PdfConverter(
|
|
24
|
+
artifact_dict=create_model_dict(),
|
|
25
|
+
processor_list=config.get_processors(),
|
|
26
|
+
renderer=config.get_renderer(),
|
|
27
|
+
config=config.generate_config_dict(),
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def convert(converter, path, num_pages):
|
|
32
|
+
text = converter(path).markdown
|
|
33
|
+
pages = [""] * num_pages
|
|
34
|
+
parts = SEP.split(text)
|
|
35
|
+
# parts = [before, id0, text0, id1, text1, ...]
|
|
36
|
+
for i in range(1, len(parts) - 1, 2):
|
|
37
|
+
idx = int(parts[i])
|
|
38
|
+
if 0 <= idx < num_pages:
|
|
39
|
+
pages[idx] = parts[i + 1].strip()
|
|
40
|
+
if len(parts) == 1:
|
|
41
|
+
pages[0] = text
|
|
42
|
+
return pages
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
if __name__ == "__main__":
|
|
46
|
+
|
|
47
|
+
def teardown():
|
|
48
|
+
# Marker 2.x starts Surya inference servers (layout, OCR error, llama.cpp);
|
|
49
|
+
# stop them before the runner exits.
|
|
50
|
+
from marker.models import shutdown_models
|
|
51
|
+
|
|
52
|
+
shutdown_models()
|
|
53
|
+
|
|
54
|
+
run(setup, convert, version("marker-pdf"), teardown=teardown)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""MinerU local analysis at effort "medium" (layout, OCR, tables, formulas; no VLM).
|
|
2
|
+
|
|
3
|
+
MinerU's "high"/"xhigh" efforts route pages through a vision-language model,
|
|
4
|
+
which is out of scope for parsearena v0.1.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
from importlib.metadata import version
|
|
9
|
+
|
|
10
|
+
from _common import run
|
|
11
|
+
|
|
12
|
+
os.environ.setdefault("MINERU_TELEMETRY_DISABLED", "1")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def setup(device, options):
|
|
16
|
+
if device != "auto":
|
|
17
|
+
os.environ.setdefault("MINERU_DEVICE_MODE", device)
|
|
18
|
+
from mineru.backend.analyze import doc_analyze
|
|
19
|
+
from mineru.render.content_list import render_content_list
|
|
20
|
+
|
|
21
|
+
return doc_analyze, render_content_list
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _render(item) -> str:
|
|
25
|
+
kind = item.get("type")
|
|
26
|
+
if kind == "table":
|
|
27
|
+
parts = [*item.get("table_caption", []), item.get("table_body", ""), *item.get("table_footnote", [])]
|
|
28
|
+
return "\n\n".join(p for p in parts if p)
|
|
29
|
+
if kind in ("image", "chart"):
|
|
30
|
+
parts = [
|
|
31
|
+
*item.get("image_caption", []),
|
|
32
|
+
*item.get("chart_caption", []),
|
|
33
|
+
*item.get("image_footnote", []),
|
|
34
|
+
]
|
|
35
|
+
return "\n\n".join(p for p in parts if p)
|
|
36
|
+
if kind == "list":
|
|
37
|
+
return "\n".join(f"- {x}" for x in item.get("list_items", []))
|
|
38
|
+
text = item.get("text", "") or ""
|
|
39
|
+
level = item.get("text_level")
|
|
40
|
+
if kind == "text" and level:
|
|
41
|
+
return "#" * min(int(level) + 1, 6) + " " + text
|
|
42
|
+
return text
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def convert(state, path, num_pages):
|
|
46
|
+
doc_analyze, render_content_list = state
|
|
47
|
+
with open(path, "rb") as fh:
|
|
48
|
+
middle_json, _ = doc_analyze(fh.read(), effort="medium")
|
|
49
|
+
pages = [[] for _ in range(num_pages)]
|
|
50
|
+
for item in render_content_list(middle_json):
|
|
51
|
+
idx = item.get("page_idx", 0)
|
|
52
|
+
text = _render(item).strip()
|
|
53
|
+
if text and 0 <= idx < num_pages:
|
|
54
|
+
pages[idx].append(text)
|
|
55
|
+
return ["\n\n".join(p) for p in pages]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if __name__ == "__main__":
|
|
59
|
+
run(setup, convert, version("mineru"))
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""pypdf baseline: the text layer of each page, nothing else."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from importlib.metadata import version
|
|
5
|
+
|
|
6
|
+
from _common import run
|
|
7
|
+
|
|
8
|
+
logging.disable(logging.WARNING)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def setup(device, options):
|
|
12
|
+
return None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def convert(state, path, num_pages):
|
|
16
|
+
from pypdf import PdfReader
|
|
17
|
+
|
|
18
|
+
return [page.extract_text() or "" for page in PdfReader(path).pages]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
if __name__ == "__main__":
|
|
22
|
+
run(setup, convert, version("pypdf"), device="cpu")
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Unstructured partition_pdf with the hi_res strategy and table structure inference."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import version
|
|
4
|
+
|
|
5
|
+
from _common import run
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def setup(device, options):
|
|
9
|
+
from unstructured.partition.pdf import partition_pdf
|
|
10
|
+
|
|
11
|
+
return partition_pdf
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _render(el) -> str:
|
|
15
|
+
category = getattr(el, "category", "")
|
|
16
|
+
if category == "Table":
|
|
17
|
+
html = getattr(el.metadata, "text_as_html", None)
|
|
18
|
+
return html or el.text
|
|
19
|
+
if category == "Title":
|
|
20
|
+
return f"## {el.text}"
|
|
21
|
+
if category == "ListItem":
|
|
22
|
+
return f"- {el.text}"
|
|
23
|
+
return el.text
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def convert(partition_pdf, path, num_pages):
|
|
27
|
+
elements = partition_pdf(filename=path, strategy="hi_res", infer_table_structure=True)
|
|
28
|
+
pages = [[] for _ in range(num_pages)]
|
|
29
|
+
for el in elements:
|
|
30
|
+
page = (getattr(el.metadata, "page_number", None) or 1) - 1
|
|
31
|
+
if 0 <= page < num_pages and el.text and el.text.strip():
|
|
32
|
+
pages[page].append(_render(el))
|
|
33
|
+
return ["\n\n".join(p) for p in pages]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
if __name__ == "__main__":
|
|
37
|
+
# hi_res runs its layout model with onnxruntime on the CPU.
|
|
38
|
+
run(setup, convert, version("unstructured"), device="cpu")
|
parsearena/text.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Normalization and the one chunker every parser output goes through."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import html
|
|
6
|
+
import re
|
|
7
|
+
import unicodedata
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
|
|
10
|
+
_TAG = re.compile(r"<[^>]{1,200}>")
|
|
11
|
+
_MD_MARKUP = re.compile(r"[*_`#|>~$\\]")
|
|
12
|
+
_HYPHEN_BREAK = re.compile(r"(\w)-\s*\n\s*(\w)")
|
|
13
|
+
_WS = re.compile(r"\s+")
|
|
14
|
+
_WORD = re.compile(r"\w+", re.UNICODE)
|
|
15
|
+
_SPACES = re.compile(r"[ \t\u00a0]{2,}")
|
|
16
|
+
_TABLE_RULE = re.compile(r"^\s*\|?[\s:|-]*-{3,}[\s:|-]*\|?\s*$", re.M)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def normalize(text: str) -> str:
|
|
20
|
+
"""Lowercase, strip markdown/HTML markup and collapse whitespace.
|
|
21
|
+
|
|
22
|
+
Used to decide whether an answer string survives parsing, so it removes
|
|
23
|
+
formatting differences between parsers (tables as pipes or HTML, bold,
|
|
24
|
+
LaTeX dollars) but keeps the characters that carry meaning.
|
|
25
|
+
"""
|
|
26
|
+
text = unicodedata.normalize("NFKC", text or "")
|
|
27
|
+
text = _HYPHEN_BREAK.sub(r"\1\2", text)
|
|
28
|
+
text = html.unescape(_TAG.sub(" ", text))
|
|
29
|
+
text = _MD_MARKUP.sub(" ", text)
|
|
30
|
+
text = text.replace("−", "-").replace("–", "-").replace("—", "-")
|
|
31
|
+
return _WS.sub(" ", text).strip().lower()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def contains(haystack_norm: str, needle: str) -> bool:
|
|
35
|
+
n = normalize(needle)
|
|
36
|
+
return bool(n) and n in haystack_norm
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def tokenize(text: str) -> list[str]:
|
|
40
|
+
return _WORD.findall(normalize(text))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class Chunk:
|
|
45
|
+
doc: str
|
|
46
|
+
index: int
|
|
47
|
+
text: str
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _split_long(block: str, max_chars: int) -> list[str]:
|
|
51
|
+
if len(block) <= max_chars:
|
|
52
|
+
return [block]
|
|
53
|
+
parts: list[str] = []
|
|
54
|
+
for line in block.split("\n"):
|
|
55
|
+
while len(line) > max_chars:
|
|
56
|
+
cut = line.rfind(" ", 0, max_chars)
|
|
57
|
+
cut = cut if cut > max_chars // 2 else max_chars
|
|
58
|
+
parts.append(line[:cut])
|
|
59
|
+
line = line[cut:].lstrip()
|
|
60
|
+
parts.append(line)
|
|
61
|
+
return [p for p in parts if p.strip()]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def chunk_text(doc: str, text: str, max_chars: int = 1200) -> list[Chunk]:
|
|
65
|
+
"""Pack paragraphs greedily into chunks of at most ``max_chars``.
|
|
66
|
+
|
|
67
|
+
Paragraphs are separated by blank lines. A paragraph that is too long (a big
|
|
68
|
+
table) is split by lines, and a line that is too long by words. The same
|
|
69
|
+
function is applied to every parser, so differences in retrieval come from
|
|
70
|
+
the parser output and not from the chunking.
|
|
71
|
+
"""
|
|
72
|
+
# Padding spaces and markdown table rules carry no content; drop them so
|
|
73
|
+
# chunk size measures text, not layout.
|
|
74
|
+
text = _TABLE_RULE.sub("", _SPACES.sub(" ", text or ""))
|
|
75
|
+
pieces: list[str] = []
|
|
76
|
+
for block in re.split(r"\n\s*\n", text):
|
|
77
|
+
block = block.strip()
|
|
78
|
+
if block:
|
|
79
|
+
pieces.extend(_split_long(block, max_chars))
|
|
80
|
+
chunks: list[str] = []
|
|
81
|
+
current = ""
|
|
82
|
+
for piece in pieces:
|
|
83
|
+
if current and len(current) + 2 + len(piece) > max_chars:
|
|
84
|
+
chunks.append(current)
|
|
85
|
+
current = piece
|
|
86
|
+
else:
|
|
87
|
+
current = f"{current}\n\n{piece}" if current else piece
|
|
88
|
+
if current:
|
|
89
|
+
chunks.append(current)
|
|
90
|
+
return [Chunk(doc=doc, index=i, text=c) for i, c in enumerate(chunks)]
|