docmd-cli 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/ARCHITECTURE.md +28 -1
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/PKG-INFO +66 -3
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/README.md +65 -2
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/__init__.py +19 -7
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/cli.py +29 -2
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/converters/marker_converter.py +26 -1
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/errors.py +24 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/postprocess/image_handling.py +11 -1
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/pyproject.toml +1 -1
- docmd_cli-0.1.2/tests/fixtures/generate_stress_fixtures.py +110 -0
- docmd_cli-0.1.2/tests/fixtures/merged_cells.pdf +74 -0
- docmd_cli-0.1.2/tests/fixtures/running_header.pdf +131 -0
- docmd_cli-0.1.2/tests/fixtures/stress.pdf +99 -0
- docmd_cli-0.1.2/tests/fixtures/with_image.pdf +85 -0
- docmd_cli-0.1.2/tests/test_marker_converter_errors.py +49 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/tests/test_postprocess.py +37 -0
- docmd_cli-0.1.2/tests/test_postprocess_integration.py +68 -0
- docmd_cli-0.1.2/tests/test_postprocess_more_integration.py +105 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/.github/workflows/ci.yml +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/.github/workflows/release.yml +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/.gitignore +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/LICENSE +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/config.py +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/converters/__init__.py +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/converters/base.py +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/converters/registry.py +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/postprocess/__init__.py +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/postprocess/heading_normalize.py +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/docmd/postprocess/table_cleanup.py +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/tests/fixtures/generate_fixtures.py +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/tests/fixtures/sample.docx +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/tests/fixtures/sample.pdf +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/tests/test_cli.py +0 -0
- {docmd_cli-0.1.0 → docmd_cli-0.1.2}/tests/test_converters.py +0 -0
|
@@ -3,6 +3,31 @@
|
|
|
3
3
|
## The pitch (keep this pinned above your desk)
|
|
4
4
|
We are not competing on conversion quality. We are selling **"hit an endpoint, get clean Markdown back"** — no Python env, no 8GB+ RAM, no GPU, no dependency hell. The open-source core proves the engine works and builds trust. The hosted API sells convenience.
|
|
5
5
|
|
|
6
|
+
## Real dependency gap found by testing (2026-09-18)
|
|
7
|
+
The pitch above says "no dependency hell." That held for text-layer PDFs, DOCX, and
|
|
8
|
+
PPTX when actually tested against real downloaded documents (an arXiv paper, an IMF
|
|
9
|
+
report). It did **not** hold for OCR: Marker's current recognition model (used for
|
|
10
|
+
both scanned-PDF OCR and equation recognition) is a VLM served through either `vllm`
|
|
11
|
+
or llama.cpp's `llama-server` binary - no plain-transformers/CPU fallback exists in
|
|
12
|
+
this version. Neither ships via pip, so `pip install docmd-cli[full]` alone cannot
|
|
13
|
+
actually OCR a scanned document or handle a PDF with equations, despite the README
|
|
14
|
+
previously implying "OCR - bundled by Marker, free" meant it worked out of the box.
|
|
15
|
+
|
|
16
|
+
Confirmed via a real test: converting a genuinely scanned PDF (downloaded from
|
|
17
|
+
archive.org) raised a missing-binary error on this dev machine (no Homebrew). Fixed
|
|
18
|
+
two ways: (1) a real fix - download a prebuilt `llama-server` binary directly from
|
|
19
|
+
llama.cpp's GitHub releases (no package manager needed) and set `LLAMA_CPP_BINARY`,
|
|
20
|
+
which then produced correct real OCR output; (2) a docmd fix - a dedicated
|
|
21
|
+
`MissingSystemDependencyError` with install instructions instead of a raw stack trace
|
|
22
|
+
when the binary is missing (see `docmd/errors.py`).
|
|
23
|
+
|
|
24
|
+
**Implications to carry into Stage 2/3 (`api/`)**: the hosted API's Docker image
|
|
25
|
+
does not currently install `llama-server` either - meaning a customer uploading a
|
|
26
|
+
scanned PDF or an equation-heavy document to the live API would hit this same error
|
|
27
|
+
today. This needs a fix in `docmd-api`'s Dockerfile before OCR can be honestly
|
|
28
|
+
advertised as working end-to-end there. Not yet fixed as of this note - flagged here
|
|
29
|
+
so it isn't lost.
|
|
30
|
+
|
|
6
31
|
## Positioning risk (checked 2026-09-18)
|
|
7
32
|
Marker's *code* is Apache-2.0 (no restriction). Marker's *model weights* use a modified
|
|
8
33
|
Open RAIL-M license: free for research, personal use, and organizations under $5M in
|
|
@@ -128,7 +153,9 @@ Treat this folder as the actual differentiator. A thin wrapper is a weekend proj
|
|
|
128
153
|
|
|
129
154
|
- `pip install docmd-cli` -> core PDF conversion (Marker bundles OCR here already; there is
|
|
130
155
|
no separate lean/no-OCR base install, since Marker's own base dependencies include
|
|
131
|
-
`surya-ocr`)
|
|
156
|
+
`surya-ocr`) - but actually *running* OCR (or equation recognition) on top of that
|
|
157
|
+
also needs the `llama-server` native binary, which no `pip install` variant can
|
|
158
|
+
provide - see "Real dependency gap found by testing" above
|
|
132
159
|
- `pip install docmd-cli[full]` -> adds DOCX/PPTX/EPUB/XLSX support via Marker's `full` extra
|
|
133
160
|
|
|
134
161
|
## License clarity
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: docmd-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Convert PDFs, DOCX, and PPTX to clean, structure-preserving Markdown.
|
|
5
5
|
Project-URL: Homepage, https://github.com/taherzribi/docmd
|
|
6
6
|
Project-URL: Issues, https://github.com/taherzribi/docmd/issues
|
|
@@ -67,7 +67,7 @@ core, usable standalone today.
|
|
|
67
67
|
| Input | Output |
|
|
68
68
|
|---|---|
|
|
69
69
|
| PDF (text-based) | Markdown with preserved headings, lists, tables |
|
|
70
|
-
| PDF (scanned) | Markdown via OCR — bundled by Marker, free |
|
|
70
|
+
| PDF (scanned) | Markdown via OCR — bundled by Marker, free, but needs [one extra native binary](#ocr-and-equations-need-one-native-binary) |
|
|
71
71
|
| DOCX | Markdown with formatting preserved (`pip install docmd-cli[full]`) |
|
|
72
72
|
| PPTX | Markdown, one section per slide (`pip install docmd-cli[full]`) |
|
|
73
73
|
|
|
@@ -107,12 +107,75 @@ native Pango/GObject/Cairo libraries, which `pip` cannot install for you:
|
|
|
107
107
|
# macOS
|
|
108
108
|
brew install pango
|
|
109
109
|
|
|
110
|
-
# Debian/Ubuntu
|
|
110
|
+
# Debian/Ubuntu (24.04 and older)
|
|
111
111
|
sudo apt-get install libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 libffi-dev shared-mime-info
|
|
112
|
+
|
|
113
|
+
# Debian trixie (13) and newer: libgdk-pixbuf2.0-0 was renamed
|
|
114
|
+
sudo apt-get install libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf-2.0-0 libffi-dev shared-mime-info
|
|
112
115
|
```
|
|
113
116
|
|
|
114
117
|
PDF conversion (the base install) does not need this.
|
|
115
118
|
|
|
119
|
+
## OCR and equations need one native binary
|
|
120
|
+
|
|
121
|
+
Found by testing docmd against a real scanned document, not documented anywhere
|
|
122
|
+
upstream at the time of writing: Marker's current OCR and equation-recognition model
|
|
123
|
+
is a vision-language model served through either `vllm` (GPU/Linux-oriented) or
|
|
124
|
+
[llama.cpp](https://github.com/ggml-org/llama.cpp)'s `llama-server` binary - there is
|
|
125
|
+
no plain-CPU/transformers fallback. `pip install docmd-cli` cannot provide either one,
|
|
126
|
+
since neither ships as a normal Python wheel.
|
|
127
|
+
|
|
128
|
+
This only matters for **scanned PDFs** (no embedded text layer) and PDFs with
|
|
129
|
+
**equations** - a normal text-layer PDF never touches this code path, and everything
|
|
130
|
+
else in this README works with just `pip install`.
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
# macOS / Linux with Homebrew
|
|
134
|
+
brew install llama.cpp
|
|
135
|
+
|
|
136
|
+
# No Homebrew: download a prebuilt binary directly, no package manager needed
|
|
137
|
+
# (pick the archive matching your OS/arch from the releases page)
|
|
138
|
+
curl -LO https://github.com/ggml-org/llama.cpp/releases/latest/download/llama-<version>-bin-macos-arm64.tar.gz
|
|
139
|
+
tar xzf llama-<version>-bin-macos-arm64.tar.gz
|
|
140
|
+
export LLAMA_CPP_BINARY=$PWD/llama-<version>/llama-server
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Without it, converting a scanned PDF or one with equations raises a
|
|
144
|
+
`MissingSystemDependencyError` with these same instructions - not a raw stack trace.
|
|
145
|
+
First real OCR run also downloads the model's GGUF weights from Hugging Face
|
|
146
|
+
(a few GB), separate from the PyTorch weights Marker already downloaded.
|
|
147
|
+
|
|
148
|
+
## Image handling
|
|
149
|
+
|
|
150
|
+
Images default to a text placeholder (`*[... omitted]*`) - no binary data, nothing to
|
|
151
|
+
resolve, safe for RAG chunking:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
markdown = convert("report.pdf") # image_mode="placeholder" by default
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
To keep real image links instead, use `image_mode="alt-text"` and pass `output_dir` so
|
|
158
|
+
the image files actually get saved somewhere the links can resolve to:
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
from docmd import convert_document
|
|
162
|
+
from docmd.config import ConvertConfig
|
|
163
|
+
|
|
164
|
+
result = convert_document(
|
|
165
|
+
"report.pdf",
|
|
166
|
+
config=ConvertConfig(image_mode="alt-text"),
|
|
167
|
+
output_dir="output/",
|
|
168
|
+
)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
From the CLI, `--image-dir` defaults to the output file's directory when `-o` is given:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
docmd convert report.pdf -o output/report.md --image-mode alt-text
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
`image_mode="skip"` drops images entirely - no placeholder, no files.
|
|
178
|
+
|
|
116
179
|
## How it works
|
|
117
180
|
|
|
118
181
|
`docmd` wraps [Marker](https://github.com/datalab-to/marker) with sane defaults and a
|
|
@@ -40,7 +40,7 @@ core, usable standalone today.
|
|
|
40
40
|
| Input | Output |
|
|
41
41
|
|---|---|
|
|
42
42
|
| PDF (text-based) | Markdown with preserved headings, lists, tables |
|
|
43
|
-
| PDF (scanned) | Markdown via OCR — bundled by Marker, free |
|
|
43
|
+
| PDF (scanned) | Markdown via OCR — bundled by Marker, free, but needs [one extra native binary](#ocr-and-equations-need-one-native-binary) |
|
|
44
44
|
| DOCX | Markdown with formatting preserved (`pip install docmd-cli[full]`) |
|
|
45
45
|
| PPTX | Markdown, one section per slide (`pip install docmd-cli[full]`) |
|
|
46
46
|
|
|
@@ -80,12 +80,75 @@ native Pango/GObject/Cairo libraries, which `pip` cannot install for you:
|
|
|
80
80
|
# macOS
|
|
81
81
|
brew install pango
|
|
82
82
|
|
|
83
|
-
# Debian/Ubuntu
|
|
83
|
+
# Debian/Ubuntu (24.04 and older)
|
|
84
84
|
sudo apt-get install libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 libffi-dev shared-mime-info
|
|
85
|
+
|
|
86
|
+
# Debian trixie (13) and newer: libgdk-pixbuf2.0-0 was renamed
|
|
87
|
+
sudo apt-get install libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf-2.0-0 libffi-dev shared-mime-info
|
|
85
88
|
```
|
|
86
89
|
|
|
87
90
|
PDF conversion (the base install) does not need this.
|
|
88
91
|
|
|
92
|
+
## OCR and equations need one native binary
|
|
93
|
+
|
|
94
|
+
Found by testing docmd against a real scanned document, not documented anywhere
|
|
95
|
+
upstream at the time of writing: Marker's current OCR and equation-recognition model
|
|
96
|
+
is a vision-language model served through either `vllm` (GPU/Linux-oriented) or
|
|
97
|
+
[llama.cpp](https://github.com/ggml-org/llama.cpp)'s `llama-server` binary - there is
|
|
98
|
+
no plain-CPU/transformers fallback. `pip install docmd-cli` cannot provide either one,
|
|
99
|
+
since neither ships as a normal Python wheel.
|
|
100
|
+
|
|
101
|
+
This only matters for **scanned PDFs** (no embedded text layer) and PDFs with
|
|
102
|
+
**equations** - a normal text-layer PDF never touches this code path, and everything
|
|
103
|
+
else in this README works with just `pip install`.
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
# macOS / Linux with Homebrew
|
|
107
|
+
brew install llama.cpp
|
|
108
|
+
|
|
109
|
+
# No Homebrew: download a prebuilt binary directly, no package manager needed
|
|
110
|
+
# (pick the archive matching your OS/arch from the releases page)
|
|
111
|
+
curl -LO https://github.com/ggml-org/llama.cpp/releases/latest/download/llama-<version>-bin-macos-arm64.tar.gz
|
|
112
|
+
tar xzf llama-<version>-bin-macos-arm64.tar.gz
|
|
113
|
+
export LLAMA_CPP_BINARY=$PWD/llama-<version>/llama-server
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Without it, converting a scanned PDF or one with equations raises a
|
|
117
|
+
`MissingSystemDependencyError` with these same instructions - not a raw stack trace.
|
|
118
|
+
First real OCR run also downloads the model's GGUF weights from Hugging Face
|
|
119
|
+
(a few GB), separate from the PyTorch weights Marker already downloaded.
|
|
120
|
+
|
|
121
|
+
## Image handling
|
|
122
|
+
|
|
123
|
+
Images default to a text placeholder (`*[... omitted]*`) - no binary data, nothing to
|
|
124
|
+
resolve, safe for RAG chunking:
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
markdown = convert("report.pdf") # image_mode="placeholder" by default
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
To keep real image links instead, use `image_mode="alt-text"` and pass `output_dir` so
|
|
131
|
+
the image files actually get saved somewhere the links can resolve to:
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from docmd import convert_document
|
|
135
|
+
from docmd.config import ConvertConfig
|
|
136
|
+
|
|
137
|
+
result = convert_document(
|
|
138
|
+
"report.pdf",
|
|
139
|
+
config=ConvertConfig(image_mode="alt-text"),
|
|
140
|
+
output_dir="output/",
|
|
141
|
+
)
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
From the CLI, `--image-dir` defaults to the output file's directory when `-o` is given:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
docmd convert report.pdf -o output/report.md --image-mode alt-text
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
`image_mode="skip"` drops images entirely - no placeholder, no files.
|
|
151
|
+
|
|
89
152
|
## How it works
|
|
90
153
|
|
|
91
154
|
`docmd` wraps [Marker](https://github.com/datalab-to/marker) with sane defaults and a
|
|
@@ -18,7 +18,7 @@ from docmd.postprocess.image_handling import apply_image_handling
|
|
|
18
18
|
from docmd.postprocess.table_cleanup import clean_tables
|
|
19
19
|
|
|
20
20
|
__all__ = ["convert", "convert_document", "ConvertConfig", "ConversionResult"]
|
|
21
|
-
__version__ = "0.1.
|
|
21
|
+
__version__ = "0.1.2"
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def convert_document(
|
|
@@ -26,6 +26,7 @@ def convert_document(
|
|
|
26
26
|
*,
|
|
27
27
|
filename: str | None = None,
|
|
28
28
|
config: ConvertConfig | None = None,
|
|
29
|
+
output_dir: str | Path | None = None,
|
|
29
30
|
) -> ConversionResult:
|
|
30
31
|
"""Convert `source` and return the full result (markdown + page count +
|
|
31
32
|
metadata), after docmd's post-processing pass has run.
|
|
@@ -33,6 +34,14 @@ def convert_document(
|
|
|
33
34
|
`source` is either a path to a file, or raw bytes - in which case
|
|
34
35
|
`filename` is required so docmd knows the format (its extension is used
|
|
35
36
|
to pick a converter; the content itself is what gets converted).
|
|
37
|
+
|
|
38
|
+
`output_dir`: only meaningful with `config.image_mode == "alt-text"`,
|
|
39
|
+
which keeps real image links in the output Markdown - those links only
|
|
40
|
+
resolve if the image files actually exist somewhere, so pass the
|
|
41
|
+
directory to save them into (typically wherever the output .md file is
|
|
42
|
+
going). Without it, `alt-text` mode's links point to image filenames
|
|
43
|
+
that were never written anywhere. `placeholder` (the default) and
|
|
44
|
+
`skip` modes ignore this - they never reference image files at all.
|
|
36
45
|
"""
|
|
37
46
|
config = config or ConvertConfig()
|
|
38
47
|
|
|
@@ -44,11 +53,11 @@ def convert_document(
|
|
|
44
53
|
try:
|
|
45
54
|
with os.fdopen(fd, "wb") as tmp:
|
|
46
55
|
tmp.write(source)
|
|
47
|
-
result = _convert_path(tmp_path, config)
|
|
56
|
+
result = _convert_path(tmp_path, config, output_dir)
|
|
48
57
|
finally:
|
|
49
58
|
os.unlink(tmp_path)
|
|
50
59
|
else:
|
|
51
|
-
result = _convert_path(str(source), config)
|
|
60
|
+
result = _convert_path(str(source), config, output_dir)
|
|
52
61
|
|
|
53
62
|
return result
|
|
54
63
|
|
|
@@ -58,13 +67,16 @@ def convert(
|
|
|
58
67
|
*,
|
|
59
68
|
filename: str | None = None,
|
|
60
69
|
config: ConvertConfig | None = None,
|
|
70
|
+
output_dir: str | Path | None = None,
|
|
61
71
|
) -> str:
|
|
62
72
|
"""Convert `source` (a file path, or bytes + filename) and return the
|
|
63
|
-
resulting Markdown as a string."""
|
|
64
|
-
return convert_document(source, filename=filename, config=config).markdown
|
|
73
|
+
resulting Markdown as a string. See `convert_document` re: `output_dir`."""
|
|
74
|
+
return convert_document(source, filename=filename, config=config, output_dir=output_dir).markdown
|
|
65
75
|
|
|
66
76
|
|
|
67
|
-
def _convert_path(
|
|
77
|
+
def _convert_path(
|
|
78
|
+
filepath: str, config: ConvertConfig, output_dir: str | Path | None
|
|
79
|
+
) -> ConversionResult:
|
|
68
80
|
converter = get_converter(filepath)
|
|
69
81
|
result = converter.convert(filepath, config)
|
|
70
82
|
|
|
@@ -73,7 +85,7 @@ def _convert_path(filepath: str, config: ConvertConfig) -> ConversionResult:
|
|
|
73
85
|
markdown = clean_tables(markdown)
|
|
74
86
|
if config.normalize_headings:
|
|
75
87
|
markdown = normalize_headings(markdown)
|
|
76
|
-
markdown = apply_image_handling(markdown, result.images, config.image_mode)
|
|
88
|
+
markdown = apply_image_handling(markdown, result.images, config.image_mode, output_dir)
|
|
77
89
|
|
|
78
90
|
return ConversionResult(
|
|
79
91
|
markdown=markdown,
|
|
@@ -47,12 +47,39 @@ def main() -> None:
|
|
|
47
47
|
show_default=True,
|
|
48
48
|
help="How to represent images in the output.",
|
|
49
49
|
)
|
|
50
|
-
|
|
50
|
+
@click.option(
|
|
51
|
+
"--image-dir",
|
|
52
|
+
"image_dir",
|
|
53
|
+
type=click.Path(file_okay=False, path_type=Path),
|
|
54
|
+
default=None,
|
|
55
|
+
help="Where to save image files in --image-mode alt-text. Defaults to "
|
|
56
|
+
"the output file's directory when -o is given; without -o, images "
|
|
57
|
+
"won't be saved unless this is set explicitly.",
|
|
58
|
+
)
|
|
59
|
+
def convert(
|
|
60
|
+
file: Path,
|
|
61
|
+
output: Path | None,
|
|
62
|
+
force_ocr: bool,
|
|
63
|
+
use_llm: bool,
|
|
64
|
+
image_mode: str,
|
|
65
|
+
image_dir: Path | None,
|
|
66
|
+
) -> None:
|
|
51
67
|
"""Convert FILE to Markdown."""
|
|
52
68
|
config = ConvertConfig(force_ocr=force_ocr, use_llm=use_llm, image_mode=image_mode)
|
|
53
69
|
|
|
70
|
+
if image_mode == "alt-text" and image_dir is None:
|
|
71
|
+
if output is not None:
|
|
72
|
+
image_dir = output.parent
|
|
73
|
+
else:
|
|
74
|
+
click.echo(
|
|
75
|
+
"warning: --image-mode alt-text with no -o/--image-dir - "
|
|
76
|
+
"image links in the output won't resolve to real files. "
|
|
77
|
+
"Pass --image-dir to save images somewhere.",
|
|
78
|
+
err=True,
|
|
79
|
+
)
|
|
80
|
+
|
|
54
81
|
try:
|
|
55
|
-
result = convert_document(str(file), config=config)
|
|
82
|
+
result = convert_document(str(file), config=config, output_dir=image_dir)
|
|
56
83
|
except DocmdError as exc:
|
|
57
84
|
click.echo(f"error: {exc}", err=True)
|
|
58
85
|
sys.exit(1)
|
|
@@ -13,7 +13,12 @@ from typing import Any
|
|
|
13
13
|
|
|
14
14
|
from docmd.config import ConvertConfig
|
|
15
15
|
from docmd.converters.base import ConversionResult
|
|
16
|
-
from docmd.errors import
|
|
16
|
+
from docmd.errors import (
|
|
17
|
+
ConversionError,
|
|
18
|
+
EncryptedDocumentError,
|
|
19
|
+
MissingExtraError,
|
|
20
|
+
MissingSystemDependencyError,
|
|
21
|
+
)
|
|
17
22
|
|
|
18
23
|
# Marker loads its models (a few hundred MB to ~1GB of weights, downloaded
|
|
19
24
|
# from Hugging Face on first use) lazily and caches them at module scope, so
|
|
@@ -21,6 +26,11 @@ from docmd.errors import ConversionError, EncryptedDocumentError, MissingExtraEr
|
|
|
21
26
|
_model_dict: dict[str, Any] | None = None
|
|
22
27
|
|
|
23
28
|
_PASSWORD_HINTS = ("password", "encrypt")
|
|
29
|
+
# Raised as surya.inference.backends.spawn.SpawnError when OCR/equation
|
|
30
|
+
# recognition needs the llama-server binary and it isn't on PATH - matched
|
|
31
|
+
# by message/type name rather than importing surya's internal exception
|
|
32
|
+
# class, so this doesn't break if that module path moves.
|
|
33
|
+
_MISSING_BINARY_HINTS = ("llama-server", "spawnerror")
|
|
24
34
|
|
|
25
35
|
|
|
26
36
|
def _get_model_dict() -> dict[str, Any]:
|
|
@@ -39,6 +49,18 @@ def _build_config_dict(config: ConvertConfig):
|
|
|
39
49
|
"output_format": "markdown",
|
|
40
50
|
"force_ocr": config.force_ocr,
|
|
41
51
|
"use_llm": config.use_llm,
|
|
52
|
+
# Marker's own default (4) spins up a multiprocessing.ProcessPoolExecutor
|
|
53
|
+
# for page-text extraction on any sufficiently multi-page PDF. On
|
|
54
|
+
# spawn-based platforms (macOS, Windows) that crashes with
|
|
55
|
+
# "An attempt has been made to start a new process before the
|
|
56
|
+
# current process has finished its bootstrapping phase" whenever the
|
|
57
|
+
# caller isn't wrapped in `if __name__ == "__main__":` - an easy trap
|
|
58
|
+
# for a library used from a plain script, a notebook, or a web
|
|
59
|
+
# server's request handler. Marker's own bundled server.py sets this
|
|
60
|
+
# to 1 for exactly this reason; docmd does the same as a library
|
|
61
|
+
# default, trading a bit of extraction parallelism for not crashing
|
|
62
|
+
# on arbitrary callers.
|
|
63
|
+
"pdftext_workers": 1,
|
|
42
64
|
}
|
|
43
65
|
if config.image_mode == "skip":
|
|
44
66
|
options["disable_image_extraction"] = True
|
|
@@ -71,8 +93,11 @@ class MarkerConverter:
|
|
|
71
93
|
rendered = converter(filepath)
|
|
72
94
|
except Exception as exc:
|
|
73
95
|
message = str(exc).lower()
|
|
96
|
+
exc_type_name = type(exc).__name__.lower()
|
|
74
97
|
if any(hint in message for hint in _PASSWORD_HINTS):
|
|
75
98
|
raise EncryptedDocumentError() from exc
|
|
99
|
+
if any(hint in message or hint in exc_type_name for hint in _MISSING_BINARY_HINTS):
|
|
100
|
+
raise MissingSystemDependencyError(str(exc)) from exc
|
|
76
101
|
raise ConversionError(
|
|
77
102
|
f"Marker failed to convert '{filepath}': {exc}", cause=exc
|
|
78
103
|
) from exc
|
|
@@ -35,6 +35,30 @@ class MissingExtraError(DocmdError):
|
|
|
35
35
|
)
|
|
36
36
|
|
|
37
37
|
|
|
38
|
+
class MissingSystemDependencyError(DocmdError):
|
|
39
|
+
"""Raised when Marker's OCR/equation recognition needs a native binary
|
|
40
|
+
that isn't installed - not a pip package, so `pip install docmd[full]`
|
|
41
|
+
can't provide it. See ARCHITECTURE.md "OCR and equation recognition need
|
|
42
|
+
a native binary" for why this exists."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, detail: str) -> None:
|
|
45
|
+
super().__init__(
|
|
46
|
+
"OCR/equation recognition needs the `llama-server` binary from "
|
|
47
|
+
"llama.cpp, which pip cannot install. This only matters for "
|
|
48
|
+
"scanned PDFs or documents with equations - a normal text-layer "
|
|
49
|
+
"PDF doesn't hit this path.\n\n"
|
|
50
|
+
"Install it with:\n"
|
|
51
|
+
" macOS: brew install llama.cpp\n"
|
|
52
|
+
" Linux: brew install llama.cpp, or download a release from\n"
|
|
53
|
+
" https://github.com/ggml-org/llama.cpp/releases\n"
|
|
54
|
+
"No Homebrew? Download the right archive from that releases "
|
|
55
|
+
"page directly (llama-<version>-bin-macos-arm64.tar.gz etc.), "
|
|
56
|
+
"then either put `llama-server` on your PATH or set "
|
|
57
|
+
"LLAMA_CPP_BINARY to its full path.\n\n"
|
|
58
|
+
f"Underlying error: {detail}"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
38
62
|
class EncryptedDocumentError(DocmdError):
|
|
39
63
|
"""Raised when the input is a password-protected / encrypted document."""
|
|
40
64
|
|
|
@@ -18,7 +18,7 @@ import re
|
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
from typing import Any
|
|
20
20
|
|
|
21
|
-
_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]
|
|
21
|
+
_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]*)\)")
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def apply_image_handling(
|
|
@@ -32,6 +32,16 @@ def apply_image_handling(
|
|
|
32
32
|
def _replace(match: re.Match[str]) -> str:
|
|
33
33
|
nonlocal counter
|
|
34
34
|
alt_text, ref = match.group(1), match.group(2)
|
|
35
|
+
|
|
36
|
+
if not ref.strip():
|
|
37
|
+
# `![]()` with no reference at all - real Marker output on some
|
|
38
|
+
# scanned pages (an OCR'd region with no recoverable image data).
|
|
39
|
+
# No mode should keep this: a placeholder/skip has nothing to
|
|
40
|
+
# describe, and alt-text mode would otherwise emit a link to
|
|
41
|
+
# nowhere (an empty href renders as a broken image for no
|
|
42
|
+
# legitimate reason, since there was never a file to begin with).
|
|
43
|
+
return ""
|
|
44
|
+
|
|
35
45
|
filename = Path(ref).name
|
|
36
46
|
|
|
37
47
|
if mode == "skip":
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Regenerates the stress-test PDF fixtures used to prove post-processing
|
|
2
|
+
fixes real Marker defects, not just hand-crafted markdown strings. See
|
|
3
|
+
tests/test_postprocess_integration.py for stress.pdf, and
|
|
4
|
+
tests/test_postprocess_more_integration.py for the ones generated here.
|
|
5
|
+
|
|
6
|
+
python tests/fixtures/generate_stress_fixtures.py
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from PIL import Image as PILImage
|
|
14
|
+
from reportlab.lib import colors
|
|
15
|
+
from reportlab.lib.pagesizes import LETTER
|
|
16
|
+
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
|
17
|
+
from reportlab.lib.units import inch
|
|
18
|
+
from reportlab.platypus import (
|
|
19
|
+
Image,
|
|
20
|
+
PageBreak,
|
|
21
|
+
Paragraph,
|
|
22
|
+
SimpleDocTemplate,
|
|
23
|
+
Spacer,
|
|
24
|
+
Table,
|
|
25
|
+
TableStyle,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
FIXTURES_DIR = Path(__file__).parent
|
|
29
|
+
styles = getSampleStyleSheet()
|
|
30
|
+
body = styles["BodyText"]
|
|
31
|
+
h_title = ParagraphStyle("H0", fontSize=28, leading=32, fontName="Helvetica-Bold")
|
|
32
|
+
h_big = ParagraphStyle("H1", fontSize=22, leading=26, fontName="Helvetica-Bold")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def generate_running_header() -> None:
|
|
36
|
+
"""4-page document with a bold header drawn at a fixed position on every
|
|
37
|
+
page (not a flowable heading) - tests whether that gets misdetected as a
|
|
38
|
+
duplicate heading, and whether structurally identical section headings
|
|
39
|
+
get assigned consistent levels throughout."""
|
|
40
|
+
|
|
41
|
+
def draw_header(canvas, _doc):
|
|
42
|
+
canvas.saveState()
|
|
43
|
+
canvas.setFont("Helvetica-Bold", 14)
|
|
44
|
+
canvas.drawString(72, 740, "CONFIDENTIAL - INTERNAL REPORT")
|
|
45
|
+
canvas.restoreState()
|
|
46
|
+
|
|
47
|
+
doc = SimpleDocTemplate(str(FIXTURES_DIR / "running_header.pdf"), pagesize=LETTER)
|
|
48
|
+
story = [Paragraph("Quarterly Risk Review", h_title), Spacer(1, 20)]
|
|
49
|
+
for section in range(1, 5):
|
|
50
|
+
story.append(Paragraph(f"Section {section}: Findings", h_big))
|
|
51
|
+
story.append(
|
|
52
|
+
Paragraph(
|
|
53
|
+
(f"This is body paragraph content for section {section}. ") * 20,
|
|
54
|
+
body,
|
|
55
|
+
)
|
|
56
|
+
)
|
|
57
|
+
story.append(PageBreak())
|
|
58
|
+
doc.build(story, onFirstPage=draw_header, onLaterPages=draw_header)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def generate_merged_cells() -> None:
|
|
62
|
+
"""A table with a spanned header cell (merged columns) - tests whether
|
|
63
|
+
that breaks raw column-count reconstruction."""
|
|
64
|
+
rows = [
|
|
65
|
+
["", "Q1 2026", "", "Q2 2026", ""],
|
|
66
|
+
["Region", "Revenue", "Growth", "Revenue", "Growth"],
|
|
67
|
+
["West", "$1.2M", "+5%", "$1.4M", "+17%"],
|
|
68
|
+
["East", "$0.9M", "+2%", "$1.0M", "+11%"],
|
|
69
|
+
]
|
|
70
|
+
doc = SimpleDocTemplate(str(FIXTURES_DIR / "merged_cells.pdf"), pagesize=LETTER)
|
|
71
|
+
table = Table(
|
|
72
|
+
rows,
|
|
73
|
+
style=TableStyle(
|
|
74
|
+
[
|
|
75
|
+
("GRID", (0, 0), (-1, -1), 0.5, colors.black),
|
|
76
|
+
("SPAN", (1, 0), (2, 0)),
|
|
77
|
+
("SPAN", (3, 0), (4, 0)),
|
|
78
|
+
("BACKGROUND", (0, 0), (-1, 1), colors.lightgrey),
|
|
79
|
+
]
|
|
80
|
+
),
|
|
81
|
+
)
|
|
82
|
+
doc.build([Paragraph("Regional Revenue", h_big), Spacer(1, 10), table])
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def generate_with_image() -> None:
|
|
86
|
+
"""A PDF with a real embedded image, to test image_handling.py against
|
|
87
|
+
real Marker image extraction rather than a synthetic dict."""
|
|
88
|
+
img_path = FIXTURES_DIR / "_gen_chart.png"
|
|
89
|
+
PILImage.new("RGB", (300, 200), color=(70, 130, 180)).save(img_path)
|
|
90
|
+
|
|
91
|
+
doc = SimpleDocTemplate(str(FIXTURES_DIR / "with_image.pdf"), pagesize=LETTER)
|
|
92
|
+
doc.build(
|
|
93
|
+
[
|
|
94
|
+
Paragraph("Revenue Chart", h_big),
|
|
95
|
+
Spacer(1, 10),
|
|
96
|
+
Paragraph("See the chart below for regional performance.", body),
|
|
97
|
+
Spacer(1, 10),
|
|
98
|
+
Image(str(img_path), width=3 * inch, height=2 * inch),
|
|
99
|
+
Spacer(1, 10),
|
|
100
|
+
Paragraph("Chart data is preliminary and subject to revision.", body),
|
|
101
|
+
]
|
|
102
|
+
)
|
|
103
|
+
img_path.unlink() # embedded in the PDF now, don't need the source file
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
if __name__ == "__main__":
|
|
107
|
+
generate_running_header()
|
|
108
|
+
generate_merged_cells()
|
|
109
|
+
generate_with_image()
|
|
110
|
+
print(f"wrote stress fixtures to {FIXTURES_DIR}")
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
%PDF-1.4
|
|
2
|
+
%���� ReportLab Generated PDF document (opensource)
|
|
3
|
+
1 0 obj
|
|
4
|
+
<<
|
|
5
|
+
/F1 2 0 R /F2 3 0 R
|
|
6
|
+
>>
|
|
7
|
+
endobj
|
|
8
|
+
2 0 obj
|
|
9
|
+
<<
|
|
10
|
+
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
|
11
|
+
>>
|
|
12
|
+
endobj
|
|
13
|
+
3 0 obj
|
|
14
|
+
<<
|
|
15
|
+
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
|
16
|
+
>>
|
|
17
|
+
endobj
|
|
18
|
+
4 0 obj
|
|
19
|
+
<<
|
|
20
|
+
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
|
|
21
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
22
|
+
>> /Rotate 0 /Trans <<
|
|
23
|
+
|
|
24
|
+
>>
|
|
25
|
+
/Type /Page
|
|
26
|
+
>>
|
|
27
|
+
endobj
|
|
28
|
+
5 0 obj
|
|
29
|
+
<<
|
|
30
|
+
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
|
31
|
+
>>
|
|
32
|
+
endobj
|
|
33
|
+
6 0 obj
|
|
34
|
+
<<
|
|
35
|
+
/Author (\(anonymous\)) /CreationDate (D:20260918162747+02'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260918162747+02'00') /Producer (ReportLab PDF Library - \(opensource\))
|
|
36
|
+
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
|
37
|
+
>>
|
|
38
|
+
endobj
|
|
39
|
+
7 0 obj
|
|
40
|
+
<<
|
|
41
|
+
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
|
42
|
+
>>
|
|
43
|
+
endobj
|
|
44
|
+
8 0 obj
|
|
45
|
+
<<
|
|
46
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 500
|
|
47
|
+
>>
|
|
48
|
+
stream
|
|
49
|
+
GasalbAP0N&A70Vp-9o2Wp=eD-b`WU86("e7cKB,5`9lcPa:4qq@P(tY`O,]Za6u")iVSsT_nZ>!'F9Npl'_)WWf&+&uo"M<S;SNJQ\51bEBr"EJNqJ)3nW0U7)Oj#S9a"BWKOQf#t8QG.r]FrE!oel[,R2:^sN$K[+:a_DUPO=;K+'BWcc!<f03nK_'+#AkN:2FjDq_&fu'&q\g6&PA-RZmS&LGZ/\-&1=ej\YEP*'kqZfH'*Q.*mPsGb;(?2K8/2H2#)SB%GBB864%J<+%_J8lZD(cfpUfJE1HFGhRYKCsD@tF+GXn4Q5DPJg5jKR,D^co^2cOum(Tj4?-KdNKH<=DaUo3YuUNtRDBD"5?an\):.+7.VApU&?5,]WJjm4jMU%^&7a/@sV.6'4E@FO!8MV%@c@,@XU"QV-bjt@#hftnF^0\s/C;r#1\/&LrLYfuJQXh9^h\l#gnMn:+3@rCY5Ca+P;!Y*lN\<ILke\iKLrW,hE#lt~>endstream
|
|
50
|
+
endobj
|
|
51
|
+
xref
|
|
52
|
+
0 9
|
|
53
|
+
0000000000 65535 f
|
|
54
|
+
0000000061 00000 n
|
|
55
|
+
0000000102 00000 n
|
|
56
|
+
0000000209 00000 n
|
|
57
|
+
0000000321 00000 n
|
|
58
|
+
0000000514 00000 n
|
|
59
|
+
0000000582 00000 n
|
|
60
|
+
0000000862 00000 n
|
|
61
|
+
0000000921 00000 n
|
|
62
|
+
trailer
|
|
63
|
+
<<
|
|
64
|
+
/ID
|
|
65
|
+
[<04e44ee2661fab79030feb6409b489d5><04e44ee2661fab79030feb6409b489d5>]
|
|
66
|
+
% ReportLab generated PDF document -- digest (opensource)
|
|
67
|
+
|
|
68
|
+
/Info 6 0 R
|
|
69
|
+
/Root 5 0 R
|
|
70
|
+
/Size 9
|
|
71
|
+
>>
|
|
72
|
+
startxref
|
|
73
|
+
1511
|
|
74
|
+
%%EOF
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
%PDF-1.4
|
|
2
|
+
%���� ReportLab Generated PDF document (opensource)
|
|
3
|
+
1 0 obj
|
|
4
|
+
<<
|
|
5
|
+
/F1 2 0 R /F2 3 0 R
|
|
6
|
+
>>
|
|
7
|
+
endobj
|
|
8
|
+
2 0 obj
|
|
9
|
+
<<
|
|
10
|
+
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
|
11
|
+
>>
|
|
12
|
+
endobj
|
|
13
|
+
3 0 obj
|
|
14
|
+
<<
|
|
15
|
+
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
|
16
|
+
>>
|
|
17
|
+
endobj
|
|
18
|
+
4 0 obj
|
|
19
|
+
<<
|
|
20
|
+
/Contents 11 0 R /MediaBox [ 0 0 612 792 ] /Parent 10 0 R /Resources <<
|
|
21
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
22
|
+
>> /Rotate 0 /Trans <<
|
|
23
|
+
|
|
24
|
+
>>
|
|
25
|
+
/Type /Page
|
|
26
|
+
>>
|
|
27
|
+
endobj
|
|
28
|
+
5 0 obj
|
|
29
|
+
<<
|
|
30
|
+
/Contents 12 0 R /MediaBox [ 0 0 612 792 ] /Parent 10 0 R /Resources <<
|
|
31
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
32
|
+
>> /Rotate 0 /Trans <<
|
|
33
|
+
|
|
34
|
+
>>
|
|
35
|
+
/Type /Page
|
|
36
|
+
>>
|
|
37
|
+
endobj
|
|
38
|
+
6 0 obj
|
|
39
|
+
<<
|
|
40
|
+
/Contents 13 0 R /MediaBox [ 0 0 612 792 ] /Parent 10 0 R /Resources <<
|
|
41
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
42
|
+
>> /Rotate 0 /Trans <<
|
|
43
|
+
|
|
44
|
+
>>
|
|
45
|
+
/Type /Page
|
|
46
|
+
>>
|
|
47
|
+
endobj
|
|
48
|
+
7 0 obj
|
|
49
|
+
<<
|
|
50
|
+
/Contents 14 0 R /MediaBox [ 0 0 612 792 ] /Parent 10 0 R /Resources <<
|
|
51
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
52
|
+
>> /Rotate 0 /Trans <<
|
|
53
|
+
|
|
54
|
+
>>
|
|
55
|
+
/Type /Page
|
|
56
|
+
>>
|
|
57
|
+
endobj
|
|
58
|
+
8 0 obj
|
|
59
|
+
<<
|
|
60
|
+
/PageMode /UseNone /Pages 10 0 R /Type /Catalog
|
|
61
|
+
>>
|
|
62
|
+
endobj
|
|
63
|
+
9 0 obj
|
|
64
|
+
<<
|
|
65
|
+
/Author (\(anonymous\)) /CreationDate (D:20260918162747+02'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260918162747+02'00') /Producer (ReportLab PDF Library - \(opensource\))
|
|
66
|
+
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
|
67
|
+
>>
|
|
68
|
+
endobj
|
|
69
|
+
10 0 obj
|
|
70
|
+
<<
|
|
71
|
+
/Count 4 /Kids [ 4 0 R 5 0 R 6 0 R 7 0 R ] /Type /Pages
|
|
72
|
+
>>
|
|
73
|
+
endobj
|
|
74
|
+
11 0 obj
|
|
75
|
+
<<
|
|
76
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 361
|
|
77
|
+
>>
|
|
78
|
+
stream
|
|
79
|
+
Gb"/chbJeX'ZTTeMXM0P7+reOn,*7oahf$h(I5uBe\Nf8g"T^lL:@V#9Xn<\1sL5>cgPXF;?g>7VUonlg^u.@#0&qZ0ZS3!1\bLn)`kj$VB`8*[1T7A[`F\ClI_a=bklZ"KF`N"iU%$i#UAp#88Vl:E?<RHkQQ(b*+,Ymh0BO6a-@13fijSb^V1/T.Gi_8?=RK^)\_An,E[*qhfafEonka+'\V(k`\GH_P[ejqrf+$%LV'#RPbkc9$a:QQU0*9sYD1,"I.2U:hB&CP@-`$n?]`j5Jst$IEeeKUP?]Hh2:Ae/Z_GI>Pc,'U/Gqa0i7o<7nWVeV7V).t45PdP??V)cFmmhn5t:`K_"\,!^:87~>endstream
|
|
80
|
+
endobj
|
|
81
|
+
12 0 obj
|
|
82
|
+
<<
|
|
83
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 319
|
|
84
|
+
>>
|
|
85
|
+
stream
|
|
86
|
+
Gb"/c]5E6T(km1*if#]lM/Bf]AEsCEUklDMIfVkK6D^%P=Sh_K/toX>L.Z9E/^*uDk<$*uF"fD[?1T<17$pIa>_)P+dZ^==6?:A3bm4uh);eA@`l*-U&dSW'JW=]Af1?[kKF,VY/)2@E-qk;\9uGS*8p?p=d&_#41X\Y*K!1F/@:II#l=-j@+?T,=L9Ll9hjJH@GBmCn)IeQ%eD!k;bqHpU`^:lP0c'$n@W$bMZO.)66eT9!9ssr<.BBeLc;"W*Q#:fW`P/A">^UekJ+^dk4!EX^HD"!R(fq6\#oTN'!I&n(gZr-VoH+7,0.TP^h#~>endstream
|
|
87
|
+
endobj
|
|
88
|
+
13 0 obj
|
|
89
|
+
<<
|
|
90
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 320
|
|
91
|
+
>>
|
|
92
|
+
stream
|
|
93
|
+
Gb"/c:JZTs(rl#lMW]dB@Tt?^Dn85$Z7okY(I5D`9NEKZn_77t6el+*-/=eE:%S@0IS\^o*6\h/f6e+A'(-tS/jNW[kt05'+ZWnibm4uh);eA@`l*-U&dSW'JbF%qb9Vu.@7'M-!dBG6$Icq0e*q4BdLGCRFPF4<)sO&'_,neS0JR0LFYVH[OI^W/6WeuXmnC6l:#;2$`ksU,;Qi`!2#dh/N5n2T@m$_tLq*\$cRLK!G%Cb35H=B8%U#>;F`uC\YB7D(/[eZ=k30_#jjclamY$P6l@Ci:@pD1d-=i(8#Mf"FY-3&8d=2jLIK3B<S+6~>endstream
|
|
94
|
+
endobj
|
|
95
|
+
14 0 obj
|
|
96
|
+
<<
|
|
97
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 319
|
|
98
|
+
>>
|
|
99
|
+
stream
|
|
100
|
+
Gb"/c]5E6T(km1*if#]lM/Bf]AEsCEUklDMIfVkK6D^%P=Sh_K/toX>L.Z9E/^*uDk<$*uF"fD[?1T<17$pIa>_)P+dZ^==6?:A3bm4uh);eA@`l*-U&dSW'JW=]Af1?[kKF,VY/)2@E-qk;\9uGS*8p?p=d&_#41X\Y*K!1F/@:II#l=-j@+?T,=L9Ll9hjJH@GBmCn)KL\5eD!k;bqHpU`^:lP0c'$n@W$bMZO.)66eT9!9ssr<.BBeLc;"W*Q#:fW`P/A">^UekJ+^dk4!EX^HD"!R(fq6\#oTN'!I&n(gZr-VoH+7,07WM^#6~>endstream
|
|
101
|
+
endobj
|
|
102
|
+
xref
|
|
103
|
+
0 15
|
|
104
|
+
0000000000 65535 f
|
|
105
|
+
0000000061 00000 n
|
|
106
|
+
0000000102 00000 n
|
|
107
|
+
0000000209 00000 n
|
|
108
|
+
0000000321 00000 n
|
|
109
|
+
0000000516 00000 n
|
|
110
|
+
0000000711 00000 n
|
|
111
|
+
0000000906 00000 n
|
|
112
|
+
0000001101 00000 n
|
|
113
|
+
0000001170 00000 n
|
|
114
|
+
0000001450 00000 n
|
|
115
|
+
0000001528 00000 n
|
|
116
|
+
0000001980 00000 n
|
|
117
|
+
0000002390 00000 n
|
|
118
|
+
0000002801 00000 n
|
|
119
|
+
trailer
|
|
120
|
+
<<
|
|
121
|
+
/ID
|
|
122
|
+
[<71bc982e6efde47f41d5e095c3f3e0e4><71bc982e6efde47f41d5e095c3f3e0e4>]
|
|
123
|
+
% ReportLab generated PDF document -- digest (opensource)
|
|
124
|
+
|
|
125
|
+
/Info 9 0 R
|
|
126
|
+
/Root 8 0 R
|
|
127
|
+
/Size 15
|
|
128
|
+
>>
|
|
129
|
+
startxref
|
|
130
|
+
3211
|
|
131
|
+
%%EOF
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
%PDF-1.4
|
|
2
|
+
%���� ReportLab Generated PDF document (opensource)
|
|
3
|
+
1 0 obj
|
|
4
|
+
<<
|
|
5
|
+
/F1 2 0 R /F2 3 0 R /F3 4 0 R
|
|
6
|
+
>>
|
|
7
|
+
endobj
|
|
8
|
+
2 0 obj
|
|
9
|
+
<<
|
|
10
|
+
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
|
11
|
+
>>
|
|
12
|
+
endobj
|
|
13
|
+
3 0 obj
|
|
14
|
+
<<
|
|
15
|
+
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
|
16
|
+
>>
|
|
17
|
+
endobj
|
|
18
|
+
4 0 obj
|
|
19
|
+
<<
|
|
20
|
+
/BaseFont /Helvetica-BoldOblique /Encoding /WinAnsiEncoding /Name /F3 /Subtype /Type1 /Type /Font
|
|
21
|
+
>>
|
|
22
|
+
endobj
|
|
23
|
+
5 0 obj
|
|
24
|
+
<<
|
|
25
|
+
/Contents 10 0 R /MediaBox [ 0 0 612 792 ] /Parent 9 0 R /Resources <<
|
|
26
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
27
|
+
>> /Rotate 0 /Trans <<
|
|
28
|
+
|
|
29
|
+
>>
|
|
30
|
+
/Type /Page
|
|
31
|
+
>>
|
|
32
|
+
endobj
|
|
33
|
+
6 0 obj
|
|
34
|
+
<<
|
|
35
|
+
/Contents 11 0 R /MediaBox [ 0 0 612 792 ] /Parent 9 0 R /Resources <<
|
|
36
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
37
|
+
>> /Rotate 0 /Trans <<
|
|
38
|
+
|
|
39
|
+
>>
|
|
40
|
+
/Type /Page
|
|
41
|
+
>>
|
|
42
|
+
endobj
|
|
43
|
+
7 0 obj
|
|
44
|
+
<<
|
|
45
|
+
/PageMode /UseNone /Pages 9 0 R /Type /Catalog
|
|
46
|
+
>>
|
|
47
|
+
endobj
|
|
48
|
+
8 0 obj
|
|
49
|
+
<<
|
|
50
|
+
/Author (\(anonymous\)) /CreationDate (D:20260918161648+02'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260918161648+02'00') /Producer (ReportLab PDF Library - \(opensource\))
|
|
51
|
+
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
|
52
|
+
>>
|
|
53
|
+
endobj
|
|
54
|
+
9 0 obj
|
|
55
|
+
<<
|
|
56
|
+
/Count 2 /Kids [ 5 0 R 6 0 R ] /Type /Pages
|
|
57
|
+
>>
|
|
58
|
+
endobj
|
|
59
|
+
10 0 obj
|
|
60
|
+
<<
|
|
61
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1689
|
|
62
|
+
>>
|
|
63
|
+
stream
|
|
64
|
+
Gat%d>AkH>&;B$?/,6=ZDEt88?QOPc$5>dYfE1p9`bWVGCq6k<+TK1:mL7&l`"<A%M7E/F_m<"-*3E`qrQ+6l0+X$?\n40VOA%.4^1G`o$sS3ln(24PSk1&Q?nSG^Ve]bQ[_"Uo^F%,tl$"2s/;hQ2B2L/)+a%&/OK.UcG5f\Fj3.+M?!VBgB6H:!D0PlLf_QOk^LJ';Ql:HL5Ki3GjQ:BagAep`o1uRMme)&,NkLL^dtX;nbnI!:-Dij_:]00_ZXGe&P.93%Z.nZ+mD"\JpAXk\](BLFhJr88&%3GGm^J6-ZFa")`7jE9etgJfSC[=Z?I$UY?*g(1FsT!R0_[%FDldJfs%V;j'5rS5b/dS/)AhY8mZP5,MtO`r.B-[^9#jViH(F,bdp8nQ<DceuPci**j&f"Pe/'YD4jHX"<d!HSnaU0QN[eI<:)3J;J%^>W5)uKX]q21sf?\eA#]N?@PGAlmZ]khH6j!WV!"Y3-j(UN(rjLKDO#Kcnf$:#!0DhrK*WWMf7Hsmboe^('"7RErGL=chJ3>&Hqhb/"q$\h\^ar$?JcJ2)%2]3K4=J7(Bfo$&"RN%aZ#M,=Hns!Aa2e.][F[\UTNP+O:4aSg;,@?!5l_GLE#f!3#T$,u+YfUq^W.!88/*R^QII/?6;I[f4TH!^EN^<0l2d4%UI1Xu+IDa\8e6NPVZG+i&0@q6+uBcKkWtNgE8#FQ!eOQM_hgX=>8fg1"uuYo2KGQ:/][N$6Lr5^6U3=5C@$!QA<qtsJ3aa^KCdmq`hPpf^n)+5\h*s1/?*VB6'>DT63@E`!"[q+W>]r9P(4jfSEr$7jJMrT"Xuqo*2*Og*.WfP\FC"BjiAK5:pJ=YOIHB8+TY.K7"tht#K;=k6NWu0o_;0W5]@?(V?A/A&=F.B*i.lFd4#*!C;VLTBaG&?HH;1u,jHO?!lBSKgpWe#lENLDTQIXj+?F3\ErqN'M9H83&'FKOl'tRN95@FR[Xd-YL^XV]U*)3>+?X>X<@_"@+CN;c-TXsf<2MqC"=-hSR1g,QN8`@93<<:Zq5!J@IBJ+YUkjs:8>7:`.Ngk+jE$HD$HSf8j>q\i_+>6,'p(gPA6(s?E0;_=B*DhF/LO9%.j-l%,,54&W/-B>#S/OJ?)45^=Y9T2FIt'+2go$Z_ZGuDW!.erJ-YOg1iuhbp4ORAk6o@*K#k0H,RPa5!l@;q2AbSaJ1S+k@](Znl?*/r']QtB>3mdEfYMO)l'N$/r4mO1JNpY1.h9m/r5&kGm&\097Sl2O3[=EkLg[j.^`KRBOTE9NS=mjHE\,34agIl'JN`ss_0!CXW'#TZKs7oaq@*KcJ;W.X'/'p.^+FHhK'N-[Z!_kRku7Y<?O,FM=s^QKCpLdm?_)U#%kNc*R0N0A40BTp"ce$raKNV`9RjdmHgT!@c;p6Z0<`,4n-+%XCtR9;&F$L!XoK["F<U.J$1;am%naZlWfRF]@&o.TTG4c5$>E<DZ<D`!?pVi5Goi3%YF=W?.CcC'*gD8URccscW2b+aP_,j'MaMP#WoeS'$GL8H#Gkh/k'9c1\n[fAV')i$H%6&9X6+\($DD4+#Gp@Yk'9c1VJ;\-V4ajNGu+jBDCuFDA(<X^>5P>+5;]hAG-,Y,ID;h[F\cgf9Ko>alhp"OO$gtn1KW^7LPdo@S\n^#5?3sco"gefo1,m1Spo>FImINnLB~>endstream
|
|
65
|
+
endobj
|
|
66
|
+
11 0 obj
|
|
67
|
+
<<
|
|
68
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1195
|
|
69
|
+
>>
|
|
70
|
+
stream
|
|
71
|
+
Gasb[>E9FX&4YRK/,[a)J=<RCD>SI"fp,SD"@V6p"?cj#/hiJlr]\%qg$U#d=<j3ClQhaZEITDF`l!j.s*qUgm*>e2iuc*jA;<f)oNnlAC\I%tqlpf"$>9'!0kFp@6FaaRdV:#Ze)0n$]A)fXV'V=/m&akQKk'j-]6pmDl*USDm<E@nTinm_h%ll`B87H"m/;8'p$VN64+(On(JcHspJNm0pX>K?5.4jKY8>fg3FV!_D>aE[qm`4e-.i#_(A>Z9iqW:f)@`ku7&&Pnh&jCn-3+5`-64N;A<_?gGRk!'J1%+).i-hEfY=TX*nJY#,O&4*f%]55n)MVeLaFKF3?a5S*B6,(4=t=1M^"_7];;("8^tfRiCF:V(PNPNKn'<"*Xk@8:KG\CFANYQp?hU^F(^0X1H$tk0$`Y\QZNA0"!m/R@*eq!Z@97%dQ518q>2SMfBs^E]*Q5`,Y96B2b"f)Mu]H>Rg%EN&[5o,jBsS_R#F(Q*K9&+(O@bP!B_rT88_T::iaVHnPJ.$'NF7j"<kJcKcUXh!)FdASaU)5)Y&qd#iBqqTn$A\16`?5.+EgS&AjW1Zd9^,KTJ>A[#Ds*E_35$PlX4p2AS>-#Q[&eOE/'V^S)4Ce=I>"0GG'l5+OY7+2&,;%LbN$:Wh!YFhY--7ur=T:kPHl?nrp]"XMKO*#9*WFGMqI%h$IM0bVC7J7rsILBSsNM&[9BT$/sZ?7)B[YgfiIA"hgr->7Ern=)u4Z.T<5C!j8.oU+Q,!l9oYJZWpbaXP&Dd\+Wm(Q1)$R\cfm"?S@_R)(I12%)6$+IGk;o[k^)E'+]<Z1N8@.(.%d?jc*9!h,VfoHjZdCfUR[Edm_F*Xq\fEFF5KW;"'(kCUtGhd^YH".`RIB82##R:,CDDsi$o*n]?Gj>6j\^0'Mdd'LO0j$>fNrtkZ_K6-hbIs67WI[9$j%3F0^igP;:"fXk?Dm&VX,ujFMHs/jdHO[L#Rp((aD_L]09n/qGcH#lcKZBpIF8:b:C[ej`KO3*;51\Y3Xa-6Dnu2nQet4V?>6tZg8EX_2/kWr<:2s,<VYQ22U(gS%)],=nRc\U@L%%QI+YHC'29k0U'UGVA>a0hXR`@iGaJO5fPQGcmo@htY]mXa7S'JH#2GIOWgT'u=DOS!sh.B%D\fJpoa]j;Ibtom8"7Wl:C*ioQH*\*Up+H<Jh(LG8*K^~>endstream
|
|
72
|
+
endobj
|
|
73
|
+
xref
|
|
74
|
+
0 12
|
|
75
|
+
0000000000 65535 f
|
|
76
|
+
0000000061 00000 n
|
|
77
|
+
0000000112 00000 n
|
|
78
|
+
0000000219 00000 n
|
|
79
|
+
0000000331 00000 n
|
|
80
|
+
0000000450 00000 n
|
|
81
|
+
0000000644 00000 n
|
|
82
|
+
0000000838 00000 n
|
|
83
|
+
0000000906 00000 n
|
|
84
|
+
0000001186 00000 n
|
|
85
|
+
0000001251 00000 n
|
|
86
|
+
0000003032 00000 n
|
|
87
|
+
trailer
|
|
88
|
+
<<
|
|
89
|
+
/ID
|
|
90
|
+
[<363e53a615617ff0e3d5d99a726c39b8><363e53a615617ff0e3d5d99a726c39b8>]
|
|
91
|
+
% ReportLab generated PDF document -- digest (opensource)
|
|
92
|
+
|
|
93
|
+
/Info 8 0 R
|
|
94
|
+
/Root 7 0 R
|
|
95
|
+
/Size 12
|
|
96
|
+
>>
|
|
97
|
+
startxref
|
|
98
|
+
4319
|
|
99
|
+
%%EOF
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
%PDF-1.4
|
|
2
|
+
%���� ReportLab Generated PDF document (opensource)
|
|
3
|
+
1 0 obj
|
|
4
|
+
<<
|
|
5
|
+
/F1 2 0 R /F2 3 0 R
|
|
6
|
+
>>
|
|
7
|
+
endobj
|
|
8
|
+
2 0 obj
|
|
9
|
+
<<
|
|
10
|
+
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
|
11
|
+
>>
|
|
12
|
+
endobj
|
|
13
|
+
3 0 obj
|
|
14
|
+
<<
|
|
15
|
+
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
|
16
|
+
>>
|
|
17
|
+
endobj
|
|
18
|
+
4 0 obj
|
|
19
|
+
<<
|
|
20
|
+
/BitsPerComponent 8 /ColorSpace /DeviceRGB /Filter [ /ASCII85Decode /FlateDecode ] /Height 200 /Length 255 /Subtype /Image
|
|
21
|
+
/Type /XObject /Width 300
|
|
22
|
+
>>
|
|
23
|
+
stream
|
|
24
|
+
Gb"0<!=8`+$j3/G$ar_:p`'OXFAZ%<<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3%!<E3&lcN@.'qUP~>endstream
|
|
25
|
+
endobj
|
|
26
|
+
5 0 obj
|
|
27
|
+
<<
|
|
28
|
+
/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
|
|
29
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] /XObject <<
|
|
30
|
+
/FormXob.33833d909d83ea6c265a8c21bc2adfc1 4 0 R
|
|
31
|
+
>>
|
|
32
|
+
>> /Rotate 0 /Trans <<
|
|
33
|
+
|
|
34
|
+
>>
|
|
35
|
+
/Type /Page
|
|
36
|
+
>>
|
|
37
|
+
endobj
|
|
38
|
+
6 0 obj
|
|
39
|
+
<<
|
|
40
|
+
/PageMode /UseNone /Pages 8 0 R /Type /Catalog
|
|
41
|
+
>>
|
|
42
|
+
endobj
|
|
43
|
+
7 0 obj
|
|
44
|
+
<<
|
|
45
|
+
/Author (\(anonymous\)) /CreationDate (D:20260918162747+02'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260918162747+02'00') /Producer (ReportLab PDF Library - \(opensource\))
|
|
46
|
+
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
|
47
|
+
>>
|
|
48
|
+
endobj
|
|
49
|
+
8 0 obj
|
|
50
|
+
<<
|
|
51
|
+
/Count 1 /Kids [ 5 0 R ] /Type /Pages
|
|
52
|
+
>>
|
|
53
|
+
endobj
|
|
54
|
+
9 0 obj
|
|
55
|
+
<<
|
|
56
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 341
|
|
57
|
+
>>
|
|
58
|
+
stream
|
|
59
|
+
Gas2Dc#/.f&;9L7`>mVRLi'qpCYIq#93Vn>)RC\Wr1o(ClJ'pXW)@>_<jLf6mPgp"?uEK(9abcYCj$e'+KrT@0Ll%nk</MJpiSAmO@m&pV_PC.IGiEG0?N%WNZoT.7EC=*pI5tac@j5j[c9OA,P$VHr9EDt4n\HD.bU8gQp"_qk*$2Aq+cV55VSBL$>TDj<5"!^IGK+Y_258j\%4@0Y%C;Y1)%tJ&gld<K$.;Wj(jprB'6rOI1\r2MP;k9=fEUndKl7SAHXZFct-cJT)/JAE@,Ug&>uSJh_)_]N`(oVomi7&VJb]pgMc;Y`,I>hA4DNkCZSG)Y*W]5#^uc+qh+V~>endstream
|
|
60
|
+
endobj
|
|
61
|
+
xref
|
|
62
|
+
0 10
|
|
63
|
+
0000000000 65535 f
|
|
64
|
+
0000000061 00000 n
|
|
65
|
+
0000000102 00000 n
|
|
66
|
+
0000000209 00000 n
|
|
67
|
+
0000000321 00000 n
|
|
68
|
+
0000000766 00000 n
|
|
69
|
+
0000001022 00000 n
|
|
70
|
+
0000001090 00000 n
|
|
71
|
+
0000001370 00000 n
|
|
72
|
+
0000001429 00000 n
|
|
73
|
+
trailer
|
|
74
|
+
<<
|
|
75
|
+
/ID
|
|
76
|
+
[<a37d360f94bbcb79be8c654f5f762036><a37d360f94bbcb79be8c654f5f762036>]
|
|
77
|
+
% ReportLab generated PDF document -- digest (opensource)
|
|
78
|
+
|
|
79
|
+
/Info 7 0 R
|
|
80
|
+
/Root 6 0 R
|
|
81
|
+
/Size 10
|
|
82
|
+
>>
|
|
83
|
+
startxref
|
|
84
|
+
1860
|
|
85
|
+
%%EOF
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Unit tests for MarkerConverter's exception classification - mocked so
|
|
2
|
+
these run fast and don't depend on whether llama-server happens to be
|
|
3
|
+
installed on the machine running the tests. See
|
|
4
|
+
docmd/converters/marker_converter.py for the real (slow, environment-
|
|
5
|
+
dependent) version of this failure, found via real OCR testing on a
|
|
6
|
+
genuinely scanned PDF from archive.org.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import docmd.converters.marker_converter as marker_converter_module
|
|
12
|
+
from docmd.config import ConvertConfig
|
|
13
|
+
from docmd.converters.marker_converter import MarkerConverter
|
|
14
|
+
from docmd.errors import MissingSystemDependencyError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class _FakeSpawnError(Exception):
|
|
18
|
+
"""Stands in for surya.inference.backends.spawn.SpawnError without
|
|
19
|
+
importing surya's internal exception class directly."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class _FakePdfConverterMissingBinary:
|
|
23
|
+
def __init__(self, **_kwargs) -> None:
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
def __call__(self, _filepath: str):
|
|
27
|
+
raise _FakeSpawnError(
|
|
28
|
+
"llama-server binary not found. Install with:\n"
|
|
29
|
+
" macOS: brew install llama.cpp"
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_missing_llama_server_binary_raises_clear_error(monkeypatch, tmp_path):
|
|
34
|
+
monkeypatch.setattr(marker_converter_module, "_get_model_dict", lambda: {})
|
|
35
|
+
monkeypatch.setattr(
|
|
36
|
+
"marker.converters.pdf.PdfConverter", _FakePdfConverterMissingBinary
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
dummy_pdf = tmp_path / "dummy.pdf"
|
|
40
|
+
dummy_pdf.write_bytes(b"%PDF-1.4\n%%EOF")
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
MarkerConverter().convert(str(dummy_pdf), ConvertConfig())
|
|
44
|
+
raise AssertionError("expected MissingSystemDependencyError")
|
|
45
|
+
except MissingSystemDependencyError as exc:
|
|
46
|
+
message = str(exc)
|
|
47
|
+
assert "llama.cpp" in message
|
|
48
|
+
assert "brew install llama.cpp" in message
|
|
49
|
+
assert "LLAMA_CPP_BINARY" in message
|
|
@@ -9,6 +9,30 @@ def test_heading_normalize_fixes_level_skip():
|
|
|
9
9
|
assert out == "# Title\n## Subsection\n"
|
|
10
10
|
|
|
11
11
|
|
|
12
|
+
def test_heading_normalize_fixes_level_skip_then_leaves_recovered_depth_alone():
|
|
13
|
+
"""The exact pattern found in a real 27-page IMF report while stress-
|
|
14
|
+
testing against downloaded (not synthetic) documents: an H1 jumps
|
|
15
|
+
straight to H4 (skipping H2/H3), gets clamped to H2 - but a *later*
|
|
16
|
+
heading that's also originally H4 is left alone, because an H3 heading
|
|
17
|
+
in between legitimately re-establishes the depth. Never reproduced
|
|
18
|
+
synthetically (see skip_headings.pdf in git history, removed after
|
|
19
|
+
Marker's own layout model wouldn't cooperate) - this encodes the real
|
|
20
|
+
structure without redistributing the copyrighted source PDF."""
|
|
21
|
+
md = (
|
|
22
|
+
"# Summary of the Economy Classification\n"
|
|
23
|
+
"#### General Features and Composition of Groups\n"
|
|
24
|
+
"### Advanced Economies\n"
|
|
25
|
+
"#### Emerging Market and Developing Economies\n"
|
|
26
|
+
)
|
|
27
|
+
out = normalize_headings(md)
|
|
28
|
+
assert out == (
|
|
29
|
+
"# Summary of the Economy Classification\n"
|
|
30
|
+
"## General Features and Composition of Groups\n"
|
|
31
|
+
"### Advanced Economies\n"
|
|
32
|
+
"#### Emerging Market and Developing Economies\n"
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
12
36
|
def test_heading_normalize_drops_empty_heading():
|
|
13
37
|
md = "# Title\n##\nBody text.\n"
|
|
14
38
|
out = normalize_headings(md)
|
|
@@ -98,3 +122,16 @@ def test_image_handling_alt_text_mode_fills_empty_alt():
|
|
|
98
122
|
md = ""
|
|
99
123
|
out = apply_image_handling(md, images={}, mode="alt-text")
|
|
100
124
|
assert out == ""
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_image_handling_drops_fully_empty_image_reference():
|
|
128
|
+
"""Real Marker output on a scanned page: `![]()` with no href at all,
|
|
129
|
+
directly adjacent to a real image reference with no separator between
|
|
130
|
+
them - found while OCR-testing a genuinely scanned PDF. The old regex
|
|
131
|
+
required at least one character inside the parens, so `![]()` never
|
|
132
|
+
matched and leaked through every mode unprocessed."""
|
|
133
|
+
md = "![]()"
|
|
134
|
+
|
|
135
|
+
assert apply_image_handling(md, images={}, mode="placeholder") == "*[image omitted]*"
|
|
136
|
+
assert apply_image_handling(md, images={}, mode="skip") == ""
|
|
137
|
+
assert apply_image_handling(md, images={}, mode="alt-text") == ""
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Integration test proving post-processing fixes a *real* Marker bug, not
|
|
2
|
+
just hand-crafted markdown strings (see tests/test_postprocess.py for those -
|
|
3
|
+
useful for fast unit coverage, but circular as evidence of real-world value,
|
|
4
|
+
since the input was written to match what the regex expects).
|
|
5
|
+
|
|
6
|
+
fixtures/stress.pdf has a 45-row table that spans a PDF page break
|
|
7
|
+
(reportlab's repeatRows=1, matching how real multi-page business reports are
|
|
8
|
+
laid out). Marker processes pages independently, so it renders this as two
|
|
9
|
+
separate Markdown tables with a duplicated header row in between - this test
|
|
10
|
+
converts the real PDF through real Marker, confirms that split actually
|
|
11
|
+
happens (so this test fails loudly if a future Marker version stops doing
|
|
12
|
+
it, rather than silently testing nothing), then confirms docmd's full
|
|
13
|
+
convert() pipeline merges them back into one continuous table.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from docmd import convert_document
|
|
19
|
+
from docmd.config import ConvertConfig
|
|
20
|
+
from docmd.converters.marker_converter import MarkerConverter
|
|
21
|
+
|
|
22
|
+
FIXTURE = Path(__file__).parent / "fixtures" / "stress.pdf"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_marker_really_does_split_a_page_spanning_table():
|
|
26
|
+
"""Documents the real upstream behavior this feature exists to fix."""
|
|
27
|
+
raw = MarkerConverter().convert(str(FIXTURE), ConvertConfig())
|
|
28
|
+
# Raw Marker output pads cell widths with extra spaces for column
|
|
29
|
+
# alignment (e.g. "| Region | Rep |"), so match loosely rather than
|
|
30
|
+
# on an exact literal string.
|
|
31
|
+
header_count = sum(
|
|
32
|
+
1
|
|
33
|
+
for line in raw.markdown.splitlines()
|
|
34
|
+
if "Region" in line and "Revenue" in line and line.strip().startswith("|")
|
|
35
|
+
)
|
|
36
|
+
assert header_count >= 2, (
|
|
37
|
+
"expected Marker to render the page-spanning table as 2+ blocks "
|
|
38
|
+
"with a repeated header - if this fails, Marker's behavior changed "
|
|
39
|
+
"and test_docmd_merges_the_split_table's premise needs revisiting"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_docmd_merges_the_split_table():
|
|
44
|
+
result = convert_document(str(FIXTURE))
|
|
45
|
+
markdown = result.markdown
|
|
46
|
+
|
|
47
|
+
# One continuous table: header appears exactly once, not once per page.
|
|
48
|
+
assert markdown.count("| Region | Rep | Deal | Revenue | Status |") == 1
|
|
49
|
+
|
|
50
|
+
# All 45 data rows survived the merge - none silently dropped, including
|
|
51
|
+
# the ones on both sides of where the original page-break duplicate
|
|
52
|
+
# header used to sit (rows 27 and 28 of 45).
|
|
53
|
+
for i in range(1, 46):
|
|
54
|
+
assert f"Deal-{1000 + i}" in markdown
|
|
55
|
+
|
|
56
|
+
# The row right after the original page-break duplicate header is still
|
|
57
|
+
# present as a *data* row, not swallowed along with the header.
|
|
58
|
+
assert "North | Rep 28 | Deal-1028" in markdown
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_docmd_promotes_first_heading_to_h1():
|
|
62
|
+
"""Source PDF's title paragraph rendered as H2 in raw Marker output (its
|
|
63
|
+
font size wasn't visually distinct enough for Marker's heading-level
|
|
64
|
+
detection) - docmd promotes a document's opening heading to H1 rather
|
|
65
|
+
than leaving it start at H2 with no root."""
|
|
66
|
+
result = convert_document(str(FIXTURE))
|
|
67
|
+
first_line = result.markdown.splitlines()[0]
|
|
68
|
+
assert first_line.startswith("# ") and "Annual Sales Report" in first_line
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""More real-Marker-bug regression tests, generated by "do many more tests" -
|
|
2
|
+
each one converts a real PDF through real Marker and checks docmd's output,
|
|
3
|
+
not hand-written markdown. See test_postprocess_integration.py for the
|
|
4
|
+
first one (page-spanning table). Honest negative results are included too,
|
|
5
|
+
not just wins - see test_merged_header_cells_survive_cleanly.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from docmd import convert_document
|
|
11
|
+
from docmd.config import ConvertConfig
|
|
12
|
+
from docmd.converters.marker_converter import MarkerConverter
|
|
13
|
+
|
|
14
|
+
FIXTURES = Path(__file__).parent / "fixtures"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_inconsistent_heading_levels_get_normalized_to_match():
|
|
18
|
+
"""running_header.pdf has 4 structurally identical section headings
|
|
19
|
+
(same paragraph style). On the machine this was first written on (macOS,
|
|
20
|
+
MPS), raw Marker assigned them *different* levels - Section 1 came back
|
|
21
|
+
H3, Sections 2-4 came back H2 - a genuine model inconsistency, not a
|
|
22
|
+
deliberately constructed skip.
|
|
23
|
+
|
|
24
|
+
That specific raw output isn't portable, though: CI (Linux, CPU) ran the
|
|
25
|
+
exact same PDF through the exact same pinned Marker version and got all
|
|
26
|
+
four sections back as H2 from the start - a real platform-dependent
|
|
27
|
+
difference in the underlying ML model's inference, not something our
|
|
28
|
+
code controls. Hard-pinning the raw "bug" broke CI twice for that
|
|
29
|
+
reason. So this only *asserts* the thing docmd actually guarantees -
|
|
30
|
+
consistent heading levels across all four sections, however Marker
|
|
31
|
+
happened to render them - and reports (without failing) whether this
|
|
32
|
+
particular run's raw output shows the original inconsistency."""
|
|
33
|
+
raw = MarkerConverter().convert(str(FIXTURES / "running_header.pdf"), ConvertConfig())
|
|
34
|
+
if "### **Section 1: Findings**" in raw.markdown:
|
|
35
|
+
print("raw Marker output reproduced the H3-vs-H2 inconsistency on this run")
|
|
36
|
+
else:
|
|
37
|
+
print("raw Marker output did not reproduce the inconsistency on this run/platform")
|
|
38
|
+
|
|
39
|
+
result = convert_document(str(FIXTURES / "running_header.pdf"))
|
|
40
|
+
levels = {
|
|
41
|
+
n: "###" if f"### **Section {n}: Findings**" in result.markdown else "##"
|
|
42
|
+
for n in range(1, 5)
|
|
43
|
+
}
|
|
44
|
+
assert len(set(levels.values())) == 1, f"inconsistent levels in docmd output: {levels}"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_running_header_does_not_leak_into_output():
|
|
48
|
+
"""The bold text drawn at a fixed position on every page ("CONFIDENTIAL
|
|
49
|
+
- INTERNAL REPORT") never appears in Marker's raw markdown at all -
|
|
50
|
+
Marker's own PageHeaderProcessor already strips repeated page headers.
|
|
51
|
+
This is an honest negative result: docmd's duplicate-heading dedup
|
|
52
|
+
wasn't needed here because Marker solved it upstream. Pinned as a
|
|
53
|
+
regression test so a future Marker version silently changing this
|
|
54
|
+
doesn't go unnoticed."""
|
|
55
|
+
raw = MarkerConverter().convert(str(FIXTURES / "running_header.pdf"), ConvertConfig())
|
|
56
|
+
assert "CONFIDENTIAL" not in raw.markdown
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_merged_header_cells_survive_cleanly():
|
|
60
|
+
"""merged_cells.pdf has a spanned header cell (colspan). Honest negative
|
|
61
|
+
result: Marker already renders this as a clean, consistent 5-column
|
|
62
|
+
table (blank cells for the spanned columns) with no ragged rows -
|
|
63
|
+
nothing for table_cleanup.py to fix here. Pinned so this doesn't
|
|
64
|
+
silently regress, and so nobody assumes this case needs a fix it
|
|
65
|
+
doesn't need."""
|
|
66
|
+
raw = MarkerConverter().convert(str(FIXTURES / "merged_cells.pdf"), ConvertConfig())
|
|
67
|
+
table_lines = [l for l in raw.markdown.splitlines() if l.strip().startswith("|")]
|
|
68
|
+
# spanned quarter-label row, column-label row, separator, 2 data rows.
|
|
69
|
+
assert len(table_lines) == 5
|
|
70
|
+
assert all(line.count("|") == 6 for line in table_lines) # 5 columns, consistent
|
|
71
|
+
|
|
72
|
+
result = convert_document(str(FIXTURES / "merged_cells.pdf"))
|
|
73
|
+
assert "West" in result.markdown and "$1.2M" in result.markdown
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_alt_text_mode_actually_saves_the_image(tmp_path):
|
|
77
|
+
"""Regression test for a real bug found while doing this testing:
|
|
78
|
+
`output_dir` was accepted by apply_image_handling() but never actually
|
|
79
|
+
threaded through from convert()/convert_document(), so alt-text mode
|
|
80
|
+
produced a markdown image link pointing to a file that was never
|
|
81
|
+
written anywhere. Fixed in docmd/__init__.py and docmd/cli.py."""
|
|
82
|
+
result = convert_document(
|
|
83
|
+
str(FIXTURES / "with_image.pdf"),
|
|
84
|
+
config=ConvertConfig(image_mode="alt-text"),
|
|
85
|
+
output_dir=tmp_path,
|
|
86
|
+
)
|
|
87
|
+
assert ")
|
|
90
|
+
assert len(saved_images) == 1
|
|
91
|
+
assert saved_images[0].name in result.markdown
|
|
92
|
+
# A real, valid JPEG - not an empty/placeholder file.
|
|
93
|
+
assert saved_images[0].read_bytes()[:2] == b"\xff\xd8"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_placeholder_mode_never_writes_image_files(tmp_path):
|
|
97
|
+
"""The default mode should never touch the filesystem, even if an
|
|
98
|
+
output_dir happens to be passed."""
|
|
99
|
+
result = convert_document(
|
|
100
|
+
str(FIXTURES / "with_image.pdf"),
|
|
101
|
+
config=ConvertConfig(image_mode="placeholder"),
|
|
102
|
+
output_dir=tmp_path,
|
|
103
|
+
)
|
|
104
|
+
assert "omitted" in result.markdown
|
|
105
|
+
assert list(tmp_path.iterdir()) == []
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|