d2md 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. d2md-0.1.0/.gitignore +26 -0
  2. d2md-0.1.0/CHANGELOG.md +43 -0
  3. d2md-0.1.0/LICENSE +21 -0
  4. d2md-0.1.0/PKG-INFO +582 -0
  5. d2md-0.1.0/README.md +537 -0
  6. d2md-0.1.0/SECURITY.md +88 -0
  7. d2md-0.1.0/bench/README.md +138 -0
  8. d2md-0.1.0/bench/detect.py +150 -0
  9. d2md-0.1.0/bench/gaps.py +80 -0
  10. d2md-0.1.0/bench/legacy_safe.py +109 -0
  11. d2md-0.1.0/bench/make_corpus.py +398 -0
  12. d2md-0.1.0/bench/matrix.py +924 -0
  13. d2md-0.1.0/bench/matrix_report.py +1226 -0
  14. d2md-0.1.0/bench/matrix_worker.py +463 -0
  15. d2md-0.1.0/bench/report.py +144 -0
  16. d2md-0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/README.md +228 -0
  17. d2md-0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/macos-arm64-cpu.json +2895 -0
  18. d2md-0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/macos-arm64-mps.json +1078 -0
  19. d2md-0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/summary.csv +220 -0
  20. d2md-0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/ubuntu-gtx1060-cpu.json +2834 -0
  21. d2md-0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/ubuntu-gtx1060-cuda.json +993 -0
  22. d2md-0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/windows-rtx3090ti-cpu.json +2810 -0
  23. d2md-0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/windows-rtx3090ti-cuda.json +969 -0
  24. d2md-0.1.0/bench/run.py +492 -0
  25. d2md-0.1.0/bench/score.py +155 -0
  26. d2md-0.1.0/bench/shipped.py +106 -0
  27. d2md-0.1.0/bench/support.py +206 -0
  28. d2md-0.1.0/bench/surya_run.py +118 -0
  29. d2md-0.1.0/bench/vlm.py +119 -0
  30. d2md-0.1.0/docs/findings.md +213 -0
  31. d2md-0.1.0/docs/ocr.md +631 -0
  32. d2md-0.1.0/docs/release-process.md +83 -0
  33. d2md-0.1.0/docs/verification.md +63 -0
  34. d2md-0.1.0/examples/README.md +78 -0
  35. d2md-0.1.0/examples/generate.py +443 -0
  36. d2md-0.1.0/examples/smoke.py +141 -0
  37. d2md-0.1.0/examples/verify.py +61 -0
  38. d2md-0.1.0/pyproject.toml +77 -0
  39. d2md-0.1.0/scripts/smoke-linux.sh +158 -0
  40. d2md-0.1.0/scripts/smoke-windows.ps1 +126 -0
  41. d2md-0.1.0/src/d2md/__init__.py +3 -0
  42. d2md-0.1.0/src/d2md/_backend_process.py +911 -0
  43. d2md-0.1.0/src/d2md/_onnx.py +29 -0
  44. d2md-0.1.0/src/d2md/capabilities.py +29 -0
  45. d2md-0.1.0/src/d2md/cli.py +1278 -0
  46. d2md-0.1.0/src/d2md/convert.py +1346 -0
  47. d2md-0.1.0/src/d2md/direct_ocr.py +114 -0
  48. d2md-0.1.0/src/d2md/encoding.py +216 -0
  49. d2md-0.1.0/src/d2md/errors.py +5 -0
  50. d2md-0.1.0/src/d2md/ocr.py +279 -0
  51. d2md-0.1.0/tests/test_backend_process.py +1117 -0
  52. d2md-0.1.0/tests/test_benchmark_legacy_security.py +461 -0
  53. d2md-0.1.0/tests/test_benchmark_matrix.py +1620 -0
  54. d2md-0.1.0/tests/test_benchmark_matrix_security.py +935 -0
  55. d2md-0.1.0/tests/test_capabilities.py +25 -0
  56. d2md-0.1.0/tests/test_cli_capabilities.py +155 -0
  57. d2md-0.1.0/tests/test_cli_import.py +41 -0
  58. d2md-0.1.0/tests/test_cli_machine_output.py +296 -0
  59. d2md-0.1.0/tests/test_cli_option_paths.py +173 -0
  60. d2md-0.1.0/tests/test_convert.py +423 -0
  61. d2md-0.1.0/tests/test_device_cli.py +83 -0
  62. d2md-0.1.0/tests/test_direct_ocr.py +194 -0
  63. d2md-0.1.0/tests/test_encoding.py +106 -0
  64. d2md-0.1.0/tests/test_examples.py +285 -0
  65. d2md-0.1.0/tests/test_install_profiles.py +156 -0
  66. d2md-0.1.0/tests/test_lang_cli.py +106 -0
  67. d2md-0.1.0/tests/test_ocr.py +248 -0
  68. d2md-0.1.0/tests/test_onnx_telemetry.py +70 -0
  69. d2md-0.1.0/tests/test_routing_is_wired.py +302 -0
  70. d2md-0.1.0/tests/test_security.py +1399 -0
  71. d2md-0.1.0/tests/test_smoke_scripts.py +102 -0
  72. d2md-0.1.0/tests/test_text_pdf.py +141 -0
d2md-0.1.0/.gitignore ADDED
@@ -0,0 +1,26 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .DS_Store
4
+ .claude/
5
+ .coverage
6
+ .coverage.*
7
+ .venv/
8
+ .venv-smoke/
9
+ .venv-smoke-profile/
10
+ .worktrees/
11
+ build/
12
+ dist/
13
+ htmlcov/
14
+ *.egg-info/
15
+ .mypy_cache/
16
+ .nox/
17
+ .pytest_cache/
18
+ .ruff_cache/
19
+ .tox/
20
+
21
+ # Generated application and benchmark output
22
+ md-out/
23
+ corpus/
24
+ bench-out/
25
+ examples/generated/
26
+ examples/converted/
@@ -0,0 +1,43 @@
1
+ # Changelog
2
+
3
+ All notable changes are recorded here.
4
+
5
+ ## [Unreleased]
6
+
7
+ No changes yet.
8
+
9
+ ## [0.1.0]
10
+
11
+ - Local conversion of text files, Office/web documents, and PDFs to Markdown,
12
+ with a lightweight default path.
13
+ - Explicit optional `ocr` and `docling` profiles for scanned documents,
14
+ layout, reading order, headings, and tables.
15
+ - Thai-aware handling for legacy TIS-620/CP874 text and measured damaged Thai
16
+ PDF text layers.
17
+ - Machine-friendly `--stdout`, `--json`, and `--capabilities` CLI modes for
18
+ shell automation and AI agents.
19
+ - Resource ceilings, symbolic-link rejection, atomic output handling on POSIX,
20
+ and explicit errors for unsupported capabilities and unsafe paths.
21
+ - Early plain-text and PDF output checks, collected-file identity binding, and
22
+ a cumulative CLI output budget for hostile or unexpectedly large batches.
23
+ - Content-aware ZIP, PDF, and image preflight so misleading filename suffixes
24
+ cannot skip format-specific limits; ZIP packages must match their declared
25
+ document family, and generic recursive ZIP conversion is disabled.
26
+ - Full-string MarkItDown and Docling jobs run in reusable subprocesses, with
27
+ output ceilings enforced before the caller accepts the result and failed
28
+ workers discarded.
29
+ - Option-looking paths require explicit `--`/`./` disambiguation and cannot
30
+ silently enable conversion capabilities.
31
+ - The benchmark-promotion schema filters private values from public evidence.
32
+ - The build backend is pinned for reproducible release builds.
33
+ - A hosted CI workflow covering Python 3.10–3.13 across Linux, macOS, and
34
+ Windows, plus wheel and source-distribution smoke-test jobs.
35
+ - A coordinated security policy and a release gate that requires private
36
+ vulnerability reporting to be enabled and verified when the repository
37
+ becomes public.
38
+ - Manual cross-platform release-candidate verification documented in
39
+ [docs/verification.md](docs/verification.md). Those results are historical
40
+ manual evidence and do not replace current-revision hosted CI.
41
+ - Prominent macOS Apple Vision OCR guidance with historical, commit-pinned
42
+ accuracy evidence, zero separate OCR-weight downloads, and an explicit tested
43
+ hardware reference rather than an unsupported minimum claim.
d2md-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 wtigero
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
d2md-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,582 @@
1
+ Metadata-Version: 2.5
2
+ Name: d2md
3
+ Version: 0.1.0
4
+ Summary: Local document-to-Markdown conversion with Thai-aware text extraction and OCR
5
+ Project-URL: Homepage, https://github.com/wtigero/d2md
6
+ Project-URL: Repository, https://github.com/wtigero/d2md
7
+ Project-URL: Issues, https://github.com/wtigero/d2md/issues
8
+ Project-URL: Changelog, https://github.com/wtigero/d2md/blob/main/CHANGELOG.md
9
+ Project-URL: Security, https://github.com/wtigero/d2md/security/policy
10
+ Author: wtigero
11
+ License: MIT
12
+ License-File: LICENSE
13
+ Keywords: docling,markdown,markitdown,ocr,pdf,rag,thai
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Natural Language :: Thai
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Topic :: Text Processing :: Markup :: Markdown
18
+ Requires-Python: <3.14,>=3.10
19
+ Requires-Dist: markitdown[docx,pptx,xlsx]>=0.1
20
+ Requires-Dist: olefile<1,>=0.47
21
+ Requires-Dist: pypdfium2<6,>=5
22
+ Requires-Dist: xlrd<3,>=2
23
+ Provides-Extra: benchmark
24
+ Requires-Dist: psutil>=5.9; extra == 'benchmark'
25
+ Provides-Extra: dev
26
+ Requires-Dist: pillow>=10; extra == 'dev'
27
+ Requires-Dist: pytest>=8; extra == 'dev'
28
+ Requires-Dist: python-docx<2,>=1; extra == 'dev'
29
+ Requires-Dist: reportlab>=4; extra == 'dev'
30
+ Requires-Dist: tomli>=2; (python_version < '3.11') and extra == 'dev'
31
+ Requires-Dist: xlwt<2,>=1.3; extra == 'dev'
32
+ Provides-Extra: docling
33
+ Requires-Dist: docling<3,>=2.119; extra == 'docling'
34
+ Requires-Dist: numpy>=1.24; (sys_platform != 'darwin') and extra == 'docling'
35
+ Requires-Dist: ocrmac<2,>=1; (sys_platform == 'darwin') and extra == 'docling'
36
+ Requires-Dist: pillow>=10; extra == 'docling'
37
+ Requires-Dist: rapidocr<4,>=3; (sys_platform != 'darwin') and extra == 'docling'
38
+ Requires-Dist: transformers<5.15; extra == 'docling'
39
+ Provides-Extra: ocr
40
+ Requires-Dist: numpy>=1.24; (sys_platform != 'darwin') and extra == 'ocr'
41
+ Requires-Dist: ocrmac<2,>=1; (sys_platform == 'darwin') and extra == 'ocr'
42
+ Requires-Dist: pillow>=10; extra == 'ocr'
43
+ Requires-Dist: rapidocr<4,>=3; (sys_platform != 'darwin') and extra == 'ocr'
44
+ Description-Content-Type: text/markdown
45
+
46
+ # d2md
47
+
48
+ Convert local documents to Markdown for reading, search, RAG, scripts, and AI
49
+ agents. The default path stays lightweight; OCR and Docling run only when both
50
+ installed and explicitly requested.
51
+
52
+ - Convert one file or a whole directory tree.
53
+ - Read normal PDFs without loading an ML stack.
54
+ - OCR scanned PDFs and images only with `--ocr`.
55
+ - Use Docling for layout, reading order, headings, and tables only with
56
+ `--docling`.
57
+ - Preserve legacy Thai TIS-620/CP874 text and detect measured Thai PDF damage.
58
+ - Fail explicitly on empty output, unavailable languages, and unsafe paths.
59
+
60
+ Conversion runs locally. On macOS, direct `--ocr` uses Apple Vision supplied by
61
+ macOS, with no API key and no separate OCR model or weight download. RapidOCR
62
+ on Linux or Windows and Docling on every platform may retrieve model files on
63
+ their first explicit use.
64
+
65
+ > **macOS highlight: accurate OCR without a model download.** Install the OCR
66
+ > profile once and direct `--ocr` uses Apple Vision supplied by macOS, locally—no
67
+ > API key and no separate OCR model or weight download. In the promoted
68
+ > benchmark for commit [`7203499`](https://github.com/wtigero/d2md/blob/v0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/README.md),
69
+ > direct Apple Vision OCR measured `0` to `0.0299` no-space character error
70
+ > rate (`0%` to `2.99%`) across ten clean fixtures, including Thai at `0.0034`
71
+ > (`0.34%`). This is historical, commit-pinned, fixture-bounded evidence rather
72
+ > than a claim about every document or a current-CI accuracy result. It applies
73
+ > to direct `--ocr`; Docling remains an optional, model-backed profile.
74
+
75
+ ## Choose a mode
76
+
77
+ Start with the lightest mode that fits the document:
78
+
79
+ | Need | Install profile | Command |
80
+ |---|---|---|
81
+ | Text already present in PDF, Office, web, or text files | Base | `d2md report.pdf` |
82
+ | Scanned PDF or image | OCR | `d2md scan.pdf --ocr` |
83
+ | Layout, headings, reading order, or tables | Docling | `d2md report.pdf --docling` |
84
+ | A scanned document with Docling structure | Docling | `d2md scan.pdf --docling --ocr` |
85
+ | Markdown for an AI or shell pipeline | Matching profile | `d2md report.pdf --stdout` |
86
+ | A machine-readable batch report | Matching profile | `d2md documents/ --json` |
87
+
88
+ Installing an optional profile never changes an ordinary command. The flags
89
+ still decide which capability is allowed to run.
90
+
91
+ ## Minimum requirements and tested resource floor
92
+
93
+ Python 3.10 through 3.13 is supported. Every profile works on CPU; a GPU is not
94
+ required. MPS, CUDA, and XPU are optional device selections for Docling only;
95
+ the published hardware evidence covers MPS and CUDA, not XPU. Direct OCR does
96
+ not load PyTorch.
97
+
98
+ | Profile | Minimum execution path | First-use model data |
99
+ |---|---|---|
100
+ | Base | Supported Python and CPU | None |
101
+ | Direct OCR on macOS | CPU plus Apple Vision | No separate OCR model download |
102
+ | Direct OCR on Linux or Windows | CPU | RapidOCR weights may be retrieved |
103
+ | Docling | CPU; acceleration is optional | Docling artifacts may be retrieved |
104
+
105
+ The [lowest-memory tested system](https://github.com/wtigero/d2md/blob/v0.1.0/docs/verification.md#hardware-and-relevant-packages)—not
106
+ a minimum requirement—was an Apple M3 with 16 GiB RAM. Base, direct Apple
107
+ Vision OCR, Docling CPU, and Docling MPS all passed there. Lower-memory hosts
108
+ may work but have not been qualified, and no fixed RAM or storage minimum is
109
+ claimed because document size and the selected backend dominate both. Leave
110
+ additional disk space for optional model caches, and retain the built-in
111
+ [safety limits](#safety-limits) for untrusted input.
112
+
113
+ ## Quick start: normal documents
114
+
115
+ Install the base tool from PyPI:
116
+
117
+ ```bash
118
+ uv tool install d2md
119
+ d2md report.pdf --stdout
120
+ ```
121
+
122
+ Or install it into an existing Python environment:
123
+
124
+ ```bash
125
+ pip install d2md
126
+ ```
127
+
128
+ This reads text already present in PDFs, Office/web documents, and text files.
129
+ It does not run OCR or load Docling.
130
+
131
+ Convert a directory tree or choose another output directory:
132
+
133
+ ```bash
134
+ d2md ~/Documents
135
+ d2md ~/Documents -o converted
136
+ ```
137
+
138
+ Output defaults to `md-out/`. For example, `reports/annual.pdf` becomes
139
+ `md-out/annual.md`.
140
+
141
+ ## Use with AI and automation
142
+
143
+ No AI SDK, API key, or model is required. An AI tool can call the same local
144
+ CLI and choose one of two machine-friendly outputs.
145
+
146
+ ### Get Markdown on standard output
147
+
148
+ Use `--stdout` when the next process needs the converted content itself:
149
+
150
+ ```bash
151
+ d2md report.pdf --stdout
152
+ d2md scan.pdf --ocr --lang thai --stdout
153
+ d2md report.pdf --docling --stdout | your-ai-command
154
+ ```
155
+
156
+ `--stdout` accepts exactly one collected input, writes no Markdown file, and
157
+ prints no progress text into the stream. Errors go to standard error and the
158
+ command exits non-zero. Because it does not write files, it cannot be combined
159
+ with `-o`, `--outdir`, or `--force`.
160
+
161
+ ### Get a JSON run report
162
+
163
+ Use `--json` for batch automation and agents that need status, paths, backend,
164
+ timing, and errors:
165
+
166
+ ```bash
167
+ d2md documents/ -o converted --json
168
+ d2md scans/ --ocr --lang thai -o converted --json
169
+ ```
170
+
171
+ `--json` still writes Markdown files. Its standard output is one JSON object
172
+ and contains no human progress lines. Backend diagnostics, if any, go to
173
+ standard error:
174
+
175
+ ```json
176
+ {
177
+ "schema_version": 1,
178
+ "ok": true,
179
+ "options": {"device": "auto", "docling": false, "language": null, "ocr": false},
180
+ "summary": {"converted": 1, "failed": 0, "skipped": 0},
181
+ "results": [
182
+ {
183
+ "source": "report.pdf",
184
+ "output": "md-out/report.md",
185
+ "status": "converted",
186
+ "backend": "pypdfium2",
187
+ "characters": 12480,
188
+ "seconds": 0.012345
189
+ }
190
+ ],
191
+ "errors": [],
192
+ "warnings": []
193
+ }
194
+ ```
195
+
196
+ The schema is versioned. After valid argument parsing, a failed run keeps the
197
+ same envelope, sets `ok` to `false`, lists structured `errors`, and returns a
198
+ non-zero exit code.
199
+
200
+ ### Let an AI inspect this machine
201
+
202
+ ```bash
203
+ d2md --capabilities --json
204
+ ```
205
+
206
+ This reports installed OCR engines, readable script groups, whether Docling is
207
+ installed, the accepted device choices, and exact commands for installing the
208
+ optional profiles. Capability discovery exits zero even when optional
209
+ components are absent, so an agent can inspect first and act without guessing.
210
+ For a shorter human-readable OCR-only view, run `d2md --engines`.
211
+
212
+ ## OCR scanned documents
213
+
214
+ Install the OCR profile and opt in with `--ocr`:
215
+
216
+ ```bash
217
+ uv tool install "d2md[ocr]"
218
+ d2md scan.pdf --ocr
219
+ ```
220
+
221
+ If the base tool is already installed, replace its environment with the OCR
222
+ profile:
223
+
224
+ ```bash
225
+ uv tool install --force "d2md[ocr]"
226
+ ```
227
+
228
+ `--ocr` permits OCR for scanned PDFs and images. A healthy PDF with a usable
229
+ text layer still uses direct text extraction; it is not re-OCRed.
230
+
231
+ When the script is known, pass `--lang` to avoid extra detection passes:
232
+
233
+ ```bash
234
+ d2md scan.pdf --ocr --lang thai
235
+ d2md scans/ --ocr --lang latin
236
+ ```
237
+
238
+ Script groups are not language counts. One Latin model can cover several
239
+ languages, while Thai, Japanese, Chinese, Korean, Cyrillic, and Arabic require
240
+ separate script handling. Availability is platform-specific:
241
+
242
+ | Platform | Engine | Script groups exposed | Clean benchmark fixtures completed |
243
+ |---|---|---|---|
244
+ | macOS | Apple Vision through `ocrmac` | Latin, Thai, Japanese, Chinese, Korean, Cyrillic, Arabic | English, German, Vietnamese, Thai, Japanese, Simplified and Traditional Chinese, Korean, Russian, Arabic |
245
+ | Linux and Windows | RapidOCR | Latin, Japanese, Chinese | English, German, Vietnamese, Japanese, Simplified and Traditional Chinese |
246
+
247
+ “Completed” means the route ran and was scored, not that every result met an
248
+ accuracy threshold. Vietnamese on RapidOCR measured a no-space character error
249
+ rate of `0.1903`, so this project does not make a high-accuracy Vietnamese claim
250
+ for Linux or Windows.
251
+
252
+ Inspect the current installation rather than guessing from the table:
253
+
254
+ ```bash
255
+ d2md --engines
256
+ d2md --capabilities --json
257
+ ```
258
+
259
+ An unsupported script is refused before the first file; it is never silently
260
+ mapped to Latin. The error lists the scripts available on that installation.
261
+ Apple Vision is supplied by macOS and needs no separate OCR weight download.
262
+ RapidOCR may retrieve model files when a configured model is used for the first
263
+ time.
264
+
265
+ ## Layout and tables with Docling
266
+
267
+ Use Docling when layout, heading, reading-order, or table reconstruction is
268
+ more important than the lightweight text-only route:
269
+
270
+ ```bash
271
+ uv tool install "d2md[docling]"
272
+ d2md report.pdf --docling
273
+ d2md scan.pdf --docling --ocr
274
+ ```
275
+
276
+ If another profile is already installed, add `--force` to the installation
277
+ command. The Docling profile includes the platform OCR dependencies.
278
+
279
+ `--docling` does not imply OCR. Add `--ocr` only for scans or images. Docling
280
+ may retrieve its model artifacts on first explicit use.
281
+
282
+ ### NVIDIA CUDA
283
+
284
+ For NVIDIA CUDA, let `uv` select matching PyTorch wheels for the machine:
285
+
286
+ ```bash
287
+ uv tool install --torch-backend auto "d2md[docling]"
288
+ ```
289
+
290
+ Add `--force` when replacing an existing installation. If `auto` selects a
291
+ wheel that excludes an older Pascal GPU such as the GTX 10 series, select the
292
+ compatible CUDA 12.6 wheels explicitly:
293
+
294
+ ```bash
295
+ uv tool install --force --torch-backend cu126 "d2md[docling]"
296
+ ```
297
+
298
+ Keep torch and torchvision on the same backend build. The commands above let
299
+ `uv` resolve the pair together; replacing only one of them can break Docling
300
+ imports. `--torch-backend` requires a recent `uv` and is currently marked
301
+ experimental by `uv`.
302
+
303
+ Device selection belongs only to Docling. Direct OCR does not use `--device`:
304
+
305
+ ```bash
306
+ d2md report.pdf --docling --device cpu
307
+ d2md scan.pdf --docling --ocr --device cuda
308
+ d2md scan.pdf --docling --ocr --device mps
309
+ ```
310
+
311
+ Supported device names are `auto`, `cpu`, `cuda`, `mps`, and `xpu`. An
312
+ explicit unavailable accelerator is a strict error and does not fall back to
313
+ CPU.
314
+
315
+ ## Installation notes
316
+
317
+ The package supports Python 3.10 through 3.13. To replace an existing profile,
318
+ repeat its command with `--force`.
319
+
320
+ ### Source and release verification
321
+
322
+ For source installs or verifying the exact `v0.1.0` release tag, use:
323
+
324
+ ```bash
325
+ uv tool install "d2md @ git+https://github.com/wtigero/d2md.git@v0.1.0"
326
+ ```
327
+
328
+ Tags and public releases are promoted through the gates in
329
+ [docs/release-process.md](https://github.com/wtigero/d2md/blob/v0.1.0/docs/release-process.md).
330
+
331
+ For repository development, create an isolated environment and install the
332
+ test plus optional dependencies:
333
+
334
+ ```bash
335
+ uv venv --python 3.12 .venv
336
+ uv pip install --python .venv/bin/python -e '.[dev,docling]'
337
+ .venv/bin/python -m pytest
338
+ ```
339
+
340
+ ## Formats and routing
341
+
342
+ Installed extras never change an unflagged command. Routing depends only on
343
+ the input type and explicit flags.
344
+
345
+ | Input or option | Route |
346
+ |---|---|
347
+ | `.txt .md .csv .json .xml .yaml .yml` | validated direct read |
348
+ | `.docx .xlsx .xls .pptx .html .htm .msg .epub` | MarkItDown |
349
+ | Healthy PDF without `--docling` | pypdfium2 text extraction |
350
+ | Scanned/damaged PDF with `--ocr` | direct page rendering plus the selected OCR engine |
351
+ | Image with `--ocr` | the selected OCR engine |
352
+ | PDF/image with `--docling` | Docling; OCR only when `--ocr` is also present |
353
+ | Explicit file (or library call) with an unknown non-archive extension | content-aware PDF/image preflight, then a MarkItDown attempt |
354
+ | Generic `.zip` or renamed ZIP archive | rejected; extract and pass a supported document directly |
355
+
356
+ A scanned PDF, damaged Thai text layer, or image without `--ocr` returns an
357
+ actionable error containing both the OCR installation command and required
358
+ flag. There is no automatic fallback into an installed optional stack.
359
+ Recursive directory discovery collects only known supported suffixes; it does
360
+ not apply the unknown-extension fallback.
361
+
362
+ ## Command reference
363
+
364
+ ```bash
365
+ d2md report.pdf
366
+ d2md --force -- *.xlsx *.pdf
367
+ d2md scans/ --ocr
368
+ d2md scans/ --ocr --lang thai
369
+ d2md reports/ --docling --device cpu
370
+ d2md report.pdf --stdout
371
+ d2md reports/ --json
372
+ d2md --capabilities --json
373
+ d2md --version
374
+ ```
375
+
376
+ Put options before `--` when shell wildcards may expand to filenames beginning
377
+ with `-`, as in the example above. `--` (or a `./` prefix) disambiguates such
378
+ legal paths from options; an option-looking path never authorizes OCR, Docling,
379
+ or any other capability.
380
+
381
+ | Option | Behavior |
382
+ |---|---|
383
+ | `-o DIR`, `--outdir DIR` | write Markdown to `DIR` instead of `md-out` |
384
+ | `-f`, `--force` | replace an existing output safely |
385
+ | `-q`, `--quiet` | print failures and the final summary only |
386
+ | `--stdout` | convert exactly one input to Markdown on standard output; write no file |
387
+ | `--json` | write files normally and print one versioned JSON run report |
388
+ | `--ocr` | permit OCR for scanned PDFs and images |
389
+ | `--docling` | use Docling for PDF/image layout and tables |
390
+ | `--lang SCRIPT` | select the OCR script; requires `--ocr` |
391
+ | `--device MODE` | select the Docling device; non-default values require `--docling` |
392
+ | `--engines` | show locally available OCR engines and configured scripts |
393
+ | `--capabilities` | show OCR, Docling, and device capabilities; combine with `--json` for agents |
394
+ | `--version` | print the installed `d2md` version and exit |
395
+ | `--unsafe-unlimited` | disable resource ceilings for trusted inputs only |
396
+
397
+ The former `--fast` option is a hidden deprecated compatibility no-op for one
398
+ release. Direct PDF text extraction is now the default.
399
+
400
+ If multiple inputs have the same basename, the first output is kept unless
401
+ `--force` is used. The command exits non-zero if any input fails:
402
+
403
+ ```bash
404
+ d2md ./inbox -o ./markdown || echo "some documents need attention"
405
+ ```
406
+
407
+ ## Safety limits
408
+
409
+ | Resource | Default limit |
410
+ |---|---:|
411
+ | Input file size | 100 MiB |
412
+ | Supported files | 10,000 |
413
+ | Discovered directory entries | 10,000 |
414
+ | Retained collection failures | 10,000 |
415
+ | PDF pages | 500 |
416
+ | Rendered pixels per page | 40 million |
417
+ | Rendered pixels per PDF | 400 million |
418
+ | Extracted/output characters, per file and per CLI run | 20 million |
419
+ | ZIP-based document members | 10,000 |
420
+ | ZIP-based expanded content | 500 MiB |
421
+
422
+ The three input-collection budgets are separate caps, not a combined total.
423
+ Except for them and the cumulative output-character cap, these ceilings apply
424
+ to each file independently. A directory run may therefore spend the input,
425
+ archive, page, pixel, and parser-work allowances again for every collected
426
+ file. Split untrusted trees into small batches and use OS-level time and memory
427
+ limits when a strict whole-run compute budget is required.
428
+
429
+ The CLI rejects links observed during input discovery, binds each selected
430
+ regular file to the identity collected for it, and safely reopens that file
431
+ before parsing. Do not recursively convert a directory tree that another
432
+ process can rename or replace while the run is in progress: portable directory
433
+ enumeration cannot pin every queued directory across the whole traversal. On
434
+ POSIX, output writes are directory-descriptor relative and atomic.
435
+ An output link is never followed; `--force` replaces the link itself rather
436
+ than its target. On Windows, use an output directory you control rather than
437
+ one below a shared attacker-writable parent: portable filesystem APIs cannot
438
+ hold the same kind of directory descriptor across the final write. Terminal
439
+ control characters in filenames and backend errors are escaped before display.
440
+
441
+ The output-character ceiling is enforced while plain text is decoded and before
442
+ an oversized direct-PDF page text layer is extracted, incrementally for direct
443
+ OCR, and cumulatively across files published by one CLI run. Automatic OCR
444
+ language detection can do extra sample rendering and inference before the final
445
+ PDF pixel tally; supply `--lang` to skip that detection for untrusted or tightly
446
+ budgeted jobs. MarkItDown and Docling full-string backends run in reusable,
447
+ independent Python workers. The worker enforces the output ceiling before the
448
+ long-lived caller accepts or materializes the result. A failure, timeout, or
449
+ protocol error discards that worker; a successful worker remains warm for reuse.
450
+ This is not a whole-host sandbox or a hard RAM/GPU boundary. Keep untrusted
451
+ Office/Docling jobs within the input and archive limits above, split hostile
452
+ files into small batches, and apply OS, container, or job-level time and memory
453
+ controls when a hard boundary is required.
454
+
455
+ The default MarkItDown and Docling backend job deadline is 30 minutes (1,800
456
+ seconds). Only the explicit trusted-input `--unsafe-unlimited` mode makes that
457
+ deadline unlimited; it does not turn the worker into a whole-host sandbox.
458
+
459
+ Preflight checks use the verified file content as well as its suffix. Renaming a
460
+ ZIP, PDF, or supported image therefore does not skip its archive, page, or pixel
461
+ checks. A ZIP-based Office document or EPUB must also contain the standard
462
+ markers for the family named by its suffix. Generic and masquerading ZIP
463
+ archives are rejected, and the fallback parser's recursive ZIP converter stays
464
+ disabled; extract an archive and pass the intended document directly.
465
+
466
+ For a known trusted archival job that exceeds these ceilings, use
467
+ `--unsafe-unlimited`. It disables only resource limits; link protections and
468
+ the platform-specific output safeguards above remain active.
469
+
470
+ ## Manual verification
471
+
472
+ Synthetic examples cover every accepted extension. See
473
+ [examples/README.md](https://github.com/wtigero/d2md/blob/v0.1.0/examples/README.md) for the manifest and direct driver.
474
+ See the dated [manual verification results](https://github.com/wtigero/d2md/blob/v0.1.0/docs/verification.md) for the
475
+ tested operating systems, profiles, devices, and fixture counts.
476
+
477
+ Run isolated profiles on Linux or macOS:
478
+
479
+ ```bash
480
+ ./scripts/smoke-linux.sh --profile base
481
+ ./scripts/smoke-linux.sh --profile ocr
482
+ ./scripts/smoke-linux.sh --profile docling --device cpu
483
+ ./scripts/smoke-linux.sh --profile docling --device cuda --require-gpu
484
+ ./scripts/smoke-linux.sh --profile docling --device mps --require-gpu
485
+ ```
486
+
487
+ Run the same profiles from Windows PowerShell:
488
+
489
+ ```powershell
490
+ .\scripts\smoke-windows.ps1 -Profile Base
491
+ .\scripts\smoke-windows.ps1 -Profile Ocr
492
+ .\scripts\smoke-windows.ps1 -Profile Docling -Device cpu
493
+ .\scripts\smoke-windows.ps1 -Profile Docling -Device cuda -RequireGpu
494
+ ```
495
+
496
+ The scripts use one full-test environment and a second profile-only
497
+ environment, print the imported package path, run dependency checks, and
498
+ verify expected markers. No CI is required for these manual release checks.
499
+
500
+ ## Controlled performance measurements
501
+
502
+ The manual smoke checks above prove functionality; they are not speed claims.
503
+ The [production benchmark matrix](https://github.com/wtigero/d2md/blob/v0.1.0/bench/README.md) measures the current
504
+ explicit routes by operating system, CPU/accelerator device, document type,
505
+ method, and language. It records initialization separately from warm timing,
506
+ quality and resource metrics, and promotes results only after all six required
507
+ hardware runs validate against the same commit and fixture hashes rebuilt from
508
+ the clean promotion checkout.
509
+
510
+ The first promoted result covers commit
511
+ [`7203499`](https://github.com/wtigero/d2md/blob/v0.1.0/bench/results/7203499e58bf8e6415b3190638d0f8a689f55924/README.md):
512
+ 219 scenarios across macOS CPU/MPS, Ubuntu CPU/GTX 1060 CUDA, and Windows
513
+ CPU/RTX 3090 Ti CUDA. All six runs used the same fixture hashes. There were no
514
+ unexpected conversion errors.
515
+
516
+ | Platform and device | Success | Explicitly unsupported |
517
+ |---|---:|---:|
518
+ | macOS CPU | 54 | 0 |
519
+ | macOS MPS | 19 | 0 |
520
+ | Ubuntu CPU | 46 | 8 |
521
+ | Ubuntu GTX 1060 CUDA | 15 | 4 |
522
+ | Windows CPU | 46 | 8 |
523
+ | Windows RTX 3090 Ti CUDA | 15 | 4 |
524
+
525
+ CPU runs contain the default, direct-OCR, and Docling routes. Accelerator runs
526
+ contain only Docling scenarios, so their smaller row counts are intentional.
527
+ Representative warm medians for the one-page synthetic fixtures are:
528
+
529
+ | Route and fixture | macOS CPU | macOS MPS | Ubuntu CPU | GTX 1060 CUDA | Windows CPU | RTX 3090 Ti CUDA |
530
+ |---|---:|---:|---:|---:|---:|---:|
531
+ | Born-digital PDF, default | 0.0008 s | — | 0.0009 s | — | 0.0105 s | — |
532
+ | Born-digital PDF, Docling | 0.294 s | 0.124 s | 0.480 s | 0.101 s | 1.688 s | 0.209 s |
533
+ | Scanned PDF, direct OCR | 0.062 s | — | 2.122 s | — | 4.457 s | — |
534
+ | Scanned PDF, Docling + OCR | 0.374 s | 0.191 s | 1.697 s | 1.159 s | 7.104 s | 5.028 s |
535
+
536
+ These are hardware-specific medians after warm-up, not promises for other
537
+ documents or machines. First Docling + OCR initialization on the image fixture
538
+ ranged from 13.3 seconds to 573.1 seconds across the six runs. The 573.1-second
539
+ Ubuntu CPU result included a 60-second PyTorch AVX2 probe timeout on that VM;
540
+ the corresponding warm median was 1.93 seconds. The promoted result contains
541
+ the exact timing range, operation count, quality score, and resource telemetry
542
+ for every document type, method, language, OS, and device.
543
+
544
+ The fixed OCR corpus contains English, German, Vietnamese, Thai, Japanese,
545
+ Simplified Chinese, Traditional Chinese, Korean, Russian, and Arabic. Direct
546
+ Apple Vision OCR completed all ten clean fixtures with no-space character error
547
+ rates from 0 to 0.0299. RapidOCR on Ubuntu and Windows completed English,
548
+ German, Vietnamese, Japanese, and both Chinese fixtures; Thai, Korean, Russian,
549
+ and Arabic were explicitly unsupported. Vietnamese completed but measured
550
+ 0.1903, so the result does not make a high-accuracy Vietnamese claim. Docling
551
+ with OCR is scored separately from direct OCR in the full result.
552
+
553
+ ## Why the Thai checks exist
554
+
555
+ Thai PDFs exported from some applications can contain a damaged `ToUnicode`
556
+ map. The page looks normal, but extracted text loses marks and can turn `จำกัด`
557
+ into `จ ากัด` without raising an error. `d2md` validates the direct PDF text
558
+ and asks for explicit OCR when the known damaged-Thai pattern is detected.
559
+
560
+ Legacy Thai `.txt` files have a similar silent-failure mode: generic encoding
561
+ detection can interpret CP874 as an unrelated encoding. `d2md` validates a
562
+ CP874 candidate by its Thai character ratio before accepting it.
563
+
564
+ The research that motivated these guards is preserved in
565
+ [docs/findings.md](https://github.com/wtigero/d2md/blob/v0.1.0/docs/findings.md). OCR corpus methodology lives in
566
+ [docs/ocr.md](https://github.com/wtigero/d2md/blob/v0.1.0/docs/ocr.md). Those research measurements are historical
567
+ evidence, not performance claims for the current installation profiles.
568
+
569
+ ## Limitations
570
+
571
+ - OCR targets printed text, not handwriting.
572
+ - The lightweight PDF route extracts text but does not reconstruct tables or
573
+ heading levels; request Docling when structure matters.
574
+ - Automatic damaged-text detection is specific to the measured Thai failure
575
+ shape. Other damaged text layers may require manual inspection and `--ocr`.
576
+ - OCR script coverage differs by platform and installed engine.
577
+ - Complex merged-cell tables and values split across line breaks can still
578
+ require cleanup after Docling conversion.
579
+
580
+ ## License
581
+
582
+ MIT