pdf-rtldoc 1.0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pdf_rtldoc-1.0.6/LICENSE +21 -0
- pdf_rtldoc-1.0.6/PKG-INFO +163 -0
- pdf_rtldoc-1.0.6/README.md +143 -0
- pdf_rtldoc-1.0.6/pdf_rtldoc.egg-info/PKG-INFO +163 -0
- pdf_rtldoc-1.0.6/pdf_rtldoc.egg-info/SOURCES.txt +19 -0
- pdf_rtldoc-1.0.6/pdf_rtldoc.egg-info/dependency_links.txt +1 -0
- pdf_rtldoc-1.0.6/pdf_rtldoc.egg-info/entry_points.txt +2 -0
- pdf_rtldoc-1.0.6/pdf_rtldoc.egg-info/requires.txt +9 -0
- pdf_rtldoc-1.0.6/pdf_rtldoc.egg-info/top_level.txt +1 -0
- pdf_rtldoc-1.0.6/pyproject.toml +44 -0
- pdf_rtldoc-1.0.6/rtldoc/__init__.py +1 -0
- pdf_rtldoc-1.0.6/rtldoc/arabic.py +204 -0
- pdf_rtldoc-1.0.6/rtldoc/cli.py +75 -0
- pdf_rtldoc-1.0.6/rtldoc/cvfallback.py +111 -0
- pdf_rtldoc-1.0.6/rtldoc/geobidi.py +262 -0
- pdf_rtldoc-1.0.6/rtldoc/layout.py +2026 -0
- pdf_rtldoc-1.0.6/rtldoc/ocr.py +115 -0
- pdf_rtldoc-1.0.6/rtldoc/pipeline.py +843 -0
- pdf_rtldoc-1.0.6/rtldoc/primitives.py +631 -0
- pdf_rtldoc-1.0.6/rtldoc/visual.py +508 -0
- pdf_rtldoc-1.0.6/setup.cfg +4 -0
pdf_rtldoc-1.0.6/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Umer Javaid
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pdf-rtldoc
|
|
3
|
+
Version: 1.0.6
|
|
4
|
+
Summary: Geometry-first PDF parser for RTL and complex layouts (Arabic textbooks, financial tables, multi-column docs)
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/umerjavaidkh/rtldoc
|
|
7
|
+
Project-URL: Repository, https://github.com/umerjavaidkh/rtldoc
|
|
8
|
+
Project-URL: Issues, https://github.com/umerjavaidkh/rtldoc/issues
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: pymupdf>=1.24
|
|
13
|
+
Requires-Dist: numpy
|
|
14
|
+
Provides-Extra: cv
|
|
15
|
+
Requires-Dist: opencv-python-headless; extra == "cv"
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: arabic-reshaper; extra == "dev"
|
|
18
|
+
Requires-Dist: python-bidi; extra == "dev"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# rtldoc
|
|
22
|
+
|
|
23
|
+
**A geometry-first PDF parser for RTL & complex layouts.** It extracts text,
|
|
24
|
+
reading order, and tables from born-digital PDFs using the glyph coordinates
|
|
25
|
+
already in the file — **no OCR, no GPU, no API**. Built for the pages that
|
|
26
|
+
quietly break Docling, Marker, and VLM parsers: Arabic/RTL, multi-column,
|
|
27
|
+
and vector tables.
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## Why it's better where it matters
|
|
32
|
+
|
|
33
|
+
- **RTL / Arabic done right.** Reading order and bidi are rebuilt from glyph
|
|
34
|
+
*positions*, not from a reading-order model trained on English. It fixes the
|
|
35
|
+
presentation-form and lam-alef bugs that silently corrupt ~⅓ of Arabic words
|
|
36
|
+
in every general parser — encoding bugs no layout model can fine-tune away.
|
|
37
|
+
- **Tables without a model.** Reads table structure from the PDF's own vector
|
|
38
|
+
rules at 100% (vs ~85–95% from a CNN), and recovers *borderless* tables by
|
|
39
|
+
column alignment.
|
|
40
|
+
- **Deterministic & auditable.** Every output traces to a rule you can point
|
|
41
|
+
at. CPU-only, **6–60 pages/sec**, $0/page — a VLM is 100–1000× the cost and
|
|
42
|
+
can't be audited.
|
|
43
|
+
- **Page visual summary, no vision model.** Opt-in (`--visual`) geometry pass
|
|
44
|
+
that turns a page's own vector drawings into a structured summary: image
|
|
45
|
+
size/color stats, table dimensions, and — for vector flowcharts/diagrams —
|
|
46
|
+
the actual boxes, their text labels, and which ones a connecting line joins.
|
|
47
|
+
Built entirely from the PDF's own drawing commands (PyMuPDF `get_drawings()`),
|
|
48
|
+
not a screenshot or a guess. HTML output renders any detected diagram as a
|
|
49
|
+
real chart via Mermaid.js, fed the extracted nodes/edges directly.
|
|
50
|
+
|
|
51
|
+
## Proven at scale
|
|
52
|
+
|
|
53
|
+
Tested on **119 PDFs / 13,557 pages** it never saw during development — an
|
|
54
|
+
Arabic teacher's guide, two SEC 10-Ks, 96+ arXiv papers (15 fields), 5 OpenStax
|
|
55
|
+
physics/chemistry/calculus textbooks (figures, geometry, exercises), the
|
|
56
|
+
3,130-page PostgreSQL 18 manual (deeply-nested reference tables, code blocks),
|
|
57
|
+
and a growing set of real-world forms, reports, and scanned documents.
|
|
58
|
+
The checks are *property-based and label-free*, so they scale to any corpus:
|
|
59
|
+
|
|
60
|
+
| Property (must hold on every page) | Result |
|
|
61
|
+
|---|---:|
|
|
62
|
+
| crashes | **0** |
|
|
63
|
+
| encoding leaks (presentation forms in output) | **0** |
|
|
64
|
+
| malformed tables | **0** |
|
|
65
|
+
| non-deterministic pages | **0** |
|
|
66
|
+
| text coverage vs the PDF's own glyph stream | **~99%** |
|
|
67
|
+
|
|
68
|
+
**Table quality, scored with TEDS** (the PubTabNet/OmniDocBench standard) on a
|
|
69
|
+
borderless financial statement — where the whole point is a hard table:
|
|
70
|
+
|
|
71
|
+
| rtldoc | pdfplumber | naive `get_text` |
|
|
72
|
+
|---:|---:|---:|
|
|
73
|
+
| **0.942** | 0.061 | 0.000 |
|
|
74
|
+
|
|
75
|
+
Everything is reproducible in [`eval/`](eval/) (harnesses, arXiv manifest,
|
|
76
|
+
saved reports).
|
|
77
|
+
|
|
78
|
+
## Use it
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install pdf-rtldoc
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
(Published on PyPI as `pdf-rtldoc` -- the plain `rtldoc` name was already
|
|
85
|
+
taken by an unrelated project. The installed command, and the module you
|
|
86
|
+
import in Python, are both still `rtldoc`.)
|
|
87
|
+
|
|
88
|
+
Or install straight from a specific release without going through PyPI:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
pip install "git+https://github.com/umerjavaidkh/rtldoc.git@v1.0.6"
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Then:
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
rtldoc parse book.pdf --md out/ --json out.json
|
|
98
|
+
rtldoc parse book.pdf --html out_html/ # real <table>/<figure>, RTL-aware dir=
|
|
99
|
+
rtldoc parse book.pdf --html out_html/ --visual # + diagram/image/table visual summary
|
|
100
|
+
rtldoc audit book.pdf # flags low-confidence pages for review
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Runtime deps are just PyMuPDF + numpy. (From a clone: `pip install -e .`.)
|
|
104
|
+
|
|
105
|
+
Zero-setup via Docker (399 MB, no compiler/GPU):
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
docker build -t rtldoc . && docker run --rm -v "$PWD:/d" rtldoc parse /d/book.pdf --md /d/out
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Output: per-page Markdown (tables as GFM, images extracted + auto-captioned)
|
|
112
|
+
plus structured JSON, or a self-contained HTML page per PDF page.
|
|
113
|
+
|
|
114
|
+
## Honest limits
|
|
115
|
+
|
|
116
|
+
- Borderless-table *grid geometry* is approximate on the hardest wide,
|
|
117
|
+
multi-level-header tables (occasional row/column structure mismatches
|
|
118
|
+
— that's the 0.942 TEDS on our graded case, not 1.0).
|
|
119
|
+
- A page dominated by a figure/table spanning the full content width can
|
|
120
|
+
still be under-counted as fewer columns than it visually has — the
|
|
121
|
+
whitespace-gutter detector requires the gap to stay empty across most
|
|
122
|
+
of the page's height, and a full-width element defeats that locally.
|
|
123
|
+
- A table cell whose own text wraps onto a later line, where that line's
|
|
124
|
+
content coincidentally re-aligns with an earlier row's columns, can
|
|
125
|
+
occasionally attach to the wrong row (`_merge_wrapped_label_rows` — a
|
|
126
|
+
narrow, tracked edge case, not a general table-detection failure).
|
|
127
|
+
- Scanned / no-text-layer pages now OCR via Tesseract (needs the `tesseract`
|
|
128
|
+
binary on PATH — `brew install tesseract` / `apt install tesseract-ocr`;
|
|
129
|
+
no Python package required). Word-level positioning, not this repo's
|
|
130
|
+
glyph-exact reading order; a page with no tesseract installed just gets
|
|
131
|
+
no blocks, as before.
|
|
132
|
+
- Best semantic typing needs a one-time per-publisher style map (~20 min).
|
|
133
|
+
- Diagram detection reconstructs simple box-and-arrow flowcharts reliably;
|
|
134
|
+
dense multi-level diagrams (deep tree/org-chart hierarchies with many
|
|
135
|
+
branches) get correct node/box detection but not yet reliable connection
|
|
136
|
+
tracing — a harder, separate problem noted for future work.
|
|
137
|
+
|
|
138
|
+
## Roadmap
|
|
139
|
+
|
|
140
|
+
- Reliable connection tracing for dense/branching diagrams (see above).
|
|
141
|
+
- A page-level chart/figure classification pass, so bar charts, legends,
|
|
142
|
+
and gridlines are recognized and set aside before table/diagram
|
|
143
|
+
detection runs, rather than relying on those detectors' own guards to
|
|
144
|
+
reject them case by case.
|
|
145
|
+
- A cell-level golden regression corpus (`eval/golden/` + `eval/regression.py`)
|
|
146
|
+
now exists and grows with each table-detection fix; still short of full
|
|
147
|
+
coverage across document types.
|
|
148
|
+
|
|
149
|
+
## Publishing a release (maintainers)
|
|
150
|
+
|
|
151
|
+
Bump `version` in `pyproject.toml`, commit, tag (`git tag -a vX.Y.Z`), push
|
|
152
|
+
the tag, then publish a GitHub Release from it. `.github/workflows/publish.yml`
|
|
153
|
+
builds and uploads to PyPI automatically when the release is published, via
|
|
154
|
+
PyPI's Trusted Publisher (OIDC) mechanism — no API token stored anywhere.
|
|
155
|
+
|
|
156
|
+
One-time setup (already done for `pdf-rtldoc`): on pypi.org, under the
|
|
157
|
+
project's *Publishing* settings, add a trusted publisher with owner
|
|
158
|
+
`umerjavaidkh`, repository `rtldoc`, workflow filename `publish.yml`, and
|
|
159
|
+
environment name `pypi`.
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
*Design rationale, the full bug log, and methodology: **[DESIGN.md](DESIGN.md)**.*
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# rtldoc
|
|
2
|
+
|
|
3
|
+
**A geometry-first PDF parser for RTL & complex layouts.** It extracts text,
|
|
4
|
+
reading order, and tables from born-digital PDFs using the glyph coordinates
|
|
5
|
+
already in the file — **no OCR, no GPU, no API**. Built for the pages that
|
|
6
|
+
quietly break Docling, Marker, and VLM parsers: Arabic/RTL, multi-column,
|
|
7
|
+
and vector tables.
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## Why it's better where it matters
|
|
12
|
+
|
|
13
|
+
- **RTL / Arabic done right.** Reading order and bidi are rebuilt from glyph
|
|
14
|
+
*positions*, not from a reading-order model trained on English. It fixes the
|
|
15
|
+
presentation-form and lam-alef bugs that silently corrupt ~⅓ of Arabic words
|
|
16
|
+
in every general parser — encoding bugs no layout model can fine-tune away.
|
|
17
|
+
- **Tables without a model.** Reads table structure from the PDF's own vector
|
|
18
|
+
rules at 100% (vs ~85–95% from a CNN), and recovers *borderless* tables by
|
|
19
|
+
column alignment.
|
|
20
|
+
- **Deterministic & auditable.** Every output traces to a rule you can point
|
|
21
|
+
at. CPU-only, **6–60 pages/sec**, $0/page — a VLM is 100–1000× the cost and
|
|
22
|
+
can't be audited.
|
|
23
|
+
- **Page visual summary, no vision model.** Opt-in (`--visual`) geometry pass
|
|
24
|
+
that turns a page's own vector drawings into a structured summary: image
|
|
25
|
+
size/color stats, table dimensions, and — for vector flowcharts/diagrams —
|
|
26
|
+
the actual boxes, their text labels, and which ones a connecting line joins.
|
|
27
|
+
Built entirely from the PDF's own drawing commands (PyMuPDF `get_drawings()`),
|
|
28
|
+
not a screenshot or a guess. HTML output renders any detected diagram as a
|
|
29
|
+
real chart via Mermaid.js, fed the extracted nodes/edges directly.
|
|
30
|
+
|
|
31
|
+
## Proven at scale
|
|
32
|
+
|
|
33
|
+
Tested on **119 PDFs / 13,557 pages** it never saw during development — an
|
|
34
|
+
Arabic teacher's guide, two SEC 10-Ks, 96+ arXiv papers (15 fields), 5 OpenStax
|
|
35
|
+
physics/chemistry/calculus textbooks (figures, geometry, exercises), the
|
|
36
|
+
3,130-page PostgreSQL 18 manual (deeply-nested reference tables, code blocks),
|
|
37
|
+
and a growing set of real-world forms, reports, and scanned documents.
|
|
38
|
+
The checks are *property-based and label-free*, so they scale to any corpus:
|
|
39
|
+
|
|
40
|
+
| Property (must hold on every page) | Result |
|
|
41
|
+
|---|---:|
|
|
42
|
+
| crashes | **0** |
|
|
43
|
+
| encoding leaks (presentation forms in output) | **0** |
|
|
44
|
+
| malformed tables | **0** |
|
|
45
|
+
| non-deterministic pages | **0** |
|
|
46
|
+
| text coverage vs the PDF's own glyph stream | **~99%** |
|
|
47
|
+
|
|
48
|
+
**Table quality, scored with TEDS** (the PubTabNet/OmniDocBench standard) on a
|
|
49
|
+
borderless financial statement — where the whole point is a hard table:
|
|
50
|
+
|
|
51
|
+
| rtldoc | pdfplumber | naive `get_text` |
|
|
52
|
+
|---:|---:|---:|
|
|
53
|
+
| **0.942** | 0.061 | 0.000 |
|
|
54
|
+
|
|
55
|
+
Everything is reproducible in [`eval/`](eval/) (harnesses, arXiv manifest,
|
|
56
|
+
saved reports).
|
|
57
|
+
|
|
58
|
+
## Use it
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install pdf-rtldoc
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
(Published on PyPI as `pdf-rtldoc` -- the plain `rtldoc` name was already
|
|
65
|
+
taken by an unrelated project. The installed command, and the module you
|
|
66
|
+
import in Python, are both still `rtldoc`.)
|
|
67
|
+
|
|
68
|
+
Or install straight from a specific release without going through PyPI:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip install "git+https://github.com/umerjavaidkh/rtldoc.git@v1.0.6"
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Then:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
rtldoc parse book.pdf --md out/ --json out.json
|
|
78
|
+
rtldoc parse book.pdf --html out_html/ # real <table>/<figure>, RTL-aware dir=
|
|
79
|
+
rtldoc parse book.pdf --html out_html/ --visual # + diagram/image/table visual summary
|
|
80
|
+
rtldoc audit book.pdf # flags low-confidence pages for review
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Runtime deps are just PyMuPDF + numpy. (From a clone: `pip install -e .`.)
|
|
84
|
+
|
|
85
|
+
Zero-setup via Docker (399 MB, no compiler/GPU):
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
docker build -t rtldoc . && docker run --rm -v "$PWD:/d" rtldoc parse /d/book.pdf --md /d/out
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Output: per-page Markdown (tables as GFM, images extracted + auto-captioned)
|
|
92
|
+
plus structured JSON, or a self-contained HTML page per PDF page.
|
|
93
|
+
|
|
94
|
+
## Honest limits
|
|
95
|
+
|
|
96
|
+
- Borderless-table *grid geometry* is approximate on the hardest wide,
|
|
97
|
+
multi-level-header tables (occasional row/column structure mismatches
|
|
98
|
+
— that's the 0.942 TEDS on our graded case, not 1.0).
|
|
99
|
+
- A page dominated by a figure/table spanning the full content width can
|
|
100
|
+
still be under-counted as fewer columns than it visually has — the
|
|
101
|
+
whitespace-gutter detector requires the gap to stay empty across most
|
|
102
|
+
of the page's height, and a full-width element defeats that locally.
|
|
103
|
+
- A table cell whose own text wraps onto a later line, where that line's
|
|
104
|
+
content coincidentally re-aligns with an earlier row's columns, can
|
|
105
|
+
occasionally attach to the wrong row (`_merge_wrapped_label_rows` — a
|
|
106
|
+
narrow, tracked edge case, not a general table-detection failure).
|
|
107
|
+
- Scanned / no-text-layer pages now OCR via Tesseract (needs the `tesseract`
|
|
108
|
+
binary on PATH — `brew install tesseract` / `apt install tesseract-ocr`;
|
|
109
|
+
no Python package required). Word-level positioning, not this repo's
|
|
110
|
+
glyph-exact reading order; a page with no tesseract installed just gets
|
|
111
|
+
no blocks, as before.
|
|
112
|
+
- Best semantic typing needs a one-time per-publisher style map (~20 min).
|
|
113
|
+
- Diagram detection reconstructs simple box-and-arrow flowcharts reliably;
|
|
114
|
+
dense multi-level diagrams (deep tree/org-chart hierarchies with many
|
|
115
|
+
branches) get correct node/box detection but not yet reliable connection
|
|
116
|
+
tracing — a harder, separate problem noted for future work.
|
|
117
|
+
|
|
118
|
+
## Roadmap
|
|
119
|
+
|
|
120
|
+
- Reliable connection tracing for dense/branching diagrams (see above).
|
|
121
|
+
- A page-level chart/figure classification pass, so bar charts, legends,
|
|
122
|
+
and gridlines are recognized and set aside before table/diagram
|
|
123
|
+
detection runs, rather than relying on those detectors' own guards to
|
|
124
|
+
reject them case by case.
|
|
125
|
+
- A cell-level golden regression corpus (`eval/golden/` + `eval/regression.py`)
|
|
126
|
+
now exists and grows with each table-detection fix; still short of full
|
|
127
|
+
coverage across document types.
|
|
128
|
+
|
|
129
|
+
## Publishing a release (maintainers)
|
|
130
|
+
|
|
131
|
+
Bump `version` in `pyproject.toml`, commit, tag (`git tag -a vX.Y.Z`), push
|
|
132
|
+
the tag, then publish a GitHub Release from it. `.github/workflows/publish.yml`
|
|
133
|
+
builds and uploads to PyPI automatically when the release is published, via
|
|
134
|
+
PyPI's Trusted Publisher (OIDC) mechanism — no API token stored anywhere.
|
|
135
|
+
|
|
136
|
+
One-time setup (already done for `pdf-rtldoc`): on pypi.org, under the
|
|
137
|
+
project's *Publishing* settings, add a trusted publisher with owner
|
|
138
|
+
`umerjavaidkh`, repository `rtldoc`, workflow filename `publish.yml`, and
|
|
139
|
+
environment name `pypi`.
|
|
140
|
+
|
|
141
|
+
---
|
|
142
|
+
|
|
143
|
+
*Design rationale, the full bug log, and methodology: **[DESIGN.md](DESIGN.md)**.*
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pdf-rtldoc
|
|
3
|
+
Version: 1.0.6
|
|
4
|
+
Summary: Geometry-first PDF parser for RTL and complex layouts (Arabic textbooks, financial tables, multi-column docs)
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/umerjavaidkh/rtldoc
|
|
7
|
+
Project-URL: Repository, https://github.com/umerjavaidkh/rtldoc
|
|
8
|
+
Project-URL: Issues, https://github.com/umerjavaidkh/rtldoc/issues
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: pymupdf>=1.24
|
|
13
|
+
Requires-Dist: numpy
|
|
14
|
+
Provides-Extra: cv
|
|
15
|
+
Requires-Dist: opencv-python-headless; extra == "cv"
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: arabic-reshaper; extra == "dev"
|
|
18
|
+
Requires-Dist: python-bidi; extra == "dev"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# rtldoc
|
|
22
|
+
|
|
23
|
+
**A geometry-first PDF parser for RTL & complex layouts.** It extracts text,
|
|
24
|
+
reading order, and tables from born-digital PDFs using the glyph coordinates
|
|
25
|
+
already in the file — **no OCR, no GPU, no API**. Built for the pages that
|
|
26
|
+
quietly break Docling, Marker, and VLM parsers: Arabic/RTL, multi-column,
|
|
27
|
+
and vector tables.
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## Why it's better where it matters
|
|
32
|
+
|
|
33
|
+
- **RTL / Arabic done right.** Reading order and bidi are rebuilt from glyph
|
|
34
|
+
*positions*, not from a reading-order model trained on English. It fixes the
|
|
35
|
+
presentation-form and lam-alef bugs that silently corrupt ~⅓ of Arabic words
|
|
36
|
+
in every general parser — encoding bugs no layout model can fine-tune away.
|
|
37
|
+
- **Tables without a model.** Reads table structure from the PDF's own vector
|
|
38
|
+
rules at 100% (vs ~85–95% from a CNN), and recovers *borderless* tables by
|
|
39
|
+
column alignment.
|
|
40
|
+
- **Deterministic & auditable.** Every output traces to a rule you can point
|
|
41
|
+
at. CPU-only, **6–60 pages/sec**, $0/page — a VLM is 100–1000× the cost and
|
|
42
|
+
can't be audited.
|
|
43
|
+
- **Page visual summary, no vision model.** Opt-in (`--visual`) geometry pass
|
|
44
|
+
that turns a page's own vector drawings into a structured summary: image
|
|
45
|
+
size/color stats, table dimensions, and — for vector flowcharts/diagrams —
|
|
46
|
+
the actual boxes, their text labels, and which ones a connecting line joins.
|
|
47
|
+
Built entirely from the PDF's own drawing commands (PyMuPDF `get_drawings()`),
|
|
48
|
+
not a screenshot or a guess. HTML output renders any detected diagram as a
|
|
49
|
+
real chart via Mermaid.js, fed the extracted nodes/edges directly.
|
|
50
|
+
|
|
51
|
+
## Proven at scale
|
|
52
|
+
|
|
53
|
+
Tested on **119 PDFs / 13,557 pages** it never saw during development — an
|
|
54
|
+
Arabic teacher's guide, two SEC 10-Ks, 96+ arXiv papers (15 fields), 5 OpenStax
|
|
55
|
+
physics/chemistry/calculus textbooks (figures, geometry, exercises), the
|
|
56
|
+
3,130-page PostgreSQL 18 manual (deeply-nested reference tables, code blocks),
|
|
57
|
+
and a growing set of real-world forms, reports, and scanned documents.
|
|
58
|
+
The checks are *property-based and label-free*, so they scale to any corpus:
|
|
59
|
+
|
|
60
|
+
| Property (must hold on every page) | Result |
|
|
61
|
+
|---|---:|
|
|
62
|
+
| crashes | **0** |
|
|
63
|
+
| encoding leaks (presentation forms in output) | **0** |
|
|
64
|
+
| malformed tables | **0** |
|
|
65
|
+
| non-deterministic pages | **0** |
|
|
66
|
+
| text coverage vs the PDF's own glyph stream | **~99%** |
|
|
67
|
+
|
|
68
|
+
**Table quality, scored with TEDS** (the PubTabNet/OmniDocBench standard) on a
|
|
69
|
+
borderless financial statement — where the whole point is a hard table:
|
|
70
|
+
|
|
71
|
+
| rtldoc | pdfplumber | naive `get_text` |
|
|
72
|
+
|---:|---:|---:|
|
|
73
|
+
| **0.942** | 0.061 | 0.000 |
|
|
74
|
+
|
|
75
|
+
Everything is reproducible in [`eval/`](eval/) (harnesses, arXiv manifest,
|
|
76
|
+
saved reports).
|
|
77
|
+
|
|
78
|
+
## Use it
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install pdf-rtldoc
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
(Published on PyPI as `pdf-rtldoc` -- the plain `rtldoc` name was already
|
|
85
|
+
taken by an unrelated project. The installed command, and the module you
|
|
86
|
+
import in Python, are both still `rtldoc`.)
|
|
87
|
+
|
|
88
|
+
Or install straight from a specific release without going through PyPI:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
pip install "git+https://github.com/umerjavaidkh/rtldoc.git@v1.0.6"
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Then:
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
rtldoc parse book.pdf --md out/ --json out.json
|
|
98
|
+
rtldoc parse book.pdf --html out_html/ # real <table>/<figure>, RTL-aware dir=
|
|
99
|
+
rtldoc parse book.pdf --html out_html/ --visual # + diagram/image/table visual summary
|
|
100
|
+
rtldoc audit book.pdf # flags low-confidence pages for review
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Runtime deps are just PyMuPDF + numpy. (From a clone: `pip install -e .`.)
|
|
104
|
+
|
|
105
|
+
Zero-setup via Docker (399 MB, no compiler/GPU):
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
docker build -t rtldoc . && docker run --rm -v "$PWD:/d" rtldoc parse /d/book.pdf --md /d/out
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Output: per-page Markdown (tables as GFM, images extracted + auto-captioned)
|
|
112
|
+
plus structured JSON, or a self-contained HTML page per PDF page.
|
|
113
|
+
|
|
114
|
+
## Honest limits
|
|
115
|
+
|
|
116
|
+
- Borderless-table *grid geometry* is approximate on the hardest wide,
|
|
117
|
+
multi-level-header tables (occasional row/column structure mismatches
|
|
118
|
+
— that's the 0.942 TEDS on our graded case, not 1.0).
|
|
119
|
+
- A page dominated by a figure/table spanning the full content width can
|
|
120
|
+
still be under-counted as fewer columns than it visually has — the
|
|
121
|
+
whitespace-gutter detector requires the gap to stay empty across most
|
|
122
|
+
of the page's height, and a full-width element defeats that locally.
|
|
123
|
+
- A table cell whose own text wraps onto a later line, where that line's
|
|
124
|
+
content coincidentally re-aligns with an earlier row's columns, can
|
|
125
|
+
occasionally attach to the wrong row (`_merge_wrapped_label_rows` — a
|
|
126
|
+
narrow, tracked edge case, not a general table-detection failure).
|
|
127
|
+
- Scanned / no-text-layer pages now OCR via Tesseract (needs the `tesseract`
|
|
128
|
+
binary on PATH — `brew install tesseract` / `apt install tesseract-ocr`;
|
|
129
|
+
no Python package required). Word-level positioning, not this repo's
|
|
130
|
+
glyph-exact reading order; a page with no tesseract installed just gets
|
|
131
|
+
no blocks, as before.
|
|
132
|
+
- Best semantic typing needs a one-time per-publisher style map (~20 min).
|
|
133
|
+
- Diagram detection reconstructs simple box-and-arrow flowcharts reliably;
|
|
134
|
+
dense multi-level diagrams (deep tree/org-chart hierarchies with many
|
|
135
|
+
branches) get correct node/box detection but not yet reliable connection
|
|
136
|
+
tracing — a harder, separate problem noted for future work.
|
|
137
|
+
|
|
138
|
+
## Roadmap
|
|
139
|
+
|
|
140
|
+
- Reliable connection tracing for dense/branching diagrams (see above).
|
|
141
|
+
- A page-level chart/figure classification pass, so bar charts, legends,
|
|
142
|
+
and gridlines are recognized and set aside before table/diagram
|
|
143
|
+
detection runs, rather than relying on those detectors' own guards to
|
|
144
|
+
reject them case by case.
|
|
145
|
+
- A cell-level golden regression corpus (`eval/golden/` + `eval/regression.py`)
|
|
146
|
+
now exists and grows with each table-detection fix; still short of full
|
|
147
|
+
coverage across document types.
|
|
148
|
+
|
|
149
|
+
## Publishing a release (maintainers)
|
|
150
|
+
|
|
151
|
+
Bump `version` in `pyproject.toml`, commit, tag (`git tag -a vX.Y.Z`), push
|
|
152
|
+
the tag, then publish a GitHub Release from it. `.github/workflows/publish.yml`
|
|
153
|
+
builds and uploads to PyPI automatically when the release is published, via
|
|
154
|
+
PyPI's Trusted Publisher (OIDC) mechanism — no API token stored anywhere.
|
|
155
|
+
|
|
156
|
+
One-time setup (already done for `pdf-rtldoc`): on pypi.org, under the
|
|
157
|
+
project's *Publishing* settings, add a trusted publisher with owner
|
|
158
|
+
`umerjavaidkh`, repository `rtldoc`, workflow filename `publish.yml`, and
|
|
159
|
+
environment name `pypi`.
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
*Design rationale, the full bug log, and methodology: **[DESIGN.md](DESIGN.md)**.*
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
pdf_rtldoc.egg-info/PKG-INFO
|
|
5
|
+
pdf_rtldoc.egg-info/SOURCES.txt
|
|
6
|
+
pdf_rtldoc.egg-info/dependency_links.txt
|
|
7
|
+
pdf_rtldoc.egg-info/entry_points.txt
|
|
8
|
+
pdf_rtldoc.egg-info/requires.txt
|
|
9
|
+
pdf_rtldoc.egg-info/top_level.txt
|
|
10
|
+
rtldoc/__init__.py
|
|
11
|
+
rtldoc/arabic.py
|
|
12
|
+
rtldoc/cli.py
|
|
13
|
+
rtldoc/cvfallback.py
|
|
14
|
+
rtldoc/geobidi.py
|
|
15
|
+
rtldoc/layout.py
|
|
16
|
+
rtldoc/ocr.py
|
|
17
|
+
rtldoc/pipeline.py
|
|
18
|
+
rtldoc/primitives.py
|
|
19
|
+
rtldoc/visual.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
rtldoc
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
# PyPI distribution name is "pdf-rtldoc" -- "rtldoc" was already taken by
|
|
7
|
+
# an unrelated SystemVerilog documentation tool. The importable module
|
|
8
|
+
# stays `rtldoc` (see [tool.setuptools] below) and the installed CLI
|
|
9
|
+
# command stays `rtldoc` (see [project.scripts]) -- neither has to match
|
|
10
|
+
# the PyPI distribution name.
|
|
11
|
+
name = "pdf-rtldoc"
|
|
12
|
+
version = "1.0.6"
|
|
13
|
+
description = "Geometry-first PDF parser for RTL and complex layouts (Arabic textbooks, financial tables, multi-column docs)"
|
|
14
|
+
readme = "README.md"
|
|
15
|
+
license = { text = "MIT" }
|
|
16
|
+
requires-python = ">=3.10"
|
|
17
|
+
dependencies = [
|
|
18
|
+
"pymupdf>=1.24",
|
|
19
|
+
"numpy",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
# cvfallback.py (pixel-based fallback for scanned/flattened pages) isn't
|
|
24
|
+
# wired into the pipeline yet -- nothing in rtldoc/ imports cv2 today.
|
|
25
|
+
cv = ["opencv-python-headless"]
|
|
26
|
+
# ocr.py's scanned-page fallback shells out to the `tesseract` CLI binary
|
|
27
|
+
# directly (apt/brew install tesseract-ocr) rather than the pytesseract
|
|
28
|
+
# Python wrapper -- no pip package needed, so there's no extras group for
|
|
29
|
+
# it. Optional at runtime: if the binary isn't on PATH, a scanned page just
|
|
30
|
+
# gets no blocks, same as before this feature existed.
|
|
31
|
+
# Only tests/make_fixture.py (the synthetic-PDF generator for local testing)
|
|
32
|
+
# needs these; the parser itself never shapes or reverses text on its own.
|
|
33
|
+
dev = ["arabic-reshaper", "python-bidi"]
|
|
34
|
+
|
|
35
|
+
[project.urls]
|
|
36
|
+
Homepage = "https://github.com/umerjavaidkh/rtldoc"
|
|
37
|
+
Repository = "https://github.com/umerjavaidkh/rtldoc"
|
|
38
|
+
Issues = "https://github.com/umerjavaidkh/rtldoc/issues"
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
rtldoc = "rtldoc.cli:main"
|
|
42
|
+
|
|
43
|
+
[tool.setuptools]
|
|
44
|
+
packages = ["rtldoc"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""rtldoc — a geometry-first parser for RTL complex-layout PDFs."""
|