bobine 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. bobine-0.2.0/LICENSE +6 -0
  2. bobine-0.2.0/LICENSES/Apache-2.0.txt +105 -0
  3. bobine-0.2.0/LICENSES/MIT.txt +21 -0
  4. bobine-0.2.0/PKG-INFO +227 -0
  5. bobine-0.2.0/README.md +175 -0
  6. bobine-0.2.0/bobine/__init__.py +84 -0
  7. bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/__init__.py +9 -0
  8. bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/main.py +214 -0
  9. bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/models.py +154 -0
  10. bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/utils.py +185 -0
  11. bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/utils_load.py +172 -0
  12. bobine-0.2.0/bobine/assets.py +90 -0
  13. bobine-0.2.0/bobine/config.py +94 -0
  14. bobine-0.2.0/bobine/converter.py +829 -0
  15. bobine-0.2.0/bobine/documents.py +180 -0
  16. bobine-0.2.0/bobine/engine.py +239 -0
  17. bobine-0.2.0/bobine/markdown.py +103 -0
  18. bobine-0.2.0/bobine/pipeline.py +278 -0
  19. bobine-0.2.0/bobine/tables.py +87 -0
  20. bobine-0.2.0/bobine/versions.py +148 -0
  21. bobine-0.2.0/bobine.egg-info/PKG-INFO +227 -0
  22. bobine-0.2.0/bobine.egg-info/SOURCES.txt +38 -0
  23. bobine-0.2.0/bobine.egg-info/dependency_links.txt +1 -0
  24. bobine-0.2.0/bobine.egg-info/requires.txt +29 -0
  25. bobine-0.2.0/bobine.egg-info/top_level.txt +1 -0
  26. bobine-0.2.0/pyproject.toml +107 -0
  27. bobine-0.2.0/setup.cfg +4 -0
  28. bobine-0.2.0/tests/test_assets.py +52 -0
  29. bobine-0.2.0/tests/test_config.py +37 -0
  30. bobine-0.2.0/tests/test_converter.py +478 -0
  31. bobine-0.2.0/tests/test_converter_flow.py +485 -0
  32. bobine-0.2.0/tests/test_documents.py +75 -0
  33. bobine-0.2.0/tests/test_engine.py +249 -0
  34. bobine-0.2.0/tests/test_formula_vendored.py +78 -0
  35. bobine-0.2.0/tests/test_integration.py +126 -0
  36. bobine-0.2.0/tests/test_markdown.py +52 -0
  37. bobine-0.2.0/tests/test_pdf_corpus.py +106 -0
  38. bobine-0.2.0/tests/test_pipeline.py +256 -0
  39. bobine-0.2.0/tests/test_tables.py +49 -0
  40. bobine-0.2.0/tests/test_versions.py +71 -0
bobine-0.2.0/LICENSE ADDED
@@ -0,0 +1,6 @@
1
+ This project is dual-licensed under the terms of either:
2
+
3
+ - MIT License (see [LICENSES/MIT.txt](LICENSES/MIT.txt))
4
+ - Apache License, Version 2.0 (see [LICENSES/Apache-2.0.txt](LICENSES/Apache-2.0.txt))
5
+
6
+ You may choose either license for your use.
@@ -0,0 +1,105 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean any term or agreement which makes use of the
10
+ following terminology, including the incorporation of the Apache
11
+ License.
12
+
13
+ "Package" shall mean the work resulting from the distribution of the
14
+ Software as defined in the License.
15
+
16
+ "Source" form shall mean the preferred form of the Software for making
17
+ modifications to it.
18
+
19
+ "Object" code form shall mean the modified version of the Software in
20
+ object code form or as a modification of the Software in object code
21
+ form.
22
+
23
+ "Work" shall mean the work of the author, or any derivative work of the
24
+ Work as defined in the License.
25
+
26
+ "Derivative works" shall mean any work that is based on the Work and
27
+ which includes the Work or a portion of it, with or without
28
+ modifications.
29
+
30
+ "Contributor" shall mean any individual or entity that submits a work
31
+ intended to be published in accordance with the License.
32
+
33
+ "Contribution" shall mean any work of authorship intended to be
34
+ published in accordance with the License.
35
+
36
+ "Contributor license agreement" shall mean any document that sets forth
37
+ the terms under which a Contributor has granted the License the rights to
38
+ use, distribute, and modify the Contribution.
39
+
40
+ "You" shall mean the licensee.
41
+
42
+ "Licensor" shall mean the entity that grants the License.
43
+
44
+ 2. Grant of License.
45
+
46
+ Subject to the conditions in this License, the Licensor grants You a
47
+ worldwide, royalty-free, non-exclusive, irrevocable license to use,
48
+ reproduce, modify, and distribute the Work as a whole or in part.
49
+
50
+ 3. Reproduction of Copyright and License Notices.
51
+
52
+ You may add your own copyright notices to your modifications and may
53
+ provide your own license notices, provided that you include the license
54
+ notices provided with the original Work.
55
+
56
+ 4. Distribution.
57
+
58
+ You may distribute copies of the Work in any medium, provided that you
59
+ include the license notices and copyright notices with the Work, and
60
+ that you retain any notices on the original Work.
61
+
62
+ 5. Modifications.
63
+
64
+ You may modify the Work and distribute your modifications, provided that:
65
+
66
+ a) You must cause the modified files to carry prominent notices stating
67
+ that you changed the files.
68
+
69
+ b) You must include the license notices with your modifications.
70
+
71
+ c) You must retain all copyright, license, notice, and attribution
72
+ notices in the original Work.
73
+
74
+ 6. Trademarks.
75
+
76
+ The name of the Licensor, the name of the Work, or the Licensor's
77
+ trademarks may not be used to endorse or promote products derived from
78
+ this License without specific prior written permission.
79
+
80
+ 7. Limitation of Liability.
81
+
82
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
83
+ WILL ANY COPYRIGHT HOLDER, AUTHOR, OR ANY OTHER CONTRIBUTOR TO THE WORK
84
+ BE LIABLE FOR ANY DAMAGES ARISING FROM THE USE OF THE WORK.
85
+
86
+ 8. Accepting Warranty or Additional Liability.
87
+
88
+ You may choose to offer and charge a fee for copying, distribution,
89
+ modification, or a warranty of the Work, provided that you do so under
90
+ the terms of a license that carries its own set of warranties and
91
+ disclaimers, and that you clearly indicate the terms of the license that
92
+ you are offering on behalf of the Work.
93
+
94
+ 9. GNU GPL Compatibility.
95
+
96
+ This License may be used in conjunction with the GNU General Public
97
+ License version 3.0 (GPLv3). You may combine the Work with code covered
98
+ by the GPLv3 under the terms of the GPLv3.
99
+
100
+ 10. Termination.
101
+
102
+ You may not attempt to modify the License or claim that your version of
103
+ the License is the Apache License.
104
+
105
+ END OF TERMS AND CONDITIONS
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 opticsWolf
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
bobine-0.2.0/PKG-INFO ADDED
@@ -0,0 +1,227 @@
1
+ Metadata-Version: 2.4
2
+ Name: bobine
3
+ Version: 0.2.0
4
+ Summary: Standalone PDF / Office / text → Markdown ingestion engine (extracted from OKFgraph)
5
+ Author-email: opticsWolf <opticswolf@protonmail.com>
6
+ License-Expression: Apache-2.0 OR MIT
7
+ Project-URL: Homepage, https://github.com/opticsWolf/bobine
8
+ Project-URL: Repository, https://github.com/opticsWolf/bobine
9
+ Project-URL: Documentation, https://github.com/opticsWolf/bobine/blob/main/README.md
10
+ Keywords: pdf,markdown,ocr,onnx,latex,ingestion,rapidocr,pdf-oxide
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Text Processing :: Markup :: Markdown
20
+ Classifier: Topic :: Text Processing :: Markup :: LaTeX
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ License-File: LICENSES/Apache-2.0.txt
25
+ License-File: LICENSES/MIT.txt
26
+ Requires-Dist: Pillow>=10.0
27
+ Provides-Extra: pdf-ingest
28
+ Requires-Dist: pdf_oxide>=0.2.1; extra == "pdf-ingest"
29
+ Requires-Dist: office_oxide>=0.1.8; extra == "pdf-ingest"
30
+ Requires-Dist: rapidocr==3.9.2; extra == "pdf-ingest"
31
+ Requires-Dist: rapid_layout==1.2.1; extra == "pdf-ingest"
32
+ Requires-Dist: rapid_table==3.0.2; extra == "pdf-ingest"
33
+ Requires-Dist: numpy>=2.0; extra == "pdf-ingest"
34
+ Provides-Extra: formula
35
+ Requires-Dist: onnxruntime>=1.17; extra == "formula"
36
+ Requires-Dist: tokenizers>=0.13.2; extra == "formula"
37
+ Requires-Dist: opencv-python>=4.5; extra == "formula"
38
+ Requires-Dist: chardet>=5.0; extra == "formula"
39
+ Requires-Dist: requests>=2.28; extra == "formula"
40
+ Requires-Dist: PyYAML>=6.0; extra == "formula"
41
+ Provides-Extra: markdown
42
+ Requires-Dist: mordant>=0.8; extra == "markdown"
43
+ Requires-Dist: python-frontmatter>=1.0; extra == "markdown"
44
+ Requires-Dist: pyyaml>=6.0; extra == "markdown"
45
+ Provides-Extra: dev
46
+ Requires-Dist: pytest>=8.0; extra == "dev"
47
+ Requires-Dist: pytest-cov>=5.0; extra == "dev"
48
+ Requires-Dist: ruff>=0.9; extra == "dev"
49
+ Requires-Dist: reportlab>=4.0; extra == "dev"
50
+ Requires-Dist: numpy>=2.0; extra == "dev"
51
+ Dynamic: license-file
52
+
53
+ # bobine
54
+
55
+ Standalone **PDF / Office / text → Markdown ingestion engine**, extracted from
56
+ the OKFgraph project as a self-contained module. Runs on a single
57
+ `onnxruntime` wheel with no CUDA-version coupling — RapidAI family + `pdf_oxide`
58
+ for PDFs, pure Python for text documents.
59
+
60
+ Dual-licensed under the terms of either the MIT License or the Apache License,
61
+ Version 2.0 — you may choose either (see [LICENSE](LICENSE)).
62
+
63
+ ## Docs
64
+
65
+ - [**Quick reference**](docs/quickref.md) — install, API, config, common tasks
66
+ - [**Architecture**](docs/architecture.md) — modules, data flow, coordinate
67
+ spaces, vendoring, testing strategy
68
+ - [**Implementation plan**](docs/IMPLEMENTATION_PLAN.md) — phases, roadmap, status
69
+ - [**Parity audit**](docs/PARITY.md) — OKFgraph extraction fidelity + drift catalogue
70
+
71
+ ## Why bobine?
72
+
73
+ The ingestion pipeline was entangled with the knowledge-graph project it served.
74
+ `bobine` moves the whole pipeline — conversion, image staging, markdown
75
+ linting, document normalization — into its own package so any consumer (a
76
+ graph, a CLI, an MCP server, a batch tool) can reuse it without importing a
77
+ database stack.
78
+
79
+ ## Installation
80
+
81
+ ```bash
82
+ pip install -e . # core (Pillow only)
83
+ pip install -e ".[pdf-ingest]" # + pdf_oxide + RapidAI ONNX passes
84
+ pip install -e ".[formula]" # + formula OCR (vendored RapidLaTeXOCR)
85
+ pip install -e ".[markdown]" # + mordant linting + frontmatter parsing
86
+ ```
87
+
88
+ Everything is optional: the package imports with zero dependencies and
89
+ degrades gracefully (no-op fast paths, clear `RuntimeError`s when a backend
90
+ is missing).
91
+
92
+ ## Quick start
93
+
94
+ ```python
95
+ from bobine import ConverterConfig, RoutingMode, ingest_document
96
+
97
+ # One PDF → staged, linted markdown in ./out (images → ./out/_assets,
98
+ # links rewritten to okf-asset://<id>)
99
+ result = ingest_document(
100
+ "paper.pdf",
101
+ "out",
102
+ config=ConverterConfig(
103
+ routing_mode=RoutingMode.SURGICAL,
104
+ ),
105
+ )
106
+ print(result.md_path, result.image_count, result.page_count)
107
+
108
+ # Text documents need no native deps at all
109
+ doc = ingest_document("notes.txt", "out")
110
+ ```
111
+
112
+ ### PDF conversion with ONNX heavy passes
113
+
114
+ `HybridConverter` routes pages through four modes:
115
+
116
+ | Mode | Behaviour |
117
+ |-----------|---------------------------------------------------------------------------|
118
+ | `NEVER` | Fast path only (pdf_oxide). No ONNX models loaded. |
119
+ | `AUTO` | Heuristics per page → full ONNX layout + OCR on flagged pages. |
120
+ | `SURGICAL`| Formula crops via RapidLaTeXOCR only; full pipeline just for scans. |
121
+ | `ALWAYS` | Every page through the full ONNX layout + OCR pipeline. |
122
+
123
+ ```python
124
+ from bobine import HybridConverter, ConverterConfig, RoutingMode
125
+
126
+ conv = HybridConverter(ConverterConfig(routing_mode=RoutingMode.AUTO))
127
+ conv.ensure_models()
128
+ md = conv.convert_pdf(
129
+ "paper.pdf", work_dir="work", should_continue=lambda: True, on_page=lambda i, n: None
130
+ )
131
+ conv.close()
132
+ ```
133
+
134
+ ### Text-type documents
135
+
136
+ ```python
137
+ from bobine import load_markdown_document, wrap_thoughts, lint_markdown
138
+
139
+ doc = load_markdown_document("note.md") # frontmatter-aware
140
+ thought = wrap_thoughts("raw reasoning…", topic="graphs")
141
+ fixed = lint_markdown(doc.body, auto_fix=True) # mordant, guarded
142
+ ```
143
+
144
+ ## Module layout
145
+
146
+ ```
147
+ bobine/
148
+ ├── __init__.py public API
149
+ ├── config.py ConverterConfig, RoutingMode
150
+ ├── engine.py OnnxRapidEngine (lazy ONNX model manager)
151
+ ├── converter.py HybridConverter (core PDF/Office pipeline)
152
+ ├── tables.py HTML table → GFM pipe-table converter
153
+ ├── assets.py okf-asset:// staging for extracted images
154
+ ├── versions.py RapidAI version pins + runtime check
155
+ ├── documents.py Document model, frontmatter, thoughts wrapper
156
+ ├── markdown.py mordant linting (guarded, no-op without it)
157
+ ├── pipeline.py convert_to_markdown / stage_images / ingest_document
158
+ └── _vendor/ third-party code, vendored with licenses intact
159
+ └── rapid_latex_ocr/ formula OCR (MIT (c) 2023 RapidAI; numpy-2 fixed)
160
+ ```
161
+
162
+ ### Formula OCR (SURGICAL mode)
163
+
164
+ The LaTeX formula recognizer is **vendored** (`bobine/_vendor/rapid_latex_ocr/`,
165
+ MIT (c) 2023 RapidAI) with the numpy-2 incompatibility fixed upstream never
166
+ addressed — no external package needed. Runtime deps come from the
167
+ `[formula]` extra; the ONNX models (~179 MB) auto-download on first use from
168
+ `github.com/RapidAI/RapidLaTeXOCR/releases/download/v0.0.0/` into
169
+ `bobine/_vendor/rapid_latex_ocr/models/` (git-ignored).
170
+
171
+ Formula regions come from the text layer (TeX math fonts / unicode math
172
+ chars), merged **line-aware** so multi-line display equations become one
173
+ crop. For text-layer-hostile PDFs (Word/InDesign/OCR output without math
174
+ fonts), set `ConverterConfig(formula_layout_fallback=True)` to ask the
175
+ layout model for equation regions instead (pulls the `rapid_layout` stack
176
+ into SURGICAL mode — off by default).
177
+
178
+ ## Output contract
179
+
180
+ `ingest_document` produces a directory that a graph/import layer can consume:
181
+
182
+ - `<stem>.md` — linted markdown with `okf-asset://<id>` image links
183
+ - `_assets/<id>.<ext>` — staged image bytes (deduped, concept-scoped ids)
184
+
185
+ `bobine` never embeds, indexes, or writes to a database. The consumer owns
186
+ embedding and storage (in OKFgraph that is `OKFRouter.import_bundle`).
187
+
188
+ ## Testing
189
+
190
+ ```bash
191
+ # unit suite (no native backends needed — fake pdf_oxide objects drive the
192
+ # converter's routing/splice/ONNX-assembly paths)
193
+ pytest
194
+
195
+ # integration suite (requires bobine[pdf-ingest] + bobine[formula])
196
+ pytest -m integration
197
+
198
+ # coverage + lint
199
+ pytest --cov=bobine --cov-report=term-missing
200
+ ruff check . && ruff format --check .
201
+ ```
202
+
203
+ Markers: `integration` (real pdf_oxide/office_oxide/RapidAI + the PDF corpus)
204
+ and `slow` (ONNX runs over real pages)
205
+
206
+ ### Test-PDF corpus
207
+
208
+ `tests/fixtures/pdf/` holds **trimmed page ranges** from three CC BY 4.0
209
+ arXiv papers (solitons physics, splitting-methods math, trust-ML tables) plus
210
+ a generated scanned page — see `tests/fixtures/SOURCES.md` for provenance and
211
+ attribution. The full untrimmed PDFs are git-ignored under
212
+ `tests/fixtures/full_pdfs/` for local tests. The scanned page is regenerable:
213
+
214
+ ```bash
215
+ uv run --with reportlab python tests/fixtures/generate_corpus.py
216
+ ```
217
+
218
+ The integration tests self-skip when backends are missing, so the bare install
219
+ always stays green. CI (`.github/workflows/ci.yml`) runs the core suite on
220
+ Python 3.10–3.13 plus an integration job. **169 tests, 92 % coverage** as of
221
+ 2026-08-09.
222
+ ## Version pinning
223
+
224
+ RapidAI packages move fast; `check_rapid_versions()` warns on first import if
225
+ an installed version drifts from the known-good list. Silence with
226
+ `BOBINE_INGEST_ALLOW_UNPINNED=1` (the legacy `OKFGRAPH_INGEST_ALLOW_UNPINNED`
227
+ is still honoured).
bobine-0.2.0/README.md ADDED
@@ -0,0 +1,175 @@
1
+ # bobine
2
+
3
+ Standalone **PDF / Office / text → Markdown ingestion engine**, extracted from
4
+ the OKFgraph project as a self-contained module. Runs on a single
5
+ `onnxruntime` wheel with no CUDA-version coupling — RapidAI family + `pdf_oxide`
6
+ for PDFs, pure Python for text documents.
7
+
8
+ Dual-licensed under the terms of either the MIT License or the Apache License,
9
+ Version 2.0 — you may choose either (see [LICENSE](LICENSE)).
10
+
11
+ ## Docs
12
+
13
+ - [**Quick reference**](docs/quickref.md) — install, API, config, common tasks
14
+ - [**Architecture**](docs/architecture.md) — modules, data flow, coordinate
15
+ spaces, vendoring, testing strategy
16
+ - [**Implementation plan**](docs/IMPLEMENTATION_PLAN.md) — phases, roadmap, status
17
+ - [**Parity audit**](docs/PARITY.md) — OKFgraph extraction fidelity + drift catalogue
18
+
19
+ ## Why bobine?
20
+
21
+ The ingestion pipeline was entangled with the knowledge-graph project it served.
22
+ `bobine` moves the whole pipeline — conversion, image staging, markdown
23
+ linting, document normalization — into its own package so any consumer (a
24
+ graph, a CLI, an MCP server, a batch tool) can reuse it without importing a
25
+ database stack.
26
+
27
+ ## Installation
28
+
29
+ ```bash
30
+ pip install -e . # core (Pillow only)
31
+ pip install -e ".[pdf-ingest]" # + pdf_oxide + RapidAI ONNX passes
32
+ pip install -e ".[formula]" # + formula OCR (vendored RapidLaTeXOCR)
33
+ pip install -e ".[markdown]" # + mordant linting + frontmatter parsing
34
+ ```
35
+
36
+ Everything is optional: the package imports with zero dependencies and
37
+ degrades gracefully (no-op fast paths, clear `RuntimeError`s when a backend
38
+ is missing).
39
+
40
+ ## Quick start
41
+
42
+ ```python
43
+ from bobine import ConverterConfig, RoutingMode, ingest_document
44
+
45
+ # One PDF → staged, linted markdown in ./out (images → ./out/_assets,
46
+ # links rewritten to okf-asset://<id>)
47
+ result = ingest_document(
48
+ "paper.pdf",
49
+ "out",
50
+ config=ConverterConfig(
51
+ routing_mode=RoutingMode.SURGICAL,
52
+ ),
53
+ )
54
+ print(result.md_path, result.image_count, result.page_count)
55
+
56
+ # Text documents need no native deps at all
57
+ doc = ingest_document("notes.txt", "out")
58
+ ```
59
+
60
+ ### PDF conversion with ONNX heavy passes
61
+
62
+ `HybridConverter` routes pages through four modes:
63
+
64
+ | Mode | Behaviour |
65
+ |-----------|---------------------------------------------------------------------------|
66
+ | `NEVER` | Fast path only (pdf_oxide). No ONNX models loaded. |
67
+ | `AUTO` | Heuristics per page → full ONNX layout + OCR on flagged pages. |
68
+ | `SURGICAL`| Formula crops via RapidLaTeXOCR only; full pipeline just for scans. |
69
+ | `ALWAYS` | Every page through the full ONNX layout + OCR pipeline. |
70
+
71
+ ```python
72
+ from bobine import HybridConverter, ConverterConfig, RoutingMode
73
+
74
+ conv = HybridConverter(ConverterConfig(routing_mode=RoutingMode.AUTO))
75
+ conv.ensure_models()
76
+ md = conv.convert_pdf(
77
+ "paper.pdf", work_dir="work", should_continue=lambda: True, on_page=lambda i, n: None
78
+ )
79
+ conv.close()
80
+ ```
81
+
82
+ ### Text-type documents
83
+
84
+ ```python
85
+ from bobine import load_markdown_document, wrap_thoughts, lint_markdown
86
+
87
+ doc = load_markdown_document("note.md") # frontmatter-aware
88
+ thought = wrap_thoughts("raw reasoning…", topic="graphs")
89
+ fixed = lint_markdown(doc.body, auto_fix=True) # mordant, guarded
90
+ ```
91
+
92
+ ## Module layout
93
+
94
+ ```
95
+ bobine/
96
+ ├── __init__.py public API
97
+ ├── config.py ConverterConfig, RoutingMode
98
+ ├── engine.py OnnxRapidEngine (lazy ONNX model manager)
99
+ ├── converter.py HybridConverter (core PDF/Office pipeline)
100
+ ├── tables.py HTML table → GFM pipe-table converter
101
+ ├── assets.py okf-asset:// staging for extracted images
102
+ ├── versions.py RapidAI version pins + runtime check
103
+ ├── documents.py Document model, frontmatter, thoughts wrapper
104
+ ├── markdown.py mordant linting (guarded, no-op without it)
105
+ ├── pipeline.py convert_to_markdown / stage_images / ingest_document
106
+ └── _vendor/ third-party code, vendored with licenses intact
107
+ └── rapid_latex_ocr/ formula OCR (MIT (c) 2023 RapidAI; numpy-2 fixed)
108
+ ```
109
+
110
+ ### Formula OCR (SURGICAL mode)
111
+
112
+ The LaTeX formula recognizer is **vendored** (`bobine/_vendor/rapid_latex_ocr/`,
113
+ MIT (c) 2023 RapidAI) with the numpy-2 incompatibility fixed upstream never
114
+ addressed — no external package needed. Runtime deps come from the
115
+ `[formula]` extra; the ONNX models (~179 MB) auto-download on first use from
116
+ `github.com/RapidAI/RapidLaTeXOCR/releases/download/v0.0.0/` into
117
+ `bobine/_vendor/rapid_latex_ocr/models/` (git-ignored).
118
+
119
+ Formula regions come from the text layer (TeX math fonts / unicode math
120
+ chars), merged **line-aware** so multi-line display equations become one
121
+ crop. For text-layer-hostile PDFs (Word/InDesign/OCR output without math
122
+ fonts), set `ConverterConfig(formula_layout_fallback=True)` to ask the
123
+ layout model for equation regions instead (pulls the `rapid_layout` stack
124
+ into SURGICAL mode — off by default).
125
+
126
+ ## Output contract
127
+
128
+ `ingest_document` produces a directory that a graph/import layer can consume:
129
+
130
+ - `<stem>.md` — linted markdown with `okf-asset://<id>` image links
131
+ - `_assets/<id>.<ext>` — staged image bytes (deduped, concept-scoped ids)
132
+
133
+ `bobine` never embeds, indexes, or writes to a database. The consumer owns
134
+ embedding and storage (in OKFgraph that is `OKFRouter.import_bundle`).
135
+
136
+ ## Testing
137
+
138
+ ```bash
139
+ # unit suite (no native backends needed — fake pdf_oxide objects drive the
140
+ # converter's routing/splice/ONNX-assembly paths)
141
+ pytest
142
+
143
+ # integration suite (requires bobine[pdf-ingest] + bobine[formula])
144
+ pytest -m integration
145
+
146
+ # coverage + lint
147
+ pytest --cov=bobine --cov-report=term-missing
148
+ ruff check . && ruff format --check .
149
+ ```
150
+
151
+ Markers: `integration` (real pdf_oxide/office_oxide/RapidAI + the PDF corpus)
152
+ and `slow` (ONNX runs over real pages)
153
+
154
+ ### Test-PDF corpus
155
+
156
+ `tests/fixtures/pdf/` holds **trimmed page ranges** from three CC BY 4.0
157
+ arXiv papers (solitons physics, splitting-methods math, trust-ML tables) plus
158
+ a generated scanned page — see `tests/fixtures/SOURCES.md` for provenance and
159
+ attribution. The full untrimmed PDFs are git-ignored under
160
+ `tests/fixtures/full_pdfs/` for local tests. The scanned page is regenerable:
161
+
162
+ ```bash
163
+ uv run --with reportlab python tests/fixtures/generate_corpus.py
164
+ ```
165
+
166
+ The integration tests self-skip when backends are missing, so the bare install
167
+ always stays green. CI (`.github/workflows/ci.yml`) runs the core suite on
168
+ Python 3.10–3.13 plus an integration job. **169 tests, 92 % coverage** as of
169
+ 2026-08-09.
170
+ ## Version pinning
171
+
172
+ RapidAI packages move fast; `check_rapid_versions()` warns on first import if
173
+ an installed version drifts from the known-good list. Silence with
174
+ `BOBINE_INGEST_ALLOW_UNPINNED=1` (the legacy `OKFGRAPH_INGEST_ALLOW_UNPINNED`
175
+ is still honoured).
@@ -0,0 +1,84 @@
1
+ """bobine — PDF / Office / text → Markdown ingestion engine.
2
+
3
+ Standalone module extracted from the OKFgraph project. Runs on a single
4
+ ``onnxruntime`` wheel with no CUDA-version coupling (RapidAI family + pdf_oxide),
5
+ and needs nothing but the stdlib for plain text / markdown documents.
6
+
7
+ Public API
8
+ ----------
9
+ Conversion (PDF/Office)
10
+ - ``ConverterConfig`` / ``RoutingMode`` — configuration and routing modes.
11
+ - ``HybridConverter`` — core conversion pipeline (Qt-independent).
12
+ - ``OnnxRapidEngine`` — lazy ONNX model manager.
13
+ - ``html_tables_to_gfm`` — HTML table → GFM pipe-table converter.
14
+
15
+ Images
16
+ - ``stage_images_as_okf_assets`` — okf-asset:// staging for extracted images.
17
+ - ``asset_id`` / ``ASSET_STORE_DIRNAME`` — asset naming convention.
18
+
19
+ Text-type documents
20
+ - ``Document`` — normalized document model (id, title, body, tags, …).
21
+ - ``load_markdown_document`` — frontmatter-aware .md loading.
22
+ - ``wrap_thoughts`` — raw reasoning text → OKF-compliant markdown.
23
+ - ``lint_markdown`` / ``lint_markdown_file`` — mordant linting (guarded).
24
+
25
+ Orchestration
26
+ - ``convert_to_markdown`` — dispatch PDF / Office / text → markdown string.
27
+ - ``stage_images`` — collect extracted images into an asset store.
28
+ - ``ingest_document`` — full pipeline: convert → write .md → stage assets
29
+ → lint, returns a ``ConvertedDocument``.
30
+
31
+ Versioning
32
+ - ``check_rapid_versions`` — runtime version check for RapidAI packages.
33
+ """
34
+
35
+ from bobine.assets import (
36
+ ASSET_STORE_DIRNAME,
37
+ asset_id,
38
+ stage_images_as_okf_assets,
39
+ )
40
+ from bobine.config import ConverterConfig, RoutingMode
41
+ from bobine.converter import HybridConverter
42
+ from bobine.documents import Document, load_markdown_document, wrap_thoughts
43
+ from bobine.engine import OnnxRapidEngine
44
+ from bobine.markdown import lint_markdown, lint_markdown_file
45
+ from bobine.pipeline import (
46
+ OFFICE_EXTS,
47
+ SUPPORTED_EXTENSIONS,
48
+ TEXT_EXTS,
49
+ ConvertedDocument,
50
+ convert_directory,
51
+ convert_to_markdown,
52
+ ingest_document,
53
+ stage_images,
54
+ )
55
+ from bobine.tables import html_tables_to_gfm
56
+ from bobine.versions import check_rapid_versions
57
+
58
+ __version__ = "0.2.0"
59
+
60
+ __all__ = [
61
+ "ASSET_STORE_DIRNAME",
62
+ "OFFICE_EXTS",
63
+ "SUPPORTED_EXTENSIONS",
64
+ "TEXT_EXTS",
65
+ "ConvertedDocument",
66
+ "ConverterConfig",
67
+ "Document",
68
+ "HybridConverter",
69
+ "OnnxRapidEngine",
70
+ "RoutingMode",
71
+ "__version__",
72
+ "asset_id",
73
+ "check_rapid_versions",
74
+ "convert_directory",
75
+ "convert_to_markdown",
76
+ "html_tables_to_gfm",
77
+ "ingest_document",
78
+ "lint_markdown",
79
+ "lint_markdown_file",
80
+ "load_markdown_document",
81
+ "stage_images",
82
+ "stage_images_as_okf_assets",
83
+ "wrap_thoughts",
84
+ ]
@@ -0,0 +1,9 @@
1
+ # -*- encoding: utf-8 -*-
2
+ # @Author: SWHL (vendored into bobine, MIT licensed)
3
+ from .main import LaTeXOCR
4
+
5
+ # Legacy alias: older builds shipped the class as `LatexOCR`; keep both so
6
+ # bobine's engine import works regardless of which build was vendored.
7
+ LatexOCR = LaTeXOCR
8
+
9
+ __all__ = ["LaTeXOCR", "LatexOCR"]