bobine 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bobine-0.2.0/LICENSE +6 -0
- bobine-0.2.0/LICENSES/Apache-2.0.txt +105 -0
- bobine-0.2.0/LICENSES/MIT.txt +21 -0
- bobine-0.2.0/PKG-INFO +227 -0
- bobine-0.2.0/README.md +175 -0
- bobine-0.2.0/bobine/__init__.py +84 -0
- bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/__init__.py +9 -0
- bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/main.py +214 -0
- bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/models.py +154 -0
- bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/utils.py +185 -0
- bobine-0.2.0/bobine/_vendor/rapid_latex_ocr/utils_load.py +172 -0
- bobine-0.2.0/bobine/assets.py +90 -0
- bobine-0.2.0/bobine/config.py +94 -0
- bobine-0.2.0/bobine/converter.py +829 -0
- bobine-0.2.0/bobine/documents.py +180 -0
- bobine-0.2.0/bobine/engine.py +239 -0
- bobine-0.2.0/bobine/markdown.py +103 -0
- bobine-0.2.0/bobine/pipeline.py +278 -0
- bobine-0.2.0/bobine/tables.py +87 -0
- bobine-0.2.0/bobine/versions.py +148 -0
- bobine-0.2.0/bobine.egg-info/PKG-INFO +227 -0
- bobine-0.2.0/bobine.egg-info/SOURCES.txt +38 -0
- bobine-0.2.0/bobine.egg-info/dependency_links.txt +1 -0
- bobine-0.2.0/bobine.egg-info/requires.txt +29 -0
- bobine-0.2.0/bobine.egg-info/top_level.txt +1 -0
- bobine-0.2.0/pyproject.toml +107 -0
- bobine-0.2.0/setup.cfg +4 -0
- bobine-0.2.0/tests/test_assets.py +52 -0
- bobine-0.2.0/tests/test_config.py +37 -0
- bobine-0.2.0/tests/test_converter.py +478 -0
- bobine-0.2.0/tests/test_converter_flow.py +485 -0
- bobine-0.2.0/tests/test_documents.py +75 -0
- bobine-0.2.0/tests/test_engine.py +249 -0
- bobine-0.2.0/tests/test_formula_vendored.py +78 -0
- bobine-0.2.0/tests/test_integration.py +126 -0
- bobine-0.2.0/tests/test_markdown.py +52 -0
- bobine-0.2.0/tests/test_pdf_corpus.py +106 -0
- bobine-0.2.0/tests/test_pipeline.py +256 -0
- bobine-0.2.0/tests/test_tables.py +49 -0
- bobine-0.2.0/tests/test_versions.py +71 -0
bobine-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean any term or agreement which makes use of the
|
|
10
|
+
following terminology, including the incorporation of the Apache
|
|
11
|
+
License.
|
|
12
|
+
|
|
13
|
+
"Package" shall mean the work resulting from the distribution of the
|
|
14
|
+
Software as defined in the License.
|
|
15
|
+
|
|
16
|
+
"Source" form shall mean the preferred form of the Software for making
|
|
17
|
+
modifications to it.
|
|
18
|
+
|
|
19
|
+
"Object" code form shall mean the modified version of the Software in
|
|
20
|
+
object code form or as a modification of the Software in object code
|
|
21
|
+
form.
|
|
22
|
+
|
|
23
|
+
"Work" shall mean the work of the author, or any derivative work of the
|
|
24
|
+
Work as defined in the License.
|
|
25
|
+
|
|
26
|
+
"Derivative works" shall mean any work that is based on the Work and
|
|
27
|
+
which includes the Work or a portion of it, with or without
|
|
28
|
+
modifications.
|
|
29
|
+
|
|
30
|
+
"Contributor" shall mean any individual or entity that submits a work
|
|
31
|
+
intended to be published in accordance with the License.
|
|
32
|
+
|
|
33
|
+
"Contribution" shall mean any work of authorship intended to be
|
|
34
|
+
published in accordance with the License.
|
|
35
|
+
|
|
36
|
+
"Contributor license agreement" shall mean any document that sets forth
|
|
37
|
+
the terms under which a Contributor has granted the License the rights to
|
|
38
|
+
use, distribute, and modify the Contribution.
|
|
39
|
+
|
|
40
|
+
"You" shall mean the licensee.
|
|
41
|
+
|
|
42
|
+
"Licensor" shall mean the entity that grants the License.
|
|
43
|
+
|
|
44
|
+
2. Grant of License.
|
|
45
|
+
|
|
46
|
+
Subject to the conditions in this License, the Licensor grants You a
|
|
47
|
+
worldwide, royalty-free, non-exclusive, irrevocable license to use,
|
|
48
|
+
reproduce, modify, and distribute the Work as a whole or in part.
|
|
49
|
+
|
|
50
|
+
3. Reproduction of Copyright and License Notices.
|
|
51
|
+
|
|
52
|
+
You may add your own copyright notices to your modifications and may
|
|
53
|
+
provide your own license notices, provided that you include the license
|
|
54
|
+
notices provided with the original Work.
|
|
55
|
+
|
|
56
|
+
4. Distribution.
|
|
57
|
+
|
|
58
|
+
You may distribute copies of the Work in any medium, provided that you
|
|
59
|
+
include the license notices and copyright notices with the Work, and
|
|
60
|
+
that you retain any notices on the original Work.
|
|
61
|
+
|
|
62
|
+
5. Modifications.
|
|
63
|
+
|
|
64
|
+
You may modify the Work and distribute your modifications, provided that:
|
|
65
|
+
|
|
66
|
+
a) You must cause the modified files to carry prominent notices stating
|
|
67
|
+
that you changed the files.
|
|
68
|
+
|
|
69
|
+
b) You must include the license notices with your modifications.
|
|
70
|
+
|
|
71
|
+
c) You must retain all copyright, license, notice, and attribution
|
|
72
|
+
notices in the original Work.
|
|
73
|
+
|
|
74
|
+
6. Trademarks.
|
|
75
|
+
|
|
76
|
+
The name of the Licensor, the name of the Work, or the Licensor's
|
|
77
|
+
trademarks may not be used to endorse or promote products derived from
|
|
78
|
+
this License without specific prior written permission.
|
|
79
|
+
|
|
80
|
+
7. Limitation of Liability.
|
|
81
|
+
|
|
82
|
+
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
|
83
|
+
WILL ANY COPYRIGHT HOLDER, AUTHOR, OR ANY OTHER CONTRIBUTOR TO THE WORK
|
|
84
|
+
BE LIABLE FOR ANY DAMAGES ARISING FROM THE USE OF THE WORK.
|
|
85
|
+
|
|
86
|
+
8. Accepting Warranty or Additional Liability.
|
|
87
|
+
|
|
88
|
+
You may choose to offer and charge a fee for copying, distribution,
|
|
89
|
+
modification, or a warranty of the Work, provided that you do so under
|
|
90
|
+
the terms of a license that carries its own set of warranties and
|
|
91
|
+
disclaimers, and that you clearly indicate the terms of the license that
|
|
92
|
+
you are offering on behalf of the Work.
|
|
93
|
+
|
|
94
|
+
9. GNU GPL Compatibility.
|
|
95
|
+
|
|
96
|
+
This License may be used in conjunction with the GNU General Public
|
|
97
|
+
License version 3.0 (GPLv3). You may combine the Work with code covered
|
|
98
|
+
by the GPLv3 under the terms of the GPLv3.
|
|
99
|
+
|
|
100
|
+
10. Termination.
|
|
101
|
+
|
|
102
|
+
You may not attempt to modify the License or claim that your version of
|
|
103
|
+
the License is the Apache License.
|
|
104
|
+
|
|
105
|
+
END OF TERMS AND CONDITIONS
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 opticsWolf
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
bobine-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bobine
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Standalone PDF / Office / text → Markdown ingestion engine (extracted from OKFgraph)
|
|
5
|
+
Author-email: opticsWolf <opticswolf@protonmail.com>
|
|
6
|
+
License-Expression: Apache-2.0 OR MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/opticsWolf/bobine
|
|
8
|
+
Project-URL: Repository, https://github.com/opticsWolf/bobine
|
|
9
|
+
Project-URL: Documentation, https://github.com/opticsWolf/bobine/blob/main/README.md
|
|
10
|
+
Keywords: pdf,markdown,ocr,onnx,latex,ingestion,rapidocr,pdf-oxide
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
20
|
+
Classifier: Topic :: Text Processing :: Markup :: LaTeX
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
License-File: LICENSES/Apache-2.0.txt
|
|
25
|
+
License-File: LICENSES/MIT.txt
|
|
26
|
+
Requires-Dist: Pillow>=10.0
|
|
27
|
+
Provides-Extra: pdf-ingest
|
|
28
|
+
Requires-Dist: pdf_oxide>=0.2.1; extra == "pdf-ingest"
|
|
29
|
+
Requires-Dist: office_oxide>=0.1.8; extra == "pdf-ingest"
|
|
30
|
+
Requires-Dist: rapidocr==3.9.2; extra == "pdf-ingest"
|
|
31
|
+
Requires-Dist: rapid_layout==1.2.1; extra == "pdf-ingest"
|
|
32
|
+
Requires-Dist: rapid_table==3.0.2; extra == "pdf-ingest"
|
|
33
|
+
Requires-Dist: numpy>=2.0; extra == "pdf-ingest"
|
|
34
|
+
Provides-Extra: formula
|
|
35
|
+
Requires-Dist: onnxruntime>=1.17; extra == "formula"
|
|
36
|
+
Requires-Dist: tokenizers>=0.13.2; extra == "formula"
|
|
37
|
+
Requires-Dist: opencv-python>=4.5; extra == "formula"
|
|
38
|
+
Requires-Dist: chardet>=5.0; extra == "formula"
|
|
39
|
+
Requires-Dist: requests>=2.28; extra == "formula"
|
|
40
|
+
Requires-Dist: PyYAML>=6.0; extra == "formula"
|
|
41
|
+
Provides-Extra: markdown
|
|
42
|
+
Requires-Dist: mordant>=0.8; extra == "markdown"
|
|
43
|
+
Requires-Dist: python-frontmatter>=1.0; extra == "markdown"
|
|
44
|
+
Requires-Dist: pyyaml>=6.0; extra == "markdown"
|
|
45
|
+
Provides-Extra: dev
|
|
46
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
47
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
48
|
+
Requires-Dist: ruff>=0.9; extra == "dev"
|
|
49
|
+
Requires-Dist: reportlab>=4.0; extra == "dev"
|
|
50
|
+
Requires-Dist: numpy>=2.0; extra == "dev"
|
|
51
|
+
Dynamic: license-file
|
|
52
|
+
|
|
53
|
+
# bobine
|
|
54
|
+
|
|
55
|
+
Standalone **PDF / Office / text → Markdown ingestion engine**, extracted from
|
|
56
|
+
the OKFgraph project as a self-contained module. Runs on a single
|
|
57
|
+
`onnxruntime` wheel with no CUDA-version coupling — RapidAI family + `pdf_oxide`
|
|
58
|
+
for PDFs, pure Python for text documents.
|
|
59
|
+
|
|
60
|
+
Dual-licensed under the terms of either the MIT License or the Apache License,
|
|
61
|
+
Version 2.0 — you may choose either (see [LICENSE](LICENSE)).
|
|
62
|
+
|
|
63
|
+
## Docs
|
|
64
|
+
|
|
65
|
+
- [**Quick reference**](docs/quickref.md) — install, API, config, common tasks
|
|
66
|
+
- [**Architecture**](docs/architecture.md) — modules, data flow, coordinate
|
|
67
|
+
spaces, vendoring, testing strategy
|
|
68
|
+
- [**Implementation plan**](docs/IMPLEMENTATION_PLAN.md) — phases, roadmap, status
|
|
69
|
+
- [**Parity audit**](docs/PARITY.md) — OKFgraph extraction fidelity + drift catalogue
|
|
70
|
+
|
|
71
|
+
## Why bobine?
|
|
72
|
+
|
|
73
|
+
The ingestion pipeline was entangled with the knowledge-graph project it served.
|
|
74
|
+
`bobine` moves the whole pipeline — conversion, image staging, markdown
|
|
75
|
+
linting, document normalization — into its own package so any consumer (a
|
|
76
|
+
graph, a CLI, an MCP server, a batch tool) can reuse it without importing a
|
|
77
|
+
database stack.
|
|
78
|
+
|
|
79
|
+
## Installation
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install -e . # core (Pillow only)
|
|
83
|
+
pip install -e ".[pdf-ingest]" # + pdf_oxide + RapidAI ONNX passes
|
|
84
|
+
pip install -e ".[formula]" # + formula OCR (vendored RapidLaTeXOCR)
|
|
85
|
+
pip install -e ".[markdown]" # + mordant linting + frontmatter parsing
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Everything is optional: the package imports with zero dependencies and
|
|
89
|
+
degrades gracefully (no-op fast paths, clear `RuntimeError`s when a backend
|
|
90
|
+
is missing).
|
|
91
|
+
|
|
92
|
+
## Quick start
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from bobine import ConverterConfig, RoutingMode, ingest_document
|
|
96
|
+
|
|
97
|
+
# One PDF → staged, linted markdown in ./out (images → ./out/_assets,
|
|
98
|
+
# links rewritten to okf-asset://<id>)
|
|
99
|
+
result = ingest_document(
|
|
100
|
+
"paper.pdf",
|
|
101
|
+
"out",
|
|
102
|
+
config=ConverterConfig(
|
|
103
|
+
routing_mode=RoutingMode.SURGICAL,
|
|
104
|
+
),
|
|
105
|
+
)
|
|
106
|
+
print(result.md_path, result.image_count, result.page_count)
|
|
107
|
+
|
|
108
|
+
# Text documents need no native deps at all
|
|
109
|
+
doc = ingest_document("notes.txt", "out")
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### PDF conversion with ONNX heavy passes
|
|
113
|
+
|
|
114
|
+
`HybridConverter` routes pages through four modes:
|
|
115
|
+
|
|
116
|
+
| Mode | Behaviour |
|
|
117
|
+
|-----------|---------------------------------------------------------------------------|
|
|
118
|
+
| `NEVER` | Fast path only (pdf_oxide). No ONNX models loaded. |
|
|
119
|
+
| `AUTO` | Heuristics per page → full ONNX layout + OCR on flagged pages. |
|
|
120
|
+
| `SURGICAL`| Formula crops via RapidLaTeXOCR only; full pipeline just for scans. |
|
|
121
|
+
| `ALWAYS` | Every page through the full ONNX layout + OCR pipeline. |
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
from bobine import HybridConverter, ConverterConfig, RoutingMode
|
|
125
|
+
|
|
126
|
+
conv = HybridConverter(ConverterConfig(routing_mode=RoutingMode.AUTO))
|
|
127
|
+
conv.ensure_models()
|
|
128
|
+
md = conv.convert_pdf(
|
|
129
|
+
"paper.pdf", work_dir="work", should_continue=lambda: True, on_page=lambda i, n: None
|
|
130
|
+
)
|
|
131
|
+
conv.close()
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Text-type documents
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
from bobine import load_markdown_document, wrap_thoughts, lint_markdown
|
|
138
|
+
|
|
139
|
+
doc = load_markdown_document("note.md") # frontmatter-aware
|
|
140
|
+
thought = wrap_thoughts("raw reasoning…", topic="graphs")
|
|
141
|
+
fixed = lint_markdown(doc.body, auto_fix=True) # mordant, guarded
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
## Module layout
|
|
145
|
+
|
|
146
|
+
```
|
|
147
|
+
bobine/
|
|
148
|
+
├── __init__.py public API
|
|
149
|
+
├── config.py ConverterConfig, RoutingMode
|
|
150
|
+
├── engine.py OnnxRapidEngine (lazy ONNX model manager)
|
|
151
|
+
├── converter.py HybridConverter (core PDF/Office pipeline)
|
|
152
|
+
├── tables.py HTML table → GFM pipe-table converter
|
|
153
|
+
├── assets.py okf-asset:// staging for extracted images
|
|
154
|
+
├── versions.py RapidAI version pins + runtime check
|
|
155
|
+
├── documents.py Document model, frontmatter, thoughts wrapper
|
|
156
|
+
├── markdown.py mordant linting (guarded, no-op without it)
|
|
157
|
+
├── pipeline.py convert_to_markdown / stage_images / ingest_document
|
|
158
|
+
└── _vendor/ third-party code, vendored with licenses intact
|
|
159
|
+
└── rapid_latex_ocr/ formula OCR (MIT (c) 2023 RapidAI; numpy-2 fixed)
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
### Formula OCR (SURGICAL mode)
|
|
163
|
+
|
|
164
|
+
The LaTeX formula recognizer is **vendored** (`bobine/_vendor/rapid_latex_ocr/`,
|
|
165
|
+
MIT (c) 2023 RapidAI) with the numpy-2 incompatibility fixed upstream never
|
|
166
|
+
addressed — no external package needed. Runtime deps come from the
|
|
167
|
+
`[formula]` extra; the ONNX models (~179 MB) auto-download on first use from
|
|
168
|
+
`github.com/RapidAI/RapidLaTeXOCR/releases/download/v0.0.0/` into
|
|
169
|
+
`bobine/_vendor/rapid_latex_ocr/models/` (git-ignored).
|
|
170
|
+
|
|
171
|
+
Formula regions come from the text layer (TeX math fonts / unicode math
|
|
172
|
+
chars), merged **line-aware** so multi-line display equations become one
|
|
173
|
+
crop. For text-layer-hostile PDFs (Word/InDesign/OCR output without math
|
|
174
|
+
fonts), set `ConverterConfig(formula_layout_fallback=True)` to ask the
|
|
175
|
+
layout model for equation regions instead (pulls the `rapid_layout` stack
|
|
176
|
+
into SURGICAL mode — off by default).
|
|
177
|
+
|
|
178
|
+
## Output contract
|
|
179
|
+
|
|
180
|
+
`ingest_document` produces a directory that a graph/import layer can consume:
|
|
181
|
+
|
|
182
|
+
- `<stem>.md` — linted markdown with `okf-asset://<id>` image links
|
|
183
|
+
- `_assets/<id>.<ext>` — staged image bytes (deduped, concept-scoped ids)
|
|
184
|
+
|
|
185
|
+
`bobine` never embeds, indexes, or writes to a database. The consumer owns
|
|
186
|
+
embedding and storage (in OKFgraph that is `OKFRouter.import_bundle`).
|
|
187
|
+
|
|
188
|
+
## Testing
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
# unit suite (no native backends needed — fake pdf_oxide objects drive the
|
|
192
|
+
# converter's routing/splice/ONNX-assembly paths)
|
|
193
|
+
pytest
|
|
194
|
+
|
|
195
|
+
# integration suite (requires bobine[pdf-ingest] + bobine[formula])
|
|
196
|
+
pytest -m integration
|
|
197
|
+
|
|
198
|
+
# coverage + lint
|
|
199
|
+
pytest --cov=bobine --cov-report=term-missing
|
|
200
|
+
ruff check . && ruff format --check .
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
Markers: `integration` (real pdf_oxide/office_oxide/RapidAI + the PDF corpus)
|
|
204
|
+
and `slow` (ONNX runs over real pages)
|
|
205
|
+
|
|
206
|
+
### Test-PDF corpus
|
|
207
|
+
|
|
208
|
+
`tests/fixtures/pdf/` holds **trimmed page ranges** from three CC BY 4.0
|
|
209
|
+
arXiv papers (solitons physics, splitting-methods math, trust-ML tables) plus
|
|
210
|
+
a generated scanned page — see `tests/fixtures/SOURCES.md` for provenance and
|
|
211
|
+
attribution. The full untrimmed PDFs are git-ignored under
|
|
212
|
+
`tests/fixtures/full_pdfs/` for local tests. The scanned page is regenerable:
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
uv run --with reportlab python tests/fixtures/generate_corpus.py
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
The integration tests self-skip when backends are missing, so the bare install
|
|
219
|
+
always stays green. CI (`.github/workflows/ci.yml`) runs the core suite on
|
|
220
|
+
Python 3.10–3.13 plus an integration job. **169 tests, 92 % coverage** as of
|
|
221
|
+
2026-08-09.
|
|
222
|
+
## Version pinning
|
|
223
|
+
|
|
224
|
+
RapidAI packages move fast; `check_rapid_versions()` warns on first import if
|
|
225
|
+
an installed version drifts from the known-good list. Silence with
|
|
226
|
+
`BOBINE_INGEST_ALLOW_UNPINNED=1` (the legacy `OKFGRAPH_INGEST_ALLOW_UNPINNED`
|
|
227
|
+
is still honoured).
|
bobine-0.2.0/README.md
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# bobine
|
|
2
|
+
|
|
3
|
+
Standalone **PDF / Office / text → Markdown ingestion engine**, extracted from
|
|
4
|
+
the OKFgraph project as a self-contained module. Runs on a single
|
|
5
|
+
`onnxruntime` wheel with no CUDA-version coupling — RapidAI family + `pdf_oxide`
|
|
6
|
+
for PDFs, pure Python for text documents.
|
|
7
|
+
|
|
8
|
+
Dual-licensed under the terms of either the MIT License or the Apache License,
|
|
9
|
+
Version 2.0 — you may choose either (see [LICENSE](LICENSE)).
|
|
10
|
+
|
|
11
|
+
## Docs
|
|
12
|
+
|
|
13
|
+
- [**Quick reference**](docs/quickref.md) — install, API, config, common tasks
|
|
14
|
+
- [**Architecture**](docs/architecture.md) — modules, data flow, coordinate
|
|
15
|
+
spaces, vendoring, testing strategy
|
|
16
|
+
- [**Implementation plan**](docs/IMPLEMENTATION_PLAN.md) — phases, roadmap, status
|
|
17
|
+
- [**Parity audit**](docs/PARITY.md) — OKFgraph extraction fidelity + drift catalogue
|
|
18
|
+
|
|
19
|
+
## Why bobine?
|
|
20
|
+
|
|
21
|
+
The ingestion pipeline was entangled with the knowledge-graph project it served.
|
|
22
|
+
`bobine` moves the whole pipeline — conversion, image staging, markdown
|
|
23
|
+
linting, document normalization — into its own package so any consumer (a
|
|
24
|
+
graph, a CLI, an MCP server, a batch tool) can reuse it without importing a
|
|
25
|
+
database stack.
|
|
26
|
+
|
|
27
|
+
## Installation
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install -e . # core (Pillow only)
|
|
31
|
+
pip install -e ".[pdf-ingest]" # + pdf_oxide + RapidAI ONNX passes
|
|
32
|
+
pip install -e ".[formula]" # + formula OCR (vendored RapidLaTeXOCR)
|
|
33
|
+
pip install -e ".[markdown]" # + mordant linting + frontmatter parsing
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Everything is optional: the package imports with zero dependencies and
|
|
37
|
+
degrades gracefully (no-op fast paths, clear `RuntimeError`s when a backend
|
|
38
|
+
is missing).
|
|
39
|
+
|
|
40
|
+
## Quick start
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from bobine import ConverterConfig, RoutingMode, ingest_document
|
|
44
|
+
|
|
45
|
+
# One PDF → staged, linted markdown in ./out (images → ./out/_assets,
|
|
46
|
+
# links rewritten to okf-asset://<id>)
|
|
47
|
+
result = ingest_document(
|
|
48
|
+
"paper.pdf",
|
|
49
|
+
"out",
|
|
50
|
+
config=ConverterConfig(
|
|
51
|
+
routing_mode=RoutingMode.SURGICAL,
|
|
52
|
+
),
|
|
53
|
+
)
|
|
54
|
+
print(result.md_path, result.image_count, result.page_count)
|
|
55
|
+
|
|
56
|
+
# Text documents need no native deps at all
|
|
57
|
+
doc = ingest_document("notes.txt", "out")
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### PDF conversion with ONNX heavy passes
|
|
61
|
+
|
|
62
|
+
`HybridConverter` routes pages through four modes:
|
|
63
|
+
|
|
64
|
+
| Mode | Behaviour |
|
|
65
|
+
|-----------|---------------------------------------------------------------------------|
|
|
66
|
+
| `NEVER` | Fast path only (pdf_oxide). No ONNX models loaded. |
|
|
67
|
+
| `AUTO` | Heuristics per page → full ONNX layout + OCR on flagged pages. |
|
|
68
|
+
| `SURGICAL`| Formula crops via RapidLaTeXOCR only; full pipeline just for scans. |
|
|
69
|
+
| `ALWAYS` | Every page through the full ONNX layout + OCR pipeline. |
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from bobine import HybridConverter, ConverterConfig, RoutingMode
|
|
73
|
+
|
|
74
|
+
conv = HybridConverter(ConverterConfig(routing_mode=RoutingMode.AUTO))
|
|
75
|
+
conv.ensure_models()
|
|
76
|
+
md = conv.convert_pdf(
|
|
77
|
+
"paper.pdf", work_dir="work", should_continue=lambda: True, on_page=lambda i, n: None
|
|
78
|
+
)
|
|
79
|
+
conv.close()
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Text-type documents
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from bobine import load_markdown_document, wrap_thoughts, lint_markdown
|
|
86
|
+
|
|
87
|
+
doc = load_markdown_document("note.md") # frontmatter-aware
|
|
88
|
+
thought = wrap_thoughts("raw reasoning…", topic="graphs")
|
|
89
|
+
fixed = lint_markdown(doc.body, auto_fix=True) # mordant, guarded
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Module layout
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
bobine/
|
|
96
|
+
├── __init__.py public API
|
|
97
|
+
├── config.py ConverterConfig, RoutingMode
|
|
98
|
+
├── engine.py OnnxRapidEngine (lazy ONNX model manager)
|
|
99
|
+
├── converter.py HybridConverter (core PDF/Office pipeline)
|
|
100
|
+
├── tables.py HTML table → GFM pipe-table converter
|
|
101
|
+
├── assets.py okf-asset:// staging for extracted images
|
|
102
|
+
├── versions.py RapidAI version pins + runtime check
|
|
103
|
+
├── documents.py Document model, frontmatter, thoughts wrapper
|
|
104
|
+
├── markdown.py mordant linting (guarded, no-op without it)
|
|
105
|
+
├── pipeline.py convert_to_markdown / stage_images / ingest_document
|
|
106
|
+
└── _vendor/ third-party code, vendored with licenses intact
|
|
107
|
+
└── rapid_latex_ocr/ formula OCR (MIT (c) 2023 RapidAI; numpy-2 fixed)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
### Formula OCR (SURGICAL mode)
|
|
111
|
+
|
|
112
|
+
The LaTeX formula recognizer is **vendored** (`bobine/_vendor/rapid_latex_ocr/`,
|
|
113
|
+
MIT (c) 2023 RapidAI) with the numpy-2 incompatibility fixed upstream never
|
|
114
|
+
addressed — no external package needed. Runtime deps come from the
|
|
115
|
+
`[formula]` extra; the ONNX models (~179 MB) auto-download on first use from
|
|
116
|
+
`github.com/RapidAI/RapidLaTeXOCR/releases/download/v0.0.0/` into
|
|
117
|
+
`bobine/_vendor/rapid_latex_ocr/models/` (git-ignored).
|
|
118
|
+
|
|
119
|
+
Formula regions come from the text layer (TeX math fonts / unicode math
|
|
120
|
+
chars), merged **line-aware** so multi-line display equations become one
|
|
121
|
+
crop. For text-layer-hostile PDFs (Word/InDesign/OCR output without math
|
|
122
|
+
fonts), set `ConverterConfig(formula_layout_fallback=True)` to ask the
|
|
123
|
+
layout model for equation regions instead (pulls the `rapid_layout` stack
|
|
124
|
+
into SURGICAL mode — off by default).
|
|
125
|
+
|
|
126
|
+
## Output contract
|
|
127
|
+
|
|
128
|
+
`ingest_document` produces a directory that a graph/import layer can consume:
|
|
129
|
+
|
|
130
|
+
- `<stem>.md` — linted markdown with `okf-asset://<id>` image links
|
|
131
|
+
- `_assets/<id>.<ext>` — staged image bytes (deduped, concept-scoped ids)
|
|
132
|
+
|
|
133
|
+
`bobine` never embeds, indexes, or writes to a database. The consumer owns
|
|
134
|
+
embedding and storage (in OKFgraph that is `OKFRouter.import_bundle`).
|
|
135
|
+
|
|
136
|
+
## Testing
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
# unit suite (no native backends needed — fake pdf_oxide objects drive the
|
|
140
|
+
# converter's routing/splice/ONNX-assembly paths)
|
|
141
|
+
pytest
|
|
142
|
+
|
|
143
|
+
# integration suite (requires bobine[pdf-ingest] + bobine[formula])
|
|
144
|
+
pytest -m integration
|
|
145
|
+
|
|
146
|
+
# coverage + lint
|
|
147
|
+
pytest --cov=bobine --cov-report=term-missing
|
|
148
|
+
ruff check . && ruff format --check .
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Markers: `integration` (real pdf_oxide/office_oxide/RapidAI + the PDF corpus)
|
|
152
|
+
and `slow` (ONNX runs over real pages)
|
|
153
|
+
|
|
154
|
+
### Test-PDF corpus
|
|
155
|
+
|
|
156
|
+
`tests/fixtures/pdf/` holds **trimmed page ranges** from three CC BY 4.0
|
|
157
|
+
arXiv papers (solitons physics, splitting-methods math, trust-ML tables) plus
|
|
158
|
+
a generated scanned page — see `tests/fixtures/SOURCES.md` for provenance and
|
|
159
|
+
attribution. The full untrimmed PDFs are git-ignored under
|
|
160
|
+
`tests/fixtures/full_pdfs/` for local tests. The scanned page is regenerable:
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
uv run --with reportlab python tests/fixtures/generate_corpus.py
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
The integration tests self-skip when backends are missing, so the bare install
|
|
167
|
+
always stays green. CI (`.github/workflows/ci.yml`) runs the core suite on
|
|
168
|
+
Python 3.10–3.13 plus an integration job. **169 tests, 92 % coverage** as of
|
|
169
|
+
2026-08-09.
|
|
170
|
+
## Version pinning
|
|
171
|
+
|
|
172
|
+
RapidAI packages move fast; `check_rapid_versions()` warns on first import if
|
|
173
|
+
an installed version drifts from the known-good list. Silence with
|
|
174
|
+
`BOBINE_INGEST_ALLOW_UNPINNED=1` (the legacy `OKFGRAPH_INGEST_ALLOW_UNPINNED`
|
|
175
|
+
is still honoured).
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""bobine — PDF / Office / text → Markdown ingestion engine.
|
|
2
|
+
|
|
3
|
+
Standalone module extracted from the OKFgraph project. Runs on a single
|
|
4
|
+
``onnxruntime`` wheel with no CUDA-version coupling (RapidAI family + pdf_oxide),
|
|
5
|
+
and needs nothing but the stdlib for plain text / markdown documents.
|
|
6
|
+
|
|
7
|
+
Public API
|
|
8
|
+
----------
|
|
9
|
+
Conversion (PDF/Office)
|
|
10
|
+
- ``ConverterConfig`` / ``RoutingMode`` — configuration and routing modes.
|
|
11
|
+
- ``HybridConverter`` — core conversion pipeline (Qt-independent).
|
|
12
|
+
- ``OnnxRapidEngine`` — lazy ONNX model manager.
|
|
13
|
+
- ``html_tables_to_gfm`` — HTML table → GFM pipe-table converter.
|
|
14
|
+
|
|
15
|
+
Images
|
|
16
|
+
- ``stage_images_as_okf_assets`` — okf-asset:// staging for extracted images.
|
|
17
|
+
- ``asset_id`` / ``ASSET_STORE_DIRNAME`` — asset naming convention.
|
|
18
|
+
|
|
19
|
+
Text-type documents
|
|
20
|
+
- ``Document`` — normalized document model (id, title, body, tags, …).
|
|
21
|
+
- ``load_markdown_document`` — frontmatter-aware .md loading.
|
|
22
|
+
- ``wrap_thoughts`` — raw reasoning text → OKF-compliant markdown.
|
|
23
|
+
- ``lint_markdown`` / ``lint_markdown_file`` — mordant linting (guarded).
|
|
24
|
+
|
|
25
|
+
Orchestration
|
|
26
|
+
- ``convert_to_markdown`` — dispatch PDF / Office / text → markdown string.
|
|
27
|
+
- ``stage_images`` — collect extracted images into an asset store.
|
|
28
|
+
- ``ingest_document`` — full pipeline: convert → write .md → stage assets
|
|
29
|
+
→ lint, returns a ``ConvertedDocument``.
|
|
30
|
+
|
|
31
|
+
Versioning
|
|
32
|
+
- ``check_rapid_versions`` — runtime version check for RapidAI packages.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from bobine.assets import (
|
|
36
|
+
ASSET_STORE_DIRNAME,
|
|
37
|
+
asset_id,
|
|
38
|
+
stage_images_as_okf_assets,
|
|
39
|
+
)
|
|
40
|
+
from bobine.config import ConverterConfig, RoutingMode
|
|
41
|
+
from bobine.converter import HybridConverter
|
|
42
|
+
from bobine.documents import Document, load_markdown_document, wrap_thoughts
|
|
43
|
+
from bobine.engine import OnnxRapidEngine
|
|
44
|
+
from bobine.markdown import lint_markdown, lint_markdown_file
|
|
45
|
+
from bobine.pipeline import (
|
|
46
|
+
OFFICE_EXTS,
|
|
47
|
+
SUPPORTED_EXTENSIONS,
|
|
48
|
+
TEXT_EXTS,
|
|
49
|
+
ConvertedDocument,
|
|
50
|
+
convert_directory,
|
|
51
|
+
convert_to_markdown,
|
|
52
|
+
ingest_document,
|
|
53
|
+
stage_images,
|
|
54
|
+
)
|
|
55
|
+
from bobine.tables import html_tables_to_gfm
|
|
56
|
+
from bobine.versions import check_rapid_versions
|
|
57
|
+
|
|
58
|
+
__version__ = "0.2.0"
|
|
59
|
+
|
|
60
|
+
__all__ = [
|
|
61
|
+
"ASSET_STORE_DIRNAME",
|
|
62
|
+
"OFFICE_EXTS",
|
|
63
|
+
"SUPPORTED_EXTENSIONS",
|
|
64
|
+
"TEXT_EXTS",
|
|
65
|
+
"ConvertedDocument",
|
|
66
|
+
"ConverterConfig",
|
|
67
|
+
"Document",
|
|
68
|
+
"HybridConverter",
|
|
69
|
+
"OnnxRapidEngine",
|
|
70
|
+
"RoutingMode",
|
|
71
|
+
"__version__",
|
|
72
|
+
"asset_id",
|
|
73
|
+
"check_rapid_versions",
|
|
74
|
+
"convert_directory",
|
|
75
|
+
"convert_to_markdown",
|
|
76
|
+
"html_tables_to_gfm",
|
|
77
|
+
"ingest_document",
|
|
78
|
+
"lint_markdown",
|
|
79
|
+
"lint_markdown_file",
|
|
80
|
+
"load_markdown_document",
|
|
81
|
+
"stage_images",
|
|
82
|
+
"stage_images_as_okf_assets",
|
|
83
|
+
"wrap_thoughts",
|
|
84
|
+
]
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
|
2
|
+
# @Author: SWHL (vendored into bobine, MIT licensed)
|
|
3
|
+
from .main import LaTeXOCR
|
|
4
|
+
|
|
5
|
+
# Legacy alias: older builds shipped the class as `LatexOCR`; keep both so
|
|
6
|
+
# bobine's engine import works regardless of which build was vendored.
|
|
7
|
+
LatexOCR = LaTeXOCR
|
|
8
|
+
|
|
9
|
+
__all__ = ["LaTeXOCR", "LatexOCR"]
|