parse-anything 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parse_anything-0.1.0/LICENSE +21 -0
- parse_anything-0.1.0/PKG-INFO +124 -0
- parse_anything-0.1.0/README.md +75 -0
- parse_anything-0.1.0/pyproject.toml +55 -0
- parse_anything-0.1.0/setup.cfg +4 -0
- parse_anything-0.1.0/src/parse_anything/__init__.py +54 -0
- parse_anything-0.1.0/src/parse_anything/__main__.py +3 -0
- parse_anything-0.1.0/src/parse_anything/cli.py +104 -0
- parse_anything-0.1.0/src/parse_anything/engines/__init__.py +3 -0
- parse_anything-0.1.0/src/parse_anything/engines/base.py +34 -0
- parse_anything-0.1.0/src/parse_anything/engines/docling_engine.py +99 -0
- parse_anything-0.1.0/src/parse_anything/engines/lite_engine.py +67 -0
- parse_anything-0.1.0/src/parse_anything/engines/markitdown_engine.py +41 -0
- parse_anything-0.1.0/src/parse_anything/engines/mineru_engine.py +121 -0
- parse_anything-0.1.0/src/parse_anything/engines/ocr_engine.py +34 -0
- parse_anything-0.1.0/src/parse_anything/engines/unstructured_engine.py +81 -0
- parse_anything-0.1.0/src/parse_anything/factory.py +117 -0
- parse_anything-0.1.0/src/parse_anything/ocr/__init__.py +18 -0
- parse_anything-0.1.0/src/parse_anything/ocr/base.py +59 -0
- parse_anything-0.1.0/src/parse_anything/ocr/paddle_engine.py +66 -0
- parse_anything-0.1.0/src/parse_anything/ocr/tesseract_engine.py +47 -0
- parse_anything-0.1.0/src/parse_anything/result.py +60 -0
- parse_anything-0.1.0/src/parse_anything.egg-info/PKG-INFO +124 -0
- parse_anything-0.1.0/src/parse_anything.egg-info/SOURCES.txt +27 -0
- parse_anything-0.1.0/src/parse_anything.egg-info/dependency_links.txt +1 -0
- parse_anything-0.1.0/src/parse_anything.egg-info/entry_points.txt +2 -0
- parse_anything-0.1.0/src/parse_anything.egg-info/requires.txt +34 -0
- parse_anything-0.1.0/src/parse_anything.egg-info/top_level.txt +1 -0
- parse_anything-0.1.0/tests/test_factory.py +37 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Yugesh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: parse-anything
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: One interface to parse any document: pick an engine (mineru, docling, unstructured, markitdown, lite) and get clean markdown out.
|
|
5
|
+
Author: Yugesh
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/yugeshsr13/parse-anything
|
|
8
|
+
Project-URL: Repository, https://github.com/yugeshsr13/parse-anything
|
|
9
|
+
Project-URL: Issues, https://github.com/yugeshsr13/parse-anything/issues
|
|
10
|
+
Keywords: pdf,parsing,ocr,docling,mineru,unstructured,markitdown,paddleocr,tesseract
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Topic :: Text Processing :: Markup
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Provides-Extra: lite
|
|
25
|
+
Requires-Dist: pymupdf4llm>=0.0.17; extra == "lite"
|
|
26
|
+
Provides-Extra: docling
|
|
27
|
+
Requires-Dist: docling>=2.15; extra == "docling"
|
|
28
|
+
Provides-Extra: unstructured
|
|
29
|
+
Requires-Dist: unstructured[all-docs]>=0.16; extra == "unstructured"
|
|
30
|
+
Provides-Extra: markitdown
|
|
31
|
+
Requires-Dist: markitdown[all]>=0.1.1; extra == "markitdown"
|
|
32
|
+
Provides-Extra: mineru
|
|
33
|
+
Requires-Dist: mineru[core]>=2.0; extra == "mineru"
|
|
34
|
+
Provides-Extra: tesseract
|
|
35
|
+
Requires-Dist: pytesseract>=0.3.10; extra == "tesseract"
|
|
36
|
+
Requires-Dist: Pillow>=10.0; extra == "tesseract"
|
|
37
|
+
Requires-Dist: pymupdf>=1.24; extra == "tesseract"
|
|
38
|
+
Provides-Extra: paddle
|
|
39
|
+
Requires-Dist: paddleocr>=3.0; extra == "paddle"
|
|
40
|
+
Requires-Dist: paddlepaddle>=3.0; extra == "paddle"
|
|
41
|
+
Requires-Dist: pymupdf>=1.24; extra == "paddle"
|
|
42
|
+
Provides-Extra: ocr
|
|
43
|
+
Requires-Dist: parse-anything[paddle,tesseract]; extra == "ocr"
|
|
44
|
+
Provides-Extra: all
|
|
45
|
+
Requires-Dist: parse-anything[docling,lite,markitdown,mineru,ocr,unstructured]; extra == "all"
|
|
46
|
+
Provides-Extra: dev
|
|
47
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
48
|
+
Dynamic: license-file
|
|
49
|
+
|
|
50
|
+
# parse-anything
|
|
51
|
+
|
|
52
|
+
One interface to parse any document. Pick an engine — **MinerU**, **Docling**, **unstructured**, **MarkItDown**, or the PyMuPDF **lite** path — pass a file path, get clean markdown/text back. OCR runs on **PaddleOCR (PP-OCRv5, default)** or **Tesseract**.
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from parse_anything import parse
|
|
56
|
+
|
|
57
|
+
result = parse("report.pdf", engine="miner-u")
|
|
58
|
+
print(result.content) # markdown
|
|
59
|
+
print(result.duration_s) # seconds
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
parse-anything report.pdf --engine miner-u -o report.md
|
|
64
|
+
parse-anything scan.png --engine ocr --ocr paddle
|
|
65
|
+
parse-anything deck.pptx # auto-picks the best installed engine
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Install
|
|
69
|
+
|
|
70
|
+
Core has **zero dependencies**; each engine is an optional extra:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install -e .[all] # everything
|
|
74
|
+
pip install -e .[mineru] # MinerU (scanned/complex PDFs)
|
|
75
|
+
pip install -e .[docling] # Docling (structure fidelity)
|
|
76
|
+
pip install -e .[unstructured] # unstructured (broadest formats)
|
|
77
|
+
pip install -e .[markitdown] # MarkItDown (office/web -> md)
|
|
78
|
+
pip install -e .[lite] # PyMuPDF fast path
|
|
79
|
+
pip install -e .[paddle] # PaddleOCR (PP-OCRv5)
|
|
80
|
+
pip install -e .[tesseract] # pytesseract (+ install the tesseract binary)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Windows Tesseract binary: [UB-Mannheim build](https://github.com/UB-Mannheim/tesseract/wiki) or `choco install tesseract`.
|
|
84
|
+
|
|
85
|
+
## Engines
|
|
86
|
+
|
|
87
|
+
| `--engine` | Backed by | Reach for it when |
|
|
88
|
+
|------------|-----------|-------------------|
|
|
89
|
+
| `mineru` / `miner-u` | [MinerU](https://github.com/opendatalab/MinerU) | scanned or complex PDFs — formulas, tables, multi-column; OCR built on PaddleOCR models |
|
|
90
|
+
| `docling` | [Docling](https://github.com/docling-project/docling) | you care about reading order, tables, office formats |
|
|
91
|
+
| `unstructured` | [unstructured](https://github.com/Unstructured-IO/unstructured) | odd formats: email, html, mixed batches |
|
|
92
|
+
| `markitdown` | [MarkItDown](https://github.com/microsoft/markitdown) | fast docx/pptx/xlsx/html → markdown (no OCR) |
|
|
93
|
+
| `lite` / `liteparse` | [PyMuPDF4LLM](https://github.com/pymupdf/RAG) | speed on PDFs that already have a text layer |
|
|
94
|
+
| `ocr` | PaddleOCR / Tesseract | images or fully scanned docs, raw text out |
|
|
95
|
+
| `auto` (default) | — | picks the best installed engine for the file type |
|
|
96
|
+
|
|
97
|
+
## OCR
|
|
98
|
+
|
|
99
|
+
`--ocr paddle` (default) uses **PaddleOCR PP-OCRv5** — the current open-source benchmark leader. `--ocr tesseract` is the portable fallback. `--ocr none` skips OCR (text-layer extraction only).
|
|
100
|
+
|
|
101
|
+
## Logging
|
|
102
|
+
|
|
103
|
+
Progress prints to stderr by default (auto-selected engine, OCR page-by-page, live MinerU subprocess output with a heartbeat on long runs). `-v` for debug detail, `-q` to silence it. As a library, nothing prints unless you call `logging.basicConfig(...)` yourself.
|
|
104
|
+
|
|
105
|
+
See [SKILLS.md](SKILLS.md) for the full skill contract (inputs, outputs, failure modes).
|
|
106
|
+
|
|
107
|
+
## Layout
|
|
108
|
+
|
|
109
|
+
```
|
|
110
|
+
src/parse_anything/
|
|
111
|
+
├── __init__.py # parse(path, engine=..., ocr=...) public API
|
|
112
|
+
├── factory.py # engine registry, aliases, auto-selection
|
|
113
|
+
├── result.py # ParseResult
|
|
114
|
+
├── cli.py # parse-anything CLI
|
|
115
|
+
├── engines/ # one module per backend (lazy imports)
|
|
116
|
+
└── ocr/ # paddle + tesseract backends, PDF rasterization
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Tests
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
pip install -e .[dev]
|
|
123
|
+
pytest
|
|
124
|
+
```
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# parse-anything
|
|
2
|
+
|
|
3
|
+
One interface to parse any document. Pick an engine — **MinerU**, **Docling**, **unstructured**, **MarkItDown**, or the PyMuPDF **lite** path — pass a file path, get clean markdown/text back. OCR runs on **PaddleOCR (PP-OCRv5, default)** or **Tesseract**.
|
|
4
|
+
|
|
5
|
+
```python
|
|
6
|
+
from parse_anything import parse
|
|
7
|
+
|
|
8
|
+
result = parse("report.pdf", engine="miner-u")
|
|
9
|
+
print(result.content) # markdown
|
|
10
|
+
print(result.duration_s) # seconds
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
parse-anything report.pdf --engine miner-u -o report.md
|
|
15
|
+
parse-anything scan.png --engine ocr --ocr paddle
|
|
16
|
+
parse-anything deck.pptx # auto-picks the best installed engine
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Install
|
|
20
|
+
|
|
21
|
+
Core has **zero dependencies**; each engine is an optional extra:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install -e .[all] # everything
|
|
25
|
+
pip install -e .[mineru] # MinerU (scanned/complex PDFs)
|
|
26
|
+
pip install -e .[docling] # Docling (structure fidelity)
|
|
27
|
+
pip install -e .[unstructured] # unstructured (broadest formats)
|
|
28
|
+
pip install -e .[markitdown] # MarkItDown (office/web -> md)
|
|
29
|
+
pip install -e .[lite] # PyMuPDF fast path
|
|
30
|
+
pip install -e .[paddle] # PaddleOCR (PP-OCRv5)
|
|
31
|
+
pip install -e .[tesseract] # pytesseract (+ install the tesseract binary)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Windows Tesseract binary: [UB-Mannheim build](https://github.com/UB-Mannheim/tesseract/wiki) or `choco install tesseract`.
|
|
35
|
+
|
|
36
|
+
## Engines
|
|
37
|
+
|
|
38
|
+
| `--engine` | Backed by | Reach for it when |
|
|
39
|
+
|------------|-----------|-------------------|
|
|
40
|
+
| `mineru` / `miner-u` | [MinerU](https://github.com/opendatalab/MinerU) | scanned or complex PDFs — formulas, tables, multi-column; OCR built on PaddleOCR models |
|
|
41
|
+
| `docling` | [Docling](https://github.com/docling-project/docling) | you care about reading order, tables, office formats |
|
|
42
|
+
| `unstructured` | [unstructured](https://github.com/Unstructured-IO/unstructured) | odd formats: email, html, mixed batches |
|
|
43
|
+
| `markitdown` | [MarkItDown](https://github.com/microsoft/markitdown) | fast docx/pptx/xlsx/html → markdown (no OCR) |
|
|
44
|
+
| `lite` / `liteparse` | [PyMuPDF4LLM](https://github.com/pymupdf/RAG) | speed on PDFs that already have a text layer |
|
|
45
|
+
| `ocr` | PaddleOCR / Tesseract | images or fully scanned docs, raw text out |
|
|
46
|
+
| `auto` (default) | — | picks the best installed engine for the file type |
|
|
47
|
+
|
|
48
|
+
## OCR
|
|
49
|
+
|
|
50
|
+
`--ocr paddle` (default) uses **PaddleOCR PP-OCRv5** — the current open-source benchmark leader. `--ocr tesseract` is the portable fallback. `--ocr none` skips OCR (text-layer extraction only).
|
|
51
|
+
|
|
52
|
+
## Logging
|
|
53
|
+
|
|
54
|
+
Progress prints to stderr by default (auto-selected engine, OCR page-by-page, live MinerU subprocess output with a heartbeat on long runs). `-v` for debug detail, `-q` to silence it. As a library, nothing prints unless you call `logging.basicConfig(...)` yourself.
|
|
55
|
+
|
|
56
|
+
See [SKILLS.md](SKILLS.md) for the full skill contract (inputs, outputs, failure modes).
|
|
57
|
+
|
|
58
|
+
## Layout
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
src/parse_anything/
|
|
62
|
+
├── __init__.py # parse(path, engine=..., ocr=...) public API
|
|
63
|
+
├── factory.py # engine registry, aliases, auto-selection
|
|
64
|
+
├── result.py # ParseResult
|
|
65
|
+
├── cli.py # parse-anything CLI
|
|
66
|
+
├── engines/ # one module per backend (lazy imports)
|
|
67
|
+
└── ocr/ # paddle + tesseract backends, PDF rasterization
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Tests
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install -e .[dev]
|
|
74
|
+
pytest
|
|
75
|
+
```
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "parse-anything"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "One interface to parse any document: pick an engine (mineru, docling, unstructured, markitdown, lite) and get clean markdown out."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Yugesh" }]
|
|
13
|
+
keywords = ["pdf", "parsing", "ocr", "docling", "mineru", "unstructured", "markitdown", "paddleocr", "tesseract"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.10",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Operating System :: OS Independent",
|
|
23
|
+
"Topic :: Text Processing :: Markup",
|
|
24
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
# Core is dependency-free: every engine is an optional extra so you only
|
|
28
|
+
# install the (heavy) backends you actually use.
|
|
29
|
+
dependencies = []
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/yugeshsr13/parse-anything"
|
|
33
|
+
Repository = "https://github.com/yugeshsr13/parse-anything"
|
|
34
|
+
Issues = "https://github.com/yugeshsr13/parse-anything/issues"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
lite = ["pymupdf4llm>=0.0.17"]
|
|
38
|
+
docling = ["docling>=2.15"]
|
|
39
|
+
unstructured = ["unstructured[all-docs]>=0.16"]
|
|
40
|
+
markitdown = ["markitdown[all]>=0.1.1"]
|
|
41
|
+
mineru = ["mineru[core]>=2.0"]
|
|
42
|
+
tesseract = ["pytesseract>=0.3.10", "Pillow>=10.0", "pymupdf>=1.24"]
|
|
43
|
+
paddle = ["paddleocr>=3.0", "paddlepaddle>=3.0", "pymupdf>=1.24"]
|
|
44
|
+
ocr = ["parse-anything[tesseract,paddle]"]
|
|
45
|
+
all = ["parse-anything[lite,docling,unstructured,markitdown,mineru,ocr]"]
|
|
46
|
+
dev = ["pytest>=8.0"]
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
parse-anything = "parse_anything.cli:main"
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["src"]
|
|
53
|
+
|
|
54
|
+
[tool.pytest.ini_options]
|
|
55
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""parse-anything: one interface over mineru, docling, unstructured, markitdown,
|
|
2
|
+
lite (PyMuPDF) and raw OCR (PaddleOCR / Tesseract).
|
|
3
|
+
|
|
4
|
+
from parse_anything import parse
|
|
5
|
+
|
|
6
|
+
result = parse("report.pdf", engine="mineru")
|
|
7
|
+
print(result.content) # markdown
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from .factory import engine_names, get_engine
|
|
14
|
+
from .result import ParseResult
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
17
|
+
|
|
18
|
+
# Library convention: don't configure handlers here, just make sure "no
|
|
19
|
+
# handlers found" warnings don't show up for callers who haven't set up
|
|
20
|
+
# logging themselves. The CLI (cli.py) calls logging.basicConfig() instead.
|
|
21
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def parse(
|
|
26
|
+
path: str | Path,
|
|
27
|
+
engine: str = "auto",
|
|
28
|
+
ocr: str = "paddle",
|
|
29
|
+
**options,
|
|
30
|
+
) -> ParseResult:
|
|
31
|
+
"""Parses a document and returns its extracted content.
|
|
32
|
+
|
|
33
|
+
path: file to parse (pdf, docx, pptx, xlsx, html, images, ...)
|
|
34
|
+
engine: "auto" (default), "mineru"/"miner-u", "docling", "unstructured",
|
|
35
|
+
"markitdown", "lite"/"liteparse", or "ocr"
|
|
36
|
+
ocr: "paddle" (default, PP-OCRv5), "tesseract", or "none"
|
|
37
|
+
options: engine-specific knobs, e.g. strategy="hi_res" (unstructured),
|
|
38
|
+
force_ocr=True (mineru), dpi=300 (ocr), lang="ch" (ocr backends)
|
|
39
|
+
"""
|
|
40
|
+
file_path = Path(path)
|
|
41
|
+
if not file_path.exists():
|
|
42
|
+
raise FileNotFoundError(f"No such file: {file_path}")
|
|
43
|
+
|
|
44
|
+
logger.info("parsing %s (engine=%s, ocr=%s)", file_path.name, engine, ocr)
|
|
45
|
+
backend = get_engine(engine, path=file_path)
|
|
46
|
+
result = backend.parse(file_path, ocr=ocr, **options)
|
|
47
|
+
logger.info(
|
|
48
|
+
"finished %s: engine=%s ocr=%s %d chars in %.2fs",
|
|
49
|
+
file_path.name, result.engine, result.ocr, len(result.content), result.duration_s,
|
|
50
|
+
)
|
|
51
|
+
return result
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
__all__ = ["parse", "ParseResult", "get_engine", "engine_names", "__version__"]
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import logging
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from . import __version__, parse
|
|
6
|
+
from .factory import engine_names
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger("parse_anything.cli")
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
12
|
+
p = argparse.ArgumentParser(
|
|
13
|
+
prog="parse-anything",
|
|
14
|
+
description="Parse any document with your engine of choice and print the extracted content.",
|
|
15
|
+
)
|
|
16
|
+
p.add_argument("path", nargs="?", help="File to parse (pdf, docx, pptx, html, png, ...)")
|
|
17
|
+
p.add_argument(
|
|
18
|
+
"-e", "--engine", default="auto",
|
|
19
|
+
help="auto | mineru (miner-u) | docling | unstructured | markitdown | lite (liteparse) | ocr",
|
|
20
|
+
)
|
|
21
|
+
p.add_argument(
|
|
22
|
+
"--ocr", default="paddle", choices=["paddle", "tesseract", "none"],
|
|
23
|
+
help="OCR backend for scanned pages/images (default: paddle / PP-OCRv5)",
|
|
24
|
+
)
|
|
25
|
+
p.add_argument("-o", "--output", help="Write result here (.md/.txt for content, .json for full result)")
|
|
26
|
+
p.add_argument(
|
|
27
|
+
"-f", "--format", default="content", choices=["content", "json"],
|
|
28
|
+
help="stdout format: raw extracted content (default) or the full JSON result",
|
|
29
|
+
)
|
|
30
|
+
p.add_argument("--strategy", help="unstructured only: fast | hi_res | ocr_only | auto")
|
|
31
|
+
p.add_argument("--force-ocr", action="store_true", help="mineru only: OCR every page")
|
|
32
|
+
p.add_argument("--lang", help="OCR language (paddle: en/ch/..., tesseract: eng/deu/...)")
|
|
33
|
+
p.add_argument("--dpi", type=int, help="ocr engine only: PDF rasterization DPI (default 200)")
|
|
34
|
+
p.add_argument("--timeout", type=int, default=900, help="mineru timeout in seconds (default 900 / 15 min)")
|
|
35
|
+
p.add_argument("--list-engines", action="store_true", help="List engines and exit")
|
|
36
|
+
p.add_argument(
|
|
37
|
+
"-v", "--verbose", action="count", default=0,
|
|
38
|
+
help="raise log verbosity to debug (resolved engine names, per-page timing, ...); "
|
|
39
|
+
"progress logs (auto-selection, mineru output, OCR page progress) print by default",
|
|
40
|
+
)
|
|
41
|
+
p.add_argument("-q", "--quiet", action="store_true", help="suppress progress logs, print only errors")
|
|
42
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
43
|
+
return p
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _configure_logging(verbose: int, quiet: bool) -> None:
|
|
47
|
+
if quiet:
|
|
48
|
+
level = logging.ERROR
|
|
49
|
+
elif verbose >= 1:
|
|
50
|
+
level = logging.DEBUG
|
|
51
|
+
else:
|
|
52
|
+
level = logging.INFO
|
|
53
|
+
logging.basicConfig(
|
|
54
|
+
level=level,
|
|
55
|
+
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
|
|
56
|
+
datefmt="%H:%M:%S",
|
|
57
|
+
stream=sys.stderr,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def main(argv: list[str] | None = None) -> int:
|
|
62
|
+
args = build_parser().parse_args(argv)
|
|
63
|
+
_configure_logging(args.verbose, args.quiet)
|
|
64
|
+
|
|
65
|
+
if args.list_engines:
|
|
66
|
+
print("\n".join(engine_names()))
|
|
67
|
+
return 0
|
|
68
|
+
if not args.path:
|
|
69
|
+
build_parser().print_help()
|
|
70
|
+
return 2
|
|
71
|
+
|
|
72
|
+
options = {}
|
|
73
|
+
if args.strategy:
|
|
74
|
+
options["strategy"] = args.strategy
|
|
75
|
+
if args.force_ocr:
|
|
76
|
+
options["force_ocr"] = True
|
|
77
|
+
if args.lang:
|
|
78
|
+
options["lang"] = args.lang
|
|
79
|
+
if args.dpi:
|
|
80
|
+
options["dpi"] = args.dpi
|
|
81
|
+
if args.timeout != 900:
|
|
82
|
+
options["timeout"] = args.timeout
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
result = parse(args.path, engine=args.engine, ocr=args.ocr, **options)
|
|
86
|
+
except (FileNotFoundError, ValueError, ImportError, RuntimeError) as exc:
|
|
87
|
+
logger.error(str(exc))
|
|
88
|
+
return 1
|
|
89
|
+
|
|
90
|
+
if args.output:
|
|
91
|
+
saved = result.save(args.output)
|
|
92
|
+
if not args.quiet:
|
|
93
|
+
print(
|
|
94
|
+
f"[{result.engine}] {result.path} -> {saved} "
|
|
95
|
+
f"({len(result.content)} chars, {result.duration_s:.1f}s)",
|
|
96
|
+
file=sys.stderr,
|
|
97
|
+
)
|
|
98
|
+
else:
|
|
99
|
+
print(result.to_json() if args.format == "json" else result.content)
|
|
100
|
+
return 0
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
if __name__ == "__main__":
|
|
104
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from ..result import ParseResult
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ParserEngine(ABC):
|
|
8
|
+
"""Common interface every parsing backend implements.
|
|
9
|
+
|
|
10
|
+
Deliberately minimal — file path in, ParseResult out — so callers can swap
|
|
11
|
+
engines (mineru, docling, unstructured, markitdown, lite) without changing
|
|
12
|
+
any downstream code.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
name: str
|
|
16
|
+
|
|
17
|
+
#: pip extra that pulls in this engine's dependencies, used in error messages.
|
|
18
|
+
extra: str
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def parse(self, path: Path, ocr: str = "paddle", **options) -> ParseResult:
|
|
22
|
+
"""Extracts content from the file.
|
|
23
|
+
|
|
24
|
+
ocr: "paddle" (default), "tesseract", or "none". Engines that bundle
|
|
25
|
+
their own OCR pipeline (mineru) or don't support choosing (markitdown)
|
|
26
|
+
document how they interpret this.
|
|
27
|
+
"""
|
|
28
|
+
raise NotImplementedError
|
|
29
|
+
|
|
30
|
+
def _missing_dep(self, package: str) -> ImportError:
|
|
31
|
+
return ImportError(
|
|
32
|
+
f"Engine '{self.name}' needs '{package}'. "
|
|
33
|
+
f"Install it with: pip install parse-anything[{self.extra}]"
|
|
34
|
+
)
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from ..result import ParseResult, Timer
|
|
5
|
+
from .base import ParserEngine
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DoclingEngine(ParserEngine):
|
|
11
|
+
"""Docling (IBM) — high-fidelity structure extraction (reading order,
|
|
12
|
+
tables, headings) for PDFs and office formats, exported as markdown.
|
|
13
|
+
|
|
14
|
+
OCR backends docling supports natively: easyocr (its default), tesseract,
|
|
15
|
+
rapidocr. It has no PaddleOCR plugin, so ocr="paddle" falls back to
|
|
16
|
+
docling's default with a note in metadata — pick engine="mineru" (whose
|
|
17
|
+
pipeline is built on PaddleOCR models) when paddle-quality OCR matters.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
name = "docling"
|
|
21
|
+
extra = "docling"
|
|
22
|
+
|
|
23
|
+
def parse(self, path: Path, ocr: str = "paddle", **options) -> ParseResult:
|
|
24
|
+
try:
|
|
25
|
+
from docling.document_converter import DocumentConverter
|
|
26
|
+
except ImportError:
|
|
27
|
+
raise self._missing_dep("docling") from None
|
|
28
|
+
|
|
29
|
+
converter, ocr_used, note = self._build_converter(ocr)
|
|
30
|
+
if note:
|
|
31
|
+
logger.warning(note)
|
|
32
|
+
logger.info("running docling on %s (ocr=%s)", path.name, ocr_used)
|
|
33
|
+
|
|
34
|
+
with Timer() as t:
|
|
35
|
+
result = converter.convert(str(path))
|
|
36
|
+
content = result.document.export_to_markdown()
|
|
37
|
+
|
|
38
|
+
metadata = {"pages": len(getattr(result.document, "pages", []) or [])}
|
|
39
|
+
if note:
|
|
40
|
+
metadata["ocr_note"] = note
|
|
41
|
+
logger.info("docling finished %s: %d pages, %d chars", path.name, metadata["pages"], len(content))
|
|
42
|
+
return ParseResult(
|
|
43
|
+
path=str(path),
|
|
44
|
+
engine=self.name,
|
|
45
|
+
content=content,
|
|
46
|
+
format="markdown",
|
|
47
|
+
ocr=ocr_used,
|
|
48
|
+
metadata=metadata,
|
|
49
|
+
duration_s=t.elapsed,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def _build_converter(self, ocr: str):
|
|
53
|
+
from docling.document_converter import DocumentConverter
|
|
54
|
+
|
|
55
|
+
if ocr == "tesseract":
|
|
56
|
+
try:
|
|
57
|
+
from docling.datamodel.base_models import InputFormat
|
|
58
|
+
from docling.datamodel.pipeline_options import (
|
|
59
|
+
PdfPipelineOptions,
|
|
60
|
+
TesseractOcrOptions,
|
|
61
|
+
)
|
|
62
|
+
from docling.document_converter import PdfFormatOption
|
|
63
|
+
|
|
64
|
+
opts = PdfPipelineOptions()
|
|
65
|
+
opts.do_ocr = True
|
|
66
|
+
opts.ocr_options = TesseractOcrOptions()
|
|
67
|
+
converter = DocumentConverter(
|
|
68
|
+
format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=opts)}
|
|
69
|
+
)
|
|
70
|
+
return converter, "tesseract", None
|
|
71
|
+
except ImportError:
|
|
72
|
+
return (
|
|
73
|
+
DocumentConverter(),
|
|
74
|
+
"easyocr (docling default)",
|
|
75
|
+
"tesseract requested but docling's tesseract option is unavailable",
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
if ocr == "none":
|
|
79
|
+
try:
|
|
80
|
+
from docling.datamodel.base_models import InputFormat
|
|
81
|
+
from docling.datamodel.pipeline_options import PdfPipelineOptions
|
|
82
|
+
from docling.document_converter import PdfFormatOption
|
|
83
|
+
|
|
84
|
+
opts = PdfPipelineOptions()
|
|
85
|
+
opts.do_ocr = False
|
|
86
|
+
converter = DocumentConverter(
|
|
87
|
+
format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=opts)}
|
|
88
|
+
)
|
|
89
|
+
return converter, None, None
|
|
90
|
+
except ImportError:
|
|
91
|
+
return DocumentConverter(), None, None
|
|
92
|
+
|
|
93
|
+
note = (
|
|
94
|
+
"docling has no PaddleOCR backend; used its default (easyocr). "
|
|
95
|
+
"Use engine='mineru' for a PaddleOCR-based pipeline."
|
|
96
|
+
if ocr in ("paddle", "paddleocr")
|
|
97
|
+
else None
|
|
98
|
+
)
|
|
99
|
+
return DocumentConverter(), "easyocr (docling default)", note
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from ..ocr import IMAGE_SUFFIXES, get_ocr
|
|
5
|
+
from ..result import ParseResult, Timer
|
|
6
|
+
from .base import ParserEngine
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LiteEngine(ParserEngine):
|
|
12
|
+
"""liteparse — the lightweight, fast path. PyMuPDF-based markdown for PDFs
|
|
13
|
+
(no ML models, near-instant), direct OCR for images, plain text extraction
|
|
14
|
+
for everything else PyMuPDF can open (epub, xps, txt).
|
|
15
|
+
|
|
16
|
+
Best when documents have a real text layer and speed matters more than
|
|
17
|
+
layout reconstruction.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
name = "lite"
|
|
21
|
+
extra = "lite"
|
|
22
|
+
|
|
23
|
+
def parse(self, path: Path, ocr: str = "paddle", **options) -> ParseResult:
|
|
24
|
+
suffix = path.suffix.lower()
|
|
25
|
+
|
|
26
|
+
if suffix in IMAGE_SUFFIXES:
|
|
27
|
+
if ocr == "none":
|
|
28
|
+
raise ValueError(f"'{path.name}' is an image; lite needs ocr='paddle' or 'tesseract' for images.")
|
|
29
|
+
logger.info("lite: routing image %s to OCR backend '%s'", path.name, ocr)
|
|
30
|
+
backend = get_ocr(ocr, lang=options.get("lang"))
|
|
31
|
+
with Timer() as t:
|
|
32
|
+
text, meta = backend.parse_file(path)
|
|
33
|
+
return ParseResult(
|
|
34
|
+
path=str(path), engine=self.name, content=text, format="text",
|
|
35
|
+
ocr=backend.name, metadata=meta, duration_s=t.elapsed,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
if suffix == ".pdf":
|
|
39
|
+
try:
|
|
40
|
+
import pymupdf4llm
|
|
41
|
+
except ImportError:
|
|
42
|
+
raise self._missing_dep("pymupdf4llm") from None
|
|
43
|
+
logger.info("lite: extracting text layer from %s via pymupdf4llm", path.name)
|
|
44
|
+
with Timer() as t:
|
|
45
|
+
content = pymupdf4llm.to_markdown(str(path))
|
|
46
|
+
logger.info("lite: finished %s, %d chars in %.2fs", path.name, len(content), t.elapsed)
|
|
47
|
+
return ParseResult(
|
|
48
|
+
path=str(path), engine=self.name, content=content, format="markdown",
|
|
49
|
+
ocr=None, metadata={}, duration_s=t.elapsed,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
# Anything else PyMuPDF understands (epub, xps, cbz, txt...)
|
|
53
|
+
try:
|
|
54
|
+
import pymupdf as fitz
|
|
55
|
+
except ImportError:
|
|
56
|
+
try:
|
|
57
|
+
import fitz
|
|
58
|
+
except ImportError:
|
|
59
|
+
raise self._missing_dep("pymupdf") from None
|
|
60
|
+
with Timer() as t:
|
|
61
|
+
with fitz.open(str(path)) as doc:
|
|
62
|
+
page_count = doc.page_count
|
|
63
|
+
content = "\n\n".join(page.get_text() for page in doc)
|
|
64
|
+
return ParseResult(
|
|
65
|
+
path=str(path), engine=self.name, content=content, format="text",
|
|
66
|
+
ocr=None, metadata={"pages": page_count}, duration_s=t.elapsed,
|
|
67
|
+
)
|