mineru-parser 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mineru_parser-0.1.0/.github/workflows/release.yml +81 -0
- mineru_parser-0.1.0/.gitignore +16 -0
- mineru_parser-0.1.0/.python-version +1 -0
- mineru_parser-0.1.0/LICENSE +21 -0
- mineru_parser-0.1.0/PKG-INFO +409 -0
- mineru_parser-0.1.0/README.md +381 -0
- mineru_parser-0.1.0/docs/specs/CLAUDE.md +110 -0
- mineru_parser-0.1.0/docs/specs/SPEC-001-mineru-api-client.changelog.md +106 -0
- mineru_parser-0.1.0/docs/specs/SPEC-001-mineru-api-client.md +965 -0
- mineru_parser-0.1.0/docs/specs/SPEC-002-result-downloader.changelog.md +76 -0
- mineru_parser-0.1.0/docs/specs/SPEC-002-result-downloader.md +1347 -0
- mineru_parser-0.1.0/docs/specs/SPEC-003-result-extractor.changelog.md +132 -0
- mineru_parser-0.1.0/docs/specs/SPEC-003-result-extractor.md +1344 -0
- mineru_parser-0.1.0/docs/specs/SPEC-004-pipeline.changelog.md +104 -0
- mineru_parser-0.1.0/docs/specs/SPEC-004-pipeline.md +1053 -0
- mineru_parser-0.1.0/docs/specs/template.md +183 -0
- mineru_parser-0.1.0/pyproject.toml +52 -0
- mineru_parser-0.1.0/src/mineru_parser/__init__.py +41 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/__init__.py +47 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/_telemetry.py +7 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/_time.py +13 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/_validation.py +161 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/api.py +233 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/async_result.py +254 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/errors.py +138 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/models.py +211 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/polling.py +309 -0
- mineru_parser-0.1.0/src/mineru_parser/_client/transport.py +289 -0
- mineru_parser-0.1.0/src/mineru_parser/_downloader/__init__.py +40 -0
- mineru_parser-0.1.0/src/mineru_parser/_downloader/downloader.py +1099 -0
- mineru_parser-0.1.0/src/mineru_parser/_downloader/errors.py +122 -0
- mineru_parser-0.1.0/src/mineru_parser/_downloader/policy.py +89 -0
- mineru_parser-0.1.0/src/mineru_parser/_downloader/security.py +380 -0
- mineru_parser-0.1.0/src/mineru_parser/_extractor/__init__.py +38 -0
- mineru_parser-0.1.0/src/mineru_parser/_extractor/_manifest.py +127 -0
- mineru_parser-0.1.0/src/mineru_parser/_extractor/_members.py +351 -0
- mineru_parser-0.1.0/src/mineru_parser/_extractor/errors.py +130 -0
- mineru_parser-0.1.0/src/mineru_parser/_extractor/extractor.py +310 -0
- mineru_parser-0.1.0/src/mineru_parser/_extractor/limits.py +35 -0
- mineru_parser-0.1.0/src/mineru_parser/_extractor/models.py +31 -0
- mineru_parser-0.1.0/src/mineru_parser/_http.py +77 -0
- mineru_parser-0.1.0/src/mineru_parser/_telemetry.py +62 -0
- mineru_parser-0.1.0/src/mineru_parser/_version.py +10 -0
- mineru_parser-0.1.0/src/mineru_parser/errors.py +15 -0
- mineru_parser-0.1.0/src/mineru_parser/logging.py +0 -0
- mineru_parser-0.1.0/src/mineru_parser/pipeline/__init__.py +29 -0
- mineru_parser-0.1.0/src/mineru_parser/pipeline/budget.py +109 -0
- mineru_parser-0.1.0/src/mineru_parser/pipeline/errors.py +64 -0
- mineru_parser-0.1.0/src/mineru_parser/pipeline/orchestrator.py +586 -0
- mineru_parser-0.1.0/src/mineru_parser/pipeline/result.py +32 -0
- mineru_parser-0.1.0/src/mineru_parser/pipeline/stages.py +22 -0
- mineru_parser-0.1.0/src/mineru_parser/schema/content_v2.py +198 -0
- mineru_parser-0.1.0/tests/data/result_001.zip +0 -0
- mineru_parser-0.1.0/tests/test_client.py +558 -0
- mineru_parser-0.1.0/tests/test_downloader.py +1781 -0
- mineru_parser-0.1.0/tests/test_extractor.py +1379 -0
- mineru_parser-0.1.0/tests/test_extractor_property.py +154 -0
- mineru_parser-0.1.0/tests/test_pipeline.py +1546 -0
- mineru_parser-0.1.0/uv.lock +503 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
name: Test (Python ${{ matrix.python-version }})
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
strategy:
|
|
16
|
+
fail-fast: false
|
|
17
|
+
matrix:
|
|
18
|
+
python-version: ["3.12", "3.13", "3.14"]
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v7
|
|
21
|
+
|
|
22
|
+
- uses: astral-sh/setup-uv@v10.3.0
|
|
23
|
+
with:
|
|
24
|
+
python-version: ${{ matrix.python-version }}
|
|
25
|
+
enable-cache: true
|
|
26
|
+
|
|
27
|
+
- name: Install dependencies
|
|
28
|
+
run: uv sync --frozen --group dev --python ${{ matrix.python-version }}
|
|
29
|
+
|
|
30
|
+
- name: Run tests
|
|
31
|
+
run: uv run --frozen --python ${{ matrix.python-version }} pytest -q
|
|
32
|
+
|
|
33
|
+
build:
|
|
34
|
+
name: Build distributions
|
|
35
|
+
needs: test
|
|
36
|
+
runs-on: ubuntu-latest
|
|
37
|
+
steps:
|
|
38
|
+
- uses: actions/checkout@v7
|
|
39
|
+
|
|
40
|
+
- uses: astral-sh/setup-uv@v10.3.0
|
|
41
|
+
with:
|
|
42
|
+
python-version: "3.13"
|
|
43
|
+
enable-cache: true
|
|
44
|
+
|
|
45
|
+
- name: Verify tag matches package version
|
|
46
|
+
run: |
|
|
47
|
+
version=$(sed -n 's/^__version__ = "\(.*\)"$/\1/p' src/mineru_parser/_version.py)
|
|
48
|
+
if [ "$GITHUB_REF_NAME" != "v$version" ]; then
|
|
49
|
+
echo "::error::Tag $GITHUB_REF_NAME does not match package version $version"
|
|
50
|
+
exit 1
|
|
51
|
+
fi
|
|
52
|
+
|
|
53
|
+
- name: Build sdist and wheel
|
|
54
|
+
run: uv build
|
|
55
|
+
|
|
56
|
+
- name: Check distributions
|
|
57
|
+
run: uvx twine check dist/*
|
|
58
|
+
|
|
59
|
+
- uses: actions/upload-artifact@v7
|
|
60
|
+
with:
|
|
61
|
+
name: dist
|
|
62
|
+
path: dist/
|
|
63
|
+
if-no-files-found: error
|
|
64
|
+
|
|
65
|
+
publish:
|
|
66
|
+
name: Publish to PyPI
|
|
67
|
+
needs: build
|
|
68
|
+
runs-on: ubuntu-latest
|
|
69
|
+
environment:
|
|
70
|
+
name: pypi
|
|
71
|
+
url: https://pypi.org/p/mineru-parser
|
|
72
|
+
permissions:
|
|
73
|
+
id-token: write
|
|
74
|
+
steps:
|
|
75
|
+
- uses: actions/download-artifact@v8
|
|
76
|
+
with:
|
|
77
|
+
name: dist
|
|
78
|
+
path: dist/
|
|
79
|
+
|
|
80
|
+
- name: Publish distributions to PyPI
|
|
81
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 代码与咖啡渍
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: mineru-parser
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: One-call MinerU document parsing: submit, wait, download and extract
|
|
5
|
+
Project-URL: Homepage, https://github.com/dream-365/mineru-parser
|
|
6
|
+
Project-URL: Repository, https://github.com/dream-365/mineru-parser
|
|
7
|
+
Project-URL: Issues, https://github.com/dream-365/mineru-parser/issues
|
|
8
|
+
Author-email: 代码与咖啡渍 <14042413+jackafx@user.noreply.gitee.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: document-parsing,layout-analysis,mineru,ocr,pdf
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Classifier: Topic :: Text Processing
|
|
22
|
+
Requires-Python: >=3.12
|
|
23
|
+
Requires-Dist: idna>=3.7
|
|
24
|
+
Requires-Dist: pydantic>=2.9
|
|
25
|
+
Requires-Dist: requests>=2.32
|
|
26
|
+
Requires-Dist: urllib3>=2.2
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# mineru-parser
|
|
30
|
+
|
|
31
|
+
`mineru-parser` 把 MinerU 的异步文档解析流程封装成一次同步 Python 调用:**输入一个 PDF URI,输出一个 `ParseResult`**——包含解析产物、任务元数据与失败原因的状态信封。
|
|
32
|
+
|
|
33
|
+
一次调用跨越四个阶段,阶段顺序、参数透传、资源生命周期与端到端预算全部由库负责:
|
|
34
|
+
|
|
35
|
+
| 阶段 | 内部实现 | 输入 | 输出 |
|
|
36
|
+
| --- | --- | --- | --- |
|
|
37
|
+
| ① 提交 | client | PDF URI | `task_id` |
|
|
38
|
+
| ② 等待 | client | `task_id` | `zip_url` |
|
|
39
|
+
| ③ 下载 | downloader | `zip_url` | `io.BytesIO`(ZIP 内存流) |
|
|
40
|
+
| ④ 解析 | extractor | ZIP 内存流 | `ExtractedResult`(`{pages, images}`) |
|
|
41
|
+
|
|
42
|
+
## 快速开始
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from mineru_parser import parse
|
|
46
|
+
|
|
47
|
+
# API Key 默认读取环境变量 MINERU_API_KEY
|
|
48
|
+
# 下面使用 MinerU 官方的公开示例 PDF(13 页 Elsevier 论文),可直接运行
|
|
49
|
+
result = parse("https://cdn-mineru.openxlab.org.cn/demo/example.pdf") # 同步阻塞,直到任务终态
|
|
50
|
+
|
|
51
|
+
if result["status"] == "success":
|
|
52
|
+
for page in result["data"]["pages"]: # list[Page]:逐页版面元素
|
|
53
|
+
for block in page:
|
|
54
|
+
print(block.type, block.bbox)
|
|
55
|
+
for image in result["data"]["images"]: # list[{"path", "bytes"}]
|
|
56
|
+
print(image["path"], len(image["bytes"]))
|
|
57
|
+
print(result["task_id"], result["data_id"])
|
|
58
|
+
else:
|
|
59
|
+
err = result["error"]
|
|
60
|
+
print(err["stage"], err["error_type"], err["message"])
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`parse()` 是**同步阻塞**调用,一次调用完成"提交 → 等待 → 下载 → 解析"。任一阶段失败都不会交出半成品:失败时 `data` 必定是 `None`。
|
|
64
|
+
|
|
65
|
+
### 已验证示例
|
|
66
|
+
|
|
67
|
+
对上面这个示例 PDF 的真实运行结果(2026-10-21):
|
|
68
|
+
|
|
69
|
+
| 项目 | 实测值 |
|
|
70
|
+
| --- | --- |
|
|
71
|
+
| 端到端耗时 | 37.5 s |
|
|
72
|
+
| 提交 / 等待 / 下载 / 解析 | 0.22 s / 36.56 s / 0.67 s / 0.03 s |
|
|
73
|
+
| 结果包 ZIP | 985,735 B |
|
|
74
|
+
| 页面数 | 13 |
|
|
75
|
+
| 版面元素 | 156 |
|
|
76
|
+
| 图片 | 20 张,共 631,443 B |
|
|
77
|
+
|
|
78
|
+
```text
|
|
79
|
+
status = success
|
|
80
|
+
task_id = 85d5b85c-c040-4223-8620-fc040f9b285b
|
|
81
|
+
data_id = None # 未传 data_id
|
|
82
|
+
error = None
|
|
83
|
+
pages=13 blocks=156 images=20
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
解析出的元素类型覆盖 `title` / `paragraph` / `table` / `chart` / `equation_interline` / `list` / `page_header` / `page_number`;标题、作者、机构、摘要、公式与参考文献均正确归一化。
|
|
87
|
+
|
|
88
|
+
> 耗时随服务端队列与文档复杂度变化,上表只用于说明量级;页面数、元素数与图片数由文档本身决定。
|
|
89
|
+
|
|
90
|
+
### `parse()` 参数
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
def parse(
|
|
94
|
+
uri: str, # 必填:PDF 的绝对 http(s) URI
|
|
95
|
+
*,
|
|
96
|
+
model_version: str = "vlm", # 透传给 MinerU
|
|
97
|
+
data_id: str | None = None, # 对账标识(不是幂等键)
|
|
98
|
+
timeout: float | None = None, # 端到端相对预算(秒)
|
|
99
|
+
deadline_epoch: float | None = None, # 端到端绝对截止(Unix 时间戳)
|
|
100
|
+
limits: ExtractionLimits | None = None, # 解压侧限额
|
|
101
|
+
raise_on_error: bool = False, # True:失败时抛异常(见「失败语义」)
|
|
102
|
+
on_update: Callable[[MinerUTaskStatus], None] | None = None,
|
|
103
|
+
on_error: Callable[[PipelineStage, BaseException], None] | None = None,
|
|
104
|
+
on_metric: MetricHook | None = None,
|
|
105
|
+
correlation_id: str | None = None, # 透传到全部日志与指标
|
|
106
|
+
) -> ParseResult: ...
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`timeout` 与 `deadline_epoch` **互斥**,同时给出会在任何网络请求之前失败。两者都不给表示**不设端到端上限**(等待阶段沿用客户端的无限等待语义)。
|
|
110
|
+
|
|
111
|
+
| 参数 | 作用 | 详见 |
|
|
112
|
+
| --- | --- | --- |
|
|
113
|
+
| `model_version` | 透传给 MinerU 的模型版本 | — |
|
|
114
|
+
| `data_id` | 对账标识;**不是幂等键** | [风险与已知限制](#风险与已知限制) |
|
|
115
|
+
| `timeout` / `deadline_epoch` | 覆盖四个阶段的端到端预算 | [端到端预算与超时](#端到端预算与超时) |
|
|
116
|
+
| `limits` | 解压侧限额(`ExtractionLimits`) | [内存预算与并发度](#内存预算与并发度) |
|
|
117
|
+
| `raise_on_error` | `True` 时失败抛异常 | [失败语义](#失败语义默认信封异常可选) |
|
|
118
|
+
| `on_update` | 任务状态 / 进度变化时触发 | — |
|
|
119
|
+
| `on_error` | 阶段失败时收到 `(stage, 原始异常)` | [阶段归因](#阶段归因pipelinestage) |
|
|
120
|
+
| `on_metric` | 指标回调 | [可观测性](#可观测性) |
|
|
121
|
+
| `correlation_id` | 透传到全部日志与指标 | [可观测性](#可观测性) |
|
|
122
|
+
|
|
123
|
+
## 返回值:`ParseResult`
|
|
124
|
+
|
|
125
|
+
`ParseResult` 是普通 `dict`,**恰好**六个键:
|
|
126
|
+
|
|
127
|
+
| 键 | 类型 | 说明 |
|
|
128
|
+
| --- | --- | --- |
|
|
129
|
+
| `status` | `Literal["success", "failed"]` | 本次调用的结果状态 |
|
|
130
|
+
| `data` | `ExtractedResult \| None` | 成功时为解析产物;**失败时必须为 `None`**(不返回部分产物) |
|
|
131
|
+
| `task_id` | `str \| None` | 本次任务的服务端 id |
|
|
132
|
+
| `data_id` | `str \| None` | 对账标识;服务端与提交参数都缺失时为 `None` |
|
|
133
|
+
| `zip_url` | `str \| None` | 结果包地址(**敏感值**,见下) |
|
|
134
|
+
| `error` | `ErrorMessage \| None` | 失败原因;**成功时必须为 `None`** |
|
|
135
|
+
|
|
136
|
+
`data` 的形状就是 `ExtractedResult`——与结果包提取器的返回值逐字相同:
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
{
|
|
140
|
+
"pages": [[LayoutElement, ...], ...], # list[Page],元素是 Pydantic 模型
|
|
141
|
+
"images": [{"path": "images/a.jpg", "bytes": b"..."}],
|
|
142
|
+
}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
- **`pages`** — 按页分组的版面元素列表。元素按 `type` 判别:`title` / `paragraph` / `table` / `image` / `equation_interline` / `chart` / `list` / `algorithm` 以及页眉页脚等页级元素。
|
|
146
|
+
- **`images`** — 结果包内顶层 `images/` 下的图片原始字节,按 canonical path 字典序排列。
|
|
147
|
+
|
|
148
|
+
**元数据来源**:`task_id` / `data_id` / `zip_url` 取自本次调用的终态结果(`MinerUTaskResult`),失败时按"**当时已知**"填充——提交成功即知 `task_id`,等待完成后才知 `zip_url`,未知的取 `None`。
|
|
149
|
+
|
|
150
|
+
> **⚠️ `zip_url` 是敏感值**:它可能携带签名凭据。库不会把它写入任何日志或指标标签,但也**不会**对其脱敏——落盘或打日志前请自行评估。
|
|
151
|
+
|
|
152
|
+
## 失败语义:默认信封,异常可选
|
|
153
|
+
|
|
154
|
+
**默认不抛异常。** 已知失败(`MinerUParserError` 家族)被封装成 `ErrorMessage` 随 `status="failed"` 返回,调用方默认无需 `try` / `except` 就能拿到成败与失败原因。
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
result = parse("https://cdn-mineru.openxlab.org.cn/demo/example.pdf")
|
|
158
|
+
|
|
159
|
+
if result["status"] == "failed":
|
|
160
|
+
error = result["error"]
|
|
161
|
+
error["stage"] # PipelineStage:失败发生在哪个阶段
|
|
162
|
+
error["error_type"] # 异常类名,如 "MinerUQuotaError"
|
|
163
|
+
error["message"] # 脱敏后的可读摘要
|
|
164
|
+
error["cause_type"] # 异常链首因的类名,无异常链时为 None
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
需要"按异常类型分支"时,显式开启异常模式:
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
from mineru_parser import MinerUParserError, parse
|
|
171
|
+
|
|
172
|
+
try:
|
|
173
|
+
result = parse("https://cdn-mineru.openxlab.org.cn/demo/example.pdf", timeout=600, raise_on_error=True)
|
|
174
|
+
except MinerUParserError as exc:
|
|
175
|
+
# 已知失败:组件异常**原样抛出**(同一对象,类型与异常链不变)
|
|
176
|
+
print(type(exc).__name__, exc)
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
### 已知 vs 兜不住
|
|
180
|
+
|
|
181
|
+
| 类别 | 判定 | 两种模式下的表现 |
|
|
182
|
+
| --- | --- | --- |
|
|
183
|
+
| **已知** | `MinerUParserError` 家族的任意子类(`MinerUError` / `ResultDownloadError` / `ResultExtractionError` / `PipelineError`) | 信封模式:封装成 `ErrorMessage`;异常模式:**原样抛出**(仅超时被翻译为 `PipelineTimeoutError`) |
|
|
184
|
+
| **兜不住** | 未知异常类型、`MemoryError`、`KeyboardInterrupt`、`SystemExit` | **两种模式都原样抛出**,不封装、不吞掉 |
|
|
185
|
+
|
|
186
|
+
判定只看**类型**,不看消息、不看内容。信封由库自行构造,可序列化、可落盘、可上报:它只带类名与脱敏摘要,**不携带原始异常对象**,也不携带文档正文、凭据或签名 URL。
|
|
187
|
+
|
|
188
|
+
> **组件异常类不对外公开**,调用方无法按具体类型导入它们(`MinerUQuotaError` 等)。异常模式下用库级根异常 `MinerUParserError` 捕获,再按 `type(exc).__name__` 分支;信封模式下直接读 `error["error_type"]`。两种模式的信息量一致。
|
|
189
|
+
|
|
190
|
+
### 两种模式的对照
|
|
191
|
+
|
|
192
|
+
| 情形 | 信封(默认) | 异常(`raise_on_error=True`) |
|
|
193
|
+
| --- | --- | --- |
|
|
194
|
+
| 提交失败(认证 / 额度 / 服务端错误) | `status="failed"`,`error.stage="submit"`,`error.error_type="MinerUQuotaError"` 等 | 原样抛出 `MinerUQuotaError` |
|
|
195
|
+
| 等待阶段超时 | `error.stage="wait"`,`error.error_type="PipelineTimeoutError"`,`error.cause_type="MinerUTimeoutError"` | `PipelineTimeoutError(stage="wait")`,`__cause__` 指向原始超时异常 |
|
|
196
|
+
| 下载被 SSRF 防护拒绝 | `error.stage="download"`,`error.error_type="ResultDownloadSecurityError"` | 原样抛出 `ResultDownloadSecurityError` |
|
|
197
|
+
| 结果包解压后超限 | `error.stage="extract"`,`error.error_type="ResultExtractionLimitError"` | 原样抛出 `ResultExtractionLimitError` |
|
|
198
|
+
| 参数非法(`timeout` 与 `deadline_epoch` 同时给出) | `error.stage="config"` | `PipelineConfigurationError(field="timeout")` |
|
|
199
|
+
| 未知异常 / `MemoryError` | **原样抛出** | **原样抛出** |
|
|
200
|
+
|
|
201
|
+
## 阶段归因:`PipelineStage`
|
|
202
|
+
|
|
203
|
+
失败发生在哪一段,以封闭枚举暴露——它是指标标签与 `on_error` 参数的唯一取值来源,低基数,可直接用于监控维度。
|
|
204
|
+
|
|
205
|
+
| 取值 | 含义 |
|
|
206
|
+
| --- | --- |
|
|
207
|
+
| `config` | 任何阶段开始前的参数校验(尚未进入任何执行阶段) |
|
|
208
|
+
| `submit` | 提交解析任务(`POST /extract/task`) |
|
|
209
|
+
| `wait` | 轮询等待任务终态 |
|
|
210
|
+
| `download` | 下载结果包 |
|
|
211
|
+
| `extract` | 解析结果包 |
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
import logging
|
|
215
|
+
|
|
216
|
+
from mineru_parser import PipelineStage, parse
|
|
217
|
+
|
|
218
|
+
logger = logging.getLogger(__name__)
|
|
219
|
+
|
|
220
|
+
def on_failure(stage: PipelineStage, exc: BaseException) -> None:
|
|
221
|
+
logger.error("parse failed at %s: %s", stage, type(exc).__name__)
|
|
222
|
+
|
|
223
|
+
result = parse(
|
|
224
|
+
"https://cdn-mineru.openxlab.org.cn/demo/example.pdf",
|
|
225
|
+
data_id="doc-42",
|
|
226
|
+
timeout=600, # 覆盖提交 / 等待 / 下载 / 解析四段
|
|
227
|
+
on_update=lambda status: print(status.state),
|
|
228
|
+
on_error=on_failure,
|
|
229
|
+
)
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
阶段信息同时进入日志(`mineru.pipeline.error`,含 `stage` 与 `error_type`)与指标(`mineru.pipeline.total{stage}`),不提供回调时也能读到。回调抛出的异常会被吞掉并记一条告警,不影响主流程。
|
|
233
|
+
|
|
234
|
+
## 端到端预算与超时
|
|
235
|
+
|
|
236
|
+
`timeout`(相对秒数)与 `deadline_epoch`(绝对 Unix 时间戳)提供**单一**端到端预算,覆盖全部四个阶段,而不是三段超时之和。
|
|
237
|
+
|
|
238
|
+
```python
|
|
239
|
+
result = parse(uri, timeout=600) # 相对预算
|
|
240
|
+
result = parse(uri, deadline_epoch=1789000000.0) # 绝对截止
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
预算在每个阶段**开始前**与**结束后**检查:
|
|
244
|
+
|
|
245
|
+
| 检查点 | 耗尽时的 `stage` |
|
|
246
|
+
| --- | --- |
|
|
247
|
+
| 提交前 | `submit` |
|
|
248
|
+
| 等待前 | `wait` |
|
|
249
|
+
| 下载前 | `download` |
|
|
250
|
+
| 解析前 | `extract` |
|
|
251
|
+
| 解析后 | `extract` |
|
|
252
|
+
|
|
253
|
+
等待阶段把**剩余预算**下传给客户端(`wait(timeout=remaining)`),客户端会在每次查询前与每次 sleep 后自行检查。
|
|
254
|
+
|
|
255
|
+
**超时的统一语义**:`parse()` 抛出的超时只有一种类型——`PipelineTimeoutError`,携带 `stage`(耗尽时所处阶段)、`task_id`,以及等待阶段超时才有的 `last_status`(最后一次已知任务状态);被翻译的原始超时异常保留在 `__cause__` 上。
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
from mineru_parser import MinerUParserError, PipelineTimeoutError, parse
|
|
259
|
+
|
|
260
|
+
try:
|
|
261
|
+
result = parse(uri, timeout=600, raise_on_error=True)
|
|
262
|
+
except PipelineTimeoutError as exc:
|
|
263
|
+
print(exc.stage, exc.task_id) # 例如 "wait"、任务 id
|
|
264
|
+
except MinerUParserError as exc:
|
|
265
|
+
print(type(exc).__name__) # 其余已知失败
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
> **⚠️ 已知限制**:除等待阶段外,管道**无法中断**正在进行的下载——若下载阶段越过端到端预算,只能在阶段结束后检出并失败(`stage="download"`)。提取阶段同理:它是纯 CPU / 内存操作,预算只能在提取**结束后**检出,此时结果会被丢弃。
|
|
269
|
+
|
|
270
|
+
## 内存预算与并发度
|
|
271
|
+
|
|
272
|
+
一次 `parse()` 的峰值内存由**压缩侧**(下载的 ZIP 缓冲,默认 50 MiB)与**解压侧**(提取限额,默认 160 + 32 + 40 + 8 MiB)共同决定:
|
|
273
|
+
|
|
274
|
+
$$M_{\text{peak}} \approx 50 + 160 + 32 \times 2 + 40 + 8 = 290\,\text{MiB}$$
|
|
275
|
+
|
|
276
|
+
部署侧应满足 $W \times M_{\text{peak}} \le M_{\text{container}}$($W$ 为并发数):
|
|
277
|
+
|
|
278
|
+
| 形态 | 校验 |
|
|
279
|
+
| --- | --- |
|
|
280
|
+
| $W = 3$、容器 1 GiB | $3 \times 290 = 870\,\text{MiB} \le 1024\,\text{MiB}$ ✓ 默认档位可用 |
|
|
281
|
+
| $W = 4$、容器 1 GiB | $4 \times 290 = 1160\,\text{MiB} > 1024\,\text{MiB}$ ✗ 默认档位**超限** |
|
|
282
|
+
|
|
283
|
+
并发 4 时需要把解压侧上限压到 $X \le 126\,\text{MiB}$:
|
|
284
|
+
|
|
285
|
+
```python
|
|
286
|
+
from mineru_parser import ExtractionLimits, parse
|
|
287
|
+
|
|
288
|
+
result = parse(
|
|
289
|
+
uri,
|
|
290
|
+
limits=ExtractionLimits(max_total_image_bytes=126 * 1024 * 1024),
|
|
291
|
+
)
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
> 压缩侧与解压侧**不是同一量纲**:下载上限约束**压缩后**字节,提取限额约束**解压后**字节。因此"下载成功但提取被限额拒绝"是正常可达状态,不是错误。
|
|
295
|
+
>
|
|
296
|
+
> 峰值数字是**推导值**,不是实测值;真实并发峰值请在目标环境标定(推导口径见 SPEC-004 §3.5)。
|
|
297
|
+
|
|
298
|
+
## 错误模型
|
|
299
|
+
|
|
300
|
+
四条失败路径互不继承;需要"一网打尽"时用库级根异常 `MinerUParserError`。
|
|
301
|
+
|
|
302
|
+
其中**管道家族**(`PipelineError` 及其两个子类)可以从顶层导入并单独捕获;**组件三族**是内部实现,调用方无法按具体类型导入,只通过 `MinerUParserError` 捕获后按类名分支(信封模式下读 `error["error_type"]`)。
|
|
303
|
+
|
|
304
|
+
```mermaid
|
|
305
|
+
flowchart TB
|
|
306
|
+
ROOT["MinerUParserError<br/>(库级根异常)"]:::root
|
|
307
|
+
CLIENT["MinerUError"]:::c1
|
|
308
|
+
DLERR["ResultDownloadError"]:::c2
|
|
309
|
+
EXERR["ResultExtractionError"]:::c3
|
|
310
|
+
PIPE["PipelineError"]:::c4
|
|
311
|
+
|
|
312
|
+
ROOT --> CLIENT
|
|
313
|
+
ROOT --> DLERR
|
|
314
|
+
ROOT --> EXERR
|
|
315
|
+
ROOT --> PIPE
|
|
316
|
+
|
|
317
|
+
CLIENT --> EXEC["MinerUExecutionError<br/>(终态失败)"]:::leaf
|
|
318
|
+
EXEC --> A1["MinerUAuthenticationError<br/>MinerUQuotaError<br/>MinerUConfigurationError"]:::leaf
|
|
319
|
+
CLIENT --> A2["MinerUTransientError<br/>MinerUTimeoutError"]:::leaf
|
|
320
|
+
|
|
321
|
+
DLERR --> B1["ResultDownloadSecurityError<br/>ResultDownloadTooLargeError<br/>ResultDownloadContentTypeError<br/>ResultDownloadHTTPError<br/>ResultDownloadTimeoutError / TransientError<br/>ResultDownloadConfigurationError"]:::leaf
|
|
322
|
+
|
|
323
|
+
EXERR --> C1["ResultExtractionConfigurationError<br/>ResultExtractionArchiveError<br/>ResultExtractionManifestError<br/>ResultExtractionValidationError<br/>ResultExtractionLimitError"]:::leaf
|
|
324
|
+
|
|
325
|
+
PIPE --> D1["PipelineConfigurationError<br/>(参数 / 配置非法)"]:::leaf
|
|
326
|
+
PIPE --> D2["PipelineTimeoutError<br/>(端到端预算耗尽)"]:::leaf
|
|
327
|
+
|
|
328
|
+
classDef root fill:#eceff1,stroke:#546e7a,stroke-width:2px,color:#263238;
|
|
329
|
+
classDef c1 fill:#e3f2fd,stroke:#1565c0,stroke-width:2px,color:#0d47a1;
|
|
330
|
+
classDef c2 fill:#fff3e0,stroke:#e65100,stroke-width:2px,color:#bf360c;
|
|
331
|
+
classDef c3 fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px,color:#1b5e20;
|
|
332
|
+
classDef c4 fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px,color:#4a148c;
|
|
333
|
+
classDef leaf fill:#ffffff,stroke:#90a4ae,stroke-width:1px,color:#37474f;
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
- **client**:`MinerUTransientError` 表示请求确定未发出或只读查询失败,可以安全重试;`MinerUExecutionError` 表示结果未知、请求被拒绝或服务端明确失败。认证、额度、配置错误均继承 `MinerUExecutionError`。
|
|
337
|
+
- **downloader**:安全、超限、内容类型、HTTP 状态、超时与重试耗尽各有独立异常类型,公共消息不泄漏原始签名 URL。
|
|
338
|
+
- **extractor**:按 `reason` / `resource` / `category` 字段分支,公共消息只使用英文固定文案,不携带 manifest 正文或图片字节。
|
|
339
|
+
- **pipeline**:只有两类自身失败——`PipelineConfigurationError`(参数非法,带 `field`)与 `PipelineTimeoutError`(预算耗尽,带 `stage`)。
|
|
340
|
+
|
|
341
|
+
## 可观测性
|
|
342
|
+
|
|
343
|
+
`parse()` 接受 `on_metric` 指标回调与 `correlation_id` 关联标识,统一走 `mineru_parser` logger。指标回调失败只记一条告警日志,不影响主流程。
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
def on_metric(name: str, value: float, tags: dict[str, str]) -> None:
|
|
347
|
+
print(name, value, tags)
|
|
348
|
+
|
|
349
|
+
result = parse(uri, on_metric=on_metric, correlation_id="doc-42")
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
管道自身的指标:
|
|
353
|
+
|
|
354
|
+
| 指标 | 标签 |
|
|
355
|
+
| --- | --- |
|
|
356
|
+
| `mineru.pipeline.total` | `outcome`、`stage` |
|
|
357
|
+
| `mineru.pipeline.duration_ms` | `outcome` |
|
|
358
|
+
| `mineru.pipeline.stage.duration_ms` | `stage`、`outcome` |
|
|
359
|
+
| `mineru.pipeline.pages` / `.images` / `.image_bytes` | `outcome` |
|
|
360
|
+
|
|
361
|
+
`outcome` 是封闭枚举:`success` / `config_error` / `timeout` / `stage_error` / `unexpected`。指标标签只允许封闭枚举——**不得**包含 URI、`task_id`、`data_id`、`zip_url`、路径或 `correlation_id`(这些只出现在日志字段里)。
|
|
362
|
+
|
|
363
|
+
日志名:`mineru.pipeline`(成功)、`mineru.pipeline.stage`(阶段切换)、`mineru.pipeline.error`(失败,含 `stage` 与 `error_type`)、`mineru.pipeline.on_error.failed`(回调异常)。三个组件的既有日志名保持不变。
|
|
364
|
+
|
|
365
|
+
## 风险与已知限制
|
|
366
|
+
|
|
367
|
+
- **`data_id` 不是幂等键**:它是**对账标识**,服务端不提供"按 `data_id` 复用任务"的能力。同一 URI 调用两次 `parse()` 会创建**两个任务、计费两次**。需要去重请在业务层自行处理。
|
|
368
|
+
- **默认没有端到端超时**:不传 `timeout` / `deadline_epoch` 时等待阶段无上界,长文档解析可能耗时数分钟。生产环境建议显式给出预算。
|
|
369
|
+
- **管道无状态,崩溃后任务不可恢复**:等待途中进程崩溃会留下一个无人认领的服务端任务(可能计费)。库不提供凭 `task_id` 恢复等待的入口,请按 `data_id` 自行对账。
|
|
370
|
+
- **每次调用自建 Session**:`parse()` 每次调用都新建 client / downloader 及其连接池,跨调用无连接池复用。高频小任务场景下 TLS 握手开销显著。
|
|
371
|
+
- **`zip_url` 是敏感值**:库不记录它,但也不脱敏;落盘或打日志前请自行评估。
|
|
372
|
+
- **峰值内存是推导值**:290 MiB 来自组件默认值的推导,真实并发峰值需在目标环境标定。
|
|
373
|
+
|
|
374
|
+
## 内部实现(不对外)
|
|
375
|
+
|
|
376
|
+
以下三个组件是 `parse()` 的内部实现,**不提供公开导入路径**,也不遵循 SemVer——它们的行为契约由各自 SPEC 约束(SPEC-001 / 002 / 003),但对调用方而言,`parse()` 就是全部。
|
|
377
|
+
|
|
378
|
+
```mermaid
|
|
379
|
+
flowchart LR
|
|
380
|
+
classDef io fill:#fff3e0,stroke:#e65100,stroke-width:2px,color:#bf360c;
|
|
381
|
+
classDef mod fill:#e3f2fd,stroke:#1565c0,stroke-width:2px,color:#0d47a1;
|
|
382
|
+
classDef out fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px,color:#1b5e20;
|
|
383
|
+
classDef srv fill:#fce4ec,stroke:#ad1457,stroke-width:2px,color:#880e4f;
|
|
384
|
+
|
|
385
|
+
URI["PDF URI"]:::io
|
|
386
|
+
CLIENT["client<br/>MinerUApiClient · AsyncResult"]:::mod
|
|
387
|
+
API["MinerU API Server"]:::srv
|
|
388
|
+
ZIPURL["zip_url"]:::io
|
|
389
|
+
DL["downloader<br/>ResultDownloader"]:::mod
|
|
390
|
+
MEM["io.BytesIO<br/>ZIP 内存流"]:::io
|
|
391
|
+
EX["extractor<br/>extract_result"]:::mod
|
|
392
|
+
RESULT["ParseResult<br/>{status, data, task_id, data_id, zip_url, error}"]:::out
|
|
393
|
+
|
|
394
|
+
URI -->|"① 提交"| CLIENT
|
|
395
|
+
CLIENT -->|"POST /extract/task"| API
|
|
396
|
+
API -->|"task_id"| CLIENT
|
|
397
|
+
CLIENT -->|"GET /extract/task/{id}"| API
|
|
398
|
+
API -->|"full_zip_url"| ZIPURL
|
|
399
|
+
ZIPURL -->|"② 下载"| DL
|
|
400
|
+
DL --> MEM
|
|
401
|
+
MEM -->|"③ 解析"| EX
|
|
402
|
+
EX --> RESULT
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
- **client** — 提交、轮询、终态解释。自动处理指数退避、随机抖动、服务端 `Retry-After` 与连续错误上限;错误按认证 / 额度 / 瞬时 / 执行 / 超时分类。
|
|
406
|
+
- **downloader** — 流式下载。默认限制单次响应体 50 MiB、总耗时 300 秒,拒绝非公网目标与 HTTPS 降级重定向,日志中的 URL 剔除 query 与 fragment。
|
|
407
|
+
- **extractor** — 解析结果包。只在 ZIP 顶层定位唯一的 `*_content_list_v2.json`,用 `DocumentContentSchema` 做 Schema 校验,收集顶层 `images/` 下的图片;解压后资源受 `ExtractionLimits` 限制(manifest 16 MiB、单图 32 MiB、图片总量 160 MiB、图片 10,000 张、ZIP 成员 100,000 个),不落盘、不解码图片。
|
|
408
|
+
|
|
409
|
+
三者的组合效应由管道保证:阶段顺序由库固定,任一阶段失败即中止后续阶段(失败于 submit / wait 时不发起下载,失败于 download 时不调用提取器);中间 ZIP 流与自建组件由管道在返回前按"流 → downloader → client"的顺序关闭,失败路径同样关闭。
|