sdpy-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kit/__init__.py +68 -0
- kit/ai/__init__.py +47 -0
- kit/ai/base_service.py +94 -0
- kit/ai/chat_service.py +110 -0
- kit/ai/embedding_service.py +133 -0
- kit/ai/exceptions.py +57 -0
- kit/ai/failover.py +244 -0
- kit/ai/image_service.py +99 -0
- kit/ai/model_builder.py +119 -0
- kit/ai/model_client.py +214 -0
- kit/ai/pool.py +411 -0
- kit/ai/profiles.py +373 -0
- kit/ai/rerank_service.py +93 -0
- kit/ai/runner.py +580 -0
- kit/ai/types.py +28 -0
- kit/common_kit.py +221 -0
- kit/config/__init__.py +44 -0
- kit/config/kit_config.py +147 -0
- kit/config/log_config.py +134 -0
- kit/db/__init__.py +70 -0
- kit/db/base_db_kit.py +406 -0
- kit/db/crud.py +361 -0
- kit/db/engine.py +230 -0
- kit/db/exceptions.py +34 -0
- kit/db/models/__init__.py +33 -0
- kit/db/models/base.py +56 -0
- kit/db/mysql_kit.py +89 -0
- kit/db/pg_kit.py +87 -0
- kit/db/session.py +85 -0
- kit/db/sqlite_kit.py +284 -0
- kit/db/transaction.py +109 -0
- kit/doc_kit.py +1502 -0
- kit/file_kit.py +429 -0
- kit/mcp_kit.py +547 -0
- kit/mineru_kit.py +918 -0
- kit/redis_kit.py +392 -0
- kit/zvec_kit.py +671 -0
- sdpy_kit-0.1.0.dist-info/METADATA +128 -0
- sdpy_kit-0.1.0.dist-info/RECORD +40 -0
- sdpy_kit-0.1.0.dist-info/WHEEL +4 -0
kit/mineru_kit.py
ADDED
|
@@ -0,0 +1,918 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Copyright (c) 2021-2026 Clark Chang. All Rights Reserved.
|
|
3
|
+
|
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
you may not use this file except in compliance with the License.
|
|
6
|
+
You may obtain a copy of the License at
|
|
7
|
+
|
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
|
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
See the License for the specific language governing permissions and
|
|
14
|
+
limitations under the License.
|
|
15
|
+
|
|
16
|
+
Project: sdpy
|
|
17
|
+
Author: Clark Chang
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
# date: 2026-08-19
|
|
21
|
+
"""MinerU 精准解析 API 工具类
|
|
22
|
+
|
|
23
|
+
封装 MinerU 官方"精准解析 API"(https://mineru.net/apiManage/docs),
|
|
24
|
+
将 PDF、图片、Office 文档、HTML 等解析为 Markdown/JSON 等结构化结果。
|
|
25
|
+
|
|
26
|
+
覆盖端点:
|
|
27
|
+
- POST /api/v4/extract/task 创建单文件解析任务(URL 方式)
|
|
28
|
+
- GET /api/v4/extract/task/{task_id} 查询单任务结果
|
|
29
|
+
- POST /api/v4/file-urls/batch 批量申请文件上传链接
|
|
30
|
+
- GET /api/v4/extract-results/batch/{batch_id} 查询批量任务结果
|
|
31
|
+
|
|
32
|
+
设计要点:
|
|
33
|
+
- 全部方法无内部可变状态,线程安全(与 DocKit 风格一致)。
|
|
34
|
+
- 不依赖实例属性者用 staticmethod,需要访问类常量者用 classmethod。
|
|
35
|
+
- 低层方法(create_task / get_task_result / create_batch / upload_files /
|
|
36
|
+
get_batch_results)直接抛异常,便于精确定位。
|
|
37
|
+
- 高层方法(parse_url / parse_files / wait_for_task / wait_for_batch /
|
|
38
|
+
download_zip)封装"提交 → 轮询 → 完成"完整流程。
|
|
39
|
+
- 可选参数均为 None 时不发送,交由服务端使用默认值。
|
|
40
|
+
- Token 通过 Settings 的 MINERU_API_KEY 注入(.env 配置)。
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
import time
|
|
44
|
+
from dataclasses import dataclass
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
from typing import Any
|
|
47
|
+
|
|
48
|
+
import requests
|
|
49
|
+
|
|
50
|
+
from kit.config.kit_config import get_settings
|
|
51
|
+
from kit.config.log_config import get_default_logger
|
|
52
|
+
|
|
53
|
+
logger = get_default_logger(__name__)
|
|
54
|
+
|
|
55
|
+
# 官方限制:单文件大小上限 200MB、单次批量申请上传链接上限 50 个
|
|
56
|
+
_MAX_FILE_SIZE_BYTES: int = 200 * 1024 * 1024
|
|
57
|
+
_MAX_BATCH_FILES: int = 50
|
|
58
|
+
|
|
59
|
+
# 支持的文件扩展名(官方文档:PDF / 图片 / Office / HTML)
|
|
60
|
+
_SUPPORTED_EXTENSIONS: frozenset[str] = frozenset(
|
|
61
|
+
{
|
|
62
|
+
# PDF
|
|
63
|
+
".pdf",
|
|
64
|
+
# 图片
|
|
65
|
+
".png",
|
|
66
|
+
".jpg",
|
|
67
|
+
".jpeg",
|
|
68
|
+
# ".jp2",
|
|
69
|
+
# ".webp",
|
|
70
|
+
# ".gif",
|
|
71
|
+
# ".bmp",
|
|
72
|
+
# Word
|
|
73
|
+
".doc",
|
|
74
|
+
".docx",
|
|
75
|
+
# PowerPoint
|
|
76
|
+
".ppt",
|
|
77
|
+
".pptx",
|
|
78
|
+
# Excel
|
|
79
|
+
".xls",
|
|
80
|
+
".xlsx",
|
|
81
|
+
# HTML
|
|
82
|
+
".html",
|
|
83
|
+
}
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
# 任务终态
|
|
87
|
+
_STATE_DONE: str = "done"
|
|
88
|
+
_STATE_FAILED: str = "failed"
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class MinerUError(Exception):
|
|
92
|
+
"""MinerU API 调用异常(配置缺失 / 网络失败 / HTTP 或业务错误码)"""
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass(frozen=True, slots=True)
|
|
96
|
+
class TaskResult:
|
|
97
|
+
"""单任务查询结果
|
|
98
|
+
|
|
99
|
+
Attributes:
|
|
100
|
+
task_id: 任务 ID
|
|
101
|
+
state: 任务状态(done 完成 / pending 排队 / running 解析中 /
|
|
102
|
+
failed 失败 / converting 格式转换中)
|
|
103
|
+
full_zip_url: 解析结果压缩包地址,state=done 时有效
|
|
104
|
+
err_msg: 失败原因,state=failed 时有效
|
|
105
|
+
data_id: 业务数据 ID(请求时传入才会返回)
|
|
106
|
+
extracted_pages: 已解析页数,state=running 时有效
|
|
107
|
+
total_pages: 文档总页数,state=running 时有效
|
|
108
|
+
start_time: 解析开始时间,state=running 时有效
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
task_id: str
|
|
112
|
+
state: str
|
|
113
|
+
full_zip_url: str | None = None
|
|
114
|
+
err_msg: str | None = None
|
|
115
|
+
data_id: str | None = None
|
|
116
|
+
extracted_pages: int | None = None
|
|
117
|
+
total_pages: int | None = None
|
|
118
|
+
start_time: str | None = None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass(frozen=True, slots=True)
|
|
122
|
+
class BatchTaskResult(TaskResult):
|
|
123
|
+
"""批量任务结果项,在 TaskResult 基础上附加文件名"""
|
|
124
|
+
|
|
125
|
+
file_name: str | None = None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@dataclass(frozen=True, slots=True)
|
|
129
|
+
class BatchFile:
|
|
130
|
+
"""批量解析请求中的单个文件描述
|
|
131
|
+
|
|
132
|
+
Attributes:
|
|
133
|
+
name: 文件名(强烈建议带正确扩展名,服务端据此识别类型)
|
|
134
|
+
data_id: 业务数据 ID(可选,用于唯一标识业务数据)
|
|
135
|
+
is_ocr: 是否启用 OCR(可选,仅 pipeline / vlm 模型有效)
|
|
136
|
+
page_ranges: 页码范围(可选),如 "2,4-6"、"2--2"
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
name: str
|
|
140
|
+
data_id: str | None = None
|
|
141
|
+
is_ocr: bool | None = None
|
|
142
|
+
page_ranges: str | None = None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass(frozen=True, slots=True)
|
|
146
|
+
class BatchUploadInfo:
|
|
147
|
+
"""批量申请上传链接结果
|
|
148
|
+
|
|
149
|
+
Attributes:
|
|
150
|
+
batch_id: 批次 ID,用于查询批量任务结果
|
|
151
|
+
file_urls: 预签名上传 URL 列表(有效期 24 小时)
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
batch_id: str
|
|
155
|
+
file_urls: list[str]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class MinerUKit:
|
|
159
|
+
"""MinerU 精准解析 API 工具类
|
|
160
|
+
|
|
161
|
+
仅实现官方"精准解析 API"(需 Token),不支持 Agent 轻量解析 API。
|
|
162
|
+
|
|
163
|
+
使用前准备:在 `.env` 中配置::
|
|
164
|
+
|
|
165
|
+
MINERU_API_KEY=你的token # 在 MinerU 官网 API 管理页面自建
|
|
166
|
+
|
|
167
|
+
使用方式::
|
|
168
|
+
|
|
169
|
+
# 1. 解析一个公网 URL(提交 + 轮询一步到位)
|
|
170
|
+
result = MinerUKit.parse_url(
|
|
171
|
+
"https://cdn-mineru.openxlab.org.cn/demo/example.pdf",
|
|
172
|
+
model_version="vlm",
|
|
173
|
+
)
|
|
174
|
+
if result.state == "done":
|
|
175
|
+
zip_path = MinerUKit.download_zip(result.full_zip_url, "./output")
|
|
176
|
+
|
|
177
|
+
# 2. 解析本地文件(批量上传 + 轮询一步到位)
|
|
178
|
+
results = MinerUKit.parse_files(["report1.pdf", "report2.docx"])
|
|
179
|
+
|
|
180
|
+
# 3. 低层分步调用
|
|
181
|
+
task_id = MinerUKit.create_task(url, model_version="vlm")
|
|
182
|
+
result = MinerUKit.wait_for_task(task_id)
|
|
183
|
+
|
|
184
|
+
注意:
|
|
185
|
+
- 单文件 ≤ 200MB;单次批量 ≤ 50 个文件(官方限制,本地强制校验)。
|
|
186
|
+
- github、aws 等国外 URL 可能因网络限制请求超时(官方说明)。
|
|
187
|
+
"""
|
|
188
|
+
|
|
189
|
+
SUPPORTED_EXTENSIONS: frozenset[str] = _SUPPORTED_EXTENSIONS
|
|
190
|
+
MAX_FILE_SIZE_BYTES: int = _MAX_FILE_SIZE_BYTES
|
|
191
|
+
MAX_BATCH_FILES: int = _MAX_BATCH_FILES
|
|
192
|
+
|
|
193
|
+
# ------------------------------------------------------------------
|
|
194
|
+
# 内部工具
|
|
195
|
+
# ------------------------------------------------------------------
|
|
196
|
+
|
|
197
|
+
@staticmethod
|
|
198
|
+
def _token() -> str:
|
|
199
|
+
"""读取 API Key,未配置时抛出 MinerUError"""
|
|
200
|
+
token = get_settings().MINERU_API_KEY
|
|
201
|
+
if not token:
|
|
202
|
+
raise MinerUError(
|
|
203
|
+
"未配置 MINERU_API_KEY,请在 .env 中设置"
|
|
204
|
+
"(在 MinerU 官网 API 管理页面自建)"
|
|
205
|
+
)
|
|
206
|
+
return token
|
|
207
|
+
|
|
208
|
+
@classmethod
|
|
209
|
+
def _headers(cls) -> dict[str, str]:
|
|
210
|
+
"""构造带鉴权的请求头"""
|
|
211
|
+
return {
|
|
212
|
+
"Content-Type": "application/json",
|
|
213
|
+
"Authorization": f"Bearer {cls._token()}",
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
@staticmethod
|
|
217
|
+
def _base_url() -> str:
|
|
218
|
+
"""读取 API 基础地址(去除尾部斜杠)"""
|
|
219
|
+
return get_settings().MINERU_BASE_API_URL.rstrip("/")
|
|
220
|
+
|
|
221
|
+
@staticmethod
|
|
222
|
+
def _request(method: str, url: str, **kwargs: Any) -> dict[str, Any]:
|
|
223
|
+
"""发送 HTTP 请求并统一校验响应,成功返回 JSON 字典
|
|
224
|
+
|
|
225
|
+
Raises:
|
|
226
|
+
MinerUError: 网络失败 / HTTP 非 200 / 业务 code 非 0
|
|
227
|
+
"""
|
|
228
|
+
timeout = get_settings().MINERU_TIMEOUT
|
|
229
|
+
try:
|
|
230
|
+
resp = requests.request(method, url, timeout=timeout, **kwargs)
|
|
231
|
+
except requests.RequestException as e:
|
|
232
|
+
logger.error(f"MinerU API 请求失败: {method} {url}: {e}")
|
|
233
|
+
raise MinerUError(f"网络请求失败: {e}") from e
|
|
234
|
+
|
|
235
|
+
if resp.status_code != 200:
|
|
236
|
+
logger.error(
|
|
237
|
+
f"MinerU API HTTP 错误: {method} {url} -> "
|
|
238
|
+
f"{resp.status_code}: {resp.text[:200]}"
|
|
239
|
+
)
|
|
240
|
+
raise MinerUError(f"HTTP {resp.status_code}: {resp.text[:200]}")
|
|
241
|
+
|
|
242
|
+
payload: dict[str, Any] = resp.json()
|
|
243
|
+
code = payload.get("code")
|
|
244
|
+
if code != 0:
|
|
245
|
+
msg = payload.get("msg", "未知错误")
|
|
246
|
+
trace_id = payload.get("trace_id", "")
|
|
247
|
+
logger.error(
|
|
248
|
+
f"MinerU API 业务错误: code={code}, msg={msg}, trace_id={trace_id}"
|
|
249
|
+
)
|
|
250
|
+
raise MinerUError(f"API 错误 code={code}: {msg}(trace_id={trace_id})")
|
|
251
|
+
return payload
|
|
252
|
+
|
|
253
|
+
@staticmethod
|
|
254
|
+
def _build_payload(params: dict[str, Any]) -> dict[str, Any]:
|
|
255
|
+
"""过滤值为 None 的参数,仅保留显式传入项"""
|
|
256
|
+
return {k: v for k, v in params.items() if v is not None}
|
|
257
|
+
|
|
258
|
+
@classmethod
|
|
259
|
+
def _check_local_file(cls, file_path: str | Path) -> Path:
|
|
260
|
+
"""校验本地待上传文件(存在 / 扩展名 / 大小),通过返回绝对路径
|
|
261
|
+
|
|
262
|
+
Raises:
|
|
263
|
+
FileNotFoundError: 文件不存在或不是常规文件
|
|
264
|
+
ValueError: 扩展名不受支持或超过 200MB 大小限制
|
|
265
|
+
"""
|
|
266
|
+
path = Path(file_path).resolve()
|
|
267
|
+
if not path.is_file():
|
|
268
|
+
raise FileNotFoundError(f"文件不存在: {path}")
|
|
269
|
+
ext = path.suffix.lower()
|
|
270
|
+
if ext not in cls.SUPPORTED_EXTENSIONS:
|
|
271
|
+
raise ValueError(
|
|
272
|
+
f"不支持的文件格式: {ext or '<无扩展名>'},"
|
|
273
|
+
f"支持的格式: {', '.join(sorted(cls.SUPPORTED_EXTENSIONS))}"
|
|
274
|
+
)
|
|
275
|
+
if path.stat().st_size > cls.MAX_FILE_SIZE_BYTES:
|
|
276
|
+
size_mb = path.stat().st_size / 1024 / 1024
|
|
277
|
+
raise ValueError(
|
|
278
|
+
f"文件大小 {size_mb:.1f}MB 超过官方限制 200MB: {path.name}"
|
|
279
|
+
)
|
|
280
|
+
return path
|
|
281
|
+
|
|
282
|
+
# ------------------------------------------------------------------
|
|
283
|
+
# 低层 API(一端点一方法)
|
|
284
|
+
# ------------------------------------------------------------------
|
|
285
|
+
|
|
286
|
+
@classmethod
|
|
287
|
+
def create_task(
|
|
288
|
+
cls,
|
|
289
|
+
url: str,
|
|
290
|
+
*,
|
|
291
|
+
model_version: str | None = None,
|
|
292
|
+
is_ocr: bool | None = None,
|
|
293
|
+
enable_formula: bool | None = None,
|
|
294
|
+
enable_table: bool | None = None,
|
|
295
|
+
language: str | None = None,
|
|
296
|
+
data_id: str | None = None,
|
|
297
|
+
callback: str | None = None,
|
|
298
|
+
seed: str | None = None,
|
|
299
|
+
extra_formats: list[str] | None = None,
|
|
300
|
+
page_ranges: str | None = None,
|
|
301
|
+
no_cache: bool | None = None,
|
|
302
|
+
cache_tolerance: int | None = None,
|
|
303
|
+
) -> str:
|
|
304
|
+
"""创建单文件解析任务(URL 方式)
|
|
305
|
+
|
|
306
|
+
对应官方端点 ``POST /api/v4/extract/task``。该接口不支持文件直接上传,
|
|
307
|
+
仅接收可公网访问的文件 URL(github、aws 等国外 URL 可能超时)。
|
|
308
|
+
|
|
309
|
+
Args:
|
|
310
|
+
url: 文件 URL(支持 pdf/doc/docx/ppt/pptx/xls/xlsx/图片/html)
|
|
311
|
+
model_version: 模型版本,可选 ``pipeline``(官方默认)/ ``vlm``(推荐)/
|
|
312
|
+
``MinerU-HTML``(解析 HTML 文件时必须指定)
|
|
313
|
+
is_ocr: 是否启用 OCR,默认服务端 false,仅 pipeline / vlm 有效
|
|
314
|
+
enable_formula: 是否开启公式识别,默认服务端 true,仅 pipeline / vlm 有效
|
|
315
|
+
enable_table: 是否开启表格识别,默认服务端 true,仅 pipeline / vlm 有效
|
|
316
|
+
language: 文档语言(默认 ``ch``),仅 pipeline / vlm 有效
|
|
317
|
+
data_id: 业务数据 ID(字母/数字/下划线/短划线/句号,≤128 字符)
|
|
318
|
+
callback: 解析结果回调通知 URL(POST/UTF-8/JSON);为空需轮询结果
|
|
319
|
+
seed: 回调签名随机字符串(使用 callback 时必须提供,≤64 字符)
|
|
320
|
+
extra_formats: 附加导出格式,仅支持 ``docx`` / ``html`` / ``latex``
|
|
321
|
+
中一个或多个(markdown、json 默认导出无须设置)
|
|
322
|
+
page_ranges: 页码范围,如 ``"2,4-6"``(第2页及第4~6页)、``"2--2"``
|
|
323
|
+
(第2页至倒数第二页)
|
|
324
|
+
no_cache: 是否绕过缓存,默认服务端 false
|
|
325
|
+
cache_tolerance: 缓存容忍时间(秒),默认服务端 900
|
|
326
|
+
|
|
327
|
+
Returns:
|
|
328
|
+
str: 任务 ID(task_id),用于查询任务结果
|
|
329
|
+
|
|
330
|
+
Raises:
|
|
331
|
+
MinerUError: 未配置 Token / 网络失败 / HTTP 或业务错误码
|
|
332
|
+
|
|
333
|
+
Example:
|
|
334
|
+
>>> task_id = MinerUKit.create_task(
|
|
335
|
+
... "https://cdn-mineru.openxlab.org.cn/demo/example.pdf",
|
|
336
|
+
... model_version="vlm",
|
|
337
|
+
... )
|
|
338
|
+
"""
|
|
339
|
+
payload = cls._build_payload(
|
|
340
|
+
{
|
|
341
|
+
"url": url,
|
|
342
|
+
"model_version": model_version,
|
|
343
|
+
"is_ocr": is_ocr,
|
|
344
|
+
"enable_formula": enable_formula,
|
|
345
|
+
"enable_table": enable_table,
|
|
346
|
+
"language": language,
|
|
347
|
+
"data_id": data_id,
|
|
348
|
+
"callback": callback,
|
|
349
|
+
"seed": seed,
|
|
350
|
+
"extra_formats": extra_formats,
|
|
351
|
+
"page_ranges": page_ranges,
|
|
352
|
+
"no_cache": no_cache,
|
|
353
|
+
"cache_tolerance": cache_tolerance,
|
|
354
|
+
}
|
|
355
|
+
)
|
|
356
|
+
logger.info(f"提交 MinerU 解析任务: {url}")
|
|
357
|
+
resp = cls._request(
|
|
358
|
+
"POST",
|
|
359
|
+
f"{cls._base_url()}/api/v4/extract/task",
|
|
360
|
+
headers=cls._headers(),
|
|
361
|
+
json=payload,
|
|
362
|
+
)
|
|
363
|
+
task_id: str = resp["data"]["task_id"]
|
|
364
|
+
logger.info(f"MinerU 任务已创建: task_id={task_id}")
|
|
365
|
+
return task_id
|
|
366
|
+
|
|
367
|
+
@classmethod
|
|
368
|
+
def get_task_result(cls, task_id: str) -> TaskResult:
|
|
369
|
+
"""查询单任务结果
|
|
370
|
+
|
|
371
|
+
对应官方端点 ``GET /api/v4/extract/task/{task_id}``。
|
|
372
|
+
|
|
373
|
+
Args:
|
|
374
|
+
task_id: 任务 ID(由 create_task / create_batch 返回)
|
|
375
|
+
|
|
376
|
+
Returns:
|
|
377
|
+
TaskResult: 任务状态与结果(state=done 时含 full_zip_url)
|
|
378
|
+
|
|
379
|
+
Raises:
|
|
380
|
+
MinerUError: 未配置 Token / 网络失败 / HTTP 或业务错误码
|
|
381
|
+
|
|
382
|
+
Example:
|
|
383
|
+
>>> result = MinerUKit.get_task_result(task_id)
|
|
384
|
+
>>> result.state
|
|
385
|
+
'done'
|
|
386
|
+
"""
|
|
387
|
+
resp = cls._request(
|
|
388
|
+
"GET",
|
|
389
|
+
f"{cls._base_url()}/api/v4/extract/task/{task_id}",
|
|
390
|
+
headers=cls._headers(),
|
|
391
|
+
)
|
|
392
|
+
data: dict[str, Any] = resp.get("data", {})
|
|
393
|
+
progress: dict[str, Any] = data.get("extract_progress") or {}
|
|
394
|
+
return TaskResult(
|
|
395
|
+
task_id=data.get("task_id", task_id),
|
|
396
|
+
state=data.get("state", ""),
|
|
397
|
+
full_zip_url=data.get("full_zip_url"),
|
|
398
|
+
err_msg=data.get("err_msg"),
|
|
399
|
+
data_id=data.get("data_id"),
|
|
400
|
+
extracted_pages=progress.get("extracted_pages"),
|
|
401
|
+
total_pages=progress.get("total_pages"),
|
|
402
|
+
start_time=progress.get("start_time"),
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
@classmethod
|
|
406
|
+
def create_batch(
|
|
407
|
+
cls,
|
|
408
|
+
files: list[BatchFile],
|
|
409
|
+
*,
|
|
410
|
+
model_version: str | None = None,
|
|
411
|
+
enable_formula: bool | None = None,
|
|
412
|
+
enable_table: bool | None = None,
|
|
413
|
+
language: str | None = None,
|
|
414
|
+
callback: str | None = None,
|
|
415
|
+
seed: str | None = None,
|
|
416
|
+
extra_formats: list[str] | None = None,
|
|
417
|
+
no_cache: bool | None = None,
|
|
418
|
+
cache_tolerance: int | None = None,
|
|
419
|
+
) -> BatchUploadInfo:
|
|
420
|
+
"""批量申请文件上传链接(本地文件解析第一步)
|
|
421
|
+
|
|
422
|
+
对应官方端点 ``POST /api/v4/file-urls/batch``。申请到的预签名 URL
|
|
423
|
+
有效期 24 小时;文件上传完成后系统**自动**提交解析任务,无须再调用
|
|
424
|
+
create_task。
|
|
425
|
+
|
|
426
|
+
Args:
|
|
427
|
+
files: 文件描述列表(≤50 个),每项含文件名与可选参数
|
|
428
|
+
model_version: 模型版本(同 create_task;HTML 文件须指定 MinerU-HTML)
|
|
429
|
+
enable_formula: 是否开启公式识别(同 create_task)
|
|
430
|
+
enable_table: 是否开启表格识别(同 create_task)
|
|
431
|
+
language: 文档语言(同 create_task)
|
|
432
|
+
callback: 解析结果回调通知 URL(同 create_task)
|
|
433
|
+
seed: 回调签名随机字符串(使用 callback 时必须提供)
|
|
434
|
+
extra_formats: 附加导出格式(同 create_task)
|
|
435
|
+
no_cache: 是否绕过缓存(同 create_task)
|
|
436
|
+
cache_tolerance: 缓存容忍时间(秒,同 create_task)
|
|
437
|
+
|
|
438
|
+
Returns:
|
|
439
|
+
BatchUploadInfo: 批次 ID 与预签名上传 URL 列表
|
|
440
|
+
|
|
441
|
+
Raises:
|
|
442
|
+
ValueError: 文件列表为空或超过 50 个,或扩展名不受支持
|
|
443
|
+
MinerUError: 未配置 Token / 网络失败 / HTTP 或业务错误码
|
|
444
|
+
|
|
445
|
+
Example:
|
|
446
|
+
>>> info = MinerUKit.create_batch(
|
|
447
|
+
... [BatchFile(name="demo.pdf")], model_version="vlm"
|
|
448
|
+
... )
|
|
449
|
+
>>> info.batch_id
|
|
450
|
+
"""
|
|
451
|
+
if not files:
|
|
452
|
+
raise ValueError("文件列表不能为空")
|
|
453
|
+
if len(files) > cls.MAX_BATCH_FILES:
|
|
454
|
+
raise ValueError(
|
|
455
|
+
f"单次批量不能超过 {cls.MAX_BATCH_FILES} 个文件,当前: {len(files)}"
|
|
456
|
+
)
|
|
457
|
+
for f in files:
|
|
458
|
+
ext = Path(f.name).suffix.lower()
|
|
459
|
+
if ext not in cls.SUPPORTED_EXTENSIONS:
|
|
460
|
+
raise ValueError(
|
|
461
|
+
f"不支持的文件格式: {f.name}(扩展名 {ext or '<无扩展名>'})"
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
file_dicts = [
|
|
465
|
+
cls._build_payload(
|
|
466
|
+
{
|
|
467
|
+
"name": f.name,
|
|
468
|
+
"data_id": f.data_id,
|
|
469
|
+
"is_ocr": f.is_ocr,
|
|
470
|
+
"page_ranges": f.page_ranges,
|
|
471
|
+
}
|
|
472
|
+
)
|
|
473
|
+
for f in files
|
|
474
|
+
]
|
|
475
|
+
payload = cls._build_payload(
|
|
476
|
+
{
|
|
477
|
+
"files": file_dicts,
|
|
478
|
+
"model_version": model_version,
|
|
479
|
+
"enable_formula": enable_formula,
|
|
480
|
+
"enable_table": enable_table,
|
|
481
|
+
"language": language,
|
|
482
|
+
"callback": callback,
|
|
483
|
+
"seed": seed,
|
|
484
|
+
"extra_formats": extra_formats,
|
|
485
|
+
"no_cache": no_cache,
|
|
486
|
+
"cache_tolerance": cache_tolerance,
|
|
487
|
+
}
|
|
488
|
+
)
|
|
489
|
+
logger.info(f"申请 MinerU 批量上传链接: {len(files)} 个文件")
|
|
490
|
+
resp = cls._request(
|
|
491
|
+
"POST",
|
|
492
|
+
f"{cls._base_url()}/api/v4/file-urls/batch",
|
|
493
|
+
headers=cls._headers(),
|
|
494
|
+
json=payload,
|
|
495
|
+
)
|
|
496
|
+
data: dict[str, Any] = resp["data"]
|
|
497
|
+
info = BatchUploadInfo(
|
|
498
|
+
batch_id=data["batch_id"],
|
|
499
|
+
file_urls=list(data.get("file_urls", [])),
|
|
500
|
+
)
|
|
501
|
+
logger.info(
|
|
502
|
+
f"MinerU 批量上传链接已获取: batch_id={info.batch_id},"
|
|
503
|
+
f"{len(info.file_urls)} 个链接"
|
|
504
|
+
)
|
|
505
|
+
return info
|
|
506
|
+
|
|
507
|
+
@classmethod
|
|
508
|
+
def upload_files(
|
|
509
|
+
cls, file_paths: list[str | Path], upload_urls: list[str]
|
|
510
|
+
) -> list[bool]:
|
|
511
|
+
"""将本地文件批量上传至预签名 URL(本地文件解析第二步)
|
|
512
|
+
|
|
513
|
+
对文件执行 PUT 上传。官方要求:上传时**不设置** Authorization 与
|
|
514
|
+
Content-Type 请求头,本方法已按此处理。逐个上传,单个失败记录日志
|
|
515
|
+
并继续,不影响其余文件。
|
|
516
|
+
|
|
517
|
+
Args:
|
|
518
|
+
file_paths: 本地文件路径列表(与 upload_urls 顺序一一对应)
|
|
519
|
+
upload_urls: create_batch 返回的预签名上传 URL 列表
|
|
520
|
+
|
|
521
|
+
Returns:
|
|
522
|
+
list[bool]: 每个文件的上传结果(True 成功 / False 失败)
|
|
523
|
+
|
|
524
|
+
Raises:
|
|
525
|
+
ValueError: 两个列表长度不一致
|
|
526
|
+
FileNotFoundError: 文件不存在(单个文件校验失败直接抛出)
|
|
527
|
+
MinerUError: 网络请求失败
|
|
528
|
+
|
|
529
|
+
Example:
|
|
530
|
+
>>> results = MinerUKit.upload_files(["demo.pdf"], info.file_urls)
|
|
531
|
+
"""
|
|
532
|
+
if len(file_paths) != len(upload_urls):
|
|
533
|
+
raise ValueError(
|
|
534
|
+
f"文件数 ({len(file_paths)}) 与上传链接数 ({len(upload_urls)}) 不一致"
|
|
535
|
+
)
|
|
536
|
+
timeout = get_settings().MINERU_TIMEOUT
|
|
537
|
+
results: list[bool] = []
|
|
538
|
+
for file_path, upload_url in zip(file_paths, upload_urls):
|
|
539
|
+
path = cls._check_local_file(file_path)
|
|
540
|
+
logger.info(f"上传文件到 MinerU: {path.name}({path.stat().st_size} 字节)")
|
|
541
|
+
try:
|
|
542
|
+
resp = requests.put(upload_url, data=path.read_bytes(), timeout=timeout)
|
|
543
|
+
except requests.RequestException as e:
|
|
544
|
+
logger.error(f"文件上传失败: {path.name}: {e}")
|
|
545
|
+
results.append(False)
|
|
546
|
+
continue
|
|
547
|
+
if resp.status_code == 200:
|
|
548
|
+
logger.info(f"✅ 文件上传成功: {path.name}")
|
|
549
|
+
results.append(True)
|
|
550
|
+
else:
|
|
551
|
+
logger.error(
|
|
552
|
+
f"文件上传失败: {path.name} -> HTTP {resp.status_code}: "
|
|
553
|
+
f"{resp.text[:200]}"
|
|
554
|
+
)
|
|
555
|
+
results.append(False)
|
|
556
|
+
return results
|
|
557
|
+
|
|
558
|
+
@classmethod
|
|
559
|
+
def get_batch_results(cls, batch_id: str) -> list[BatchTaskResult]:
|
|
560
|
+
"""查询批量任务结果
|
|
561
|
+
|
|
562
|
+
对应官方端点 ``GET /api/v4/extract-results/batch/{batch_id}``。
|
|
563
|
+
注意:查询过早时部分任务可能尚未创建(结果列表可能少于上传文件数),
|
|
564
|
+
等待全部完成请使用 wait_for_batch。
|
|
565
|
+
|
|
566
|
+
Args:
|
|
567
|
+
batch_id: 批次 ID(由 create_batch 返回)
|
|
568
|
+
|
|
569
|
+
Returns:
|
|
570
|
+
list[BatchTaskResult]: 批次内各文件的当前状态
|
|
571
|
+
|
|
572
|
+
Raises:
|
|
573
|
+
MinerUError: 未配置 Token / 网络失败 / HTTP 或业务错误码
|
|
574
|
+
|
|
575
|
+
Example:
|
|
576
|
+
>>> results = MinerUKit.get_batch_results(info.batch_id)
|
|
577
|
+
>>> [(r.file_name, r.state) for r in results]
|
|
578
|
+
"""
|
|
579
|
+
resp = cls._request(
|
|
580
|
+
"GET",
|
|
581
|
+
f"{cls._base_url()}/api/v4/extract-results/batch/{batch_id}",
|
|
582
|
+
headers=cls._headers(),
|
|
583
|
+
)
|
|
584
|
+
data: dict[str, Any] = resp.get("data", {})
|
|
585
|
+
raw_list: list[dict[str, Any]] = data.get("extract_result", [])
|
|
586
|
+
results: list[BatchTaskResult] = []
|
|
587
|
+
for item in raw_list:
|
|
588
|
+
progress: dict[str, Any] = item.get("extract_progress") or {}
|
|
589
|
+
results.append(
|
|
590
|
+
BatchTaskResult(
|
|
591
|
+
task_id=item.get("task_id", ""),
|
|
592
|
+
state=item.get("state", ""),
|
|
593
|
+
full_zip_url=item.get("full_zip_url"),
|
|
594
|
+
err_msg=item.get("err_msg"),
|
|
595
|
+
data_id=item.get("data_id"),
|
|
596
|
+
extracted_pages=progress.get("extracted_pages"),
|
|
597
|
+
total_pages=progress.get("total_pages"),
|
|
598
|
+
start_time=progress.get("start_time"),
|
|
599
|
+
file_name=item.get("file_name"),
|
|
600
|
+
)
|
|
601
|
+
)
|
|
602
|
+
return results
|
|
603
|
+
|
|
604
|
+
# ------------------------------------------------------------------
|
|
605
|
+
# 高层便利 API(提交 → 轮询 → 完成)
|
|
606
|
+
# ------------------------------------------------------------------
|
|
607
|
+
|
|
608
|
+
@classmethod
|
|
609
|
+
def wait_for_task(
|
|
610
|
+
cls,
|
|
611
|
+
task_id: str,
|
|
612
|
+
*,
|
|
613
|
+
poll_interval: float | None = None,
|
|
614
|
+
timeout: int | None = None,
|
|
615
|
+
) -> TaskResult:
|
|
616
|
+
"""轮询等待单任务到达终态(done / failed)
|
|
617
|
+
|
|
618
|
+
Args:
|
|
619
|
+
task_id: 任务 ID
|
|
620
|
+
poll_interval: 轮询间隔秒数,None 时使用 MINERU_POLL_INTERVAL
|
|
621
|
+
timeout: 总等待秒数,None 时使用 MINERU_MAX_WAIT
|
|
622
|
+
|
|
623
|
+
Returns:
|
|
624
|
+
TaskResult: 终态结果(state=done 或 failed)
|
|
625
|
+
|
|
626
|
+
Raises:
|
|
627
|
+
MinerUError: 超过总等待时间仍未到达终态
|
|
628
|
+
|
|
629
|
+
Example:
|
|
630
|
+
>>> result = MinerUKit.wait_for_task(task_id, timeout=300)
|
|
631
|
+
"""
|
|
632
|
+
settings = get_settings()
|
|
633
|
+
interval = (
|
|
634
|
+
poll_interval
|
|
635
|
+
if poll_interval is not None
|
|
636
|
+
else settings.MINERU_POLL_INTERVAL
|
|
637
|
+
)
|
|
638
|
+
max_wait = timeout if timeout is not None else settings.MINERU_MAX_WAIT
|
|
639
|
+
deadline = time.monotonic() + max_wait
|
|
640
|
+
while True:
|
|
641
|
+
result = cls.get_task_result(task_id)
|
|
642
|
+
if result.state == _STATE_DONE:
|
|
643
|
+
logger.info(f"✅ MinerU 任务完成: {task_id}")
|
|
644
|
+
return result
|
|
645
|
+
if result.state == _STATE_FAILED:
|
|
646
|
+
logger.error(f"❌ MinerU 任务失败: {task_id}: {result.err_msg}")
|
|
647
|
+
return result
|
|
648
|
+
if result.state == "running" and result.total_pages:
|
|
649
|
+
logger.info(
|
|
650
|
+
f"MinerU 解析中: {task_id} "
|
|
651
|
+
f"已解析 {result.extracted_pages or 0}/{result.total_pages} 页"
|
|
652
|
+
)
|
|
653
|
+
else:
|
|
654
|
+
logger.debug(f"MinerU 任务状态: {task_id} -> {result.state}")
|
|
655
|
+
if time.monotonic() >= deadline:
|
|
656
|
+
raise MinerUError(
|
|
657
|
+
f"等待任务 {task_id} 超时({max_wait} 秒),"
|
|
658
|
+
f"最后状态: {result.state}"
|
|
659
|
+
)
|
|
660
|
+
time.sleep(interval)
|
|
661
|
+
|
|
662
|
+
@classmethod
|
|
663
|
+
def wait_for_batch(
|
|
664
|
+
cls,
|
|
665
|
+
batch_id: str,
|
|
666
|
+
*,
|
|
667
|
+
expected_count: int | None = None,
|
|
668
|
+
poll_interval: float | None = None,
|
|
669
|
+
timeout: int | None = None,
|
|
670
|
+
) -> list[BatchTaskResult]:
|
|
671
|
+
"""轮询等待批量任务全部到达终态
|
|
672
|
+
|
|
673
|
+
Args:
|
|
674
|
+
batch_id: 批次 ID
|
|
675
|
+
expected_count: 期望的文件数量。官方在文件上传后才会自动创建任务,
|
|
676
|
+
查询过早时结果列表可能不完整;传入该值可确保全部任务出现且
|
|
677
|
+
到达终态后才返回(parse_files 会自动传入)
|
|
678
|
+
poll_interval: 轮询间隔秒数,None 时使用 MINERU_POLL_INTERVAL
|
|
679
|
+
timeout: 总等待秒数,None 时使用 MINERU_MAX_WAIT
|
|
680
|
+
|
|
681
|
+
Returns:
|
|
682
|
+
list[BatchTaskResult]: 批次内全部文件的终态结果
|
|
683
|
+
|
|
684
|
+
Raises:
|
|
685
|
+
MinerUError: 超过总等待时间仍未全部到达终态
|
|
686
|
+
|
|
687
|
+
Example:
|
|
688
|
+
>>> results = MinerUKit.wait_for_batch(info.batch_id, expected_count=2)
|
|
689
|
+
"""
|
|
690
|
+
settings = get_settings()
|
|
691
|
+
interval = (
|
|
692
|
+
poll_interval
|
|
693
|
+
if poll_interval is not None
|
|
694
|
+
else settings.MINERU_POLL_INTERVAL
|
|
695
|
+
)
|
|
696
|
+
max_wait = timeout if timeout is not None else settings.MINERU_MAX_WAIT
|
|
697
|
+
deadline = time.monotonic() + max_wait
|
|
698
|
+
while True:
|
|
699
|
+
results = cls.get_batch_results(batch_id)
|
|
700
|
+
all_terminal = bool(results) and all(
|
|
701
|
+
r.state in (_STATE_DONE, _STATE_FAILED) for r in results
|
|
702
|
+
)
|
|
703
|
+
count_ok = expected_count is None or len(results) >= expected_count
|
|
704
|
+
if all_terminal and count_ok:
|
|
705
|
+
done = sum(1 for r in results if r.state == _STATE_DONE)
|
|
706
|
+
failed = sum(1 for r in results if r.state == _STATE_FAILED)
|
|
707
|
+
logger.info(
|
|
708
|
+
f"✅ MinerU 批量任务完成: batch_id={batch_id},"
|
|
709
|
+
f"成功 {done} 个,失败 {failed} 个"
|
|
710
|
+
)
|
|
711
|
+
return results
|
|
712
|
+
if time.monotonic() >= deadline:
|
|
713
|
+
raise MinerUError(
|
|
714
|
+
f"等待批量任务 {batch_id} 超时({max_wait} 秒),"
|
|
715
|
+
f"当前可见 {len(results)} 个任务"
|
|
716
|
+
)
|
|
717
|
+
logger.debug(
|
|
718
|
+
f"MinerU 批量任务进行中: batch_id={batch_id},"
|
|
719
|
+
f"当前可见 {len(results)} 个任务"
|
|
720
|
+
)
|
|
721
|
+
time.sleep(interval)
|
|
722
|
+
|
|
723
|
+
@classmethod
|
|
724
|
+
def parse_url(
|
|
725
|
+
cls,
|
|
726
|
+
url: str,
|
|
727
|
+
*,
|
|
728
|
+
model_version: str | None = None,
|
|
729
|
+
is_ocr: bool | None = None,
|
|
730
|
+
enable_formula: bool | None = None,
|
|
731
|
+
enable_table: bool | None = None,
|
|
732
|
+
language: str | None = None,
|
|
733
|
+
data_id: str | None = None,
|
|
734
|
+
extra_formats: list[str] | None = None,
|
|
735
|
+
page_ranges: str | None = None,
|
|
736
|
+
no_cache: bool | None = None,
|
|
737
|
+
cache_tolerance: int | None = None,
|
|
738
|
+
poll_interval: float | None = None,
|
|
739
|
+
timeout: int | None = None,
|
|
740
|
+
) -> TaskResult:
|
|
741
|
+
"""解析公网 URL 文件(提交 + 轮询一步到位)
|
|
742
|
+
|
|
743
|
+
create_task 与 wait_for_task 的组合封装,参数含义见二者 docstring
|
|
744
|
+
(callback / seed 回调相关参数不参与本方法,如需回调请使用低层方法)。
|
|
745
|
+
|
|
746
|
+
Args:
|
|
747
|
+
url: 文件 URL
|
|
748
|
+
model_version: 模型版本(HTML 文件须指定 MinerU-HTML)
|
|
749
|
+
is_ocr: 是否启用 OCR
|
|
750
|
+
enable_formula: 是否开启公式识别
|
|
751
|
+
enable_table: 是否开启表格识别
|
|
752
|
+
language: 文档语言
|
|
753
|
+
data_id: 业务数据 ID
|
|
754
|
+
extra_formats: 附加导出格式(docx/html/latex)
|
|
755
|
+
page_ranges: 页码范围
|
|
756
|
+
no_cache: 是否绕过缓存
|
|
757
|
+
cache_tolerance: 缓存容忍时间(秒)
|
|
758
|
+
poll_interval: 轮询间隔秒数
|
|
759
|
+
timeout: 轮询总超时秒数
|
|
760
|
+
|
|
761
|
+
Returns:
|
|
762
|
+
TaskResult: 终态结果(state=done 时含 full_zip_url)
|
|
763
|
+
|
|
764
|
+
Raises:
|
|
765
|
+
MinerUError: 未配置 Token / 网络失败 / 轮询超时
|
|
766
|
+
|
|
767
|
+
Example:
|
|
768
|
+
>>> result = MinerUKit.parse_url(
|
|
769
|
+
... "https://cdn-mineru.openxlab.org.cn/demo/example.pdf",
|
|
770
|
+
... model_version="vlm",
|
|
771
|
+
... )
|
|
772
|
+
>>> print(result.state, result.full_zip_url)
|
|
773
|
+
"""
|
|
774
|
+
task_id = cls.create_task(
|
|
775
|
+
url,
|
|
776
|
+
model_version=model_version,
|
|
777
|
+
is_ocr=is_ocr,
|
|
778
|
+
enable_formula=enable_formula,
|
|
779
|
+
enable_table=enable_table,
|
|
780
|
+
language=language,
|
|
781
|
+
data_id=data_id,
|
|
782
|
+
extra_formats=extra_formats,
|
|
783
|
+
page_ranges=page_ranges,
|
|
784
|
+
no_cache=no_cache,
|
|
785
|
+
cache_tolerance=cache_tolerance,
|
|
786
|
+
)
|
|
787
|
+
return cls.wait_for_task(task_id, poll_interval=poll_interval, timeout=timeout)
|
|
788
|
+
|
|
789
|
+
@classmethod
|
|
790
|
+
def parse_files(
|
|
791
|
+
cls,
|
|
792
|
+
file_paths: list[str | Path],
|
|
793
|
+
*,
|
|
794
|
+
model_version: str | None = None,
|
|
795
|
+
is_ocr: bool | None = None,
|
|
796
|
+
enable_formula: bool | None = None,
|
|
797
|
+
enable_table: bool | None = None,
|
|
798
|
+
language: str | None = None,
|
|
799
|
+
page_ranges: str | None = None,
|
|
800
|
+
extra_formats: list[str] | None = None,
|
|
801
|
+
poll_interval: float | None = None,
|
|
802
|
+
timeout: int | None = None,
|
|
803
|
+
) -> list[BatchTaskResult]:
|
|
804
|
+
"""解析本地文件(申请链接 + 上传 + 轮询一步到位)
|
|
805
|
+
|
|
806
|
+
create_batch + upload_files + wait_for_batch 的组合封装。
|
|
807
|
+
每个文件共享相同的解析参数(页码范围等);如需每个文件独立参数,
|
|
808
|
+
请使用低层方法自行组合。
|
|
809
|
+
|
|
810
|
+
Args:
|
|
811
|
+
file_paths: 本地文件路径列表(1~50 个,每个 ≤200MB)
|
|
812
|
+
model_version: 模型版本(HTML 文件须指定 MinerU-HTML)
|
|
813
|
+
is_ocr: 是否启用 OCR(全部文件共享)
|
|
814
|
+
enable_formula: 是否开启公式识别
|
|
815
|
+
enable_table: 是否开启表格识别
|
|
816
|
+
language: 文档语言
|
|
817
|
+
page_ranges: 页码范围(全部文件共享)
|
|
818
|
+
extra_formats: 附加导出格式(docx/html/latex)
|
|
819
|
+
poll_interval: 轮询间隔秒数
|
|
820
|
+
timeout: 轮询总超时秒数
|
|
821
|
+
|
|
822
|
+
Returns:
|
|
823
|
+
list[BatchTaskResult]: 全部文件的终态结果(按 file_name 关联)
|
|
824
|
+
|
|
825
|
+
Raises:
|
|
826
|
+
ValueError: 文件数量 / 格式 / 大小不符合官方限制
|
|
827
|
+
FileNotFoundError: 文件不存在
|
|
828
|
+
MinerUError: 未配置 Token / 网络失败 / 轮询超时
|
|
829
|
+
|
|
830
|
+
Example:
|
|
831
|
+
>>> results = MinerUKit.parse_files(
|
|
832
|
+
... ["a.pdf", "b.docx"], model_version="vlm"
|
|
833
|
+
... )
|
|
834
|
+
>>> for r in results:
|
|
835
|
+
... print(r.file_name, r.state, r.full_zip_url)
|
|
836
|
+
"""
|
|
837
|
+
files = [
|
|
838
|
+
BatchFile(name=Path(p).name, is_ocr=is_ocr, page_ranges=page_ranges)
|
|
839
|
+
for p in file_paths
|
|
840
|
+
]
|
|
841
|
+
info = cls.create_batch(
|
|
842
|
+
files,
|
|
843
|
+
model_version=model_version,
|
|
844
|
+
enable_formula=enable_formula,
|
|
845
|
+
enable_table=enable_table,
|
|
846
|
+
language=language,
|
|
847
|
+
extra_formats=extra_formats,
|
|
848
|
+
)
|
|
849
|
+
cls.upload_files(file_paths, info.file_urls)
|
|
850
|
+
return cls.wait_for_batch(
|
|
851
|
+
info.batch_id,
|
|
852
|
+
expected_count=len(files),
|
|
853
|
+
poll_interval=poll_interval,
|
|
854
|
+
timeout=timeout,
|
|
855
|
+
)
|
|
856
|
+
|
|
857
|
+
@staticmethod
|
|
858
|
+
def download_zip(zip_url: str, dest_dir: str | Path) -> str:
|
|
859
|
+
"""下载解析结果压缩包到本地
|
|
860
|
+
|
|
861
|
+
结果 zip 内含 Markdown / JSON(可选 docx/html/latex),文件结构说明见
|
|
862
|
+
https://opendatalab.github.io/MinerU/reference/output_files/
|
|
863
|
+
|
|
864
|
+
Args:
|
|
865
|
+
zip_url: 结果压缩包地址(full_zip_url)
|
|
866
|
+
dest_dir: 目标目录(不存在时自动创建),文件名为 task_id.zip 或
|
|
867
|
+
从 URL 推断
|
|
868
|
+
|
|
869
|
+
Returns:
|
|
870
|
+
str: 保存的本地文件绝对路径
|
|
871
|
+
|
|
872
|
+
Raises:
|
|
873
|
+
MinerUError: 网络失败或 HTTP 非 200
|
|
874
|
+
|
|
875
|
+
Example:
|
|
876
|
+
>>> MinerUKit.download_zip(result.full_zip_url, "./output")
|
|
877
|
+
"""
|
|
878
|
+
dest = Path(dest_dir)
|
|
879
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
880
|
+
filename = zip_url.rstrip("/").split("/")[-1] or "mineru_result.zip"
|
|
881
|
+
if not filename.endswith(".zip"):
|
|
882
|
+
filename = f"{filename}.zip"
|
|
883
|
+
target = (dest / filename).resolve()
|
|
884
|
+
timeout = get_settings().MINERU_TIMEOUT
|
|
885
|
+
logger.info(f"下载 MinerU 结果包: {zip_url}")
|
|
886
|
+
try:
|
|
887
|
+
resp = requests.get(zip_url, timeout=timeout, stream=True)
|
|
888
|
+
except requests.RequestException as e:
|
|
889
|
+
logger.error(f"下载失败: {e}")
|
|
890
|
+
raise MinerUError(f"结果包下载失败: {e}") from e
|
|
891
|
+
if resp.status_code != 200:
|
|
892
|
+
raise MinerUError(f"结果包下载失败: HTTP {resp.status_code}")
|
|
893
|
+
with open(target, "wb") as f:
|
|
894
|
+
for chunk in resp.iter_content(chunk_size=1024 * 1024):
|
|
895
|
+
if chunk:
|
|
896
|
+
f.write(chunk)
|
|
897
|
+
logger.info(f"✅ 结果包已保存: {target}({target.stat().st_size} 字节)")
|
|
898
|
+
return str(target)
|
|
899
|
+
|
|
900
|
+
# ------------------------------------------------------------------
|
|
901
|
+
# 校验辅助
|
|
902
|
+
# ------------------------------------------------------------------
|
|
903
|
+
|
|
904
|
+
@staticmethod
|
|
905
|
+
def is_supported(file_path: str | Path) -> bool:
|
|
906
|
+
"""检查文件格式是否受支持(依据扩展名)
|
|
907
|
+
|
|
908
|
+
Args:
|
|
909
|
+
file_path: 文件路径或文件名
|
|
910
|
+
|
|
911
|
+
Returns:
|
|
912
|
+
bool: 扩展名在支持列表中返回 True
|
|
913
|
+
|
|
914
|
+
Example:
|
|
915
|
+
>>> MinerUKit.is_supported("report.pdf")
|
|
916
|
+
True
|
|
917
|
+
"""
|
|
918
|
+
return Path(file_path).suffix.lower() in MinerUKit.SUPPORTED_EXTENSIONS
|