yimo-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yimo/__init__.py +26 -0
- yimo/kit/__init__.py +98 -0
- yimo/kit/ai/__init__.py +110 -0
- yimo/kit/ai/_internal/__init__.py +27 -0
- yimo/kit/ai/_internal/failover.py +535 -0
- yimo/kit/ai/_internal/pool.py +526 -0
- yimo/kit/ai/_internal/rerank_transport.py +137 -0
- yimo/kit/ai/_internal/retry_after.py +51 -0
- yimo/kit/ai/_internal/sdk_client.py +463 -0
- yimo/kit/ai/_internal/service_base.py +115 -0
- yimo/kit/ai/_internal/types.py +28 -0
- yimo/kit/ai/agent/__init__.py +127 -0
- yimo/kit/ai/agent/approval.py +66 -0
- yimo/kit/ai/agent/errors.py +128 -0
- yimo/kit/ai/agent/loop.py +643 -0
- yimo/kit/ai/agent/mcp.py +278 -0
- yimo/kit/ai/agent/registry.py +203 -0
- yimo/kit/ai/agent/session.py +150 -0
- yimo/kit/ai/agent/skill.py +370 -0
- yimo/kit/ai/agent/types.py +161 -0
- yimo/kit/ai/chat_service.py +724 -0
- yimo/kit/ai/decision/__init__.py +57 -0
- yimo/kit/ai/decision/decima.py +853 -0
- yimo/kit/ai/decision/laya.py +552 -0
- yimo/kit/ai/embedding_service.py +198 -0
- yimo/kit/ai/exceptions.py +109 -0
- yimo/kit/ai/image_service.py +84 -0
- yimo/kit/ai/messages.py +105 -0
- yimo/kit/ai/model_builder.py +121 -0
- yimo/kit/ai/model_client.py +62 -0
- yimo/kit/ai/profiles.py +577 -0
- yimo/kit/ai/rerank_service.py +136 -0
- yimo/kit/ai/runner.py +1127 -0
- yimo/kit/common_kit.py +221 -0
- yimo/kit/config/__init__.py +44 -0
- yimo/kit/config/kit_config.py +210 -0
- yimo/kit/config/log_config.py +171 -0
- yimo/kit/crawl/__init__.py +71 -0
- yimo/kit/crawl/article_io.py +325 -0
- yimo/kit/crawl/article_models.py +172 -0
- yimo/kit/crawl/deepseek_kit.py +673 -0
- yimo/kit/crawl/web_article_kit.py +638 -0
- yimo/kit/crawl/wx_article_kit.py +692 -0
- yimo/kit/db/__init__.py +101 -0
- yimo/kit/db/audit.py +625 -0
- yimo/kit/db/base_db_kit.py +1265 -0
- yimo/kit/db/crud.py +423 -0
- yimo/kit/db/engine.py +235 -0
- yimo/kit/db/exceptions.py +45 -0
- yimo/kit/db/models/__init__.py +33 -0
- yimo/kit/db/models/base.py +56 -0
- yimo/kit/db/mysql_kit.py +348 -0
- yimo/kit/db/pg_kit.py +442 -0
- yimo/kit/db/query.py +338 -0
- yimo/kit/db/session.py +85 -0
- yimo/kit/db/sqlite_kit.py +462 -0
- yimo/kit/db/transaction.py +109 -0
- yimo/kit/docs/__init__.py +34 -0
- yimo/kit/docs/anydoc_kit.py +490 -0
- yimo/kit/docs/docx_kit.py +895 -0
- yimo/kit/docs/md2doc_kit.py +1801 -0
- yimo/kit/docs/mineru_kit.py +1497 -0
- yimo/kit/docs/rc_docparser_kit.py +654 -0
- yimo/kit/file_kit.py +518 -0
- yimo/kit/license_kit.py +533 -0
- yimo/kit/redis_kit.py +392 -0
- yimo_kit-0.1.0.dist-info/METADATA +1673 -0
- yimo_kit-0.1.0.dist-info/RECORD +70 -0
- yimo_kit-0.1.0.dist-info/WHEEL +4 -0
- yimo_kit-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,654 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Copyright (c) 2021-2026 溢墨修心(Yimo). All Rights Reserved.
|
|
3
|
+
|
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
you may not use this file except in compliance with the License.
|
|
6
|
+
You may obtain a copy of the License at
|
|
7
|
+
|
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
|
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
See the License for the specific language governing permissions and
|
|
14
|
+
limitations under the License.
|
|
15
|
+
|
|
16
|
+
Project: yimo
|
|
17
|
+
Author: 溢墨修心(Yimo)
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
# date: 2026-10-19
|
|
21
|
+
"""RcDocParserKit —— 离线 docx → Markdown + Images 工具类(基于 rc-docparser)
|
|
22
|
+
|
|
23
|
+
支持面:`.docx`。
|
|
24
|
+
产物:`<output_dir>/<stem>.md` + `<output_dir>/images/`,md 内以
|
|
25
|
+
`images/<文件名>` 相对引用。
|
|
26
|
+
|
|
27
|
+
实现内容:
|
|
28
|
+
- 只接收 `.docx`;对外提供 `is_supported / convert_to_md / convert_to_dir / convert`
|
|
29
|
+
四件套,结果对象字段与 `MarkerResult` 对齐(docx 无分页,`page_count` 恒为 None)。
|
|
30
|
+
- 标题层级:先把段落上的 `w:outlineLvl`(以及「标题 N」样式名)补成 `Heading N`,
|
|
31
|
+
另存临时副本后再交给库解析,否则中文 Word 文档的标题会整篇退化为正文;
|
|
32
|
+
「图/表 + 编号」开头或超长段落不视为标题(避免作者误设大纲级别的图注变成标题)。
|
|
33
|
+
- 产物规整:去掉库自动加的 `# <stem>` 与 `> Source: ...` 头
|
|
34
|
+
(含时间戳、标题与文件名重复)、标题层级回退一级(库自留一级标题位)、
|
|
35
|
+
图片引用统一改写为 `images/<文件名>`(库在 Windows 上输出反斜杠相对路径)。
|
|
36
|
+
- 图片:库按出现顺序产出 `img-NNN-<sha1 前 10>.<ext>`,本模块复制到 `images/`;
|
|
37
|
+
`extract_images=False` 时不落图片并删除 md 中的图片引用行。
|
|
38
|
+
- 表格:用 python-docx 把每张表重建为**等宽 GFM 网格**,合并单元格的覆盖位留空
|
|
39
|
+
(库原生产物会把同一单元格文本复制到每个覆盖列/行);表格数量不匹配时保留原产物。
|
|
40
|
+
- 临时副本与解析工作目录均创建在产物目录内的临时目录,结束后清理,不修改源文件。
|
|
41
|
+
|
|
42
|
+
前提条件:
|
|
43
|
+
- 纯本地解析:不需要 GPU、模型权重、OCR、GTK 等系统库。
|
|
44
|
+
- 中文标题依赖 `w:outlineLvl` 或「标题 N」样式;两者都没有(仅靠字号/加粗)时
|
|
45
|
+
无法还原层级。
|
|
46
|
+
- 已知限制:页眉页脚与脚注不提取;合并单元格按「覆盖位留空」展平(span 语义不再标出),
|
|
47
|
+
单元格内嵌图片/公式只保留文字;未带 `w:numPr` 的列表样式(如 List Bullet)
|
|
48
|
+
按普通段落输出。
|
|
49
|
+
|
|
50
|
+
依赖:
|
|
51
|
+
- `rc-docparser>=0.2.2`(`pip install yimo-kit[docx-parse]`);
|
|
52
|
+
未安装时导入报错并提示该命令。
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
import re
|
|
56
|
+
import shutil
|
|
57
|
+
import tempfile
|
|
58
|
+
from dataclasses import dataclass
|
|
59
|
+
from pathlib import Path
|
|
60
|
+
|
|
61
|
+
from yimo.kit.config.log_config import get_default_logger
|
|
62
|
+
|
|
63
|
+
logger = get_default_logger(__name__)
|
|
64
|
+
|
|
65
|
+
try:
|
|
66
|
+
from docparser import WorkspaceLayout, parse_docx
|
|
67
|
+
from docx import Document
|
|
68
|
+
from docx.enum.style import WD_STYLE_TYPE
|
|
69
|
+
from docx.oxml.ns import qn
|
|
70
|
+
from docx.text.paragraph import Paragraph
|
|
71
|
+
except ImportError as e:
|
|
72
|
+
raise ImportError(
|
|
73
|
+
"rc_docparser_kit 需要可选依赖,请安装:pip install yimo-kit[docx-parse]"
|
|
74
|
+
) from e
|
|
75
|
+
|
|
76
|
+
# 只支持 docx(库核心另支持 xlsx/txt/md/csv,pptx/epub 需额外 extras,均不在范围内)
|
|
77
|
+
_SUPPORTED_EXTENSIONS: frozenset[str] = frozenset({".docx"})
|
|
78
|
+
|
|
79
|
+
# 库自动加的来源行前缀(落盘前剥离)
|
|
80
|
+
_SOURCE_LINE_PREFIX = "> Source:"
|
|
81
|
+
|
|
82
|
+
# 库把 docx 的 Heading N 输出为 N+1 级,这里回退一级
|
|
83
|
+
_HEADING_RE = re.compile(r"^(#{2,7})(\s)")
|
|
84
|
+
|
|
85
|
+
# 中文 Word 常见的标题样式名(库只认英文 heading N)
|
|
86
|
+
_STYLE_HEADING_RE = re.compile(r"^(?:heading|标题)\s*(\d)", re.IGNORECASE)
|
|
87
|
+
|
|
88
|
+
# 标题启发式:图注/表注常被作者误设大纲级别
|
|
89
|
+
_CAPTION_LIKE_RE = re.compile(
|
|
90
|
+
r"^\s*(?:图|表|fig(?:ure)?|table)\s*[.\-]?\s*\d", re.IGNORECASE
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
# 标题文本长度上限(超过视为误设大纲级别的正文/图注)
|
|
94
|
+
_MAX_HEADING_CHARS = 60
|
|
95
|
+
|
|
96
|
+
# GFM 表格行(用于定位库输出中的表格块)
|
|
97
|
+
_TABLE_LINE_PREFIX = "|"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class RcDocParserError(Exception):
|
|
101
|
+
"""docx 解析异常(依赖调用失败 / 产物缺失 / 落盘失败)"""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass(frozen=True, slots=True)
|
|
105
|
+
class RcDocParserResult:
|
|
106
|
+
"""docx 解析结果对象(字段与 MarkerResult 对齐)
|
|
107
|
+
|
|
108
|
+
Attributes:
|
|
109
|
+
success: 解析是否成功
|
|
110
|
+
markdown: 解析出的 Markdown 文本,失败时为空字符串
|
|
111
|
+
file_path: 源文件绝对路径
|
|
112
|
+
output_dir: 产物目录绝对路径;未落盘时为 None
|
|
113
|
+
md_path: 落盘的 Markdown 文件绝对路径;未落盘时为 None
|
|
114
|
+
image_paths: 落盘的图片文件绝对路径元组(位于 `<output_dir>/images/`)
|
|
115
|
+
page_count: docx 无分页概念,恒为 None(保持字段对齐)
|
|
116
|
+
error: 失败原因描述(异常类型与信息),成功时为 None
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
success: bool
|
|
120
|
+
markdown: str
|
|
121
|
+
file_path: str
|
|
122
|
+
output_dir: str | None = None
|
|
123
|
+
md_path: str | None = None
|
|
124
|
+
image_paths: tuple[str, ...] = ()
|
|
125
|
+
page_count: int | None = None
|
|
126
|
+
error: str | None = None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class RcDocParserKit:
|
|
130
|
+
"""离线 docx 解析工具类,将 docx 转为 Markdown + Images
|
|
131
|
+
|
|
132
|
+
仅支持 `.docx`;产物形态与 `yimo.kit.docs.marker_kit` 一致:
|
|
133
|
+
`<out>/<stem>.md` + `<out>/images/`,md 内以 `images/<文件名>` 相对引用。
|
|
134
|
+
|
|
135
|
+
使用前安装可选依赖::
|
|
136
|
+
|
|
137
|
+
pip install yimo-kit[docx-parse]
|
|
138
|
+
|
|
139
|
+
使用方式::
|
|
140
|
+
|
|
141
|
+
# 1. 落盘 md + images(推荐):源文件同目录下建同名子目录
|
|
142
|
+
md_path = RcDocParserKit.convert_to_dir("report.docx")
|
|
143
|
+
|
|
144
|
+
# 2. 只取 Markdown 文本(图片不落盘,引用仍指向 images/)
|
|
145
|
+
md = RcDocParserKit.convert_to_md("report.docx")
|
|
146
|
+
|
|
147
|
+
# 3. 安全转换(不抛异常,返回结果对象)
|
|
148
|
+
result = RcDocParserKit.convert("report.docx", "out/doc")
|
|
149
|
+
if result.success:
|
|
150
|
+
print(result.md_path, result.image_paths)
|
|
151
|
+
|
|
152
|
+
注意:
|
|
153
|
+
- 纯本地、无模型/OCR/系统库依赖;
|
|
154
|
+
- 图片文件名由库生成(`img-NNN-<sha1 前 10>.<ext>`),同名不同内容不会互相覆盖;
|
|
155
|
+
- 合并单元格表格会退化为 GFM 表格,跨行跨列信息丢失。
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
SUPPORTED_EXTENSIONS: frozenset[str] = _SUPPORTED_EXTENSIONS
|
|
159
|
+
|
|
160
|
+
# ------------------------------------------------------------------
|
|
161
|
+
# 校验与内部工具
|
|
162
|
+
# ------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
@staticmethod
|
|
165
|
+
def is_supported(file_path: str | Path) -> bool:
|
|
166
|
+
"""检查文件格式是否受支持(依据扩展名)
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
file_path: 文件路径或文件名
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
bool: 扩展名为 `.docx` 时返回 True
|
|
173
|
+
|
|
174
|
+
Example:
|
|
175
|
+
>>> RcDocParserKit.is_supported("report.docx")
|
|
176
|
+
True
|
|
177
|
+
>>> RcDocParserKit.is_supported("report.pdf")
|
|
178
|
+
False
|
|
179
|
+
"""
|
|
180
|
+
return Path(file_path).suffix.lower() in RcDocParserKit.SUPPORTED_EXTENSIONS
|
|
181
|
+
|
|
182
|
+
@staticmethod
|
|
183
|
+
def _validate_path(file_path: str | Path) -> Path:
|
|
184
|
+
"""校验待解析文件(存在 / 是文件 / 扩展名受支持),返回绝对路径
|
|
185
|
+
|
|
186
|
+
Raises:
|
|
187
|
+
FileNotFoundError: 文件不存在
|
|
188
|
+
ValueError: 路径不是常规文件,或扩展名不受支持
|
|
189
|
+
"""
|
|
190
|
+
path = Path(file_path).resolve()
|
|
191
|
+
if not path.exists():
|
|
192
|
+
raise FileNotFoundError(f"文件不存在: {path}")
|
|
193
|
+
if not path.is_file():
|
|
194
|
+
raise ValueError(f"路径不是有效文件: {path}")
|
|
195
|
+
if not RcDocParserKit.is_supported(path):
|
|
196
|
+
raise ValueError(
|
|
197
|
+
f"不支持的文件格式: {path.suffix or '<无扩展名>'},"
|
|
198
|
+
f"RcDocParserKit 仅处理 .docx,"
|
|
199
|
+
f"支持的格式: {', '.join(sorted(RcDocParserKit.SUPPORTED_EXTENSIONS))}"
|
|
200
|
+
)
|
|
201
|
+
return path
|
|
202
|
+
|
|
203
|
+
@staticmethod
|
|
204
|
+
def _resolve_out_dir(src: Path, output_dir: str | Path | None) -> Path:
|
|
205
|
+
"""解析产物目录:缺省为源文件同目录下与文件名同名的子目录"""
|
|
206
|
+
if output_dir is None:
|
|
207
|
+
return (src.parent / src.stem).resolve()
|
|
208
|
+
return Path(output_dir).resolve()
|
|
209
|
+
|
|
210
|
+
@staticmethod
|
|
211
|
+
def _strip_auto_header(markdown: str, stem: str) -> str:
|
|
212
|
+
"""去掉库自动加的 `# <stem>` 与 `> Source: ...` 头
|
|
213
|
+
|
|
214
|
+
只删这两类行(及其后的空行),不误删文档自身的引用块。
|
|
215
|
+
"""
|
|
216
|
+
lines = markdown.splitlines()
|
|
217
|
+
if lines and lines[0].strip() == f"# {stem}":
|
|
218
|
+
lines = lines[1:]
|
|
219
|
+
while lines and not lines[0].strip():
|
|
220
|
+
lines = lines[1:]
|
|
221
|
+
if lines and lines[0].startswith(_SOURCE_LINE_PREFIX):
|
|
222
|
+
lines = lines[1:]
|
|
223
|
+
while lines and not lines[0].strip():
|
|
224
|
+
lines = lines[1:]
|
|
225
|
+
return "\n".join(lines)
|
|
226
|
+
|
|
227
|
+
@staticmethod
|
|
228
|
+
def _shift_headings(markdown: str) -> str:
|
|
229
|
+
"""把库输出的 Heading N+1 回退为 Heading N(对齐原始文档层级)"""
|
|
230
|
+
shifted: list[str] = []
|
|
231
|
+
for line in markdown.splitlines():
|
|
232
|
+
match = _HEADING_RE.match(line)
|
|
233
|
+
if match:
|
|
234
|
+
line = match.group(1)[1:] + match.group(2) + line[match.end() :]
|
|
235
|
+
shifted.append(line)
|
|
236
|
+
return "\n".join(shifted)
|
|
237
|
+
|
|
238
|
+
@staticmethod
|
|
239
|
+
def _ref_variants(rel: str) -> set[str]:
|
|
240
|
+
"""同一相对路径在不同平台的写法(库在 Windows 上输出反斜杠)"""
|
|
241
|
+
return {rel, rel.replace("\\", "/"), rel.replace("/", "\\")}
|
|
242
|
+
|
|
243
|
+
@classmethod
|
|
244
|
+
def _localize_image_refs(cls, markdown: str, mapping: dict[str, str]) -> str:
|
|
245
|
+
"""把库输出的相对图片路径改写为 `images/<文件名>`"""
|
|
246
|
+
for rel, target in mapping.items():
|
|
247
|
+
for variant in cls._ref_variants(rel):
|
|
248
|
+
markdown = markdown.replace(f"]({variant})", f"]({target})")
|
|
249
|
+
return markdown
|
|
250
|
+
|
|
251
|
+
@classmethod
|
|
252
|
+
def _drop_image_refs(cls, markdown: str, rels: list[str]) -> str:
|
|
253
|
+
"""删除图片引用行(extract_images=False / 图片复制失败时使用)"""
|
|
254
|
+
variants: set[str] = set()
|
|
255
|
+
for rel in rels:
|
|
256
|
+
variants |= cls._ref_variants(rel)
|
|
257
|
+
kept = [
|
|
258
|
+
line
|
|
259
|
+
for line in markdown.splitlines()
|
|
260
|
+
if not (
|
|
261
|
+
line.lstrip().startswith("" in line for variant in variants)
|
|
263
|
+
)
|
|
264
|
+
]
|
|
265
|
+
return "\n".join(kept)
|
|
266
|
+
|
|
267
|
+
# ------------------------------------------------------------------
|
|
268
|
+
# 表格重建(合并单元格 → 等宽 GFM)
|
|
269
|
+
# ------------------------------------------------------------------
|
|
270
|
+
|
|
271
|
+
@staticmethod
|
|
272
|
+
def _tc_text(tc) -> str:
|
|
273
|
+
"""从 `w:tc` 取文本:段落用空格连接,转义竖线(GFM 单元格不能换行)"""
|
|
274
|
+
parts = [Paragraph(p, None).text.strip() for p in tc.findall(qn("w:p"))]
|
|
275
|
+
text = " ".join(part for part in parts if part)
|
|
276
|
+
return text.replace("|", "\\|")
|
|
277
|
+
|
|
278
|
+
@classmethod
|
|
279
|
+
def _table_grid(cls, table) -> list[list[str]]:
|
|
280
|
+
"""把 docx 表格重建为等宽矩形网格(合并格的覆盖位留空)
|
|
281
|
+
|
|
282
|
+
直接读 OOXML:每个 `w:tr` 的 `w:tc` 列表是「逻辑单元格」(不含网格展开),
|
|
283
|
+
`w:gridSpan` 给出横向跨度,`w:vMerge`(无 val 或 `continue`)表示纵向续接。
|
|
284
|
+
横向与纵向的覆盖位一律留空,避免同一单元格文本被复制到每一列/每一行。
|
|
285
|
+
"""
|
|
286
|
+
rows: list[list[str]] = []
|
|
287
|
+
for tr in table._tbl.tr_lst:
|
|
288
|
+
cells: list[str] = []
|
|
289
|
+
for tc in tr.tc_lst:
|
|
290
|
+
tc_pr = tc.tcPr
|
|
291
|
+
span = 1
|
|
292
|
+
if tc_pr is not None and tc_pr.gridSpan is not None:
|
|
293
|
+
span = max(1, int(tc_pr.gridSpan.val))
|
|
294
|
+
continued = False
|
|
295
|
+
if tc_pr is not None and tc_pr.vMerge is not None:
|
|
296
|
+
continued = tc_pr.vMerge.val in (None, "continue")
|
|
297
|
+
cells.append("" if continued else cls._tc_text(tc))
|
|
298
|
+
cells.extend([""] * (span - 1))
|
|
299
|
+
rows.append(cells)
|
|
300
|
+
|
|
301
|
+
width = max((len(row) for row in rows), default=0)
|
|
302
|
+
return [row + [""] * (width - len(row)) for row in rows]
|
|
303
|
+
|
|
304
|
+
@staticmethod
|
|
305
|
+
def _render_gfm_table(rows: list[list[str]]) -> list[str]:
|
|
306
|
+
"""把矩形网格渲染为 GFM 表格行(首行作表头)"""
|
|
307
|
+
if not rows:
|
|
308
|
+
return []
|
|
309
|
+
width = max(len(row) for row in rows)
|
|
310
|
+
lines = [
|
|
311
|
+
"| " + " | ".join(rows[0]) + " |",
|
|
312
|
+
"| " + " | ".join(["---"] * width) + " |",
|
|
313
|
+
]
|
|
314
|
+
for row in rows[1:]:
|
|
315
|
+
lines.append("| " + " | ".join(row) + " |")
|
|
316
|
+
return lines
|
|
317
|
+
|
|
318
|
+
@classmethod
|
|
319
|
+
def _rebuild_tables(cls, markdown: str, src: Path) -> str:
|
|
320
|
+
"""用 python-docx 重建的等宽 GFM 表格替换库输出的表格块
|
|
321
|
+
|
|
322
|
+
库按 `row.cells` 取文本,合并单元格会重复出现在每个覆盖格位;这里按顺序
|
|
323
|
+
一一替换。表格数量不一致、或读取 docx 失败时**保留原产物**并 warning,
|
|
324
|
+
不影响整体转换。
|
|
325
|
+
"""
|
|
326
|
+
try:
|
|
327
|
+
doc = Document(str(src))
|
|
328
|
+
except Exception as e: # noqa: BLE001 - 表格重建失败不应影响正文
|
|
329
|
+
logger.warning(f"读取 docx 表格失败,保留库输出: {e}")
|
|
330
|
+
return markdown
|
|
331
|
+
if not doc.tables:
|
|
332
|
+
return markdown
|
|
333
|
+
|
|
334
|
+
lines = markdown.splitlines()
|
|
335
|
+
blocks: list[tuple[int, int]] = []
|
|
336
|
+
index = 0
|
|
337
|
+
while index < len(lines):
|
|
338
|
+
if lines[index].lstrip().startswith(_TABLE_LINE_PREFIX):
|
|
339
|
+
start = index
|
|
340
|
+
while index < len(lines) and lines[index].lstrip().startswith(
|
|
341
|
+
_TABLE_LINE_PREFIX
|
|
342
|
+
):
|
|
343
|
+
index += 1
|
|
344
|
+
blocks.append((start, index))
|
|
345
|
+
else:
|
|
346
|
+
index += 1
|
|
347
|
+
|
|
348
|
+
if len(blocks) != len(doc.tables):
|
|
349
|
+
logger.warning(
|
|
350
|
+
f"表格数量不匹配(md {len(blocks)} / docx {len(doc.tables)}),"
|
|
351
|
+
f"保留原始表格"
|
|
352
|
+
)
|
|
353
|
+
return markdown
|
|
354
|
+
|
|
355
|
+
rebuilt: list[str] = []
|
|
356
|
+
cursor = 0
|
|
357
|
+
for (start, end), table in zip(blocks, doc.tables):
|
|
358
|
+
rebuilt.extend(lines[cursor:start])
|
|
359
|
+
rebuilt.extend(cls._render_gfm_table(cls._table_grid(table)))
|
|
360
|
+
cursor = end
|
|
361
|
+
rebuilt.extend(lines[cursor:])
|
|
362
|
+
|
|
363
|
+
suffix = "\n" if markdown.endswith("\n") else ""
|
|
364
|
+
return "\n".join(rebuilt) + suffix
|
|
365
|
+
|
|
366
|
+
@staticmethod
|
|
367
|
+
def _outline_level(paragraph) -> int | None:
|
|
368
|
+
"""读取段落的 `w:outlineLvl`(0 基)并转为 Heading 级别,无则 None"""
|
|
369
|
+
ppr = paragraph._element.find(qn("w:pPr"))
|
|
370
|
+
if ppr is None:
|
|
371
|
+
return None
|
|
372
|
+
node = ppr.find(qn("w:outlineLvl"))
|
|
373
|
+
if node is None:
|
|
374
|
+
return None
|
|
375
|
+
raw = node.get(qn("w:val"))
|
|
376
|
+
if raw is None or not str(raw).isdigit():
|
|
377
|
+
return None
|
|
378
|
+
return int(raw) + 1
|
|
379
|
+
|
|
380
|
+
@classmethod
|
|
381
|
+
def _prepare_source(cls, src: Path, work_root: Path) -> tuple[Path, int]:
|
|
382
|
+
"""生成「标题样式已补齐」的临时副本,返回 (解析用路径, 补齐段落数)
|
|
383
|
+
|
|
384
|
+
中文 Word 文档常把标题写成自定义样式(如 `Body Text`)+ `w:outlineLvl`,
|
|
385
|
+
而库只按样式名 `heading N` 判定标题,会整篇丢失层级。这里按 `outlineLvl` /
|
|
386
|
+
中文「标题 N」样式名把段落样式改为 `Heading N` 后另存副本,原文不动;
|
|
387
|
+
无需补齐时直接返回原路径。
|
|
388
|
+
"""
|
|
389
|
+
doc = Document(str(src))
|
|
390
|
+
patched = 0
|
|
391
|
+
for para in doc.paragraphs:
|
|
392
|
+
level = cls._outline_level(para)
|
|
393
|
+
if level is None:
|
|
394
|
+
match = _STYLE_HEADING_RE.match((para.style.name or "").strip())
|
|
395
|
+
level = int(match.group(1)) if match else None
|
|
396
|
+
if level is None:
|
|
397
|
+
continue
|
|
398
|
+
text = (para.text or "").strip()
|
|
399
|
+
if not text or _CAPTION_LIKE_RE.match(text):
|
|
400
|
+
continue
|
|
401
|
+
if len(text) > _MAX_HEADING_CHARS:
|
|
402
|
+
continue
|
|
403
|
+
level = min(6, max(1, level))
|
|
404
|
+
name = f"Heading {level}"
|
|
405
|
+
try:
|
|
406
|
+
style = doc.styles[name]
|
|
407
|
+
except KeyError:
|
|
408
|
+
style = doc.styles.add_style(name, WD_STYLE_TYPE.PARAGRAPH)
|
|
409
|
+
para.style = style
|
|
410
|
+
patched += 1
|
|
411
|
+
|
|
412
|
+
if not patched:
|
|
413
|
+
return src, 0
|
|
414
|
+
work_root.mkdir(parents=True, exist_ok=True)
|
|
415
|
+
target = work_root / src.name
|
|
416
|
+
doc.save(str(target))
|
|
417
|
+
logger.info(f"已按大纲级别补齐 {patched} 个标题样式(临时副本,原文不改)")
|
|
418
|
+
return target, patched
|
|
419
|
+
|
|
420
|
+
@classmethod
|
|
421
|
+
def _run(cls, src: Path, work_root: Path) -> tuple[str, list[dict], Path]:
|
|
422
|
+
"""调用库解析一次,返回 (原始 md, 图片清单, 图片源目录)
|
|
423
|
+
|
|
424
|
+
Raises:
|
|
425
|
+
RcDocParserError: 临时副本生成 / 库调用 / 产物读取失败
|
|
426
|
+
"""
|
|
427
|
+
layout = WorkspaceLayout(
|
|
428
|
+
raw_dir=work_root / "raw",
|
|
429
|
+
parsed_dir=work_root / "parsed",
|
|
430
|
+
assets_dir=work_root / "assets",
|
|
431
|
+
cache_dir=work_root / "cache",
|
|
432
|
+
)
|
|
433
|
+
try:
|
|
434
|
+
parse_src, _patched = cls._prepare_source(src, work_root / "patched")
|
|
435
|
+
payload = parse_docx(parse_src, layout)
|
|
436
|
+
md_file = layout.parsed_dir_for(parse_src) / "document.md"
|
|
437
|
+
markdown = md_file.read_text(encoding="utf-8")
|
|
438
|
+
except RcDocParserError:
|
|
439
|
+
raise
|
|
440
|
+
except Exception as e: # noqa: BLE001 - 库无统一异常基类
|
|
441
|
+
raise RcDocParserError(f"docx 解析失败: {type(e).__name__}: {e}") from e
|
|
442
|
+
|
|
443
|
+
return (
|
|
444
|
+
markdown,
|
|
445
|
+
list(payload.get("images") or []),
|
|
446
|
+
layout.assets_dir_for(parse_src),
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
@classmethod
|
|
450
|
+
def _write_outputs(
|
|
451
|
+
cls, src: Path, out_dir: Path, *, extract_images: bool
|
|
452
|
+
) -> tuple[str, tuple[str, ...]]:
|
|
453
|
+
"""解析并落盘,返回 (md 路径, 图片路径元组)"""
|
|
454
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
455
|
+
work_root = Path(tempfile.mkdtemp(prefix=".rc-docparser-", dir=out_dir))
|
|
456
|
+
image_paths: list[str] = []
|
|
457
|
+
try:
|
|
458
|
+
raw_md, images, assets_dir = cls._run(src, work_root)
|
|
459
|
+
mapping: dict[str, str] = {}
|
|
460
|
+
if extract_images and images:
|
|
461
|
+
images_dir = out_dir / "images"
|
|
462
|
+
images_dir.mkdir(parents=True, exist_ok=True)
|
|
463
|
+
for img in images:
|
|
464
|
+
rel = str(img.get("asset_path") or "")
|
|
465
|
+
if not rel:
|
|
466
|
+
continue
|
|
467
|
+
name = Path(rel).name
|
|
468
|
+
source_file = assets_dir / name
|
|
469
|
+
if not source_file.is_file():
|
|
470
|
+
logger.warning(f"库未写出图片,跳过并移除引用: {name}")
|
|
471
|
+
continue
|
|
472
|
+
shutil.copyfile(source_file, images_dir / name)
|
|
473
|
+
mapping[rel] = f"images/{name}"
|
|
474
|
+
image_paths.append(str(images_dir / name))
|
|
475
|
+
|
|
476
|
+
markdown = cls._render(
|
|
477
|
+
raw_md,
|
|
478
|
+
images,
|
|
479
|
+
src.stem,
|
|
480
|
+
mapping=mapping,
|
|
481
|
+
extract_images=extract_images,
|
|
482
|
+
)
|
|
483
|
+
markdown = cls._rebuild_tables(markdown, src)
|
|
484
|
+
md_path = out_dir / f"{src.stem}.md"
|
|
485
|
+
md_path.write_text(markdown, encoding="utf-8")
|
|
486
|
+
finally:
|
|
487
|
+
shutil.rmtree(work_root, ignore_errors=True)
|
|
488
|
+
|
|
489
|
+
return str(md_path), tuple(image_paths)
|
|
490
|
+
|
|
491
|
+
@classmethod
|
|
492
|
+
def _render(
|
|
493
|
+
cls,
|
|
494
|
+
markdown: str,
|
|
495
|
+
images: list[dict],
|
|
496
|
+
stem: str,
|
|
497
|
+
*,
|
|
498
|
+
mapping: dict[str, str],
|
|
499
|
+
extract_images: bool,
|
|
500
|
+
) -> str:
|
|
501
|
+
"""规整 md:图片引用 / 去自动头 / 标题回退一级"""
|
|
502
|
+
if extract_images:
|
|
503
|
+
markdown = cls._localize_image_refs(markdown, mapping)
|
|
504
|
+
# 复制失败的图片仍可能保留原相对引用,统一删除避免出现死链
|
|
505
|
+
missing = [
|
|
506
|
+
str(img.get("asset_path") or "")
|
|
507
|
+
for img in images
|
|
508
|
+
if str(img.get("asset_path") or "") not in mapping
|
|
509
|
+
]
|
|
510
|
+
if missing:
|
|
511
|
+
markdown = cls._drop_image_refs(markdown, missing)
|
|
512
|
+
else:
|
|
513
|
+
markdown = cls._drop_image_refs(
|
|
514
|
+
markdown, [str(img.get("asset_path") or "") for img in images]
|
|
515
|
+
)
|
|
516
|
+
markdown = cls._strip_auto_header(markdown, stem)
|
|
517
|
+
markdown = cls._shift_headings(markdown)
|
|
518
|
+
return markdown.rstrip() + "\n"
|
|
519
|
+
|
|
520
|
+
# ------------------------------------------------------------------
|
|
521
|
+
# 公开 API
|
|
522
|
+
# ------------------------------------------------------------------
|
|
523
|
+
|
|
524
|
+
@classmethod
|
|
525
|
+
def convert_to_md(cls, file_path: str | Path) -> str:
|
|
526
|
+
"""将 docx 解析为 Markdown 文本(图片不落盘)
|
|
527
|
+
|
|
528
|
+
Args:
|
|
529
|
+
file_path: docx 文件路径(支持 str 或 Path)
|
|
530
|
+
|
|
531
|
+
Returns:
|
|
532
|
+
str: 解析出的 Markdown 文本(图片引用为 `images/<文件名>`)
|
|
533
|
+
|
|
534
|
+
Raises:
|
|
535
|
+
FileNotFoundError: 文件不存在
|
|
536
|
+
ValueError: 路径非法 / 扩展名不受支持
|
|
537
|
+
RcDocParserError: 解析失败或未产出 markdown
|
|
538
|
+
|
|
539
|
+
Example:
|
|
540
|
+
>>> md = RcDocParserKit.convert_to_md("report.docx")
|
|
541
|
+
"""
|
|
542
|
+
src = cls._validate_path(file_path)
|
|
543
|
+
logger.info(f"开始 docx 解析(仅文本): {src.name}")
|
|
544
|
+
with tempfile.TemporaryDirectory(prefix="rc-docparser-") as tmp:
|
|
545
|
+
raw_md, images, _assets_dir = cls._run(src, Path(tmp))
|
|
546
|
+
|
|
547
|
+
mapping = {
|
|
548
|
+
str(img["asset_path"]): f"images/{Path(str(img['asset_path'])).name}"
|
|
549
|
+
for img in images
|
|
550
|
+
if img.get("asset_path")
|
|
551
|
+
}
|
|
552
|
+
markdown = cls._render(
|
|
553
|
+
raw_md, images, src.stem, mapping=mapping, extract_images=True
|
|
554
|
+
)
|
|
555
|
+
markdown = cls._rebuild_tables(markdown, src)
|
|
556
|
+
logger.info(f"docx 解析成功: {src.name}({len(markdown)} 字符)")
|
|
557
|
+
return markdown
|
|
558
|
+
|
|
559
|
+
@classmethod
|
|
560
|
+
def convert_to_dir(
|
|
561
|
+
cls,
|
|
562
|
+
src_path: str | Path,
|
|
563
|
+
output_dir: str | Path | None = None,
|
|
564
|
+
*,
|
|
565
|
+
extract_images: bool = True,
|
|
566
|
+
) -> str:
|
|
567
|
+
"""解析 docx 并将 Markdown + 图片落盘到目录
|
|
568
|
+
|
|
569
|
+
产物写入 output_dir:`<源文件名>.md` 与 `images/` 图片目录(md 内以
|
|
570
|
+
`images/<文件名>` 相对引用);无图片时不创建空 `images/`。
|
|
571
|
+
|
|
572
|
+
Args:
|
|
573
|
+
src_path: 源 docx 路径
|
|
574
|
+
output_dir: 产物目录;None 时使用源文件同目录下与文件名同名的子目录
|
|
575
|
+
extract_images: 是否落盘图片(False 时仅写 md,并删除图片引用)
|
|
576
|
+
|
|
577
|
+
Returns:
|
|
578
|
+
str: 落盘的 Markdown 文件绝对路径
|
|
579
|
+
|
|
580
|
+
Raises:
|
|
581
|
+
FileNotFoundError: 源文件不存在
|
|
582
|
+
ValueError: 路径非法 / 扩展名不受支持
|
|
583
|
+
RcDocParserError: 解析失败
|
|
584
|
+
OSError: 产物写入失败
|
|
585
|
+
|
|
586
|
+
Example:
|
|
587
|
+
>>> RcDocParserKit.convert_to_dir("report.docx") # -> 同目录 report/
|
|
588
|
+
>>> RcDocParserKit.convert_to_dir("report.docx", "out/doc")
|
|
589
|
+
"""
|
|
590
|
+
src = cls._validate_path(src_path)
|
|
591
|
+
out_dir = cls._resolve_out_dir(src, output_dir)
|
|
592
|
+
logger.info(f"开始 docx 解析: {src.name} -> {out_dir}")
|
|
593
|
+
md_path, image_paths = cls._write_outputs(
|
|
594
|
+
src, out_dir, extract_images=extract_images
|
|
595
|
+
)
|
|
596
|
+
logger.info(f"docx 解析成功: {src.name}({len(image_paths)} 图)-> {md_path}")
|
|
597
|
+
return md_path
|
|
598
|
+
|
|
599
|
+
@classmethod
|
|
600
|
+
def convert(
|
|
601
|
+
cls,
|
|
602
|
+
src_path: str | Path,
|
|
603
|
+
output_dir: str | Path | None = None,
|
|
604
|
+
*,
|
|
605
|
+
extract_images: bool = True,
|
|
606
|
+
) -> RcDocParserResult:
|
|
607
|
+
"""安全转换 docx,返回包含状态的结果对象(不抛转换异常)
|
|
608
|
+
|
|
609
|
+
产物落盘规则同 convert_to_dir。
|
|
610
|
+
|
|
611
|
+
Args:
|
|
612
|
+
src_path: 源 docx 路径
|
|
613
|
+
output_dir: 产物目录;None 时使用源文件同目录下与文件名同名的子目录
|
|
614
|
+
extract_images: 是否落盘图片
|
|
615
|
+
|
|
616
|
+
Returns:
|
|
617
|
+
RcDocParserResult: 解析结果(含 markdown / md_path / image_paths / error)
|
|
618
|
+
|
|
619
|
+
Example:
|
|
620
|
+
>>> result = RcDocParserKit.convert("report.docx", "out/doc")
|
|
621
|
+
>>> if result.success:
|
|
622
|
+
... print(result.md_path, len(result.image_paths))
|
|
623
|
+
"""
|
|
624
|
+
src = Path(src_path).resolve()
|
|
625
|
+
out_dir = cls._resolve_out_dir(src, output_dir)
|
|
626
|
+
try:
|
|
627
|
+
path = cls._validate_path(src)
|
|
628
|
+
md_path, image_paths = cls._write_outputs(
|
|
629
|
+
path, out_dir, extract_images=extract_images
|
|
630
|
+
)
|
|
631
|
+
except Exception as e: # noqa: BLE001 - 安全版统一封装为结果对象
|
|
632
|
+
logger.warning(f"docx 解析失败: {src.name}({type(e).__name__}): {e}")
|
|
633
|
+
return RcDocParserResult(
|
|
634
|
+
success=False,
|
|
635
|
+
markdown="",
|
|
636
|
+
file_path=str(src),
|
|
637
|
+
output_dir=str(out_dir),
|
|
638
|
+
error=f"{type(e).__name__}: {e}",
|
|
639
|
+
)
|
|
640
|
+
|
|
641
|
+
md_file = Path(md_path)
|
|
642
|
+
logger.info(
|
|
643
|
+
f"docx 解析成功: {src.name}({md_file.stat().st_size} 字节,"
|
|
644
|
+
f"{len(image_paths)} 图)-> {md_path}"
|
|
645
|
+
)
|
|
646
|
+
return RcDocParserResult(
|
|
647
|
+
success=True,
|
|
648
|
+
markdown=md_file.read_text(encoding="utf-8"),
|
|
649
|
+
file_path=str(path),
|
|
650
|
+
output_dir=str(out_dir),
|
|
651
|
+
md_path=md_path,
|
|
652
|
+
image_paths=image_paths,
|
|
653
|
+
page_count=None,
|
|
654
|
+
)
|