treesearchlib 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- treesearch/__init__.py +53 -0
- treesearch/__main__.py +6 -0
- treesearch/_bin/pst-extract.exe +0 -0
- treesearch/cli.py +554 -0
- treesearch/config.py +206 -0
- treesearch/fts.py +2293 -0
- treesearch/heuristics.py +425 -0
- treesearch/indexer.py +2038 -0
- treesearch/parsers/__init__.py +62 -0
- treesearch/parsers/anydoc_parser.py +193 -0
- treesearch/parsers/ast_parser.py +136 -0
- treesearch/parsers/docx_parser.py +304 -0
- treesearch/parsers/email_html_md.py +60 -0
- treesearch/parsers/excel_parser.py +218 -0
- treesearch/parsers/html_parser.py +172 -0
- treesearch/parsers/image_metadata.py +345 -0
- treesearch/parsers/image_parser.py +59 -0
- treesearch/parsers/image_store.py +182 -0
- treesearch/parsers/markitdown_parser.py +258 -0
- treesearch/parsers/mhtml_parser.py +108 -0
- treesearch/parsers/pdf_parser.py +409 -0
- treesearch/parsers/pst_attachment_store.py +156 -0
- treesearch/parsers/pst_parser.py +733 -0
- treesearch/parsers/registry.py +405 -0
- treesearch/parsers/treesitter_parser.py +433 -0
- treesearch/pathutil.py +227 -0
- treesearch/py.typed +0 -0
- treesearch/ripgrep.py +159 -0
- treesearch/search.py +935 -0
- treesearch/tokenizer.py +176 -0
- treesearch/tree.py +393 -0
- treesearch/tree_searcher.py +1006 -0
- treesearch/treesearch.py +574 -0
- treesearch/watch.py +305 -0
- treesearchlib-1.1.0.dist-info/METADATA +124 -0
- treesearchlib-1.1.0.dist-info/RECORD +39 -0
- treesearchlib-1.1.0.dist-info/WHEEL +5 -0
- treesearchlib-1.1.0.dist-info/entry_points.txt +2 -0
- treesearchlib-1.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
@author:XuMing(xuming624@qq.com)
|
|
4
|
+
@description: Parser registry for file-type-aware indexing and pre-filter routing.
|
|
5
|
+
|
|
6
|
+
Provides a unified interface for file parsers and automatic routing of
|
|
7
|
+
file types to appropriate parser + pre-filter combinations.
|
|
8
|
+
"""
|
|
9
|
+
from .registry import (
|
|
10
|
+
ParserRegistry,
|
|
11
|
+
get_parser,
|
|
12
|
+
SOURCE_TYPE_MAP,
|
|
13
|
+
PREFILTER_ROUTING,
|
|
14
|
+
get_prefilters_for_source_type,
|
|
15
|
+
)
|
|
16
|
+
from .ast_parser import parse_python_structure
|
|
17
|
+
from .anydoc_parser import anydoc_to_tree, ANYDOC_EXTENSIONS
|
|
18
|
+
|
|
19
|
+
# PDF parser (optional)
|
|
20
|
+
try:
|
|
21
|
+
from .pdf_parser import extract_pdf_text, PDF_EXTENSIONS
|
|
22
|
+
_has_pdf = True
|
|
23
|
+
except ImportError:
|
|
24
|
+
_has_pdf = False
|
|
25
|
+
|
|
26
|
+
# Excel parser (optional)
|
|
27
|
+
try:
|
|
28
|
+
from .excel_parser import excel_to_tree, EXCEL_EXTENSIONS
|
|
29
|
+
_has_excel = True
|
|
30
|
+
except ImportError:
|
|
31
|
+
_has_excel = False
|
|
32
|
+
|
|
33
|
+
# Tree-sitter parser (optional)
|
|
34
|
+
try:
|
|
35
|
+
from .treesitter_parser import (
|
|
36
|
+
parse_treesitter_structure,
|
|
37
|
+
treesitter_code_to_tree,
|
|
38
|
+
EXT_TO_LANGUAGE,
|
|
39
|
+
)
|
|
40
|
+
_has_treesitter = True
|
|
41
|
+
except ImportError:
|
|
42
|
+
_has_treesitter = False
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"ParserRegistry",
|
|
46
|
+
"get_parser",
|
|
47
|
+
"SOURCE_TYPE_MAP",
|
|
48
|
+
"PREFILTER_ROUTING",
|
|
49
|
+
"get_prefilters_for_source_type",
|
|
50
|
+
"parse_python_structure",
|
|
51
|
+
"anydoc_to_tree",
|
|
52
|
+
"ANYDOC_EXTENSIONS",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
if _has_pdf:
|
|
56
|
+
__all__ += ["extract_pdf_text", "PDF_EXTENSIONS"]
|
|
57
|
+
|
|
58
|
+
if _has_excel:
|
|
59
|
+
__all__ += ["excel_to_tree", "EXCEL_EXTENSIONS"]
|
|
60
|
+
|
|
61
|
+
if _has_treesitter:
|
|
62
|
+
__all__ += ["parse_treesitter_structure", "treesitter_code_to_tree", "EXT_TO_LANGUAGE"]
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
@author:XuMing(xuming624@qq.com)
|
|
4
|
+
@description: anydoc-based parser for TreeSearch.
|
|
5
|
+
|
|
6
|
+
Uses Firecrawl's ``anydoc`` (pure Rust, PyPI: firecrawl-anydoc) to convert
|
|
7
|
+
legacy/office-adjacent documents to Markdown, then feeds the result into the
|
|
8
|
+
existing ``md_to_tree`` pipeline.
|
|
9
|
+
|
|
10
|
+
Covers: .doc/.docm (Word 97-2003), .ppt/.pps/.pot (PowerPoint 97-2003),
|
|
11
|
+
.xls (Excel 97-2003), .rtf, .epub.
|
|
12
|
+
|
|
13
|
+
Requires dependency: ``pip install firecrawl-anydoc``(主依赖,ADR-0013)。
|
|
14
|
+
|
|
15
|
+
Notes:
|
|
16
|
+
- anydoc 按内容识别格式(扩展名标错也能转),无需显式 format 参数。
|
|
17
|
+
- ppt 老格式输出为扁平段落(无 slide 边界/标题),直接走 md_to_tree,
|
|
18
|
+
不做 pptx 式的根节点包裹(markitdown 每 slide 一个 ``#`` 标题才可包裹)。
|
|
19
|
+
- anydoc 的 Markdown 序列化器不输出图片引用(仅保留 alt 文本),内嵌图片
|
|
20
|
+
经 ``to_document().assets`` 提取落盘后统一附加到文档末尾。
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import io
|
|
24
|
+
import logging
|
|
25
|
+
import os
|
|
26
|
+
import re
|
|
27
|
+
import zipfile
|
|
28
|
+
from typing import TYPE_CHECKING, Optional
|
|
29
|
+
|
|
30
|
+
if TYPE_CHECKING:
|
|
31
|
+
from .image_store import ImagePart, ImageStore
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
# Extensions handled by anydoc
|
|
36
|
+
ANYDOC_EXTENSIONS = {
|
|
37
|
+
".doc", ".docm", # Word 97-2003
|
|
38
|
+
".ppt", ".pps", ".pot", # PowerPoint 97-2003
|
|
39
|
+
".xls", # Excel 97-2003
|
|
40
|
+
".rtf",
|
|
41
|
+
".epub",
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
# anydoc Asset.media_type → 落盘扩展名(仅 image/* 会落盘,其余跳过)
|
|
45
|
+
_MEDIA_TYPE_TO_EXT = {
|
|
46
|
+
"image/png": "png",
|
|
47
|
+
"image/jpeg": "jpg",
|
|
48
|
+
"image/gif": "gif",
|
|
49
|
+
"image/webp": "webp",
|
|
50
|
+
"image/bmp": "bmp",
|
|
51
|
+
"image/tiff": "tiff",
|
|
52
|
+
"image/svg+xml": "svg",
|
|
53
|
+
"image/x-emf": "emf",
|
|
54
|
+
"image/x-wmf": "wmf",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _convert(data: bytes) -> tuple[str, object]:
|
|
59
|
+
"""anydoc 转换:(markdown, document)。ConvertError 记具体原因后抛出。"""
|
|
60
|
+
import anydoc
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
md_content = anydoc.to_markdown_bytes(data)
|
|
64
|
+
document = anydoc.to_document(data)
|
|
65
|
+
except anydoc.EncryptedError as e:
|
|
66
|
+
logger.warning("anydoc: encrypted document, skipped: %s", e)
|
|
67
|
+
raise
|
|
68
|
+
except anydoc.ConvertError as e:
|
|
69
|
+
logger.warning("anydoc conversion failed (%s): %s", type(e).__name__, e)
|
|
70
|
+
raise
|
|
71
|
+
return md_content or "", document
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _check_epub_drm(data: bytes) -> None:
|
|
75
|
+
"""epub(zip) 含 META-INF/encryption.xml → 全书 DRM 加密,抛清晰错误。
|
|
76
|
+
|
|
77
|
+
anydoc 对加密 epub 只会报误导性的 "malformed document: no rootfile entry"
|
|
78
|
+
(container.xml 本身也是密文)。此处预检给出真实原因;error message 会被
|
|
79
|
+
indexer 记入 failed_files.last_error 并在 UI 展示。
|
|
80
|
+
"""
|
|
81
|
+
try:
|
|
82
|
+
with zipfile.ZipFile(io.BytesIO(data)) as z:
|
|
83
|
+
if "META-INF/encryption.xml" not in z.namelist():
|
|
84
|
+
return
|
|
85
|
+
vendor = ""
|
|
86
|
+
try:
|
|
87
|
+
enc_xml = z.read("META-INF/encryption.xml").decode("utf-8", errors="replace")
|
|
88
|
+
m = re.search(r"<Proprietary>([^<]+)</Proprietary>", enc_xml)
|
|
89
|
+
if m:
|
|
90
|
+
vendor = f"({m.group(1)})"
|
|
91
|
+
except Exception: # noqa: BLE001
|
|
92
|
+
pass
|
|
93
|
+
except zipfile.BadZipFile:
|
|
94
|
+
return # 非 zip 留给 anydoc 自己报错
|
|
95
|
+
raise RuntimeError(f"EPUB 含 DRM 加密{vendor},内容为密文,不支持解析")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _extract_images(document: object, image_store: "ImageStore", rel_path: str) -> list[str]:
|
|
99
|
+
"""提取 assets 中的图片落盘,返回 inline_md 列表(文档序)。
|
|
100
|
+
|
|
101
|
+
仅 image/* 媒体类型落盘;OLE 对象等非图 asset 跳过。
|
|
102
|
+
anydoc 不提供图片在正文中的位置,disp_w 未知(前端退回 naturalWidth)。
|
|
103
|
+
"""
|
|
104
|
+
from .image_store import ImagePart
|
|
105
|
+
|
|
106
|
+
parts: list["ImagePart"] = []
|
|
107
|
+
for asset in document.assets:
|
|
108
|
+
media_type = (asset.media_type or "").lower()
|
|
109
|
+
if not media_type.startswith("image/"):
|
|
110
|
+
logger.debug("skip non-image asset: %s (%s)", asset.origin_part, media_type)
|
|
111
|
+
continue
|
|
112
|
+
parts.append(
|
|
113
|
+
ImagePart(
|
|
114
|
+
blob=asset.data,
|
|
115
|
+
ext=_MEDIA_TYPE_TO_EXT.get(media_type, "png"),
|
|
116
|
+
source_ref=str(asset.id),
|
|
117
|
+
disp_w=None,
|
|
118
|
+
)
|
|
119
|
+
)
|
|
120
|
+
if not parts:
|
|
121
|
+
return []
|
|
122
|
+
refs = image_store.extract_for_doc(rel_path, parts)
|
|
123
|
+
return [refs[p.source_ref].inline_md for p in parts if p.source_ref in refs]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
async def anydoc_to_tree(
|
|
127
|
+
file_path: str,
|
|
128
|
+
*,
|
|
129
|
+
model: Optional[str] = None,
|
|
130
|
+
if_add_node_summary: bool = True,
|
|
131
|
+
summary_chars_threshold: int = 600,
|
|
132
|
+
if_add_doc_description: bool = False,
|
|
133
|
+
if_add_node_text: bool = False,
|
|
134
|
+
if_add_node_id: bool = True,
|
|
135
|
+
image_store: "ImageStore | None" = None,
|
|
136
|
+
rel_path: str | None = None,
|
|
137
|
+
**kwargs,
|
|
138
|
+
) -> dict:
|
|
139
|
+
"""Build a tree index from a document via anydoc.
|
|
140
|
+
|
|
141
|
+
Converts the document to Markdown using ``anydoc``, extracts embedded
|
|
142
|
+
images to ImageStore, then delegates to ``md_to_tree`` for structure
|
|
143
|
+
extraction.
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
{'doc_name': str, 'structure': list, 'source_path': str}
|
|
147
|
+
"""
|
|
148
|
+
try:
|
|
149
|
+
import anydoc # noqa: F401
|
|
150
|
+
except ImportError:
|
|
151
|
+
raise ImportError(
|
|
152
|
+
"anydoc support requires 'firecrawl-anydoc'. "
|
|
153
|
+
"Install with: pip install firecrawl-anydoc"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
doc_name = os.path.splitext(os.path.basename(file_path))[0]
|
|
157
|
+
logger.debug("Parsing with anydoc: %s", file_path)
|
|
158
|
+
|
|
159
|
+
with open(file_path, "rb") as f:
|
|
160
|
+
data = f.read()
|
|
161
|
+
|
|
162
|
+
if file_path.lower().endswith(".epub"):
|
|
163
|
+
_check_epub_drm(data)
|
|
164
|
+
|
|
165
|
+
md_content, document = _convert(data)
|
|
166
|
+
if not md_content.strip():
|
|
167
|
+
logger.warning("anydoc returned empty content for: %s", file_path)
|
|
168
|
+
md_content = ""
|
|
169
|
+
|
|
170
|
+
# 内嵌图片提取:anydoc 的 md 不含图片位置,统一附加到文档末尾
|
|
171
|
+
if image_store is not None and rel_path:
|
|
172
|
+
try:
|
|
173
|
+
image_mds = _extract_images(document, image_store, rel_path)
|
|
174
|
+
if image_mds:
|
|
175
|
+
md_content = md_content.rstrip() + "\n\n" + "\n\n".join(image_mds)
|
|
176
|
+
except Exception as e: # noqa: BLE001
|
|
177
|
+
logger.warning("anydoc image extraction failed for %s: %s", file_path, e)
|
|
178
|
+
|
|
179
|
+
from ..indexer import md_to_tree
|
|
180
|
+
|
|
181
|
+
tree_result = await md_to_tree(
|
|
182
|
+
md_content=md_content,
|
|
183
|
+
model=model,
|
|
184
|
+
if_add_node_summary=if_add_node_summary,
|
|
185
|
+
summary_chars_threshold=summary_chars_threshold,
|
|
186
|
+
if_add_doc_description=if_add_doc_description,
|
|
187
|
+
if_add_node_text=if_add_node_text,
|
|
188
|
+
if_add_node_id=if_add_node_id,
|
|
189
|
+
**kwargs,
|
|
190
|
+
)
|
|
191
|
+
tree_result["doc_name"] = doc_name
|
|
192
|
+
tree_result["source_path"] = os.path.abspath(file_path)
|
|
193
|
+
return tree_result
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
@author:XuMing(xuming624@qq.com)
|
|
4
|
+
@description: Python AST-based code structure parser.
|
|
5
|
+
|
|
6
|
+
Uses the built-in ``ast`` module to extract classes and functions from
|
|
7
|
+
Python source files, replacing regex-based detection for `.py` files.
|
|
8
|
+
Falls back to regex if AST parsing fails (e.g. syntax errors).
|
|
9
|
+
"""
|
|
10
|
+
import ast
|
|
11
|
+
import logging
|
|
12
|
+
import warnings
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def parse_python_structure(source: str) -> list[dict]:
|
|
18
|
+
"""Parse Python source code using AST and return a flat heading list.
|
|
19
|
+
|
|
20
|
+
Returns:
|
|
21
|
+
list of {'title': str, 'line_num': int, 'level': int}
|
|
22
|
+
level 1 = class, level 2 = function/method
|
|
23
|
+
"""
|
|
24
|
+
try:
|
|
25
|
+
with warnings.catch_warnings():
|
|
26
|
+
warnings.simplefilter("ignore", SyntaxWarning)
|
|
27
|
+
tree = ast.parse(source)
|
|
28
|
+
except SyntaxError as e:
|
|
29
|
+
logger.debug("AST parse failed (syntax error at line %s), falling back to regex", e.lineno)
|
|
30
|
+
return []
|
|
31
|
+
|
|
32
|
+
headings = []
|
|
33
|
+
|
|
34
|
+
for node in ast.walk(tree):
|
|
35
|
+
if isinstance(node, ast.ClassDef):
|
|
36
|
+
# Build class signature with bases
|
|
37
|
+
bases = ", ".join(_name_of(b) for b in node.bases) if node.bases else ""
|
|
38
|
+
title = f"class {node.name}({bases})" if bases else f"class {node.name}"
|
|
39
|
+
headings.append({
|
|
40
|
+
"title": title,
|
|
41
|
+
"line_num": node.lineno,
|
|
42
|
+
"level": 1,
|
|
43
|
+
})
|
|
44
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
45
|
+
prefix = "async def" if isinstance(node, ast.AsyncFunctionDef) else "def"
|
|
46
|
+
args_str = _format_args(node.args)
|
|
47
|
+
# Return annotation
|
|
48
|
+
ret = ""
|
|
49
|
+
if node.returns:
|
|
50
|
+
ret = f" -> {_name_of(node.returns)}"
|
|
51
|
+
title = f"{prefix} {node.name}({args_str}){ret}"
|
|
52
|
+
# Truncate long signatures
|
|
53
|
+
if len(title) > 120:
|
|
54
|
+
title = title[:117] + "..."
|
|
55
|
+
headings.append({
|
|
56
|
+
"title": title,
|
|
57
|
+
"line_num": node.lineno,
|
|
58
|
+
"level": 2,
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
# Sort by line number (ast.walk doesn't guarantee order)
|
|
62
|
+
headings.sort(key=lambda h: h["line_num"])
|
|
63
|
+
return headings
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _name_of(node) -> str:
|
|
67
|
+
"""Extract a readable name from an AST node."""
|
|
68
|
+
if isinstance(node, ast.Name):
|
|
69
|
+
return node.id
|
|
70
|
+
elif isinstance(node, ast.Attribute):
|
|
71
|
+
return f"{_name_of(node.value)}.{node.attr}"
|
|
72
|
+
elif isinstance(node, ast.Constant):
|
|
73
|
+
return repr(node.value)
|
|
74
|
+
elif isinstance(node, ast.Subscript):
|
|
75
|
+
return f"{_name_of(node.value)}[{_name_of(node.slice)}]"
|
|
76
|
+
elif isinstance(node, ast.Tuple):
|
|
77
|
+
return ", ".join(_name_of(e) for e in node.elts)
|
|
78
|
+
elif isinstance(node, ast.List):
|
|
79
|
+
return "[" + ", ".join(_name_of(e) for e in node.elts) + "]"
|
|
80
|
+
elif isinstance(node, ast.Starred):
|
|
81
|
+
return f"*{_name_of(node.value)}"
|
|
82
|
+
elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
|
|
83
|
+
return f"{_name_of(node.left)} | {_name_of(node.right)}"
|
|
84
|
+
return "..."
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _format_args(args: ast.arguments) -> str:
|
|
88
|
+
"""Format function arguments to a readable string."""
|
|
89
|
+
parts = []
|
|
90
|
+
|
|
91
|
+
# Positional-only
|
|
92
|
+
for a in args.posonlyargs:
|
|
93
|
+
parts.append(a.arg)
|
|
94
|
+
if args.posonlyargs:
|
|
95
|
+
parts.append("/")
|
|
96
|
+
|
|
97
|
+
# Regular args
|
|
98
|
+
defaults_offset = len(args.args) - len(args.defaults)
|
|
99
|
+
for i, a in enumerate(args.args):
|
|
100
|
+
name = a.arg
|
|
101
|
+
if a.annotation:
|
|
102
|
+
name += f": {_name_of(a.annotation)}"
|
|
103
|
+
di = i - defaults_offset
|
|
104
|
+
if di >= 0 and di < len(args.defaults):
|
|
105
|
+
name += f"={_name_of(args.defaults[di])}"
|
|
106
|
+
parts.append(name)
|
|
107
|
+
|
|
108
|
+
# *args
|
|
109
|
+
if args.vararg:
|
|
110
|
+
parts.append(f"*{args.vararg.arg}")
|
|
111
|
+
elif args.kwonlyargs:
|
|
112
|
+
parts.append("*")
|
|
113
|
+
|
|
114
|
+
# Keyword-only
|
|
115
|
+
for i, a in enumerate(args.kwonlyargs):
|
|
116
|
+
name = a.arg
|
|
117
|
+
if a.annotation:
|
|
118
|
+
name += f": {_name_of(a.annotation)}"
|
|
119
|
+
if i < len(args.kw_defaults) and args.kw_defaults[i] is not None:
|
|
120
|
+
name += f"={_name_of(args.kw_defaults[i])}"
|
|
121
|
+
parts.append(name)
|
|
122
|
+
|
|
123
|
+
# **kwargs
|
|
124
|
+
if args.kwarg:
|
|
125
|
+
parts.append(f"**{args.kwarg.arg}")
|
|
126
|
+
|
|
127
|
+
result = ", ".join(parts)
|
|
128
|
+
# Skip 'self' / 'cls' at the beginning for readability
|
|
129
|
+
for prefix in ("self, ", "cls, "):
|
|
130
|
+
if result.startswith(prefix):
|
|
131
|
+
result = result[len(prefix):]
|
|
132
|
+
break
|
|
133
|
+
if result in ("self", "cls"):
|
|
134
|
+
result = ""
|
|
135
|
+
|
|
136
|
+
return result
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
@author:XuMing(xuming624@qq.com)
|
|
4
|
+
@description: DOCX parser for TreeSearch.
|
|
5
|
+
|
|
6
|
+
Requires optional dependency: ``pip install python-docx``
|
|
7
|
+
Extracts paragraphs and headings from DOCX and builds tree structure.
|
|
8
|
+
"""
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
from typing import Optional, TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from .image_store import ImagePart, ImageStore
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _escape_md_cell(s: str) -> str:
|
|
20
|
+
"""转义 md table 单元格:``|`` → ``\\|``,换行 → 空格,回车删除。
|
|
21
|
+
|
|
22
|
+
与 csv/xlsx 在 preview_synthesizer._escape_md_cell 的口径一致。
|
|
23
|
+
"""
|
|
24
|
+
return s.replace("|", "\\|").replace("\n", " ").replace("\r", "")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _table_to_text(table) -> str:
|
|
28
|
+
"""Convert a docx Table to a GitHub-flavored markdown table string.
|
|
29
|
+
|
|
30
|
+
首行视为 header,输出形如::
|
|
31
|
+
|
|
32
|
+
| h1 | h2 |
|
|
33
|
+
| --- | --- |
|
|
34
|
+
| v1 | v2 |
|
|
35
|
+
|
|
36
|
+
这样前端 ``<md-viewer>`` 才能识别为真正的表格(缺 separator 行只会被
|
|
37
|
+
当成软换行段落)。单元格里的 ``|`` 转义为 ``\\|``,换行替换为空格。
|
|
38
|
+
"""
|
|
39
|
+
rows = []
|
|
40
|
+
ncols = 0
|
|
41
|
+
for row in table.rows:
|
|
42
|
+
cells = [_escape_md_cell(cell.text.strip()) for cell in row.cells]
|
|
43
|
+
ncols = max(ncols, len(cells))
|
|
44
|
+
rows.append("| " + " | ".join(cells) + " |")
|
|
45
|
+
if not rows or ncols == 0:
|
|
46
|
+
return ""
|
|
47
|
+
separator = "| " + " | ".join(["---"] * ncols) + " |"
|
|
48
|
+
return "\n".join([rows[0], separator, *rows[1:]])
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _paragraph_image_rids(para) -> list[str]:
|
|
52
|
+
"""段落内所有图片的 rId(按文档顺序,含 inline + anchor)。"""
|
|
53
|
+
from docx.oxml.ns import qn
|
|
54
|
+
return [blip.get(qn("r:embed"))
|
|
55
|
+
for blip in para._p.findall(".//" + qn("a:blip"))
|
|
56
|
+
if blip.get(qn("r:embed"))]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _list_marker(para) -> str:
|
|
60
|
+
"""列表段落 → md 列表符号前缀(按 ilvl 缩进);非列表 → ''。
|
|
61
|
+
|
|
62
|
+
``para.text`` 不含项目符号/编号(numPr 由 Word 渲染期生成),导致 docx
|
|
63
|
+
列表在索引与预览里退化为普通段落。检测段落级 ``w:numPr``(Word 列表的
|
|
64
|
+
常规编码)或 List 系样式名,补回 ``- `` 前缀——与 anydoc 的 docx→md
|
|
65
|
+
输出口径一致(统一 ``-``,不区分有序/无序)。标题样式直接跳过(编号
|
|
66
|
+
标题不应变成列表项)。
|
|
67
|
+
"""
|
|
68
|
+
from docx.oxml.ns import qn
|
|
69
|
+
|
|
70
|
+
style_name = ((para.style.name if para.style else "") or "").lower()
|
|
71
|
+
if style_name.startswith("heading"):
|
|
72
|
+
return ""
|
|
73
|
+
|
|
74
|
+
pPr = para._p.pPr
|
|
75
|
+
numPr = pPr.find(qn("w:numPr")) if pPr is not None else None
|
|
76
|
+
if numPr is None:
|
|
77
|
+
return "- " if "list" in style_name else ""
|
|
78
|
+
ilvl = 0
|
|
79
|
+
ilvl_el = numPr.find(qn("w:ilvl"))
|
|
80
|
+
if ilvl_el is not None:
|
|
81
|
+
try:
|
|
82
|
+
ilvl = int(ilvl_el.get(qn("w:val"), 0))
|
|
83
|
+
except (ValueError, TypeError):
|
|
84
|
+
ilvl = 0
|
|
85
|
+
return " " * ilvl + "- "
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _docx_part_blob_ext(doc, rid: str) -> tuple[bytes, str] | None:
|
|
89
|
+
"""按 rId 取 image part 的 (blob, ext),失败返回 None。"""
|
|
90
|
+
try:
|
|
91
|
+
part = doc.part.related_parts.get(rid)
|
|
92
|
+
if part is None:
|
|
93
|
+
return None
|
|
94
|
+
blob = part.blob
|
|
95
|
+
ext = ""
|
|
96
|
+
try:
|
|
97
|
+
ext = part.image.ext
|
|
98
|
+
except Exception:
|
|
99
|
+
partname = getattr(part, "partname", None)
|
|
100
|
+
ext = partname.ext.lstrip(".") if partname and partname.ext else ""
|
|
101
|
+
return blob, (ext or "png")
|
|
102
|
+
except Exception as e:
|
|
103
|
+
logger.warning("Failed to extract docx image part %s: %s", rid, e)
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _docx_rid_disp_w_map(doc) -> dict[str, int]:
|
|
108
|
+
"""遍历 body 内所有 drawing,返回 {rId: 显示宽 px}(EMU÷9525)。
|
|
109
|
+
|
|
110
|
+
每个 ``<w:drawing>`` 内 ``<wp:extent cx>`` 与 ``<a:blip r:embed>`` 关联,
|
|
111
|
+
覆盖 inline 与 anchor 两种锚定方式;取不到 extent 的图不在映射里(disp_w=None)。
|
|
112
|
+
"""
|
|
113
|
+
from docx.oxml.ns import qn
|
|
114
|
+
|
|
115
|
+
mapping: dict[str, int] = {}
|
|
116
|
+
for drawing in doc.element.body.findall(".//" + qn("w:drawing")):
|
|
117
|
+
extent = drawing.find(".//" + qn("wp:extent"))
|
|
118
|
+
blip = drawing.find(".//" + qn("a:blip"))
|
|
119
|
+
if extent is None or blip is None:
|
|
120
|
+
continue
|
|
121
|
+
rid = blip.get(qn("r:embed"))
|
|
122
|
+
cx = extent.get("cx")
|
|
123
|
+
if not rid or not cx:
|
|
124
|
+
continue
|
|
125
|
+
try:
|
|
126
|
+
mapping[rid] = round(int(cx) / 9525)
|
|
127
|
+
except (ValueError, TypeError):
|
|
128
|
+
continue
|
|
129
|
+
return mapping
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _extract_docx_headings(
|
|
133
|
+
docx_path: str,
|
|
134
|
+
image_store: "ImageStore | None" = None,
|
|
135
|
+
rel_path: str | None = None,
|
|
136
|
+
) -> tuple[list[dict], list[str]]:
|
|
137
|
+
"""Extract headings and text from a DOCX file.
|
|
138
|
+
|
|
139
|
+
Iterates document body elements in order so that tables are
|
|
140
|
+
interleaved with paragraphs at the correct position.
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
(headings, lines) where headings is a list of
|
|
144
|
+
{'title': str, 'line_num': int, 'level': int}
|
|
145
|
+
"""
|
|
146
|
+
try:
|
|
147
|
+
from docx import Document as DocxDocument
|
|
148
|
+
from docx.oxml.ns import qn
|
|
149
|
+
from docx.table import Table
|
|
150
|
+
from docx.text.paragraph import Paragraph
|
|
151
|
+
except ImportError:
|
|
152
|
+
raise ImportError(
|
|
153
|
+
"DOCX support requires 'python-docx'. Install with: pip install python-docx"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
doc = DocxDocument(docx_path)
|
|
157
|
+
lines = []
|
|
158
|
+
headings = []
|
|
159
|
+
para_image_rids: list[list[str]] = []
|
|
160
|
+
|
|
161
|
+
for child in doc.element.body:
|
|
162
|
+
tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
|
|
163
|
+
|
|
164
|
+
if tag == "p":
|
|
165
|
+
para = Paragraph(child, doc)
|
|
166
|
+
text = para.text.strip()
|
|
167
|
+
if text:
|
|
168
|
+
text = _list_marker(para) + text
|
|
169
|
+
rids = _paragraph_image_rids(para)
|
|
170
|
+
line_num = len(lines) + 1
|
|
171
|
+
lines.append(text)
|
|
172
|
+
para_image_rids.append(rids)
|
|
173
|
+
|
|
174
|
+
style_name = (para.style.name if para.style else "") or ""
|
|
175
|
+
style_name = style_name.lower()
|
|
176
|
+
if style_name.startswith("heading"):
|
|
177
|
+
try:
|
|
178
|
+
level = int(style_name.split()[-1])
|
|
179
|
+
except (ValueError, IndexError):
|
|
180
|
+
level = 1
|
|
181
|
+
if text:
|
|
182
|
+
headings.append({
|
|
183
|
+
"title": text,
|
|
184
|
+
"line_num": line_num,
|
|
185
|
+
"level": level,
|
|
186
|
+
})
|
|
187
|
+
|
|
188
|
+
elif tag == "tbl":
|
|
189
|
+
table = Table(child, doc)
|
|
190
|
+
table_text = _table_to_text(table)
|
|
191
|
+
if table_text.strip():
|
|
192
|
+
line_num = len(lines) + 1
|
|
193
|
+
lines.append(table_text)
|
|
194
|
+
para_image_rids.append([])
|
|
195
|
+
|
|
196
|
+
# 图片提取 + 段落级锚定
|
|
197
|
+
if image_store is not None and rel_path:
|
|
198
|
+
unique_rids: list[str] = []
|
|
199
|
+
seen: set[str] = set()
|
|
200
|
+
for rids in para_image_rids:
|
|
201
|
+
for r in rids:
|
|
202
|
+
if r and r not in seen:
|
|
203
|
+
seen.add(r)
|
|
204
|
+
unique_rids.append(r)
|
|
205
|
+
disp_map = _docx_rid_disp_w_map(doc)
|
|
206
|
+
parts_list = []
|
|
207
|
+
for rid in unique_rids:
|
|
208
|
+
be = _docx_part_blob_ext(doc, rid)
|
|
209
|
+
if be is not None:
|
|
210
|
+
from .image_store import ImagePart
|
|
211
|
+
parts_list.append(
|
|
212
|
+
ImagePart(
|
|
213
|
+
blob=be[0], ext=be[1], source_ref=rid,
|
|
214
|
+
disp_w=disp_map.get(rid),
|
|
215
|
+
)
|
|
216
|
+
)
|
|
217
|
+
refs = image_store.extract_for_doc(rel_path, parts_list) if parts_list else {}
|
|
218
|
+
for i, rids in enumerate(para_image_rids):
|
|
219
|
+
mds = [refs[r].inline_md for r in rids if r in refs]
|
|
220
|
+
if mds:
|
|
221
|
+
base = lines[i]
|
|
222
|
+
lines[i] = (base + "\n\n" + " ".join(mds)) if base else " ".join(mds)
|
|
223
|
+
|
|
224
|
+
return headings, lines
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
async def docx_to_tree(
|
|
228
|
+
docx_path: str,
|
|
229
|
+
*,
|
|
230
|
+
model: Optional[str] = None,
|
|
231
|
+
if_add_node_summary: bool = True,
|
|
232
|
+
summary_chars_threshold: int = 600,
|
|
233
|
+
if_add_doc_description: bool = False,
|
|
234
|
+
if_add_node_text: bool = False,
|
|
235
|
+
if_add_node_id: bool = True,
|
|
236
|
+
image_store: "ImageStore | None" = None,
|
|
237
|
+
rel_path: str | None = None,
|
|
238
|
+
**kwargs,
|
|
239
|
+
) -> dict:
|
|
240
|
+
"""Build a tree index from a DOCX file.
|
|
241
|
+
|
|
242
|
+
Extracts headings via ``python-docx`` paragraph styles. If no headings
|
|
243
|
+
are found, falls back to ``text_to_tree`` with plain text content.
|
|
244
|
+
|
|
245
|
+
Returns:
|
|
246
|
+
{'doc_name': str, 'structure': list, 'source_path': str}
|
|
247
|
+
"""
|
|
248
|
+
doc_name = os.path.splitext(os.path.basename(docx_path))[0]
|
|
249
|
+
logger.debug("Parsing DOCX: %s", docx_path)
|
|
250
|
+
|
|
251
|
+
headings, lines = _extract_docx_headings(docx_path, image_store=image_store, rel_path=rel_path)
|
|
252
|
+
|
|
253
|
+
if not headings:
|
|
254
|
+
# No DOCX headings found, fall back to text_to_tree.
|
|
255
|
+
# 用 \n\n 分段(过滤空段),让 text_to_tree 和下游 preview 都能正确识别段落边界。
|
|
256
|
+
text_content = "\n\n".join(ln for ln in lines if ln.strip())
|
|
257
|
+
from ..indexer import text_to_tree
|
|
258
|
+
result = await text_to_tree(
|
|
259
|
+
text_content=text_content,
|
|
260
|
+
model=model,
|
|
261
|
+
if_add_node_summary=if_add_node_summary,
|
|
262
|
+
summary_chars_threshold=summary_chars_threshold,
|
|
263
|
+
if_add_doc_description=if_add_doc_description,
|
|
264
|
+
if_add_node_text=if_add_node_text,
|
|
265
|
+
if_add_node_id=if_add_node_id,
|
|
266
|
+
**kwargs,
|
|
267
|
+
)
|
|
268
|
+
result["doc_name"] = doc_name
|
|
269
|
+
result["source_path"] = os.path.abspath(docx_path)
|
|
270
|
+
return result
|
|
271
|
+
|
|
272
|
+
# Build nodes from headings
|
|
273
|
+
from ..indexer import _build_tree, _finalize_tree
|
|
274
|
+
|
|
275
|
+
nodes = []
|
|
276
|
+
for i, hd in enumerate(headings):
|
|
277
|
+
start = hd["line_num"] - 1
|
|
278
|
+
end = headings[i + 1]["line_num"] - 1 if i + 1 < len(headings) else len(lines)
|
|
279
|
+
# 用 \n\n 分隔非空段落,保留 docx 段落结构。
|
|
280
|
+
# 否则 preview synthesizer 输出单 \n,CommonMark 渲染会把多段合并成一段。
|
|
281
|
+
# 表格项作为整体参与连接(内部保持单 \n),与相邻段落正确分段。
|
|
282
|
+
paragraphs = [ln for ln in lines[start:end] if ln.strip()]
|
|
283
|
+
text = "\n\n".join(paragraphs).strip()
|
|
284
|
+
nodes.append({
|
|
285
|
+
"title": hd["title"],
|
|
286
|
+
"line_num": hd["line_num"],
|
|
287
|
+
"line_start": hd["line_num"],
|
|
288
|
+
"line_end": end,
|
|
289
|
+
"level": hd["level"],
|
|
290
|
+
"text": text,
|
|
291
|
+
})
|
|
292
|
+
|
|
293
|
+
tree = _build_tree(nodes)
|
|
294
|
+
|
|
295
|
+
return _finalize_tree(
|
|
296
|
+
tree, doc_name,
|
|
297
|
+
source_path=os.path.abspath(docx_path),
|
|
298
|
+
source_type="docx",
|
|
299
|
+
if_add_node_id=if_add_node_id,
|
|
300
|
+
if_add_node_summary=if_add_node_summary,
|
|
301
|
+
summary_chars_threshold=summary_chars_threshold,
|
|
302
|
+
if_add_node_text=if_add_node_text,
|
|
303
|
+
if_add_doc_description=if_add_doc_description,
|
|
304
|
+
)
|