yimo-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yimo/__init__.py +26 -0
- yimo/kit/__init__.py +98 -0
- yimo/kit/ai/__init__.py +110 -0
- yimo/kit/ai/_internal/__init__.py +27 -0
- yimo/kit/ai/_internal/failover.py +535 -0
- yimo/kit/ai/_internal/pool.py +526 -0
- yimo/kit/ai/_internal/rerank_transport.py +137 -0
- yimo/kit/ai/_internal/retry_after.py +51 -0
- yimo/kit/ai/_internal/sdk_client.py +463 -0
- yimo/kit/ai/_internal/service_base.py +115 -0
- yimo/kit/ai/_internal/types.py +28 -0
- yimo/kit/ai/agent/__init__.py +127 -0
- yimo/kit/ai/agent/approval.py +66 -0
- yimo/kit/ai/agent/errors.py +128 -0
- yimo/kit/ai/agent/loop.py +643 -0
- yimo/kit/ai/agent/mcp.py +278 -0
- yimo/kit/ai/agent/registry.py +203 -0
- yimo/kit/ai/agent/session.py +150 -0
- yimo/kit/ai/agent/skill.py +370 -0
- yimo/kit/ai/agent/types.py +161 -0
- yimo/kit/ai/chat_service.py +724 -0
- yimo/kit/ai/decision/__init__.py +57 -0
- yimo/kit/ai/decision/decima.py +853 -0
- yimo/kit/ai/decision/laya.py +552 -0
- yimo/kit/ai/embedding_service.py +198 -0
- yimo/kit/ai/exceptions.py +109 -0
- yimo/kit/ai/image_service.py +84 -0
- yimo/kit/ai/messages.py +105 -0
- yimo/kit/ai/model_builder.py +121 -0
- yimo/kit/ai/model_client.py +62 -0
- yimo/kit/ai/profiles.py +577 -0
- yimo/kit/ai/rerank_service.py +136 -0
- yimo/kit/ai/runner.py +1127 -0
- yimo/kit/common_kit.py +221 -0
- yimo/kit/config/__init__.py +44 -0
- yimo/kit/config/kit_config.py +210 -0
- yimo/kit/config/log_config.py +171 -0
- yimo/kit/crawl/__init__.py +71 -0
- yimo/kit/crawl/article_io.py +325 -0
- yimo/kit/crawl/article_models.py +172 -0
- yimo/kit/crawl/deepseek_kit.py +673 -0
- yimo/kit/crawl/web_article_kit.py +638 -0
- yimo/kit/crawl/wx_article_kit.py +692 -0
- yimo/kit/db/__init__.py +101 -0
- yimo/kit/db/audit.py +625 -0
- yimo/kit/db/base_db_kit.py +1265 -0
- yimo/kit/db/crud.py +423 -0
- yimo/kit/db/engine.py +235 -0
- yimo/kit/db/exceptions.py +45 -0
- yimo/kit/db/models/__init__.py +33 -0
- yimo/kit/db/models/base.py +56 -0
- yimo/kit/db/mysql_kit.py +348 -0
- yimo/kit/db/pg_kit.py +442 -0
- yimo/kit/db/query.py +338 -0
- yimo/kit/db/session.py +85 -0
- yimo/kit/db/sqlite_kit.py +462 -0
- yimo/kit/db/transaction.py +109 -0
- yimo/kit/docs/__init__.py +34 -0
- yimo/kit/docs/anydoc_kit.py +490 -0
- yimo/kit/docs/docx_kit.py +895 -0
- yimo/kit/docs/md2doc_kit.py +1801 -0
- yimo/kit/docs/mineru_kit.py +1497 -0
- yimo/kit/docs/rc_docparser_kit.py +654 -0
- yimo/kit/file_kit.py +518 -0
- yimo/kit/license_kit.py +533 -0
- yimo/kit/redis_kit.py +392 -0
- yimo_kit-0.1.0.dist-info/METADATA +1673 -0
- yimo_kit-0.1.0.dist-info/RECORD +70 -0
- yimo_kit-0.1.0.dist-info/WHEEL +4 -0
- yimo_kit-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,692 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Copyright (c) 2021-2026 溢墨修心(Yimo). All Rights Reserved.
|
|
3
|
+
|
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
you may not use this file except in compliance with the License.
|
|
6
|
+
You may obtain a copy of the License at
|
|
7
|
+
|
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
|
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
See the License for the specific language governing permissions and
|
|
14
|
+
limitations under the License.
|
|
15
|
+
|
|
16
|
+
Project: yimo
|
|
17
|
+
Author: 溢墨修心(Yimo)
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
# date: 2026-09-24
|
|
21
|
+
"""微信公众号文章抓取工具类 WxArticleKit
|
|
22
|
+
|
|
23
|
+
输入单篇公众号文章 URL(mp.weixin.qq.com/s/...),抓取正文并输出结构化信息:
|
|
24
|
+
|
|
25
|
+
- `fetch(url)`:抓取 + 解析(不落盘)→ `ArticleResult`(Markdown + 元信息);
|
|
26
|
+
- `save(url, output_dir, file_name=None, write_meta=True)`:抓取 → 存储到
|
|
27
|
+
`<output_dir>/<stem>/` 下(落盘由 `yimo.kit.crawl.article_io` 统一完成):
|
|
28
|
+
- `<stem>.md`:正文 Markdown(含 YAML 风格元信息头,发布时间为中文日期);
|
|
29
|
+
- `images/`:图片转存目录(md 引用同步本地化,失败保留原 URL);
|
|
30
|
+
- `meta.json`:结构化元信息(默认产出,`write_meta=False` 关闭)。
|
|
31
|
+
省略 `file_name` 时目录按本地时间 `yymmddhhmmss` 自动命名(同秒追加 `-2`…)。
|
|
32
|
+
|
|
33
|
+
设计要点:
|
|
34
|
+
- 零第三方依赖:HTTP 抓取用标准库 urllib;HTML→Markdown 为模块内私有块级转换器
|
|
35
|
+
(原 research/html2md.py 的微信正文所需子集),模块自包含,不依赖 research/,
|
|
36
|
+
也不依赖其它 kit 模块的可选能力。
|
|
37
|
+
- 模型共用:`ArticleInfo` / `ArticleResult` / `SavedOutput` / `ArticleImage` 等
|
|
38
|
+
已收敛到 `yimo.kit.crawl.article_models`(三个 `*ArticleKit` 共用同一套平台中立模型,
|
|
39
|
+
便于多平台抓取后写入同一张表),本模块仍从该处 import 同名模型,旧引用方式不变;
|
|
40
|
+
失败时 `ArticleResult.reason` 为类型化原因(如 `risk_control` / `no_content`)。
|
|
41
|
+
- 反爬纪律:浏览器 UA + 合理超时;无登录态;命中风控特征(环境异常/访问过于频繁)
|
|
42
|
+
时返回结构化错误,不破解验证码。
|
|
43
|
+
- 除调用方错误(非法 URL)外不抛异常:抓取/下载失败以 `ok=False` 或空值返回并记日志。
|
|
44
|
+
|
|
45
|
+
使用方式(按需引用,无需经 kit 包顶层导出):
|
|
46
|
+
|
|
47
|
+
from yimo.kit.crawl.wx_article_kit import WxArticleKit
|
|
48
|
+
|
|
49
|
+
result = WxArticleKit.fetch("https://mp.weixin.qq.com/s/xxxx")
|
|
50
|
+
if result.ok:
|
|
51
|
+
print(result.info.author, result.markdown)
|
|
52
|
+
|
|
53
|
+
saved = WxArticleKit.save(url, output_dir="output", file_name="12345")
|
|
54
|
+
# output/12345/ 下:12345.md + images/ + meta.json
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
import html
|
|
58
|
+
import re
|
|
59
|
+
import socket
|
|
60
|
+
import urllib.error
|
|
61
|
+
import urllib.parse
|
|
62
|
+
import urllib.request
|
|
63
|
+
from datetime import datetime, timedelta, timezone
|
|
64
|
+
from http.client import HTTPResponse
|
|
65
|
+
from pathlib import Path
|
|
66
|
+
|
|
67
|
+
from yimo.kit.config.log_config import get_default_logger
|
|
68
|
+
|
|
69
|
+
# 落盘公共层(md + images/ + meta.json,三个 Kit 共用同一契约)
|
|
70
|
+
from yimo.kit.crawl import article_io
|
|
71
|
+
|
|
72
|
+
# 通用模型与 WebArticleKit 共用(多平台抓取后写入同一张表)
|
|
73
|
+
from yimo.kit.crawl.article_models import (
|
|
74
|
+
REASON_FETCH_FAILED,
|
|
75
|
+
REASON_NO_CONTENT,
|
|
76
|
+
REASON_RISK_CONTROL,
|
|
77
|
+
ArticleImage,
|
|
78
|
+
ArticleInfo,
|
|
79
|
+
ArticleLink,
|
|
80
|
+
ArticleResult,
|
|
81
|
+
CodeBlock,
|
|
82
|
+
SavedOutput,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
logger = get_default_logger(__name__)
|
|
86
|
+
|
|
87
|
+
# 来源标识(通用模型 ArticleInfo 的 source / site_host 字段)
|
|
88
|
+
_SOURCE_NAME = "微信公众平台"
|
|
89
|
+
_SITE_HOST = "mp.weixin.qq.com"
|
|
90
|
+
|
|
91
|
+
# Asia/Shanghai(东八区,无夏令时)固定偏移
|
|
92
|
+
_TZ_SHANGHAI = timezone(timedelta(hours=8))
|
|
93
|
+
|
|
94
|
+
# 请求默认超时(秒)
|
|
95
|
+
_DEFAULT_TIMEOUT = 20.0
|
|
96
|
+
|
|
97
|
+
# 反爬:伪装常规浏览器(无登录态)
|
|
98
|
+
_DEFAULT_UA = (
|
|
99
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
100
|
+
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
# 正文区边界:<div id="js_content"> 起始,id="js_pc_qr_code"(推广/二维码区)为硬边界
|
|
104
|
+
_JS_CONTENT_RE = re.compile(r'<div[^>]*id="js_content"[^>]*>', re.I)
|
|
105
|
+
_JS_PC_QR_RE = re.compile(r'id="js_pc_qr_code"')
|
|
106
|
+
|
|
107
|
+
# 元信息提取
|
|
108
|
+
_TITLE_RE = re.compile(
|
|
109
|
+
r'<h1[^>]*id="activity-name"[^>]*>\s*'
|
|
110
|
+
r'<span[^>]*class="js_title_inner"[^>]*>(.*?)</span>',
|
|
111
|
+
re.S,
|
|
112
|
+
)
|
|
113
|
+
_OG_TITLE_RE = re.compile(r'<meta[^>]*property="og:title"[^>]*content="([^"]*)"')
|
|
114
|
+
_NAME_RE = re.compile(r'id="js_name"[^>]*>(.*?)</', re.S)
|
|
115
|
+
_CT_RE = re.compile(r'var ct = "(\d+)"')
|
|
116
|
+
_SUMMARY_RE = re.compile(r'<meta[^>]*name="description"[^>]*content="([^"]*)"')
|
|
117
|
+
|
|
118
|
+
# 风控特征(命中即返回结构化错误,不破解验证码)
|
|
119
|
+
_RISK_MARKERS = ("环境异常", "访问过于频繁", "请完成验证", "user verify")
|
|
120
|
+
|
|
121
|
+
# 图片格式(从 URL 的 wx_fmt 参数识别)
|
|
122
|
+
_IMG_FMT_RE = re.compile(r"wx_fmt=(\w+)")
|
|
123
|
+
|
|
124
|
+
# 顶层块标签(用于正文区切块;pre 需整体保留)
|
|
125
|
+
_BLOCK_TAGS = ("pre", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "ol", "ul", "p")
|
|
126
|
+
|
|
127
|
+
# 内联样式标签(正则,左标记,右标记)
|
|
128
|
+
_INLINE_STYLES: list[tuple[str, str, str]] = [
|
|
129
|
+
(r"<strong\b[^>]*>(.*?)</strong>", "**", "**"),
|
|
130
|
+
(r"<b\b[^>]*>(.*?)</b>", "**", "**"),
|
|
131
|
+
(r"<em\b[^>]*>(.*?)</em>", "*", "*"),
|
|
132
|
+
(r"<i\b[^>]*>(.*?)</i>", "*", "*"),
|
|
133
|
+
(r"<code\b[^>]*>(.*?)</code>", "`", "`"),
|
|
134
|
+
]
|
|
135
|
+
|
|
136
|
+
# 转义会破坏 Markdown 语法的字符
|
|
137
|
+
_MD_ESCAPE_RE = re.compile(r"([\\`*_{}\[\]<>])")
|
|
138
|
+
|
|
139
|
+
# 代码块占位符(不含 MD 特殊字符,故不会被后续转义规则破坏)
|
|
140
|
+
_CODE_PLACEHOLDER_RE = re.compile(r"(\x01CB\d+\x01)")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
# 模型定义已收敛到 yimo.kit.crawl.article_models(与 WebArticleKit 共用,字段为平台中立超集);
|
|
144
|
+
# 本模块仍从该处 import 同名模型,故 `yimo.kit.crawl.wx_article_kit.ArticleInfo` 等旧引用方式不变。
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class WxArticleKit:
|
|
148
|
+
"""微信公众号文章抓取工具类
|
|
149
|
+
|
|
150
|
+
使用方式:
|
|
151
|
+
result = WxArticleKit.fetch("https://mp.weixin.qq.com/s/xxxx")
|
|
152
|
+
if result.ok:
|
|
153
|
+
print(result.markdown, result.info.author)
|
|
154
|
+
|
|
155
|
+
saved = WxArticleKit.save(url, output_dir="output", file_name="12345")
|
|
156
|
+
# output/12345/ 下:12345.md + images/
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
@classmethod
|
|
160
|
+
def fetch(cls, url: str, timeout: float = _DEFAULT_TIMEOUT) -> ArticleResult:
|
|
161
|
+
"""抓取并解析单篇文章(不落盘)
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
url: 微信公众号文章链接
|
|
165
|
+
timeout: 请求超时(秒)
|
|
166
|
+
|
|
167
|
+
Returns:
|
|
168
|
+
ArticleResult:ok=True 时 info/markdown 有值;失败时 ok=False 且
|
|
169
|
+
error 为原因描述(不抛异常)
|
|
170
|
+
|
|
171
|
+
Raises:
|
|
172
|
+
ValueError: url 为空或非 http(s)
|
|
173
|
+
"""
|
|
174
|
+
if not url or not url.strip().lower().startswith(("http://", "https://")):
|
|
175
|
+
raise ValueError("url 必须是合法的 http(s) 链接")
|
|
176
|
+
|
|
177
|
+
page = cls._fetch_html(url, timeout)
|
|
178
|
+
if not page:
|
|
179
|
+
return ArticleResult(
|
|
180
|
+
False, "抓取返回空内容(可能被风控拦截)", None, "", REASON_FETCH_FAILED
|
|
181
|
+
)
|
|
182
|
+
if any(marker in page for marker in _RISK_MARKERS):
|
|
183
|
+
return ArticleResult(
|
|
184
|
+
False,
|
|
185
|
+
"命中微信风控特征(环境异常/访问过于频繁),未破解验证码",
|
|
186
|
+
None,
|
|
187
|
+
"",
|
|
188
|
+
REASON_RISK_CONTROL,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
body_html = cls._extract_body(page)
|
|
192
|
+
if not body_html:
|
|
193
|
+
return ArticleResult(
|
|
194
|
+
False, "未找到正文容器 js_content", None, "", REASON_NO_CONTENT
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
title, author, publish_time, summary = cls._extract_meta(page)
|
|
198
|
+
markdown, images, links, codes = cls._to_markdown(body_html, base_url=url)
|
|
199
|
+
if not markdown.strip():
|
|
200
|
+
return ArticleResult(False, "正文转换后为空", None, "", REASON_NO_CONTENT)
|
|
201
|
+
|
|
202
|
+
info = ArticleInfo(
|
|
203
|
+
url=url,
|
|
204
|
+
title=title,
|
|
205
|
+
author=author,
|
|
206
|
+
publish_time=publish_time,
|
|
207
|
+
source=_SOURCE_NAME,
|
|
208
|
+
site_host=_SITE_HOST,
|
|
209
|
+
summary=summary,
|
|
210
|
+
images=images,
|
|
211
|
+
links=links,
|
|
212
|
+
code_blocks=codes,
|
|
213
|
+
)
|
|
214
|
+
return ArticleResult(True, "", info, markdown)
|
|
215
|
+
|
|
216
|
+
@classmethod
|
|
217
|
+
def save(
|
|
218
|
+
cls,
|
|
219
|
+
url: str,
|
|
220
|
+
output_dir: str | Path,
|
|
221
|
+
file_name: str | None = None,
|
|
222
|
+
write_meta: bool = True,
|
|
223
|
+
timeout: float = _DEFAULT_TIMEOUT,
|
|
224
|
+
) -> SavedOutput:
|
|
225
|
+
"""抓取并存储到 `<output_dir>/<file_name>/`(`<file_name>.md` + `images/`)
|
|
226
|
+
|
|
227
|
+
落盘统一由 `yimo.kit.crawl.article_io` 完成(三个 Kit 同一契约):
|
|
228
|
+
正文与 `images/` 必定产出,`meta.json` 默认产出(`write_meta=False` 关闭)。
|
|
229
|
+
|
|
230
|
+
Args:
|
|
231
|
+
url: 微信公众号文章链接
|
|
232
|
+
output_dir: 输出根目录(不存在自动创建)
|
|
233
|
+
file_name: 子目录名与 Markdown 文件名主干;`None` 时按本地时间
|
|
234
|
+
`%y%m%d%H%M%S` 自动命名(同秒碰撞追加 `-2`…),带 `.md` 后缀会自动去除
|
|
235
|
+
write_meta: 是否产出 meta.json(默认 True)
|
|
236
|
+
timeout: 请求超时(秒)
|
|
237
|
+
|
|
238
|
+
Returns:
|
|
239
|
+
SavedOutput:ok=True 时含文章子目录与各产物路径;失败时 ok=False 带
|
|
240
|
+
error/reason;`warnings` 透传自 `fetch`
|
|
241
|
+
|
|
242
|
+
Raises:
|
|
243
|
+
ValueError: 显式传入的 file_name 为空
|
|
244
|
+
"""
|
|
245
|
+
stem = article_io.resolve_stem(file_name, output_dir)
|
|
246
|
+
|
|
247
|
+
result = cls.fetch(url, timeout)
|
|
248
|
+
if not result.ok:
|
|
249
|
+
return SavedOutput(False, result.error, reason=result.reason)
|
|
250
|
+
|
|
251
|
+
return article_io.save_article(
|
|
252
|
+
result,
|
|
253
|
+
output_dir,
|
|
254
|
+
stem,
|
|
255
|
+
write_meta,
|
|
256
|
+
ext_resolver=cls._image_ext,
|
|
257
|
+
label="微信文章",
|
|
258
|
+
timeout=timeout,
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
@staticmethod
|
|
262
|
+
def _image_ext(url: str) -> str:
|
|
263
|
+
"""微信图片扩展名:优先取 CDN 的 `wx_fmt=` 参数(图片 URL 路径无扩展名)"""
|
|
264
|
+
matched = _IMG_FMT_RE.search(url)
|
|
265
|
+
return article_io.normalize_image_ext(matched.group(1) if matched else "")
|
|
266
|
+
|
|
267
|
+
# ------------------------------------------------------------------
|
|
268
|
+
# 抓取
|
|
269
|
+
# ------------------------------------------------------------------
|
|
270
|
+
@staticmethod
|
|
271
|
+
def _fetch_html(url: str, timeout: float) -> str:
|
|
272
|
+
"""标准库 GET 抓取页面(模拟浏览器 UA),失败返回空串"""
|
|
273
|
+
request = urllib.request.Request(
|
|
274
|
+
url,
|
|
275
|
+
headers={
|
|
276
|
+
"User-Agent": _DEFAULT_UA,
|
|
277
|
+
"Accept-Language": "zh-CN,zh;q=0.9",
|
|
278
|
+
"Accept": (
|
|
279
|
+
"text/html,application/xhtml+xml," "application/xml;q=0.9,*/*;q=0.8"
|
|
280
|
+
),
|
|
281
|
+
},
|
|
282
|
+
)
|
|
283
|
+
try:
|
|
284
|
+
with urllib.request.urlopen(request, timeout=timeout) as resp:
|
|
285
|
+
raw = resp.read() if isinstance(resp, HTTPResponse) else b""
|
|
286
|
+
except (
|
|
287
|
+
urllib.error.URLError,
|
|
288
|
+
urllib.error.HTTPError,
|
|
289
|
+
socket.timeout,
|
|
290
|
+
TimeoutError,
|
|
291
|
+
ConnectionError,
|
|
292
|
+
) as exc:
|
|
293
|
+
logger.warning(f"页面抓取失败: {url}({exc})")
|
|
294
|
+
return ""
|
|
295
|
+
|
|
296
|
+
# 微信公众号页面固定 UTF-8,异常时按 GBK 兜底
|
|
297
|
+
try:
|
|
298
|
+
return raw.decode("utf-8")
|
|
299
|
+
except UnicodeDecodeError:
|
|
300
|
+
return raw.decode("gbk", errors="replace")
|
|
301
|
+
|
|
302
|
+
# ------------------------------------------------------------------
|
|
303
|
+
# 正文区与元信息提取
|
|
304
|
+
# ------------------------------------------------------------------
|
|
305
|
+
@staticmethod
|
|
306
|
+
def _extract_body(page: str) -> str:
|
|
307
|
+
"""切片 #js_content 起始 → 尾部二维码区,返回正文 HTML"""
|
|
308
|
+
matched = _JS_CONTENT_RE.search(page)
|
|
309
|
+
if not matched:
|
|
310
|
+
return ""
|
|
311
|
+
start = matched.end()
|
|
312
|
+
|
|
313
|
+
# 正文尾部硬边界(js_pc_qr_code 出现位置),缺失时退化为取到页尾
|
|
314
|
+
qr = _JS_PC_QR_RE.search(page, start)
|
|
315
|
+
end = qr.start() if qr else len(page)
|
|
316
|
+
if end <= start:
|
|
317
|
+
return ""
|
|
318
|
+
|
|
319
|
+
# end 落在 <div ... id="js_pc_qr_code"> 中间时,向前回退到该 div 的 <
|
|
320
|
+
div_start = page.rfind("<", start, end)
|
|
321
|
+
if div_start > start:
|
|
322
|
+
end = div_start
|
|
323
|
+
|
|
324
|
+
body = page[start:end]
|
|
325
|
+
# 兜底:正文区不应含脚本,遇 <script 截断(防页面尾部 JS 混入)
|
|
326
|
+
script = re.search(r"<script\b", body, re.I)
|
|
327
|
+
return body[: script.start()] if script else body
|
|
328
|
+
|
|
329
|
+
@staticmethod
|
|
330
|
+
def _extract_meta(page: str) -> tuple[str, str, str, str]:
|
|
331
|
+
"""提取元信息 → (标题, 作者, 发布时间, 摘要)
|
|
332
|
+
|
|
333
|
+
标题优先 `span.js_title_inner`,为空则回落 `og:title`;
|
|
334
|
+
作者取 `#js_name` 内部文本;发布时间取页面 `var ct`(Unix 秒)按
|
|
335
|
+
Asia/Shanghai 转 ISO 8601;摘要取 `meta[name=description]`。
|
|
336
|
+
"""
|
|
337
|
+
title = ""
|
|
338
|
+
matched = _TITLE_RE.search(page)
|
|
339
|
+
if matched:
|
|
340
|
+
title = matched.group(1).strip()
|
|
341
|
+
if not title:
|
|
342
|
+
matched = _OG_TITLE_RE.search(page)
|
|
343
|
+
title = html.unescape(matched.group(1)).strip() if matched else ""
|
|
344
|
+
|
|
345
|
+
author = ""
|
|
346
|
+
matched = _NAME_RE.search(page)
|
|
347
|
+
if matched:
|
|
348
|
+
author = html.unescape(re.sub(r"<[^>]+>", "", matched.group(1))).strip()
|
|
349
|
+
|
|
350
|
+
publish_time = ""
|
|
351
|
+
matched = _CT_RE.search(page)
|
|
352
|
+
if matched:
|
|
353
|
+
stamp = datetime.fromtimestamp(int(matched.group(1)), _TZ_SHANGHAI)
|
|
354
|
+
publish_time = stamp.isoformat()
|
|
355
|
+
|
|
356
|
+
summary = ""
|
|
357
|
+
matched = _SUMMARY_RE.search(page)
|
|
358
|
+
if matched:
|
|
359
|
+
summary = html.unescape(matched.group(1)).strip()
|
|
360
|
+
|
|
361
|
+
return title, author, publish_time, summary
|
|
362
|
+
|
|
363
|
+
# ------------------------------------------------------------------
|
|
364
|
+
# 正文 HTML → Markdown(零依赖块级转换器)
|
|
365
|
+
# ------------------------------------------------------------------
|
|
366
|
+
@classmethod
|
|
367
|
+
def _to_markdown(
|
|
368
|
+
cls, body_html: str, base_url: str = ""
|
|
369
|
+
) -> tuple[str, list[ArticleImage], list[ArticleLink], list[CodeBlock]]:
|
|
370
|
+
"""正文 HTML → GFM Markdown,并同步收集图片/链接/代码块清单
|
|
371
|
+
|
|
372
|
+
Args:
|
|
373
|
+
body_html: 正文 HTML 片段(#js_content 切片)
|
|
374
|
+
base_url: 相对链接/图片地址的补齐基址(通常为原文 URL)
|
|
375
|
+
|
|
376
|
+
Returns:
|
|
377
|
+
(markdown, images, links, code_blocks)
|
|
378
|
+
"""
|
|
379
|
+
images: list[ArticleImage] = []
|
|
380
|
+
links: list[ArticleLink] = []
|
|
381
|
+
|
|
382
|
+
# 预处理:剥离 HTML 注释;代码块先抽出为占位符,避免被内联规则破坏
|
|
383
|
+
text = re.sub(r"<!--.*?-->", "", body_html, flags=re.S)
|
|
384
|
+
text, codes = cls._extract_code_blocks(text)
|
|
385
|
+
|
|
386
|
+
# 逐块转换(容器块递归展开),块间空行分隔
|
|
387
|
+
blocks = [
|
|
388
|
+
cls._convert_block(block, base_url, images, links)
|
|
389
|
+
for block in cls._split_blocks(text)
|
|
390
|
+
]
|
|
391
|
+
markdown = "\n\n".join(block for block in blocks if block)
|
|
392
|
+
|
|
393
|
+
# 还原代码块为围栏块
|
|
394
|
+
for index, code_block in enumerate(codes):
|
|
395
|
+
fence = f"```{code_block.language}\n{code_block.code}\n```"
|
|
396
|
+
markdown = markdown.replace(f"\x01CB{index}\x01", fence)
|
|
397
|
+
|
|
398
|
+
# 后处理:收敛多余空行与首尾空白
|
|
399
|
+
return re.sub(r"\n{3,}", "\n\n", markdown).strip(), images, links, codes
|
|
400
|
+
|
|
401
|
+
@staticmethod
|
|
402
|
+
def _extract_code_blocks(text: str) -> tuple[str, list[CodeBlock]]:
|
|
403
|
+
"""把 pre 块抽出为占位符,返回 (占位文本, 代码块清单)
|
|
404
|
+
|
|
405
|
+
语言优先取 class 中的 `language-*` / `code-snippet__*`,探测不到记 `text`;
|
|
406
|
+
代码内仅做安全反义(` ` → 空格、`&` → `&`),保留 `<`/`>`
|
|
407
|
+
实体,避免渲染时 `<xxx>` 被误当 HTML 标签。
|
|
408
|
+
"""
|
|
409
|
+
codes: list[CodeBlock] = []
|
|
410
|
+
|
|
411
|
+
def repl(matched: re.Match[str]) -> str:
|
|
412
|
+
raw = matched.group(0)
|
|
413
|
+
lang_match = re.search(
|
|
414
|
+
r'class="[^"]*(?:language-|code-snippet__)([\w+-]+)', raw
|
|
415
|
+
)
|
|
416
|
+
# 微信真实页面 code 内嵌 span/section 模拟样式,甚至无 code 标签,
|
|
417
|
+
# 故统一剥标签仅保留文本(换行保留)
|
|
418
|
+
code_match = re.search(r"<code\b[^>]*>(.*?)</code>", raw, re.S)
|
|
419
|
+
code = code_match.group(1) if code_match else raw
|
|
420
|
+
code = re.sub(r"</(?:section|p|div)>|<br\s*/?>", "\n", code, flags=re.I)
|
|
421
|
+
code = re.sub(r"<[^>]+>", "", code)
|
|
422
|
+
code = code.replace(" ", " ").replace("&", "&")
|
|
423
|
+
codes.append(
|
|
424
|
+
CodeBlock(
|
|
425
|
+
lang_match.group(1) if lang_match else "text", code.strip("\n")
|
|
426
|
+
)
|
|
427
|
+
)
|
|
428
|
+
# 占位符用控制字符包裹(真实正文极少含 \x01)
|
|
429
|
+
return f"\x01CB{len(codes) - 1}\x01"
|
|
430
|
+
|
|
431
|
+
return re.sub(r"<pre\b[^>]*>.*?</pre>", repl, text, flags=re.S), codes
|
|
432
|
+
|
|
433
|
+
@staticmethod
|
|
434
|
+
def _split_blocks(text: str) -> list[str]:
|
|
435
|
+
"""按顶层块标签切分,逐块返回原始 HTML 片段
|
|
436
|
+
|
|
437
|
+
用配对计数扫描:遇 `<tag ...>` 深度 +1,遇 `</tag>` 深度 -1,归零即一个
|
|
438
|
+
完整块,天然支持嵌套(ul > li > ul 等)。仅块级标签参与计数,
|
|
439
|
+
img/br/span/li 等自闭合或行内标签不计入,否则深度永不归零。
|
|
440
|
+
"""
|
|
441
|
+
tag_re = re.compile(r"<(/?)(\w+)\b[^>]*>")
|
|
442
|
+
parts: list[str] = []
|
|
443
|
+
pos = 0
|
|
444
|
+
depth = 0
|
|
445
|
+
current: list[str] = []
|
|
446
|
+
|
|
447
|
+
def flush() -> None:
|
|
448
|
+
nonlocal current
|
|
449
|
+
block = "".join(current).strip()
|
|
450
|
+
if block:
|
|
451
|
+
parts.append(block)
|
|
452
|
+
current = []
|
|
453
|
+
|
|
454
|
+
for matched in tag_re.finditer(text):
|
|
455
|
+
if matched.start() > pos:
|
|
456
|
+
current.append(text[pos : matched.start()])
|
|
457
|
+
current.append(matched.group(0))
|
|
458
|
+
pos = matched.end()
|
|
459
|
+
tag = matched.group(2).lower()
|
|
460
|
+
if tag not in _BLOCK_TAGS:
|
|
461
|
+
continue
|
|
462
|
+
if matched.group(1) == "/":
|
|
463
|
+
depth = max(0, depth - 1)
|
|
464
|
+
if depth == 0:
|
|
465
|
+
flush()
|
|
466
|
+
else:
|
|
467
|
+
depth += 1
|
|
468
|
+
if pos < len(text):
|
|
469
|
+
current.append(text[pos:])
|
|
470
|
+
flush()
|
|
471
|
+
return parts
|
|
472
|
+
|
|
473
|
+
@classmethod
|
|
474
|
+
def _convert_block(
|
|
475
|
+
cls,
|
|
476
|
+
block: str,
|
|
477
|
+
base_url: str,
|
|
478
|
+
images: list[ArticleImage],
|
|
479
|
+
links: list[ArticleLink],
|
|
480
|
+
) -> str:
|
|
481
|
+
"""转换单个块为 Markdown 行"""
|
|
482
|
+
block = block.strip()
|
|
483
|
+
if not block:
|
|
484
|
+
return ""
|
|
485
|
+
|
|
486
|
+
# 块内夹带代码块占位符时(占位符两侧附着其它内容),按占位符切分逐段
|
|
487
|
+
# 转换,保持占位符位置语义
|
|
488
|
+
parts = _CODE_PLACEHOLDER_RE.split(block)
|
|
489
|
+
if len(parts) > 1:
|
|
490
|
+
out = [
|
|
491
|
+
(
|
|
492
|
+
part
|
|
493
|
+
if _CODE_PLACEHOLDER_RE.fullmatch(part)
|
|
494
|
+
else cls._convert_block(part.strip(), base_url, images, links)
|
|
495
|
+
)
|
|
496
|
+
for part in parts
|
|
497
|
+
if part.strip()
|
|
498
|
+
]
|
|
499
|
+
return "\n\n".join(item for item in out if item)
|
|
500
|
+
|
|
501
|
+
# 标题
|
|
502
|
+
matched = re.match(r"<h([1-6])\b[^>]*>(.*?)</h\1>", block, re.S)
|
|
503
|
+
if matched:
|
|
504
|
+
content = cls._inline(matched.group(2), base_url, images, links).strip()
|
|
505
|
+
return f"{'#' * int(matched.group(1))} {content}"
|
|
506
|
+
|
|
507
|
+
# 容器块(section/div,微信正文常见多层嵌套):递归切分内部块
|
|
508
|
+
matched = re.match(r"<(section|div)\b[^>]*>(.*)</\1>$", block, re.S)
|
|
509
|
+
if matched:
|
|
510
|
+
inner = cls._split_blocks(matched.group(2))
|
|
511
|
+
if inner:
|
|
512
|
+
blocks = [cls._convert_block(b, base_url, images, links) for b in inner]
|
|
513
|
+
return "\n\n".join(b for b in blocks if b)
|
|
514
|
+
|
|
515
|
+
# 引用
|
|
516
|
+
matched = re.match(r"<blockquote\b[^>]*>(.*?)</blockquote>", block, re.S)
|
|
517
|
+
if matched:
|
|
518
|
+
inner = cls._convert_block(matched.group(1), base_url, images, links)
|
|
519
|
+
return "\n".join(f"> {line}" for line in inner.split("\n"))
|
|
520
|
+
|
|
521
|
+
# 列表
|
|
522
|
+
if re.match(r"<(ul|ol)\b[^>]*>", block):
|
|
523
|
+
return cls._convert_list(block, base_url, images, links)
|
|
524
|
+
|
|
525
|
+
# 段落
|
|
526
|
+
matched = re.match(r"<p\b[^>]*>(.*?)</p>", block, re.S)
|
|
527
|
+
if matched:
|
|
528
|
+
return cls._inline(matched.group(1), base_url, images, links).strip()
|
|
529
|
+
|
|
530
|
+
# 图片
|
|
531
|
+
if block.startswith("<img"):
|
|
532
|
+
return cls._convert_img(block, base_url, images)
|
|
533
|
+
|
|
534
|
+
# 链接块
|
|
535
|
+
if block.startswith("<a"):
|
|
536
|
+
return cls._convert_anchor(block, base_url, images, links)
|
|
537
|
+
|
|
538
|
+
# 其它:按内联处理
|
|
539
|
+
return cls._inline(block, base_url, images, links)
|
|
540
|
+
|
|
541
|
+
@staticmethod
|
|
542
|
+
def _iter_list_items(fragment: str) -> list[str]:
|
|
543
|
+
"""用配对计数提取顶层 li 内容(支持 li 内嵌套 li/ul)"""
|
|
544
|
+
items: list[str] = []
|
|
545
|
+
depth = 0
|
|
546
|
+
start: int | None = None
|
|
547
|
+
for matched in re.finditer(r"<(/?)li\b[^>]*>", fragment, re.I):
|
|
548
|
+
if matched.group(1) == "/":
|
|
549
|
+
depth -= 1
|
|
550
|
+
if depth == 0 and start is not None:
|
|
551
|
+
items.append(fragment[start : matched.start()])
|
|
552
|
+
start = None
|
|
553
|
+
else:
|
|
554
|
+
if depth == 0:
|
|
555
|
+
start = matched.end()
|
|
556
|
+
depth += 1
|
|
557
|
+
return items
|
|
558
|
+
|
|
559
|
+
@classmethod
|
|
560
|
+
def _convert_list(
|
|
561
|
+
cls,
|
|
562
|
+
block: str,
|
|
563
|
+
base_url: str,
|
|
564
|
+
images: list[ArticleImage],
|
|
565
|
+
links: list[ArticleLink],
|
|
566
|
+
) -> str:
|
|
567
|
+
"""转换 ul/ol 列表为 Markdown 列表(支持多层嵌套)"""
|
|
568
|
+
lines: list[str] = []
|
|
569
|
+
|
|
570
|
+
def walk(fragment: str, level: int, ordered: bool) -> None:
|
|
571
|
+
counter = 0
|
|
572
|
+
for item in cls._iter_list_items(fragment):
|
|
573
|
+
counter += 1
|
|
574
|
+
prefix = f"{' ' * level}{'- ' if not ordered else f'{counter}. '}"
|
|
575
|
+
# li 内从第一个子列表标签开始视为嵌套区
|
|
576
|
+
sub_tag = re.search(r"<(ul|ol)\b[^>]*>", item)
|
|
577
|
+
if not sub_tag:
|
|
578
|
+
lines.append(
|
|
579
|
+
f"{prefix}{cls._inline(item, base_url, images, links).strip()}"
|
|
580
|
+
)
|
|
581
|
+
continue
|
|
582
|
+
main = item[: sub_tag.start()]
|
|
583
|
+
main_html = cls._inline(main, base_url, images, links).strip()
|
|
584
|
+
if main_html:
|
|
585
|
+
lines.append(f"{prefix}{main_html}")
|
|
586
|
+
walk(item[sub_tag.start() :], level + 1, ordered)
|
|
587
|
+
|
|
588
|
+
walk(block, 0, block.strip().startswith("<ol"))
|
|
589
|
+
return "\n".join(lines)
|
|
590
|
+
|
|
591
|
+
@classmethod
|
|
592
|
+
def _inline(
|
|
593
|
+
cls,
|
|
594
|
+
text: str,
|
|
595
|
+
base_url: str,
|
|
596
|
+
images: list[ArticleImage],
|
|
597
|
+
links: list[ArticleLink],
|
|
598
|
+
) -> str:
|
|
599
|
+
"""转换内联 HTML 为行内 Markdown"""
|
|
600
|
+
if not text.strip():
|
|
601
|
+
return ""
|
|
602
|
+
|
|
603
|
+
protected: dict[str, str] = {}
|
|
604
|
+
|
|
605
|
+
# 先处理图片(可能嵌在段落里)。pattern 不能带 /> 结尾:URL 中的斜杠
|
|
606
|
+
# 会触发回溯导致标签只匹配到一半
|
|
607
|
+
def img_repl(matched: re.Match[str]) -> str:
|
|
608
|
+
md = cls._convert_img(matched.group(0), base_url, images)
|
|
609
|
+
return cls._seal(md, protected) if md else ""
|
|
610
|
+
|
|
611
|
+
text = re.sub(r"<img\b[^>]*>", img_repl, text, flags=re.S)
|
|
612
|
+
|
|
613
|
+
# 处理链接
|
|
614
|
+
def anchor_repl(matched: re.Match[str]) -> str:
|
|
615
|
+
md = cls._convert_anchor(matched.group(0), base_url, images, links)
|
|
616
|
+
return cls._seal(md, protected) if md else ""
|
|
617
|
+
|
|
618
|
+
text = re.sub(r"<a\b[^>]*>(.*?)</a>", anchor_repl, text, flags=re.S)
|
|
619
|
+
|
|
620
|
+
# 处理行内样式(粗体/斜体/行内代码;内层 span/font 等标签剥离)
|
|
621
|
+
def style_repl(left: str, right: str, matched: re.Match[str]) -> str:
|
|
622
|
+
inner = html.unescape(re.sub(r"<[^>]+>", "", matched.group(1)))
|
|
623
|
+
return cls._seal(f"{left}{inner}{right}", protected)
|
|
624
|
+
|
|
625
|
+
for pattern, left, right in _INLINE_STYLES:
|
|
626
|
+
text = re.sub(
|
|
627
|
+
pattern,
|
|
628
|
+
lambda m, left=left, right=right: style_repl(left, right, m),
|
|
629
|
+
text,
|
|
630
|
+
flags=re.S,
|
|
631
|
+
)
|
|
632
|
+
|
|
633
|
+
# 换行 <br>
|
|
634
|
+
text = re.sub(r"<br\s*/?>", "\n", text)
|
|
635
|
+
|
|
636
|
+
# 剥离剩余标签(span/font/div/section/center 等,保留内容)后反转义实体
|
|
637
|
+
text = html.unescape(re.sub(r"<[^>]+>", "", text))
|
|
638
|
+
|
|
639
|
+
# 只对剩余纯文本做 MD 特殊字符转义
|
|
640
|
+
text = _MD_ESCAPE_RE.sub(r"\\\1", text)
|
|
641
|
+
|
|
642
|
+
# 还原生成的 Markdown 片段
|
|
643
|
+
for token, original in protected.items():
|
|
644
|
+
text = text.replace(token, original)
|
|
645
|
+
|
|
646
|
+
# 合并空白(留出换行);兜底清理未还原的 seal 令牌(嵌套 <a> 等边缘 DOM)
|
|
647
|
+
text = re.sub(r"[ \t]+", " ", text)
|
|
648
|
+
text = re.sub(r"\x00\d*\x00", "", text)
|
|
649
|
+
return text.strip()
|
|
650
|
+
|
|
651
|
+
@staticmethod
|
|
652
|
+
def _convert_img(tag: str, base_url: str, images: list[ArticleImage]) -> str:
|
|
653
|
+
"""转换 <img> 为 Markdown 图片语法,并记录到 images 清单"""
|
|
654
|
+
original_match = re.search(r'data-src="([^"]*)"', tag)
|
|
655
|
+
original = original_match.group(1) if original_match else ""
|
|
656
|
+
if not original:
|
|
657
|
+
original_match = re.search(r'src="([^"]*)"', tag)
|
|
658
|
+
original = original_match.group(1) if original_match else ""
|
|
659
|
+
if not original:
|
|
660
|
+
return ""
|
|
661
|
+
src = urllib.parse.urljoin(base_url, html.unescape(original))
|
|
662
|
+
alt_match = re.search(r'alt="([^"]*)"', tag)
|
|
663
|
+
alt = html.unescape(alt_match.group(1)) if alt_match else ""
|
|
664
|
+
images.append(ArticleImage(src, original, alt))
|
|
665
|
+
return f""
|
|
666
|
+
|
|
667
|
+
@classmethod
|
|
668
|
+
def _convert_anchor(
|
|
669
|
+
cls,
|
|
670
|
+
tag: str,
|
|
671
|
+
base_url: str,
|
|
672
|
+
images: list[ArticleImage],
|
|
673
|
+
links: list[ArticleLink],
|
|
674
|
+
) -> str:
|
|
675
|
+
"""转换 <a> 为 Markdown 链接,并记录到 links 清单"""
|
|
676
|
+
href_match = re.search(r'href="([^"]*)"', tag)
|
|
677
|
+
href = href_match.group(1) if href_match else ""
|
|
678
|
+
content_match = re.search(r">(.*?)</a>", tag, re.S)
|
|
679
|
+
content = content_match.group(1) if content_match else ""
|
|
680
|
+
text = cls._inline(content, base_url, images, links).strip() if content else ""
|
|
681
|
+
if not href:
|
|
682
|
+
return text
|
|
683
|
+
href = urllib.parse.urljoin(base_url, html.unescape(href))
|
|
684
|
+
links.append(ArticleLink(text, href))
|
|
685
|
+
return f"[{text or href}]({href})"
|
|
686
|
+
|
|
687
|
+
@staticmethod
|
|
688
|
+
def _seal(text: str, protected: dict[str, str]) -> str:
|
|
689
|
+
"""把 MD 标记暂存为占位符,防止后续转义规则误伤"""
|
|
690
|
+
token = f"\u0000{len(protected)}\u0000"
|
|
691
|
+
protected[token] = text
|
|
692
|
+
return token
|