yimo-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yimo/__init__.py +26 -0
- yimo/kit/__init__.py +98 -0
- yimo/kit/ai/__init__.py +110 -0
- yimo/kit/ai/_internal/__init__.py +27 -0
- yimo/kit/ai/_internal/failover.py +535 -0
- yimo/kit/ai/_internal/pool.py +526 -0
- yimo/kit/ai/_internal/rerank_transport.py +137 -0
- yimo/kit/ai/_internal/retry_after.py +51 -0
- yimo/kit/ai/_internal/sdk_client.py +463 -0
- yimo/kit/ai/_internal/service_base.py +115 -0
- yimo/kit/ai/_internal/types.py +28 -0
- yimo/kit/ai/agent/__init__.py +127 -0
- yimo/kit/ai/agent/approval.py +66 -0
- yimo/kit/ai/agent/errors.py +128 -0
- yimo/kit/ai/agent/loop.py +643 -0
- yimo/kit/ai/agent/mcp.py +278 -0
- yimo/kit/ai/agent/registry.py +203 -0
- yimo/kit/ai/agent/session.py +150 -0
- yimo/kit/ai/agent/skill.py +370 -0
- yimo/kit/ai/agent/types.py +161 -0
- yimo/kit/ai/chat_service.py +724 -0
- yimo/kit/ai/decision/__init__.py +57 -0
- yimo/kit/ai/decision/decima.py +853 -0
- yimo/kit/ai/decision/laya.py +552 -0
- yimo/kit/ai/embedding_service.py +198 -0
- yimo/kit/ai/exceptions.py +109 -0
- yimo/kit/ai/image_service.py +84 -0
- yimo/kit/ai/messages.py +105 -0
- yimo/kit/ai/model_builder.py +121 -0
- yimo/kit/ai/model_client.py +62 -0
- yimo/kit/ai/profiles.py +577 -0
- yimo/kit/ai/rerank_service.py +136 -0
- yimo/kit/ai/runner.py +1127 -0
- yimo/kit/common_kit.py +221 -0
- yimo/kit/config/__init__.py +44 -0
- yimo/kit/config/kit_config.py +210 -0
- yimo/kit/config/log_config.py +171 -0
- yimo/kit/crawl/__init__.py +71 -0
- yimo/kit/crawl/article_io.py +325 -0
- yimo/kit/crawl/article_models.py +172 -0
- yimo/kit/crawl/deepseek_kit.py +673 -0
- yimo/kit/crawl/web_article_kit.py +638 -0
- yimo/kit/crawl/wx_article_kit.py +692 -0
- yimo/kit/db/__init__.py +101 -0
- yimo/kit/db/audit.py +625 -0
- yimo/kit/db/base_db_kit.py +1265 -0
- yimo/kit/db/crud.py +423 -0
- yimo/kit/db/engine.py +235 -0
- yimo/kit/db/exceptions.py +45 -0
- yimo/kit/db/models/__init__.py +33 -0
- yimo/kit/db/models/base.py +56 -0
- yimo/kit/db/mysql_kit.py +348 -0
- yimo/kit/db/pg_kit.py +442 -0
- yimo/kit/db/query.py +338 -0
- yimo/kit/db/session.py +85 -0
- yimo/kit/db/sqlite_kit.py +462 -0
- yimo/kit/db/transaction.py +109 -0
- yimo/kit/docs/__init__.py +34 -0
- yimo/kit/docs/anydoc_kit.py +490 -0
- yimo/kit/docs/docx_kit.py +895 -0
- yimo/kit/docs/md2doc_kit.py +1801 -0
- yimo/kit/docs/mineru_kit.py +1497 -0
- yimo/kit/docs/rc_docparser_kit.py +654 -0
- yimo/kit/file_kit.py +518 -0
- yimo/kit/license_kit.py +533 -0
- yimo/kit/redis_kit.py +392 -0
- yimo_kit-0.1.0.dist-info/METADATA +1673 -0
- yimo_kit-0.1.0.dist-info/RECORD +70 -0
- yimo_kit-0.1.0.dist-info/WHEEL +4 -0
- yimo_kit-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,638 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Copyright (c) 2021-2026 溢墨修心(Yimo). All Rights Reserved.
|
|
3
|
+
|
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
you may not use this file except in compliance with the License.
|
|
6
|
+
You may obtain a copy of the License at
|
|
7
|
+
|
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
|
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
See the License for the specific language governing permissions and
|
|
14
|
+
limitations under the License.
|
|
15
|
+
|
|
16
|
+
Project: yimo
|
|
17
|
+
Author: 溢墨修心(Yimo)
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
# date: 2026-10-14
|
|
21
|
+
"""网页文章抓取并存储工具类 WebArticleKit
|
|
22
|
+
|
|
23
|
+
输入任意文章 URL,抓取正文并输出结构化信息(形态与 `WxArticleKit` 一致,但二者独立):
|
|
24
|
+
|
|
25
|
+
- `fetch(url)`:抓取 + 解析(不落盘)→ `ArticleResult`(Markdown + 元信息);
|
|
26
|
+
- `save(url, output_dir, file_name=None, write_meta=True)`:抓取 → 存储到
|
|
27
|
+
`<output_dir>/<stem>/` 下(落盘由 `yimo.kit.crawl.article_io` 统一完成):
|
|
28
|
+
- `<stem>.md`:正文 Markdown(YAML 元信息头,发布时间为中文日期);
|
|
29
|
+
- `images/`:图片转存目录(md 引用同步本地化,失败保留原 URL);
|
|
30
|
+
- `meta.json`:结构化元信息(默认产出,`write_meta=False` 关闭)。
|
|
31
|
+
省略 `file_name` 时目录按本地时间 `yymmddhhmmss` 自动命名(同秒追加 `-2`…)。
|
|
32
|
+
|
|
33
|
+
模型(`ArticleInfo` / `ArticleResult` / `SavedOutput` / `ArticleImage` 等)由三个
|
|
34
|
+
`*ArticleKit` **共用 `yimo.kit.crawl.article_models` 的同一套平台中立定义**,便于多平台抓取后
|
|
35
|
+
写入同一张表;`publish_time` 在结构化模型里保持 ISO 8601,仅 YAML 头渲染为中文日期。
|
|
36
|
+
|
|
37
|
+
质量信号(均不改变成功语义):
|
|
38
|
+
|
|
39
|
+
- `ArticleResult.reason` / `SavedOutput.reason`:失败时给出**类型化分类**
|
|
40
|
+
(`unsupported_source` / `fetch_blocked` / `fetch_failed` / `no_content` /
|
|
41
|
+
`risk_control`),便于业务侧分支处理;
|
|
42
|
+
- `ArticleResult.warnings` / `SavedOutput.warnings`:**非致命**抽取自检提示
|
|
43
|
+
(正文疑似过短、页面含图片但产物无图),同时记入 warning 日志;
|
|
44
|
+
- 标题统一去除站点后缀(如 CSDN 的 `标题-CSDN博客`);刻意不做「与页面 h1 择优」。
|
|
45
|
+
|
|
46
|
+
设计要点:
|
|
47
|
+
|
|
48
|
+
- 纯 HTTP:不启动浏览器、无登录态;正文抽取由 trafilatura 完成(可选依赖,
|
|
49
|
+
见下),技术栈对调用方透明。
|
|
50
|
+
- 站点规则表只允许两类条目——**「转交」**(该来源有专属工具类)与**「接口适配」**
|
|
51
|
+
(站点自曝无需签名的结构化接口);**不做逐站 DOM 补丁**,其余一律走通用路径。
|
|
52
|
+
- 微信:正文解析有特殊性(页面字段原生解析更准),已由
|
|
53
|
+
`yimo.kit.crawl.wx_article_kit.WxArticleKit` 独立实现,本工具类**不覆盖**,命中即返回提示;
|
|
54
|
+
- 头条:文章页是空 body 的 JS 空壳,通用路径必然失败,故**明确支持**:走移动端
|
|
55
|
+
JSON 接口(`m.toutiao.com/i{gid}/info/`);
|
|
56
|
+
- 其它依赖 JavaScript 动态渲染或登录态的站点(如问答类):**按类型说明失败原因**,
|
|
57
|
+
不维护域名名单。
|
|
58
|
+
- 反爬纪律:浏览器 UA + 中文 Accept-Language + 合理超时;命中风控特征返回结构化错误,
|
|
59
|
+
不破解验证码、不携带登录态。
|
|
60
|
+
- 除调用方错误(非法 URL / 空 file_name)外不抛异常:抓取/抽取/下载失败以
|
|
61
|
+
`ok=False` 或空值返回并记日志。
|
|
62
|
+
|
|
63
|
+
可选依赖:需安装 `pip install "yimo-kit[web-article]"`(trafilatura)。
|
|
64
|
+
|
|
65
|
+
使用方式(按需引用,无需经 kit 包顶层导出):
|
|
66
|
+
|
|
67
|
+
from yimo.kit.crawl.web_article_kit import WebArticleKit
|
|
68
|
+
|
|
69
|
+
result = WebArticleKit.fetch("https://blog.csdn.net/xxx/article/details/123")
|
|
70
|
+
if result.ok:
|
|
71
|
+
print(result.info.source, result.info.title, result.markdown)
|
|
72
|
+
|
|
73
|
+
saved = WebArticleKit.save(url, output_dir="output", file_name="12345")
|
|
74
|
+
# output/12345/ 下:12345.md + images/ + meta.json
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
import json
|
|
78
|
+
import re
|
|
79
|
+
import socket
|
|
80
|
+
import urllib.error
|
|
81
|
+
import urllib.parse
|
|
82
|
+
import urllib.request
|
|
83
|
+
from datetime import datetime, timedelta, timezone
|
|
84
|
+
from pathlib import Path
|
|
85
|
+
from typing import NamedTuple
|
|
86
|
+
|
|
87
|
+
from yimo.kit.config.kit_config import get_settings
|
|
88
|
+
from yimo.kit.config.log_config import get_default_logger
|
|
89
|
+
|
|
90
|
+
# 落盘公共层(md + images/ + meta.json,三个 Kit 共用同一契约)
|
|
91
|
+
from yimo.kit.crawl import article_io
|
|
92
|
+
|
|
93
|
+
# 通用模型与 WxArticleKit 共用(多平台抓取后可写入同一张表)
|
|
94
|
+
from yimo.kit.crawl.article_models import (
|
|
95
|
+
REASON_FETCH_BLOCKED,
|
|
96
|
+
REASON_FETCH_FAILED,
|
|
97
|
+
REASON_NO_CONTENT,
|
|
98
|
+
REASON_RISK_CONTROL,
|
|
99
|
+
REASON_UNSUPPORTED_SOURCE,
|
|
100
|
+
ArticleImage,
|
|
101
|
+
ArticleInfo,
|
|
102
|
+
ArticleLink,
|
|
103
|
+
ArticleResult,
|
|
104
|
+
CodeBlock,
|
|
105
|
+
SavedOutput,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
logger = get_default_logger(__name__)
|
|
109
|
+
|
|
110
|
+
try:
|
|
111
|
+
import trafilatura
|
|
112
|
+
from trafilatura import metadata as trafilatura_metadata
|
|
113
|
+
except ImportError as e: # pragma: no cover - 缺依赖时的友好提示
|
|
114
|
+
raise ImportError(
|
|
115
|
+
"web_article_kit 需要可选依赖,请安装:pip install yimo-kit[web-article]"
|
|
116
|
+
) from e
|
|
117
|
+
|
|
118
|
+
# Asia/Shanghai(东八区,无夏令时)固定偏移
|
|
119
|
+
_TZ_SHANGHAI = timezone(timedelta(hours=8))
|
|
120
|
+
|
|
121
|
+
# 请求默认超时(秒)与浏览器 UA(自抓页面与图片下载共用;trafilatura 只做抽取)
|
|
122
|
+
_DEFAULT_TIMEOUT = 20.0
|
|
123
|
+
_DEFAULT_UA = (
|
|
124
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
125
|
+
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
# Markdown 图片语法(含可选 title):
|
|
129
|
+
_MD_IMG_RE = re.compile(r'!\[[^\]]*\]\((https?://[^)\s]+)(?:\s+"[^"]*")?\)')
|
|
130
|
+
# Markdown 链接语法: [text](url)
|
|
131
|
+
_MD_LINK_RE = re.compile(r'(?<!!)\[([^\]]*)\]\((https?://[^)\s]+)(?:\s+"[^"]*")?\)')
|
|
132
|
+
# Markdown 围栏代码块:```lang\ncode\n```
|
|
133
|
+
_MD_CODE_RE = re.compile(r"^```([^\n`]*)\n(.*?)^```", re.S | re.M)
|
|
134
|
+
|
|
135
|
+
# 风控特征(命中即返回结构化错误,不破解验证码)
|
|
136
|
+
_RISK_MARKERS = ("环境异常", "访问过于频繁", "请完成验证", "user verify")
|
|
137
|
+
|
|
138
|
+
# 站点规则表:只允许两类条目——「转交」与「接口适配」,见模块 docstring
|
|
139
|
+
_WECHAT_HOST_RE = re.compile(r"://mp\.weixin\.qq\.com/")
|
|
140
|
+
_TOUTIAO_HOST_RE = re.compile(r"://(?:www|m)\.toutiao\.com/")
|
|
141
|
+
_SITE_RULES: tuple[tuple[str, re.Pattern[str]], ...] = (
|
|
142
|
+
("wechat", _WECHAT_HOST_RE),
|
|
143
|
+
("toutiao", _TOUTIAO_HOST_RE),
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
# 微信转交提示(该来源由 WxArticleKit 独立实现)
|
|
147
|
+
_WECHAT_HINT = (
|
|
148
|
+
"该来源有专属解析,请使用 yimo.kit.crawl.wx_article_kit.WxArticleKit"
|
|
149
|
+
"(本工具类不覆盖微信文章,二者各自独立)"
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
# 头条:文章页是空 body 的 JS 空壳(正文由 JS 注入),但移动端 JSON 接口
|
|
153
|
+
# 无需签名即可返回结构化正文,故走站点专属接口适配分支
|
|
154
|
+
_TOUTIAO_GID_RE = re.compile(r"(?:/article/|/i|group_id=)(\d+)")
|
|
155
|
+
_TOUTIAO_API = "https://m.toutiao.com/i{gid}/info/"
|
|
156
|
+
|
|
157
|
+
# 抓取被拒的状态码:正文通常依赖 JavaScript 渲染或登录态,纯 HTTP 不可用
|
|
158
|
+
_BLOCKED_STATUS = {401, 403, 412, 418, 429}
|
|
159
|
+
|
|
160
|
+
# 标题站点后缀分隔符(如 CSDN 的 `标题-CSDN博客`)
|
|
161
|
+
_TITLE_SEPS = ("-", "_", "|", "|", "—", "–", "·")
|
|
162
|
+
|
|
163
|
+
# 正文自检阈值统一由配置 CRAWL_MIN_CONTENT_CHARS 提供(默认 500,0 关闭)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
# 模型定义已收敛到 yimo.kit.crawl.article_models(与 WxArticleKit 共用同一套平台中立模型);
|
|
167
|
+
# 本模块从该处 import 同名模型,故 `yimo.kit.crawl.web_article_kit.ArticleInfo` 等引用方式不变。
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class _ExtractOutcome(NamedTuple):
|
|
171
|
+
"""内部分支的抽取结果(info / markdown / 失败原因 / 非致命提示)"""
|
|
172
|
+
|
|
173
|
+
info: ArticleInfo | None = None
|
|
174
|
+
markdown: str = ""
|
|
175
|
+
error: str = ""
|
|
176
|
+
reason: str = ""
|
|
177
|
+
warnings: tuple[str, ...] = ()
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class WebArticleKit:
|
|
181
|
+
"""网页文章抓取并存储工具类(无状态,方法均为 `classmethod`)
|
|
182
|
+
|
|
183
|
+
使用方式:
|
|
184
|
+
result = WebArticleKit.fetch("https://blog.csdn.net/xxx/article/details/123")
|
|
185
|
+
if result.ok:
|
|
186
|
+
print(result.info.source, result.info.title, result.markdown)
|
|
187
|
+
|
|
188
|
+
saved = WebArticleKit.save(url, output_dir="output", file_name="12345")
|
|
189
|
+
# output/12345/ 下:12345.md + images/
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
@classmethod
|
|
193
|
+
def fetch(cls, url: str, timeout: float = _DEFAULT_TIMEOUT) -> ArticleResult:
|
|
194
|
+
"""抓取并解析单篇文章(不落盘)
|
|
195
|
+
|
|
196
|
+
Args:
|
|
197
|
+
url: 文章链接(微信文章请改用 `yimo.kit.crawl.wx_article_kit.WxArticleKit`)
|
|
198
|
+
timeout: 请求超时(秒)
|
|
199
|
+
|
|
200
|
+
Returns:
|
|
201
|
+
ArticleResult:ok=True 时 info/markdown 有值;失败时 ok=False 且
|
|
202
|
+
error 为**类型化**原因说明、reason 为类型化分类(不抛异常);
|
|
203
|
+
`warnings` 为非致命自检提示(不影响 ok)
|
|
204
|
+
|
|
205
|
+
Raises:
|
|
206
|
+
ValueError: url 为空或非 http(s)
|
|
207
|
+
"""
|
|
208
|
+
if not url or not url.strip().lower().startswith(("http://", "https://")):
|
|
209
|
+
raise ValueError("url 必须是合法的 http(s) 链接")
|
|
210
|
+
|
|
211
|
+
# 站点规则表分发(只允许「转交」与「接口适配」两类条目)
|
|
212
|
+
if cls._match_site(url) == "wechat":
|
|
213
|
+
return ArticleResult(
|
|
214
|
+
False, _WECHAT_HINT, None, "", REASON_UNSUPPORTED_SOURCE
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
outcome = (
|
|
218
|
+
cls._extract_toutiao(url, timeout)
|
|
219
|
+
if cls._match_site(url) == "toutiao"
|
|
220
|
+
else cls._extract_generic(url, timeout)
|
|
221
|
+
)
|
|
222
|
+
if not outcome.info or not outcome.markdown.strip():
|
|
223
|
+
return ArticleResult(
|
|
224
|
+
False,
|
|
225
|
+
outcome.error or "正文抽取为空",
|
|
226
|
+
None,
|
|
227
|
+
"",
|
|
228
|
+
outcome.reason or REASON_NO_CONTENT,
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
# 自检提示只记录,不改变成功语义
|
|
232
|
+
for message in outcome.warnings:
|
|
233
|
+
logger.warning(f"抽取自检提示: {url}({message})")
|
|
234
|
+
return ArticleResult(
|
|
235
|
+
True, "", outcome.info, outcome.markdown, "", outcome.warnings
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
@classmethod
|
|
239
|
+
def save(
|
|
240
|
+
cls,
|
|
241
|
+
url: str,
|
|
242
|
+
output_dir: str | Path,
|
|
243
|
+
file_name: str | None = None,
|
|
244
|
+
write_meta: bool = True,
|
|
245
|
+
timeout: float = _DEFAULT_TIMEOUT,
|
|
246
|
+
) -> SavedOutput:
|
|
247
|
+
"""抓取并存储到 `<output_dir>/<file_name>/`(`<file_name>.md` + `images/`)
|
|
248
|
+
|
|
249
|
+
落盘统一由 `yimo.kit.crawl.article_io` 完成(三个 Kit 同一契约):
|
|
250
|
+
正文与 `images/` 必定产出,`meta.json` 默认产出(`write_meta=False` 关闭)。
|
|
251
|
+
|
|
252
|
+
Args:
|
|
253
|
+
url: 文章链接
|
|
254
|
+
output_dir: 输出根目录(不存在自动创建)
|
|
255
|
+
file_name: 子目录名与 Markdown 文件名主干;`None` 时按本地时间
|
|
256
|
+
`%y%m%d%H%M%S` 自动命名(同秒碰撞追加 `-2`…),带 `.md` 后缀会自动去除
|
|
257
|
+
write_meta: 是否产出 meta.json(默认 True)
|
|
258
|
+
timeout: 请求超时(秒)
|
|
259
|
+
|
|
260
|
+
Returns:
|
|
261
|
+
SavedOutput:ok=True 时含文章子目录与各产物路径;失败时 ok=False 带 error
|
|
262
|
+
|
|
263
|
+
Raises:
|
|
264
|
+
ValueError: 显式传入的 file_name 为空
|
|
265
|
+
"""
|
|
266
|
+
stem = article_io.resolve_stem(file_name, output_dir)
|
|
267
|
+
|
|
268
|
+
result = cls.fetch(url, timeout)
|
|
269
|
+
if not result.ok:
|
|
270
|
+
return SavedOutput(False, result.error, reason=result.reason)
|
|
271
|
+
|
|
272
|
+
return article_io.save_article(
|
|
273
|
+
result, output_dir, stem, write_meta, label="网页文章", timeout=timeout
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
# ------------------------------------------------------------------
|
|
277
|
+
# 站点分发与抽取
|
|
278
|
+
# ------------------------------------------------------------------
|
|
279
|
+
@staticmethod
|
|
280
|
+
def _match_site(url: str) -> str:
|
|
281
|
+
"""命中站点规则表返回规则名,未命中返回空串"""
|
|
282
|
+
for name, pattern in _SITE_RULES:
|
|
283
|
+
if pattern.search(url):
|
|
284
|
+
return name
|
|
285
|
+
return ""
|
|
286
|
+
|
|
287
|
+
@classmethod
|
|
288
|
+
def _extract_generic(cls, url: str, timeout: float) -> _ExtractOutcome:
|
|
289
|
+
"""通用分支:自抓 HTML → 预处理 → trafilatura 抽取"""
|
|
290
|
+
page, error, reason = cls._fetch_html(url, timeout)
|
|
291
|
+
if not page:
|
|
292
|
+
return _ExtractOutcome(error=error, reason=reason)
|
|
293
|
+
if any(marker in page for marker in _RISK_MARKERS):
|
|
294
|
+
return _ExtractOutcome(
|
|
295
|
+
error="命中站点风控特征(访问过于频繁/环境异常),已停止抓取",
|
|
296
|
+
reason=REASON_RISK_CONTROL,
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
# 懒加载图片(data-src 为真实地址、src 为占位/JS 拼接)提升为 src:
|
|
300
|
+
# 通用处理,大量站点使用懒加载,不处理会整篇丢图
|
|
301
|
+
page = cls._promote_data_src(page)
|
|
302
|
+
|
|
303
|
+
title, author, publish_time, source, summary = cls._extract_meta(page, url)
|
|
304
|
+
markdown = cls._to_markdown(page, url)
|
|
305
|
+
if not markdown:
|
|
306
|
+
return _ExtractOutcome(
|
|
307
|
+
error=(
|
|
308
|
+
"未能识别正文:该页面可能不是文章页,或正文由 JavaScript 动态渲染"
|
|
309
|
+
"(纯 HTTP 无法获取)"
|
|
310
|
+
),
|
|
311
|
+
reason=REASON_NO_CONTENT,
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
info = cls._build_info(
|
|
315
|
+
url, title, author, publish_time, source, summary, markdown
|
|
316
|
+
)
|
|
317
|
+
return _ExtractOutcome(
|
|
318
|
+
info=info, markdown=markdown, warnings=cls._self_check(page, info, markdown)
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
@classmethod
|
|
322
|
+
def _extract_toutiao(cls, url: str, timeout: float) -> _ExtractOutcome:
|
|
323
|
+
"""头条分支(站点接口适配):走 m.toutiao.com 的 JSON 接口
|
|
324
|
+
|
|
325
|
+
文章页 HTML 为空 body 的纯 JS 注入,通用抽取必然失败;该接口无需签名即可
|
|
326
|
+
返回 title / source / publish_time / content(正文 HTML)。
|
|
327
|
+
"""
|
|
328
|
+
matched = _TOUTIAO_GID_RE.search(url)
|
|
329
|
+
if not matched:
|
|
330
|
+
return _ExtractOutcome(
|
|
331
|
+
error="未能从头条 URL 中解析出文章 id(group_id)",
|
|
332
|
+
reason=REASON_NO_CONTENT,
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
raw, error, reason = cls._fetch_html(
|
|
336
|
+
_TOUTIAO_API.format(gid=matched.group(1)), timeout
|
|
337
|
+
)
|
|
338
|
+
if not raw:
|
|
339
|
+
return _ExtractOutcome(error=error, reason=reason)
|
|
340
|
+
try:
|
|
341
|
+
payload = json.loads(raw)
|
|
342
|
+
except json.JSONDecodeError:
|
|
343
|
+
return _ExtractOutcome(
|
|
344
|
+
error="头条接口返回非 JSON(可能被风控拦截)",
|
|
345
|
+
reason=REASON_FETCH_FAILED,
|
|
346
|
+
)
|
|
347
|
+
data = payload.get("data") or {}
|
|
348
|
+
content = data.get("content") or ""
|
|
349
|
+
if not content.strip():
|
|
350
|
+
return _ExtractOutcome(
|
|
351
|
+
error="头条接口未返回正文内容(文章可能已删除或需登录)",
|
|
352
|
+
reason=REASON_NO_CONTENT,
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
# 正文片段规整:补图片扩展名(trafilatura 只认路径扩展名)+ 完整文档容器
|
|
356
|
+
content = cls._ensure_image_ext(content)
|
|
357
|
+
content = cls._wrap_article_body(content)
|
|
358
|
+
|
|
359
|
+
markdown = cls._to_markdown(content, url)
|
|
360
|
+
if not markdown:
|
|
361
|
+
return _ExtractOutcome(error="头条正文转换后为空", reason=REASON_NO_CONTENT)
|
|
362
|
+
|
|
363
|
+
media_user = data.get("media_user") or {}
|
|
364
|
+
stamp = data.get("publish_time")
|
|
365
|
+
publish_time = (
|
|
366
|
+
datetime.fromtimestamp(int(stamp), _TZ_SHANGHAI).isoformat()
|
|
367
|
+
if stamp
|
|
368
|
+
else ""
|
|
369
|
+
)
|
|
370
|
+
info = cls._build_info(
|
|
371
|
+
url=url,
|
|
372
|
+
title=data.get("title") or "",
|
|
373
|
+
author=media_user.get("screen_name") or data.get("source") or "",
|
|
374
|
+
publish_time=publish_time,
|
|
375
|
+
source="今日头条",
|
|
376
|
+
summary=data.get("abstract") or "",
|
|
377
|
+
markdown=markdown,
|
|
378
|
+
)
|
|
379
|
+
return _ExtractOutcome(
|
|
380
|
+
info=info,
|
|
381
|
+
markdown=markdown,
|
|
382
|
+
warnings=cls._self_check(content, info, markdown),
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
@classmethod
|
|
386
|
+
def _build_info(
|
|
387
|
+
cls,
|
|
388
|
+
url: str,
|
|
389
|
+
title: str,
|
|
390
|
+
author: str,
|
|
391
|
+
publish_time: str,
|
|
392
|
+
source: str,
|
|
393
|
+
summary: str,
|
|
394
|
+
markdown: str,
|
|
395
|
+
) -> ArticleInfo:
|
|
396
|
+
"""组装 ArticleInfo(含图片/链接/代码块清单;图片 local 在落盘阶段填充)
|
|
397
|
+
|
|
398
|
+
标题在此统一去除站点后缀(如 CSDN 的 `标题-CSDN博客`)。
|
|
399
|
+
"""
|
|
400
|
+
return ArticleInfo(
|
|
401
|
+
url=url,
|
|
402
|
+
title=cls._strip_source_suffix(title or "", source or ""),
|
|
403
|
+
author=author or "",
|
|
404
|
+
publish_time=publish_time or "",
|
|
405
|
+
source=source or "",
|
|
406
|
+
site_host=urllib.parse.urlparse(url).netloc,
|
|
407
|
+
summary=summary or "",
|
|
408
|
+
images=cls._collect_images(markdown),
|
|
409
|
+
links=cls._collect_links(markdown),
|
|
410
|
+
code_blocks=cls._collect_code_blocks(markdown),
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
# ------------------------------------------------------------------
|
|
414
|
+
# 后处理与自检
|
|
415
|
+
# ------------------------------------------------------------------
|
|
416
|
+
@staticmethod
|
|
417
|
+
def _strip_source_suffix(title: str, source: str) -> str:
|
|
418
|
+
"""去掉标题尾部的站点后缀(如 `标题-CSDN博客`)
|
|
419
|
+
|
|
420
|
+
仅当尾部与已知 `source`(来源平台名)匹配时剥离,避免误伤正文标题;
|
|
421
|
+
刻意**不做**「与页面 h1 择优」——h1 可能是导航文案,误伤风险高于收益。
|
|
422
|
+
"""
|
|
423
|
+
if not title or not source:
|
|
424
|
+
return title
|
|
425
|
+
for sep in _TITLE_SEPS:
|
|
426
|
+
for suffix in (f"{sep}{source}", f"{sep} {source}", f" {sep} {source}"):
|
|
427
|
+
if title.endswith(suffix):
|
|
428
|
+
return title[: -len(suffix)].strip()
|
|
429
|
+
return title
|
|
430
|
+
|
|
431
|
+
@staticmethod
|
|
432
|
+
def _self_check(page: str, info: ArticleInfo, markdown: str) -> tuple[str, ...]:
|
|
433
|
+
"""抽取自检(**非致命**):正文过短 / 页面含图片但产物无图
|
|
434
|
+
|
|
435
|
+
长度阈值取自 `CRAWL_MIN_CONTENT_CHARS`(默认 500,设 `0` 关闭);
|
|
436
|
+
只产出提示供调用方判断质量,**不改变 `ok` 语义**;两类问题在过去实测中
|
|
437
|
+
都出现过静默发生(degraded 抽取无任何异常信号),故显式暴露。
|
|
438
|
+
"""
|
|
439
|
+
warnings: list[str] = []
|
|
440
|
+
min_chars = get_settings().CRAWL_MIN_CONTENT_CHARS
|
|
441
|
+
if min_chars > 0 and len(markdown) < min_chars:
|
|
442
|
+
warnings.append(f"正文仅 {len(markdown)} 字符,若原文本身较短可忽略")
|
|
443
|
+
image_tags = len(re.findall(r"<img\b", page, re.I))
|
|
444
|
+
if image_tags and not info.images:
|
|
445
|
+
warnings.append(
|
|
446
|
+
f"页面含 {image_tags} 个 <img> 但产物未提取到任何图片,图片可能被过滤"
|
|
447
|
+
)
|
|
448
|
+
return tuple(warnings)
|
|
449
|
+
|
|
450
|
+
# ------------------------------------------------------------------
|
|
451
|
+
# 抓取与元数据
|
|
452
|
+
# ------------------------------------------------------------------
|
|
453
|
+
@staticmethod
|
|
454
|
+
def _fetch_html(url: str, timeout: float) -> tuple[str, str]:
|
|
455
|
+
"""标准库 GET 抓取页面(模拟浏览器 UA)
|
|
456
|
+
|
|
457
|
+
Returns:
|
|
458
|
+
(html, error, reason):成功时 error/reason 为空;失败时 html 为空,
|
|
459
|
+
error 为类型化说明、reason 为分类(`fetch_blocked` / `fetch_failed`)
|
|
460
|
+
"""
|
|
461
|
+
request = urllib.request.Request(
|
|
462
|
+
url,
|
|
463
|
+
headers={
|
|
464
|
+
"User-Agent": _DEFAULT_UA,
|
|
465
|
+
"Accept-Language": "zh-CN,zh;q=0.9",
|
|
466
|
+
"Accept": (
|
|
467
|
+
"text/html,application/xhtml+xml," "application/xml;q=0.9,*/*;q=0.8"
|
|
468
|
+
),
|
|
469
|
+
},
|
|
470
|
+
)
|
|
471
|
+
try:
|
|
472
|
+
with urllib.request.urlopen(request, timeout=timeout) as resp:
|
|
473
|
+
raw = resp.read()
|
|
474
|
+
except urllib.error.HTTPError as exc:
|
|
475
|
+
blocked = exc.code in _BLOCKED_STATUS
|
|
476
|
+
hint = (
|
|
477
|
+
";该状态通常表示站点拒绝直接抓取,正文依赖 JavaScript 动态渲染"
|
|
478
|
+
"或需登录态,纯 HTTP 方式不可用"
|
|
479
|
+
if blocked
|
|
480
|
+
else ""
|
|
481
|
+
)
|
|
482
|
+
logger.warning(f"页面抓取失败: {url}(HTTP {exc.code}{hint})")
|
|
483
|
+
return (
|
|
484
|
+
"",
|
|
485
|
+
f"页面抓取失败(HTTP {exc.code}){hint}",
|
|
486
|
+
REASON_FETCH_BLOCKED if blocked else REASON_FETCH_FAILED,
|
|
487
|
+
)
|
|
488
|
+
except (urllib.error.URLError, socket.timeout, TimeoutError, OSError) as exc:
|
|
489
|
+
logger.warning(f"页面抓取失败: {url}({exc})")
|
|
490
|
+
return "", f"页面抓取失败({exc})", REASON_FETCH_FAILED
|
|
491
|
+
|
|
492
|
+
try:
|
|
493
|
+
return raw.decode("utf-8"), "", ""
|
|
494
|
+
except UnicodeDecodeError:
|
|
495
|
+
return raw.decode("gbk", errors="replace"), "", ""
|
|
496
|
+
|
|
497
|
+
@staticmethod
|
|
498
|
+
def _extract_meta(page: str, url: str) -> tuple[str, str, str, str, str]:
|
|
499
|
+
"""提取元信息 → (标题, 作者, 发布时间, 来源平台, 摘要)
|
|
500
|
+
|
|
501
|
+
`publish_time` 为 ISO 8601 或 `YYYY-MM-DD`(trafilatura 精度所限);
|
|
502
|
+
提取不到时返回空串(不返回 None)。
|
|
503
|
+
"""
|
|
504
|
+
try:
|
|
505
|
+
meta = trafilatura_metadata.extract_metadata(page, url)
|
|
506
|
+
except (ValueError, TypeError) as exc: # 元数据解析失败不影响正文抽取
|
|
507
|
+
logger.warning(f"元信息提取失败(忽略): {url}({exc})")
|
|
508
|
+
return "", "", "", "", ""
|
|
509
|
+
if meta is None:
|
|
510
|
+
return "", "", "", "", ""
|
|
511
|
+
return (
|
|
512
|
+
meta.title or "",
|
|
513
|
+
meta.author or "",
|
|
514
|
+
meta.date or "",
|
|
515
|
+
meta.sitename or "",
|
|
516
|
+
meta.description or "",
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
@classmethod
|
|
520
|
+
def _to_markdown(cls, page: str, url: str) -> str:
|
|
521
|
+
"""正文 → GFM Markdown
|
|
522
|
+
|
|
523
|
+
`with_metadata=False`:元数据由 `_extract_meta` 单独提供,开启会让
|
|
524
|
+
trafilatura 在正文里再输出一份 YAML 头(与自产头重复)。
|
|
525
|
+
"""
|
|
526
|
+
markdown = trafilatura.extract(
|
|
527
|
+
page,
|
|
528
|
+
url=url,
|
|
529
|
+
output_format="markdown",
|
|
530
|
+
with_metadata=False,
|
|
531
|
+
include_images=True,
|
|
532
|
+
include_tables=True,
|
|
533
|
+
include_links=True,
|
|
534
|
+
)
|
|
535
|
+
return cls._clean_markdown(markdown or "")
|
|
536
|
+
|
|
537
|
+
@staticmethod
|
|
538
|
+
def _clean_markdown(markdown: str) -> str:
|
|
539
|
+
"""收敛产物噪声:删除空标题行(页面中的空标题元素会被渲染成裸 `##`),
|
|
540
|
+
并把 3 个以上连续换行压成段落间空行"""
|
|
541
|
+
markdown = re.sub(r"(?m)^#{1,6}[ \t]*$\n?", "", markdown)
|
|
542
|
+
return re.sub(r"\n{3,}", "\n\n", markdown).strip()
|
|
543
|
+
|
|
544
|
+
# ------------------------------------------------------------------
|
|
545
|
+
# 正文片段预处理(通用兜底 + 头条接口适配)
|
|
546
|
+
# ------------------------------------------------------------------
|
|
547
|
+
@staticmethod
|
|
548
|
+
def _promote_data_src(page: str) -> str:
|
|
549
|
+
"""把 <img> 的 data-src(懒加载真实地址)提升为 src,移除占位 src"""
|
|
550
|
+
|
|
551
|
+
def repl(matched: re.Match[str]) -> str:
|
|
552
|
+
tag = matched.group(0)
|
|
553
|
+
lazy = re.search(r'data-src="([^"]*)"', tag)
|
|
554
|
+
if not lazy:
|
|
555
|
+
return tag
|
|
556
|
+
tag = re.sub(r'\ssrc="[^"]*"', "", tag) # 移除占位/JS 拼接 src
|
|
557
|
+
return tag.replace("data-src=", "src=", 1)
|
|
558
|
+
|
|
559
|
+
return re.sub(r"<img\b[^>]*>", repl, page)
|
|
560
|
+
|
|
561
|
+
@staticmethod
|
|
562
|
+
def _ensure_image_ext(content: str) -> str:
|
|
563
|
+
"""给路径无图片扩展名的 img src 追加尾随扩展名参数
|
|
564
|
+
|
|
565
|
+
此类图片 URL 形如 `...~tplv-tt-large.image?x-expires=..&x-signature=..`,
|
|
566
|
+
路径无扩展名,而 trafilatura 仅按路径扩展名识别图片,不处理会被整体过滤。
|
|
567
|
+
实测:把扩展名插到 `?` 之前会破坏 CDN 签名(403),**追加尾随查询参数**
|
|
568
|
+
则下载正常(签名只覆盖原有参数);真实格式在下载时由响应 Content-Type 判定,
|
|
569
|
+
故此处的占位扩展名不影响落盘结果。
|
|
570
|
+
"""
|
|
571
|
+
|
|
572
|
+
def fix_src(matched: re.Match[str]) -> str:
|
|
573
|
+
prefix, url = matched.group(1), matched.group(2)
|
|
574
|
+
suffix = Path(urllib.parse.urlparse(url).path).suffix
|
|
575
|
+
if article_io.normalize_image_ext(suffix):
|
|
576
|
+
return matched.group(0)
|
|
577
|
+
sep = "&" if "?" in url else "?"
|
|
578
|
+
return f'{prefix}{url}{sep}ext=.jpg"'
|
|
579
|
+
|
|
580
|
+
def fix_tag(matched: re.Match[str]) -> str:
|
|
581
|
+
return re.sub(
|
|
582
|
+
r'((?:data-src|src)=")(https?://[^"]+)"', fix_src, matched.group(0)
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
return re.sub(r"<img\b[^>]*>", fix_tag, content)
|
|
586
|
+
|
|
587
|
+
@staticmethod
|
|
588
|
+
def _wrap_article_body(content: str) -> str:
|
|
589
|
+
"""把正文规整为「完整文档 + `id="articleBody"` 容器」
|
|
590
|
+
|
|
591
|
+
两件事缺一不可:
|
|
592
|
+
- 套 `articleBody` 容器:该 id 在 trafilatura 的正文容器词表内,否则主抽取器
|
|
593
|
+
匹配不到、退化为兜底路径(只保留段落,图片全丢);
|
|
594
|
+
- **必须是完整文档(含 `<body>`)**:主抽取器按「后代」匹配(不含自身),
|
|
595
|
+
实测部分文章返回**裸片段**(直接以 `<img>` 开头),若把容器当根元素包装,
|
|
596
|
+
容器自身匹配不到,同样丢图。
|
|
597
|
+
"""
|
|
598
|
+
if 'id="articleBody"' in content:
|
|
599
|
+
return content
|
|
600
|
+
if "<body>" in content:
|
|
601
|
+
head, _, rest = content.partition("<body>")
|
|
602
|
+
body, sep, tail = rest.rpartition("</body>")
|
|
603
|
+
if not sep:
|
|
604
|
+
return f'{head}<body><div id="articleBody">{rest}</div></body>'
|
|
605
|
+
return f'{head}<body><div id="articleBody">{body}</div>{sep}{tail}'
|
|
606
|
+
return f'<html><body><div id="articleBody">{content}</div></body></html>'
|
|
607
|
+
|
|
608
|
+
# ------------------------------------------------------------------
|
|
609
|
+
# 清单收集(图片 / 链接 / 代码块)
|
|
610
|
+
# ------------------------------------------------------------------
|
|
611
|
+
@staticmethod
|
|
612
|
+
def _collect_images(markdown: str) -> list[ArticleImage]:
|
|
613
|
+
"""从 Markdown 收集远程图片清单(同 URL 去重,保留出现顺序)"""
|
|
614
|
+
images: list[ArticleImage] = []
|
|
615
|
+
for src in dict.fromkeys(_MD_IMG_RE.findall(markdown)):
|
|
616
|
+
images.append(ArticleImage(src=src, original=src))
|
|
617
|
+
return images
|
|
618
|
+
|
|
619
|
+
@staticmethod
|
|
620
|
+
def _collect_links(markdown: str) -> list[ArticleLink]:
|
|
621
|
+
"""从 Markdown 收集链接清单(去重,按 (文本, 地址) 计)"""
|
|
622
|
+
links: list[ArticleLink] = []
|
|
623
|
+
seen: set[tuple[str, str]] = set()
|
|
624
|
+
for text, href in _MD_LINK_RE.findall(markdown):
|
|
625
|
+
key = (text.strip(), href)
|
|
626
|
+
if key in seen:
|
|
627
|
+
continue
|
|
628
|
+
seen.add(key)
|
|
629
|
+
links.append(ArticleLink(text=text.strip(), href=href))
|
|
630
|
+
return links
|
|
631
|
+
|
|
632
|
+
@staticmethod
|
|
633
|
+
def _collect_code_blocks(markdown: str) -> list[CodeBlock]:
|
|
634
|
+
"""从 Markdown 收集代码块清单(语言缺失时记 `text`)"""
|
|
635
|
+
return [
|
|
636
|
+
CodeBlock(language=(lang.strip() or "text"), code=code.strip("\n"))
|
|
637
|
+
for lang, code in _MD_CODE_RE.findall(markdown)
|
|
638
|
+
]
|