yimo-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. yimo/__init__.py +26 -0
  2. yimo/kit/__init__.py +98 -0
  3. yimo/kit/ai/__init__.py +110 -0
  4. yimo/kit/ai/_internal/__init__.py +27 -0
  5. yimo/kit/ai/_internal/failover.py +535 -0
  6. yimo/kit/ai/_internal/pool.py +526 -0
  7. yimo/kit/ai/_internal/rerank_transport.py +137 -0
  8. yimo/kit/ai/_internal/retry_after.py +51 -0
  9. yimo/kit/ai/_internal/sdk_client.py +463 -0
  10. yimo/kit/ai/_internal/service_base.py +115 -0
  11. yimo/kit/ai/_internal/types.py +28 -0
  12. yimo/kit/ai/agent/__init__.py +127 -0
  13. yimo/kit/ai/agent/approval.py +66 -0
  14. yimo/kit/ai/agent/errors.py +128 -0
  15. yimo/kit/ai/agent/loop.py +643 -0
  16. yimo/kit/ai/agent/mcp.py +278 -0
  17. yimo/kit/ai/agent/registry.py +203 -0
  18. yimo/kit/ai/agent/session.py +150 -0
  19. yimo/kit/ai/agent/skill.py +370 -0
  20. yimo/kit/ai/agent/types.py +161 -0
  21. yimo/kit/ai/chat_service.py +724 -0
  22. yimo/kit/ai/decision/__init__.py +57 -0
  23. yimo/kit/ai/decision/decima.py +853 -0
  24. yimo/kit/ai/decision/laya.py +552 -0
  25. yimo/kit/ai/embedding_service.py +198 -0
  26. yimo/kit/ai/exceptions.py +109 -0
  27. yimo/kit/ai/image_service.py +84 -0
  28. yimo/kit/ai/messages.py +105 -0
  29. yimo/kit/ai/model_builder.py +121 -0
  30. yimo/kit/ai/model_client.py +62 -0
  31. yimo/kit/ai/profiles.py +577 -0
  32. yimo/kit/ai/rerank_service.py +136 -0
  33. yimo/kit/ai/runner.py +1127 -0
  34. yimo/kit/common_kit.py +221 -0
  35. yimo/kit/config/__init__.py +44 -0
  36. yimo/kit/config/kit_config.py +210 -0
  37. yimo/kit/config/log_config.py +171 -0
  38. yimo/kit/crawl/__init__.py +71 -0
  39. yimo/kit/crawl/article_io.py +325 -0
  40. yimo/kit/crawl/article_models.py +172 -0
  41. yimo/kit/crawl/deepseek_kit.py +673 -0
  42. yimo/kit/crawl/web_article_kit.py +638 -0
  43. yimo/kit/crawl/wx_article_kit.py +692 -0
  44. yimo/kit/db/__init__.py +101 -0
  45. yimo/kit/db/audit.py +625 -0
  46. yimo/kit/db/base_db_kit.py +1265 -0
  47. yimo/kit/db/crud.py +423 -0
  48. yimo/kit/db/engine.py +235 -0
  49. yimo/kit/db/exceptions.py +45 -0
  50. yimo/kit/db/models/__init__.py +33 -0
  51. yimo/kit/db/models/base.py +56 -0
  52. yimo/kit/db/mysql_kit.py +348 -0
  53. yimo/kit/db/pg_kit.py +442 -0
  54. yimo/kit/db/query.py +338 -0
  55. yimo/kit/db/session.py +85 -0
  56. yimo/kit/db/sqlite_kit.py +462 -0
  57. yimo/kit/db/transaction.py +109 -0
  58. yimo/kit/docs/__init__.py +34 -0
  59. yimo/kit/docs/anydoc_kit.py +490 -0
  60. yimo/kit/docs/docx_kit.py +895 -0
  61. yimo/kit/docs/md2doc_kit.py +1801 -0
  62. yimo/kit/docs/mineru_kit.py +1497 -0
  63. yimo/kit/docs/rc_docparser_kit.py +654 -0
  64. yimo/kit/file_kit.py +518 -0
  65. yimo/kit/license_kit.py +533 -0
  66. yimo/kit/redis_kit.py +392 -0
  67. yimo_kit-0.1.0.dist-info/METADATA +1673 -0
  68. yimo_kit-0.1.0.dist-info/RECORD +70 -0
  69. yimo_kit-0.1.0.dist-info/WHEEL +4 -0
  70. yimo_kit-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,673 @@
1
+ """
2
+ Copyright (c) 2021-2026 溢墨修心(Yimo). All Rights Reserved.
3
+
4
+ Licensed under the Apache License, Version 2.0 (the "License");
5
+ you may not use this file except in compliance with the License.
6
+ You may obtain a copy of the License at
7
+
8
+ http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ Unless required by applicable law or agreed to in writing, software
11
+ distributed under the License is distributed on an "AS IS" BASIS,
12
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ See the License for the specific language governing permissions and
14
+ limitations under the License.
15
+
16
+ Project: yimo
17
+ Author: 溢墨修心(Yimo)
18
+ """
19
+
20
+ # date: 2026-10-21
21
+ """DeepSeek Search / Fetch API 工具类 DeepSeekKit
22
+
23
+ 封装官方两个只读接口(<https://api-docs.deepseek.com/zh-cn/guides/search>):
24
+
25
+ - `search(query, ...)`:`POST /v1/tools/search`,按 query 搜索网页 →
26
+ `SearchResult`(10 字段/条,含 `rerank_score` / `authority` 质量信号);
27
+ - `fetch(url)`:`POST /v1/tools/fetch`,抓取单个网页 → `ArticleResult`
28
+ (复用 `yimo.kit.crawl.article_models` 的公共模型,与另两个 `*ArticleKit` 同构);
29
+ - `save(url, output_dir, file_name=None, write_meta=True)`:`fetch` + 统一落盘
30
+ (由 `yimo.kit.crawl.article_io` 完成,三个 Kit 同一契约):
31
+
32
+ - `<output_dir>/<stem>/<stem>.md`:YAML 头 + 正文 Markdown;
33
+ - `images/`:正文含图则转存(md 引用本地化,失败保留原 URL);
34
+ - `meta.json`:结构化元信息(默认产出,`write_meta=False` 关闭)。
35
+ 省略 `file_name` 时目录按本地时间 `yymmddhhmmss` 自动命名(同秒追加 `-2`…)。
36
+
37
+ 设计要点:
38
+
39
+ - **零第三方依赖**(标准库 `urllib`),基础安装即可用,无需 extras;
40
+ - API Key 走配置(`DEEPSEEK_API_KEY`,见 `yimo.kit.config.kit_config`),不硬编码;
41
+ - 未配置 Key 属调用前置错误,抛 `DeepSeekKitError`(与 `MinerUKit` 一致);
42
+ - 重试按官方建议:仅 **408 / 429 / 5xx** 重试(指数退避),其余 4xx 不重试;
43
+ - 状态码映射到公共 `reason`:401/402 → `auth_failed`、403 → `fetch_blocked`、
44
+ 404 → `no_content`、429 → `rate_limited`、408/5xx → `fetch_failed`。
45
+
46
+ 正文质量(**重要,实测结论**):
47
+
48
+ - 接口返回的 `content` 是**重度反斜杠转义的伪 Markdown**(`\\#`、`\\:`、`\\|`、
49
+ `\\-`、`\\(`、`\\.` 等),落盘前统一做**反转义**,否则标题层级失效、表格与
50
+ 代码行全部带乱码;
51
+ - 该接口**不返回代码块围栏**(实测 0 个 ```,代码被压成普通段落后再被转义),
52
+ 反转义无法区分"原文真实转义"与"服务端加的转义",代码中的 `\\.` 会被还原为
53
+ `.`;需要高保真正文(代码块、图片)请改用
54
+ `yimo.kit.crawl.web_article_kit.WebArticleKit`;
55
+ - 图片实测大概率缺失(图片密集文章 `![` 出现 0 次),`images/` 多为空目录。
56
+
57
+ 使用方式(按需 import,无需经 kit 包顶层导出):
58
+
59
+ from yimo.kit.crawl.deepseek_kit import DeepSeekKit
60
+
61
+ found = DeepSeekKit.search("DeepSeek V4 发布", count=5)
62
+ for item in found.items:
63
+ print(item.title, item.url, item.rerank_score)
64
+
65
+ saved = DeepSeekKit.save("https://api-docs.deepseek.com/zh-cn/guides/search",
66
+ output_dir="output", file_name="ds-search-guide")
67
+ # output/ds-search-guide/ 下:ds-search-guide.md + images/ + meta.json
68
+ """
69
+
70
+ import json
71
+ import random
72
+ import re
73
+ import socket
74
+ import string
75
+ import time
76
+ import urllib.error
77
+ import urllib.parse
78
+ import urllib.request
79
+ from dataclasses import dataclass
80
+ from pathlib import Path
81
+ from typing import Any
82
+
83
+ from yimo.kit.config.kit_config import get_settings
84
+ from yimo.kit.config.log_config import get_default_logger
85
+
86
+ # 落盘公共层(md + images/ + meta.json,三个 Kit 共用同一契约)
87
+ from yimo.kit.crawl import article_io
88
+ from yimo.kit.crawl.article_models import (
89
+ REASON_AUTH_FAILED,
90
+ REASON_FETCH_BLOCKED,
91
+ REASON_FETCH_FAILED,
92
+ REASON_NO_CONTENT,
93
+ REASON_RATE_LIMITED,
94
+ ArticleImage,
95
+ ArticleInfo,
96
+ ArticleLink,
97
+ ArticleResult,
98
+ CodeBlock,
99
+ SavedOutput,
100
+ )
101
+
102
+ logger = get_default_logger(__name__)
103
+
104
+ # 接口路径(基础地址由 DEEPSEEK_BASE_URL 提供)
105
+ _SEARCH_PATH = "/v1/tools/search"
106
+ _FETCH_PATH = "/v1/tools/fetch"
107
+
108
+ # 官方限制:query 1~128 字符、count 1~50、url 长度 ≤8192
109
+ _MAX_QUERY_CHARS = 128
110
+ _MAX_COUNT = 50
111
+ _MAX_URL_CHARS = 8192
112
+
113
+ # 可重试状态码(官方建议:408/429/5xx 等待后重试,其它 4xx 不改请求勿重试)
114
+ _RETRYABLE_STATUS = frozenset({408, 429, 500, 502, 503, 504})
115
+
116
+ # 状态码 → (reason, 中文说明);未收录的按"请求不合法"处理
117
+ _STATUS_MAP: dict[int, tuple[str, str]] = {
118
+ 401: (REASON_AUTH_FAILED, "API Key 缺失或无效"),
119
+ 402: (REASON_AUTH_FAILED, "账号余额不足,请充值后重试"),
120
+ 403: (REASON_FETCH_BLOCKED, "请求被安全策略拒绝"),
121
+ 404: (REASON_NO_CONTENT, "未能获取可用的网页内容"),
122
+ 429: (REASON_RATE_LIMITED, "超过 QPS 限制"),
123
+ }
124
+
125
+ # ASCII 标点集合:CommonMark 允许用反斜杠转义的字符
126
+ _ASCII_PUNCT = frozenset(string.punctuation)
127
+
128
+ # 平台中立来源名(写入 ArticleInfo.source)
129
+ _SOURCE = "DeepSeek Web Fetch"
130
+
131
+ # Markdown 图片语法(含可选 title):![alt](src "title")
132
+ _MD_IMG_RE = re.compile(r'!\[[^\]]*\]\((https?://[^)\s]+)(?:\s+"[^"]*")?\)')
133
+ # Markdown 链接语法: [text](url)
134
+ _MD_LINK_RE = re.compile(r'(?<!!)\[([^\]]*)\]\((https?://[^)\s]+)(?:\s+"[^"]*")?\)')
135
+ # Markdown 围栏代码块:```lang\ncode\n```
136
+ _MD_CODE_RE = re.compile(r"^```([^\n`]*)\n(.*?)^```", re.S | re.M)
137
+ # 空标题行(页面空标题元素会被渲染成裸 `##`)
138
+ _EMPTY_HEADING_RE = re.compile(r"(?m)^#{1,6}[ \t]*$\n?")
139
+ # 3 个以上连续换行
140
+ _BLANK_LINES_RE = re.compile(r"\n{3,}")
141
+
142
+ # 图片扩展名推断、下载 UA、正文自检阈值均下沉到 yimo.kit.crawl.article_io / 配置
143
+ # (CRAWL_MIN_CONTENT_CHARS,默认 500,0 关闭)
144
+
145
+
146
+ class DeepSeekKitError(Exception):
147
+ """DeepSeekKit 调用前置错误(配置缺失等)"""
148
+
149
+
150
+ @dataclass(frozen=True, slots=True)
151
+ class SearchItem:
152
+ """单条搜索结果(字段与接口原始返回一一对应)
153
+
154
+ Attributes:
155
+ title: 标题
156
+ url: 结果链接
157
+ snippet: 短摘要(通常 100~200 字符,适合前端展现)
158
+ summary: 长摘要(实测可达 2000+ 字符,适合给模型阅读)
159
+ content: 网页正文(`need_content=False` 或正文缺失时为空串)
160
+ site_name: 站点名称,缺失时为空串
161
+ site_icon: 站点图标 URL,缺失时为空串
162
+ date: 发布时间(UTC 的 RFC 3339),未知时为空串
163
+ authority: 网页权威性 0~4(`-1` 表示未知),≥3 为极权威
164
+ rerank_score: 排序分数 0~3(≥1.5 为优质结果),非数值时为 0.0
165
+ """
166
+
167
+ title: str
168
+ url: str
169
+ snippet: str = ""
170
+ summary: str = ""
171
+ content: str = ""
172
+ site_name: str = ""
173
+ site_icon: str = ""
174
+ date: str = ""
175
+ authority: int = -1
176
+ rerank_score: float = 0.0
177
+
178
+ def to_dict(self) -> dict[str, Any]:
179
+ """序列化为可 JSON 化的 dict(业务侧自行落库/转存)"""
180
+ return {
181
+ "title": self.title,
182
+ "url": self.url,
183
+ "snippet": self.snippet,
184
+ "summary": self.summary,
185
+ "content": self.content,
186
+ "site_name": self.site_name,
187
+ "site_icon": self.site_icon,
188
+ "date": self.date,
189
+ "authority": self.authority,
190
+ "rerank_score": self.rerank_score,
191
+ }
192
+
193
+
194
+ @dataclass(frozen=True, slots=True)
195
+ class SearchResult:
196
+ """search() 的返回结果
197
+
198
+ `ok` 表示**请求是否成功**,不代表有结果:搜索正常结束但无匹配时
199
+ `ok=True`、`items` 为空,并在 `warnings` 中提示。
200
+ """
201
+
202
+ ok: bool
203
+ error: str = ""
204
+ reason: str = ""
205
+ items: tuple[SearchItem, ...] = ()
206
+ warnings: tuple[str, ...] = ()
207
+
208
+
209
+ class DeepSeekKit:
210
+ """DeepSeek Search / Fetch API 工具类(无状态,方法均为 `classmethod`)
211
+
212
+ 使用前在 `.env` 或环境变量中配置::
213
+
214
+ DEEPSEEK_API_KEY=sk-xxx
215
+
216
+ 使用方式::
217
+
218
+ found = DeepSeekKit.search("DeepSeek V4 发布", count=5, need_content=False)
219
+ if found.ok:
220
+ for item in found.items:
221
+ print(item.title, item.url, item.rerank_score)
222
+
223
+ result = DeepSeekKit.fetch("https://example.com/article")
224
+ if result.ok:
225
+ print(result.info.title, result.markdown)
226
+
227
+ saved = DeepSeekKit.save(url, output_dir="output", file_name="example")
228
+ # output/example/ 下:example.md + images/
229
+ """
230
+
231
+ # ------------------------------------------------------------------
232
+ # 公开方法
233
+ # ------------------------------------------------------------------
234
+ @classmethod
235
+ def search(
236
+ cls,
237
+ query: str,
238
+ count: int = 10,
239
+ need_content: bool = False,
240
+ ori_question: str | None = None,
241
+ localization: str | None = None,
242
+ timeout: float | None = None,
243
+ retries: int | None = None,
244
+ ) -> SearchResult:
245
+ """网页搜索(不落盘)
246
+
247
+ Args:
248
+ query: 搜索词,1~128 字符且不能全为空白;支持 `site:` / `-site:` /
249
+ `"..."` / `lang:` / `after:` / `before:` 等官方搜索语法
250
+ count: 返回条数,1~50(只影响条数,不影响计费)
251
+ need_content: 是否返回网页正文(`content` 字段)
252
+ ori_question: 触发搜索的用户原始问题(1~128 字符),有助于理解意图
253
+ localization: 用于优先返回本地化结果的 IP 地址
254
+ timeout: 单次请求超时(秒),None 取 `DEEPSEEK_TIMEOUT`
255
+ retries: 408/429/5xx 重试次数,None 取 `DEEPSEEK_MAX_RETRIES`
256
+
257
+ Returns:
258
+ SearchResult:`ok=True` 时 `items` 为结果元组(可能为空,
259
+ 此时 `warnings` 提示无匹配);失败时 `ok=False` 且 `error`/`reason`
260
+ 为类型化说明(不抛网络异常)
261
+
262
+ Raises:
263
+ ValueError: query 为空/超长、count 越界
264
+ DeepSeekKitError: 未配置 `DEEPSEEK_API_KEY`
265
+ """
266
+ text = (query or "").strip()
267
+ if not text:
268
+ raise ValueError("query 不能为空或全为空白")
269
+ if len(query) > _MAX_QUERY_CHARS:
270
+ raise ValueError(f"query 长度不能超过 {_MAX_QUERY_CHARS} 个字符")
271
+ if not 1 <= count <= _MAX_COUNT:
272
+ raise ValueError(f"count 必须在 1~{_MAX_COUNT} 之间")
273
+ if ori_question is not None and len(ori_question) > _MAX_QUERY_CHARS:
274
+ raise ValueError(
275
+ f"ori_question 长度不能超过 {_MAX_QUERY_CHARS} 个字符"
276
+ )
277
+
278
+ payload: dict[str, Any] = {
279
+ "query": query,
280
+ "count": count,
281
+ "need_content": need_content,
282
+ }
283
+ # 可选参数为 None 时不写入请求体,交由服务端取默认值
284
+ if ori_question:
285
+ payload["ori_question"] = ori_question
286
+ if localization:
287
+ payload["localization"] = localization
288
+
289
+ data, error, reason = cls._post(_SEARCH_PATH, payload, timeout, retries)
290
+ if data is None:
291
+ return SearchResult(False, error, reason)
292
+
293
+ items = tuple(
294
+ cls._parse_item(raw)
295
+ for raw in (data.get("results") or [])
296
+ if isinstance(raw, dict)
297
+ )
298
+ warnings = () if items else ("搜索无匹配结果",)
299
+ return SearchResult(True, "", "", items, warnings)
300
+
301
+ @classmethod
302
+ def fetch(
303
+ cls,
304
+ url: str,
305
+ timeout: float | None = None,
306
+ retries: int | None = None,
307
+ ) -> ArticleResult:
308
+ """抓取单篇网页并转为 Markdown(不落盘)
309
+
310
+ Args:
311
+ url: 目标网页(必须含 host 的 http(s) 链接)
312
+ timeout: 单次请求超时(秒),None 取 `DEEPSEEK_TIMEOUT`
313
+ retries: 408/429/5xx 重试次数,None 取 `DEEPSEEK_MAX_RETRIES`
314
+
315
+ Returns:
316
+ ArticleResult:`ok=True` 时 `info`/`markdown` 有值(正文已反转义);
317
+ 失败时 `ok=False` 且 `reason` 为类型化分类;`warnings` 为非致命提示
318
+
319
+ Raises:
320
+ ValueError: url 为空、非 http(s) 或不含主机名
321
+ DeepSeekKitError: 未配置 `DEEPSEEK_API_KEY`
322
+ """
323
+ target = (url or "").strip()
324
+ if not target.lower().startswith(("http://", "https://")):
325
+ raise ValueError("url 必须是合法的 http(s) 链接")
326
+ if len(target) > _MAX_URL_CHARS:
327
+ raise ValueError(f"url 长度不能超过 {_MAX_URL_CHARS} 个字符")
328
+ if not urllib.parse.urlparse(target).hostname:
329
+ raise ValueError("url 必须包含主机名(host)")
330
+
331
+ data, error, reason = cls._post(_FETCH_PATH, {"url": target}, timeout, retries)
332
+ if data is None:
333
+ return ArticleResult(False, error, None, "", reason)
334
+
335
+ raw_content = data.get("content")
336
+ markdown = cls._normalize_markdown(
337
+ raw_content if isinstance(raw_content, str) else ""
338
+ )
339
+ if not markdown:
340
+ return ArticleResult(
341
+ False, "Fetch 未返回可用的 Markdown 正文", None, "", REASON_NO_CONTENT
342
+ )
343
+
344
+ raw_title = data.get("title")
345
+ info = cls._build_info(
346
+ target, raw_title if isinstance(raw_title, str) else "", markdown
347
+ )
348
+ # 自检提示:既返回给调用方,也写 WARNING 日志(与 WebArticleKit 一致)
349
+ warnings = cls._self_check(markdown)
350
+ for message in warnings:
351
+ logger.warning(f"抽取自检提示: {target}({message})")
352
+ return ArticleResult(True, "", info, markdown, "", warnings)
353
+
354
+ @classmethod
355
+ def save(
356
+ cls,
357
+ url: str,
358
+ output_dir: str | Path,
359
+ file_name: str | None = None,
360
+ write_meta: bool = True,
361
+ timeout: float | None = None,
362
+ retries: int | None = None,
363
+ ) -> SavedOutput:
364
+ """Fetch 并存储到 `<output_dir>/<file_name>/`(`<file_name>.md` + `images/`)
365
+
366
+ 落盘统一由 `yimo.kit.crawl.article_io` 完成(三个 Kit 同一契约):
367
+ 正文与 `images/` 必定产出,`meta.json` 默认产出(`write_meta=False` 关闭)。
368
+
369
+ Args:
370
+ url: 目标网页
371
+ output_dir: 输出根目录(不存在自动创建)
372
+ file_name: 子目录名与 Markdown 文件名主干;`None` 时按本地时间
373
+ `%y%m%d%H%M%S` 自动命名(同秒碰撞追加 `-2`…),带 `.md` 后缀会自动去除
374
+ write_meta: 是否产出 meta.json(默认 True)
375
+ timeout: 单次请求超时(秒),None 取 `DEEPSEEK_TIMEOUT`
376
+ retries: 408/429/5xx 重试次数,None 取 `DEEPSEEK_MAX_RETRIES`
377
+
378
+ Returns:
379
+ SavedOutput:`ok=True` 时含文章子目录与各产物路径;失败时 `ok=False`
380
+ 带 `error`/`reason`
381
+
382
+ Raises:
383
+ ValueError: url 非法、显式传入的 file_name 为空
384
+ DeepSeekKitError: 未配置 `DEEPSEEK_API_KEY`
385
+ """
386
+ stem = article_io.resolve_stem(file_name, output_dir)
387
+
388
+ result = cls.fetch(url, timeout, retries)
389
+ if not result.ok:
390
+ return SavedOutput(False, result.error, reason=result.reason)
391
+
392
+ wait = timeout if timeout is not None else get_settings().DEEPSEEK_TIMEOUT
393
+ return article_io.save_article(
394
+ result,
395
+ output_dir,
396
+ stem,
397
+ write_meta,
398
+ label="DeepSeek Fetch 内容",
399
+ timeout=wait,
400
+ )
401
+
402
+ # ------------------------------------------------------------------
403
+ # 请求与错误处理
404
+ # ------------------------------------------------------------------
405
+ @staticmethod
406
+ def _api_key() -> str:
407
+ """读取 API Key,未配置时抛出 DeepSeekKitError"""
408
+ key = get_settings().DEEPSEEK_API_KEY
409
+ if not key:
410
+ raise DeepSeekKitError(
411
+ "未配置 DEEPSEEK_API_KEY,请在 .env 或环境变量中设置"
412
+ "(DeepSeek 开放平台创建)"
413
+ )
414
+ return key
415
+
416
+ @classmethod
417
+ def _post(
418
+ cls,
419
+ path: str,
420
+ payload: dict[str, Any],
421
+ timeout: float | None,
422
+ retries: int | None,
423
+ ) -> tuple[dict[str, Any] | None, str, str]:
424
+ """发送 JSON POST 并统一校验响应
425
+
426
+ Returns:
427
+ (data, error, reason):成功时 data 为响应字典、后两项为空串;
428
+ 失败时 data 为 None,error 为类型化说明、reason 为分类
429
+
430
+ Raises:
431
+ DeepSeekKitError: 未配置 API Key(调用前置错误)
432
+ """
433
+ settings = get_settings()
434
+ key = cls._api_key()
435
+ url = settings.DEEPSEEK_BASE_URL.rstrip("/") + path
436
+ wait = settings.DEEPSEEK_TIMEOUT if timeout is None else timeout
437
+ retry_times = (
438
+ settings.DEEPSEEK_MAX_RETRIES if retries is None else max(retries, 0)
439
+ )
440
+ attempts = retry_times + 1
441
+ body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
442
+ headers = {
443
+ "Authorization": f"Bearer {key}",
444
+ "Content-Type": "application/json",
445
+ }
446
+
447
+ last_error = "请求失败"
448
+ last_reason = REASON_FETCH_FAILED
449
+ for attempt in range(attempts):
450
+ request = urllib.request.Request(
451
+ url, data=body, headers=headers, method="POST"
452
+ )
453
+ try:
454
+ with urllib.request.urlopen(request, timeout=wait) as resp:
455
+ raw = resp.read()
456
+ except urllib.error.HTTPError as exc:
457
+ error, reason = cls._describe_http_error(exc, path)
458
+ retryable = exc.code in _RETRYABLE_STATUS
459
+ logger.warning(
460
+ f"DeepSeek API 失败: POST {path} -> HTTP {exc.code}({error})"
461
+ )
462
+ last_error, last_reason = error, reason
463
+ if not retryable or attempt >= attempts - 1:
464
+ return None, error, reason
465
+ except (
466
+ urllib.error.URLError,
467
+ socket.timeout,
468
+ TimeoutError,
469
+ OSError,
470
+ ) as exc:
471
+ if isinstance(exc, TimeoutError) or isinstance(
472
+ getattr(exc, "reason", None), (socket.timeout, TimeoutError)
473
+ ):
474
+ error, reason = f"请求超时({exc})", REASON_FETCH_FAILED
475
+ else:
476
+ error, reason = f"网络请求失败({exc})", REASON_FETCH_FAILED
477
+ logger.warning(f"DeepSeek API 失败: POST {path} -> {error}")
478
+ last_error, last_reason = error, reason
479
+ if attempt >= attempts - 1:
480
+ return None, error, reason
481
+ else:
482
+ return cls._parse_body(raw, path)
483
+
484
+ # 指数退避后重试(最多 8 秒 + 抖动)
485
+ delay = min(2**attempt, 8) + random.uniform(0, 0.5)
486
+ logger.info(
487
+ f"DeepSeek API 将重试(第 {attempt + 1}/{retry_times} 次): "
488
+ f"POST {path},等待 {delay:.1f}s"
489
+ )
490
+ time.sleep(delay)
491
+
492
+ return None, last_error, last_reason
493
+
494
+ @classmethod
495
+ def _parse_body(
496
+ cls, raw: bytes, path: str
497
+ ) -> tuple[dict[str, Any] | None, str, str]:
498
+ """解析 200 响应体:必须是 JSON 对象"""
499
+ try:
500
+ data = json.loads(raw.decode("utf-8"))
501
+ except (UnicodeDecodeError, json.JSONDecodeError):
502
+ logger.warning(f"DeepSeek API 返回非 JSON: POST {path}")
503
+ return None, "接口返回非 JSON(可能被网关拦截)", REASON_FETCH_FAILED
504
+ if not isinstance(data, dict):
505
+ logger.warning(f"DeepSeek API 返回非对象: POST {path}")
506
+ return None, "接口响应顶层不是 JSON 对象", REASON_FETCH_FAILED
507
+ return data, "", ""
508
+
509
+ @staticmethod
510
+ def _parse_item(raw: dict[str, Any]) -> SearchItem:
511
+ """把接口原始条目归一为 SearchItem(`null` 与非法类型统一为空值)
512
+
513
+ `authority` 非整数时记 `-1`(未知),`rerank_score` 非数值时记 `0.0`。
514
+ """
515
+
516
+ def text(key: str) -> str:
517
+ value = raw.get(key)
518
+ return value if isinstance(value, str) else ""
519
+
520
+ authority = raw.get("authority")
521
+ score = raw.get("rerank_score")
522
+ return SearchItem(
523
+ title=text("title"),
524
+ url=text("url"),
525
+ snippet=text("snippet"),
526
+ summary=text("summary"),
527
+ content=text("content"),
528
+ site_name=text("site_name"),
529
+ site_icon=text("site_icon"),
530
+ date=text("date"),
531
+ authority=authority if isinstance(authority, int) else -1,
532
+ rerank_score=float(score) if isinstance(score, (int, float)) else 0.0,
533
+ )
534
+
535
+ @classmethod
536
+ def _describe_http_error(
537
+ cls, exc: urllib.error.HTTPError, path: str
538
+ ) -> tuple[str, str]:
539
+ """把 HTTPError 映射为 (中文说明, reason)
540
+
541
+ 状态码未收录时按"请求不合法"处理(reason=`fetch_failed`,不重试)。
542
+ """
543
+ reason, label = _STATUS_MAP.get(
544
+ exc.code, (REASON_FETCH_FAILED, "请求被拒绝(请求不合法)")
545
+ )
546
+ detail = cls._error_message(exc)
547
+ suffix = f":{detail}" if detail else ""
548
+ return f"{label}(HTTP {exc.code}){suffix}", reason
549
+
550
+ @staticmethod
551
+ def _error_message(exc: urllib.error.HTTPError) -> str:
552
+ """尽力从错误响应体提取 `error.message`(OpenAI 兼容格式)"""
553
+ try:
554
+ raw = exc.read()
555
+ except (OSError, ValueError):
556
+ return ""
557
+ if not raw:
558
+ return ""
559
+ text = raw.decode("utf-8", errors="replace")
560
+ try:
561
+ payload = json.loads(text)
562
+ except json.JSONDecodeError:
563
+ return text[:200]
564
+ if not isinstance(payload, dict):
565
+ return text[:200]
566
+ error = payload.get("error")
567
+ if isinstance(error, dict):
568
+ message = error.get("message")
569
+ if isinstance(message, str) and message:
570
+ return message[:200]
571
+ return text[:200]
572
+
573
+ # ------------------------------------------------------------------
574
+ # 正文处理
575
+ # ------------------------------------------------------------------
576
+ @classmethod
577
+ def _normalize_markdown(cls, text: str) -> str:
578
+ """反转义 + 结构收敛(Fetch 正文必须经过这一步才可读)"""
579
+ if not text:
580
+ return ""
581
+ return cls._clean_markdown(cls._unescape_markdown(text))
582
+
583
+ @staticmethod
584
+ def _unescape_markdown(text: str) -> str:
585
+ """还原 Markdown 反斜杠转义(CommonMark:`\\` + ASCII 标点 → 标点)
586
+
587
+ 接口返回的正文把所有标点都做了转义(`\\#`、`\\:`、`\\|`、`\\-`、`\\(`),
588
+ 不还原则标题层级失效、表格与代码行全是乱码。
589
+
590
+ 注意:接口不返回代码块围栏,代码内容同样被转义,故此处**不做代码块豁免**
591
+ (无法与实际正文区分);代价是原文真实需要的 `\\.` 等会被还原。
592
+ """
593
+ out: list[str] = []
594
+ index = 0
595
+ length = len(text)
596
+ while index < length:
597
+ char = text[index]
598
+ if (
599
+ char == "\\"
600
+ and index + 1 < length
601
+ and text[index + 1] in _ASCII_PUNCT
602
+ ):
603
+ out.append(text[index + 1])
604
+ index += 2
605
+ continue
606
+ out.append(char)
607
+ index += 1
608
+ return "".join(out)
609
+
610
+ @staticmethod
611
+ def _clean_markdown(markdown: str) -> str:
612
+ """删除空标题行、把 3 个以上连续换行压成段落间空行"""
613
+ text = _EMPTY_HEADING_RE.sub("", markdown)
614
+ return _BLANK_LINES_RE.sub("\n\n", text).strip()
615
+
616
+ @classmethod
617
+ def _build_info(cls, url: str, title: str, markdown: str) -> ArticleInfo:
618
+ """组装 ArticleInfo(接口不返回作者/时间,相应字段留空)"""
619
+ return ArticleInfo(
620
+ url=url,
621
+ title=title.strip(),
622
+ source=_SOURCE,
623
+ site_host=urllib.parse.urlparse(url).netloc,
624
+ images=cls._collect_images(markdown),
625
+ links=cls._collect_links(markdown),
626
+ code_blocks=cls._collect_code_blocks(markdown),
627
+ )
628
+
629
+ @staticmethod
630
+ def _self_check(markdown: str) -> tuple[str, ...]:
631
+ """抽取自检(**非致命**):正文过短时给出提示,不改变 `ok` 语义
632
+
633
+ 阈值取自 `CRAWL_MIN_CONTENT_CHARS`(默认 500,设 `0` 关闭)。
634
+ """
635
+ min_chars = get_settings().CRAWL_MIN_CONTENT_CHARS
636
+ if min_chars > 0 and len(markdown) < min_chars:
637
+ return (f"正文仅 {len(markdown)} 字符,若原文本身较短可忽略",)
638
+ return ()
639
+
640
+ # ------------------------------------------------------------------
641
+ # 清单收集(图片 / 链接 / 代码块)
642
+ # ------------------------------------------------------------------
643
+ @staticmethod
644
+ def _collect_images(markdown: str) -> list[ArticleImage]:
645
+ """从 Markdown 收集远程图片清单(同 URL 去重,保留出现顺序)"""
646
+ return [
647
+ ArticleImage(src=src, original=src)
648
+ for src in dict.fromkeys(_MD_IMG_RE.findall(markdown))
649
+ ]
650
+
651
+ @staticmethod
652
+ def _collect_links(markdown: str) -> list[ArticleLink]:
653
+ """从 Markdown 收集链接清单(去重,按 (文本, 地址) 计)"""
654
+ links: list[ArticleLink] = []
655
+ seen: set[tuple[str, str]] = set()
656
+ for text, href in _MD_LINK_RE.findall(markdown):
657
+ key = (text.strip(), href)
658
+ if key in seen:
659
+ continue
660
+ seen.add(key)
661
+ links.append(ArticleLink(text=text.strip(), href=href))
662
+ return links
663
+
664
+ @staticmethod
665
+ def _collect_code_blocks(markdown: str) -> list[CodeBlock]:
666
+ """从 Markdown 收集代码块清单(语言缺失时记 `text`)
667
+
668
+ 接口通常不返回代码块围栏,故该清单可能为空。
669
+ """
670
+ return [
671
+ CodeBlock(language=(lang.strip() or "text"), code=code.strip("\n"))
672
+ for lang, code in _MD_CODE_RE.findall(markdown)
673
+ ]