web-forager 0.0.0-stage → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +112 -0
- package/CHANGELOG.md +454 -0
- package/LICENSE +21 -0
- package/README.md +477 -3
- package/README_zh.md +477 -0
- package/SKILL.md +351 -0
- package/THIRD_PARTY_LICENSES.txt +35 -0
- package/browser/bridges/playwright-bridge.js +375 -0
- package/browser/launch_chrome.py +144 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +79 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +115 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +116 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +56 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +272 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +105 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +432 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +308 -0
- package/docs/REFERENCE_zh.md +309 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +97 -0
- package/mcp_server.py +597 -0
- package/package.json +80 -6
- package/requirements.txt +12 -0
- package/run.py +852 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +874 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +4220 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +135 -0
- package/scripts/scholar.py +354 -0
- package/scripts/sitemap.py +180 -0
- package/scripts/untrusted.py +95 -0
|
@@ -0,0 +1,874 @@
|
|
|
1
|
+
"""Bilibili site adapter — robots-compliant HTML extraction.
|
|
2
|
+
|
|
3
|
+
Why HTML instead of the JSON API?
|
|
4
|
+
---------------------------------
|
|
5
|
+
``https://api.bilibili.com/robots.txt`` disallows every path for generic
|
|
6
|
+
agents (``User-agent: * / Disallow: /``). We therefore read the public pages
|
|
7
|
+
that robots.txt *does* allow:
|
|
8
|
+
|
|
9
|
+
* ``https://www.bilibili.com/`` — home feed, server-rendered cards
|
|
10
|
+
* ``https://www.bilibili.com/video/BV…`` — video page, embedded
|
|
11
|
+
``window.__INITIAL_STATE__`` JSON
|
|
12
|
+
|
|
13
|
+
Keyword search is resolved through Bing, then filtered to Bilibili video URLs.
|
|
14
|
+
The direct search page (``search.bilibili.com``) is disallowed by its own
|
|
15
|
+
robots.txt and is only used when the caller explicitly disables robots checks.
|
|
16
|
+
|
|
17
|
+
No third-party package beyond requests / beautifulsoup4 is used here.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import re
|
|
24
|
+
import urllib.parse
|
|
25
|
+
from datetime import datetime, timezone
|
|
26
|
+
from typing import Any, Iterable, Optional
|
|
27
|
+
|
|
28
|
+
from . import engine
|
|
29
|
+
|
|
30
|
+
BILI_HOME = "https://www.bilibili.com/"
|
|
31
|
+
BILI_VIDEO = "https://www.bilibili.com/video/{bvid}/"
|
|
32
|
+
BILI_SEARCH = "https://search.bilibili.com/all?keyword={query}"
|
|
33
|
+
BILI_API_SEARCH = "https://api.bilibili.com/x/web-interface/search/type"
|
|
34
|
+
|
|
35
|
+
_BV_RE = re.compile(r"(BV[0-9A-Za-z]{10})")
|
|
36
|
+
_AV_RE = re.compile(r"\bav(\d+)\b", re.I)
|
|
37
|
+
_TITLE_SUFFIX_RE = re.compile(r"[_\-|]\s*哔哩哔哩.*$", re.I)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# ---------------------------------------------------------------------------
|
|
42
|
+
# Identifiers and formatting
|
|
43
|
+
# ---------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def extract_bvid(value: str) -> str:
|
|
47
|
+
"""Return a BV id from a URL, a bare BV id, or an av number."""
|
|
48
|
+
|
|
49
|
+
text = str(value or "").strip()
|
|
50
|
+
match = _BV_RE.search(text)
|
|
51
|
+
if match:
|
|
52
|
+
return match.group(1)
|
|
53
|
+
match = _AV_RE.search(text)
|
|
54
|
+
if match:
|
|
55
|
+
return f"av{match.group(1)}"
|
|
56
|
+
raise engine.InvalidURLError(f"no Bilibili video id found in: {value!r}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def video_url(bvid_or_url: str) -> str:
|
|
60
|
+
text = str(bvid_or_url or "").strip()
|
|
61
|
+
if text.lower().startswith(("http://", "https://")):
|
|
62
|
+
return engine.normalize_url(text)
|
|
63
|
+
bvid = extract_bvid(text)
|
|
64
|
+
if bvid.startswith("av"):
|
|
65
|
+
return f"https://www.bilibili.com/video/{bvid}/"
|
|
66
|
+
return BILI_VIDEO.format(bvid=bvid)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def is_video_url(url: str) -> bool:
|
|
70
|
+
try:
|
|
71
|
+
parts = engine.normalize_url(url)
|
|
72
|
+
except engine.InvalidURLError:
|
|
73
|
+
return False
|
|
74
|
+
return "bilibili.com" in parts and "/video/" in parts
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def is_b23_url(url: str) -> bool:
|
|
78
|
+
try:
|
|
79
|
+
parts = engine.normalize_url(url)
|
|
80
|
+
except engine.InvalidURLError:
|
|
81
|
+
return False
|
|
82
|
+
return "b23.tv" in parts
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def format_duration(seconds: Any) -> str:
|
|
86
|
+
try:
|
|
87
|
+
total = int(seconds)
|
|
88
|
+
except (TypeError, ValueError):
|
|
89
|
+
return ""
|
|
90
|
+
if total < 0:
|
|
91
|
+
return ""
|
|
92
|
+
hours, remainder = divmod(total, 3600)
|
|
93
|
+
minutes, secs = divmod(remainder, 60)
|
|
94
|
+
if hours:
|
|
95
|
+
return f"{hours}:{minutes:02d}:{secs:02d}"
|
|
96
|
+
return f"{minutes}:{secs:02d}"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _iso_timestamp(value: Any) -> str:
|
|
100
|
+
try:
|
|
101
|
+
stamp = int(value)
|
|
102
|
+
except (TypeError, ValueError):
|
|
103
|
+
return ""
|
|
104
|
+
if stamp <= 0:
|
|
105
|
+
return ""
|
|
106
|
+
return datetime.fromtimestamp(stamp, tz=timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def clean_title(value: str) -> str:
|
|
110
|
+
text = engine._clean_result_text(value)
|
|
111
|
+
return _TITLE_SUFFIX_RE.sub("", text).strip()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# ---------------------------------------------------------------------------
|
|
115
|
+
# Embedded JSON
|
|
116
|
+
# ---------------------------------------------------------------------------
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _balanced_json(text: str, start: int) -> str:
|
|
120
|
+
"""Return the balanced JSON object beginning at ``start``."""
|
|
121
|
+
|
|
122
|
+
depth = 0
|
|
123
|
+
in_string = False
|
|
124
|
+
escaped = False
|
|
125
|
+
for index in range(start, len(text)):
|
|
126
|
+
char = text[index]
|
|
127
|
+
if in_string:
|
|
128
|
+
if escaped:
|
|
129
|
+
escaped = False
|
|
130
|
+
elif char == "\\":
|
|
131
|
+
escaped = True
|
|
132
|
+
elif char == '"':
|
|
133
|
+
in_string = False
|
|
134
|
+
continue
|
|
135
|
+
if char == '"':
|
|
136
|
+
in_string = True
|
|
137
|
+
elif char == "{":
|
|
138
|
+
depth += 1
|
|
139
|
+
elif char == "}":
|
|
140
|
+
depth -= 1
|
|
141
|
+
if depth == 0:
|
|
142
|
+
return text[start : index + 1]
|
|
143
|
+
return ""
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def extract_initial_state(html: str) -> dict[str, Any]:
|
|
147
|
+
"""Extract ``window.__INITIAL_STATE__ = {...}`` from a Bilibili page."""
|
|
148
|
+
|
|
149
|
+
if not html:
|
|
150
|
+
return {}
|
|
151
|
+
marker = "__INITIAL_STATE__"
|
|
152
|
+
position = html.find(marker)
|
|
153
|
+
while position != -1:
|
|
154
|
+
equals = html.find("=", position)
|
|
155
|
+
brace = html.find("{", equals)
|
|
156
|
+
if equals != -1 and brace != -1 and brace - equals < 40:
|
|
157
|
+
payload = _balanced_json(html, brace)
|
|
158
|
+
if payload:
|
|
159
|
+
try:
|
|
160
|
+
data = json.loads(payload)
|
|
161
|
+
except json.JSONDecodeError:
|
|
162
|
+
data = None
|
|
163
|
+
if isinstance(data, dict):
|
|
164
|
+
return data
|
|
165
|
+
position = html.find(marker, position + len(marker))
|
|
166
|
+
return {}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# ---------------------------------------------------------------------------
|
|
170
|
+
# Parsers
|
|
171
|
+
# ---------------------------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
_DEAD_TITLES = ("视频已失效", "已失效视频", "该视频已失效", "视频失效", "稿件不可见")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _is_dead_video(item: dict[str, Any]) -> bool:
|
|
178
|
+
title = re.sub(r"\s+", "", str(item.get("title") or ""))
|
|
179
|
+
if any(marker in title for marker in _DEAD_TITLES):
|
|
180
|
+
return True
|
|
181
|
+
# A non-empty redirect field usually means the original video is gone.
|
|
182
|
+
return bool(item.get("redirect_url") or item.get("redirected_url"))
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _related_items(raw: Iterable[Any]) -> list[dict[str, Any]]:
|
|
186
|
+
related: list[dict[str, Any]] = []
|
|
187
|
+
seen: set[str] = set()
|
|
188
|
+
for item in raw or []:
|
|
189
|
+
if not isinstance(item, dict):
|
|
190
|
+
continue
|
|
191
|
+
bvid = item.get("bvid") or ""
|
|
192
|
+
if not bvid or bvid in seen:
|
|
193
|
+
continue
|
|
194
|
+
if _is_dead_video(item):
|
|
195
|
+
continue
|
|
196
|
+
seen.add(bvid)
|
|
197
|
+
owner = item.get("owner") or {}
|
|
198
|
+
stat = item.get("stat") or {}
|
|
199
|
+
related.append(
|
|
200
|
+
{
|
|
201
|
+
"bvid": bvid,
|
|
202
|
+
"url": BILI_VIDEO.format(bvid=bvid),
|
|
203
|
+
"title": clean_title(item.get("title") or ""),
|
|
204
|
+
"uploader": owner.get("name") or "",
|
|
205
|
+
"duration": format_duration(item.get("duration")),
|
|
206
|
+
"duration_seconds": item.get("duration") or 0,
|
|
207
|
+
"view": stat.get("view") or 0,
|
|
208
|
+
"cover": item.get("pic") or "",
|
|
209
|
+
}
|
|
210
|
+
)
|
|
211
|
+
return related
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def parse_video_state(state: dict[str, Any], url: str = "") -> dict[str, Any]:
|
|
215
|
+
"""Normalise the embedded video state into a stable record."""
|
|
216
|
+
|
|
217
|
+
data = state.get("videoData") or {}
|
|
218
|
+
if not isinstance(data, dict) or not data:
|
|
219
|
+
return {}
|
|
220
|
+
|
|
221
|
+
owner = data.get("owner") or {}
|
|
222
|
+
stat = data.get("stat") or {}
|
|
223
|
+
bvid = data.get("bvid") or (extract_bvid(url) if url else "")
|
|
224
|
+
pages = data.get("pages") or []
|
|
225
|
+
parts = [
|
|
226
|
+
{
|
|
227
|
+
"page": page.get("page"),
|
|
228
|
+
"title": page.get("part") or "",
|
|
229
|
+
"duration_seconds": page.get("duration") or 0,
|
|
230
|
+
"duration": format_duration(page.get("duration")),
|
|
231
|
+
}
|
|
232
|
+
for page in pages
|
|
233
|
+
if isinstance(page, dict)
|
|
234
|
+
]
|
|
235
|
+
|
|
236
|
+
tags: list[str] = []
|
|
237
|
+
for tag in state.get("tags") or []:
|
|
238
|
+
if isinstance(tag, dict):
|
|
239
|
+
name = tag.get("tag_name") or tag.get("name")
|
|
240
|
+
if name:
|
|
241
|
+
tags.append(str(name))
|
|
242
|
+
elif isinstance(tag, str):
|
|
243
|
+
tags.append(tag)
|
|
244
|
+
|
|
245
|
+
related = _related_items(state.get("related") or data.get("related") or [])
|
|
246
|
+
|
|
247
|
+
record: dict[str, Any] = {
|
|
248
|
+
"bvid": bvid,
|
|
249
|
+
"aid": data.get("aid"),
|
|
250
|
+
"url": BILI_VIDEO.format(bvid=bvid) if bvid and not url else (url or ""),
|
|
251
|
+
"title": clean_title(data.get("title") or ""),
|
|
252
|
+
"description": (data.get("desc") or "").strip(),
|
|
253
|
+
"uploader": {
|
|
254
|
+
"name": owner.get("name") or "",
|
|
255
|
+
"mid": owner.get("mid"),
|
|
256
|
+
"url": f"https://space.bilibili.com/{owner.get('mid')}" if owner.get("mid") else "",
|
|
257
|
+
},
|
|
258
|
+
"published": _iso_timestamp(data.get("pubdate")),
|
|
259
|
+
"published_ts": data.get("pubdate") or 0,
|
|
260
|
+
"duration": format_duration(data.get("duration")),
|
|
261
|
+
"duration_seconds": data.get("duration") or 0,
|
|
262
|
+
"category": data.get("tname") or "",
|
|
263
|
+
"cover": data.get("pic") or "",
|
|
264
|
+
"stats": {
|
|
265
|
+
"view": stat.get("view") or 0,
|
|
266
|
+
"danmaku": stat.get("danmaku") or 0,
|
|
267
|
+
"reply": stat.get("reply") or 0,
|
|
268
|
+
"favorite": stat.get("favorite") or 0,
|
|
269
|
+
"coin": stat.get("coin") or 0,
|
|
270
|
+
"share": stat.get("share") or 0,
|
|
271
|
+
"like": stat.get("like") or 0,
|
|
272
|
+
},
|
|
273
|
+
"parts": parts,
|
|
274
|
+
"tags": tags,
|
|
275
|
+
"related": related,
|
|
276
|
+
}
|
|
277
|
+
record["markdown"] = video_markdown(record)
|
|
278
|
+
return record
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def video_markdown(record: dict[str, Any]) -> str:
|
|
282
|
+
lines: list[str] = [f"# {record.get('title') or record.get('bvid') or 'Bilibili video'}", ""]
|
|
283
|
+
uploader = (record.get("uploader") or {}).get("name") or ""
|
|
284
|
+
stats = record.get("stats") or {}
|
|
285
|
+
meta = " | ".join(
|
|
286
|
+
part
|
|
287
|
+
for part in (
|
|
288
|
+
f"UP主:{uploader}" if uploader else "",
|
|
289
|
+
f"发布:{record.get('published', '')[:10]}" if record.get("published") else "",
|
|
290
|
+
f"时长:{record.get('duration')}" if record.get("duration") else "",
|
|
291
|
+
f"播放:{stats.get('view')}" if stats.get("view") else "",
|
|
292
|
+
f"弹幕:{stats.get('danmaku')}" if stats.get("danmaku") else "",
|
|
293
|
+
)
|
|
294
|
+
if part
|
|
295
|
+
)
|
|
296
|
+
if meta:
|
|
297
|
+
lines.extend([f"> {meta}", ""])
|
|
298
|
+
if record.get("description"):
|
|
299
|
+
lines.extend([record["description"], ""])
|
|
300
|
+
parts = record.get("parts") or []
|
|
301
|
+
if len(parts) > 1:
|
|
302
|
+
lines.append("## 分P")
|
|
303
|
+
lines.append("")
|
|
304
|
+
for part in parts:
|
|
305
|
+
lines.append(f"- P{part.get('page')} {part.get('title')} ({part.get('duration')})")
|
|
306
|
+
lines.append("")
|
|
307
|
+
related = record.get("related") or []
|
|
308
|
+
if related:
|
|
309
|
+
lines.append("## 相关推荐")
|
|
310
|
+
lines.append("")
|
|
311
|
+
for item in related[:20]:
|
|
312
|
+
suffix = f" · UP主:{item['uploader']}" if item.get("uploader") else ""
|
|
313
|
+
lines.append(f"- [{item['title']}]({item['url']}){suffix}")
|
|
314
|
+
lines.append("")
|
|
315
|
+
return "\n".join(lines).strip()
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def parse_video_cards(html: str, base: str = BILI_HOME, limit: int = 20) -> list[dict[str, Any]]:
|
|
319
|
+
"""Parse ``.bili-video-card`` blocks from the home page or search page."""
|
|
320
|
+
|
|
321
|
+
soup = engine.make_soup(html)
|
|
322
|
+
cards: list[dict[str, Any]] = []
|
|
323
|
+
seen: set[str] = set()
|
|
324
|
+
|
|
325
|
+
for card in soup.select(".bili-video-card"):
|
|
326
|
+
anchor = card.select_one("a[href*='/video/']")
|
|
327
|
+
if not anchor:
|
|
328
|
+
continue
|
|
329
|
+
href = engine._attr_text(anchor, "href")
|
|
330
|
+
url = engine.absolute_url(href, base)
|
|
331
|
+
if not url:
|
|
332
|
+
continue
|
|
333
|
+
try:
|
|
334
|
+
bvid = extract_bvid(url)
|
|
335
|
+
except engine.InvalidURLError:
|
|
336
|
+
continue
|
|
337
|
+
if bvid in seen:
|
|
338
|
+
continue
|
|
339
|
+
seen.add(bvid)
|
|
340
|
+
|
|
341
|
+
title_tag = card.select_one(".bili-video-card__info--tit")
|
|
342
|
+
title = ""
|
|
343
|
+
if title_tag is not None:
|
|
344
|
+
title = engine._attr_text(title_tag, "title") or title_tag.get_text(" ", strip=True)
|
|
345
|
+
if not title:
|
|
346
|
+
title = engine._attr_text(anchor, "title") or anchor.get_text(" ", strip=True)
|
|
347
|
+
title = clean_title(title)
|
|
348
|
+
|
|
349
|
+
author_tag = card.select_one(".bili-video-card__info--author")
|
|
350
|
+
author = author_tag.get_text(" ", strip=True) if author_tag is not None else ""
|
|
351
|
+
|
|
352
|
+
duration_tag = card.select_one(".bili-video-card__stats__duration")
|
|
353
|
+
duration = duration_tag.get_text(" ", strip=True) if duration_tag is not None else ""
|
|
354
|
+
|
|
355
|
+
stat_items = [
|
|
356
|
+
item.get_text(" ", strip=True) for item in card.select(".bili-video-card__stats--item")
|
|
357
|
+
]
|
|
358
|
+
date_tag = card.select_one(".bili-video-card__info--date")
|
|
359
|
+
published_text = date_tag.get_text(" ", strip=True) if date_tag is not None else ""
|
|
360
|
+
cover_tag = card.select_one("img")
|
|
361
|
+
cover = engine._attr_text(cover_tag, "src") if cover_tag is not None else ""
|
|
362
|
+
|
|
363
|
+
cards.append(
|
|
364
|
+
{
|
|
365
|
+
"bvid": bvid,
|
|
366
|
+
"url": BILI_VIDEO.format(bvid=bvid),
|
|
367
|
+
"title": title,
|
|
368
|
+
"uploader": author,
|
|
369
|
+
"duration": duration,
|
|
370
|
+
"published_text": published_text,
|
|
371
|
+
"stats_text": stat_items,
|
|
372
|
+
"cover": engine.absolute_url(cover, base) if cover else "",
|
|
373
|
+
"markdown": f"- [{title}]({BILI_VIDEO.format(bvid=bvid)})"
|
|
374
|
+
+ (f" · UP主:{author}" if author else "")
|
|
375
|
+
+ (f" · {duration}" if duration else "")
|
|
376
|
+
+ (f" · {published_text}" if published_text else ""),
|
|
377
|
+
}
|
|
378
|
+
)
|
|
379
|
+
if len(cards) >= limit:
|
|
380
|
+
break
|
|
381
|
+
return cards
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _https_url(value: str) -> str:
|
|
385
|
+
if value.startswith("//"):
|
|
386
|
+
return "https:" + value
|
|
387
|
+
return value.replace("http://", "https://")
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def parse_api_search(payload: Any, limit: int = 20) -> list[dict[str, Any]]:
|
|
391
|
+
"""Map Bilibili's JSON search payload onto the card shape.
|
|
392
|
+
|
|
393
|
+
The endpoint answers ``{"code": 0, "data": {"result": [...]}}``; any other
|
|
394
|
+
code (``-412`` is the anti-bot refusal) is reported as a FetchError so the
|
|
395
|
+
caller can fall back to the search page.
|
|
396
|
+
"""
|
|
397
|
+
|
|
398
|
+
if not isinstance(payload, dict):
|
|
399
|
+
raise engine.FetchError("Bilibili search returned a non-JSON payload")
|
|
400
|
+
if payload.get("code") != 0:
|
|
401
|
+
raise engine.FetchError(
|
|
402
|
+
f"Bilibili search API code {payload.get('code')}: "
|
|
403
|
+
f"{payload.get('message') or 'no message'}"
|
|
404
|
+
)
|
|
405
|
+
data = payload.get("data") or {}
|
|
406
|
+
items = data.get("result") or []
|
|
407
|
+
cards: list[dict[str, Any]] = []
|
|
408
|
+
seen: set[str] = set()
|
|
409
|
+
for item in items:
|
|
410
|
+
if not isinstance(item, dict):
|
|
411
|
+
continue
|
|
412
|
+
bvid = str(item.get("bvid") or "").strip()
|
|
413
|
+
if not bvid and item.get("arcurl"):
|
|
414
|
+
match = _BV_RE.search(str(item["arcurl"]))
|
|
415
|
+
bvid = match.group(1) if match else ""
|
|
416
|
+
if not bvid or bvid in seen:
|
|
417
|
+
continue
|
|
418
|
+
seen.add(bvid)
|
|
419
|
+
title = engine._join_cjk(clean_title(item.get("title") or ""))
|
|
420
|
+
uploader = str(item.get("author") or "").strip()
|
|
421
|
+
duration = str(item.get("duration") or "").strip()
|
|
422
|
+
published = ""
|
|
423
|
+
if item.get("pubdate"):
|
|
424
|
+
try:
|
|
425
|
+
published = datetime.fromtimestamp(
|
|
426
|
+
int(item["pubdate"]), tz=timezone.utc
|
|
427
|
+
).strftime("%Y-%m-%d")
|
|
428
|
+
except (OverflowError, OSError, ValueError):
|
|
429
|
+
published = ""
|
|
430
|
+
stats = []
|
|
431
|
+
if item.get("play") is not None:
|
|
432
|
+
stats.append(f"播放 {item['play']}")
|
|
433
|
+
if item.get("danmaku") is not None:
|
|
434
|
+
stats.append(f"弹幕 {item['danmaku']}")
|
|
435
|
+
url = BILI_VIDEO.format(bvid=bvid)
|
|
436
|
+
cards.append(
|
|
437
|
+
{
|
|
438
|
+
"bvid": bvid,
|
|
439
|
+
"url": url,
|
|
440
|
+
"title": title,
|
|
441
|
+
"uploader": uploader,
|
|
442
|
+
"duration": duration,
|
|
443
|
+
"published_text": published,
|
|
444
|
+
"stats_text": stats,
|
|
445
|
+
"cover": _https_url(str(item.get("pic") or "")),
|
|
446
|
+
"markdown": f"- [{title}]({url})"
|
|
447
|
+
+ (f" · UP主:{uploader}" if uploader else "")
|
|
448
|
+
+ (f" · {duration}" if duration else "")
|
|
449
|
+
+ (f" · {published}" if published else ""),
|
|
450
|
+
}
|
|
451
|
+
)
|
|
452
|
+
if len(cards) >= limit:
|
|
453
|
+
break
|
|
454
|
+
return cards
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
# ---------------------------------------------------------------------------
|
|
458
|
+
# Fetching
|
|
459
|
+
# ---------------------------------------------------------------------------
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _metadata_from_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
463
|
+
return {
|
|
464
|
+
"title": record.get("title", ""),
|
|
465
|
+
"authors": [record.get("uploader", {}).get("name", "")] if record.get("uploader", {}).get("name") else [],
|
|
466
|
+
"published": record.get("published", ""),
|
|
467
|
+
"modified": "",
|
|
468
|
+
"journal": "bilibili",
|
|
469
|
+
"volume": "",
|
|
470
|
+
"issue": "",
|
|
471
|
+
"first_page": "",
|
|
472
|
+
"last_page": "",
|
|
473
|
+
"doi": "",
|
|
474
|
+
"arxiv_id": "",
|
|
475
|
+
"abstract": record.get("description", ""),
|
|
476
|
+
"keywords": record.get("tags", []),
|
|
477
|
+
"language": "zh-CN",
|
|
478
|
+
"site_name": "bilibili",
|
|
479
|
+
"canonical_url": record.get("url", ""),
|
|
480
|
+
"og_type": "video",
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _fetch_raw(url: str, fetcher: engine.HttpFetcher) -> engine.HttpResponse:
|
|
485
|
+
return fetcher.fetch(url)
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def fetch_video(
|
|
489
|
+
target: str,
|
|
490
|
+
config: Optional[engine.FetchConfig] = None,
|
|
491
|
+
fetcher: Optional[engine.HttpFetcher] = None,
|
|
492
|
+
) -> dict[str, Any]:
|
|
493
|
+
"""Fetch one Bilibili video page and return a normalised record."""
|
|
494
|
+
|
|
495
|
+
config = config or engine.FetchConfig()
|
|
496
|
+
own_fetcher = fetcher is None
|
|
497
|
+
fetcher = fetcher or engine.HttpFetcher(config)
|
|
498
|
+
|
|
499
|
+
try:
|
|
500
|
+
text = str(target or "").strip()
|
|
501
|
+
if text.lower().startswith(("http://", "https://")):
|
|
502
|
+
url = engine.normalize_url(text)
|
|
503
|
+
else:
|
|
504
|
+
url = video_url(text)
|
|
505
|
+
|
|
506
|
+
response = _fetch_raw(url, fetcher)
|
|
507
|
+
final_url = response.final_url
|
|
508
|
+
|
|
509
|
+
if is_b23_url(final_url):
|
|
510
|
+
# b23.tv redirects to the canonical video page.
|
|
511
|
+
response = _fetch_raw(final_url, fetcher)
|
|
512
|
+
final_url = response.final_url
|
|
513
|
+
|
|
514
|
+
record = parse_video_state(extract_initial_state(response.text), final_url)
|
|
515
|
+
if not record:
|
|
516
|
+
metadata = engine.extract_metadata(response.text, url=final_url)
|
|
517
|
+
markdown = engine.html_to_markdown(response.text, url=final_url)
|
|
518
|
+
record = {
|
|
519
|
+
"bvid": extract_bvid(final_url) if _BV_RE.search(final_url) else "",
|
|
520
|
+
"aid": None,
|
|
521
|
+
"url": final_url,
|
|
522
|
+
"title": metadata.get("title", ""),
|
|
523
|
+
"description": metadata.get("abstract", ""),
|
|
524
|
+
"uploader": {"name": "", "mid": None, "url": ""},
|
|
525
|
+
"published": metadata.get("published", ""),
|
|
526
|
+
"published_ts": 0,
|
|
527
|
+
"duration": "",
|
|
528
|
+
"duration_seconds": 0,
|
|
529
|
+
"category": "",
|
|
530
|
+
"cover": "",
|
|
531
|
+
"stats": {},
|
|
532
|
+
"parts": [],
|
|
533
|
+
"tags": metadata.get("keywords", []),
|
|
534
|
+
"related": [],
|
|
535
|
+
"markdown": markdown,
|
|
536
|
+
"fallback": True,
|
|
537
|
+
"warnings": ["embedded __INITIAL_STATE__ not found; used generic extraction"],
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
record.setdefault("warnings", [])
|
|
541
|
+
record["metadata"] = _metadata_from_record(record)
|
|
542
|
+
record["final_url"] = final_url
|
|
543
|
+
record["fetched_at"] = engine.utc_now()
|
|
544
|
+
return record
|
|
545
|
+
finally:
|
|
546
|
+
if own_fetcher:
|
|
547
|
+
fetcher.close()
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def feed(
|
|
551
|
+
config: Optional[engine.FetchConfig] = None,
|
|
552
|
+
limit: int = 20,
|
|
553
|
+
enrich: int = 0,
|
|
554
|
+
url: str = BILI_HOME,
|
|
555
|
+
fetcher: Optional[engine.HttpFetcher] = None,
|
|
556
|
+
) -> dict[str, Any]:
|
|
557
|
+
"""Read the server-rendered home feed (robots.txt permits ``/``)."""
|
|
558
|
+
|
|
559
|
+
config = config or engine.FetchConfig()
|
|
560
|
+
own_fetcher = fetcher is None
|
|
561
|
+
fetcher = fetcher or engine.HttpFetcher(config)
|
|
562
|
+
|
|
563
|
+
try:
|
|
564
|
+
response = fetcher.fetch(url)
|
|
565
|
+
cards = parse_video_cards(response.text, base=response.final_url, limit=limit)
|
|
566
|
+
videos: list[dict[str, Any]] = []
|
|
567
|
+
for card in cards[: max(0, enrich)]:
|
|
568
|
+
try:
|
|
569
|
+
videos.append(fetch_video(card["url"], config, fetcher=fetcher))
|
|
570
|
+
except (engine.RobotsDenied, engine.FetchError, engine.ExtractionError) as exc:
|
|
571
|
+
card["error"] = f"{exc.__class__.__name__}: {exc}"
|
|
572
|
+
return {
|
|
573
|
+
"mode": "feed",
|
|
574
|
+
"source": response.final_url,
|
|
575
|
+
"cards": cards,
|
|
576
|
+
"videos": videos,
|
|
577
|
+
"results": videos or cards,
|
|
578
|
+
}
|
|
579
|
+
finally:
|
|
580
|
+
if own_fetcher:
|
|
581
|
+
fetcher.close()
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _search_via_engines(
|
|
585
|
+
query: str, config: engine.FetchConfig, fetcher: engine.HttpFetcher
|
|
586
|
+
) -> list[dict[str, Any]]:
|
|
587
|
+
"""Best-effort discovery: ask the web search engines, keep Bilibili video URLs."""
|
|
588
|
+
|
|
589
|
+
candidates: list[dict[str, Any]] = []
|
|
590
|
+
seen: set[str] = set()
|
|
591
|
+
for search_query in (f"哔哩哔哩 {query}", f"B站 {query} 视频"):
|
|
592
|
+
try:
|
|
593
|
+
results = engine.search(
|
|
594
|
+
search_query,
|
|
595
|
+
config,
|
|
596
|
+
engine=config.search_engine,
|
|
597
|
+
limit=config.num_results,
|
|
598
|
+
fetcher=fetcher,
|
|
599
|
+
)
|
|
600
|
+
except engine.FetchError:
|
|
601
|
+
continue
|
|
602
|
+
for item in results:
|
|
603
|
+
url = item.get("url", "")
|
|
604
|
+
if "bilibili.com" not in url or "/video/" not in url:
|
|
605
|
+
continue
|
|
606
|
+
try:
|
|
607
|
+
bvid = extract_bvid(url)
|
|
608
|
+
except engine.InvalidURLError:
|
|
609
|
+
continue
|
|
610
|
+
if bvid in seen:
|
|
611
|
+
continue
|
|
612
|
+
seen.add(bvid)
|
|
613
|
+
candidates.append(
|
|
614
|
+
{
|
|
615
|
+
"bvid": bvid,
|
|
616
|
+
"url": BILI_VIDEO.format(bvid=bvid),
|
|
617
|
+
"title": clean_title(item.get("title", "")),
|
|
618
|
+
"snippet": item.get("snippet", ""),
|
|
619
|
+
"source": item.get("source", config.search_engine),
|
|
620
|
+
}
|
|
621
|
+
)
|
|
622
|
+
if len(candidates) >= config.num_results:
|
|
623
|
+
break
|
|
624
|
+
return candidates
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def _api_search(
|
|
628
|
+
query: str, config: engine.FetchConfig, fetcher: engine.HttpFetcher
|
|
629
|
+
) -> list[dict[str, Any]]:
|
|
630
|
+
"""Ask Bilibili's JSON search endpoint.
|
|
631
|
+
|
|
632
|
+
The endpoint needs the ``buvid3`` cookie that one visit to the home page
|
|
633
|
+
sets — without it the API answers ``-412``. The HttpFetcher's session keeps
|
|
634
|
+
cookies, so the same fetcher performs both requests.
|
|
635
|
+
"""
|
|
636
|
+
|
|
637
|
+
try:
|
|
638
|
+
# Never from the cache: a cached anti-bot page supplies no cookie.
|
|
639
|
+
fetcher.fetch(BILI_HOME, check_robots=False, use_cache=False)
|
|
640
|
+
except engine.FetchError:
|
|
641
|
+
pass # the search call may still succeed on some networks
|
|
642
|
+
url = BILI_API_SEARCH + "?search_type=video&keyword=" + urllib.parse.quote_plus(
|
|
643
|
+
query
|
|
644
|
+
) + "&page=1"
|
|
645
|
+
response = fetcher.fetch(url, check_robots=False, use_cache=False)
|
|
646
|
+
return parse_api_search(json.loads(response.text), limit=config.num_results)
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def _filter_by_query(
|
|
650
|
+
cards: list[dict[str, Any]], query: str, limit: int
|
|
651
|
+
) -> list[dict[str, Any]]:
|
|
652
|
+
"""Keep only cards that mention the query text.
|
|
653
|
+
|
|
654
|
+
A search page can answer with a recommendation rail instead of results —
|
|
655
|
+
the cards look normal and carry nothing to do with the query. Dropping them
|
|
656
|
+
is what turns a silent wrong answer into an honest empty one.
|
|
657
|
+
"""
|
|
658
|
+
|
|
659
|
+
tokens = engine._query_tokens(query)
|
|
660
|
+
if not tokens:
|
|
661
|
+
return cards[:limit]
|
|
662
|
+
kept = [card for card in cards if engine.result_coverage(card, tokens) > 0]
|
|
663
|
+
return kept[:limit]
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _search_direct(
|
|
667
|
+
query: str,
|
|
668
|
+
config: engine.FetchConfig,
|
|
669
|
+
fetcher: engine.HttpFetcher,
|
|
670
|
+
check_robots: bool = True,
|
|
671
|
+
) -> tuple[list[dict[str, Any]], str]:
|
|
672
|
+
"""Query Bilibili directly, JSON first and the search page second.
|
|
673
|
+
|
|
674
|
+
Both hosts are disallowed by their own robots.txt for generic agents, so
|
|
675
|
+
this is reached only when robots checks are off globally or
|
|
676
|
+
``--direct-search`` narrows the override to this one query; video pages are
|
|
677
|
+
still fetched with robots checks enabled. The second return value names the
|
|
678
|
+
host that answered, or the one that was read when neither produced cards.
|
|
679
|
+
"""
|
|
680
|
+
|
|
681
|
+
try:
|
|
682
|
+
cards = _api_search(query, config, fetcher)
|
|
683
|
+
except (engine.FetchError, ValueError, json.JSONDecodeError):
|
|
684
|
+
cards = []
|
|
685
|
+
if cards:
|
|
686
|
+
return cards, "api.bilibili.com"
|
|
687
|
+
url = BILI_SEARCH.format(query=urllib.parse.quote_plus(query))
|
|
688
|
+
response = fetcher.fetch(url, check_robots=check_robots)
|
|
689
|
+
cards = parse_video_cards(response.text, base=response.final_url, limit=config.num_results)
|
|
690
|
+
return _filter_by_query(cards, query, config.num_results), "search.bilibili.com"
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def search(
|
|
694
|
+
query: str,
|
|
695
|
+
config: Optional[engine.FetchConfig] = None,
|
|
696
|
+
fetcher: Optional[engine.HttpFetcher] = None,
|
|
697
|
+
allow_direct: bool = False,
|
|
698
|
+
) -> dict[str, Any]:
|
|
699
|
+
"""Search Bilibili videos and optionally fetch the top results."""
|
|
700
|
+
|
|
701
|
+
config = config or engine.FetchConfig()
|
|
702
|
+
query = (query or "").strip()
|
|
703
|
+
if not query:
|
|
704
|
+
raise ValueError("query must not be empty")
|
|
705
|
+
reason = engine.doxxing_reason(query)
|
|
706
|
+
if reason:
|
|
707
|
+
raise engine.ProhibitedQuery(
|
|
708
|
+
f"refused: {reason}. web-forager does not collect or assemble "
|
|
709
|
+
"profiles of identifiable people; search for a topic, a paper "
|
|
710
|
+
"title or an organisation instead."
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
own_fetcher = fetcher is None
|
|
714
|
+
fetcher = fetcher or engine.HttpFetcher(config)
|
|
715
|
+
|
|
716
|
+
try:
|
|
717
|
+
if not config.respect_robots:
|
|
718
|
+
cards, route = _search_direct(query, config, fetcher, check_robots=False)
|
|
719
|
+
note = (
|
|
720
|
+
"direct search used because robots checks are disabled globally; "
|
|
721
|
+
f"{route} answered"
|
|
722
|
+
)
|
|
723
|
+
elif allow_direct:
|
|
724
|
+
cards, route = _search_direct(query, config, fetcher, check_robots=False)
|
|
725
|
+
note = (
|
|
726
|
+
f"--direct-search asked {route} directly; that host disallows "
|
|
727
|
+
"generic agents in robots.txt, and video pages are still fetched "
|
|
728
|
+
"with robots checks enabled"
|
|
729
|
+
)
|
|
730
|
+
else:
|
|
731
|
+
cards = _search_via_engines(query, config, fetcher)
|
|
732
|
+
route = f"search:{config.search_engine}"
|
|
733
|
+
note = (
|
|
734
|
+
"search.bilibili.com disallows all agents in robots.txt; "
|
|
735
|
+
f"results were resolved through the {config.search_engine} search backend "
|
|
736
|
+
"and filtered to Bilibili video URLs"
|
|
737
|
+
)
|
|
738
|
+
if not cards:
|
|
739
|
+
note += (
|
|
740
|
+
"; the search engine returned no video URLs — pass --url with a seed video, "
|
|
741
|
+
"or --direct-search to query the Bilibili search page explicitly"
|
|
742
|
+
)
|
|
743
|
+
|
|
744
|
+
if not cards and route in ("search.bilibili.com", "api.bilibili.com"):
|
|
745
|
+
note += (
|
|
746
|
+
"; no video card mentioned the query. Bilibili's JSON search "
|
|
747
|
+
"refuses a User-Agent it does not recognise, and the search page "
|
|
748
|
+
"builds its cards in JavaScript — use browser mode's search-dig, "
|
|
749
|
+
"or pass --url with a seed video"
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
videos: list[dict[str, Any]] = []
|
|
753
|
+
for card in cards[: max(0, config.max_pages)]:
|
|
754
|
+
try:
|
|
755
|
+
videos.append(fetch_video(card["url"], config, fetcher=fetcher))
|
|
756
|
+
except (engine.RobotsDenied, engine.FetchError, engine.ExtractionError) as exc:
|
|
757
|
+
card["error"] = f"{exc.__class__.__name__}: {exc}"
|
|
758
|
+
|
|
759
|
+
return {
|
|
760
|
+
"mode": "search",
|
|
761
|
+
"query": query,
|
|
762
|
+
"route": route,
|
|
763
|
+
"note": note,
|
|
764
|
+
"search_results": cards,
|
|
765
|
+
"videos": videos,
|
|
766
|
+
"results": videos or cards,
|
|
767
|
+
}
|
|
768
|
+
finally:
|
|
769
|
+
if own_fetcher:
|
|
770
|
+
fetcher.close()
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def related(
|
|
774
|
+
target: str,
|
|
775
|
+
config: Optional[engine.FetchConfig] = None,
|
|
776
|
+
limit: Optional[int] = None,
|
|
777
|
+
enrich: int = 0,
|
|
778
|
+
fetcher: Optional[engine.HttpFetcher] = None,
|
|
779
|
+
) -> dict[str, Any]:
|
|
780
|
+
"""Return the recommendation list of one video."""
|
|
781
|
+
|
|
782
|
+
config = config or engine.FetchConfig()
|
|
783
|
+
own_fetcher = fetcher is None
|
|
784
|
+
fetcher = fetcher or engine.HttpFetcher(config)
|
|
785
|
+
|
|
786
|
+
try:
|
|
787
|
+
record = fetch_video(target, config, fetcher=fetcher)
|
|
788
|
+
items = record.get("related", [])[: limit or config.num_results]
|
|
789
|
+
videos: list[dict[str, Any]] = []
|
|
790
|
+
for item in items[: max(0, enrich)]:
|
|
791
|
+
try:
|
|
792
|
+
videos.append(fetch_video(item["url"], config, fetcher=fetcher))
|
|
793
|
+
except (engine.RobotsDenied, engine.FetchError, engine.ExtractionError) as exc:
|
|
794
|
+
item["error"] = f"{exc.__class__.__name__}: {exc}"
|
|
795
|
+
return {
|
|
796
|
+
"mode": "related",
|
|
797
|
+
"seed": record,
|
|
798
|
+
"related": items,
|
|
799
|
+
"videos": videos,
|
|
800
|
+
"results": videos or items,
|
|
801
|
+
}
|
|
802
|
+
finally:
|
|
803
|
+
if own_fetcher:
|
|
804
|
+
fetcher.close()
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def surf(
|
|
808
|
+
seed: str,
|
|
809
|
+
config: Optional[engine.FetchConfig] = None,
|
|
810
|
+
depth: int = 1,
|
|
811
|
+
per_level: Optional[int] = None,
|
|
812
|
+
fetcher: Optional[engine.HttpFetcher] = None,
|
|
813
|
+
allow_direct: bool = False,
|
|
814
|
+
) -> dict[str, Any]:
|
|
815
|
+
"""Walk the recommendation graph from a video or a keyword query."""
|
|
816
|
+
|
|
817
|
+
config = config or engine.FetchConfig()
|
|
818
|
+
own_fetcher = fetcher is None
|
|
819
|
+
fetcher = fetcher or engine.HttpFetcher(config)
|
|
820
|
+
per_level = per_level or config.max_pages
|
|
821
|
+
|
|
822
|
+
try:
|
|
823
|
+
text = str(seed or "").strip()
|
|
824
|
+
seeds: list[str] = []
|
|
825
|
+
search_payload: dict[str, Any] | None = None
|
|
826
|
+
if text.lower().startswith(("http://", "https://")) or _BV_RE.search(text) or _AV_RE.search(text):
|
|
827
|
+
seeds = [video_url(text)]
|
|
828
|
+
else:
|
|
829
|
+
search_payload = search(text, config, fetcher=fetcher, allow_direct=allow_direct)
|
|
830
|
+
seeds = [card["url"] for card in search_payload.get("search_results", [])[:per_level]]
|
|
831
|
+
|
|
832
|
+
visited: set[str] = set()
|
|
833
|
+
videos: list[dict[str, Any]] = []
|
|
834
|
+
edges: list[dict[str, str]] = []
|
|
835
|
+
queue = seeds
|
|
836
|
+
|
|
837
|
+
for _ in range(max(0, depth) + 1):
|
|
838
|
+
next_queue: list[str] = []
|
|
839
|
+
for url in queue:
|
|
840
|
+
try:
|
|
841
|
+
bvid = extract_bvid(url)
|
|
842
|
+
except engine.InvalidURLError:
|
|
843
|
+
continue
|
|
844
|
+
if bvid in visited:
|
|
845
|
+
continue
|
|
846
|
+
visited.add(bvid)
|
|
847
|
+
try:
|
|
848
|
+
record = fetch_video(url, config, fetcher=fetcher)
|
|
849
|
+
except (engine.RobotsDenied, engine.FetchError, engine.ExtractionError) as exc:
|
|
850
|
+
videos.append({"url": url, "bvid": bvid, "error": f"{exc.__class__.__name__}: {exc}"})
|
|
851
|
+
continue
|
|
852
|
+
videos.append(record)
|
|
853
|
+
for item in record.get("related", [])[:per_level]:
|
|
854
|
+
if item["bvid"] in visited:
|
|
855
|
+
continue
|
|
856
|
+
edges.append({"from": record.get("bvid", bvid), "to": item["bvid"], "title": item.get("title", "")})
|
|
857
|
+
next_queue.append(item["url"])
|
|
858
|
+
queue = next_queue[:per_level]
|
|
859
|
+
if not queue:
|
|
860
|
+
break
|
|
861
|
+
|
|
862
|
+
return {
|
|
863
|
+
"mode": "surf",
|
|
864
|
+
"seed": text,
|
|
865
|
+
"depth": depth,
|
|
866
|
+
"search": search_payload,
|
|
867
|
+
"edges": edges,
|
|
868
|
+
"visited": sorted(visited),
|
|
869
|
+
"videos": videos,
|
|
870
|
+
"results": videos,
|
|
871
|
+
}
|
|
872
|
+
finally:
|
|
873
|
+
if own_fetcher:
|
|
874
|
+
fetcher.close()
|