web-forager 0.0.0-stage → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +119 -0
  18. package/CHANGELOG.md +508 -0
  19. package/LICENSE +21 -0
  20. package/README.md +498 -2
  21. package/README_zh.md +498 -0
  22. package/SKILL.md +451 -0
  23. package/THIRD_PARTY_LICENSES.txt +40 -0
  24. package/browser/bridges/playwright-bridge.js +382 -0
  25. package/browser/launch_chrome.py +193 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +90 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +141 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +130 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +215 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +252 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +107 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +441 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/research.md +35 -0
  58. package/commands/scholar.md +28 -0
  59. package/commands/surf.md +28 -0
  60. package/cordis.patch.yml +19 -0
  61. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  62. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  63. package/docs/DEPENDENCIES.md +40 -0
  64. package/docs/INSTALL_COMMANDS.md +113 -0
  65. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  66. package/docs/REFERENCE.md +349 -0
  67. package/docs/REFERENCE_zh.md +342 -0
  68. package/index.js +7 -0
  69. package/install.py +254 -0
  70. package/manifest.json +98 -0
  71. package/mcp_server.py +850 -0
  72. package/package.json +82 -6
  73. package/requirements.txt +15 -0
  74. package/run.py +908 -0
  75. package/scripts/__init__.py +1 -0
  76. package/scripts/bilibili.py +882 -0
  77. package/scripts/documents.py +437 -0
  78. package/scripts/engine.py +5141 -0
  79. package/scripts/feeds.py +180 -0
  80. package/scripts/pdf.py +156 -0
  81. package/scripts/research.py +2087 -0
  82. package/scripts/scholar.py +381 -0
  83. package/scripts/sitemap.py +195 -0
  84. package/scripts/untrusted.py +312 -0
@@ -0,0 +1,882 @@
1
+ """Bilibili site adapter — robots-compliant HTML extraction.
2
+
3
+ Why HTML instead of the JSON API?
4
+ ---------------------------------
5
+ ``https://api.bilibili.com/robots.txt`` disallows every path for generic
6
+ agents (``User-agent: * / Disallow: /``). We therefore read the public pages
7
+ that robots.txt *does* allow:
8
+
9
+ * ``https://www.bilibili.com/`` — home feed, server-rendered cards
10
+ * ``https://www.bilibili.com/video/BV…`` — video page, embedded
11
+ ``window.__INITIAL_STATE__`` JSON
12
+
13
+ Keyword search is resolved through Bing, then filtered to Bilibili video URLs.
14
+ The direct search page (``search.bilibili.com``) is disallowed by its own
15
+ robots.txt and is only used when the caller explicitly disables robots checks.
16
+
17
+ No third-party package beyond requests / beautifulsoup4 is used here.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import json
23
+ import re
24
+ import urllib.parse
25
+ from datetime import datetime, timezone
26
+ from typing import Any, Iterable, Optional
27
+
28
+ from . import engine
29
+
30
+ BILI_HOME = "https://www.bilibili.com/"
31
+ BILI_VIDEO = "https://www.bilibili.com/video/{bvid}/"
32
+ BILI_SEARCH = "https://search.bilibili.com/all?keyword={query}"
33
+ BILI_API_SEARCH = "https://api.bilibili.com/x/web-interface/search/type"
34
+
35
+ _BV_RE = re.compile(r"(BV[0-9A-Za-z]{10})")
36
+ _AV_RE = re.compile(r"\bav(\d+)\b", re.I)
37
+ _TITLE_SUFFIX_RE = re.compile(r"[_\-|]\s*哔哩哔哩.*$", re.I)
38
+
39
+
40
+
41
+ # ---------------------------------------------------------------------------
42
+ # Identifiers and formatting
43
+ # ---------------------------------------------------------------------------
44
+
45
+
46
+ def extract_bvid(value: str) -> str:
47
+ """Return a BV id from a URL, a bare BV id, or an av number."""
48
+
49
+ text = str(value or "").strip()
50
+ match = _BV_RE.search(text)
51
+ if match:
52
+ return match.group(1)
53
+ match = _AV_RE.search(text)
54
+ if match:
55
+ return f"av{match.group(1)}"
56
+ raise engine.InvalidURLError(f"no Bilibili video id found in: {value!r}")
57
+
58
+
59
+ def video_url(bvid_or_url: str) -> str:
60
+ text = str(bvid_or_url or "").strip()
61
+ if text.lower().startswith(("http://", "https://")):
62
+ return engine.normalize_url(text)
63
+ bvid = extract_bvid(text)
64
+ if bvid.startswith("av"):
65
+ return f"https://www.bilibili.com/video/{bvid}/"
66
+ return BILI_VIDEO.format(bvid=bvid)
67
+
68
+
69
+ def is_video_url(url: str) -> bool:
70
+ try:
71
+ parts = engine.normalize_url(url)
72
+ except engine.InvalidURLError:
73
+ return False
74
+ return "bilibili.com" in parts and "/video/" in parts
75
+
76
+
77
+ def is_b23_url(url: str) -> bool:
78
+ try:
79
+ parts = engine.normalize_url(url)
80
+ except engine.InvalidURLError:
81
+ return False
82
+ return "b23.tv" in parts
83
+
84
+
85
+ def format_duration(seconds: Any) -> str:
86
+ try:
87
+ total = int(seconds)
88
+ except (TypeError, ValueError):
89
+ return ""
90
+ if total < 0:
91
+ return ""
92
+ hours, remainder = divmod(total, 3600)
93
+ minutes, secs = divmod(remainder, 60)
94
+ if hours:
95
+ return f"{hours}:{minutes:02d}:{secs:02d}"
96
+ return f"{minutes}:{secs:02d}"
97
+
98
+
99
+ def _iso_timestamp(value: Any) -> str:
100
+ try:
101
+ stamp = int(value)
102
+ except (TypeError, ValueError):
103
+ return ""
104
+ if stamp <= 0:
105
+ return ""
106
+ return datetime.fromtimestamp(stamp, tz=timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
107
+
108
+
109
+ def clean_title(value: str) -> str:
110
+ text = engine._clean_result_text(value)
111
+ return _TITLE_SUFFIX_RE.sub("", text).strip()
112
+
113
+
114
+ # ---------------------------------------------------------------------------
115
+ # Embedded JSON
116
+ # ---------------------------------------------------------------------------
117
+
118
+
119
+ def _balanced_json(text: str, start: int) -> str:
120
+ """Return the balanced JSON object beginning at ``start``."""
121
+
122
+ depth = 0
123
+ in_string = False
124
+ escaped = False
125
+ for index in range(start, len(text)):
126
+ char = text[index]
127
+ if in_string:
128
+ if escaped:
129
+ escaped = False
130
+ elif char == "\\":
131
+ escaped = True
132
+ elif char == '"':
133
+ in_string = False
134
+ continue
135
+ if char == '"':
136
+ in_string = True
137
+ elif char == "{":
138
+ depth += 1
139
+ elif char == "}":
140
+ depth -= 1
141
+ if depth == 0:
142
+ return text[start : index + 1]
143
+ return ""
144
+
145
+
146
+ def extract_initial_state(html: str) -> dict[str, Any]:
147
+ """Extract ``window.__INITIAL_STATE__ = {...}`` from a Bilibili page."""
148
+
149
+ if not html:
150
+ return {}
151
+ marker = "__INITIAL_STATE__"
152
+ position = html.find(marker)
153
+ while position != -1:
154
+ equals = html.find("=", position)
155
+ brace = html.find("{", equals)
156
+ if equals != -1 and brace != -1 and brace - equals < 40:
157
+ payload = _balanced_json(html, brace)
158
+ if payload:
159
+ try:
160
+ data = json.loads(payload)
161
+ except json.JSONDecodeError:
162
+ data = None
163
+ if isinstance(data, dict):
164
+ return data
165
+ position = html.find(marker, position + len(marker))
166
+ return {}
167
+
168
+
169
+ # ---------------------------------------------------------------------------
170
+ # Parsers
171
+ # ---------------------------------------------------------------------------
172
+
173
+
174
+ _DEAD_TITLES = ("视频已失效", "已失效视频", "该视频已失效", "视频失效", "稿件不可见")
175
+
176
+
177
+ def _is_dead_video(item: dict[str, Any]) -> bool:
178
+ title = re.sub(r"\s+", "", str(item.get("title") or ""))
179
+ if any(marker in title for marker in _DEAD_TITLES):
180
+ return True
181
+ # A non-empty redirect field usually means the original video is gone.
182
+ return bool(item.get("redirect_url") or item.get("redirected_url"))
183
+
184
+
185
+ def _related_items(raw: Iterable[Any]) -> list[dict[str, Any]]:
186
+ related: list[dict[str, Any]] = []
187
+ seen: set[str] = set()
188
+ for item in raw or []:
189
+ if not isinstance(item, dict):
190
+ continue
191
+ bvid = item.get("bvid") or ""
192
+ if not bvid or bvid in seen:
193
+ continue
194
+ if _is_dead_video(item):
195
+ continue
196
+ seen.add(bvid)
197
+ owner = item.get("owner") or {}
198
+ stat = item.get("stat") or {}
199
+ related.append(
200
+ {
201
+ "bvid": bvid,
202
+ "url": BILI_VIDEO.format(bvid=bvid),
203
+ "title": clean_title(item.get("title") or ""),
204
+ "uploader": owner.get("name") or "",
205
+ "duration": format_duration(item.get("duration")),
206
+ "duration_seconds": item.get("duration") or 0,
207
+ "view": stat.get("view") or 0,
208
+ "cover": item.get("pic") or "",
209
+ }
210
+ )
211
+ return related
212
+
213
+
214
+ def parse_video_state(state: dict[str, Any], url: str = "") -> dict[str, Any]:
215
+ """Normalise the embedded video state into a stable record."""
216
+
217
+ data = state.get("videoData") or {}
218
+ if not isinstance(data, dict) or not data:
219
+ return {}
220
+
221
+ owner = data.get("owner") or {}
222
+ stat = data.get("stat") or {}
223
+ bvid = data.get("bvid") or (extract_bvid(url) if url else "")
224
+ pages = data.get("pages") or []
225
+ parts = [
226
+ {
227
+ "page": page.get("page"),
228
+ "title": page.get("part") or "",
229
+ "duration_seconds": page.get("duration") or 0,
230
+ "duration": format_duration(page.get("duration")),
231
+ }
232
+ for page in pages
233
+ if isinstance(page, dict)
234
+ ]
235
+
236
+ tags: list[str] = []
237
+ for tag in state.get("tags") or []:
238
+ if isinstance(tag, dict):
239
+ name = tag.get("tag_name") or tag.get("name")
240
+ if name:
241
+ tags.append(str(name))
242
+ elif isinstance(tag, str):
243
+ tags.append(tag)
244
+
245
+ related = _related_items(state.get("related") or data.get("related") or [])
246
+
247
+ record: dict[str, Any] = {
248
+ "bvid": bvid,
249
+ "aid": data.get("aid"),
250
+ "url": BILI_VIDEO.format(bvid=bvid) if bvid and not url else (url or ""),
251
+ "title": clean_title(data.get("title") or ""),
252
+ "description": (data.get("desc") or "").strip(),
253
+ "uploader": {
254
+ "name": owner.get("name") or "",
255
+ "mid": owner.get("mid"),
256
+ "url": f"https://space.bilibili.com/{owner.get('mid')}" if owner.get("mid") else "",
257
+ },
258
+ "published": _iso_timestamp(data.get("pubdate")),
259
+ "published_ts": data.get("pubdate") or 0,
260
+ "duration": format_duration(data.get("duration")),
261
+ "duration_seconds": data.get("duration") or 0,
262
+ "category": data.get("tname") or "",
263
+ "cover": data.get("pic") or "",
264
+ "stats": {
265
+ "view": stat.get("view") or 0,
266
+ "danmaku": stat.get("danmaku") or 0,
267
+ "reply": stat.get("reply") or 0,
268
+ "favorite": stat.get("favorite") or 0,
269
+ "coin": stat.get("coin") or 0,
270
+ "share": stat.get("share") or 0,
271
+ "like": stat.get("like") or 0,
272
+ },
273
+ "parts": parts,
274
+ "tags": tags,
275
+ "related": related,
276
+ }
277
+ record["markdown"] = video_markdown(record)
278
+ return record
279
+
280
+
281
+ def video_markdown(record: dict[str, Any]) -> str:
282
+ lines: list[str] = [f"# {record.get('title') or record.get('bvid') or 'Bilibili video'}", ""]
283
+ uploader = (record.get("uploader") or {}).get("name") or ""
284
+ stats = record.get("stats") or {}
285
+ meta = " | ".join(
286
+ part
287
+ for part in (
288
+ f"UP主:{uploader}" if uploader else "",
289
+ f"发布:{record.get('published', '')[:10]}" if record.get("published") else "",
290
+ f"时长:{record.get('duration')}" if record.get("duration") else "",
291
+ f"播放:{stats.get('view')}" if stats.get("view") else "",
292
+ f"弹幕:{stats.get('danmaku')}" if stats.get("danmaku") else "",
293
+ )
294
+ if part
295
+ )
296
+ if meta:
297
+ lines.extend([f"> {meta}", ""])
298
+ if record.get("description"):
299
+ lines.extend([record["description"], ""])
300
+ parts = record.get("parts") or []
301
+ if len(parts) > 1:
302
+ lines.append("## 分P")
303
+ lines.append("")
304
+ for part in parts:
305
+ lines.append(f"- P{part.get('page')} {part.get('title')} ({part.get('duration')})")
306
+ lines.append("")
307
+ related = record.get("related") or []
308
+ if related:
309
+ lines.append("## 相关推荐")
310
+ lines.append("")
311
+ for item in related[:20]:
312
+ suffix = f" · UP主:{item['uploader']}" if item.get("uploader") else ""
313
+ lines.append(f"- [{item['title']}]({item['url']}){suffix}")
314
+ lines.append("")
315
+ return "\n".join(lines).strip()
316
+
317
+
318
+ def parse_video_cards(html: str, base: str = BILI_HOME, limit: int = 20) -> list[dict[str, Any]]:
319
+ """Parse ``.bili-video-card`` blocks from the home page or search page."""
320
+
321
+ soup = engine.make_soup(html)
322
+ cards: list[dict[str, Any]] = []
323
+ seen: set[str] = set()
324
+
325
+ for card in soup.select(".bili-video-card"):
326
+ anchor = card.select_one("a[href*='/video/']")
327
+ if not anchor:
328
+ continue
329
+ href = engine._attr_text(anchor, "href")
330
+ url = engine.absolute_url(href, base)
331
+ if not url:
332
+ continue
333
+ try:
334
+ bvid = extract_bvid(url)
335
+ except engine.InvalidURLError:
336
+ continue
337
+ if bvid in seen:
338
+ continue
339
+ seen.add(bvid)
340
+
341
+ title_tag = card.select_one(".bili-video-card__info--tit")
342
+ title = ""
343
+ if title_tag is not None:
344
+ title = engine._attr_text(title_tag, "title") or title_tag.get_text(" ", strip=True)
345
+ if not title:
346
+ title = engine._attr_text(anchor, "title") or anchor.get_text(" ", strip=True)
347
+ title = clean_title(title)
348
+
349
+ author_tag = card.select_one(".bili-video-card__info--author")
350
+ author = author_tag.get_text(" ", strip=True) if author_tag is not None else ""
351
+
352
+ duration_tag = card.select_one(".bili-video-card__stats__duration")
353
+ duration = duration_tag.get_text(" ", strip=True) if duration_tag is not None else ""
354
+
355
+ stat_items = [
356
+ item.get_text(" ", strip=True) for item in card.select(".bili-video-card__stats--item")
357
+ ]
358
+ date_tag = card.select_one(".bili-video-card__info--date")
359
+ published_text = date_tag.get_text(" ", strip=True) if date_tag is not None else ""
360
+ cover_tag = card.select_one("img")
361
+ cover = engine._attr_text(cover_tag, "src") if cover_tag is not None else ""
362
+
363
+ cards.append(
364
+ {
365
+ "bvid": bvid,
366
+ "url": BILI_VIDEO.format(bvid=bvid),
367
+ "title": title,
368
+ "uploader": author,
369
+ "duration": duration,
370
+ "published_text": published_text,
371
+ "stats_text": stat_items,
372
+ "cover": engine.absolute_url(cover, base) if cover else "",
373
+ "markdown": f"- [{title}]({BILI_VIDEO.format(bvid=bvid)})"
374
+ + (f" · UP主:{author}" if author else "")
375
+ + (f" · {duration}" if duration else "")
376
+ + (f" · {published_text}" if published_text else ""),
377
+ }
378
+ )
379
+ if len(cards) >= limit:
380
+ break
381
+ return cards
382
+
383
+
384
+ def _https_url(value: str) -> str:
385
+ if value.startswith("//"):
386
+ return "https:" + value
387
+ return value.replace("http://", "https://")
388
+
389
+
390
+ def parse_api_search(payload: Any, limit: int = 20) -> list[dict[str, Any]]:
391
+ """Map Bilibili's JSON search payload onto the card shape.
392
+
393
+ The endpoint answers ``{"code": 0, "data": {"result": [...]}}``; any other
394
+ code (``-412`` is the anti-bot refusal) is reported as a FetchError so the
395
+ caller can fall back to the search page.
396
+ """
397
+
398
+ if not isinstance(payload, dict):
399
+ raise engine.FetchError("Bilibili search returned a non-JSON payload")
400
+ if payload.get("code") != 0:
401
+ raise engine.FetchError(
402
+ f"Bilibili search API code {payload.get('code')}: "
403
+ f"{payload.get('message') or 'no message'}"
404
+ )
405
+ data = payload.get("data") or {}
406
+ items = data.get("result") or []
407
+ cards: list[dict[str, Any]] = []
408
+ seen: set[str] = set()
409
+ for item in items:
410
+ if not isinstance(item, dict):
411
+ continue
412
+ bvid = str(item.get("bvid") or "").strip()
413
+ if not bvid and item.get("arcurl"):
414
+ match = _BV_RE.search(str(item["arcurl"]))
415
+ bvid = match.group(1) if match else ""
416
+ if not bvid or bvid in seen:
417
+ continue
418
+ seen.add(bvid)
419
+ title = engine._join_cjk(clean_title(item.get("title") or ""))
420
+ uploader = str(item.get("author") or "").strip()
421
+ duration = str(item.get("duration") or "").strip()
422
+ published = ""
423
+ if item.get("pubdate"):
424
+ try:
425
+ published = datetime.fromtimestamp(
426
+ int(item["pubdate"]), tz=timezone.utc
427
+ ).strftime("%Y-%m-%d")
428
+ except (OverflowError, OSError, ValueError):
429
+ published = ""
430
+ stats = []
431
+ if item.get("play") is not None:
432
+ stats.append(f"播放 {item['play']}")
433
+ if item.get("danmaku") is not None:
434
+ stats.append(f"弹幕 {item['danmaku']}")
435
+ url = BILI_VIDEO.format(bvid=bvid)
436
+ cards.append(
437
+ {
438
+ "bvid": bvid,
439
+ "url": url,
440
+ "title": title,
441
+ "uploader": uploader,
442
+ "duration": duration,
443
+ "published_text": published,
444
+ "stats_text": stats,
445
+ "cover": _https_url(str(item.get("pic") or "")),
446
+ "markdown": f"- [{title}]({url})"
447
+ + (f" · UP主:{uploader}" if uploader else "")
448
+ + (f" · {duration}" if duration else "")
449
+ + (f" · {published}" if published else ""),
450
+ }
451
+ )
452
+ if len(cards) >= limit:
453
+ break
454
+ return cards
455
+
456
+
457
+ # ---------------------------------------------------------------------------
458
+ # Fetching
459
+ # ---------------------------------------------------------------------------
460
+
461
+
462
+ def _metadata_from_record(record: dict[str, Any]) -> dict[str, Any]:
463
+ return {
464
+ "title": record.get("title", ""),
465
+ "authors": [record.get("uploader", {}).get("name", "")] if record.get("uploader", {}).get("name") else [],
466
+ "published": record.get("published", ""),
467
+ "modified": "",
468
+ "journal": "bilibili",
469
+ "volume": "",
470
+ "issue": "",
471
+ "first_page": "",
472
+ "last_page": "",
473
+ "doi": "",
474
+ "arxiv_id": "",
475
+ "abstract": record.get("description", ""),
476
+ "keywords": record.get("tags", []),
477
+ "language": "zh-CN",
478
+ "site_name": "bilibili",
479
+ "canonical_url": record.get("url", ""),
480
+ "og_type": "video",
481
+ }
482
+
483
+
484
+ def _fetch_raw(url: str, fetcher: engine.HttpFetcher) -> engine.HttpResponse:
485
+ return fetcher.fetch(url)
486
+
487
+
488
+ def fetch_video(
489
+ target: str,
490
+ config: Optional[engine.FetchConfig] = None,
491
+ fetcher: Optional[engine.HttpFetcher] = None,
492
+ ) -> dict[str, Any]:
493
+ """Fetch one Bilibili video page and return a normalised record."""
494
+
495
+ config = config or engine.FetchConfig()
496
+ own_fetcher = fetcher is None
497
+ fetcher = fetcher or engine.HttpFetcher(config)
498
+
499
+ try:
500
+ text = str(target or "").strip()
501
+ if text.lower().startswith(("http://", "https://")):
502
+ url = engine.normalize_url(text)
503
+ else:
504
+ url = video_url(text)
505
+
506
+ response = _fetch_raw(url, fetcher)
507
+ final_url = response.final_url
508
+
509
+ if is_b23_url(final_url):
510
+ # b23.tv redirects to the canonical video page.
511
+ response = _fetch_raw(final_url, fetcher)
512
+ final_url = response.final_url
513
+
514
+ record = parse_video_state(extract_initial_state(response.text), final_url)
515
+ if not record:
516
+ metadata = engine.extract_metadata(response.text, url=final_url)
517
+ markdown = engine.html_to_markdown(response.text, url=final_url)
518
+ record = {
519
+ "bvid": extract_bvid(final_url) if _BV_RE.search(final_url) else "",
520
+ "aid": None,
521
+ "url": final_url,
522
+ "title": metadata.get("title", ""),
523
+ "description": metadata.get("abstract", ""),
524
+ "uploader": {"name": "", "mid": None, "url": ""},
525
+ "published": metadata.get("published", ""),
526
+ "published_ts": 0,
527
+ "duration": "",
528
+ "duration_seconds": 0,
529
+ "category": "",
530
+ "cover": "",
531
+ "stats": {},
532
+ "parts": [],
533
+ "tags": metadata.get("keywords", []),
534
+ "related": [],
535
+ "markdown": markdown,
536
+ "fallback": True,
537
+ "warnings": ["embedded __INITIAL_STATE__ not found; used generic extraction"],
538
+ }
539
+
540
+ record.setdefault("warnings", [])
541
+ record["metadata"] = _metadata_from_record(record)
542
+ record["final_url"] = final_url
543
+ record["fetched_at"] = engine.utc_now()
544
+ return record
545
+ finally:
546
+ if own_fetcher:
547
+ fetcher.close()
548
+
549
+
550
+ def feed(
551
+ config: Optional[engine.FetchConfig] = None,
552
+ limit: int = 20,
553
+ enrich: int = 0,
554
+ url: str = BILI_HOME,
555
+ fetcher: Optional[engine.HttpFetcher] = None,
556
+ ) -> dict[str, Any]:
557
+ """Read the server-rendered home feed (robots.txt permits ``/``)."""
558
+
559
+ config = config or engine.FetchConfig()
560
+ own_fetcher = fetcher is None
561
+ fetcher = fetcher or engine.HttpFetcher(config)
562
+
563
+ try:
564
+ response = fetcher.fetch(url)
565
+ cards = parse_video_cards(response.text, base=response.final_url, limit=limit)
566
+ videos: list[dict[str, Any]] = []
567
+ for card in cards[: max(0, enrich)]:
568
+ try:
569
+ videos.append(fetch_video(card["url"], config, fetcher=fetcher))
570
+ except (engine.RobotsDenied, engine.FetchError, engine.ExtractionError) as exc:
571
+ card["error"] = f"{exc.__class__.__name__}: {exc}"
572
+ return {
573
+ "mode": "feed",
574
+ "source": response.final_url,
575
+ "cards": cards,
576
+ "videos": videos,
577
+ "results": videos or cards,
578
+ }
579
+ finally:
580
+ if own_fetcher:
581
+ fetcher.close()
582
+
583
+
584
+ def _search_via_engines(
585
+ query: str, config: engine.FetchConfig, fetcher: engine.HttpFetcher
586
+ ) -> list[dict[str, Any]]:
587
+ """Best-effort discovery: ask the web search engines, keep Bilibili video URLs."""
588
+
589
+ candidates: list[dict[str, Any]] = []
590
+ seen: set[str] = set()
591
+ for search_query in (f"哔哩哔哩 {query}", f"B站 {query} 视频"):
592
+ try:
593
+ results = engine.search(
594
+ search_query,
595
+ config,
596
+ engine=config.search_engine,
597
+ limit=config.num_results,
598
+ fetcher=fetcher,
599
+ )
600
+ except engine.FetchError:
601
+ continue
602
+ for item in results:
603
+ url = item.get("url", "")
604
+ if "bilibili.com" not in url or "/video/" not in url:
605
+ continue
606
+ try:
607
+ bvid = extract_bvid(url)
608
+ except engine.InvalidURLError:
609
+ continue
610
+ if bvid in seen:
611
+ continue
612
+ seen.add(bvid)
613
+ candidates.append(
614
+ {
615
+ "bvid": bvid,
616
+ "url": BILI_VIDEO.format(bvid=bvid),
617
+ "title": clean_title(item.get("title", "")),
618
+ "snippet": item.get("snippet", ""),
619
+ "source": item.get("source", config.search_engine),
620
+ }
621
+ )
622
+ if len(candidates) >= config.num_results:
623
+ break
624
+ return candidates
625
+
626
+
627
+ def _api_search(
628
+ query: str, config: engine.FetchConfig, fetcher: engine.HttpFetcher
629
+ ) -> list[dict[str, Any]]:
630
+ """Ask Bilibili's JSON search endpoint.
631
+
632
+ The endpoint needs the ``buvid3`` cookie that one visit to the home page
633
+ sets — without it the API answers ``-412``. The HttpFetcher's session keeps
634
+ cookies, so the same fetcher performs both requests.
635
+ """
636
+
637
+ try:
638
+ # Never from the cache: a cached anti-bot page supplies no cookie.
639
+ fetcher.fetch(BILI_HOME, check_robots=False, use_cache=False)
640
+ except engine.FetchError:
641
+ pass # the search call may still succeed on some networks
642
+ url = BILI_API_SEARCH + "?search_type=video&keyword=" + urllib.parse.quote_plus(
643
+ query
644
+ ) + "&page=1"
645
+ response = fetcher.fetch(url, check_robots=False, use_cache=False)
646
+ return parse_api_search(json.loads(response.text), limit=config.num_results)
647
+
648
+
649
+ def _filter_by_query(
650
+ cards: list[dict[str, Any]], query: str, limit: int
651
+ ) -> list[dict[str, Any]]:
652
+ """Keep only cards that mention the query text.
653
+
654
+ A search page can answer with a recommendation rail instead of results —
655
+ the cards look normal and carry nothing to do with the query. Dropping them
656
+ is what turns a silent wrong answer into an honest empty one.
657
+ """
658
+
659
+ tokens = engine._query_tokens(query)
660
+ if not tokens:
661
+ return cards[:limit]
662
+ kept = [card for card in cards if engine.result_coverage(card, tokens) > 0]
663
+ return kept[:limit]
664
+
665
+
666
+ def _search_direct(
667
+ query: str,
668
+ config: engine.FetchConfig,
669
+ fetcher: engine.HttpFetcher,
670
+ check_robots: bool = True,
671
+ ) -> tuple[list[dict[str, Any]], str]:
672
+ """Query Bilibili directly, JSON first and the search page second.
673
+
674
+ Both hosts are disallowed by their own robots.txt for generic agents, so
675
+ this is reached only when robots checks are off globally or
676
+ ``--direct-search`` narrows the override to this one query; video pages are
677
+ still fetched with robots checks enabled. The second return value names the
678
+ host that answered, or the one that was read when neither produced cards.
679
+ """
680
+
681
+ try:
682
+ cards = _api_search(query, config, fetcher)
683
+ except (engine.FetchError, ValueError, json.JSONDecodeError):
684
+ cards = []
685
+ if cards:
686
+ return cards, "api.bilibili.com"
687
+ url = BILI_SEARCH.format(query=urllib.parse.quote_plus(query))
688
+ response = fetcher.fetch(url, check_robots=check_robots)
689
+ cards = parse_video_cards(response.text, base=response.final_url, limit=config.num_results)
690
+ return _filter_by_query(cards, query, config.num_results), "search.bilibili.com"
691
+
692
+
693
+ def search(
694
+ query: str,
695
+ config: Optional[engine.FetchConfig] = None,
696
+ fetcher: Optional[engine.HttpFetcher] = None,
697
+ allow_direct: bool = False,
698
+ ) -> dict[str, Any]:
699
+ """Search Bilibili videos and optionally fetch the top results."""
700
+
701
+ config = config or engine.FetchConfig()
702
+ query = (query or "").strip()
703
+ if not query:
704
+ raise ValueError("query must not be empty")
705
+ reason = engine.doxxing_reason(query)
706
+ if reason:
707
+ raise engine.ProhibitedQuery(
708
+ f"refused: {reason}. web-forager does not collect or assemble "
709
+ "profiles of identifiable people; search for a topic, a paper "
710
+ "title or an organisation instead."
711
+ )
712
+
713
+ own_fetcher = fetcher is None
714
+ fetcher = fetcher or engine.HttpFetcher(config)
715
+
716
+ try:
717
+ if not config.respect_robots:
718
+ cards, route = _search_direct(query, config, fetcher, check_robots=False)
719
+ note = (
720
+ "direct search used because robots checks are disabled globally; "
721
+ f"{route} answered"
722
+ )
723
+ elif allow_direct:
724
+ if config.query_robots == "respect":
725
+ raise engine.RobotsDenied(
726
+ "--direct-search queries api.bilibili.com and search.bilibili.com, "
727
+ "which disallow generic agents in robots.txt. Pass "
728
+ "--query-robots warn (record the override) or ignore (skip the "
729
+ "check) only after you are authorised to query them."
730
+ )
731
+ cards, route = _search_direct(query, config, fetcher, check_robots=False)
732
+ note = (
733
+ f"--direct-search asked {route} directly under the user-selected "
734
+ "query-robots policy; video pages are still fetched with robots "
735
+ "checks enabled"
736
+ )
737
+ else:
738
+ cards = _search_via_engines(query, config, fetcher)
739
+ route = f"search:{config.search_engine}"
740
+ note = (
741
+ "search.bilibili.com disallows all agents in robots.txt; "
742
+ f"results were resolved through the {config.search_engine} search backend "
743
+ "and filtered to Bilibili video URLs"
744
+ )
745
+ if not cards:
746
+ note += (
747
+ "; the search engine returned no video URLs — pass --url with a seed video, "
748
+ "or --direct-search to query the Bilibili search page explicitly"
749
+ )
750
+
751
+ if not cards and route in ("search.bilibili.com", "api.bilibili.com"):
752
+ note += (
753
+ "; no video card mentioned the query. Bilibili's JSON search "
754
+ "refuses a User-Agent it does not recognise, and the search page "
755
+ "builds its cards in JavaScript — use browser mode's search-dig, "
756
+ "or pass --url with a seed video"
757
+ )
758
+
759
+ videos: list[dict[str, Any]] = []
760
+ for card in cards[: max(0, config.max_pages)]:
761
+ try:
762
+ videos.append(fetch_video(card["url"], config, fetcher=fetcher))
763
+ except (engine.RobotsDenied, engine.FetchError, engine.ExtractionError) as exc:
764
+ card["error"] = f"{exc.__class__.__name__}: {exc}"
765
+
766
+ return {
767
+ "mode": "search",
768
+ "query": query,
769
+ "query_robots": config.query_robots,
770
+ "route": route,
771
+ "note": note,
772
+ "search_results": cards,
773
+ "videos": videos,
774
+ "results": videos or cards,
775
+ }
776
+ finally:
777
+ if own_fetcher:
778
+ fetcher.close()
779
+
780
+
781
+ def related(
782
+ target: str,
783
+ config: Optional[engine.FetchConfig] = None,
784
+ limit: Optional[int] = None,
785
+ enrich: int = 0,
786
+ fetcher: Optional[engine.HttpFetcher] = None,
787
+ ) -> dict[str, Any]:
788
+ """Return the recommendation list of one video."""
789
+
790
+ config = config or engine.FetchConfig()
791
+ own_fetcher = fetcher is None
792
+ fetcher = fetcher or engine.HttpFetcher(config)
793
+
794
+ try:
795
+ record = fetch_video(target, config, fetcher=fetcher)
796
+ items = record.get("related", [])[: limit or config.num_results]
797
+ videos: list[dict[str, Any]] = []
798
+ for item in items[: max(0, enrich)]:
799
+ try:
800
+ videos.append(fetch_video(item["url"], config, fetcher=fetcher))
801
+ except (engine.RobotsDenied, engine.FetchError, engine.ExtractionError) as exc:
802
+ item["error"] = f"{exc.__class__.__name__}: {exc}"
803
+ return {
804
+ "mode": "related",
805
+ "seed": record,
806
+ "related": items,
807
+ "videos": videos,
808
+ "results": videos or items,
809
+ }
810
+ finally:
811
+ if own_fetcher:
812
+ fetcher.close()
813
+
814
+
815
+ def surf(
816
+ seed: str,
817
+ config: Optional[engine.FetchConfig] = None,
818
+ depth: int = 1,
819
+ per_level: Optional[int] = None,
820
+ fetcher: Optional[engine.HttpFetcher] = None,
821
+ allow_direct: bool = False,
822
+ ) -> dict[str, Any]:
823
+ """Walk the recommendation graph from a video or a keyword query."""
824
+
825
+ config = config or engine.FetchConfig()
826
+ own_fetcher = fetcher is None
827
+ fetcher = fetcher or engine.HttpFetcher(config)
828
+ per_level = per_level or config.max_pages
829
+
830
+ try:
831
+ text = str(seed or "").strip()
832
+ seeds: list[str] = []
833
+ search_payload: dict[str, Any] | None = None
834
+ if text.lower().startswith(("http://", "https://")) or _BV_RE.search(text) or _AV_RE.search(text):
835
+ seeds = [video_url(text)]
836
+ else:
837
+ search_payload = search(text, config, fetcher=fetcher, allow_direct=allow_direct)
838
+ seeds = [card["url"] for card in search_payload.get("search_results", [])[:per_level]]
839
+
840
+ visited: set[str] = set()
841
+ videos: list[dict[str, Any]] = []
842
+ edges: list[dict[str, str]] = []
843
+ queue = seeds
844
+
845
+ for _ in range(max(0, depth) + 1):
846
+ next_queue: list[str] = []
847
+ for url in queue:
848
+ try:
849
+ bvid = extract_bvid(url)
850
+ except engine.InvalidURLError:
851
+ continue
852
+ if bvid in visited:
853
+ continue
854
+ visited.add(bvid)
855
+ try:
856
+ record = fetch_video(url, config, fetcher=fetcher)
857
+ except (engine.RobotsDenied, engine.FetchError, engine.ExtractionError) as exc:
858
+ videos.append({"url": url, "bvid": bvid, "error": f"{exc.__class__.__name__}: {exc}"})
859
+ continue
860
+ videos.append(record)
861
+ for item in record.get("related", [])[:per_level]:
862
+ if item["bvid"] in visited:
863
+ continue
864
+ edges.append({"from": record.get("bvid", bvid), "to": item["bvid"], "title": item.get("title", "")})
865
+ next_queue.append(item["url"])
866
+ queue = next_queue[:per_level]
867
+ if not queue:
868
+ break
869
+
870
+ return {
871
+ "mode": "surf",
872
+ "seed": text,
873
+ "depth": depth,
874
+ "search": search_payload,
875
+ "edges": edges,
876
+ "visited": sorted(visited),
877
+ "videos": videos,
878
+ "results": videos,
879
+ }
880
+ finally:
881
+ if own_fetcher:
882
+ fetcher.close()