openmd-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. openmd/__init__.py +3 -0
  2. openmd/__main__.py +5 -0
  3. openmd/adapters/__init__.py +1 -0
  4. openmd/adapters/base.py +95 -0
  5. openmd/adapters/bilibili/__init__.py +11 -0
  6. openmd/adapters/bilibili/adapter.py +104 -0
  7. openmd/adapters/bilibili/discovery.py +33 -0
  8. openmd/adapters/douyin/__init__.py +11 -0
  9. openmd/adapters/douyin/adapter.py +117 -0
  10. openmd/adapters/douyin/discovery.py +36 -0
  11. openmd/adapters/douyin/sidecar.py +77 -0
  12. openmd/adapters/generic_media/__init__.py +14 -0
  13. openmd/adapters/high_privacy.py +77 -0
  14. openmd/adapters/local_audio/__init__.py +14 -0
  15. openmd/adapters/local_audio/adapter.py +169 -0
  16. openmd/adapters/local_audio/asr.py +58 -0
  17. openmd/adapters/local_audio/backends/__init__.py +1 -0
  18. openmd/adapters/local_audio/backends/auto.py +177 -0
  19. openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
  20. openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
  21. openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
  22. openmd/adapters/opencli.py +770 -0
  23. openmd/adapters/pdf/__init__.py +11 -0
  24. openmd/adapters/pdf/adapter.py +153 -0
  25. openmd/adapters/pdf/backends/__init__.py +1 -0
  26. openmd/adapters/pdf/backends/markitdown.py +165 -0
  27. openmd/adapters/pdf/parser.py +56 -0
  28. openmd/adapters/qq_group/__init__.py +11 -0
  29. openmd/adapters/qq_group/adapter.py +289 -0
  30. openmd/adapters/qq_group/parsers/__init__.py +1 -0
  31. openmd/adapters/qq_group/parsers/html.py +60 -0
  32. openmd/adapters/qq_group/parsers/json.py +71 -0
  33. openmd/adapters/registry.py +149 -0
  34. openmd/adapters/screenshot/__init__.py +11 -0
  35. openmd/adapters/screenshot/adapter.py +154 -0
  36. openmd/adapters/screenshot/backends/__init__.py +1 -0
  37. openmd/adapters/screenshot/backends/paddle.py +15 -0
  38. openmd/adapters/screenshot/backends/rapidocr.py +176 -0
  39. openmd/adapters/screenshot/backends/tesseract.py +15 -0
  40. openmd/adapters/screenshot/ocr.py +84 -0
  41. openmd/adapters/wechat_group/__init__.py +11 -0
  42. openmd/adapters/wechat_group/adapter.py +160 -0
  43. openmd/adapters/wechat_group/wx_cli.py +268 -0
  44. openmd/adapters/wechat_moments/__init__.py +11 -0
  45. openmd/adapters/wechat_moments/adapter.py +167 -0
  46. openmd/adapters/wechat_moments/wx_cli.py +265 -0
  47. openmd/adapters/weixin/__init__.py +11 -0
  48. openmd/adapters/weixin/adapter.py +148 -0
  49. openmd/adapters/weixin/discovery.py +510 -0
  50. openmd/adapters/weixin/report.py +278 -0
  51. openmd/adapters/xiaohongshu.py +158 -0
  52. openmd/backends/__init__.py +1 -0
  53. openmd/backends/base.py +204 -0
  54. openmd/backends/bilibili.py +1480 -0
  55. openmd/backends/bilibili_wbi.py +277 -0
  56. openmd/backends/discourse.py +426 -0
  57. openmd/backends/douyin.py +868 -0
  58. openmd/backends/douyin_detail.py +124 -0
  59. openmd/backends/douyin_providers.py +602 -0
  60. openmd/backends/executor.py +546 -0
  61. openmd/backends/feishu.py +861 -0
  62. openmd/backends/generic_media.py +77 -0
  63. openmd/backends/interactive.py +60 -0
  64. openmd/backends/ocr.py +41 -0
  65. openmd/backends/opencli.py +82 -0
  66. openmd/backends/pdf.py +122 -0
  67. openmd/backends/registry.py +234 -0
  68. openmd/backends/screenshot.py +79 -0
  69. openmd/backends/video_metadata.py +167 -0
  70. openmd/backends/web_article.py +363 -0
  71. openmd/backends/wechat_group.py +35 -0
  72. openmd/backends/wechat_moments.py +35 -0
  73. openmd/backends/weixin.py +767 -0
  74. openmd/backends/x.py +455 -0
  75. openmd/backends/xiaoe.py +940 -0
  76. openmd/backends/xiaohongshu.py +911 -0
  77. openmd/backends/xiaohongshu_providers.py +876 -0
  78. openmd/backends/xiaoyuzhou.py +1017 -0
  79. openmd/backends/youtube.py +280 -0
  80. openmd/backends/youtube_discovery.py +1245 -0
  81. openmd/backends/youtube_transcript.py +213 -0
  82. openmd/browser_extension/background.js +231 -0
  83. openmd/browser_extension/icons/icon-128.png +0 -0
  84. openmd/browser_extension/icons/icon-16.png +0 -0
  85. openmd/browser_extension/icons/icon-32.png +0 -0
  86. openmd/browser_extension/icons/icon-48.png +0 -0
  87. openmd/browser_extension/icons/icon-512.png +0 -0
  88. openmd/browser_extension/idle.html +6 -0
  89. openmd/browser_extension/manifest.json +43 -0
  90. openmd/browser_extension/options.html +42 -0
  91. openmd/browser_extension/options.js +117 -0
  92. openmd/browser_extension/popup.css +71 -0
  93. openmd/browser_extension/popup.html +17 -0
  94. openmd/browser_extension/popup.js +59 -0
  95. openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
  96. openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
  97. openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
  98. openmd/browser_host.py +503 -0
  99. openmd/browser_setup.py +106 -0
  100. openmd/cli.py +2012 -0
  101. openmd/console.py +32 -0
  102. openmd/core/__init__.py +1 -0
  103. openmd/core/capabilities.py +162 -0
  104. openmd/core/content_validation.py +323 -0
  105. openmd/core/coverage.py +163 -0
  106. openmd/core/detector.py +247 -0
  107. openmd/core/doctor.py +104 -0
  108. openmd/core/execution_policy.py +164 -0
  109. openmd/core/filename.py +64 -0
  110. openmd/core/input_validation.py +91 -0
  111. openmd/core/local_config.py +96 -0
  112. openmd/core/manifest.py +156 -0
  113. openmd/core/models.py +168 -0
  114. openmd/core/opencli_output.py +34 -0
  115. openmd/core/operation_contract.py +60 -0
  116. openmd/core/output.py +194 -0
  117. openmd/core/package_contract.py +335 -0
  118. openmd/core/package_validator.py +489 -0
  119. openmd/core/package_writer.py +666 -0
  120. openmd/core/planner.py +345 -0
  121. openmd/core/preferences.py +61 -0
  122. openmd/core/readable.py +129 -0
  123. openmd/core/receipt.py +285 -0
  124. openmd/core/renderer.py +185 -0
  125. openmd/core/runlog.py +208 -0
  126. openmd/core/secrets.py +313 -0
  127. openmd/doctor_view.py +68 -0
  128. openmd/interactive.py +453 -0
  129. openmd/ocr/__init__.py +1 -0
  130. openmd/ocr/breakers.py +84 -0
  131. openmd/ocr/config.py +54 -0
  132. openmd/ocr/documents.py +522 -0
  133. openmd/ocr/endpoint.py +72 -0
  134. openmd/ocr/layout.py +92 -0
  135. openmd/ocr/layout_filter.py +118 -0
  136. openmd/ocr/layout_worker.py +89 -0
  137. openmd/ocr/native_math.py +220 -0
  138. openmd/ocr/office.py +305 -0
  139. openmd/ocr/ollama.py +200 -0
  140. openmd/ocr/otsl.py +51 -0
  141. openmd/ocr/ovis-prompt.txt +1 -0
  142. openmd/ocr/pipeline.py +384 -0
  143. openmd/ocr/settings.json +25 -0
  144. openmd/ocr/tiling.py +631 -0
  145. openmd/post_video.py +154 -0
  146. openmd/probe.py +397 -0
  147. openmd/routing/__init__.py +1 -0
  148. openmd/routing/cards/builtin.json +620 -0
  149. openmd/routing/cards/defuddle.json +70 -0
  150. openmd/routing/cards/x-tweet-fetcher.json +71 -0
  151. openmd/routing/cards/xyz-dl.json +60 -0
  152. openmd/routing/cards/yutto.json +68 -0
  153. openmd/routing/entry_shim.py +71 -0
  154. openmd/routing/lifecycle.py +459 -0
  155. openmd/routing/loader.py +138 -0
  156. openmd/routing/providers.py +158 -0
  157. openmd/routing/readers.py +108 -0
  158. openmd/routing/routes.json +504 -0
  159. openmd/routing/runner.py +192 -0
  160. openmd/routing/view.py +40 -0
  161. openmd/runtime/__init__.py +1 -0
  162. openmd/runtime/_windows_job.py +234 -0
  163. openmd/runtime/asr.py +217 -0
  164. openmd/runtime/asr_worker.py +119 -0
  165. openmd/runtime/gpu_lock.py +81 -0
  166. openmd/runtime/gpu_quarantine.py +85 -0
  167. openmd/runtime/interpreters.py +407 -0
  168. openmd/runtime/managed_tools.py +204 -0
  169. openmd/runtime/models.py +202 -0
  170. openmd/runtime/services.py +175 -0
  171. openmd/runtime/tempdirs.py +227 -0
  172. openmd_cli-0.1.0.dist-info/METADATA +168 -0
  173. openmd_cli-0.1.0.dist-info/RECORD +177 -0
  174. openmd_cli-0.1.0.dist-info/WHEEL +5 -0
  175. openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
  176. openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
  177. openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1480 @@
1
+ """Bilibili subtitle extraction and collection discovery providers.
2
+
3
+ The providers in this module keep Bilibili work isolated from shared registries:
4
+ ``openmd.backends.registry`` discovers this file by platform naming convention.
5
+ Real network/browser work is only registered when the matching CLI runtime gate
6
+ is enabled by the caller.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from contextlib import contextmanager
12
+ import csv
13
+ import io
14
+ import json
15
+ import re
16
+ import shutil
17
+ import subprocess
18
+ import tempfile
19
+ import time
20
+ from dataclasses import dataclass, replace
21
+ from datetime import datetime, timezone
22
+ from pathlib import Path
23
+ from typing import Any, Callable
24
+ from urllib.error import HTTPError, URLError
25
+ from urllib.parse import urlencode, urlparse
26
+ from urllib.request import Request, urlopen
27
+
28
+ from openmd.core.opencli_output import load_opencli_json
29
+ from openmd.runtime.asr import get_asr_backend
30
+ from openmd.backends.base import (
31
+ BackendCapability,
32
+ BackendDescriptor,
33
+ BackendExtractRequest,
34
+ BackendHealth,
35
+ DiscoveryRequest,
36
+ DiscoveryResult,
37
+ )
38
+ from openmd.backends.executor import BackendProviderRegistry, DiscoveryProviderRegistry
39
+ from openmd.backends.registry import BackendRuntimeOptions
40
+ from openmd.backends.bilibili_wbi import (
41
+ BilibiliWbiSigner,
42
+ WbiSigningError,
43
+ bilibili_cookie_header_for_url,
44
+ bilibili_headers,
45
+ )
46
+ from openmd.core.detector import extract_first_url
47
+ from openmd.core.models import ContentItem, SourceLevel, TranscriptResult
48
+ from openmd.core.secrets import sanitize_error_summary
49
+ from openmd.runtime.tempdirs import owned_temporary_directory
50
+ from openmd.runtime.interpreters import run_interruptibly
51
+
52
+
53
+ BILIBILI_HARVEST_BACKEND = "bilibili-harvest.core"
54
+ BILIBILI_OPENCLI_SUBTITLE_BACKEND = "opencli.bilibili.subtitle"
55
+ BILIBILI_YTDLP_MEDIA_BACKEND = "yt-dlp.bilibili"
56
+ BILIBILI_ASR_FALLBACK_BACKEND = "asr.fallback"
57
+ BILIBILI_NATIVE_DISCOVERY_BACKEND = "bilibili-harvest.discovery"
58
+ BILIBILI_OPENCLI_DISCOVERY_BACKEND = "opencli.bilibili.user-videos"
59
+
60
+ # Historical backend IDs retain "bilibili-harvest", but the current Openmd
61
+ # implementation is a public Bilibili API wrapper, not the Bilibili-Harvest
62
+ # local app/browser-extension pipeline.
63
+
64
+ _BVID_RE = re.compile(r"(BV[0-9A-Za-z]+)")
65
+ _SPACE_UID_RE = re.compile(r"(?:space\.bilibili\.com/|^)(\d{3,})")
66
+ _AI_CAPTION_HINT_RE = re.compile(
67
+ "ai|auto|automatic|auto[_ -]?caption|"
68
+ "\u81ea\u52a8|\u667a\u80fd\u5b57\u5e55|"
69
+ "ai\u5b57\u5e55|ai\u751f\u6210",
70
+ re.IGNORECASE,
71
+ )
72
+ _NATIVE_CAPTION_HINT_RE = re.compile(
73
+ "native|manual|author|uploaded|"
74
+ "\u5b98\u65b9\u5b57\u5e55|\u4eba\u5de5|\u4f5c\u8005|up\u4e3b",
75
+ re.IGNORECASE,
76
+ )
77
+
78
+ CommandRunner = Callable[[list[str], int], tuple[int, str, str]]
79
+ HttpFetcher = Callable[[str, dict[str, str], int], tuple[int, str]]
80
+
81
+
82
+ @dataclass(frozen=True)
83
+ class _SubtitleSegment:
84
+ start: float
85
+ end: float | None
86
+ content: str
87
+
88
+
89
+ @dataclass(frozen=True)
90
+ class _MediaWorkspace:
91
+ path: Path
92
+ retention: str
93
+
94
+
95
+ def register_backend_providers(
96
+ registry: BackendProviderRegistry,
97
+ options: BackendRuntimeOptions,
98
+ ) -> None:
99
+ """Register Bilibili extraction providers behind explicit runtime gates."""
100
+ requested = options.backend_override
101
+ if options.enable_network_backends and requested in {None, BILIBILI_HARVEST_BACKEND}:
102
+ registry.register(BILIBILI_HARVEST_BACKEND, BilibiliNativeSubtitleProvider())
103
+
104
+ if options.enable_browser_backends and requested in {None, BILIBILI_OPENCLI_SUBTITLE_BACKEND}:
105
+ registry.register(
106
+ BILIBILI_OPENCLI_SUBTITLE_BACKEND,
107
+ OpenCliBilibiliSubtitleProvider(),
108
+ )
109
+
110
+ if requested in {None, BILIBILI_YTDLP_MEDIA_BACKEND}:
111
+ registry.register(
112
+ BILIBILI_YTDLP_MEDIA_BACKEND,
113
+ BilibiliMediaDownloadGateProvider(options),
114
+ )
115
+
116
+ if requested in {None, BILIBILI_ASR_FALLBACK_BACKEND}:
117
+ registry.register(
118
+ BILIBILI_ASR_FALLBACK_BACKEND,
119
+ BilibiliMediaAsrFallbackProvider(options),
120
+ )
121
+
122
+
123
+ def register_discovery_providers(
124
+ registry: DiscoveryProviderRegistry,
125
+ options: BackendRuntimeOptions,
126
+ ) -> None:
127
+ """Register Bilibili discovery providers without touching shared registries."""
128
+ requested = options.requested_discovery_provider
129
+ native_requested = requested in {None, BILIBILI_NATIVE_DISCOVERY_BACKEND}
130
+ opencli_requested = requested in {None, BILIBILI_OPENCLI_DISCOVERY_BACKEND}
131
+
132
+ native_registered = False
133
+ if options.enable_network_backends and native_requested:
134
+ registry.register(
135
+ BILIBILI_NATIVE_DISCOVERY_BACKEND,
136
+ BilibiliNativeUserVideosProvider(),
137
+ platforms=("bilibili",),
138
+ default_for_platforms=requested is None,
139
+ )
140
+ native_registered = True
141
+
142
+ if options.enable_browser_backends and opencli_requested:
143
+ registry.register(
144
+ BILIBILI_OPENCLI_DISCOVERY_BACKEND,
145
+ OpenCliBilibiliUserVideosProvider(),
146
+ platforms=("bilibili",),
147
+ default_for_platforms=requested is None and not native_registered,
148
+ )
149
+
150
+
151
+ class BilibiliNativeSubtitleProvider:
152
+ """Wrap public Bilibili subtitle APIs; not the local/browser Harvest pipeline."""
153
+
154
+ name = BILIBILI_HARVEST_BACKEND
155
+
156
+ def __init__(
157
+ self,
158
+ *,
159
+ fetcher: HttpFetcher | None = None,
160
+ timeout: int = 30,
161
+ wbi_signer: BilibiliWbiSigner | None = None,
162
+ max_ai_retries: int = 3,
163
+ ai_retry_interval: float = 0.0,
164
+ ) -> None:
165
+ self._fetcher = fetcher or _default_fetch
166
+ self._timeout = timeout
167
+ self._wbi_signer = wbi_signer or BilibiliWbiSigner(
168
+ fetcher=self._fetcher,
169
+ timeout=timeout,
170
+ )
171
+ self._max_ai_retries = max(1, max_ai_retries)
172
+ self._ai_retry_interval = max(0.0, ai_retry_interval)
173
+
174
+ def descriptor(self) -> BackendDescriptor:
175
+ capability = BackendCapability(
176
+ platforms=("bilibili",),
177
+ source_types=("video",),
178
+ input_types=("single",),
179
+ source_levels=("native_caption", "auto_caption", "partial_extraction"),
180
+ requires_network=True,
181
+ )
182
+ return BackendDescriptor(
183
+ name=self.name,
184
+ layer="platform_native",
185
+ capabilities=(capability,),
186
+ description=(
187
+ "Bilibili public API subtitle wrapper; not the Bilibili-Harvest "
188
+ "local/browser pipeline, no media download or ASR."
189
+ ),
190
+ provider="openmd.backends.bilibili",
191
+ )
192
+
193
+ def health(self) -> BackendHealth:
194
+ return BackendHealth(ok=True, dependencies={"network": "enabled by caller"})
195
+
196
+ def extract(self, request: BackendExtractRequest) -> TranscriptResult:
197
+ item = request.item
198
+ bvid = extract_bvid(item.url)
199
+ if bvid is None:
200
+ return _failed_result(self.name, "invalid_input", "Bilibili BVID not found.")
201
+
202
+ try:
203
+ view = _fetch_json(
204
+ self._fetcher,
205
+ "https://api.bilibili.com/x/web-interface/view",
206
+ {"bvid": bvid},
207
+ self._timeout,
208
+ )
209
+ view_data = _bilibili_api_data(view)
210
+ except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
211
+ return _failed_result(self.name, "bilibili_view_fetch_failed", exc)
212
+
213
+ aid = str(view_data.get("aid") or "").strip()
214
+ cid = str(view_data.get("cid") or "").strip()
215
+ if not cid:
216
+ pages = view_data.get("pages")
217
+ if isinstance(pages, list) and pages:
218
+ cid = str((pages[0] or {}).get("cid") or "").strip()
219
+ if not cid:
220
+ return _failed_result(
221
+ self.name,
222
+ "bilibili_cid_missing",
223
+ "Bilibili view metadata did not include a cid.",
224
+ metadata={"bvid": bvid, "aid": aid},
225
+ )
226
+
227
+ subtitles: list[dict[str, Any]] = []
228
+ endpoint_used = "none"
229
+ player_warnings: list[str] = []
230
+ for endpoint, url, params in _player_subtitle_attempts(aid=aid, cid=cid, bvid=bvid):
231
+ request_params = params
232
+ if endpoint == "wbi_v2":
233
+ try:
234
+ request_params = self._wbi_signer.sign_params(params, timeout=self._timeout)
235
+ except (HTTPError, URLError, OSError, ValueError, WbiSigningError) as exc:
236
+ player_warnings.append(
237
+ f"{endpoint}_signing: {sanitize_error_summary(exc)}; unsigned request attempted"
238
+ )
239
+
240
+ for retry_index in range(self._max_ai_retries):
241
+ try:
242
+ player = _fetch_json(self._fetcher, url, request_params, self._timeout)
243
+ player_data = _bilibili_api_data(player)
244
+ except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
245
+ player_warnings.append(f"{endpoint}: {sanitize_error_summary(exc)}")
246
+ break
247
+
248
+ subtitles = _subtitle_entries_from_player(player_data)
249
+ if subtitles:
250
+ endpoint_used = endpoint
251
+ break
252
+ if retry_index + 1 < self._max_ai_retries and self._ai_retry_interval:
253
+ time.sleep(self._ai_retry_interval)
254
+ if subtitles:
255
+ break
256
+ player_warnings.append(f"{endpoint}: no subtitle tracks")
257
+
258
+ if not subtitles:
259
+ return _failed_result(
260
+ self.name,
261
+ "bilibili_subtitle_missing",
262
+ "Bilibili metadata did not expose native or AI subtitles.",
263
+ title=_safe_str(view_data.get("title")),
264
+ author=_owner_name(view_data),
265
+ published_at=_format_unix_timestamp(view_data.get("pubdate")),
266
+ metadata={"bvid": bvid, "aid": aid, "cid": cid, "player_warnings": player_warnings},
267
+ notes=[
268
+ (
269
+ "Openmd's current bilibili-harvest.core backend uses public "
270
+ "Bilibili APIs only; it does not invoke the Bilibili-Harvest "
271
+ "local app/browser-extension pipeline."
272
+ ),
273
+ *_warning_notes(player_warnings),
274
+ ],
275
+ )
276
+
277
+ subtitle = _choose_subtitle(subtitles)
278
+ subtitle_track_type = _subtitle_track_type(subtitle)
279
+ subtitle_url = _normalize_subtitle_url(_safe_str(subtitle.get("subtitle_url")))
280
+ ai_rescue_note: str | None = None
281
+ if not subtitle_url and subtitle_track_type == "ai":
282
+ try:
283
+ subtitle_url = _resolve_ai_subtitle_url(
284
+ self._fetcher,
285
+ aid=aid,
286
+ cid=cid,
287
+ timeout=self._timeout,
288
+ )
289
+ except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
290
+ player_warnings.append(f"ai_subtitle_search_stat: {sanitize_error_summary(exc)}")
291
+ if subtitle_url:
292
+ ai_rescue_note = "AI subtitle URL was recovered via x/player/v2/ai/subtitle/search/stat."
293
+
294
+ if not subtitle_url:
295
+ return _failed_result(
296
+ self.name,
297
+ "bilibili_subtitle_url_missing",
298
+ "Selected Bilibili subtitle did not include a subtitle_url.",
299
+ metadata={"bvid": bvid, "aid": aid, "cid": cid, "track_type": subtitle_track_type},
300
+ notes=[
301
+ "AI subtitle URL rescue via x/player/v2/ai/subtitle/search/stat was attempted when applicable.",
302
+ *_warning_notes(player_warnings),
303
+ ],
304
+ )
305
+
306
+ try:
307
+ subtitle_payload = _fetch_json(self._fetcher, subtitle_url, {}, self._timeout)
308
+ segments = _segments_from_subtitle_payload(subtitle_payload)
309
+ except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
310
+ return _failed_result(self.name, "bilibili_subtitle_fetch_failed", exc)
311
+
312
+ if not segments:
313
+ return _failed_result(
314
+ self.name,
315
+ "bilibili_subtitle_empty",
316
+ "Bilibili subtitle payload did not contain usable subtitle lines.",
317
+ metadata={"bvid": bvid, "aid": aid, "cid": cid},
318
+ )
319
+
320
+ source_level = _source_level_from_subtitle_meta(subtitle)
321
+ source_notes = [
322
+ "Bilibili public API subtitle discovery tried signed x/player/wbi/v2 before x/player/v2 when aid was available.",
323
+ f"subtitle_endpoint_used={endpoint_used}",
324
+ f"async_ai_subtitle_retries={self._max_ai_retries}",
325
+ "Bilibili danmaku/XML subtitle tracks were filtered before track selection.",
326
+ _source_level_note(source_level),
327
+ "This backend does not invoke the Bilibili-Harvest local app/browser-extension pipeline.",
328
+ "No media, raw subtitle JSON, raw response, cookies, headers, or ASR artifacts were persisted.",
329
+ ]
330
+ if ai_rescue_note:
331
+ source_notes.append(ai_rescue_note)
332
+ source_notes.extend(_warning_notes(player_warnings))
333
+ return TranscriptResult(
334
+ text=_format_segments(segments),
335
+ status="verified",
336
+ source_level=source_level,
337
+ backend=self.name,
338
+ source_notes=source_notes,
339
+ title=_safe_str(view_data.get("title")),
340
+ author=_owner_name(view_data),
341
+ published_at=_format_unix_timestamp(view_data.get("pubdate")),
342
+ metadata={
343
+ "bvid": bvid,
344
+ "aid": aid,
345
+ "cid": cid,
346
+ "caption_language": _safe_str(subtitle.get("lan") or subtitle.get("lan_doc")),
347
+ "caption_track_type": subtitle_track_type,
348
+ "subtitle_endpoint_used": endpoint_used,
349
+ "segment_count": len(segments),
350
+ },
351
+ )
352
+
353
+
354
+ class OpenCliBilibiliSubtitleProvider:
355
+ """Controlled OpenCLI fallback for Bilibili subtitle extraction."""
356
+
357
+ name = BILIBILI_OPENCLI_SUBTITLE_BACKEND
358
+
359
+ def __init__(
360
+ self,
361
+ *,
362
+ runner: CommandRunner | None = None,
363
+ timeout: int = 120,
364
+ ) -> None:
365
+ self._runner = runner or _run_command
366
+ self._timeout = timeout
367
+
368
+ def descriptor(self) -> BackendDescriptor:
369
+ capability = BackendCapability(
370
+ platforms=("bilibili",),
371
+ source_types=("video",),
372
+ input_types=("single",),
373
+ source_levels=("native_caption", "auto_caption", "partial_extraction"),
374
+ requires_network=True,
375
+ )
376
+ return BackendDescriptor(
377
+ name=self.name,
378
+ layer="browser_cli_fallback",
379
+ capabilities=(capability,),
380
+ description="OpenCLI bilibili subtitle fallback; registered only with browser backend opt-in.",
381
+ provider="opencli",
382
+ is_fallback=True,
383
+ )
384
+
385
+ def health(self) -> BackendHealth:
386
+ ok = shutil.which("opencli") is not None
387
+ return BackendHealth(
388
+ ok=ok,
389
+ dependencies={"opencli": "ok" if ok else "missing"},
390
+ warnings=() if ok else ("opencli executable was not found on PATH",),
391
+ )
392
+
393
+ def extract(self, request: BackendExtractRequest) -> TranscriptResult:
394
+ item = request.item
395
+ bvid = extract_bvid(item.url)
396
+ if bvid is None:
397
+ return _failed_result(self.name, "invalid_input", "Bilibili BVID not found.")
398
+
399
+ try:
400
+ exit_code, stdout, stderr = self._runner(
401
+ ["opencli", "bilibili", "subtitle", bvid, "-f", "json"],
402
+ self._timeout,
403
+ )
404
+ except FileNotFoundError as exc:
405
+ return _failed_result(self.name, "backend_not_found", exc)
406
+ except subprocess.TimeoutExpired as exc:
407
+ return _failed_result(self.name, "backend_timeout", exc)
408
+ except OSError as exc:
409
+ return _failed_result(self.name, "backend_os_error", exc)
410
+
411
+ if exit_code != 0:
412
+ return _failed_result(
413
+ self.name,
414
+ f"backend_exit_{exit_code}",
415
+ stderr or "OpenCLI bilibili subtitle failed.",
416
+ )
417
+
418
+ segments = parse_bilibili_subtitle_segments(stdout)
419
+ if not segments:
420
+ return _failed_result(
421
+ self.name,
422
+ "empty_result",
423
+ "OpenCLI bilibili subtitle returned no parseable subtitle rows.",
424
+ )
425
+
426
+ source_level = _source_level_from_text(stdout, default="auto_caption")
427
+ return TranscriptResult(
428
+ text=_format_segments(segments),
429
+ status="verified",
430
+ source_level=source_level,
431
+ backend=self.name,
432
+ source_notes=[
433
+ "OpenCLI bilibili subtitle output was parsed and normalized in memory.",
434
+ _source_level_note(source_level),
435
+ "No raw stdout/stderr, raw subtitle rows, media, cookies, or headers were persisted.",
436
+ ],
437
+ metadata={"bvid": bvid, "segment_count": len(segments)},
438
+ )
439
+
440
+
441
+ class BilibiliMediaDownloadGateProvider:
442
+ """Document the media-download stage without performing hidden extraction."""
443
+
444
+ name = BILIBILI_YTDLP_MEDIA_BACKEND
445
+
446
+ def __init__(self, options: BackendRuntimeOptions) -> None:
447
+ self._options = options
448
+
449
+ def descriptor(self) -> BackendDescriptor:
450
+ capability = BackendCapability(
451
+ platforms=("bilibili",),
452
+ source_types=("video",),
453
+ input_types=("single",),
454
+ source_levels=("partial_extraction",),
455
+ requires_network=True,
456
+ optional_config=("enable_media_download_backends", "enable_asr_backends"),
457
+ )
458
+ return BackendDescriptor(
459
+ name=self.name,
460
+ layer="media_download_gate",
461
+ capabilities=(capability,),
462
+ description=(
463
+ "Bilibili media download gate. The text-producing fallback is "
464
+ "asr.fallback and requires explicit media-download plus ASR opt-in."
465
+ ),
466
+ provider="openmd.backends.bilibili",
467
+ is_fallback=True,
468
+ )
469
+
470
+ def health(self) -> BackendHealth:
471
+ if not self._options.enable_media_download_backends:
472
+ return BackendHealth(ok=True, warnings=("media download fallback is not enabled",))
473
+ ok = shutil.which("yt-dlp") is not None
474
+ return BackendHealth(
475
+ ok=ok,
476
+ dependencies={"yt-dlp": "ok" if ok else "missing"},
477
+ error_code=None if ok else "media_download_backend_missing",
478
+ error_summary=None if ok else "yt-dlp executable was not found on PATH.",
479
+ )
480
+
481
+ def extract(self, request: BackendExtractRequest) -> TranscriptResult:
482
+ item = request.item
483
+ bvid = extract_bvid(item.url)
484
+ return _failed_result(
485
+ self.name,
486
+ "bilibili_media_download_stage_requires_asr",
487
+ "Media download is an intermediate stage; rerun with --enable-media-download --enable-asr to attempt ASR fallback.",
488
+ title=item.title,
489
+ author=item.author,
490
+ published_at=item.published_at,
491
+ metadata={
492
+ "bvid": bvid,
493
+ "failure_class": "user_resolvable",
494
+ "media_download_enabled": self._options.enable_media_download_backends,
495
+ "asr_enabled": self._options.enable_asr_backends,
496
+ },
497
+ notes=[
498
+ "No media download or ASR action was attempted by the standalone media-download stage.",
499
+ "Use the final asr.fallback provider for approved temporary media download plus local ASR.",
500
+ ],
501
+ )
502
+
503
+
504
+ class BilibiliMediaAsrFallbackProvider:
505
+ """Explicit temporary-media + local-ASR fallback for no-subtitle Bilibili items."""
506
+
507
+ name = BILIBILI_ASR_FALLBACK_BACKEND
508
+
509
+ def __init__(
510
+ self,
511
+ options: BackendRuntimeOptions,
512
+ *,
513
+ runner: CommandRunner | None = None,
514
+ asr_backend: object | None = None,
515
+ timeout: int = 1800,
516
+ ) -> None:
517
+ self._options = options
518
+ self._runner = runner or _run_command
519
+ self._asr_backend = asr_backend or get_asr_backend("asr.auto")
520
+ self._timeout = timeout
521
+
522
+ def descriptor(self) -> BackendDescriptor:
523
+ capability = BackendCapability(
524
+ platforms=("bilibili",),
525
+ source_types=("video",),
526
+ input_types=("single",),
527
+ source_levels=("media_asr", "partial_extraction"),
528
+ requires_network=True,
529
+ optional_config=("enable_media_download_backends", "enable_asr_backends"),
530
+ )
531
+ return BackendDescriptor(
532
+ name=self.name,
533
+ layer="media_asr_fallback",
534
+ capabilities=(capability,),
535
+ description=(
536
+ "Bilibili no-subtitle fallback via temporary yt-dlp media download "
537
+ "and local ASR; registered safely but runs only with explicit gates."
538
+ ),
539
+ provider="openmd.backends.bilibili",
540
+ is_fallback=True,
541
+ )
542
+
543
+ def health(self) -> BackendHealth:
544
+ if not (self._options.enable_media_download_backends and self._options.enable_asr_backends):
545
+ return BackendHealth(ok=True, warnings=("Bilibili media ASR fallback is not enabled",))
546
+ yt_dlp_ok = shutil.which("yt-dlp") is not None
547
+ is_available = getattr(self._asr_backend, "is_available", None)
548
+ asr_ok = bool(is_available()) if callable(is_available) else True
549
+ dependencies = {
550
+ "yt-dlp": "ok" if yt_dlp_ok else "missing",
551
+ getattr(self._asr_backend, "name", "asr_backend"): "ok" if asr_ok else "missing",
552
+ }
553
+ return BackendHealth(
554
+ ok=yt_dlp_ok and asr_ok,
555
+ dependencies=dependencies,
556
+ error_code=None if yt_dlp_ok and asr_ok else "bilibili_media_asr_dependency_missing",
557
+ error_summary=None if yt_dlp_ok and asr_ok else "yt-dlp or the configured ASR backend is missing.",
558
+ )
559
+
560
+ def extract(self, request: BackendExtractRequest) -> TranscriptResult:
561
+ item = request.item
562
+ bvid = extract_bvid(item.url)
563
+ if bvid is None:
564
+ return _failed_result(self.name, "invalid_input", "Bilibili BVID not found.")
565
+
566
+ missing_flags = _missing_media_asr_flags(self._options)
567
+ if missing_flags:
568
+ return _failed_result(
569
+ self.name,
570
+ "bilibili_media_asr_not_enabled",
571
+ "Bilibili no-subtitle ASR fallback requires explicit --enable-media-download and --enable-asr.",
572
+ title=item.title,
573
+ author=item.author,
574
+ published_at=item.published_at,
575
+ metadata={
576
+ "bvid": bvid,
577
+ "failure_class": "user_resolvable",
578
+ "missing_flags": missing_flags,
579
+ "media_retention": "none",
580
+ },
581
+ notes=[
582
+ "No media download or ASR action was attempted because the explicit fallback gates are closed.",
583
+ "Raw media is not retained by default; add --keep-media only for approved debugging.",
584
+ ],
585
+ )
586
+
587
+ with _media_workspace(self._options) as workspace:
588
+ media_path = self._download_media(item, bvid, workspace.path)
589
+ if isinstance(media_path, TranscriptResult):
590
+ return media_path
591
+
592
+ transcribe = getattr(self._asr_backend, "transcribe", None)
593
+ if not callable(transcribe):
594
+ return _failed_result(
595
+ self.name,
596
+ "asr_backend_missing",
597
+ "Configured ASR backend does not expose transcribe(path).",
598
+ metadata={"bvid": bvid, "failure_class": "user_resolvable"},
599
+ )
600
+ asr_result = transcribe(media_path)
601
+ if not isinstance(asr_result, TranscriptResult) or not asr_result.has_usable_text:
602
+ return _failed_result(
603
+ self.name,
604
+ getattr(asr_result, "error_code", None) or "bilibili_media_asr_failed",
605
+ getattr(asr_result, "error_summary", None) or "Bilibili media ASR produced no usable text.",
606
+ title=item.title or getattr(asr_result, "title", None),
607
+ author=item.author,
608
+ published_at=item.published_at,
609
+ metadata={
610
+ "bvid": bvid,
611
+ "failure_class": "user_resolvable",
612
+ "asr_backend": getattr(self._asr_backend, "name", "unknown"),
613
+ "media_retention": workspace.retention,
614
+ },
615
+ notes=[
616
+ "Temporary media download was attempted under explicit operator gates.",
617
+ *_source_notes_from(asr_result),
618
+ ],
619
+ )
620
+
621
+ metadata = dict(asr_result.metadata)
622
+ if workspace.retention != "kept":
623
+ metadata.pop("source_ref", None)
624
+ metadata.update(
625
+ {
626
+ "profile": "media",
627
+ "bvid": bvid,
628
+ "media_download_backend": BILIBILI_YTDLP_MEDIA_BACKEND,
629
+ "asr_backend": getattr(self._asr_backend, "name", asr_result.backend),
630
+ "media_retention": workspace.retention,
631
+ "raw_media_retained": workspace.retention == "kept",
632
+ }
633
+ )
634
+ return replace(
635
+ asr_result,
636
+ source_level="media_asr",
637
+ backend=getattr(self._asr_backend, "name", asr_result.backend),
638
+ title=item.title or asr_result.title,
639
+ author=item.author or asr_result.author,
640
+ published_at=item.published_at or asr_result.published_at,
641
+ source_notes=[
642
+ "Bilibili no-subtitle fallback used explicit temporary media download plus local ASR.",
643
+ "Raw media was retained only when --keep-media was set.",
644
+ *asr_result.source_notes,
645
+ ],
646
+ metadata=metadata,
647
+ )
648
+
649
+ def _download_media(
650
+ self,
651
+ item: ContentItem,
652
+ bvid: str,
653
+ media_dir: Path,
654
+ ) -> Path | TranscriptResult:
655
+ output_template = media_dir / f"{bvid}.%(ext)s"
656
+ command = [
657
+ "yt-dlp",
658
+ "--no-playlist",
659
+ "--extract-audio",
660
+ "--audio-format",
661
+ "wav",
662
+ "--output",
663
+ str(output_template),
664
+ item.url,
665
+ ]
666
+ try:
667
+ exit_code, stdout, stderr = self._runner(command, self._timeout)
668
+ except FileNotFoundError as exc:
669
+ return _failed_result(
670
+ self.name,
671
+ "media_download_backend_missing",
672
+ exc,
673
+ metadata={"bvid": bvid, "failure_class": "user_resolvable"},
674
+ )
675
+ except subprocess.TimeoutExpired as exc:
676
+ return _failed_result(
677
+ self.name,
678
+ "media_download_timeout",
679
+ exc,
680
+ metadata={"bvid": bvid, "failure_class": "runtime_transient"},
681
+ )
682
+ except OSError as exc:
683
+ return _failed_result(
684
+ self.name,
685
+ "media_download_os_error",
686
+ exc,
687
+ metadata={"bvid": bvid, "failure_class": "runtime_transient"},
688
+ )
689
+
690
+ if exit_code != 0:
691
+ return _failed_result(
692
+ self.name,
693
+ f"media_download_exit_{exit_code}",
694
+ stderr or stdout or "yt-dlp media download failed.",
695
+ metadata={"bvid": bvid, "failure_class": "runtime_transient"},
696
+ )
697
+
698
+ media_path = _find_downloaded_media_file(media_dir, bvid)
699
+ if media_path is None:
700
+ return _failed_result(
701
+ self.name,
702
+ "media_download_empty",
703
+ "yt-dlp completed but no media file was found in the temporary workspace.",
704
+ metadata={"bvid": bvid, "failure_class": "runtime_transient"},
705
+ )
706
+ return media_path
707
+
708
+
709
+ class BilibiliNativeUserVideosProvider:
710
+ """Discover public Bilibili user videos without extracting item subtitles."""
711
+
712
+ name = BILIBILI_NATIVE_DISCOVERY_BACKEND
713
+
714
+ def __init__(
715
+ self,
716
+ *,
717
+ fetcher: HttpFetcher | None = None,
718
+ timeout: int = 30,
719
+ wbi_signer: BilibiliWbiSigner | None = None,
720
+ ) -> None:
721
+ self._fetcher = fetcher or _default_fetch
722
+ self._timeout = timeout
723
+ self._wbi_signer = wbi_signer or BilibiliWbiSigner(
724
+ fetcher=self._fetcher,
725
+ timeout=timeout,
726
+ )
727
+
728
+ def discover(self, request: DiscoveryRequest) -> DiscoveryResult:
729
+ uid = extract_bilibili_uid(request.input_url)
730
+ if uid is None:
731
+ return DiscoveryResult.failure(
732
+ provider=self.name,
733
+ error_code="invalid_input",
734
+ error_summary="Bilibili space UID not found for collection discovery.",
735
+ )
736
+
737
+ limit = _bounded_limit(request.limit)
738
+ params = {"mid": uid, "pn": "1", "ps": str(limit), "order": "pubdate"}
739
+ warnings: list[str] = []
740
+ endpoint_used = "x/space/arc/search"
741
+ videos: list[dict[str, Any]] = []
742
+
743
+ try:
744
+ signed_params = self._wbi_signer.sign_params(params, timeout=self._timeout)
745
+ signed_payload = _fetch_json(
746
+ self._fetcher,
747
+ "https://api.bilibili.com/x/space/wbi/arc/search",
748
+ signed_params,
749
+ self._timeout,
750
+ )
751
+ signed_data = _bilibili_api_data(signed_payload)
752
+ videos = _video_entries_from_space_payload(signed_data)
753
+ endpoint_used = "x/space/wbi/arc/search"
754
+ except (HTTPError, URLError, OSError, ValueError, WbiSigningError, json.JSONDecodeError) as exc:
755
+ warnings.append(f"x/space/wbi/arc/search: {sanitize_error_summary(exc)}")
756
+
757
+ if not videos:
758
+ if endpoint_used == "x/space/wbi/arc/search":
759
+ warnings.append("x/space/wbi/arc/search: no usable video rows")
760
+ try:
761
+ payload = _fetch_json(
762
+ self._fetcher,
763
+ "https://api.bilibili.com/x/space/arc/search",
764
+ params,
765
+ self._timeout,
766
+ )
767
+ data = _bilibili_api_data(payload)
768
+ videos = _video_entries_from_space_payload(data)
769
+ endpoint_used = "x/space/arc/search"
770
+ except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
771
+ warnings.append(f"x/space/arc/search: {sanitize_error_summary(exc)}")
772
+
773
+ if not videos:
774
+ return DiscoveryResult.failure(
775
+ provider=self.name,
776
+ error_code="bilibili_discovery_fetch_failed",
777
+ error_summary=sanitize_error_summary("; ".join(warnings) or "Bilibili discovery returned no usable rows."),
778
+ metadata={"uid": uid, "limit": limit, "endpoint_used": endpoint_used},
779
+ )
780
+
781
+ items = [_content_item_from_video_entry(entry, request, uid) for entry in videos]
782
+ items = [item for item in items if item.url][:limit]
783
+ if not items:
784
+ return DiscoveryResult.failure(
785
+ provider=self.name,
786
+ error_code="empty_result",
787
+ error_summary="Bilibili user videos discovery returned no usable video rows.",
788
+ metadata={"uid": uid, "limit": limit},
789
+ )
790
+
791
+ return DiscoveryResult.success(
792
+ provider=self.name,
793
+ items=items,
794
+ source_notes=[
795
+ "Bilibili public user videos API discovery only; item extraction was not attempted.",
796
+ f"discovery_endpoint_used={endpoint_used}",
797
+ "Discovery metadata was normalized without persisting raw API responses.",
798
+ (
799
+ "Bilibili UP discovery tries signed x/space/wbi/arc/search before "
800
+ "falling back to public x/space/arc/search; callers can use the "
801
+ "OpenCLI fallback when native public discovery is unavailable."
802
+ ),
803
+ *_warning_notes(warnings),
804
+ ],
805
+ metadata={
806
+ "uid": uid,
807
+ "limit": limit,
808
+ "discovery_endpoint_used": endpoint_used,
809
+ "normalized_url": f"https://space.bilibili.com/{uid}",
810
+ "stability_note": "Tries signed x/space/wbi/arc/search first, then public x/space/arc/search fallback.",
811
+ },
812
+ )
813
+
814
+
815
+ class OpenCliBilibiliUserVideosProvider:
816
+ """Controlled OpenCLI fallback for Bilibili UP video discovery."""
817
+
818
+ name = BILIBILI_OPENCLI_DISCOVERY_BACKEND
819
+
820
+ def __init__(
821
+ self,
822
+ *,
823
+ runner: CommandRunner | None = None,
824
+ timeout: int = 120,
825
+ ) -> None:
826
+ self._runner = runner or _run_command
827
+ self._timeout = timeout
828
+
829
+ def discover(self, request: DiscoveryRequest) -> DiscoveryResult:
830
+ uid = extract_bilibili_uid(request.input_url)
831
+ if uid is None:
832
+ return DiscoveryResult.failure(
833
+ provider=self.name,
834
+ error_code="invalid_input",
835
+ error_summary="Bilibili space UID not found for OpenCLI discovery.",
836
+ )
837
+
838
+ limit = _bounded_limit(request.limit)
839
+ try:
840
+ exit_code, stdout, stderr = self._runner(
841
+ ["opencli", "bilibili", "user-videos", uid, "--limit", str(limit)],
842
+ self._timeout,
843
+ )
844
+ except FileNotFoundError as exc:
845
+ return DiscoveryResult.failure(
846
+ provider=self.name,
847
+ error_code="backend_not_found",
848
+ error_summary=sanitize_error_summary(exc),
849
+ )
850
+ except subprocess.TimeoutExpired as exc:
851
+ return DiscoveryResult.failure(
852
+ provider=self.name,
853
+ error_code="backend_timeout",
854
+ error_summary=sanitize_error_summary(exc),
855
+ )
856
+ except OSError as exc:
857
+ return DiscoveryResult.failure(
858
+ provider=self.name,
859
+ error_code="backend_os_error",
860
+ error_summary=sanitize_error_summary(exc),
861
+ )
862
+
863
+ if exit_code != 0:
864
+ return DiscoveryResult.failure(
865
+ provider=self.name,
866
+ error_code=f"backend_exit_{exit_code}",
867
+ error_summary=sanitize_error_summary(stderr or "OpenCLI bilibili user-videos failed."),
868
+ )
869
+
870
+ rows = parse_opencli_bilibili_user_videos(stdout)
871
+ items = [_content_item_from_opencli_row(row, request, uid) for row in rows]
872
+ items = [item for item in items if item.url][:limit]
873
+ if not items:
874
+ return DiscoveryResult.failure(
875
+ provider=self.name,
876
+ error_code="empty_result",
877
+ error_summary="OpenCLI bilibili user-videos returned no parseable video rows.",
878
+ metadata={"uid": uid, "limit": limit},
879
+ )
880
+
881
+ return DiscoveryResult.success(
882
+ provider=self.name,
883
+ items=items,
884
+ source_notes=[
885
+ "OpenCLI bilibili user-videos discovery only; item extraction was not attempted.",
886
+ "Raw stdout/stderr was parsed in memory and not persisted.",
887
+ ],
888
+ metadata={"uid": uid, "limit": limit, "normalized_url": f"https://space.bilibili.com/{uid}"},
889
+ )
890
+
891
+
892
+ def extract_bvid(value: str) -> str | None:
893
+ match = _BVID_RE.search(extract_first_url(value)) or _BVID_RE.search(value)
894
+ return match.group(1) if match else None
895
+
896
+
897
+ def extract_bilibili_uid(value: str) -> str | None:
898
+ url = extract_first_url(value)
899
+ parsed = urlparse(url if "://" in url else f"https://{url}")
900
+ host_path = f"{parsed.netloc}{parsed.path}"
901
+ match = _SPACE_UID_RE.search(host_path)
902
+ if match:
903
+ return match.group(1)
904
+ if value.strip().isdigit():
905
+ return value.strip()
906
+ return None
907
+
908
+
909
+ def parse_bilibili_subtitle_segments(text: str) -> list[_SubtitleSegment]:
910
+ rows = _parse_json_subtitle_rows(text) or _parse_tabular_rows(text)
911
+ return _segments_from_rows(rows)
912
+
913
+
914
+ def parse_opencli_bilibili_user_videos(text: str) -> list[dict[str, str]]:
915
+ rows = _parse_json_subtitle_rows(text) or _parse_tabular_rows(text)
916
+ return [row for row in rows if _row_lookup(row, "url") or extract_bvid(str(row))]
917
+
918
+
919
+ def _missing_media_asr_flags(options: BackendRuntimeOptions) -> list[str]:
920
+ missing: list[str] = []
921
+ if not options.enable_media_download_backends:
922
+ missing.append("--enable-media-download")
923
+ if not options.enable_asr_backends:
924
+ missing.append("--enable-asr")
925
+ return missing
926
+
927
+
928
+ @contextmanager
929
+ def _media_workspace(options: BackendRuntimeOptions):
930
+ if options.keep_media_artifacts:
931
+ media_dir = Path(options.media_artifact_dir or "out/raw-media")
932
+ media_dir.mkdir(parents=True, exist_ok=True)
933
+ yield _MediaWorkspace(path=media_dir, retention="kept")
934
+ return
935
+
936
+ with owned_temporary_directory(prefix="openmd-bilibili-asr-") as tmp_dir:
937
+ yield _MediaWorkspace(path=Path(tmp_dir), retention="deleted")
938
+
939
+
940
+ def _find_downloaded_media_file(media_dir: Path, bvid: str) -> Path | None:
941
+ suffix_rank = {
942
+ ".wav": 0,
943
+ ".m4a": 1,
944
+ ".mp3": 2,
945
+ ".opus": 3,
946
+ ".aac": 4,
947
+ ".flac": 5,
948
+ ".mp4": 6,
949
+ }
950
+ candidates = [path for path in media_dir.iterdir() if path.is_file()]
951
+ if not candidates:
952
+ return None
953
+ matched = [path for path in candidates if bvid in path.name] or candidates
954
+ return sorted(
955
+ matched,
956
+ key=lambda path: (suffix_rank.get(path.suffix.lower(), 100), path.name),
957
+ )[0]
958
+
959
+
960
+ def _source_notes_from(value: object) -> list[str]:
961
+ if isinstance(value, TranscriptResult):
962
+ return list(value.source_notes)
963
+ return []
964
+
965
+
966
+ def _default_fetch(url: str, params: dict[str, str], timeout: int) -> tuple[int, str]:
967
+ query = urlencode(params)
968
+ full_url = f"{url}?{query}" if query else url
969
+ request = Request(
970
+ full_url,
971
+ headers=bilibili_headers(cookie_header=bilibili_cookie_header_for_url(url)),
972
+ )
973
+ with urlopen(request, timeout=timeout) as response:
974
+ raw = response.read().decode("utf-8", errors="replace")
975
+ return int(getattr(response, "status", 200)), raw
976
+
977
+
978
+ def _run_command(command: list[str], timeout: int) -> tuple[int, str, str]:
979
+ resolved = shutil.which(command[0]) or command[0]
980
+ completed = run_interruptibly(
981
+ [resolved, *command[1:]],
982
+ capture_output=True,
983
+ text=True,
984
+ timeout=timeout,
985
+ check=False,
986
+ encoding="utf-8",
987
+ errors="replace",
988
+ )
989
+ return completed.returncode, completed.stdout or "", completed.stderr or ""
990
+
991
+
992
+ def _fetch_json(
993
+ fetcher: HttpFetcher,
994
+ url: str,
995
+ params: dict[str, str],
996
+ timeout: int,
997
+ ) -> Any:
998
+ status, body = fetcher(url, params, timeout)
999
+ if status < 200 or status >= 300:
1000
+ raise ValueError(f"Bilibili request failed with HTTP {status}.")
1001
+ return json.loads(body)
1002
+
1003
+
1004
+ def _bilibili_api_data(payload: Any) -> dict[str, Any]:
1005
+ if not isinstance(payload, dict):
1006
+ raise ValueError("Bilibili API payload was not a JSON object.")
1007
+ code = payload.get("code", 0)
1008
+ if code not in {0, "0", None}:
1009
+ message = _safe_str(payload.get("message") or payload.get("msg") or code)
1010
+ raise ValueError(f"Bilibili API returned code {code}: {message}")
1011
+ data = payload.get("data")
1012
+ if not isinstance(data, dict):
1013
+ raise ValueError("Bilibili API payload did not include an object data field.")
1014
+ return data
1015
+
1016
+
1017
+ def _subtitle_entries_from_player(player_data: dict[str, Any]) -> list[dict[str, Any]]:
1018
+ subtitle = player_data.get("subtitle")
1019
+ if not isinstance(subtitle, dict):
1020
+ return []
1021
+ entries = subtitle.get("subtitles")
1022
+ if not isinstance(entries, list):
1023
+ return []
1024
+ return [
1025
+ entry
1026
+ for entry in entries
1027
+ if isinstance(entry, dict) and not _is_danmaku_subtitle_entry(entry)
1028
+ ]
1029
+
1030
+
1031
+ def _choose_subtitle(entries: list[dict[str, Any]]) -> dict[str, Any]:
1032
+ dedup: dict[str, dict[str, Any]] = {}
1033
+ for entry in entries:
1034
+ lang = _normalize_lang(_safe_str(entry.get("lan") or entry.get("lan_doc")))
1035
+ key = lang.lower()
1036
+ existing = dedup.get(key)
1037
+ if existing is None or (
1038
+ _subtitle_track_type(existing) == "ai" and _subtitle_track_type(entry) == "uploader"
1039
+ ):
1040
+ dedup[key] = entry
1041
+
1042
+ candidates = list(dedup.values()) or entries
1043
+ for preferred in ("zh-CN", "zh-Hans", "zh", "ai-zh"):
1044
+ for entry in candidates:
1045
+ if _normalize_lang(_safe_str(entry.get("lan") or entry.get("lan_doc"))).lower() == preferred.lower():
1046
+ return entry
1047
+
1048
+ zh_like = [
1049
+ entry
1050
+ for entry in candidates
1051
+ if _normalize_lang(_safe_str(entry.get("lan") or entry.get("lan_doc"))).lower().startswith(("zh", "ai-zh"))
1052
+ ]
1053
+ if zh_like:
1054
+ return sorted(zh_like, key=_subtitle_sort_key)[0]
1055
+ return sorted(candidates, key=_subtitle_sort_key)[0]
1056
+
1057
+
1058
+ def _normalize_subtitle_url(value: str | None) -> str | None:
1059
+ if not value:
1060
+ return None
1061
+ if value.startswith("//"):
1062
+ return f"https:{value}"
1063
+ if value.startswith("http://") or value.startswith("https://"):
1064
+ return value
1065
+ if value.startswith("/"):
1066
+ return f"https://api.bilibili.com{value}"
1067
+ return None
1068
+
1069
+
1070
+ def _player_subtitle_attempts(
1071
+ *,
1072
+ aid: str,
1073
+ cid: str,
1074
+ bvid: str,
1075
+ ) -> list[tuple[str, str, dict[str, str]]]:
1076
+ attempts: list[tuple[str, str, dict[str, str]]] = []
1077
+ if aid:
1078
+ attempts.append(
1079
+ (
1080
+ "wbi_v2",
1081
+ "https://api.bilibili.com/x/player/wbi/v2",
1082
+ {"aid": aid, "cid": cid},
1083
+ )
1084
+ )
1085
+ attempts.append(
1086
+ (
1087
+ "v2",
1088
+ "https://api.bilibili.com/x/player/v2",
1089
+ {"cid": cid, "bvid": bvid},
1090
+ )
1091
+ )
1092
+ return attempts
1093
+
1094
+
1095
+ def _normalize_lang(raw_lang: str) -> str:
1096
+ raw = raw_lang.strip()
1097
+ if not raw:
1098
+ return "unknown"
1099
+ lower = raw.lower()
1100
+ mapping = {
1101
+ "zh-cn": "zh-CN",
1102
+ "zh-hans": "zh-Hans",
1103
+ "zh-hant": "zh-Hant",
1104
+ "zh-tw": "zh-Hant",
1105
+ "zh-hk": "zh-Hant",
1106
+ "zh": "zh",
1107
+ "ai-zh": "ai-zh",
1108
+ }
1109
+ if lower in mapping:
1110
+ return mapping[lower]
1111
+ if lower.startswith("ai-zh"):
1112
+ return "ai-zh"
1113
+ if lower.startswith("zh-cn"):
1114
+ return "zh-CN"
1115
+ if lower.startswith("zh-hans"):
1116
+ return "zh-Hans"
1117
+ if lower.startswith("zh"):
1118
+ return "zh"
1119
+ return raw
1120
+
1121
+
1122
+ def _is_danmaku_subtitle_entry(entry: dict[str, Any]) -> bool:
1123
+ raw_lang = _safe_str(entry.get("lan") or entry.get("lan_doc")).lower()
1124
+ if raw_lang == "danmaku":
1125
+ return True
1126
+ subtitle_url = _normalize_subtitle_url(_safe_str(entry.get("subtitle_url")))
1127
+ if not subtitle_url:
1128
+ return False
1129
+ return urlparse(subtitle_url).path.lower().endswith(".xml")
1130
+
1131
+
1132
+ def _subtitle_track_type(entry: dict[str, Any]) -> str:
1133
+ raw_lang = _safe_str(entry.get("lan"))
1134
+ values = " ".join(_safe_str(value) for value in entry.values())
1135
+ ai_status = entry.get("ai_status")
1136
+ if (
1137
+ raw_lang.lower().startswith("ai-")
1138
+ or bool(entry.get("ai_type"))
1139
+ or ai_status not in {None, 0, "0", False}
1140
+ or _AI_CAPTION_HINT_RE.search(values)
1141
+ ):
1142
+ return "ai"
1143
+ return "uploader"
1144
+
1145
+
1146
+ def _subtitle_sort_key(entry: dict[str, Any]) -> tuple[int, str]:
1147
+ source_rank = 0 if _subtitle_track_type(entry) == "uploader" else 1
1148
+ return source_rank, _normalize_lang(_safe_str(entry.get("lan") or entry.get("lan_doc"))).lower()
1149
+
1150
+
1151
+ def _resolve_ai_subtitle_url(
1152
+ fetcher: HttpFetcher,
1153
+ *,
1154
+ aid: str,
1155
+ cid: str,
1156
+ timeout: int,
1157
+ ) -> str | None:
1158
+ if not aid or not cid:
1159
+ return None
1160
+ payload = _fetch_json(
1161
+ fetcher,
1162
+ "https://api.bilibili.com/x/player/v2/ai/subtitle/search/stat",
1163
+ {"aid": aid, "cid": cid},
1164
+ timeout,
1165
+ )
1166
+ data = _bilibili_api_data(payload)
1167
+ return _normalize_subtitle_url(_safe_str(data.get("subtitle_url")))
1168
+
1169
+
1170
+ def _segments_from_subtitle_payload(payload: Any) -> list[_SubtitleSegment]:
1171
+ if isinstance(payload, dict):
1172
+ body = payload.get("body") or payload.get("segments") or payload.get("data")
1173
+ else:
1174
+ body = payload
1175
+ if not isinstance(body, list):
1176
+ return []
1177
+ return _segments_from_rows([row for row in body if isinstance(row, dict)])
1178
+
1179
+
1180
+ def _parse_json_subtitle_rows(text: str) -> list[dict[str, Any]]:
1181
+ try:
1182
+ payload = load_opencli_json(text)
1183
+ except json.JSONDecodeError:
1184
+ return []
1185
+ if isinstance(payload, dict):
1186
+ body = payload.get("body") or payload.get("segments") or payload.get("data")
1187
+ if isinstance(body, list):
1188
+ return [row for row in body if isinstance(row, dict)]
1189
+ return [payload]
1190
+ if isinstance(payload, list):
1191
+ return [row for row in payload if isinstance(row, dict)]
1192
+ return []
1193
+
1194
+
1195
+ def _parse_tabular_rows(text: str) -> list[dict[str, str]]:
1196
+ return _parse_console_or_markdown_table(text) or _parse_csv_rows(text)
1197
+
1198
+
1199
+ def _parse_console_or_markdown_table(text: str) -> list[dict[str, str]]:
1200
+ lines = [line.strip() for line in text.splitlines() if line.strip()]
1201
+ header: list[str] | None = None
1202
+ rows: list[dict[str, str]] = []
1203
+ delimiter = "\u2502" if any("\u2502" in line for line in lines) else "|"
1204
+
1205
+ for line in lines:
1206
+ if delimiter not in line or set(line) <= {
1207
+ delimiter,
1208
+ "-",
1209
+ "+",
1210
+ "\u2500",
1211
+ "\u250c",
1212
+ "\u2510",
1213
+ "\u2514",
1214
+ "\u2518",
1215
+ "\u252c",
1216
+ "\u2534",
1217
+ "\u253c",
1218
+ " ",
1219
+ }:
1220
+ continue
1221
+ cells = [cell.strip().strip("'\"") for cell in line.strip(delimiter).split(delimiter)]
1222
+ cells = [cell for cell in cells if cell or delimiter == "|"]
1223
+ if not cells:
1224
+ continue
1225
+ normalized = [_normalize_header(cell) for cell in cells]
1226
+ if any(cell in {"from", "to", "content", "title", "url"} for cell in normalized):
1227
+ header = normalized
1228
+ continue
1229
+ if header is None or len(cells) < len(header):
1230
+ continue
1231
+ row = {header[index]: cells[index].strip() for index in range(len(header))}
1232
+ if any(value for value in row.values()) and set("".join(row.values())) != {"-"}:
1233
+ rows.append(row)
1234
+ return rows
1235
+
1236
+
1237
+ def _parse_csv_rows(text: str) -> list[dict[str, str]]:
1238
+ try:
1239
+ reader = csv.DictReader(io.StringIO(text))
1240
+ if not reader.fieldnames:
1241
+ return []
1242
+ fieldnames = [_normalize_header(field or "") for field in reader.fieldnames]
1243
+ rows: list[dict[str, str]] = []
1244
+ for raw_row in reader:
1245
+ row = {
1246
+ fieldnames[index]: (raw_row.get(reader.fieldnames[index] or "") or "").strip()
1247
+ for index in range(len(fieldnames))
1248
+ }
1249
+ if any(row.values()):
1250
+ rows.append(row)
1251
+ return rows
1252
+ except csv.Error:
1253
+ return []
1254
+
1255
+
1256
+ def _segments_from_rows(rows: list[dict[str, Any]]) -> list[_SubtitleSegment]:
1257
+ segments: list[_SubtitleSegment] = []
1258
+ for row in rows:
1259
+ start = _coerce_seconds(_row_lookup(row, "from", "start", "begin"))
1260
+ end = _coerce_seconds(_row_lookup(row, "to", "end"))
1261
+ content = _safe_str(_row_lookup(row, "content", "text", "body"))
1262
+ if start is None or not content:
1263
+ continue
1264
+ segments.append(_SubtitleSegment(start=start, end=end, content=content))
1265
+ return segments
1266
+
1267
+
1268
+ def _format_segments(segments: list[_SubtitleSegment]) -> str:
1269
+ return "".join(f"[{_format_timestamp(segment.start)}] {segment.content.strip()}\n" for segment in segments)
1270
+
1271
+
1272
+ def _format_timestamp(seconds: float) -> str:
1273
+ total = max(0, int(seconds))
1274
+ hours, remainder = divmod(total, 3600)
1275
+ minutes, secs = divmod(remainder, 60)
1276
+ return f"{hours:02d}:{minutes:02d}:{secs:02d}"
1277
+
1278
+
1279
+ def _coerce_seconds(value: object) -> float | None:
1280
+ text = _safe_str(value)
1281
+ if not text:
1282
+ return None
1283
+ text = text.strip().strip("s\u79d2")
1284
+ if ":" in text:
1285
+ parts = text.split(":")
1286
+ try:
1287
+ numbers = [float(part) for part in parts]
1288
+ except ValueError:
1289
+ return None
1290
+ if len(numbers) == 3:
1291
+ return numbers[0] * 3600 + numbers[1] * 60 + numbers[2]
1292
+ if len(numbers) == 2:
1293
+ return numbers[0] * 60 + numbers[1]
1294
+ try:
1295
+ return float(text)
1296
+ except ValueError:
1297
+ return None
1298
+
1299
+
1300
+ def _source_level_from_subtitle_meta(meta: dict[str, Any]) -> SourceLevel:
1301
+ if _subtitle_track_type(meta) == "ai":
1302
+ return "auto_caption"
1303
+ values = " ".join(_safe_str(value) for value in meta.values())
1304
+ if _NATIVE_CAPTION_HINT_RE.search(values):
1305
+ return "native_caption"
1306
+ return "native_caption"
1307
+
1308
+
1309
+ def _source_level_from_text(text: str, *, default: SourceLevel) -> SourceLevel:
1310
+ if _AI_CAPTION_HINT_RE.search(text):
1311
+ return "auto_caption"
1312
+ if _NATIVE_CAPTION_HINT_RE.search(text):
1313
+ return "native_caption"
1314
+ return default
1315
+
1316
+
1317
+ def _source_level_note(source_level: SourceLevel) -> str:
1318
+ if source_level == "auto_caption":
1319
+ return "source_level=auto_caption: Bilibili metadata/output indicated AI or automatic captions, or OpenCLI did not expose manual-caption metadata."
1320
+ if source_level == "native_caption":
1321
+ return "source_level=native_caption: Bilibili metadata/output indicated native/manual captions."
1322
+ return "source_level=partial_extraction: no usable caption text was produced."
1323
+
1324
+
1325
+ def _video_entries_from_space_payload(data: dict[str, Any]) -> list[dict[str, Any]]:
1326
+ listing = data.get("list")
1327
+ if isinstance(listing, dict) and isinstance(listing.get("vlist"), list):
1328
+ return [entry for entry in listing["vlist"] if isinstance(entry, dict)]
1329
+ if isinstance(data.get("vlist"), list):
1330
+ return [entry for entry in data["vlist"] if isinstance(entry, dict)]
1331
+ return []
1332
+
1333
+
1334
+ def _content_item_from_video_entry(
1335
+ entry: dict[str, Any],
1336
+ request: DiscoveryRequest,
1337
+ uid: str,
1338
+ ) -> ContentItem:
1339
+ bvid = _safe_str(entry.get("bvid")) or extract_bvid(_safe_str(entry.get("arcurl")))
1340
+ url = _safe_str(entry.get("arcurl")) or (f"https://www.bilibili.com/video/{bvid}/" if bvid else "")
1341
+ return ContentItem(
1342
+ platform="bilibili",
1343
+ input_type="single",
1344
+ url=url,
1345
+ id=bvid,
1346
+ title=_safe_str(entry.get("title")),
1347
+ author=_safe_str(entry.get("author")) or None,
1348
+ published_at=_format_unix_timestamp(entry.get("created")),
1349
+ source_type="video",
1350
+ raw_meta={
1351
+ "collection_url": request.input_url,
1352
+ "collection_uid": uid,
1353
+ "discovery_limit": request.limit,
1354
+ "discovery_provider": BILIBILI_NATIVE_DISCOVERY_BACKEND,
1355
+ },
1356
+ )
1357
+
1358
+
1359
+ def _content_item_from_opencli_row(
1360
+ row: dict[str, str],
1361
+ request: DiscoveryRequest,
1362
+ uid: str,
1363
+ ) -> ContentItem:
1364
+ row_text = " ".join(row.values())
1365
+ url = _row_lookup(row, "url") or _safe_str(re.search(r"https?://\S+", row_text).group(0) if re.search(r"https?://\S+", row_text) else "")
1366
+ bvid = extract_bvid(url or row_text)
1367
+ if not url and bvid:
1368
+ url = f"https://www.bilibili.com/video/{bvid}/"
1369
+ raw_meta = {
1370
+ "collection_url": request.input_url,
1371
+ "collection_uid": uid,
1372
+ "discovery_limit": request.limit,
1373
+ "discovery_provider": BILIBILI_OPENCLI_DISCOVERY_BACKEND,
1374
+ }
1375
+ for key in ("rank", "plays", "likes"):
1376
+ value = _row_lookup(row, key)
1377
+ if value:
1378
+ raw_meta[key] = value
1379
+ return ContentItem(
1380
+ platform="bilibili",
1381
+ input_type="single",
1382
+ url=url,
1383
+ id=bvid,
1384
+ title=_row_lookup(row, "title") or None,
1385
+ published_at=_row_lookup(row, "date", "published_at") or None,
1386
+ source_type="video",
1387
+ raw_meta=raw_meta,
1388
+ )
1389
+
1390
+
1391
+ def _bounded_limit(limit: int | None) -> int:
1392
+ if limit is None:
1393
+ return 20
1394
+ return max(0, min(limit, 50))
1395
+
1396
+
1397
+ def _normalize_header(value: str) -> str:
1398
+ normalized = value.strip().lower().replace(" ", "_").replace("-", "_")
1399
+ aliases = {
1400
+ "(index)": "index",
1401
+ "index": "index",
1402
+ "from": "from",
1403
+ "start": "start",
1404
+ "to": "to",
1405
+ "end": "end",
1406
+ "content": "content",
1407
+ "text": "text",
1408
+ "rank": "rank",
1409
+ "title": "title",
1410
+ "plays": "plays",
1411
+ "likes": "likes",
1412
+ "date": "date",
1413
+ "url": "url",
1414
+ }
1415
+ return aliases.get(normalized, normalized)
1416
+
1417
+
1418
+ def _row_lookup(row: dict[str, Any], *keys: str) -> str:
1419
+ normalized = {_normalize_header(str(key)): value for key, value in row.items()}
1420
+ for key in keys:
1421
+ value = normalized.get(_normalize_header(key))
1422
+ text = _safe_str(value)
1423
+ if text:
1424
+ return text
1425
+ return ""
1426
+
1427
+
1428
+ def _owner_name(view_data: dict[str, Any]) -> str | None:
1429
+ owner = view_data.get("owner")
1430
+ if isinstance(owner, dict):
1431
+ return _safe_str(owner.get("name")) or None
1432
+ return None
1433
+
1434
+
1435
+ def _format_unix_timestamp(value: object) -> str | None:
1436
+ try:
1437
+ seconds = int(str(value))
1438
+ except (TypeError, ValueError):
1439
+ return None
1440
+ return datetime.fromtimestamp(seconds, tz=timezone.utc).date().isoformat()
1441
+
1442
+
1443
+ def _safe_str(value: object) -> str:
1444
+ if value is None:
1445
+ return ""
1446
+ return str(value).strip()
1447
+
1448
+
1449
+ def _warning_notes(warnings: list[str]) -> list[str]:
1450
+ return [f"Bilibili subtitle discovery warning: {warning}" for warning in warnings]
1451
+
1452
+
1453
+ def _failed_result(
1454
+ backend: str,
1455
+ code: str,
1456
+ error: object,
1457
+ *,
1458
+ title: str | None = None,
1459
+ author: str | None = None,
1460
+ published_at: str | None = None,
1461
+ metadata: dict[str, Any] | None = None,
1462
+ notes: list[str] | None = None,
1463
+ ) -> TranscriptResult:
1464
+ return TranscriptResult(
1465
+ text="",
1466
+ status="failed",
1467
+ source_level="partial_extraction",
1468
+ backend=backend,
1469
+ source_notes=[
1470
+ "Bilibili provider returned structured failure.",
1471
+ _source_level_note("partial_extraction"),
1472
+ *(notes or []),
1473
+ ],
1474
+ error_code=code,
1475
+ error_summary=sanitize_error_summary(error),
1476
+ title=title,
1477
+ author=author,
1478
+ published_at=published_at,
1479
+ metadata=metadata or {},
1480
+ )