openmd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openmd/__init__.py +3 -0
- openmd/__main__.py +5 -0
- openmd/adapters/__init__.py +1 -0
- openmd/adapters/base.py +95 -0
- openmd/adapters/bilibili/__init__.py +11 -0
- openmd/adapters/bilibili/adapter.py +104 -0
- openmd/adapters/bilibili/discovery.py +33 -0
- openmd/adapters/douyin/__init__.py +11 -0
- openmd/adapters/douyin/adapter.py +117 -0
- openmd/adapters/douyin/discovery.py +36 -0
- openmd/adapters/douyin/sidecar.py +77 -0
- openmd/adapters/generic_media/__init__.py +14 -0
- openmd/adapters/high_privacy.py +77 -0
- openmd/adapters/local_audio/__init__.py +14 -0
- openmd/adapters/local_audio/adapter.py +169 -0
- openmd/adapters/local_audio/asr.py +58 -0
- openmd/adapters/local_audio/backends/__init__.py +1 -0
- openmd/adapters/local_audio/backends/auto.py +177 -0
- openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
- openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
- openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
- openmd/adapters/opencli.py +770 -0
- openmd/adapters/pdf/__init__.py +11 -0
- openmd/adapters/pdf/adapter.py +153 -0
- openmd/adapters/pdf/backends/__init__.py +1 -0
- openmd/adapters/pdf/backends/markitdown.py +165 -0
- openmd/adapters/pdf/parser.py +56 -0
- openmd/adapters/qq_group/__init__.py +11 -0
- openmd/adapters/qq_group/adapter.py +289 -0
- openmd/adapters/qq_group/parsers/__init__.py +1 -0
- openmd/adapters/qq_group/parsers/html.py +60 -0
- openmd/adapters/qq_group/parsers/json.py +71 -0
- openmd/adapters/registry.py +149 -0
- openmd/adapters/screenshot/__init__.py +11 -0
- openmd/adapters/screenshot/adapter.py +154 -0
- openmd/adapters/screenshot/backends/__init__.py +1 -0
- openmd/adapters/screenshot/backends/paddle.py +15 -0
- openmd/adapters/screenshot/backends/rapidocr.py +176 -0
- openmd/adapters/screenshot/backends/tesseract.py +15 -0
- openmd/adapters/screenshot/ocr.py +84 -0
- openmd/adapters/wechat_group/__init__.py +11 -0
- openmd/adapters/wechat_group/adapter.py +160 -0
- openmd/adapters/wechat_group/wx_cli.py +268 -0
- openmd/adapters/wechat_moments/__init__.py +11 -0
- openmd/adapters/wechat_moments/adapter.py +167 -0
- openmd/adapters/wechat_moments/wx_cli.py +265 -0
- openmd/adapters/weixin/__init__.py +11 -0
- openmd/adapters/weixin/adapter.py +148 -0
- openmd/adapters/weixin/discovery.py +510 -0
- openmd/adapters/weixin/report.py +278 -0
- openmd/adapters/xiaohongshu.py +158 -0
- openmd/backends/__init__.py +1 -0
- openmd/backends/base.py +204 -0
- openmd/backends/bilibili.py +1480 -0
- openmd/backends/bilibili_wbi.py +277 -0
- openmd/backends/discourse.py +426 -0
- openmd/backends/douyin.py +868 -0
- openmd/backends/douyin_detail.py +124 -0
- openmd/backends/douyin_providers.py +602 -0
- openmd/backends/executor.py +546 -0
- openmd/backends/feishu.py +861 -0
- openmd/backends/generic_media.py +77 -0
- openmd/backends/interactive.py +60 -0
- openmd/backends/ocr.py +41 -0
- openmd/backends/opencli.py +82 -0
- openmd/backends/pdf.py +122 -0
- openmd/backends/registry.py +234 -0
- openmd/backends/screenshot.py +79 -0
- openmd/backends/video_metadata.py +167 -0
- openmd/backends/web_article.py +363 -0
- openmd/backends/wechat_group.py +35 -0
- openmd/backends/wechat_moments.py +35 -0
- openmd/backends/weixin.py +767 -0
- openmd/backends/x.py +455 -0
- openmd/backends/xiaoe.py +940 -0
- openmd/backends/xiaohongshu.py +911 -0
- openmd/backends/xiaohongshu_providers.py +876 -0
- openmd/backends/xiaoyuzhou.py +1017 -0
- openmd/backends/youtube.py +280 -0
- openmd/backends/youtube_discovery.py +1245 -0
- openmd/backends/youtube_transcript.py +213 -0
- openmd/browser_extension/background.js +231 -0
- openmd/browser_extension/icons/icon-128.png +0 -0
- openmd/browser_extension/icons/icon-16.png +0 -0
- openmd/browser_extension/icons/icon-32.png +0 -0
- openmd/browser_extension/icons/icon-48.png +0 -0
- openmd/browser_extension/icons/icon-512.png +0 -0
- openmd/browser_extension/idle.html +6 -0
- openmd/browser_extension/manifest.json +43 -0
- openmd/browser_extension/options.html +42 -0
- openmd/browser_extension/options.js +117 -0
- openmd/browser_extension/popup.css +71 -0
- openmd/browser_extension/popup.html +17 -0
- openmd/browser_extension/popup.js +59 -0
- openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
- openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
- openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
- openmd/browser_host.py +503 -0
- openmd/browser_setup.py +106 -0
- openmd/cli.py +2012 -0
- openmd/console.py +32 -0
- openmd/core/__init__.py +1 -0
- openmd/core/capabilities.py +162 -0
- openmd/core/content_validation.py +323 -0
- openmd/core/coverage.py +163 -0
- openmd/core/detector.py +247 -0
- openmd/core/doctor.py +104 -0
- openmd/core/execution_policy.py +164 -0
- openmd/core/filename.py +64 -0
- openmd/core/input_validation.py +91 -0
- openmd/core/local_config.py +96 -0
- openmd/core/manifest.py +156 -0
- openmd/core/models.py +168 -0
- openmd/core/opencli_output.py +34 -0
- openmd/core/operation_contract.py +60 -0
- openmd/core/output.py +194 -0
- openmd/core/package_contract.py +335 -0
- openmd/core/package_validator.py +489 -0
- openmd/core/package_writer.py +666 -0
- openmd/core/planner.py +345 -0
- openmd/core/preferences.py +61 -0
- openmd/core/readable.py +129 -0
- openmd/core/receipt.py +285 -0
- openmd/core/renderer.py +185 -0
- openmd/core/runlog.py +208 -0
- openmd/core/secrets.py +313 -0
- openmd/doctor_view.py +68 -0
- openmd/interactive.py +453 -0
- openmd/ocr/__init__.py +1 -0
- openmd/ocr/breakers.py +84 -0
- openmd/ocr/config.py +54 -0
- openmd/ocr/documents.py +522 -0
- openmd/ocr/endpoint.py +72 -0
- openmd/ocr/layout.py +92 -0
- openmd/ocr/layout_filter.py +118 -0
- openmd/ocr/layout_worker.py +89 -0
- openmd/ocr/native_math.py +220 -0
- openmd/ocr/office.py +305 -0
- openmd/ocr/ollama.py +200 -0
- openmd/ocr/otsl.py +51 -0
- openmd/ocr/ovis-prompt.txt +1 -0
- openmd/ocr/pipeline.py +384 -0
- openmd/ocr/settings.json +25 -0
- openmd/ocr/tiling.py +631 -0
- openmd/post_video.py +154 -0
- openmd/probe.py +397 -0
- openmd/routing/__init__.py +1 -0
- openmd/routing/cards/builtin.json +620 -0
- openmd/routing/cards/defuddle.json +70 -0
- openmd/routing/cards/x-tweet-fetcher.json +71 -0
- openmd/routing/cards/xyz-dl.json +60 -0
- openmd/routing/cards/yutto.json +68 -0
- openmd/routing/entry_shim.py +71 -0
- openmd/routing/lifecycle.py +459 -0
- openmd/routing/loader.py +138 -0
- openmd/routing/providers.py +158 -0
- openmd/routing/readers.py +108 -0
- openmd/routing/routes.json +504 -0
- openmd/routing/runner.py +192 -0
- openmd/routing/view.py +40 -0
- openmd/runtime/__init__.py +1 -0
- openmd/runtime/_windows_job.py +234 -0
- openmd/runtime/asr.py +217 -0
- openmd/runtime/asr_worker.py +119 -0
- openmd/runtime/gpu_lock.py +81 -0
- openmd/runtime/gpu_quarantine.py +85 -0
- openmd/runtime/interpreters.py +407 -0
- openmd/runtime/managed_tools.py +204 -0
- openmd/runtime/models.py +202 -0
- openmd/runtime/services.py +175 -0
- openmd/runtime/tempdirs.py +227 -0
- openmd_cli-0.1.0.dist-info/METADATA +168 -0
- openmd_cli-0.1.0.dist-info/RECORD +177 -0
- openmd_cli-0.1.0.dist-info/WHEEL +5 -0
- openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
- openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1480 @@
|
|
|
1
|
+
"""Bilibili subtitle extraction and collection discovery providers.
|
|
2
|
+
|
|
3
|
+
The providers in this module keep Bilibili work isolated from shared registries:
|
|
4
|
+
``openmd.backends.registry`` discovers this file by platform naming convention.
|
|
5
|
+
Real network/browser work is only registered when the matching CLI runtime gate
|
|
6
|
+
is enabled by the caller.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from contextlib import contextmanager
|
|
12
|
+
import csv
|
|
13
|
+
import io
|
|
14
|
+
import json
|
|
15
|
+
import re
|
|
16
|
+
import shutil
|
|
17
|
+
import subprocess
|
|
18
|
+
import tempfile
|
|
19
|
+
import time
|
|
20
|
+
from dataclasses import dataclass, replace
|
|
21
|
+
from datetime import datetime, timezone
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Callable
|
|
24
|
+
from urllib.error import HTTPError, URLError
|
|
25
|
+
from urllib.parse import urlencode, urlparse
|
|
26
|
+
from urllib.request import Request, urlopen
|
|
27
|
+
|
|
28
|
+
from openmd.core.opencli_output import load_opencli_json
|
|
29
|
+
from openmd.runtime.asr import get_asr_backend
|
|
30
|
+
from openmd.backends.base import (
|
|
31
|
+
BackendCapability,
|
|
32
|
+
BackendDescriptor,
|
|
33
|
+
BackendExtractRequest,
|
|
34
|
+
BackendHealth,
|
|
35
|
+
DiscoveryRequest,
|
|
36
|
+
DiscoveryResult,
|
|
37
|
+
)
|
|
38
|
+
from openmd.backends.executor import BackendProviderRegistry, DiscoveryProviderRegistry
|
|
39
|
+
from openmd.backends.registry import BackendRuntimeOptions
|
|
40
|
+
from openmd.backends.bilibili_wbi import (
|
|
41
|
+
BilibiliWbiSigner,
|
|
42
|
+
WbiSigningError,
|
|
43
|
+
bilibili_cookie_header_for_url,
|
|
44
|
+
bilibili_headers,
|
|
45
|
+
)
|
|
46
|
+
from openmd.core.detector import extract_first_url
|
|
47
|
+
from openmd.core.models import ContentItem, SourceLevel, TranscriptResult
|
|
48
|
+
from openmd.core.secrets import sanitize_error_summary
|
|
49
|
+
from openmd.runtime.tempdirs import owned_temporary_directory
|
|
50
|
+
from openmd.runtime.interpreters import run_interruptibly
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
BILIBILI_HARVEST_BACKEND = "bilibili-harvest.core"
|
|
54
|
+
BILIBILI_OPENCLI_SUBTITLE_BACKEND = "opencli.bilibili.subtitle"
|
|
55
|
+
BILIBILI_YTDLP_MEDIA_BACKEND = "yt-dlp.bilibili"
|
|
56
|
+
BILIBILI_ASR_FALLBACK_BACKEND = "asr.fallback"
|
|
57
|
+
BILIBILI_NATIVE_DISCOVERY_BACKEND = "bilibili-harvest.discovery"
|
|
58
|
+
BILIBILI_OPENCLI_DISCOVERY_BACKEND = "opencli.bilibili.user-videos"
|
|
59
|
+
|
|
60
|
+
# Historical backend IDs retain "bilibili-harvest", but the current Openmd
|
|
61
|
+
# implementation is a public Bilibili API wrapper, not the Bilibili-Harvest
|
|
62
|
+
# local app/browser-extension pipeline.
|
|
63
|
+
|
|
64
|
+
_BVID_RE = re.compile(r"(BV[0-9A-Za-z]+)")
|
|
65
|
+
_SPACE_UID_RE = re.compile(r"(?:space\.bilibili\.com/|^)(\d{3,})")
|
|
66
|
+
_AI_CAPTION_HINT_RE = re.compile(
|
|
67
|
+
"ai|auto|automatic|auto[_ -]?caption|"
|
|
68
|
+
"\u81ea\u52a8|\u667a\u80fd\u5b57\u5e55|"
|
|
69
|
+
"ai\u5b57\u5e55|ai\u751f\u6210",
|
|
70
|
+
re.IGNORECASE,
|
|
71
|
+
)
|
|
72
|
+
_NATIVE_CAPTION_HINT_RE = re.compile(
|
|
73
|
+
"native|manual|author|uploaded|"
|
|
74
|
+
"\u5b98\u65b9\u5b57\u5e55|\u4eba\u5de5|\u4f5c\u8005|up\u4e3b",
|
|
75
|
+
re.IGNORECASE,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
CommandRunner = Callable[[list[str], int], tuple[int, str, str]]
|
|
79
|
+
HttpFetcher = Callable[[str, dict[str, str], int], tuple[int, str]]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass(frozen=True)
|
|
83
|
+
class _SubtitleSegment:
|
|
84
|
+
start: float
|
|
85
|
+
end: float | None
|
|
86
|
+
content: str
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass(frozen=True)
|
|
90
|
+
class _MediaWorkspace:
|
|
91
|
+
path: Path
|
|
92
|
+
retention: str
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def register_backend_providers(
|
|
96
|
+
registry: BackendProviderRegistry,
|
|
97
|
+
options: BackendRuntimeOptions,
|
|
98
|
+
) -> None:
|
|
99
|
+
"""Register Bilibili extraction providers behind explicit runtime gates."""
|
|
100
|
+
requested = options.backend_override
|
|
101
|
+
if options.enable_network_backends and requested in {None, BILIBILI_HARVEST_BACKEND}:
|
|
102
|
+
registry.register(BILIBILI_HARVEST_BACKEND, BilibiliNativeSubtitleProvider())
|
|
103
|
+
|
|
104
|
+
if options.enable_browser_backends and requested in {None, BILIBILI_OPENCLI_SUBTITLE_BACKEND}:
|
|
105
|
+
registry.register(
|
|
106
|
+
BILIBILI_OPENCLI_SUBTITLE_BACKEND,
|
|
107
|
+
OpenCliBilibiliSubtitleProvider(),
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
if requested in {None, BILIBILI_YTDLP_MEDIA_BACKEND}:
|
|
111
|
+
registry.register(
|
|
112
|
+
BILIBILI_YTDLP_MEDIA_BACKEND,
|
|
113
|
+
BilibiliMediaDownloadGateProvider(options),
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
if requested in {None, BILIBILI_ASR_FALLBACK_BACKEND}:
|
|
117
|
+
registry.register(
|
|
118
|
+
BILIBILI_ASR_FALLBACK_BACKEND,
|
|
119
|
+
BilibiliMediaAsrFallbackProvider(options),
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def register_discovery_providers(
|
|
124
|
+
registry: DiscoveryProviderRegistry,
|
|
125
|
+
options: BackendRuntimeOptions,
|
|
126
|
+
) -> None:
|
|
127
|
+
"""Register Bilibili discovery providers without touching shared registries."""
|
|
128
|
+
requested = options.requested_discovery_provider
|
|
129
|
+
native_requested = requested in {None, BILIBILI_NATIVE_DISCOVERY_BACKEND}
|
|
130
|
+
opencli_requested = requested in {None, BILIBILI_OPENCLI_DISCOVERY_BACKEND}
|
|
131
|
+
|
|
132
|
+
native_registered = False
|
|
133
|
+
if options.enable_network_backends and native_requested:
|
|
134
|
+
registry.register(
|
|
135
|
+
BILIBILI_NATIVE_DISCOVERY_BACKEND,
|
|
136
|
+
BilibiliNativeUserVideosProvider(),
|
|
137
|
+
platforms=("bilibili",),
|
|
138
|
+
default_for_platforms=requested is None,
|
|
139
|
+
)
|
|
140
|
+
native_registered = True
|
|
141
|
+
|
|
142
|
+
if options.enable_browser_backends and opencli_requested:
|
|
143
|
+
registry.register(
|
|
144
|
+
BILIBILI_OPENCLI_DISCOVERY_BACKEND,
|
|
145
|
+
OpenCliBilibiliUserVideosProvider(),
|
|
146
|
+
platforms=("bilibili",),
|
|
147
|
+
default_for_platforms=requested is None and not native_registered,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class BilibiliNativeSubtitleProvider:
|
|
152
|
+
"""Wrap public Bilibili subtitle APIs; not the local/browser Harvest pipeline."""
|
|
153
|
+
|
|
154
|
+
name = BILIBILI_HARVEST_BACKEND
|
|
155
|
+
|
|
156
|
+
def __init__(
|
|
157
|
+
self,
|
|
158
|
+
*,
|
|
159
|
+
fetcher: HttpFetcher | None = None,
|
|
160
|
+
timeout: int = 30,
|
|
161
|
+
wbi_signer: BilibiliWbiSigner | None = None,
|
|
162
|
+
max_ai_retries: int = 3,
|
|
163
|
+
ai_retry_interval: float = 0.0,
|
|
164
|
+
) -> None:
|
|
165
|
+
self._fetcher = fetcher or _default_fetch
|
|
166
|
+
self._timeout = timeout
|
|
167
|
+
self._wbi_signer = wbi_signer or BilibiliWbiSigner(
|
|
168
|
+
fetcher=self._fetcher,
|
|
169
|
+
timeout=timeout,
|
|
170
|
+
)
|
|
171
|
+
self._max_ai_retries = max(1, max_ai_retries)
|
|
172
|
+
self._ai_retry_interval = max(0.0, ai_retry_interval)
|
|
173
|
+
|
|
174
|
+
def descriptor(self) -> BackendDescriptor:
|
|
175
|
+
capability = BackendCapability(
|
|
176
|
+
platforms=("bilibili",),
|
|
177
|
+
source_types=("video",),
|
|
178
|
+
input_types=("single",),
|
|
179
|
+
source_levels=("native_caption", "auto_caption", "partial_extraction"),
|
|
180
|
+
requires_network=True,
|
|
181
|
+
)
|
|
182
|
+
return BackendDescriptor(
|
|
183
|
+
name=self.name,
|
|
184
|
+
layer="platform_native",
|
|
185
|
+
capabilities=(capability,),
|
|
186
|
+
description=(
|
|
187
|
+
"Bilibili public API subtitle wrapper; not the Bilibili-Harvest "
|
|
188
|
+
"local/browser pipeline, no media download or ASR."
|
|
189
|
+
),
|
|
190
|
+
provider="openmd.backends.bilibili",
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
def health(self) -> BackendHealth:
|
|
194
|
+
return BackendHealth(ok=True, dependencies={"network": "enabled by caller"})
|
|
195
|
+
|
|
196
|
+
def extract(self, request: BackendExtractRequest) -> TranscriptResult:
|
|
197
|
+
item = request.item
|
|
198
|
+
bvid = extract_bvid(item.url)
|
|
199
|
+
if bvid is None:
|
|
200
|
+
return _failed_result(self.name, "invalid_input", "Bilibili BVID not found.")
|
|
201
|
+
|
|
202
|
+
try:
|
|
203
|
+
view = _fetch_json(
|
|
204
|
+
self._fetcher,
|
|
205
|
+
"https://api.bilibili.com/x/web-interface/view",
|
|
206
|
+
{"bvid": bvid},
|
|
207
|
+
self._timeout,
|
|
208
|
+
)
|
|
209
|
+
view_data = _bilibili_api_data(view)
|
|
210
|
+
except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
|
|
211
|
+
return _failed_result(self.name, "bilibili_view_fetch_failed", exc)
|
|
212
|
+
|
|
213
|
+
aid = str(view_data.get("aid") or "").strip()
|
|
214
|
+
cid = str(view_data.get("cid") or "").strip()
|
|
215
|
+
if not cid:
|
|
216
|
+
pages = view_data.get("pages")
|
|
217
|
+
if isinstance(pages, list) and pages:
|
|
218
|
+
cid = str((pages[0] or {}).get("cid") or "").strip()
|
|
219
|
+
if not cid:
|
|
220
|
+
return _failed_result(
|
|
221
|
+
self.name,
|
|
222
|
+
"bilibili_cid_missing",
|
|
223
|
+
"Bilibili view metadata did not include a cid.",
|
|
224
|
+
metadata={"bvid": bvid, "aid": aid},
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
subtitles: list[dict[str, Any]] = []
|
|
228
|
+
endpoint_used = "none"
|
|
229
|
+
player_warnings: list[str] = []
|
|
230
|
+
for endpoint, url, params in _player_subtitle_attempts(aid=aid, cid=cid, bvid=bvid):
|
|
231
|
+
request_params = params
|
|
232
|
+
if endpoint == "wbi_v2":
|
|
233
|
+
try:
|
|
234
|
+
request_params = self._wbi_signer.sign_params(params, timeout=self._timeout)
|
|
235
|
+
except (HTTPError, URLError, OSError, ValueError, WbiSigningError) as exc:
|
|
236
|
+
player_warnings.append(
|
|
237
|
+
f"{endpoint}_signing: {sanitize_error_summary(exc)}; unsigned request attempted"
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
for retry_index in range(self._max_ai_retries):
|
|
241
|
+
try:
|
|
242
|
+
player = _fetch_json(self._fetcher, url, request_params, self._timeout)
|
|
243
|
+
player_data = _bilibili_api_data(player)
|
|
244
|
+
except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
|
|
245
|
+
player_warnings.append(f"{endpoint}: {sanitize_error_summary(exc)}")
|
|
246
|
+
break
|
|
247
|
+
|
|
248
|
+
subtitles = _subtitle_entries_from_player(player_data)
|
|
249
|
+
if subtitles:
|
|
250
|
+
endpoint_used = endpoint
|
|
251
|
+
break
|
|
252
|
+
if retry_index + 1 < self._max_ai_retries and self._ai_retry_interval:
|
|
253
|
+
time.sleep(self._ai_retry_interval)
|
|
254
|
+
if subtitles:
|
|
255
|
+
break
|
|
256
|
+
player_warnings.append(f"{endpoint}: no subtitle tracks")
|
|
257
|
+
|
|
258
|
+
if not subtitles:
|
|
259
|
+
return _failed_result(
|
|
260
|
+
self.name,
|
|
261
|
+
"bilibili_subtitle_missing",
|
|
262
|
+
"Bilibili metadata did not expose native or AI subtitles.",
|
|
263
|
+
title=_safe_str(view_data.get("title")),
|
|
264
|
+
author=_owner_name(view_data),
|
|
265
|
+
published_at=_format_unix_timestamp(view_data.get("pubdate")),
|
|
266
|
+
metadata={"bvid": bvid, "aid": aid, "cid": cid, "player_warnings": player_warnings},
|
|
267
|
+
notes=[
|
|
268
|
+
(
|
|
269
|
+
"Openmd's current bilibili-harvest.core backend uses public "
|
|
270
|
+
"Bilibili APIs only; it does not invoke the Bilibili-Harvest "
|
|
271
|
+
"local app/browser-extension pipeline."
|
|
272
|
+
),
|
|
273
|
+
*_warning_notes(player_warnings),
|
|
274
|
+
],
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
subtitle = _choose_subtitle(subtitles)
|
|
278
|
+
subtitle_track_type = _subtitle_track_type(subtitle)
|
|
279
|
+
subtitle_url = _normalize_subtitle_url(_safe_str(subtitle.get("subtitle_url")))
|
|
280
|
+
ai_rescue_note: str | None = None
|
|
281
|
+
if not subtitle_url and subtitle_track_type == "ai":
|
|
282
|
+
try:
|
|
283
|
+
subtitle_url = _resolve_ai_subtitle_url(
|
|
284
|
+
self._fetcher,
|
|
285
|
+
aid=aid,
|
|
286
|
+
cid=cid,
|
|
287
|
+
timeout=self._timeout,
|
|
288
|
+
)
|
|
289
|
+
except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
|
|
290
|
+
player_warnings.append(f"ai_subtitle_search_stat: {sanitize_error_summary(exc)}")
|
|
291
|
+
if subtitle_url:
|
|
292
|
+
ai_rescue_note = "AI subtitle URL was recovered via x/player/v2/ai/subtitle/search/stat."
|
|
293
|
+
|
|
294
|
+
if not subtitle_url:
|
|
295
|
+
return _failed_result(
|
|
296
|
+
self.name,
|
|
297
|
+
"bilibili_subtitle_url_missing",
|
|
298
|
+
"Selected Bilibili subtitle did not include a subtitle_url.",
|
|
299
|
+
metadata={"bvid": bvid, "aid": aid, "cid": cid, "track_type": subtitle_track_type},
|
|
300
|
+
notes=[
|
|
301
|
+
"AI subtitle URL rescue via x/player/v2/ai/subtitle/search/stat was attempted when applicable.",
|
|
302
|
+
*_warning_notes(player_warnings),
|
|
303
|
+
],
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
try:
|
|
307
|
+
subtitle_payload = _fetch_json(self._fetcher, subtitle_url, {}, self._timeout)
|
|
308
|
+
segments = _segments_from_subtitle_payload(subtitle_payload)
|
|
309
|
+
except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
|
|
310
|
+
return _failed_result(self.name, "bilibili_subtitle_fetch_failed", exc)
|
|
311
|
+
|
|
312
|
+
if not segments:
|
|
313
|
+
return _failed_result(
|
|
314
|
+
self.name,
|
|
315
|
+
"bilibili_subtitle_empty",
|
|
316
|
+
"Bilibili subtitle payload did not contain usable subtitle lines.",
|
|
317
|
+
metadata={"bvid": bvid, "aid": aid, "cid": cid},
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
source_level = _source_level_from_subtitle_meta(subtitle)
|
|
321
|
+
source_notes = [
|
|
322
|
+
"Bilibili public API subtitle discovery tried signed x/player/wbi/v2 before x/player/v2 when aid was available.",
|
|
323
|
+
f"subtitle_endpoint_used={endpoint_used}",
|
|
324
|
+
f"async_ai_subtitle_retries={self._max_ai_retries}",
|
|
325
|
+
"Bilibili danmaku/XML subtitle tracks were filtered before track selection.",
|
|
326
|
+
_source_level_note(source_level),
|
|
327
|
+
"This backend does not invoke the Bilibili-Harvest local app/browser-extension pipeline.",
|
|
328
|
+
"No media, raw subtitle JSON, raw response, cookies, headers, or ASR artifacts were persisted.",
|
|
329
|
+
]
|
|
330
|
+
if ai_rescue_note:
|
|
331
|
+
source_notes.append(ai_rescue_note)
|
|
332
|
+
source_notes.extend(_warning_notes(player_warnings))
|
|
333
|
+
return TranscriptResult(
|
|
334
|
+
text=_format_segments(segments),
|
|
335
|
+
status="verified",
|
|
336
|
+
source_level=source_level,
|
|
337
|
+
backend=self.name,
|
|
338
|
+
source_notes=source_notes,
|
|
339
|
+
title=_safe_str(view_data.get("title")),
|
|
340
|
+
author=_owner_name(view_data),
|
|
341
|
+
published_at=_format_unix_timestamp(view_data.get("pubdate")),
|
|
342
|
+
metadata={
|
|
343
|
+
"bvid": bvid,
|
|
344
|
+
"aid": aid,
|
|
345
|
+
"cid": cid,
|
|
346
|
+
"caption_language": _safe_str(subtitle.get("lan") or subtitle.get("lan_doc")),
|
|
347
|
+
"caption_track_type": subtitle_track_type,
|
|
348
|
+
"subtitle_endpoint_used": endpoint_used,
|
|
349
|
+
"segment_count": len(segments),
|
|
350
|
+
},
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
class OpenCliBilibiliSubtitleProvider:
|
|
355
|
+
"""Controlled OpenCLI fallback for Bilibili subtitle extraction."""
|
|
356
|
+
|
|
357
|
+
name = BILIBILI_OPENCLI_SUBTITLE_BACKEND
|
|
358
|
+
|
|
359
|
+
def __init__(
|
|
360
|
+
self,
|
|
361
|
+
*,
|
|
362
|
+
runner: CommandRunner | None = None,
|
|
363
|
+
timeout: int = 120,
|
|
364
|
+
) -> None:
|
|
365
|
+
self._runner = runner or _run_command
|
|
366
|
+
self._timeout = timeout
|
|
367
|
+
|
|
368
|
+
def descriptor(self) -> BackendDescriptor:
|
|
369
|
+
capability = BackendCapability(
|
|
370
|
+
platforms=("bilibili",),
|
|
371
|
+
source_types=("video",),
|
|
372
|
+
input_types=("single",),
|
|
373
|
+
source_levels=("native_caption", "auto_caption", "partial_extraction"),
|
|
374
|
+
requires_network=True,
|
|
375
|
+
)
|
|
376
|
+
return BackendDescriptor(
|
|
377
|
+
name=self.name,
|
|
378
|
+
layer="browser_cli_fallback",
|
|
379
|
+
capabilities=(capability,),
|
|
380
|
+
description="OpenCLI bilibili subtitle fallback; registered only with browser backend opt-in.",
|
|
381
|
+
provider="opencli",
|
|
382
|
+
is_fallback=True,
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
def health(self) -> BackendHealth:
|
|
386
|
+
ok = shutil.which("opencli") is not None
|
|
387
|
+
return BackendHealth(
|
|
388
|
+
ok=ok,
|
|
389
|
+
dependencies={"opencli": "ok" if ok else "missing"},
|
|
390
|
+
warnings=() if ok else ("opencli executable was not found on PATH",),
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
def extract(self, request: BackendExtractRequest) -> TranscriptResult:
|
|
394
|
+
item = request.item
|
|
395
|
+
bvid = extract_bvid(item.url)
|
|
396
|
+
if bvid is None:
|
|
397
|
+
return _failed_result(self.name, "invalid_input", "Bilibili BVID not found.")
|
|
398
|
+
|
|
399
|
+
try:
|
|
400
|
+
exit_code, stdout, stderr = self._runner(
|
|
401
|
+
["opencli", "bilibili", "subtitle", bvid, "-f", "json"],
|
|
402
|
+
self._timeout,
|
|
403
|
+
)
|
|
404
|
+
except FileNotFoundError as exc:
|
|
405
|
+
return _failed_result(self.name, "backend_not_found", exc)
|
|
406
|
+
except subprocess.TimeoutExpired as exc:
|
|
407
|
+
return _failed_result(self.name, "backend_timeout", exc)
|
|
408
|
+
except OSError as exc:
|
|
409
|
+
return _failed_result(self.name, "backend_os_error", exc)
|
|
410
|
+
|
|
411
|
+
if exit_code != 0:
|
|
412
|
+
return _failed_result(
|
|
413
|
+
self.name,
|
|
414
|
+
f"backend_exit_{exit_code}",
|
|
415
|
+
stderr or "OpenCLI bilibili subtitle failed.",
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
segments = parse_bilibili_subtitle_segments(stdout)
|
|
419
|
+
if not segments:
|
|
420
|
+
return _failed_result(
|
|
421
|
+
self.name,
|
|
422
|
+
"empty_result",
|
|
423
|
+
"OpenCLI bilibili subtitle returned no parseable subtitle rows.",
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
source_level = _source_level_from_text(stdout, default="auto_caption")
|
|
427
|
+
return TranscriptResult(
|
|
428
|
+
text=_format_segments(segments),
|
|
429
|
+
status="verified",
|
|
430
|
+
source_level=source_level,
|
|
431
|
+
backend=self.name,
|
|
432
|
+
source_notes=[
|
|
433
|
+
"OpenCLI bilibili subtitle output was parsed and normalized in memory.",
|
|
434
|
+
_source_level_note(source_level),
|
|
435
|
+
"No raw stdout/stderr, raw subtitle rows, media, cookies, or headers were persisted.",
|
|
436
|
+
],
|
|
437
|
+
metadata={"bvid": bvid, "segment_count": len(segments)},
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
class BilibiliMediaDownloadGateProvider:
|
|
442
|
+
"""Document the media-download stage without performing hidden extraction."""
|
|
443
|
+
|
|
444
|
+
name = BILIBILI_YTDLP_MEDIA_BACKEND
|
|
445
|
+
|
|
446
|
+
def __init__(self, options: BackendRuntimeOptions) -> None:
|
|
447
|
+
self._options = options
|
|
448
|
+
|
|
449
|
+
def descriptor(self) -> BackendDescriptor:
|
|
450
|
+
capability = BackendCapability(
|
|
451
|
+
platforms=("bilibili",),
|
|
452
|
+
source_types=("video",),
|
|
453
|
+
input_types=("single",),
|
|
454
|
+
source_levels=("partial_extraction",),
|
|
455
|
+
requires_network=True,
|
|
456
|
+
optional_config=("enable_media_download_backends", "enable_asr_backends"),
|
|
457
|
+
)
|
|
458
|
+
return BackendDescriptor(
|
|
459
|
+
name=self.name,
|
|
460
|
+
layer="media_download_gate",
|
|
461
|
+
capabilities=(capability,),
|
|
462
|
+
description=(
|
|
463
|
+
"Bilibili media download gate. The text-producing fallback is "
|
|
464
|
+
"asr.fallback and requires explicit media-download plus ASR opt-in."
|
|
465
|
+
),
|
|
466
|
+
provider="openmd.backends.bilibili",
|
|
467
|
+
is_fallback=True,
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
def health(self) -> BackendHealth:
|
|
471
|
+
if not self._options.enable_media_download_backends:
|
|
472
|
+
return BackendHealth(ok=True, warnings=("media download fallback is not enabled",))
|
|
473
|
+
ok = shutil.which("yt-dlp") is not None
|
|
474
|
+
return BackendHealth(
|
|
475
|
+
ok=ok,
|
|
476
|
+
dependencies={"yt-dlp": "ok" if ok else "missing"},
|
|
477
|
+
error_code=None if ok else "media_download_backend_missing",
|
|
478
|
+
error_summary=None if ok else "yt-dlp executable was not found on PATH.",
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
def extract(self, request: BackendExtractRequest) -> TranscriptResult:
|
|
482
|
+
item = request.item
|
|
483
|
+
bvid = extract_bvid(item.url)
|
|
484
|
+
return _failed_result(
|
|
485
|
+
self.name,
|
|
486
|
+
"bilibili_media_download_stage_requires_asr",
|
|
487
|
+
"Media download is an intermediate stage; rerun with --enable-media-download --enable-asr to attempt ASR fallback.",
|
|
488
|
+
title=item.title,
|
|
489
|
+
author=item.author,
|
|
490
|
+
published_at=item.published_at,
|
|
491
|
+
metadata={
|
|
492
|
+
"bvid": bvid,
|
|
493
|
+
"failure_class": "user_resolvable",
|
|
494
|
+
"media_download_enabled": self._options.enable_media_download_backends,
|
|
495
|
+
"asr_enabled": self._options.enable_asr_backends,
|
|
496
|
+
},
|
|
497
|
+
notes=[
|
|
498
|
+
"No media download or ASR action was attempted by the standalone media-download stage.",
|
|
499
|
+
"Use the final asr.fallback provider for approved temporary media download plus local ASR.",
|
|
500
|
+
],
|
|
501
|
+
)
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
class BilibiliMediaAsrFallbackProvider:
|
|
505
|
+
"""Explicit temporary-media + local-ASR fallback for no-subtitle Bilibili items."""
|
|
506
|
+
|
|
507
|
+
name = BILIBILI_ASR_FALLBACK_BACKEND
|
|
508
|
+
|
|
509
|
+
def __init__(
|
|
510
|
+
self,
|
|
511
|
+
options: BackendRuntimeOptions,
|
|
512
|
+
*,
|
|
513
|
+
runner: CommandRunner | None = None,
|
|
514
|
+
asr_backend: object | None = None,
|
|
515
|
+
timeout: int = 1800,
|
|
516
|
+
) -> None:
|
|
517
|
+
self._options = options
|
|
518
|
+
self._runner = runner or _run_command
|
|
519
|
+
self._asr_backend = asr_backend or get_asr_backend("asr.auto")
|
|
520
|
+
self._timeout = timeout
|
|
521
|
+
|
|
522
|
+
def descriptor(self) -> BackendDescriptor:
|
|
523
|
+
capability = BackendCapability(
|
|
524
|
+
platforms=("bilibili",),
|
|
525
|
+
source_types=("video",),
|
|
526
|
+
input_types=("single",),
|
|
527
|
+
source_levels=("media_asr", "partial_extraction"),
|
|
528
|
+
requires_network=True,
|
|
529
|
+
optional_config=("enable_media_download_backends", "enable_asr_backends"),
|
|
530
|
+
)
|
|
531
|
+
return BackendDescriptor(
|
|
532
|
+
name=self.name,
|
|
533
|
+
layer="media_asr_fallback",
|
|
534
|
+
capabilities=(capability,),
|
|
535
|
+
description=(
|
|
536
|
+
"Bilibili no-subtitle fallback via temporary yt-dlp media download "
|
|
537
|
+
"and local ASR; registered safely but runs only with explicit gates."
|
|
538
|
+
),
|
|
539
|
+
provider="openmd.backends.bilibili",
|
|
540
|
+
is_fallback=True,
|
|
541
|
+
)
|
|
542
|
+
|
|
543
|
+
def health(self) -> BackendHealth:
|
|
544
|
+
if not (self._options.enable_media_download_backends and self._options.enable_asr_backends):
|
|
545
|
+
return BackendHealth(ok=True, warnings=("Bilibili media ASR fallback is not enabled",))
|
|
546
|
+
yt_dlp_ok = shutil.which("yt-dlp") is not None
|
|
547
|
+
is_available = getattr(self._asr_backend, "is_available", None)
|
|
548
|
+
asr_ok = bool(is_available()) if callable(is_available) else True
|
|
549
|
+
dependencies = {
|
|
550
|
+
"yt-dlp": "ok" if yt_dlp_ok else "missing",
|
|
551
|
+
getattr(self._asr_backend, "name", "asr_backend"): "ok" if asr_ok else "missing",
|
|
552
|
+
}
|
|
553
|
+
return BackendHealth(
|
|
554
|
+
ok=yt_dlp_ok and asr_ok,
|
|
555
|
+
dependencies=dependencies,
|
|
556
|
+
error_code=None if yt_dlp_ok and asr_ok else "bilibili_media_asr_dependency_missing",
|
|
557
|
+
error_summary=None if yt_dlp_ok and asr_ok else "yt-dlp or the configured ASR backend is missing.",
|
|
558
|
+
)
|
|
559
|
+
|
|
560
|
+
def extract(self, request: BackendExtractRequest) -> TranscriptResult:
|
|
561
|
+
item = request.item
|
|
562
|
+
bvid = extract_bvid(item.url)
|
|
563
|
+
if bvid is None:
|
|
564
|
+
return _failed_result(self.name, "invalid_input", "Bilibili BVID not found.")
|
|
565
|
+
|
|
566
|
+
missing_flags = _missing_media_asr_flags(self._options)
|
|
567
|
+
if missing_flags:
|
|
568
|
+
return _failed_result(
|
|
569
|
+
self.name,
|
|
570
|
+
"bilibili_media_asr_not_enabled",
|
|
571
|
+
"Bilibili no-subtitle ASR fallback requires explicit --enable-media-download and --enable-asr.",
|
|
572
|
+
title=item.title,
|
|
573
|
+
author=item.author,
|
|
574
|
+
published_at=item.published_at,
|
|
575
|
+
metadata={
|
|
576
|
+
"bvid": bvid,
|
|
577
|
+
"failure_class": "user_resolvable",
|
|
578
|
+
"missing_flags": missing_flags,
|
|
579
|
+
"media_retention": "none",
|
|
580
|
+
},
|
|
581
|
+
notes=[
|
|
582
|
+
"No media download or ASR action was attempted because the explicit fallback gates are closed.",
|
|
583
|
+
"Raw media is not retained by default; add --keep-media only for approved debugging.",
|
|
584
|
+
],
|
|
585
|
+
)
|
|
586
|
+
|
|
587
|
+
with _media_workspace(self._options) as workspace:
|
|
588
|
+
media_path = self._download_media(item, bvid, workspace.path)
|
|
589
|
+
if isinstance(media_path, TranscriptResult):
|
|
590
|
+
return media_path
|
|
591
|
+
|
|
592
|
+
transcribe = getattr(self._asr_backend, "transcribe", None)
|
|
593
|
+
if not callable(transcribe):
|
|
594
|
+
return _failed_result(
|
|
595
|
+
self.name,
|
|
596
|
+
"asr_backend_missing",
|
|
597
|
+
"Configured ASR backend does not expose transcribe(path).",
|
|
598
|
+
metadata={"bvid": bvid, "failure_class": "user_resolvable"},
|
|
599
|
+
)
|
|
600
|
+
asr_result = transcribe(media_path)
|
|
601
|
+
if not isinstance(asr_result, TranscriptResult) or not asr_result.has_usable_text:
|
|
602
|
+
return _failed_result(
|
|
603
|
+
self.name,
|
|
604
|
+
getattr(asr_result, "error_code", None) or "bilibili_media_asr_failed",
|
|
605
|
+
getattr(asr_result, "error_summary", None) or "Bilibili media ASR produced no usable text.",
|
|
606
|
+
title=item.title or getattr(asr_result, "title", None),
|
|
607
|
+
author=item.author,
|
|
608
|
+
published_at=item.published_at,
|
|
609
|
+
metadata={
|
|
610
|
+
"bvid": bvid,
|
|
611
|
+
"failure_class": "user_resolvable",
|
|
612
|
+
"asr_backend": getattr(self._asr_backend, "name", "unknown"),
|
|
613
|
+
"media_retention": workspace.retention,
|
|
614
|
+
},
|
|
615
|
+
notes=[
|
|
616
|
+
"Temporary media download was attempted under explicit operator gates.",
|
|
617
|
+
*_source_notes_from(asr_result),
|
|
618
|
+
],
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
metadata = dict(asr_result.metadata)
|
|
622
|
+
if workspace.retention != "kept":
|
|
623
|
+
metadata.pop("source_ref", None)
|
|
624
|
+
metadata.update(
|
|
625
|
+
{
|
|
626
|
+
"profile": "media",
|
|
627
|
+
"bvid": bvid,
|
|
628
|
+
"media_download_backend": BILIBILI_YTDLP_MEDIA_BACKEND,
|
|
629
|
+
"asr_backend": getattr(self._asr_backend, "name", asr_result.backend),
|
|
630
|
+
"media_retention": workspace.retention,
|
|
631
|
+
"raw_media_retained": workspace.retention == "kept",
|
|
632
|
+
}
|
|
633
|
+
)
|
|
634
|
+
return replace(
|
|
635
|
+
asr_result,
|
|
636
|
+
source_level="media_asr",
|
|
637
|
+
backend=getattr(self._asr_backend, "name", asr_result.backend),
|
|
638
|
+
title=item.title or asr_result.title,
|
|
639
|
+
author=item.author or asr_result.author,
|
|
640
|
+
published_at=item.published_at or asr_result.published_at,
|
|
641
|
+
source_notes=[
|
|
642
|
+
"Bilibili no-subtitle fallback used explicit temporary media download plus local ASR.",
|
|
643
|
+
"Raw media was retained only when --keep-media was set.",
|
|
644
|
+
*asr_result.source_notes,
|
|
645
|
+
],
|
|
646
|
+
metadata=metadata,
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
def _download_media(
|
|
650
|
+
self,
|
|
651
|
+
item: ContentItem,
|
|
652
|
+
bvid: str,
|
|
653
|
+
media_dir: Path,
|
|
654
|
+
) -> Path | TranscriptResult:
|
|
655
|
+
output_template = media_dir / f"{bvid}.%(ext)s"
|
|
656
|
+
command = [
|
|
657
|
+
"yt-dlp",
|
|
658
|
+
"--no-playlist",
|
|
659
|
+
"--extract-audio",
|
|
660
|
+
"--audio-format",
|
|
661
|
+
"wav",
|
|
662
|
+
"--output",
|
|
663
|
+
str(output_template),
|
|
664
|
+
item.url,
|
|
665
|
+
]
|
|
666
|
+
try:
|
|
667
|
+
exit_code, stdout, stderr = self._runner(command, self._timeout)
|
|
668
|
+
except FileNotFoundError as exc:
|
|
669
|
+
return _failed_result(
|
|
670
|
+
self.name,
|
|
671
|
+
"media_download_backend_missing",
|
|
672
|
+
exc,
|
|
673
|
+
metadata={"bvid": bvid, "failure_class": "user_resolvable"},
|
|
674
|
+
)
|
|
675
|
+
except subprocess.TimeoutExpired as exc:
|
|
676
|
+
return _failed_result(
|
|
677
|
+
self.name,
|
|
678
|
+
"media_download_timeout",
|
|
679
|
+
exc,
|
|
680
|
+
metadata={"bvid": bvid, "failure_class": "runtime_transient"},
|
|
681
|
+
)
|
|
682
|
+
except OSError as exc:
|
|
683
|
+
return _failed_result(
|
|
684
|
+
self.name,
|
|
685
|
+
"media_download_os_error",
|
|
686
|
+
exc,
|
|
687
|
+
metadata={"bvid": bvid, "failure_class": "runtime_transient"},
|
|
688
|
+
)
|
|
689
|
+
|
|
690
|
+
if exit_code != 0:
|
|
691
|
+
return _failed_result(
|
|
692
|
+
self.name,
|
|
693
|
+
f"media_download_exit_{exit_code}",
|
|
694
|
+
stderr or stdout or "yt-dlp media download failed.",
|
|
695
|
+
metadata={"bvid": bvid, "failure_class": "runtime_transient"},
|
|
696
|
+
)
|
|
697
|
+
|
|
698
|
+
media_path = _find_downloaded_media_file(media_dir, bvid)
|
|
699
|
+
if media_path is None:
|
|
700
|
+
return _failed_result(
|
|
701
|
+
self.name,
|
|
702
|
+
"media_download_empty",
|
|
703
|
+
"yt-dlp completed but no media file was found in the temporary workspace.",
|
|
704
|
+
metadata={"bvid": bvid, "failure_class": "runtime_transient"},
|
|
705
|
+
)
|
|
706
|
+
return media_path
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
class BilibiliNativeUserVideosProvider:
|
|
710
|
+
"""Discover public Bilibili user videos without extracting item subtitles."""
|
|
711
|
+
|
|
712
|
+
name = BILIBILI_NATIVE_DISCOVERY_BACKEND
|
|
713
|
+
|
|
714
|
+
def __init__(
|
|
715
|
+
self,
|
|
716
|
+
*,
|
|
717
|
+
fetcher: HttpFetcher | None = None,
|
|
718
|
+
timeout: int = 30,
|
|
719
|
+
wbi_signer: BilibiliWbiSigner | None = None,
|
|
720
|
+
) -> None:
|
|
721
|
+
self._fetcher = fetcher or _default_fetch
|
|
722
|
+
self._timeout = timeout
|
|
723
|
+
self._wbi_signer = wbi_signer or BilibiliWbiSigner(
|
|
724
|
+
fetcher=self._fetcher,
|
|
725
|
+
timeout=timeout,
|
|
726
|
+
)
|
|
727
|
+
|
|
728
|
+
def discover(self, request: DiscoveryRequest) -> DiscoveryResult:
|
|
729
|
+
uid = extract_bilibili_uid(request.input_url)
|
|
730
|
+
if uid is None:
|
|
731
|
+
return DiscoveryResult.failure(
|
|
732
|
+
provider=self.name,
|
|
733
|
+
error_code="invalid_input",
|
|
734
|
+
error_summary="Bilibili space UID not found for collection discovery.",
|
|
735
|
+
)
|
|
736
|
+
|
|
737
|
+
limit = _bounded_limit(request.limit)
|
|
738
|
+
params = {"mid": uid, "pn": "1", "ps": str(limit), "order": "pubdate"}
|
|
739
|
+
warnings: list[str] = []
|
|
740
|
+
endpoint_used = "x/space/arc/search"
|
|
741
|
+
videos: list[dict[str, Any]] = []
|
|
742
|
+
|
|
743
|
+
try:
|
|
744
|
+
signed_params = self._wbi_signer.sign_params(params, timeout=self._timeout)
|
|
745
|
+
signed_payload = _fetch_json(
|
|
746
|
+
self._fetcher,
|
|
747
|
+
"https://api.bilibili.com/x/space/wbi/arc/search",
|
|
748
|
+
signed_params,
|
|
749
|
+
self._timeout,
|
|
750
|
+
)
|
|
751
|
+
signed_data = _bilibili_api_data(signed_payload)
|
|
752
|
+
videos = _video_entries_from_space_payload(signed_data)
|
|
753
|
+
endpoint_used = "x/space/wbi/arc/search"
|
|
754
|
+
except (HTTPError, URLError, OSError, ValueError, WbiSigningError, json.JSONDecodeError) as exc:
|
|
755
|
+
warnings.append(f"x/space/wbi/arc/search: {sanitize_error_summary(exc)}")
|
|
756
|
+
|
|
757
|
+
if not videos:
|
|
758
|
+
if endpoint_used == "x/space/wbi/arc/search":
|
|
759
|
+
warnings.append("x/space/wbi/arc/search: no usable video rows")
|
|
760
|
+
try:
|
|
761
|
+
payload = _fetch_json(
|
|
762
|
+
self._fetcher,
|
|
763
|
+
"https://api.bilibili.com/x/space/arc/search",
|
|
764
|
+
params,
|
|
765
|
+
self._timeout,
|
|
766
|
+
)
|
|
767
|
+
data = _bilibili_api_data(payload)
|
|
768
|
+
videos = _video_entries_from_space_payload(data)
|
|
769
|
+
endpoint_used = "x/space/arc/search"
|
|
770
|
+
except (HTTPError, URLError, OSError, ValueError, json.JSONDecodeError) as exc:
|
|
771
|
+
warnings.append(f"x/space/arc/search: {sanitize_error_summary(exc)}")
|
|
772
|
+
|
|
773
|
+
if not videos:
|
|
774
|
+
return DiscoveryResult.failure(
|
|
775
|
+
provider=self.name,
|
|
776
|
+
error_code="bilibili_discovery_fetch_failed",
|
|
777
|
+
error_summary=sanitize_error_summary("; ".join(warnings) or "Bilibili discovery returned no usable rows."),
|
|
778
|
+
metadata={"uid": uid, "limit": limit, "endpoint_used": endpoint_used},
|
|
779
|
+
)
|
|
780
|
+
|
|
781
|
+
items = [_content_item_from_video_entry(entry, request, uid) for entry in videos]
|
|
782
|
+
items = [item for item in items if item.url][:limit]
|
|
783
|
+
if not items:
|
|
784
|
+
return DiscoveryResult.failure(
|
|
785
|
+
provider=self.name,
|
|
786
|
+
error_code="empty_result",
|
|
787
|
+
error_summary="Bilibili user videos discovery returned no usable video rows.",
|
|
788
|
+
metadata={"uid": uid, "limit": limit},
|
|
789
|
+
)
|
|
790
|
+
|
|
791
|
+
return DiscoveryResult.success(
|
|
792
|
+
provider=self.name,
|
|
793
|
+
items=items,
|
|
794
|
+
source_notes=[
|
|
795
|
+
"Bilibili public user videos API discovery only; item extraction was not attempted.",
|
|
796
|
+
f"discovery_endpoint_used={endpoint_used}",
|
|
797
|
+
"Discovery metadata was normalized without persisting raw API responses.",
|
|
798
|
+
(
|
|
799
|
+
"Bilibili UP discovery tries signed x/space/wbi/arc/search before "
|
|
800
|
+
"falling back to public x/space/arc/search; callers can use the "
|
|
801
|
+
"OpenCLI fallback when native public discovery is unavailable."
|
|
802
|
+
),
|
|
803
|
+
*_warning_notes(warnings),
|
|
804
|
+
],
|
|
805
|
+
metadata={
|
|
806
|
+
"uid": uid,
|
|
807
|
+
"limit": limit,
|
|
808
|
+
"discovery_endpoint_used": endpoint_used,
|
|
809
|
+
"normalized_url": f"https://space.bilibili.com/{uid}",
|
|
810
|
+
"stability_note": "Tries signed x/space/wbi/arc/search first, then public x/space/arc/search fallback.",
|
|
811
|
+
},
|
|
812
|
+
)
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
class OpenCliBilibiliUserVideosProvider:
|
|
816
|
+
"""Controlled OpenCLI fallback for Bilibili UP video discovery."""
|
|
817
|
+
|
|
818
|
+
name = BILIBILI_OPENCLI_DISCOVERY_BACKEND
|
|
819
|
+
|
|
820
|
+
def __init__(
|
|
821
|
+
self,
|
|
822
|
+
*,
|
|
823
|
+
runner: CommandRunner | None = None,
|
|
824
|
+
timeout: int = 120,
|
|
825
|
+
) -> None:
|
|
826
|
+
self._runner = runner or _run_command
|
|
827
|
+
self._timeout = timeout
|
|
828
|
+
|
|
829
|
+
def discover(self, request: DiscoveryRequest) -> DiscoveryResult:
|
|
830
|
+
uid = extract_bilibili_uid(request.input_url)
|
|
831
|
+
if uid is None:
|
|
832
|
+
return DiscoveryResult.failure(
|
|
833
|
+
provider=self.name,
|
|
834
|
+
error_code="invalid_input",
|
|
835
|
+
error_summary="Bilibili space UID not found for OpenCLI discovery.",
|
|
836
|
+
)
|
|
837
|
+
|
|
838
|
+
limit = _bounded_limit(request.limit)
|
|
839
|
+
try:
|
|
840
|
+
exit_code, stdout, stderr = self._runner(
|
|
841
|
+
["opencli", "bilibili", "user-videos", uid, "--limit", str(limit)],
|
|
842
|
+
self._timeout,
|
|
843
|
+
)
|
|
844
|
+
except FileNotFoundError as exc:
|
|
845
|
+
return DiscoveryResult.failure(
|
|
846
|
+
provider=self.name,
|
|
847
|
+
error_code="backend_not_found",
|
|
848
|
+
error_summary=sanitize_error_summary(exc),
|
|
849
|
+
)
|
|
850
|
+
except subprocess.TimeoutExpired as exc:
|
|
851
|
+
return DiscoveryResult.failure(
|
|
852
|
+
provider=self.name,
|
|
853
|
+
error_code="backend_timeout",
|
|
854
|
+
error_summary=sanitize_error_summary(exc),
|
|
855
|
+
)
|
|
856
|
+
except OSError as exc:
|
|
857
|
+
return DiscoveryResult.failure(
|
|
858
|
+
provider=self.name,
|
|
859
|
+
error_code="backend_os_error",
|
|
860
|
+
error_summary=sanitize_error_summary(exc),
|
|
861
|
+
)
|
|
862
|
+
|
|
863
|
+
if exit_code != 0:
|
|
864
|
+
return DiscoveryResult.failure(
|
|
865
|
+
provider=self.name,
|
|
866
|
+
error_code=f"backend_exit_{exit_code}",
|
|
867
|
+
error_summary=sanitize_error_summary(stderr or "OpenCLI bilibili user-videos failed."),
|
|
868
|
+
)
|
|
869
|
+
|
|
870
|
+
rows = parse_opencli_bilibili_user_videos(stdout)
|
|
871
|
+
items = [_content_item_from_opencli_row(row, request, uid) for row in rows]
|
|
872
|
+
items = [item for item in items if item.url][:limit]
|
|
873
|
+
if not items:
|
|
874
|
+
return DiscoveryResult.failure(
|
|
875
|
+
provider=self.name,
|
|
876
|
+
error_code="empty_result",
|
|
877
|
+
error_summary="OpenCLI bilibili user-videos returned no parseable video rows.",
|
|
878
|
+
metadata={"uid": uid, "limit": limit},
|
|
879
|
+
)
|
|
880
|
+
|
|
881
|
+
return DiscoveryResult.success(
|
|
882
|
+
provider=self.name,
|
|
883
|
+
items=items,
|
|
884
|
+
source_notes=[
|
|
885
|
+
"OpenCLI bilibili user-videos discovery only; item extraction was not attempted.",
|
|
886
|
+
"Raw stdout/stderr was parsed in memory and not persisted.",
|
|
887
|
+
],
|
|
888
|
+
metadata={"uid": uid, "limit": limit, "normalized_url": f"https://space.bilibili.com/{uid}"},
|
|
889
|
+
)
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def extract_bvid(value: str) -> str | None:
|
|
893
|
+
match = _BVID_RE.search(extract_first_url(value)) or _BVID_RE.search(value)
|
|
894
|
+
return match.group(1) if match else None
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
def extract_bilibili_uid(value: str) -> str | None:
|
|
898
|
+
url = extract_first_url(value)
|
|
899
|
+
parsed = urlparse(url if "://" in url else f"https://{url}")
|
|
900
|
+
host_path = f"{parsed.netloc}{parsed.path}"
|
|
901
|
+
match = _SPACE_UID_RE.search(host_path)
|
|
902
|
+
if match:
|
|
903
|
+
return match.group(1)
|
|
904
|
+
if value.strip().isdigit():
|
|
905
|
+
return value.strip()
|
|
906
|
+
return None
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
def parse_bilibili_subtitle_segments(text: str) -> list[_SubtitleSegment]:
|
|
910
|
+
rows = _parse_json_subtitle_rows(text) or _parse_tabular_rows(text)
|
|
911
|
+
return _segments_from_rows(rows)
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
def parse_opencli_bilibili_user_videos(text: str) -> list[dict[str, str]]:
|
|
915
|
+
rows = _parse_json_subtitle_rows(text) or _parse_tabular_rows(text)
|
|
916
|
+
return [row for row in rows if _row_lookup(row, "url") or extract_bvid(str(row))]
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def _missing_media_asr_flags(options: BackendRuntimeOptions) -> list[str]:
|
|
920
|
+
missing: list[str] = []
|
|
921
|
+
if not options.enable_media_download_backends:
|
|
922
|
+
missing.append("--enable-media-download")
|
|
923
|
+
if not options.enable_asr_backends:
|
|
924
|
+
missing.append("--enable-asr")
|
|
925
|
+
return missing
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
@contextmanager
|
|
929
|
+
def _media_workspace(options: BackendRuntimeOptions):
|
|
930
|
+
if options.keep_media_artifacts:
|
|
931
|
+
media_dir = Path(options.media_artifact_dir or "out/raw-media")
|
|
932
|
+
media_dir.mkdir(parents=True, exist_ok=True)
|
|
933
|
+
yield _MediaWorkspace(path=media_dir, retention="kept")
|
|
934
|
+
return
|
|
935
|
+
|
|
936
|
+
with owned_temporary_directory(prefix="openmd-bilibili-asr-") as tmp_dir:
|
|
937
|
+
yield _MediaWorkspace(path=Path(tmp_dir), retention="deleted")
|
|
938
|
+
|
|
939
|
+
|
|
940
|
+
def _find_downloaded_media_file(media_dir: Path, bvid: str) -> Path | None:
|
|
941
|
+
suffix_rank = {
|
|
942
|
+
".wav": 0,
|
|
943
|
+
".m4a": 1,
|
|
944
|
+
".mp3": 2,
|
|
945
|
+
".opus": 3,
|
|
946
|
+
".aac": 4,
|
|
947
|
+
".flac": 5,
|
|
948
|
+
".mp4": 6,
|
|
949
|
+
}
|
|
950
|
+
candidates = [path for path in media_dir.iterdir() if path.is_file()]
|
|
951
|
+
if not candidates:
|
|
952
|
+
return None
|
|
953
|
+
matched = [path for path in candidates if bvid in path.name] or candidates
|
|
954
|
+
return sorted(
|
|
955
|
+
matched,
|
|
956
|
+
key=lambda path: (suffix_rank.get(path.suffix.lower(), 100), path.name),
|
|
957
|
+
)[0]
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def _source_notes_from(value: object) -> list[str]:
|
|
961
|
+
if isinstance(value, TranscriptResult):
|
|
962
|
+
return list(value.source_notes)
|
|
963
|
+
return []
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _default_fetch(url: str, params: dict[str, str], timeout: int) -> tuple[int, str]:
|
|
967
|
+
query = urlencode(params)
|
|
968
|
+
full_url = f"{url}?{query}" if query else url
|
|
969
|
+
request = Request(
|
|
970
|
+
full_url,
|
|
971
|
+
headers=bilibili_headers(cookie_header=bilibili_cookie_header_for_url(url)),
|
|
972
|
+
)
|
|
973
|
+
with urlopen(request, timeout=timeout) as response:
|
|
974
|
+
raw = response.read().decode("utf-8", errors="replace")
|
|
975
|
+
return int(getattr(response, "status", 200)), raw
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
def _run_command(command: list[str], timeout: int) -> tuple[int, str, str]:
|
|
979
|
+
resolved = shutil.which(command[0]) or command[0]
|
|
980
|
+
completed = run_interruptibly(
|
|
981
|
+
[resolved, *command[1:]],
|
|
982
|
+
capture_output=True,
|
|
983
|
+
text=True,
|
|
984
|
+
timeout=timeout,
|
|
985
|
+
check=False,
|
|
986
|
+
encoding="utf-8",
|
|
987
|
+
errors="replace",
|
|
988
|
+
)
|
|
989
|
+
return completed.returncode, completed.stdout or "", completed.stderr or ""
|
|
990
|
+
|
|
991
|
+
|
|
992
|
+
def _fetch_json(
|
|
993
|
+
fetcher: HttpFetcher,
|
|
994
|
+
url: str,
|
|
995
|
+
params: dict[str, str],
|
|
996
|
+
timeout: int,
|
|
997
|
+
) -> Any:
|
|
998
|
+
status, body = fetcher(url, params, timeout)
|
|
999
|
+
if status < 200 or status >= 300:
|
|
1000
|
+
raise ValueError(f"Bilibili request failed with HTTP {status}.")
|
|
1001
|
+
return json.loads(body)
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
def _bilibili_api_data(payload: Any) -> dict[str, Any]:
|
|
1005
|
+
if not isinstance(payload, dict):
|
|
1006
|
+
raise ValueError("Bilibili API payload was not a JSON object.")
|
|
1007
|
+
code = payload.get("code", 0)
|
|
1008
|
+
if code not in {0, "0", None}:
|
|
1009
|
+
message = _safe_str(payload.get("message") or payload.get("msg") or code)
|
|
1010
|
+
raise ValueError(f"Bilibili API returned code {code}: {message}")
|
|
1011
|
+
data = payload.get("data")
|
|
1012
|
+
if not isinstance(data, dict):
|
|
1013
|
+
raise ValueError("Bilibili API payload did not include an object data field.")
|
|
1014
|
+
return data
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
def _subtitle_entries_from_player(player_data: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1018
|
+
subtitle = player_data.get("subtitle")
|
|
1019
|
+
if not isinstance(subtitle, dict):
|
|
1020
|
+
return []
|
|
1021
|
+
entries = subtitle.get("subtitles")
|
|
1022
|
+
if not isinstance(entries, list):
|
|
1023
|
+
return []
|
|
1024
|
+
return [
|
|
1025
|
+
entry
|
|
1026
|
+
for entry in entries
|
|
1027
|
+
if isinstance(entry, dict) and not _is_danmaku_subtitle_entry(entry)
|
|
1028
|
+
]
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def _choose_subtitle(entries: list[dict[str, Any]]) -> dict[str, Any]:
|
|
1032
|
+
dedup: dict[str, dict[str, Any]] = {}
|
|
1033
|
+
for entry in entries:
|
|
1034
|
+
lang = _normalize_lang(_safe_str(entry.get("lan") or entry.get("lan_doc")))
|
|
1035
|
+
key = lang.lower()
|
|
1036
|
+
existing = dedup.get(key)
|
|
1037
|
+
if existing is None or (
|
|
1038
|
+
_subtitle_track_type(existing) == "ai" and _subtitle_track_type(entry) == "uploader"
|
|
1039
|
+
):
|
|
1040
|
+
dedup[key] = entry
|
|
1041
|
+
|
|
1042
|
+
candidates = list(dedup.values()) or entries
|
|
1043
|
+
for preferred in ("zh-CN", "zh-Hans", "zh", "ai-zh"):
|
|
1044
|
+
for entry in candidates:
|
|
1045
|
+
if _normalize_lang(_safe_str(entry.get("lan") or entry.get("lan_doc"))).lower() == preferred.lower():
|
|
1046
|
+
return entry
|
|
1047
|
+
|
|
1048
|
+
zh_like = [
|
|
1049
|
+
entry
|
|
1050
|
+
for entry in candidates
|
|
1051
|
+
if _normalize_lang(_safe_str(entry.get("lan") or entry.get("lan_doc"))).lower().startswith(("zh", "ai-zh"))
|
|
1052
|
+
]
|
|
1053
|
+
if zh_like:
|
|
1054
|
+
return sorted(zh_like, key=_subtitle_sort_key)[0]
|
|
1055
|
+
return sorted(candidates, key=_subtitle_sort_key)[0]
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
def _normalize_subtitle_url(value: str | None) -> str | None:
|
|
1059
|
+
if not value:
|
|
1060
|
+
return None
|
|
1061
|
+
if value.startswith("//"):
|
|
1062
|
+
return f"https:{value}"
|
|
1063
|
+
if value.startswith("http://") or value.startswith("https://"):
|
|
1064
|
+
return value
|
|
1065
|
+
if value.startswith("/"):
|
|
1066
|
+
return f"https://api.bilibili.com{value}"
|
|
1067
|
+
return None
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
def _player_subtitle_attempts(
|
|
1071
|
+
*,
|
|
1072
|
+
aid: str,
|
|
1073
|
+
cid: str,
|
|
1074
|
+
bvid: str,
|
|
1075
|
+
) -> list[tuple[str, str, dict[str, str]]]:
|
|
1076
|
+
attempts: list[tuple[str, str, dict[str, str]]] = []
|
|
1077
|
+
if aid:
|
|
1078
|
+
attempts.append(
|
|
1079
|
+
(
|
|
1080
|
+
"wbi_v2",
|
|
1081
|
+
"https://api.bilibili.com/x/player/wbi/v2",
|
|
1082
|
+
{"aid": aid, "cid": cid},
|
|
1083
|
+
)
|
|
1084
|
+
)
|
|
1085
|
+
attempts.append(
|
|
1086
|
+
(
|
|
1087
|
+
"v2",
|
|
1088
|
+
"https://api.bilibili.com/x/player/v2",
|
|
1089
|
+
{"cid": cid, "bvid": bvid},
|
|
1090
|
+
)
|
|
1091
|
+
)
|
|
1092
|
+
return attempts
|
|
1093
|
+
|
|
1094
|
+
|
|
1095
|
+
def _normalize_lang(raw_lang: str) -> str:
|
|
1096
|
+
raw = raw_lang.strip()
|
|
1097
|
+
if not raw:
|
|
1098
|
+
return "unknown"
|
|
1099
|
+
lower = raw.lower()
|
|
1100
|
+
mapping = {
|
|
1101
|
+
"zh-cn": "zh-CN",
|
|
1102
|
+
"zh-hans": "zh-Hans",
|
|
1103
|
+
"zh-hant": "zh-Hant",
|
|
1104
|
+
"zh-tw": "zh-Hant",
|
|
1105
|
+
"zh-hk": "zh-Hant",
|
|
1106
|
+
"zh": "zh",
|
|
1107
|
+
"ai-zh": "ai-zh",
|
|
1108
|
+
}
|
|
1109
|
+
if lower in mapping:
|
|
1110
|
+
return mapping[lower]
|
|
1111
|
+
if lower.startswith("ai-zh"):
|
|
1112
|
+
return "ai-zh"
|
|
1113
|
+
if lower.startswith("zh-cn"):
|
|
1114
|
+
return "zh-CN"
|
|
1115
|
+
if lower.startswith("zh-hans"):
|
|
1116
|
+
return "zh-Hans"
|
|
1117
|
+
if lower.startswith("zh"):
|
|
1118
|
+
return "zh"
|
|
1119
|
+
return raw
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def _is_danmaku_subtitle_entry(entry: dict[str, Any]) -> bool:
|
|
1123
|
+
raw_lang = _safe_str(entry.get("lan") or entry.get("lan_doc")).lower()
|
|
1124
|
+
if raw_lang == "danmaku":
|
|
1125
|
+
return True
|
|
1126
|
+
subtitle_url = _normalize_subtitle_url(_safe_str(entry.get("subtitle_url")))
|
|
1127
|
+
if not subtitle_url:
|
|
1128
|
+
return False
|
|
1129
|
+
return urlparse(subtitle_url).path.lower().endswith(".xml")
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
def _subtitle_track_type(entry: dict[str, Any]) -> str:
|
|
1133
|
+
raw_lang = _safe_str(entry.get("lan"))
|
|
1134
|
+
values = " ".join(_safe_str(value) for value in entry.values())
|
|
1135
|
+
ai_status = entry.get("ai_status")
|
|
1136
|
+
if (
|
|
1137
|
+
raw_lang.lower().startswith("ai-")
|
|
1138
|
+
or bool(entry.get("ai_type"))
|
|
1139
|
+
or ai_status not in {None, 0, "0", False}
|
|
1140
|
+
or _AI_CAPTION_HINT_RE.search(values)
|
|
1141
|
+
):
|
|
1142
|
+
return "ai"
|
|
1143
|
+
return "uploader"
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def _subtitle_sort_key(entry: dict[str, Any]) -> tuple[int, str]:
|
|
1147
|
+
source_rank = 0 if _subtitle_track_type(entry) == "uploader" else 1
|
|
1148
|
+
return source_rank, _normalize_lang(_safe_str(entry.get("lan") or entry.get("lan_doc"))).lower()
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
def _resolve_ai_subtitle_url(
|
|
1152
|
+
fetcher: HttpFetcher,
|
|
1153
|
+
*,
|
|
1154
|
+
aid: str,
|
|
1155
|
+
cid: str,
|
|
1156
|
+
timeout: int,
|
|
1157
|
+
) -> str | None:
|
|
1158
|
+
if not aid or not cid:
|
|
1159
|
+
return None
|
|
1160
|
+
payload = _fetch_json(
|
|
1161
|
+
fetcher,
|
|
1162
|
+
"https://api.bilibili.com/x/player/v2/ai/subtitle/search/stat",
|
|
1163
|
+
{"aid": aid, "cid": cid},
|
|
1164
|
+
timeout,
|
|
1165
|
+
)
|
|
1166
|
+
data = _bilibili_api_data(payload)
|
|
1167
|
+
return _normalize_subtitle_url(_safe_str(data.get("subtitle_url")))
|
|
1168
|
+
|
|
1169
|
+
|
|
1170
|
+
def _segments_from_subtitle_payload(payload: Any) -> list[_SubtitleSegment]:
|
|
1171
|
+
if isinstance(payload, dict):
|
|
1172
|
+
body = payload.get("body") or payload.get("segments") or payload.get("data")
|
|
1173
|
+
else:
|
|
1174
|
+
body = payload
|
|
1175
|
+
if not isinstance(body, list):
|
|
1176
|
+
return []
|
|
1177
|
+
return _segments_from_rows([row for row in body if isinstance(row, dict)])
|
|
1178
|
+
|
|
1179
|
+
|
|
1180
|
+
def _parse_json_subtitle_rows(text: str) -> list[dict[str, Any]]:
|
|
1181
|
+
try:
|
|
1182
|
+
payload = load_opencli_json(text)
|
|
1183
|
+
except json.JSONDecodeError:
|
|
1184
|
+
return []
|
|
1185
|
+
if isinstance(payload, dict):
|
|
1186
|
+
body = payload.get("body") or payload.get("segments") or payload.get("data")
|
|
1187
|
+
if isinstance(body, list):
|
|
1188
|
+
return [row for row in body if isinstance(row, dict)]
|
|
1189
|
+
return [payload]
|
|
1190
|
+
if isinstance(payload, list):
|
|
1191
|
+
return [row for row in payload if isinstance(row, dict)]
|
|
1192
|
+
return []
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
def _parse_tabular_rows(text: str) -> list[dict[str, str]]:
|
|
1196
|
+
return _parse_console_or_markdown_table(text) or _parse_csv_rows(text)
|
|
1197
|
+
|
|
1198
|
+
|
|
1199
|
+
def _parse_console_or_markdown_table(text: str) -> list[dict[str, str]]:
|
|
1200
|
+
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
|
1201
|
+
header: list[str] | None = None
|
|
1202
|
+
rows: list[dict[str, str]] = []
|
|
1203
|
+
delimiter = "\u2502" if any("\u2502" in line for line in lines) else "|"
|
|
1204
|
+
|
|
1205
|
+
for line in lines:
|
|
1206
|
+
if delimiter not in line or set(line) <= {
|
|
1207
|
+
delimiter,
|
|
1208
|
+
"-",
|
|
1209
|
+
"+",
|
|
1210
|
+
"\u2500",
|
|
1211
|
+
"\u250c",
|
|
1212
|
+
"\u2510",
|
|
1213
|
+
"\u2514",
|
|
1214
|
+
"\u2518",
|
|
1215
|
+
"\u252c",
|
|
1216
|
+
"\u2534",
|
|
1217
|
+
"\u253c",
|
|
1218
|
+
" ",
|
|
1219
|
+
}:
|
|
1220
|
+
continue
|
|
1221
|
+
cells = [cell.strip().strip("'\"") for cell in line.strip(delimiter).split(delimiter)]
|
|
1222
|
+
cells = [cell for cell in cells if cell or delimiter == "|"]
|
|
1223
|
+
if not cells:
|
|
1224
|
+
continue
|
|
1225
|
+
normalized = [_normalize_header(cell) for cell in cells]
|
|
1226
|
+
if any(cell in {"from", "to", "content", "title", "url"} for cell in normalized):
|
|
1227
|
+
header = normalized
|
|
1228
|
+
continue
|
|
1229
|
+
if header is None or len(cells) < len(header):
|
|
1230
|
+
continue
|
|
1231
|
+
row = {header[index]: cells[index].strip() for index in range(len(header))}
|
|
1232
|
+
if any(value for value in row.values()) and set("".join(row.values())) != {"-"}:
|
|
1233
|
+
rows.append(row)
|
|
1234
|
+
return rows
|
|
1235
|
+
|
|
1236
|
+
|
|
1237
|
+
def _parse_csv_rows(text: str) -> list[dict[str, str]]:
|
|
1238
|
+
try:
|
|
1239
|
+
reader = csv.DictReader(io.StringIO(text))
|
|
1240
|
+
if not reader.fieldnames:
|
|
1241
|
+
return []
|
|
1242
|
+
fieldnames = [_normalize_header(field or "") for field in reader.fieldnames]
|
|
1243
|
+
rows: list[dict[str, str]] = []
|
|
1244
|
+
for raw_row in reader:
|
|
1245
|
+
row = {
|
|
1246
|
+
fieldnames[index]: (raw_row.get(reader.fieldnames[index] or "") or "").strip()
|
|
1247
|
+
for index in range(len(fieldnames))
|
|
1248
|
+
}
|
|
1249
|
+
if any(row.values()):
|
|
1250
|
+
rows.append(row)
|
|
1251
|
+
return rows
|
|
1252
|
+
except csv.Error:
|
|
1253
|
+
return []
|
|
1254
|
+
|
|
1255
|
+
|
|
1256
|
+
def _segments_from_rows(rows: list[dict[str, Any]]) -> list[_SubtitleSegment]:
|
|
1257
|
+
segments: list[_SubtitleSegment] = []
|
|
1258
|
+
for row in rows:
|
|
1259
|
+
start = _coerce_seconds(_row_lookup(row, "from", "start", "begin"))
|
|
1260
|
+
end = _coerce_seconds(_row_lookup(row, "to", "end"))
|
|
1261
|
+
content = _safe_str(_row_lookup(row, "content", "text", "body"))
|
|
1262
|
+
if start is None or not content:
|
|
1263
|
+
continue
|
|
1264
|
+
segments.append(_SubtitleSegment(start=start, end=end, content=content))
|
|
1265
|
+
return segments
|
|
1266
|
+
|
|
1267
|
+
|
|
1268
|
+
def _format_segments(segments: list[_SubtitleSegment]) -> str:
|
|
1269
|
+
return "".join(f"[{_format_timestamp(segment.start)}] {segment.content.strip()}\n" for segment in segments)
|
|
1270
|
+
|
|
1271
|
+
|
|
1272
|
+
def _format_timestamp(seconds: float) -> str:
|
|
1273
|
+
total = max(0, int(seconds))
|
|
1274
|
+
hours, remainder = divmod(total, 3600)
|
|
1275
|
+
minutes, secs = divmod(remainder, 60)
|
|
1276
|
+
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
def _coerce_seconds(value: object) -> float | None:
|
|
1280
|
+
text = _safe_str(value)
|
|
1281
|
+
if not text:
|
|
1282
|
+
return None
|
|
1283
|
+
text = text.strip().strip("s\u79d2")
|
|
1284
|
+
if ":" in text:
|
|
1285
|
+
parts = text.split(":")
|
|
1286
|
+
try:
|
|
1287
|
+
numbers = [float(part) for part in parts]
|
|
1288
|
+
except ValueError:
|
|
1289
|
+
return None
|
|
1290
|
+
if len(numbers) == 3:
|
|
1291
|
+
return numbers[0] * 3600 + numbers[1] * 60 + numbers[2]
|
|
1292
|
+
if len(numbers) == 2:
|
|
1293
|
+
return numbers[0] * 60 + numbers[1]
|
|
1294
|
+
try:
|
|
1295
|
+
return float(text)
|
|
1296
|
+
except ValueError:
|
|
1297
|
+
return None
|
|
1298
|
+
|
|
1299
|
+
|
|
1300
|
+
def _source_level_from_subtitle_meta(meta: dict[str, Any]) -> SourceLevel:
|
|
1301
|
+
if _subtitle_track_type(meta) == "ai":
|
|
1302
|
+
return "auto_caption"
|
|
1303
|
+
values = " ".join(_safe_str(value) for value in meta.values())
|
|
1304
|
+
if _NATIVE_CAPTION_HINT_RE.search(values):
|
|
1305
|
+
return "native_caption"
|
|
1306
|
+
return "native_caption"
|
|
1307
|
+
|
|
1308
|
+
|
|
1309
|
+
def _source_level_from_text(text: str, *, default: SourceLevel) -> SourceLevel:
|
|
1310
|
+
if _AI_CAPTION_HINT_RE.search(text):
|
|
1311
|
+
return "auto_caption"
|
|
1312
|
+
if _NATIVE_CAPTION_HINT_RE.search(text):
|
|
1313
|
+
return "native_caption"
|
|
1314
|
+
return default
|
|
1315
|
+
|
|
1316
|
+
|
|
1317
|
+
def _source_level_note(source_level: SourceLevel) -> str:
|
|
1318
|
+
if source_level == "auto_caption":
|
|
1319
|
+
return "source_level=auto_caption: Bilibili metadata/output indicated AI or automatic captions, or OpenCLI did not expose manual-caption metadata."
|
|
1320
|
+
if source_level == "native_caption":
|
|
1321
|
+
return "source_level=native_caption: Bilibili metadata/output indicated native/manual captions."
|
|
1322
|
+
return "source_level=partial_extraction: no usable caption text was produced."
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _video_entries_from_space_payload(data: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1326
|
+
listing = data.get("list")
|
|
1327
|
+
if isinstance(listing, dict) and isinstance(listing.get("vlist"), list):
|
|
1328
|
+
return [entry for entry in listing["vlist"] if isinstance(entry, dict)]
|
|
1329
|
+
if isinstance(data.get("vlist"), list):
|
|
1330
|
+
return [entry for entry in data["vlist"] if isinstance(entry, dict)]
|
|
1331
|
+
return []
|
|
1332
|
+
|
|
1333
|
+
|
|
1334
|
+
def _content_item_from_video_entry(
|
|
1335
|
+
entry: dict[str, Any],
|
|
1336
|
+
request: DiscoveryRequest,
|
|
1337
|
+
uid: str,
|
|
1338
|
+
) -> ContentItem:
|
|
1339
|
+
bvid = _safe_str(entry.get("bvid")) or extract_bvid(_safe_str(entry.get("arcurl")))
|
|
1340
|
+
url = _safe_str(entry.get("arcurl")) or (f"https://www.bilibili.com/video/{bvid}/" if bvid else "")
|
|
1341
|
+
return ContentItem(
|
|
1342
|
+
platform="bilibili",
|
|
1343
|
+
input_type="single",
|
|
1344
|
+
url=url,
|
|
1345
|
+
id=bvid,
|
|
1346
|
+
title=_safe_str(entry.get("title")),
|
|
1347
|
+
author=_safe_str(entry.get("author")) or None,
|
|
1348
|
+
published_at=_format_unix_timestamp(entry.get("created")),
|
|
1349
|
+
source_type="video",
|
|
1350
|
+
raw_meta={
|
|
1351
|
+
"collection_url": request.input_url,
|
|
1352
|
+
"collection_uid": uid,
|
|
1353
|
+
"discovery_limit": request.limit,
|
|
1354
|
+
"discovery_provider": BILIBILI_NATIVE_DISCOVERY_BACKEND,
|
|
1355
|
+
},
|
|
1356
|
+
)
|
|
1357
|
+
|
|
1358
|
+
|
|
1359
|
+
def _content_item_from_opencli_row(
|
|
1360
|
+
row: dict[str, str],
|
|
1361
|
+
request: DiscoveryRequest,
|
|
1362
|
+
uid: str,
|
|
1363
|
+
) -> ContentItem:
|
|
1364
|
+
row_text = " ".join(row.values())
|
|
1365
|
+
url = _row_lookup(row, "url") or _safe_str(re.search(r"https?://\S+", row_text).group(0) if re.search(r"https?://\S+", row_text) else "")
|
|
1366
|
+
bvid = extract_bvid(url or row_text)
|
|
1367
|
+
if not url and bvid:
|
|
1368
|
+
url = f"https://www.bilibili.com/video/{bvid}/"
|
|
1369
|
+
raw_meta = {
|
|
1370
|
+
"collection_url": request.input_url,
|
|
1371
|
+
"collection_uid": uid,
|
|
1372
|
+
"discovery_limit": request.limit,
|
|
1373
|
+
"discovery_provider": BILIBILI_OPENCLI_DISCOVERY_BACKEND,
|
|
1374
|
+
}
|
|
1375
|
+
for key in ("rank", "plays", "likes"):
|
|
1376
|
+
value = _row_lookup(row, key)
|
|
1377
|
+
if value:
|
|
1378
|
+
raw_meta[key] = value
|
|
1379
|
+
return ContentItem(
|
|
1380
|
+
platform="bilibili",
|
|
1381
|
+
input_type="single",
|
|
1382
|
+
url=url,
|
|
1383
|
+
id=bvid,
|
|
1384
|
+
title=_row_lookup(row, "title") or None,
|
|
1385
|
+
published_at=_row_lookup(row, "date", "published_at") or None,
|
|
1386
|
+
source_type="video",
|
|
1387
|
+
raw_meta=raw_meta,
|
|
1388
|
+
)
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
def _bounded_limit(limit: int | None) -> int:
|
|
1392
|
+
if limit is None:
|
|
1393
|
+
return 20
|
|
1394
|
+
return max(0, min(limit, 50))
|
|
1395
|
+
|
|
1396
|
+
|
|
1397
|
+
def _normalize_header(value: str) -> str:
|
|
1398
|
+
normalized = value.strip().lower().replace(" ", "_").replace("-", "_")
|
|
1399
|
+
aliases = {
|
|
1400
|
+
"(index)": "index",
|
|
1401
|
+
"index": "index",
|
|
1402
|
+
"from": "from",
|
|
1403
|
+
"start": "start",
|
|
1404
|
+
"to": "to",
|
|
1405
|
+
"end": "end",
|
|
1406
|
+
"content": "content",
|
|
1407
|
+
"text": "text",
|
|
1408
|
+
"rank": "rank",
|
|
1409
|
+
"title": "title",
|
|
1410
|
+
"plays": "plays",
|
|
1411
|
+
"likes": "likes",
|
|
1412
|
+
"date": "date",
|
|
1413
|
+
"url": "url",
|
|
1414
|
+
}
|
|
1415
|
+
return aliases.get(normalized, normalized)
|
|
1416
|
+
|
|
1417
|
+
|
|
1418
|
+
def _row_lookup(row: dict[str, Any], *keys: str) -> str:
|
|
1419
|
+
normalized = {_normalize_header(str(key)): value for key, value in row.items()}
|
|
1420
|
+
for key in keys:
|
|
1421
|
+
value = normalized.get(_normalize_header(key))
|
|
1422
|
+
text = _safe_str(value)
|
|
1423
|
+
if text:
|
|
1424
|
+
return text
|
|
1425
|
+
return ""
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
def _owner_name(view_data: dict[str, Any]) -> str | None:
|
|
1429
|
+
owner = view_data.get("owner")
|
|
1430
|
+
if isinstance(owner, dict):
|
|
1431
|
+
return _safe_str(owner.get("name")) or None
|
|
1432
|
+
return None
|
|
1433
|
+
|
|
1434
|
+
|
|
1435
|
+
def _format_unix_timestamp(value: object) -> str | None:
|
|
1436
|
+
try:
|
|
1437
|
+
seconds = int(str(value))
|
|
1438
|
+
except (TypeError, ValueError):
|
|
1439
|
+
return None
|
|
1440
|
+
return datetime.fromtimestamp(seconds, tz=timezone.utc).date().isoformat()
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
def _safe_str(value: object) -> str:
|
|
1444
|
+
if value is None:
|
|
1445
|
+
return ""
|
|
1446
|
+
return str(value).strip()
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
def _warning_notes(warnings: list[str]) -> list[str]:
|
|
1450
|
+
return [f"Bilibili subtitle discovery warning: {warning}" for warning in warnings]
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def _failed_result(
|
|
1454
|
+
backend: str,
|
|
1455
|
+
code: str,
|
|
1456
|
+
error: object,
|
|
1457
|
+
*,
|
|
1458
|
+
title: str | None = None,
|
|
1459
|
+
author: str | None = None,
|
|
1460
|
+
published_at: str | None = None,
|
|
1461
|
+
metadata: dict[str, Any] | None = None,
|
|
1462
|
+
notes: list[str] | None = None,
|
|
1463
|
+
) -> TranscriptResult:
|
|
1464
|
+
return TranscriptResult(
|
|
1465
|
+
text="",
|
|
1466
|
+
status="failed",
|
|
1467
|
+
source_level="partial_extraction",
|
|
1468
|
+
backend=backend,
|
|
1469
|
+
source_notes=[
|
|
1470
|
+
"Bilibili provider returned structured failure.",
|
|
1471
|
+
_source_level_note("partial_extraction"),
|
|
1472
|
+
*(notes or []),
|
|
1473
|
+
],
|
|
1474
|
+
error_code=code,
|
|
1475
|
+
error_summary=sanitize_error_summary(error),
|
|
1476
|
+
title=title,
|
|
1477
|
+
author=author,
|
|
1478
|
+
published_at=published_at,
|
|
1479
|
+
metadata=metadata or {},
|
|
1480
|
+
)
|