kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
|
@@ -0,0 +1,602 @@
|
|
|
1
|
+
"""T20.2 — subtitle parsing shared by every platform adapter.
|
|
2
|
+
|
|
3
|
+
Formats
|
|
4
|
+
-------
|
|
5
|
+
* ``json3`` — YouTube's JSON3 caption payload. The event/``segs`` logic is the
|
|
6
|
+
one the legacy :class:`~ytk.ingestion.providers.ytdlp.YtDlpSubtitleProvider`
|
|
7
|
+
used (kept byte-compatible here; the legacy provider is intentionally left
|
|
8
|
+
untouched so all v1 tests keep passing).
|
|
9
|
+
* ``srt`` / ``vtt`` — parsed with ``pysubs2`` when importable, otherwise with a
|
|
10
|
+
small in-file fallback parser so the base install keeps working. Both paths
|
|
11
|
+
keep millisecond precision (floats rounded to 3 decimals).
|
|
12
|
+
* ``plain`` — a minimal ``start<TAB>end<TAB>text`` form. Untimed plain text is
|
|
13
|
+
rejected instead of inventing timestamps (v2 never fabricates time ranges).
|
|
14
|
+
* inline ``data`` (Bilibili serves SRT inline in yt-dlp's ``data`` field) and
|
|
15
|
+
``source_url`` payloads are both supported; URL fetches are delegated to an
|
|
16
|
+
injectable callable so unit tests never touch the network.
|
|
17
|
+
|
|
18
|
+
Danmaku / live chat is never narration (A05): such tracks are refused here and
|
|
19
|
+
excluded from :func:`select_track`, forcing the audio/ASR path instead.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import html
|
|
25
|
+
import json
|
|
26
|
+
import re
|
|
27
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from typing import Literal
|
|
30
|
+
from urllib.parse import urlsplit
|
|
31
|
+
|
|
32
|
+
from ytk.errors import AnalysisBlockedError, ContentIntegrityError
|
|
33
|
+
from ytk.ingestion.adapters.base import AcquiredBlock, SubtitleDescriptor
|
|
34
|
+
from ytk.models.content import TimeRange, TimeRangeLocator
|
|
35
|
+
|
|
36
|
+
SubtitleFormat = Literal["json3", "srt", "vtt", "plain"]
|
|
37
|
+
SubtitleKind = Literal["manual", "automatic", "inline"]
|
|
38
|
+
SubtitleProcessor = Literal[
|
|
39
|
+
"subtitle_manual",
|
|
40
|
+
"subtitle_auto",
|
|
41
|
+
"subtitle_json3",
|
|
42
|
+
"subtitle_srt",
|
|
43
|
+
"subtitle_vtt",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
#: Preferred transport format when several payloads exist for one language.
|
|
47
|
+
FORMAT_RANK: Mapping[SubtitleFormat, int] = {"json3": 0, "srt": 1, "vtt": 2, "plain": 3}
|
|
48
|
+
|
|
49
|
+
#: ``ext`` value (as exposed by yt-dlp) -> parser format.
|
|
50
|
+
EXT_TO_FORMAT: Mapping[str, SubtitleFormat] = {
|
|
51
|
+
"json3": "json3",
|
|
52
|
+
"srt": "srt",
|
|
53
|
+
"vtt": "vtt",
|
|
54
|
+
"webvtt": "vtt",
|
|
55
|
+
"txt": "plain",
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
#: Track names/languages that never carry the author's narration.
|
|
59
|
+
_NON_NARRATION_MARKERS: tuple[str, ...] = (
|
|
60
|
+
"danmaku",
|
|
61
|
+
"弹幕",
|
|
62
|
+
"bullet",
|
|
63
|
+
"barrage",
|
|
64
|
+
"live_chat",
|
|
65
|
+
"live-chat",
|
|
66
|
+
"livechat",
|
|
67
|
+
"live chat",
|
|
68
|
+
"live_comments",
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
_TIMESTAMP_RE = r"(?:\d{1,2}:)?\d{1,3}:\d{2}[.,]\d{1,3}"
|
|
72
|
+
_FALLBACK_CUE_RE = re.compile(
|
|
73
|
+
rf"(?P<start>{_TIMESTAMP_RE})\s*-->\s*(?P<end>{_TIMESTAMP_RE})[^\n]*\n(?P<text>.*?)(?=\n\s*\n|\Z)",
|
|
74
|
+
re.DOTALL,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
FetchCallable = Callable[[str], str]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class TrackInfo:
|
|
82
|
+
"""One selectable subtitle track discovered in provider metadata."""
|
|
83
|
+
|
|
84
|
+
name: str
|
|
85
|
+
language: str | None = None
|
|
86
|
+
kind: SubtitleKind = "manual"
|
|
87
|
+
format: SubtitleFormat = "json3"
|
|
88
|
+
url: str | None = None
|
|
89
|
+
inline_data: str | None = None
|
|
90
|
+
is_narration: bool = True
|
|
91
|
+
ext: str = ""
|
|
92
|
+
|
|
93
|
+
def to_descriptor(self) -> SubtitleDescriptor:
|
|
94
|
+
"""Build the acquisition-contract descriptor for this track."""
|
|
95
|
+
if self.kind == "inline" or self.inline_data is not None:
|
|
96
|
+
if self.inline_data is None:
|
|
97
|
+
raise ContentIntegrityError(f"inline subtitle track {self.name!r} has no data")
|
|
98
|
+
return SubtitleDescriptor(
|
|
99
|
+
kind="inline",
|
|
100
|
+
language=self.language,
|
|
101
|
+
format=self.format,
|
|
102
|
+
inline_data=self.inline_data,
|
|
103
|
+
is_narration=self.is_narration,
|
|
104
|
+
)
|
|
105
|
+
if not self.url:
|
|
106
|
+
raise ContentIntegrityError(f"subtitle track {self.name!r} has neither data nor a url")
|
|
107
|
+
return SubtitleDescriptor(
|
|
108
|
+
kind="manual" if self.kind == "manual" else "automatic",
|
|
109
|
+
language=self.language,
|
|
110
|
+
format=self.format,
|
|
111
|
+
source_url=self.url,
|
|
112
|
+
is_narration=self.is_narration,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass(frozen=True)
|
|
117
|
+
class ParsedSubtitle:
|
|
118
|
+
"""Validated subtitle blocks plus the provenance the evidence layer needs."""
|
|
119
|
+
|
|
120
|
+
source_kind: SubtitleKind
|
|
121
|
+
language: str | None
|
|
122
|
+
format: SubtitleFormat
|
|
123
|
+
blocks: list[AcquiredBlock]
|
|
124
|
+
processor: SubtitleProcessor
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def is_narration_track(name: str, ext: str) -> bool:
|
|
128
|
+
"""False for danmaku/live-chat style payloads that are not the narration.
|
|
129
|
+
|
|
130
|
+
``yt-dlp`` exposes Bilibili danmaku as ``ext=xml``; live chat surfaces as a
|
|
131
|
+
``live_chat`` language key on YouTube. Both are excluded. An unnamed XML
|
|
132
|
+
track is treated as danmaku (the only XML payload an extractor emits today);
|
|
133
|
+
TTML/XML subtitles would need a name/language to be trusted.
|
|
134
|
+
"""
|
|
135
|
+
raw_name = name or ""
|
|
136
|
+
normalized_name = re.sub(r"[\s\-]+", "_", raw_name.strip().casefold())
|
|
137
|
+
normalized_ext = (ext or "").strip().casefold()
|
|
138
|
+
if normalized_ext == "xml" and (not normalized_name or "danmaku" in normalized_name or "弹幕" in raw_name):
|
|
139
|
+
return False
|
|
140
|
+
return not any(marker in normalized_name for marker in _NON_NARRATION_MARKERS)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def language_matches(track_language: str | None, requested: str | None) -> bool:
|
|
144
|
+
"""True when ``track_language`` serves ``requested`` (base tag aware)."""
|
|
145
|
+
|
|
146
|
+
def _normalize(value: str | None) -> str:
|
|
147
|
+
return (value or "").strip().replace("_", "-").casefold()
|
|
148
|
+
|
|
149
|
+
track = _normalize(track_language)
|
|
150
|
+
wanted = _normalize(requested)
|
|
151
|
+
if not track or not wanted:
|
|
152
|
+
return False
|
|
153
|
+
if track == wanted:
|
|
154
|
+
return True
|
|
155
|
+
return track.split("-")[0] == wanted.split("-")[0]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _track_sort_key(track: TrackInfo) -> tuple[int, int]:
|
|
159
|
+
# Inline payloads win over URL payloads of the same language (no extra
|
|
160
|
+
# network round-trip, and Bilibili only exposes inline data).
|
|
161
|
+
inline_rank = 0 if track.inline_data is not None else 1
|
|
162
|
+
return (inline_rank, FORMAT_RANK.get(track.format, len(FORMAT_RANK)))
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def select_track(available: Sequence[TrackInfo], languages: Sequence[str]) -> TrackInfo | None:
|
|
166
|
+
"""Pick the best narration track: manual first, then language priority.
|
|
167
|
+
|
|
168
|
+
Mirrors the legacy YouTube behaviour (manual captions are preferred over
|
|
169
|
+
automatic ones regardless of language, the requested language order decides
|
|
170
|
+
within a class) and prefers inline data over a URL for the same language.
|
|
171
|
+
"""
|
|
172
|
+
usable = [
|
|
173
|
+
track
|
|
174
|
+
for track in available
|
|
175
|
+
if track.is_narration
|
|
176
|
+
and is_narration_track(track.name or "", track.ext)
|
|
177
|
+
and is_narration_track(track.language or "", track.ext)
|
|
178
|
+
and (track.inline_data is not None or bool(track.url))
|
|
179
|
+
]
|
|
180
|
+
if not usable:
|
|
181
|
+
return None
|
|
182
|
+
|
|
183
|
+
manual = [track for track in usable if track.kind != "automatic"]
|
|
184
|
+
automatic = [track for track in usable if track.kind == "automatic"]
|
|
185
|
+
requested = [language for language in languages if language]
|
|
186
|
+
|
|
187
|
+
for group in (manual, automatic):
|
|
188
|
+
if not group:
|
|
189
|
+
continue
|
|
190
|
+
for language in requested:
|
|
191
|
+
matched = [track for track in group if language_matches(track.language, language)]
|
|
192
|
+
if matched:
|
|
193
|
+
return min(matched, key=_track_sort_key)
|
|
194
|
+
# No requested language is available: still better than ASR.
|
|
195
|
+
return min(group, key=_track_sort_key)
|
|
196
|
+
return None
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def merge_adjacent_blocks(blocks: Sequence[AcquiredBlock], *, gap: float = 0.35) -> list[AcquiredBlock]:
|
|
200
|
+
"""Merge consecutive timed caption blocks separated by at most ``gap`` seconds.
|
|
201
|
+
|
|
202
|
+
Rolling caption streams emit one block per line; merging keeps short gaps
|
|
203
|
+
together. Overlapping blocks (negative gap) are merged as well. Blocks
|
|
204
|
+
with non-time locators, different kinds or different languages are never
|
|
205
|
+
merged, and ``order`` is re-indexed on the merged sequence.
|
|
206
|
+
"""
|
|
207
|
+
if gap < 0:
|
|
208
|
+
raise ValueError("gap must be >= 0")
|
|
209
|
+
|
|
210
|
+
merged: list[AcquiredBlock] = []
|
|
211
|
+
for block in blocks:
|
|
212
|
+
previous = merged[-1] if merged else None
|
|
213
|
+
if previous is not None and _can_merge(previous, block, gap=gap):
|
|
214
|
+
start = previous.locator.time.start # type: ignore[union-attr]
|
|
215
|
+
end = max(previous.locator.time.end, block.locator.time.end) # type: ignore[union-attr]
|
|
216
|
+
merged[-1] = AcquiredBlock(
|
|
217
|
+
kind=previous.kind,
|
|
218
|
+
raw_text=_join_text(previous.raw_text, block.raw_text),
|
|
219
|
+
cleaned_text=None,
|
|
220
|
+
language=previous.language or block.language,
|
|
221
|
+
order=previous.order,
|
|
222
|
+
locator=TimeRangeLocator(time=TimeRange(start=start, end=end)),
|
|
223
|
+
)
|
|
224
|
+
continue
|
|
225
|
+
merged.append(
|
|
226
|
+
AcquiredBlock(
|
|
227
|
+
kind=block.kind,
|
|
228
|
+
raw_text=block.raw_text,
|
|
229
|
+
cleaned_text=block.cleaned_text,
|
|
230
|
+
language=block.language,
|
|
231
|
+
order=block.order,
|
|
232
|
+
locator=block.locator,
|
|
233
|
+
)
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
return [
|
|
237
|
+
block.model_copy(update={"order": index}) if block.order != index else block
|
|
238
|
+
for index, block in enumerate(merged)
|
|
239
|
+
]
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _can_merge(previous: AcquiredBlock, current: AcquiredBlock, *, gap: float) -> bool:
|
|
243
|
+
if getattr(previous.locator, "kind", None) != "time_range":
|
|
244
|
+
return False
|
|
245
|
+
if getattr(current.locator, "kind", None) != "time_range":
|
|
246
|
+
return False
|
|
247
|
+
if previous.kind != current.kind:
|
|
248
|
+
return False
|
|
249
|
+
if previous.language and current.language and previous.language != current.language:
|
|
250
|
+
return False
|
|
251
|
+
return current.locator.time.start - previous.locator.time.end <= gap # type: ignore[union-attr]
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _join_text(left: str, right: str) -> str:
|
|
255
|
+
left, right = left.strip(), right.strip()
|
|
256
|
+
if not left:
|
|
257
|
+
return right
|
|
258
|
+
if not right:
|
|
259
|
+
return left
|
|
260
|
+
# No artificial space between CJK characters.
|
|
261
|
+
if _is_cjk(left[-1]) and _is_cjk(right[0]):
|
|
262
|
+
return left + right
|
|
263
|
+
return f"{left} {right}"
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _is_cjk(char: str) -> bool:
|
|
267
|
+
return "\u4e00" <= char <= "\u9fff" or "\u3040" <= char <= "\u30ff" or "\uac00" <= char <= "\ud7af"
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def parse_subtitle(
|
|
271
|
+
desc: SubtitleDescriptor,
|
|
272
|
+
*,
|
|
273
|
+
source_id: str,
|
|
274
|
+
fetch: FetchCallable | None = None,
|
|
275
|
+
) -> ParsedSubtitle:
|
|
276
|
+
"""Parse one subtitle descriptor into located caption blocks.
|
|
277
|
+
|
|
278
|
+
``fetch`` is injected by callers/tests; the default is a bounded ``httpx``
|
|
279
|
+
GET used only outside the offline test suite.
|
|
280
|
+
"""
|
|
281
|
+
if not source_id or not source_id.strip():
|
|
282
|
+
raise ValueError("source_id is required to build subtitle blocks")
|
|
283
|
+
if not desc.is_narration:
|
|
284
|
+
raise AnalysisBlockedError(
|
|
285
|
+
f"subtitle track for {source_id} is danmaku/live-chat, not narration; "
|
|
286
|
+
"use the audio/ASR path instead"
|
|
287
|
+
)
|
|
288
|
+
if not is_narration_track(desc.language or "", EXT_TO_FORMAT.get(desc.format, desc.format)):
|
|
289
|
+
raise AnalysisBlockedError(
|
|
290
|
+
f"subtitle track for {source_id} is not narration (language={desc.language!r})"
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
payload = desc.inline_data
|
|
294
|
+
if payload is None:
|
|
295
|
+
if not desc.source_url:
|
|
296
|
+
raise ContentIntegrityError("subtitle descriptor has neither inline data nor a url")
|
|
297
|
+
payload = (fetch or _default_fetch)(desc.source_url)
|
|
298
|
+
if not isinstance(payload, str) or not payload.strip():
|
|
299
|
+
raise ContentIntegrityError(f"subtitle payload for {source_id} is empty")
|
|
300
|
+
|
|
301
|
+
events = _extract_events(payload, desc.format)
|
|
302
|
+
blocks = [
|
|
303
|
+
AcquiredBlock(
|
|
304
|
+
kind="caption",
|
|
305
|
+
raw_text=text,
|
|
306
|
+
cleaned_text=None,
|
|
307
|
+
language=desc.language,
|
|
308
|
+
order=index,
|
|
309
|
+
locator=TimeRangeLocator(
|
|
310
|
+
time=TimeRange(start=round(start, 3), end=round(end, 3))
|
|
311
|
+
),
|
|
312
|
+
)
|
|
313
|
+
for index, (text, start, end) in enumerate(events)
|
|
314
|
+
]
|
|
315
|
+
if not blocks:
|
|
316
|
+
raise ContentIntegrityError(f"subtitle payload for {source_id} contains no caption text")
|
|
317
|
+
|
|
318
|
+
return ParsedSubtitle(
|
|
319
|
+
source_kind=desc.kind,
|
|
320
|
+
language=desc.language,
|
|
321
|
+
format=desc.format,
|
|
322
|
+
blocks=blocks,
|
|
323
|
+
processor=_processor_for(desc.kind, desc.format),
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _processor_for(kind: SubtitleKind, fmt: SubtitleFormat) -> SubtitleProcessor:
|
|
328
|
+
if kind == "manual":
|
|
329
|
+
return "subtitle_manual"
|
|
330
|
+
if kind == "automatic":
|
|
331
|
+
return "subtitle_auto"
|
|
332
|
+
if fmt == "json3":
|
|
333
|
+
return "subtitle_json3"
|
|
334
|
+
if fmt == "vtt":
|
|
335
|
+
return "subtitle_vtt"
|
|
336
|
+
# SRT and the minimal plain timing form are both cue-based text.
|
|
337
|
+
return "subtitle_srt"
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
# ---------------------------------------------------------------------------
|
|
341
|
+
# Event extraction per format
|
|
342
|
+
# ---------------------------------------------------------------------------
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _extract_events(payload: str, fmt: SubtitleFormat) -> list[tuple[str, float, float]]:
|
|
346
|
+
if not payload.strip():
|
|
347
|
+
raise ContentIntegrityError("subtitle payload is empty")
|
|
348
|
+
if fmt == "json3":
|
|
349
|
+
events = parse_json3_events(payload)
|
|
350
|
+
elif fmt in {"srt", "vtt"}:
|
|
351
|
+
_reject_obvious_truncation(payload, fmt)
|
|
352
|
+
events = _parse_ssa(payload, fmt)
|
|
353
|
+
if fmt == "vtt":
|
|
354
|
+
events = _dedupe_rolling_captions(events)
|
|
355
|
+
elif fmt == "plain":
|
|
356
|
+
events = _parse_plain(payload)
|
|
357
|
+
else: # pragma: no cover - the Literal already constrains callers
|
|
358
|
+
raise ContentIntegrityError(f"unsupported subtitle format: {fmt}")
|
|
359
|
+
if not events:
|
|
360
|
+
raise ContentIntegrityError(f"{fmt} subtitle contains no caption text")
|
|
361
|
+
_validate_events(events)
|
|
362
|
+
return events
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def parse_json3_events(payload: str | Mapping[str, object]) -> list[tuple[str, float, float]]:
|
|
366
|
+
"""JSON3 ``events`` -> ``(text, start_s, end_s)`` with ms precision."""
|
|
367
|
+
if isinstance(payload, str):
|
|
368
|
+
try:
|
|
369
|
+
data: object = json.loads(payload)
|
|
370
|
+
except json.JSONDecodeError as exc:
|
|
371
|
+
raise ContentIntegrityError(f"invalid JSON3 subtitle: {exc.msg}") from exc
|
|
372
|
+
else:
|
|
373
|
+
data = payload
|
|
374
|
+
if not isinstance(data, Mapping) or not isinstance(data.get("events"), list):
|
|
375
|
+
raise ContentIntegrityError("invalid JSON3 subtitle: missing events list")
|
|
376
|
+
|
|
377
|
+
events: list[tuple[str, float, float]] = []
|
|
378
|
+
for event in data["events"]:
|
|
379
|
+
if not isinstance(event, Mapping):
|
|
380
|
+
continue
|
|
381
|
+
segs = event.get("segs")
|
|
382
|
+
if not isinstance(segs, list):
|
|
383
|
+
continue
|
|
384
|
+
text = _normalize_text(
|
|
385
|
+
"".join(str(seg.get("utf8", "")) for seg in segs if isinstance(seg, Mapping))
|
|
386
|
+
)
|
|
387
|
+
if not text:
|
|
388
|
+
continue
|
|
389
|
+
try:
|
|
390
|
+
start_ms = float(event.get("tStartMs", 0.0) or 0.0)
|
|
391
|
+
duration_ms = float(event.get("dDurationMs", 0.0) or 0.0)
|
|
392
|
+
except (TypeError, ValueError) as exc:
|
|
393
|
+
raise ContentIntegrityError("invalid JSON3 subtitle event timing") from exc
|
|
394
|
+
if start_ms < 0 or duration_ms < 0:
|
|
395
|
+
raise ContentIntegrityError("JSON3 subtitle event has a negative timestamp")
|
|
396
|
+
start = round(start_ms / 1000.0, 3)
|
|
397
|
+
end = round(start + duration_ms / 1000.0, 3)
|
|
398
|
+
events.append((text, start, end))
|
|
399
|
+
return events
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _load_pysubs2() -> object | None:
|
|
403
|
+
"""Import ``pysubs2`` lazily; ``None`` means "use the fallback parser"."""
|
|
404
|
+
try:
|
|
405
|
+
import pysubs2 # noqa: PLC0415
|
|
406
|
+
except Exception: # pragma: no cover - optional dependency
|
|
407
|
+
return None
|
|
408
|
+
return pysubs2
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _parse_ssa(payload: str, fmt: SubtitleFormat) -> list[tuple[str, float, float]]:
|
|
412
|
+
pysubs2 = _load_pysubs2()
|
|
413
|
+
if pysubs2 is None:
|
|
414
|
+
return _fallback_cues(payload, fmt)
|
|
415
|
+
|
|
416
|
+
format_name = "srt" if fmt == "srt" else "vtt"
|
|
417
|
+
try:
|
|
418
|
+
subs = pysubs2.SSAFile.from_string(payload, format_=format_name) # type: ignore[attr-defined]
|
|
419
|
+
except Exception as exc: # pysubs2 raises its own FormatAutodetectionError subclasses
|
|
420
|
+
raise ContentIntegrityError(f"invalid {fmt.upper()} subtitle: {exc}") from exc
|
|
421
|
+
|
|
422
|
+
events: list[tuple[str, float, float]] = []
|
|
423
|
+
for line in subs:
|
|
424
|
+
text = _normalize_text(getattr(line, "plaintext", "") or "")
|
|
425
|
+
if not text:
|
|
426
|
+
continue
|
|
427
|
+
events.append((text, line.start / 1000.0, line.end / 1000.0))
|
|
428
|
+
return events
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _fallback_cues(payload: str, fmt: SubtitleFormat) -> list[tuple[str, float, float]]:
|
|
432
|
+
"""Minimal SRT/VTT reader used when ``pysubs2`` is not installed."""
|
|
433
|
+
if fmt == "vtt":
|
|
434
|
+
payload = re.sub(r"(?m)^WEBVTT[^\n]*\n", "", payload, count=1)
|
|
435
|
+
events: list[tuple[str, float, float]] = []
|
|
436
|
+
for match in _FALLBACK_CUE_RE.finditer(payload):
|
|
437
|
+
text = _strip_cue_markup(match.group("text"))
|
|
438
|
+
text = _normalize_text(text)
|
|
439
|
+
if not text:
|
|
440
|
+
continue
|
|
441
|
+
start = _parse_timestamp(match.group("start"))
|
|
442
|
+
end = _parse_timestamp(match.group("end"))
|
|
443
|
+
events.append((text, start, end))
|
|
444
|
+
if not events and "-->" in payload:
|
|
445
|
+
raise ContentIntegrityError(f"{fmt.upper()} subtitle appears truncated or malformed")
|
|
446
|
+
return events
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _parse_timestamp(value: str) -> float:
|
|
450
|
+
normalized = value.strip().replace(",", ".")
|
|
451
|
+
parts = normalized.split(":")
|
|
452
|
+
try:
|
|
453
|
+
numbers = [float(part) for part in parts]
|
|
454
|
+
except ValueError as exc: # pragma: no cover - regex constrains the shape
|
|
455
|
+
raise ContentIntegrityError(f"invalid subtitle timestamp: {value!r}") from exc
|
|
456
|
+
if len(numbers) == 3:
|
|
457
|
+
hours, minutes, seconds = numbers[0], numbers[1], numbers[2]
|
|
458
|
+
elif len(numbers) == 2:
|
|
459
|
+
hours, minutes, seconds = 0.0, numbers[0], numbers[1]
|
|
460
|
+
else: # pragma: no cover - regex constrains the shape
|
|
461
|
+
raise ContentIntegrityError(f"invalid subtitle timestamp: {value!r}")
|
|
462
|
+
return hours * 3600.0 + minutes * 60.0 + seconds
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _strip_cue_markup(text: str) -> str:
|
|
466
|
+
# Generic type parameters and comparisons are source knowledge, not HTML.
|
|
467
|
+
text = re.sub(r"</?(?:b|i|u|ruby|rt)\s*>|</?c(?:\.[\w-]+)*\s*>|<v(?:\s+[^>]*)?>|</v>|<lang(?:\s+[^>]*)?>|</lang>|<font(?:\s+[^>]*)?>|</font>|<(?:\d{2}:)?\d{2}:\d{2}\.\d{3}>", "", text, flags=re.IGNORECASE)
|
|
468
|
+
text = re.sub(r"\{\\[^}]*\}", "", text) # ASS override tags
|
|
469
|
+
return html.unescape(text)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _parse_plain(payload: str) -> list[tuple[str, float, float]]:
|
|
473
|
+
events: list[tuple[str, float, float]] = []
|
|
474
|
+
for line in payload.splitlines():
|
|
475
|
+
line = line.strip()
|
|
476
|
+
if not line:
|
|
477
|
+
continue
|
|
478
|
+
parts = line.split("\t")
|
|
479
|
+
if len(parts) < 3:
|
|
480
|
+
raise ContentIntegrityError(
|
|
481
|
+
"plain subtitle lines need explicit timing: 'start<TAB>end<TAB>text'"
|
|
482
|
+
)
|
|
483
|
+
raw_start, raw_end = parts[0].strip(), parts[1].strip()
|
|
484
|
+
text = _normalize_text("\t".join(parts[2:]))
|
|
485
|
+
if not text:
|
|
486
|
+
continue
|
|
487
|
+
try:
|
|
488
|
+
start, end = float(raw_start), float(raw_end)
|
|
489
|
+
except ValueError as exc:
|
|
490
|
+
raise ContentIntegrityError(
|
|
491
|
+
"plain subtitle lines need explicit timing: 'start<TAB>end<TAB>text'"
|
|
492
|
+
) from exc
|
|
493
|
+
events.append((text, start, end))
|
|
494
|
+
return events
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _reject_obvious_truncation(payload: str, fmt: SubtitleFormat) -> None:
|
|
498
|
+
"""Reject a cleared-down cue header: the last timed cue has no text after it."""
|
|
499
|
+
lines = payload.splitlines()
|
|
500
|
+
last_cue = max((index for index, line in enumerate(lines) if "-->" in line), default=-1)
|
|
501
|
+
if last_cue < 0:
|
|
502
|
+
if payload.strip():
|
|
503
|
+
raise ContentIntegrityError(f"{fmt.upper()} subtitle has no timed cues")
|
|
504
|
+
return
|
|
505
|
+
trailing = [line for line in lines[last_cue + 1 :] if line.strip()]
|
|
506
|
+
if not trailing:
|
|
507
|
+
raise ContentIntegrityError(
|
|
508
|
+
f"subtitle appears truncated: the last {fmt.upper()} cue has no text"
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _dedupe_rolling_captions(
|
|
513
|
+
events: list[tuple[str, float, float]],
|
|
514
|
+
) -> list[tuple[str, float, float]]:
|
|
515
|
+
"""Drop the repeated prefix YouTube-style rolling captions prepend."""
|
|
516
|
+
result: list[tuple[str, float, float]] = []
|
|
517
|
+
previous: str | None = None
|
|
518
|
+
for raw_text, start, end in events:
|
|
519
|
+
text = raw_text
|
|
520
|
+
if previous:
|
|
521
|
+
if text == previous:
|
|
522
|
+
text = ""
|
|
523
|
+
elif text.startswith(previous + "\n"):
|
|
524
|
+
text = text[len(previous) + 1 :]
|
|
525
|
+
elif text.startswith(previous + " "):
|
|
526
|
+
text = text[len(previous) + 1 :]
|
|
527
|
+
previous = raw_text
|
|
528
|
+
text = _normalize_text(text)
|
|
529
|
+
if not text:
|
|
530
|
+
continue
|
|
531
|
+
result.append((text, start, end))
|
|
532
|
+
return result
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _validate_events(events: list[tuple[str, float, float]]) -> None:
|
|
536
|
+
previous_start = -1.0
|
|
537
|
+
for text, start, end in events:
|
|
538
|
+
if not text.strip():
|
|
539
|
+
raise ContentIntegrityError("subtitle cue text must not be blank")
|
|
540
|
+
if start < 0 or end < 0:
|
|
541
|
+
raise ContentIntegrityError("subtitle cue has a negative timestamp")
|
|
542
|
+
if end < start:
|
|
543
|
+
raise ContentIntegrityError("subtitle cue end is before its start")
|
|
544
|
+
if start < previous_start:
|
|
545
|
+
# Out-of-order cues are tolerated by sorting later, but a backwards
|
|
546
|
+
# jump of a whole minute usually means two files were concatenated.
|
|
547
|
+
if previous_start - start > 60.0:
|
|
548
|
+
raise ContentIntegrityError("subtitle cues are not in chronological order")
|
|
549
|
+
previous_start = max(previous_start, start)
|
|
550
|
+
events.sort(key=lambda event: event[1])
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _normalize_text(text: str) -> str:
|
|
554
|
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
555
|
+
text = "\n".join(" ".join(line.split()) for line in text.split("\n"))
|
|
556
|
+
return text.strip()
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _default_fetch(url: str, *, timeout: float = 30.0) -> str:
|
|
560
|
+
"""Bounded GET used in production only; unit tests inject ``fetch``."""
|
|
561
|
+
|
|
562
|
+
parts = urlsplit(url)
|
|
563
|
+
if parts.scheme not in {"http", "https"} or not parts.hostname:
|
|
564
|
+
raise ContentIntegrityError(f"refusing to fetch a non-http(s) subtitle url: {parts.scheme}")
|
|
565
|
+
from ytk.ingestion.http_client import make_client
|
|
566
|
+
|
|
567
|
+
with make_client(url=url, timeout=timeout) as client:
|
|
568
|
+
response = client.get(url, headers={"User-Agent": "ytk/2.0 (+subtitles)"})
|
|
569
|
+
status = getattr(response, "status_code", 200)
|
|
570
|
+
if status in (401, 403):
|
|
571
|
+
from ytk.errors import AnalysisBlockedError
|
|
572
|
+
|
|
573
|
+
raise AnalysisBlockedError(
|
|
574
|
+
f"subtitle URL rejected the request (HTTP {status}); access/permission blocked "
|
|
575
|
+
"(AUTH_REQUIRED)"
|
|
576
|
+
)
|
|
577
|
+
if status == 429:
|
|
578
|
+
from ytk.errors import AnalysisBlockedError
|
|
579
|
+
|
|
580
|
+
raise AnalysisBlockedError(
|
|
581
|
+
"subtitle endpoint returned HTTP 429 (RATE_LIMITED); retry after the platform "
|
|
582
|
+
"cooldown, this is not a missing-subtitle condition"
|
|
583
|
+
)
|
|
584
|
+
response.raise_for_status()
|
|
585
|
+
return response.text
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
__all__ = [
|
|
589
|
+
"EXT_TO_FORMAT",
|
|
590
|
+
"FORMAT_RANK",
|
|
591
|
+
"ParsedSubtitle",
|
|
592
|
+
"SubtitleFormat",
|
|
593
|
+
"SubtitleKind",
|
|
594
|
+
"SubtitleProcessor",
|
|
595
|
+
"TrackInfo",
|
|
596
|
+
"is_narration_track",
|
|
597
|
+
"language_matches",
|
|
598
|
+
"merge_adjacent_blocks",
|
|
599
|
+
"parse_json3_events",
|
|
600
|
+
"parse_subtitle",
|
|
601
|
+
"select_track",
|
|
602
|
+
]
|