kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
@@ -0,0 +1,602 @@
1
+ """T20.2 — subtitle parsing shared by every platform adapter.
2
+
3
+ Formats
4
+ -------
5
+ * ``json3`` — YouTube's JSON3 caption payload. The event/``segs`` logic is the
6
+ one the legacy :class:`~ytk.ingestion.providers.ytdlp.YtDlpSubtitleProvider`
7
+ used (kept byte-compatible here; the legacy provider is intentionally left
8
+ untouched so all v1 tests keep passing).
9
+ * ``srt`` / ``vtt`` — parsed with ``pysubs2`` when importable, otherwise with a
10
+ small in-file fallback parser so the base install keeps working. Both paths
11
+ keep millisecond precision (floats rounded to 3 decimals).
12
+ * ``plain`` — a minimal ``start<TAB>end<TAB>text`` form. Untimed plain text is
13
+ rejected instead of inventing timestamps (v2 never fabricates time ranges).
14
+ * inline ``data`` (Bilibili serves SRT inline in yt-dlp's ``data`` field) and
15
+ ``source_url`` payloads are both supported; URL fetches are delegated to an
16
+ injectable callable so unit tests never touch the network.
17
+
18
+ Danmaku / live chat is never narration (A05): such tracks are refused here and
19
+ excluded from :func:`select_track`, forcing the audio/ASR path instead.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import html
25
+ import json
26
+ import re
27
+ from collections.abc import Callable, Mapping, Sequence
28
+ from dataclasses import dataclass
29
+ from typing import Literal
30
+ from urllib.parse import urlsplit
31
+
32
+ from ytk.errors import AnalysisBlockedError, ContentIntegrityError
33
+ from ytk.ingestion.adapters.base import AcquiredBlock, SubtitleDescriptor
34
+ from ytk.models.content import TimeRange, TimeRangeLocator
35
+
36
+ SubtitleFormat = Literal["json3", "srt", "vtt", "plain"]
37
+ SubtitleKind = Literal["manual", "automatic", "inline"]
38
+ SubtitleProcessor = Literal[
39
+ "subtitle_manual",
40
+ "subtitle_auto",
41
+ "subtitle_json3",
42
+ "subtitle_srt",
43
+ "subtitle_vtt",
44
+ ]
45
+
46
+ #: Preferred transport format when several payloads exist for one language.
47
+ FORMAT_RANK: Mapping[SubtitleFormat, int] = {"json3": 0, "srt": 1, "vtt": 2, "plain": 3}
48
+
49
+ #: ``ext`` value (as exposed by yt-dlp) -> parser format.
50
+ EXT_TO_FORMAT: Mapping[str, SubtitleFormat] = {
51
+ "json3": "json3",
52
+ "srt": "srt",
53
+ "vtt": "vtt",
54
+ "webvtt": "vtt",
55
+ "txt": "plain",
56
+ }
57
+
58
+ #: Track names/languages that never carry the author's narration.
59
+ _NON_NARRATION_MARKERS: tuple[str, ...] = (
60
+ "danmaku",
61
+ "弹幕",
62
+ "bullet",
63
+ "barrage",
64
+ "live_chat",
65
+ "live-chat",
66
+ "livechat",
67
+ "live chat",
68
+ "live_comments",
69
+ )
70
+
71
+ _TIMESTAMP_RE = r"(?:\d{1,2}:)?\d{1,3}:\d{2}[.,]\d{1,3}"
72
+ _FALLBACK_CUE_RE = re.compile(
73
+ rf"(?P<start>{_TIMESTAMP_RE})\s*-->\s*(?P<end>{_TIMESTAMP_RE})[^\n]*\n(?P<text>.*?)(?=\n\s*\n|\Z)",
74
+ re.DOTALL,
75
+ )
76
+
77
+ FetchCallable = Callable[[str], str]
78
+
79
+
80
+ @dataclass(frozen=True)
81
+ class TrackInfo:
82
+ """One selectable subtitle track discovered in provider metadata."""
83
+
84
+ name: str
85
+ language: str | None = None
86
+ kind: SubtitleKind = "manual"
87
+ format: SubtitleFormat = "json3"
88
+ url: str | None = None
89
+ inline_data: str | None = None
90
+ is_narration: bool = True
91
+ ext: str = ""
92
+
93
+ def to_descriptor(self) -> SubtitleDescriptor:
94
+ """Build the acquisition-contract descriptor for this track."""
95
+ if self.kind == "inline" or self.inline_data is not None:
96
+ if self.inline_data is None:
97
+ raise ContentIntegrityError(f"inline subtitle track {self.name!r} has no data")
98
+ return SubtitleDescriptor(
99
+ kind="inline",
100
+ language=self.language,
101
+ format=self.format,
102
+ inline_data=self.inline_data,
103
+ is_narration=self.is_narration,
104
+ )
105
+ if not self.url:
106
+ raise ContentIntegrityError(f"subtitle track {self.name!r} has neither data nor a url")
107
+ return SubtitleDescriptor(
108
+ kind="manual" if self.kind == "manual" else "automatic",
109
+ language=self.language,
110
+ format=self.format,
111
+ source_url=self.url,
112
+ is_narration=self.is_narration,
113
+ )
114
+
115
+
116
+ @dataclass(frozen=True)
117
+ class ParsedSubtitle:
118
+ """Validated subtitle blocks plus the provenance the evidence layer needs."""
119
+
120
+ source_kind: SubtitleKind
121
+ language: str | None
122
+ format: SubtitleFormat
123
+ blocks: list[AcquiredBlock]
124
+ processor: SubtitleProcessor
125
+
126
+
127
+ def is_narration_track(name: str, ext: str) -> bool:
128
+ """False for danmaku/live-chat style payloads that are not the narration.
129
+
130
+ ``yt-dlp`` exposes Bilibili danmaku as ``ext=xml``; live chat surfaces as a
131
+ ``live_chat`` language key on YouTube. Both are excluded. An unnamed XML
132
+ track is treated as danmaku (the only XML payload an extractor emits today);
133
+ TTML/XML subtitles would need a name/language to be trusted.
134
+ """
135
+ raw_name = name or ""
136
+ normalized_name = re.sub(r"[\s\-]+", "_", raw_name.strip().casefold())
137
+ normalized_ext = (ext or "").strip().casefold()
138
+ if normalized_ext == "xml" and (not normalized_name or "danmaku" in normalized_name or "弹幕" in raw_name):
139
+ return False
140
+ return not any(marker in normalized_name for marker in _NON_NARRATION_MARKERS)
141
+
142
+
143
+ def language_matches(track_language: str | None, requested: str | None) -> bool:
144
+ """True when ``track_language`` serves ``requested`` (base tag aware)."""
145
+
146
+ def _normalize(value: str | None) -> str:
147
+ return (value or "").strip().replace("_", "-").casefold()
148
+
149
+ track = _normalize(track_language)
150
+ wanted = _normalize(requested)
151
+ if not track or not wanted:
152
+ return False
153
+ if track == wanted:
154
+ return True
155
+ return track.split("-")[0] == wanted.split("-")[0]
156
+
157
+
158
+ def _track_sort_key(track: TrackInfo) -> tuple[int, int]:
159
+ # Inline payloads win over URL payloads of the same language (no extra
160
+ # network round-trip, and Bilibili only exposes inline data).
161
+ inline_rank = 0 if track.inline_data is not None else 1
162
+ return (inline_rank, FORMAT_RANK.get(track.format, len(FORMAT_RANK)))
163
+
164
+
165
+ def select_track(available: Sequence[TrackInfo], languages: Sequence[str]) -> TrackInfo | None:
166
+ """Pick the best narration track: manual first, then language priority.
167
+
168
+ Mirrors the legacy YouTube behaviour (manual captions are preferred over
169
+ automatic ones regardless of language, the requested language order decides
170
+ within a class) and prefers inline data over a URL for the same language.
171
+ """
172
+ usable = [
173
+ track
174
+ for track in available
175
+ if track.is_narration
176
+ and is_narration_track(track.name or "", track.ext)
177
+ and is_narration_track(track.language or "", track.ext)
178
+ and (track.inline_data is not None or bool(track.url))
179
+ ]
180
+ if not usable:
181
+ return None
182
+
183
+ manual = [track for track in usable if track.kind != "automatic"]
184
+ automatic = [track for track in usable if track.kind == "automatic"]
185
+ requested = [language for language in languages if language]
186
+
187
+ for group in (manual, automatic):
188
+ if not group:
189
+ continue
190
+ for language in requested:
191
+ matched = [track for track in group if language_matches(track.language, language)]
192
+ if matched:
193
+ return min(matched, key=_track_sort_key)
194
+ # No requested language is available: still better than ASR.
195
+ return min(group, key=_track_sort_key)
196
+ return None
197
+
198
+
199
+ def merge_adjacent_blocks(blocks: Sequence[AcquiredBlock], *, gap: float = 0.35) -> list[AcquiredBlock]:
200
+ """Merge consecutive timed caption blocks separated by at most ``gap`` seconds.
201
+
202
+ Rolling caption streams emit one block per line; merging keeps short gaps
203
+ together. Overlapping blocks (negative gap) are merged as well. Blocks
204
+ with non-time locators, different kinds or different languages are never
205
+ merged, and ``order`` is re-indexed on the merged sequence.
206
+ """
207
+ if gap < 0:
208
+ raise ValueError("gap must be >= 0")
209
+
210
+ merged: list[AcquiredBlock] = []
211
+ for block in blocks:
212
+ previous = merged[-1] if merged else None
213
+ if previous is not None and _can_merge(previous, block, gap=gap):
214
+ start = previous.locator.time.start # type: ignore[union-attr]
215
+ end = max(previous.locator.time.end, block.locator.time.end) # type: ignore[union-attr]
216
+ merged[-1] = AcquiredBlock(
217
+ kind=previous.kind,
218
+ raw_text=_join_text(previous.raw_text, block.raw_text),
219
+ cleaned_text=None,
220
+ language=previous.language or block.language,
221
+ order=previous.order,
222
+ locator=TimeRangeLocator(time=TimeRange(start=start, end=end)),
223
+ )
224
+ continue
225
+ merged.append(
226
+ AcquiredBlock(
227
+ kind=block.kind,
228
+ raw_text=block.raw_text,
229
+ cleaned_text=block.cleaned_text,
230
+ language=block.language,
231
+ order=block.order,
232
+ locator=block.locator,
233
+ )
234
+ )
235
+
236
+ return [
237
+ block.model_copy(update={"order": index}) if block.order != index else block
238
+ for index, block in enumerate(merged)
239
+ ]
240
+
241
+
242
+ def _can_merge(previous: AcquiredBlock, current: AcquiredBlock, *, gap: float) -> bool:
243
+ if getattr(previous.locator, "kind", None) != "time_range":
244
+ return False
245
+ if getattr(current.locator, "kind", None) != "time_range":
246
+ return False
247
+ if previous.kind != current.kind:
248
+ return False
249
+ if previous.language and current.language and previous.language != current.language:
250
+ return False
251
+ return current.locator.time.start - previous.locator.time.end <= gap # type: ignore[union-attr]
252
+
253
+
254
+ def _join_text(left: str, right: str) -> str:
255
+ left, right = left.strip(), right.strip()
256
+ if not left:
257
+ return right
258
+ if not right:
259
+ return left
260
+ # No artificial space between CJK characters.
261
+ if _is_cjk(left[-1]) and _is_cjk(right[0]):
262
+ return left + right
263
+ return f"{left} {right}"
264
+
265
+
266
+ def _is_cjk(char: str) -> bool:
267
+ return "\u4e00" <= char <= "\u9fff" or "\u3040" <= char <= "\u30ff" or "\uac00" <= char <= "\ud7af"
268
+
269
+
270
+ def parse_subtitle(
271
+ desc: SubtitleDescriptor,
272
+ *,
273
+ source_id: str,
274
+ fetch: FetchCallable | None = None,
275
+ ) -> ParsedSubtitle:
276
+ """Parse one subtitle descriptor into located caption blocks.
277
+
278
+ ``fetch`` is injected by callers/tests; the default is a bounded ``httpx``
279
+ GET used only outside the offline test suite.
280
+ """
281
+ if not source_id or not source_id.strip():
282
+ raise ValueError("source_id is required to build subtitle blocks")
283
+ if not desc.is_narration:
284
+ raise AnalysisBlockedError(
285
+ f"subtitle track for {source_id} is danmaku/live-chat, not narration; "
286
+ "use the audio/ASR path instead"
287
+ )
288
+ if not is_narration_track(desc.language or "", EXT_TO_FORMAT.get(desc.format, desc.format)):
289
+ raise AnalysisBlockedError(
290
+ f"subtitle track for {source_id} is not narration (language={desc.language!r})"
291
+ )
292
+
293
+ payload = desc.inline_data
294
+ if payload is None:
295
+ if not desc.source_url:
296
+ raise ContentIntegrityError("subtitle descriptor has neither inline data nor a url")
297
+ payload = (fetch or _default_fetch)(desc.source_url)
298
+ if not isinstance(payload, str) or not payload.strip():
299
+ raise ContentIntegrityError(f"subtitle payload for {source_id} is empty")
300
+
301
+ events = _extract_events(payload, desc.format)
302
+ blocks = [
303
+ AcquiredBlock(
304
+ kind="caption",
305
+ raw_text=text,
306
+ cleaned_text=None,
307
+ language=desc.language,
308
+ order=index,
309
+ locator=TimeRangeLocator(
310
+ time=TimeRange(start=round(start, 3), end=round(end, 3))
311
+ ),
312
+ )
313
+ for index, (text, start, end) in enumerate(events)
314
+ ]
315
+ if not blocks:
316
+ raise ContentIntegrityError(f"subtitle payload for {source_id} contains no caption text")
317
+
318
+ return ParsedSubtitle(
319
+ source_kind=desc.kind,
320
+ language=desc.language,
321
+ format=desc.format,
322
+ blocks=blocks,
323
+ processor=_processor_for(desc.kind, desc.format),
324
+ )
325
+
326
+
327
+ def _processor_for(kind: SubtitleKind, fmt: SubtitleFormat) -> SubtitleProcessor:
328
+ if kind == "manual":
329
+ return "subtitle_manual"
330
+ if kind == "automatic":
331
+ return "subtitle_auto"
332
+ if fmt == "json3":
333
+ return "subtitle_json3"
334
+ if fmt == "vtt":
335
+ return "subtitle_vtt"
336
+ # SRT and the minimal plain timing form are both cue-based text.
337
+ return "subtitle_srt"
338
+
339
+
340
+ # ---------------------------------------------------------------------------
341
+ # Event extraction per format
342
+ # ---------------------------------------------------------------------------
343
+
344
+
345
+ def _extract_events(payload: str, fmt: SubtitleFormat) -> list[tuple[str, float, float]]:
346
+ if not payload.strip():
347
+ raise ContentIntegrityError("subtitle payload is empty")
348
+ if fmt == "json3":
349
+ events = parse_json3_events(payload)
350
+ elif fmt in {"srt", "vtt"}:
351
+ _reject_obvious_truncation(payload, fmt)
352
+ events = _parse_ssa(payload, fmt)
353
+ if fmt == "vtt":
354
+ events = _dedupe_rolling_captions(events)
355
+ elif fmt == "plain":
356
+ events = _parse_plain(payload)
357
+ else: # pragma: no cover - the Literal already constrains callers
358
+ raise ContentIntegrityError(f"unsupported subtitle format: {fmt}")
359
+ if not events:
360
+ raise ContentIntegrityError(f"{fmt} subtitle contains no caption text")
361
+ _validate_events(events)
362
+ return events
363
+
364
+
365
+ def parse_json3_events(payload: str | Mapping[str, object]) -> list[tuple[str, float, float]]:
366
+ """JSON3 ``events`` -> ``(text, start_s, end_s)`` with ms precision."""
367
+ if isinstance(payload, str):
368
+ try:
369
+ data: object = json.loads(payload)
370
+ except json.JSONDecodeError as exc:
371
+ raise ContentIntegrityError(f"invalid JSON3 subtitle: {exc.msg}") from exc
372
+ else:
373
+ data = payload
374
+ if not isinstance(data, Mapping) or not isinstance(data.get("events"), list):
375
+ raise ContentIntegrityError("invalid JSON3 subtitle: missing events list")
376
+
377
+ events: list[tuple[str, float, float]] = []
378
+ for event in data["events"]:
379
+ if not isinstance(event, Mapping):
380
+ continue
381
+ segs = event.get("segs")
382
+ if not isinstance(segs, list):
383
+ continue
384
+ text = _normalize_text(
385
+ "".join(str(seg.get("utf8", "")) for seg in segs if isinstance(seg, Mapping))
386
+ )
387
+ if not text:
388
+ continue
389
+ try:
390
+ start_ms = float(event.get("tStartMs", 0.0) or 0.0)
391
+ duration_ms = float(event.get("dDurationMs", 0.0) or 0.0)
392
+ except (TypeError, ValueError) as exc:
393
+ raise ContentIntegrityError("invalid JSON3 subtitle event timing") from exc
394
+ if start_ms < 0 or duration_ms < 0:
395
+ raise ContentIntegrityError("JSON3 subtitle event has a negative timestamp")
396
+ start = round(start_ms / 1000.0, 3)
397
+ end = round(start + duration_ms / 1000.0, 3)
398
+ events.append((text, start, end))
399
+ return events
400
+
401
+
402
+ def _load_pysubs2() -> object | None:
403
+ """Import ``pysubs2`` lazily; ``None`` means "use the fallback parser"."""
404
+ try:
405
+ import pysubs2 # noqa: PLC0415
406
+ except Exception: # pragma: no cover - optional dependency
407
+ return None
408
+ return pysubs2
409
+
410
+
411
+ def _parse_ssa(payload: str, fmt: SubtitleFormat) -> list[tuple[str, float, float]]:
412
+ pysubs2 = _load_pysubs2()
413
+ if pysubs2 is None:
414
+ return _fallback_cues(payload, fmt)
415
+
416
+ format_name = "srt" if fmt == "srt" else "vtt"
417
+ try:
418
+ subs = pysubs2.SSAFile.from_string(payload, format_=format_name) # type: ignore[attr-defined]
419
+ except Exception as exc: # pysubs2 raises its own FormatAutodetectionError subclasses
420
+ raise ContentIntegrityError(f"invalid {fmt.upper()} subtitle: {exc}") from exc
421
+
422
+ events: list[tuple[str, float, float]] = []
423
+ for line in subs:
424
+ text = _normalize_text(getattr(line, "plaintext", "") or "")
425
+ if not text:
426
+ continue
427
+ events.append((text, line.start / 1000.0, line.end / 1000.0))
428
+ return events
429
+
430
+
431
+ def _fallback_cues(payload: str, fmt: SubtitleFormat) -> list[tuple[str, float, float]]:
432
+ """Minimal SRT/VTT reader used when ``pysubs2`` is not installed."""
433
+ if fmt == "vtt":
434
+ payload = re.sub(r"(?m)^WEBVTT[^\n]*\n", "", payload, count=1)
435
+ events: list[tuple[str, float, float]] = []
436
+ for match in _FALLBACK_CUE_RE.finditer(payload):
437
+ text = _strip_cue_markup(match.group("text"))
438
+ text = _normalize_text(text)
439
+ if not text:
440
+ continue
441
+ start = _parse_timestamp(match.group("start"))
442
+ end = _parse_timestamp(match.group("end"))
443
+ events.append((text, start, end))
444
+ if not events and "-->" in payload:
445
+ raise ContentIntegrityError(f"{fmt.upper()} subtitle appears truncated or malformed")
446
+ return events
447
+
448
+
449
+ def _parse_timestamp(value: str) -> float:
450
+ normalized = value.strip().replace(",", ".")
451
+ parts = normalized.split(":")
452
+ try:
453
+ numbers = [float(part) for part in parts]
454
+ except ValueError as exc: # pragma: no cover - regex constrains the shape
455
+ raise ContentIntegrityError(f"invalid subtitle timestamp: {value!r}") from exc
456
+ if len(numbers) == 3:
457
+ hours, minutes, seconds = numbers[0], numbers[1], numbers[2]
458
+ elif len(numbers) == 2:
459
+ hours, minutes, seconds = 0.0, numbers[0], numbers[1]
460
+ else: # pragma: no cover - regex constrains the shape
461
+ raise ContentIntegrityError(f"invalid subtitle timestamp: {value!r}")
462
+ return hours * 3600.0 + minutes * 60.0 + seconds
463
+
464
+
465
+ def _strip_cue_markup(text: str) -> str:
466
+ # Generic type parameters and comparisons are source knowledge, not HTML.
467
+ text = re.sub(r"</?(?:b|i|u|ruby|rt)\s*>|</?c(?:\.[\w-]+)*\s*>|<v(?:\s+[^>]*)?>|</v>|<lang(?:\s+[^>]*)?>|</lang>|<font(?:\s+[^>]*)?>|</font>|<(?:\d{2}:)?\d{2}:\d{2}\.\d{3}>", "", text, flags=re.IGNORECASE)
468
+ text = re.sub(r"\{\\[^}]*\}", "", text) # ASS override tags
469
+ return html.unescape(text)
470
+
471
+
472
+ def _parse_plain(payload: str) -> list[tuple[str, float, float]]:
473
+ events: list[tuple[str, float, float]] = []
474
+ for line in payload.splitlines():
475
+ line = line.strip()
476
+ if not line:
477
+ continue
478
+ parts = line.split("\t")
479
+ if len(parts) < 3:
480
+ raise ContentIntegrityError(
481
+ "plain subtitle lines need explicit timing: 'start<TAB>end<TAB>text'"
482
+ )
483
+ raw_start, raw_end = parts[0].strip(), parts[1].strip()
484
+ text = _normalize_text("\t".join(parts[2:]))
485
+ if not text:
486
+ continue
487
+ try:
488
+ start, end = float(raw_start), float(raw_end)
489
+ except ValueError as exc:
490
+ raise ContentIntegrityError(
491
+ "plain subtitle lines need explicit timing: 'start<TAB>end<TAB>text'"
492
+ ) from exc
493
+ events.append((text, start, end))
494
+ return events
495
+
496
+
497
+ def _reject_obvious_truncation(payload: str, fmt: SubtitleFormat) -> None:
498
+ """Reject a cleared-down cue header: the last timed cue has no text after it."""
499
+ lines = payload.splitlines()
500
+ last_cue = max((index for index, line in enumerate(lines) if "-->" in line), default=-1)
501
+ if last_cue < 0:
502
+ if payload.strip():
503
+ raise ContentIntegrityError(f"{fmt.upper()} subtitle has no timed cues")
504
+ return
505
+ trailing = [line for line in lines[last_cue + 1 :] if line.strip()]
506
+ if not trailing:
507
+ raise ContentIntegrityError(
508
+ f"subtitle appears truncated: the last {fmt.upper()} cue has no text"
509
+ )
510
+
511
+
512
+ def _dedupe_rolling_captions(
513
+ events: list[tuple[str, float, float]],
514
+ ) -> list[tuple[str, float, float]]:
515
+ """Drop the repeated prefix YouTube-style rolling captions prepend."""
516
+ result: list[tuple[str, float, float]] = []
517
+ previous: str | None = None
518
+ for raw_text, start, end in events:
519
+ text = raw_text
520
+ if previous:
521
+ if text == previous:
522
+ text = ""
523
+ elif text.startswith(previous + "\n"):
524
+ text = text[len(previous) + 1 :]
525
+ elif text.startswith(previous + " "):
526
+ text = text[len(previous) + 1 :]
527
+ previous = raw_text
528
+ text = _normalize_text(text)
529
+ if not text:
530
+ continue
531
+ result.append((text, start, end))
532
+ return result
533
+
534
+
535
+ def _validate_events(events: list[tuple[str, float, float]]) -> None:
536
+ previous_start = -1.0
537
+ for text, start, end in events:
538
+ if not text.strip():
539
+ raise ContentIntegrityError("subtitle cue text must not be blank")
540
+ if start < 0 or end < 0:
541
+ raise ContentIntegrityError("subtitle cue has a negative timestamp")
542
+ if end < start:
543
+ raise ContentIntegrityError("subtitle cue end is before its start")
544
+ if start < previous_start:
545
+ # Out-of-order cues are tolerated by sorting later, but a backwards
546
+ # jump of a whole minute usually means two files were concatenated.
547
+ if previous_start - start > 60.0:
548
+ raise ContentIntegrityError("subtitle cues are not in chronological order")
549
+ previous_start = max(previous_start, start)
550
+ events.sort(key=lambda event: event[1])
551
+
552
+
553
+ def _normalize_text(text: str) -> str:
554
+ text = text.replace("\r\n", "\n").replace("\r", "\n")
555
+ text = "\n".join(" ".join(line.split()) for line in text.split("\n"))
556
+ return text.strip()
557
+
558
+
559
+ def _default_fetch(url: str, *, timeout: float = 30.0) -> str:
560
+ """Bounded GET used in production only; unit tests inject ``fetch``."""
561
+
562
+ parts = urlsplit(url)
563
+ if parts.scheme not in {"http", "https"} or not parts.hostname:
564
+ raise ContentIntegrityError(f"refusing to fetch a non-http(s) subtitle url: {parts.scheme}")
565
+ from ytk.ingestion.http_client import make_client
566
+
567
+ with make_client(url=url, timeout=timeout) as client:
568
+ response = client.get(url, headers={"User-Agent": "ytk/2.0 (+subtitles)"})
569
+ status = getattr(response, "status_code", 200)
570
+ if status in (401, 403):
571
+ from ytk.errors import AnalysisBlockedError
572
+
573
+ raise AnalysisBlockedError(
574
+ f"subtitle URL rejected the request (HTTP {status}); access/permission blocked "
575
+ "(AUTH_REQUIRED)"
576
+ )
577
+ if status == 429:
578
+ from ytk.errors import AnalysisBlockedError
579
+
580
+ raise AnalysisBlockedError(
581
+ "subtitle endpoint returned HTTP 429 (RATE_LIMITED); retry after the platform "
582
+ "cooldown, this is not a missing-subtitle condition"
583
+ )
584
+ response.raise_for_status()
585
+ return response.text
586
+
587
+
588
+ __all__ = [
589
+ "EXT_TO_FORMAT",
590
+ "FORMAT_RANK",
591
+ "ParsedSubtitle",
592
+ "SubtitleFormat",
593
+ "SubtitleKind",
594
+ "SubtitleProcessor",
595
+ "TrackInfo",
596
+ "is_narration_track",
597
+ "language_matches",
598
+ "merge_adjacent_blocks",
599
+ "parse_json3_events",
600
+ "parse_subtitle",
601
+ "select_track",
602
+ ]