kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
@@ -0,0 +1,212 @@
1
+ """Transcript bundle cache — repeat runs of the same video skip the network.
2
+
3
+ Pattern source: github/blog/WriteHERE ``recursive/cache.py`` (hash-keyed
4
+ cache, atomic writes). A live fetch walks the whole provider chain
5
+ (transcript-api → yt-dlp subtitles → whisper-cpp ASR → MiniMax video
6
+ understanding); any of those can be slow, flaky, or metered. The cache
7
+ key is the video id — YouTube captions are effectively immutable — with
8
+ the requested language tuple stored in the envelope so a different
9
+ ``YTK_TRANSCRIPT_LANGUAGES`` never serves a stale variant.
10
+
11
+ Failure policy: a cache that cannot be read (missing, corrupt, wrong
12
+ schema) is a MISS, never an error — the cache must not make extraction
13
+ less reliable than it already was.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import json
18
+ import logging
19
+ import os
20
+ import tempfile
21
+ from pathlib import Path
22
+
23
+ from ytk.models.transcript import TranscriptBundle
24
+
25
+ logger = logging.getLogger(__name__)
26
+
27
+ _SCHEMA = 1
28
+ _DEFAULT_CACHE_DIR = Path.home() / ".cache" / "ytk" / "transcripts"
29
+ # Bundles from these sources are config/model-dependent, never cached.
30
+ _NON_CACHEABLE_SOURCES = frozenset({"asr", "minimax"})
31
+ # Rough bound: ~500 caption JSONs; each is typically < 1 MB.
32
+ _MAX_ENTRIES = 500
33
+
34
+
35
+ class TranscriptCache:
36
+ """File-per-video JSON cache with atomic writes."""
37
+
38
+ def __init__(
39
+ self,
40
+ cache_dir: Path | str | None = None,
41
+ *,
42
+ enabled: bool = True,
43
+ ) -> None:
44
+ self._cache_dir = Path(cache_dir) if cache_dir is not None else _DEFAULT_CACHE_DIR
45
+ self._enabled = enabled
46
+
47
+ def get(self, video_id: str, requested_languages: tuple[str, ...]) -> TranscriptBundle | None:
48
+ if not self._enabled:
49
+ return None
50
+ path = self._path_for(video_id)
51
+ try:
52
+ envelope = json.loads(path.read_text(encoding="utf-8"))
53
+ except FileNotFoundError:
54
+ return None
55
+ except (OSError, json.JSONDecodeError) as exc:
56
+ logger.warning("transcript cache read failed (%s): treating as miss", exc)
57
+ return None
58
+ if not isinstance(envelope, dict) or envelope.get("schema") != _SCHEMA:
59
+ return None
60
+ if tuple(envelope.get("requested_languages") or ()) != tuple(requested_languages):
61
+ return None
62
+ try:
63
+ bundle = TranscriptBundle.model_validate(envelope.get("bundle"))
64
+ except Exception as exc: # pydantic ValidationError et al.
65
+ logger.warning("transcript cache payload invalid (%s): treating as miss", exc)
66
+ return None
67
+ if bundle.video_id != video_id:
68
+ # Sanitizer collisions (e.g. punctuation-only ids collapsing to
69
+ # the same file) must never serve another video's transcript
70
+ # (adversarial review 2026-08-15 finding).
71
+ logger.warning(
72
+ "transcript cache identity mismatch: file %s holds %s — treating as miss",
73
+ video_id,
74
+ bundle.video_id,
75
+ )
76
+ return None
77
+ return bundle
78
+
79
+ def put(self, bundle: TranscriptBundle, requested_languages: tuple[str, ...]) -> Path | None:
80
+ if not self._enabled:
81
+ return None
82
+ # Deterministic caption sources only (review P1-5): ASR/MiniMax
83
+ # bundles depend on model/config — caching them would serve stale
84
+ # output forever after a whisper upgrade or when real captions land.
85
+ if bundle.transcript_source in _NON_CACHEABLE_SOURCES:
86
+ logger.info(
87
+ "transcript cache: skipping non-cacheable source=%s for %s",
88
+ bundle.transcript_source,
89
+ bundle.video_id,
90
+ )
91
+ return None
92
+ envelope = {
93
+ "schema": _SCHEMA,
94
+ "requested_languages": list(requested_languages),
95
+ "bundle": json.loads(bundle.model_dump_json()),
96
+ }
97
+ payload = json.dumps(envelope, ensure_ascii=False)
98
+ path = self._path_for(bundle.video_id)
99
+ # Failure policy on the write side too (review P1-1/P2-1): mkdir and
100
+ # mkstemp are inside the guard — an unwritable cache dir must degrade
101
+ # to "no cache", never crash the pipeline.
102
+ try:
103
+ self._cache_dir.mkdir(parents=True, exist_ok=True)
104
+ # Atomic write: temp file in the same directory, then os.replace —
105
+ # a concurrent reader sees either the old file or the new one,
106
+ # never a half-written payload.
107
+ # Sanitized id in the temp prefix too: the RAW id may contain
108
+ # path-hostile chars (mkstemp would fail → silent no-cache).
109
+ safe_id = self._path_for(bundle.video_id).stem
110
+ fd, tmp_name = tempfile.mkstemp(
111
+ dir=self._cache_dir, prefix=f".{safe_id}.", suffix=".tmp"
112
+ )
113
+ try:
114
+ with os.fdopen(fd, "w", encoding="utf-8") as handle:
115
+ handle.write(payload)
116
+ os.replace(tmp_name, path)
117
+ except OSError:
118
+ try:
119
+ os.unlink(tmp_name)
120
+ except OSError:
121
+ pass
122
+ raise
123
+ except OSError:
124
+ logger.warning("transcript cache write failed for %s", bundle.video_id)
125
+ return None
126
+ self._evict_if_needed()
127
+ return path
128
+
129
+ def _evict_if_needed(self) -> None:
130
+ """Keep the cache bounded (review P2-3): newest ``_MAX_ENTRIES`` by
131
+ mtime survive; older files are removed best-effort."""
132
+ try:
133
+ entries = sorted(
134
+ (p for p in self._cache_dir.glob("*.json")),
135
+ key=lambda p: p.stat().st_mtime,
136
+ reverse=True,
137
+ )
138
+ for stale in entries[_MAX_ENTRIES:]:
139
+ stale.unlink(missing_ok=True)
140
+ except OSError:
141
+ logger.warning("transcript cache eviction failed (ignored)")
142
+
143
+ def stats(self) -> dict:
144
+ """Introspection for `ytk cache status`: entries, bytes, per-file detail.
145
+
146
+ Best-effort: unreadable files are skipped, never raise.
147
+ """
148
+ detail: list[dict] = []
149
+ total = 0
150
+ try:
151
+ paths = sorted(self._cache_dir.glob("*.json"))
152
+ except OSError:
153
+ paths = []
154
+ for path in paths:
155
+ try:
156
+ envelope = json.loads(path.read_text(encoding="utf-8"))
157
+ size = path.stat().st_size
158
+ # Valid JSON but a non-dict bundle (e.g. "notadict") used to
159
+ # crash stats() — and with it `ytk cache status` AND the whole
160
+ # `ytk doctor` run (adversarial review 2026-08-15 finding:
161
+ # verified end-to-end; one corrupt file bricked doctor).
162
+ if not isinstance(envelope, dict):
163
+ continue
164
+ bundle = envelope.get("bundle", {})
165
+ if not isinstance(bundle, dict):
166
+ continue
167
+ segments = bundle.get("segments", [])
168
+ if not isinstance(segments, list):
169
+ continue # null/garbage segments — not a valid entry
170
+ detail.append(
171
+ {
172
+ "video_id": path.stem,
173
+ "bytes": size,
174
+ "languages": envelope.get("requested_languages", []),
175
+ "source": bundle.get("transcript_source"),
176
+ "segments": len(segments) if isinstance(segments, list) else 0,
177
+ }
178
+ )
179
+ total += size
180
+ except (OSError, json.JSONDecodeError, TypeError, ValueError):
181
+ continue
182
+ return {"dir": str(self._cache_dir), "entries": len(detail), "total_bytes": total, "detail": detail}
183
+
184
+ def clear(self) -> int:
185
+ """Remove every cached entry (and crashed-write .tmp strays).
186
+
187
+ Returns how many cache entries were removed.
188
+ """
189
+ removed = 0
190
+ try:
191
+ paths = list(self._cache_dir.glob("*.json"))
192
+ strays = list(self._cache_dir.glob(".*.tmp"))
193
+ except OSError:
194
+ return 0
195
+ for path in paths:
196
+ try:
197
+ path.unlink()
198
+ removed += 1
199
+ except OSError:
200
+ logger.warning("transcript cache: failed to remove %s", path)
201
+ for stray in strays:
202
+ try:
203
+ stray.unlink()
204
+ except OSError:
205
+ pass
206
+ return removed
207
+
208
+ def _path_for(self, video_id: str) -> Path:
209
+ # The URL parser validates LENGTH only — this sanitizer is the actual
210
+ # traversal defense (strips '.', '/', unicode oddities accepted).
211
+ safe = "".join(c for c in video_id if (c.isascii() and c.isalnum()) or c in "-_")
212
+ return self._cache_dir / f"{safe}.json"
@@ -0,0 +1,53 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from urllib.parse import ParseResult, parse_qs, urlparse
5
+
6
+ YOUTUBE_WATCH_HOSTS = {"www.youtube.com", "youtube.com"}
7
+ YOUTUBE_SHORT_HOSTS = YOUTUBE_WATCH_HOSTS | {"youtu.be"}
8
+ VIDEO_ID_LENGTH = 11
9
+
10
+
11
+ @dataclass(frozen=True)
12
+ class ParsedYouTubeUrl:
13
+ raw_url: str
14
+ canonical_url: str
15
+ video_id: str
16
+
17
+
18
+ def parse_youtube_url(url: str) -> ParsedYouTubeUrl:
19
+ parsed_url = urlparse(url)
20
+ host = parsed_url.netloc.lower()
21
+
22
+ if host in YOUTUBE_WATCH_HOSTS:
23
+ video_id = _parse_www_youtube_video_id(parsed_url)
24
+ elif host == "youtu.be":
25
+ video_id = _parse_short_youtube_video_id(parsed_url)
26
+ else:
27
+ raise ValueError(f"unsupported YouTube host: {host or '<missing>'}")
28
+
29
+ _validate_video_id(video_id)
30
+ canonical_url = f"https://www.youtube.com/watch?v={video_id}"
31
+ return ParsedYouTubeUrl(raw_url=url, canonical_url=canonical_url, video_id=video_id)
32
+
33
+
34
+ def _parse_www_youtube_video_id(parsed_url: ParseResult) -> str:
35
+ path = parsed_url.path.rstrip("/")
36
+ if path == "/watch":
37
+ query = parse_qs(parsed_url.query)
38
+ return query.get("v", [""])[0]
39
+ if path.startswith("/shorts/"):
40
+ return path.split("/", maxsplit=2)[2]
41
+ raise ValueError(f"unsupported YouTube URL path: {parsed_url.path or '/'}")
42
+
43
+
44
+ def _parse_short_youtube_video_id(parsed_url: ParseResult) -> str:
45
+ video_id = parsed_url.path.strip("/")
46
+ if not video_id:
47
+ raise ValueError("missing YouTube video id")
48
+ return video_id
49
+
50
+
51
+ def _validate_video_id(video_id: str) -> None:
52
+ if len(video_id) != VIDEO_ID_LENGTH:
53
+ raise ValueError(f"invalid YouTube video id length: {len(video_id)}")
@@ -0,0 +1,292 @@
1
+ """Bounded local video samples and explicitly attributed user visual observations.
2
+
3
+ No network, model download or automatic visual interpretation occurs here.
4
+ Decoded frame timestamps and file digests bind observations to actual samples;
5
+ reviewing samples never establishes full-video coverage or original remote identity.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import math
11
+ import os
12
+ from pathlib import Path, PurePosixPath, PureWindowsPath
13
+ import re
14
+ import subprocess
15
+ import tempfile
16
+ from typing import Any
17
+
18
+ from ytk.errors import UserFacingError
19
+ from ytk.ingestion.media import sha256_file as _sha256_file
20
+ from ytk.models.content import VideoFrameLocator, make_block_id
21
+
22
+ MAX_FRAMES = 12
23
+ MANIFEST_NAME = 'frames-manifest.json'
24
+ _JSON_LIMIT = 2_000_000
25
+ _SHOWINFO_TIME = re.compile(r'\bn:\s*0\s+pts:\s*[-\d]+\s+pts_time:([-\d.e+]+)')
26
+
27
+
28
+ class VisualInputError(UserFacingError):
29
+ def __init__(self, message: str, *, reason_code: str = 'VISUAL_NOTES_INVALID') -> None:
30
+ super().__init__(message)
31
+ self.reason_code = reason_code
32
+
33
+
34
+ def _local_file(path: Path | str) -> Path:
35
+ candidate = Path(path).expanduser()
36
+ if candidate.name.startswith('.env') or candidate.is_symlink() or not candidate.is_file():
37
+ raise VisualInputError('Provide an existing regular local file, without symbolic links.')
38
+ if candidate.stat().st_size == 0:
39
+ raise VisualInputError('The local file is empty.')
40
+ return candidate.resolve()
41
+
42
+
43
+ def sha256_file(path: Path | str) -> str:
44
+ return _sha256_file(_local_file(path))
45
+
46
+
47
+ def _run(command: list[str], *, timeout: float = 120) -> subprocess.CompletedProcess[str]:
48
+ try:
49
+ result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False)
50
+ except FileNotFoundError:
51
+ raise VisualInputError('Install FFmpeg and ffprobe in this runtime before working with local video.', reason_code='MEDIA_TOOL_UNAVAILABLE') from None
52
+ except subprocess.TimeoutExpired:
53
+ raise VisualInputError('Local media decoding exceeded its time budget.', reason_code='MEDIA_TIMEOUT') from None
54
+ if result.returncode != 0:
55
+ # Decoder stderr can include private file paths or external metadata.
56
+ raise VisualInputError('The local file could not be decoded as readable media.', reason_code='MEDIA_DECODE_FAILED')
57
+ return result
58
+
59
+
60
+ def _number(value: object, *, positive: bool = False) -> float:
61
+ if isinstance(value, bool):
62
+ raise ValueError('boolean is not a media timestamp')
63
+ number = float(str(value))
64
+ if not math.isfinite(number) or number < 0 or (positive and number <= 0):
65
+ raise ValueError('invalid media timestamp')
66
+ return number
67
+
68
+
69
+ def probe_media(media_file: Path | str, *, ffprobe_binary: str = 'ffprobe') -> dict[str, Any]:
70
+ source = _local_file(media_file)
71
+ result = _run([ffprobe_binary, '-v', 'error', '-protocol_whitelist', 'file,pipe',
72
+ '-show_format', '-show_streams', '-of', 'json', str(source)])
73
+ try:
74
+ payload = json.loads(result.stdout)
75
+ streams = [stream for stream in payload.get('streams', []) if isinstance(stream, dict)]
76
+ video = [stream for stream in streams if stream.get('codec_type') == 'video'
77
+ and not (stream.get('disposition') or {}).get('attached_pic')]
78
+ audio = [stream for stream in streams if stream.get('codec_type') == 'audio']
79
+ duration = (payload.get('format') or {}).get('duration')
80
+ if duration is None:
81
+ values = [_number(stream['duration']) for stream in streams if stream.get('duration') is not None]
82
+ duration = max(values) if values else None
83
+ duration = _number(duration, positive=True) if duration is not None else None
84
+ start = (video[0].get('start_time') if video else None) or (payload.get('format') or {}).get('start_time') or 0
85
+ start_time = float(start)
86
+ if not math.isfinite(start_time):
87
+ raise ValueError
88
+ except (ValueError, TypeError, KeyError, AttributeError):
89
+ raise VisualInputError('ffprobe returned invalid local-media metadata.', reason_code='MEDIA_DECODE_FAILED') from None
90
+ return {'duration': duration, 'start_time': start_time,
91
+ 'audio_streams': audio, 'video_streams': video}
92
+
93
+
94
+ def _decode_frame(source: Path, target: Path, requested: float, start_time: float,
95
+ ffmpeg_binary: str = 'ffmpeg') -> float | None:
96
+ result = _run([
97
+ ffmpeg_binary, '-hide_banner', '-nostdin', '-loglevel', 'info',
98
+ '-protocol_whitelist', 'file,pipe', '-copyts', '-ss', f'{requested:.6f}',
99
+ '-i', str(source), '-map', '0:v:0', '-an', '-sn', '-frames:v', '1',
100
+ '-vf', "scale='min(iw,1280)':'min(ih,1280)':force_original_aspect_ratio=decrease,showinfo",
101
+ '-fps_mode', 'passthrough', '-q:v', '2', '-n', str(target),
102
+ ])
103
+ match = _SHOWINFO_TIME.search(result.stderr)
104
+ return float(match[1]) - start_time if target.is_file() and match else None
105
+
106
+
107
+ def export_frames(media_file: Path | str, outdir: Path | str, max_frames: int = 8, *,
108
+ timestamps: list[float] | None = None,
109
+ ffmpeg_binary: str = 'ffmpeg', ffprobe_binary: str = 'ffprobe') -> dict[str, Any]:
110
+ if isinstance(max_frames, bool) or not isinstance(max_frames, int) or not 1 <= max_frames <= MAX_FRAMES:
111
+ raise VisualInputError('Choose between 1 and 12 representative frames.', reason_code='INVALID_FRAME_BUDGET')
112
+ source = _local_file(media_file)
113
+ destination = Path(outdir).expanduser()
114
+ if destination.is_symlink() or (destination.exists() and (not destination.is_dir() or any(destination.iterdir()))):
115
+ raise VisualInputError('Frame export needs a new or empty directory without symbolic links.', reason_code='OUTPUT_UNAVAILABLE')
116
+ info = probe_media(source, ffprobe_binary=ffprobe_binary)
117
+ if not info['video_streams'] or not info['duration']:
118
+ raise VisualInputError('This file has no video stream with a known duration.', reason_code='NO_VIDEO_STREAM')
119
+ duration = info['duration']
120
+ if timestamps is not None:
121
+ if (not 1 <= len(timestamps) <= MAX_FRAMES or any(
122
+ isinstance(stamp,bool) or not isinstance(stamp,(int,float)) or not math.isfinite(stamp)
123
+ or stamp < 0 or stamp >= duration for stamp in timestamps
124
+ ) or sorted(set(timestamps)) != timestamps):
125
+ raise VisualInputError('Choose 1–12 increasing, distinct timestamps inside the video.', reason_code='INVALID_FRAME_TIMESTAMPS')
126
+ video_hash = sha256_file(source)
127
+ count = len(timestamps) if timestamps is not None else max_frames
128
+ frame_count = info['video_streams'][0].get('nb_frames')
129
+ if timestamps is None and frame_count is not None and str(frame_count).isdigit() and int(frame_count) > 0:
130
+ count = min(count, int(frame_count))
131
+ destination.parent.mkdir(parents=True, exist_ok=True)
132
+ with tempfile.TemporaryDirectory(prefix='.kd-frames-', dir=destination.parent) as work:
133
+ stage = Path(work)
134
+ frames: list[dict[str, Any]] = []
135
+ skipped = []
136
+ for index in range(count):
137
+ requested = timestamps[index] if timestamps is not None else duration * index / count
138
+ filename = f'frame_{len(frames):04d}.jpg'
139
+ frame_path = stage / filename
140
+ timestamp = _decode_frame(source, frame_path, requested, info['start_time'], ffmpeg_binary)
141
+ if timestamp is None:
142
+ skipped.append(round(requested, 6))
143
+ continue
144
+ if not math.isfinite(timestamp) or timestamp < -0.001 or timestamp >= duration:
145
+ raise VisualInputError('Decoded frame timestamp does not belong to this video.', reason_code='FRAME_TIMESTAMP_INVALID')
146
+ image_info = probe_media(frame_path, ffprobe_binary=ffprobe_binary)
147
+ image = image_info['video_streams'][0] if image_info['video_streams'] else {}
148
+ if not image.get('width') or not image.get('height'):
149
+ raise VisualInputError('The decoded representative frame is unreadable.', reason_code='MEDIA_DECODE_FAILED')
150
+ frames.append({'index': len(frames), 'file': filename,
151
+ 'timestamp': round(max(timestamp, 0), 6), 'requested_timestamp': round(requested, 6),
152
+ 'sha256': sha256_file(frame_path), 'width': image['width'], 'height': image['height']})
153
+ if not frames or sha256_file(source) != video_hash:
154
+ raise VisualInputError('No reliable frame samples were produced, or the input changed.', reason_code='FRAME_EXPORT_FAILED')
155
+ manifest = {'schema_version': 1, 'kind': 'video_frame_samples', 'video_sha256': video_hash,
156
+ 'duration': duration, 'sampling': ('specified_targets_actual_decoded_timestamps' if timestamps is not None else 'uniform_targets_actual_decoded_timestamps'),
157
+ 'sampled': True, 'full_video_reviewed': False, 'max_frames': len(timestamps) if timestamps is not None else max_frames,
158
+ 'skipped_targets': skipped, 'frames': frames}
159
+ (stage / MANIFEST_NAME).write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
160
+ # All decoding completed before publishing; the manifest is moved last.
161
+ if destination.exists() and (destination.is_symlink() or any(destination.iterdir())):
162
+ raise VisualInputError('The output directory changed during export.', reason_code='OUTPUT_UNAVAILABLE')
163
+ destination.mkdir(parents=True, exist_ok=True)
164
+ owned = []
165
+ try:
166
+ for name in [*(frame['file'] for frame in frames), MANIFEST_NAME]:
167
+ target = destination / name
168
+ if target.exists() or target.is_symlink():
169
+ raise VisualInputError('A frame output already exists.', reason_code='OUTPUT_UNAVAILABLE')
170
+ os.replace(stage / name, target)
171
+ owned.append(target)
172
+ except (OSError, VisualInputError):
173
+ for target in owned:
174
+ target.unlink(missing_ok=True)
175
+ raise VisualInputError('The frame package could not be saved; existing files were preserved.', reason_code='OUTPUT_UNAVAILABLE') from None
176
+ return manifest
177
+
178
+
179
+ def _json_file(path: Path) -> dict[str, Any]:
180
+ source = _local_file(path)
181
+ if source.stat().st_size > _JSON_LIMIT:
182
+ raise VisualInputError('The visual review JSON exceeds the size limit.')
183
+ try:
184
+ payload = json.loads(source.read_text(encoding='utf-8'))
185
+ except (UnicodeError, ValueError):
186
+ raise VisualInputError('The visual review file is not valid UTF-8 JSON.') from None
187
+ if not isinstance(payload, dict):
188
+ raise VisualInputError('The visual review JSON must contain an object.')
189
+ return payload
190
+
191
+
192
+ def _relative_file(root: Path, value: object) -> Path:
193
+ if not isinstance(value, str) or not value or '\\' in value:
194
+ raise VisualInputError('A frame-package file path is invalid.')
195
+ relative = PurePosixPath(value)
196
+ if relative.is_absolute() or PureWindowsPath(value).drive or '..' in relative.parts:
197
+ raise VisualInputError('Frame-package paths must stay inside their package directory.')
198
+ candidate = root
199
+ for component in relative.parts:
200
+ candidate = candidate / component
201
+ if candidate.is_symlink():
202
+ raise VisualInputError('Symbolic links are not valid frame-package files.')
203
+ resolved = _local_file(candidate)
204
+ if not resolved.is_relative_to(root.resolve()):
205
+ raise VisualInputError('A frame-package path escapes its directory.')
206
+ return resolved
207
+
208
+
209
+ def load_visual_notes(media_file: Path | str, notes_file: Path | str) -> tuple[list[dict[str, Any]], dict[str, Any]]:
210
+ """Validate user-provided sample observations; never infer unseen content."""
211
+ source = _local_file(media_file)
212
+ review_path = _local_file(notes_file)
213
+ notes = _json_file(review_path)
214
+ manifest_path = _relative_file(review_path.parent, notes.get('frames_manifest'))
215
+ manifest = _json_file(manifest_path)
216
+ video_hash = sha256_file(source)
217
+ reviewer = notes.get('reviewer')
218
+ if not isinstance(reviewer, str) or not reviewer.strip() or len(reviewer) > 256:
219
+ raise VisualInputError('Name the person or model providing the sample observations.')
220
+ if video_hash != notes.get('video_sha256') or video_hash != manifest.get('video_sha256'):
221
+ raise VisualInputError('The visual review or frame package belongs to a different video.')
222
+ try:
223
+ if manifest.get('schema_version') != 1 or manifest.get('kind') != 'video_frame_samples' or manifest.get('sampled') is not True:
224
+ raise ValueError
225
+ duration = _number(manifest['duration'], positive=True)
226
+ media_info = probe_media(source)
227
+ actual_duration = media_info['duration']
228
+ if actual_duration is None or abs(duration-actual_duration) > 0.01:
229
+ raise ValueError
230
+ frames = manifest['frames']
231
+ if not isinstance(frames, list) or not 1 <= len(frames) <= MAX_FRAMES:
232
+ raise ValueError
233
+ indexed = {}
234
+ frame_paths: dict[int, Path] = {}
235
+ for frame in frames:
236
+ index = frame['index']
237
+ if isinstance(index, bool) or not isinstance(index, int) or index < 0 or index in indexed:
238
+ raise ValueError
239
+ timestamp = _number(frame['timestamp'])
240
+ if timestamp >= duration:
241
+ raise ValueError
242
+ image = _relative_file(manifest_path.parent, frame['file'])
243
+ with image.open('rb') as handle:
244
+ image_header = handle.read(2)
245
+ if image.suffix.lower() != '.jpg' or image.stat().st_size > 10_000_000 or image_header != b'\xff\xd8' or sha256_file(image) != frame['sha256']:
246
+ raise ValueError
247
+ indexed[index] = frame
248
+ frame_paths[index] = image
249
+ observations = notes['observations']
250
+ if not isinstance(observations, list) or not 1 <= len(observations) <= len(frames):
251
+ raise ValueError
252
+ blocks: list[dict[str, Any]] = []
253
+ seen = set()
254
+ source_id = f'local:{video_hash}'
255
+ for observation in observations:
256
+ index = observation['frame_index']
257
+ text = observation['text']
258
+ if isinstance(index, bool) or not isinstance(index, int) or index not in indexed or index in seen:
259
+ raise ValueError
260
+ if not isinstance(text, str) or not text.strip() or len(text) > 8000:
261
+ raise ValueError
262
+ seen.add(index)
263
+ frame = indexed[index]
264
+ # Bind a reviewed frame's claimed time and bytes to an actual decode
265
+ # of the supplied video, rather than trusting editable JSON fields.
266
+ requested = _number(frame['requested_timestamp'])
267
+ if requested >= duration:
268
+ raise ValueError
269
+ with tempfile.TemporaryDirectory(prefix='kd-verify-frame-') as work:
270
+ replay = Path(work) / 'frame.jpg'
271
+ actual_time = _decode_frame(source, replay, requested, media_info['start_time'])
272
+ if actual_time is None or abs(actual_time-frame['timestamp']) > 0.00001 or sha256_file(replay) != frame['sha256']:
273
+ raise ValueError
274
+ image_streams = probe_media(replay)['video_streams']
275
+ if not image_streams or frame.get('width') != image_streams[0].get('width') or frame.get('height') != image_streams[0].get('height'):
276
+ raise ValueError
277
+ locator = VideoFrameLocator(time=frame['timestamp'], image_width=frame.get('width'), image_height=frame.get('height'))
278
+ blocks.append({'block_id': make_block_id(source_id, locator, text), 'source_id': source_id,
279
+ 'kind': 'ocr', 'raw_text': text, 'order': len(blocks), 'locator': locator.model_dump(exclude_none=True)})
280
+ except (KeyError, TypeError, ValueError, AttributeError):
281
+ raise VisualInputError('The visual review has invalid frames, timestamps, observations, or file digests.') from None
282
+ return blocks, {'acquisition_mode': 'user_import', 'video_sha256': video_hash,
283
+ 'sampled': True, 'original_source_verified': False,
284
+ 'frame_assets': [{'local_path': str(frame_paths[index]),
285
+ 'media_type': 'frame', 'order': index} for index in sorted(seen)],
286
+ 'coverage': {'visuals': 'sampled_user_review'},
287
+ 'visual_review': {'reviewer': reviewer.strip(), 'frames_reviewed': sorted(seen),
288
+ 'frames_available': len(frames), 'manifest_sha256': sha256_file(manifest_path),
289
+ 'interpretation': 'user_provided_observations_not_automatic_ocr'}}
290
+
291
+
292
+ __all__ = ['export_frames', 'load_visual_notes', 'probe_media', 'sha256_file']