vidX 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. vidx/__init__.py +0 -0
  2. vidx/analyzers/__init__.py +78 -0
  3. vidx/analyzers/audio.py +125 -0
  4. vidx/analyzers/audiomap.py +163 -0
  5. vidx/analyzers/captions.py +63 -0
  6. vidx/analyzers/detail.py +146 -0
  7. vidx/analyzers/glitch.py +106 -0
  8. vidx/analyzers/laugh.py +112 -0
  9. vidx/analyzers/probe.py +60 -0
  10. vidx/analyzers/shots.py +269 -0
  11. vidx/analyzers/transcript.py +352 -0
  12. vidx/analyzers/visual.py +189 -0
  13. vidx/api.py +190 -0
  14. vidx/assets/sfx/drone_01.m4a +0 -0
  15. vidx/assets/sfx/drone_02.m4a +0 -0
  16. vidx/assets/sfx/drone_03.m4a +0 -0
  17. vidx/assets/sfx/drone_04.m4a +0 -0
  18. vidx/assets/sfx/drone_05.m4a +0 -0
  19. vidx/assets/sfx/drone_06.m4a +0 -0
  20. vidx/assets/sfx/impact_01.m4a +0 -0
  21. vidx/assets/sfx/impact_02.m4a +0 -0
  22. vidx/assets/sfx/impact_03.m4a +0 -0
  23. vidx/assets/sfx/impact_04.m4a +0 -0
  24. vidx/assets/sfx/impact_05.m4a +0 -0
  25. vidx/assets/sfx/impact_06.m4a +0 -0
  26. vidx/assets/sfx/riser_01.m4a +0 -0
  27. vidx/assets/sfx/riser_02.m4a +0 -0
  28. vidx/assets/sfx/riser_03.m4a +0 -0
  29. vidx/assets/sfx/riser_04.m4a +0 -0
  30. vidx/assets/sfx/riser_05.m4a +0 -0
  31. vidx/assets/sfx/whoosh_01.m4a +0 -0
  32. vidx/assets/sfx/whoosh_02.m4a +0 -0
  33. vidx/assets/sfx/whoosh_03.m4a +0 -0
  34. vidx/checks.py +161 -0
  35. vidx/cli.py +188 -0
  36. vidx/color.py +95 -0
  37. vidx/config.py +51 -0
  38. vidx/cuts.py +224 -0
  39. vidx/detect.py +199 -0
  40. vidx/diff.py +152 -0
  41. vidx/dsl.py +571 -0
  42. vidx/index.py +93 -0
  43. vidx/mcp_server.py +158 -0
  44. vidx/models.py +18 -0
  45. vidx/otio_export.py +53 -0
  46. vidx/query.py +579 -0
  47. vidx/render.py +600 -0
  48. vidx/store.py +318 -0
  49. vidx/thumbs.py +80 -0
  50. vidx/timeline.py +286 -0
  51. vidx/util.py +119 -0
  52. vidx-0.2.0.dist-info/METADATA +291 -0
  53. vidx-0.2.0.dist-info/RECORD +56 -0
  54. vidx-0.2.0.dist-info/WHEEL +4 -0
  55. vidx-0.2.0.dist-info/entry_points.txt +3 -0
  56. vidx-0.2.0.dist-info/licenses/LICENSE +21 -0
vidx/__init__.py ADDED
File without changes
@@ -0,0 +1,78 @@
1
+ """Pluggable analyzers. Add a new one by appending to ANALYZERS (order matters)."""
2
+ from __future__ import annotations
3
+
4
+ import sys
5
+ import time
6
+ from dataclasses import dataclass, field
7
+ from pathlib import Path
8
+ from typing import Callable, Protocol
9
+
10
+ from .. import store, util
11
+ from ..store import VideoInfo
12
+
13
+
14
+ @dataclass
15
+ class Context:
16
+ force: bool = False
17
+ captions: bool = False
18
+ glitch: bool = False
19
+ log: Callable[[str], None] = lambda m: print(m, file=sys.stderr, flush=True)
20
+ timings: dict[str, float] = field(default_factory=dict)
21
+
22
+ def audio16k(self, video: VideoInfo) -> Path | None:
23
+ if not video.has_audio:
24
+ return None
25
+ return util.extract_audio16k(video.path, video.dir / "audio16k.wav")
26
+
27
+
28
+ @dataclass
29
+ class AnalyzerResult:
30
+ info: dict = field(default_factory=dict) # small facts, stored in analyzer_runs.info
31
+ skipped: str | None = None
32
+
33
+
34
+ class Analyzer(Protocol):
35
+ name: str
36
+ optional: bool # only runs when its flag is on (e.g. captions)
37
+
38
+ def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult: ...
39
+
40
+
41
+ def registry() -> list[Analyzer]:
42
+ from .audio import AudioEvents
43
+ from .audiomap import AudioMap
44
+ from .captions import Captions
45
+ from .detail import Detail
46
+ from .glitch import Glitch
47
+ from .laugh import Laugh
48
+ from .shots import Shots
49
+ from .transcript import Transcript
50
+ from .visual import Visual
51
+
52
+ # shots first (the rest work per shot); laugh after transcript (uses words as a speech mask)
53
+ return [Shots(), Detail(), Visual(), Transcript(), AudioEvents(), AudioMap(), Laugh(), Glitch(), Captions()]
54
+
55
+
56
+ # Per-shot results: these must be redone whenever shot boundaries change.
57
+ SHOT_DEPENDENT = ("detail", "visual", "audiomap", "glitch", "captions")
58
+
59
+
60
+ def run_all(video: VideoInfo, ctx: Context) -> dict[str, float]:
61
+ runs = store.analyzer_runs(video.id)
62
+ for a in registry():
63
+ if a.optional and not getattr(ctx, a.name, False):
64
+ continue
65
+ version = getattr(a, "version", 1)
66
+ if not ctx.force and a.name in runs and runs[a.name].get("v", 1) >= version:
67
+ ctx.log(f" {a.name:<11} cached")
68
+ continue
69
+ t0 = time.perf_counter()
70
+ res = a.run(video, ctx)
71
+ dt = time.perf_counter() - t0
72
+ ctx.timings[a.name] = dt
73
+ store.mark_analyzer(video.id, a.name, dt, {**res.info, "v": version})
74
+ if a.name == "shots": # new boundaries invalidate everything stored per shot
75
+ for name in SHOT_DEPENDENT:
76
+ runs.pop(name, None)
77
+ ctx.log(f" {a.name:<11} {dt:6.1f}s" + (f" ({res.skipped})" if res.skipped else ""))
78
+ return ctx.timings
@@ -0,0 +1,125 @@
1
+ """silence / loud / black / freeze events from ONE ffmpeg decode pass."""
2
+ from __future__ import annotations
3
+
4
+ import re
5
+ import subprocess
6
+ import sys
7
+
8
+ from .. import config, store
9
+ from ..store import Event, VideoInfo
10
+ from . import AnalyzerResult, Context
11
+
12
+ KINDS = ["silence", "loud", "black", "freeze"]
13
+ _num = r"(-?[\d.]+(?:e-?\d+)?)"
14
+
15
+
16
+ def detect_cmd(path: str, has_audio: bool, hwaccel: bool = True, video: bool = True,
17
+ black_min: float = config.BLACK_MIN, freeze_min: float = config.FREEZE_MIN) -> list[str]:
18
+ cmd = ["ffmpeg", "-nostdin", "-hide_banner", "-nostats"]
19
+ if hwaccel and sys.platform == "darwin":
20
+ cmd += ["-hwaccel", "videotoolbox"]
21
+ cmd += ["-i", path]
22
+ if video:
23
+ cmd += ["-vf", f"scale=320:-2,cropdetect=limit=24:round=2:reset=1,blackdetect=d={black_min}:pix_th=0.10,"
24
+ f"freezedetect=n=-60dB:d={freeze_min}"]
25
+ else:
26
+ cmd += ["-vn"]
27
+ if has_audio:
28
+ cmd += ["-af", f"silencedetect=n={config.SILENCE_DB}dB:d={config.SILENCE_MIN},ebur128=framelog=info"]
29
+ else:
30
+ cmd += ["-an"]
31
+ return cmd + ["-f", "null", "-"]
32
+
33
+
34
+ def parse_detect_log(log: str, duration: float) -> tuple[list[Event], dict]:
35
+ ev: list[Event] = []
36
+ # black
37
+ for m in re.finditer(rf"black_start:{_num} black_end:{_num} black_duration:{_num}", log):
38
+ a, b, d = map(float, m.groups())
39
+ ev.append(Event("black", a, b, round(d, 2)))
40
+ # freeze (start / end on separate lines)
41
+ starts = [float(x) for x in re.findall(rf"freeze_start: {_num}", log)]
42
+ ends = [float(x) for x in re.findall(rf"freeze_end: {_num}", log)]
43
+ for i, a in enumerate(starts):
44
+ b = ends[i] if i < len(ends) else duration
45
+ ev.append(Event("freeze", a, b, round(b - a, 2)))
46
+ # silence
47
+ s_starts = [float(x) for x in re.findall(rf"silence_start: {_num}", log)]
48
+ s_ends = [float(x) for x in re.findall(rf"silence_end: {_num}", log)]
49
+ for i, a in enumerate(s_starts):
50
+ a = max(a, 0.0)
51
+ b = s_ends[i] if i < len(s_ends) else duration
52
+ ev.append(Event("silence", a, b, round(b - a, 2)))
53
+ # loudness: momentary every 100ms + integrated summary
54
+ mom = [(float(t), float(m)) for t, m in re.findall(rf"t: {_num}\s+TARGET:\S+ LUFS\s+M:\s*{_num}", log)]
55
+ integ = re.findall(rf"Integrated loudness:\s*I:\s*{_num} LUFS", log)
56
+ lufs = float(integ[-1]) if integ else None
57
+ if mom and lufs is not None and lufs > -70:
58
+ thr = lufs + config.LOUD_OVER_INTEGRATED
59
+ run_a, peak, last = None, -200.0, None
60
+ for t, m in mom + [(duration + 1, -200.0)]:
61
+ t0 = t - 0.4 # momentary window is 400ms ending at t
62
+ if m >= thr:
63
+ if run_a is None:
64
+ run_a, peak = max(t0, 0.0), m
65
+ peak = max(peak, m)
66
+ last = t
67
+ elif run_a is not None and (last is None or t - last > 0.3):
68
+ if last - run_a >= 0.3:
69
+ ev.append(Event("loud", round(run_a, 2), round(last, 2), round(peak, 1)))
70
+ run_a = None
71
+ ev.sort(key=lambda e: (e.start, e.kind))
72
+ return ev, {"lufs": lufs, "crop320": dominant_crop(log)}
73
+
74
+
75
+ def dominant_crop(log: str, min_share: float = 0.6) -> list[int] | None:
76
+ """Per-frame cropdetect -> the letterbox most frames agree on. Title cards / credits that fill
77
+ the frame shouldn't cancel a film's 2.35:1 bars, so height and width are voted separately."""
78
+ from collections import Counter
79
+
80
+ crops = [tuple(map(int, m)) for m in re.findall(r"crop=(\d+):(\d+):(\d+):(\d+)", log)]
81
+ if len(crops) < 10:
82
+ return None
83
+ n = len(crops)
84
+ (h, hc), = Counter(c[1] for c in crops).most_common(1)
85
+ (w, wc), = Counter(c[0] for c in crops).most_common(1)
86
+ if hc / n < min_share or wc / n < min_share:
87
+ return None
88
+ (y, _), = Counter(c[3] for c in crops if c[1] == h).most_common(1)
89
+ (x, _), = Counter(c[2] for c in crops if c[0] == w).most_common(1)
90
+ return [w, h, x, y]
91
+
92
+
93
+ def active_area(crop320: list[int] | None, width: int, height: int) -> list[int] | None:
94
+ """Baked-in letterbox / pillarbox in display pixels [w, h, x, y], or None if the frame is full.
95
+ cropdetect ran on a 320px-wide copy, so scale back up."""
96
+ if not crop320:
97
+ return None
98
+ f = width / 320
99
+ w, h, x, y = (int(round(v * f / 2)) * 2 for v in crop320)
100
+ w, h = min(w, width), min(h, height)
101
+ if w < width * 0.3 or h < height * 0.3: # mostly black video: don't trust it
102
+ return None
103
+ if w >= width * 0.97 and h >= height * 0.97:
104
+ return None
105
+ return [w, h, x, y]
106
+
107
+
108
+ class AudioEvents:
109
+ name = "audio"
110
+ optional = False
111
+
112
+ def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult:
113
+ def go(hw: bool) -> subprocess.CompletedProcess:
114
+ return subprocess.run(detect_cmd(video.path, video.has_audio, hw), capture_output=True, text=True)
115
+
116
+ p = go(True)
117
+ if p.returncode != 0:
118
+ p = go(False)
119
+ events, info = parse_detect_log(p.stderr, video.duration)
120
+ store.put_events(video.id, KINDS, events)
121
+ store.update_meta(video.id, lufs=info["lufs"],
122
+ active=active_area(info.get("crop320"), video.width, video.height))
123
+ counts = {k: sum(e.kind == k for e in events) for k in KINDS}
124
+ info.pop("crop320", None)
125
+ return AnalyzerResult(info={**info, **counts})
@@ -0,0 +1,163 @@
1
+ """Label each stretch of audio as speech / music / sfx, find beats, and add loudness per shot.
2
+
3
+ Speech comes from Silero VAD, which ships inside faster-whisper (offline). Everything else is signal
4
+ processing: sustained tonal sound is music, short noisy bursts are sfx. Silence stays with the audio
5
+ analyzer. Audio is read in chunks, so memory stays flat on long files.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import numpy as np
10
+
11
+ from .. import store, util
12
+ from ..store import Event, VideoInfo
13
+ from . import AnalyzerResult, Context
14
+
15
+ KINDS = ["speech", "music", "sfx", "beat"]
16
+ HOP = 0.02 # 20 ms analysis frames
17
+ WIN = 0.5 # labelling resolution
18
+ SILENT_DB = -50.0
19
+ BPM_RANGE = (70, 180)
20
+
21
+
22
+ def frame_features(x: np.ndarray, sr: int) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
23
+ """Per-frame RMS, spectral flatness and positive spectral flux (the onset signal)."""
24
+ n = int(sr * HOP)
25
+ m = len(x) // n
26
+ if m < 2:
27
+ return np.zeros(0), np.zeros(0), np.zeros(0)
28
+ fr = x[: m * n].reshape(m, n).astype(np.float32)
29
+ rms = np.sqrt((fr.astype(np.float64) ** 2).mean(1) + 1e-12)
30
+ spec = np.abs(np.fft.rfft(fr * np.hanning(n).astype(np.float32), axis=1)) + 1e-7
31
+ flat = np.exp(np.log(spec).mean(1)) / spec.mean(1)
32
+ logs = np.log1p(spec * 100)
33
+ flux = np.maximum(np.diff(logs, axis=0, prepend=logs[:1]), 0).mean(1)
34
+ return rms, flat, flux
35
+
36
+
37
+ def label_windows(rms: np.ndarray, flat: np.ndarray, flux: np.ndarray, speech: np.ndarray) -> list[str]:
38
+ """One label per WIN seconds. speech: per-frame bool from the VAD."""
39
+ per = int(WIN / HOP)
40
+ labels = []
41
+ for i in range(0, len(rms) - per + 1, per):
42
+ sl = slice(i, i + per)
43
+ db = 20 * np.log10(rms[sl].mean() + 1e-9)
44
+ if speech[sl].mean() > 0.5:
45
+ labels.append("speech")
46
+ elif db < SILENT_DB:
47
+ labels.append("silence")
48
+ elif np.median(flat[sl]) < 0.12:
49
+ labels.append("music") # energy concentrated in tones
50
+ else:
51
+ labels.append("sfx") # noisy / transient
52
+ return labels
53
+
54
+
55
+ def smooth(labels: list[str], min_run: int = 3) -> list[str]:
56
+ """Short sfx islands inside music or speech are usually noise in the classifier, except a real
57
+ hit, which survives because it stands between different neighbours."""
58
+ out = list(labels)
59
+ i = 0
60
+ while i < len(out):
61
+ j = i
62
+ while j + 1 < len(out) and out[j + 1] == out[i]:
63
+ j += 1
64
+ if j - i + 1 < min_run and 0 < i and j + 1 < len(out) and out[i - 1] == out[j + 1]:
65
+ for k in range(i, j + 1):
66
+ out[k] = out[i - 1]
67
+ i = j + 1
68
+ return out
69
+
70
+
71
+ def stretches(labels: list[str]) -> list[Event]:
72
+ ev: list[Event] = []
73
+ for i, l in enumerate(labels):
74
+ if l == "silence":
75
+ continue
76
+ a, b = i * WIN, (i + 1) * WIN
77
+ if ev and ev[-1].kind == l and abs(ev[-1].end - a) < 1e-6:
78
+ ev[-1].end = b
79
+ else:
80
+ ev.append(Event(l, a, b, None))
81
+ for e in ev:
82
+ e.value = round(e.end - e.start, 2)
83
+ return ev
84
+
85
+
86
+ def track_beats(flux: np.ndarray) -> tuple[float, float, np.ndarray]:
87
+ """-> (bpm, strength 0-1, beat times). Autocorrelation picks the tempo, then dynamic programming
88
+ (Ellis 2007) places beats on strong onsets at that spacing."""
89
+ if len(flux) < int(8 / HOP):
90
+ return 0.0, 0.0, np.zeros(0)
91
+ env = flux - flux.mean()
92
+ env = np.maximum(env, 0)
93
+ sd = env.std() or 1.0
94
+ env = env / sd
95
+ n = len(env)
96
+ size = 1 << int(np.ceil(np.log2(2 * n)))
97
+ f = np.fft.rfft(env, size)
98
+ ac = np.fft.irfft(f * np.conj(f), size)[:n]
99
+ lo, hi = int(60 / BPM_RANGE[1] / HOP), int(60 / BPM_RANGE[0] / HOP)
100
+ if hi >= n or ac[0] <= 0:
101
+ return 0.0, 0.0, np.zeros(0)
102
+ lag = lo + int(np.argmax(ac[lo:hi]))
103
+ strength = float(ac[lag] / ac[0])
104
+ score = env.copy()
105
+ back = np.full(n, -1)
106
+ span = np.arange(-2 * lag, -lag // 2)
107
+ pen = -100 * np.log(-span / lag) ** 2 / 100
108
+ for i in range(int(lag * 2), n):
109
+ idx = i + span
110
+ cand = score[idx] + pen
111
+ k = int(np.argmax(cand))
112
+ score[i] += cand[k]
113
+ back[i] = idx[k]
114
+ i = int(np.argmax(score[-lag * 2:])) + n - lag * 2
115
+ beats = []
116
+ while i >= 0:
117
+ beats.append(i)
118
+ i = back[i]
119
+ return round(60 / (lag * HOP), 1), round(strength, 2), np.asarray(beats[::-1]) * HOP
120
+
121
+
122
+ class AudioMap:
123
+ name = "audiomap"
124
+ optional = False
125
+
126
+ def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult:
127
+ wav = ctx.audio16k(video)
128
+ if wav is None:
129
+ return AnalyzerResult(skipped="no audio stream")
130
+ from faster_whisper.vad import VadOptions, get_speech_timestamps
131
+
132
+ reader = util.WavReader(wav)
133
+ sr = reader.sr
134
+ parts = []
135
+ speech_spans: list[tuple[float, float]] = []
136
+ for t0, x in reader.chunks(300.0): # 300 s is a multiple of HOP, so frames line up
137
+ parts.append(frame_features(x, sr))
138
+ for s in get_speech_timestamps(x, VadOptions(min_silence_duration_ms=300)):
139
+ speech_spans.append((t0 + s["start"] / sr, t0 + s["end"] / sr))
140
+ del x
141
+ rms, flat, flux = (np.concatenate([p[i] for p in parts]) for i in range(3))
142
+ speech = np.zeros(len(rms), bool)
143
+ for a, b in speech_spans:
144
+ speech[int(a / HOP): int(b / HOP) + 1] = True
145
+ events = stretches(smooth(label_windows(rms, flat, flux, speech)))
146
+
147
+ music_s = sum(e.end - e.start for e in events if e.kind == "music")
148
+ bpm, strength, beats = track_beats(flux)
149
+ keep_beats = strength >= 0.25 and music_s >= 0.1 * reader.duration
150
+ if keep_beats:
151
+ events += [Event("beat", round(float(t), 2), round(float(t) + 0.05, 2), None) for t in beats]
152
+ store.put_events(video.id, KINDS, events)
153
+ store.update_meta(video.id, bpm=bpm if keep_beats else None, beat_strength=strength)
154
+
155
+ db = 20 * np.log10(rms + 1e-9)
156
+ updates = {}
157
+ for s in store.get_shots(video.id):
158
+ seg = db[int(s.start / HOP): max(int(s.start / HOP) + 1, int(s.end / HOP))]
159
+ if len(seg):
160
+ updates[s.idx] = {"audio": {"db": round(float(seg.mean()), 1), "peak_db": round(float(seg.max()), 1)}}
161
+ store.merge_shot_detail(video.id, updates)
162
+ counts = {k: sum(e.kind == k for e in events) for k in KINDS}
163
+ return AnalyzerResult(info={**counts, "bpm": bpm if keep_beats else None})
@@ -0,0 +1,63 @@
1
+ """Optional one-line shot captions with Florence-2-base (`index_video(captions=True)`).
2
+
3
+ Install extras first: `uv sync --extra captions`. Captions take precedence over CLIP tags in
4
+ overview lines. One middle frame per shot.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ from .. import config, store, thumbs
9
+ from ..store import VideoInfo
10
+ from . import AnalyzerResult, Context
11
+
12
+ MODEL_ID = "florence-community/Florence-2-base" # native transformers (>=4.56) port of microsoft/Florence-2-base
13
+ LEGACY_ID = "microsoft/Florence-2-base" # trust_remote_code version for older transformers
14
+
15
+
16
+ def _load():
17
+ import torch
18
+ import transformers
19
+
20
+ dev = config.device()
21
+ dtype = torch.float16 if dev == "cuda" else torch.float32
22
+ if hasattr(transformers, "Florence2ForConditionalGeneration"):
23
+ model = transformers.Florence2ForConditionalGeneration.from_pretrained(MODEL_ID, dtype=dtype)
24
+ proc = transformers.AutoProcessor.from_pretrained(MODEL_ID)
25
+ else:
26
+ model = transformers.AutoModelForCausalLM.from_pretrained(LEGACY_ID, trust_remote_code=True, torch_dtype=dtype)
27
+ proc = transformers.AutoProcessor.from_pretrained(LEGACY_ID, trust_remote_code=True)
28
+ return model.to(dev).eval(), proc, dev, dtype, torch
29
+
30
+
31
+ def _clean(text: str) -> str:
32
+ t = text.replace("</s>", "").replace("<s>", "").replace("<pad>", "").strip()
33
+ for pre in ("The image shows ", "The image is ", "In this image we can see ", "This image shows "):
34
+ if t.startswith(pre):
35
+ t = t[len(pre):]
36
+ t = t.split(". ")[0].rstrip(".")
37
+ return t[:80]
38
+
39
+
40
+ class Captions:
41
+ name = "captions"
42
+ optional = True # runs only when ctx.captions is True
43
+
44
+ def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult:
45
+ try:
46
+ model, proc, dev, dtype, torch = _load()
47
+ except ImportError as e:
48
+ return AnalyzerResult(skipped=f"captions need extras: uv sync --extra captions ({e.name})")
49
+ shots = store.get_shots(video.id)
50
+ frames = thumbs.grab_frames(video.path, [(s.start + s.end) / 2 for s in shots], long_side=512)
51
+ caps = {}
52
+ task = "<CAPTION>"
53
+ with torch.no_grad():
54
+ for s, im in zip(shots, frames):
55
+ if im is None:
56
+ continue
57
+ inp = proc(text=task, images=im, return_tensors="pt").to(dev)
58
+ inp["pixel_values"] = inp["pixel_values"].to(dtype)
59
+ ids = model.generate(input_ids=inp["input_ids"], pixel_values=inp["pixel_values"],
60
+ max_new_tokens=40, num_beams=3)
61
+ caps[s.idx] = _clean(proc.batch_decode(ids, skip_special_tokens=True)[0])
62
+ store.update_shot_field(video.id, "caption", caps)
63
+ return AnalyzerResult(info={"captions": len(caps), "model": MODEL_ID})
@@ -0,0 +1,146 @@
1
+ """Per-shot keyframes (start / middle / end) and on-screen text + watermark positions.
2
+
3
+ Text comes from RapidOCR, whose models ship inside its wheel (offline). Text seen in the same place
4
+ across most of the video is a watermark; a logo with no readable text is found as edges that stay
5
+ put while the shots around them change. Positions are 0-1 fractions of the frame.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import difflib
10
+ from collections import Counter
11
+
12
+ import numpy as np
13
+
14
+ from .. import store, thumbs
15
+ from ..store import VideoInfo
16
+ from . import AnalyzerResult, Context
17
+
18
+ KEYFRAME_SIDE = 384
19
+ OCR_SIDE = 960
20
+ MAX_OCR_SAMPLES = 120
21
+ WATERMARK_SHARE = 0.3 # OCR misses faint marks on dark frames, so a third of samples is already a lot
22
+ MIN_SCORE = 0.8
23
+
24
+
25
+ def keyframe_times(start: float, end: float, fps: float) -> list[float]:
26
+ inset = min(2.0 / fps, (end - start) / 4)
27
+ return [start + inset, (start + end) / 2, end - inset]
28
+
29
+
30
+ def ocr_times(shots, duration: float, cap: int = MAX_OCR_SAMPLES) -> list[float]:
31
+ """Each shot's middle, plus one sample every 10s inside long takes."""
32
+ ts: list[float] = []
33
+ for s in shots:
34
+ n = max(1, int(s.dur // 10))
35
+ ts += [s.start + s.dur * (i + 0.5) / n for i in range(n)]
36
+ if len(ts) > cap:
37
+ ts = [ts[i] for i in np.linspace(0, len(ts) - 1, cap).round().astype(int)]
38
+ return ts
39
+
40
+
41
+ def _similar(a: str, b: str) -> bool:
42
+ """OCR spells a small watermark differently on every frame, and often reads only part of it."""
43
+ return a in b or b in a or difflib.SequenceMatcher(None, a, b, autojunk=False).ratio() >= 0.6
44
+
45
+
46
+ def cluster_text(hits: list[tuple], n_samples: int, step: float, multi_shot: bool = True) -> list[tuple]:
47
+ """hits: (t, x, y, w, h, text) -> overlay rows, grouping the same text in the same place."""
48
+ groups: list[dict] = []
49
+ for t, x, y, w, h, text in sorted(hits):
50
+ key = "".join(text.lower().split())
51
+ for g in groups:
52
+ if abs(g["x"] - x) < 0.05 and abs(g["y"] - y) < 0.05 and _similar(g["key"], key):
53
+ g["ts"].append(t)
54
+ g["names"][text] += 1
55
+ g["w"], g["h"] = max(g["w"], w), max(g["h"], h)
56
+ break
57
+ else:
58
+ groups.append({"key": key, "x": x, "y": y, "w": w, "h": h, "ts": [t], "names": Counter({text: 1})})
59
+ rows = []
60
+ for g in groups:
61
+ share = len(set(g["ts"])) / max(n_samples, 1)
62
+ kind = "watermark" if share >= WATERMARK_SHARE and n_samples >= 4 and multi_shot else "text"
63
+ rows.append((kind, round(max(min(g["ts"]) - step / 2, 0.0), 2), round(max(g["ts"]) + step / 2, 2),
64
+ round(g["x"], 3), round(g["y"], 3), round(g["w"], 3), round(g["h"], 3),
65
+ g["names"].most_common(1)[0][0]))
66
+ return rows
67
+
68
+
69
+ def static_logo_boxes(edge_maps: list[np.ndarray], keep: float = 0.8, max_area: float = 0.06) -> list[tuple]:
70
+ """Boxes (x, y, w, h fractions) of edges that persist across samples from DIFFERENT shots."""
71
+ import cv2
72
+
73
+ if len(edge_maps) < 6:
74
+ return []
75
+ persist = (np.mean(np.stack(edge_maps) > 0, axis=0) >= keep).astype(np.uint8)
76
+ persist = cv2.dilate(persist, np.ones((3, 3), np.uint8))
77
+ h, w = persist.shape
78
+ n, _, stats, _ = cv2.connectedComponentsWithStats(persist)
79
+ boxes = []
80
+ for i in range(1, n):
81
+ x, y, bw, bh, area = stats[i]
82
+ touches_edge = x <= 1 or y <= 1 or x + bw >= w - 1 or y + bh >= h - 1
83
+ if touches_edge or bw * bh > max_area * w * h or area < 30:
84
+ continue # letterbox lines reach the frame edge; huge regions are static backgrounds
85
+ boxes.append((float(x / w), float(y / h), float(bw / w), float(bh / h)))
86
+ return boxes
87
+
88
+
89
+ class Detail:
90
+ name = "detail"
91
+ optional = False
92
+
93
+ def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult:
94
+ import cv2
95
+ from rapidocr_onnxruntime import RapidOCR
96
+
97
+ shots = store.get_shots(video.id)
98
+ kdir = video.dir / "keyframes"
99
+ kdir.mkdir(exist_ok=True)
100
+ updates: dict[int, dict] = {}
101
+ edge_maps = []
102
+ for s in shots:
103
+ ts = keyframe_times(s.start, s.end, video.fps)
104
+ frames = thumbs.grab_frames(video.path, ts, long_side=KEYFRAME_SIDE, workers=3)
105
+ saved = []
106
+ for tag, t, f in zip("ame", ts, frames):
107
+ if f is None:
108
+ continue
109
+ f.save(kdir / f"s{s.idx}_{tag}.jpg", quality=82)
110
+ saved.append(round(t, 2))
111
+ updates[s.idx] = {"keyframes": saved}
112
+ if frames[1] is not None and s.dur >= 0.5:
113
+ g = cv2.cvtColor(np.asarray(frames[1]), cv2.COLOR_RGB2GRAY)
114
+ edge_maps.append(cv2.Canny(g, 80, 160))
115
+ store.merge_shot_detail(video.id, updates)
116
+
117
+ engine = RapidOCR()
118
+ ts = ocr_times(shots, video.duration)
119
+ step = video.duration / max(len(ts), 1)
120
+ hits: list[tuple] = []
121
+ for i in range(0, len(ts), 8): # small batches: never hold many full frames at once
122
+ batch = ts[i:i + 8]
123
+ for t, f in zip(batch, thumbs.grab_frames(video.path, batch, long_side=OCR_SIDE, workers=2)):
124
+ if f is None:
125
+ continue
126
+ arr = np.asarray(f)[..., ::-1]
127
+ res, _ = engine(arr)
128
+ H, W = arr.shape[:2]
129
+ for box, text, score in res or []:
130
+ if float(score) < MIN_SCORE or sum(ch.isalnum() for ch in text) < 3:
131
+ continue
132
+ b = np.asarray(box, np.float32)
133
+ x0, y0, x1, y1 = b[:, 0].min(), b[:, 1].min(), b[:, 0].max(), b[:, 1].max()
134
+ hits.append((t, float(x0 / W), float(y0 / H), float((x1 - x0) / W), float((y1 - y0) / H), text.strip()))
135
+ rows = cluster_text(hits, len(ts), step, multi_shot=len(shots) >= 4) # in one take, room text never moves either
136
+ if len(shots) >= 6: # a static background in a one-take video would look like a logo
137
+ placed = [(r[3], r[4], r[5], r[6]) for r in rows if r[0] == "watermark"]
138
+ for x, y, w, h in static_logo_boxes(edge_maps):
139
+ if not any(abs(px - x) < 0.08 and abs(py - y) < 0.08 for px, py, _, _ in placed):
140
+ rows.append(("watermark", 0.0, round(video.duration, 2), round(x, 3), round(y, 3),
141
+ round(w, 3), round(h, 3), ""))
142
+ store.put_overlays(video.id, rows)
143
+ marks = [r for r in rows if r[0] == "watermark"]
144
+ store.update_meta(video.id, watermarks=[{"box": list(r[3:7]), "text": r[7]} for r in marks])
145
+ return AnalyzerResult(info={"keyframes": sum(len(u["keyframes"]) for u in updates.values()),
146
+ "text": len(rows) - len(marks), "watermarks": len(marks)})