vidX 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vidx/__init__.py +0 -0
- vidx/analyzers/__init__.py +78 -0
- vidx/analyzers/audio.py +125 -0
- vidx/analyzers/audiomap.py +163 -0
- vidx/analyzers/captions.py +63 -0
- vidx/analyzers/detail.py +146 -0
- vidx/analyzers/glitch.py +106 -0
- vidx/analyzers/laugh.py +112 -0
- vidx/analyzers/probe.py +60 -0
- vidx/analyzers/shots.py +269 -0
- vidx/analyzers/transcript.py +352 -0
- vidx/analyzers/visual.py +189 -0
- vidx/api.py +190 -0
- vidx/assets/sfx/drone_01.m4a +0 -0
- vidx/assets/sfx/drone_02.m4a +0 -0
- vidx/assets/sfx/drone_03.m4a +0 -0
- vidx/assets/sfx/drone_04.m4a +0 -0
- vidx/assets/sfx/drone_05.m4a +0 -0
- vidx/assets/sfx/drone_06.m4a +0 -0
- vidx/assets/sfx/impact_01.m4a +0 -0
- vidx/assets/sfx/impact_02.m4a +0 -0
- vidx/assets/sfx/impact_03.m4a +0 -0
- vidx/assets/sfx/impact_04.m4a +0 -0
- vidx/assets/sfx/impact_05.m4a +0 -0
- vidx/assets/sfx/impact_06.m4a +0 -0
- vidx/assets/sfx/riser_01.m4a +0 -0
- vidx/assets/sfx/riser_02.m4a +0 -0
- vidx/assets/sfx/riser_03.m4a +0 -0
- vidx/assets/sfx/riser_04.m4a +0 -0
- vidx/assets/sfx/riser_05.m4a +0 -0
- vidx/assets/sfx/whoosh_01.m4a +0 -0
- vidx/assets/sfx/whoosh_02.m4a +0 -0
- vidx/assets/sfx/whoosh_03.m4a +0 -0
- vidx/checks.py +161 -0
- vidx/cli.py +188 -0
- vidx/color.py +95 -0
- vidx/config.py +51 -0
- vidx/cuts.py +224 -0
- vidx/detect.py +199 -0
- vidx/diff.py +152 -0
- vidx/dsl.py +571 -0
- vidx/index.py +93 -0
- vidx/mcp_server.py +158 -0
- vidx/models.py +18 -0
- vidx/otio_export.py +53 -0
- vidx/query.py +579 -0
- vidx/render.py +600 -0
- vidx/store.py +318 -0
- vidx/thumbs.py +80 -0
- vidx/timeline.py +286 -0
- vidx/util.py +119 -0
- vidx-0.2.0.dist-info/METADATA +291 -0
- vidx-0.2.0.dist-info/RECORD +56 -0
- vidx-0.2.0.dist-info/WHEEL +4 -0
- vidx-0.2.0.dist-info/entry_points.txt +3 -0
- vidx-0.2.0.dist-info/licenses/LICENSE +21 -0
vidx/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Pluggable analyzers. Add a new one by appending to ANALYZERS (order matters)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Callable, Protocol
|
|
9
|
+
|
|
10
|
+
from .. import store, util
|
|
11
|
+
from ..store import VideoInfo
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Context:
|
|
16
|
+
force: bool = False
|
|
17
|
+
captions: bool = False
|
|
18
|
+
glitch: bool = False
|
|
19
|
+
log: Callable[[str], None] = lambda m: print(m, file=sys.stderr, flush=True)
|
|
20
|
+
timings: dict[str, float] = field(default_factory=dict)
|
|
21
|
+
|
|
22
|
+
def audio16k(self, video: VideoInfo) -> Path | None:
|
|
23
|
+
if not video.has_audio:
|
|
24
|
+
return None
|
|
25
|
+
return util.extract_audio16k(video.path, video.dir / "audio16k.wav")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class AnalyzerResult:
|
|
30
|
+
info: dict = field(default_factory=dict) # small facts, stored in analyzer_runs.info
|
|
31
|
+
skipped: str | None = None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Analyzer(Protocol):
|
|
35
|
+
name: str
|
|
36
|
+
optional: bool # only runs when its flag is on (e.g. captions)
|
|
37
|
+
|
|
38
|
+
def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult: ...
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def registry() -> list[Analyzer]:
|
|
42
|
+
from .audio import AudioEvents
|
|
43
|
+
from .audiomap import AudioMap
|
|
44
|
+
from .captions import Captions
|
|
45
|
+
from .detail import Detail
|
|
46
|
+
from .glitch import Glitch
|
|
47
|
+
from .laugh import Laugh
|
|
48
|
+
from .shots import Shots
|
|
49
|
+
from .transcript import Transcript
|
|
50
|
+
from .visual import Visual
|
|
51
|
+
|
|
52
|
+
# shots first (the rest work per shot); laugh after transcript (uses words as a speech mask)
|
|
53
|
+
return [Shots(), Detail(), Visual(), Transcript(), AudioEvents(), AudioMap(), Laugh(), Glitch(), Captions()]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# Per-shot results: these must be redone whenever shot boundaries change.
|
|
57
|
+
SHOT_DEPENDENT = ("detail", "visual", "audiomap", "glitch", "captions")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def run_all(video: VideoInfo, ctx: Context) -> dict[str, float]:
|
|
61
|
+
runs = store.analyzer_runs(video.id)
|
|
62
|
+
for a in registry():
|
|
63
|
+
if a.optional and not getattr(ctx, a.name, False):
|
|
64
|
+
continue
|
|
65
|
+
version = getattr(a, "version", 1)
|
|
66
|
+
if not ctx.force and a.name in runs and runs[a.name].get("v", 1) >= version:
|
|
67
|
+
ctx.log(f" {a.name:<11} cached")
|
|
68
|
+
continue
|
|
69
|
+
t0 = time.perf_counter()
|
|
70
|
+
res = a.run(video, ctx)
|
|
71
|
+
dt = time.perf_counter() - t0
|
|
72
|
+
ctx.timings[a.name] = dt
|
|
73
|
+
store.mark_analyzer(video.id, a.name, dt, {**res.info, "v": version})
|
|
74
|
+
if a.name == "shots": # new boundaries invalidate everything stored per shot
|
|
75
|
+
for name in SHOT_DEPENDENT:
|
|
76
|
+
runs.pop(name, None)
|
|
77
|
+
ctx.log(f" {a.name:<11} {dt:6.1f}s" + (f" ({res.skipped})" if res.skipped else ""))
|
|
78
|
+
return ctx.timings
|
vidx/analyzers/audio.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""silence / loud / black / freeze events from ONE ffmpeg decode pass."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from .. import config, store
|
|
9
|
+
from ..store import Event, VideoInfo
|
|
10
|
+
from . import AnalyzerResult, Context
|
|
11
|
+
|
|
12
|
+
KINDS = ["silence", "loud", "black", "freeze"]
|
|
13
|
+
_num = r"(-?[\d.]+(?:e-?\d+)?)"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def detect_cmd(path: str, has_audio: bool, hwaccel: bool = True, video: bool = True,
|
|
17
|
+
black_min: float = config.BLACK_MIN, freeze_min: float = config.FREEZE_MIN) -> list[str]:
|
|
18
|
+
cmd = ["ffmpeg", "-nostdin", "-hide_banner", "-nostats"]
|
|
19
|
+
if hwaccel and sys.platform == "darwin":
|
|
20
|
+
cmd += ["-hwaccel", "videotoolbox"]
|
|
21
|
+
cmd += ["-i", path]
|
|
22
|
+
if video:
|
|
23
|
+
cmd += ["-vf", f"scale=320:-2,cropdetect=limit=24:round=2:reset=1,blackdetect=d={black_min}:pix_th=0.10,"
|
|
24
|
+
f"freezedetect=n=-60dB:d={freeze_min}"]
|
|
25
|
+
else:
|
|
26
|
+
cmd += ["-vn"]
|
|
27
|
+
if has_audio:
|
|
28
|
+
cmd += ["-af", f"silencedetect=n={config.SILENCE_DB}dB:d={config.SILENCE_MIN},ebur128=framelog=info"]
|
|
29
|
+
else:
|
|
30
|
+
cmd += ["-an"]
|
|
31
|
+
return cmd + ["-f", "null", "-"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def parse_detect_log(log: str, duration: float) -> tuple[list[Event], dict]:
|
|
35
|
+
ev: list[Event] = []
|
|
36
|
+
# black
|
|
37
|
+
for m in re.finditer(rf"black_start:{_num} black_end:{_num} black_duration:{_num}", log):
|
|
38
|
+
a, b, d = map(float, m.groups())
|
|
39
|
+
ev.append(Event("black", a, b, round(d, 2)))
|
|
40
|
+
# freeze (start / end on separate lines)
|
|
41
|
+
starts = [float(x) for x in re.findall(rf"freeze_start: {_num}", log)]
|
|
42
|
+
ends = [float(x) for x in re.findall(rf"freeze_end: {_num}", log)]
|
|
43
|
+
for i, a in enumerate(starts):
|
|
44
|
+
b = ends[i] if i < len(ends) else duration
|
|
45
|
+
ev.append(Event("freeze", a, b, round(b - a, 2)))
|
|
46
|
+
# silence
|
|
47
|
+
s_starts = [float(x) for x in re.findall(rf"silence_start: {_num}", log)]
|
|
48
|
+
s_ends = [float(x) for x in re.findall(rf"silence_end: {_num}", log)]
|
|
49
|
+
for i, a in enumerate(s_starts):
|
|
50
|
+
a = max(a, 0.0)
|
|
51
|
+
b = s_ends[i] if i < len(s_ends) else duration
|
|
52
|
+
ev.append(Event("silence", a, b, round(b - a, 2)))
|
|
53
|
+
# loudness: momentary every 100ms + integrated summary
|
|
54
|
+
mom = [(float(t), float(m)) for t, m in re.findall(rf"t: {_num}\s+TARGET:\S+ LUFS\s+M:\s*{_num}", log)]
|
|
55
|
+
integ = re.findall(rf"Integrated loudness:\s*I:\s*{_num} LUFS", log)
|
|
56
|
+
lufs = float(integ[-1]) if integ else None
|
|
57
|
+
if mom and lufs is not None and lufs > -70:
|
|
58
|
+
thr = lufs + config.LOUD_OVER_INTEGRATED
|
|
59
|
+
run_a, peak, last = None, -200.0, None
|
|
60
|
+
for t, m in mom + [(duration + 1, -200.0)]:
|
|
61
|
+
t0 = t - 0.4 # momentary window is 400ms ending at t
|
|
62
|
+
if m >= thr:
|
|
63
|
+
if run_a is None:
|
|
64
|
+
run_a, peak = max(t0, 0.0), m
|
|
65
|
+
peak = max(peak, m)
|
|
66
|
+
last = t
|
|
67
|
+
elif run_a is not None and (last is None or t - last > 0.3):
|
|
68
|
+
if last - run_a >= 0.3:
|
|
69
|
+
ev.append(Event("loud", round(run_a, 2), round(last, 2), round(peak, 1)))
|
|
70
|
+
run_a = None
|
|
71
|
+
ev.sort(key=lambda e: (e.start, e.kind))
|
|
72
|
+
return ev, {"lufs": lufs, "crop320": dominant_crop(log)}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def dominant_crop(log: str, min_share: float = 0.6) -> list[int] | None:
|
|
76
|
+
"""Per-frame cropdetect -> the letterbox most frames agree on. Title cards / credits that fill
|
|
77
|
+
the frame shouldn't cancel a film's 2.35:1 bars, so height and width are voted separately."""
|
|
78
|
+
from collections import Counter
|
|
79
|
+
|
|
80
|
+
crops = [tuple(map(int, m)) for m in re.findall(r"crop=(\d+):(\d+):(\d+):(\d+)", log)]
|
|
81
|
+
if len(crops) < 10:
|
|
82
|
+
return None
|
|
83
|
+
n = len(crops)
|
|
84
|
+
(h, hc), = Counter(c[1] for c in crops).most_common(1)
|
|
85
|
+
(w, wc), = Counter(c[0] for c in crops).most_common(1)
|
|
86
|
+
if hc / n < min_share or wc / n < min_share:
|
|
87
|
+
return None
|
|
88
|
+
(y, _), = Counter(c[3] for c in crops if c[1] == h).most_common(1)
|
|
89
|
+
(x, _), = Counter(c[2] for c in crops if c[0] == w).most_common(1)
|
|
90
|
+
return [w, h, x, y]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def active_area(crop320: list[int] | None, width: int, height: int) -> list[int] | None:
|
|
94
|
+
"""Baked-in letterbox / pillarbox in display pixels [w, h, x, y], or None if the frame is full.
|
|
95
|
+
cropdetect ran on a 320px-wide copy, so scale back up."""
|
|
96
|
+
if not crop320:
|
|
97
|
+
return None
|
|
98
|
+
f = width / 320
|
|
99
|
+
w, h, x, y = (int(round(v * f / 2)) * 2 for v in crop320)
|
|
100
|
+
w, h = min(w, width), min(h, height)
|
|
101
|
+
if w < width * 0.3 or h < height * 0.3: # mostly black video: don't trust it
|
|
102
|
+
return None
|
|
103
|
+
if w >= width * 0.97 and h >= height * 0.97:
|
|
104
|
+
return None
|
|
105
|
+
return [w, h, x, y]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class AudioEvents:
|
|
109
|
+
name = "audio"
|
|
110
|
+
optional = False
|
|
111
|
+
|
|
112
|
+
def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult:
|
|
113
|
+
def go(hw: bool) -> subprocess.CompletedProcess:
|
|
114
|
+
return subprocess.run(detect_cmd(video.path, video.has_audio, hw), capture_output=True, text=True)
|
|
115
|
+
|
|
116
|
+
p = go(True)
|
|
117
|
+
if p.returncode != 0:
|
|
118
|
+
p = go(False)
|
|
119
|
+
events, info = parse_detect_log(p.stderr, video.duration)
|
|
120
|
+
store.put_events(video.id, KINDS, events)
|
|
121
|
+
store.update_meta(video.id, lufs=info["lufs"],
|
|
122
|
+
active=active_area(info.get("crop320"), video.width, video.height))
|
|
123
|
+
counts = {k: sum(e.kind == k for e in events) for k in KINDS}
|
|
124
|
+
info.pop("crop320", None)
|
|
125
|
+
return AnalyzerResult(info={**info, **counts})
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Label each stretch of audio as speech / music / sfx, find beats, and add loudness per shot.
|
|
2
|
+
|
|
3
|
+
Speech comes from Silero VAD, which ships inside faster-whisper (offline). Everything else is signal
|
|
4
|
+
processing: sustained tonal sound is music, short noisy bursts are sfx. Silence stays with the audio
|
|
5
|
+
analyzer. Audio is read in chunks, so memory stays flat on long files.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
from .. import store, util
|
|
12
|
+
from ..store import Event, VideoInfo
|
|
13
|
+
from . import AnalyzerResult, Context
|
|
14
|
+
|
|
15
|
+
KINDS = ["speech", "music", "sfx", "beat"]
|
|
16
|
+
HOP = 0.02 # 20 ms analysis frames
|
|
17
|
+
WIN = 0.5 # labelling resolution
|
|
18
|
+
SILENT_DB = -50.0
|
|
19
|
+
BPM_RANGE = (70, 180)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def frame_features(x: np.ndarray, sr: int) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
23
|
+
"""Per-frame RMS, spectral flatness and positive spectral flux (the onset signal)."""
|
|
24
|
+
n = int(sr * HOP)
|
|
25
|
+
m = len(x) // n
|
|
26
|
+
if m < 2:
|
|
27
|
+
return np.zeros(0), np.zeros(0), np.zeros(0)
|
|
28
|
+
fr = x[: m * n].reshape(m, n).astype(np.float32)
|
|
29
|
+
rms = np.sqrt((fr.astype(np.float64) ** 2).mean(1) + 1e-12)
|
|
30
|
+
spec = np.abs(np.fft.rfft(fr * np.hanning(n).astype(np.float32), axis=1)) + 1e-7
|
|
31
|
+
flat = np.exp(np.log(spec).mean(1)) / spec.mean(1)
|
|
32
|
+
logs = np.log1p(spec * 100)
|
|
33
|
+
flux = np.maximum(np.diff(logs, axis=0, prepend=logs[:1]), 0).mean(1)
|
|
34
|
+
return rms, flat, flux
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def label_windows(rms: np.ndarray, flat: np.ndarray, flux: np.ndarray, speech: np.ndarray) -> list[str]:
|
|
38
|
+
"""One label per WIN seconds. speech: per-frame bool from the VAD."""
|
|
39
|
+
per = int(WIN / HOP)
|
|
40
|
+
labels = []
|
|
41
|
+
for i in range(0, len(rms) - per + 1, per):
|
|
42
|
+
sl = slice(i, i + per)
|
|
43
|
+
db = 20 * np.log10(rms[sl].mean() + 1e-9)
|
|
44
|
+
if speech[sl].mean() > 0.5:
|
|
45
|
+
labels.append("speech")
|
|
46
|
+
elif db < SILENT_DB:
|
|
47
|
+
labels.append("silence")
|
|
48
|
+
elif np.median(flat[sl]) < 0.12:
|
|
49
|
+
labels.append("music") # energy concentrated in tones
|
|
50
|
+
else:
|
|
51
|
+
labels.append("sfx") # noisy / transient
|
|
52
|
+
return labels
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def smooth(labels: list[str], min_run: int = 3) -> list[str]:
|
|
56
|
+
"""Short sfx islands inside music or speech are usually noise in the classifier, except a real
|
|
57
|
+
hit, which survives because it stands between different neighbours."""
|
|
58
|
+
out = list(labels)
|
|
59
|
+
i = 0
|
|
60
|
+
while i < len(out):
|
|
61
|
+
j = i
|
|
62
|
+
while j + 1 < len(out) and out[j + 1] == out[i]:
|
|
63
|
+
j += 1
|
|
64
|
+
if j - i + 1 < min_run and 0 < i and j + 1 < len(out) and out[i - 1] == out[j + 1]:
|
|
65
|
+
for k in range(i, j + 1):
|
|
66
|
+
out[k] = out[i - 1]
|
|
67
|
+
i = j + 1
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def stretches(labels: list[str]) -> list[Event]:
|
|
72
|
+
ev: list[Event] = []
|
|
73
|
+
for i, l in enumerate(labels):
|
|
74
|
+
if l == "silence":
|
|
75
|
+
continue
|
|
76
|
+
a, b = i * WIN, (i + 1) * WIN
|
|
77
|
+
if ev and ev[-1].kind == l and abs(ev[-1].end - a) < 1e-6:
|
|
78
|
+
ev[-1].end = b
|
|
79
|
+
else:
|
|
80
|
+
ev.append(Event(l, a, b, None))
|
|
81
|
+
for e in ev:
|
|
82
|
+
e.value = round(e.end - e.start, 2)
|
|
83
|
+
return ev
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def track_beats(flux: np.ndarray) -> tuple[float, float, np.ndarray]:
|
|
87
|
+
"""-> (bpm, strength 0-1, beat times). Autocorrelation picks the tempo, then dynamic programming
|
|
88
|
+
(Ellis 2007) places beats on strong onsets at that spacing."""
|
|
89
|
+
if len(flux) < int(8 / HOP):
|
|
90
|
+
return 0.0, 0.0, np.zeros(0)
|
|
91
|
+
env = flux - flux.mean()
|
|
92
|
+
env = np.maximum(env, 0)
|
|
93
|
+
sd = env.std() or 1.0
|
|
94
|
+
env = env / sd
|
|
95
|
+
n = len(env)
|
|
96
|
+
size = 1 << int(np.ceil(np.log2(2 * n)))
|
|
97
|
+
f = np.fft.rfft(env, size)
|
|
98
|
+
ac = np.fft.irfft(f * np.conj(f), size)[:n]
|
|
99
|
+
lo, hi = int(60 / BPM_RANGE[1] / HOP), int(60 / BPM_RANGE[0] / HOP)
|
|
100
|
+
if hi >= n or ac[0] <= 0:
|
|
101
|
+
return 0.0, 0.0, np.zeros(0)
|
|
102
|
+
lag = lo + int(np.argmax(ac[lo:hi]))
|
|
103
|
+
strength = float(ac[lag] / ac[0])
|
|
104
|
+
score = env.copy()
|
|
105
|
+
back = np.full(n, -1)
|
|
106
|
+
span = np.arange(-2 * lag, -lag // 2)
|
|
107
|
+
pen = -100 * np.log(-span / lag) ** 2 / 100
|
|
108
|
+
for i in range(int(lag * 2), n):
|
|
109
|
+
idx = i + span
|
|
110
|
+
cand = score[idx] + pen
|
|
111
|
+
k = int(np.argmax(cand))
|
|
112
|
+
score[i] += cand[k]
|
|
113
|
+
back[i] = idx[k]
|
|
114
|
+
i = int(np.argmax(score[-lag * 2:])) + n - lag * 2
|
|
115
|
+
beats = []
|
|
116
|
+
while i >= 0:
|
|
117
|
+
beats.append(i)
|
|
118
|
+
i = back[i]
|
|
119
|
+
return round(60 / (lag * HOP), 1), round(strength, 2), np.asarray(beats[::-1]) * HOP
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class AudioMap:
|
|
123
|
+
name = "audiomap"
|
|
124
|
+
optional = False
|
|
125
|
+
|
|
126
|
+
def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult:
|
|
127
|
+
wav = ctx.audio16k(video)
|
|
128
|
+
if wav is None:
|
|
129
|
+
return AnalyzerResult(skipped="no audio stream")
|
|
130
|
+
from faster_whisper.vad import VadOptions, get_speech_timestamps
|
|
131
|
+
|
|
132
|
+
reader = util.WavReader(wav)
|
|
133
|
+
sr = reader.sr
|
|
134
|
+
parts = []
|
|
135
|
+
speech_spans: list[tuple[float, float]] = []
|
|
136
|
+
for t0, x in reader.chunks(300.0): # 300 s is a multiple of HOP, so frames line up
|
|
137
|
+
parts.append(frame_features(x, sr))
|
|
138
|
+
for s in get_speech_timestamps(x, VadOptions(min_silence_duration_ms=300)):
|
|
139
|
+
speech_spans.append((t0 + s["start"] / sr, t0 + s["end"] / sr))
|
|
140
|
+
del x
|
|
141
|
+
rms, flat, flux = (np.concatenate([p[i] for p in parts]) for i in range(3))
|
|
142
|
+
speech = np.zeros(len(rms), bool)
|
|
143
|
+
for a, b in speech_spans:
|
|
144
|
+
speech[int(a / HOP): int(b / HOP) + 1] = True
|
|
145
|
+
events = stretches(smooth(label_windows(rms, flat, flux, speech)))
|
|
146
|
+
|
|
147
|
+
music_s = sum(e.end - e.start for e in events if e.kind == "music")
|
|
148
|
+
bpm, strength, beats = track_beats(flux)
|
|
149
|
+
keep_beats = strength >= 0.25 and music_s >= 0.1 * reader.duration
|
|
150
|
+
if keep_beats:
|
|
151
|
+
events += [Event("beat", round(float(t), 2), round(float(t) + 0.05, 2), None) for t in beats]
|
|
152
|
+
store.put_events(video.id, KINDS, events)
|
|
153
|
+
store.update_meta(video.id, bpm=bpm if keep_beats else None, beat_strength=strength)
|
|
154
|
+
|
|
155
|
+
db = 20 * np.log10(rms + 1e-9)
|
|
156
|
+
updates = {}
|
|
157
|
+
for s in store.get_shots(video.id):
|
|
158
|
+
seg = db[int(s.start / HOP): max(int(s.start / HOP) + 1, int(s.end / HOP))]
|
|
159
|
+
if len(seg):
|
|
160
|
+
updates[s.idx] = {"audio": {"db": round(float(seg.mean()), 1), "peak_db": round(float(seg.max()), 1)}}
|
|
161
|
+
store.merge_shot_detail(video.id, updates)
|
|
162
|
+
counts = {k: sum(e.kind == k for e in events) for k in KINDS}
|
|
163
|
+
return AnalyzerResult(info={**counts, "bpm": bpm if keep_beats else None})
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Optional one-line shot captions with Florence-2-base (`index_video(captions=True)`).
|
|
2
|
+
|
|
3
|
+
Install extras first: `uv sync --extra captions`. Captions take precedence over CLIP tags in
|
|
4
|
+
overview lines. One middle frame per shot.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from .. import config, store, thumbs
|
|
9
|
+
from ..store import VideoInfo
|
|
10
|
+
from . import AnalyzerResult, Context
|
|
11
|
+
|
|
12
|
+
MODEL_ID = "florence-community/Florence-2-base" # native transformers (>=4.56) port of microsoft/Florence-2-base
|
|
13
|
+
LEGACY_ID = "microsoft/Florence-2-base" # trust_remote_code version for older transformers
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _load():
|
|
17
|
+
import torch
|
|
18
|
+
import transformers
|
|
19
|
+
|
|
20
|
+
dev = config.device()
|
|
21
|
+
dtype = torch.float16 if dev == "cuda" else torch.float32
|
|
22
|
+
if hasattr(transformers, "Florence2ForConditionalGeneration"):
|
|
23
|
+
model = transformers.Florence2ForConditionalGeneration.from_pretrained(MODEL_ID, dtype=dtype)
|
|
24
|
+
proc = transformers.AutoProcessor.from_pretrained(MODEL_ID)
|
|
25
|
+
else:
|
|
26
|
+
model = transformers.AutoModelForCausalLM.from_pretrained(LEGACY_ID, trust_remote_code=True, torch_dtype=dtype)
|
|
27
|
+
proc = transformers.AutoProcessor.from_pretrained(LEGACY_ID, trust_remote_code=True)
|
|
28
|
+
return model.to(dev).eval(), proc, dev, dtype, torch
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _clean(text: str) -> str:
|
|
32
|
+
t = text.replace("</s>", "").replace("<s>", "").replace("<pad>", "").strip()
|
|
33
|
+
for pre in ("The image shows ", "The image is ", "In this image we can see ", "This image shows "):
|
|
34
|
+
if t.startswith(pre):
|
|
35
|
+
t = t[len(pre):]
|
|
36
|
+
t = t.split(". ")[0].rstrip(".")
|
|
37
|
+
return t[:80]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Captions:
|
|
41
|
+
name = "captions"
|
|
42
|
+
optional = True # runs only when ctx.captions is True
|
|
43
|
+
|
|
44
|
+
def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult:
|
|
45
|
+
try:
|
|
46
|
+
model, proc, dev, dtype, torch = _load()
|
|
47
|
+
except ImportError as e:
|
|
48
|
+
return AnalyzerResult(skipped=f"captions need extras: uv sync --extra captions ({e.name})")
|
|
49
|
+
shots = store.get_shots(video.id)
|
|
50
|
+
frames = thumbs.grab_frames(video.path, [(s.start + s.end) / 2 for s in shots], long_side=512)
|
|
51
|
+
caps = {}
|
|
52
|
+
task = "<CAPTION>"
|
|
53
|
+
with torch.no_grad():
|
|
54
|
+
for s, im in zip(shots, frames):
|
|
55
|
+
if im is None:
|
|
56
|
+
continue
|
|
57
|
+
inp = proc(text=task, images=im, return_tensors="pt").to(dev)
|
|
58
|
+
inp["pixel_values"] = inp["pixel_values"].to(dtype)
|
|
59
|
+
ids = model.generate(input_ids=inp["input_ids"], pixel_values=inp["pixel_values"],
|
|
60
|
+
max_new_tokens=40, num_beams=3)
|
|
61
|
+
caps[s.idx] = _clean(proc.batch_decode(ids, skip_special_tokens=True)[0])
|
|
62
|
+
store.update_shot_field(video.id, "caption", caps)
|
|
63
|
+
return AnalyzerResult(info={"captions": len(caps), "model": MODEL_ID})
|
vidx/analyzers/detail.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Per-shot keyframes (start / middle / end) and on-screen text + watermark positions.
|
|
2
|
+
|
|
3
|
+
Text comes from RapidOCR, whose models ship inside its wheel (offline). Text seen in the same place
|
|
4
|
+
across most of the video is a watermark; a logo with no readable text is found as edges that stay
|
|
5
|
+
put while the shots around them change. Positions are 0-1 fractions of the frame.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import difflib
|
|
10
|
+
from collections import Counter
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
from .. import store, thumbs
|
|
15
|
+
from ..store import VideoInfo
|
|
16
|
+
from . import AnalyzerResult, Context
|
|
17
|
+
|
|
18
|
+
KEYFRAME_SIDE = 384
|
|
19
|
+
OCR_SIDE = 960
|
|
20
|
+
MAX_OCR_SAMPLES = 120
|
|
21
|
+
WATERMARK_SHARE = 0.3 # OCR misses faint marks on dark frames, so a third of samples is already a lot
|
|
22
|
+
MIN_SCORE = 0.8
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def keyframe_times(start: float, end: float, fps: float) -> list[float]:
|
|
26
|
+
inset = min(2.0 / fps, (end - start) / 4)
|
|
27
|
+
return [start + inset, (start + end) / 2, end - inset]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def ocr_times(shots, duration: float, cap: int = MAX_OCR_SAMPLES) -> list[float]:
|
|
31
|
+
"""Each shot's middle, plus one sample every 10s inside long takes."""
|
|
32
|
+
ts: list[float] = []
|
|
33
|
+
for s in shots:
|
|
34
|
+
n = max(1, int(s.dur // 10))
|
|
35
|
+
ts += [s.start + s.dur * (i + 0.5) / n for i in range(n)]
|
|
36
|
+
if len(ts) > cap:
|
|
37
|
+
ts = [ts[i] for i in np.linspace(0, len(ts) - 1, cap).round().astype(int)]
|
|
38
|
+
return ts
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _similar(a: str, b: str) -> bool:
|
|
42
|
+
"""OCR spells a small watermark differently on every frame, and often reads only part of it."""
|
|
43
|
+
return a in b or b in a or difflib.SequenceMatcher(None, a, b, autojunk=False).ratio() >= 0.6
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def cluster_text(hits: list[tuple], n_samples: int, step: float, multi_shot: bool = True) -> list[tuple]:
|
|
47
|
+
"""hits: (t, x, y, w, h, text) -> overlay rows, grouping the same text in the same place."""
|
|
48
|
+
groups: list[dict] = []
|
|
49
|
+
for t, x, y, w, h, text in sorted(hits):
|
|
50
|
+
key = "".join(text.lower().split())
|
|
51
|
+
for g in groups:
|
|
52
|
+
if abs(g["x"] - x) < 0.05 and abs(g["y"] - y) < 0.05 and _similar(g["key"], key):
|
|
53
|
+
g["ts"].append(t)
|
|
54
|
+
g["names"][text] += 1
|
|
55
|
+
g["w"], g["h"] = max(g["w"], w), max(g["h"], h)
|
|
56
|
+
break
|
|
57
|
+
else:
|
|
58
|
+
groups.append({"key": key, "x": x, "y": y, "w": w, "h": h, "ts": [t], "names": Counter({text: 1})})
|
|
59
|
+
rows = []
|
|
60
|
+
for g in groups:
|
|
61
|
+
share = len(set(g["ts"])) / max(n_samples, 1)
|
|
62
|
+
kind = "watermark" if share >= WATERMARK_SHARE and n_samples >= 4 and multi_shot else "text"
|
|
63
|
+
rows.append((kind, round(max(min(g["ts"]) - step / 2, 0.0), 2), round(max(g["ts"]) + step / 2, 2),
|
|
64
|
+
round(g["x"], 3), round(g["y"], 3), round(g["w"], 3), round(g["h"], 3),
|
|
65
|
+
g["names"].most_common(1)[0][0]))
|
|
66
|
+
return rows
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def static_logo_boxes(edge_maps: list[np.ndarray], keep: float = 0.8, max_area: float = 0.06) -> list[tuple]:
|
|
70
|
+
"""Boxes (x, y, w, h fractions) of edges that persist across samples from DIFFERENT shots."""
|
|
71
|
+
import cv2
|
|
72
|
+
|
|
73
|
+
if len(edge_maps) < 6:
|
|
74
|
+
return []
|
|
75
|
+
persist = (np.mean(np.stack(edge_maps) > 0, axis=0) >= keep).astype(np.uint8)
|
|
76
|
+
persist = cv2.dilate(persist, np.ones((3, 3), np.uint8))
|
|
77
|
+
h, w = persist.shape
|
|
78
|
+
n, _, stats, _ = cv2.connectedComponentsWithStats(persist)
|
|
79
|
+
boxes = []
|
|
80
|
+
for i in range(1, n):
|
|
81
|
+
x, y, bw, bh, area = stats[i]
|
|
82
|
+
touches_edge = x <= 1 or y <= 1 or x + bw >= w - 1 or y + bh >= h - 1
|
|
83
|
+
if touches_edge or bw * bh > max_area * w * h or area < 30:
|
|
84
|
+
continue # letterbox lines reach the frame edge; huge regions are static backgrounds
|
|
85
|
+
boxes.append((float(x / w), float(y / h), float(bw / w), float(bh / h)))
|
|
86
|
+
return boxes
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class Detail:
|
|
90
|
+
name = "detail"
|
|
91
|
+
optional = False
|
|
92
|
+
|
|
93
|
+
def run(self, video: VideoInfo, ctx: Context) -> AnalyzerResult:
|
|
94
|
+
import cv2
|
|
95
|
+
from rapidocr_onnxruntime import RapidOCR
|
|
96
|
+
|
|
97
|
+
shots = store.get_shots(video.id)
|
|
98
|
+
kdir = video.dir / "keyframes"
|
|
99
|
+
kdir.mkdir(exist_ok=True)
|
|
100
|
+
updates: dict[int, dict] = {}
|
|
101
|
+
edge_maps = []
|
|
102
|
+
for s in shots:
|
|
103
|
+
ts = keyframe_times(s.start, s.end, video.fps)
|
|
104
|
+
frames = thumbs.grab_frames(video.path, ts, long_side=KEYFRAME_SIDE, workers=3)
|
|
105
|
+
saved = []
|
|
106
|
+
for tag, t, f in zip("ame", ts, frames):
|
|
107
|
+
if f is None:
|
|
108
|
+
continue
|
|
109
|
+
f.save(kdir / f"s{s.idx}_{tag}.jpg", quality=82)
|
|
110
|
+
saved.append(round(t, 2))
|
|
111
|
+
updates[s.idx] = {"keyframes": saved}
|
|
112
|
+
if frames[1] is not None and s.dur >= 0.5:
|
|
113
|
+
g = cv2.cvtColor(np.asarray(frames[1]), cv2.COLOR_RGB2GRAY)
|
|
114
|
+
edge_maps.append(cv2.Canny(g, 80, 160))
|
|
115
|
+
store.merge_shot_detail(video.id, updates)
|
|
116
|
+
|
|
117
|
+
engine = RapidOCR()
|
|
118
|
+
ts = ocr_times(shots, video.duration)
|
|
119
|
+
step = video.duration / max(len(ts), 1)
|
|
120
|
+
hits: list[tuple] = []
|
|
121
|
+
for i in range(0, len(ts), 8): # small batches: never hold many full frames at once
|
|
122
|
+
batch = ts[i:i + 8]
|
|
123
|
+
for t, f in zip(batch, thumbs.grab_frames(video.path, batch, long_side=OCR_SIDE, workers=2)):
|
|
124
|
+
if f is None:
|
|
125
|
+
continue
|
|
126
|
+
arr = np.asarray(f)[..., ::-1]
|
|
127
|
+
res, _ = engine(arr)
|
|
128
|
+
H, W = arr.shape[:2]
|
|
129
|
+
for box, text, score in res or []:
|
|
130
|
+
if float(score) < MIN_SCORE or sum(ch.isalnum() for ch in text) < 3:
|
|
131
|
+
continue
|
|
132
|
+
b = np.asarray(box, np.float32)
|
|
133
|
+
x0, y0, x1, y1 = b[:, 0].min(), b[:, 1].min(), b[:, 0].max(), b[:, 1].max()
|
|
134
|
+
hits.append((t, float(x0 / W), float(y0 / H), float((x1 - x0) / W), float((y1 - y0) / H), text.strip()))
|
|
135
|
+
rows = cluster_text(hits, len(ts), step, multi_shot=len(shots) >= 4) # in one take, room text never moves either
|
|
136
|
+
if len(shots) >= 6: # a static background in a one-take video would look like a logo
|
|
137
|
+
placed = [(r[3], r[4], r[5], r[6]) for r in rows if r[0] == "watermark"]
|
|
138
|
+
for x, y, w, h in static_logo_boxes(edge_maps):
|
|
139
|
+
if not any(abs(px - x) < 0.08 and abs(py - y) < 0.08 for px, py, _, _ in placed):
|
|
140
|
+
rows.append(("watermark", 0.0, round(video.duration, 2), round(x, 3), round(y, 3),
|
|
141
|
+
round(w, 3), round(h, 3), ""))
|
|
142
|
+
store.put_overlays(video.id, rows)
|
|
143
|
+
marks = [r for r in rows if r[0] == "watermark"]
|
|
144
|
+
store.update_meta(video.id, watermarks=[{"box": list(r[3:7]), "text": r[7]} for r in marks])
|
|
145
|
+
return AnalyzerResult(info={"keyframes": sum(len(u["keyframes"]) for u in updates.values()),
|
|
146
|
+
"text": len(rows) - len(marks), "watermarks": len(marks)})
|