leanscene 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of leanscene might be problematic. Click here for more details.
- leanscene/__init__.py +10 -0
- leanscene/__main__.py +3 -0
- leanscene/cli.py +88 -0
- leanscene/detect.py +223 -0
- leanscene/model.py +102 -0
- leanscene/weights/leanscene.pt +0 -0
- leanscene-0.1.0.dist-info/METADATA +193 -0
- leanscene-0.1.0.dist-info/RECORD +12 -0
- leanscene-0.1.0.dist-info/WHEEL +5 -0
- leanscene-0.1.0.dist-info/entry_points.txt +2 -0
- leanscene-0.1.0.dist-info/licenses/LICENSE +21 -0
- leanscene-0.1.0.dist-info/top_level.txt +1 -0
leanscene/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""LeanScene: a 0.5M-parameter network for fast shot transition detection."""
|
|
2
|
+
|
|
3
|
+
from .detect import (CUT_THRESHOLD, FADE_CLOSE, FADE_THRESHOLD, MODELS, Result, Transition, default_device, detect,
|
|
4
|
+
find_shots, find_transitions, frame_probs, label_frames, load_model)
|
|
5
|
+
from .model import LeanScene
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
|
|
9
|
+
__all__ = ["CUT_THRESHOLD", "FADE_CLOSE", "FADE_THRESHOLD", "MODELS", "LeanScene", "Result", "Transition",
|
|
10
|
+
"default_device", "detect", "find_shots", "find_transitions", "frame_probs", "label_frames", "load_model"]
|
leanscene/__main__.py
ADDED
leanscene/cli.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Command line: leanscene VIDEO [VIDEO ...]"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import csv
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .detect import CUT_THRESHOLD, FADE_THRESHOLD, detect, load_model
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _clock(seconds: float) -> str:
|
|
17
|
+
m, s = divmod(seconds, 60)
|
|
18
|
+
h, m = divmod(int(m), 60)
|
|
19
|
+
return f"{h:02d}:{m:02d}:{s:06.3f}"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _table(path: str, d: dict, out) -> None:
|
|
23
|
+
timed = d["fps"] is not None
|
|
24
|
+
fps = f"{d['fps']:.3f} fps" if timed else "fps unknown"
|
|
25
|
+
print(f"{path}: {d['frames']} frames, {fps}, {len(d['shots'])} shots, {len(d['transitions'])} transitions", file=out)
|
|
26
|
+
print(f"{'shot':>5} {'start':>8} {'end':>8}" + (f" {'start time':>12} {'end time':>12}" if timed else ""), file=out)
|
|
27
|
+
for i, s in enumerate(d["shots"], 1):
|
|
28
|
+
times = f" {_clock(s['start_time']):>12} {_clock(s['end_time']):>12}" if timed else ""
|
|
29
|
+
print(f"{i:>5} {s['start']:>8} {s['end']:>8}{times}", file=out)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def main(argv=None) -> int:
|
|
33
|
+
ap = argparse.ArgumentParser(prog="leanscene", description="Detect shot transitions (cuts and gradual transitions) in videos.")
|
|
34
|
+
ap.add_argument("videos", nargs="+", help="video files")
|
|
35
|
+
ap.add_argument("-f", "--format", choices=["table", "json", "csv"], default="table",
|
|
36
|
+
help="table: one row per shot (default). json: shots and transitions. csv: one row per shot")
|
|
37
|
+
ap.add_argument("-o", "--output", help="write to this file instead of standard output")
|
|
38
|
+
ap.add_argument("--model", default="leanscene", help="a checkpoint path, to use weights other than the bundled ones")
|
|
39
|
+
ap.add_argument("--device", help="cpu, cuda or mps (default: cuda when available, else cpu)")
|
|
40
|
+
ap.add_argument("--cut", type=float, default=CUT_THRESHOLD, help=f"cut threshold (default {CUT_THRESHOLD})")
|
|
41
|
+
ap.add_argument("--fade", type=float, default=FADE_THRESHOLD, help=f"gradual-transition threshold (default {FADE_THRESHOLD})")
|
|
42
|
+
ap.add_argument("--probs", metavar="FILE.npy",
|
|
43
|
+
help="also save the per-frame probabilities (frames x 3: same, cut, fade); one video only")
|
|
44
|
+
ap.add_argument("--version", action="version", version=f"leanscene {__version__}")
|
|
45
|
+
args = ap.parse_args(argv)
|
|
46
|
+
if args.probs and len(args.videos) > 1:
|
|
47
|
+
ap.error("--probs takes a single video")
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
model = load_model(args.model, args.device)
|
|
51
|
+
except ValueError as e:
|
|
52
|
+
ap.error(str(e))
|
|
53
|
+
results, failed = {}, False
|
|
54
|
+
for path in args.videos:
|
|
55
|
+
try:
|
|
56
|
+
r = detect(path, model=model, cut=args.cut, fade=args.fade)
|
|
57
|
+
except (FileNotFoundError, ValueError) as e:
|
|
58
|
+
print(f"leanscene: {path}: {e if isinstance(e, ValueError) else 'no such file'}", file=sys.stderr)
|
|
59
|
+
failed = True
|
|
60
|
+
continue
|
|
61
|
+
if r.frames == 0:
|
|
62
|
+
print(f"leanscene: {path}: no frames could be decoded", file=sys.stderr)
|
|
63
|
+
failed = True
|
|
64
|
+
continue
|
|
65
|
+
results[path] = r.to_dict()
|
|
66
|
+
if args.probs:
|
|
67
|
+
np.save(args.probs, r.probs)
|
|
68
|
+
|
|
69
|
+
out = open(args.output, "w", newline="") if args.output else sys.stdout
|
|
70
|
+
try:
|
|
71
|
+
if args.format == "json":
|
|
72
|
+
json.dump(results if len(args.videos) > 1 else next(iter(results.values()), {}), out, indent=1)
|
|
73
|
+
out.write("\n")
|
|
74
|
+
elif args.format == "csv":
|
|
75
|
+
w = csv.writer(out)
|
|
76
|
+
w.writerow(["video", "shot", "start_frame", "end_frame", "start_time", "end_time"])
|
|
77
|
+
for path, d in results.items():
|
|
78
|
+
for i, s in enumerate(d["shots"], 1):
|
|
79
|
+
w.writerow([path, i, s["start"], s["end"], s.get("start_time", ""), s.get("end_time", "")])
|
|
80
|
+
else:
|
|
81
|
+
for n, (path, d) in enumerate(results.items()):
|
|
82
|
+
if n:
|
|
83
|
+
print(file=out)
|
|
84
|
+
_table(path, d, out)
|
|
85
|
+
finally:
|
|
86
|
+
if args.output:
|
|
87
|
+
out.close()
|
|
88
|
+
return 1 if failed else 0
|
leanscene/detect.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Run LeanScene over a video: per-frame probabilities, the decision rule, shots."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from importlib import resources
|
|
8
|
+
from typing import Iterable, Iterator, Optional, Union
|
|
9
|
+
|
|
10
|
+
import cv2
|
|
11
|
+
import numpy as np
|
|
12
|
+
import torch
|
|
13
|
+
|
|
14
|
+
from .model import FRAME_SIZE, LeanScene
|
|
15
|
+
|
|
16
|
+
# The decision rule used for every number in the paper. It was fixed before the test sets
|
|
17
|
+
# were scored.
|
|
18
|
+
CUT_THRESHOLD = 0.7
|
|
19
|
+
FADE_THRESHOLD = 0.8
|
|
20
|
+
FADE_CLOSE = 8 # fade frames separated by at most this many frames are joined into one run
|
|
21
|
+
|
|
22
|
+
MODELS = {"leanscene": "leanscene.pt"} # bundled weights: the paper's model
|
|
23
|
+
|
|
24
|
+
Source = Union[str, "os.PathLike[str]", Iterable[np.ndarray]]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def default_device() -> torch.device:
|
|
28
|
+
"""CUDA when available, otherwise the CPU.
|
|
29
|
+
|
|
30
|
+
Apple's MPS backend is faster but is not chosen automatically: in our runs a process that
|
|
31
|
+
fed it many different input lengths kept growing in memory. Pass device="mps" to use it.
|
|
32
|
+
"""
|
|
33
|
+
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def load_model(name: Union[str, "os.PathLike[str]"] = "leanscene", device=None) -> LeanScene:
|
|
37
|
+
"""Load bundled weights by name (see MODELS) or a checkpoint file by path."""
|
|
38
|
+
if str(name) in MODELS:
|
|
39
|
+
with resources.as_file(resources.files("leanscene") / "weights" / MODELS[str(name)]) as path:
|
|
40
|
+
ck = torch.load(path, map_location="cpu", weights_only=True)
|
|
41
|
+
elif os.path.exists(name):
|
|
42
|
+
ck = torch.load(name, map_location="cpu", weights_only=True)
|
|
43
|
+
else:
|
|
44
|
+
raise ValueError(f"unknown model {str(name)!r}: expected one of {sorted(MODELS)} or a checkpoint path")
|
|
45
|
+
model = LeanScene(**ck["config"])
|
|
46
|
+
model.load_state_dict(ck["model"])
|
|
47
|
+
return model.to(torch.device(device) if device is not None else default_device()).eval()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _frames(source: Source) -> Iterator[np.ndarray]:
|
|
51
|
+
"""Yield every frame as FRAME_SIZE x FRAME_SIZE BGR uint8."""
|
|
52
|
+
size = (FRAME_SIZE, FRAME_SIZE)
|
|
53
|
+
if isinstance(source, (str, os.PathLike)):
|
|
54
|
+
if not os.path.exists(source):
|
|
55
|
+
raise FileNotFoundError(str(source))
|
|
56
|
+
cap = cv2.VideoCapture(str(source))
|
|
57
|
+
if not cap.isOpened():
|
|
58
|
+
raise ValueError(f"cannot open {source} as a video")
|
|
59
|
+
try:
|
|
60
|
+
while True:
|
|
61
|
+
ok, frame = cap.read()
|
|
62
|
+
if not ok:
|
|
63
|
+
break
|
|
64
|
+
yield cv2.resize(frame, size)
|
|
65
|
+
finally:
|
|
66
|
+
cap.release()
|
|
67
|
+
else:
|
|
68
|
+
for frame in source:
|
|
69
|
+
frame = np.asarray(frame)
|
|
70
|
+
if frame.ndim != 3 or frame.shape[2] != 3 or frame.dtype != np.uint8:
|
|
71
|
+
raise ValueError(f"frames must be HxWx3 uint8 in BGR order, got {frame.shape} {frame.dtype}")
|
|
72
|
+
yield frame if frame.shape[:2] == size else cv2.resize(frame, size)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@torch.inference_mode()
|
|
76
|
+
def frame_probs(source: Source, model: Optional[LeanScene] = None, device=None, batch: Optional[int] = None,
|
|
77
|
+
window: int = 4096) -> np.ndarray:
|
|
78
|
+
"""Per-frame probabilities of same / cut / fade, as a float32 array of shape (frames, 3).
|
|
79
|
+
|
|
80
|
+
source: a video path, or an iterable of HxWx3 uint8 frames in BGR order (OpenCV's).
|
|
81
|
+
The video is streamed: frames are encoded in batches of `batch` and the temporal head runs
|
|
82
|
+
over `window` frames at a time with more context on each side than its receptive field, so
|
|
83
|
+
memory stays flat on long videos and the result equals a single pass up to float rounding.
|
|
84
|
+
"""
|
|
85
|
+
if model is None:
|
|
86
|
+
model = load_model(device=device)
|
|
87
|
+
elif device is not None:
|
|
88
|
+
model = model.to(device)
|
|
89
|
+
model.eval()
|
|
90
|
+
dev = next(model.parameters()).device
|
|
91
|
+
if batch is None:
|
|
92
|
+
# Measured on an Apple M1 Pro: small batches are about twice as fast on the CPU.
|
|
93
|
+
batch = 8 if dev.type == "cpu" else 512
|
|
94
|
+
ctx = max(64, model.receptive_field // 2 + 1)
|
|
95
|
+
|
|
96
|
+
out: list[np.ndarray] = []
|
|
97
|
+
chunks: list[torch.Tensor] = [] # features not yet labelled; the first `done` rows are left context
|
|
98
|
+
held, done = 0, 0
|
|
99
|
+
|
|
100
|
+
def encode(frames: list[np.ndarray]) -> None:
|
|
101
|
+
nonlocal held
|
|
102
|
+
x = torch.from_numpy(np.stack(frames)).to(dev).permute(0, 3, 1, 2).float().div_(255)
|
|
103
|
+
chunks.append(model.frame_features(x))
|
|
104
|
+
held += len(frames)
|
|
105
|
+
|
|
106
|
+
def label(final: bool) -> None:
|
|
107
|
+
nonlocal chunks, held, done
|
|
108
|
+
hi = held if final else held - ctx
|
|
109
|
+
if hi <= done:
|
|
110
|
+
return
|
|
111
|
+
f = torch.cat(chunks)
|
|
112
|
+
out.append(torch.softmax(model.head(f[None])[0], 1)[done:hi].float().cpu().numpy())
|
|
113
|
+
keep = max(0, hi - ctx)
|
|
114
|
+
chunks, held, done = [f[keep:]], held - keep, hi - keep
|
|
115
|
+
|
|
116
|
+
pending: list[np.ndarray] = []
|
|
117
|
+
for frame in _frames(source):
|
|
118
|
+
pending.append(frame)
|
|
119
|
+
if len(pending) == batch:
|
|
120
|
+
encode(pending)
|
|
121
|
+
pending = []
|
|
122
|
+
if held - done >= window + ctx:
|
|
123
|
+
label(False)
|
|
124
|
+
if pending:
|
|
125
|
+
encode(pending)
|
|
126
|
+
label(True)
|
|
127
|
+
return np.concatenate(out) if out else np.zeros((0, 3), np.float32)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def label_frames(probs: np.ndarray, cut: float = CUT_THRESHOLD, fade: float = FADE_THRESHOLD,
|
|
131
|
+
close: int = FADE_CLOSE) -> np.ndarray:
|
|
132
|
+
"""Apply the decision rule: (frames, 3) probabilities -> labels 0 same, 1 cut, 2 fade."""
|
|
133
|
+
probs = np.asarray(probs)
|
|
134
|
+
labels = np.zeros(len(probs), np.int64)
|
|
135
|
+
labels[probs[:, 2] >= fade] = 2
|
|
136
|
+
labels[probs[:, 1] >= cut] = 1
|
|
137
|
+
fades = np.flatnonzero(labels == 2)
|
|
138
|
+
for a, b in zip(fades[:-1], fades[1:]):
|
|
139
|
+
if 1 < b - a <= close + 1:
|
|
140
|
+
labels[a:b] = 2
|
|
141
|
+
return labels
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass(frozen=True)
|
|
145
|
+
class Transition:
|
|
146
|
+
"""A maximal run of frames not labelled same. start and end are inclusive frame indices.
|
|
147
|
+
|
|
148
|
+
kind is "cut" when the run holds no fade frame, else "gradual". For a cut, `end` is the
|
|
149
|
+
first frame of the new shot; start < end only when a few junk frames sit between two shots.
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
start: int
|
|
153
|
+
end: int
|
|
154
|
+
kind: str
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def find_transitions(labels: np.ndarray) -> list[Transition]:
|
|
158
|
+
labels = np.asarray(labels)
|
|
159
|
+
on = np.concatenate([[0], (labels != 0).astype(np.int8), [0]])
|
|
160
|
+
edges = np.flatnonzero(np.diff(on))
|
|
161
|
+
return [Transition(int(s), int(e) - 1, "gradual" if (labels[s:e] == 2).any() else "cut")
|
|
162
|
+
for s, e in zip(edges[::2], edges[1::2])]
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def find_shots(transitions: Iterable[Transition], frames: int) -> list[tuple[int, int]]:
|
|
166
|
+
"""Shots as inclusive (first frame, last frame) pairs.
|
|
167
|
+
|
|
168
|
+
A cut starts the new shot on its last frame. The frames of a gradual transition belong to
|
|
169
|
+
neither neighbour.
|
|
170
|
+
"""
|
|
171
|
+
shots, start = [], 0
|
|
172
|
+
for t in transitions:
|
|
173
|
+
if t.start - 1 >= start:
|
|
174
|
+
shots.append((start, t.start - 1))
|
|
175
|
+
start = t.end if t.kind == "cut" else t.end + 1
|
|
176
|
+
if start <= frames - 1:
|
|
177
|
+
shots.append((start, frames - 1))
|
|
178
|
+
return shots
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@dataclass
|
|
182
|
+
class Result:
|
|
183
|
+
frames: int
|
|
184
|
+
fps: Optional[float] # None when the source is not a file or the file does not say
|
|
185
|
+
transitions: list[Transition]
|
|
186
|
+
shots: list[tuple[int, int]]
|
|
187
|
+
probs: np.ndarray = field(repr=False) # (frames, 3): same, cut, fade
|
|
188
|
+
|
|
189
|
+
def to_dict(self) -> dict:
|
|
190
|
+
"""JSON-ready. Times are in seconds and present only when fps is known; a shot's
|
|
191
|
+
end time is the end of its last frame."""
|
|
192
|
+
def t(frame: int):
|
|
193
|
+
return round(frame / self.fps, 4)
|
|
194
|
+
|
|
195
|
+
shots = [{"start": a, "end": b} for a, b in self.shots]
|
|
196
|
+
transitions = [{"start": x.start, "end": x.end, "kind": x.kind} for x in self.transitions]
|
|
197
|
+
if self.fps:
|
|
198
|
+
for d in shots + transitions:
|
|
199
|
+
d["start_time"], d["end_time"] = t(d["start"]), t(d["end"] + 1)
|
|
200
|
+
return {"frames": self.frames, "fps": self.fps, "shots": shots, "transitions": transitions}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _fps(path) -> Optional[float]:
|
|
204
|
+
cap = cv2.VideoCapture(str(path))
|
|
205
|
+
try:
|
|
206
|
+
fps = cap.get(cv2.CAP_PROP_FPS)
|
|
207
|
+
finally:
|
|
208
|
+
cap.release()
|
|
209
|
+
return float(fps) if fps and fps > 0 else None
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def detect(source: Source, model: Optional[LeanScene] = None, device=None, cut: float = CUT_THRESHOLD,
|
|
213
|
+
fade: float = FADE_THRESHOLD, close: int = FADE_CLOSE, batch: Optional[int] = None) -> Result:
|
|
214
|
+
"""Find the shot transitions of a video.
|
|
215
|
+
|
|
216
|
+
source: a video path, or an iterable of HxWx3 uint8 BGR frames.
|
|
217
|
+
model: a loaded model to reuse across videos (load_model()); loaded on each call otherwise.
|
|
218
|
+
cut, fade: probability thresholds. Raise them for fewer false alarms, lower them to miss less.
|
|
219
|
+
"""
|
|
220
|
+
probs = frame_probs(source, model=model, device=device, batch=batch)
|
|
221
|
+
transitions = find_transitions(label_frames(probs, cut, fade, close))
|
|
222
|
+
fps = _fps(source) if isinstance(source, (str, os.PathLike)) else None
|
|
223
|
+
return Result(len(probs), fps, transitions, find_shots(transitions, len(probs)), probs)
|
leanscene/model.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""The LeanScene network: a per-frame encoder and a dilated 1D temporal head."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn as nn
|
|
7
|
+
import torch.nn.functional as F
|
|
8
|
+
|
|
9
|
+
FRAME_SIZE = 100 # frames are resized to FRAME_SIZE x FRAME_SIZE before the network sees them
|
|
10
|
+
HIST_BINS = 4 # per channel; the joint colour histogram has HIST_BINS ** 3 = 64 values
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _conv_bn(i: int, o: int, stride: int) -> nn.Sequential:
|
|
14
|
+
return nn.Sequential(nn.Conv2d(i, o, 3, stride, 1, bias=False), nn.BatchNorm2d(o), nn.ReLU(inplace=True))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class _TBlock(nn.Module):
|
|
18
|
+
"""Residual block of one dilated 1D convolution."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, c: int, dilation: int):
|
|
21
|
+
super().__init__()
|
|
22
|
+
self.conv = nn.Conv1d(c, c, 3, padding=dilation, dilation=dilation, bias=False)
|
|
23
|
+
self.bn = nn.BatchNorm1d(c)
|
|
24
|
+
|
|
25
|
+
def forward(self, x):
|
|
26
|
+
return x + F.relu(self.bn(self.conv(x)))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def color_hist(x: torch.Tensor, bins: int = HIST_BINS) -> torch.Tensor:
|
|
30
|
+
"""x: (N,3,H,W) in [0,1] -> (N, bins**3) normalised joint colour histogram."""
|
|
31
|
+
q = (x * (bins - 0.001)).long()
|
|
32
|
+
idx = (q[:, 0] * bins + q[:, 1]) * bins + q[:, 2]
|
|
33
|
+
return F.one_hot(idx.flatten(1), bins**3).float().mean(1)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class LeanScene(nn.Module):
|
|
37
|
+
"""Labels every frame of a video as same (0), cut (1) or fade (2).
|
|
38
|
+
|
|
39
|
+
Each frame is encoded once: a four-layer 2D CNN gives a global feature and a 2x2 grid of
|
|
40
|
+
features, and two descriptors come straight from the pixels (a 64-bin colour histogram
|
|
41
|
+
and an 8x8 colour thumbnail). The temporal head sees those features together with their
|
|
42
|
+
change since the previous frame, and runs residual dilated 1D convolutions over the
|
|
43
|
+
sequence.
|
|
44
|
+
|
|
45
|
+
A cut is labelled on the first frame of the new shot; fade marks the frames inside a
|
|
46
|
+
gradual transition.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, feat: int = 128, hid: int = 128, dilations=(1, 2, 4, 8, 16), spatial: bool = True,
|
|
50
|
+
width: float = 1.0):
|
|
51
|
+
super().__init__()
|
|
52
|
+
self.config = {"feat": feat, "hid": hid, "dilations": list(dilations), "spatial": spatial, "width": width}
|
|
53
|
+
c = [max(8, int(round(ch * width))) for ch in (24, 48, 96)]
|
|
54
|
+
self.enc = nn.Sequential(_conv_bn(3, c[0], 2), _conv_bn(c[0], c[1], 2), _conv_bn(c[1], c[2], 2),
|
|
55
|
+
_conv_bn(c[2], feat, 2))
|
|
56
|
+
self.feat, self.spatial = feat, spatial
|
|
57
|
+
self.sdim = 3 * 8 * 8 + feat * 2 * 2 if spatial else 0
|
|
58
|
+
extra = 0
|
|
59
|
+
if spatial:
|
|
60
|
+
self.sproj = nn.Sequential(nn.Conv1d(self.sdim, 64, 1), nn.ReLU(inplace=True))
|
|
61
|
+
extra = 64
|
|
62
|
+
self.inp = nn.Conv1d(feat + 64 + 1 + extra, hid, 1)
|
|
63
|
+
self.blocks = nn.Sequential(*[_TBlock(hid, d) for d in dilations])
|
|
64
|
+
self.out = nn.Conv1d(hid, 3, 1)
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def receptive_field(self) -> int:
|
|
68
|
+
"""Number of frames that can influence one frame's label."""
|
|
69
|
+
return 1 + 2 * sum(self.config["dilations"])
|
|
70
|
+
|
|
71
|
+
def frame_features(self, x: torch.Tensor) -> torch.Tensor:
|
|
72
|
+
"""x: (N,3,H,W) float in [0,1], BGR channel order -> (N, D) per-frame features."""
|
|
73
|
+
fmap = self.enc(x)
|
|
74
|
+
parts = [fmap.mean((2, 3)), color_hist(x)]
|
|
75
|
+
if self.spatial:
|
|
76
|
+
# Fixed-kernel pooling on a centre crop of divisible size, so it also runs on
|
|
77
|
+
# backends without adaptive pooling for non-divisible sizes.
|
|
78
|
+
h, w = x.shape[2] // 8 * 8, x.shape[3] // 8 * 8
|
|
79
|
+
top, left = (x.shape[2] - h) // 2, (x.shape[3] - w) // 2
|
|
80
|
+
xc = x[:, :, top:top + h, left:left + w]
|
|
81
|
+
fh, fw = fmap.shape[2] // 2 * 2, fmap.shape[3] // 2 * 2
|
|
82
|
+
parts += [F.avg_pool2d(xc, (h // 8, w // 8)).flatten(1),
|
|
83
|
+
F.avg_pool2d(fmap[:, :, :fh, :fw], (fh // 2, fw // 2)).flatten(1)]
|
|
84
|
+
return torch.cat(parts, 1)
|
|
85
|
+
|
|
86
|
+
def head(self, f: torch.Tensor) -> torch.Tensor:
|
|
87
|
+
"""f: (B,S,D) frame features in temporal order -> (B,S,3) logits."""
|
|
88
|
+
f = f.transpose(1, 2) # (B, D, S)
|
|
89
|
+
k = self.feat + 64
|
|
90
|
+
hist = f[:, self.feat:k]
|
|
91
|
+
diff = torch.cat([torch.zeros_like(hist[:, :, :1]), (hist[:, :, 1:] - hist[:, :, :-1]).abs()], 2)
|
|
92
|
+
x = [f[:, :k], diff.sum(1, keepdim=True)]
|
|
93
|
+
if self.spatial:
|
|
94
|
+
sp = f[:, k:]
|
|
95
|
+
sdiff = torch.cat([torch.zeros_like(sp[:, :, :1]), (sp[:, :, 1:] - sp[:, :, :-1]).abs()], 2)
|
|
96
|
+
x.append(self.sproj(sdiff))
|
|
97
|
+
return self.out(self.blocks(self.inp(torch.cat(x, 1)))).transpose(1, 2).contiguous()
|
|
98
|
+
|
|
99
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
100
|
+
"""x: (B,S,3,H,W) float in [0,1], BGR -> (B,S,3) logits."""
|
|
101
|
+
b, s = x.shape[:2]
|
|
102
|
+
return self.head(self.frame_features(x.flatten(0, 1)).reshape(b, s, -1))
|
|
Binary file
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: leanscene
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A 0.5M-parameter network for fast shot transition detection in video
|
|
5
|
+
Author: Lucas Jackson
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/lucasjacks0n/LeanScene
|
|
8
|
+
Keywords: shot boundary detection,scene detection,video,shot transition detection
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Topic :: Multimedia :: Video
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Requires-Python: >=3.9
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: numpy
|
|
16
|
+
Requires-Dist: opencv-python-headless
|
|
17
|
+
Requires-Dist: torch>=2.0
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest; extra == "dev"
|
|
20
|
+
Provides-Extra: export
|
|
21
|
+
Requires-Dist: onnx; extra == "export"
|
|
22
|
+
Requires-Dist: onnxruntime; extra == "export"
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
<h1 align="center">
|
|
26
|
+
<picture>
|
|
27
|
+
<source media="(prefers-color-scheme: dark)" srcset="logo/leanscene-wordmark-dark.svg">
|
|
28
|
+
<img src="logo/leanscene-wordmark.svg" alt="LeanScene" width="440">
|
|
29
|
+
</picture>
|
|
30
|
+
</h1>
|
|
31
|
+
|
|
32
|
+
A 0.5M-parameter network that finds shot transitions (cuts, dissolves, fades) in video.
|
|
33
|
+
It matches TransNet V2 on that model's three benchmarks at 40x fewer operations per frame.
|
|
34
|
+
|
|
35
|
+
```sh
|
|
36
|
+
pip install leanscene
|
|
37
|
+
leanscene video.mp4
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
video.mp4: 14775 frames, 25.000 fps, 51 shots, 50 transitions
|
|
42
|
+
shot start end start time end time
|
|
43
|
+
1 0 343 00:00:00.000 00:00:13.760
|
|
44
|
+
2 344 2403 00:00:13.760 00:01:36.160
|
|
45
|
+
...
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The weights (2 MB) ship inside the package; nothing is downloaded at run time. The paper is in
|
|
49
|
+
preparation.
|
|
50
|
+
|
|
51
|
+
## Python
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
import leanscene
|
|
55
|
+
|
|
56
|
+
result = leanscene.detect("video.mp4")
|
|
57
|
+
result.shots # [(0, 343), (344, 2403), (2414, 3315), ...] inclusive frame ranges, 0-based
|
|
58
|
+
result.transitions # [Transition(start=344, end=344, kind='cut'), Transition(start=2404, end=2413, kind='gradual'), ...]
|
|
59
|
+
result.probs # (frames, 3) float32: same, cut, fade
|
|
60
|
+
result.to_dict() # JSON-ready, with times in seconds
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
To process many videos, load the model once:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
model = leanscene.load_model() # or load_model(device="cuda")
|
|
67
|
+
for path in paths:
|
|
68
|
+
result = leanscene.detect(path, model=model)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
`detect` also takes an iterable of `HxWx3` uint8 frames in BGR order instead of a path, and
|
|
72
|
+
`cut=` / `fade=` thresholds (defaults 0.7 and 0.8, the rule used for every number below).
|
|
73
|
+
`frame_probs` returns only the probabilities; `label_frames`, `find_transitions` and
|
|
74
|
+
`find_shots` are the steps `detect` applies to them.
|
|
75
|
+
|
|
76
|
+
A cut is reported on the first frame of the new shot. A gradual transition is the run of
|
|
77
|
+
frames inside it, and those frames belong to neither neighbouring shot.
|
|
78
|
+
|
|
79
|
+
## Command line
|
|
80
|
+
|
|
81
|
+
| Option | Effect |
|
|
82
|
+
|---|---|
|
|
83
|
+
| `-f table\|json\|csv` | Output format. `table` (default) and `csv` list shots; `json` has shots and transitions |
|
|
84
|
+
| `-o FILE` | Write to a file instead of standard output |
|
|
85
|
+
| `--model PATH` | A checkpoint file, to use weights other than the bundled ones |
|
|
86
|
+
| `--device DEVICE` | `cpu`, `cuda` or `mps`. Default: `cuda` when available, else `cpu` |
|
|
87
|
+
| `--cut X`, `--fade X` | Thresholds (0.7, 0.8). Higher gives fewer false alarms, lower misses less |
|
|
88
|
+
| `--probs FILE.npy` | Also save the per-frame probabilities |
|
|
89
|
+
|
|
90
|
+
Several videos can be given at once. `python -m leanscene` is the same command.
|
|
91
|
+
|
|
92
|
+
## Results
|
|
93
|
+
|
|
94
|
+
F1 over transitions with TransNet V2's protocol (two-frame tolerance), one decision rule fixed
|
|
95
|
+
before the test sets were scored. TransNet V2 is its published weights run through the same
|
|
96
|
+
decoding and scoring code.
|
|
97
|
+
|
|
98
|
+
| | BBC Planet Earth | ClipShots | RAI | Parameters | MACs per frame |
|
|
99
|
+
|---|---|---|---|---|---|
|
|
100
|
+
| TransNet V2 (as published) | 96.2 | 77.9 | 93.9 | 7.62 M | 832 M |
|
|
101
|
+
| TransNet V2 (same protocol) | 96.1 | 77.7 | 92.6 | | |
|
|
102
|
+
| LeanScene, mean of 5 seeds | 97.3 ± 0.2 | 79.1 ± 0.5 | 93.5 ± 1.2 | 0.49 M | 20.9 M |
|
|
103
|
+
| **The bundled weights** (one of the 5 seeds) | 97.25 | 79.00 | 93.47 | | |
|
|
104
|
+
|
|
105
|
+
What these numbers do and do not show:
|
|
106
|
+
|
|
107
|
+
- A paired bootstrap over videos separates LeanScene from TransNet V2 on **BBC only**
|
|
108
|
+
(+1.13, 95% interval +0.48 to +2.13). On ClipShots and RAI the intervals include zero:
|
|
109
|
+
that is parity, not a win. TransNet V2's published RAI number is above ours.
|
|
110
|
+
- The BBC lead shrinks to +0.33 if TransNet V2's threshold is tuned on the test set.
|
|
111
|
+
- The BBC lead depends on the hand labels in [`labels/`](labels/README.md). Trained without
|
|
112
|
+
them, the model scores 95.4 on BBC, 80.0 on ClipShots and 93.2 on RAI (mean of 3 seeds).
|
|
113
|
+
- The test sets were looked at during development. No benchmark here was unseen.
|
|
114
|
+
|
|
115
|
+
Model-only throughput on an Apple M1 Pro (8 threads, 6,000 frames, decoding excluded):
|
|
116
|
+
TransNet V2 159 frames/s on the CPU; LeanScene 570 frames/s in batches of 512 and 1,225 in
|
|
117
|
+
batches of 8 on the CPU, and 3,947 on the GPU (MPS). The CPU speed-up is 3.6x to 7.7x, far less
|
|
118
|
+
than the 40x reduction in operations. No CUDA timing has been measured.
|
|
119
|
+
|
|
120
|
+
## Weights
|
|
121
|
+
|
|
122
|
+
The package ships one set of weights (2 MB, sha256 `091853e5b35fbfd4…`, research run
|
|
123
|
+
`seqnet_v24_seed2`). It was trained on synthetic transitions built from single-shot
|
|
124
|
+
skateboarding clips, ClipShots training videos, and 1,245 hand labels on skateboarding videos.
|
|
125
|
+
The hand labels are in [`labels/`](labels/README.md), as YouTube ids and frame indices. The
|
|
126
|
+
clips behind the synthetic transitions are not released.
|
|
127
|
+
|
|
128
|
+
The paper trains the model five times from different random seeds and reports the mean. The
|
|
129
|
+
bundled file is the seed with the best F1 on the validation set (906 ClipShots training videos
|
|
130
|
+
held out from training), chosen without looking at the test sets.
|
|
131
|
+
|
|
132
|
+
## How it works
|
|
133
|
+
|
|
134
|
+
Each frame is resized to 100x100 and encoded once: a four-layer 2D CNN gives a global feature
|
|
135
|
+
and a 2x2 grid of features, and two descriptors come straight from the pixels (a 64-bin colour
|
|
136
|
+
histogram and an 8x8 colour thumbnail). A temporal head of five residual dilated 1D
|
|
137
|
+
convolutions (dilations 1, 2, 4, 8, 16; receptive field 63 frames) sees those features and
|
|
138
|
+
their change since the previous frame, and labels every frame *same*, *cut* or *fade*.
|
|
139
|
+
|
|
140
|
+
The video is streamed, so memory stays flat on long files: frames are encoded in batches and
|
|
141
|
+
the head runs over 4,096 frames at a time with 64 frames of context on each side, which gives
|
|
142
|
+
the same output as a single pass.
|
|
143
|
+
|
|
144
|
+
## Labels
|
|
145
|
+
|
|
146
|
+
[`labels/`](labels/README.md) holds the 1,245 hand labels (cuts, fades and confirmed
|
|
147
|
+
non-transitions on 466 skateboarding videos) the model was trained on.
|
|
148
|
+
|
|
149
|
+
## Browser demo
|
|
150
|
+
|
|
151
|
+
[`demo/`](demo/README.md) runs the same model in a browser tab with ONNX Runtime Web: drop a
|
|
152
|
+
video on the page and it shows the transitions on a timeline. Nothing is uploaded.
|
|
153
|
+
|
|
154
|
+
## Reproducing the numbers
|
|
155
|
+
|
|
156
|
+
[`eval/`](eval/README.md) scores the bundled weights on the three benchmarks with the
|
|
157
|
+
paper's protocol.
|
|
158
|
+
|
|
159
|
+
## Limits
|
|
160
|
+
|
|
161
|
+
- Videos are decoded with OpenCV. Times are frame index divided by the container's frame
|
|
162
|
+
rate, so they drift on variable-frame-rate files; frame indices are exact.
|
|
163
|
+
- `--device mps` is not the default on Apple hardware: a process that fed MPS many different
|
|
164
|
+
input lengths kept growing in memory in our runs.
|
|
165
|
+
- Training code is not in this repository.
|
|
166
|
+
|
|
167
|
+
## Development
|
|
168
|
+
|
|
169
|
+
```sh
|
|
170
|
+
python3 -m venv .venv
|
|
171
|
+
.venv/bin/python -m pip install -e ".[dev,export]"
|
|
172
|
+
.venv/bin/python -m pytest
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
`.venv/` is about 1 GB (mostly PyTorch), is gitignored, and is safe to delete; the two commands
|
|
176
|
+
above recreate it. The `export` extra is only needed for `demo/export_onnx.py`.
|
|
177
|
+
|
|
178
|
+
## Citation
|
|
179
|
+
|
|
180
|
+
```bibtex
|
|
181
|
+
@misc{jackson2026leanscene,
|
|
182
|
+
title = {LeanScene: A 0.5M-Parameter Network for Fast Shot Transition Detection},
|
|
183
|
+
author = {Lucas Jackson},
|
|
184
|
+
year = {2026},
|
|
185
|
+
note = {In preparation}
|
|
186
|
+
}
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
## Licence
|
|
190
|
+
|
|
191
|
+
- Code and model weights: MIT, see [LICENSE](LICENSE). `eval/evaluate.py` contains two functions
|
|
192
|
+
adapted from [TransNet V2](https://github.com/soCzech/TransNetV2) (MIT).
|
|
193
|
+
- The labels in [`labels/`](labels/README.md): [CC BY 4.0](labels/LICENSE). Use them freely, with credit.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
leanscene/__init__.py,sha256=Flyjg5nLC5t6B_cfs1I6n_5AJtXjURwfuvRDx3K6ofo,563
|
|
2
|
+
leanscene/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
|
|
3
|
+
leanscene/cli.py,sha256=jhXWZFvuJ9kce7p7nM30l3RnaSR_CaXcrK6uc-_NbJc,3959
|
|
4
|
+
leanscene/detect.py,sha256=v8oNAorkmxikzf8pARGuh86Vf5AI9ELhZFMFJco69Dg,8882
|
|
5
|
+
leanscene/model.py,sha256=7W7Genf3tiOv5Bg-idELAD9KGWSBikWJb-gcXmOW7Bw,4718
|
|
6
|
+
leanscene/weights/leanscene.pt,sha256=CRhT5bNfv9Q-wrpQbIiE2-PX_1_Ipkf1VHHHdk-qFkE,1982643
|
|
7
|
+
leanscene-0.1.0.dist-info/licenses/LICENSE,sha256=OYxHFdPHM2r92qrhcyalg0VP1tcu2lmDfEWUPEGbEWk,1070
|
|
8
|
+
leanscene-0.1.0.dist-info/METADATA,sha256=x0s_Tx7Dd-R3jIYp4mqBk2lSOReT5X5yqtjBcMf62S4,8041
|
|
9
|
+
leanscene-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
10
|
+
leanscene-0.1.0.dist-info/entry_points.txt,sha256=2V97gXubJLi5L4A_9a0441aw3E4fyJ2PeTCH2azFj8A,49
|
|
11
|
+
leanscene-0.1.0.dist-info/top_level.txt,sha256=qENOyNGlkcBIR0TVchkA-z-EoMAMkoL8kqcGDrlkUh8,10
|
|
12
|
+
leanscene-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lucas Jackson
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
leanscene
|