leanscene 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- leanscene-0.1.1/LICENSE +21 -0
- leanscene-0.1.1/PKG-INFO +247 -0
- leanscene-0.1.1/README.md +223 -0
- leanscene-0.1.1/pyproject.toml +43 -0
- leanscene-0.1.1/setup.cfg +4 -0
- leanscene-0.1.1/src/leanscene/__init__.py +10 -0
- leanscene-0.1.1/src/leanscene/__main__.py +3 -0
- leanscene-0.1.1/src/leanscene/cli.py +153 -0
- leanscene-0.1.1/src/leanscene/detect.py +230 -0
- leanscene-0.1.1/src/leanscene/model.py +102 -0
- leanscene-0.1.1/src/leanscene/weights/leanscene.pt +0 -0
- leanscene-0.1.1/src/leanscene.egg-info/PKG-INFO +247 -0
- leanscene-0.1.1/src/leanscene.egg-info/SOURCES.txt +18 -0
- leanscene-0.1.1/src/leanscene.egg-info/dependency_links.txt +1 -0
- leanscene-0.1.1/src/leanscene.egg-info/entry_points.txt +2 -0
- leanscene-0.1.1/src/leanscene.egg-info/requires.txt +10 -0
- leanscene-0.1.1/src/leanscene.egg-info/top_level.txt +1 -0
- leanscene-0.1.1/tests/test_detect.py +99 -0
- leanscene-0.1.1/tests/test_rule.py +49 -0
- leanscene-0.1.1/tests/test_training.py +71 -0
leanscene-0.1.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lucas Jackson
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
leanscene-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: leanscene
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Fast shot transition detection in 0.39M parameters: higher F1 than TransNet V2 on its three benchmarks, with 115x fewer operations
|
|
5
|
+
Author: Lucas Jackson
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/lucasjacks0n/LeanScene
|
|
8
|
+
Keywords: shot boundary detection,scene detection,video,shot transition detection
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Topic :: Multimedia :: Video
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: numpy
|
|
16
|
+
Requires-Dist: opencv-python-headless
|
|
17
|
+
Requires-Dist: torch>=2.0
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest; extra == "dev"
|
|
20
|
+
Provides-Extra: export
|
|
21
|
+
Requires-Dist: onnx; extra == "export"
|
|
22
|
+
Requires-Dist: onnxruntime; extra == "export"
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
<h1 align="center">
|
|
26
|
+
<picture>
|
|
27
|
+
<source media="(prefers-color-scheme: dark)" srcset="logo/leanscene-wordmark-dark.svg">
|
|
28
|
+
<img src="logo/leanscene-wordmark.svg" alt="LeanScene" width="440">
|
|
29
|
+
</picture>
|
|
30
|
+
</h1>
|
|
31
|
+
|
|
32
|
+
A 0.39M-parameter network that finds shot transitions (cuts, dissolves, fades) in video.
|
|
33
|
+
It scores higher than TransNet V2 on all three of that model's benchmarks, with 115x fewer
|
|
34
|
+
operations per frame.
|
|
35
|
+
|
|
36
|
+
**[Try it in your browser](https://lucasjackson.dev/leanscene/)**: drop a video and see every
|
|
37
|
+
transition on a timeline. It runs in the tab; nothing is uploaded.
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
# Python
|
|
41
|
+
pip install git+https://github.com/lucasjacks0n/LeanScene
|
|
42
|
+
leanscene video.mp4
|
|
43
|
+
|
|
44
|
+
# Node.js
|
|
45
|
+
npm install leanscene
|
|
46
|
+
npx leanscene video.mp4
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The Node.js package is [on npm](https://www.npmjs.com/package/leanscene). A PyPI release
|
|
50
|
+
(`pip install leanscene`) is on its way; until then, install the Python package from GitHub as above.
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
video.mp4: 14775 frames, 25.000 fps, 51 shots, 50 transitions
|
|
54
|
+
shot start end start time end time
|
|
55
|
+
1 0 343 00:00:00.000 00:00:13.760
|
|
56
|
+
2 344 2403 00:00:13.760 00:01:36.160
|
|
57
|
+
...
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The weights (1.6 MB) ship inside the package; nothing is downloaded at run time. The paper is in
|
|
61
|
+
preparation.
|
|
62
|
+
|
|
63
|
+
## Python
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
import leanscene
|
|
67
|
+
|
|
68
|
+
result = leanscene.detect("video.mp4")
|
|
69
|
+
result.shots # [(0, 343), (344, 2403), (2414, 3315), ...] inclusive frame ranges, 0-based
|
|
70
|
+
result.transitions # [Transition(start=344, end=344, kind='cut'), Transition(start=2404, end=2413, kind='gradual'), ...]
|
|
71
|
+
result.probs # (frames, 3) float32: same, cut, fade
|
|
72
|
+
result.to_dict() # JSON-ready, with times in seconds
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
To process many videos, load the model once:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
model = leanscene.load_model() # or load_model(device="cuda")
|
|
79
|
+
for path in paths:
|
|
80
|
+
result = leanscene.detect(path, model=model)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`detect` also takes an iterable of `HxWx3` uint8 frames in BGR order instead of a path, and
|
|
84
|
+
`cut=` / `fade=` thresholds (defaults 0.7 and 0.8, the rule used for every number below).
|
|
85
|
+
`frame_probs` returns only the probabilities; `label_frames`, `find_transitions` and
|
|
86
|
+
`find_shots` are the steps `detect` applies to them.
|
|
87
|
+
|
|
88
|
+
A cut is reported on the first frame of the new shot. A gradual transition is the run of
|
|
89
|
+
frames inside it, and those frames belong to neither neighbouring shot.
|
|
90
|
+
|
|
91
|
+
## Command line
|
|
92
|
+
|
|
93
|
+
| Option | Effect |
|
|
94
|
+
|---|---|
|
|
95
|
+
| `-f table\|json\|csv` | Output format. `table` (default) and `csv` list shots; `json` has shots and transitions |
|
|
96
|
+
| `-o FILE` | Write to a file instead of standard output |
|
|
97
|
+
| `--model PATH` | A checkpoint file, to use weights other than the bundled ones |
|
|
98
|
+
| `--device DEVICE` | `cpu`, `cuda` or `mps`. Default: `cuda` when available, else `cpu` |
|
|
99
|
+
| `--cut X`, `--fade X` | Thresholds (0.7, 0.8). Higher gives fewer false alarms, lower misses less |
|
|
100
|
+
| `--probs FILE.npy` | Also save the per-frame probabilities |
|
|
101
|
+
| `-q`, `--quiet` | No progress line |
|
|
102
|
+
|
|
103
|
+
In a terminal it shows a live progress line (frames done, frames per second, time left). The line goes to
|
|
104
|
+
standard error, so piped or redirected output stays clean. From Python, pass `progress=` a function that
|
|
105
|
+
takes the number of frames read so far.
|
|
106
|
+
|
|
107
|
+
Several videos can be given at once. `python -m leanscene` is the same command.
|
|
108
|
+
|
|
109
|
+
## Node.js
|
|
110
|
+
|
|
111
|
+
[`js/`](https://github.com/lucasjacks0n/LeanScene/blob/main/js/README.md) is the same model as an [npm package](https://www.npmjs.com/package/leanscene) for Node, with TypeScript types and
|
|
112
|
+
the same command line. It decodes with ffmpeg and runs the network with ONNX Runtime.
|
|
113
|
+
|
|
114
|
+
```sh
|
|
115
|
+
npm install leanscene
|
|
116
|
+
npx leanscene video.mp4
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
```ts
|
|
120
|
+
import { detect } from "leanscene";
|
|
121
|
+
const { shots, transitions } = await detect("video.mp4");
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
On 8 test videos it found 527 transitions where the Python package found 524, at 1,922 frames
|
|
125
|
+
per second end to end on a laptop CPU against 1,106; its README has the comparison.
|
|
126
|
+
|
|
127
|
+
## Results
|
|
128
|
+
|
|
129
|
+
LeanScene scores higher than TransNet V2 on all three benchmarks, at a twentieth of the size and with 115x
|
|
130
|
+
fewer operations per frame.
|
|
131
|
+
|
|
132
|
+
| | BBC Planet Earth | ClipShots | RAI | Parameters | MACs per frame |
|
|
133
|
+
|---|---|---|---|---|---|
|
|
134
|
+
| TransNet V2 | 96.1 | 77.7 | 92.6 | 7.62 M | 832 M |
|
|
135
|
+
| **LeanScene** (mean of 5 seeds) | **97.4** ± 0.1 | **79.1** ± 0.2 | **93.7** ± 0.5 | **0.39 M** | **7.2 M** |
|
|
136
|
+
| The bundled weights (one of the 5 seeds) | 97.45 | 78.82 | 93.03 | | |
|
|
137
|
+
| LeanScene at full width (not shipped; mean of 5 seeds) | 97.3 ± 0.2 | 79.1 ± 0.5 | 93.5 ± 1.2 | 0.49 M | 20.9 M |
|
|
138
|
+
|
|
139
|
+
F1 over transitions with TransNet V2's own protocol (two-frame tolerance). Both models are run
|
|
140
|
+
through the same decoding and scoring code, and LeanScene's decision rule was fixed before the
|
|
141
|
+
test sets were scored.
|
|
142
|
+
|
|
143
|
+
It is fast as well as small. On an Apple M1 Pro the full-width model alone runs at 1,225
|
|
144
|
+
frames/s on the CPU and 3,947 on the GPU, against 159 for TransNet V2 on the CPU. The shipped
|
|
145
|
+
model does about a third of the full-width model's operations.
|
|
146
|
+
|
|
147
|
+
<details>
|
|
148
|
+
<summary>Fine print</summary>
|
|
149
|
+
|
|
150
|
+
- TransNet V2's paper reports 96.2, 77.9 and 93.9. The row above is its published weights
|
|
151
|
+
measured by us; we could not reproduce its RAI number.
|
|
152
|
+
- Every one of the five seeds is above TransNet V2 on every benchmark.
|
|
153
|
+
- A paired bootstrap over videos puts the BBC lead clear of zero (+1.28, 95% interval +0.50 to
|
|
154
|
+
+2.47). The ClipShots and RAI leads (+1.36 and +1.13) have intervals that include zero.
|
|
155
|
+
- The BBC lead shrinks to +0.48 if TransNet V2's threshold is tuned on the test set, and it
|
|
156
|
+
depends on the hand labels in [`labels/`](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/README.md): trained without them, the
|
|
157
|
+
full-width model scores 95.4 on BBC, 80.0 on ClipShots and 93.2 on RAI (mean of 3 seeds).
|
|
158
|
+
- The test sets were looked at during development.
|
|
159
|
+
- Speeds are model only (8 threads, 6,000 frames, decoding excluded); LeanScene's CPU figure is
|
|
160
|
+
with batches of 8 frames (570 frames/s with batches of 512). They were measured on the
|
|
161
|
+
full-width model; the shipped model has not been timed yet, and no CUDA timing has been measured.
|
|
162
|
+
|
|
163
|
+
</details>
|
|
164
|
+
|
|
165
|
+
## Weights
|
|
166
|
+
|
|
167
|
+
The package ships one set of weights (1.6 MB, sha256 `b0d1cc2a77f8aa91…`, research run
|
|
168
|
+
`seqnet_w05_seed2`). It was trained on synthetic transitions built from single-shot
|
|
169
|
+
skateboarding clips, ClipShots training videos, and 1,245 hand labels on skateboarding videos.
|
|
170
|
+
The hand labels are in [`labels/`](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/README.md), as YouTube ids and frame indices. The
|
|
171
|
+
clips behind the synthetic transitions are not released.
|
|
172
|
+
|
|
173
|
+
The paper trains the model five times from different random seeds and reports the mean. The
|
|
174
|
+
bundled file is the seed with the best F1 on the validation set (906 ClipShots training videos
|
|
175
|
+
held out from training), chosen without looking at the test sets.
|
|
176
|
+
|
|
177
|
+
## How it works
|
|
178
|
+
|
|
179
|
+
Each frame is resized to 100x100 and encoded once: a four-layer 2D CNN gives a global feature
|
|
180
|
+
and a 2x2 grid of features, and two descriptors come straight from the pixels (a 64-bin colour
|
|
181
|
+
histogram and an 8x8 colour thumbnail). A temporal head of five residual dilated 1D
|
|
182
|
+
convolutions (dilations 1, 2, 4, 8, 16; receptive field 63 frames) sees those features and
|
|
183
|
+
their change since the previous frame, and labels every frame *same*, *cut* or *fade*.
|
|
184
|
+
|
|
185
|
+
The video is streamed, so memory stays flat on long files: frames are encoded in batches and
|
|
186
|
+
the head runs over 4,096 frames at a time with 64 frames of context on each side, which gives
|
|
187
|
+
the same output as a single pass.
|
|
188
|
+
|
|
189
|
+
## Labels
|
|
190
|
+
|
|
191
|
+
[`labels/`](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/README.md) holds the 1,245 hand labels (cuts, fades and confirmed
|
|
192
|
+
non-transitions on 466 skateboarding videos) the model was trained on.
|
|
193
|
+
|
|
194
|
+
## Browser demo
|
|
195
|
+
|
|
196
|
+
Live at **[lucasjackson.dev/leanscene](https://lucasjackson.dev/leanscene/)**.
|
|
197
|
+
[`demo/`](https://github.com/lucasjacks0n/LeanScene/blob/main/demo/README.md) runs the same model in a browser tab with ONNX Runtime Web: drop a
|
|
198
|
+
video on the page and it shows the transitions on a timeline, then lets you search the shots by
|
|
199
|
+
describing them (one frame per shot is embedded with MobileCLIP, also in the tab). The video is
|
|
200
|
+
never uploaded.
|
|
201
|
+
|
|
202
|
+
## Reproducing the numbers
|
|
203
|
+
|
|
204
|
+
[`eval/`](https://github.com/lucasjacks0n/LeanScene/blob/main/eval/README.md) scores the bundled weights on the three benchmarks with the
|
|
205
|
+
paper's protocol.
|
|
206
|
+
|
|
207
|
+
## Training
|
|
208
|
+
|
|
209
|
+
[`training/`](https://github.com/lucasjacks0n/LeanScene/blob/main/training/README.md) is the code that trained the bundled weights: the trainer, the
|
|
210
|
+
synthetic transitions, and the mining and cleaning of hard negatives, with the command for
|
|
211
|
+
each step of the paper's recipe. The single-shot clips behind the synthetic sequences are not
|
|
212
|
+
distributed, so bring your own to train.
|
|
213
|
+
|
|
214
|
+
## Limits
|
|
215
|
+
|
|
216
|
+
- Videos are decoded with OpenCV. Times are frame index divided by the container's frame
|
|
217
|
+
rate, so they drift on variable-frame-rate files; frame indices are exact.
|
|
218
|
+
- `--device mps` is not the default on Apple hardware: a process that fed MPS many different
|
|
219
|
+
input lengths kept growing in memory in our runs.
|
|
220
|
+
|
|
221
|
+
## Development
|
|
222
|
+
|
|
223
|
+
```sh
|
|
224
|
+
python3 -m venv .venv
|
|
225
|
+
.venv/bin/python -m pip install -e ".[dev,export]"
|
|
226
|
+
.venv/bin/python -m pytest
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
`.venv/` is about 1 GB (mostly PyTorch), is gitignored, and is safe to delete; the two commands
|
|
230
|
+
above recreate it. The `export` extra is only needed for `demo/export_onnx.py`.
|
|
231
|
+
|
|
232
|
+
## Citation
|
|
233
|
+
|
|
234
|
+
```bibtex
|
|
235
|
+
@misc{jackson2026leanscene,
|
|
236
|
+
title = {LeanScene: A 0.39M-Parameter Network for Fast Shot Transition Detection},
|
|
237
|
+
author = {Lucas Jackson},
|
|
238
|
+
year = {2026},
|
|
239
|
+
note = {In preparation}
|
|
240
|
+
}
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
## Licence
|
|
244
|
+
|
|
245
|
+
- Code and model weights: MIT, see [LICENSE](https://github.com/lucasjacks0n/LeanScene/blob/main/LICENSE). `eval/evaluate.py` contains two functions
|
|
246
|
+
adapted from [TransNet V2](https://github.com/soCzech/TransNetV2) (MIT).
|
|
247
|
+
- The labels in [`labels/`](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/README.md): [CC BY 4.0](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/LICENSE). Use them freely, with credit.
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
<h1 align="center">
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: dark)" srcset="logo/leanscene-wordmark-dark.svg">
|
|
4
|
+
<img src="logo/leanscene-wordmark.svg" alt="LeanScene" width="440">
|
|
5
|
+
</picture>
|
|
6
|
+
</h1>
|
|
7
|
+
|
|
8
|
+
A 0.39M-parameter network that finds shot transitions (cuts, dissolves, fades) in video.
|
|
9
|
+
It scores higher than TransNet V2 on all three of that model's benchmarks, with 115x fewer
|
|
10
|
+
operations per frame.
|
|
11
|
+
|
|
12
|
+
**[Try it in your browser](https://lucasjackson.dev/leanscene/)**: drop a video and see every
|
|
13
|
+
transition on a timeline. It runs in the tab; nothing is uploaded.
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
# Python
|
|
17
|
+
pip install git+https://github.com/lucasjacks0n/LeanScene
|
|
18
|
+
leanscene video.mp4
|
|
19
|
+
|
|
20
|
+
# Node.js
|
|
21
|
+
npm install leanscene
|
|
22
|
+
npx leanscene video.mp4
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
The Node.js package is [on npm](https://www.npmjs.com/package/leanscene). A PyPI release
|
|
26
|
+
(`pip install leanscene`) is on its way; until then, install the Python package from GitHub as above.
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
video.mp4: 14775 frames, 25.000 fps, 51 shots, 50 transitions
|
|
30
|
+
shot start end start time end time
|
|
31
|
+
1 0 343 00:00:00.000 00:00:13.760
|
|
32
|
+
2 344 2403 00:00:13.760 00:01:36.160
|
|
33
|
+
...
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The weights (1.6 MB) ship inside the package; nothing is downloaded at run time. The paper is in
|
|
37
|
+
preparation.
|
|
38
|
+
|
|
39
|
+
## Python
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
import leanscene
|
|
43
|
+
|
|
44
|
+
result = leanscene.detect("video.mp4")
|
|
45
|
+
result.shots # [(0, 343), (344, 2403), (2414, 3315), ...] inclusive frame ranges, 0-based
|
|
46
|
+
result.transitions # [Transition(start=344, end=344, kind='cut'), Transition(start=2404, end=2413, kind='gradual'), ...]
|
|
47
|
+
result.probs # (frames, 3) float32: same, cut, fade
|
|
48
|
+
result.to_dict() # JSON-ready, with times in seconds
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
To process many videos, load the model once:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
model = leanscene.load_model() # or load_model(device="cuda")
|
|
55
|
+
for path in paths:
|
|
56
|
+
result = leanscene.detect(path, model=model)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
`detect` also takes an iterable of `HxWx3` uint8 frames in BGR order instead of a path, and
|
|
60
|
+
`cut=` / `fade=` thresholds (defaults 0.7 and 0.8, the rule used for every number below).
|
|
61
|
+
`frame_probs` returns only the probabilities; `label_frames`, `find_transitions` and
|
|
62
|
+
`find_shots` are the steps `detect` applies to them.
|
|
63
|
+
|
|
64
|
+
A cut is reported on the first frame of the new shot. A gradual transition is the run of
|
|
65
|
+
frames inside it, and those frames belong to neither neighbouring shot.
|
|
66
|
+
|
|
67
|
+
## Command line
|
|
68
|
+
|
|
69
|
+
| Option | Effect |
|
|
70
|
+
|---|---|
|
|
71
|
+
| `-f table\|json\|csv` | Output format. `table` (default) and `csv` list shots; `json` has shots and transitions |
|
|
72
|
+
| `-o FILE` | Write to a file instead of standard output |
|
|
73
|
+
| `--model PATH` | A checkpoint file, to use weights other than the bundled ones |
|
|
74
|
+
| `--device DEVICE` | `cpu`, `cuda` or `mps`. Default: `cuda` when available, else `cpu` |
|
|
75
|
+
| `--cut X`, `--fade X` | Thresholds (0.7, 0.8). Higher gives fewer false alarms, lower misses less |
|
|
76
|
+
| `--probs FILE.npy` | Also save the per-frame probabilities |
|
|
77
|
+
| `-q`, `--quiet` | No progress line |
|
|
78
|
+
|
|
79
|
+
In a terminal it shows a live progress line (frames done, frames per second, time left). The line goes to
|
|
80
|
+
standard error, so piped or redirected output stays clean. From Python, pass `progress=` a function that
|
|
81
|
+
takes the number of frames read so far.
|
|
82
|
+
|
|
83
|
+
Several videos can be given at once. `python -m leanscene` is the same command.
|
|
84
|
+
|
|
85
|
+
## Node.js
|
|
86
|
+
|
|
87
|
+
[`js/`](https://github.com/lucasjacks0n/LeanScene/blob/main/js/README.md) is the same model as an [npm package](https://www.npmjs.com/package/leanscene) for Node, with TypeScript types and
|
|
88
|
+
the same command line. It decodes with ffmpeg and runs the network with ONNX Runtime.
|
|
89
|
+
|
|
90
|
+
```sh
|
|
91
|
+
npm install leanscene
|
|
92
|
+
npx leanscene video.mp4
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
```ts
|
|
96
|
+
import { detect } from "leanscene";
|
|
97
|
+
const { shots, transitions } = await detect("video.mp4");
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
On 8 test videos it found 527 transitions where the Python package found 524, at 1,922 frames
|
|
101
|
+
per second end to end on a laptop CPU against 1,106; its README has the comparison.
|
|
102
|
+
|
|
103
|
+
## Results
|
|
104
|
+
|
|
105
|
+
LeanScene scores higher than TransNet V2 on all three benchmarks, at a twentieth of the size and with 115x
|
|
106
|
+
fewer operations per frame.
|
|
107
|
+
|
|
108
|
+
| | BBC Planet Earth | ClipShots | RAI | Parameters | MACs per frame |
|
|
109
|
+
|---|---|---|---|---|---|
|
|
110
|
+
| TransNet V2 | 96.1 | 77.7 | 92.6 | 7.62 M | 832 M |
|
|
111
|
+
| **LeanScene** (mean of 5 seeds) | **97.4** ± 0.1 | **79.1** ± 0.2 | **93.7** ± 0.5 | **0.39 M** | **7.2 M** |
|
|
112
|
+
| The bundled weights (one of the 5 seeds) | 97.45 | 78.82 | 93.03 | | |
|
|
113
|
+
| LeanScene at full width (not shipped; mean of 5 seeds) | 97.3 ± 0.2 | 79.1 ± 0.5 | 93.5 ± 1.2 | 0.49 M | 20.9 M |
|
|
114
|
+
|
|
115
|
+
F1 over transitions with TransNet V2's own protocol (two-frame tolerance). Both models are run
|
|
116
|
+
through the same decoding and scoring code, and LeanScene's decision rule was fixed before the
|
|
117
|
+
test sets were scored.
|
|
118
|
+
|
|
119
|
+
It is fast as well as small. On an Apple M1 Pro the full-width model alone runs at 1,225
|
|
120
|
+
frames/s on the CPU and 3,947 on the GPU, against 159 for TransNet V2 on the CPU. The shipped
|
|
121
|
+
model does about a third of the full-width model's operations.
|
|
122
|
+
|
|
123
|
+
<details>
|
|
124
|
+
<summary>Fine print</summary>
|
|
125
|
+
|
|
126
|
+
- TransNet V2's paper reports 96.2, 77.9 and 93.9. The row above is its published weights
|
|
127
|
+
measured by us; we could not reproduce its RAI number.
|
|
128
|
+
- Every one of the five seeds is above TransNet V2 on every benchmark.
|
|
129
|
+
- A paired bootstrap over videos puts the BBC lead clear of zero (+1.28, 95% interval +0.50 to
|
|
130
|
+
+2.47). The ClipShots and RAI leads (+1.36 and +1.13) have intervals that include zero.
|
|
131
|
+
- The BBC lead shrinks to +0.48 if TransNet V2's threshold is tuned on the test set, and it
|
|
132
|
+
depends on the hand labels in [`labels/`](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/README.md): trained without them, the
|
|
133
|
+
full-width model scores 95.4 on BBC, 80.0 on ClipShots and 93.2 on RAI (mean of 3 seeds).
|
|
134
|
+
- The test sets were looked at during development.
|
|
135
|
+
- Speeds are model only (8 threads, 6,000 frames, decoding excluded); LeanScene's CPU figure is
|
|
136
|
+
with batches of 8 frames (570 frames/s with batches of 512). They were measured on the
|
|
137
|
+
full-width model; the shipped model has not been timed yet, and no CUDA timing has been measured.
|
|
138
|
+
|
|
139
|
+
</details>
|
|
140
|
+
|
|
141
|
+
## Weights
|
|
142
|
+
|
|
143
|
+
The package ships one set of weights (1.6 MB, sha256 `b0d1cc2a77f8aa91…`, research run
|
|
144
|
+
`seqnet_w05_seed2`). It was trained on synthetic transitions built from single-shot
|
|
145
|
+
skateboarding clips, ClipShots training videos, and 1,245 hand labels on skateboarding videos.
|
|
146
|
+
The hand labels are in [`labels/`](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/README.md), as YouTube ids and frame indices. The
|
|
147
|
+
clips behind the synthetic transitions are not released.
|
|
148
|
+
|
|
149
|
+
The paper trains the model five times from different random seeds and reports the mean. The
|
|
150
|
+
bundled file is the seed with the best F1 on the validation set (906 ClipShots training videos
|
|
151
|
+
held out from training), chosen without looking at the test sets.
|
|
152
|
+
|
|
153
|
+
## How it works
|
|
154
|
+
|
|
155
|
+
Each frame is resized to 100x100 and encoded once: a four-layer 2D CNN gives a global feature
|
|
156
|
+
and a 2x2 grid of features, and two descriptors come straight from the pixels (a 64-bin colour
|
|
157
|
+
histogram and an 8x8 colour thumbnail). A temporal head of five residual dilated 1D
|
|
158
|
+
convolutions (dilations 1, 2, 4, 8, 16; receptive field 63 frames) sees those features and
|
|
159
|
+
their change since the previous frame, and labels every frame *same*, *cut* or *fade*.
|
|
160
|
+
|
|
161
|
+
The video is streamed, so memory stays flat on long files: frames are encoded in batches and
|
|
162
|
+
the head runs over 4,096 frames at a time with 64 frames of context on each side, which gives
|
|
163
|
+
the same output as a single pass.
|
|
164
|
+
|
|
165
|
+
## Labels
|
|
166
|
+
|
|
167
|
+
[`labels/`](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/README.md) holds the 1,245 hand labels (cuts, fades and confirmed
|
|
168
|
+
non-transitions on 466 skateboarding videos) the model was trained on.
|
|
169
|
+
|
|
170
|
+
## Browser demo
|
|
171
|
+
|
|
172
|
+
Live at **[lucasjackson.dev/leanscene](https://lucasjackson.dev/leanscene/)**.
|
|
173
|
+
[`demo/`](https://github.com/lucasjacks0n/LeanScene/blob/main/demo/README.md) runs the same model in a browser tab with ONNX Runtime Web: drop a
|
|
174
|
+
video on the page and it shows the transitions on a timeline, then lets you search the shots by
|
|
175
|
+
describing them (one frame per shot is embedded with MobileCLIP, also in the tab). The video is
|
|
176
|
+
never uploaded.
|
|
177
|
+
|
|
178
|
+
## Reproducing the numbers
|
|
179
|
+
|
|
180
|
+
[`eval/`](https://github.com/lucasjacks0n/LeanScene/blob/main/eval/README.md) scores the bundled weights on the three benchmarks with the
|
|
181
|
+
paper's protocol.
|
|
182
|
+
|
|
183
|
+
## Training
|
|
184
|
+
|
|
185
|
+
[`training/`](https://github.com/lucasjacks0n/LeanScene/blob/main/training/README.md) is the code that trained the bundled weights: the trainer, the
|
|
186
|
+
synthetic transitions, and the mining and cleaning of hard negatives, with the command for
|
|
187
|
+
each step of the paper's recipe. The single-shot clips behind the synthetic sequences are not
|
|
188
|
+
distributed, so bring your own to train.
|
|
189
|
+
|
|
190
|
+
## Limits
|
|
191
|
+
|
|
192
|
+
- Videos are decoded with OpenCV. Times are frame index divided by the container's frame
|
|
193
|
+
rate, so they drift on variable-frame-rate files; frame indices are exact.
|
|
194
|
+
- `--device mps` is not the default on Apple hardware: a process that fed MPS many different
|
|
195
|
+
input lengths kept growing in memory in our runs.
|
|
196
|
+
|
|
197
|
+
## Development
|
|
198
|
+
|
|
199
|
+
```sh
|
|
200
|
+
python3 -m venv .venv
|
|
201
|
+
.venv/bin/python -m pip install -e ".[dev,export]"
|
|
202
|
+
.venv/bin/python -m pytest
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
`.venv/` is about 1 GB (mostly PyTorch), is gitignored, and is safe to delete; the two commands
|
|
206
|
+
above recreate it. The `export` extra is only needed for `demo/export_onnx.py`.
|
|
207
|
+
|
|
208
|
+
## Citation
|
|
209
|
+
|
|
210
|
+
```bibtex
|
|
211
|
+
@misc{jackson2026leanscene,
|
|
212
|
+
title = {LeanScene: A 0.39M-Parameter Network for Fast Shot Transition Detection},
|
|
213
|
+
author = {Lucas Jackson},
|
|
214
|
+
year = {2026},
|
|
215
|
+
note = {In preparation}
|
|
216
|
+
}
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
## Licence
|
|
220
|
+
|
|
221
|
+
- Code and model weights: MIT, see [LICENSE](https://github.com/lucasjacks0n/LeanScene/blob/main/LICENSE). `eval/evaluate.py` contains two functions
|
|
222
|
+
adapted from [TransNet V2](https://github.com/soCzech/TransNetV2) (MIT).
|
|
223
|
+
- The labels in [`labels/`](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/README.md): [CC BY 4.0](https://github.com/lucasjacks0n/LeanScene/blob/main/labels/LICENSE). Use them freely, with credit.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "leanscene"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "Fast shot transition detection in 0.39M parameters: higher F1 than TransNet V2 on its three benchmarks, with 115x fewer operations"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Lucas Jackson" }]
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
keywords = ["shot boundary detection", "scene detection", "video", "shot transition detection"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Topic :: Multimedia :: Video",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
19
|
+
]
|
|
20
|
+
dependencies = [
|
|
21
|
+
"numpy",
|
|
22
|
+
"opencv-python-headless",
|
|
23
|
+
"torch>=2.0",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.optional-dependencies]
|
|
27
|
+
dev = ["pytest"]
|
|
28
|
+
export = ["onnx", "onnxruntime"]
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
leanscene = "leanscene.cli:main"
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Repository = "https://github.com/lucasjacks0n/LeanScene"
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.packages.find]
|
|
37
|
+
where = ["src"]
|
|
38
|
+
|
|
39
|
+
[tool.setuptools.package-data]
|
|
40
|
+
leanscene = ["weights/*.pt"]
|
|
41
|
+
|
|
42
|
+
[tool.pytest.ini_options]
|
|
43
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""LeanScene: a 0.39M-parameter network for fast shot transition detection."""
|
|
2
|
+
|
|
3
|
+
from .detect import (CUT_THRESHOLD, FADE_CLOSE, FADE_THRESHOLD, MODELS, Result, Transition, default_device, detect,
|
|
4
|
+
find_shots, find_transitions, frame_probs, label_frames, load_model)
|
|
5
|
+
from .model import LeanScene
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.1"
|
|
8
|
+
|
|
9
|
+
__all__ = ["CUT_THRESHOLD", "FADE_CLOSE", "FADE_THRESHOLD", "MODELS", "LeanScene", "Result", "Transition",
|
|
10
|
+
"default_device", "detect", "find_shots", "find_transitions", "frame_probs", "label_frames", "load_model"]
|