iterframes 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {iterframes-0.4.0 → iterframes-0.5.0}/.github/workflows/ci.yaml +70 -1
- iterframes-0.5.0/.release-please-manifest.json +3 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/AGENTS.md +33 -9
- {iterframes-0.4.0 → iterframes-0.5.0}/CHANGELOG.md +13 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/Cargo.lock +1 -1
- {iterframes-0.4.0 → iterframes-0.5.0}/Cargo.toml +1 -1
- iterframes-0.5.0/PKG-INFO +196 -0
- iterframes-0.5.0/README.md +172 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/build.rs +17 -1
- {iterframes-0.4.0 → iterframes-0.5.0}/docs/development.md +34 -4
- {iterframes-0.4.0 → iterframes-0.5.0}/docs/index.md +16 -1
- {iterframes-0.4.0 → iterframes-0.5.0}/docs/reference.md +80 -13
- iterframes-0.5.0/iterframes/__init__.py +149 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/pyproject.toml +4 -1
- {iterframes-0.4.0 → iterframes-0.5.0}/scripts/build-ffmpeg.sh +46 -6
- iterframes-0.5.0/src/decoder.rs +637 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/src/ffmpeg.rs +202 -25
- {iterframes-0.4.0 → iterframes-0.5.0}/src/lib.rs +167 -42
- iterframes-0.5.0/tests/conftest.py +57 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/tests/test_benchmark.py +49 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/tests/test_read.py +107 -15
- iterframes-0.5.0/tests/test_select.py +293 -0
- iterframes-0.4.0/.release-please-manifest.json +0 -3
- iterframes-0.4.0/PKG-INFO +0 -106
- iterframes-0.4.0/README.md +0 -82
- iterframes-0.4.0/iterframes/__init__.py +0 -88
- iterframes-0.4.0/src/decoder.rs +0 -183
- iterframes-0.4.0/tests/conftest.py +0 -39
- {iterframes-0.4.0 → iterframes-0.5.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/.gitignore +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/.pre-commit-config.yaml +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/.readthedocs.yaml +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/CLAUDE.md +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/LICENSE +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/mkdocs.yml +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/release-please-config.json +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/src/dlpack.rs +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/tests/data/video_480x270.mp4 +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/tests/data/video_av1_480x270.mp4 +0 -0
- {iterframes-0.4.0 → iterframes-0.5.0}/uv.lock +0 -0
|
@@ -147,6 +147,75 @@ jobs:
|
|
|
147
147
|
uv pip install --reinstall-package iterframes dist/*.whl
|
|
148
148
|
uv run --no-sync pytest
|
|
149
149
|
|
|
150
|
+
windows:
|
|
151
|
+
runs-on: ${{ matrix.platform.runner }}
|
|
152
|
+
strategy:
|
|
153
|
+
matrix:
|
|
154
|
+
platform:
|
|
155
|
+
- runner: windows-2025
|
|
156
|
+
target: x64
|
|
157
|
+
defaults:
|
|
158
|
+
run:
|
|
159
|
+
# MSYS2 runs the FFmpeg build script; path-type inherit keeps MSVC,
|
|
160
|
+
# cargo, and uv on PATH, after MSYS2's own tools.
|
|
161
|
+
shell: msys2 {0}
|
|
162
|
+
steps:
|
|
163
|
+
- uses: actions/checkout@v7
|
|
164
|
+
- name: Set up MSVC
|
|
165
|
+
uses: ilammy/msvc-dev-cmd@v1
|
|
166
|
+
with:
|
|
167
|
+
arch: ${{ matrix.platform.target }}
|
|
168
|
+
- name: Set up MSYS2
|
|
169
|
+
uses: msys2/setup-msys2@v2
|
|
170
|
+
with:
|
|
171
|
+
msystem: UCRT64
|
|
172
|
+
path-type: inherit
|
|
173
|
+
# Native pkg-config and nasm; FFmpeg and dav1d compile with MSVC.
|
|
174
|
+
install: >-
|
|
175
|
+
make diffutils curl tar xz
|
|
176
|
+
mingw-w64-ucrt-x86_64-pkgconf
|
|
177
|
+
mingw-w64-ucrt-x86_64-nasm
|
|
178
|
+
- name: Remove MSYS2's link
|
|
179
|
+
# It would shadow MSVC's link.exe, for rustc among others.
|
|
180
|
+
run: rm -f /usr/bin/link.exe
|
|
181
|
+
- name: Install uv
|
|
182
|
+
uses: astral-sh/setup-uv@v10.1.0
|
|
183
|
+
- name: Install meson and ninja
|
|
184
|
+
# With a native Python, so that meson sees Windows and uses MSVC.
|
|
185
|
+
run: |
|
|
186
|
+
uv tool install meson
|
|
187
|
+
uv tool install ninja
|
|
188
|
+
echo "$(cygpath -u "$(uv tool dir --bin)")" >> "$GITHUB_PATH"
|
|
189
|
+
- name: Cache FFmpeg
|
|
190
|
+
uses: actions/cache@v6
|
|
191
|
+
with:
|
|
192
|
+
path: build/ffmpeg
|
|
193
|
+
key: ffmpeg-windows-${{ matrix.platform.target }}-${{ hashFiles('scripts/build-ffmpeg.sh') }}
|
|
194
|
+
- name: Build wheels
|
|
195
|
+
run: |
|
|
196
|
+
uv sync --frozen --no-install-project
|
|
197
|
+
uv run --no-sync maturin build --release --out dist
|
|
198
|
+
- name: Show the FFmpeg build log
|
|
199
|
+
if: failure()
|
|
200
|
+
run: |
|
|
201
|
+
tail -n 200 build/ffmpeg.log
|
|
202
|
+
cat build/ffmpeg/lib/pkgconfig/*.pc
|
|
203
|
+
- name: Upload wheels
|
|
204
|
+
uses: actions/upload-artifact@v7
|
|
205
|
+
with:
|
|
206
|
+
name: wheels-windows-${{ matrix.platform.target }}
|
|
207
|
+
path: dist
|
|
208
|
+
- name: Run tests
|
|
209
|
+
run: |
|
|
210
|
+
for python in 3.11 3.14; do
|
|
211
|
+
uv sync --frozen --no-install-project --python $python
|
|
212
|
+
uv pip install --reinstall-package iterframes dist/*.whl
|
|
213
|
+
uv run --no-sync pytest
|
|
214
|
+
done
|
|
215
|
+
- name: Lint Rust
|
|
216
|
+
# The only job that compiles the Windows code paths.
|
|
217
|
+
run: cargo clippy --all-targets -- -D warnings
|
|
218
|
+
|
|
150
219
|
sdist:
|
|
151
220
|
runs-on: ubuntu-latest
|
|
152
221
|
steps:
|
|
@@ -166,7 +235,7 @@ jobs:
|
|
|
166
235
|
name: Attach to the GitHub release
|
|
167
236
|
runs-on: ubuntu-latest
|
|
168
237
|
if: ${{ needs.release-please.outputs.release_created == 'true' }}
|
|
169
|
-
needs: [release-please, lint, linux, macos, sdist]
|
|
238
|
+
needs: [release-please, lint, linux, macos, windows, sdist]
|
|
170
239
|
permissions:
|
|
171
240
|
# Used to sign the attestations
|
|
172
241
|
id-token: write
|
|
@@ -17,7 +17,7 @@ decodes the next ones. Keep that true, and say so in the docs.
|
|
|
17
17
|
|
|
18
18
|
The package uses maturin's mixed layout: the compiled module is installed
|
|
19
19
|
as `iterframes.iterframes`, and `iterframes/__init__.py` wraps its
|
|
20
|
-
`FrameReader` into `read` and `
|
|
20
|
+
`FrameReader` into `read` and `read_batches`.
|
|
21
21
|
|
|
22
22
|
## How decoding works
|
|
23
23
|
|
|
@@ -36,6 +36,24 @@ as `iterframes.iterframes`, and `iterframes/__init__.py` wraps its
|
|
|
36
36
|
Python code holds it. Never attach to Python there (no `Python::attach`,
|
|
37
37
|
no Python objects in `decoder.rs`); `test_benchmark.py` checks the
|
|
38
38
|
overlap.
|
|
39
|
+
- With `batch_size`, the thread allocates one `ffmpeg::Buffer` per batch
|
|
40
|
+
and swscale writes each frame into its slice (`Scaler::run_into`); the
|
|
41
|
+
batch is sent when full, or at the end of the video unless `drop_last`.
|
|
42
|
+
Python gets it as a `Batch`, a 4-dimensional buffer, with no copy. The
|
|
43
|
+
channel then holds batches: `prefetch_frames` rounds up to whole ones.
|
|
44
|
+
- `frames`, or `start`, `stop`, and `step`, pick the frames to decode by
|
|
45
|
+
number. `start=0` with `step=1` is `Selection::First`, which reads the
|
|
46
|
+
video straight through and stops early, with no index and no seek: keep
|
|
47
|
+
that path, since it is the common one. Otherwise the thread indexes the
|
|
48
|
+
file first (`index` in
|
|
49
|
+
`src/decoder.rs`), demuxing every packet without decoding it to map each
|
|
50
|
+
frame to its timestamp and mark the key frames, and `Seeker` decodes each
|
|
51
|
+
frame asked for from the key frame before it, or goes on from the frame
|
|
52
|
+
it decoded last when that is closer, or when the stream does not seek. Frames passed on the way are never
|
|
53
|
+
converted to RGB. The index leaves out the frames before the first key
|
|
54
|
+
frame, which decoding cannot return either, so frame numbers are those of
|
|
55
|
+
a plain `read`. A stream that cannot seek, such as raw H.264, is opened
|
|
56
|
+
again and read from the start (`Source::open`).
|
|
39
57
|
- Errors travel through the channel and become Python exceptions in
|
|
40
58
|
`impl From<Error> for PyErr`. A closed channel means the end of the video.
|
|
41
59
|
- Dropping the reader closes the channel; the thread notices on its next
|
|
@@ -54,28 +72,33 @@ as `iterframes.iterframes`, and `iterframes/__init__.py` wraps its
|
|
|
54
72
|
as `AVERROR` are written by hand in the `sys` module of `src/ffmpeg.rs`.
|
|
55
73
|
- The build disables autodetection, so the wheel depends on libc and
|
|
56
74
|
system frameworks only; dav1d is added for AV1.
|
|
75
|
+
- On Windows the script runs in MSYS2 but compiles with MSVC
|
|
76
|
+
(`--toolchain=msvc`, `-MD`), because the wheels target
|
|
77
|
+
`x86_64-pc-windows-msvc`; it copies each `libx.a` to `x.lib` for MSVC's
|
|
78
|
+
linker. `build.rs` finds `bash.exe` on `PATH` itself, since Rust would
|
|
79
|
+
otherwise pick WSL's from the system directory.
|
|
57
80
|
- Keep the build LGPL: never pass `--enable-gpl` or `--enable-nonfree`.
|
|
58
81
|
- Hardware decoding (`device="mps"` / `"cuda"`, PyTorch's names, mapped
|
|
59
82
|
to FFmpeg's in `hardware_devices` in `src/lib.rs`): VideoToolbox on
|
|
60
|
-
macOS; on Linux
|
|
61
|
-
decoders, which load the NVIDIA driver
|
|
62
|
-
GPU. Both add no library to the wheel. The macOS build needs clang's
|
|
83
|
+
macOS; on Linux and Windows
|
|
84
|
+
the `*_cuvid` decoders, which load the NVIDIA driver at run time and
|
|
85
|
+
resize on the GPU. Both add no library to the wheel. The macOS build needs clang's
|
|
63
86
|
compiler-rt for `@available`, which `build.rs` links.
|
|
64
87
|
- NVDEC has never run on a GPU in this project: CI has none, and the
|
|
65
88
|
`cuda` tests skip when the device does not open. The same goes for
|
|
66
89
|
`on_device=True` (`CudaFrame`, `Plane`, `src/dlpack.rs`), which waits on
|
|
67
|
-
cuvid's copy with CUDA driver calls found through dlopen
|
|
68
|
-
(`ffmpeg::cuda`).
|
|
90
|
+
cuvid's copy with CUDA driver calls found through dlopen, or
|
|
91
|
+
`LoadLibraryA` on Windows (`ffmpeg::cuda`).
|
|
69
92
|
|
|
70
93
|
## Layout
|
|
71
94
|
|
|
72
95
|
| Path | Contents |
|
|
73
96
|
| --- | --- |
|
|
74
|
-
| `src/lib.rs` | PyO3 module: `Frame`, `FrameReader`, error mapping, module init |
|
|
97
|
+
| `src/lib.rs` | PyO3 module: `Frame`, `Batch`, `FrameReader`, error mapping, module init |
|
|
75
98
|
| `src/decoder.rs` | Decoding thread |
|
|
76
99
|
| `src/ffmpeg.rs` | Safe wrappers over the FFmpeg calls the crate needs |
|
|
77
100
|
| `src/dlpack.rs` | DLPack capsules for the planes of `CudaFrame` |
|
|
78
|
-
| `iterframes/__init__.py` | `read`, `
|
|
101
|
+
| `iterframes/__init__.py` | `read`, `read_batches` |
|
|
79
102
|
| `build.rs` | Builds and links FFmpeg, generates its bindings |
|
|
80
103
|
| `scripts/build-ffmpeg.sh` | Static FFmpeg and dav1d, run by `build.rs` |
|
|
81
104
|
| `tests/` | pytest suite; frames are compared with PyAV |
|
|
@@ -110,7 +133,8 @@ uv run --no-sync pytest
|
|
|
110
133
|
```
|
|
111
134
|
|
|
112
135
|
CI runs the same four, plus `ruff check .` and `ruff format --check .` for
|
|
113
|
-
the Python files.
|
|
136
|
+
the Python files. Only the `windows` job compiles the `cfg(windows)`
|
|
137
|
+
code, and runs clippy on it. `pre-commit run -a` covers the formatters and linters
|
|
114
138
|
locally.
|
|
115
139
|
|
|
116
140
|
## Docs
|
|
@@ -6,6 +6,19 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
|
|
6
6
|
From 0.4.0 on, [release-please](https://github.com/googleapis/release-please)
|
|
7
7
|
writes each entry from the Conventional Commits since the previous release.
|
|
8
8
|
|
|
9
|
+
## [0.5.0](https://github.com/alesanfra/iterframes/compare/v0.4.0...v0.5.0) (2026-09-20)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
### ⚠ BREAKING CHANGES
|
|
13
|
+
|
|
14
|
+
* `read_all` is removed; use `list(iterframes.read(...))`.
|
|
15
|
+
|
|
16
|
+
### Features
|
|
17
|
+
|
|
18
|
+
* build wheels for Windows ([#4](https://github.com/alesanfra/iterframes/issues/4)) ([39702a5](https://github.com/alesanfra/iterframes/commit/39702a5b55333824956827c5cf23b50cf150fb22))
|
|
19
|
+
* read frames by number, in the style of decord ([#10](https://github.com/alesanfra/iterframes/issues/10)) ([6d0627e](https://github.com/alesanfra/iterframes/commit/6d0627e65d357667ba28f274c780c3a58da541cb))
|
|
20
|
+
* read frames in batches with read_batches ([#6](https://github.com/alesanfra/iterframes/issues/6)) ([136ae14](https://github.com/alesanfra/iterframes/commit/136ae14ad6cccfa3459f69518a610dd94957d4c1))
|
|
21
|
+
|
|
9
22
|
## [0.4.0](https://github.com/alesanfra/iterframes/compare/v0.3.0...v0.4.0) (2026-09-19)
|
|
10
23
|
|
|
11
24
|
The rewrite: static FFmpeg wheels, frames without copies, and hardware
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: iterframes
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Classifier: Intended Audience :: Science/Research
|
|
5
|
+
Classifier: Programming Language :: Python :: 3
|
|
6
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
7
|
+
Classifier: Programming Language :: Rust
|
|
8
|
+
Classifier: Topic :: Multimedia :: Video
|
|
9
|
+
Requires-Dist: numpy>=1.21
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Summary: Iterate over the frames of a video as NumPy arrays, decoded on a background thread while you process them
|
|
12
|
+
Keywords: video,decoder,ffmpeg,frames,numpy,video-reader,pytorch,deep-learning,computer-vision,nvdec,videotoolbox
|
|
13
|
+
Home-Page: https://github.com/alesanfra/iterframes
|
|
14
|
+
Author-email: Alessio Sanfratello <sanfra90@gmail.com>
|
|
15
|
+
License-Expression: LGPL-3.0-only
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
18
|
+
Project-URL: Changelog, https://github.com/alesanfra/iterframes/blob/main/CHANGELOG.md
|
|
19
|
+
Project-URL: Documentation, https://iterframes.readthedocs.io
|
|
20
|
+
Project-URL: Homepage, https://github.com/alesanfra/iterframes
|
|
21
|
+
Project-URL: Issues, https://github.com/alesanfra/iterframes/issues
|
|
22
|
+
Project-URL: Source, https://github.com/alesanfra/iterframes
|
|
23
|
+
|
|
24
|
+
# iterframes
|
|
25
|
+
|
|
26
|
+
[](https://pypi.org/project/iterframes/)
|
|
27
|
+
[](https://github.com/alesanfra/iterframes/actions/workflows/ci.yaml)
|
|
28
|
+
[](https://iterframes.readthedocs.io)
|
|
29
|
+
|
|
30
|
+
**Video frames as NumPy arrays, decoded while your code is busy with the
|
|
31
|
+
last one.**
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
import iterframes
|
|
35
|
+
|
|
36
|
+
for frame in iterframes.read("video.mp4", height=224, width=224):
|
|
37
|
+
model(frame) # (224, 224, 3) uint8 RGB, resized by FFmpeg, no copy
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
`model` runs on one frame while a Rust thread decodes the next ones. That
|
|
41
|
+
thread never takes the GIL, so decoding overlaps with your work instead of
|
|
42
|
+
adding to it, even when your code is pure Python. `pip install iterframes`
|
|
43
|
+
brings FFmpeg with it: nothing else to install, no system packages, no
|
|
44
|
+
`ffmpeg` binary to call.
|
|
45
|
+
|
|
46
|
+
The documentation is at
|
|
47
|
+
[iterframes.readthedocs.io](https://iterframes.readthedocs.io): the
|
|
48
|
+
[reference](https://iterframes.readthedocs.io/en/latest/reference/)
|
|
49
|
+
documents every argument and error, and the
|
|
50
|
+
[development guide](https://iterframes.readthedocs.io/en/latest/development/)
|
|
51
|
+
covers building from source.
|
|
52
|
+
|
|
53
|
+
## Numbers
|
|
54
|
+
|
|
55
|
+
One video, 901 frames of 480x270 H.264, decoded and converted to RGB,
|
|
56
|
+
best of seven runs on an Apple silicon Mac with the bundled FFmpeg 9.0.2.
|
|
57
|
+
The benchmarks are in
|
|
58
|
+
[`tests/test_benchmark.py`](https://github.com/alesanfra/iterframes/blob/main/tests/test_benchmark.py);
|
|
59
|
+
run them on your own videos before believing them.
|
|
60
|
+
|
|
61
|
+
| | Time |
|
|
62
|
+
| --- | --- |
|
|
63
|
+
| Every frame, iterframes | **0.046 s** |
|
|
64
|
+
| Every frame, PyAV, same decode and conversion | 0.208 s |
|
|
65
|
+
| 32 frames at random positions | 0.036 s |
|
|
66
|
+
| First 10 frames (`stop=10`) | 0.003 s |
|
|
67
|
+
| Second half (`start=450`) | 0.028 s |
|
|
68
|
+
|
|
69
|
+
In the overlap benchmark, which upscales the same video to 1080p, adding
|
|
70
|
+
per-frame work as expensive as decoding took 16% longer in total, not
|
|
71
|
+
twice as long: the decoding had already happened.
|
|
72
|
+
|
|
73
|
+
## What you get
|
|
74
|
+
|
|
75
|
+
- **Frames without copies.** RGB `uint8` arrays of shape
|
|
76
|
+
`(height, width, 3)`, straight out of FFmpeg's buffers.
|
|
77
|
+
- **Batches as one array.** `read_batches` decodes into a single
|
|
78
|
+
`(batch, height, width, 3)` block, ready for a model.
|
|
79
|
+
- **Resizing while decoding**, not a `cv2.resize` afterwards.
|
|
80
|
+
- **Random access.** Read frames by number, without decoding the rest.
|
|
81
|
+
- **Hardware decoding** on NVIDIA GPUs (NVDEC) and Apple silicon
|
|
82
|
+
(VideoToolbox), in the wheels. On NVIDIA the frames can stay on the GPU
|
|
83
|
+
for PyTorch.
|
|
84
|
+
- **Wheels** for Linux (x86_64, aarch64), macOS (Apple silicon), and
|
|
85
|
+
Windows, for every CPython from 3.11 on.
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
# Batches of 12 frames, decoded into one (12, 224, 224, 3) array
|
|
89
|
+
for batch in iterframes.read_batches("video.mp4", 12, height=224, width=224):
|
|
90
|
+
...
|
|
91
|
+
|
|
92
|
+
# Stop whenever you like; the decoder stops with the loop
|
|
93
|
+
for index, frame in enumerate(iterframes.read("video.mp4")):
|
|
94
|
+
if index == 100:
|
|
95
|
+
break
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## Random access
|
|
99
|
+
|
|
100
|
+
Sample clips for training, or grab a thumbnail, without reading the whole
|
|
101
|
+
file:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
clip = list(iterframes.read("video.mp4", frames=[0, 30, 60]))
|
|
105
|
+
every_fifth = iterframes.read("video.mp4", start=100, stop=200, step=5)
|
|
106
|
+
last = next(iterframes.read("video.mp4", frames=[-1]))
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Both work with `read_batches`. Negative numbers count from the end, and
|
|
110
|
+
frame `n` is the one `read` yields `n`-th. iterframes indexes the file
|
|
111
|
+
once, without decoding it, then decodes each frame from the key frame
|
|
112
|
+
before it; frames asked for in order cost no more than reading the video
|
|
113
|
+
straight through. See
|
|
114
|
+
[Reading frames by number](https://iterframes.readthedocs.io/en/latest/reference/#reading-frames-by-number).
|
|
115
|
+
|
|
116
|
+
## Hardware decoding
|
|
117
|
+
|
|
118
|
+
Pass `device` to decode on a GPU instead of the CPU, which then stays free
|
|
119
|
+
for your model. The names are PyTorch's:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
# NVIDIA GPU on Linux and Windows; with height and width, the GPU resizes too
|
|
123
|
+
for frame in iterframes.read("video.mp4", height=224, width=224, device="cuda"):
|
|
124
|
+
...
|
|
125
|
+
|
|
126
|
+
# Apple silicon (VideoToolbox)
|
|
127
|
+
for frame in iterframes.read("video.mp4", device="mps"):
|
|
128
|
+
...
|
|
129
|
+
|
|
130
|
+
# Whatever the machine has, else the CPU
|
|
131
|
+
for frame in iterframes.read("video.mp4", device="auto"):
|
|
132
|
+
...
|
|
133
|
+
|
|
134
|
+
print(iterframes.DEVICES) # ('cpu', 'mps') on a Mac
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
The frames still arrive as NumPy arrays in memory. A GPU saves CPU time
|
|
138
|
+
but is not always faster than the CPU decoder, so measure both; see
|
|
139
|
+
[Hardware decoding](https://iterframes.readthedocs.io/en/latest/reference/#hardware-decoding).
|
|
140
|
+
|
|
141
|
+
With an NVIDIA GPU, `on_device=True` keeps the frames on it, in NV12, for
|
|
142
|
+
PyTorch and other libraries to take without a copy:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
import torch
|
|
146
|
+
|
|
147
|
+
for frame in iterframes.read("video.mp4", device="cuda", on_device=True):
|
|
148
|
+
y = torch.from_dlpack(frame.y) # (height, width) uint8, on the GPU
|
|
149
|
+
uv = torch.from_dlpack(frame.uv) # (height / 2, width / 2, 2)
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
[Frames on the GPU](https://iterframes.readthedocs.io/en/latest/reference/#frames-on-the-gpu) shows how to
|
|
153
|
+
convert them to RGB there.
|
|
154
|
+
|
|
155
|
+
## Compared with OpenCV, decord, and PyAV
|
|
156
|
+
|
|
157
|
+
| | iterframes | OpenCV `VideoCapture` | decord | PyAV |
|
|
158
|
+
| --- | --- | --- | --- | --- |
|
|
159
|
+
| Decoding runs | Ahead of your code, on a background thread | When you call `read()` | When you index the reader | When you ask for the next frame |
|
|
160
|
+
| Pixels | RGB | BGR | RGB | Any format FFmpeg supports |
|
|
161
|
+
| Resize while decoding | Yes | No, with `cv2.resize` after | Yes | Yes, with `reformat` |
|
|
162
|
+
| Batches as one array | Yes, `read_batches` | No | Yes, `get_batch` | No |
|
|
163
|
+
| Seeking and random access | Yes, `frames=[...]` or `start`, `stop`, `step` | Yes | Yes, fast | Yes |
|
|
164
|
+
| Audio, encoding, muxing | No | Encoding with `VideoWriter` | Audio reading | Yes |
|
|
165
|
+
| Hardware decoding | NVIDIA, Apple silicon, in the wheels | Depends on the build and backend | NVIDIA, when built from source | Depends on the build |
|
|
166
|
+
| Latest wheels | Linux, macOS, Windows, CPython 3.11+ | Linux, macOS, Windows | x86_64 only, last release in 2021 | Linux, macOS, Windows |
|
|
167
|
+
|
|
168
|
+
Pick iterframes to read videos into a model, in order or by frame number,
|
|
169
|
+
and keep decoding out of your loop's way. Pick PyAV when you need the rest
|
|
170
|
+
of FFmpeg: audio, encoding, streams, or precise control over the decoder.
|
|
171
|
+
OpenCV is the natural choice when the rest of the pipeline already uses
|
|
172
|
+
it.
|
|
173
|
+
|
|
174
|
+
## Installation
|
|
175
|
+
|
|
176
|
+
```console
|
|
177
|
+
pip install iterframes
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
The wheels bundle FFmpeg and work on any CPython from 3.11 on, on Linux
|
|
181
|
+
(x86_64, aarch64), macOS (Apple silicon), and Windows (x86_64). Other
|
|
182
|
+
platforms build from source, FFmpeg included.
|
|
183
|
+
|
|
184
|
+
## Contributing
|
|
185
|
+
|
|
186
|
+
Contributions are welcome. The
|
|
187
|
+
[development guide](https://iterframes.readthedocs.io/en/latest/development/#help-wanted)
|
|
188
|
+
lists features that are waiting for someone to build them.
|
|
189
|
+
|
|
190
|
+
## License
|
|
191
|
+
|
|
192
|
+
iterframes is released under the [LGPL-3.0](https://github.com/alesanfra/iterframes/blob/main/LICENSE). The wheels include
|
|
193
|
+
[FFmpeg](https://ffmpeg.org/), built under the LGPL, and
|
|
194
|
+
[dav1d](https://code.videolan.org/videolan/dav1d), under the BSD 2-clause
|
|
195
|
+
license.
|
|
196
|
+
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# iterframes
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/iterframes/)
|
|
4
|
+
[](https://github.com/alesanfra/iterframes/actions/workflows/ci.yaml)
|
|
5
|
+
[](https://iterframes.readthedocs.io)
|
|
6
|
+
|
|
7
|
+
**Video frames as NumPy arrays, decoded while your code is busy with the
|
|
8
|
+
last one.**
|
|
9
|
+
|
|
10
|
+
```python
|
|
11
|
+
import iterframes
|
|
12
|
+
|
|
13
|
+
for frame in iterframes.read("video.mp4", height=224, width=224):
|
|
14
|
+
model(frame) # (224, 224, 3) uint8 RGB, resized by FFmpeg, no copy
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
`model` runs on one frame while a Rust thread decodes the next ones. That
|
|
18
|
+
thread never takes the GIL, so decoding overlaps with your work instead of
|
|
19
|
+
adding to it, even when your code is pure Python. `pip install iterframes`
|
|
20
|
+
brings FFmpeg with it: nothing else to install, no system packages, no
|
|
21
|
+
`ffmpeg` binary to call.
|
|
22
|
+
|
|
23
|
+
The documentation is at
|
|
24
|
+
[iterframes.readthedocs.io](https://iterframes.readthedocs.io): the
|
|
25
|
+
[reference](https://iterframes.readthedocs.io/en/latest/reference/)
|
|
26
|
+
documents every argument and error, and the
|
|
27
|
+
[development guide](https://iterframes.readthedocs.io/en/latest/development/)
|
|
28
|
+
covers building from source.
|
|
29
|
+
|
|
30
|
+
## Numbers
|
|
31
|
+
|
|
32
|
+
One video, 901 frames of 480x270 H.264, decoded and converted to RGB,
|
|
33
|
+
best of seven runs on an Apple silicon Mac with the bundled FFmpeg 9.0.2.
|
|
34
|
+
The benchmarks are in
|
|
35
|
+
[`tests/test_benchmark.py`](https://github.com/alesanfra/iterframes/blob/main/tests/test_benchmark.py);
|
|
36
|
+
run them on your own videos before believing them.
|
|
37
|
+
|
|
38
|
+
| | Time |
|
|
39
|
+
| --- | --- |
|
|
40
|
+
| Every frame, iterframes | **0.046 s** |
|
|
41
|
+
| Every frame, PyAV, same decode and conversion | 0.208 s |
|
|
42
|
+
| 32 frames at random positions | 0.036 s |
|
|
43
|
+
| First 10 frames (`stop=10`) | 0.003 s |
|
|
44
|
+
| Second half (`start=450`) | 0.028 s |
|
|
45
|
+
|
|
46
|
+
In the overlap benchmark, which upscales the same video to 1080p, adding
|
|
47
|
+
per-frame work as expensive as decoding took 16% longer in total, not
|
|
48
|
+
twice as long: the decoding had already happened.
|
|
49
|
+
|
|
50
|
+
## What you get
|
|
51
|
+
|
|
52
|
+
- **Frames without copies.** RGB `uint8` arrays of shape
|
|
53
|
+
`(height, width, 3)`, straight out of FFmpeg's buffers.
|
|
54
|
+
- **Batches as one array.** `read_batches` decodes into a single
|
|
55
|
+
`(batch, height, width, 3)` block, ready for a model.
|
|
56
|
+
- **Resizing while decoding**, not a `cv2.resize` afterwards.
|
|
57
|
+
- **Random access.** Read frames by number, without decoding the rest.
|
|
58
|
+
- **Hardware decoding** on NVIDIA GPUs (NVDEC) and Apple silicon
|
|
59
|
+
(VideoToolbox), in the wheels. On NVIDIA the frames can stay on the GPU
|
|
60
|
+
for PyTorch.
|
|
61
|
+
- **Wheels** for Linux (x86_64, aarch64), macOS (Apple silicon), and
|
|
62
|
+
Windows, for every CPython from 3.11 on.
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
# Batches of 12 frames, decoded into one (12, 224, 224, 3) array
|
|
66
|
+
for batch in iterframes.read_batches("video.mp4", 12, height=224, width=224):
|
|
67
|
+
...
|
|
68
|
+
|
|
69
|
+
# Stop whenever you like; the decoder stops with the loop
|
|
70
|
+
for index, frame in enumerate(iterframes.read("video.mp4")):
|
|
71
|
+
if index == 100:
|
|
72
|
+
break
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Random access
|
|
76
|
+
|
|
77
|
+
Sample clips for training, or grab a thumbnail, without reading the whole
|
|
78
|
+
file:
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
clip = list(iterframes.read("video.mp4", frames=[0, 30, 60]))
|
|
82
|
+
every_fifth = iterframes.read("video.mp4", start=100, stop=200, step=5)
|
|
83
|
+
last = next(iterframes.read("video.mp4", frames=[-1]))
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Both work with `read_batches`. Negative numbers count from the end, and
|
|
87
|
+
frame `n` is the one `read` yields `n`-th. iterframes indexes the file
|
|
88
|
+
once, without decoding it, then decodes each frame from the key frame
|
|
89
|
+
before it; frames asked for in order cost no more than reading the video
|
|
90
|
+
straight through. See
|
|
91
|
+
[Reading frames by number](https://iterframes.readthedocs.io/en/latest/reference/#reading-frames-by-number).
|
|
92
|
+
|
|
93
|
+
## Hardware decoding
|
|
94
|
+
|
|
95
|
+
Pass `device` to decode on a GPU instead of the CPU, which then stays free
|
|
96
|
+
for your model. The names are PyTorch's:
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
# NVIDIA GPU on Linux and Windows; with height and width, the GPU resizes too
|
|
100
|
+
for frame in iterframes.read("video.mp4", height=224, width=224, device="cuda"):
|
|
101
|
+
...
|
|
102
|
+
|
|
103
|
+
# Apple silicon (VideoToolbox)
|
|
104
|
+
for frame in iterframes.read("video.mp4", device="mps"):
|
|
105
|
+
...
|
|
106
|
+
|
|
107
|
+
# Whatever the machine has, else the CPU
|
|
108
|
+
for frame in iterframes.read("video.mp4", device="auto"):
|
|
109
|
+
...
|
|
110
|
+
|
|
111
|
+
print(iterframes.DEVICES) # ('cpu', 'mps') on a Mac
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
The frames still arrive as NumPy arrays in memory. A GPU saves CPU time
|
|
115
|
+
but is not always faster than the CPU decoder, so measure both; see
|
|
116
|
+
[Hardware decoding](https://iterframes.readthedocs.io/en/latest/reference/#hardware-decoding).
|
|
117
|
+
|
|
118
|
+
With an NVIDIA GPU, `on_device=True` keeps the frames on it, in NV12, for
|
|
119
|
+
PyTorch and other libraries to take without a copy:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
import torch
|
|
123
|
+
|
|
124
|
+
for frame in iterframes.read("video.mp4", device="cuda", on_device=True):
|
|
125
|
+
y = torch.from_dlpack(frame.y) # (height, width) uint8, on the GPU
|
|
126
|
+
uv = torch.from_dlpack(frame.uv) # (height / 2, width / 2, 2)
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
[Frames on the GPU](https://iterframes.readthedocs.io/en/latest/reference/#frames-on-the-gpu) shows how to
|
|
130
|
+
convert them to RGB there.
|
|
131
|
+
|
|
132
|
+
## Compared with OpenCV, decord, and PyAV
|
|
133
|
+
|
|
134
|
+
| | iterframes | OpenCV `VideoCapture` | decord | PyAV |
|
|
135
|
+
| --- | --- | --- | --- | --- |
|
|
136
|
+
| Decoding runs | Ahead of your code, on a background thread | When you call `read()` | When you index the reader | When you ask for the next frame |
|
|
137
|
+
| Pixels | RGB | BGR | RGB | Any format FFmpeg supports |
|
|
138
|
+
| Resize while decoding | Yes | No, with `cv2.resize` after | Yes | Yes, with `reformat` |
|
|
139
|
+
| Batches as one array | Yes, `read_batches` | No | Yes, `get_batch` | No |
|
|
140
|
+
| Seeking and random access | Yes, `frames=[...]` or `start`, `stop`, `step` | Yes | Yes, fast | Yes |
|
|
141
|
+
| Audio, encoding, muxing | No | Encoding with `VideoWriter` | Audio reading | Yes |
|
|
142
|
+
| Hardware decoding | NVIDIA, Apple silicon, in the wheels | Depends on the build and backend | NVIDIA, when built from source | Depends on the build |
|
|
143
|
+
| Latest wheels | Linux, macOS, Windows, CPython 3.11+ | Linux, macOS, Windows | x86_64 only, last release in 2021 | Linux, macOS, Windows |
|
|
144
|
+
|
|
145
|
+
Pick iterframes to read videos into a model, in order or by frame number,
|
|
146
|
+
and keep decoding out of your loop's way. Pick PyAV when you need the rest
|
|
147
|
+
of FFmpeg: audio, encoding, streams, or precise control over the decoder.
|
|
148
|
+
OpenCV is the natural choice when the rest of the pipeline already uses
|
|
149
|
+
it.
|
|
150
|
+
|
|
151
|
+
## Installation
|
|
152
|
+
|
|
153
|
+
```console
|
|
154
|
+
pip install iterframes
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
The wheels bundle FFmpeg and work on any CPython from 3.11 on, on Linux
|
|
158
|
+
(x86_64, aarch64), macOS (Apple silicon), and Windows (x86_64). Other
|
|
159
|
+
platforms build from source, FFmpeg included.
|
|
160
|
+
|
|
161
|
+
## Contributing
|
|
162
|
+
|
|
163
|
+
Contributions are welcome. The
|
|
164
|
+
[development guide](https://iterframes.readthedocs.io/en/latest/development/#help-wanted)
|
|
165
|
+
lists features that are waiting for someone to build them.
|
|
166
|
+
|
|
167
|
+
## License
|
|
168
|
+
|
|
169
|
+
iterframes is released under the [LGPL-3.0](https://github.com/alesanfra/iterframes/blob/main/LICENSE). The wheels include
|
|
170
|
+
[FFmpeg](https://ffmpeg.org/), built under the LGPL, and
|
|
171
|
+
[dav1d](https://code.videolan.org/videolan/dav1d), under the BSD 2-clause
|
|
172
|
+
license.
|
|
@@ -52,7 +52,7 @@ fn build_ffmpeg(script: &Path, prefix: &Path) {
|
|
|
52
52
|
lock.lock().expect("cannot lock the FFmpeg build");
|
|
53
53
|
|
|
54
54
|
let log = prefix.with_extension("log");
|
|
55
|
-
let status = Command::new(
|
|
55
|
+
let status = Command::new(bash())
|
|
56
56
|
.arg(script)
|
|
57
57
|
.arg(prefix)
|
|
58
58
|
.stdout(File::create(&log).expect("cannot create the build log"))
|
|
@@ -88,6 +88,22 @@ fn build_ffmpeg(script: &Path, prefix: &Path) {
|
|
|
88
88
|
}
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
+
/// The shell that runs the build script. On Windows, Rust looks for a bare
|
|
92
|
+
/// `bash` in the system directory before `PATH`, and would find WSL's
|
|
93
|
+
/// there instead of MSYS2's.
|
|
94
|
+
fn bash() -> PathBuf {
|
|
95
|
+
if cfg!(windows) {
|
|
96
|
+
let path = env::var_os("PATH").unwrap_or_default();
|
|
97
|
+
if let Some(bash) = env::split_paths(&path)
|
|
98
|
+
.map(|dir| dir.join("bash.exe"))
|
|
99
|
+
.find(|bash| bash.is_file())
|
|
100
|
+
{
|
|
101
|
+
return bash;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
PathBuf::from("bash")
|
|
105
|
+
}
|
|
106
|
+
|
|
91
107
|
/// Link the static libraries and return their include paths.
|
|
92
108
|
fn link_ffmpeg(prefix: &Path) -> Vec<PathBuf> {
|
|
93
109
|
// SAFETY: the build script is single-threaded.
|