parallel-matplotlib-animation 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parallel_animate/__init__.py +2 -0
- parallel_animate/animator.py +256 -110
- {parallel_matplotlib_animation-0.1.2.dist-info → parallel_matplotlib_animation-0.1.3.dist-info}/METADATA +28 -17
- parallel_matplotlib_animation-0.1.3.dist-info/RECORD +7 -0
- {parallel_matplotlib_animation-0.1.2.dist-info → parallel_matplotlib_animation-0.1.3.dist-info}/WHEEL +1 -1
- parallel_animate/examples/__init__.py +0 -0
- parallel_animate/examples/multi_panel_animation.py +0 -144
- parallel_animate/examples/nondeterministic_video_loader.py +0 -46
- parallel_animate/examples/scaling_test.py +0 -96
- parallel_animate/examples/simple_wave_animation.py +0 -37
- parallel_animate/examples/very_complex_animation.py +0 -634
- parallel_matplotlib_animation-0.1.2.dist-info/RECORD +0 -13
- {parallel_matplotlib_animation-0.1.2.dist-info → parallel_matplotlib_animation-0.1.3.dist-info}/licenses/LICENSE +0 -0
parallel_animate/__init__.py
CHANGED
parallel_animate/animator.py
CHANGED
|
@@ -2,13 +2,15 @@ import matplotlib
|
|
|
2
2
|
|
|
3
3
|
matplotlib.use("Agg")
|
|
4
4
|
|
|
5
|
+
import sys
|
|
5
6
|
import tempfile
|
|
6
7
|
import itertools
|
|
7
8
|
import logging
|
|
9
|
+
import queue
|
|
8
10
|
import multiprocessing as mp
|
|
11
|
+
import numpy as np
|
|
9
12
|
import matplotlib.pyplot as plt
|
|
10
|
-
import
|
|
11
|
-
from PIL import Image
|
|
13
|
+
import pvio
|
|
12
14
|
from tqdm import tqdm
|
|
13
15
|
from pathlib import Path
|
|
14
16
|
from abc import ABC, abstractmethod
|
|
@@ -36,6 +38,25 @@ class IndexedFrameParams:
|
|
|
36
38
|
class Animator(ABC):
|
|
37
39
|
"""
|
|
38
40
|
Base class for creating matplotlib animations with efficient parallel rendering.
|
|
41
|
+
|
|
42
|
+
Performance note: every frame is fully rasterised — ``update()`` mutates the
|
|
43
|
+
figure and the whole canvas is then re-drawn and captured. Matplotlib's
|
|
44
|
+
blitting optimisation (re-drawing only the artists that changed via
|
|
45
|
+
``draw_artist``/``restore_region``) is intentionally not used: frames are
|
|
46
|
+
rendered independently and, in parallel mode, in separate worker processes,
|
|
47
|
+
so there is no persistent inter-frame canvas state to blit against. Speedups
|
|
48
|
+
come from rendering frames concurrently across workers rather than from
|
|
49
|
+
incremental redraws. To keep each frame cheap, do expensive, frame-invariant
|
|
50
|
+
setup once in ``setup()`` and only touch what changes in ``update()``.
|
|
51
|
+
|
|
52
|
+
Pickling note: in parallel mode the whole Animator instance is pickled and
|
|
53
|
+
copied into every worker process. Anything stored on ``self`` (e.g. in
|
|
54
|
+
``__init__``) is therefore serialised once per worker — stashing a large
|
|
55
|
+
array such as an entire video tensor on an instance attribute multiplies its
|
|
56
|
+
memory across workers and adds pickling overhead. Keep heavy, per-frame data
|
|
57
|
+
out of instance attributes: pass it through ``param_by_frame`` (the intended
|
|
58
|
+
channel), or load/construct it lazily inside ``setup()`` so each worker
|
|
59
|
+
builds its own copy instead of receiving one over the pickle boundary.
|
|
39
60
|
"""
|
|
40
61
|
|
|
41
62
|
@abstractmethod
|
|
@@ -81,9 +102,10 @@ class Animator(ABC):
|
|
|
81
102
|
plotting_log_interval: int | None = None,
|
|
82
103
|
saving_log_interval: int | None = None,
|
|
83
104
|
savefig_params: dict[str, Any] | None = None,
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
105
|
+
video_mode: str = "auto",
|
|
106
|
+
video_quality: int | None = None,
|
|
107
|
+
video_preset: str | None = None,
|
|
108
|
+
video_extra_ffmpeg_params: list[str] | None = None,
|
|
87
109
|
reuse_figure_object: bool = True,
|
|
88
110
|
preload_factor: int = 8,
|
|
89
111
|
) -> None:
|
|
@@ -106,20 +128,38 @@ class Animator(ABC):
|
|
|
106
128
|
worker (None = no interval logging).
|
|
107
129
|
saving_log_interval (int | None): Log progress every N frames when merging
|
|
108
130
|
frames into video (None = no interval logging).
|
|
109
|
-
savefig_params (dict[str, Any]):
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
131
|
+
savefig_params (dict[str, Any]): Rendering options for each frame
|
|
132
|
+
(default: {}). For speed, frames are grabbed directly from the
|
|
133
|
+
Agg canvas buffer as raw RGB pixels (no PNG encode/decode or
|
|
134
|
+
image-file round-trip), so only ``dpi`` is honoured here — it
|
|
135
|
+
sets the raster resolution. Other ``savefig``-specific options
|
|
136
|
+
such as ``bbox_inches`` do not apply to a direct buffer grab.
|
|
137
|
+
video_mode (str): Encoding backend selection passed to parallel-video-io,
|
|
138
|
+
one of "auto", "gpu", or "cpu" (default: "auto"). "auto" uses the GPU
|
|
139
|
+
encoder (FFmpeg/NVENC) when a CUDA device is available and transparently
|
|
140
|
+
falls back to CPU (libx264) otherwise. "gpu" forces NVENC and "cpu"
|
|
141
|
+
forces libx264. Output is always an H.264 MP4.
|
|
142
|
+
video_quality (int | None): H.264 quantiser scale in the range 0-51, where
|
|
143
|
+
lower means higher quality / larger files. If None (default), use
|
|
144
|
+
parallel-video-io's visually-lossless default.
|
|
145
|
+
video_preset (str | None): Encoder preset. If None (default), use the
|
|
146
|
+
encoder's own default. Accepts libx264 presets ("ultrafast" … "placebo")
|
|
147
|
+
in CPU mode or NVENC presets ("p1" … "p7") in GPU mode.
|
|
148
|
+
video_extra_ffmpeg_params (list[str] | None): Extra raw FFmpeg arguments
|
|
149
|
+
appended to the encode command for advanced tuning (default: None).
|
|
115
150
|
reuse_figure_object (bool): If False, the figure will be re-created for each
|
|
116
151
|
frame (i.e. setup() called every frame). There is basically no reason to
|
|
117
152
|
set this to False. Use only for testing and benchmarking.
|
|
118
153
|
preload_factor (int): Number of workloads to prefetch in the task queue per
|
|
119
154
|
worker (approximately). I.e. each worker will have up to this many
|
|
120
|
-
frames (on average) queued ahead of time to work on.
|
|
155
|
+
frames (on average) queued ahead of time to work on. Higher values
|
|
156
|
+
smooth throughput but hold more frames' params in the queue at once;
|
|
157
|
+
with large per-frame params (e.g. images) this raises peak memory by
|
|
158
|
+
roughly num_workers * preload_factor frames' worth.
|
|
121
159
|
"""
|
|
122
|
-
#
|
|
160
|
+
# Use the length of param_by_frame as the frame count when not given. Some
|
|
161
|
+
# iterables (e.g. generators) have no length; that's fine, but the progress
|
|
162
|
+
# bar then can't show a completion percentage.
|
|
123
163
|
if n_frames is None:
|
|
124
164
|
try:
|
|
125
165
|
n_frames = len(param_by_frame)
|
|
@@ -129,6 +169,12 @@ class Animator(ABC):
|
|
|
129
169
|
"Progress bar won't show completion percentage."
|
|
130
170
|
)
|
|
131
171
|
|
|
172
|
+
# Resolve auto (None) progress-bar disabling to a concrete bool up front so
|
|
173
|
+
# tqdm and pvio agree: when output isn't a TTY, both stay quiet. tqdm writes
|
|
174
|
+
# to stderr by default, so mirror its auto-detection on stderr.
|
|
175
|
+
if disable_progress_bar is None:
|
|
176
|
+
disable_progress_bar = not sys.stderr.isatty()
|
|
177
|
+
|
|
132
178
|
# Determine number of workers
|
|
133
179
|
if num_workers == 0:
|
|
134
180
|
raise ValueError(
|
|
@@ -146,8 +192,6 @@ class Animator(ABC):
|
|
|
146
192
|
# Avoid mutable default arguments by creating defaults here
|
|
147
193
|
if savefig_params is None:
|
|
148
194
|
savefig_params = {}
|
|
149
|
-
if video_params is None:
|
|
150
|
-
video_params = {"crf": "23", "preset": "slow"}
|
|
151
195
|
|
|
152
196
|
if num_workers == 1:
|
|
153
197
|
_logger.info("Running in serial mode")
|
|
@@ -174,14 +218,15 @@ class Animator(ABC):
|
|
|
174
218
|
preload_factor,
|
|
175
219
|
)
|
|
176
220
|
|
|
177
|
-
_logger.info("Creating video with
|
|
221
|
+
_logger.info("Creating video with parallel-video-io")
|
|
178
222
|
_merge_frames_into_video(
|
|
179
223
|
frames_dir,
|
|
180
224
|
output_file,
|
|
181
225
|
fps,
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
226
|
+
video_mode,
|
|
227
|
+
video_quality,
|
|
228
|
+
video_preset,
|
|
229
|
+
video_extra_ffmpeg_params,
|
|
185
230
|
disable_progress_bar,
|
|
186
231
|
_logger,
|
|
187
232
|
log_interval=saving_log_interval,
|
|
@@ -215,28 +260,18 @@ class Animator(ABC):
|
|
|
215
260
|
if reuse_figure_object:
|
|
216
261
|
fig = self._setup_and_check()
|
|
217
262
|
|
|
218
|
-
seen_frame_ids: set[int] = set()
|
|
219
263
|
frames_processed = 0
|
|
220
264
|
for frame_idx, params in tqdm(
|
|
221
|
-
|
|
265
|
+
_resolved_frames(param_by_frame, n_frames),
|
|
222
266
|
total=n_frames,
|
|
223
267
|
disable=disable_progress_bar,
|
|
268
|
+
desc="Rendering frames",
|
|
224
269
|
):
|
|
225
|
-
if isinstance(params, IndexedFrameParams):
|
|
226
|
-
if params.frame_id in seen_frame_ids:
|
|
227
|
-
raise ValueError(
|
|
228
|
-
f"Duplicate frame_id {params.frame_id} in param_by_frame"
|
|
229
|
-
)
|
|
230
|
-
seen_frame_ids.add(params.frame_id)
|
|
231
|
-
frame_idx = params.frame_id
|
|
232
|
-
params = params.params
|
|
233
|
-
|
|
234
270
|
if not reuse_figure_object:
|
|
235
271
|
fig = self._setup_and_check()
|
|
236
272
|
|
|
237
273
|
self.update(frame_idx, params)
|
|
238
|
-
|
|
239
|
-
fig.savefig(frame_path, **savefig_params)
|
|
274
|
+
_save_frame(fig, frames_dir, frame_idx, savefig_params)
|
|
240
275
|
if not reuse_figure_object:
|
|
241
276
|
plt.close(fig)
|
|
242
277
|
|
|
@@ -288,34 +323,35 @@ class Animator(ABC):
|
|
|
288
323
|
# workers by ~preload_factor frames. Note: tqdm here tracks enqueue speed,
|
|
289
324
|
# which approximates render progress but hits 100% while workers finish the
|
|
290
325
|
# last queued frames.
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
326
|
+
#
|
|
327
|
+
# Because the queue is bounded, a worker that dies (e.g. update() raised)
|
|
328
|
+
# stops draining it; without the liveness check in _put_or_abort the queue
|
|
329
|
+
# would fill and this loop would block forever instead of surfacing the
|
|
330
|
+
# error. _put_or_abort polls worker health while it waits for space.
|
|
331
|
+
# Any failure once workers are running (a validation error here, or a
|
|
332
|
+
# worker crash surfaced by _put_or_abort) must tear the workers down,
|
|
333
|
+
# otherwise they block forever on the queue and leak as orphans.
|
|
334
|
+
try:
|
|
335
|
+
for frame_idx, params in tqdm(
|
|
336
|
+
_resolved_frames(param_by_frame, n_frames),
|
|
337
|
+
total=n_frames,
|
|
338
|
+
disable=disable_progress_bar,
|
|
339
|
+
desc="Rendering frames",
|
|
340
|
+
):
|
|
341
|
+
_put_or_abort(task_queue, (frame_idx, params), workers)
|
|
342
|
+
|
|
343
|
+
# Send sentinel values to signal workers to exit
|
|
344
|
+
for _ in range(num_workers):
|
|
345
|
+
_put_or_abort(task_queue, None, workers)
|
|
346
|
+
|
|
347
|
+
# Wait for all workers to finish and check for errors
|
|
348
|
+
for p in workers:
|
|
349
|
+
p.join()
|
|
350
|
+
except BaseException:
|
|
351
|
+
_abort_workers(workers)
|
|
352
|
+
raise
|
|
353
|
+
|
|
354
|
+
failed = _failed_workers(workers)
|
|
319
355
|
if failed:
|
|
320
356
|
details = ", ".join(
|
|
321
357
|
f"pid {pid} exited with code {code}" for pid, code in failed
|
|
@@ -325,6 +361,129 @@ class Animator(ABC):
|
|
|
325
361
|
_logger.info("All workers completed")
|
|
326
362
|
|
|
327
363
|
|
|
364
|
+
def _resolved_frames(param_by_frame: Iterable[Any], n_frames: int | None):
|
|
365
|
+
"""Yield ``(frame_idx, params)`` for each frame, shared by serial and parallel.
|
|
366
|
+
|
|
367
|
+
Resolves :class:`IndexedFrameParams` (its ``frame_id`` overrides the positional
|
|
368
|
+
index) and rejects duplicate frame indices: each frame is written to one file
|
|
369
|
+
keyed by its index, so a repeat — whether two identical ``frame_id``s or a
|
|
370
|
+
``frame_id`` colliding with a positional (enumerate) index — would silently
|
|
371
|
+
overwrite an earlier frame.
|
|
372
|
+
"""
|
|
373
|
+
seen_frame_ids: set[int] = set()
|
|
374
|
+
for frame_idx, params in enumerate(itertools.islice(param_by_frame, n_frames)):
|
|
375
|
+
if isinstance(params, IndexedFrameParams):
|
|
376
|
+
frame_idx = params.frame_id
|
|
377
|
+
params = params.params
|
|
378
|
+
|
|
379
|
+
if frame_idx in seen_frame_ids:
|
|
380
|
+
raise ValueError(f"Duplicate frame index {frame_idx} in param_by_frame")
|
|
381
|
+
seen_frame_ids.add(frame_idx)
|
|
382
|
+
|
|
383
|
+
yield frame_idx, params
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _figure_to_rgb_array(fig: plt.Figure, savefig_params: dict[str, Any]) -> np.ndarray:
|
|
387
|
+
"""Rasterise ``fig`` to a contiguous ``(H, W, 3)`` uint8 RGB array.
|
|
388
|
+
|
|
389
|
+
Pixels are read straight from the Agg canvas buffer, skipping PNG
|
|
390
|
+
encode/decode and any image-file round-trip. Only the ``dpi`` entry of
|
|
391
|
+
``savefig_params`` affects the result (it sets the raster resolution); other
|
|
392
|
+
``savefig``-specific options do not apply to a direct buffer grab.
|
|
393
|
+
"""
|
|
394
|
+
dpi = savefig_params.get("dpi")
|
|
395
|
+
if dpi is not None and dpi != "figure" and fig.get_dpi() != dpi:
|
|
396
|
+
fig.set_dpi(dpi)
|
|
397
|
+
fig.canvas.draw()
|
|
398
|
+
rgba = np.asarray(fig.canvas.buffer_rgba())
|
|
399
|
+
# Drop the alpha channel and return a contiguous array the encoder can consume.
|
|
400
|
+
return np.ascontiguousarray(rgba[:, :, :3])
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _save_frame(
|
|
404
|
+
fig: plt.Figure,
|
|
405
|
+
frames_dir: Path | str,
|
|
406
|
+
frame_idx: int,
|
|
407
|
+
savefig_params: dict[str, Any],
|
|
408
|
+
) -> None:
|
|
409
|
+
"""Render ``fig`` and persist it as a raw ``.npy`` frame in ``frames_dir``."""
|
|
410
|
+
frame = _figure_to_rgb_array(fig, savefig_params)
|
|
411
|
+
frame_path = Path(frames_dir) / f"frame_{frame_idx:09d}.npy"
|
|
412
|
+
np.save(frame_path, frame)
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
class _NpyFrameSequence:
|
|
416
|
+
"""Re-iterable, lazily-loading view over frames saved as ``.npy`` files.
|
|
417
|
+
|
|
418
|
+
pvio's ``write_frames_to_video`` accepts any re-iterable yielding uint8
|
|
419
|
+
``(H, W, C)`` arrays and may iterate it more than once (GPU→CPU fallback),
|
|
420
|
+
so frames are loaded from disk on demand rather than held in memory. It also
|
|
421
|
+
indexes ``frames[0]`` to determine the output size, hence ``__getitem__``.
|
|
422
|
+
"""
|
|
423
|
+
|
|
424
|
+
def __init__(self, paths: list[Path | str]):
|
|
425
|
+
self._paths = list(paths)
|
|
426
|
+
|
|
427
|
+
def __len__(self) -> int:
|
|
428
|
+
return len(self._paths)
|
|
429
|
+
|
|
430
|
+
def __getitem__(self, index: int) -> np.ndarray:
|
|
431
|
+
return np.load(self._paths[index])
|
|
432
|
+
|
|
433
|
+
def __iter__(self):
|
|
434
|
+
for path in self._paths:
|
|
435
|
+
yield np.load(path)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _failed_workers(workers: list[mp.Process]) -> list[tuple[int | None, int | None]]:
|
|
439
|
+
"""Return (pid, exitcode) for every worker that has exited with a non-zero code.
|
|
440
|
+
|
|
441
|
+
Workers that are still running (exitcode is None) or exited cleanly (0) are
|
|
442
|
+
excluded.
|
|
443
|
+
"""
|
|
444
|
+
return [
|
|
445
|
+
(p.pid, p.exitcode)
|
|
446
|
+
for p in workers
|
|
447
|
+
if p.exitcode is not None and p.exitcode != 0
|
|
448
|
+
]
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _abort_workers(workers: list[mp.Process]) -> None:
|
|
452
|
+
"""Terminate any still-running workers and reap them all.
|
|
453
|
+
|
|
454
|
+
Called when one worker has already failed: the survivors are blocked waiting
|
|
455
|
+
on a queue that will never be drained, so there is nothing to salvage.
|
|
456
|
+
"""
|
|
457
|
+
for p in workers:
|
|
458
|
+
if p.is_alive():
|
|
459
|
+
p.terminate()
|
|
460
|
+
for p in workers:
|
|
461
|
+
p.join()
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _put_or_abort(task_queue: mp.Queue, item: Any, workers: list[mp.Process]) -> None:
|
|
465
|
+
"""Put ``item`` on ``task_queue``, aborting if a worker dies while we wait.
|
|
466
|
+
|
|
467
|
+
The queue is bounded, so ``put`` blocks once it is full. If a worker has
|
|
468
|
+
crashed it will never drain the queue again, which would block the producer
|
|
469
|
+
forever. Instead we put with a timeout and, whenever the queue stays full,
|
|
470
|
+
check whether any worker has died; if so we tear down the remaining workers
|
|
471
|
+
and raise rather than hang.
|
|
472
|
+
"""
|
|
473
|
+
while True:
|
|
474
|
+
try:
|
|
475
|
+
task_queue.put(item, timeout=1.0)
|
|
476
|
+
return
|
|
477
|
+
except queue.Full:
|
|
478
|
+
failed = _failed_workers(workers)
|
|
479
|
+
if failed:
|
|
480
|
+
_abort_workers(workers)
|
|
481
|
+
details = ", ".join(
|
|
482
|
+
f"pid {pid} exited with code {code}" for pid, code in failed
|
|
483
|
+
)
|
|
484
|
+
raise RuntimeError(f"One or more worker processes failed: {details}")
|
|
485
|
+
|
|
486
|
+
|
|
328
487
|
def _worker_process(
|
|
329
488
|
animator: Animator,
|
|
330
489
|
worker_id: int,
|
|
@@ -341,6 +500,10 @@ def _worker_process(
|
|
|
341
500
|
1. Calls setup() once to initialize the figure (unless reuse_figure_object is False)
|
|
342
501
|
2. Repeatedly pulls individual frames from the task queue
|
|
343
502
|
3. Renders each frame
|
|
503
|
+
|
|
504
|
+
The Animator instance is pickled and sent to every worker. See the Animator
|
|
505
|
+
class docstring for the implications of stashing large data on instance
|
|
506
|
+
attributes.
|
|
344
507
|
"""
|
|
345
508
|
fig = None
|
|
346
509
|
if reuse_figure_object:
|
|
@@ -356,8 +519,7 @@ def _worker_process(
|
|
|
356
519
|
if not reuse_figure_object:
|
|
357
520
|
fig = animator._setup_and_check()
|
|
358
521
|
animator.update(frame_idx, params)
|
|
359
|
-
|
|
360
|
-
fig.savefig(frame_path, **savefig_params)
|
|
522
|
+
_save_frame(fig, frames_dir, frame_idx, savefig_params)
|
|
361
523
|
frames_processed += 1
|
|
362
524
|
if not reuse_figure_object:
|
|
363
525
|
plt.close(fig)
|
|
@@ -375,67 +537,51 @@ def _merge_frames_into_video(
|
|
|
375
537
|
frames_dir: Path | str,
|
|
376
538
|
output_file: Path | str,
|
|
377
539
|
fps: int,
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
540
|
+
video_mode: str,
|
|
541
|
+
video_quality: int | None,
|
|
542
|
+
video_preset: str | None,
|
|
543
|
+
video_extra_ffmpeg_params: list[str] | None,
|
|
381
544
|
disable_progress_bar: bool | None,
|
|
382
545
|
logger: logging.Logger,
|
|
383
546
|
log_interval: int | None,
|
|
384
547
|
) -> None:
|
|
385
|
-
"""Use
|
|
386
|
-
|
|
387
|
-
|
|
548
|
+
"""Use parallel-video-io (pvio) to merge frames into an H.264 MP4.
|
|
549
|
+
|
|
550
|
+
Frames are stored as raw ``.npy`` RGB arrays (see :func:`_save_frame`) and
|
|
551
|
+
streamed to pvio lazily, so no PNG decode step is needed. pvio handles
|
|
552
|
+
even-dimension and pixel-format requirements internally, and picks the GPU
|
|
553
|
+
(NVENC) or CPU (libx264) encoder according to ``video_mode``.
|
|
554
|
+
"""
|
|
555
|
+
# Gather frame files in sorted (i.e. frame-index) order
|
|
556
|
+
frame_files = sorted(Path(frames_dir).glob("frame_*.npy"))
|
|
388
557
|
|
|
389
558
|
if not frame_files:
|
|
390
559
|
raise RuntimeError("No frames found in temporary directory")
|
|
391
560
|
|
|
392
|
-
|
|
393
|
-
with Image.open(frame_files[0]) as first_img:
|
|
394
|
-
width, height = first_img.size
|
|
395
|
-
# Make width/height even (required by many codecs)
|
|
396
|
-
width = (width // 2) * 2
|
|
397
|
-
height = (height // 2) * 2
|
|
561
|
+
Path(output_file).parent.mkdir(parents=True, exist_ok=True)
|
|
398
562
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
for i, frame_path in tqdm(
|
|
410
|
-
enumerate(frame_files),
|
|
411
|
-
total=len(frame_files),
|
|
412
|
-
desc="Merging frames",
|
|
413
|
-
disable=disable_progress_bar,
|
|
414
|
-
):
|
|
415
|
-
with Image.open(frame_path) as img:
|
|
416
|
-
img = img.convert("RGB")
|
|
417
|
-
# Ensure image has the right size
|
|
418
|
-
if img.size != (width, height):
|
|
419
|
-
img = img.resize((width, height))
|
|
420
|
-
|
|
421
|
-
video_frame = av.VideoFrame.from_image(img)
|
|
422
|
-
video_frame = video_frame.reformat(width, height, format=video_pixfmt)
|
|
423
|
-
for packet in stream.encode(video_frame):
|
|
424
|
-
container.mux(packet)
|
|
563
|
+
encode_kwargs: dict[str, Any] = {
|
|
564
|
+
"mode": video_mode,
|
|
565
|
+
"preset": video_preset,
|
|
566
|
+
"extra_ffmpeg_params": video_extra_ffmpeg_params,
|
|
567
|
+
"log_interval": log_interval,
|
|
568
|
+
"quiet": bool(disable_progress_bar),
|
|
569
|
+
}
|
|
570
|
+
# Only override pvio's default quality when the user requested a specific value.
|
|
571
|
+
if video_quality is not None:
|
|
572
|
+
encode_kwargs["quality"] = video_quality
|
|
425
573
|
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
container.close()
|
|
574
|
+
try:
|
|
575
|
+
pvio.write_frames_to_video(
|
|
576
|
+
output_file,
|
|
577
|
+
_NpyFrameSequence(frame_files),
|
|
578
|
+
fps,
|
|
579
|
+
**encode_kwargs,
|
|
580
|
+
)
|
|
435
581
|
|
|
436
582
|
size_mb = Path(output_file).stat().st_size / (1024 * 1024)
|
|
437
583
|
logger.info(f"Video created: {output_file} ({size_mb:.2f} MB)")
|
|
438
584
|
|
|
439
585
|
except Exception as e:
|
|
440
|
-
logger.critical(f"
|
|
586
|
+
logger.critical(f"parallel-video-io failed to create video: {e}")
|
|
441
587
|
raise
|
|
@@ -1,16 +1,23 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parallel-matplotlib-animation
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Animate matplotlib figures into videos, in parallel but with efficient caching
|
|
5
|
-
Project-URL: Repository, https://github.com/
|
|
5
|
+
Project-URL: Repository, https://github.com/nely-epfl/parallel-matplotlib-animation
|
|
6
6
|
Author-email: Sibo Wang-Chen <sibo.wang@epfl.ch>
|
|
7
7
|
License-File: LICENSE
|
|
8
|
-
Requires-Python:
|
|
9
|
-
Requires-Dist: av>=14.0
|
|
8
|
+
Requires-Python: <3.15,>=3.10
|
|
10
9
|
Requires-Dist: matplotlib>=3.10.0
|
|
11
10
|
Requires-Dist: numpy<3,>=2
|
|
12
|
-
Requires-Dist:
|
|
11
|
+
Requires-Dist: parallel-video-io==0.1.8
|
|
13
12
|
Requires-Dist: tqdm>=4.67
|
|
13
|
+
Provides-Extra: benchmark
|
|
14
|
+
Requires-Dist: pandas<3.0,>=2.0; extra == 'benchmark'
|
|
15
|
+
Requires-Dist: plotly<7.0,>=6.8; extra == 'benchmark'
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: mkdocs-material<10.0,>=9.5; extra == 'dev'
|
|
18
|
+
Requires-Dist: mkdocstrings[python]>=0.29; extra == 'dev'
|
|
19
|
+
Requires-Dist: pytest<10.0,>=9.0; extra == 'dev'
|
|
20
|
+
Requires-Dist: ruff>=0.15; extra == 'dev'
|
|
14
21
|
Description-Content-Type: text/markdown
|
|
15
22
|
|
|
16
23
|
# parallel-matplotlib-animation
|
|
@@ -36,7 +43,7 @@ Renders matplotlib animations by:
|
|
|
36
43
|
1. Creating a bunch of worker processes, and creating matplotlib resources (plt.Figure, plt.Axes, artists, etc.) once per worker
|
|
37
44
|
2. Distributing frames across workers via a dynamic queue
|
|
38
45
|
3. Rendering the assigned frames from each worker, but updating the data only (without redrawing the whole plot from scratch)
|
|
39
|
-
4. Encoding frames to video with
|
|
46
|
+
4. Encoding frames to video with [parallel-video-io](https://github.com/sibocw/parallel-video-io) (FFmpeg under the hood, with automatic GPU/NVENC acceleration when available)
|
|
40
47
|
|
|
41
48
|
**Key design: Figure reuse.** In each worker process, `setup()` runs once to create the figure, then `update()` modifies it repeatedly. This brings the best of:
|
|
42
49
|
- Serial processing: avoids the overhead of recreating complex layouts for every frame
|
|
@@ -52,7 +59,7 @@ from parallel_animate import Animator
|
|
|
52
59
|
# Step 1: Create a child class of parallel_animate.Animator
|
|
53
60
|
class WaveAnimation(Animator):
|
|
54
61
|
|
|
55
|
-
# Step 2: Define how the plot should be
|
|
62
|
+
# Step 2: Define how the plot should be set up
|
|
56
63
|
def setup(self):
|
|
57
64
|
fig, ax = plt.subplots()
|
|
58
65
|
self.x = np.linspace(0, 4 * np.pi, 200)
|
|
@@ -73,12 +80,12 @@ class WaveAnimation(Animator):
|
|
|
73
80
|
# Step 4: Define a list of input parameters, one for each frame
|
|
74
81
|
params = [{"phase": 2 * np.pi * i / 60} for i in range(60)]
|
|
75
82
|
|
|
76
|
-
# Step 5: Make video in parallel
|
|
83
|
+
# Step 5: Make the video in parallel
|
|
77
84
|
anim = WaveAnimation()
|
|
78
85
|
anim.make_video("wave.mp4", param_by_frame=params, fps=30, num_workers=4)
|
|
79
86
|
```
|
|
80
87
|
|
|
81
|
-
<
|
|
88
|
+
<video src="https://sibocw.github.io/parallel-matplotlib-animation/output/simple_wave_animation.mp4" width="480" autoplay loop muted playsinline controls></video>
|
|
82
89
|
|
|
83
90
|
## Usage
|
|
84
91
|
This library has a single class: `parallel_animate.Animator`. To make an animation, you must create your own class inheriting from it and define the following methods:
|
|
@@ -94,37 +101,41 @@ Once you have defined your animator class, there is a single method that you nee
|
|
|
94
101
|
- `fps` (int): Frame rate of the output video
|
|
95
102
|
- `n_frames` (int or None): Number of frames to render. If None, use the length of `param_by_frame`. If param_by_frame does not have `__len__` implemented and `n_frames` is None, the progress bar won't show completion percentage.
|
|
96
103
|
- `num_workers` (int): Number of worker processes to be spawned. If -1, use all CPU cores. If -2, use all but one CPU cores, etc. If 1, no child process is created and the video is made in the main process itself. Default is -1.
|
|
97
|
-
-
|
|
104
|
+
- `video_mode` (str): Encoder selection passed to parallel-video-io: `"auto"` (default) uses the GPU encoder (FFmpeg/NVENC) when a CUDA device is available and falls back to CPU (libx264) otherwise; `"gpu"` forces NVENC and `"cpu"` forces libx264. Output is always an H.264 MP4.
|
|
105
|
+
- `video_quality` (int or None), `video_preset` (str or None), `video_extra_ffmpeg_params` (list of str or None): Optional encoding-quality controls forwarded to parallel-video-io. Leave as `None` to use its sensible defaults.
|
|
106
|
+
- See the docstring for `parallel_animate.animator` directly for the remaining less commonly used, optional parameters. These control logging, figure reuse, prefetching, etc.
|
|
107
|
+
|
|
108
|
+
> **Note:** parallel-video-io is currently Linux-only.
|
|
98
109
|
|
|
99
110
|
### Special case: frame params arriving out-of-order in `param_by_frame`
|
|
100
111
|
In some cases, frames in `param_by_frame` might be out of order. We can handle these scenarios by populating `param_by_frame` with a special `parallel_animate.IndexedFrameParams` dataclass, which specifies the frame index that overrides the ordering in `param_by_frame`. This can be useful when, for example, the animator needs to draw frames that are decoded from a video, and the dataloader for that video might return frames in nondeterministic order because it's parallelized.
|
|
101
112
|
|
|
102
|
-
See `
|
|
113
|
+
See `examples/nondeterministic_video_loader.py` for details.
|
|
103
114
|
|
|
104
115
|
## Examples
|
|
105
116
|
|
|
106
|
-
See [`
|
|
117
|
+
See [`examples/`](https://github.com/sibocw/parallel-matplotlib-animation/blob/main/examples/). Run all of them (except the benchmark) with `./examples/run_all.sh`.
|
|
107
118
|
|
|
108
119
|
`simple_wave_animation.py`: The example above
|
|
109
120
|
|
|
110
121
|
`multi_panel_animation.py`: 5 subplots with different plot types
|
|
111
122
|
|
|
112
|
-
<
|
|
123
|
+
<video src="https://sibocw.github.io/parallel-matplotlib-animation/output/multi_panel_animation.mp4" width="480" autoplay loop muted playsinline controls></video>
|
|
113
124
|
|
|
114
125
|
`very_complex_animation.py`: 14 subplots with GridSpec layout
|
|
115
126
|
|
|
116
|
-
<
|
|
127
|
+
<video src="https://sibocw.github.io/parallel-matplotlib-animation/output/very_complex_animation.mp4" width="480" autoplay loop muted playsinline controls></video>
|
|
117
128
|
|
|
118
129
|
`nondeterministic_video_loader.py`: handling frames that arrive out of order
|
|
119
130
|
|
|
120
|
-
<
|
|
131
|
+
<video src="https://sibocw.github.io/parallel-matplotlib-animation/output/nondeterministic_video_loader.mp4" width="240" autoplay loop muted playsinline controls></video>
|
|
121
132
|
|
|
122
133
|
|
|
123
134
|
## Performance test
|
|
124
135
|
|
|
125
|
-
A [strong scaling test](https://hpc-wiki.info/hpc/Scaling_tests#Strong_Scaling) is implemented in `
|
|
136
|
+
A [strong scaling test](https://hpc-wiki.info/hpc/Scaling_tests#Strong_Scaling) is implemented in `examples/scaling_test.py`. Here's the result on my 8-core (16-thread) Intel Core i9-11900K Processor:
|
|
126
137
|
|
|
127
|
-
|
|
138
|
+
See the [interactive scaling figure](https://sibocw.github.io/parallel-matplotlib-animation/benchmark/) on the documentation site.
|
|
128
139
|
|
|
129
140
|
The left-most blue dot indicates serial processing with resources reuse. The black line indicates ideal scaling (zero overhead) if all frames are rendered completely independently in parallel (as is the case in all parallel matplotlib animation libraries I found). Blue dots at 1+ workers are what's implemented in this library.
|
|
130
141
|
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
parallel_animate/__init__.py,sha256=GodSG4KP4d8DR_7y3UxAILW8IZRgmOCmPvaMIzgsJ_g,97
|
|
2
|
+
parallel_animate/animator.py,sha256=Qvz9jC65Q3Fjpn2Vxuf-VitshqZmqYzlCgDXtK_-mUY,23352
|
|
3
|
+
parallel_animate/util.py,sha256=Ym0OyySuU23AE9AWxjBe9pZHizkuBYhlo7wCn7AdzB0,2157
|
|
4
|
+
parallel_matplotlib_animation-0.1.3.dist-info/METADATA,sha256=37ZIZqalX8tTo-ZqMb8DOFqyxZnFT7oRDurE2LadoeE,8839
|
|
5
|
+
parallel_matplotlib_animation-0.1.3.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
6
|
+
parallel_matplotlib_animation-0.1.3.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
7
|
+
parallel_matplotlib_animation-0.1.3.dist-info/RECORD,,
|
|
File without changes
|