torchcodec 0.8.0__cp313-cp313-macosx_12_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of torchcodec might be problematic. Click here for more details.

Files changed (82) hide show
  1. torchcodec/.dylibs/libc++.1.0.dylib +0 -0
  2. torchcodec/.dylibs/libpython3.13.dylib +0 -0
  3. torchcodec/__init__.py +16 -0
  4. torchcodec/_core/AVIOContextHolder.cpp +60 -0
  5. torchcodec/_core/AVIOContextHolder.h +64 -0
  6. torchcodec/_core/AVIOFileLikeContext.cpp +98 -0
  7. torchcodec/_core/AVIOFileLikeContext.h +55 -0
  8. torchcodec/_core/AVIOTensorContext.cpp +123 -0
  9. torchcodec/_core/AVIOTensorContext.h +43 -0
  10. torchcodec/_core/BetaCudaDeviceInterface.cpp +636 -0
  11. torchcodec/_core/BetaCudaDeviceInterface.h +191 -0
  12. torchcodec/_core/CMakeLists.txt +325 -0
  13. torchcodec/_core/CUDACommon.cpp +315 -0
  14. torchcodec/_core/CUDACommon.h +46 -0
  15. torchcodec/_core/Cache.h +138 -0
  16. torchcodec/_core/CpuDeviceInterface.cpp +347 -0
  17. torchcodec/_core/CpuDeviceInterface.h +132 -0
  18. torchcodec/_core/CudaDeviceInterface.cpp +357 -0
  19. torchcodec/_core/CudaDeviceInterface.h +64 -0
  20. torchcodec/_core/DeviceInterface.cpp +117 -0
  21. torchcodec/_core/DeviceInterface.h +148 -0
  22. torchcodec/_core/Encoder.cpp +807 -0
  23. torchcodec/_core/Encoder.h +173 -0
  24. torchcodec/_core/FFMPEGCommon.cpp +608 -0
  25. torchcodec/_core/FFMPEGCommon.h +245 -0
  26. torchcodec/_core/FilterGraph.cpp +149 -0
  27. torchcodec/_core/FilterGraph.h +59 -0
  28. torchcodec/_core/Frame.cpp +42 -0
  29. torchcodec/_core/Frame.h +72 -0
  30. torchcodec/_core/Metadata.h +72 -0
  31. torchcodec/_core/NVDECCache.cpp +70 -0
  32. torchcodec/_core/NVDECCache.h +104 -0
  33. torchcodec/_core/SingleStreamDecoder.cpp +1719 -0
  34. torchcodec/_core/SingleStreamDecoder.h +405 -0
  35. torchcodec/_core/StreamOptions.h +63 -0
  36. torchcodec/_core/Transform.cpp +60 -0
  37. torchcodec/_core/Transform.h +59 -0
  38. torchcodec/_core/ValidationUtils.cpp +35 -0
  39. torchcodec/_core/ValidationUtils.h +21 -0
  40. torchcodec/_core/__init__.py +41 -0
  41. torchcodec/_core/_metadata.py +317 -0
  42. torchcodec/_core/custom_ops.cpp +875 -0
  43. torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +360 -0
  44. torchcodec/_core/nvcuvid_include/cuviddec.h +1374 -0
  45. torchcodec/_core/nvcuvid_include/nvcuvid.h +610 -0
  46. torchcodec/_core/ops.py +498 -0
  47. torchcodec/_core/pybind_ops.cpp +50 -0
  48. torchcodec/_frame.py +145 -0
  49. torchcodec/_internally_replaced_utils.py +67 -0
  50. torchcodec/_samplers/__init__.py +7 -0
  51. torchcodec/_samplers/video_clip_sampler.py +418 -0
  52. torchcodec/decoders/__init__.py +12 -0
  53. torchcodec/decoders/_audio_decoder.py +177 -0
  54. torchcodec/decoders/_decoder_utils.py +112 -0
  55. torchcodec/decoders/_video_decoder.py +500 -0
  56. torchcodec/encoders/__init__.py +1 -0
  57. torchcodec/encoders/_audio_encoder.py +150 -0
  58. torchcodec/libtorchcodec_core4.dylib +0 -0
  59. torchcodec/libtorchcodec_core5.dylib +0 -0
  60. torchcodec/libtorchcodec_core6.dylib +0 -0
  61. torchcodec/libtorchcodec_core7.dylib +0 -0
  62. torchcodec/libtorchcodec_core8.dylib +0 -0
  63. torchcodec/libtorchcodec_custom_ops4.dylib +0 -0
  64. torchcodec/libtorchcodec_custom_ops5.dylib +0 -0
  65. torchcodec/libtorchcodec_custom_ops6.dylib +0 -0
  66. torchcodec/libtorchcodec_custom_ops7.dylib +0 -0
  67. torchcodec/libtorchcodec_custom_ops8.dylib +0 -0
  68. torchcodec/libtorchcodec_pybind_ops4.so +0 -0
  69. torchcodec/libtorchcodec_pybind_ops5.so +0 -0
  70. torchcodec/libtorchcodec_pybind_ops6.so +0 -0
  71. torchcodec/libtorchcodec_pybind_ops7.so +0 -0
  72. torchcodec/libtorchcodec_pybind_ops8.so +0 -0
  73. torchcodec/samplers/__init__.py +2 -0
  74. torchcodec/samplers/_common.py +84 -0
  75. torchcodec/samplers/_index_based.py +287 -0
  76. torchcodec/samplers/_time_based.py +358 -0
  77. torchcodec/version.py +2 -0
  78. torchcodec-0.8.0.dist-info/METADATA +253 -0
  79. torchcodec-0.8.0.dist-info/RECORD +82 -0
  80. torchcodec-0.8.0.dist-info/WHEEL +5 -0
  81. torchcodec-0.8.0.dist-info/licenses/LICENSE +28 -0
  82. torchcodec-0.8.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,358 @@
1
+ from typing import Literal, Optional
2
+
3
+ import torch
4
+
5
+ from torchcodec import FrameBatch
6
+ from torchcodec.samplers._common import (
7
+ _FRAMEBATCH_RETURN_DOCS,
8
+ _POLICY_FUNCTION_TYPE,
9
+ _POLICY_FUNCTIONS,
10
+ _reshape_4d_framebatch_into_5d,
11
+ _validate_common_params,
12
+ )
13
+
14
+
15
+ def _validate_params_time_based(
16
+ *,
17
+ decoder,
18
+ num_clips,
19
+ seconds_between_clip_starts,
20
+ seconds_between_frames,
21
+ ):
22
+
23
+ if (num_clips is None and seconds_between_clip_starts is None) or (
24
+ num_clips is not None and seconds_between_clip_starts is not None
25
+ ):
26
+ raise ValueError("This is internal only and should never happen.")
27
+
28
+ if seconds_between_clip_starts is not None and seconds_between_clip_starts <= 0:
29
+ raise ValueError(
30
+ f"seconds_between_clip_starts ({seconds_between_clip_starts}) must be > 0"
31
+ )
32
+
33
+ if num_clips is not None and num_clips <= 0:
34
+ raise ValueError(f"num_clips ({num_clips}) must be > 0")
35
+
36
+ if decoder.metadata.average_fps is None:
37
+ raise ValueError(
38
+ "Could not infer average fps from video metadata. "
39
+ "Try using an index-based sampler instead."
40
+ )
41
+
42
+ # Note that metadata.begin_stream_seconds is a property that will always yield a valid
43
+ # value; if it is not present in the actual metadata, the metadata object will return 0.
44
+ # Hence, we do not test for it here and only test metadata.end_stream_seconds.
45
+ if decoder.metadata.end_stream_seconds is None:
46
+ raise ValueError(
47
+ "Could not infer stream end from video metadata. "
48
+ "Try using an index-based sampler instead."
49
+ )
50
+
51
+ average_frame_duration_seconds = 1 / decoder.metadata.average_fps
52
+ if seconds_between_frames is None:
53
+ seconds_between_frames = average_frame_duration_seconds
54
+ elif seconds_between_frames <= 0:
55
+ raise ValueError(
56
+ f"seconds_between_clip_starts ({seconds_between_clip_starts}) must be > 0, got"
57
+ )
58
+
59
+ return seconds_between_frames
60
+
61
+
62
+ def _validate_sampling_range_time_based(
63
+ *,
64
+ num_frames_per_clip,
65
+ seconds_between_frames,
66
+ sampling_range_start,
67
+ sampling_range_end,
68
+ begin_stream_seconds,
69
+ end_stream_seconds,
70
+ ):
71
+
72
+ if sampling_range_start is None:
73
+ sampling_range_start = begin_stream_seconds
74
+ else:
75
+ if sampling_range_start < begin_stream_seconds:
76
+ raise ValueError(
77
+ f"sampling_range_start ({sampling_range_start}) must be at least {begin_stream_seconds}"
78
+ )
79
+ if sampling_range_start >= end_stream_seconds:
80
+ raise ValueError(
81
+ f"sampling_range_start ({sampling_range_start}) must be smaller than {end_stream_seconds}"
82
+ )
83
+
84
+ if sampling_range_end is None:
85
+ # We allow a clip to start anywhere within
86
+ # [sampling_range_start, sampling_range_end)
87
+ # When sampling_range_end is None, we want to automatically set it to
88
+ # the largest possible value such that the sampled frames in any clip
89
+ # are within the bounds of the video duration (in other words, we don't
90
+ # want to have to resort to the `policy`).
91
+ # I.e. we want to guarantee that for all frames in any clip we have
92
+ # pts < end_stream_seconds.
93
+ #
94
+ # The frames of a clip will be sampled at the following pts:
95
+ # clip_timestamps = [
96
+ # clip_start + 0 * seconds_between_frames,
97
+ # clip_start + 1 * seconds_between_frames,
98
+ # clip_start + 2 * seconds_between_frames,
99
+ # ...
100
+ # clip_start + (num_frames_per_clip - 1) * seconds_between_frames,
101
+ # ]
102
+ # To guarantee that any such value is < end_stream_seconds, we only need
103
+ # to guarantee that
104
+ # clip_start < end_stream_seconds - (num_frames_per_clip - 1) * seconds_between_frames
105
+ #
106
+ # So that's the value of sampling_range_end we want to use.
107
+ sampling_range_end = (
108
+ end_stream_seconds - (num_frames_per_clip - 1) * seconds_between_frames
109
+ )
110
+ elif sampling_range_end <= begin_stream_seconds:
111
+ raise ValueError(
112
+ f"sampling_range_end ({sampling_range_end}) must be at least {begin_stream_seconds}"
113
+ )
114
+
115
+ if sampling_range_start >= sampling_range_end:
116
+ raise ValueError(
117
+ f"sampling_range_start ({sampling_range_start}) must be smaller than sampling_range_end ({sampling_range_end})"
118
+ )
119
+
120
+ sampling_range_end = min(sampling_range_end, end_stream_seconds)
121
+
122
+ return sampling_range_start, sampling_range_end
123
+
124
+
125
+ def _build_all_clips_timestamps(
126
+ *,
127
+ clip_start_seconds: torch.Tensor, # 1D float tensor
128
+ num_frames_per_clip: int,
129
+ seconds_between_frames: float,
130
+ end_stream_seconds: float,
131
+ policy_fun: _POLICY_FUNCTION_TYPE,
132
+ ) -> list[float]:
133
+
134
+ all_clips_timestamps: list[float] = []
135
+ for start_seconds in clip_start_seconds:
136
+ clip_timestamps = [
137
+ timestamp
138
+ for i in range(num_frames_per_clip)
139
+ if (timestamp := start_seconds + i * seconds_between_frames)
140
+ < end_stream_seconds
141
+ ]
142
+
143
+ if len(clip_timestamps) < num_frames_per_clip:
144
+ clip_timestamps = policy_fun(clip_timestamps, num_frames_per_clip)
145
+ all_clips_timestamps += clip_timestamps
146
+
147
+ return all_clips_timestamps
148
+
149
+
150
+ def _generic_time_based_sampler(
151
+ kind: Literal["random", "regular"],
152
+ decoder,
153
+ *,
154
+ num_clips: Optional[int], # mutually exclusive with seconds_between_clip_starts
155
+ seconds_between_clip_starts: Optional[float],
156
+ num_frames_per_clip: int,
157
+ seconds_between_frames: Optional[float],
158
+ # None means "begining", which may not always be 0
159
+ sampling_range_start: Optional[float],
160
+ sampling_range_end: Optional[float], # interval is [start, end).
161
+ policy: Literal["repeat_last", "wrap", "error"] = "repeat_last",
162
+ ) -> FrameBatch:
163
+ # Note: *everywhere*, sampling_range_end denotes the upper bound of where a
164
+ # clip can start. This is an *open* upper bound, i.e. we will make sure no
165
+ # clip starts exactly at (or above) sampling_range_end.
166
+
167
+ _validate_common_params(
168
+ decoder=decoder,
169
+ num_frames_per_clip=num_frames_per_clip,
170
+ policy=policy,
171
+ )
172
+
173
+ seconds_between_frames = _validate_params_time_based(
174
+ decoder=decoder,
175
+ num_clips=num_clips,
176
+ seconds_between_clip_starts=seconds_between_clip_starts,
177
+ seconds_between_frames=seconds_between_frames,
178
+ )
179
+
180
+ sampling_range_start, sampling_range_end = _validate_sampling_range_time_based(
181
+ num_frames_per_clip=num_frames_per_clip,
182
+ seconds_between_frames=seconds_between_frames,
183
+ sampling_range_start=sampling_range_start,
184
+ sampling_range_end=sampling_range_end,
185
+ begin_stream_seconds=decoder.metadata.begin_stream_seconds,
186
+ end_stream_seconds=decoder.metadata.end_stream_seconds,
187
+ )
188
+
189
+ if kind == "random":
190
+ assert num_clips is not None # appease type-checker
191
+ sampling_range_width = sampling_range_end - sampling_range_start
192
+ # torch.rand() returns in [0, 1)
193
+ # which ensures all clip starts are < sampling_range_end
194
+ clip_start_seconds = (
195
+ torch.rand(num_clips) * sampling_range_width + sampling_range_start
196
+ )
197
+ else:
198
+ assert seconds_between_clip_starts is not None # appease type-checker
199
+ clip_start_seconds = torch.arange(
200
+ sampling_range_start,
201
+ sampling_range_end, # excluded
202
+ seconds_between_clip_starts,
203
+ )
204
+ # As mentioned in the docs, torch.arange may return values
205
+ # equal to or above `end` because of floating precision errors.
206
+ # Here, we manually ensure all values are strictly lower than `sample_range_end`
207
+ if clip_start_seconds[-1] >= sampling_range_end:
208
+ clip_start_seconds = clip_start_seconds[
209
+ clip_start_seconds < sampling_range_end
210
+ ]
211
+
212
+ num_clips = len(clip_start_seconds)
213
+
214
+ all_clips_timestamps = _build_all_clips_timestamps(
215
+ clip_start_seconds=clip_start_seconds,
216
+ num_frames_per_clip=num_frames_per_clip,
217
+ seconds_between_frames=seconds_between_frames,
218
+ end_stream_seconds=decoder.metadata.end_stream_seconds,
219
+ policy_fun=_POLICY_FUNCTIONS[policy],
220
+ )
221
+
222
+ frames = decoder.get_frames_played_at(seconds=all_clips_timestamps)
223
+ return _reshape_4d_framebatch_into_5d(
224
+ frames=frames,
225
+ num_clips=num_clips,
226
+ num_frames_per_clip=num_frames_per_clip,
227
+ )
228
+
229
+
230
+ def clips_at_random_timestamps(
231
+ decoder,
232
+ *,
233
+ num_clips: int = 1,
234
+ num_frames_per_clip: int = 1,
235
+ seconds_between_frames: Optional[float] = None,
236
+ # None means "begining", which may not always be 0
237
+ sampling_range_start: Optional[float] = None,
238
+ sampling_range_end: Optional[float] = None, # interval is [start, end).
239
+ policy: Literal["repeat_last", "wrap", "error"] = "repeat_last",
240
+ ) -> FrameBatch:
241
+ # See docstring below
242
+ torch._C._log_api_usage_once("torchcodec.samplers.clips_at_random_timestamps")
243
+ return _generic_time_based_sampler(
244
+ kind="random",
245
+ decoder=decoder,
246
+ num_clips=num_clips,
247
+ seconds_between_clip_starts=None,
248
+ num_frames_per_clip=num_frames_per_clip,
249
+ seconds_between_frames=seconds_between_frames,
250
+ sampling_range_start=sampling_range_start,
251
+ sampling_range_end=sampling_range_end,
252
+ policy=policy,
253
+ )
254
+
255
+
256
+ def clips_at_regular_timestamps(
257
+ decoder,
258
+ *,
259
+ seconds_between_clip_starts: float,
260
+ num_frames_per_clip: int = 1,
261
+ seconds_between_frames: Optional[float] = None,
262
+ # None means "begining", which may not always be 0
263
+ sampling_range_start: Optional[float] = None,
264
+ sampling_range_end: Optional[float] = None, # interval is [start, end).
265
+ policy: Literal["repeat_last", "wrap", "error"] = "repeat_last",
266
+ ) -> FrameBatch:
267
+ # See docstring below
268
+ torch._C._log_api_usage_once("torchcodec.samplers.clips_at_regular_timestamps")
269
+ return _generic_time_based_sampler(
270
+ kind="regular",
271
+ decoder=decoder,
272
+ num_clips=None,
273
+ seconds_between_clip_starts=seconds_between_clip_starts,
274
+ num_frames_per_clip=num_frames_per_clip,
275
+ seconds_between_frames=seconds_between_frames,
276
+ sampling_range_start=sampling_range_start,
277
+ sampling_range_end=sampling_range_end,
278
+ policy=policy,
279
+ )
280
+
281
+
282
+ _COMMON_DOCS = """
283
+ {maybe_note}
284
+
285
+ Args:
286
+ decoder (VideoDecoder): The :class:`~torchcodec.decoders.VideoDecoder`
287
+ instance to sample clips from.
288
+ {num_clips_or_seconds_between_clip_starts}
289
+ num_frames_per_clip (int, optional): The number of frames per clips. Default: 1.
290
+ seconds_between_frames (float or None, optional): The time (in seconds)
291
+ between each frame within a clip. More accurately, this defines the
292
+ time between the *frame sampling point*, i.e. the timestamps at
293
+ which we sample the frames. Because frames span intervals in time ,
294
+ the resulting start of frames within a clip may not be exactly
295
+ spaced by ``seconds_between_frames`` - but on average, they will be.
296
+ Default is None, which is set to the average frame duration
297
+ (``1/average_fps``).
298
+ sampling_range_start (float or None, optional): The start of the
299
+ sampling range, which defines the first timestamp (in seconds) that
300
+ a clip may *start* at. Default: None, which corresponds to the start
301
+ of the video. (Note: some videos start at negative values, which is
302
+ why the default is not 0).
303
+ sampling_range_end (float or None, optional): The end of the sampling
304
+ range, which defines the last timestamp (in seconds) that a clip may
305
+ *start* at. This value is exclusive, i.e. a clip may only start within
306
+ [``sampling_range_start``, ``sampling_range_end``). If None
307
+ (default), the value is set automatically such that the clips never
308
+ span beyond the end of the video, i.e. it is set to
309
+ ``end_video_seconds - (num_frames_per_clip - 1) *
310
+ seconds_between_frames``. When a clip spans beyond the end of the
311
+ video, the ``policy`` parameter defines how to construct such clip.
312
+ policy (str, optional): Defines how to construct clips that span beyond
313
+ the end of the video. This is best described with an example:
314
+ assuming the last valid (seekable) timestamp in a video is 10.9, and
315
+ a clip was sampled to start at timestamp 10.5, with
316
+ ``num_frames_per_clip=5`` and ``seconds_between_frames=0.2``, the
317
+ sampling timestamps of the frames in the clip are supposed to be
318
+ [10.5, 10.7, 10.9, 11.1, 11.2]. But 11.1 and 11.2 are invalid
319
+ timestamps, so the ``policy`` parameter defines how to replace those
320
+ frames, with valid sampling timestamps:
321
+
322
+ - "repeat_last": repeats the last valid frame of the clip. We would
323
+ get frames sampled at timestamps [10.5, 10.7, 10.9, 10.9, 10.9].
324
+ - "wrap": wraps around to the beginning of the clip. We would get
325
+ frames sampled at timestamps [10.5, 10.7, 10.9, 10.5, 10.7].
326
+ - "error": raises an error.
327
+
328
+ Default is "repeat_last". Note that when ``sampling_range_end=None``
329
+ (default), this policy parameter is unlikely to be relevant.
330
+
331
+ {return_docs}
332
+ """
333
+
334
+
335
+ _NUM_CLIPS_DOCS = """
336
+ num_clips (int, optional): The number of clips to return. Default: 1.
337
+ """
338
+ clips_at_random_timestamps.__doc__ = f"""Sample :term:`clips` at random timestamps.
339
+ {_COMMON_DOCS.format(maybe_note="", num_clips_or_seconds_between_clip_starts=_NUM_CLIPS_DOCS, return_docs=_FRAMEBATCH_RETURN_DOCS)}
340
+ """
341
+
342
+
343
+ _SECONDS_BETWEEN_CLIP_STARTS = """
344
+ seconds_between_clip_starts (float): The space (in seconds) between each
345
+ clip start.
346
+ """
347
+
348
+ _NOTE_DOCS = """
349
+ .. note::
350
+ For consistency with existing sampling APIs (such as torchvision), this
351
+ sampler takes a ``seconds_between_clip_starts`` parameter instead of
352
+ ``num_clips``. If you find that supporting ``num_clips`` would be
353
+ useful, please let us know by `opening a feature request
354
+ <https://github.com/pytorch/torchcodec/issues?q=is:open+is:issue>`_.
355
+ """
356
+ clips_at_regular_timestamps.__doc__ = f"""Sample :term:`clips` at regular (equally-spaced) timestamps.
357
+ {_COMMON_DOCS.format(maybe_note=_NOTE_DOCS, num_clips_or_seconds_between_clip_starts=_SECONDS_BETWEEN_CLIP_STARTS, return_docs=_FRAMEBATCH_RETURN_DOCS)}
358
+ """
torchcodec/version.py ADDED
@@ -0,0 +1,2 @@
1
+ # Note that this file is generated during install.
2
+ __version__ = '0.8.0'
@@ -0,0 +1,253 @@
1
+ Metadata-Version: 2.4
2
+ Name: torchcodec
3
+ Version: 0.8.0
4
+ Summary: A video decoder for PyTorch
5
+ Author-email: PyTorch Team <packages@pytorch.org>
6
+ Project-URL: GitHub, https://github.com/pytorch/torchcodec
7
+ Project-URL: Documentation, https://pytorch.org/torchcodec/stable/index.html
8
+ Requires-Python: >=3.8
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Provides-Extra: dev
12
+ Requires-Dist: numpy; extra == "dev"
13
+ Requires-Dist: pytest; extra == "dev"
14
+ Requires-Dist: pillow; extra == "dev"
15
+ Dynamic: license-file
16
+
17
+ [**Installation**](#installing-torchcodec) | [**Simple Example**](#using-torchcodec) | [**Detailed Example**](https://pytorch.org/torchcodec/stable/generated_examples/) | [**Documentation**](https://pytorch.org/torchcodec) | [**Contributing**](CONTRIBUTING.md) | [**License**](#license)
18
+
19
+ # TorchCodec
20
+
21
+ TorchCodec is a Python library for decoding video and audio data into PyTorch
22
+ tensors, on CPU and CUDA GPU. It also supports audio encoding, and video
23
+ encoding will come soon! It aims to be fast, easy to use, and well integrated
24
+ into the PyTorch ecosystem. If you want to use PyTorch to train ML models on
25
+ videos and audio, TorchCodec is how you turn these into data.
26
+
27
+ We achieve these capabilities through:
28
+
29
+ * Pythonic APIs that mirror Python and PyTorch conventions.
30
+ * Relying on [FFmpeg](https://www.ffmpeg.org/) to do the decoding and encoding.
31
+ TorchCodec uses the version of FFmpeg you already have installed. FFmpeg is a
32
+ mature library with broad coverage available on most systems. It is, however,
33
+ not easy to use. TorchCodec abstracts FFmpeg's complexity to ensure it is used
34
+ correctly and efficiently.
35
+ * Returning data as PyTorch tensors, ready to be fed into PyTorch transforms
36
+ or used directly to train models.
37
+
38
+ ## Using TorchCodec
39
+
40
+ Here's a condensed summary of what you can do with TorchCodec. For more detailed
41
+ examples, [check out our
42
+ documentation](https://pytorch.org/torchcodec/stable/generated_examples/)!
43
+
44
+ #### Decoding
45
+
46
+ ```python
47
+ from torchcodec.decoders import VideoDecoder
48
+
49
+ device = "cpu" # or e.g. "cuda" !
50
+ decoder = VideoDecoder("path/to/video.mp4", device=device)
51
+
52
+ decoder.metadata
53
+ # VideoStreamMetadata:
54
+ # num_frames: 250
55
+ # duration_seconds: 10.0
56
+ # bit_rate: 31315.0
57
+ # codec: h264
58
+ # average_fps: 25.0
59
+ # ... (truncated output)
60
+
61
+ # Simple Indexing API
62
+ decoder[0] # uint8 tensor of shape [C, H, W]
63
+ decoder[0 : -1 : 20] # uint8 stacked tensor of shape [N, C, H, W]
64
+
65
+ # Indexing, with PTS and duration info:
66
+ decoder.get_frames_at(indices=[2, 100])
67
+ # FrameBatch:
68
+ # data (shape): torch.Size([2, 3, 270, 480])
69
+ # pts_seconds: tensor([0.0667, 3.3367], dtype=torch.float64)
70
+ # duration_seconds: tensor([0.0334, 0.0334], dtype=torch.float64)
71
+
72
+ # Time-based indexing with PTS and duration info
73
+ decoder.get_frames_played_at(seconds=[0.5, 10.4])
74
+ # FrameBatch:
75
+ # data (shape): torch.Size([2, 3, 270, 480])
76
+ # pts_seconds: tensor([ 0.4671, 10.3770], dtype=torch.float64)
77
+ # duration_seconds: tensor([0.0334, 0.0334], dtype=torch.float64)
78
+ ```
79
+
80
+ #### Clip sampling
81
+
82
+ ```python
83
+
84
+ from torchcodec.samplers import clips_at_regular_timestamps
85
+
86
+ clips_at_regular_timestamps(
87
+ decoder,
88
+ seconds_between_clip_starts=1.5,
89
+ num_frames_per_clip=4,
90
+ seconds_between_frames=0.1
91
+ )
92
+ # FrameBatch:
93
+ # data (shape): torch.Size([9, 4, 3, 270, 480])
94
+ # pts_seconds: tensor([[ 0.0000, 0.0667, 0.1668, 0.2669],
95
+ # [ 1.4681, 1.5682, 1.6683, 1.7684],
96
+ # [ 2.9696, 3.0697, 3.1698, 3.2699],
97
+ # ... (truncated), dtype=torch.float64)
98
+ # duration_seconds: tensor([[0.0334, 0.0334, 0.0334, 0.0334],
99
+ # [0.0334, 0.0334, 0.0334, 0.0334],
100
+ # [0.0334, 0.0334, 0.0334, 0.0334],
101
+ # ... (truncated), dtype=torch.float64)
102
+ ```
103
+
104
+ You can use the following snippet to generate a video with FFmpeg and tryout
105
+ TorchCodec:
106
+
107
+ ```bash
108
+ fontfile=/usr/share/fonts/dejavu-sans-mono-fonts/DejaVuSansMono-Bold.ttf
109
+ output_video_file=/tmp/output_video.mp4
110
+
111
+ ffmpeg -f lavfi -i \
112
+ color=size=640x400:duration=10:rate=25:color=blue \
113
+ -vf "drawtext=fontfile=${fontfile}:fontsize=30:fontcolor=white:x=(w-text_w)/2:y=(h-text_h)/2:text='Frame %{frame_num}'" \
114
+ ${output_video_file}
115
+ ```
116
+
117
+ ## Installing TorchCodec
118
+ ### Installing CPU-only TorchCodec
119
+
120
+ 1. Install the latest stable version of PyTorch following the
121
+ [official instructions](https://pytorch.org/get-started/locally/). For other
122
+ versions, refer to the table below for compatibility between versions of
123
+ `torch` and `torchcodec`.
124
+
125
+ 2. Install FFmpeg, if it's not already installed. Linux distributions usually
126
+ come with FFmpeg pre-installed. TorchCodec supports all major FFmpeg versions
127
+ in [4, 7].
128
+
129
+ If FFmpeg is not already installed, or you need a more recent version, an
130
+ easy way to install it is to use `conda`:
131
+
132
+ ```bash
133
+ conda install "ffmpeg<8"
134
+ # or
135
+ conda install "ffmpeg<8" -c conda-forge
136
+ ```
137
+
138
+ 3. Install TorchCodec:
139
+
140
+ ```bash
141
+ pip install torchcodec
142
+ ```
143
+
144
+ The following table indicates the compatibility between versions of
145
+ `torchcodec`, `torch` and Python.
146
+
147
+ | `torchcodec` | `torch` | Python |
148
+ | ------------------ | ------------------ | ------------------- |
149
+ | `main` / `nightly` | `main` / `nightly` | `>=3.10`, `<=3.13` |
150
+ | `0.7` | `2.8` | `>=3.9`, `<=3.13` |
151
+ | `0.6` | `2.8` | `>=3.9`, `<=3.13` |
152
+ | `0.5` | `2.7` | `>=3.9`, `<=3.13` |
153
+ | `0.4` | `2.7` | `>=3.9`, `<=3.13` |
154
+ | `0.3` | `2.7` | `>=3.9`, `<=3.13` |
155
+ | `0.2` | `2.6` | `>=3.9`, `<=3.13` |
156
+ | `0.1` | `2.5` | `>=3.9`, `<=3.12` |
157
+ | `0.0.3` | `2.4` | `>=3.8`, `<=3.12` |
158
+
159
+ ### Installing CUDA-enabled TorchCodec
160
+
161
+ First, make sure you have a GPU that has NVDEC hardware that can decode the
162
+ format you want. Refer to Nvidia's GPU support matrix for more details
163
+ [here](https://developer.nvidia.com/video-encode-and-decode-gpu-support-matrix-new).
164
+
165
+ 1. Install FFmpeg with NVDEC support.
166
+ TorchCodec with CUDA should work with FFmpeg versions in [4, 7].
167
+
168
+ If FFmpeg is not already installed, or you need a more recent version, an
169
+ easy way to install it is to use `conda`:
170
+
171
+ ```bash
172
+ conda install "ffmpeg<8"
173
+ # or
174
+ conda install "ffmpeg<8" -c conda-forge
175
+ ```
176
+
177
+ After installing FFmpeg make sure it has NVDEC support when you list the supported
178
+ decoders:
179
+
180
+ ```bash
181
+ ffmpeg -decoders | grep -i nvidia
182
+ # This should show a line like this:
183
+ # V..... h264_cuvid Nvidia CUVID H264 decoder (codec h264)
184
+ ```
185
+
186
+ To check that FFmpeg libraries work with NVDEC correctly you can decode a sample video:
187
+
188
+ ```bash
189
+ ffmpeg -hwaccel cuda -hwaccel_output_format cuda -i test/resources/nasa_13013.mp4 -f null -
190
+ ```
191
+
192
+ #### Linux
193
+
194
+ 2. Install Pytorch corresponding to your CUDA Toolkit using the
195
+ [official instructions](https://pytorch.org/get-started/locally/). You'll
196
+ need the `libnpp` and `libnvrtc` CUDA libraries, which are usually part of
197
+ the CUDA Toolkit.
198
+
199
+
200
+ 3. Install TorchCodec
201
+
202
+ Pass in an `--index-url` parameter that corresponds to your CUDA Toolkit
203
+ version, for example:
204
+
205
+ ```bash
206
+ # This corresponds to CUDA Toolkit version 12.6. It should be the same one
207
+ # you used when you installed PyTorch (If you installed PyTorch with pip).
208
+ pip install torchcodec --index-url=https://download.pytorch.org/whl/cu126
209
+ ```
210
+
211
+ Note that without passing in the `--index-url` parameter, `pip` installs
212
+ the CPU-only version of TorchCodec.
213
+
214
+ #### Windows
215
+
216
+ 2. On Windows (experimental support), you'll need to rely on `conda` to install
217
+ both pytorch and TorchCodec:
218
+
219
+ ```bash
220
+ conda install -c conda-forge "torchcodec=*=*cuda*"
221
+ ```
222
+
223
+ ## Benchmark Results
224
+
225
+ The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_data.py) on a lightly loaded 22-core machine with an Nvidia A100 with
226
+ 5 [NVDEC decoders](https://docs.nvidia.com/video-technologies/video-codec-sdk/12.1/nvdec-application-note/index.html#).
227
+
228
+ ![benchmark_results](./benchmarks/decoders/benchmark_readme_chart.png)
229
+
230
+ The top row is a [Mandelbrot](https://ffmpeg.org/ffmpeg-filters.html#mandelbrot) video
231
+ generated from FFmpeg that has a resolution of 1280x720 at 60 fps and is 120 seconds long.
232
+ The bottom row is [promotional video from NASA](https://download.pytorch.org/torchaudio/tutorial-assets/stream-api/NASAs_Most_Scientifically_Complex_Space_Observatory_Requires_Precision-MP4_small.mp4)
233
+ that has a resolution of 960x540 at 29.7 fps and is 206 seconds long. Both videos were
234
+ encoded with libx264 and yuv420p pixel format. All decoders, except for TorchVision, used FFmpeg 6.1.2. TorchVision used FFmpeg 4.2.2.
235
+
236
+ For TorchCodec, the "approx" label means that it was using [approximate mode](https://pytorch.org/torchcodec/stable/generated_examples/approximate_mode.html)
237
+ for seeking.
238
+
239
+ ## Contributing
240
+
241
+ We welcome contributions to TorchCodec! Please see our [contributing
242
+ guide](CONTRIBUTING.md) for more details.
243
+
244
+ ## License
245
+
246
+ TorchCodec is released under the [BSD 3 license](./LICENSE).
247
+
248
+ However, TorchCodec may be used with code not written by Meta which may be
249
+ distributed under different licenses.
250
+
251
+ For example, if you build TorchCodec with ENABLE_CUDA=1 or use the CUDA-enabled
252
+ release of torchcodec, please review CUDA's license here:
253
+ [Nvidia licenses](https://docs.nvidia.com/cuda/eula/index.html).