mvdata 0.9.0__tar.gz

mvdata-0.9.0/PKG-INFO

@@ -0,0 +1,52 @@
+Metadata-Version: 2.4
+Name: mvdata
+Version: 0.9.0
+Summary: Gracia Dataset Convention - Python library for working with multi-view video datasets
+Author: Gracia Team
+License: MIT
+Project-URL: Homepage, https://github.com/gracia-labs/mvdata
+Project-URL: Documentation, https://github.com/gracia-labs/mvdata/tree/main/docs
+Project-URL: Repository, https://github.com/gracia-labs/mvdata
+Project-URL: Issues, https://github.com/gracia-labs/mvdata/issues
+Keywords: dataset,multi-view,video,avif,computer-vision
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Image Processing
+Classifier: Topic :: Multimedia :: Video
+Requires-Python: <3.13,>=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=1.20.0
+Requires-Dist: Pillow>=8.0.0
+Requires-Dist: pyavif>=0.0.2
+Requires-Dist: av>=10.0.0
+Provides-Extra: pynvc
+Requires-Dist: PyNvVideoCodec>=2.1.0; extra == "pynvc"
+Provides-Extra: cuda
+Requires-Dist: cupy-cuda12x>=13.0.0; extra == "cuda"
+Provides-Extra: s3
+Requires-Dist: boto3>=1.26.0; extra == "s3"
+Provides-Extra: gcs
+Requires-Dist: google-cloud-storage>=2.0.0; extra == "gcs"
+Provides-Extra: cloud
+Requires-Dist: boto3>=1.26.0; extra == "cloud"
+Requires-Dist: google-cloud-storage>=2.0.0; extra == "cloud"
+Provides-Extra: dev
+Requires-Dist: ipykernel>=7.2.0; extra == "dev"
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=22.0.0; extra == "dev"
+Requires-Dist: isort>=5.10.0; extra == "dev"
+Requires-Dist: mypy>=0.990; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+# mvdata
+
+Python library for working with Gracia multi-view video datasets.
+
+The package provides readers, writers, conversion tools, and GPU-aware video
+decode helpers for the dataset layouts documented in the `docs` directory.

mvdata-0.9.0/README.md

@@ -0,0 +1,6 @@
+# mvdata
+
+Python library for working with Gracia multi-view video datasets.
+
+The package provides readers, writers, conversion tools, and GPU-aware video
+decode helpers for the dataset layouts documented in the `docs` directory.

mvdata-0.9.0/mvdata/__init__.py

@@ -0,0 +1,169 @@
+"""
+Gracia Dataset Convention - Python library for working with multi-view video datasets.
+"""
+
+from typing import Any
+
+from .cloud_storage import CloudStorageProviderType, CloudStorageUrl
+from .dataset_base import Dataset, FrameReader, auto_detect_dataset
+from .downloader import (
+    DatasetDownloader,
+    LegacyDatasetDownloader,
+    MultiVideoDatasetDownloader,
+    RangedDatasetDownloader,
+    infer_and_download_dataset,
+    register_downloader,
+)
+from .legacy_writer import LegacyDatasetWriter
+from .per_frame import LegacyDataset, LegacyFrameReader
+from .writer_base import DatasetWriter
+from .gpu_policy import (
+    clear_gpu_override,
+    gpu_disabled,
+    gpu_enabled,
+    gpu_enabled_override,
+    gpu_usage_allowed,
+    gpu_nvenc_disabled_globally,
+    nvdec_decode_allowed,
+    nvdec_decoding_usable,
+    nvenc_encode_allowed,
+    nvenc_encoding_usable,
+    pynvvideocodec_importable,
+)
+from .gpu_support import (
+    GpuDecodeMediaSupport,
+    GpuDecodeSupportReport,
+    gpu_decode_support,
+    supports_gpu_decode,
+)
+from .image_metrics import calculate_psnr
+from .multivideo_decode_benchmark import (
+    MultiVideoDecodeBenchmarkResult,
+    NvdecRuntimeInfo,
+    benchmark_multivideo_decode,
+    inspect_nvdec_runtime,
+    summarize_multivideo_dataset,
+)
+
+
+def _build_missing_class(class_name: str, install_hint: str, import_error: Exception):
+    """Create a placeholder class that raises a clear dependency error when instantiated."""
+
+    def _raise_missing(self, *args: Any, **kwargs: Any) -> None:
+        raise ImportError(
+            f"{class_name} is unavailable because an optional dependency is missing. "
+            f"Install it with: {install_hint}"
+        ) from import_error
+
+    missing_class = type(class_name, (), {"__init__": _raise_missing})
+    missing_class.__module__ = __name__
+    return missing_class
+
+
+try:
+    from .ranged import RangedDataset, RangedFrameReader, RangedMp4FrameReader
+    from .ranged_writer import RangedDatasetWriter, RangedNvencSettings
+except Exception as exc:  # pragma: no cover - exercised in environments without pyavif
+    RangedDataset = _build_missing_class("RangedDataset", "pip install pyavif", exc)
+    RangedFrameReader = _build_missing_class("RangedFrameReader", "pip install pyavif", exc)
+    RangedMp4FrameReader = _build_missing_class("RangedMp4FrameReader", "pip install pyavif", exc)
+    RangedDatasetWriter = _build_missing_class("RangedDatasetWriter", "pip install pyavif", exc)
+    RangedNvencSettings = _build_missing_class("RangedNvencSettings", "pip install pyavif", exc)
+
+try:
+    from .multivideo import MultiVideoDataset, MultiVideoFrameReader
+    from .multivideo_writer import MultiVideoDatasetWriter
+except Exception as exc:  # pragma: no cover - exercised in environments without av/ffmpeg
+    MultiVideoDataset = _build_missing_class("MultiVideoDataset", "pip install av", exc)
+    MultiVideoFrameReader = _build_missing_class("MultiVideoFrameReader", "pip install av", exc)
+    MultiVideoDatasetWriter = _build_missing_class("MultiVideoDatasetWriter", "pip install av", exc)
+
+try:
+    from .multivideo_slicer import (
+        MultiVideoSliceEligibility,
+        MultiVideoSliceEligibilityMetadata,
+        MultiVideoSliceError,
+        MultiVideoSlicePlan,
+        MultiVideoSliceRange,
+        MultiVideoStreamEligibility,
+        MultiVideoStreamSliceInfo,
+        MultiVideoToRangedSlicer,
+    )
+except Exception as exc:  # pragma: no cover - exercised in environments without av/pyavif
+    MultiVideoToRangedSlicer = _build_missing_class(
+        "MultiVideoToRangedSlicer", "pip install av pyavif", exc
+    )
+    MultiVideoSliceEligibility = _build_missing_class(
+        "MultiVideoSliceEligibility", "pip install av pyavif", exc
+    )
+    MultiVideoSliceEligibilityMetadata = _build_missing_class(
+        "MultiVideoSliceEligibilityMetadata", "pip install av pyavif", exc
+    )
+    MultiVideoSliceError = _build_missing_class(
+        "MultiVideoSliceError", "pip install av pyavif", exc
+    )
+    MultiVideoSlicePlan = _build_missing_class("MultiVideoSlicePlan", "pip install av pyavif", exc)
+    MultiVideoSliceRange = _build_missing_class(
+        "MultiVideoSliceRange", "pip install av pyavif", exc
+    )
+    MultiVideoStreamEligibility = _build_missing_class(
+        "MultiVideoStreamEligibility", "pip install av pyavif", exc
+    )
+    MultiVideoStreamSliceInfo = _build_missing_class(
+        "MultiVideoStreamSliceInfo", "pip install av pyavif", exc
+    )
+
+__all__ = [
+    'Dataset',
+    'FrameReader',
+    'auto_detect_dataset',
+    'LegacyDataset',
+    'LegacyFrameReader',
+    'RangedDataset',
+    'RangedFrameReader',
+    'RangedMp4FrameReader',
+    'MultiVideoDataset',
+    'MultiVideoFrameReader',
+    'DatasetWriter',
+    'RangedDatasetWriter',
+    'RangedNvencSettings',
+    'LegacyDatasetWriter',
+    'MultiVideoDatasetWriter',
+    'MultiVideoToRangedSlicer',
+    'MultiVideoSliceEligibility',
+    'MultiVideoSliceEligibilityMetadata',
+    'MultiVideoSliceError',
+    'MultiVideoSlicePlan',
+    'MultiVideoSliceRange',
+    'MultiVideoStreamEligibility',
+    'MultiVideoStreamSliceInfo',
+    'calculate_psnr',
+    'NvdecRuntimeInfo',
+    'MultiVideoDecodeBenchmarkResult',
+    'inspect_nvdec_runtime',
+    'summarize_multivideo_dataset',
+    'benchmark_multivideo_decode',
+    'GpuDecodeMediaSupport',
+    'GpuDecodeSupportReport',
+    'gpu_decode_support',
+    'supports_gpu_decode',
+    'gpu_enabled',
+    'clear_gpu_override',
+    'gpu_disabled',
+    'gpu_enabled_override',
+    'gpu_usage_allowed',
+    'gpu_nvenc_disabled_globally',
+    'nvdec_decode_allowed',
+    'nvenc_encode_allowed',
+    'nvdec_decoding_usable',
+    'nvenc_encoding_usable',
+    'pynvvideocodec_importable',
+    'DatasetDownloader',
+    'RangedDatasetDownloader',
+    'LegacyDatasetDownloader',
+    'MultiVideoDatasetDownloader',
+    'infer_and_download_dataset',
+    'register_downloader',
+    'CloudStorageUrl',
+    'CloudStorageProviderType',
+]

mvdata-0.9.0/mvdata/cloud_storage.py

@@ -0,0 +1,204 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+from pathlib import Path
+from typing import List, Optional, Tuple
+from urllib.parse import urlparse
+
+
+class CloudStorageProviderType(Enum):
+    AWS_S3 = 's3'
+    GOOGLE_STORAGE = 'gs'
+
+
+class CloudStorageUrl:
+    def __init__(self, url: str):
+        parsed = urlparse(url)
+        if parsed.scheme not in ("s3", "gs"):
+            raise ValueError(f"Invalid cloud URI scheme: {parsed.scheme}. Expected 's3' or 'gs'")
+
+        if not parsed.netloc:
+            raise ValueError(f"Invalid cloud URI: missing bucket name in {url}")
+
+        self.schema: str = parsed.scheme
+        self.bucket: str = parsed.netloc
+        path = parsed.path.lstrip("/")
+        if path and not path.endswith('/'):
+            path += "/"
+        self.path: str = path
+        self.cloud: CloudStorageProviderType = (
+            CloudStorageProviderType.AWS_S3
+            if parsed.scheme == "s3"
+            else CloudStorageProviderType.GOOGLE_STORAGE
+        )
+
+    @staticmethod
+    def from_parts(schema: str, bucket: str, path: str = "") -> "CloudStorageUrl":
+        path = path.lstrip("/")
+        return CloudStorageUrl(f"{schema}://{bucket}/{path}")
+
+
+class BaseCloudStorageProvider(ABC):
+    @abstractmethod
+    def list_objects(self, bucket: str, prefix: str, delimiter: Optional[str] = None) -> Tuple[List[str], List[str]]:
+        pass
+
+    @abstractmethod
+    def download_file(self, bucket: str, key: str, local_path: Path) -> None:
+        pass
+
+
+class AwsS3CloudStorageProvider(BaseCloudStorageProvider):
+    def __init__(self):
+        try:
+            import boto3
+            from botocore.exceptions import ClientError, NoCredentialsError
+        except ImportError as exc:
+            raise ImportError(
+                "boto3 is required for S3 support. "
+                "Install it with: pip install mvdata[s3]"
+            ) from exc
+
+        self._ClientError = ClientError
+
+        try:
+            self._client = boto3.client("s3")
+        except NoCredentialsError as exc:
+            raise RuntimeError(
+                "AWS credentials not found. Configure credentials using AWS CLI, "
+                "environment variables, or IAM role."
+            ) from exc
+
+    def list_objects(self, bucket: str, prefix: str, delimiter: Optional[str] = None) -> Tuple[List[str], List[str]]:
+        objects = []
+        common_prefixes = []
+
+        try:
+            paginator = self._client.get_paginator("list_objects_v2")
+
+            page_kwargs = {
+                "Bucket": bucket,
+                "Prefix": prefix,
+            }
+            if delimiter:
+                page_kwargs["Delimiter"] = delimiter
+
+            for page in paginator.paginate(**page_kwargs):
+                if "Contents" in page:
+                    objects.extend([obj["Key"] for obj in page["Contents"]])
+
+                if delimiter and "CommonPrefixes" in page:
+                    common_prefixes.extend([cp["Prefix"] for cp in page["CommonPrefixes"]])
+
+            return objects, common_prefixes
+
+        except self._ClientError as e:
+            raise RuntimeError(f"S3 access error: {e}") from e
+
+    def download_file(self, bucket: str, key: str, local_path: Path) -> None:
+        local_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            self._client.download_file(bucket, key, str(local_path))
+        except self._ClientError as e:
+            raise RuntimeError(f"S3 download error for {key}: {e}") from e
+
+
+class GoogleStorageCloudStorageProvider(BaseCloudStorageProvider):
+    def __init__(self):
+        try:
+            from google.cloud import storage
+        except ImportError as exc:
+            raise ImportError(
+                "google-cloud-storage is required for GCS support. "
+                "Install it with: pip install mvdata[gcs]"
+            ) from exc
+
+        from google.api_core.exceptions import Forbidden, NotFound, GoogleAPICallError  # type: ignore[import-untyped]
+        from google.auth.exceptions import DefaultCredentialsError  # type: ignore[import-untyped]
+
+        self._Forbidden = Forbidden
+        self._NotFound = NotFound
+        self._GoogleAPICallError = GoogleAPICallError
+
+        try:
+            self._client = storage.Client()
+        except DefaultCredentialsError as exc:
+            raise RuntimeError(
+                "GCS credentials not found. Configure credentials using "
+                "GOOGLE_APPLICATION_CREDENTIALS environment variable, "
+                "gcloud auth, or service account."
+            ) from exc
+
+    def list_objects(self, bucket: str, prefix: str, delimiter: Optional[str] = None) -> Tuple[List[str], List[str]]:
+        objects = []
+        common_prefixes = []
+
+        try:
+            bucket_obj = self._client.bucket(bucket)
+            blobs = self._client.list_blobs(bucket_obj, prefix=prefix, delimiter=delimiter)
+
+            for blob in blobs:
+                objects.append(blob.name)
+
+            if delimiter:
+                common_prefixes = list(blobs.prefixes)
+
+            return objects, common_prefixes
+
+        except (self._Forbidden, self._NotFound) as e:
+            raise RuntimeError(f"GCS access error: {e}") from e
+        except self._GoogleAPICallError as e:
+            raise RuntimeError(f"GCS access error: {e}") from e
+
+    def download_file(self, bucket: str, key: str, local_path: Path) -> None:
+        local_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            bucket_obj = self._client.bucket(bucket)
+            blob = bucket_obj.blob(key)
+            blob.download_to_filename(str(local_path))
+        except (self._Forbidden, self._NotFound) as e:
+            raise RuntimeError(f"GCS download error for {key}: {e}") from e
+        except self._GoogleAPICallError as e:
+            raise RuntimeError(f"GCS download error for {key}: {e}") from e
+
+
+class CloudStorageProvider:
+    _providers: dict[CloudStorageProviderType, BaseCloudStorageProvider] = {}
+
+    @classmethod
+    def _get_provider(cls, provider_type: CloudStorageProviderType) -> BaseCloudStorageProvider:
+        if provider_type not in cls._providers:
+            if provider_type == CloudStorageProviderType.AWS_S3:
+                cls._providers[provider_type] = AwsS3CloudStorageProvider()
+            else:
+                cls._providers[provider_type] = GoogleStorageCloudStorageProvider()
+        return cls._providers[provider_type]
+
+    @classmethod
+    def list_objects(cls, url: CloudStorageUrl, delimiter: Optional[str] = None) -> Tuple[List[str], List[str]]:
+        """
+        List objects in cloud storage bucket with given prefix.
+
+        Args:
+            url: CloudStorageUrl with schema, bucket, and path prefix
+            delimiter: Optional delimiter for hierarchical listing (e.g., '/')
+
+        Returns:
+            Tuple of (object_keys, common_prefixes)
+        """
+        provider = cls._get_provider(url.cloud)
+        return provider.list_objects(url.bucket, url.path, delimiter)
+
+    @classmethod
+    def download_file(cls, url: CloudStorageUrl, local_path: Path) -> None:
+        """
+        Download a single file from cloud storage.
+
+        Args:
+            url: CloudStorageUrl with schema, bucket, and path to the object
+            local_path: Local file path to save to
+        """
+        key = url.path.rstrip("/")
+        provider = cls._get_provider(url.cloud)
+        provider.download_file(url.bucket, key, local_path)

mvdata-0.9.0/mvdata/codec/__init__.py

@@ -0,0 +1,126 @@
+"""NVENC / NVDEC / PyAV codec helpers for mvdata.
+
+This subpackage is split into small modules:
+
+- :mod:`mvdata.codec._imports` – lazy optional-dep imports and constants
+- :mod:`mvdata.codec.probe`    – bit-depth probing and GPU capability checks
+- :mod:`mvdata.codec.frames`   – decoded-frame → HWC RGB (numpy and cupy)
+- :mod:`mvdata.codec.encode`   – RGB → NVENC elementary stream → MP4
+- :mod:`mvdata.codec.decode`   – NVDEC / PyAV decode of MP4
+- :mod:`mvdata.codec.select`   – round-trip PSNR and auto codec selection
+
+The flat namespace here is preserved for backward compatibility with callers
+that use ``from mvdata.codec import X`` or the top-level ``mvdata.nvenc_codec``
+shim.
+"""
+
+from ._imports import (
+    AUTO_NVENC_CHROMA_SUBSAMPLING,
+    AUTO_NVENC_CODEC_CANDIDATES,
+    AUTO_NVENC_MIN_PSNR_DB,
+    AUTO_NVENC_PRESET,
+    AUTO_NVENC_RANGE_MODE,
+    _PYNVC_API_REF,
+    normalize_codec_name,
+    try_import_av,
+    try_import_cupy,
+    try_import_pynvvideocodec,
+    try_import_torch,
+)
+from .decode import (
+    _try_open_nvdec,
+    decode_mp4_to_rgb,
+    decode_mp4_to_rgb_nvdec,
+    decode_mp4_to_rgb_pyav,
+    preferred_decoder_for_mp4,
+)
+from .encode import (
+    assert_yuv444_encode_supported,
+    best_quality_extra_kwargs,
+    create_nvenc_encoder,
+    encode_rgb_frames_to_elementary_stream,
+    encode_rgb_iter_to_elementary_stream,
+    encode_rgb_iter_to_mp4,
+    encode_rgb_sequence_to_mp4,
+    ffmpeg_executable,
+    mux_elementary_to_mp4,
+    pad_rgb_for_nvenc,
+    resolve_nvenc_bitrate_bps,
+    rgb_hwc_to_nvenc_bgra_hwc4,
+    rgb_hwc_to_yuv444_planar,
+    suggest_nvenc_vbr_bitrate_bps,
+)
+from .frames import (
+    _decoded_frame_to_rgb_cupy,
+    _decoded_frame_to_rgb_numpy,
+    _normalize_rgb_numpy_output,
+    _pyav_frame_to_rgb,
+    numpy_to_cupy_rgb,
+)
+from .probe import (
+    infer_video_bit_depth_from_frame,
+    infer_video_bit_depth_from_pixel_format_name,
+    infer_video_bit_depth_from_stream,
+    infer_video_bit_depth_from_video_format,
+    nvdec_decode_compatibility_issue,
+    nvdec_decode_compatibility_issue_for_path,
+    nvenc_encode_compatibility_issue,
+    nvenc_encode_compatibility_issue_for_rgb,
+    probe_video_bit_depth,
+    probe_video_bit_depth_pyav,
+    probe_video_stream_metadata,
+    pynvc_chroma_format,
+    pynvc_codec_enum,
+    safe_get_decoder_caps,
+    safe_get_encoder_caps,
+)
+from .select import roundtrip_nvenc_stream_psnr, select_auto_nvenc_candidate_for_rgb
+
+__all__ = [
+    "AUTO_NVENC_CHROMA_SUBSAMPLING",
+    "AUTO_NVENC_CODEC_CANDIDATES",
+    "AUTO_NVENC_MIN_PSNR_DB",
+    "AUTO_NVENC_PRESET",
+    "AUTO_NVENC_RANGE_MODE",
+    "assert_yuv444_encode_supported",
+    "best_quality_extra_kwargs",
+    "create_nvenc_encoder",
+    "decode_mp4_to_rgb",
+    "decode_mp4_to_rgb_nvdec",
+    "decode_mp4_to_rgb_pyav",
+    "encode_rgb_frames_to_elementary_stream",
+    "encode_rgb_iter_to_elementary_stream",
+    "encode_rgb_iter_to_mp4",
+    "encode_rgb_sequence_to_mp4",
+    "ffmpeg_executable",
+    "infer_video_bit_depth_from_frame",
+    "infer_video_bit_depth_from_pixel_format_name",
+    "infer_video_bit_depth_from_stream",
+    "infer_video_bit_depth_from_video_format",
+    "mux_elementary_to_mp4",
+    "normalize_codec_name",
+    "numpy_to_cupy_rgb",
+    "nvdec_decode_compatibility_issue",
+    "nvdec_decode_compatibility_issue_for_path",
+    "nvenc_encode_compatibility_issue",
+    "nvenc_encode_compatibility_issue_for_rgb",
+    "pad_rgb_for_nvenc",
+    "preferred_decoder_for_mp4",
+    "probe_video_bit_depth",
+    "probe_video_bit_depth_pyav",
+    "probe_video_stream_metadata",
+    "pynvc_chroma_format",
+    "pynvc_codec_enum",
+    "resolve_nvenc_bitrate_bps",
+    "rgb_hwc_to_nvenc_bgra_hwc4",
+    "rgb_hwc_to_yuv444_planar",
+    "roundtrip_nvenc_stream_psnr",
+    "safe_get_decoder_caps",
+    "safe_get_encoder_caps",
+    "select_auto_nvenc_candidate_for_rgb",
+    "suggest_nvenc_vbr_bitrate_bps",
+    "try_import_av",
+    "try_import_cupy",
+    "try_import_pynvvideocodec",
+    "try_import_torch",
+]

mvdata-0.9.0/mvdata/codec/_imports.py

@@ -0,0 +1,42 @@
+"""Lazy optional-dependency imports and codec name normalisation."""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+_PYNVC_API_REF = "PyNvVideoCodec_API_Reference.pdf"
+
+AUTO_NVENC_CODEC_CANDIDATES = ("av1", "hevc", "h264")
+AUTO_NVENC_CHROMA_SUBSAMPLING = "444"
+AUTO_NVENC_PRESET = "P7"
+AUTO_NVENC_RANGE_MODE = "full"
+AUTO_NVENC_MIN_PSNR_DB = 38.0
+
+
+def try_import_pynvvideocodec() -> Any:
+    return importlib.import_module("PyNvVideoCodec")
+
+
+def _try_import(name: str) -> Any | None:
+    try:
+        return importlib.import_module(name)
+    except ImportError:
+        return None
+
+
+def try_import_av() -> Any | None:
+    return _try_import("av")
+
+
+def try_import_torch() -> Any | None:
+    return _try_import("torch")
+
+
+def try_import_cupy() -> Any | None:
+    return _try_import("cupy")
+
+
+def normalize_codec_name(codec: str) -> str:
+    c = codec.strip().lower()
+    return "hevc" if c == "h265" else c