macrodata-refiner 0.3.1__tar.gz → 0.3.2__tar.gz

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.3.1
+Version: 0.3.2
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -27,9 +27,11 @@ Provides-Extra: video
 Requires-Dist: av; extra == "video"
 Requires-Dist: pillow; extra == "video"
 Provides-Extra: hf
-Requires-Dist: datasets>=3.0.0; extra == "hf"
 Requires-Dist: huggingface-hub>=1.4.1; extra == "hf"
 Requires-Dist: hf>=1.7.1; extra == "hf"
+Provides-Extra: datasets
+Requires-Dist: macrodata-refiner[hf]; extra == "datasets"
+Requires-Dist: datasets>=3.0.0; extra == "datasets"
 Provides-Extra: hand-tracking
 Requires-Dist: macrodata-refiner[hf]; extra == "hand-tracking"
 Requires-Dist: macrodata-refiner[video]; extra == "hand-tracking"
@@ -49,6 +51,8 @@ Requires-Dist: mcap-ros2-support; extra == "mcap"
 Requires-Dist: pillow; extra == "mcap"
 Provides-Extra: s3
 Requires-Dist: s3fs; extra == "s3"
+Provides-Extra: gcs
+Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: tensorflow
 Requires-Dist: tensorflow; extra == "tensorflow"
 Provides-Extra: tfds
@@ -59,6 +63,7 @@ Requires-Dist: macrodata-refiner[all]; extra == "testing"
 Requires-Dist: pytest>=8.0.0; extra == "testing"
 Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
 Provides-Extra: all
+Requires-Dist: macrodata-refiner[datasets]; extra == "all"
 Requires-Dist: macrodata-refiner[hdf5]; extra == "all"
 Requires-Dist: macrodata-refiner[hf]; extra == "all"
 Requires-Dist: macrodata-refiner[mcap]; extra == "all"
@@ -66,6 +71,7 @@ Requires-Dist: macrodata-refiner[video]; extra == "all"
 Requires-Dist: macrodata-refiner[zarr]; extra == "all"
 Requires-Dist: macrodata-refiner[text]; extra == "all"
 Requires-Dist: macrodata-refiner[s3]; extra == "all"
+Requires-Dist: macrodata-refiner[gcs]; extra == "all"
 Requires-Dist: macrodata-refiner[tfds]; extra == "all"
 Dynamic: license-file
 

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "macrodata-refiner"
-version = "0.3.1"
+version = "0.3.2"
 description = "Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets"
 readme = "README.md"
 license = "Apache-2.0"
@@ -35,10 +35,13 @@ video = [
     "pillow",
 ]
 hf = [
-    "datasets>=3.0.0",
     "huggingface-hub>=1.4.1",
     "hf>=1.7.1",
 ]
+datasets = [
+    "macrodata-refiner[hf]",
+    "datasets>=3.0.0",
+]
 hand_tracking = [
     "macrodata-refiner[hf]",
     "macrodata-refiner[video]",
@@ -64,6 +67,9 @@ mcap = [
 s3 = [
     "s3fs",
 ]
+gcs = [
+    "gcsfs",
+]
 tensorflow = [
     "tensorflow",
 ]
@@ -77,6 +83,7 @@ testing = [
     "pytest-cov>=5.0.0",
 ]
 all = [
+    "macrodata-refiner[datasets]",
     "macrodata-refiner[hdf5]",
     "macrodata-refiner[hf]",
     "macrodata-refiner[mcap]",
@@ -84,6 +91,7 @@ all = [
     "macrodata-refiner[zarr]",
     "macrodata-refiner[text]",
     "macrodata-refiner[s3]",
+    "macrodata-refiner[gcs]",
     "macrodata-refiner[tfds]",
 ]
 

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/macrodata_refiner.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.3.1
+Version: 0.3.2
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -27,9 +27,11 @@ Provides-Extra: video
 Requires-Dist: av; extra == "video"
 Requires-Dist: pillow; extra == "video"
 Provides-Extra: hf
-Requires-Dist: datasets>=3.0.0; extra == "hf"
 Requires-Dist: huggingface-hub>=1.4.1; extra == "hf"
 Requires-Dist: hf>=1.7.1; extra == "hf"
+Provides-Extra: datasets
+Requires-Dist: macrodata-refiner[hf]; extra == "datasets"
+Requires-Dist: datasets>=3.0.0; extra == "datasets"
 Provides-Extra: hand-tracking
 Requires-Dist: macrodata-refiner[hf]; extra == "hand-tracking"
 Requires-Dist: macrodata-refiner[video]; extra == "hand-tracking"
@@ -49,6 +51,8 @@ Requires-Dist: mcap-ros2-support; extra == "mcap"
 Requires-Dist: pillow; extra == "mcap"
 Provides-Extra: s3
 Requires-Dist: s3fs; extra == "s3"
+Provides-Extra: gcs
+Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: tensorflow
 Requires-Dist: tensorflow; extra == "tensorflow"
 Provides-Extra: tfds
@@ -59,6 +63,7 @@ Requires-Dist: macrodata-refiner[all]; extra == "testing"
 Requires-Dist: pytest>=8.0.0; extra == "testing"
 Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
 Provides-Extra: all
+Requires-Dist: macrodata-refiner[datasets]; extra == "all"
 Requires-Dist: macrodata-refiner[hdf5]; extra == "all"
 Requires-Dist: macrodata-refiner[hf]; extra == "all"
 Requires-Dist: macrodata-refiner[mcap]; extra == "all"
@@ -66,6 +71,7 @@ Requires-Dist: macrodata-refiner[video]; extra == "all"
 Requires-Dist: macrodata-refiner[zarr]; extra == "all"
 Requires-Dist: macrodata-refiner[text]; extra == "all"
 Requires-Dist: macrodata-refiner[s3]; extra == "all"
+Requires-Dist: macrodata-refiner[gcs]; extra == "all"
 Requires-Dist: macrodata-refiner[tfds]; extra == "all"
 Dynamic: license-file
 

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/macrodata_refiner.egg-info/SOURCES.txt

@@ -73,6 +73,7 @@ src/refiner/io/__init__.py
 src/refiner/io/datafile.py
 src/refiner/io/datafolder.py
 src/refiner/io/fileset.py
+src/refiner/io/utils.py
 src/refiner/launchers/__init__.py
 src/refiner/launchers/base.py
 src/refiner/launchers/cloud.py

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/macrodata_refiner.egg-info/requires.txt

@@ -11,6 +11,7 @@ msgspec>=0.20.0
 pydantic>=2.0.0
 
 [all]
+macrodata-refiner[datasets]
 macrodata-refiner[hdf5]
 macrodata-refiner[hf]
 macrodata-refiner[mcap]
@@ -18,8 +19,16 @@ macrodata-refiner[video]
 macrodata-refiner[zarr]
 macrodata-refiner[text]
 macrodata-refiner[s3]
+macrodata-refiner[gcs]
 macrodata-refiner[tfds]
 
+[datasets]
+macrodata-refiner[hf]
+datasets>=3.0.0
+
+[gcs]
+gcsfs
+
 [hand_tracking]
 macrodata-refiner[hf]
 macrodata-refiner[video]
@@ -29,7 +38,6 @@ ego-vision[models]>=0.1.25
 h5py
 
 [hf]
-datasets>=3.0.0
 huggingface-hub>=1.4.1
 hf>=1.7.1
 

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/io/datafile.py

@@ -11,6 +11,7 @@ from typing import Any, TypeAlias, Union, cast
 from fsspec import AbstractFileSystem, url_to_fs
 from fsspec.implementations.http import HTTPFileSystem
 from fsspec.implementations.local import LocalFileSystem
+from refiner.io.utils import required_refiner_extras
 
 DataFilePath: TypeAlias = str | PathLike[str]
 DataFileSpec: TypeAlias = tuple[DataFilePath, AbstractFileSystem]
@@ -146,6 +147,9 @@ class DataFile:
     def abs_path(self) -> str:
         return self.fs.unstrip_protocol(self.path).removeprefix("file://")
 
+    def required_refiner_extras(self) -> tuple[str, ...]:
+        return required_refiner_extras(self.path, self.fs)
+
     @property
     def is_local(self) -> bool:
         return isinstance(self.fs, LocalFileSystem)

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/io/datafolder.py

@@ -7,6 +7,7 @@ from fsspec.implementations.dirfs import DirFileSystem
 from fsspec.implementations.local import LocalFileSystem
 
 from refiner.io.datafile import DataFile, _storage_options_for_path
+from refiner.io.utils import required_refiner_extras
 
 DataFolderPath: TypeAlias = str | PathLike[str]
 DataFolderSpec: TypeAlias = tuple[DataFolderPath, AbstractFileSystem]
@@ -102,6 +103,9 @@ class DataFolder(DirFileSystem):
         # make sure we strip file:// and similar
         return self.fs.unstrip_protocol(self._join(path)).removeprefix("file://")
 
+    def required_refiner_extras(self) -> tuple[str, ...]:
+        return required_refiner_extras(self.path, self.fs)
+
     def abs_paths(self, paths: str | Iterable[str]) -> str | list[str]:
         """
         Transform  a list of relative paths into a list of complete paths (including fs protocol and base path)

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/io/fileset.py

@@ -8,8 +8,13 @@ from typing import Any, Literal, TypeAlias, Union, cast
 
 from fsspec import AbstractFileSystem, url_to_fs
 
-from refiner.io.datafile import DataFile, DataFileSpec, _storage_options_for_path
+from refiner.io.datafile import (
+    DataFile,
+    DataFileSpec,
+    _storage_options_for_path,
+)
 from refiner.io.datafolder import DataFolder, DataFolderSpec
+from refiner.io.utils import required_refiner_extras
 
 DataFileSetInput: TypeAlias = Union[
     str, PathLike[str], DataFileSpec, DataFolderSpec, DataFile, DataFolder
@@ -22,6 +27,9 @@ class _PathSource:
     path: str
     fs: AbstractFileSystem
 
+    def required_refiner_extras(self) -> tuple[str, ...]:
+        return required_refiner_extras(self.path, self.fs)
+
 
 @dataclass(frozen=True, slots=True)
 class DataFileSet:
@@ -174,6 +182,17 @@ class DataFileSet:
             raise TypeError("DataFileSet entries are not all folders")
         return cast(tuple[DataFolder, ...], entries)
 
+    def required_refiner_extras(self) -> tuple[str, ...]:
+        return tuple(
+            sorted(
+                {
+                    extra
+                    for entry in self.entries
+                    for extra in entry.required_refiner_extras()
+                }
+            )
+        )
+
     @property
     def resolved_entries(self) -> tuple[DataFile | DataFolder, ...]:
         entries: list[DataFile | DataFolder] = []

macrodata_refiner-0.3.2/src/refiner/io/utils.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from fsspec import AbstractFileSystem
+
+
+_PROTOCOL_REFINER_EXTRAS = {
+    "s3": "s3",
+    "s3a": "s3",
+    "hf": "hf",
+    "gcs": "gcs",
+    "gs": "gcs",
+}
+
+
+def required_refiner_extras(path: str, fs: AbstractFileSystem) -> tuple[str, ...]:
+    protocol = fs.protocol
+    protocols = (protocol,) if isinstance(protocol, str) else tuple(protocol)
+    path_protocol, sep, _rest = path.partition("://")
+    return tuple(
+        sorted(
+            {
+                extra
+                for item in (*protocols, path_protocol if sep else None)
+                if item is not None
+                and (extra := _PROTOCOL_REFINER_EXTRAS.get(str(item).lower()))
+                is not None
+            }
+        )
+    )

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/launchers/cloud.py

@@ -103,9 +103,11 @@ class CloudLauncher(BaseLauncher):
         gpu: Optional GPU runtime request for cloud scheduling.
         sync_local_dependencies: Whether to include packages detected from the
             local environment in the cloud runtime.
-        extra_dependencies: Additional packages to install in the cloud runtime.
-            Entries are requirement strings. These take precedence over packages
-            detected from the local environment.
+        dependencies: Additional packages to install in the cloud runtime.
+            Entries are requirement strings.
+        refiner_extras: Additional macrodata-refiner extras to install in the
+            cloud runtime. Built-in blocks automatically declare the extras they
+            require; pass this for extras used outside those blocks.
         secrets: Optional secret sources mounted into the cloud runtime.
         env: Optional plain environment variables mounted into the cloud runtime.
     """
@@ -119,8 +121,9 @@ class CloudLauncher(BaseLauncher):
         cpus_per_worker: int | None = None,
         mem_mb_per_worker: int | None = None,
         gpu: GPU | None = None,
-        sync_local_dependencies: bool = True,
-        extra_dependencies: Sequence[str] | None = None,
+        sync_local_dependencies: bool = False,
+        dependencies: Sequence[str] | None = None,
+        refiner_extras: Sequence[str] | None = None,
         secrets: SecretInput | None = None,
         env: dict[str, object | None] | None = None,
         continue_from_job: str | None = None,
@@ -141,7 +144,8 @@ class CloudLauncher(BaseLauncher):
         self.cpus_per_worker = cpus_per_worker
         self.mem_mb_per_worker = mem_mb_per_worker
         self.sync_local_dependencies = sync_local_dependencies
-        self.extra_dependencies = extra_dependencies
+        self.dependencies = dependencies
+        self.refiner_extras = refiner_extras
         self.secrets = normalize_secret_sources(secrets)
         self.env = env
         self.continue_from_job = normalized_continue_from_job
@@ -153,12 +157,14 @@ class CloudLauncher(BaseLauncher):
         return raw.strip().lower() in {"1", "true", "yes", "on"}
 
     def _resolve_cloud_manifest(
-        self, *, secret_values: tuple[str, ...]
+        self, *, secret_values: tuple[str, ...], stages: list[PlannedStage]
     ) -> dict[str, object]:
         manifest = build_run_manifest(
             secret_values=secret_values,
             capture_dependencies=self.sync_local_dependencies,
-            extra_dependencies=self.extra_dependencies,
+            dependencies=self.dependencies,
+            refiner_extras=self.refiner_extras,
+            pipeline_stages=stages,
         )
         environment = manifest.get("environment")
         if environment is None:
@@ -286,7 +292,10 @@ class CloudLauncher(BaseLauncher):
         resolved_secret_sources, secret_values = resolve_secret_sources(self.secrets)
         resolved_env = resolve_env_mapping(self.env) if self.env else None
         stages = self._resolved_stages()
-        manifest = self._resolve_cloud_manifest(secret_values=secret_values)
+        manifest = self._resolve_cloud_manifest(
+            secret_values=secret_values,
+            stages=stages,
+        )
         plan = self._compiled_plan(stages, secret_values=secret_values)
         try:
             pipeline_payloads = self._upload_stage_payloads(

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/pipeline/pipeline.py

@@ -708,8 +708,9 @@ class RefinerPipeline:
         cpus_per_worker: int | None = None,
         mem_mb_per_worker: int | None = None,
         gpu: GPU | None = None,
-        sync_local_dependencies: bool = True,
-        extra_dependencies: Sequence[str] | None = None,
+        sync_local_dependencies: bool = False,
+        dependencies: Sequence[str] | None = None,
+        refiner_extras: Sequence[str] | None = None,
         secrets: SecretInput | None = None,
         env: Mapping[str, object | None] | None = None,
         continue_from_job: str | None = None,
@@ -725,10 +726,13 @@ class RefinerPipeline:
             gpu: Optional structured GPU request.
             sync_local_dependencies: Include packages detected from the local
                 environment in the cloud runtime.
-            extra_dependencies: Additional packages to install in the cloud runtime.
+            dependencies: Additional packages to install in the cloud runtime.
                 Entries are requirement strings such as `"torch"` or
-                `"ego-vision[models]==0.1.2"`. These take precedence over packages
-                detected from the local environment.
+                `"ego-vision[models]==0.1.2"`.
+            refiner_extras: Additional macrodata-refiner extras to install in
+                the cloud runtime. Built-in blocks automatically declare the
+                extras they require; pass this for extras used outside those
+                blocks.
             secrets: Secret sources to mount inside the cloud image. A mapping keeps
                 the legacy behavior; `None` values are loaded from the submitting
                 environment. `Secrets.env(...)` references stored workspace secrets.
@@ -750,7 +754,8 @@ class RefinerPipeline:
             mem_mb_per_worker=mem_mb_per_worker,
             gpu=gpu,
             sync_local_dependencies=sync_local_dependencies,
-            extra_dependencies=extra_dependencies,
+            dependencies=dependencies,
+            refiner_extras=refiner_extras,
             secrets=secrets,
             env=dict(env) if env is not None else None,
             continue_from_job=continue_from_job,

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/pipeline/planning.py

@@ -329,10 +329,19 @@ def _builtin_description(fn: Any) -> dict[str, Any] | None:
     return {"name": name, "args": args, "services": tuple(parsed_services)}
 
 
-def describe_builtin(name: str, **args: Any) -> Any:
+def describe_builtin(
+    name: str, *, refiner_extras: tuple[str, ...] = (), **args: Any
+) -> Any:
     def _decorate(fn: Any) -> Any:
         setattr(
-            fn, _REFINER_BUILTIN_CALL_ATTR, {"name": name, "args": args, "services": ()}
+            fn,
+            _REFINER_BUILTIN_CALL_ATTR,
+            {
+                "name": name,
+                "args": args,
+                "services": (),
+                "refiner_extras": refiner_extras,
+            },
         )
         return fn
 

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/pipeline/sinks/base.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import ABC
-from typing import Any
+from typing import Any, cast
 
 import pyarrow as pa
 
@@ -56,6 +56,27 @@ class BaseSink(ABC):
         """
         return None
 
+    def required_refiner_extras(self) -> tuple[str, ...]:
+        """macrodata-refiner extras required by this sink."""
+        return tuple(
+            sorted(
+                {
+                    *self._declared_refiner_extras(),
+                    *self._io_refiner_extras(),
+                }
+            )
+        )
+
+    def _declared_refiner_extras(self) -> tuple[str, ...]:
+        """Feature extras declared by this sink."""
+        return ()
+
+    def _io_refiner_extras(self) -> tuple[str, ...]:
+        """Storage extras required by this sink's output, if it has one."""
+        if not hasattr(self, "output"):
+            return ()
+        return cast(Any, self).output.required_refiner_extras()
+
     def build_reducer(self) -> "BaseSink | None":
         """Return an optional 1-worker reducer sink for launched execution.
 

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/pipeline/sinks/lerobot.py

@@ -36,7 +36,6 @@ from refiner.robotics.lerobot_format import (
     infer_feature_info,
 )
 from refiner.robotics.row import RoboticsRow
-from refiner.utils import check_required_dependencies
 from refiner.worker.context import get_active_worker_token
 from refiner.worker.metrics.api import register_gauge
 
@@ -101,7 +100,6 @@ class LeRobotWriterSink(BaseSink):
         quantile_bins: int = 5000,
         force_recompute_video_stats: bool = False,
     ):
-        check_required_dependencies("write_lerobot", ["av"], dist="robotics")
         self.output = DataFolder.resolve(output)
         self.data_files_size_in_mb = data_files_size_in_mb
         self.video_files_size_in_mb = video_files_size_in_mb
@@ -123,6 +121,9 @@ class LeRobotWriterSink(BaseSink):
         )
         self._episodes_in_flight_registered = False
 
+    def _declared_refiner_extras(self) -> tuple[str, ...]:
+        return ("video",)
+
     def write_shard_block(self, shard_id: str, block: Block) -> None:
         """Submit one async write task per episode row in the shard-local block."""
         if not self._episodes_in_flight_registered:

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/pipeline/sinks/reducer/zarr.py

@@ -15,7 +15,6 @@ from refiner.pipeline.sinks.zarr import (
     _render_store_relpath,
     _zarr_store,
 )
-from refiner.utils import check_required_dependencies
 from refiner.worker.context import get_active_stage_index, get_finalized_workers
 from refiner.worker.lifecycle import sort_finalized_workers
 
@@ -30,7 +29,6 @@ class ZarrReducerSink(FileCleanupReducerSink):
         array_chunk_bytes: int = _DEFAULT_ARRAY_CHUNK_BYTES,
         reduce_to_single_store: bool = True,
     ) -> None:
-        check_required_dependencies("write_zarr", ["zarr"], dist="zarr")
         super().__init__(
             output=output,
             filename_template=(
@@ -43,6 +41,9 @@ class ZarrReducerSink(FileCleanupReducerSink):
         self.array_chunk_bytes = array_chunk_bytes
         self.reduce_to_single_store = reduce_to_single_store
 
+    def _declared_refiner_extras(self) -> tuple[str, ...]:
+        return ("zarr",)
+
     def write_shard_block(self, shard_id: str, block: Block) -> None:
         self._run_cleanup()
         if self.reduce_to_single_store:

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/pipeline/sinks/zarr.py

@@ -42,7 +42,6 @@ class ZarrSink(BaseSink):
         array_chunk_bytes: int = _DEFAULT_ARRAY_CHUNK_BYTES,
         reduce_to_single_store: bool = True,
     ):
-        check_required_dependencies("write_zarr", ["zarr"], dist="zarr")
         if video_frame_batch_size <= 0:
             raise ValueError("video_frame_batch_size must be greater than zero")
         if array_chunk_bytes <= 0:
@@ -70,6 +69,9 @@ class ZarrSink(BaseSink):
         self._stores: dict[str, _ZarrWriteState] = {}
         self._default_arrays: dict[str, str] | None = None
 
+    def _declared_refiner_extras(self) -> tuple[str, ...]:
+        return ("zarr",)
+
     def write_shard_block(self, shard_id: str, block: Block) -> int:
         count = 0
         pending_arrays: dict[str, list[np.ndarray]] = {}
@@ -311,6 +313,7 @@ class ZarrSink(BaseSink):
         store = self._stores.get(relpath)
         if store is not None:
             return store
+        check_required_dependencies("write_zarr", ["zarr"], dist="zarr")
         import zarr
 
         store = _ZarrWriteState(
@@ -535,6 +538,7 @@ def _matching_length(lengths: list[int]) -> int | None:
 
 
 def _zarr_store(output: DataFolder, path: str = "", *, mode: str = "r"):
+    check_required_dependencies("write_zarr", ["zarr"], dist="zarr")
     import zarr
 
     return zarr.storage.FSStore(

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/pipeline/sources/base.py

@@ -47,6 +47,25 @@ class BaseSource(ABC):
         """Optional source metadata for planning/observability."""
         return {}
 
+    def required_refiner_extras(self) -> tuple[str, ...]:
+        """macrodata-refiner extras required by this source."""
+        return tuple(
+            sorted(
+                {
+                    *self._declared_refiner_extras(),
+                    *self._io_refiner_extras(),
+                }
+            )
+        )
+
+    def _declared_refiner_extras(self) -> tuple[str, ...]:
+        """Feature extras declared by this source."""
+        return ()
+
+    def _io_refiner_extras(self) -> tuple[str, ...]:
+        """Storage extras required by this source's normalized IO handles."""
+        return ()
+
 
 __all__ = ["BaseSource"]
 

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/pipeline/sources/readers/base.py

@@ -117,6 +117,9 @@ class BaseReader(BaseSource):
             "file_path_column": self.file_path_column,
         }
 
+    def _io_refiner_extras(self) -> tuple[str, ...]:
+        return self.fileset.required_refiner_extras()
+
     def _with_file_path(
         self, row: dict[str, Any], source_file: DataFile
     ) -> dict[str, Any]:

{macrodata_refiner-0.3.1 → macrodata_refiner-0.3.2}/src/refiner/pipeline/sources/readers/hdf5.py

@@ -119,6 +119,9 @@ class Hdf5Reader(BaseReader):
         )
         return description
 
+    def _declared_refiner_extras(self) -> tuple[str, ...]:
+        return ("hdf5",)
+
     def _validate_column_names(self) -> None:
         for name, path in self.datasets.items():
             if path.startswith("/"):