xarray-binfile 0.1.0__py3-none-any.whl

xarray_binfile/__init__.py

@@ -0,0 +1,22 @@
+"""Read and write raw binary files through xarray."""
+
+from xarray_binfile._version import __version__, __version_tuple__
+from xarray_binfile.read.entrypoint import RawBinaryEntrypoint
+from xarray_binfile.read.file_metadata import ReadSpecs, ReadSpecsGetterProtocol
+
+# The write accessor classes are xarray plugins: importing their module
+# registers ``.binary_engine`` on Dataset and DataArray, which is the only
+# supported way to reach them. The read backend, in contrast, is registered
+# automatically through the ``xarray.backends`` entry point.
+from xarray_binfile.write import accessor as _accessor
+from xarray_binfile.write.file_metadata import WriteSpecs, WriteSpecsGetterProtocol
+
+__all__ = [
+    "RawBinaryEntrypoint",
+    "ReadSpecs",
+    "ReadSpecsGetterProtocol",
+    "WriteSpecs",
+    "WriteSpecsGetterProtocol",
+    "__version__",
+    "__version_tuple__",
+]

xarray_binfile/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

xarray_binfile/read/__init__.py

@@ -0,0 +1,2 @@
+from xarray_binfile.read.entrypoint import RawBinaryEntrypoint
+from xarray_binfile.read.file_metadata import ReadSpecs, ReadSpecsGetterProtocol

xarray_binfile/read/array.py

@@ -0,0 +1,201 @@
+"""
+Defines a backend array for reading binary files in Xarray.
+"""
+
+import numpy as np
+import xarray as xr
+from xarray.backends import BackendArray
+from xarray.core import indexing
+
+from xarray_binfile.read.file_metadata import ReadSpecs
+
+
+def _is_coord_sliced(size: int, slice_spec: slice | int) -> bool:
+    """
+    Check if a coordinate is sliced.
+    Args:
+        size: Size of the coordinate.
+        slice_spec: Slice specification or integer index.
+    Returns:
+        True if the coordinate is sliced, False otherwise.
+    """
+    if not isinstance(slice_spec, slice):
+        return True
+    return any(
+        (
+            (slice_spec.start or 0) != 0,
+            (slice_spec.stop or size) != size,
+            (slice_spec.step or 1) != 1,
+        ),
+    )
+
+
+class BinaryEngineBackendArray(BackendArray):
+    """
+    Backend array for reading binary files in Xarray.
+
+    Attributes:
+        metadata: Metadata describing the binary file.
+        dtype: Data type of the array.
+        shape: Shape of the array.
+    """
+
+    def __init__(self, metadata: ReadSpecs):
+        """
+        Initializes the backend array.
+
+        Args:
+            metadata: Metadata describing the binary file.
+
+        Raises:
+            ValueError: If the file size does not match the size expected
+                from the metadata shape and dtype.
+            FileNotFoundError: If the binary file does not exist.
+        """
+        self.metadata = metadata
+
+        # Attributes required by BackendArray
+        self.dtype = np.dtype(self.metadata.dtype)
+        self.shape = self.metadata.shape
+
+        self._validate_file_size()
+
+    def _validate_file_size(self) -> None:
+        """
+        Ensure the file size matches the metadata shape and dtype.
+
+        Raw binary files carry no metadata, so a size mismatch is the earliest
+        possible signal that the declared shape, dtype, or byte order is wrong.
+        Failing fast here avoids confusing downstream errors or silently
+        misread data.
+
+        Raises:
+            ValueError: If the file size does not match the expected size.
+        """
+        expected_bytes = int(np.prod(self.shape, dtype=np.int64)) * self.dtype.itemsize
+        actual_bytes = self.metadata.filepath.stat().st_size
+        if actual_bytes != expected_bytes:
+            error_message = (
+                f"Size mismatch for {self.metadata.filepath}: file has "
+                f"{actual_bytes} bytes, but the read specs expect "
+                f"{expected_bytes} bytes (shape {self.shape}, dtype {self.dtype}). "
+                "Check the shape, dtype, and byte order declared by the read specs getter."
+            )
+            raise ValueError(error_message)
+
+    def __getitem__(self, key: indexing.ExplicitIndexer) -> np.typing.ArrayLike:
+        """
+        Retrieves data from the array using explicit indexing.
+
+        Args:
+            key: Indexing key specifying the data to retrieve.
+
+        Returns:
+            The retrieved data.
+        """
+        return indexing.explicit_indexing_adapter(
+            key=key,
+            shape=self.metadata.shape,
+            indexing_support=indexing.IndexingSupport.BASIC,
+            raw_indexing_method=self._raw_indexing_method,
+        )
+
+    def _raw_indexing_method(self, key: tuple[slice | int, ...]) -> np.typing.ArrayLike:
+        """
+        Performs raw indexing on the binary file.
+
+        Args:
+            key: Tuple of slices or integers specifying the indices to read.
+
+        Returns:
+            The data read from the binary file.
+        """
+        with open(self.metadata.filepath, "rb") as file:
+            return self._read_binary_at_slices(file, key)
+
+    def _is_sliced(self, key: tuple[slice | int, ...]) -> bool:
+        """
+        Checks if the key is a slice of the original array.
+
+        Args:
+            key: Tuple of slices or integers specifying the indices to check.
+
+        Returns:
+            True if the key is a slice, False otherwise.
+        """
+
+        return any(
+            _is_coord_sliced(s, k)
+            for k, s in zip(key, self.metadata.shape, strict=True)
+        )
+
+    def _wrap_numpy_fromfile(self, file) -> np.typing.NDArray:
+        """
+        Reads the entire binary file into a NumPy array.
+
+        Args:
+            file: The binary file to read.
+
+        Returns:
+            The data read from the file.
+        """
+        return np.fromfile(
+            file, dtype=self.metadata.dtype, count=np.prod(self.metadata.shape)
+        ).reshape(self.metadata.shape)
+
+    def _wrap_numpy_memmap(
+        self, file, key: tuple[slice | int, ...]
+    ) -> np.typing.NDArray:
+        """
+        Reads a portion of the binary file using memory mapping.
+
+        Args:
+            file: The binary file to read.
+            key: Tuple of slices or integers specifying the indices to read.
+
+        Returns:
+            The data read from the file.
+        """
+        memory_map = np.memmap(
+            file,
+            dtype=self.metadata.dtype,
+            mode="r",
+            shape=self.metadata.shape,
+            order="C",
+        )
+        return np.asarray(memory_map[key])  # ensure we actually read the data
+
+    def _read_binary_at_slices(
+        self, file, key: tuple[slice | int, ...]
+    ) -> np.typing.NDArray:
+        """
+        Reads a binary file at specific locations based on the key tuple of slices.
+
+        Args:
+            file: The binary file to read.
+            key: Tuple of slices or integers specifying the indices to read.
+
+        Returns:
+            The array data read from the file at the specified slices.
+        """
+        if self._is_sliced(key):
+            return self._wrap_numpy_memmap(file, key)
+        return self._wrap_numpy_fromfile(file)
+
+    def get_xarray_dataset(self) -> xr.Dataset:
+        """
+        Converts the backend array to an Xarray Dataset.
+
+        Returns:
+            The Xarray Dataset representation of the backend array.
+        """
+        return xr.Dataset(
+            data_vars={
+                self.metadata.name: (
+                    self.metadata.dims,
+                    indexing.LazilyIndexedArray(self),
+                )
+            },
+            coords=self.metadata.coords,
+            attrs=self.metadata.attrs,
+        )

xarray_binfile/read/entrypoint.py

@@ -0,0 +1,74 @@
+"""
+Backend for reading binary files in Xarray.
+
+References:
+    * https://docs.xarray.dev/en/latest/internals/how-to-add-new-backend.html
+    * https://github.com/pydata/xarray/discussions/6406
+"""
+
+import os
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+from xarray import Dataset
+from xarray.backends import BackendEntrypoint
+
+from xarray_binfile.read.array import BinaryEngineBackendArray
+from xarray_binfile.read.file_metadata import ReadSpecsGetterProtocol
+
+
+class RawBinaryEntrypoint(BackendEntrypoint):
+    """
+    Backend entry point for reading binary files in Xarray.
+
+    Attributes:
+        open_dataset_parameters: Parameters accepted by the `open_dataset` method.
+        description: Description of the backend.
+        url: URL to the backend documentation.
+    """
+
+    open_dataset_parameters = ("filename_or_obj", "read_specs_getter", "drop_variables")
+    description = "Read and write raw binary files using the familiar interface from the Xarray library."
+    url = "https://docs.fschuch.com/xarray-binfile/"
+
+    def open_dataset(  # type: ignore[override]
+        self,
+        filename_or_obj: str | os.PathLike[Any],
+        *,
+        read_specs_getter: ReadSpecsGetterProtocol,
+        drop_variables: str | Iterable[str] | None = None,
+    ) -> Dataset:
+        """
+        Open a dataset from a binary file.
+
+        Args:
+            filename_or_obj: Path to the binary file or a file-like object.
+            read_specs_getter: A callable that generates read specifications for the binary file.
+            drop_variables: Variables to drop from the dataset. Defaults to None.
+
+        Returns:
+            The opened Xarray dataset.
+
+        Raises:
+            ValueError: If `filename_or_obj` is not a valid file path.
+            ValueError: If there is an error reading the metadata from the file path.
+        """
+        try:
+            file_path = Path(filename_or_obj)
+        except TypeError as err:
+            error_message = f"Expected a file path or file-like object, but got: {filename_or_obj!r}"
+            raise ValueError(error_message) from err
+        try:
+            file_metadata = read_specs_getter(path=file_path)
+        except Exception as err:
+            error_message = f"Error reading metadata from {file_path}: {err}"
+            raise ValueError(error_message) from err
+        if (
+            isinstance(drop_variables, str) and file_metadata.name == drop_variables
+        ) or (
+            isinstance(drop_variables, Iterable)
+            and file_metadata.name in drop_variables
+        ):
+            return Dataset()
+        return BinaryEngineBackendArray(metadata=file_metadata).get_xarray_dataset()

xarray_binfile/read/file_metadata.py

@@ -0,0 +1,86 @@
+"""
+Defines metadata structures and protocols for reading binary files.
+"""
+
+from dataclasses import dataclass
+from functools import cached_property
+from pathlib import Path
+from typing import Protocol
+
+from xarray_binfile.typing import AttributesLike, CoordsLike, DTypeLike
+
+
+@dataclass(frozen=True)
+class ReadSpecs:
+    """
+    Immutable metadata contract for interpreting one binary file.
+
+    A raw binary file does not carry enough semantic information to be decoded
+    safely by itself. This object provides that missing context so the backend
+    can expose the data as a labeled xarray variable.
+
+    Notes:
+        - Dimension order is inferred from ``coords`` key order.
+        - ``shape`` is inferred from coordinate lengths.
+        - ``dtype`` should be explicit about byte order when portability matters
+          (for example ``"<f4"`` for little-endian float32).
+
+    Attributes:
+        filepath: Path to the binary file.
+        dtype: Data type of the binary file.
+        coords: Coordinates of the data in the binary file.
+        name: Name of the dataset or variable.
+        attrs: Additional attributes for the dataset or variable.
+    """
+
+    filepath: Path
+    dtype: DTypeLike
+    coords: CoordsLike
+    name: str
+    attrs: AttributesLike | None = None
+
+    @cached_property
+    def shape(self) -> tuple[int, ...]:
+        """
+        Gets the shape of the data based on the coordinates.
+
+        Returns:
+            Shape of the data.
+        """
+        return tuple(len(i) for i in self.coords.values())
+
+    @cached_property
+    def dims(self) -> tuple[str, ...]:
+        """
+        Gets the dimension names of the data.
+
+        Returns:
+            Dimension names.
+        """
+        return tuple(self.coords.keys())
+
+
+class ReadSpecsGetterProtocol(Protocol):
+    """
+    Structural protocol for read spec getter implementations.
+
+    Any callable matching ``(path: Path) -> ReadSpecs`` can be used as a read
+    specs getter. No inheritance is required.
+
+    Typical responsibilities:
+        - Parse filename conventions.
+        - Resolve variable identity (for example variable name, step index).
+        - Provide explicit dtype, coordinates, and optional attrs.
+    """
+
+    def __call__(self, path: Path) -> ReadSpecs:
+        """
+        Generate read specifications for a binary file.
+
+        Args:
+            path: Path to the binary file.
+
+        Returns:
+            The metadata required by the binary backend to decode ``path``.
+        """
+        ...

xarray_binfile/tutorial/__init__.py

@@ -0,0 +1,2 @@
+from xarray_binfile.tutorial.dataset_generator import DatasetGenerator
+from xarray_binfile.tutorial.file_metadata import FileSpecsGetter

xarray_binfile/tutorial/dataset_generator.py

@@ -0,0 +1,87 @@
+"""
+Provides a utility for generating xarray Datasets with random data.
+"""
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import xarray as xr
+
+from xarray_binfile.read.file_metadata import ReadSpecs, ReadSpecsGetterProtocol
+
+
+@dataclass(frozen=True)
+class DatasetGenerator:
+    """
+    Utility that generates synthetic datasets from read specs metadata.
+
+    This helper is intended for tutorials and tests where you want deterministic
+    mock data that follows the same metadata conventions expected by the backend.
+
+    Attributes:
+        read_specs_getter: A callable that generates read specifications for binary files.
+        random_generator: A random number generator for creating random data.
+    """
+
+    read_specs_getter: ReadSpecsGetterProtocol
+    random_generator = np.random.Generator(np.random.PCG64(1234))
+
+    def _get_numpy_array(self, metadata: ReadSpecs) -> np.ndarray:
+        """
+        Generate a random NumPy array matching ``metadata`` shape and dtype.
+
+        Args:
+            metadata: Metadata describing the array.
+
+        Returns:
+            A random NumPy array.
+        """
+        return self.random_generator.random(size=metadata.shape, dtype=metadata.dtype)
+
+    def _get_xarray_array(self, metadata: ReadSpecs) -> xr.DataArray:
+        """
+        Generate a random xarray DataArray matching ``metadata``.
+
+        Args:
+            metadata: Metadata describing the array.
+
+        Returns:
+            A random xarray DataArray.
+        """
+        return xr.DataArray(
+            data=self._get_numpy_array(metadata),
+            coords=metadata.coords,
+            attrs=metadata.attrs,
+        )
+
+    def _get_dataset(self, metadata: ReadSpecs) -> xr.Dataset:
+        """
+        Wrap the generated DataArray into a single-variable Dataset.
+
+        Args:
+            metadata: Metadata describing the dataset.
+
+        Returns:
+            A random xarray Dataset.
+        """
+        return self._get_xarray_array(metadata).to_dataset(name=metadata.name)
+
+    def __call__(self, iter_filepath: Iterator[Path]) -> xr.Dataset:
+        """
+        Generate and merge datasets for the provided file paths.
+
+        Each path is converted to ``ReadSpecs`` using ``read_specs_getter``.
+        A synthetic variable is generated for each metadata object, and all
+        variables are merged into one Dataset.
+
+        Args:
+            iter_filepath: An iterator over file paths.
+
+        Returns:
+            A merged xarray Dataset.
+        """
+        metadata = map(self.read_specs_getter, iter_filepath)
+        datasets = map(self._get_dataset, metadata)
+        return xr.merge(datasets, join="outer", compat="no_conflicts")

xarray_binfile/tutorial/file_metadata.py

@@ -0,0 +1,114 @@
+"""
+Defines utilities for generating file metadata for reading and writing binary files.
+"""
+
+import re
+from collections.abc import Iterator
+from dataclasses import dataclass
+from functools import cached_property
+from pathlib import Path
+
+import numpy as np
+from xarray import DataArray
+
+from xarray_binfile.read.file_metadata import ReadSpecs
+from xarray_binfile.typing import ArrayLike, DTypeLike
+from xarray_binfile.write.file_metadata import WriteSpecs
+
+
+@dataclass(frozen=True)
+class FileSpecsGetter:
+    """
+    Reference implementation of both read and write spec getter protocols.
+
+    This helper is intended for tutorials, tests, and simple projects with a
+    filename convention that encodes variable name and step index.
+
+    It is intentionally narrow: the default regex expects time-indexed files
+    (for example ``ux-0001.bin``). For mixed layouts that include static files
+    (for example ``epsi.bin``), implement a custom getter that still follows
+    the same read/write protocol contracts.
+
+    It implements:
+        - ``reader(path) -> ReadSpecs``
+        - ``writer(data_array) -> Iterator[WriteSpecs]``
+
+    Attributes:
+        base_coords: Base coordinates for the data.
+        dtype: Data type of the binary file. Defaults to np.float64.
+        filename_template: Template for generating filenames.
+        filename_regex: Regular expression for parsing filenames.
+    """
+
+    base_coords: dict[str, ArrayLike]
+    dtype: DTypeLike = np.float64
+    filename_template: str = "{name}-{digits:04}.bin"
+    filename_regex: re.Pattern = re.compile(r"(?P<name>\w+)-(?P<digits>\d{4})\.bin")
+
+    def reader(self, path: Path) -> ReadSpecs:
+        """
+        Build ``ReadSpecs`` from a file path.
+
+        The implementation parses ``path.name`` using ``filename_regex`` and
+        appends a single-value ``time`` coordinate based on the extracted step
+        number.
+
+        Args:
+            path: Path to the binary file.
+
+        Returns:
+            ReadSpecs: Metadata for reading the binary file.
+
+        Raises:
+            ValueError: If the filename does not match the expected pattern.
+        """
+        match = self.filename_regex.match(path.name)
+        if not match:
+            error_message = f"Invalid filename: {path.name}"
+            raise ValueError(error_message)
+
+        name, digits = match.groups()
+        time = np.array([int(digits)], dtype=np.int64)
+
+        return ReadSpecs(
+            filepath=path.resolve(),
+            dtype=self.dtype,
+            coords=self.base_coords | {"time": time},
+            name=name,
+        )
+
+    def writer(self, data_array: DataArray) -> Iterator[WriteSpecs]:
+        """
+        Yield ``WriteSpecs`` for one DataArray.
+
+        The default behavior writes one file per time coordinate value using
+        ``filename_template`` and transposes each slice to ``base_coords``
+        dimension order before serialization. Each slice is cast to ``dtype``
+        on write, so files always round-trip with :meth:`reader`.
+
+        Args:
+            data_array: The data array to generate write specifications for.
+
+        Returns:
+            An iterator over write specifications.
+        """
+        for time in data_array.coords["time"]:
+            yield WriteSpecs(
+                filename=self.filename_template.format(
+                    name=data_array.name, digits=int(time)
+                ),
+                sub_array=data_array.sel(time=time).transpose(
+                    *self._base_dims, missing_dims="raise"
+                ),
+                dtype=self.dtype,
+            )
+
+    @cached_property
+    def _base_dims(self) -> tuple[str, ...]:
+        """
+        Return the canonical base dimension order.
+
+        Returns:
+            A tuple of base dimension names.
+        """
+        return tuple(self.base_coords.keys())

xarray_binfile/typing.py

@@ -0,0 +1,12 @@
+"""Type hints shared across the xarray_binfile package."""
+
+import typing
+
+import numpy.typing
+
+# TODO: Type aliases could look better on the docs https://github.com/sphinx-doc/sphinx/issues/10785#issuecomment-1897551241
+
+ArrayLike = numpy.typing.ArrayLike
+DTypeLike = numpy.typing.DTypeLike
+AttributesLike: typing.TypeAlias = typing.Mapping[typing.Any, typing.Any]
+CoordsLike: typing.TypeAlias = typing.Mapping[str, numpy.typing.ArrayLike]

xarray_binfile/write/__init__.py

@@ -0,0 +1,2 @@
+from xarray_binfile.write.accessor import BinaryEngineDataArray, BinaryEngineDataset
+from xarray_binfile.write.file_metadata import WriteSpecs, WriteSpecsGetterProtocol

xarray_binfile/write/accessor.py

@@ -0,0 +1,117 @@
+"""
+Provides accessors for writing xarray Dataset and DataArray objects to binary files.
+"""
+
+import os
+import tempfile
+from pathlib import Path
+
+import xarray as xr
+
+from xarray_binfile.write.file_metadata import WriteSpecsGetterProtocol
+
+
+@xr.register_dataset_accessor("binary_engine")
+class BinaryEngineDataset:
+    """
+    An accessor with extra utilities for xarray.Dataset.
+    """
+
+    def __init__(self, data_set: xr.Dataset):
+        """
+        Initializes the BinaryEngineDataset accessor.
+
+        Args:
+            data_set: The dataset to attach the accessor to.
+        """
+        self._data_set = data_set
+
+    def to_file(
+        self,
+        write_specs_getter: WriteSpecsGetterProtocol,
+        directory: Path | None = None,
+    ) -> None:
+        """
+        Writes the dataset to binary files.
+
+        Every data variable is delegated to
+        :meth:`BinaryEngineDataArray.to_file`, so the same eager, atomic,
+        whole-file write semantics apply: each output file is fully
+        materialized in memory (triggering a Dask compute for lazy variables),
+        written in a single pass, and moved into place only once complete.
+        See that method for guidance on sizing files and on alternatives when
+        streaming or partial writes are needed.
+
+        Args:
+            write_specs_getter: A callable that generates write specifications for the data arrays.
+            directory: The directory where the binary files will be written. Defaults to the current working directory.
+        """
+        for data_array in self._data_set.data_vars.values():
+            data_array.binary_engine.to_file(write_specs_getter, directory)
+
+
+@xr.register_dataarray_accessor("binary_engine")
+class BinaryEngineDataArray:
+    """
+    An accessor with extra utilities for xarray.DataArray.
+    """
+
+    def __init__(self, data_array: xr.DataArray):
+        """
+        Initializes the BinaryEngineDataArray accessor.
+
+        Args:
+            data_array: The data array to attach the accessor to.
+        """
+        self._data_array = data_array
+
+    def to_file(
+        self,
+        write_specs_getter: WriteSpecsGetterProtocol,
+        directory: Path | None = None,
+    ) -> None:
+        """
+        Writes the data array to binary files.
+
+        Writes are eager and whole-file only. For each write specification,
+        the entire ``sub_array`` is loaded into memory (triggering a Dask
+        compute for lazy data) and the target file is written in full, in a
+        single pass. There is no partial, appending, or resuming write mode:
+        re-writing a file always replaces its whole content instead of trying
+        to guess or patch existing bytes, which avoids leaving files in a
+        partially-updated, corrupted state.
+
+        Writes are also atomic per file: the bytes are first serialized into
+        a unique temporary directory created inside the destination directory
+        (so the final move stays on the same filesystem), and each file is
+        moved to its final path with :func:`os.replace` only once it is
+        complete. An interrupted write never leaves a truncated file at the
+        destination, and the temporary directory is removed automatically.
+
+        Plan the write specifications so that every individual output file
+        fits comfortably in memory, for example by splitting the array into
+        one file per time step. If you need streaming, incremental, or
+        partial writes, prefer one of the other file formats supported by
+        xarray, such as NetCDF or Zarr.
+
+        Each file is written with the in-memory dtype and native byte order,
+        unless the write specification sets ``dtype``, in which case the
+        values are cast right before serialization.
+
+        Args:
+            write_specs_getter: A callable that generates write specifications for the data array.
+            directory: The directory where the binary files will be written. Defaults to the current working directory.
+        """
+        _directory = directory or Path.cwd()
+        with tempfile.TemporaryDirectory(
+            dir=_directory, prefix=".binary_engine-"
+        ) as temporary_directory:
+            for details in write_specs_getter(self._data_array):
+                new_type = (
+                    details.dtype
+                    if details.dtype is not None
+                    else details.sub_array.dtype
+                )
+                temporary_file = Path(temporary_directory) / details.filename
+                details.sub_array.values.astype(new_type).tofile(temporary_file)
+                os.replace(temporary_file, _directory / details.filename)

xarray_binfile/write/file_metadata.py

@@ -0,0 +1,62 @@
+"""
+Defines metadata structures and protocols for writing binary files.
+"""
+
+from collections.abc import Iterator
+from typing import NamedTuple, Protocol
+
+import xarray as xr
+
+from xarray_binfile.typing import DTypeLike
+
+
+class WriteSpecs(NamedTuple):
+    """
+    Immutable write instruction for one output binary file.
+
+    Each item defines both the output filename and the exact DataArray slice to
+    serialize into that file.
+
+    Attributes:
+        filename: The name of the binary file.
+        sub_array: The portion of the DataArray to be written.
+        dtype: Optional on-disk data type. When set, the sub-array is cast to
+            this dtype (including byte order, for example ``"<f4"``) right
+            before serialization. When ``None``, the in-memory dtype and
+            native byte order are written as-is, so make sure they match what
+            your read specs getter declares.
+    """
+
+    filename: str
+    sub_array: xr.DataArray
+    dtype: DTypeLike | None = None
+
+
+class WriteSpecsGetterProtocol(Protocol):
+    """
+    Structural protocol for write spec getter implementations.
+
+    Any callable matching
+    ``(data_array: xr.DataArray) -> Iterator[WriteSpecs]`` can be used as a
+    write specs getter. No inheritance is required.
+
+    Typical responsibilities:
+        - Define filename conventions.
+        - Split a DataArray into one or more per-file slices.
+        - Ensure each slice is transposed to the expected on-disk dimension order.
+        - Keep each per-file slice small enough to fit in memory, since every
+          ``sub_array`` is fully materialized (Dask compute) before its file
+          is written in a single pass.
+    """
+
+    def __call__(self, data_array: xr.DataArray) -> Iterator[WriteSpecs]:
+        """
+        Generate write instructions for a DataArray.
+
+        Args:
+            data_array: The data array for which to generate write specifications.
+
+        Returns:
+            An iterator over per-file write instructions.
+        """
+        ...

xarray_binfile-0.1.0.dist-info/METADATA

@@ -0,0 +1,101 @@
+Metadata-Version: 2.4
+Name: xarray-binfile
+Version: 0.1.0
+Summary: Read and write raw binary files using the familiar interface from the Xarray library
+Project-URL: Source, https://github.com/fschuch/xarray-binfile
+Project-URL: Tracker, https://github.com/fschuch/xarray-binfile/issues
+Project-URL: Changelog, https://docs.fschuch.com/xarray-binfile/references/what-is-new.html
+Project-URL: Documentation, https://docs.fschuch.com/xarray-binfile/
+Author-email: fschuch <me@fschuch.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Requires-Python: >=3.10
+Requires-Dist: dask>=2024.0.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: xarray>=2024.1.0
+Provides-Extra: benchmarks
+Requires-Dist: pytest-memray>=1.7.0; (sys_platform != 'win32') and extra == 'benchmarks'
+Provides-Extra: docs
+Requires-Dist: graphviz>=0.20.3; extra == 'docs'
+Requires-Dist: jupyter-book==1.0.4.post1; extra == 'docs'
+Requires-Dist: matplotlib>=3.9.0; extra == 'docs'
+Requires-Dist: sphinx-autobuild==2024.10.3; extra == 'docs'
+Requires-Dist: sphinx-github-changelog==1.7.2; extra == 'docs'
+Requires-Dist: sphinx==7.4.7; extra == 'docs'
+Provides-Extra: tests
+Requires-Dist: coverage[toml]>=7.5.3; extra == 'tests'
+Requires-Dist: hypothesis>=6.130.4; extra == 'tests'
+Requires-Dist: pre-commit>=3.5.0; extra == 'tests'
+Requires-Dist: pytest-benchmark>=5.1.0; extra == 'tests'
+Requires-Dist: pytest-cov>=5.0.0; extra == 'tests'
+Requires-Dist: pytest>=8.2.2; extra == 'tests'
+Provides-Extra: tests-extra
+Requires-Dist: pytest-randomly==4.1.0; extra == 'tests-extra'
+Requires-Dist: pytest-rerunfailures==16.3; extra == 'tests-extra'
+Requires-Dist: pytest-xdist==3.8.0; extra == 'tests-extra'
+Description-Content-Type: text/markdown
+
+# Xarray-binfile
+
+<p align="center">
+<a href="https://github.com/fschuch/xarray-binfile"><img src="https://raw.githubusercontent.com/fschuch/xarray-binfile/refs/heads/main/docs/logo.png" alt="Xarray-binfile logo" width="320"></a>
+</p>
+<p align="center">
+    <em>Custom Xarray file engine to handle raw binary files</em>
+</p>
+
+______________________________________________________________________
+
+- QA:
+  [![CI](https://github.com/fschuch/xarray-binfile/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/fschuch/xarray-binfile/actions/workflows/ci.yaml)
+  [![CodeQL](https://github.com/fschuch/xarray-binfile/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/fschuch/xarray-binfile/actions/workflows/github-code-scanning/codeql)
+  [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/fschuch/xarray-binfile/main.svg)](https://results.pre-commit.ci/latest/github/fschuch/xarray-binfile/main)
+  [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=fschuch_xarray-binfile&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=fschuch_xarray-binfile)
+  [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=fschuch_xarray-binfile&metric=coverage)](https://sonarcloud.io/summary/new_code?id=fschuch_xarray-binfile)
+  [![CodeFactor](https://www.codefactor.io/repository/github/fschuch/xarray-binfile/badge)](https://www.codefactor.io/repository/github/fschuch/xarray-binfile)
+
+- Docs:
+  [![Docs](https://github.com/fschuch/xarray-binfile/actions/workflows/docs.yaml/badge.svg?branch=main)](https://docs.fschuch.com/xarray-binfile)
+
+- Package:
+  [![PyPI - Version](https://img.shields.io/pypi/v/xarray-binfile.svg?logo=pypi&label=PyPI)](https://pypi.org/project/xarray-binfile/)
+  [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/xarray-binfile.svg?logo=python&label=Python)](https://pypi.org/project/xarray-binfile/)
+
+- Meta:
+  [![Wizard Template](https://img.shields.io/badge/Wizard-Template-%23447CAA)](https://github.com/fschuch/wizard-template)
+  [![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
+  [![Hatch project](https://img.shields.io/badge/%F0%9F%A5%9A-Hatch-4051b5.svg)](https://github.com/pypa/hatch)
+  [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+  ![GitHub License](https://img.shields.io/github/license/fschuch/xarray-binfile?color=blue)
+  [![EffVer Versioning](https://img.shields.io/badge/version_scheme-EffVer-0097a7)](https://jacobtomlinson.dev/effver)
+
+______________________________________________________________________
+
+## Overview
+
+Xarray-binfile is a Python package that integrates raw binary files with xarray. You provide the metadata that describes each binary file, and the package exposes those files through xarray datasets and data arrays with optional lazy chunking through Dask.
+
+The package was first designed around binary outputs from the Fortran framework [2DECOMP&FFT](https://github.com/2decomp-fft/2decomp-fft) and the CFD solver [Xcompact3d](https://github.com/xcompact3d/Incompact3d), which is built on top of it. It is not limited to those ecosystems and can be used with any raw binary convention once the corresponding metadata mapping is provided.
+
+It is aimed at workflows where raw binary files are part of an existing convention, such as simulation outputs or NumPy `.tofile()` dumps. Once opened, the data can be indexed, reduced, plotted, and computed with the usual xarray and Dask APIs.
+
+See the documentation for end-to-end examples covering lazy loading, Dask-backed computations, task-graph visualization, writing derived results back to `.bin` files, parallel and larger-than-memory workflows, benchmarking on your own data, batch dtype conversion, probe time-series export to CSV, and quick matplotlib plots.
+
+> [!CAUTION]
+> Raw binary files are machine- and convention-dependent. Differences such as little-endian vs big-endian byte order, word size, record layout, and memory ordering can change how bytes should be interpreted. Always verify dtype and endianness when sharing files across machines or toolchains.
+
+## Copyright and License
+
+© 2025 [Felipe N. Schuch](https://github.com/fschuch).
+All content is under [MIT License](https://github.com/fschuch/xarray-binfile/blob/main/LICENSE).

xarray_binfile-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+xarray_binfile/__init__.py,sha256=w4kVH6JRQGV653J8o43o3wPA93Fp5tYbIHiLC4XzE-s,874
+xarray_binfile/_version.py,sha256=n_5vdJsPNu7wZ57LGuRL585uvll-hiuvZUBWzdG0RQU,520
+xarray_binfile/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xarray_binfile/typing.py,sha256=oSm8QBMfBzXSfdViJixNYfMYEqARJB2N1FjejU_BXM8,441
+xarray_binfile/read/__init__.py,sha256=Rmvab90D3K9OcJSp6jJUulSEcg_0QbKQaUUVWM20wqE,144
+xarray_binfile/read/array.py,sha256=I2YRptyQoZhZRdZwqz9kd4zGqUyaWRMoKCb3hlnRcDg,6300
+xarray_binfile/read/entrypoint.py,sha256=Kz_8nIzmbasJJ25BU26_eD6bGhMn-OYMq9lDMgR_Yyc,2719
+xarray_binfile/read/file_metadata.py,sha256=2mCWxEo0cCSuXiJPCzXlb7052KwcWuXmXS1ycyuaHec,2512
+xarray_binfile/tutorial/__init__.py,sha256=1XMAc-gELAxmiHBDaSX9cwnhRm3OEISvF-_vSviiBzM,137
+xarray_binfile/tutorial/dataset_generator.py,sha256=efDd8XifxA0DMu6mjGq7qcjl86pbwGhrG70fMGM6K6c,2750
+xarray_binfile/tutorial/file_metadata.py,sha256=fIeZicVPuRsSopmLl5dw4THnq-788v4VKTF5bFszkls,3843
+xarray_binfile/write/__init__.py,sha256=zckfWb6NXP8OE7r38Iz2r3crr1_rL0OOM5kpQ-RmD-Q,169
+xarray_binfile/write/accessor.py,sha256=KpM6oEj_i0gtC2qHYEweFy2oNdcLvOiRsRlgzzDWcAY,4665
+xarray_binfile/write/file_metadata.py,sha256=XrB82ZzWBBDIrzS59BguBQbjmHmfhCjmQ1LXhuE_dSA,2022
+xarray_binfile-0.1.0.dist-info/METADATA,sha256=IplVKBOkNgoJi9NgN9XmcGfoCbp9WeTzcWxSsv8bYvg,6685
+xarray_binfile-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+xarray_binfile-0.1.0.dist-info/entry_points.txt,sha256=SPu6_MRtBw578qcwsBcjsaQeW-P-sutg3HfHeKFzA0k,79
+xarray_binfile-0.1.0.dist-info/licenses/LICENSE,sha256=hsXQT5P2j7Kz8HdmUWgf7uiJ1FCY_EI-3Gxht-DVITU,1073
+xarray_binfile-0.1.0.dist-info/RECORD,,

xarray_binfile-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

xarray_binfile-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[xarray.backends]
+binfile = xarray_binfile.read.entrypoint:RawBinaryEntrypoint

xarray_binfile-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Felipe N. Schuch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.