xarray-binfile 0.1.0b0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_binfile/__init__.py +3 -0
- xarray_binfile/_version.py +21 -0
- xarray_binfile/py.typed +0 -0
- xarray_binfile/read/__init__.py +2 -0
- xarray_binfile/read/array.py +165 -0
- xarray_binfile/read/entrypoint.py +74 -0
- xarray_binfile/read/file_metadata.py +68 -0
- xarray_binfile/tutorial/__init__.py +2 -0
- xarray_binfile/tutorial/dataset_generator.py +80 -0
- xarray_binfile/tutorial/file_metadata.py +92 -0
- xarray_binfile/typing.py +12 -0
- xarray_binfile/write/__init__.py +2 -0
- xarray_binfile/write/accessor.py +72 -0
- xarray_binfile/write/file_metadata.py +46 -0
- xarray_binfile-0.1.0b0.dist-info/METADATA +88 -0
- xarray_binfile-0.1.0b0.dist-info/RECORD +19 -0
- xarray_binfile-0.1.0b0.dist-info/WHEEL +4 -0
- xarray_binfile-0.1.0b0.dist-info/entry_points.txt +2 -0
- xarray_binfile-0.1.0b0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
|
|
5
|
+
|
|
6
|
+
TYPE_CHECKING = False
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from typing import Tuple
|
|
9
|
+
from typing import Union
|
|
10
|
+
|
|
11
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
12
|
+
else:
|
|
13
|
+
VERSION_TUPLE = object
|
|
14
|
+
|
|
15
|
+
version: str
|
|
16
|
+
__version__: str
|
|
17
|
+
__version_tuple__: VERSION_TUPLE
|
|
18
|
+
version_tuple: VERSION_TUPLE
|
|
19
|
+
|
|
20
|
+
__version__ = version = '0.1.0b0'
|
|
21
|
+
__version_tuple__ = version_tuple = (0, 1, 0, 'b0')
|
xarray_binfile/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Defines a backend array for reading binary files in Xarray.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import xarray as xr
|
|
7
|
+
from xarray.backends import BackendArray
|
|
8
|
+
from xarray.core import indexing
|
|
9
|
+
|
|
10
|
+
from xarray_binfile.read.file_metadata import ReadSpecs
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _is_coord_sliced(size: int, slice_spec: slice) -> bool:
|
|
14
|
+
"""
|
|
15
|
+
Check if a coordinate is sliced.
|
|
16
|
+
Args:
|
|
17
|
+
size: Size of the coordinate.
|
|
18
|
+
slice_spec: Slice specification.
|
|
19
|
+
Returns:
|
|
20
|
+
True if the coordinate is sliced, False otherwise.
|
|
21
|
+
"""
|
|
22
|
+
return any(
|
|
23
|
+
(
|
|
24
|
+
(slice_spec.start or 0) != 0,
|
|
25
|
+
(slice_spec.stop or size) != size,
|
|
26
|
+
(slice_spec.step or 1) != 1,
|
|
27
|
+
),
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class BinaryEngineBackendArray(BackendArray):
|
|
32
|
+
"""
|
|
33
|
+
Backend array for reading binary files in Xarray.
|
|
34
|
+
|
|
35
|
+
Attributes:
|
|
36
|
+
metadata: Metadata describing the binary file.
|
|
37
|
+
dtype: Data type of the array.
|
|
38
|
+
shape: Shape of the array.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self, metadata: ReadSpecs):
|
|
42
|
+
"""
|
|
43
|
+
Initializes the backend array.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
metadata: Metadata describing the binary file.
|
|
47
|
+
"""
|
|
48
|
+
self.metadata = metadata
|
|
49
|
+
|
|
50
|
+
# Attributes required by BackendArray
|
|
51
|
+
self.dtype = self.metadata.dtype
|
|
52
|
+
self.shape = self.metadata.shape
|
|
53
|
+
|
|
54
|
+
def __getitem__(self, key: indexing.ExplicitIndexer) -> np.typing.ArrayLike:
|
|
55
|
+
"""
|
|
56
|
+
Retrieves data from the array using explicit indexing.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
key: Indexing key specifying the data to retrieve.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
The retrieved data.
|
|
63
|
+
"""
|
|
64
|
+
return indexing.explicit_indexing_adapter(
|
|
65
|
+
key=key,
|
|
66
|
+
shape=self.metadata.shape,
|
|
67
|
+
indexing_support=indexing.IndexingSupport.VECTORIZED,
|
|
68
|
+
raw_indexing_method=self._raw_indexing_method,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
def _raw_indexing_method(self, key: tuple[slice, ...]) -> np.typing.ArrayLike:
|
|
72
|
+
"""
|
|
73
|
+
Performs raw indexing on the binary file.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
key: Tuple of slices specifying the indices to read.
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
The data read from the binary file.
|
|
80
|
+
"""
|
|
81
|
+
with open(self.metadata.filepath, "rb") as file:
|
|
82
|
+
return self._read_binary_at_slices(file, key)
|
|
83
|
+
|
|
84
|
+
def _is_sliced(self, key: tuple[slice, ...]) -> bool:
|
|
85
|
+
"""
|
|
86
|
+
Checks if the key is a slice of the original array.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
key: Tuple of slices specifying the indices to check.
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
True if the key is a slice, False otherwise.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
return any(
|
|
96
|
+
_is_coord_sliced(s, k)
|
|
97
|
+
for k, s in zip(key, self.metadata.shape, strict=True)
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
def _wrap_numpy_fromfile(self, file) -> np.typing.NDArray:
|
|
101
|
+
"""
|
|
102
|
+
Reads the entire binary file into a NumPy array.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
file: The binary file to read.
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
The data read from the file.
|
|
109
|
+
"""
|
|
110
|
+
return np.fromfile(
|
|
111
|
+
file, dtype=self.metadata.dtype, count=np.prod(self.metadata.shape)
|
|
112
|
+
).reshape(self.metadata.shape)
|
|
113
|
+
|
|
114
|
+
def _wrap_numpy_memmap(self, file, key: tuple[slice, ...]) -> np.typing.NDArray:
|
|
115
|
+
"""
|
|
116
|
+
Reads a portion of the binary file using memory mapping.
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
file: The binary file to read.
|
|
120
|
+
key: Tuple of slices specifying the indices to read.
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
The data read from the file.
|
|
124
|
+
"""
|
|
125
|
+
memory_map = np.memmap(
|
|
126
|
+
file,
|
|
127
|
+
dtype=self.metadata.dtype,
|
|
128
|
+
mode="r",
|
|
129
|
+
shape=self.metadata.shape,
|
|
130
|
+
order="C",
|
|
131
|
+
)
|
|
132
|
+
return np.asarray(memory_map[key]) # ensure we actually read the data
|
|
133
|
+
|
|
134
|
+
def _read_binary_at_slices(self, file, key: tuple[slice, ...]) -> np.typing.NDArray:
|
|
135
|
+
"""
|
|
136
|
+
Reads a binary file at specific locations based on the key tuple of slices.
|
|
137
|
+
|
|
138
|
+
Args:
|
|
139
|
+
file: The binary file to read.
|
|
140
|
+
key: Tuple of slices specifying the indices to read.
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
The array data read from the file at the specified slices.
|
|
144
|
+
"""
|
|
145
|
+
if self._is_sliced(key):
|
|
146
|
+
return self._wrap_numpy_memmap(file, key)
|
|
147
|
+
return self._wrap_numpy_fromfile(file)
|
|
148
|
+
|
|
149
|
+
def get_xarray_dataset(self) -> xr.Dataset:
|
|
150
|
+
"""
|
|
151
|
+
Converts the backend array to an Xarray Dataset.
|
|
152
|
+
|
|
153
|
+
Returns:
|
|
154
|
+
The Xarray Dataset representation of the backend array.
|
|
155
|
+
"""
|
|
156
|
+
return xr.Dataset(
|
|
157
|
+
data_vars={
|
|
158
|
+
self.metadata.name: (
|
|
159
|
+
self.metadata.dims,
|
|
160
|
+
indexing.LazilyIndexedArray(self),
|
|
161
|
+
)
|
|
162
|
+
},
|
|
163
|
+
coords=self.metadata.coords,
|
|
164
|
+
attrs=self.metadata.attrs,
|
|
165
|
+
)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Backend for reading binary files in Xarray.
|
|
3
|
+
|
|
4
|
+
References:
|
|
5
|
+
* https://docs.xarray.dev/en/latest/internals/how-to-add-new-backend.html
|
|
6
|
+
* https://github.com/pydata/xarray/discussions/6406
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from collections.abc import Iterable
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from xarray import Dataset
|
|
15
|
+
from xarray.backends import BackendEntrypoint
|
|
16
|
+
|
|
17
|
+
from xarray_binfile.read.array import BinaryEngineBackendArray
|
|
18
|
+
from xarray_binfile.read.file_metadata import ReadSpecsGetterProtocol
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class RawBinaryEntrypoint(BackendEntrypoint):
|
|
22
|
+
"""
|
|
23
|
+
Backend entry point for reading binary files in Xarray.
|
|
24
|
+
|
|
25
|
+
Attributes:
|
|
26
|
+
open_dataset_parameters: Parameters accepted by the `open_dataset` method.
|
|
27
|
+
description: Description of the backend.
|
|
28
|
+
url: URL to the backend documentation.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
open_dataset_parameters = ("filename_or_obj", "drop_variables, read_specs_getter")
|
|
32
|
+
description = "Read and write raw binary files using the familiar interface from the Xarray library."
|
|
33
|
+
url = "https://docs.fschuch.com/xarray-binfile/"
|
|
34
|
+
|
|
35
|
+
def open_dataset( # type: ignore[override]
|
|
36
|
+
self,
|
|
37
|
+
filename_or_obj: str | os.PathLike[Any],
|
|
38
|
+
*,
|
|
39
|
+
read_specs_getter: ReadSpecsGetterProtocol,
|
|
40
|
+
drop_variables: str | Iterable[str] | None = None,
|
|
41
|
+
) -> Dataset:
|
|
42
|
+
"""
|
|
43
|
+
Open a dataset from a binary file.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
filename_or_obj: Path to the binary file or a file-like object.
|
|
47
|
+
read_specs_getter: A callable that generates read specifications for the binary file.
|
|
48
|
+
drop_variables: Variables to drop from the dataset. Defaults to None.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
The opened Xarray dataset.
|
|
52
|
+
|
|
53
|
+
Raises:
|
|
54
|
+
ValueError: If `filename_or_obj` is not a valid file path.
|
|
55
|
+
ValueError: If there is an error reading the metadata from the file path.
|
|
56
|
+
"""
|
|
57
|
+
try:
|
|
58
|
+
file_path = Path(filename_or_obj)
|
|
59
|
+
except TypeError as err:
|
|
60
|
+
error_message = f"Expected a file path or file-like object, but got: {filename_or_obj!r}"
|
|
61
|
+
raise ValueError(error_message) from err
|
|
62
|
+
try:
|
|
63
|
+
file_metadata = read_specs_getter(path=file_path)
|
|
64
|
+
except Exception as err:
|
|
65
|
+
error_message = f"Error reading metadata from {file_path}: {err}"
|
|
66
|
+
raise ValueError(error_message) from err
|
|
67
|
+
if (
|
|
68
|
+
isinstance(drop_variables, str) and file_metadata.name == drop_variables
|
|
69
|
+
) or (
|
|
70
|
+
isinstance(drop_variables, Iterable)
|
|
71
|
+
and file_metadata.name in drop_variables
|
|
72
|
+
):
|
|
73
|
+
return Dataset()
|
|
74
|
+
return BinaryEngineBackendArray(metadata=file_metadata).get_xarray_dataset()
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Defines metadata structures and protocols for reading binary files.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from functools import cached_property
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Protocol
|
|
9
|
+
|
|
10
|
+
from xarray_binfile.typing import AttributesLike, CoordsLike, DTypeLike
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class ReadSpecs:
|
|
15
|
+
"""
|
|
16
|
+
Metadata for reading a binary file.
|
|
17
|
+
|
|
18
|
+
Attributes:
|
|
19
|
+
filepath: Path to the binary file.
|
|
20
|
+
dtype: Data type of the binary file.
|
|
21
|
+
coords: Coordinates of the data in the binary file.
|
|
22
|
+
name: Name of the dataset or variable.
|
|
23
|
+
attrs: Additional attributes for the dataset or variable.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
filepath: Path
|
|
27
|
+
dtype: DTypeLike
|
|
28
|
+
coords: CoordsLike
|
|
29
|
+
name: str
|
|
30
|
+
attrs: AttributesLike | None = None
|
|
31
|
+
|
|
32
|
+
@cached_property
|
|
33
|
+
def shape(self) -> tuple[int, ...]:
|
|
34
|
+
"""
|
|
35
|
+
Gets the shape of the data based on the coordinates.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
Shape of the data.
|
|
39
|
+
"""
|
|
40
|
+
return tuple(len(i) for i in self.coords.values())
|
|
41
|
+
|
|
42
|
+
@cached_property
|
|
43
|
+
def dims(self) -> tuple[str, ...]:
|
|
44
|
+
"""
|
|
45
|
+
Gets the dimension names of the data.
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Dimension names.
|
|
49
|
+
"""
|
|
50
|
+
return tuple(self.coords.keys())
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class ReadSpecsGetterProtocol(Protocol):
|
|
54
|
+
"""
|
|
55
|
+
Protocol for generating read specifications for a binary file.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __call__(self, path: Path) -> ReadSpecs:
|
|
59
|
+
"""
|
|
60
|
+
Generates read specifications for a binary file.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
path: Path to the binary file.
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
Metadata for reading the binary file.
|
|
67
|
+
"""
|
|
68
|
+
...
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Provides a utility for generating xarray Datasets with random data.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import xarray as xr
|
|
11
|
+
|
|
12
|
+
from xarray_binfile.read.file_metadata import ReadSpecs, ReadSpecsGetterProtocol
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class DatasetGenerator:
|
|
17
|
+
"""
|
|
18
|
+
Generates xarray Datasets with random data based on metadata.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
read_specs_getter: A callable that generates read specifications for binary files.
|
|
22
|
+
random_generator: A random number generator for creating random data.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
read_specs_getter: ReadSpecsGetterProtocol
|
|
26
|
+
random_generator = np.random.Generator(np.random.PCG64(1234))
|
|
27
|
+
|
|
28
|
+
def _get_numpy_array(self, metadata: ReadSpecs) -> np.ndarray:
|
|
29
|
+
"""
|
|
30
|
+
Generates a random NumPy array based on metadata.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
metadata: Metadata describing the array.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
A random NumPy array.
|
|
37
|
+
"""
|
|
38
|
+
return self.random_generator.random(size=metadata.shape, dtype=metadata.dtype)
|
|
39
|
+
|
|
40
|
+
def _get_xarray_array(self, metadata: ReadSpecs) -> xr.DataArray:
|
|
41
|
+
"""
|
|
42
|
+
Generates a random xarray DataArray based on metadata.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
metadata: Metadata describing the array.
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
A random xarray DataArray.
|
|
49
|
+
"""
|
|
50
|
+
return xr.DataArray(
|
|
51
|
+
data=self._get_numpy_array(metadata),
|
|
52
|
+
coords=metadata.coords,
|
|
53
|
+
attrs=metadata.attrs,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
def _get_dataset(self, metadata: ReadSpecs) -> xr.Dataset:
|
|
57
|
+
"""
|
|
58
|
+
Generates a random xarray Dataset based on metadata.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
metadata: Metadata describing the dataset.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
A random xarray Dataset.
|
|
65
|
+
"""
|
|
66
|
+
return self._get_xarray_array(metadata).to_dataset(name=metadata.name)
|
|
67
|
+
|
|
68
|
+
def __call__(self, iter_filepath: Iterator[Path]) -> xr.Dataset:
|
|
69
|
+
"""
|
|
70
|
+
Generates a merged xarray Dataset from multiple file paths.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
iter_filepath: An iterator over file paths.
|
|
74
|
+
|
|
75
|
+
Returns:
|
|
76
|
+
A merged xarray Dataset.
|
|
77
|
+
"""
|
|
78
|
+
metadata = map(self.read_specs_getter, iter_filepath)
|
|
79
|
+
datasets = map(self._get_dataset, metadata)
|
|
80
|
+
return xr.merge(datasets)
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Defines utilities for generating file metadata for reading and writing binary files.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from functools import cached_property
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
from xarray import DataArray
|
|
13
|
+
|
|
14
|
+
from xarray_binfile.read.file_metadata import ReadSpecs
|
|
15
|
+
from xarray_binfile.typing import ArrayLike, DTypeLike
|
|
16
|
+
from xarray_binfile.write.file_metadata import WriteSpecs
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class FileSpecsGetter:
|
|
21
|
+
"""
|
|
22
|
+
Generates file metadata for reading and writing binary files.
|
|
23
|
+
|
|
24
|
+
Attributes:
|
|
25
|
+
base_coords: Base coordinates for the data.
|
|
26
|
+
dtype: Data type of the binary file. Defaults to np.float64.
|
|
27
|
+
filename_template: Template for generating filenames.
|
|
28
|
+
filename_regex: Regular expression for parsing filenames.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
base_coords: dict[str, ArrayLike]
|
|
32
|
+
dtype: DTypeLike = np.float64
|
|
33
|
+
filename_template: str = "{name}-{digits:04}.bin"
|
|
34
|
+
filename_regex: re.Pattern = re.compile(r"(?P<name>\w+)-(?P<digits>\d{4})\.bin")
|
|
35
|
+
|
|
36
|
+
def reader(self, path: Path) -> ReadSpecs:
|
|
37
|
+
"""
|
|
38
|
+
Generate read specifications for a binary file.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
path: Path to the binary file.
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
ReadSpecs: Metadata for reading the binary file.
|
|
45
|
+
|
|
46
|
+
Raises:
|
|
47
|
+
ValueError: If the filename does not match the expected pattern.
|
|
48
|
+
"""
|
|
49
|
+
match = self.filename_regex.match(path.name)
|
|
50
|
+
if not match:
|
|
51
|
+
error_message = f"Invalid filename: {path.name}"
|
|
52
|
+
raise ValueError(error_message)
|
|
53
|
+
|
|
54
|
+
name, digits = match.groups()
|
|
55
|
+
time = np.array([int(digits)], dtype=np.int64)
|
|
56
|
+
|
|
57
|
+
return ReadSpecs(
|
|
58
|
+
filepath=path.resolve(),
|
|
59
|
+
dtype=self.dtype,
|
|
60
|
+
coords=self.base_coords | {"time": time},
|
|
61
|
+
name=name,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def writer(self, data_array: DataArray) -> Iterator[WriteSpecs]:
|
|
65
|
+
"""
|
|
66
|
+
Generate write specifications for a DataArray.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
data_array: The data array to generate write specifications for.
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
An iterator over write specifications.
|
|
73
|
+
"""
|
|
74
|
+
for time in data_array.coords["time"]:
|
|
75
|
+
yield WriteSpecs(
|
|
76
|
+
filename=self.filename_template.format(
|
|
77
|
+
name=data_array.name, digits=int(time)
|
|
78
|
+
),
|
|
79
|
+
sub_array=data_array.sel(time=time).transpose(
|
|
80
|
+
*self._base_dims, missing_dims="raise"
|
|
81
|
+
),
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
@cached_property
|
|
85
|
+
def _base_dims(self) -> tuple[str, ...]:
|
|
86
|
+
"""
|
|
87
|
+
Get the base dimensions from the coordinates.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
A tuple of base dimension names.
|
|
91
|
+
"""
|
|
92
|
+
return tuple(self.base_coords.keys())
|
xarray_binfile/typing.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Module for type hints used in the binary_engine_xarray package."""
|
|
2
|
+
|
|
3
|
+
import typing
|
|
4
|
+
|
|
5
|
+
import numpy.typing
|
|
6
|
+
|
|
7
|
+
# TODO: Type aliases could look better on the docs https://github.com/sphinx-doc/sphinx/issues/10785#issuecomment-1897551241
|
|
8
|
+
|
|
9
|
+
ArrayLike = numpy.typing.ArrayLike
|
|
10
|
+
DTypeLike = numpy.typing.DTypeLike
|
|
11
|
+
AttributesLike: typing.TypeAlias = typing.Mapping[typing.Any, typing.Any]
|
|
12
|
+
CoordsLike: typing.TypeAlias = typing.Mapping[str, numpy.typing.ArrayLike]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Provides accessors for writing xarray Dataset and DataArray objects to binary files.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import xarray as xr
|
|
8
|
+
|
|
9
|
+
from xarray_binfile.write.file_metadata import WriteSpecsGetterProtocol
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@xr.register_dataset_accessor("binary_engine")
|
|
13
|
+
class BinaryEngineDataset:
|
|
14
|
+
"""
|
|
15
|
+
An accessor with extra utilities for xarray.Dataset.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self, data_set: xr.Dataset):
|
|
19
|
+
"""
|
|
20
|
+
Initializes the BinaryEngineDataset accessor.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
data_set: The dataset to attach the accessor to.
|
|
24
|
+
"""
|
|
25
|
+
self._data_set = data_set
|
|
26
|
+
|
|
27
|
+
def to_file(
|
|
28
|
+
self,
|
|
29
|
+
write_specs_getter: WriteSpecsGetterProtocol,
|
|
30
|
+
directory: Path | None = None,
|
|
31
|
+
) -> None:
|
|
32
|
+
"""
|
|
33
|
+
Writes the dataset to binary files.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
write_specs_getter: A callable that generates write specifications for the data arrays.
|
|
37
|
+
directory: The directory where the binary files will be written. Defaults to the current working directory.
|
|
38
|
+
"""
|
|
39
|
+
for data_array in self._data_set.data_vars.values():
|
|
40
|
+
data_array.binary_engine.to_file(write_specs_getter, directory)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@xr.register_dataarray_accessor("binary_engine")
|
|
44
|
+
class BinaryEngineDataArray:
|
|
45
|
+
"""
|
|
46
|
+
An accessor with extra utilities for xarray.DataArray.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, data_array: xr.DataArray):
|
|
50
|
+
"""
|
|
51
|
+
Initializes the BinaryEngineDataArray accessor.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
data_array: The data array to attach the accessor to.
|
|
55
|
+
"""
|
|
56
|
+
self._data_array = data_array
|
|
57
|
+
|
|
58
|
+
def to_file(
|
|
59
|
+
self,
|
|
60
|
+
write_specs_getter: WriteSpecsGetterProtocol,
|
|
61
|
+
directory: Path | None = None,
|
|
62
|
+
) -> None:
|
|
63
|
+
"""
|
|
64
|
+
Writes the data array to binary files.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
write_specs_getter: A callable that generates write specifications for the data array.
|
|
68
|
+
directory: The directory where the binary files will be written. Defaults to the current working directory.
|
|
69
|
+
"""
|
|
70
|
+
_directory = directory or Path.cwd()
|
|
71
|
+
for details in write_specs_getter(self._data_array):
|
|
72
|
+
details.sub_array.to_numpy().tofile(_directory / details.filename)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Defines metadata structures and protocols for writing binary files.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
from typing import NamedTuple, Protocol
|
|
7
|
+
|
|
8
|
+
import xarray as xr
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class WriteSpecs(NamedTuple):
|
|
12
|
+
"""
|
|
13
|
+
Metadata for writing a portion of a DataArray to a binary file.
|
|
14
|
+
|
|
15
|
+
Attributes
|
|
16
|
+
----------
|
|
17
|
+
filename : str
|
|
18
|
+
The name of the binary file.
|
|
19
|
+
sub_array : xr.DataArray
|
|
20
|
+
The portion of the DataArray to be written.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
filename: str
|
|
24
|
+
sub_array: xr.DataArray
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class WriteSpecsGetterProtocol(Protocol):
|
|
28
|
+
"""
|
|
29
|
+
Protocol for generating write specifications for a DataArray.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __call__(self, data_array: xr.DataArray) -> Iterator[WriteSpecs]:
|
|
33
|
+
"""
|
|
34
|
+
Generate write specifications for a DataArray.
|
|
35
|
+
|
|
36
|
+
Parameters
|
|
37
|
+
----------
|
|
38
|
+
data_array : xr.DataArray
|
|
39
|
+
The data array for which to generate write specifications.
|
|
40
|
+
|
|
41
|
+
Returns
|
|
42
|
+
-------
|
|
43
|
+
Iterator[WriteSpecs]
|
|
44
|
+
An iterator over the write specifications.
|
|
45
|
+
"""
|
|
46
|
+
...
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xarray-binfile
|
|
3
|
+
Version: 0.1.0b0
|
|
4
|
+
Summary: Read and write raw binary files using the familiar interface from the Xarray library
|
|
5
|
+
Project-URL: Source, https://github.com/fschuch/xarray-binfile
|
|
6
|
+
Project-URL: Tracker, https://github.com/fschuch/xarray-binfile/issues
|
|
7
|
+
Project-URL: Changelog, https://fschuch.github.io/xarray-binfile/references/what-is-new.html
|
|
8
|
+
Project-URL: Documentation, https://fschuch.github.io/xarray-binfile/
|
|
9
|
+
Author-email: fschuch <me@fschuch.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: dask>=2024.0.0
|
|
24
|
+
Requires-Dist: xarray>=2024.1.0
|
|
25
|
+
Provides-Extra: docs
|
|
26
|
+
Requires-Dist: docutils==0.21.2; extra == 'docs'
|
|
27
|
+
Requires-Dist: jupyter-book==1.0.4.post1; extra == 'docs'
|
|
28
|
+
Requires-Dist: sphinx-autobuild==2024.10.3; extra == 'docs'
|
|
29
|
+
Requires-Dist: sphinx-github-changelog==1.7.1; extra == 'docs'
|
|
30
|
+
Requires-Dist: sphinx==7.4.7; extra == 'docs'
|
|
31
|
+
Provides-Extra: tests
|
|
32
|
+
Requires-Dist: coverage[toml]>=7.5.3; extra == 'tests'
|
|
33
|
+
Requires-Dist: hypothesis>=6.130.4; extra == 'tests'
|
|
34
|
+
Requires-Dist: pre-commit>=3.5.0; extra == 'tests'
|
|
35
|
+
Requires-Dist: pytest-benchmark>=5.1.0; extra == 'tests'
|
|
36
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == 'tests'
|
|
37
|
+
Requires-Dist: pytest-memray>=1.7.0; (sys_platform != 'win32') and extra == 'tests'
|
|
38
|
+
Requires-Dist: pytest>=8.2.2; extra == 'tests'
|
|
39
|
+
Provides-Extra: tests-extra
|
|
40
|
+
Requires-Dist: pytest-randomly==3.16.0; extra == 'tests-extra'
|
|
41
|
+
Requires-Dist: pytest-rerunfailures==15.1; extra == 'tests-extra'
|
|
42
|
+
Requires-Dist: pytest-xdist==3.8.0; extra == 'tests-extra'
|
|
43
|
+
Description-Content-Type: text/markdown
|
|
44
|
+
|
|
45
|
+
# Xarray-binfile
|
|
46
|
+
|
|
47
|
+
<p align="center">
|
|
48
|
+
<a href="https://github.com/fschuch/xarray-binfile"><img src="https://raw.githubusercontent.com/fschuch/xarray-binfile/refs/heads/main/docs/logo.png" alt="Wizard template logo" width="320"></a>
|
|
49
|
+
</p>
|
|
50
|
+
<p align="center">
|
|
51
|
+
<em>Custom Xarray file engine to handle raw binary files</em>
|
|
52
|
+
</p>
|
|
53
|
+
|
|
54
|
+
______________________________________________________________________
|
|
55
|
+
|
|
56
|
+
- QA:
|
|
57
|
+
[](https://github.com/fschuch/xarray-binfile/actions/workflows/ci.yaml)
|
|
58
|
+
[](https://github.com/fschuch/xarray-binfile/actions/workflows/github-code-scanning/codeql)
|
|
59
|
+
[](https://results.pre-commit.ci/latest/github/fschuch/xarray-binfile/main)
|
|
60
|
+
[](https://sonarcloud.io/summary/new_code?id=fschuch_xarray-binfile)
|
|
61
|
+
[](https://sonarcloud.io/summary/new_code?id=fschuch_xarray-binfile)
|
|
62
|
+
[](https://www.codefactor.io/repository/github/fschuch/xarray-binfile)
|
|
63
|
+
|
|
64
|
+
<!-- - Docs:
|
|
65
|
+
[](https://docs.fschuch.com/xarray-binfile) -->
|
|
66
|
+
|
|
67
|
+
- Package:
|
|
68
|
+
[](https://pypi.org/project/xarray-binfile/)
|
|
69
|
+
[](https://pypi.org/project/xarray-binfile/)
|
|
70
|
+
|
|
71
|
+
- Meta:
|
|
72
|
+
[](https://github.com/fschuch/wizard-template)
|
|
73
|
+
[](https://mypy-lang.org/)
|
|
74
|
+
[](https://github.com/pypa/hatch)
|
|
75
|
+
[](https://github.com/astral-sh/ruff)
|
|
76
|
+

|
|
77
|
+
[](https://jacobtomlinson.dev/effver)
|
|
78
|
+
|
|
79
|
+
______________________________________________________________________
|
|
80
|
+
|
|
81
|
+
## Overview
|
|
82
|
+
|
|
83
|
+
Xarray-binfile is a Python package designed to extend the capabilities of Xarray, a powerful library for working with labeled multi-dimensional arrays. This package provides a custom file engine specifically for reading and writing raw binary files, enabling users to leverage Xarray's data structures while working with binary data formats.
|
|
84
|
+
|
|
85
|
+
## Copyright and License
|
|
86
|
+
|
|
87
|
+
© 2025 [Felipe N. Schuch](https://github.com/fschuch).
|
|
88
|
+
All content is under [MIT License](https://github.com/fschuch/xarray-binfile/blob/main/LICENSE).
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
xarray_binfile/__init__.py,sha256=8e2AniDmmZDP4ACal8BLZUQAZH3bdYigoyuwoMwU3-A,127
|
|
2
|
+
xarray_binfile/_version.py,sha256=3IdK5-9L8GfdJbn5XMY6rBIueY27XuOo8ogyssGipNE,519
|
|
3
|
+
xarray_binfile/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
xarray_binfile/typing.py,sha256=OULN2kY8eJtkJRyT_D3HG_MH8sZ9JYTYpAdCJJ7s0ww,452
|
|
5
|
+
xarray_binfile/read/__init__.py,sha256=Rmvab90D3K9OcJSp6jJUulSEcg_0QbKQaUUVWM20wqE,144
|
|
6
|
+
xarray_binfile/read/array.py,sha256=kzEnu8Vq_YZ2Fv1GoHmTnt_vSqt-oehN7mzvqTKUYWo,4786
|
|
7
|
+
xarray_binfile/read/entrypoint.py,sha256=KQ8fBr47hu4K-knTpLwd0iwurWvRM_YttoT2hzxbvKo,2717
|
|
8
|
+
xarray_binfile/read/file_metadata.py,sha256=hs3pZT3f8sAs0rxxTh-bCHKPdZQS1aLTEYBDlaquXPc,1639
|
|
9
|
+
xarray_binfile/tutorial/__init__.py,sha256=1XMAc-gELAxmiHBDaSX9cwnhRm3OEISvF-_vSviiBzM,137
|
|
10
|
+
xarray_binfile/tutorial/dataset_generator.py,sha256=rTCi6_Kx80NgCYjyILgXe1pNJoTH7QbGhNVjMt7MLOM,2315
|
|
11
|
+
xarray_binfile/tutorial/file_metadata.py,sha256=t6RpDsIdUdWg6-zk9EsnjYuinLyro3xR4oW2IFId0s8,2810
|
|
12
|
+
xarray_binfile/write/__init__.py,sha256=zckfWb6NXP8OE7r38Iz2r3crr1_rL0OOM5kpQ-RmD-Q,169
|
|
13
|
+
xarray_binfile/write/accessor.py,sha256=FTeXtpVB1698ga1uIm3ifeA3DwVbN3yQezxDipI5f5k,2212
|
|
14
|
+
xarray_binfile/write/file_metadata.py,sha256=800GtnSDyUo_qEIuYPh5GHPaTtzVFhtnCJG3tck6YIg,1059
|
|
15
|
+
xarray_binfile-0.1.0b0.dist-info/METADATA,sha256=MzDa1apSAJn_e_jQDfJWUNvzmXnpSBghPuNJMnIRqBU,5350
|
|
16
|
+
xarray_binfile-0.1.0b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
17
|
+
xarray_binfile-0.1.0b0.dist-info/entry_points.txt,sha256=SPu6_MRtBw578qcwsBcjsaQeW-P-sutg3HfHeKFzA0k,79
|
|
18
|
+
xarray_binfile-0.1.0b0.dist-info/licenses/LICENSE,sha256=hsXQT5P2j7Kz8HdmUWgf7uiJ1FCY_EI-3Gxht-DVITU,1073
|
|
19
|
+
xarray_binfile-0.1.0b0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Felipe N. Schuch
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|