ncfunc 0.1.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ncfunc-0.1.0/README.md → ncfunc-0.2.1/PKG-INFO +42 -1
- ncfunc-0.1.0/PKG-INFO → ncfunc-0.2.1/README.md +29 -13
- {ncfunc-0.1.0 → ncfunc-0.2.1}/pyproject.toml +6 -3
- {ncfunc-0.1.0 → ncfunc-0.2.1}/pyproject.toml.orig +6 -3
- {ncfunc-0.1.0 → ncfunc-0.2.1}/src/ncfunc/__init__.py +2 -0
- {ncfunc-0.1.0 → ncfunc-0.2.1}/src/ncfunc/core.py +15 -8
- ncfunc-0.2.1/src/ncfunc/read_mf.py +62 -0
|
@@ -1,3 +1,16 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: ncfunc
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Add your description here
|
|
5
|
+
Author: warmpool
|
|
6
|
+
Author-email: warmpool <mail@mail.com>
|
|
7
|
+
Requires-Dist: datenum>=0.1.0
|
|
8
|
+
Requires-Dist: netcdf4>=1.6
|
|
9
|
+
Requires-Dist: numpy>=1.24
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Project-URL: Homepage, https://github.com/warmpool/py-ncfunc
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
1
14
|
# ncfunc
|
|
2
15
|
|
|
3
16
|
Functional-style reading and writing of NetCDF data files.
|
|
@@ -18,7 +31,9 @@ variable name, dimension name, etc.
|
|
|
18
31
|
[datenum](https://pypi.org/project/datenum/) serials, including fixed-length
|
|
19
32
|
calendars (`360_day`, `365_day`)
|
|
20
33
|
- **Bounds-based subsetting** - `read_within` slices variables by coordinate
|
|
21
|
-
ranges instead of index arithmetic
|
|
34
|
+
ranges instead of index arithmetic
|
|
35
|
+
- **Multi-file reads** - `read_mf_within` reads the same variable across many
|
|
36
|
+
files and combines the results along a time or spatial axis
|
|
22
37
|
- **Cached metadata** - structure queries reuse a path-keyed cache that
|
|
23
38
|
auto-refreshes when a file's mtime changes (`DatasetMeta`)
|
|
24
39
|
- **One-shot writer** - `save()` creates dimensions, coordinates and the
|
|
@@ -56,6 +71,7 @@ file = "tests/ersst_2022-2024.nc"
|
|
|
56
71
|
| `read(path, var, subsets?)` | read a variable into `np.ndarray` |
|
|
57
72
|
| `read_time(path, ...)` | read and decode a time coordinate to datenum values |
|
|
58
73
|
| `read_within(path, var, withins, ...)` | bounds-based subset read with coordinates |
|
|
74
|
+
| `read_mf_within(paths, var, withins, ...)` | bounds-based read across many files, combined along an axis |
|
|
59
75
|
| `write(path, var, data, subsets?)` | write into an existing variable |
|
|
60
76
|
| `create(path, var, dim_specs, ...)` | create a variable (+ dimensions), idempotently |
|
|
61
77
|
| `save(path, data, ...)` | create + write a variable and its coordinates in one call |
|
|
@@ -132,6 +148,27 @@ The time dimension is found by guessing a time-named coordinate; point at it
|
|
|
132
148
|
explicitly with `idim_time=<index>` if the guess would be wrong, or disable
|
|
133
149
|
time handling with `decode_time=False`.
|
|
134
150
|
|
|
151
|
+
### Read across many files
|
|
152
|
+
|
|
153
|
+
`read_mf_within` applies the same bounds-based read to every file in a list and
|
|
154
|
+
combines the results. With `stacked_along=None` a new leading axis is prepended
|
|
155
|
+
(one slice per file, in path order); with an integer `stacked_along` the values
|
|
156
|
+
(and that coordinate) are concatenated along the given dimension - typically
|
|
157
|
+
the time axis of files split by time:
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
tiles = [f"era5_2022-{m:02d}.nc" for m in range(1, 13)]
|
|
161
|
+
tas, (time, lat, lon) = ncf.read_mf_within(
|
|
162
|
+
tiles,
|
|
163
|
+
"tas",
|
|
164
|
+
withins=((None, None), (-30.0, 30.0), (150.0, 210.0)),
|
|
165
|
+
stacked_along=0, # concatenate along the time axis
|
|
166
|
+
)
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Non-stacked coordinates are taken from the last file, so tiles are expected to
|
|
170
|
+
share them.
|
|
171
|
+
|
|
135
172
|
### Write data
|
|
136
173
|
|
|
137
174
|
`save` writes a variable plus its dimensions in one shot. The first entry is
|
|
@@ -160,11 +197,15 @@ For finer control, use the pieces directly:
|
|
|
160
197
|
|
|
161
198
|
```python
|
|
162
199
|
ncf.create(path, "tas", {"time": 3, "lon": 4}) # idempotent; missing dims created
|
|
200
|
+
ncf.create(path, "tas", {"time": 3, "lon": 4}, chunksizes=(1, 4)) # chunk the variable
|
|
163
201
|
ncf.write(path, "tas", data) # full write, shape must match
|
|
164
202
|
ncf.write(path, "tas", data, ((slice(0, 1), slice(None)),)) # or by slices
|
|
165
203
|
ncf.write_attr(path, "/", "history", "created today") # '/' = root attribute
|
|
166
204
|
```
|
|
167
205
|
|
|
206
|
+
`create` accepts an optional `chunksizes` (one size per dimension) to set the
|
|
207
|
+
variable's HDF5 chunk shape; it defaults to netCDF4's own chunking.
|
|
208
|
+
|
|
168
209
|
## Development
|
|
169
210
|
|
|
170
211
|
```sh
|
|
@@ -1,15 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.3
|
|
2
|
-
Name: ncfunc
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary: Add your description here
|
|
5
|
-
Author: lkkbox
|
|
6
|
-
Author-email: lkkbox <mail@mail.com>
|
|
7
|
-
Requires-Dist: datenum>=0.1.0
|
|
8
|
-
Requires-Dist: netcdf4>=1.7.3
|
|
9
|
-
Requires-Dist: numpy>=1.24
|
|
10
|
-
Requires-Python: >=3.10
|
|
11
|
-
Description-Content-Type: text/markdown
|
|
12
|
-
|
|
13
1
|
# ncfunc
|
|
14
2
|
|
|
15
3
|
Functional-style reading and writing of NetCDF data files.
|
|
@@ -30,7 +18,9 @@ variable name, dimension name, etc.
|
|
|
30
18
|
[datenum](https://pypi.org/project/datenum/) serials, including fixed-length
|
|
31
19
|
calendars (`360_day`, `365_day`)
|
|
32
20
|
- **Bounds-based subsetting** - `read_within` slices variables by coordinate
|
|
33
|
-
ranges instead of index arithmetic
|
|
21
|
+
ranges instead of index arithmetic
|
|
22
|
+
- **Multi-file reads** - `read_mf_within` reads the same variable across many
|
|
23
|
+
files and combines the results along a time or spatial axis
|
|
34
24
|
- **Cached metadata** - structure queries reuse a path-keyed cache that
|
|
35
25
|
auto-refreshes when a file's mtime changes (`DatasetMeta`)
|
|
36
26
|
- **One-shot writer** - `save()` creates dimensions, coordinates and the
|
|
@@ -68,6 +58,7 @@ file = "tests/ersst_2022-2024.nc"
|
|
|
68
58
|
| `read(path, var, subsets?)` | read a variable into `np.ndarray` |
|
|
69
59
|
| `read_time(path, ...)` | read and decode a time coordinate to datenum values |
|
|
70
60
|
| `read_within(path, var, withins, ...)` | bounds-based subset read with coordinates |
|
|
61
|
+
| `read_mf_within(paths, var, withins, ...)` | bounds-based read across many files, combined along an axis |
|
|
71
62
|
| `write(path, var, data, subsets?)` | write into an existing variable |
|
|
72
63
|
| `create(path, var, dim_specs, ...)` | create a variable (+ dimensions), idempotently |
|
|
73
64
|
| `save(path, data, ...)` | create + write a variable and its coordinates in one call |
|
|
@@ -144,6 +135,27 @@ The time dimension is found by guessing a time-named coordinate; point at it
|
|
|
144
135
|
explicitly with `idim_time=<index>` if the guess would be wrong, or disable
|
|
145
136
|
time handling with `decode_time=False`.
|
|
146
137
|
|
|
138
|
+
### Read across many files
|
|
139
|
+
|
|
140
|
+
`read_mf_within` applies the same bounds-based read to every file in a list and
|
|
141
|
+
combines the results. With `stacked_along=None` a new leading axis is prepended
|
|
142
|
+
(one slice per file, in path order); with an integer `stacked_along` the values
|
|
143
|
+
(and that coordinate) are concatenated along the given dimension - typically
|
|
144
|
+
the time axis of files split by time:
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
tiles = [f"era5_2022-{m:02d}.nc" for m in range(1, 13)]
|
|
148
|
+
tas, (time, lat, lon) = ncf.read_mf_within(
|
|
149
|
+
tiles,
|
|
150
|
+
"tas",
|
|
151
|
+
withins=((None, None), (-30.0, 30.0), (150.0, 210.0)),
|
|
152
|
+
stacked_along=0, # concatenate along the time axis
|
|
153
|
+
)
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Non-stacked coordinates are taken from the last file, so tiles are expected to
|
|
157
|
+
share them.
|
|
158
|
+
|
|
147
159
|
### Write data
|
|
148
160
|
|
|
149
161
|
`save` writes a variable plus its dimensions in one shot. The first entry is
|
|
@@ -172,11 +184,15 @@ For finer control, use the pieces directly:
|
|
|
172
184
|
|
|
173
185
|
```python
|
|
174
186
|
ncf.create(path, "tas", {"time": 3, "lon": 4}) # idempotent; missing dims created
|
|
187
|
+
ncf.create(path, "tas", {"time": 3, "lon": 4}, chunksizes=(1, 4)) # chunk the variable
|
|
175
188
|
ncf.write(path, "tas", data) # full write, shape must match
|
|
176
189
|
ncf.write(path, "tas", data, ((slice(0, 1), slice(None)),)) # or by slices
|
|
177
190
|
ncf.write_attr(path, "/", "history", "created today") # '/' = root attribute
|
|
178
191
|
```
|
|
179
192
|
|
|
193
|
+
`create` accepts an optional `chunksizes` (one size per dimension) to set the
|
|
194
|
+
variable's HDF5 chunk shape; it defaults to netCDF4's own chunking.
|
|
195
|
+
|
|
180
196
|
## Development
|
|
181
197
|
|
|
182
198
|
```sh
|
|
@@ -1,22 +1,25 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ncfunc"
|
|
3
|
-
version = "0.1
|
|
3
|
+
version = "0.2.1"
|
|
4
4
|
description = "Add your description here"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
7
7
|
dependencies = [
|
|
8
8
|
"datenum>=0.1.0",
|
|
9
|
-
"netcdf4>=1.
|
|
9
|
+
"netcdf4>=1.6",
|
|
10
10
|
"numpy>=1.24",
|
|
11
11
|
]
|
|
12
12
|
|
|
13
13
|
[[project.authors]]
|
|
14
|
-
name = "
|
|
14
|
+
name = "warmpool"
|
|
15
15
|
email = "mail@mail.com"
|
|
16
16
|
|
|
17
17
|
[project.scripts]
|
|
18
18
|
ncfunc = "ncfunc:main"
|
|
19
19
|
|
|
20
|
+
[project.urls]
|
|
21
|
+
Homepage = "https://github.com/warmpool/py-ncfunc"
|
|
22
|
+
|
|
20
23
|
[build-system]
|
|
21
24
|
requires = ["uv_build>=0.12.1,<0.13.0"]
|
|
22
25
|
build-backend = "uv_build"
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ncfunc"
|
|
3
|
-
version = "0.1
|
|
3
|
+
version = "0.2.1"
|
|
4
4
|
description = "Add your description here"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
7
|
-
{ name = "
|
|
7
|
+
{ name = "warmpool", email = "mail@mail.com" }
|
|
8
8
|
]
|
|
9
9
|
requires-python = ">=3.10"
|
|
10
10
|
dependencies = [
|
|
11
11
|
"datenum>=0.1.0",
|
|
12
|
-
"netcdf4>=1.
|
|
12
|
+
"netcdf4>=1.6",
|
|
13
13
|
"numpy>=1.24",
|
|
14
14
|
]
|
|
15
15
|
|
|
@@ -27,3 +27,6 @@ dev = [
|
|
|
27
27
|
|
|
28
28
|
[tool.ruff.lint]
|
|
29
29
|
extend-select = ["N802", "N803", "N806", "N815", "N816"]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/warmpool/py-ncfunc"
|
|
@@ -16,6 +16,7 @@ from .core import (
|
|
|
16
16
|
write,
|
|
17
17
|
write_attr,
|
|
18
18
|
)
|
|
19
|
+
from .read_mf import read_mf_within
|
|
19
20
|
|
|
20
21
|
__all__ = [
|
|
21
22
|
"DatasetMeta",
|
|
@@ -26,6 +27,7 @@ __all__ = [
|
|
|
26
27
|
"dim_names",
|
|
27
28
|
"ndim",
|
|
28
29
|
"read",
|
|
30
|
+
"read_mf_within",
|
|
29
31
|
"read_time",
|
|
30
32
|
"read_within",
|
|
31
33
|
"save",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import warnings
|
|
4
|
-
from collections.abc import Mapping
|
|
4
|
+
from collections.abc import Mapping, Sequence
|
|
5
5
|
from dataclasses import dataclass, field
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Any, ClassVar, Literal, cast
|
|
@@ -19,6 +19,7 @@ DTypeSpec = (
|
|
|
19
19
|
| str
|
|
20
20
|
| type[int | float | complex | bool | str | np.bool_ | np.number | np.str_]
|
|
21
21
|
)
|
|
22
|
+
MAX_DATASET_CACHE: int = 128
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
# --------------metadata assets-----------------------
|
|
@@ -48,7 +49,6 @@ class DatasetMeta:
|
|
|
48
49
|
variables: dict[str, VariableMeta]
|
|
49
50
|
attributes: dict[str, Any]
|
|
50
51
|
cached_registry: ClassVar[dict[str, DatasetMeta]] = {}
|
|
51
|
-
MAX_CACHE: ClassVar[int] = 64
|
|
52
52
|
|
|
53
53
|
def __init__(self, path: PathLike) -> None:
|
|
54
54
|
"""
|
|
@@ -65,7 +65,7 @@ class DatasetMeta:
|
|
|
65
65
|
self._read(path)
|
|
66
66
|
|
|
67
67
|
# -- uncaching the old entries
|
|
68
|
-
while len(self.cached_registry) >=
|
|
68
|
+
while len(self.cached_registry) >= MAX_DATASET_CACHE:
|
|
69
69
|
self.cached_registry.pop(next(iter(self.cached_registry)))
|
|
70
70
|
|
|
71
71
|
# -- caching
|
|
@@ -723,11 +723,14 @@ def write_attr(path: PathLike, var_name: str, attr_name: str, value: Any) -> Non
|
|
|
723
723
|
the file if var_name == '/'
|
|
724
724
|
the variable else
|
|
725
725
|
|
|
726
|
-
|
|
727
|
-
|
|
726
|
+
only a root write creates the file when the path does not exist; a variable
|
|
727
|
+
write raises FileNotFoundError instead, leaving no file behind.
|
|
728
|
+
|
|
729
|
+
raises KeyError if the variable does not exist
|
|
728
730
|
"""
|
|
729
|
-
# check var_name here; KeyError and FileNotFoundError surface
|
|
730
|
-
|
|
731
|
+
if var_name != "/": # check var_name here; KeyError and FileNotFoundError surface
|
|
732
|
+
_get_variable_meta(path, var_name)
|
|
733
|
+
|
|
731
734
|
with nc.Dataset(path, "a") as h_file:
|
|
732
735
|
if var_name == "/":
|
|
733
736
|
h_file.setncattr(attr_name, value)
|
|
@@ -802,6 +805,7 @@ def create(
|
|
|
802
805
|
dtype: DTypeSpec = np.float32,
|
|
803
806
|
complevel: Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] | None = 4,
|
|
804
807
|
shuffle: bool = True,
|
|
808
|
+
chunksizes: Sequence[int] | None = None,
|
|
805
809
|
) -> None:
|
|
806
810
|
"""
|
|
807
811
|
create a variable and its dimensions in a file, skipping existing parts
|
|
@@ -820,7 +824,9 @@ def create(
|
|
|
820
824
|
created coordinate variables recognised as lon/lat/level/time-like.
|
|
821
825
|
dtype accepts anything np.dtype understands ('f4', np.float32, ...);
|
|
822
826
|
complevel > 0 enables zlib compression with the given level and
|
|
823
|
-
shuffle filter.
|
|
827
|
+
shuffle filter. chunksizes sets the requested HDF5 chunk shape of the
|
|
828
|
+
variable (per dimension, in order); None leaves chunking to netCDF4's
|
|
829
|
+
default.
|
|
824
830
|
|
|
825
831
|
raises ValueError when an existing variable disagrees with dim_specs
|
|
826
832
|
"""
|
|
@@ -862,6 +868,7 @@ def create(
|
|
|
862
868
|
"compression": "zlib",
|
|
863
869
|
"complevel": complevel,
|
|
864
870
|
"shuffle": shuffle,
|
|
871
|
+
"chunksizes": chunksizes,
|
|
865
872
|
}
|
|
866
873
|
|
|
867
874
|
with nc.Dataset(path, "a") as h_file:
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
from collections.abc import Sequence
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
from . import core
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def read_mf_within(
|
|
10
|
+
paths: Sequence[core.PathLike],
|
|
11
|
+
var_name: str,
|
|
12
|
+
withins: tuple[tuple[float | None, float | None], ...],
|
|
13
|
+
idim_time: int | None = None,
|
|
14
|
+
decode_time: bool = True,
|
|
15
|
+
stacked_along: None | int = None,
|
|
16
|
+
) -> tuple[np.ndarray, tuple[np.ndarray, ...]]:
|
|
17
|
+
"""
|
|
18
|
+
read a variable within bounds from multiple files and combine the results
|
|
19
|
+
|
|
20
|
+
each file is read with core.read_within() under the same withins, then the
|
|
21
|
+
per-file values are joined. with stacked_along=None a new leading axis of
|
|
22
|
+
length len(paths) is prepended (one slice per file, in path order); with an
|
|
23
|
+
integer stacked_along the values are concatenated along that dimension and
|
|
24
|
+
the matching coordinate array is concatenated too.
|
|
25
|
+
|
|
26
|
+
the remaining dimensions' coordinates are taken from the last file; the
|
|
27
|
+
combined data is expected to align, so callers should ensure the
|
|
28
|
+
non-stacked coordinates are identical across files.
|
|
29
|
+
|
|
30
|
+
returns (values, coords) with coords the same shape as core.read_within()
|
|
31
|
+
but for the stacked dimension replaced or extended by concatenation.
|
|
32
|
+
|
|
33
|
+
raises RuntimeError when fewer than 2 paths are given, ValueError /
|
|
34
|
+
KeyError / FileNotFoundError as reported by core.read_within()
|
|
35
|
+
"""
|
|
36
|
+
if len(paths) == 0:
|
|
37
|
+
raise RuntimeError("zero path is given.")
|
|
38
|
+
elif len(paths) == 1:
|
|
39
|
+
raise RuntimeError("Only 1 path is given. Use `read_within()` instead.")
|
|
40
|
+
|
|
41
|
+
paths = tuple(Path(p) for p in paths)
|
|
42
|
+
|
|
43
|
+
values = []
|
|
44
|
+
stacked_dim = []
|
|
45
|
+
|
|
46
|
+
for path in paths:
|
|
47
|
+
value, dims = core.read_within(path, var_name, withins, idim_time, decode_time)
|
|
48
|
+
values.append(value)
|
|
49
|
+
|
|
50
|
+
if stacked_along is not None:
|
|
51
|
+
stacked_dim.append(dims[stacked_along])
|
|
52
|
+
|
|
53
|
+
if stacked_along is None:
|
|
54
|
+
value_out = np.stack(values, axis=0)
|
|
55
|
+
dims_out = (np.arange(len(paths)), *dims)
|
|
56
|
+
else:
|
|
57
|
+
value_out = np.concatenate(values, axis=stacked_along)
|
|
58
|
+
dims_out = list(dims)
|
|
59
|
+
dims_out[stacked_along] = np.concatenate(stacked_dim)
|
|
60
|
+
dims_out = tuple(dims_out)
|
|
61
|
+
|
|
62
|
+
return value_out, dims_out
|