h5netcdf 1.6.3__py3-none-any.whl → 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of h5netcdf might be problematic. Click here for more details.
- h5netcdf/_version.py +16 -3
- h5netcdf/attrs.py +23 -2
- h5netcdf/core.py +193 -53
- h5netcdf/dimensions.py +21 -5
- h5netcdf/tests/conftest.py +12 -0
- h5netcdf/tests/test_h5netcdf.py +345 -163
- h5netcdf/utils.py +205 -0
- {h5netcdf-1.6.3.dist-info → h5netcdf-1.7.0.dist-info}/METADATA +5 -5
- h5netcdf-1.7.0.dist-info/RECORD +16 -0
- h5netcdf-1.6.3.dist-info/RECORD +0 -16
- {h5netcdf-1.6.3.dist-info → h5netcdf-1.7.0.dist-info}/WHEEL +0 -0
- {h5netcdf-1.6.3.dist-info → h5netcdf-1.7.0.dist-info}/licenses/AUTHORS.txt +0 -0
- {h5netcdf-1.6.3.dist-info → h5netcdf-1.7.0.dist-info}/licenses/LICENSE +0 -0
- {h5netcdf-1.6.3.dist-info → h5netcdf-1.7.0.dist-info}/top_level.txt +0 -0
h5netcdf/_version.py
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
# file generated by setuptools-scm
|
|
2
2
|
# don't change, don't track in version control
|
|
3
3
|
|
|
4
|
-
__all__ = [
|
|
4
|
+
__all__ = [
|
|
5
|
+
"__version__",
|
|
6
|
+
"__version_tuple__",
|
|
7
|
+
"version",
|
|
8
|
+
"version_tuple",
|
|
9
|
+
"__commit_id__",
|
|
10
|
+
"commit_id",
|
|
11
|
+
]
|
|
5
12
|
|
|
6
13
|
TYPE_CHECKING = False
|
|
7
14
|
if TYPE_CHECKING:
|
|
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
|
|
|
9
16
|
from typing import Union
|
|
10
17
|
|
|
11
18
|
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
+
COMMIT_ID = Union[str, None]
|
|
12
20
|
else:
|
|
13
21
|
VERSION_TUPLE = object
|
|
22
|
+
COMMIT_ID = object
|
|
14
23
|
|
|
15
24
|
version: str
|
|
16
25
|
__version__: str
|
|
17
26
|
__version_tuple__: VERSION_TUPLE
|
|
18
27
|
version_tuple: VERSION_TUPLE
|
|
28
|
+
commit_id: COMMIT_ID
|
|
29
|
+
__commit_id__: COMMIT_ID
|
|
19
30
|
|
|
20
|
-
__version__ = version = '1.
|
|
21
|
-
__version_tuple__ = version_tuple = (1,
|
|
31
|
+
__version__ = version = '1.7.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (1, 7, 0)
|
|
33
|
+
|
|
34
|
+
__commit_id__ = commit_id = None
|
h5netcdf/attrs.py
CHANGED
|
@@ -2,6 +2,8 @@ from collections.abc import MutableMapping
|
|
|
2
2
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
|
|
5
|
+
from .utils import _create_string_attribute
|
|
6
|
+
|
|
5
7
|
_HIDDEN_ATTRS = frozenset(
|
|
6
8
|
[
|
|
7
9
|
"REFERENCE_LIST",
|
|
@@ -17,10 +19,11 @@ _HIDDEN_ATTRS = frozenset(
|
|
|
17
19
|
|
|
18
20
|
|
|
19
21
|
class Attributes(MutableMapping):
|
|
20
|
-
def __init__(self, h5attrs, check_dtype, h5py_pckg):
|
|
22
|
+
def __init__(self, h5attrs, check_dtype, h5py_pckg, format="NETCDF4"):
|
|
21
23
|
self._h5attrs = h5attrs
|
|
22
24
|
self._check_dtype = check_dtype
|
|
23
25
|
self._h5py = h5py_pckg
|
|
26
|
+
self._format = format
|
|
24
27
|
|
|
25
28
|
def __getitem__(self, key):
|
|
26
29
|
if key in _HIDDEN_ATTRS:
|
|
@@ -83,7 +86,25 @@ class Attributes(MutableMapping):
|
|
|
83
86
|
dtype = np.asarray(value).dtype
|
|
84
87
|
|
|
85
88
|
self._check_dtype(dtype)
|
|
86
|
-
|
|
89
|
+
|
|
90
|
+
if (
|
|
91
|
+
dtype.kind in {"S", "U"} # for strings
|
|
92
|
+
and dtype.metadata is None # but not special h5py strings
|
|
93
|
+
and not isinstance(value, (list, self._h5py.Empty))
|
|
94
|
+
and self._h5py.__name__ == "h5py"
|
|
95
|
+
):
|
|
96
|
+
# create with low level API to get fixed length strings
|
|
97
|
+
# as netcdf4-python/netcdf-c does
|
|
98
|
+
_create_string_attribute(self._h5attrs._id, key, value)
|
|
99
|
+
# always for CLASSIC mode
|
|
100
|
+
elif self._format == "NETCDF4_CLASSIC":
|
|
101
|
+
self._h5attrs[key] = np.atleast_1d(value)
|
|
102
|
+
else:
|
|
103
|
+
# netcdf4-python/netcdf-c writes non-string scalars as simple dataset
|
|
104
|
+
# converting to 1D
|
|
105
|
+
if np.isscalar(value) and dtype.kind not in {"S", "U"}:
|
|
106
|
+
value = np.atleast_1d(value)
|
|
107
|
+
self._h5attrs[key] = value
|
|
87
108
|
|
|
88
109
|
def __delitem__(self, key):
|
|
89
110
|
del self._h5attrs[key]
|
h5netcdf/core.py
CHANGED
|
@@ -13,7 +13,15 @@ from packaging import version
|
|
|
13
13
|
from . import __version__
|
|
14
14
|
from .attrs import Attributes
|
|
15
15
|
from .dimensions import Dimension, Dimensions
|
|
16
|
-
from .utils import
|
|
16
|
+
from .utils import (
|
|
17
|
+
CompatibilityError,
|
|
18
|
+
Frozen,
|
|
19
|
+
_commit_enum_type,
|
|
20
|
+
_create_classic_string_dataset,
|
|
21
|
+
_create_enum_dataset,
|
|
22
|
+
_create_enum_dataset_attribute,
|
|
23
|
+
_create_string_attribute,
|
|
24
|
+
)
|
|
17
25
|
|
|
18
26
|
try:
|
|
19
27
|
import h5pyd
|
|
@@ -36,10 +44,6 @@ def _name_from_dimension(dim):
|
|
|
36
44
|
return dim[0].name.split("/")[-1]
|
|
37
45
|
|
|
38
46
|
|
|
39
|
-
class CompatibilityError(Exception):
|
|
40
|
-
"""Raised when using features that are not part of the NetCDF4 API."""
|
|
41
|
-
|
|
42
|
-
|
|
43
47
|
def _invalid_netcdf_feature(feature, allow):
|
|
44
48
|
if not allow:
|
|
45
49
|
msg = (
|
|
@@ -348,7 +352,9 @@ class BaseVariable(BaseObject):
|
|
|
348
352
|
[self._parent._all_dimensions[d]._dimid for d in dims],
|
|
349
353
|
"int32",
|
|
350
354
|
)
|
|
351
|
-
|
|
355
|
+
# add _Netcdf4Coordinates for multi-dimensional coordinate variables
|
|
356
|
+
# or for (one-dimensional) coordinates
|
|
357
|
+
if len(coord_ids) >= 1:
|
|
352
358
|
self._h5ds.attrs["_Netcdf4Coordinates"] = coord_ids
|
|
353
359
|
|
|
354
360
|
def _ensure_dim_id(self):
|
|
@@ -361,12 +367,21 @@ class BaseVariable(BaseObject):
|
|
|
361
367
|
self._h5ds.attrs["_Netcdf4Dimid"] = dim.attrs["_Netcdf4Dimid"]
|
|
362
368
|
|
|
363
369
|
def _maybe_resize_dimensions(self, key, value):
|
|
364
|
-
"""Resize according to given (expanded) key with respect to variable dimensions
|
|
370
|
+
"""Resize according to given (expanded) key with respect to variable dimensions.
|
|
371
|
+
|
|
372
|
+
Parameters
|
|
373
|
+
----------
|
|
374
|
+
key : Tuple[slice]
|
|
375
|
+
Indexing key
|
|
376
|
+
value : array-like
|
|
377
|
+
Values to be written.
|
|
378
|
+
"""
|
|
365
379
|
new_shape = ()
|
|
366
|
-
v =
|
|
380
|
+
v = np.asarray(value)
|
|
367
381
|
for i, dim in enumerate(self.dimensions):
|
|
368
382
|
# is unlimited dimensions (check in all dimensions)
|
|
369
383
|
if self._parent._all_dimensions[dim].isunlimited():
|
|
384
|
+
current_dim_size = len(self._parent._all_dimensions[dim])
|
|
370
385
|
if key[i].stop is None:
|
|
371
386
|
# if stop is None, get dimensions from value,
|
|
372
387
|
# they must match with variable dimension
|
|
@@ -375,16 +390,25 @@ class BaseVariable(BaseObject):
|
|
|
375
390
|
if v.ndim == self.ndim:
|
|
376
391
|
new_max = max(v.shape[i], self._h5ds.shape[i])
|
|
377
392
|
elif v.ndim == 0:
|
|
378
|
-
# for
|
|
393
|
+
# for scalar values we take the current dimension size
|
|
394
|
+
# (check in all dimensions)
|
|
379
395
|
new_max = self._parent._all_dimensions[dim].size
|
|
396
|
+
# but for compatibility with netcdf4-python/netcdf-c
|
|
397
|
+
# we set at least 1
|
|
398
|
+
if new_max == 0:
|
|
399
|
+
new_max = 1
|
|
380
400
|
else:
|
|
381
401
|
raise IndexError("shape of data does not conform to slice")
|
|
402
|
+
# if slice stop is negative, we need to check the value size
|
|
403
|
+
elif key[i].stop < 0:
|
|
404
|
+
new_max = v.shape[i] - key[i].stop
|
|
382
405
|
else:
|
|
383
406
|
new_max = max(key[i].stop, self._h5ds.shape[i])
|
|
384
407
|
# resize unlimited dimension if needed but no other variables
|
|
385
408
|
# this is in line with `netcdf4-python` which only resizes
|
|
386
409
|
# the dimension and this variable
|
|
387
|
-
|
|
410
|
+
# todo: check above assumptions with latest netcdf4-python/netcdf-c
|
|
411
|
+
if current_dim_size < new_max and self.name == dim:
|
|
388
412
|
self._parent.resize_dimension(dim, new_max)
|
|
389
413
|
new_shape += (new_max,)
|
|
390
414
|
else:
|
|
@@ -412,8 +436,19 @@ class BaseVariable(BaseObject):
|
|
|
412
436
|
string_info
|
|
413
437
|
and string_info.length is not None
|
|
414
438
|
and string_info.length > 1
|
|
415
|
-
)
|
|
439
|
+
):
|
|
440
|
+
# fixed length string
|
|
441
|
+
value = fillvalue
|
|
442
|
+
elif string_info and string_info.length is None:
|
|
443
|
+
# variable length string
|
|
416
444
|
value = fillvalue
|
|
445
|
+
elif enum_info:
|
|
446
|
+
value = fillvalue
|
|
447
|
+
if self._root._h5py.__name__ == "h5py":
|
|
448
|
+
_create_enum_dataset_attribute(
|
|
449
|
+
self, "_FillValue", value, self.datatype
|
|
450
|
+
)
|
|
451
|
+
return
|
|
417
452
|
else:
|
|
418
453
|
value = self.dtype.type(fillvalue)
|
|
419
454
|
|
|
@@ -499,16 +534,17 @@ class BaseVariable(BaseObject):
|
|
|
499
534
|
for k in key0
|
|
500
535
|
]
|
|
501
536
|
# second convert to max shape
|
|
537
|
+
# we take the minimum of shape vs max_index to not return
|
|
538
|
+
# slices larger than expected data
|
|
502
539
|
max_shape = tuple(
|
|
503
|
-
|
|
504
|
-
shape[i] if k is None else max(h5ds_shape[i], k)
|
|
505
|
-
for i, k in enumerate(max_index)
|
|
506
|
-
]
|
|
540
|
+
s if k is None else min(s, k) for s, k in zip(shape, max_index)
|
|
507
541
|
)
|
|
508
542
|
|
|
509
543
|
# check if hdf5 dataset dimensions are smaller than
|
|
510
544
|
# their respective netcdf dimensions
|
|
511
545
|
sdiff = [d0 - d1 for d0, d1 in zip(max_shape, h5ds_shape)]
|
|
546
|
+
# set negative values to zero
|
|
547
|
+
sdiff = np.maximum(sdiff, 0)
|
|
512
548
|
# create padding only if hdf5 dataset is smaller than netcdf dimension
|
|
513
549
|
if sum(sdiff):
|
|
514
550
|
padding = [(0, s) for s in sdiff]
|
|
@@ -583,13 +619,28 @@ class BaseVariable(BaseObject):
|
|
|
583
619
|
):
|
|
584
620
|
self._h5ds[key] = value.view(view)
|
|
585
621
|
else:
|
|
586
|
-
|
|
622
|
+
# write with low-level API for CLASSIC format
|
|
623
|
+
if (
|
|
624
|
+
self._root._format == "NETCDF4_CLASSIC"
|
|
625
|
+
and self.dtype.kind in ["S", "U"]
|
|
626
|
+
and self._root._h5py.__name__ == "h5py"
|
|
627
|
+
):
|
|
628
|
+
# h5py expects np.ndarray
|
|
629
|
+
value = np.asanyarray(value)
|
|
630
|
+
self._h5ds.id.write(
|
|
631
|
+
h5py.h5s.ALL, h5py.h5s.ALL, value, mtype=self._h5ds.id.get_type()
|
|
632
|
+
)
|
|
633
|
+
else:
|
|
634
|
+
self._h5ds[key] = value
|
|
587
635
|
|
|
588
636
|
@property
|
|
589
637
|
def attrs(self):
|
|
590
638
|
"""Return variable attributes."""
|
|
591
639
|
return Attributes(
|
|
592
|
-
self._h5ds.attrs,
|
|
640
|
+
self._h5ds.attrs,
|
|
641
|
+
self._root._check_valid_netcdf_dtype,
|
|
642
|
+
self._root._h5py,
|
|
643
|
+
format=self._root._format,
|
|
593
644
|
)
|
|
594
645
|
|
|
595
646
|
_cls_name = "h5netcdf.Variable"
|
|
@@ -677,7 +728,7 @@ def _unlabeled_dimension_mix(h5py_dataset):
|
|
|
677
728
|
if not dimlist:
|
|
678
729
|
status = "nodim"
|
|
679
730
|
else:
|
|
680
|
-
dimset =
|
|
731
|
+
dimset = {len(j) for j in dimlist}
|
|
681
732
|
# either all dimensions have exactly one scale
|
|
682
733
|
# or all dimensions have no scale
|
|
683
734
|
if dimset ^ {0} == set():
|
|
@@ -788,7 +839,7 @@ def _check_fillvalue(group, fillvalue, dtype):
|
|
|
788
839
|
# 1. we need to warn the user that writing enums with default values
|
|
789
840
|
# which are defined in the enum dict will mask those values
|
|
790
841
|
if (h5fillvalue or 0) in dtype.enum_dict.values():
|
|
791
|
-
reverse =
|
|
842
|
+
reverse = {v: k for k, v in dtype.enum_dict.items()}
|
|
792
843
|
msg = (
|
|
793
844
|
f"Creating variable with default fill_value {h5fillvalue or 0!r}"
|
|
794
845
|
f" which IS defined in enum type {dtype!r}."
|
|
@@ -979,6 +1030,13 @@ class Group(Mapping):
|
|
|
979
1030
|
|
|
980
1031
|
@dimensions.setter
|
|
981
1032
|
def dimensions(self, value):
|
|
1033
|
+
if self._format == "NETCDF4_CLASSIC":
|
|
1034
|
+
unlimited_dims = list(filter(lambda s: s in [None, 0], value.values()))
|
|
1035
|
+
if len(unlimited_dims) > 1:
|
|
1036
|
+
raise CompatibilityError(
|
|
1037
|
+
"NETCDF4_CLASSIC format only allows one unlimited dimension."
|
|
1038
|
+
)
|
|
1039
|
+
|
|
982
1040
|
for k, v in self._all_dimensions.maps[0].items():
|
|
983
1041
|
if k in value:
|
|
984
1042
|
if v != value[k]:
|
|
@@ -1104,8 +1162,10 @@ class Group(Mapping):
|
|
|
1104
1162
|
# dimension scale without a corresponding variable.
|
|
1105
1163
|
# Keep the references, to re-attach later
|
|
1106
1164
|
refs = None
|
|
1165
|
+
dimid = None
|
|
1107
1166
|
if h5name in self._dimensions and h5name in self._h5group:
|
|
1108
1167
|
refs = self._dimensions[name]._scale_refs
|
|
1168
|
+
dimid = self._dimensions[name]._h5ds.attrs.get("_Netcdf4Dimid", None)
|
|
1109
1169
|
self._dimensions[name]._detach_scale()
|
|
1110
1170
|
del self._h5group[name]
|
|
1111
1171
|
|
|
@@ -1115,15 +1175,28 @@ class Group(Mapping):
|
|
|
1115
1175
|
fillvalue, h5fillvalue = _check_fillvalue(self, fillvalue, dtype)
|
|
1116
1176
|
|
|
1117
1177
|
# create hdf5 variable
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
dtype
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1178
|
+
# for classic format string types write with low level API
|
|
1179
|
+
if (
|
|
1180
|
+
self._root._format == "NETCDF4_CLASSIC"
|
|
1181
|
+
and np.dtype(dtype).kind in ["S", "U"]
|
|
1182
|
+
and self._root._h5py.__name__ == "h5py"
|
|
1183
|
+
):
|
|
1184
|
+
_create_classic_string_dataset(
|
|
1185
|
+
self._h5group._id, h5name, data, shape, chunks
|
|
1186
|
+
)
|
|
1187
|
+
elif self._root._h5py.__name__ == "h5py" and isinstance(dtype, EnumType):
|
|
1188
|
+
# use low level API for creating ENUMS
|
|
1189
|
+
_create_enum_dataset(self, h5name, shape, dtype, h5fillvalue)
|
|
1190
|
+
else:
|
|
1191
|
+
self._h5group.create_dataset(
|
|
1192
|
+
h5name,
|
|
1193
|
+
shape,
|
|
1194
|
+
dtype=dtype,
|
|
1195
|
+
data=data,
|
|
1196
|
+
chunks=chunks,
|
|
1197
|
+
fillvalue=h5fillvalue,
|
|
1198
|
+
**kwargs,
|
|
1199
|
+
)
|
|
1127
1200
|
|
|
1128
1201
|
# create variable class instance
|
|
1129
1202
|
variable = self._variable_cls(self, h5name, dimensions)
|
|
@@ -1135,9 +1208,12 @@ class Group(Mapping):
|
|
|
1135
1208
|
|
|
1136
1209
|
# Re-create dim-scale and re-attach references to coordinate variable.
|
|
1137
1210
|
if name in self._all_dimensions and h5name in self._h5group:
|
|
1138
|
-
|
|
1211
|
+
if dimid is not None:
|
|
1212
|
+
self._all_dimensions[name]._create_scale(dimid=dimid)
|
|
1139
1213
|
if refs is not None:
|
|
1140
1214
|
self._all_dimensions[name]._attach_scale(refs)
|
|
1215
|
+
# re-attach coords for dimension scales
|
|
1216
|
+
variable._attach_coords()
|
|
1141
1217
|
|
|
1142
1218
|
# In case of data variables attach dim_scales and coords.
|
|
1143
1219
|
if name in self.variables and h5name not in self._dimensions:
|
|
@@ -1146,9 +1222,12 @@ class Group(Mapping):
|
|
|
1146
1222
|
|
|
1147
1223
|
# This is a bit of a hack, netCDF4 attaches _Netcdf4Dimid to every variable
|
|
1148
1224
|
# when a variable is first written to, after variable creation.
|
|
1149
|
-
#
|
|
1150
|
-
|
|
1151
|
-
|
|
1225
|
+
# Last known behaviour since netcdf4-python 1.7.2 and netcdf-c 4.9.2
|
|
1226
|
+
if (None in maxshape and maxshape[0] is not None) or (
|
|
1227
|
+
None not in maxshape
|
|
1228
|
+
and len(variable._h5ds.attrs.get("_Netcdf4Coordinates", [])) >= 1
|
|
1229
|
+
):
|
|
1230
|
+
variable._ensure_dim_id()
|
|
1152
1231
|
|
|
1153
1232
|
# add fillvalue attribute to variable
|
|
1154
1233
|
if fillvalue is not None:
|
|
@@ -1211,7 +1290,6 @@ class Group(Mapping):
|
|
|
1211
1290
|
var : h5netcdf.Variable
|
|
1212
1291
|
Variable class instance
|
|
1213
1292
|
"""
|
|
1214
|
-
|
|
1215
1293
|
# if root-variable
|
|
1216
1294
|
if name.startswith("/"):
|
|
1217
1295
|
# handling default fillvalues for legacyapi
|
|
@@ -1220,6 +1298,12 @@ class Group(Mapping):
|
|
|
1220
1298
|
|
|
1221
1299
|
if fillvalue is None and isinstance(self._parent._root, Dataset):
|
|
1222
1300
|
fillvalue = _get_default_fillvalue(dtype)
|
|
1301
|
+
|
|
1302
|
+
if self._root._format == "NETCDF4_CLASSIC" and len(dimensions) == 0:
|
|
1303
|
+
raise CompatibilityError(
|
|
1304
|
+
"NETCDF4_CLASSIC format does not allow variables without dimensions."
|
|
1305
|
+
)
|
|
1306
|
+
|
|
1223
1307
|
return self._root.create_variable(
|
|
1224
1308
|
name[1:],
|
|
1225
1309
|
dimensions,
|
|
@@ -1271,10 +1355,8 @@ class Group(Mapping):
|
|
|
1271
1355
|
return item
|
|
1272
1356
|
|
|
1273
1357
|
def __iter__(self):
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
for name in self.variables:
|
|
1277
|
-
yield name
|
|
1358
|
+
yield from self.groups
|
|
1359
|
+
yield from self.variables
|
|
1278
1360
|
|
|
1279
1361
|
def __len__(self):
|
|
1280
1362
|
return len(self.variables) + len(self.groups)
|
|
@@ -1345,7 +1427,10 @@ class Group(Mapping):
|
|
|
1345
1427
|
@property
|
|
1346
1428
|
def attrs(self):
|
|
1347
1429
|
return Attributes(
|
|
1348
|
-
self._h5group.attrs,
|
|
1430
|
+
self._h5group.attrs,
|
|
1431
|
+
self._root._check_valid_netcdf_dtype,
|
|
1432
|
+
self._root._h5py,
|
|
1433
|
+
format=self._root._format,
|
|
1349
1434
|
)
|
|
1350
1435
|
|
|
1351
1436
|
_cls_name = "h5netcdf.Group"
|
|
@@ -1399,8 +1484,14 @@ class Group(Mapping):
|
|
|
1399
1484
|
enum_dict: dict
|
|
1400
1485
|
A Python dictionary containing the Enum field/value pairs.
|
|
1401
1486
|
"""
|
|
1402
|
-
|
|
1403
|
-
|
|
1487
|
+
# to correspond with netcdf4-python/netcdf-c we need to create
|
|
1488
|
+
# with low level API, to keep enums ordered by value
|
|
1489
|
+
# works only for h5py
|
|
1490
|
+
if self._root._h5py.__name__ == "h5py":
|
|
1491
|
+
_commit_enum_type(self, datatype_name, enum_dict, datatype)
|
|
1492
|
+
else:
|
|
1493
|
+
et = self._root._h5py.enum_dtype(enum_dict, basetype=datatype)
|
|
1494
|
+
self._h5group[datatype_name] = et
|
|
1404
1495
|
# create enumtype class instance
|
|
1405
1496
|
enumtype = self._enumtype_cls(self, datatype_name)
|
|
1406
1497
|
self._enumtypes[datatype_name] = enumtype
|
|
@@ -1444,7 +1535,15 @@ class Group(Mapping):
|
|
|
1444
1535
|
|
|
1445
1536
|
|
|
1446
1537
|
class File(Group):
|
|
1447
|
-
def __init__(
|
|
1538
|
+
def __init__(
|
|
1539
|
+
self,
|
|
1540
|
+
path,
|
|
1541
|
+
mode="r",
|
|
1542
|
+
format="NETCDF4",
|
|
1543
|
+
invalid_netcdf=False,
|
|
1544
|
+
phony_dims=None,
|
|
1545
|
+
**kwargs,
|
|
1546
|
+
):
|
|
1448
1547
|
"""NetCDF4 file constructor.
|
|
1449
1548
|
|
|
1450
1549
|
Parameters
|
|
@@ -1456,6 +1555,10 @@ class File(Group):
|
|
|
1456
1555
|
mode: "r", "r+", "a", "w"
|
|
1457
1556
|
A valid file access mode. Defaults to "r".
|
|
1458
1557
|
|
|
1558
|
+
format: "NETCDF4", "NETCDF4_CLASSIC"
|
|
1559
|
+
The format of the file to create. Only relevant when creating a new
|
|
1560
|
+
file (mode "w"). Defaults to "NETCDF4".
|
|
1561
|
+
|
|
1459
1562
|
invalid_netcdf: bool
|
|
1460
1563
|
Allow writing netCDF4 with data types and attributes that would
|
|
1461
1564
|
otherwise not generate netCDF4 files that can be read by other
|
|
@@ -1527,7 +1630,7 @@ class File(Group):
|
|
|
1527
1630
|
mode = "r+"
|
|
1528
1631
|
self._h5py = h5pyd
|
|
1529
1632
|
try:
|
|
1530
|
-
self.
|
|
1633
|
+
self.__h5file = self._h5py.File(
|
|
1531
1634
|
path, mode, track_order=track_order, **kwargs
|
|
1532
1635
|
)
|
|
1533
1636
|
self._preexisting_file = mode != "w"
|
|
@@ -1535,7 +1638,7 @@ class File(Group):
|
|
|
1535
1638
|
# if file does not exist, create it
|
|
1536
1639
|
if _mode == "a":
|
|
1537
1640
|
mode = "w"
|
|
1538
|
-
self.
|
|
1641
|
+
self.__h5file = self._h5py.File(
|
|
1539
1642
|
path, mode, track_order=track_order, **kwargs
|
|
1540
1643
|
)
|
|
1541
1644
|
self._preexisting_file = False
|
|
@@ -1551,19 +1654,19 @@ class File(Group):
|
|
|
1551
1654
|
else:
|
|
1552
1655
|
self._preexisting_file = os.path.exists(path) and mode != "w"
|
|
1553
1656
|
self._h5py = h5py
|
|
1554
|
-
self.
|
|
1657
|
+
self.__h5file = self._h5py.File(
|
|
1555
1658
|
path, mode, track_order=track_order, **kwargs
|
|
1556
1659
|
)
|
|
1557
1660
|
elif isinstance(path, h5py.File):
|
|
1558
1661
|
self._preexisting_file = mode in {"r", "r+", "a"}
|
|
1559
1662
|
self._h5py = h5py
|
|
1560
|
-
self.
|
|
1663
|
+
self.__h5file = path
|
|
1561
1664
|
# h5py File passed in: let the caller decide when to close it
|
|
1562
1665
|
self._close_h5file = False
|
|
1563
1666
|
else: # file-like object
|
|
1564
1667
|
self._preexisting_file = mode in {"r", "r+", "a"}
|
|
1565
1668
|
self._h5py = h5py
|
|
1566
|
-
self.
|
|
1669
|
+
self.__h5file = self._h5py.File(
|
|
1567
1670
|
path, mode, track_order=track_order, **kwargs
|
|
1568
1671
|
)
|
|
1569
1672
|
except Exception:
|
|
@@ -1572,7 +1675,16 @@ class File(Group):
|
|
|
1572
1675
|
else:
|
|
1573
1676
|
self._closed = False
|
|
1574
1677
|
|
|
1678
|
+
if self._preexisting_file:
|
|
1679
|
+
format = (
|
|
1680
|
+
"NETCDF4_CLASSIC"
|
|
1681
|
+
if self._h5file.attrs.get("_nc3_strict")
|
|
1682
|
+
else "NETCDF4"
|
|
1683
|
+
)
|
|
1684
|
+
|
|
1685
|
+
self._filename = self._h5file.filename
|
|
1575
1686
|
self._mode = mode
|
|
1687
|
+
self._format = format
|
|
1576
1688
|
self._writable = mode != "r"
|
|
1577
1689
|
self._root_ref = weakref.ref(self)
|
|
1578
1690
|
self._h5path = "/"
|
|
@@ -1589,6 +1701,9 @@ class File(Group):
|
|
|
1589
1701
|
"phony_dims='access' for per access naming."
|
|
1590
1702
|
)
|
|
1591
1703
|
|
|
1704
|
+
if format not in ["NETCDF4", "NETCDF4_CLASSIC"]:
|
|
1705
|
+
raise ValueError(f"unknown format {format!r}")
|
|
1706
|
+
|
|
1592
1707
|
# string decoding
|
|
1593
1708
|
if "legacy" in self._cls_name:
|
|
1594
1709
|
if self.decode_vlen_strings is not None:
|
|
@@ -1639,6 +1754,11 @@ class File(Group):
|
|
|
1639
1754
|
description = "boolean"
|
|
1640
1755
|
elif self._h5py.check_dtype(ref=dtype) is not None:
|
|
1641
1756
|
description = "reference"
|
|
1757
|
+
elif (
|
|
1758
|
+
dtype in [int, np.int64, np.uint64, np.uint32, np.uint16, np.uint8]
|
|
1759
|
+
and self._format == "NETCDF4_CLASSIC"
|
|
1760
|
+
):
|
|
1761
|
+
description = f"{dtype} (CLASSIC)"
|
|
1642
1762
|
else:
|
|
1643
1763
|
description = None
|
|
1644
1764
|
|
|
@@ -1660,6 +1780,10 @@ class File(Group):
|
|
|
1660
1780
|
def parent(self):
|
|
1661
1781
|
return None
|
|
1662
1782
|
|
|
1783
|
+
@property
|
|
1784
|
+
def data_model(self):
|
|
1785
|
+
return self._format
|
|
1786
|
+
|
|
1663
1787
|
@property
|
|
1664
1788
|
def _root(self):
|
|
1665
1789
|
return self
|
|
@@ -1673,12 +1797,20 @@ class File(Group):
|
|
|
1673
1797
|
f"hdf5={self._h5py.version.hdf5_version},"
|
|
1674
1798
|
f"{self._h5py.__name__}={self._h5py.__version__}"
|
|
1675
1799
|
)
|
|
1676
|
-
self.
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1800
|
+
if self._format == "NETCDF4_CLASSIC" and self._h5py.__name__ == "h5py":
|
|
1801
|
+
_create_string_attribute(
|
|
1802
|
+
self.attrs._h5attrs._id, "_NCProperties", _NC_PROPERTIES
|
|
1803
|
+
)
|
|
1804
|
+
else:
|
|
1805
|
+
self.attrs._h5attrs["_NCProperties"] = np.array(
|
|
1806
|
+
_NC_PROPERTIES,
|
|
1807
|
+
dtype=self._h5py.string_dtype(
|
|
1808
|
+
encoding="ascii", length=len(_NC_PROPERTIES)
|
|
1809
|
+
),
|
|
1810
|
+
)
|
|
1811
|
+
if self._format == "NETCDF4_CLASSIC":
|
|
1812
|
+
self.attrs._h5attrs["_nc3_strict"] = np.array(1, np.int32)
|
|
1813
|
+
|
|
1682
1814
|
if self.invalid_netcdf:
|
|
1683
1815
|
# see https://github.com/h5netcdf/h5netcdf/issues/165
|
|
1684
1816
|
# warn user if .nc file extension is used for invalid netcdf features
|
|
@@ -1693,15 +1825,23 @@ class File(Group):
|
|
|
1693
1825
|
# remove _NCProperties if invalid_netcdf if exists
|
|
1694
1826
|
if "_NCProperties" in self.attrs._h5attrs:
|
|
1695
1827
|
del self.attrs._h5attrs["_NCProperties"]
|
|
1828
|
+
if "_nc3_strict" in self.attrs._h5attrs:
|
|
1829
|
+
del self.attrs._h5attrs["_nc3_strict"]
|
|
1696
1830
|
|
|
1697
1831
|
sync = flush
|
|
1698
1832
|
|
|
1833
|
+
@property
|
|
1834
|
+
def _h5file(self):
|
|
1835
|
+
if self._closed:
|
|
1836
|
+
raise ValueError(f"I/O operation on {self}: {self._filename!r}")
|
|
1837
|
+
return self.__h5file
|
|
1838
|
+
|
|
1699
1839
|
def close(self):
|
|
1700
1840
|
if not self._closed:
|
|
1701
1841
|
self.flush()
|
|
1702
1842
|
if self._close_h5file:
|
|
1703
1843
|
self._h5file.close()
|
|
1704
|
-
self.
|
|
1844
|
+
self.__h5file = None
|
|
1705
1845
|
self._closed = True
|
|
1706
1846
|
|
|
1707
1847
|
__del__ = close
|
h5netcdf/dimensions.py
CHANGED
|
@@ -4,6 +4,8 @@ from collections.abc import MutableMapping
|
|
|
4
4
|
|
|
5
5
|
import numpy as np
|
|
6
6
|
|
|
7
|
+
from .utils import CompatibilityError
|
|
8
|
+
|
|
7
9
|
|
|
8
10
|
class Dimensions(MutableMapping):
|
|
9
11
|
def __init__(self, group):
|
|
@@ -23,9 +25,21 @@ class Dimensions(MutableMapping):
|
|
|
23
25
|
raise RuntimeError("H5NetCDF: Write to read only")
|
|
24
26
|
if name in self._objects:
|
|
25
27
|
raise ValueError(f"dimension {name!r} already exists")
|
|
28
|
+
if (
|
|
29
|
+
size in [0, None]
|
|
30
|
+
and self._unlimited()
|
|
31
|
+
and self._group._format == "NETCDF4_CLASSIC"
|
|
32
|
+
):
|
|
33
|
+
raise CompatibilityError(
|
|
34
|
+
"Only one unlimited dimension allowed in the NETCDF4_CLASSIC format."
|
|
35
|
+
)
|
|
26
36
|
|
|
27
37
|
self._objects[name] = Dimension(self._group, name, size, create_h5ds=True)
|
|
28
38
|
|
|
39
|
+
def _unlimited(self):
|
|
40
|
+
"""Return a tuple of unlimited dimensions."""
|
|
41
|
+
return tuple(dim for dim in self._objects.values() if dim.isunlimited())
|
|
42
|
+
|
|
29
43
|
def add_phony(self, name, size):
|
|
30
44
|
self._objects[name] = Dimension(
|
|
31
45
|
self._group, name, size, create_h5ds=False, phony=True
|
|
@@ -78,6 +92,7 @@ class Dimension:
|
|
|
78
92
|
self._h5path = _join_h5paths(parent.name, name)
|
|
79
93
|
self._name = name
|
|
80
94
|
self._size = 0 if size is None else size
|
|
95
|
+
|
|
81
96
|
if self._phony:
|
|
82
97
|
self._root._phony_dim_count += 1
|
|
83
98
|
else:
|
|
@@ -165,7 +180,7 @@ class Dimension:
|
|
|
165
180
|
"""Return dimension scale references"""
|
|
166
181
|
return list(self._h5ds.attrs.get("REFERENCE_LIST", []))
|
|
167
182
|
|
|
168
|
-
def _create_scale(self):
|
|
183
|
+
def _create_scale(self, dimid=None):
|
|
169
184
|
"""Create dimension scale for this dimension"""
|
|
170
185
|
if self._name not in self._parent._h5group:
|
|
171
186
|
kwargs = {}
|
|
@@ -179,7 +194,10 @@ class Dimension:
|
|
|
179
194
|
dtype=">f4",
|
|
180
195
|
**kwargs,
|
|
181
196
|
)
|
|
182
|
-
|
|
197
|
+
# fallback to init-time dimid
|
|
198
|
+
if dimid is None:
|
|
199
|
+
dimid = self._dimid
|
|
200
|
+
self._h5ds.attrs["_Netcdf4Dimid"] = np.array(dimid, dtype=np.int32)
|
|
183
201
|
|
|
184
202
|
if len(self._h5ds.shape) > 1:
|
|
185
203
|
dims = self._parent._variables[self._name].dimensions
|
|
@@ -188,10 +206,8 @@ class Dimension:
|
|
|
188
206
|
)
|
|
189
207
|
self._h5ds.attrs["_Netcdf4Coordinates"] = coord_ids
|
|
190
208
|
|
|
191
|
-
# need special handling for size in case of
|
|
209
|
+
# need special handling for size in case of tuple
|
|
192
210
|
size = self._size
|
|
193
|
-
if not size:
|
|
194
|
-
size = 1
|
|
195
211
|
if isinstance(size, tuple):
|
|
196
212
|
size = size[0]
|
|
197
213
|
dimlen = bytes(f"{size:10}", "ascii")
|
h5netcdf/tests/conftest.py
CHANGED
|
@@ -6,6 +6,8 @@ from shutil import rmtree
|
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
8
|
|
|
9
|
+
from h5netcdf.utils import h5dump as _h5dump
|
|
10
|
+
|
|
9
11
|
try:
|
|
10
12
|
from h5pyd import Folder
|
|
11
13
|
from hsds.hsds_app import HsdsApp
|
|
@@ -81,3 +83,13 @@ def hsds_up():
|
|
|
81
83
|
pass
|
|
82
84
|
|
|
83
85
|
rmtree(root_dir, ignore_errors=True)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@pytest.fixture
|
|
89
|
+
def h5dump():
|
|
90
|
+
return _h5dump
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@pytest.fixture(params=["NETCDF4", "NETCDF4_CLASSIC"])
|
|
94
|
+
def data_model(request):
|
|
95
|
+
return dict(format=request.param)
|