h5yaml 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

h5yaml/lib/chunksizes.py

@@ -35,20 +35,28 @@ def guess_chunks(dims: ArrayLike[int], dtype_sz: int) -> str | tuple[int]:
 
     """
     fixed_size = dtype_sz
-    for val in [x for x in dims if x > 0]:
-        fixed_size *= val
-
-    if 0 in dims:  # variable with an unlimited dimension
-        udim = dims.index(0)
-    else:  # variable has no unlimited dimension
-        udim = 0
-        if fixed_size < 65536:
+    if len(dims) > 1:
+        for val in [x for x in dims[1:] if x > 0]:
+            fixed_size *= val
+
+    # first variables without an unlimited dimension
+    if 0 not in dims:
+        if fixed_size < 400000:
             return "contiguous"
 
+        res = list(dims)
+        res[0] = max(1, 2048000 // fixed_size)
+        return tuple(res)
+
+    # then variables with an unlimited dimension
     if len(dims) == 1:
         return (1024,)
 
+    udim = dims.index(0)
     res = list(dims)
-    res[udim] = min(1024, (2048 * 1024) // (fixed_size // max(1, dims[0])))
+    if fixed_size < 400000:
+        res[udim] = 1024
+    else:
+        res[udim] = max(1, 2048000 // fixed_size)
 
     return tuple(res)

h5yaml/yaml_h5py.py

@@ -23,6 +23,8 @@ import numpy as np
 from h5yaml.conf_from_yaml import conf_from_yaml
 from h5yaml.lib.chunksizes import guess_chunks
 
+# - helper function ------------------------------------
+
 
 # - class definition -----------------------------------
 class H5Yaml:
@@ -53,22 +55,20 @@ class H5Yaml:
 
     def __dimensions(self: H5Yaml, fid: h5py.File) -> None:
         """Add dimensions to HDF5 product."""
-        for key, value in self.h5_def["dimensions"].items():
+        for key, val in self.h5_def["dimensions"].items():
             fillvalue = None
-            if "_FillValue" in value:
+            if "_FillValue" in val:
                 fillvalue = (
-                    np.nan if value["_FillValue"] == "NaN" else int(value["_FillValue"])
+                    np.nan if val["_FillValue"] == "NaN" else int(val["_FillValue"])
                 )
 
-            if value["_size"] == 0:
-                ds_chunk = value.get("_chunks", (50,))
+            if val["_size"] == 0:
+                ds_chunk = val.get("_chunks", (50,))
                 dset = fid.create_dataset(
                     key,
                     shape=(0,),
                     dtype=(
-                        h5py.string_dtype()
-                        if value["_dtype"] == "str"
-                        else value["_dtype"]
+                        h5py.string_dtype() if val["_dtype"] == "str" else val["_dtype"]
                     ),
                     chunks=ds_chunk if isinstance(ds_chunk, tuple) else tuple(ds_chunk),
                     maxshape=(None,),
@@ -77,21 +77,48 @@ class H5Yaml:
             else:
                 dset = fid.create_dataset(
                     key,
-                    shape=(value["_size"],),
-                    dtype=value["_dtype"],
+                    shape=(val["_size"],),
+                    dtype=val["_dtype"],
                 )
-                if "_values" in value:
-                    dset[:] = value["_values"]
+                if "_values" in val:
+                    dset[:] = val["_values"]
 
             dset.make_scale(
                 Path(key).name
-                if "long_name" in value
+                if "long_name" in val
                 else "This is a netCDF dimension but not a netCDF variable."
             )
-            for attr, attr_val in value.items():
+            for attr, attr_val in val.items():
                 if attr.startswith("_"):
                     continue
-                dset.attrs[attr] = attr_val
+                if attr in ("valid_min", "valid_max"):
+                    match val["_dtype"]:
+                        case "i1":
+                            dset.attrs[attr] = np.int8(attr_val)
+                        case "i2":
+                            dset.attrs[attr] = np.int16(attr_val)
+                        case "i4":
+                            dset.attrs[attr] = np.int32(attr_val)
+                        case "i8":
+                            dset.attrs[attr] = np.int64(attr_val)
+                        case "u1":
+                            dset.attrs[attr] = np.uint8(attr_val)
+                        case "u2":
+                            dset.attrs[attr] = np.uint16(attr_val)
+                        case "u4":
+                            dset.attrs[attr] = np.uint32(attr_val)
+                        case "u8":
+                            dset.attrs[attr] = np.uint64(attr_val)
+                        case "f2":
+                            dset.attrs[attr] = np.float16(attr_val)
+                        case "f4":
+                            dset.attrs[attr] = np.float32(attr_val)
+                        case "f8":
+                            dset.attrs[attr] = np.float64(attr_val)
+                        case _:
+                            dset.attrs[attr] = attr_val
+                else:
+                    dset.attrs[attr] = attr_val
 
     def __compounds(self: H5Yaml, fid: h5py.File) -> dict[str, str | int | float]:
         """Add compound datatypes to HDF5 product."""
@@ -112,14 +139,14 @@ class H5Yaml:
                 for key, value in res.items():
                     self.h5_def["compounds"][key] = value
 
-        for key, value in self.h5_def["compounds"].items():
+        for key, val in self.h5_def["compounds"].items():
             compounds[key] = {
                 "dtype": [],
                 "units": [],
                 "names": [],
             }
 
-            for _key, _val in value.items():
+            for _key, _val in val.items():
                 compounds[key]["dtype"].append((_key, _val[0]))
                 if len(_val) == 3:
                     compounds[key]["units"].append(_val[1])
@@ -156,12 +183,19 @@ class H5Yaml:
                     np.nan if val["_FillValue"] == "NaN" else int(val["_FillValue"])
                 )
 
-            compression = None
-            shuffle = False
-            # currently only gzip compression is supported
-            if "_compression" in val:
-                compression = val["_compression"]
-                shuffle = True
+            # check for scalar dataset
+            if val["_dims"][0] == "scalar":
+                dset = fid.create_dataset(
+                    key,
+                    (),
+                    dtype=ds_dtype,
+                    fillvalue=fillvalue,
+                )
+                for attr, attr_val in val.items():
+                    if attr.startswith("_"):
+                        continue
+                    dset.attrs[attr] = attr_val
+                continue
 
             n_udim = 0
             ds_shape = ()
@@ -194,6 +228,13 @@ class H5Yaml:
                     fillvalue=fillvalue,
                 )
             else:
+                compression = None
+                shuffle = False
+                # currently only gzip compression is supported
+                if "_compression" in val:
+                    compression = val["_compression"]
+                    shuffle = True
+
                 if val.get("_vlen"):
                     ds_dtype = h5py.vlen_dtype(ds_dtype)
                     fillvalue = None
@@ -217,7 +258,36 @@ class H5Yaml:
             for attr, attr_val in val.items():
                 if attr.startswith("_"):
                     continue
-                dset.attrs[attr] = attr_val
+                if attr in ("valid_min", "valid_max"):
+                    match val["_dtype"]:
+                        case "i1":
+                            dset.attrs[attr] = np.int8(attr_val)
+                        case "i2":
+                            dset.attrs[attr] = np.int16(attr_val)
+                        case "i4":
+                            dset.attrs[attr] = np.int32(attr_val)
+                        case "i8":
+                            dset.attrs[attr] = np.int64(attr_val)
+                        case "u1":
+                            dset.attrs[attr] = np.uint8(attr_val)
+                        case "u2":
+                            dset.attrs[attr] = np.uint16(attr_val)
+                        case "u4":
+                            dset.attrs[attr] = np.uint32(attr_val)
+                        case "u8":
+                            dset.attrs[attr] = np.uint64(attr_val)
+                        case "f2":
+                            dset.attrs[attr] = np.float16(attr_val)
+                        case "f4":
+                            dset.attrs[attr] = np.float32(attr_val)
+                        case "f8":
+                            dset.attrs[attr] = np.float64(attr_val)
+                        case _:
+                            dset.attrs[attr] = attr_val
+                elif attr == "flag_values":
+                    dset.attrs[attr] = np.array(attr_val, dtype="u1")
+                else:
+                    dset.attrs[attr] = attr_val
 
             if compounds is not None and val["_dtype"] in compounds:
                 if compounds[val["_dtype"]]["units"]:

h5yaml/yaml_nc.py

@@ -15,6 +15,7 @@ __all__ = ["NcYaml"]
 
 import logging
 from importlib.resources import files
+from pathlib import PurePosixPath
 from typing import TYPE_CHECKING
 
 import numpy as np
@@ -47,12 +48,20 @@ class NcYaml:
     def __groups(self: NcYaml, fid: Dataset) -> None:
         """Create groups in HDF5 product."""
         for key in self.h5_def["groups"]:
-            _ = fid.createGroup(key)
+            pkey = PurePosixPath(key)
+            if pkey.is_absolute():
+                _ = fid[pkey.parent].createGroup(pkey.name)
+            else:
+                _ = fid.createGroup(key)
 
     def __dimensions(self: NcYaml, fid: Dataset) -> None:
         """Add dimensions to HDF5 product."""
         for key, value in self.h5_def["dimensions"].items():
-            _ = fid.createDimension(key, value["_size"])
+            pkey = PurePosixPath(key)
+            if pkey.is_absolute():
+                _ = fid[pkey.parent].createDimension(pkey.name, value["_size"])
+            else:
+                _ = fid.createDimension(key, value["_size"])
 
             if "long_name" not in value:
                 continue
@@ -63,13 +72,22 @@ class NcYaml:
                     np.nan if value["_FillValue"] == "NaN" else int(value["_FillValue"])
                 )
 
-            dset = fid.createVariable(
-                key,
-                value["_dtype"],
-                dimensions=(key,),
-                fill_value=fillvalue,
-                contiguous=value["_size"] != 0,
-            )
+            if pkey.is_absolute():
+                dset = fid[pkey.parent].createVariable(
+                    pkey.name,
+                    value["_dtype"],
+                    dimensions=(pkey.name,),
+                    fill_value=fillvalue,
+                    contiguous=value["_size"] != 0,
+                )
+            else:
+                dset = fid.createVariable(
+                    key,
+                    value["_dtype"],
+                    dimensions=(key,),
+                    fill_value=fillvalue,
+                    contiguous=value["_size"] != 0,
+                )
             dset.setncatts({k: v for k, v in value.items() if not k.startswith("_")})
 
     def __compounds(self: NcYaml, fid: Dataset) -> dict[str, str | int | float]:
@@ -145,11 +163,17 @@ class NcYaml:
                 compression = "zlib"
                 complevel = val["_compression"]
 
+            var_dims = []
             n_udim = 0
             ds_shape = ()
             ds_maxshape = ()
             for coord in val["_dims"]:
-                dim_sz = fid.dimensions[coord].size
+                pcoord = PurePosixPath(coord)
+                var_dims.append(pcoord.name if pcoord.is_absolute() else coord)
+                if pcoord.is_absolute():
+                    dim_sz = fid[pcoord.parent].dimensions[pcoord.name].size
+                else:
+                    dim_sz = fid.dimensions[coord].size
                 n_udim += int(dim_sz == 0)
                 ds_shape += (dim_sz,)
                 ds_maxshape += (dim_sz if dim_sz > 0 else None,)
@@ -163,12 +187,18 @@ class NcYaml:
                 val["_chunks"] if "_chunks" in val else guess_chunks(ds_shape, sz_dtype)
             )
 
+            pkey = PurePosixPath(key)
+            var_grp = fid[pkey.parent] if pkey.is_absolute() else fid
+            var_name = pkey.name if pkey.is_absolute() else key
+            if val["_dtype"] in fid.cmptypes:
+                val["_dtype"] = fid.cmptypes[val["_dtype"]]
+
             # create the variable
             if ds_chunk == "contiguous":
-                dset = fid.createVariable(
-                    key,
+                dset = var_grp.createVariable(
+                    var_name,
                     val["_dtype"],
-                    dimensions=(key,),
+                    dimensions=var_dims,
                     fill_value=fillvalue,
                     contiguous=True,
                 )
@@ -181,13 +211,10 @@ class NcYaml:
                     if ds_maxshape == (None,):
                         ds_chunk = (16,)
 
-                if key in fid.cmptypes:
-                    val["_dtype"] = fid.cmptypes[key]
-
-                dset = fid.createVariable(
-                    key,
+                dset = var_grp.createVariable(
+                    var_name,
                     val["_dtype"],
-                    dimensions=val["_dims"],
+                    dimensions=var_dims,
                     fill_value=fillvalue,
                     contiguous=False,
                     compression=compression,

{h5yaml-0.0.3.dist-info → h5yaml-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: h5yaml
-Version: 0.0.3
+Version: 0.0.4
 Summary: Use YAML configuration file to generate HDF5/netCDF4 formated files.
 Project-URL: Homepage, https://github.com/rmvanhees/h5_yaml
 Project-URL: Source, https://github.com/rmvanhees/h5_yaml
@@ -14,28 +14,51 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
-Requires-Python: >=3.12
+Requires-Python: >=3.9
 Requires-Dist: h5py>=3.13
 Requires-Dist: netcdf4>=1.7
 Requires-Dist: numpy>=2.2
 Requires-Dist: pyyaml>=6.0
 Description-Content-Type: text/markdown
 
-# H5_YAML
+# H5YAML
+[![image](https://img.shields.io/pypi/v/h5yaml.svg?label=release)](https://github.com/rmvanhees/h5yaml/)
+[![image](https://img.shields.io/pypi/l/h5yaml.svg)](https://github.com/rmvanhees/h5yaml/LICENSE)
+[![image](https://img.shields.io/pypi/dm/h5yaml.svg)](https://pypi.org/project/h5yaml/)
+[![image](https://img.shields.io/pypi/status/h5yaml.svg?label=status)](https://pypi.org/project/h5yaml/)
 
 ## Description
-Use YAML configuration file to generate HDF5/netCDF4 formated files.
+This package let you generate [HDF5](https://docs.h5py.org/en/stable/)/[netCDF4](https://unidata.github.io/netcdf4-python/)
+formatted files as defined in a [YAML](https://yaml.org/) configuration file. This has several advantages: 
 
-The class `NcYaml` must be used when strict conformance to the netCDF4 format is
-required. However, the python netCDF4 implementation does not allow variable-length
-data to have a compound data-type. The class `H5Yaml` does not have this restiction
-and will generate HDF5 formated files which can be read by netCDF4 software.
+ * you define the layout of your HDF5/netCDF4 file using YAML which is human-readable and has intuitive syntax.
+ * you can reuse the YAML configuration file to to have all your product have a consistent layout.
+ * you can make updates by only changing the YAML configuration file
+ * you can have the layout of your HDF5/netCDF4 file as a python dictionary, thus without accessing any HDF5/netCDF4 file
+
+The `H5YAML` package has two classes to generate a HDF5/netCDF4 formatted file.
+
+ 1. The class `H5Yaml` uses the [h5py](https://pypi.org/project/h5py/) package, which is a Pythonic interface to
+    the HDF5 binary data format.
+    Let 'h5_def.yaml' be your YAML configuration file then ```H5Yaml("h5_def.yaml").create("foo.h5")``` will create
+	the HDF5 file 'foo.h5'. This can be read by netCDF4 software, because it uses dimension-scales to each dataset.
+ 2. The class `NcYaml` uses the [netCDF4](https://pypi.org/project/netCDF4/) package, which provides an object-oriented
+    python interface to the netCDF version 4 library.
+    Let 'nc_def.yaml' be your YAML configuration file then ```NcYaml("nc_def.yaml").create("foo.nc")``` will create
+	the netCDF4/HDF5 file 'foo.nc'
+
+The class `NcYaml` must be used when strict conformance to the netCDF4 format is required.
+However, package `netCDF4` has some limitations, which `h5py` has not, for example it does
+not allow variable-length variables to have a compound data-type.
 
 ## Installation
-Relases of the code, starting from version 0.1, will be made available via PyPi.
+Releases of the code, starting from version 0.1, will be made available via PyPI.
 
 ## Usage
 
@@ -54,7 +77,7 @@ The YAML file should be structured as follows:
      - science_data
    ```
 
- * The section 'dimensions' is obligatory, you shouold define the dimensions for each
+ * The section 'dimensions' is obligatory, you should define the dimensions for each
    variable in your file. The 'dimensions' section may look like this:
 
    ```
@@ -144,7 +167,7 @@ The YAML file should be structured as follows:
 ### Notes and ToDo:
 
  * The usage of older versions of h5py may result in broken netCDF4 files
- * Explain usage of parameter '_chunks', which is currently not correcly implemented.
+ * Explain usage of parameter '_chunks', which is currently not correctly implemented.
  * Explain that the usage of variable length data-sets may break netCDF4 compatibility
 
 ## Support [TBW]
@@ -161,6 +184,3 @@ The code is developed by R.M. van Hees (SRON)
 
 * Copyright: SRON (https://www.sron.nl).
 * License: BSD-3-clause
-
-## Project status
-Beta

h5yaml-0.0.4.dist-info/RECORD

@@ -0,0 +1,9 @@
+h5yaml/conf_from_yaml.py,sha256=FT5oS4yqDFUYZqgria_OrmMK52ZcrTVoAPyPkLrHstc,982
+h5yaml/yaml_h5py.py,sha256=MVPUqD5fNY3VYIi3ebaEY7uSR1sgYw-8DvIjrOWIaIk,11838
+h5yaml/yaml_nc.py,sha256=37NQ9FPuCrnOTN5R4otn0R-Sdwej1-cPwhODgY2L6pY,9202
+h5yaml/Data/h5_testing.yaml,sha256=_x3qBC8RNQ1h4c6B1JOH_y0L9DiDLsOEfyr60IcvpoI,2358
+h5yaml/lib/chunksizes.py,sha256=sAYiTTL8ecBivBDxw-XIJu2yMlt0zNHTr-KS7tBFp2o,1392
+h5yaml-0.0.4.dist-info/METADATA,sha256=Vvpv1h1t0cYvV5hBiXzdKe2CtU3TT2hWW0qwrIC8vLc,7160
+h5yaml-0.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+h5yaml-0.0.4.dist-info/licenses/LICENSE,sha256=MoOwtPnC77nFaIwRIAIE6fKhrzMd3G18mOXDPtAH8G0,1509
+h5yaml-0.0.4.dist-info/RECORD,,

h5yaml-0.0.3.dist-info/RECORD

@@ -1,9 +0,0 @@
-h5yaml/conf_from_yaml.py,sha256=FT5oS4yqDFUYZqgria_OrmMK52ZcrTVoAPyPkLrHstc,982
-h5yaml/yaml_h5py.py,sha256=m5vOdVour3FwnUjewGukIra6C_c0F61yqnsvUJN-KtM,8591
-h5yaml/yaml_nc.py,sha256=KG2y497If2tJmSSmF6bvXl7ePKorptEUEJULPexBIQw,7980
-h5yaml/Data/h5_testing.yaml,sha256=_x3qBC8RNQ1h4c6B1JOH_y0L9DiDLsOEfyr60IcvpoI,2358
-h5yaml/lib/chunksizes.py,sha256=aOXkLqTk5GgE-uk80QqHYbLB-FzBMnOrMm6ixH4QAUc,1225
-h5yaml-0.0.3.dist-info/METADATA,sha256=CQm7-Uxop-K7v5-WotW6DiKFPtYJXwEi9OJfJKC0o7I,5370
-h5yaml-0.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-h5yaml-0.0.3.dist-info/licenses/LICENSE,sha256=MoOwtPnC77nFaIwRIAIE6fKhrzMd3G18mOXDPtAH8G0,1509
-h5yaml-0.0.3.dist-info/RECORD,,