h5yaml 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

h5yaml/Data/h5_compound.yaml

@@ -19,18 +19,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-stats_dtype:
-  time:
-    - u8
-    - seconds since 1970-01-01T00:00:00
-    - timestamp
-  index: [u2, '1', index]
-  tbl_id: [u1, '1', binning id]
-  saa: [u1, '1', saa-flag]
-  coad: [u1, '1', co-addings]
-  texp: [f4, ms, exposure time]
-  lat: [f4, degree, latitude]
-  lon: [f4, degree, longitude]
-  avg: [f4, '1', '$S - S_{ref}$']
-  unc: [f4, '1', '\u03c3($S - S_{ref}$)']
-  dark_offs: [f4, '1', dark-offset]
+compounds:
+  stats_dtype:
+    time:
+      - u8
+      - seconds since 1970-01-01T00:00:00
+      - timestamp
+    index: [u2, '1', index]
+    tbl_id: [u1, '1', binning id]
+    saa: [u1, '1', saa-flag]
+    coad: [u1, '1', co-addings]
+    texp: [f4, ms, exposure time]
+    lat: [f4, degree, latitude]
+    lon: [f4, degree, longitude]
+    avg: [f4, '1', '$S - S_{ref}$']
+    unc: [f4, '1', '\u03c3($S - S_{ref}$)']
+    dark_offs: [f4, '1', dark-offset]

h5yaml/Data/h5_testing.yaml

@@ -25,6 +25,8 @@ groups:
   - group_01
   - group_02
   - group_03
+  - processing_control
+  - processing_control/input_parameters
 
 # Define dimensions
 # Note dimensions with an attribute 'long_name' will also be generated as variable
@@ -91,6 +93,26 @@ variables:
     units: '1'
     valid_min: 0
     valid_max: 65534
+  /group_00/detector_images_chunked:
+    _dtype: u2
+    _dims: [number_of_images, column, row]
+    _FillValue: 65535
+    _chunks: [1, 640, 512]
+    long_name: Detector pixel values
+    comment: unbinned full-frame data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
+  /group_00/detector_images_autochunk:
+    _dtype: u2
+    _dims: [number_of_images, column, row]
+    _chunks: True
+    _FillValue: 65535
+    long_name: Detector pixel values
+    comment: unbinned full-frame data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
   # ---------- GROUP 01 ----------
   /group_01/detector_images:
     _dtype: u2
@@ -199,7 +221,7 @@ variables:
     valid_max: 999.9
   /group_03/ds_10:
     _dtype: f4
-    _dims: [number_of_images]
+    _dims: [number_of_images, /group_03/viewport]
     long_name: float dataset
     units: '1'
     valid_min: -999.9

h5yaml/Data/h5_unsupported.yaml

@@ -39,12 +39,6 @@ dimensions:
     valid_min: 0
     valid_max: 92400
     
-# Define compound types
-# - compound elements must have a data-type, and can have a unit and long_name 
-compounds:
-  - h5_compound.yaml
-  - h5_nocompound.yaml
-
 # Define variables
 variables:
   ds_01:

h5yaml/Data/nc_testing.yaml

@@ -25,6 +25,8 @@ groups:
   - group_01
   - group_02
   - group_03
+  - processing_control
+  - processing_control/input_parameters
 
 # Define dimensions
 # Note dimensions with an attribute 'long_name' will also be generated as variable
@@ -87,6 +89,16 @@ variables:
     units: '1'
     valid_min: 0
     valid_max: 65534
+  /group_00/detector_images_chunked:
+    _dtype: u2
+    _dims: [number_of_images, column, row]
+    _FillValue: 65535
+    _chunks: [1, 640, 512]
+    long_name: Detector pixel values
+    comment: unbinned full-frame data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
   # ---------- GROUP 01 ----------
   /group_01/detector_images:
     _dtype: u2
@@ -190,7 +202,7 @@ variables:
   #  valid_max: 999.9
   /group_03/ds_10:
     _dtype: f4
-    _dims: [number_of_images]
+    _dims: [number_of_images, /group_03/viewport]
     long_name: float dataset
     units: '1'
     valid_min: -999.9

h5yaml/conf_from_yaml.py

@@ -45,7 +45,7 @@ def conf_from_yaml(file_path: Path | str) -> dict:
 
     """
     if isinstance(file_path, str):
-        file_path = Path(str)
+        file_path = Path(file_path)
 
     if not file_path.is_file():
         raise FileNotFoundError(f"{file_path} not found")
@@ -53,7 +53,7 @@ def conf_from_yaml(file_path: Path | str) -> dict:
     with file_path.open("r", encoding="ascii") as fid:
         try:
             settings = yaml.safe_load(fid)
-        except yaml.YAMLError as exc:
-            raise RuntimeError from exc
+        except yaml.parser.ParserError as exc:
+            raise RuntimeError(f"Failed to parse {file_path}") from exc
 
     return settings

h5yaml/yaml_h5.py

@@ -32,7 +32,6 @@ import numpy as np
 
 from .conf_from_yaml import conf_from_yaml
 from .lib.adjust_attr import adjust_attr
-from .lib.chunksizes import guess_chunks
 
 
 # - class definition -----------------------------------
@@ -41,30 +40,74 @@ class H5Yaml:
 
     Parameters
     ----------
-    h5_yaml_fl :  Path
+    h5_yaml_fl :  Path | str | list[Path | str]
        YAML files with the HDF5 format definition
 
     """
 
-    def __init__(self: H5Yaml, h5_yaml_fl: Path) -> None:
+    def __init__(self: H5Yaml, h5_yaml_fl: Path | str | list[Path | str]) -> None:
         """Construct a H5Yaml instance."""
         self.logger = logging.getLogger("h5yaml.H5Yaml")
+        self._h5_def = {
+            "groups": set(),
+            "attrs_global": {},
+            "attrs_groups": {},
+            "compounds": {},
+            "dimensions": {},
+            "variables": {},
+        }
+
+        for yaml_fl in h5_yaml_fl if isinstance(h5_yaml_fl, list) else [h5_yaml_fl]:
+            try:
+                config = conf_from_yaml(yaml_fl)
+            except RuntimeError as exc:
+                raise RuntimeError from exc
+
+            for key in self._h5_def:
+                if key in config:
+                    self._h5_def[key] |= (
+                        set(config[key]) if key == "groups" else config[key]
+                    )
+
+    def __attrs(self: H5Yaml, fid: h5py.File) -> None:
+        """Create global and group attributes.
 
-        try:
-            self._h5_def = conf_from_yaml(h5_yaml_fl)
-        except RuntimeError as exc:
-            raise RuntimeError from exc
+        Parameters
+        ----------
+        fid :  h5py.File
+           HDF5 file pointer (mode 'r+')
+
+        """
+        for key, value in self._h5_def["attrs_global"].items():
+            if key not in fid.attrs and value != "TBW":
+                fid.attrs[key] = value
 
-        self.yaml_dir = h5_yaml_fl.parent
+        for key, value in self._h5_def["attrs_groups"].items():
+            if key not in fid.attrs and value != "TBW":
+                fid[str(Path(key).parent)].attrs[Path(key).name] = value
 
     def __groups(self: H5Yaml, fid: h5py.File) -> None:
-        """Create groups in HDF5 product."""
-        for key in self.h5_def["groups"]:
-            _ = fid.create_group(key)
+        """Create groups in HDF5 product.
+
+        Parameters
+        ----------
+        fid :  h5py.File
+           HDF5 file pointer (mode 'r+')
+
+        """
+        for key in self._h5_def["groups"]:
+            _ = fid.require_group(key)
 
     def __dimensions(self: H5Yaml, fid: h5py.File) -> None:
-        """Add dimensions to HDF5 product."""
-        for key, val in self.h5_def["dimensions"].items():
+        """Add dimensions to HDF5 product.
+
+        Parameters
+        ----------
+        fid :  h5py.File
+           HDF5 file pointer (mode 'r+')
+
+        """
+        for key, val in self._h5_def["dimensions"].items():
             fillvalue = None
             if "_FillValue" in val:
                 fillvalue = (
@@ -72,12 +115,14 @@ class H5Yaml:
                 )
 
             if val["_size"] == 0:
-                ds_chunk = val.get("_chunks", (50,))
+                ds_chunk = val.get("_chunks")
+                if ds_chunk is not None and not isinstance(ds_chunk, bool):
+                    ds_chunk = tuple(ds_chunk)
                 dset = fid.create_dataset(
                     key,
                     shape=(0,),
                     dtype="T" if val["_dtype"] == "str" else val["_dtype"],
-                    chunks=ds_chunk if isinstance(ds_chunk, tuple) else tuple(ds_chunk),
+                    chunks=ds_chunk,
                     maxshape=(None,),
                     fillvalue=fillvalue,
                 )
@@ -101,62 +146,32 @@ class H5Yaml:
                 if not attr.startswith("_"):
                     dset.attrs[attr] = adjust_attr(val["_dtype"], attr, attr_val)
 
-    def __compounds(self: H5Yaml, fid: h5py.File) -> dict[str, str | int | float]:
-        """Add compound datatypes to HDF5 product."""
-        if "compounds" not in self.h5_def:
-            return {}
-
-        compounds = {}
-        if isinstance(self.h5_def["compounds"], list):
-            file_list = self.h5_def["compounds"].copy()
-            self.h5_def["compounds"] = {}
-            for name in file_list:
-                if not (yaml_fl := self.yaml_dir / name).is_file():
-                    continue
-                try:
-                    res = conf_from_yaml(yaml_fl)
-                except RuntimeError as exc:
-                    raise RuntimeError from exc
-                for key, value in res.items():
-                    self.h5_def["compounds"][key] = value
-
-        for key, val in self.h5_def["compounds"].items():
-            compounds[key] = {
-                "dtype": [],
-                "units": [],
-                "names": [],
-            }
-
-            for _key, _val in val.items():
-                compounds[key]["dtype"].append((_key, _val[0]))
-                if len(_val) == 3:
-                    compounds[key]["units"].append(_val[1])
-                compounds[key]["names"].append(_val[2] if len(_val) == 3 else _val[1])
-
-            fid[key] = np.dtype(compounds[key]["dtype"])
-
-        return compounds
-
-    def __variables(
-        self: H5Yaml, fid: h5py.File, compounds: dict[str, str | int | float] | None
-    ) -> None:
+    def __compounds(self: H5Yaml, fid: h5py.File) -> None:
+        """Add compound datatypes to HDF5 product.
+
+        Parameters
+        ----------
+        fid :  h5py.File
+           HDF5 file pointer (mode 'r+')
+
+        """
+        for key, val in self._h5_def["compounds"].items():
+            fid[key] = np.dtype([(k, v[0]) for k, v in val.items()])
+
+    def __variables(self: H5Yaml, fid: h5py.File) -> None:
         """Add datasets to HDF5 product.
 
         Parameters
         ----------
         fid :  h5py.File
            HDF5 file pointer (mode 'r+')
-        compounds :  dict[str, str | int | float]
-           Definition of the compound(s) in the product
 
         """
-        for key, val in self.h5_def["variables"].items():
+        for key, val in self._h5_def["variables"].items():
             if val["_dtype"] in fid:
                 ds_dtype = fid[val["_dtype"]]
-                dtype_size = fid[val["_dtype"]].dtype.itemsize
             else:
                 ds_dtype = "T" if val["_dtype"] == "str" else val["_dtype"]
-                dtype_size = np.dtype(val["_dtype"]).itemsize
 
             fillvalue = None
             if "_FillValue" in val:
@@ -190,15 +205,13 @@ class H5Yaml:
             if n_udim > 1:
                 raise ValueError(f"{key} has more than one unlimited dimension")
 
-            # obtain chunk-size settings
-            ds_chunk = (
-                val["_chunks"]
-                if "_chunks" in val
-                else guess_chunks(ds_shape, dtype_size)
-            )
+            if None in ds_maxshape and val.get("_chunks") == "contiguous":
+                raise KeyError(
+                    "you can not create a contiguous dataset with unlimited dimensions."
+                )
 
             # create the variable
-            if ds_chunk == "contiguous":
+            if val.get("_chunks") == "contiguous":
                 dset = fid.create_dataset(
                     key,
                     ds_shape,
@@ -208,6 +221,9 @@ class H5Yaml:
                     fillvalue=fillvalue,
                 )
             else:
+                ds_chunk = val.get("_chunks")
+                if ds_chunk is not None and not isinstance(ds_chunk, bool):
+                    ds_chunk = tuple(ds_chunk)
                 compression = None
                 shuffle = False
                 # currently only gzip compression is supported
@@ -225,14 +241,12 @@ class H5Yaml:
                         fid[ds_name] = h5py.vlen_dtype(ds_dtype)
                     ds_dtype = fid[ds_name]
                     fillvalue = None
-                    if ds_maxshape == (None,):
-                        ds_chunk = (16,)
 
                 dset = fid.create_dataset(
                     key,
                     ds_shape,
                     dtype=ds_dtype,
-                    chunks=ds_chunk if isinstance(ds_chunk, tuple) else tuple(ds_chunk),
+                    chunks=ds_chunk,
                     maxshape=ds_maxshape,
                     fillvalue=fillvalue,
                     compression=compression,
@@ -246,11 +260,14 @@ class H5Yaml:
                 if not attr.startswith("_"):
                     dset.attrs[attr] = adjust_attr(val["_dtype"], attr, attr_val)
 
-            if compounds is not None and val["_dtype"] in compounds:
-                if compounds[val["_dtype"]]["units"]:
-                    dset.attrs["units"] = compounds[val["_dtype"]]["units"]
-                if compounds[val["_dtype"]]["names"]:
-                    dset.attrs["long_name"] = compounds[val["_dtype"]]["names"]
+            if val["_dtype"] in self._h5_def["compounds"]:
+                compound = self._h5_def["compounds"][val["_dtype"]]
+                res = [v[2] for k, v in compound.items() if len(v) == 3]
+                if res:
+                    dset.attrs["units"] = [v[1] for k, v in compound.items()]
+                    dset.attrs["long_name"] = res
+                else:
+                    dset.attrs["long_name"] = [v[1] for k, v in compound.items()]
 
     @property
     def h5_def(self: H5Yaml) -> dict:
@@ -260,10 +277,11 @@ class H5Yaml:
     def diskless(self: H5Yaml) -> h5py.File:
         """Create a HDF5/netCDF4 file in memory."""
         fid = h5py.File.in_memory()
-        if "groups" in self.h5_def:
-            self.__groups(fid)
+        self.__groups(fid)
         self.__dimensions(fid)
-        self.__variables(fid, self.__compounds(fid))
+        self.__compounds(fid)
+        self.__variables(fid)
+        self.__attrs(fid)
         return fid
 
     def create(self: H5Yaml, l1a_name: Path | str) -> None:
@@ -277,9 +295,10 @@ class H5Yaml:
         """
         try:
             with h5py.File(l1a_name, "w") as fid:
-                if "groups" in self.h5_def:
-                    self.__groups(fid)
+                self.__groups(fid)
                 self.__dimensions(fid)
-                self.__variables(fid, self.__compounds(fid))
+                self.__compounds(fid)
+                self.__variables(fid)
+                self.__attrs(fid)
         except PermissionError as exc:
             raise RuntimeError(f"failed create {l1a_name}") from exc

h5yaml/yaml_nc.py

@@ -18,15 +18,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-"""Create HDF5/netCDF4 formatted file from a YAML configuration file using netCDF4."""
+"""Create netCDF4 formatted file from a YAML configuration file using netCDF4."""
 
 from __future__ import annotations
 
 __all__ = ["NcYaml"]
 
 import logging
-from pathlib import PurePosixPath
-from typing import TYPE_CHECKING
+from pathlib import Path, PurePosixPath
 
 import numpy as np
 
@@ -35,38 +34,91 @@ from netCDF4 import Dataset
 
 from .conf_from_yaml import conf_from_yaml
 from .lib.adjust_attr import adjust_attr
-from .lib.chunksizes import guess_chunks
-
-if TYPE_CHECKING:
-    from pathlib import Path
 
 
 # - class definition -----------------------------------
 class NcYaml:
-    """Class to create a HDF5/netCDF4 formated file from a YAML configuration file."""
+    """Class to create a netCDF4 formated file from a YAML configuration file.
+
+    Parameters
+    ----------
+    nc_yaml_fl :  Path | str | list[Path | str]
+       YAML files with the netCDF4 format definition
+
+    """
 
-    def __init__(self: NcYaml, nc_yaml_fl: Path) -> None:
+    def __init__(self: NcYaml, nc_yaml_fl: Path | str | list[Path | str]) -> None:
         """Construct a NcYaml instance."""
         self.logger = logging.getLogger("h5yaml.NcYaml")
+        self._nc_def = {
+            "groups": set(),
+            "attrs_global": {},
+            "attrs_groups": {},
+            "compounds": {},
+            "dimensions": {},
+            "variables": {},
+        }
+
+        for yaml_fl in nc_yaml_fl if isinstance(nc_yaml_fl, list) else [nc_yaml_fl]:
+            try:
+                config = conf_from_yaml(yaml_fl)
+            except RuntimeError as exc:
+                raise RuntimeError from exc
+
+            for key in self._nc_def:
+                if key in config:
+                    self._nc_def[key] |= (
+                        set(config[key]) if key == "groups" else config[key]
+                    )
+
+    def __attrs(self: NcYaml, fid: Dataset) -> None:
+        """Create global and group attributes.
 
-        try:
-            self._nc_def = conf_from_yaml(nc_yaml_fl)
-        except RuntimeError as exc:
-            raise RuntimeError from exc
+        Parameters
+        ----------
+        fid :  netCDF4.Dataset
+           netCDF4 Dataset (mode 'r+')
 
-        self.yaml_dir = nc_yaml_fl.parent
+        """
+        for key, val in self.nc_def["attrs_global"].items():
+            if val == "TBW":
+                continue
+
+            if isinstance(val, str):
+                fid.setncattr_string(key, val)
+            else:
+                fid.setncattr(key, val)
+
+        for key, val in self.nc_def["attrs_groups"].items():
+            if val == "TBW":
+                continue
+
+            if isinstance(val, str):
+                fid[str(Path(key).parent)].setncattr_string(Path(key).name, val)
+            else:
+                fid[str(Path(key).parent)].setncattr(Path(key).name, val)
 
     def __groups(self: NcYaml, fid: Dataset) -> None:
-        """Create groups in HDF5 product."""
+        """Create groups in a netCDF4 product.
+
+        Parameters
+        ----------
+        fid :  netCDF4.Dataset
+           netCDF4 Dataset (mode 'r+')
+
+        """
         for key in self.nc_def["groups"]:
-            pkey = PurePosixPath(key)
-            if pkey.is_absolute():
-                _ = fid[pkey.parent].createGroup(pkey.name)
-            else:
-                _ = fid.createGroup(key)
+            _ = fid.createGroup(key)
 
     def __dimensions(self: NcYaml, fid: Dataset) -> None:
-        """Add dimensions to HDF5 product."""
+        """Add dimensions to a netCDF4 product.
+
+        Parameters
+        ----------
+        fid :  netCDF4.Dataset
+           netCDF4 Dataset (mode 'r+')
+
+        """
         for key, value in self.nc_def["dimensions"].items():
             pkey = PurePosixPath(key)
             if pkey.is_absolute():
@@ -113,56 +165,26 @@ class NcYaml:
                 }
             )
 
-    def __compounds(self: NcYaml, fid: Dataset) -> dict[str, str | int | float]:
-        """Add compound datatypes to HDF5 product."""
-        if "compounds" not in self.nc_def:
-            return {}
-
-        compounds = {}
-        if isinstance(self.nc_def["compounds"], list):
-            file_list = self.nc_def["compounds"].copy()
-            self.nc_def["compounds"] = {}
-            for name in file_list:
-                if not (yaml_fl := self.yaml_dir / name).is_file():
-                    continue
-                try:
-                    res = conf_from_yaml(yaml_fl)
-                except RuntimeError as exc:
-                    raise RuntimeError from exc
-                for key, value in res.items():
-                    self.nc_def["compounds"][key] = value
-
-        for key, value in self.nc_def["compounds"].items():
-            compounds[key] = {
-                "dtype": [],
-                "units": [],
-                "names": [],
-            }
-
-            for _key, _val in value.items():
-                compounds[key]["dtype"].append((_key, _val[0]))
-                if len(_val) == 3:
-                    compounds[key]["units"].append(_val[1])
-                compounds[key]["names"].append(_val[2] if len(_val) == 3 else _val[1])
-
-            comp_t = np.dtype(compounds[key]["dtype"])
-            _ = fid.createCompoundType(comp_t, key)
+    def __compounds(self: NcYaml, fid: Dataset) -> None:
+        """Add compound datatypes to a netCDF4 product.
 
-        return compounds
+        Parameters
+        ----------
+        fid :  netCDF4.Dataset
+           netCDF4 Dataset (mode 'r+')
+
+        """
+        for key, val in self.nc_def["compounds"].items():
+            comp_t = np.dtype([(k, v[0]) for k, v in val.items()])
+            _ = fid.createCompoundType(comp_t, key)
 
-    def __variables(
-        self: NcYaml,
-        fid: Dataset,
-        compounds: dict[str, str | int | float] | None,
-    ) -> None:
-        """Add datasets to HDF5 product.
+    def __variables(self: NcYaml, fid: Dataset) -> None:
+        """Add datasets to a netCDF4 product.
 
         Parameters
         ----------
         fid :  netCDF4.Dataset
-           HDF5 file pointer (mode 'r+')
-        compounds :  dict[str, str | int | float]
-           Definition of the compound(s) in the product
+           netCDF4 Dataset (mode 'r+')
 
         """
         for key, val in self.nc_def["variables"].items():
@@ -172,10 +194,8 @@ class NcYaml:
 
             if val["_dtype"] in fid.cmptypes:
                 ds_dtype = fid.cmptypes[val["_dtype"]].dtype
-                sz_dtype = ds_dtype.itemsize
             else:
                 ds_dtype = val["_dtype"]
-                sz_dtype = np.dtype(val["_dtype"]).itemsize
 
             fillvalue = None
             if "_FillValue" in val:
@@ -226,16 +246,16 @@ class NcYaml:
             if n_udim > 1:
                 raise ValueError("more than one unlimited dimension")
 
-            # obtain chunk-size settings
-            ds_chunk = (
-                val["_chunks"] if "_chunks" in val else guess_chunks(ds_shape, sz_dtype)
-            )
+            if None in ds_maxshape and val.get("_chunks") == "contiguous":
+                raise KeyError(
+                    "you can not create a contiguous dataset with unlimited dimensions."
+                )
 
             if val["_dtype"] in fid.cmptypes:
                 val["_dtype"] = fid.cmptypes[val["_dtype"]]
 
             # create the variable
-            if ds_chunk == "contiguous":
+            if val.get("_chunks") == "contiguous":
                 dset = var_grp.createVariable(
                     var_name,
                     val["_dtype"],
@@ -244,25 +264,24 @@ class NcYaml:
                     contiguous=True,
                 )
             else:
+                ds_chunk = val.get("_chunks")
+                if ds_chunk is not None and not isinstance(ds_chunk, bool):
+                    ds_chunk = tuple(ds_chunk)
                 if val.get("_vlen"):
                     if val["_dtype"] in fid.cmptypes:
                         raise ValueError("can not have vlen with compounds")
                     val["_dtype"] = fid.createVLType(ds_dtype, val["_dtype"])
                     fillvalue = None
-                    if ds_maxshape == (None,):
-                        ds_chunk = (16,)
 
                 dset = var_grp.createVariable(
                     var_name,
                     str if val["_dtype"] == "str" else val["_dtype"],
                     dimensions=var_dims,
                     fill_value=fillvalue,
-                    contiguous=False,
                     compression=compression,
                     complevel=complevel,
-                    chunksizes=(
-                        ds_chunk if isinstance(ds_chunk, tuple) else tuple(ds_chunk)
-                    ),
+                    chunksizes=ds_chunk,
+                    contiguous=False,
                 )
             dset.setncatts(
                 {
@@ -272,38 +291,45 @@ class NcYaml:
                 }
             )
 
-            if compounds is not None and val["_dtype"] in compounds:
-                if compounds[val["_dtype"]]["units"]:
-                    dset.attrs["units"] = compounds[val["_dtype"]]["units"]
-                if compounds[val["_dtype"]]["names"]:
-                    dset.attrs["long_name"] = compounds[val["_dtype"]]["names"]
+            if val["_dtype"] in self._nc_def["compounds"]:
+                compound = self._nc_def["compounds"][val["_dtype"]]
+                res = [v[2] for k, v in compound.items() if len(v) == 3]
+                if res:
+                    dset.attrs["units"] = [v[1] for k, v in compound.items()]
+                    dset.attrs["long_name"] = res
+                else:
+                    dset.attrs["long_name"] = [v[1] for k, v in compound.items()]
 
     @property
     def nc_def(self: NcYaml) -> dict:
-        """Return definition of the HDF5/netCDF4 product."""
+        """Return definition of the netCDF4 product."""
         return self._nc_def
 
     def diskless(self: NcYaml) -> Dataset:
-        """Create a HDF5/netCDF4 file in memory."""
+        """Create a netCDF4 file in memory."""
         fid = Dataset("diskless_test.nc", "w", diskless=True, persistent=False)
         self.__groups(fid)
         self.__dimensions(fid)
-        self.__variables(fid, self.__compounds(fid))
+        self.__compounds(fid)
+        self.__variables(fid)
+        self.__attrs(fid)
         return fid
 
     def create(self: NcYaml, l1a_name: Path | str) -> None:
-        """Create a HDF5/netCDF4 file (overwrite if exist).
+        """Create a netCDF4 file (overwrite if exist).
 
         Parameters
         ----------
         l1a_name :  Path | str
-           Full name of the HDF5/netCDF4 file to be generated
+           Full name of the netCDF4 file to be generated
 
         """
         try:
             with Dataset(l1a_name, "w") as fid:
                 self.__groups(fid)
                 self.__dimensions(fid)
-                self.__variables(fid, self.__compounds(fid))
+                self.__compounds(fid)
+                self.__variables(fid)
+                self.__attrs(fid)
         except PermissionError as exc:
             raise RuntimeError(f"failed to create {l1a_name}") from exc

{h5yaml-0.2.0.dist-info → h5yaml-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: h5yaml
-Version: 0.2.0
+Version: 0.3.0
 Summary: Use YAML configuration file to generate HDF5/netCDF4 formated files.
 Project-URL: Homepage, https://github.com/rmvanhees/h5_yaml
 Project-URL: Source, https://github.com/rmvanhees/h5_yaml
@@ -61,23 +61,39 @@ However, package `netCDF4` has some limitations, which `h5py` has not, for examp
 not allow variable-length variables to have a compound data-type.
 
 ## Installation
-Releases of the code, starting from version 0.1, will be made available via PyPI.
+The package `h5yaml` is available from PyPI. To install it use `pip`:
+
+> $ pip install [--user] h5yaml
+
+The module `h5yaml` requires Python3.10+ and Python modules: h5py (v3.14+), netCDF4 (v1.7+) and numpy (v2.0+).
+
+**Note**: the packages `h5py` and `netCDF4` come with their own HDF5 libraries. If these are different then they may
+collide and result in a *''HDF5 error''*.
+If this is the case then you have to install the development packages of HDF5 and netCDF4 (or compile them from source).
+And reinstall `h5py` and `netCDF4` using the commands:
+
+> $ pip uninstall h5py; pip install --no-binary=h5py h5py
+> $ pip uninstall netCDF4; pip install --no-binary=netCDF4 netCDF4
 
 ## Usage
 
 The YAML file should be structured as follows:
 
- * The top level are: 'groups', 'dimensions', 'compounds' and 'variables'
+ * The top level are: 'groups', 'dimensions', 'compounds', 'variables', 'attrs\_global' and 'attrs\_groups'.
+ * > 'attrs\_global' and 'attrs\_groups' are added in version 0.3.0
+ * The names of the attributes, groups, dimensions, compounds and viariable should be specified as PosixPaths, however:
+   * The names of groups should never start with a slash (always erlative to root);
+   * All other elements which are stored in root should also not start with a slash;
+   * But these elements require a starting slash (absolute paths) when they are stored not the root. 
  * The section 'groups' are optional, but you should provide each group you want to use
    in your file. The 'groups' section in the YAML file may look like this:
-
    ```
    groups:
      - engineering_data
      - image_attributes
      - navigation_data
-     - processing_control
      - science_data
+     - processing_control/input_data
    ```
 
  * The section 'dimensions' is obligatory, you should define the dimensions for each
@@ -133,14 +149,6 @@ The YAML file should be structured as follows:
        dark_offs: [f4, '1', dark-offset]
    ```
 
-   Alternatively, provide a list with names of YAML files which contain the definitions
-   of the compounds.
-
-   ```
-   compounds:
-     - h5_nomhk_tm.yaml
-     - h5_science_hk.yaml
-   ```
  * The 'variables' are defined by their data-type ('_dtype') and dimensions ('_dims'),
    and optionally chunk sizes ('_chunks'), compression ('_compression'), variable length
    ('_vlen'). In addition, each variable can have as many attributes as you like,
@@ -148,6 +156,16 @@ The YAML file should be structured as follows:
 
    ```
    variables:
+     /science_data/detector_images:
+       _dtype: u2
+       _dims: [number_of_images, samples_per_image]
+	   _compression: 3
+       _FillValue: 65535
+       long_name: Detector pixel values
+       coverage_content_type: image
+       units: '1'
+       valid_min: 0
+       valid_max: 65534
      /image_attributes/nr_coadditions:
        _dtype: u2
        _dims: [number_of_images]
@@ -163,20 +181,18 @@ The YAML file should be structured as follows:
        units: seconds
      stats_163:
        _dtype: stats_dtype
-       _vlen: True
        _dims: [days]
+       _vlen: True
        comment: detector map statistics (MPS=163)
    ```
 
-### Notes and ToDo:
+### Notes and ToDo
 
- * The usage of older versions of h5py may result in broken netCDF4 files
- * Explain usage of parameter '_chunks', which is currently not correctly implemented.
- * Explain that the usage of variable length data-sets may break netCDF4 compatibility
+* The layout of a HDF5 or netCDF4 file can be complex. From version 0.3.0, you can split the file definition over several YAML files and provide a list with the names of YAML files as input to H5Yaml and NcYaml.  
 
 ## Support [TBW]
 
-## Roadmap
+## Road map
 
  * Release v0.1 : stable API to read your YAML files and generate the HDF5/netCDF4 file
 

h5yaml-0.3.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+h5yaml/__init__.py,sha256=NdNciPgYnbq-aVM6QqNGNZtdL72rTGLAMrDy0Yw7ckk,751
+h5yaml/conf_from_yaml.py,sha256=GVbWR-I0_sKRxrXmgLxbnTJvAXz5OtFtNYu4Pp3LaaI,1607
+h5yaml/yaml_h5.py,sha256=R_WqvK2korrR_nCY7MlmycRcD5Hc4yeJC6B4rvuwulk,10399
+h5yaml/yaml_nc.py,sha256=G4kvn_Ec72ND8phXgNbFWY7EML-pgiMcBVwZroPtrQY,11330
+h5yaml/Data/h5_compound.yaml,sha256=pAVGyhGpbbFgsb1NoTQZsttPLK1zktTZRufFdSaX78U,1172
+h5yaml/Data/h5_testing.yaml,sha256=s-kUjHiXKr4IOVf2vqz8mUr1vcU61wxKwZFuQCUbemA,6246
+h5yaml/Data/h5_unsupported.yaml,sha256=EfFztuUpuXDl_7wgwIqelwE_gdvu35zKT-YtsUfGQeM,1342
+h5yaml/Data/nc_testing.yaml,sha256=C30hXo73GG4BifIr7oymkbJ4Bh8hxKKDDMnFOO5VED8,5806
+h5yaml/lib/__init__.py,sha256=NdNciPgYnbq-aVM6QqNGNZtdL72rTGLAMrDy0Yw7ckk,751
+h5yaml/lib/adjust_attr.py,sha256=4dHEGwwIa3a3hihyuSX8jCsC08fYcz_9XWA1pBwiwfc,2284
+h5yaml-0.3.0.dist-info/METADATA,sha256=jFRcsa-2kTu8QIFtKmKWaURD3DcQwrfyiXzNvzsQXnU,8446
+h5yaml-0.3.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+h5yaml-0.3.0.dist-info/licenses/LICENSE,sha256=rLarIZOYK5jHuUjMnFbgdI_Tb_4_HAAKSOOIhwiWlE4,11356
+h5yaml-0.3.0.dist-info/RECORD,,

{h5yaml-0.2.0.dist-info → h5yaml-0.3.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

h5yaml/lib/chunksizes.py

@@ -1,73 +0,0 @@
-#
-# This file is part of Python package: `h5yaml`
-#
-#     https://github.com/rmvanhees/pyxarr.git
-#
-# Copyright (c) 2025 - R.M. van Hees (SRON)
-#    All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-"""Obtain chunksizes for HDF5 datasets."""
-
-from __future__ import annotations
-
-__all__ = ["guess_chunks"]
-
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from numpy.typing import ArrayLike
-
-
-def guess_chunks(dims: ArrayLike[int], dtype_sz: int) -> str | tuple[int]:
-    """Perform an educated guess for the dataset chunk sizes.
-
-    Parameters
-    ----------
-    dims :  ArrayLike[int]
-       Dimensions of the variable
-    dtype_sz :  int
-       The element size of the data-type of the variable
-
-    Returns
-    -------
-    "contiguous" or tuple with chunk-sizes
-
-    """
-    fixed_size = dtype_sz
-    if len(dims) > 1:
-        for val in [x for x in dims[1:] if x > 0]:
-            fixed_size *= val
-
-    # first variables without an unlimited dimension
-    if 0 not in dims:
-        if fixed_size < 400000:
-            return "contiguous"
-
-        res = list(dims)
-        res[0] = max(1, 2048000 // fixed_size)
-        return tuple(res)
-
-    # then variables with an unlimited dimension
-    if len(dims) == 1:
-        return (1024,)
-
-    udim = dims.index(0)
-    res = list(dims)
-    if fixed_size < 400000:
-        res[udim] = 1024
-    else:
-        res[udim] = max(1, 2048000 // fixed_size)
-
-    return tuple(res)

h5yaml-0.2.0.dist-info/RECORD

@@ -1,15 +0,0 @@
-h5yaml/__init__.py,sha256=NdNciPgYnbq-aVM6QqNGNZtdL72rTGLAMrDy0Yw7ckk,751
-h5yaml/conf_from_yaml.py,sha256=-2ar_gUmc5qvD1KlcctnpPY8G5c4TTXOF2tfrgcT9m4,1560
-h5yaml/yaml_h5.py,sha256=jVZAL5Cu6dqgdn25cgVBP3g7yx2wdE-cmCbkaOmQKZ4,10153
-h5yaml/yaml_nc.py,sha256=M6g4ZTPMlPmGZjyn3mLFnoQpmynGwgA0HtlUXSGNvvw,10963
-h5yaml/Data/h5_compound.yaml,sha256=z3dMCJDRAw14boRp0zT74bz_oFi21yu8coUoKOW-d2Q,1131
-h5yaml/Data/h5_testing.yaml,sha256=NhXeXjdblh3bv1hPjCl5DvIhEXo3EpD4mlgaZDElsJc,5626
-h5yaml/Data/h5_unsupported.yaml,sha256=v4HYhiTikFt6UoEUJBnmSse_WeHbmBgqF2e1bCJEfLw,1502
-h5yaml/Data/nc_testing.yaml,sha256=zuXcYrcuCankndt5e4nRPj2-xed97IA9yvfpn89XQgw,5451
-h5yaml/lib/__init__.py,sha256=NdNciPgYnbq-aVM6QqNGNZtdL72rTGLAMrDy0Yw7ckk,751
-h5yaml/lib/adjust_attr.py,sha256=4dHEGwwIa3a3hihyuSX8jCsC08fYcz_9XWA1pBwiwfc,2284
-h5yaml/lib/chunksizes.py,sha256=R1kdaKWF0Hol_maZ0tPDoUyWIH5RatP7d2J1yBA8bkk,1949
-h5yaml-0.2.0.dist-info/METADATA,sha256=IZxpXl9fI3Z7pa1DoRumjxPZXn8N_ykkJEdaUNnFJlw,7280
-h5yaml-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-h5yaml-0.2.0.dist-info/licenses/LICENSE,sha256=rLarIZOYK5jHuUjMnFbgdI_Tb_4_HAAKSOOIhwiWlE4,11356
-h5yaml-0.2.0.dist-info/RECORD,,