h5yaml 0.0.4__py3-none-any.whl → 0.1.0__py3-none-any.whl

h5yaml/Data/h5_testing.yaml

@@ -45,7 +45,10 @@ dimensions:
 # - compound elements must have a data-type, and can have a unit and long_name 
 compounds:
   stats_dtype:
-    time: [u8, seconds since 1970-01-01T00:00:00, timestamp]
+    time:
+      - u8
+      - seconds since 1970-01-01T00:00:00
+      - timestamp
     index: [u2, '1', index]
     tbl_id: [u1, '1', binning id]
     saa: [u1, '1', saa-flag]
@@ -84,7 +87,6 @@ variables:
     valid_max: 65534
   /group_01/stats:
     _dtype: stats_dtype
-    _vlen: True
     _dims: [time]
     comment: detector map statistics
   /group_02/detector_images:
@@ -97,3 +99,8 @@ variables:
     units: '1'
     valid_min: 0
     valid_max: 65534
+  /group_02/stats:
+    _dtype: stats_dtype
+    _vlen: True
+    _dims: [time]
+    comment: detector map statistics (vlen)

h5yaml/Data/nc_testing.yaml

@@ -0,0 +1,97 @@
+# YAML
+#
+# Configuration file to test the implementation of classes H5Yaml and NcYaml
+#
+# This file is part of h5_yaml:
+#    https://github.com/rmvanhees/h5_yaml.git
+#
+# Copyright (c) 2025 SRON
+#    All Rights Reserved
+#
+# License:  BSD-3-Clause
+#
+# Define groups
+groups:
+  - group_00
+  - group_01
+  - group_02
+
+# Define dimensions
+# Note dimensions with an attribute 'long_name' will also be generated as variable
+dimensions:
+  number_of_images:
+    _dtype: u2
+    _size: 0
+  samples_per_image:
+    _dtype: u4
+    _size: 203500
+  column:
+    _dtype: u2
+    _size: 640
+  row:
+    _dtype: u2
+    _size: 512
+  time:
+    _dtype: f8
+    _size: 0
+    _FillValue: -32767
+    long_name: Attitude sample time (seconds of day)
+    calendar: proleptic_gregorian
+    units: seconds since %Y-%m-%d %H:%M:%S
+    valid_min: 0
+    valid_max: 92400
+
+# Define compound types
+# - compound elements must have a data-type, and can have a unit and long_name 
+compounds:
+  stats_dtype:
+    time:
+      - u8
+      - seconds since 1970-01-01T00:00:00
+      - timestamp
+    index: [u2, '1', index]
+    tbl_id: [u1, '1', binning id]
+    saa: [u1, '1', saa-flag]
+    coad: [u1, '1', co-addings]
+    texp: [f4, ms, exposure time]
+    lat: [f4, degree, latitude]
+    lon: [f4, degree, longitude]
+    avg: [f4, '1', '$S - S_{ref}$']
+    unc: [f4, '1', '\u03c3($S - S_{ref}$)']
+    dark_offs: [f4, '1', dark-offset]
+
+# Define variables
+variables:
+  /group_00/detector_images:
+    _dtype: u2
+    _dims: [number_of_images, column, row]
+    _FillValue: 65535
+    long_name: Detector pixel values
+    comment: unbinned full-frame data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
+  /group_01/detector_images:
+    _dtype: u2
+    _dims: [number_of_images, samples_per_image]
+    _FillValue: 65535
+    _compression: 1
+    long_name: Detector pixel values
+    comment: variable binned data (filled to the largest samples_per_image)
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
+  /group_01/stats:
+    _dtype: stats_dtype
+    _dims: [time]
+    comment: detector map statistics
+  /group_02/detector_images:
+    _dtype: u2
+    _dims: [number_of_images]
+    _vlen: True
+    _FillValue: 65535
+    long_name: Detector pixel values
+    comment: variable binned (vlen) data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534

h5yaml/{yaml_h5py.py → yaml_h5.py}

@@ -23,7 +23,43 @@ import numpy as np
 from h5yaml.conf_from_yaml import conf_from_yaml
 from h5yaml.lib.chunksizes import guess_chunks
 
+
 # - helper function ------------------------------------
+def adjust_attr(attr_key: str, attr_val: np.ndarray, attr_dtype: str) -> np.ndarray:
+    """..."""
+    if attr_key in ("valid_min", "valid_max", "valid_range"):
+        match attr_dtype:
+            case "i1":
+                res = np.int8(attr_val)
+            case "i2":
+                res = np.int16(attr_val)
+            case "i4":
+                res = np.int32(attr_val)
+            case "i8":
+                res = np.int64(attr_val)
+            case "u1":
+                res = np.uint8(attr_val)
+            case "u2":
+                res = np.uint16(attr_val)
+            case "u4":
+                res = np.uint32(attr_val)
+            case "u8":
+                res = np.uint64(attr_val)
+            case "f2":
+                res = np.float16(attr_val)
+            case "f4":
+                res = np.float32(attr_val)
+            case "f8":
+                res = np.float64(attr_val)
+            case _:
+                res = attr_val
+
+        return res
+
+    if attr_key == "flag_values":
+        return np.array(attr_val, dtype="u1")
+
+    return attr_val
 
 
 # - class definition -----------------------------------
@@ -82,6 +118,8 @@ class H5Yaml:
                 )
                 if "_values" in val:
                     dset[:] = val["_values"]
+                elif "_range" in val:
+                    dset[:] = np.arange(*val["_range"], dtype=val["_dtype"])
 
             dset.make_scale(
                 Path(key).name
@@ -89,36 +127,8 @@ class H5Yaml:
                 else "This is a netCDF dimension but not a netCDF variable."
             )
             for attr, attr_val in val.items():
-                if attr.startswith("_"):
-                    continue
-                if attr in ("valid_min", "valid_max"):
-                    match val["_dtype"]:
-                        case "i1":
-                            dset.attrs[attr] = np.int8(attr_val)
-                        case "i2":
-                            dset.attrs[attr] = np.int16(attr_val)
-                        case "i4":
-                            dset.attrs[attr] = np.int32(attr_val)
-                        case "i8":
-                            dset.attrs[attr] = np.int64(attr_val)
-                        case "u1":
-                            dset.attrs[attr] = np.uint8(attr_val)
-                        case "u2":
-                            dset.attrs[attr] = np.uint16(attr_val)
-                        case "u4":
-                            dset.attrs[attr] = np.uint32(attr_val)
-                        case "u8":
-                            dset.attrs[attr] = np.uint64(attr_val)
-                        case "f2":
-                            dset.attrs[attr] = np.float16(attr_val)
-                        case "f4":
-                            dset.attrs[attr] = np.float32(attr_val)
-                        case "f8":
-                            dset.attrs[attr] = np.float64(attr_val)
-                        case _:
-                            dset.attrs[attr] = attr_val
-                else:
-                    dset.attrs[attr] = attr_val
+                if not attr.startswith("_"):
+                    dset.attrs[attr] = adjust_attr(attr, attr_val, val["_dtype"])
 
     def __compounds(self: H5Yaml, fid: h5py.File) -> dict[str, str | int | float]:
         """Add compound datatypes to HDF5 product."""
@@ -192,9 +202,8 @@ class H5Yaml:
                     fillvalue=fillvalue,
                 )
                 for attr, attr_val in val.items():
-                    if attr.startswith("_"):
-                        continue
-                    dset.attrs[attr] = attr_val
+                    if not attr.startswith("_"):
+                        dset.attrs[attr] = adjust_attr(attr, attr_val, val["_dtype"])
                 continue
 
             n_udim = 0
@@ -236,7 +245,14 @@ class H5Yaml:
                     shuffle = True
 
                 if val.get("_vlen"):
-                    ds_dtype = h5py.vlen_dtype(ds_dtype)
+                    ds_name = (
+                        val["_dtype"].split("_")[0]
+                        if "_" in val["_dtype"]
+                        else val["_dtype"]
+                    ) + "_vlen"
+                    if ds_name not in fid:
+                        fid[ds_name] = h5py.vlen_dtype(ds_dtype)
+                    ds_dtype = fid[ds_name]
                     fillvalue = None
                     if ds_maxshape == (None,):
                         ds_chunk = (16,)
@@ -256,38 +272,8 @@ class H5Yaml:
                 dset.dims[ii].attach_scale(fid[coord])
 
             for attr, attr_val in val.items():
-                if attr.startswith("_"):
-                    continue
-                if attr in ("valid_min", "valid_max"):
-                    match val["_dtype"]:
-                        case "i1":
-                            dset.attrs[attr] = np.int8(attr_val)
-                        case "i2":
-                            dset.attrs[attr] = np.int16(attr_val)
-                        case "i4":
-                            dset.attrs[attr] = np.int32(attr_val)
-                        case "i8":
-                            dset.attrs[attr] = np.int64(attr_val)
-                        case "u1":
-                            dset.attrs[attr] = np.uint8(attr_val)
-                        case "u2":
-                            dset.attrs[attr] = np.uint16(attr_val)
-                        case "u4":
-                            dset.attrs[attr] = np.uint32(attr_val)
-                        case "u8":
-                            dset.attrs[attr] = np.uint64(attr_val)
-                        case "f2":
-                            dset.attrs[attr] = np.float16(attr_val)
-                        case "f4":
-                            dset.attrs[attr] = np.float32(attr_val)
-                        case "f8":
-                            dset.attrs[attr] = np.float64(attr_val)
-                        case _:
-                            dset.attrs[attr] = attr_val
-                elif attr == "flag_values":
-                    dset.attrs[attr] = np.array(attr_val, dtype="u1")
-                else:
-                    dset.attrs[attr] = attr_val
+                if not attr.startswith("_"):
+                    dset.attrs[attr] = adjust_attr(attr, attr_val, val["_dtype"])
 
             if compounds is not None and val["_dtype"] in compounds:
                 if compounds[val["_dtype"]]["units"]:
@@ -311,7 +297,8 @@ class H5Yaml:
         """
         try:
             with h5py.File(l1a_name, "w") as fid:
-                self.__groups(fid)
+                if "groups" in self.h5_def:
+                    self.__groups(fid)
                 self.__dimensions(fid)
                 self.__variables(fid, self.__compounds(fid))
         except PermissionError as exc:

h5yaml/yaml_nc.py

@@ -258,7 +258,7 @@ class NcYaml:
 def tests() -> None:
     """..."""
     print("Calling NcYaml")
-    NcYaml(files("h5yaml.Data") / "h5_testing.yaml").create("test_yaml.nc")
+    NcYaml(files("h5yaml.Data") / "nc_testing.yaml").create("test_yaml.nc")
 
 
 if __name__ == "__main__":

{h5yaml-0.0.4.dist-info → h5yaml-0.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: h5yaml
-Version: 0.0.4
+Version: 0.1.0
 Summary: Use YAML configuration file to generate HDF5/netCDF4 formated files.
 Project-URL: Homepage, https://github.com/rmvanhees/h5_yaml
 Project-URL: Source, https://github.com/rmvanhees/h5_yaml
@@ -9,18 +9,18 @@ Author-email: Richard van Hees <r.m.van.hees@sron.nl>
 License-Expression: BSD-3-Clause
 License-File: LICENSE
 Keywords: HDF5,YAML,netCDF4
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: h5py>=3.13
 Requires-Dist: netcdf4>=1.7
 Requires-Dist: numpy>=2.2
@@ -133,10 +133,11 @@ The YAML file should be structured as follows:
    Alternatively, provide a list with names of YAML files which contain the definitions
    of the compounds.
 
+   ```
    compounds:
      - h5_nomhk_tm.yaml
      - h5_science_hk.yaml
-
+   ```
  * The 'variables' are defined by their data-type ('_dtype') and dimensions ('_dims'),
    and optionally chunk sizes ('_chunks'), compression ('_compression'), variable length
    ('_vlen'). In addition, each variable can have as many attributes as you like,

h5yaml-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+h5yaml/conf_from_yaml.py,sha256=FT5oS4yqDFUYZqgria_OrmMK52ZcrTVoAPyPkLrHstc,982
+h5yaml/yaml_h5.py,sha256=VNMyfcGaiWwqzAxy8M-l_nYHLbODIW8cxoppfZJc22k,10639
+h5yaml/yaml_nc.py,sha256=9GqddTE5qmpumwoR5sJ0LZUDKW8RfnFi9LG98bm7tH8,9202
+h5yaml/Data/h5_testing.yaml,sha256=zX0ipEZv3WILEpgniLlCDJz2zo2wTS6vAvTA2zZNQkY,2483
+h5yaml/Data/nc_testing.yaml,sha256=v1jd2yJHS9qBJikZFmxp-RfNQZ1r6NmwPMhjeUmPOus,2295
+h5yaml/lib/chunksizes.py,sha256=sAYiTTL8ecBivBDxw-XIJu2yMlt0zNHTr-KS7tBFp2o,1392
+h5yaml-0.1.0.dist-info/METADATA,sha256=hjvtnjNMAJAYK0hoksSMnyecJzzxaMKKd1fvZQGr2Kc,7188
+h5yaml-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+h5yaml-0.1.0.dist-info/licenses/LICENSE,sha256=MoOwtPnC77nFaIwRIAIE6fKhrzMd3G18mOXDPtAH8G0,1509
+h5yaml-0.1.0.dist-info/RECORD,,

h5yaml-0.0.4.dist-info/RECORD

@@ -1,9 +0,0 @@
-h5yaml/conf_from_yaml.py,sha256=FT5oS4yqDFUYZqgria_OrmMK52ZcrTVoAPyPkLrHstc,982
-h5yaml/yaml_h5py.py,sha256=MVPUqD5fNY3VYIi3ebaEY7uSR1sgYw-8DvIjrOWIaIk,11838
-h5yaml/yaml_nc.py,sha256=37NQ9FPuCrnOTN5R4otn0R-Sdwej1-cPwhODgY2L6pY,9202
-h5yaml/Data/h5_testing.yaml,sha256=_x3qBC8RNQ1h4c6B1JOH_y0L9DiDLsOEfyr60IcvpoI,2358
-h5yaml/lib/chunksizes.py,sha256=sAYiTTL8ecBivBDxw-XIJu2yMlt0zNHTr-KS7tBFp2o,1392
-h5yaml-0.0.4.dist-info/METADATA,sha256=Vvpv1h1t0cYvV5hBiXzdKe2CtU3TT2hWW0qwrIC8vLc,7160
-h5yaml-0.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-h5yaml-0.0.4.dist-info/licenses/LICENSE,sha256=MoOwtPnC77nFaIwRIAIE6fKhrzMd3G18mOXDPtAH8G0,1509
-h5yaml-0.0.4.dist-info/RECORD,,