h5yaml 0.0.4__tar.gz → 0.1.0__tar.gz

h5yaml-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.12.8
+    hooks:
+      - id: ruff-check
+        args: [ --fix ]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: check-yaml

{h5yaml-0.0.4 → h5yaml-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: h5yaml
-Version: 0.0.4
+Version: 0.1.0
 Summary: Use YAML configuration file to generate HDF5/netCDF4 formated files.
 Project-URL: Homepage, https://github.com/rmvanhees/h5_yaml
 Project-URL: Source, https://github.com/rmvanhees/h5_yaml
@@ -9,18 +9,18 @@ Author-email: Richard van Hees <r.m.van.hees@sron.nl>
 License-Expression: BSD-3-Clause
 License-File: LICENSE
 Keywords: HDF5,YAML,netCDF4
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: h5py>=3.13
 Requires-Dist: netcdf4>=1.7
 Requires-Dist: numpy>=2.2
@@ -133,10 +133,11 @@ The YAML file should be structured as follows:
    Alternatively, provide a list with names of YAML files which contain the definitions
    of the compounds.
 
+   ```
    compounds:
      - h5_nomhk_tm.yaml
      - h5_science_hk.yaml
-
+   ```
  * The 'variables' are defined by their data-type ('_dtype') and dimensions ('_dims'),
    and optionally chunk sizes ('_chunks'), compression ('_compression'), variable length
    ('_vlen'). In addition, each variable can have as many attributes as you like,

{h5yaml-0.0.4 → h5yaml-0.1.0}/README.md

@@ -104,10 +104,11 @@ The YAML file should be structured as follows:
    Alternatively, provide a list with names of YAML files which contain the definitions
    of the compounds.
 
+   ```
    compounds:
      - h5_nomhk_tm.yaml
      - h5_science_hk.yaml
-
+   ```
  * The 'variables' are defined by their data-type ('_dtype') and dimensions ('_dims'),
    and optionally chunk sizes ('_chunks'), compression ('_compression'), variable length
    ('_vlen'). In addition, each variable can have as many attributes as you like,

{h5yaml-0.0.4 → h5yaml-0.1.0}/pyproject.toml

@@ -14,18 +14,18 @@ license = "BSD-3-Clause"
 authors = [
   {name = "Richard van Hees", email = "r.m.van.hees@sron.nl"}
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 classifiers = [
-   "Development Status :: 4 - Beta",
+   "Development Status :: 5 - Production/Stable",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3 :: Only",
-   "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
+   "Programming Language :: Python :: 3.14",
    "Topic :: Scientific/Engineering :: Atmospheric Science",
 ]
 keywords = [

h5yaml-0.1.0/src/h5yaml/Data/h5_testing.yaml

@@ -0,0 +1,106 @@
+# YAML
+#
+# Configuration file to test the implementation of classes H5Yaml and NcYaml
+#
+# This file is part of h5_yaml:
+#    https://github.com/rmvanhees/h5_yaml.git
+#
+# Copyright (c) 2025 SRON
+#    All Rights Reserved
+#
+# License:  BSD-3-Clause
+#
+# Define groups
+groups:
+  - group_00
+  - group_01
+  - group_02
+
+# Define dimensions
+# Note dimensions with an attribute 'long_name' will also be generated as variable
+dimensions:
+  number_of_images:
+    _dtype: u2
+    _size: 0
+  samples_per_image:
+    _dtype: u4
+    _size: 203500
+  column:
+    _dtype: u2
+    _size: 640
+  row:
+    _dtype: u2
+    _size: 512
+  time:
+    _dtype: f8
+    _size: 0
+    _FillValue: -32767
+    long_name: Attitude sample time (seconds of day)
+    calendar: proleptic_gregorian
+    units: seconds since %Y-%m-%d %H:%M:%S
+    valid_min: 0
+    valid_max: 92400
+
+# Define compound types
+# - compound elements must have a data-type, and can have a unit and long_name 
+compounds:
+  stats_dtype:
+    time:
+      - u8
+      - seconds since 1970-01-01T00:00:00
+      - timestamp
+    index: [u2, '1', index]
+    tbl_id: [u1, '1', binning id]
+    saa: [u1, '1', saa-flag]
+    coad: [u1, '1', co-addings]
+    texp: [f4, ms, exposure time]
+    lat: [f4, degree, latitude]
+    lon: [f4, degree, longitude]
+    avg: [f4, '1', '$S - S_{ref}$']
+    unc: [f4, '1', '\u03c3($S - S_{ref}$)']
+    dark_offs: [f4, '1', dark-offset]
+
+  geo_dtype:
+    lat: [f4, latitude]
+    lon: [f4, longitude]
+    
+# Define variables
+variables:
+  /group_00/detector_images:
+    _dtype: u2
+    _dims: [number_of_images, column, row]
+    _FillValue: 65535
+    long_name: Detector pixel values
+    comment: unbinned full-frame data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
+  /group_01/detector_images:
+    _dtype: u2
+    _dims: [number_of_images, samples_per_image]
+    _FillValue: 65535
+    _compression: 1
+    long_name: Detector pixel values
+    comment: variable binned data (filled to the largest samples_per_image)
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
+  /group_01/stats:
+    _dtype: stats_dtype
+    _dims: [time]
+    comment: detector map statistics
+  /group_02/detector_images:
+    _dtype: u2
+    _dims: [number_of_images]
+    _vlen: True
+    _FillValue: 65535
+    long_name: Detector pixel values
+    comment: variable binned (vlen) data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
+  /group_02/stats:
+    _dtype: stats_dtype
+    _vlen: True
+    _dims: [time]
+    comment: detector map statistics (vlen)

h5yaml-0.0.4/src/h5yaml/Data/h5_testing.yaml → h5yaml-0.1.0/src/h5yaml/Data/nc_testing.yaml

@@ -45,7 +45,10 @@ dimensions:
 # - compound elements must have a data-type, and can have a unit and long_name 
 compounds:
   stats_dtype:
-    time: [u8, seconds since 1970-01-01T00:00:00, timestamp]
+    time:
+      - u8
+      - seconds since 1970-01-01T00:00:00
+      - timestamp
     index: [u2, '1', index]
     tbl_id: [u1, '1', binning id]
     saa: [u1, '1', saa-flag]
@@ -57,10 +60,6 @@ compounds:
     unc: [f4, '1', '\u03c3($S - S_{ref}$)']
     dark_offs: [f4, '1', dark-offset]
 
-  geo_dtype:
-    lat: [f4, latitude]
-    lon: [f4, longitude]
-    
 # Define variables
 variables:
   /group_00/detector_images:
@@ -84,7 +83,6 @@ variables:
     valid_max: 65534
   /group_01/stats:
     _dtype: stats_dtype
-    _vlen: True
     _dims: [time]
     comment: detector map statistics
   /group_02/detector_images:

h5yaml-0.0.4/src/h5yaml/yaml_h5py.py → h5yaml-0.1.0/src/h5yaml/yaml_h5.py

@@ -23,7 +23,43 @@ import numpy as np
 from h5yaml.conf_from_yaml import conf_from_yaml
 from h5yaml.lib.chunksizes import guess_chunks
 
+
 # - helper function ------------------------------------
+def adjust_attr(attr_key: str, attr_val: np.ndarray, attr_dtype: str) -> np.ndarray:
+    """..."""
+    if attr_key in ("valid_min", "valid_max", "valid_range"):
+        match attr_dtype:
+            case "i1":
+                res = np.int8(attr_val)
+            case "i2":
+                res = np.int16(attr_val)
+            case "i4":
+                res = np.int32(attr_val)
+            case "i8":
+                res = np.int64(attr_val)
+            case "u1":
+                res = np.uint8(attr_val)
+            case "u2":
+                res = np.uint16(attr_val)
+            case "u4":
+                res = np.uint32(attr_val)
+            case "u8":
+                res = np.uint64(attr_val)
+            case "f2":
+                res = np.float16(attr_val)
+            case "f4":
+                res = np.float32(attr_val)
+            case "f8":
+                res = np.float64(attr_val)
+            case _:
+                res = attr_val
+
+        return res
+
+    if attr_key == "flag_values":
+        return np.array(attr_val, dtype="u1")
+
+    return attr_val
 
 
 # - class definition -----------------------------------
@@ -82,6 +118,8 @@ class H5Yaml:
                 )
                 if "_values" in val:
                     dset[:] = val["_values"]
+                elif "_range" in val:
+                    dset[:] = np.arange(*val["_range"], dtype=val["_dtype"])
 
             dset.make_scale(
                 Path(key).name
@@ -89,36 +127,8 @@ class H5Yaml:
                 else "This is a netCDF dimension but not a netCDF variable."
             )
             for attr, attr_val in val.items():
-                if attr.startswith("_"):
-                    continue
-                if attr in ("valid_min", "valid_max"):
-                    match val["_dtype"]:
-                        case "i1":
-                            dset.attrs[attr] = np.int8(attr_val)
-                        case "i2":
-                            dset.attrs[attr] = np.int16(attr_val)
-                        case "i4":
-                            dset.attrs[attr] = np.int32(attr_val)
-                        case "i8":
-                            dset.attrs[attr] = np.int64(attr_val)
-                        case "u1":
-                            dset.attrs[attr] = np.uint8(attr_val)
-                        case "u2":
-                            dset.attrs[attr] = np.uint16(attr_val)
-                        case "u4":
-                            dset.attrs[attr] = np.uint32(attr_val)
-                        case "u8":
-                            dset.attrs[attr] = np.uint64(attr_val)
-                        case "f2":
-                            dset.attrs[attr] = np.float16(attr_val)
-                        case "f4":
-                            dset.attrs[attr] = np.float32(attr_val)
-                        case "f8":
-                            dset.attrs[attr] = np.float64(attr_val)
-                        case _:
-                            dset.attrs[attr] = attr_val
-                else:
-                    dset.attrs[attr] = attr_val
+                if not attr.startswith("_"):
+                    dset.attrs[attr] = adjust_attr(attr, attr_val, val["_dtype"])
 
     def __compounds(self: H5Yaml, fid: h5py.File) -> dict[str, str | int | float]:
         """Add compound datatypes to HDF5 product."""
@@ -192,9 +202,8 @@ class H5Yaml:
                     fillvalue=fillvalue,
                 )
                 for attr, attr_val in val.items():
-                    if attr.startswith("_"):
-                        continue
-                    dset.attrs[attr] = attr_val
+                    if not attr.startswith("_"):
+                        dset.attrs[attr] = adjust_attr(attr, attr_val, val["_dtype"])
                 continue
 
             n_udim = 0
@@ -236,7 +245,14 @@ class H5Yaml:
                     shuffle = True
 
                 if val.get("_vlen"):
-                    ds_dtype = h5py.vlen_dtype(ds_dtype)
+                    ds_name = (
+                        val["_dtype"].split("_")[0]
+                        if "_" in val["_dtype"]
+                        else val["_dtype"]
+                    ) + "_vlen"
+                    if ds_name not in fid:
+                        fid[ds_name] = h5py.vlen_dtype(ds_dtype)
+                    ds_dtype = fid[ds_name]
                     fillvalue = None
                     if ds_maxshape == (None,):
                         ds_chunk = (16,)
@@ -256,38 +272,8 @@ class H5Yaml:
                 dset.dims[ii].attach_scale(fid[coord])
 
             for attr, attr_val in val.items():
-                if attr.startswith("_"):
-                    continue
-                if attr in ("valid_min", "valid_max"):
-                    match val["_dtype"]:
-                        case "i1":
-                            dset.attrs[attr] = np.int8(attr_val)
-                        case "i2":
-                            dset.attrs[attr] = np.int16(attr_val)
-                        case "i4":
-                            dset.attrs[attr] = np.int32(attr_val)
-                        case "i8":
-                            dset.attrs[attr] = np.int64(attr_val)
-                        case "u1":
-                            dset.attrs[attr] = np.uint8(attr_val)
-                        case "u2":
-                            dset.attrs[attr] = np.uint16(attr_val)
-                        case "u4":
-                            dset.attrs[attr] = np.uint32(attr_val)
-                        case "u8":
-                            dset.attrs[attr] = np.uint64(attr_val)
-                        case "f2":
-                            dset.attrs[attr] = np.float16(attr_val)
-                        case "f4":
-                            dset.attrs[attr] = np.float32(attr_val)
-                        case "f8":
-                            dset.attrs[attr] = np.float64(attr_val)
-                        case _:
-                            dset.attrs[attr] = attr_val
-                elif attr == "flag_values":
-                    dset.attrs[attr] = np.array(attr_val, dtype="u1")
-                else:
-                    dset.attrs[attr] = attr_val
+                if not attr.startswith("_"):
+                    dset.attrs[attr] = adjust_attr(attr, attr_val, val["_dtype"])
 
             if compounds is not None and val["_dtype"] in compounds:
                 if compounds[val["_dtype"]]["units"]:
@@ -311,7 +297,8 @@ class H5Yaml:
         """
         try:
             with h5py.File(l1a_name, "w") as fid:
-                self.__groups(fid)
+                if "groups" in self.h5_def:
+                    self.__groups(fid)
                 self.__dimensions(fid)
                 self.__variables(fid, self.__compounds(fid))
         except PermissionError as exc:

{h5yaml-0.0.4 → h5yaml-0.1.0}/src/h5yaml/yaml_nc.py

@@ -258,7 +258,7 @@ class NcYaml:
 def tests() -> None:
     """..."""
     print("Calling NcYaml")
-    NcYaml(files("h5yaml.Data") / "h5_testing.yaml").create("test_yaml.nc")
+    NcYaml(files("h5yaml.Data") / "nc_testing.yaml").create("test_yaml.nc")
 
 
 if __name__ == "__main__":