h5yaml 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

h5yaml/Data/h5_testing.yaml

@@ -25,6 +25,8 @@ groups:
   - group_01
   - group_02
   - group_03
+  - processing_control
+  - /processing_control/input_parameters
 
 # Define dimensions
 # Note dimensions with an attribute 'long_name' will also be generated as variable
@@ -91,6 +93,26 @@ variables:
     units: '1'
     valid_min: 0
     valid_max: 65534
+  /group_00/detector_images_chunked:
+    _dtype: u2
+    _dims: [number_of_images, column, row]
+    _FillValue: 65535
+    _chunks: [1, 640, 512]
+    long_name: Detector pixel values
+    comment: unbinned full-frame data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
+  /group_00/detector_images_autochunk:
+    _dtype: u2
+    _dims: [number_of_images, column, row]
+    _chunks: True
+    _FillValue: 65535
+    long_name: Detector pixel values
+    comment: unbinned full-frame data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
   # ---------- GROUP 01 ----------
   /group_01/detector_images:
     _dtype: u2
@@ -199,7 +221,7 @@ variables:
     valid_max: 999.9
   /group_03/ds_10:
     _dtype: f4
-    _dims: [number_of_images]
+    _dims: [number_of_images, /group_03/viewport]
     long_name: float dataset
     units: '1'
     valid_min: -999.9

h5yaml/Data/nc_testing.yaml

@@ -25,6 +25,8 @@ groups:
   - group_01
   - group_02
   - group_03
+  - processing_control
+  - /processing_control/input_parameters
 
 # Define dimensions
 # Note dimensions with an attribute 'long_name' will also be generated as variable
@@ -87,6 +89,16 @@ variables:
     units: '1'
     valid_min: 0
     valid_max: 65534
+  /group_00/detector_images_chunked:
+    _dtype: u2
+    _dims: [number_of_images, column, row]
+    _FillValue: 65535
+    _chunks: [1, 640, 512]
+    long_name: Detector pixel values
+    comment: unbinned full-frame data
+    units: '1'
+    valid_min: 0
+    valid_max: 65534
   # ---------- GROUP 01 ----------
   /group_01/detector_images:
     _dtype: u2
@@ -190,7 +202,7 @@ variables:
   #  valid_max: 999.9
   /group_03/ds_10:
     _dtype: f4
-    _dims: [number_of_images]
+    _dims: [number_of_images, /group_03/viewport]
     long_name: float dataset
     units: '1'
     valid_min: -999.9

h5yaml/conf_from_yaml.py

@@ -45,7 +45,7 @@ def conf_from_yaml(file_path: Path | str) -> dict:
 
     """
     if isinstance(file_path, str):
-        file_path = Path(str)
+        file_path = Path(file_path)
 
     if not file_path.is_file():
         raise FileNotFoundError(f"{file_path} not found")
@@ -53,7 +53,7 @@ def conf_from_yaml(file_path: Path | str) -> dict:
     with file_path.open("r", encoding="ascii") as fid:
         try:
             settings = yaml.safe_load(fid)
-        except yaml.YAMLError as exc:
-            raise RuntimeError from exc
+        except yaml.parser.ParserError as exc:
+            raise RuntimeError(f"Failed to parse {file_path}") from exc
 
     return settings

h5yaml/yaml_h5.py

@@ -32,7 +32,6 @@ import numpy as np
 
 from .conf_from_yaml import conf_from_yaml
 from .lib.adjust_attr import adjust_attr
-from .lib.chunksizes import guess_chunks
 
 
 # - class definition -----------------------------------
@@ -41,12 +40,12 @@ class H5Yaml:
 
     Parameters
     ----------
-    h5_yaml_fl :  Path
-       YAML files with the HDF5 format definition
+    h5_yaml_fl :  Path | str
+       YAML file with the HDF5 format definition
 
     """
 
-    def __init__(self: H5Yaml, h5_yaml_fl: Path) -> None:
+    def __init__(self: H5Yaml, h5_yaml_fl: Path | str) -> None:
         """Construct a H5Yaml instance."""
         self.logger = logging.getLogger("h5yaml.H5Yaml")
 
@@ -72,12 +71,14 @@ class H5Yaml:
                 )
 
             if val["_size"] == 0:
-                ds_chunk = val.get("_chunks", (50,))
+                ds_chunk = val.get("_chunks")
+                if ds_chunk is not None and not isinstance(ds_chunk, bool):
+                    ds_chunk = tuple(ds_chunk)
                 dset = fid.create_dataset(
                     key,
                     shape=(0,),
                     dtype="T" if val["_dtype"] == "str" else val["_dtype"],
-                    chunks=ds_chunk if isinstance(ds_chunk, tuple) else tuple(ds_chunk),
+                    chunks=ds_chunk,
                     maxshape=(None,),
                     fillvalue=fillvalue,
                 )
@@ -153,10 +154,8 @@ class H5Yaml:
         for key, val in self.h5_def["variables"].items():
             if val["_dtype"] in fid:
                 ds_dtype = fid[val["_dtype"]]
-                dtype_size = fid[val["_dtype"]].dtype.itemsize
             else:
                 ds_dtype = "T" if val["_dtype"] == "str" else val["_dtype"]
-                dtype_size = np.dtype(val["_dtype"]).itemsize
 
             fillvalue = None
             if "_FillValue" in val:
@@ -190,15 +189,13 @@ class H5Yaml:
             if n_udim > 1:
                 raise ValueError(f"{key} has more than one unlimited dimension")
 
-            # obtain chunk-size settings
-            ds_chunk = (
-                val["_chunks"]
-                if "_chunks" in val
-                else guess_chunks(ds_shape, dtype_size)
-            )
+            if None in ds_maxshape and val.get("_chunks") == "contiguous":
+                raise KeyError(
+                    "you can not create a contiguous dataset with unlimited dimensions."
+                )
 
             # create the variable
-            if ds_chunk == "contiguous":
+            if val.get("_chunks") == "contiguous":
                 dset = fid.create_dataset(
                     key,
                     ds_shape,
@@ -208,6 +205,9 @@ class H5Yaml:
                     fillvalue=fillvalue,
                 )
             else:
+                ds_chunk = val.get("_chunks")
+                if ds_chunk is not None and not isinstance(ds_chunk, bool):
+                    ds_chunk = tuple(ds_chunk)
                 compression = None
                 shuffle = False
                 # currently only gzip compression is supported
@@ -225,14 +225,12 @@ class H5Yaml:
                         fid[ds_name] = h5py.vlen_dtype(ds_dtype)
                     ds_dtype = fid[ds_name]
                     fillvalue = None
-                    if ds_maxshape == (None,):
-                        ds_chunk = (16,)
 
                 dset = fid.create_dataset(
                     key,
                     ds_shape,
                     dtype=ds_dtype,
-                    chunks=ds_chunk if isinstance(ds_chunk, tuple) else tuple(ds_chunk),
+                    chunks=ds_chunk,
                     maxshape=ds_maxshape,
                     fillvalue=fillvalue,
                     compression=compression,

h5yaml/yaml_nc.py

@@ -35,7 +35,6 @@ from netCDF4 import Dataset
 
 from .conf_from_yaml import conf_from_yaml
 from .lib.adjust_attr import adjust_attr
-from .lib.chunksizes import guess_chunks
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -43,9 +42,16 @@ if TYPE_CHECKING:
 
 # - class definition -----------------------------------
 class NcYaml:
-    """Class to create a HDF5/netCDF4 formated file from a YAML configuration file."""
+    """Class to create a HDF5/netCDF4 formated file from a YAML configuration file.
 
-    def __init__(self: NcYaml, nc_yaml_fl: Path) -> None:
+    Parameters
+    ----------
+    nc_yaml_fl :  Path | str
+       YAML file with the netCDF4 format definition
+
+    """
+
+    def __init__(self: NcYaml, nc_yaml_fl: Path | str) -> None:
         """Construct a NcYaml instance."""
         self.logger = logging.getLogger("h5yaml.NcYaml")
 
@@ -172,10 +178,8 @@ class NcYaml:
 
             if val["_dtype"] in fid.cmptypes:
                 ds_dtype = fid.cmptypes[val["_dtype"]].dtype
-                sz_dtype = ds_dtype.itemsize
             else:
                 ds_dtype = val["_dtype"]
-                sz_dtype = np.dtype(val["_dtype"]).itemsize
 
             fillvalue = None
             if "_FillValue" in val:
@@ -226,16 +230,16 @@ class NcYaml:
             if n_udim > 1:
                 raise ValueError("more than one unlimited dimension")
 
-            # obtain chunk-size settings
-            ds_chunk = (
-                val["_chunks"] if "_chunks" in val else guess_chunks(ds_shape, sz_dtype)
-            )
+            if None in ds_maxshape and val.get("_chunks") == "contiguous":
+                raise KeyError(
+                    "you can not create a contiguous dataset with unlimited dimensions."
+                )
 
             if val["_dtype"] in fid.cmptypes:
                 val["_dtype"] = fid.cmptypes[val["_dtype"]]
 
             # create the variable
-            if ds_chunk == "contiguous":
+            if val.get("_chunks") == "contiguous":
                 dset = var_grp.createVariable(
                     var_name,
                     val["_dtype"],
@@ -244,25 +248,24 @@ class NcYaml:
                     contiguous=True,
                 )
             else:
+                ds_chunk = val.get("_chunks")
+                if ds_chunk is not None and not isinstance(ds_chunk, bool):
+                    ds_chunk = tuple(ds_chunk)
                 if val.get("_vlen"):
                     if val["_dtype"] in fid.cmptypes:
                         raise ValueError("can not have vlen with compounds")
                     val["_dtype"] = fid.createVLType(ds_dtype, val["_dtype"])
                     fillvalue = None
-                    if ds_maxshape == (None,):
-                        ds_chunk = (16,)
 
                 dset = var_grp.createVariable(
                     var_name,
                     str if val["_dtype"] == "str" else val["_dtype"],
                     dimensions=var_dims,
                     fill_value=fillvalue,
-                    contiguous=False,
                     compression=compression,
                     complevel=complevel,
-                    chunksizes=(
-                        ds_chunk if isinstance(ds_chunk, tuple) else tuple(ds_chunk)
-                    ),
+                    chunksizes=ds_chunk,
+                    contiguous=False,
                 )
             dset.setncatts(
                 {

{h5yaml-0.2.0.dist-info → h5yaml-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: h5yaml
-Version: 0.2.0
+Version: 0.2.1
 Summary: Use YAML configuration file to generate HDF5/netCDF4 formated files.
 Project-URL: Homepage, https://github.com/rmvanhees/h5_yaml
 Project-URL: Source, https://github.com/rmvanhees/h5_yaml
@@ -61,7 +61,19 @@ However, package `netCDF4` has some limitations, which `h5py` has not, for examp
 not allow variable-length variables to have a compound data-type.
 
 ## Installation
-Releases of the code, starting from version 0.1, will be made available via PyPI.
+The package `h5yaml` is available from PyPI. To install it use `pip`:
+
+> $ pip install [--user] h5yaml
+
+The module `h5yaml` requires Python3.10+ and Python modules: h5py (v3.14+), netCDF4 (v1.7+) and numpy (v2.0+).
+
+**Note**: the packages `h5py` and `netCDF4` come with their own HDF5 libraries. If these are different then they may
+collide and result in a *''HDF5 error''*.
+If this is the case then you have to install the development packages of HDF5 and netCDF4 (or compile them from source).
+And reinstall `h5py` and `netCDF4` using the commands:
+
+> $ pip uninstall h5py; pip install --no-binary=h5py h5py
+> $ pip uninstall netCDF4; pip install --no-binary=netCDF4 netCDF4
 
 ## Usage
 
@@ -176,7 +188,7 @@ The YAML file should be structured as follows:
 
 ## Support [TBW]
 
-## Roadmap
+## Road map
 
  * Release v0.1 : stable API to read your YAML files and generate the HDF5/netCDF4 file
 

h5yaml-0.2.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+h5yaml/__init__.py,sha256=NdNciPgYnbq-aVM6QqNGNZtdL72rTGLAMrDy0Yw7ckk,751
+h5yaml/conf_from_yaml.py,sha256=GVbWR-I0_sKRxrXmgLxbnTJvAXz5OtFtNYu4Pp3LaaI,1607
+h5yaml/yaml_h5.py,sha256=K9KkdHNDadrrwlR5EJWuzf1yFZcycJLm3EaiM4nLHkw,10112
+h5yaml/yaml_nc.py,sha256=0xJ0dYnIXEhUVZGEkv9GZZu4Xo7K4zXPGiG49j8RHKs,10977
+h5yaml/Data/h5_compound.yaml,sha256=z3dMCJDRAw14boRp0zT74bz_oFi21yu8coUoKOW-d2Q,1131
+h5yaml/Data/h5_testing.yaml,sha256=BSrjenq_L5g8GGoldtU_NbWzLSymwAcvFOh26jtoynM,6247
+h5yaml/Data/h5_unsupported.yaml,sha256=v4HYhiTikFt6UoEUJBnmSse_WeHbmBgqF2e1bCJEfLw,1502
+h5yaml/Data/nc_testing.yaml,sha256=zKDRkYpWVM3_vDpsu-ZxHAGDlITICb_nwjDKsFzPgcQ,5807
+h5yaml/lib/__init__.py,sha256=NdNciPgYnbq-aVM6QqNGNZtdL72rTGLAMrDy0Yw7ckk,751
+h5yaml/lib/adjust_attr.py,sha256=4dHEGwwIa3a3hihyuSX8jCsC08fYcz_9XWA1pBwiwfc,2284
+h5yaml-0.2.1.dist-info/METADATA,sha256=Z-be-4WMz4o1RY4WiY3o-pnfEq72cdvTkDnmTQUnaJY,7876
+h5yaml-0.2.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+h5yaml-0.2.1.dist-info/licenses/LICENSE,sha256=rLarIZOYK5jHuUjMnFbgdI_Tb_4_HAAKSOOIhwiWlE4,11356
+h5yaml-0.2.1.dist-info/RECORD,,

h5yaml/lib/chunksizes.py

@@ -1,73 +0,0 @@
-#
-# This file is part of Python package: `h5yaml`
-#
-#     https://github.com/rmvanhees/pyxarr.git
-#
-# Copyright (c) 2025 - R.M. van Hees (SRON)
-#    All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-"""Obtain chunksizes for HDF5 datasets."""
-
-from __future__ import annotations
-
-__all__ = ["guess_chunks"]
-
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from numpy.typing import ArrayLike
-
-
-def guess_chunks(dims: ArrayLike[int], dtype_sz: int) -> str | tuple[int]:
-    """Perform an educated guess for the dataset chunk sizes.
-
-    Parameters
-    ----------
-    dims :  ArrayLike[int]
-       Dimensions of the variable
-    dtype_sz :  int
-       The element size of the data-type of the variable
-
-    Returns
-    -------
-    "contiguous" or tuple with chunk-sizes
-
-    """
-    fixed_size = dtype_sz
-    if len(dims) > 1:
-        for val in [x for x in dims[1:] if x > 0]:
-            fixed_size *= val
-
-    # first variables without an unlimited dimension
-    if 0 not in dims:
-        if fixed_size < 400000:
-            return "contiguous"
-
-        res = list(dims)
-        res[0] = max(1, 2048000 // fixed_size)
-        return tuple(res)
-
-    # then variables with an unlimited dimension
-    if len(dims) == 1:
-        return (1024,)
-
-    udim = dims.index(0)
-    res = list(dims)
-    if fixed_size < 400000:
-        res[udim] = 1024
-    else:
-        res[udim] = max(1, 2048000 // fixed_size)
-
-    return tuple(res)

h5yaml-0.2.0.dist-info/RECORD

@@ -1,15 +0,0 @@
-h5yaml/__init__.py,sha256=NdNciPgYnbq-aVM6QqNGNZtdL72rTGLAMrDy0Yw7ckk,751
-h5yaml/conf_from_yaml.py,sha256=-2ar_gUmc5qvD1KlcctnpPY8G5c4TTXOF2tfrgcT9m4,1560
-h5yaml/yaml_h5.py,sha256=jVZAL5Cu6dqgdn25cgVBP3g7yx2wdE-cmCbkaOmQKZ4,10153
-h5yaml/yaml_nc.py,sha256=M6g4ZTPMlPmGZjyn3mLFnoQpmynGwgA0HtlUXSGNvvw,10963
-h5yaml/Data/h5_compound.yaml,sha256=z3dMCJDRAw14boRp0zT74bz_oFi21yu8coUoKOW-d2Q,1131
-h5yaml/Data/h5_testing.yaml,sha256=NhXeXjdblh3bv1hPjCl5DvIhEXo3EpD4mlgaZDElsJc,5626
-h5yaml/Data/h5_unsupported.yaml,sha256=v4HYhiTikFt6UoEUJBnmSse_WeHbmBgqF2e1bCJEfLw,1502
-h5yaml/Data/nc_testing.yaml,sha256=zuXcYrcuCankndt5e4nRPj2-xed97IA9yvfpn89XQgw,5451
-h5yaml/lib/__init__.py,sha256=NdNciPgYnbq-aVM6QqNGNZtdL72rTGLAMrDy0Yw7ckk,751
-h5yaml/lib/adjust_attr.py,sha256=4dHEGwwIa3a3hihyuSX8jCsC08fYcz_9XWA1pBwiwfc,2284
-h5yaml/lib/chunksizes.py,sha256=R1kdaKWF0Hol_maZ0tPDoUyWIH5RatP7d2J1yBA8bkk,1949
-h5yaml-0.2.0.dist-info/METADATA,sha256=IZxpXl9fI3Z7pa1DoRumjxPZXn8N_ykkJEdaUNnFJlw,7280
-h5yaml-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-h5yaml-0.2.0.dist-info/licenses/LICENSE,sha256=rLarIZOYK5jHuUjMnFbgdI_Tb_4_HAAKSOOIhwiWlE4,11356
-h5yaml-0.2.0.dist-info/RECORD,,