pyhdf5-handler 0.6__tar.gz → 0.8__tar.gz

{pyhdf5_handler-0.6 → pyhdf5_handler-0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyhdf5_handler
-Version: 0.6
+Version: 0.8
 Summary: A Python library to read and write data to HDF5 format. This library is based on the h5py (https://docs.h5py.org/en/stable/index.html) and has been developed by Maxime Jay-Allemand at Hydris hydrologie (https://www.hydris-hydrologie.fr/)
 Project-URL: Homepage, https://codeberg.org/maximejay/pyhdf5_handler
 Project-URL: Issues, https://codeberg.org/maximejay/pyhdf5_handler/issues
@@ -169,6 +169,29 @@ pyhdf5_handler.save_dict_to_hdf5file(
 )
 ```
 
+# Exclude object type
+
+Many python object are very complex and huge and there is no reason to parse and save it into an hdf5file. Pyhdf5_handler may crash handling such python object like geopandas.GeoDataFrame or shapely.Polygon. These module, geopandas and shapely, are excluded. To exclude other modules, you may use the global constant variable `pyhdf5_handler.EXCLUDE_PYTHON_OBJ`. The key will be keept, but the dataset will contain a string with the mention "excluded data type <class 'shapely.geometry.polygon.Polygon'>".
+
+```python
+pyhdf5_handler.EXCLUDE_PYTHON_OBJ
+['geopandas', 'shapely']
+pyhdf5_handler.EXCLUDE_PYTHON_OBJ.append("numpy")  # will exclude numpy also
+mydict = {
+        "g": geopandas.GeoDataFrame(),
+        "s": shapely.Polygon(),
+        "n_exclude": np.zeros(0),
+    }
+pyhdf5_handler.save_dict_to_hdf5file("./test.hdf5", mydict)
+pyhdf5_handler.get_hdf5file_item(
+        path_to_hdf5="./test.hdf5",
+        location="./",
+        item="n_exclude",
+        search_attrs=False,
+    )
+["excluded data type <class 'numpy.ndarray'>"]
+```
+
 #### Viewing the content of an HDF5 file
 
 Two methods can be used to view the content of an hdf5 file. The method `pyhdf5_handler.src.hdf5_handler.hdf5file_view` print the full recursive arborescence of an hdf5 file, including  dataset and attribute:

{pyhdf5_handler-0.6 → pyhdf5_handler-0.8}/README.md

@@ -152,6 +152,29 @@ pyhdf5_handler.save_dict_to_hdf5file(
 )
 ```
 
+# Exclude object type
+
+Many python object are very complex and huge and there is no reason to parse and save it into an hdf5file. Pyhdf5_handler may crash handling such python object like geopandas.GeoDataFrame or shapely.Polygon. These module, geopandas and shapely, are excluded. To exclude other modules, you may use the global constant variable `pyhdf5_handler.EXCLUDE_PYTHON_OBJ`. The key will be keept, but the dataset will contain a string with the mention "excluded data type <class 'shapely.geometry.polygon.Polygon'>".
+
+```python
+pyhdf5_handler.EXCLUDE_PYTHON_OBJ
+['geopandas', 'shapely']
+pyhdf5_handler.EXCLUDE_PYTHON_OBJ.append("numpy")  # will exclude numpy also
+mydict = {
+        "g": geopandas.GeoDataFrame(),
+        "s": shapely.Polygon(),
+        "n_exclude": np.zeros(0),
+    }
+pyhdf5_handler.save_dict_to_hdf5file("./test.hdf5", mydict)
+pyhdf5_handler.get_hdf5file_item(
+        path_to_hdf5="./test.hdf5",
+        location="./",
+        item="n_exclude",
+        search_attrs=False,
+    )
+["excluded data type <class 'numpy.ndarray'>"]
+```
+
 #### Viewing the content of an HDF5 file
 
 Two methods can be used to view the content of an hdf5 file. The method `pyhdf5_handler.src.hdf5_handler.hdf5file_view` print the full recursive arborescence of an hdf5 file, including  dataset and attribute:

{pyhdf5_handler-0.6 → pyhdf5_handler-0.8}/Release_note.txt

@@ -1,3 +1,14 @@
+v0.8 : 19-06-2026
+- relase 0.8
+- Add support for pd.DataFrame()
+- improve function map_dict_to_object
+- Fix : return in hdf5_dataset_creator after parsing an unknown object
+- Add function to map a dictionary to an object with a given structure
+
+v0.7 : 12-03-2026
+- Exclude some python object when parsing a dict or an object. For instance Geopandas and shapely are excluded, because pyhdf5_handler crash when handled these complex python object. Other module can be exclude using the global variable   EXCLUDE_PYTHON_OBJ.
+- Fix: Max recursion counter was set to zeros at the begining of the recursive function.
+
 v0.6 : 11-02-2026
 - Fix : handle np.int64/np.float64 type like. Pyhdf5_handler will correctly return unique float or int stored in a dataset.
 

{pyhdf5_handler-0.6 → pyhdf5_handler-0.8}/pyhdf5_handler/__init__.py

@@ -4,4 +4,6 @@
 
 from .src.hdf5_handler import close_all_hdf5_file, open_hdf5, add_hdf5_sub_group, hdf5_dataset_creator, save_dict_to_hdf5, save_dict_to_hdf5file, save_object_to_hdf5file, read_hdf5file_as_dict, read_hdf5_as_dict, hdf5_read_dataset, get_hdf5file_attribute, get_hdf5file_dataset, get_hdf5file_item, get_hdf5_item, search_in_hdf5file, search_in_hdf5, hdf5file_view, hdf5file_ls, hdf5_ls, hdf5_view
 
-from .src.object_handler import generate_dict_structure, generate_object_structure, read_object_as_dict
+from .src.object_handler import generate_dict_structure, generate_object_structure, read_object_as_dict, map_dict_to_object
+
+from .src.constant import EXCLUDE_PYTHON_OBJ

{pyhdf5_handler-0.6 → pyhdf5_handler-0.8}/pyhdf5_handler/src/__init__.py

@@ -1,3 +1,4 @@
 from .hdf5_handler import *
 from .object_handler import *
+from .constant import *
 

pyhdf5_handler-0.8/pyhdf5_handler/src/constant.py

@@ -0,0 +1,3 @@
+EXCLUDE_PYTHON_OBJ=["geopandas", "shapely"]
+
+

{pyhdf5_handler-0.6 → pyhdf5_handler-0.8}/pyhdf5_handler/src/hdf5_handler.py

@@ -10,6 +10,8 @@ import time
 import importlib
 
 from ..src import object_handler
+from ..src import constant
+
 import gc
 import re
 
@@ -97,7 +99,9 @@ def open_hdf5(path, read_only=False, replace=False, wait_time=0):
                 print(f"File {path} does not exist.")
                 return f
             else:
-                print(f"The file {path} is unvailable, waiting {wait}/{wait_time}s")
+                print(
+                    f"The file {path} is unvailable, waiting {wait}/{wait_time}s"
+                )
 
             wait = wait + 1
 
@@ -214,13 +218,19 @@ def _dump_object_to_hdf5_from_dict_attribute(hdf5, instance, dict_attr):
                     sub_instance = instance
 
             if isinstance(value, dict):
-                _dump_object_to_hdf5_from_dict_attribute(hdf5[attr], sub_instance, value)
+                _dump_object_to_hdf5_from_dict_attribute(
+                    hdf5[attr], sub_instance, value
+                )
 
             elif isinstance(value, list):
-                _dump_object_to_hdf5_from_list_attribute(hdf5[attr], sub_instance, value)
+                _dump_object_to_hdf5_from_list_attribute(
+                    hdf5[attr], sub_instance, value
+                )
 
             elif isinstance(value, str):
-                _dump_object_to_hdf5_from_str_attribute(hdf5[attr], sub_instance, value)
+                _dump_object_to_hdf5_from_str_attribute(
+                    hdf5[attr], sub_instance, value
+                )
 
             else:
 
@@ -390,6 +400,25 @@ def _hdf5_handle_DatetimeIndex(name: str, value: pd.DatetimeIndex):
     return dataset
 
 
+def _hdf5_handle_PandaDataFrame(
+    hdf5: h5py.File, name: str, value: pd.DataFrame
+):
+
+    hdf5 = add_hdf5_sub_group(hdf5, subgroup=name)
+    hdf5_data = hdf5[name]
+
+    hdf5_data = add_hdf5_sub_group(hdf5_data, subgroup="pd_DataFrame")
+    hdf5_data = hdf5_data["pd_DataFrame"]
+
+    keys = _hdf5_handle_array("columns", np.array(list(value.columns)))
+    _hdf5_create_dataset(hdf5_data, keys)
+
+    dataset = _hdf5_handle_array("array", value.to_numpy())
+    _hdf5_create_dataset(hdf5_data, dataset)
+
+    return
+
+
 def _hdf5_handle_list(name: str, value: list | tuple):
 
     arr = np.array(value)
@@ -399,6 +428,32 @@ def _hdf5_handle_list(name: str, value: list | tuple):
     return dataset
 
 
+def _hdf5_handle_exclude_obj(name: str, value: list | tuple):
+
+    dtype = type(value)
+
+    dataset = {
+        "name": name,
+        "attr_value": str(dtype),
+        "dataset_value": f"excluded data type {str(dtype)}",
+        "shape": 1,
+        "dtype": h5py.string_dtype(encoding="utf-8"),
+    }
+
+    return dataset
+
+
+def _hdf5_skip_cls(value):
+
+    type_str = str(type(value))
+    module_name = type_str.split("'")[1].split(".")[0]
+
+    if module_name in constant.EXCLUDE_PYTHON_OBJ:
+        return True
+    else:
+        return False
+
+
 def _hdf5_handle_array(name: str, value: np.ndarray):
 
     dtype_attr = type(value)
@@ -477,8 +532,11 @@ def hdf5_dataset_creator(hdf5: h5py.File, name: str, value):
         value to write in the hdf5
 
     """
-    # save ndarray datast
-    if isinstance(value, str):
+
+    if _hdf5_skip_cls(value):
+        dataset = _hdf5_handle_exclude_obj(name, value)
+
+    elif isinstance(value, str):
         dataset = _hdf5_handle_str(name, value)
 
     elif isinstance(value, numbers.Number):
@@ -493,6 +551,12 @@ def hdf5_dataset_creator(hdf5: h5py.File, name: str, value):
     elif isinstance(value, pd.DatetimeIndex):
         dataset = _hdf5_handle_DatetimeIndex(name, value)
 
+    # TODO : To be tested
+    elif isinstance(value, pd.DataFrame):
+
+        _hdf5_handle_PandaDataFrame(hdf5, name, value)
+        return
+
     elif isinstance(value, list):
         dataset = _hdf5_handle_list(name, value)
 
@@ -515,6 +579,8 @@ def hdf5_dataset_creator(hdf5: h5py.File, name: str, value):
 
         save_dict_to_hdf5(hdf5[name], newdict)
 
+        return
+
     _hdf5_create_dataset(hdf5, dataset)
 
 
@@ -572,6 +638,34 @@ def _dump_ndarray_to_hdf5(hdf5, value):
     )
 
 
+def _read_pd_dataframe(hdf5):
+    """
+    read a pandas dataframe data structure from hdf5 file
+
+    Parameters
+    ----------
+
+    hdf5 : h5py.File
+        an hdf5 object at the roots of the ndarray datastructure
+
+    Return
+    ------
+
+    pd.DataFrame : the pandas dataframe
+
+    """
+
+    if "pd_DataFrame" in list(hdf5.keys()):
+        columns = hdf5["pd_DataFrame/columns"][:]
+        array = hdf5["pd_DataFrame/array"][:]
+
+    newdict = {}
+    for i, col in enumerate(columns):
+        newdict.update({col.decode(): array[:, i]})
+
+    return pd.DataFrame(newdict)
+
+
 def _read_ndarray_datastructure(hdf5):
     """
     read a ndarray data structure from hdf5 file
@@ -826,7 +920,11 @@ def save_object_to_hdf5file(
 
 
 def read_hdf5file_as_dict(
-    path_to_hdf5, location="./", wait_time=0, read_attrs=True, read_dataset_attrs=False
+    path_to_hdf5,
+    location="./",
+    wait_time=0,
+    read_attrs=True,
+    read_dataset_attrs=False,
 ):
     """
 
@@ -868,7 +966,9 @@ def read_hdf5file_as_dict(
         return None
 
     dictionary = read_hdf5_as_dict(
-        hdf5[location], read_attrs=read_attrs, read_dataset_attrs=read_dataset_attrs
+        hdf5[location],
+        read_attrs=read_attrs,
+        read_dataset_attrs=read_dataset_attrs,
     )
 
     hdf5.close()
@@ -907,7 +1007,9 @@ def read_hdf5_as_dict(hdf5, read_attrs=True, read_dataset_attrs=False):
 
     """
 
-    if not isinstance(hdf5, (h5py.File, h5py.Group, h5py.Dataset, h5py.Datatype)):
+    if not isinstance(
+        hdf5, (h5py.File, h5py.Group, h5py.Dataset, h5py.Datatype)
+    ):
         print("Error: input arg is not an instance of hdf5.File()")
         return {}
 
@@ -920,7 +1022,12 @@ def read_hdf5_as_dict(hdf5, read_attrs=True, read_dataset_attrs=False):
             if key == "ndarray_ds":
 
                 # dictionary.update({key: _read_ndarray_datastructure(hdf5)})
-                return _read_ndarray_datastructure(hdf5)
+                values = _read_ndarray_datastructure(hdf5)
+                dictionary.update({key: values})
+
+            elif key == "pd_DataFrame":
+                values = _read_pd_dataframe(hdf5)
+                dictionary.update({key: values})
 
             else:
 
@@ -941,14 +1048,17 @@ def read_hdf5_as_dict(hdf5, read_attrs=True, read_dataset_attrs=False):
     list_attribute = []
     if read_attrs or read_dataset_attrs:
         tmp_list_attribute = list(hdf5.attrs.keys())
-        hdf5_item_matching_attributes = ["_" + element for element in list(hdf5.keys())]
+        hdf5_item_matching_attributes = [
+            "_" + element for element in list(hdf5.keys())
+        ]
 
     if read_attrs:
 
         list_attribute.extend(
             list(
                 filter(
-                    lambda l: l not in hdf5_item_matching_attributes, tmp_list_attribute
+                    lambda l: l not in hdf5_item_matching_attributes,
+                    tmp_list_attribute,
                 )
             )
         )
@@ -956,7 +1066,12 @@ def read_hdf5_as_dict(hdf5, read_attrs=True, read_dataset_attrs=False):
     if read_dataset_attrs:
 
         list_attribute.extend(
-            list(filter(lambda l: l in hdf5_item_matching_attributes, tmp_list_attribute))
+            list(
+                filter(
+                    lambda l: l in hdf5_item_matching_attributes,
+                    tmp_list_attribute,
+                )
+            )
         )
 
     for key in list_attribute:
@@ -1065,7 +1180,11 @@ def hdf5_read_dataset(item, expected_type=None):
 
         values = None
 
-    elif expected_type in (str(pd.Timestamp), str(np.datetime64), str(datetime.datetime)):
+    elif expected_type in (
+        str(pd.Timestamp),
+        str(np.datetime64),
+        str(datetime.datetime),
+    ):
 
         if expected_type == str(pd.Timestamp):
             values = pd.Timestamp(item[0].decode())
@@ -1149,7 +1268,9 @@ def get_hdf5file_attribute(
     return return_attribute
 
 
-def get_hdf5file_dataset(path_to_hdf5=str(), location="./", dataset=None, wait_time=0):
+def get_hdf5file_dataset(
+    path_to_hdf5=str(), location="./", dataset=None, wait_time=0
+):
     """
     Get the value of an attribute in the hdf5file
 
@@ -1201,7 +1322,11 @@ def get_hdf5file_dataset(path_to_hdf5=str(), location="./", dataset=None, wait_t
 
 
 def get_hdf5file_item(
-    path_to_hdf5=str(), location="./", item=None, wait_time=0, search_attrs=False
+    path_to_hdf5=str(),
+    location="./",
+    item=None,
+    wait_time=0,
+    search_attrs=False,
 ):
     """
 
@@ -1244,7 +1369,10 @@ def get_hdf5file_item(
         return None
 
     hdf5_item = get_hdf5_item(
-        hdf5_instance=hdf5, location=location, item=item, search_attrs=search_attrs
+        hdf5_instance=hdf5,
+        location=location,
+        item=item,
+        search_attrs=search_attrs,
     )
 
     hdf5.close()
@@ -1252,7 +1380,9 @@ def get_hdf5file_item(
     return hdf5_item
 
 
-def get_hdf5_item(hdf5_instance=None, location="./", item=None, search_attrs=False):
+def get_hdf5_item(
+    hdf5_instance=None, location="./", item=None, search_attrs=False
+):
     """
 
     Get a custom item in an hdf5file
@@ -1320,6 +1450,10 @@ def get_hdf5_item(hdf5_instance=None, location="./", item=None, search_attrs=Fal
 
                 return _read_ndarray_datastructure(hdf5)
 
+            elif item == "pd_DataFrame":
+
+                return _read_pd_dataframe(hdf5)
+
             else:
 
                 returned_dict = read_hdf5_as_dict(hdf5_item)
@@ -1391,7 +1525,9 @@ def search_in_hdf5file(
     if hdf5 is None:
         return None
 
-    results = search_in_hdf5(hdf5, key, location=location, search_attrs=search_attrs)
+    results = search_in_hdf5(
+        hdf5, key, location=location, search_attrs=search_attrs
+    )
 
     hdf5.close()
 
@@ -1513,7 +1649,12 @@ def search_in_hdf5(hdf5_base, key=None, location="./", search_attrs=False):
                     values = item[:]
 
                 result.append(
-                    {"path": location, "key": key, "datatype": "dataset", "value": values}
+                    {
+                        "path": location,
+                        "key": key,
+                        "datatype": "dataset",
+                        "value": values,
+                    }
                 )
 
     return result
@@ -1745,14 +1886,17 @@ def hdf5_view(
     list_attribute = []
     if list_attrs or list_dataset_attrs:
         tmp_list_attribute = list(hdf5.attrs.keys())
-        list_keys_matching_attributes = ["_" + element for element in list(hdf5.keys())]
+        list_keys_matching_attributes = [
+            "_" + element for element in list(hdf5.keys())
+        ]
 
     if list_attrs:
 
         list_attribute.extend(
             list(
                 filter(
-                    lambda l: l not in list_keys_matching_attributes, tmp_list_attribute
+                    lambda l: l not in list_keys_matching_attributes,
+                    tmp_list_attribute,
                 )
             )
         )
@@ -1760,14 +1904,20 @@ def hdf5_view(
     if list_dataset_attrs:
 
         list_attribute.extend(
-            list(filter(lambda l: l in list_keys_matching_attributes, tmp_list_attribute))
+            list(
+                filter(
+                    lambda l: l in list_keys_matching_attributes,
+                    tmp_list_attribute,
+                )
+            )
         )
 
     for key in list_attribute:
         values = hdf5.attrs[key]
         sub_location = os.path.join(location, key)
         if isinstance(
-            values, (int, float, np.int64, np.float64, np.int32, np.float32, np.bool)
+            values,
+            (int, float, np.int64, np.float64, np.int32, np.float32, np.bool),
         ):
             result.append(
                 f"{level_base}| {sub_location}, attribute, type={type(hdf5.attrs[key])}, value={values}"

{pyhdf5_handler-0.6 → pyhdf5_handler-0.8}/pyhdf5_handler/src/object_handler.py

@@ -1,41 +1,69 @@
-
 from __future__ import annotations
 
 import numpy as np
 import numbers
 import pandas as pd
 import datetime
+from ..src import constant
 
 
 def _isinstance_pandas(value):
-    pandas_classes = [getattr(pd, item) for item in dir(pd) if isinstance(getattr(pd, item), type)]
+    pandas_classes = [
+        getattr(pd, item)
+        for item in dir(pd)
+        if isinstance(getattr(pd, item), type)
+    ]
     for cls in pandas_classes:
         if isinstance(value, cls):
             return True
     return False
 
+
 def _isinstance_numpy(value):
-    numpy_classes = [getattr(np, item) for item in dir(np) if isinstance(getattr(np, item), type)]
+    numpy_classes = [
+        getattr(np, item)
+        for item in dir(np)
+        if isinstance(getattr(np, item), type)
+    ]
     for cls in numpy_classes:
         if isinstance(value, cls):
             return True
     return False
 
+
 def _isinstance_datetime(value):
-    datetime_classes = [getattr(datetime, item) for item in dir(datetime) if isinstance(getattr(datetime, item), type)]
+    datetime_classes = [
+        getattr(datetime, item)
+        for item in dir(datetime)
+        if isinstance(getattr(datetime, item), type)
+    ]
     for cls in datetime_classes:
         if isinstance(value, cls):
             return True
     return False
 
-def generate_dict_structure(dictionary,recursion_counter=0,recursion_limit=100,include_method=True):
+
+def _isinstance_exclude_obj(value):
+
+    type_str = str(type(value))
+    module_name = type_str.split("'")[1].split(".")[0]
+
+    if module_name in constant.EXCLUDE_PYTHON_OBJ:
+        return True
+    else:
+        return False
+
+
+def generate_dict_structure(
+    dictionary, recursion_counter=0, recursion_limit=100, include_method=True
+):
     """
-    
+
     this function create a full dictionnary containing all the structure of an dictionnary in order to save it to an hdf5
 
     Parameters
     ----------
-    
+
     instance : python dictionary
         a custom dictionary.
     recursion_limit : int
@@ -47,70 +75,148 @@ def generate_dict_structure(dictionary,recursion_counter=0,recursion_limit=100,i
 
     Returns
     -------
-    
+
     list or dict :
         A list or dictionary matching the structure of the python object.
-    
+
     """
-    key_data={}
+    key_data = {}
     key_list = list()
-    recursion_counter=0
-    for attr,value in dictionary.items():
-        
+    recursion_counter = 0
+    for attr, value in dictionary.items():
+
         try:
-            if isinstance(value,dict):
-                
-                subkey_data=generate_dict_structure(value)
-                if len(subkey_data)>0:
-                    key_data.update({attr:subkey_data})
-            
+            if _isinstance_exclude_obj(value):
+                next(attrs)
+            elif isinstance(value, dict):
+                subkey_data = generate_dict_structure(value)
+                if len(subkey_data) > 0:
+                    key_data.update({attr: subkey_data})
+
             elif isinstance(value, (list, tuple, numbers.Number, str)):
                 key_list.append(attr)
-            
+
             elif _isinstance_pandas(value):
                 key_list.append(attr)
-                
+
             elif _isinstance_datetime(value):
                 key_list.append(attr)
-                
+
             elif _isinstance_numpy(value):
                 key_list.append(attr)
-            
+
             elif type(value) == "method":
                 if include_method:
                     key_list.append(attr)
                 else:
                     next(attr)
-            
             else:
-                
-                recursion_counter = recursion_counter+1
-                
+
+                recursion_counter = recursion_counter + 1
+
                 if recursion_counter > recursion_limit:
-                    print(f"recursion counter exceed the limit of {recursion_limit}... return")
+                    print(
+                        f"recursion counter exceed the limit of {recursion_limit}... return"
+                    )
                     return
-                
-                subkey_data = generate_object_structure(value,recursion_counter=recursion_counter,recursion_limit=recursion_limit,include_method=include_method)
+
+                subkey_data = generate_object_structure(
+                    value,
+                    recursion_counter=recursion_counter,
+                    recursion_limit=recursion_limit,
+                    include_method=include_method,
+                )
                 if len(subkey_data) > 0:
                     key_data.update({attr: subkey_data})
 
         except:
             pass
-    
+
     for attr, value in key_data.items():
         key_list.append({attr: value})
-    
+
     return key_list
 
 
-def generate_object_structure(instance,recursion_counter=0,recursion_limit=100,include_method=True):
+def generate_object_from_dictionary(dictionary):
+
+    class Object:
+        def __init__(self):
+            pass
+
+    obj = Object()
+
+    if isinstance(dictionary, dict):
+        for attr, value in dictionary.items():
+
+            if isinstance(value, (dict, list, tuple)):
+                setattr(obj, attr, generate_object_from_dictionary(value))
+            else:
+                setattr(obj, attr, value)
+    elif isinstance(dictionary, (list, tuple)):
+        return dictionary
+        # for attr in dictionary:
+        #    setattr(obj, attr, None)
+
+    return obj
+
+
+def map_dict_to_object(structure, dictionary, obj):
+    """
+    Map a dictionary to an object knowing its structure.
+    Structure must be generated with generate_dict_structure or generate_object_structure. If the object or an attribute of this object is
+    None, a empty object is created to be able to set new attribute.
+    """
+
+    class Object:
+        def __init__(self):
+            pass
+
+    if obj is None:
+        obj = Object()
+
+    if isinstance(structure, dict):
+
+        for attr, value in structure.items():
+            if isinstance(value, (dict, list, tuple)):
+
+                if not hasattr(obj, attr):
+                    setattr(obj, str(attr), Object())
+                else:
+                    if getattr(obj, attr) is None:
+                        setattr(obj, str(attr), Object())
+
+                map_dict_to_object(value, dictionary[attr], getattr(obj, attr))
+            else:
+                v = None
+                if attr in dictionary.keys():
+                    v = dictionary[attr]
+
+                setattr(obj, str(attr), v)
+
+    elif isinstance(structure, (list, tuple)):
+
+        for attr in structure:
+            if isinstance(attr, (dict, list, tuple)):
+                map_dict_to_object(attr, dictionary, obj)
+            else:
+                v = None
+                if attr in dictionary.keys():
+                    v = dictionary[attr]
+
+                setattr(obj, str(attr), v)
+
+
+def generate_object_structure(
+    instance, recursion_counter=0, recursion_limit=100, include_method=True
+):
     """
-    
+
     this function create a full dictionnary containing all the structure of an object in order to save it to an hdf5
 
     Parameters
     ----------
-    
+
     instance : object
         a custom python object.
     recursion_limit : int
@@ -122,33 +228,36 @@ def generate_object_structure(instance,recursion_counter=0,recursion_limit=100,i
 
     Returns
     -------
-    
+
     list or dict :
         A list or dictionary matching the structure of the python object.
-    
+
     """
     key_data = {}
     key_list = list()
     return_list = False
-    recursion_counter = 0
+    # recursion_counter += 1
     for attr in dir(instance):
-        
+
         if not attr.startswith("_") and not attr in ["from_handle", "copy"]:
-            
+
             try:
                 value = getattr(instance, attr)
-                
-                if isinstance(value, (list, tuple)):
+
+                if _isinstance_exclude_obj(value):
+                    next(attr)
+
+                elif isinstance(value, (list, tuple)):
                     key_list.append(attr)
                     return_list = True
-                
+
                 elif _isinstance_numpy(value):
                     key_list.append(attr)
                     return_list = True
-                
+
                 elif isinstance(value, dict):
-                    
-                    depp_key_data=generate_dict_structure(value)
+
+                    depp_key_data = generate_dict_structure(value)
                     if len(depp_key_data) > 0:
                         key_data.update({attr: depp_key_data})
 
@@ -166,24 +275,35 @@ def generate_object_structure(instance,recursion_counter=0,recursion_limit=100,i
                         return_list = True
                     else:
                         next(attr)
-                
+
                 elif _isinstance_pandas(value):
                     key_list.append(attr)
                     return_list = True
-                
+
                 elif _isinstance_datetime(value):
                     key_list.append(attr)
                     return_list = True
-                    
+
+                # ~ elif value is None:
+                # ~ key_list.append(attr)
+                # ~ return_list = True
+
                 else:
-                    
-                    recursion_counter = recursion_counter+1
-                    
+
+                    recursion_counter = recursion_counter + 1
+
                     if recursion_counter > recursion_limit:
-                        print(f"recursion counter exceed the limit of {recursion_limit}... return")
+                        print(
+                            f"recursion counter exceed the limit of {recursion_limit}... return"
+                        )
                         return
-                    
-                    depp_key_data = generate_object_structure(value, recursion_counter=recursion_counter,recursion_limit=recursion_limit,include_method=include_method)
+
+                    depp_key_data = generate_object_structure(
+                        value,
+                        recursion_counter=recursion_counter,
+                        recursion_limit=recursion_limit,
+                        include_method=include_method,
+                    )
 
                     if len(depp_key_data) > 0:
                         key_data.update({attr: depp_key_data})
@@ -203,14 +323,14 @@ def generate_object_structure(instance,recursion_counter=0,recursion_limit=100,i
         return key_data
 
 
-def read_object_as_dict(instance, recursion_counter=0,recursion_limit=100):
+def read_object_as_dict(instance, recursion_counter=0, recursion_limit=100):
     """
-    
+
     create a dictionary from a custom python object
 
     Parameters
     ----------
-    
+
     instance : object
         an custom python object
     recursion_limit : int
@@ -220,32 +340,35 @@ def read_object_as_dict(instance, recursion_counter=0,recursion_limit=100):
 
     Return
     ------
-    
+
     key_data: dict
         an dictionary containing all keys and atributes of the object
-    
+
     """
     key_data = {}
-    recursion_counter = 0
+    # recursion_counter = 0
     for attr in dir(instance):
-        #print(attr)
+        # print(attr)
         if not attr.startswith("_") and not attr in ["from_handle", "copy"]:
             try:
                 value = getattr(instance, attr)
-                
-                if isinstance(value, (list, tuple)):
-                    
+
+                if _isinstance_exclude_obj(value):
+                    next(attr)
+
+                elif isinstance(value, (list, tuple)):
+
                     if isinstance(value, list):
-                        value = np.array(value).astype('U')
+                        value = np.array(value).astype("U")
 
                     if value.dtype == "object" or value.dtype.char == "U":
                         value = value.astype("U")
-                    
+
                     key_data.update({attr: value})
-                
+
                 elif isinstance(value, dict):
                     key_data.update({attr: value})
-                
+
                 elif isinstance(value, numbers.Number):
                     key_data.update({attr: value})
 
@@ -254,33 +377,38 @@ def read_object_as_dict(instance, recursion_counter=0,recursion_limit=100):
 
                 elif type(value) == "method":
                     next(attr)
-                
+
                 elif _isinstance_pandas(value):
                     if value.dtype == "object" or value.dtype.char == "U":
                         value = value.astype("U")
                     key_data.update({attr: value})
-                
+
                 elif _isinstance_datetime(value):
                     if value.dtype == "object" or value.dtype.char == "U":
                         value = value.astype("U")
                     key_data.update({attr: value})
-                
+
                 elif _isinstance_numpy(value):
                     if value.dtype == "object" or value.dtype.char == "U":
                         value = value.astype("U")
                     key_data.update({attr: value})
 
                 else:
-                    
-                    recursion_counter = recursion_counter+1
-                    
+
+                    recursion_counter = recursion_counter + 1
+
                     if recursion_counter > recursion_limit:
-                        print(f"recursion counter exceed the limit of {recursion_limit}... return")
+                        print(
+                            f"recursion counter exceed the limit of {recursion_limit}... return"
+                        )
                         return
-                    
+
                     depp_key_data = read_object_as_dict(
-                        value, recursion_counter=recursion_counter,recursion_limit=recursion_limit)
-                    
+                        value,
+                        recursion_counter=recursion_counter,
+                        recursion_limit=recursion_limit,
+                    )
+
                     if len(depp_key_data) > 0:
                         key_data.update({attr: depp_key_data})
 

{pyhdf5_handler-0.6 → pyhdf5_handler-0.8}/pyhdf5_handler/tutorial/hdf5_io_tests.py

@@ -3,6 +3,8 @@ if __name__ == "__main__":
     import pyhdf5_handler
     import datetime
     import pandas as pd
+    import geopandas
+    import shapely
 
     # open an hdf5 database, test.hdf5.
     hdf5 = pyhdf5_handler.open_hdf5("./test.hdf5")
@@ -20,7 +22,9 @@ if __name__ == "__main__":
         hdf5, "timestamp_numpy", np.datetime64("2019-09-22T17:38:30")
     )
     pyhdf5_handler.hdf5_dataset_creator(
-        hdf5, "timestamp_datetime", datetime.datetime.fromisoformat("2019-09-22T17:38:30")
+        hdf5,
+        "timestamp_datetime",
+        datetime.datetime.fromisoformat("2019-09-22T17:38:30"),
     )
     pyhdf5_handler.hdf5_dataset_creator(
         hdf5, "timestamp_pandas", pd.Timestamp("2019-09-22T17:38:30")
@@ -31,7 +35,10 @@ if __name__ == "__main__":
     pyhdf5_handler.hdf5_dataset_creator(
         hdf5,
         "list_date_numpy",
-        [np.datetime64("2019-09-22 17:38:30"), np.datetime64("2019-09-22 18:38:30")],
+        [
+            np.datetime64("2019-09-22 17:38:30"),
+            np.datetime64("2019-09-22 18:38:30"),
+        ],
     )
     pyhdf5_handler.hdf5_dataset_creator(
         hdf5,
@@ -44,10 +51,28 @@ if __name__ == "__main__":
     pyhdf5_handler.hdf5_dataset_creator(
         hdf5,
         "list_date_pandas",
-        [pd.Timestamp("2019-09-22 17:38:30"), pd.Timestamp("2019-09-22 17:38:30")],
+        [
+            pd.Timestamp("2019-09-22 17:38:30"),
+            pd.Timestamp("2019-09-22 17:38:30"),
+        ],
+    )
+    pyhdf5_handler.hdf5_dataset_creator(
+        hdf5,
+        "list_date_range_pandas",
+        pd.date_range(start="1/1/2018", end="1/08/2018"),
+    )
+
+    pyhdf5_handler.hdf5_dataset_creator(
+        hdf5,
+        "panda_dataframe_onecolumn",
+        pd.DataFrame({"column1": np.array([1, 2, 3])}),
     )
     pyhdf5_handler.hdf5_dataset_creator(
-        hdf5, "list_date_range_pandas", pd.date_range(start="1/1/2018", end="1/08/2018")
+        hdf5,
+        "panda_dataframe",
+        pd.DataFrame(
+            {"column1": np.array([1, 2, 3]), "column2": np.array([4, 5, 6])}
+        ),
     )
 
     # write a python dictionary in the hdf5 database
@@ -86,7 +111,9 @@ if __name__ == "__main__":
     data = pyhdf5_handler.read_hdf5_as_dict(hdf5, read_attrs=True)
 
     # read a specific item
-    pyhdf5_handler.hdf5_read_dataset(item=hdf5["str"], expected_type=hdf5.attrs["_str"])
+    pyhdf5_handler.hdf5_read_dataset(
+        item=hdf5["str"], expected_type=hdf5.attrs["_str"]
+    )
     pyhdf5_handler.hdf5_read_dataset(
         item=hdf5["numpy_numbers"], expected_type=hdf5.attrs["_numpy_numbers"]
     )
@@ -94,7 +121,18 @@ if __name__ == "__main__":
         item=hdf5["numbers"], expected_type=hdf5.attrs["_numbers"]
     )
     pyhdf5_handler.hdf5_read_dataset(
-        item=hdf5["list_date_numpy"], expected_type=hdf5.attrs["_list_date_numpy"]
+        item=hdf5["list_date_numpy"],
+        expected_type=hdf5.attrs["_list_date_numpy"],
+    )
+
+    # getting specific item
+    pyhdf5_handler.get_hdf5_item(
+        hdf5_instance=hdf5,
+        location="./panda_dataframe",
+    )
+    pyhdf5_handler.get_hdf5_item(
+        hdf5_instance=hdf5,
+        location="./structured_array",
     )
 
     # close the hdf5
@@ -104,10 +142,40 @@ if __name__ == "__main__":
     pyhdf5_handler.hdf5file_ls("./test.hdf5")
     pyhdf5_handler.hdf5file_ls("./test.hdf5", location="structured_array")
 
-    data = pyhdf5_handler.read_hdf5file_as_dict("./test.hdf5", read_attrs=False)
+    data = pyhdf5_handler.read_hdf5file_as_dict(
+        "./test.hdf5", read_attrs=False
+    )
 
     pyhdf5_handler.save_dict_to_hdf5file("./test.hdf5", data)
 
+    # complex class with exluded datatype
+    pyhdf5_handler.EXCLUDE_PYTHON_OBJ.append("numpy")  # exclude numpy also
+    mydict = {
+        "g": geopandas.GeoDataFrame(),
+        "s": shapely.Polygon(),
+        "n_exclude": np.zeros(0),
+    }
+    pyhdf5_handler.save_dict_to_hdf5file("./test.hdf5", mydict)
+    pyhdf5_handler.get_hdf5file_item(
+        path_to_hdf5="./test.hdf5",
+        location="./",
+        item="g",
+        search_attrs=False,
+    )
+    pyhdf5_handler.get_hdf5file_item(
+        path_to_hdf5="./test.hdf5",
+        location="./",
+        item="s",
+        search_attrs=False,
+    )
+    pyhdf5_handler.get_hdf5file_item(
+        path_to_hdf5="./test.hdf5",
+        location="./",
+        item="n_exclude",
+        search_attrs=False,
+    )
+    pyhdf5_handler.EXCLUDE_PYTHON_OBJ.remove("numpy")
+
     res = pyhdf5_handler.search_in_hdf5file(
         "./test.hdf5", key="date_range", location="./", wait_time=0
     )
@@ -124,15 +192,24 @@ if __name__ == "__main__":
     )
 
     pyhdf5_handler.get_hdf5file_item(
-        path_to_hdf5="./test.hdf5", location="./", item="list_mixte", search_attrs=False
+        path_to_hdf5="./test.hdf5",
+        location="./",
+        item="list_mixte",
+        search_attrs=False,
     )
 
     pyhdf5_handler.get_hdf5file_item(
-        path_to_hdf5="./test.hdf5", location="./", item="attribute", search_attrs=True
+        path_to_hdf5="./test.hdf5",
+        location="./",
+        item="attribute",
+        search_attrs=True,
     )
 
     pyhdf5_handler.get_hdf5file_attribute(
-        path_to_hdf5="./test.hdf5", location="./", attribute="_list_num", wait_time=0
+        path_to_hdf5="./test.hdf5",
+        location="./",
+        attribute="_list_num",
+        wait_time=0,
     )
 
     pyhdf5_handler.get_hdf5file_attribute(

{pyhdf5_handler-0.6 → pyhdf5_handler-0.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pyhdf5_handler"
-version = "0.6"
+version = "0.8"
 authors = [
   { name="Maxime Jay-Allemand", email="maxime.jay.allemand@hydris-hydrologie.fr" },
 ]

pyhdf5_handler-0.6/test_fichier_julie.py

@@ -1,42 +0,0 @@
-import pyhdf5_handler
-import numpy as np
-
-#states
-pyhdf5_handler.hdf5file_view("202309010300_states.hdf5")
-
-res=pyhdf5_handler.read_hdf5file_as_dict("202309010300_states.hdf5")
-res.keys()
-res["202309010300"].keys()
-res["202309010300"]["keys"][0].decode()
-res["202309010300"]["keys"][:].astype("str")
-
-states=res["202309010300"]["values"]
-
-states=pyhdf5_handler.get_hdf5file_item("202309010300_states.hdf5",location="./202309010300",item="values")
-
-search=pyhdf5_handler.search_in_hdf5file("202309010300_states.hdf5","values")
-res=search[0]
-res.keys()
-res["path"]
-res["key"]
-res["datatype"]
-res["value"]
-
-#prévis ensemble:
-pyhdf5_handler.hdf5file_view("20230901030000_qens.hdf5")
-
-res=pyhdf5_handler.read_hdf5file_as_dict("20230901030000_qens.hdf5")
-
-res.keys()
-res["20230901030000"].keys()
-Q=res["20230901030000"]["member0"]
-
-Q=pyhdf5_handler.get_hdf5file_item("20230901030000_qens.hdf5",location="./20230901030000",item="member8")
-
-search=pyhdf5_handler.search_in_hdf5file("20230901030000_qens.hdf5","member8")
-res=search[0]
-res.keys()
-res["path"]
-res["key"]
-res["datatype"]
-res["value"]