pyhdf5-handler 0.8__tar.gz → 0.10__tar.gz

{pyhdf5_handler-0.8 → pyhdf5_handler-0.10}/.spyproject/config/backups/workspace.ini.bak

@@ -4,7 +4,7 @@ save_data_on_exit = True
 save_history = True
 save_non_project_files = False
 project_type = 'empty-project-type'
-recent_files = ['pyhdf5_handler/src/hdf5_handler.py', 'pyhdf5_handler/src/object_handler.py', 'pyhdf5_handler/tutorial/hdf5_io_tests.py', 'pyhdf5_handler/tutorial/complementary_test.py', 'test_fichier_julie.py']
+recent_files = ['pyhdf5_handler/src/hdf5_handler.py', 'pyhdf5_handler/src/object_handler.py', 'pyhdf5_handler/tutorial/hdf5_io_tests.py', 'pyhdf5_handler/tutorial/complementary_test.py']
 
 [main]
 version = 0.2.0

{pyhdf5_handler-0.8 → pyhdf5_handler-0.10}/.spyproject/config/workspace.ini

@@ -4,7 +4,7 @@ save_data_on_exit = True
 save_history = True
 save_non_project_files = False
 project_type = 'empty-project-type'
-recent_files = ['pyhdf5_handler/src/hdf5_handler.py', 'pyhdf5_handler/src/object_handler.py', 'pyhdf5_handler/tutorial/hdf5_io_tests.py', 'pyhdf5_handler/tutorial/complementary_test.py', 'test_fichier_julie.py']
+recent_files = ['pyhdf5_handler/src/hdf5_handler.py', 'pyhdf5_handler/src/object_handler.py', 'pyhdf5_handler/tutorial/hdf5_io_tests.py', 'pyhdf5_handler/tutorial/complementary_test.py']
 
 [main]
 version = 0.2.0

{pyhdf5_handler-0.8 → pyhdf5_handler-0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyhdf5_handler
-Version: 0.8
+Version: 0.10
 Summary: A Python library to read and write data to HDF5 format. This library is based on the h5py (https://docs.h5py.org/en/stable/index.html) and has been developed by Maxime Jay-Allemand at Hydris hydrologie (https://www.hydris-hydrologie.fr/)
 Project-URL: Homepage, https://codeberg.org/maximejay/pyhdf5_handler
 Project-URL: Issues, https://codeberg.org/maximejay/pyhdf5_handler/issues
@@ -29,7 +29,7 @@ Read and write to hdf5 are currently limited to the following supported Python t
 * Pandas DatetimeIndex
 * Numpy array
 * Structured numpy array
-
+* pandas DataFrame
 
 Basically, data are stored in hdf5 file using the h5py pythonic interface to the binary HDF5 format. All data are stored in dataset using Numpy. If the data type is not supported by the HDF5 format, the data will be converted to a supported type (byte for string). For each stored dataset, an attribute, containing the type of the original data, is auto-generated. This attribute help to convert back the data to its original type when reading the HDF5. 
 
@@ -111,6 +111,15 @@ pyhdf5_handler.save_dict_to_hdf5file("./test.hdf5", dictionary)
 ```
 Function argument `location` can be used to save the dictionary in a specific group. the group will be created if not exist. The function argument `replace` can be used to replace the HDF5 file, by default pyhdf5_handler will append to it.
 
+Reading an hdf5 is easy. Just use the function `pyhdf5_handler.src.hdf5_handler.read_hdf5file_as_dict`:
+
+```python
+data = pyhdf5_handler.read_hdf5file_as_dict(
+        "./test.hdf5", read_attrs=False
+    )
+```
+
+
 #### Writing a Python "object" (class) into an HDF5 file
 
 The function `pyhdf5_handler.src.hdf5_handler.save_object_to_hdf5file` lets you write any Python class-object into an hdf5 file. The hdf5 file will mimic the structure of this class object.

{pyhdf5_handler-0.8 → pyhdf5_handler-0.10}/README.md

@@ -12,7 +12,7 @@ Read and write to hdf5 are currently limited to the following supported Python t
 * Pandas DatetimeIndex
 * Numpy array
 * Structured numpy array
-
+* pandas DataFrame
 
 Basically, data are stored in hdf5 file using the h5py pythonic interface to the binary HDF5 format. All data are stored in dataset using Numpy. If the data type is not supported by the HDF5 format, the data will be converted to a supported type (byte for string). For each stored dataset, an attribute, containing the type of the original data, is auto-generated. This attribute help to convert back the data to its original type when reading the HDF5. 
 
@@ -94,6 +94,15 @@ pyhdf5_handler.save_dict_to_hdf5file("./test.hdf5", dictionary)
 ```
 Function argument `location` can be used to save the dictionary in a specific group. the group will be created if not exist. The function argument `replace` can be used to replace the HDF5 file, by default pyhdf5_handler will append to it.
 
+Reading an hdf5 is easy. Just use the function `pyhdf5_handler.src.hdf5_handler.read_hdf5file_as_dict`:
+
+```python
+data = pyhdf5_handler.read_hdf5file_as_dict(
+        "./test.hdf5", read_attrs=False
+    )
+```
+
+
 #### Writing a Python "object" (class) into an HDF5 file
 
 The function `pyhdf5_handler.src.hdf5_handler.save_object_to_hdf5file` lets you write any Python class-object into an hdf5 file. The hdf5 file will mimic the structure of this class object.

{pyhdf5_handler-0.8 → pyhdf5_handler-0.10}/Release_note.txt

@@ -1,3 +1,11 @@
+
+v0.10: Fri Jul 31 17:30:46 2026 +0200
+- Improvement : writting and reading a pandas dataframe and a numpy ndarray no longer save and read it in a specific subgroup.
+
+
+v0.9 : 23-07-2026
+- fix: handle dtype with panda dataframe: read/write
+
 v0.8 : 19-06-2026
 - relase 0.8
 - Add support for pd.DataFrame()

{pyhdf5_handler-0.8 → pyhdf5_handler-0.10}/pyhdf5_handler/src/hdf5_handler.py

@@ -407,8 +407,7 @@ def _hdf5_handle_PandaDataFrame(
     hdf5 = add_hdf5_sub_group(hdf5, subgroup=name)
     hdf5_data = hdf5[name]
 
-    hdf5_data = add_hdf5_sub_group(hdf5_data, subgroup="pd_DataFrame")
-    hdf5_data = hdf5_data["pd_DataFrame"]
+    hdf5_data.attrs["_Pandas_DataFrame"] = 1
 
     keys = _hdf5_handle_array("columns", np.array(list(value.columns)))
     _hdf5_create_dataset(hdf5_data, keys)
@@ -416,6 +415,18 @@ def _hdf5_handle_PandaDataFrame(
     dataset = _hdf5_handle_array("array", value.to_numpy())
     _hdf5_create_dataset(hdf5_data, dataset)
 
+    dtype = []
+    for col in value.columns:
+        if pd.api.types.is_string_dtype(value[col]):
+            dtype_name = "str"
+        else:
+            dtype_name = value[col].dtype.name
+
+        dtype.append(dtype_name)
+
+    dataset = _hdf5_handle_list("dtype", dtype)
+    _hdf5_create_dataset(hdf5_data, dataset)
+
     return
 
 
@@ -492,7 +503,10 @@ def _hdf5_handle_array(name: str, value: np.ndarray):
 def _hdf5_handle_ndarray(hdf5: h5py.File, name: str, value: np.ndarray):
 
     hdf5 = add_hdf5_sub_group(hdf5, subgroup=name)
-    _dump_ndarray_to_hdf5(hdf5[name], value)
+    hdf5_data = hdf5[name]
+
+    hdf5_data.attrs["_numpy_ndarray"] = 1
+    _dump_ndarray_to_hdf5(hdf5_data, value)
 
 
 def _hdf5_create_dataset(hdf5: h5py.File, dataset: dict):
@@ -599,13 +613,12 @@ def _dump_ndarray_to_hdf5(hdf5, value):
 
     """
     # save ndarray datastructure
-
-    hdf5 = add_hdf5_sub_group(hdf5, subgroup="ndarray_ds")
-    hdf5_data = hdf5["ndarray_ds"]
+    # hdf5 = add_hdf5_sub_group(hdf5, subgroup="ndarray_ds")
+    # hdf5_data = hdf5["ndarray_ds"]
 
     for item in value.dtype.names:
 
-        hdf5_dataset_creator(hdf5=hdf5_data, name=item, value=value[item])
+        hdf5_dataset_creator(hdf5=hdf5, name=item, value=value[item])
 
     index = np.array(value.dtype.descr)[:, 0]
     dtype = np.array(value.dtype.descr)[:, 1]
@@ -613,10 +626,10 @@ def _dump_ndarray_to_hdf5(hdf5, value):
     dtype = dtype.astype("O")
     data_type = h5py.string_dtype(encoding="utf-8")
 
-    if "ndarray_dtype" in hdf5_data.keys():
-        del hdf5_data["ndarray_dtype"]
+    if "ndarray_dtype" in hdf5.keys():
+        del hdf5["ndarray_dtype"]
 
-    hdf5_data.create_dataset(
+    hdf5.create_dataset(
         "ndarray_dtype",
         shape=dtype.shape,
         dtype=data_type,
@@ -625,10 +638,10 @@ def _dump_ndarray_to_hdf5(hdf5, value):
         chunks=True,
     )
 
-    if "ndarray_indexes" in hdf5_data.keys():
-        del hdf5_data["ndarray_indexes"]
+    if "ndarray_indexes" in hdf5.keys():
+        del hdf5["ndarray_indexes"]
 
-    hdf5_data.create_dataset(
+    hdf5.create_dataset(
         "ndarray_indexes",
         shape=index.shape,
         dtype=data_type,
@@ -655,13 +668,14 @@ def _read_pd_dataframe(hdf5):
 
     """
 
-    if "pd_DataFrame" in list(hdf5.keys()):
-        columns = hdf5["pd_DataFrame/columns"][:]
-        array = hdf5["pd_DataFrame/array"][:]
+    if "_Pandas_DataFrame" in list(hdf5.attrs.keys()):
+        columns = hdf5["columns"][:]
+        array = hdf5["array"][:]
+        dtype = hdf5["dtype"][:]
 
     newdict = {}
     for i, col in enumerate(columns):
-        newdict.update({col.decode(): array[:, i]})
+        newdict.update({col.decode(): array[:, i].astype(dtype[i])})
 
     return pd.DataFrame(newdict)
 
@@ -683,39 +697,36 @@ def _read_ndarray_datastructure(hdf5):
 
     """
 
-    if "ndarray_ds" in list(hdf5.keys()):
+    # if "ndarray_ds" in list(hdf5.keys()):
+    decoded_item = list()
+    for it in hdf5["ndarray_dtype"][:]:
+        decoded_item.append(it.decode())
+    list_dtypes = decoded_item
 
-        decoded_item = list()
-        for it in hdf5["ndarray_ds/ndarray_dtype"][:]:
-            decoded_item.append(it.decode())
-        list_dtypes = decoded_item
+    decoded_item = list()
+    for it in hdf5["ndarray_indexes"][:]:
+        decoded_item.append(it.decode())
+    list_indexes = decoded_item
 
-        decoded_item = list()
-        for it in hdf5["ndarray_ds/ndarray_indexes"][:]:
-            decoded_item.append(it.decode())
-        list_indexes = decoded_item
+    len_data = len(hdf5[f"{list_indexes[0]}"][:])
 
-        len_data = len(hdf5[f"ndarray_ds/{list_indexes[0]}"][:])
+    list_datatype = list()
+    for i in range(len(list_indexes)):
+        list_datatype.append((list_indexes[i], list_dtypes[i]))
 
-        list_datatype = list()
-        for i in range(len(list_indexes)):
-            list_datatype.append((list_indexes[i], list_dtypes[i]))
+    datatype = np.dtype(list_datatype)
 
-        datatype = np.dtype(list_datatype)
+    ndarray = np.zeros(len_data, dtype=datatype)
 
-        ndarray = np.zeros(len_data, dtype=datatype)
+    for i in range(len(list_indexes)):
 
-        for i in range(len(list_indexes)):
+        expected_type = list_dtypes[i]
 
-            expected_type = list_dtypes[i]
+        values = hdf5_read_dataset(hdf5[f"{list_indexes[i]}"], expected_type)
 
-            values = hdf5_read_dataset(
-                hdf5[f"ndarray_ds/{list_indexes[i]}"], expected_type
-            )
+        ndarray[list_indexes[i]] = values
 
-            ndarray[list_indexes[i]] = values
-
-        return ndarray
+    return ndarray
 
 
 def save_dict_to_hdf5(hdf5, dictionary):
@@ -1019,14 +1030,13 @@ def read_hdf5_as_dict(hdf5, read_attrs=True, read_dataset_attrs=False):
 
         if str(type(item)).find("group") != -1:
 
-            if key == "ndarray_ds":
-
-                # dictionary.update({key: _read_ndarray_datastructure(hdf5)})
-                values = _read_ndarray_datastructure(hdf5)
+            if "_Pandas_DataFrame" in list(item.attrs.keys()):
+                values = _read_pd_dataframe(item)
                 dictionary.update({key: values})
 
-            elif key == "pd_DataFrame":
-                values = _read_pd_dataframe(hdf5)
+            elif "_numpy_ndarray" in list(item.attrs.keys()):
+                values = _read_ndarray_datastructure(item)
+                # values = _read_ndarray_datastructure(hdf5)
                 dictionary.update({key: values})
 
             else:
@@ -1446,13 +1456,13 @@ def get_hdf5_item(
 
         if str(type(hdf5_item)).find("group") != -1:
 
-            if item == "ndarray_ds":
-
-                return _read_ndarray_datastructure(hdf5)
+            # if item == "ndarray_ds":
+            if "_numpy_ndarray" in list(hdf5_item.attrs.keys()):
 
-            elif item == "pd_DataFrame":
+                return _read_ndarray_datastructure(hdf5_item)
 
-                return _read_pd_dataframe(hdf5)
+            elif "_Pandas_DataFrame" in list(hdf5_item.attrs.keys()):
+                return _read_pd_dataframe(hdf5_item)
 
             else:
 

{pyhdf5_handler-0.8 → pyhdf5_handler-0.10}/pyhdf5_handler/tutorial/hdf5_io_tests.py

@@ -74,6 +74,21 @@ if __name__ == "__main__":
             {"column1": np.array([1, 2, 3]), "column2": np.array([4, 5, 6])}
         ),
     )
+    pyhdf5_handler.hdf5_dataset_creator(
+        hdf5,
+        "mix_dtype_panda_dataframe",
+        pd.DataFrame(
+            {
+                "column1": np.array(["A", "B", "C"]),
+                "column2": np.array([4, 5, 6]),
+            }
+        ),
+    )
+    pyhdf5_handler.hdf5_dataset_creator(
+        hdf5,
+        "empty_dataframe",
+        pd.DataFrame({}),
+    )
 
     # write a python dictionary in the hdf5 database
     dictionary = {
@@ -86,6 +101,12 @@ if __name__ == "__main__":
             "array": np.array([1, 2, 3, 4]),
             "date_range": pd.date_range(start="1/1/2018", end="1/08/2018"),
             "list_mixte": [1.0, np.datetime64("2019-09-22 17:38:30")],
+            "pandas_df": pd.DataFrame(
+                {
+                    "column1": np.array([1, 2, 3]),
+                    "column2": np.array([4, 5, 6]),
+                }
+            ),
         }
     }
 
@@ -130,6 +151,10 @@ if __name__ == "__main__":
         hdf5_instance=hdf5,
         location="./panda_dataframe",
     )
+    pyhdf5_handler.get_hdf5_item(
+        hdf5_instance=hdf5,
+        location="./mix_dtype_panda_dataframe",
+    )
     pyhdf5_handler.get_hdf5_item(
         hdf5_instance=hdf5,
         location="./structured_array",
@@ -142,8 +167,34 @@ if __name__ == "__main__":
     pyhdf5_handler.hdf5file_ls("./test.hdf5")
     pyhdf5_handler.hdf5file_ls("./test.hdf5", location="structured_array")
 
-    data = pyhdf5_handler.read_hdf5file_as_dict(
-        "./test.hdf5", read_attrs=False
+    pyhdf5_handler.save_dict_to_hdf5file(
+        "./panda.hdf5",
+        {
+            "mypandas": pd.DataFrame(
+                {
+                    "column1": np.array(["A", "B", "C"]),
+                    "column2": np.array([4, 5, 6]),
+                }
+            )
+        },
+    )
+
+    pd_data = pyhdf5_handler.read_hdf5file_as_dict(
+        "./panda.hdf5", read_attrs=False
+    )
+
+    pyhdf5_handler.save_dict_to_hdf5file(
+        "./ndarray.hdf5",
+        {"myndarray": people},
+    )
+
+    ndarray = pyhdf5_handler.read_hdf5file_as_dict(
+        "./ndarray.hdf5", read_attrs=False
+    )
+
+    pyhdf5_handler.get_hdf5file_item(
+        path_to_hdf5="./test.hdf5",
+        location="./mix_dtype_panda_dataframe",
     )
 
     pyhdf5_handler.save_dict_to_hdf5file("./test.hdf5", data)
@@ -214,7 +265,7 @@ if __name__ == "__main__":
 
     pyhdf5_handler.get_hdf5file_attribute(
         path_to_hdf5="./test.hdf5",
-        location="./structured_array/ndarray_ds",
+        location="./structured_array/",
         attribute="_name",
         wait_time=0,
     )

{pyhdf5_handler-0.8 → pyhdf5_handler-0.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pyhdf5_handler"
-version = "0.8"
+version = "0.10"
 authors = [
   { name="Maxime Jay-Allemand", email="maxime.jay.allemand@hydris-hydrologie.fr" },
 ]