f9columnar 0.2.3__tar.gz → 0.2.4__tar.gz

{f9columnar-0.2.3 → f9columnar-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: f9columnar
-Version: 0.2.3
+Version: 0.2.4
 Summary: Columnar analysis utils.
 License: MIT
 Author: Jan Gavranovic
@@ -42,13 +42,13 @@ Description-Content-Type: text/markdown
 
 # F9 Columnar
 
-### What is this?
+A lightweight Python package for batch processing of ROOT and HDF5 event data.
 
-This is a lightweight Python package for processing event data stored in ROOT or HDF5 files in a batch-oriented manner. It is designed for datasets that are too large to fit into memory or too slow to process in a single thread. Built on top of PyTorch and Awkward Arrays, the package leverages PyTorch's DataLoader with an IterableDataset for efficient parallel processing. It implements a columnar event loop that returns batches of events, following the same structure as a standard PyTorch training loop over epochs.
+### Project description
 
-### Why?
+This package is designed for efficient handling of large datasets. Built on PyTorch, Awkward Array, and Uproot, it utilizes PyTorch's DataLoader with an IterableDataset to enable parallel processing. It implements a columnar event loop, returning batches of events in a format compatible with standard PyTorch training loops over multiple epochs.
 
-The primary goal of this package is to provide a simple and efficient solution for loading and processing large datasets, particularly for machine learning applications. It includes [`RootDataLoader`](f9columnar/root_dataloader.py) and [`Hdf5DataLoader`](f9columnar/hdf5_dataloader.py) classes for reading data from ROOT and HDF5 files. Additionally, it supports parallel data processing through processor classes, which can be chained together to perform complex calculations and histogramming.
+Optimized for machine learning applications, the package provides `RootDataLoader` and `Hdf5DataLoader` classes for data loading from ROOT and HDF5 files. Additionally, it supports parallel data processing through a modular pipeline of processor classes, allowing users to chain operations for complex computations and histogramming.
 
 ##  Setup
 

{f9columnar-0.2.3 → f9columnar-0.2.4}/README.md

@@ -1,12 +1,12 @@
 # F9 Columnar
 
-### What is this?
+A lightweight Python package for batch processing of ROOT and HDF5 event data.
 
-This is a lightweight Python package for processing event data stored in ROOT or HDF5 files in a batch-oriented manner. It is designed for datasets that are too large to fit into memory or too slow to process in a single thread. Built on top of PyTorch and Awkward Arrays, the package leverages PyTorch's DataLoader with an IterableDataset for efficient parallel processing. It implements a columnar event loop that returns batches of events, following the same structure as a standard PyTorch training loop over epochs.
+### Project description
 
-### Why?
+This package is designed for efficient handling of large datasets. Built on PyTorch, Awkward Array, and Uproot, it utilizes PyTorch's DataLoader with an IterableDataset to enable parallel processing. It implements a columnar event loop, returning batches of events in a format compatible with standard PyTorch training loops over multiple epochs.
 
-The primary goal of this package is to provide a simple and efficient solution for loading and processing large datasets, particularly for machine learning applications. It includes [`RootDataLoader`](f9columnar/root_dataloader.py) and [`Hdf5DataLoader`](f9columnar/hdf5_dataloader.py) classes for reading data from ROOT and HDF5 files. Additionally, it supports parallel data processing through processor classes, which can be chained together to perform complex calculations and histogramming.
+Optimized for machine learning applications, the package provides `RootDataLoader` and `Hdf5DataLoader` classes for data loading from ROOT and HDF5 files. Additionally, it supports parallel data processing through a modular pipeline of processor classes, allowing users to chain operations for complex computations and histogramming.
 
 ##  Setup
 

{f9columnar-0.2.3 → f9columnar-0.2.4}/f9columnar/hdf5_writer.py

@@ -443,9 +443,17 @@ class NtupleHdf5Writer(ArraysHdf5Writer):
         write_mc: bool = True,
         write_data: bool = True,
         name: str = "datasetHDF5Writer",
+        dataset_names: list[str] | None = None,
         **hdf5_kwargs: Any,
     ) -> None:
-        super().__init__(file_path, dataset_names=None, name=name)
+
+        if dataset_names is None:
+            dataset_names = ["mc", "data"]
+
+        if len(dataset_names) != 2:
+            raise ValueError("Dataset names must be a list of two strings!")
+
+        super().__init__(file_path, dataset_names=dataset_names, name=name)
         self.save_node = save_node
         self.chunk_shape = chunk_shape
 
@@ -463,25 +471,26 @@ class NtupleHdf5Writer(ArraysHdf5Writer):
             self.data_column_names = data_column_names
 
         metadata = {}
+        mc_group, data_group = self.dataset_names[0], self.dataset_names[1]
 
         if write_mc:
             self.create_datasets(
-                dataset_names=["mc"],
+                dataset_names=[mc_group],
                 shape=(self.chunk_shape, len(self.mc_column_names)),
                 maxshape=(None, len(self.mc_column_names)),
                 **hdf5_kwargs,
             )
-            metadata["mc_columns"] = self.mc_column_names
+            metadata[f"{mc_group}_columns"] = self.mc_column_names
 
         if write_data:
             self.create_datasets(
                 mode="a" if self.write_mc else "w",
-                dataset_names=["data"],
+                dataset_names=[data_group],
                 shape=(self.chunk_shape, len(self.data_column_names)),
                 maxshape=(None, len(self.data_column_names)),
                 **hdf5_kwargs,
             )
-            metadata["data_columns"] = self.data_column_names
+            metadata[f"{data_group}_columns"] = self.data_column_names
 
         self.add_metadata(metadata)
 
@@ -498,12 +507,12 @@ class NtupleHdf5Writer(ArraysHdf5Writer):
 
         if self.is_data and self.write_data:
             self._current_idx, self._current_shape = self._current_mc_idx, self._current_mc_shape
-            self.write_arrays(arrays, "data", self.data_column_names, self.chunk_shape)
+            self.write_arrays(arrays, self.dataset_names[1], self.data_column_names, self.chunk_shape)
             self._current_mc_idx, self._current_mc_shape = self._current_idx, self._current_shape
 
         if not self.is_data and self.write_mc:
             self._current_idx, self._current_shape = self._current_data_idx, self._current_data_shape
-            self.write_arrays(arrays, "mc", self.mc_column_names, self.chunk_shape)
+            self.write_arrays(arrays, self.dataset_names[0], self.mc_column_names, self.chunk_shape)
             self._current_data_idx, self._current_data_shape = self._current_idx, self._current_shape
 
         return {"processors": processors}
@@ -522,9 +531,19 @@ class NtupleHdf5PileWriter(ArraysHdf5PileWriter):
         write_mc: bool = True,
         write_data: bool = True,
         name: str = "datasetHDF5Writer",
+        dataset_names_map: dict[str, str] | None = None,
         **hdf5_kwargs: Any,
     ) -> None:
-        super().__init__(file_path, n_piles, pile_assignment, dataset_names=["mc", "data"], name=name)
+
+        if dataset_names_map is None:
+            dataset_names = ["mc", "data"]
+        else:
+            dataset_names = [dataset_names_map["mc"], dataset_names_map["data"]]
+
+        if len(dataset_names) != 2:
+            raise ValueError("Dataset names must be a list of two strings!")
+
+        super().__init__(file_path, n_piles, pile_assignment, dataset_names=dataset_names, name=name)
         self.save_node = save_node
         self.chunk_shape = chunk_shape
 
@@ -542,25 +561,26 @@ class NtupleHdf5PileWriter(ArraysHdf5PileWriter):
             self.data_column_names = data_column_names
 
         metadata: dict[str, Any] = {}
+        mc_group, data_group = self.dataset_names[0], self.dataset_names[1]
 
         if write_mc:
             self.create_datasets(
-                dataset_names=self.pile_datasets_dct["mc"],
+                dataset_names=self.pile_datasets_dct[mc_group],
                 shape=(self.chunk_shape, len(self.mc_column_names)),
                 maxshape=(None, len(self.mc_column_names)),
                 **hdf5_kwargs,
             )
-            metadata["mc_columns"] = self.mc_column_names
+            metadata[f"{mc_group}_columns"] = self.mc_column_names
 
         if write_data:
             self.create_datasets(
                 mode="a" if self.write_mc else "w",
-                dataset_names=self.pile_datasets_dct["data"],
+                dataset_names=self.pile_datasets_dct[data_group],
                 shape=(self.chunk_shape, len(self.data_column_names)),
                 maxshape=(None, len(self.data_column_names)),
                 **hdf5_kwargs,
             )
-            metadata["data_columns"] = self.data_column_names
+            metadata[f"{data_group}_columns"] = self.data_column_names
 
         metadata["piles"] = self.pile_datasets_dct
 
@@ -575,10 +595,10 @@ class NtupleHdf5PileWriter(ArraysHdf5PileWriter):
             raise AttributeError("Arrays attribute not found in the processor!")
 
         if self.is_data and self.write_data:
-            self.write_arrays(arrays, "data", self.data_column_names, self.chunk_shape)
+            self.write_arrays(arrays, self.dataset_names[1], self.data_column_names, self.chunk_shape)
 
         if not self.is_data and self.write_mc:
-            self.write_arrays(arrays, "mc", self.mc_column_names, self.chunk_shape)
+            self.write_arrays(arrays, self.dataset_names[0], self.mc_column_names, self.chunk_shape)
 
         return {"processors": processors}
 

{f9columnar-0.2.3 → f9columnar-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "f9columnar"
-version = "0.2.3"
+version = "0.2.4"
 description = "Columnar analysis utils."
 authors = ["Jan Gavranovic <jan.gavranovic@cern.ch>"]
 readme = "README.md"