f9columnar 0.3.2__tar.gz → 0.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {f9columnar-0.3.2 → f9columnar-0.3.4}/PKG-INFO +6 -5
  2. {f9columnar-0.3.2 → f9columnar-0.3.4}/README.md +2 -2
  3. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/arrays.py +37 -10
  4. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/hdf5_dataloader.py +0 -3
  5. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/dataloader_helpers.py +5 -2
  6. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/dataset_scaling.py +32 -24
  7. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/hdf5_dataloader.py +56 -20
  8. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/processors_collection.py +15 -8
  9. {f9columnar-0.3.2 → f9columnar-0.3.4}/pyproject.toml +2 -2
  10. {f9columnar-0.3.2 → f9columnar-0.3.4}/LICENSE +0 -0
  11. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/__init__.py +0 -0
  12. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/PMGxsecDB_mc16.txt +0 -0
  13. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/PMGxsecDB_mc21.txt +0 -0
  14. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/campaigns.json +0 -0
  15. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/luminosity.json +0 -0
  16. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/periods.json +0 -0
  17. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/truth_classification.json +0 -0
  18. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/dataset_builder.py +0 -0
  19. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/hdf5_writer.py +0 -0
  20. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/histograms.py +0 -0
  21. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/__init__.py +0 -0
  22. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/hdf5_parallel_writer.py +0 -0
  23. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/hdf5_writer.py +0 -0
  24. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/imbalanced_sampler.py +0 -0
  25. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/scalers.py +0 -0
  26. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/processors.py +0 -0
  27. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/root_dataloader.py +0 -0
  28. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/run.py +0 -0
  29. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/__init__.py +0 -0
  30. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_driver.py +0 -0
  31. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_handler.py +0 -0
  32. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_merger.py +0 -0
  33. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_run.py +0 -0
  34. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_run.sh +0 -0
  35. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/__init__.py +0 -0
  36. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/ak_helpers.py +0 -0
  37. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/config_utils.py +0 -0
  38. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/helpers.py +0 -0
  39. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/loggers.py +0 -0
  40. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/regex_helpers.py +0 -0
  41. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/rucio_db.py +0 -0
  42. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/rucio_utils.py +0 -0
  43. {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/xsec_db.py +0 -0
@@ -1,8 +1,9 @@
1
- Metadata-Version: 2.3
1
+ Metadata-Version: 2.4
2
2
  Name: f9columnar
3
- Version: 0.3.2
3
+ Version: 0.3.4
4
4
  Summary: Columnar analysis utils.
5
5
  License: MIT
6
+ License-File: LICENSE
6
7
  Author: Jan Gavranovic
7
8
  Author-email: jan.gavranovic@cern.ch
8
9
  Requires-Python: >=3.10,<3.13
@@ -33,7 +34,7 @@ Requires-Dist: scipy (>=1.14.1,<2.0.0)
33
34
  Requires-Dist: seaborn (>=0.13.2,<0.14.0)
34
35
  Requires-Dist: sympy (>=1.13.1,<2.0.0)
35
36
  Requires-Dist: tables (>=3.10.1,<4.0.0)
36
- Requires-Dist: torch (==2.7.0) ; extra == "torch"
37
+ Requires-Dist: torch (==2.8.0) ; extra == "torch"
37
38
  Requires-Dist: tqdm (>=4.65.0,<5.0.0)
38
39
  Requires-Dist: uncertainties (>=3.2.2,<4.0.0)
39
40
  Requires-Dist: uproot (>=5.3.10,<6.0.0)
@@ -47,7 +48,7 @@ Description-Content-Type: text/markdown
47
48
 
48
49
  [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
49
50
  [![python](https://img.shields.io/badge/-Python_3.12-3776AB?logo=python&logoColor=white)](https://www.python.org/)
50
- [![pytorch](https://img.shields.io/badge/-PyTorch_2.7-EE4C2C?logo=pytorch&logoColor=white)](https://pytorch.org/)
51
+ [![pytorch](https://img.shields.io/badge/-PyTorch_2.8-EE4C2C?logo=pytorch&logoColor=white)](https://pytorch.org/)
51
52
  [![License](https://img.shields.io/badge/license-MIT-green)](./LICENSE)
52
53
  [![pytorch](https://img.shields.io/pypi/v/f9columnar)](https://pypi.org/project/f9columnar/)
53
54
 
@@ -59,7 +60,7 @@ A lightweight Python package for batch processing of ROOT and HDF5 event data.
59
60
 
60
61
  This package is designed for efficient handling of large datasets. Built on PyTorch, Awkward Array, and Uproot, it utilizes PyTorch's DataLoader with an IterableDataset to enable parallel processing. It implements a columnar event loop, returning batches of events in a format compatible with standard PyTorch training loops over multiple epochs.
61
62
 
62
- Optimized for machine learning applications, the package provides `RootDataLoader` and `Hdf5DataLoader` classes for data loading from ROOT and HDF5 files. Additionally, it supports parallel data processing through a modular pipeline of processor classes, allowing users to chain operations for complex computations and histogramming.
63
+ It is optimized for machine learning applications, the package provides `RootDataLoader` and `Hdf5DataLoader` classes for data loading from ROOT and HDF5 files. Additionally, it supports parallel data processing through a modular pipeline of processor classes, allowing users to chain operations for complex computations and histogramming.
63
64
 
64
65
  ## Setup
65
66
 
@@ -4,7 +4,7 @@
4
4
 
5
5
  [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
6
6
  [![python](https://img.shields.io/badge/-Python_3.12-3776AB?logo=python&logoColor=white)](https://www.python.org/)
7
- [![pytorch](https://img.shields.io/badge/-PyTorch_2.7-EE4C2C?logo=pytorch&logoColor=white)](https://pytorch.org/)
7
+ [![pytorch](https://img.shields.io/badge/-PyTorch_2.8-EE4C2C?logo=pytorch&logoColor=white)](https://pytorch.org/)
8
8
  [![License](https://img.shields.io/badge/license-MIT-green)](./LICENSE)
9
9
  [![pytorch](https://img.shields.io/pypi/v/f9columnar)](https://pypi.org/project/f9columnar/)
10
10
 
@@ -16,7 +16,7 @@ A lightweight Python package for batch processing of ROOT and HDF5 event data.
16
16
 
17
17
  This package is designed for efficient handling of large datasets. Built on PyTorch, Awkward Array, and Uproot, it utilizes PyTorch's DataLoader with an IterableDataset to enable parallel processing. It implements a columnar event loop, returning batches of events in a format compatible with standard PyTorch training loops over multiple epochs.
18
18
 
19
- Optimized for machine learning applications, the package provides `RootDataLoader` and `Hdf5DataLoader` classes for data loading from ROOT and HDF5 files. Additionally, it supports parallel data processing through a modular pipeline of processor classes, allowing users to chain operations for complex computations and histogramming.
19
+ It is optimized for machine learning applications, the package provides `RootDataLoader` and `Hdf5DataLoader` classes for data loading from ROOT and HDF5 files. Additionally, it supports parallel data processing through a modular pipeline of processor classes, allowing users to chain operations for complex computations and histogramming.
20
20
 
21
21
  ## Setup
22
22
 
@@ -4,7 +4,7 @@ import copy
4
4
  import logging
5
5
  from dataclasses import dataclass
6
6
  from dataclasses import field as dataclass_field
7
- from typing import Any
7
+ from typing import Any, Self
8
8
 
9
9
  import awkward as ak
10
10
  import numpy as np
@@ -25,12 +25,12 @@ class BaseArrays:
25
25
  def shape(self) -> tuple[int, int]:
26
26
  return (len(self.array), len(self.fields))
27
27
 
28
- def mask(self, mask: ak.Array | np.ndarray, inplace: bool = True) -> ak.Array | BaseArrays:
28
+ def mask(self, mask: ak.Array | np.ndarray, inplace: bool = True) -> Self:
29
29
  if inplace:
30
30
  self.array = self.array[mask]
31
31
  return self
32
32
  else:
33
- return self.array[mask]
33
+ return self.__class__(self.array[mask])
34
34
 
35
35
  def _check_type(self, value: Any) -> bool:
36
36
  raise NotImplementedError
@@ -71,9 +71,9 @@ class BaseArrays:
71
71
  return self
72
72
 
73
73
  def __len__(self) -> int:
74
- if type(self.array) is ak.Array:
74
+ if isinstance(self.array, ak.Array):
75
75
  return len(self.array)
76
- elif type(self.array) is ak.Record:
76
+ elif isinstance(self.array, ak.Record):
77
77
  return 1
78
78
  else:
79
79
  raise ValueError("Invalid array type!")
@@ -169,9 +169,7 @@ class Arrays:
169
169
  def shape(self) -> tuple[int, int, int]:
170
170
  return (len(self), len(self.flat_fields), len(self.jagged_fields))
171
171
 
172
- def remove_empty(
173
- self, return_mask: bool = False, all_fields: bool = True
174
- ) -> Arrays | ak.Array | tuple[ak.Array | None, ak.Array] | None:
172
+ def remove_empty(self, return_mask: bool = False, all_fields: bool = True) -> Arrays | ak.Array | None:
175
173
  if self.jagged_arrays is None:
176
174
  if return_mask:
177
175
  return None
@@ -185,7 +183,7 @@ class Arrays:
185
183
  else:
186
184
  return self.mask_flat_inplace(non_empty_mask)
187
185
 
188
- def mask_flat(self, mask: ak.Array | np.ndarray) -> tuple[ak.Array | None, ak.Array | None]:
186
+ def mask_flat(self, mask: ak.Array | np.ndarray) -> tuple[FlatArrays | None, JaggedArrays | None]:
189
187
  if self.flat_arrays is not None:
190
188
  flat_masked = self.flat_arrays.mask(mask, inplace=False)
191
189
  else:
@@ -207,7 +205,7 @@ class Arrays:
207
205
 
208
206
  return self
209
207
 
210
- def mask_jagged(self, mask: ak.Array | np.ndarray) -> ak.Array | None:
208
+ def mask_jagged(self, mask: ak.Array | np.ndarray) -> JaggedArrays | None:
211
209
  if self.jagged_arrays is not None:
212
210
  jagged_masked = self.jagged_arrays.mask(mask, inplace=False)
213
211
  else:
@@ -661,3 +659,32 @@ class ArraysProcessor(Processor):
661
659
  arrays = self.arrays_handler.make_array_groups(arrays, self.groups)
662
660
 
663
661
  return {"arrays": arrays}
662
+
663
+
664
+ def arrays_to_ak(arrays: Arrays) -> ak.Array:
665
+ fields: dict[str, ak.Array] = {}
666
+
667
+ if arrays.flat_arrays is not None:
668
+ for field in arrays.flat_arrays.fields:
669
+ fields[field] = arrays.flat_arrays.array[field]
670
+
671
+ if arrays.jagged_arrays is not None:
672
+ for field in arrays.jagged_arrays.fields:
673
+ fields[field] = arrays.jagged_arrays.array[field]
674
+
675
+ return ak.zip(fields, depth_limit=1)
676
+
677
+
678
+ def group_arrays_to_ak(group_arrays: GroupArrays) -> ak.Array:
679
+ group_fields: dict[str, ak.Array] = {}
680
+
681
+ for _, arrays in group_arrays.group_arrays.items():
682
+ if arrays.flat_arrays is not None:
683
+ for field in arrays.flat_arrays.fields:
684
+ group_fields[field] = arrays.flat_arrays.array[field]
685
+
686
+ if arrays.jagged_arrays is not None:
687
+ for field in arrays.jagged_arrays.fields:
688
+ group_fields[field] = arrays.jagged_arrays.array[field]
689
+
690
+ return ak.zip(group_fields, depth_limit=1)
@@ -141,9 +141,6 @@ class Hdf5Iterator:
141
141
  else:
142
142
  arrays = dataset[self._current_start_entry : self._current_stop_entry]
143
143
 
144
- if self.shuffle:
145
- np.random.shuffle(arrays)
146
-
147
144
  return arrays
148
145
 
149
146
  def __next__(self) -> tuple[Any, dict[str, Any]]:
@@ -87,7 +87,10 @@ def get_hdf5_metadata(hdf5_file_path: str | list[str], resolve_path: bool = Fals
87
87
  hdf5_file_path = resolve_hdf5_path(hdf5_file_path)
88
88
 
89
89
  with h5py.File(hdf5_file_path, "r") as f:
90
- metadata = json.loads(f["metadata"][()])
90
+ try:
91
+ metadata = json.loads(f["metadata"][()])
92
+ except KeyError:
93
+ return {}
91
94
 
92
95
  return metadata
93
96
 
@@ -377,7 +380,7 @@ def get_column_selection(hdf5_files: str | list[str], column_names: list[str]) -
377
380
  selection = ColumnSelection()
378
381
 
379
382
  for dataset_name in used_columns_dct.keys():
380
- if dataset_name in pad_values:
383
+ if pad_values is not None and dataset_name in pad_values:
381
384
  pad_value = float(pad_values[dataset_name])
382
385
  else:
383
386
  pad_value = None
@@ -22,6 +22,7 @@ class DatasetScaler:
22
22
  scaler_save_path: str | None = None,
23
23
  n_max: int | float | None = None,
24
24
  extra_hash: str | None = None,
25
+ reuse_scalers: bool = True,
25
26
  scaler_kwargs: dict[str, Any] | None = None,
26
27
  dataset_kwargs: dict[str, Any] | None = None,
27
28
  dataloader_kwargs: dict[str, Any] | None = None,
@@ -41,10 +42,10 @@ class DatasetScaler:
41
42
  by default None.
42
43
  n_max : int | float | None, optional
43
44
  Maximum number of events to use for feature scaling. If None, all events will be used, by default None.
44
- perform_closure : bool, optional
45
- Whether to perform feature scaling closure. If True, the `closure_config` must be provided, by default False.
46
- closure_config : dict[str, dict[str, Any]] | None, optional
47
- Configuration for the feature scaling closure. Needed if `perform_closure` is True.
45
+ extra_hash : str | None, optional
46
+ Extra hash to append to the scaler filename, by default None.
47
+ reuse_scalers : bool, optional
48
+ Whether to reuse the same scaler for all objects in a dataset, by default True.
48
49
  scaler_kwargs : dict[str, Any] | None, optional
49
50
  Additional keyword arguments to pass to the sklearn feature scaler, by default None.
50
51
  dataset_kwargs : dict[str, Any] | None, optional
@@ -76,7 +77,7 @@ class DatasetScaler:
76
77
  dataloader_kwargs,
77
78
  )
78
79
 
79
- self.dataset_scalers = self._get_scalers(scaler_type, scaler_save_path, n_max, scaler_kwargs)
80
+ self.dataset_scalers = self._get_scalers(scaler_type, scaler_save_path, n_max, scaler_kwargs, reuse_scalers)
80
81
  logging.info(f"Scaling datasets: {list(self.dataset_scalers.keys())}")
81
82
 
82
83
  def _get_dataloader(
@@ -104,6 +105,7 @@ class DatasetScaler:
104
105
  scaler_save_path: str,
105
106
  n_max: int | float | None = None,
106
107
  scaler_kwargs: dict[str, Any] | None = None,
108
+ reuse_scalers: bool = False,
107
109
  ) -> dict[str, dict[str, list[FeatureScaler | None]]]:
108
110
  if scaler_kwargs is None:
109
111
  scaler_kwargs = {}
@@ -116,21 +118,22 @@ class DatasetScaler:
116
118
  numer_scalers: list[FeatureScaler | None] = []
117
119
  categ_scalers: list[FeatureScaler | None] = []
118
120
 
119
- for _ in range(n_objects):
120
- if len(column_selection.numer_columns) != 0:
121
- numer_feature_scaler = NumericalFeatureScaler(
122
- scaler_type,
123
- n_max=n_max,
124
- save_path=scaler_save_path,
125
- **scaler_kwargs,
126
- )
127
- else:
128
- numer_feature_scaler = None
129
-
130
- if len(column_selection.categ_columns) != 0:
131
- categ_feature_scaler = CategoricalFeatureScaler("categorical", scaler_save_path)
132
- else:
133
- categ_feature_scaler = None
121
+ for n_obj in range(n_objects):
122
+ if (reuse_scalers and n_obj == 0) or (not reuse_scalers):
123
+ if len(column_selection.numer_columns) != 0:
124
+ numer_feature_scaler = NumericalFeatureScaler(
125
+ scaler_type,
126
+ n_max=n_max,
127
+ save_path=scaler_save_path,
128
+ **scaler_kwargs,
129
+ )
130
+ else:
131
+ numer_feature_scaler = None
132
+
133
+ if len(column_selection.categ_columns) != 0:
134
+ categ_feature_scaler = CategoricalFeatureScaler("categorical", scaler_save_path)
135
+ else:
136
+ categ_feature_scaler = None
134
137
 
135
138
  numer_scalers.append(numer_feature_scaler)
136
139
  categ_scalers.append(categ_feature_scaler)
@@ -145,19 +148,21 @@ class DatasetScaler:
145
148
  def _get_padding_mask(self, X: np.ndarray, dataset_name: str) -> np.ndarray | None:
146
149
  return padding_mask(X, self.selection[dataset_name].pad_value)
147
150
 
148
- def _scale_dataset_idx(self, ds_batch: WeightedDatasetBatch, dataset_name: str, idx: int) -> bool:
151
+ def _scale_dataset_idx(
152
+ self, ds_batch: WeightedDatasetBatch, dataset_name: str, idx: int, scale_categ_padding: bool = True
153
+ ) -> bool:
149
154
  X = ds_batch[dataset_name].X.numpy()
150
155
 
151
156
  if dataset_name != "events":
152
157
  X = X[:, idx, :]
153
158
  padding_mask = self._get_padding_mask(X, dataset_name)
154
-
155
- if padding_mask is not None:
156
- X = X[padding_mask]
157
159
  else:
158
160
  padding_mask = None
159
161
 
160
162
  if self.selection[dataset_name].has_categ_columns:
163
+ if padding_mask is not None and not scale_categ_padding:
164
+ X = X[padding_mask]
165
+
161
166
  categ_feature_scaler = self.dataset_scalers[dataset_name]["categ"][idx]
162
167
 
163
168
  if categ_feature_scaler is None:
@@ -169,6 +174,9 @@ class DatasetScaler:
169
174
  categ_feature_scaler.fit(X_categ)
170
175
 
171
176
  if self.selection[dataset_name].has_numer_columns:
177
+ if padding_mask is not None:
178
+ X = X[padding_mask]
179
+
172
180
  numer_feature_scaler = self.dataset_scalers[dataset_name]["numer"][idx]
173
181
 
174
182
  if numer_feature_scaler is None:
@@ -28,6 +28,10 @@ WeightedBatchType = tuple[
28
28
  torch.Tensor,
29
29
  dict[str, Any],
30
30
  ]
31
+ FullWeightedBatchType = tuple[
32
+ dict[str, tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]],
33
+ dict[str, Any],
34
+ ]
31
35
 
32
36
 
33
37
  @dataclass(frozen=True, slots=True, repr=False, eq=False)
@@ -165,6 +169,16 @@ class WeightedBatch:
165
169
  if isinstance(self._y_aux, np.ndarray):
166
170
  self._y_aux = torch.from_numpy(self._y_aux)
167
171
 
172
+ n_samples = self._X.shape[0]
173
+ if self._y is not None and self._y.shape[0] != n_samples:
174
+ raise RuntimeError("X and y must have the same first dimension!")
175
+
176
+ if self._w is not None and self._w.shape[0] != n_samples:
177
+ raise RuntimeError("X and w must have the same first dimension!")
178
+
179
+ if self._y_aux is not None and self._y_aux.shape[0] != n_samples:
180
+ raise RuntimeError("X and y_aux must have the same first dimension!")
181
+
168
182
  @property
169
183
  def X(self) -> torch.Tensor:
170
184
  return self._X # type: ignore
@@ -185,8 +199,7 @@ class WeightedBatch:
185
199
  def feature_shape(self) -> tuple[int, ...]:
186
200
  return tuple(self._X.shape)
187
201
 
188
- @property
189
- def batch(self) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
202
+ def make_batch(self) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
190
203
  return self._X, self._y, self._w, self._y_aux # type: ignore
191
204
 
192
205
  def mask(self, mask: np.ndarray | torch.Tensor) -> WeightedBatch:
@@ -283,6 +296,14 @@ class WeightedDatasetBatch:
283
296
 
284
297
  return self.__class__(_masked_datasets)
285
298
 
299
+ def make_batch(
300
+ self,
301
+ ) -> dict[str, tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]]:
302
+ batch = {}
303
+ for key, ds_batch in self._datasets.items():
304
+ batch[key] = ds_batch.make_batch()
305
+ return batch
306
+
286
307
  def __repr__(self) -> str:
287
308
  return f"WeightedDatasetBatch({self.keys()})"
288
309
 
@@ -393,7 +414,7 @@ class MLHdf5Iterator(Hdf5Iterator):
393
414
  postfix: str | None = None,
394
415
  extra_hash: str = "",
395
416
  ) -> tuple[NumericalFeatureScaler | None, CategoricalFeatureScaler | None]:
396
- if scaler_path is None and scaler_type is None:
417
+ if scaler_path is None or scaler_type is None:
397
418
  return None, None
398
419
 
399
420
  numer_feature_scaler: NumericalFeatureScaler | None
@@ -422,7 +443,7 @@ class MLHdf5Iterator(Hdf5Iterator):
422
443
 
423
444
  return numer_feature_scaler, categ_feature_scaler
424
445
 
425
- def _scale(self, X: np.ndarray, dataset_name: str, is_numer: bool) -> np.ndarray:
446
+ def _scale(self, X: np.ndarray, dataset_name: str, is_numer: bool, scale_categ_padding: bool = True) -> np.ndarray:
426
447
  scalers: list[NumericalFeatureScaler | None] | list[CategoricalFeatureScaler | None]
427
448
 
428
449
  if is_numer:
@@ -432,6 +453,9 @@ class MLHdf5Iterator(Hdf5Iterator):
432
453
 
433
454
  n_objects = self.selection[dataset_name].n_objects
434
455
 
456
+ if len(scalers) != n_objects:
457
+ raise RuntimeError(f"Scaler list length mismatch for dataset {dataset_name}.")
458
+
435
459
  if n_objects == 1:
436
460
  scaler = scalers[0]
437
461
  if scaler is not None:
@@ -443,7 +467,9 @@ class MLHdf5Iterator(Hdf5Iterator):
443
467
  X_i = X[:, i, :]
444
468
  pad_value = self.selection[dataset_name].pad_value
445
469
 
446
- if pad_value is not None:
470
+ if pad_value is not None and scale_categ_padding and not is_numer:
471
+ X_objects.append(scaler.transform(X_i))
472
+ elif pad_value is not None:
447
473
  mask = padding_mask(X_i, pad_value)
448
474
 
449
475
  X_full = np.full_like(X_i, pad_value)
@@ -504,16 +530,16 @@ class MLHdf5Iterator(Hdf5Iterator):
504
530
 
505
531
  if X_numer_dataset_stack is not None and X_categ_dataset_stack is not None:
506
532
  X_dataset_stack = np.concatenate([X_numer_dataset_stack, X_categ_dataset_stack], axis=-1)
507
- numer_idx = np.arange(numer_shape[1])
508
- categ_idx = np.arange(numer_shape[1], numer_shape[1] + categ_shape[1])
533
+ numer_idx = np.arange(numer_shape[-1])
534
+ categ_idx = np.arange(numer_shape[-1], numer_shape[-1] + categ_shape[-1])
509
535
 
510
536
  elif X_numer_dataset_stack is not None:
511
537
  X_dataset_stack = X_numer_dataset_stack
512
- categ_idx, numer_idx = None, np.arange(X_numer_dataset_stack.shape[1])
538
+ categ_idx, numer_idx = None, np.arange(numer_shape[-1])
513
539
 
514
540
  elif X_categ_dataset_stack is not None:
515
541
  X_dataset_stack = X_categ_dataset_stack
516
- categ_idx, numer_idx = np.arange(X_categ_dataset_stack.shape[1]), None
542
+ categ_idx, numer_idx = np.arange(categ_shape[-1]), None
517
543
 
518
544
  else:
519
545
  raise RuntimeError("Both X_numer and X_categ are None!")
@@ -629,10 +655,24 @@ def events_collate_fn(batch: Any) -> WeightedBatchType:
629
655
  return ds.X, ds.y, ds.w, ds.y_aux, reports
630
656
 
631
657
 
632
- def default_setup_func(stacked_datasets: StackedDatasets, ml_iterator: MLHdf5Iterator) -> WeightedDatasetBatch:
658
+ def full_collate_fn(batch: Any) -> Any:
659
+ datasets_batch, reports = batch[0].make_batch(), batch[1]
660
+ return datasets_batch, reports
661
+
662
+
663
+ def default_setup_func(stacked_datasets: StackedDatasets, ml_iterator: MLHdf5Iterator) -> WeightedDatasetBatch | None:
633
664
  weighted_datase_batch = WeightedDatasetBatch()
634
665
 
635
- for ds_name, ds in stacked_datasets.items():
666
+ dataset_keys = stacked_datasets.keys()
667
+ if "events" in dataset_keys:
668
+ dataset_keys.remove("events")
669
+ dataset_keys.insert(0, "events")
670
+
671
+ mask_unmapped: np.ndarray | None = None
672
+
673
+ for ds_name in dataset_keys:
674
+ ds = stacked_datasets[ds_name]
675
+
636
676
  X = ds.X.astype(np.float32)
637
677
  y = ds.get_extra("label_type", None)
638
678
  w = ds.get_extra("weights", None)
@@ -643,18 +683,14 @@ def default_setup_func(stacked_datasets: StackedDatasets, ml_iterator: MLHdf5Ite
643
683
  if remap_labels is not None:
644
684
  max_label = ml_iterator.dataset_kwargs["max_label"]
645
685
  y, mask_unmapped = remap_labels_lookup(y, max_label, remap_labels)
646
- X = X[mask_unmapped]
647
- if w is not None:
648
- w = w[mask_unmapped]
649
686
 
650
- if ml_iterator.shuffle:
651
- shuffle_idx = np.random.permutation(len(X))
652
- X[:] = X[shuffle_idx]
687
+ if y.shape[0] == 0:
688
+ return None
653
689
 
654
- if y is not None:
655
- y[:] = y[shuffle_idx]
690
+ if mask_unmapped is not None:
691
+ X = X[mask_unmapped]
656
692
  if w is not None:
657
- w[:] = w[shuffle_idx]
693
+ w = w[mask_unmapped]
658
694
 
659
695
  weighted_datase_batch[ds_name] = WeightedBatch(X, y, w, y_aux)
660
696
 
@@ -28,19 +28,26 @@ class ProcessorsCollection:
28
28
  def __getitem__(self, name: str) -> Processor:
29
29
  return self.objects[name]
30
30
 
31
+ def _add_processor(self, processor: Processor) -> None:
32
+ # Check if processor being added is unique
33
+ if processor in self.objects.values():
34
+ raise ValueError(f"{processor.name} already exists in the {self.collection_name} collection!")
35
+
36
+ # Check if name of unique processor being added is unique
37
+ if processor.name in self.objects:
38
+ raise ValueError(
39
+ f'Processor with name "{processor.name}" already exists in the {self.collection_name} collection! Choose a unique name.'
40
+ )
41
+
42
+ self.objects[processor.name] = processor
43
+
31
44
  def add(self, *objs: Processor | ProcessorsCollection) -> ProcessorsCollection:
32
45
  for obj in objs:
33
46
  if isinstance(obj, ProcessorsCollection):
34
47
  for v in obj.objects.values():
35
- if v in self.objects:
36
- raise ValueError(f"{v.name} already exists in the {self.collection_name} collection!")
37
- else:
38
- self.objects[v.name] = v
48
+ self._add_processor(v)
39
49
  elif isinstance(obj, Processor):
40
- if obj in self.objects:
41
- raise ValueError(f"{obj.name} already exists in the {self.collection_name} collection!")
42
- else:
43
- self.objects[obj.name] = obj
50
+ self._add_processor(obj)
44
51
  else:
45
52
  raise ValueError(f"Invalid object type {type(obj)}!")
46
53
 
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "f9columnar"
3
- version = "0.3.2"
3
+ version = "0.3.4"
4
4
  description = "Columnar analysis utils."
5
5
  authors = ["Jan Gavranovic <jan.gavranovic@cern.ch>"]
6
6
  readme = "README.md"
@@ -9,7 +9,7 @@ license = "MIT"
9
9
 
10
10
  [tool.poetry.dependencies]
11
11
  python = ">=3.10,<3.13"
12
- torch = { version = "2.7.0", optional = true }
12
+ torch = { version = "2.8.0", optional = true }
13
13
  numpy = "^2.0.2"
14
14
  pandas = "^2.2.2"
15
15
  scipy = "^1.14.1"
File without changes
File without changes