f9columnar 0.3.2__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {f9columnar-0.3.2 → f9columnar-0.3.4}/PKG-INFO +6 -5
- {f9columnar-0.3.2 → f9columnar-0.3.4}/README.md +2 -2
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/arrays.py +37 -10
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/hdf5_dataloader.py +0 -3
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/dataloader_helpers.py +5 -2
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/dataset_scaling.py +32 -24
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/hdf5_dataloader.py +56 -20
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/processors_collection.py +15 -8
- {f9columnar-0.3.2 → f9columnar-0.3.4}/pyproject.toml +2 -2
- {f9columnar-0.3.2 → f9columnar-0.3.4}/LICENSE +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/__init__.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/PMGxsecDB_mc16.txt +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/PMGxsecDB_mc21.txt +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/campaigns.json +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/luminosity.json +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/periods.json +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/data/truth_classification.json +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/dataset_builder.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/hdf5_writer.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/histograms.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/__init__.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/hdf5_parallel_writer.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/hdf5_writer.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/imbalanced_sampler.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/ml/scalers.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/processors.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/root_dataloader.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/run.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/__init__.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_driver.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_handler.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_merger.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_run.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/submit/act_run.sh +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/__init__.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/ak_helpers.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/config_utils.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/helpers.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/loggers.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/regex_helpers.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/rucio_db.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/rucio_utils.py +0 -0
- {f9columnar-0.3.2 → f9columnar-0.3.4}/f9columnar/utils/xsec_db.py +0 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: f9columnar
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary: Columnar analysis utils.
|
|
5
5
|
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
6
7
|
Author: Jan Gavranovic
|
|
7
8
|
Author-email: jan.gavranovic@cern.ch
|
|
8
9
|
Requires-Python: >=3.10,<3.13
|
|
@@ -33,7 +34,7 @@ Requires-Dist: scipy (>=1.14.1,<2.0.0)
|
|
|
33
34
|
Requires-Dist: seaborn (>=0.13.2,<0.14.0)
|
|
34
35
|
Requires-Dist: sympy (>=1.13.1,<2.0.0)
|
|
35
36
|
Requires-Dist: tables (>=3.10.1,<4.0.0)
|
|
36
|
-
Requires-Dist: torch (==2.
|
|
37
|
+
Requires-Dist: torch (==2.8.0) ; extra == "torch"
|
|
37
38
|
Requires-Dist: tqdm (>=4.65.0,<5.0.0)
|
|
38
39
|
Requires-Dist: uncertainties (>=3.2.2,<4.0.0)
|
|
39
40
|
Requires-Dist: uproot (>=5.3.10,<6.0.0)
|
|
@@ -47,7 +48,7 @@ Description-Content-Type: text/markdown
|
|
|
47
48
|
|
|
48
49
|
[](https://github.com/astral-sh/ruff)
|
|
49
50
|
[](https://www.python.org/)
|
|
50
|
-
[](https://pytorch.org/)
|
|
51
52
|
[](./LICENSE)
|
|
52
53
|
[](https://pypi.org/project/f9columnar/)
|
|
53
54
|
|
|
@@ -59,7 +60,7 @@ A lightweight Python package for batch processing of ROOT and HDF5 event data.
|
|
|
59
60
|
|
|
60
61
|
This package is designed for efficient handling of large datasets. Built on PyTorch, Awkward Array, and Uproot, it utilizes PyTorch's DataLoader with an IterableDataset to enable parallel processing. It implements a columnar event loop, returning batches of events in a format compatible with standard PyTorch training loops over multiple epochs.
|
|
61
62
|
|
|
62
|
-
|
|
63
|
+
It is optimized for machine learning applications, the package provides `RootDataLoader` and `Hdf5DataLoader` classes for data loading from ROOT and HDF5 files. Additionally, it supports parallel data processing through a modular pipeline of processor classes, allowing users to chain operations for complex computations and histogramming.
|
|
63
64
|
|
|
64
65
|
## Setup
|
|
65
66
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
[](https://github.com/astral-sh/ruff)
|
|
6
6
|
[](https://www.python.org/)
|
|
7
|
-
[](https://pytorch.org/)
|
|
8
8
|
[](./LICENSE)
|
|
9
9
|
[](https://pypi.org/project/f9columnar/)
|
|
10
10
|
|
|
@@ -16,7 +16,7 @@ A lightweight Python package for batch processing of ROOT and HDF5 event data.
|
|
|
16
16
|
|
|
17
17
|
This package is designed for efficient handling of large datasets. Built on PyTorch, Awkward Array, and Uproot, it utilizes PyTorch's DataLoader with an IterableDataset to enable parallel processing. It implements a columnar event loop, returning batches of events in a format compatible with standard PyTorch training loops over multiple epochs.
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
It is optimized for machine learning applications, the package provides `RootDataLoader` and `Hdf5DataLoader` classes for data loading from ROOT and HDF5 files. Additionally, it supports parallel data processing through a modular pipeline of processor classes, allowing users to chain operations for complex computations and histogramming.
|
|
20
20
|
|
|
21
21
|
## Setup
|
|
22
22
|
|
|
@@ -4,7 +4,7 @@ import copy
|
|
|
4
4
|
import logging
|
|
5
5
|
from dataclasses import dataclass
|
|
6
6
|
from dataclasses import field as dataclass_field
|
|
7
|
-
from typing import Any
|
|
7
|
+
from typing import Any, Self
|
|
8
8
|
|
|
9
9
|
import awkward as ak
|
|
10
10
|
import numpy as np
|
|
@@ -25,12 +25,12 @@ class BaseArrays:
|
|
|
25
25
|
def shape(self) -> tuple[int, int]:
|
|
26
26
|
return (len(self.array), len(self.fields))
|
|
27
27
|
|
|
28
|
-
def mask(self, mask: ak.Array | np.ndarray, inplace: bool = True) ->
|
|
28
|
+
def mask(self, mask: ak.Array | np.ndarray, inplace: bool = True) -> Self:
|
|
29
29
|
if inplace:
|
|
30
30
|
self.array = self.array[mask]
|
|
31
31
|
return self
|
|
32
32
|
else:
|
|
33
|
-
return self.array[mask]
|
|
33
|
+
return self.__class__(self.array[mask])
|
|
34
34
|
|
|
35
35
|
def _check_type(self, value: Any) -> bool:
|
|
36
36
|
raise NotImplementedError
|
|
@@ -71,9 +71,9 @@ class BaseArrays:
|
|
|
71
71
|
return self
|
|
72
72
|
|
|
73
73
|
def __len__(self) -> int:
|
|
74
|
-
if
|
|
74
|
+
if isinstance(self.array, ak.Array):
|
|
75
75
|
return len(self.array)
|
|
76
|
-
elif
|
|
76
|
+
elif isinstance(self.array, ak.Record):
|
|
77
77
|
return 1
|
|
78
78
|
else:
|
|
79
79
|
raise ValueError("Invalid array type!")
|
|
@@ -169,9 +169,7 @@ class Arrays:
|
|
|
169
169
|
def shape(self) -> tuple[int, int, int]:
|
|
170
170
|
return (len(self), len(self.flat_fields), len(self.jagged_fields))
|
|
171
171
|
|
|
172
|
-
def remove_empty(
|
|
173
|
-
self, return_mask: bool = False, all_fields: bool = True
|
|
174
|
-
) -> Arrays | ak.Array | tuple[ak.Array | None, ak.Array] | None:
|
|
172
|
+
def remove_empty(self, return_mask: bool = False, all_fields: bool = True) -> Arrays | ak.Array | None:
|
|
175
173
|
if self.jagged_arrays is None:
|
|
176
174
|
if return_mask:
|
|
177
175
|
return None
|
|
@@ -185,7 +183,7 @@ class Arrays:
|
|
|
185
183
|
else:
|
|
186
184
|
return self.mask_flat_inplace(non_empty_mask)
|
|
187
185
|
|
|
188
|
-
def mask_flat(self, mask: ak.Array | np.ndarray) -> tuple[
|
|
186
|
+
def mask_flat(self, mask: ak.Array | np.ndarray) -> tuple[FlatArrays | None, JaggedArrays | None]:
|
|
189
187
|
if self.flat_arrays is not None:
|
|
190
188
|
flat_masked = self.flat_arrays.mask(mask, inplace=False)
|
|
191
189
|
else:
|
|
@@ -207,7 +205,7 @@ class Arrays:
|
|
|
207
205
|
|
|
208
206
|
return self
|
|
209
207
|
|
|
210
|
-
def mask_jagged(self, mask: ak.Array | np.ndarray) ->
|
|
208
|
+
def mask_jagged(self, mask: ak.Array | np.ndarray) -> JaggedArrays | None:
|
|
211
209
|
if self.jagged_arrays is not None:
|
|
212
210
|
jagged_masked = self.jagged_arrays.mask(mask, inplace=False)
|
|
213
211
|
else:
|
|
@@ -661,3 +659,32 @@ class ArraysProcessor(Processor):
|
|
|
661
659
|
arrays = self.arrays_handler.make_array_groups(arrays, self.groups)
|
|
662
660
|
|
|
663
661
|
return {"arrays": arrays}
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def arrays_to_ak(arrays: Arrays) -> ak.Array:
|
|
665
|
+
fields: dict[str, ak.Array] = {}
|
|
666
|
+
|
|
667
|
+
if arrays.flat_arrays is not None:
|
|
668
|
+
for field in arrays.flat_arrays.fields:
|
|
669
|
+
fields[field] = arrays.flat_arrays.array[field]
|
|
670
|
+
|
|
671
|
+
if arrays.jagged_arrays is not None:
|
|
672
|
+
for field in arrays.jagged_arrays.fields:
|
|
673
|
+
fields[field] = arrays.jagged_arrays.array[field]
|
|
674
|
+
|
|
675
|
+
return ak.zip(fields, depth_limit=1)
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def group_arrays_to_ak(group_arrays: GroupArrays) -> ak.Array:
|
|
679
|
+
group_fields: dict[str, ak.Array] = {}
|
|
680
|
+
|
|
681
|
+
for _, arrays in group_arrays.group_arrays.items():
|
|
682
|
+
if arrays.flat_arrays is not None:
|
|
683
|
+
for field in arrays.flat_arrays.fields:
|
|
684
|
+
group_fields[field] = arrays.flat_arrays.array[field]
|
|
685
|
+
|
|
686
|
+
if arrays.jagged_arrays is not None:
|
|
687
|
+
for field in arrays.jagged_arrays.fields:
|
|
688
|
+
group_fields[field] = arrays.jagged_arrays.array[field]
|
|
689
|
+
|
|
690
|
+
return ak.zip(group_fields, depth_limit=1)
|
|
@@ -87,7 +87,10 @@ def get_hdf5_metadata(hdf5_file_path: str | list[str], resolve_path: bool = Fals
|
|
|
87
87
|
hdf5_file_path = resolve_hdf5_path(hdf5_file_path)
|
|
88
88
|
|
|
89
89
|
with h5py.File(hdf5_file_path, "r") as f:
|
|
90
|
-
|
|
90
|
+
try:
|
|
91
|
+
metadata = json.loads(f["metadata"][()])
|
|
92
|
+
except KeyError:
|
|
93
|
+
return {}
|
|
91
94
|
|
|
92
95
|
return metadata
|
|
93
96
|
|
|
@@ -377,7 +380,7 @@ def get_column_selection(hdf5_files: str | list[str], column_names: list[str]) -
|
|
|
377
380
|
selection = ColumnSelection()
|
|
378
381
|
|
|
379
382
|
for dataset_name in used_columns_dct.keys():
|
|
380
|
-
if dataset_name in pad_values:
|
|
383
|
+
if pad_values is not None and dataset_name in pad_values:
|
|
381
384
|
pad_value = float(pad_values[dataset_name])
|
|
382
385
|
else:
|
|
383
386
|
pad_value = None
|
|
@@ -22,6 +22,7 @@ class DatasetScaler:
|
|
|
22
22
|
scaler_save_path: str | None = None,
|
|
23
23
|
n_max: int | float | None = None,
|
|
24
24
|
extra_hash: str | None = None,
|
|
25
|
+
reuse_scalers: bool = True,
|
|
25
26
|
scaler_kwargs: dict[str, Any] | None = None,
|
|
26
27
|
dataset_kwargs: dict[str, Any] | None = None,
|
|
27
28
|
dataloader_kwargs: dict[str, Any] | None = None,
|
|
@@ -41,10 +42,10 @@ class DatasetScaler:
|
|
|
41
42
|
by default None.
|
|
42
43
|
n_max : int | float | None, optional
|
|
43
44
|
Maximum number of events to use for feature scaling. If None, all events will be used, by default None.
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
45
|
+
extra_hash : str | None, optional
|
|
46
|
+
Extra hash to append to the scaler filename, by default None.
|
|
47
|
+
reuse_scalers : bool, optional
|
|
48
|
+
Whether to reuse the same scaler for all objects in a dataset, by default True.
|
|
48
49
|
scaler_kwargs : dict[str, Any] | None, optional
|
|
49
50
|
Additional keyword arguments to pass to the sklearn feature scaler, by default None.
|
|
50
51
|
dataset_kwargs : dict[str, Any] | None, optional
|
|
@@ -76,7 +77,7 @@ class DatasetScaler:
|
|
|
76
77
|
dataloader_kwargs,
|
|
77
78
|
)
|
|
78
79
|
|
|
79
|
-
self.dataset_scalers = self._get_scalers(scaler_type, scaler_save_path, n_max, scaler_kwargs)
|
|
80
|
+
self.dataset_scalers = self._get_scalers(scaler_type, scaler_save_path, n_max, scaler_kwargs, reuse_scalers)
|
|
80
81
|
logging.info(f"Scaling datasets: {list(self.dataset_scalers.keys())}")
|
|
81
82
|
|
|
82
83
|
def _get_dataloader(
|
|
@@ -104,6 +105,7 @@ class DatasetScaler:
|
|
|
104
105
|
scaler_save_path: str,
|
|
105
106
|
n_max: int | float | None = None,
|
|
106
107
|
scaler_kwargs: dict[str, Any] | None = None,
|
|
108
|
+
reuse_scalers: bool = False,
|
|
107
109
|
) -> dict[str, dict[str, list[FeatureScaler | None]]]:
|
|
108
110
|
if scaler_kwargs is None:
|
|
109
111
|
scaler_kwargs = {}
|
|
@@ -116,21 +118,22 @@ class DatasetScaler:
|
|
|
116
118
|
numer_scalers: list[FeatureScaler | None] = []
|
|
117
119
|
categ_scalers: list[FeatureScaler | None] = []
|
|
118
120
|
|
|
119
|
-
for
|
|
120
|
-
if
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
121
|
+
for n_obj in range(n_objects):
|
|
122
|
+
if (reuse_scalers and n_obj == 0) or (not reuse_scalers):
|
|
123
|
+
if len(column_selection.numer_columns) != 0:
|
|
124
|
+
numer_feature_scaler = NumericalFeatureScaler(
|
|
125
|
+
scaler_type,
|
|
126
|
+
n_max=n_max,
|
|
127
|
+
save_path=scaler_save_path,
|
|
128
|
+
**scaler_kwargs,
|
|
129
|
+
)
|
|
130
|
+
else:
|
|
131
|
+
numer_feature_scaler = None
|
|
132
|
+
|
|
133
|
+
if len(column_selection.categ_columns) != 0:
|
|
134
|
+
categ_feature_scaler = CategoricalFeatureScaler("categorical", scaler_save_path)
|
|
135
|
+
else:
|
|
136
|
+
categ_feature_scaler = None
|
|
134
137
|
|
|
135
138
|
numer_scalers.append(numer_feature_scaler)
|
|
136
139
|
categ_scalers.append(categ_feature_scaler)
|
|
@@ -145,19 +148,21 @@ class DatasetScaler:
|
|
|
145
148
|
def _get_padding_mask(self, X: np.ndarray, dataset_name: str) -> np.ndarray | None:
|
|
146
149
|
return padding_mask(X, self.selection[dataset_name].pad_value)
|
|
147
150
|
|
|
148
|
-
def _scale_dataset_idx(
|
|
151
|
+
def _scale_dataset_idx(
|
|
152
|
+
self, ds_batch: WeightedDatasetBatch, dataset_name: str, idx: int, scale_categ_padding: bool = True
|
|
153
|
+
) -> bool:
|
|
149
154
|
X = ds_batch[dataset_name].X.numpy()
|
|
150
155
|
|
|
151
156
|
if dataset_name != "events":
|
|
152
157
|
X = X[:, idx, :]
|
|
153
158
|
padding_mask = self._get_padding_mask(X, dataset_name)
|
|
154
|
-
|
|
155
|
-
if padding_mask is not None:
|
|
156
|
-
X = X[padding_mask]
|
|
157
159
|
else:
|
|
158
160
|
padding_mask = None
|
|
159
161
|
|
|
160
162
|
if self.selection[dataset_name].has_categ_columns:
|
|
163
|
+
if padding_mask is not None and not scale_categ_padding:
|
|
164
|
+
X = X[padding_mask]
|
|
165
|
+
|
|
161
166
|
categ_feature_scaler = self.dataset_scalers[dataset_name]["categ"][idx]
|
|
162
167
|
|
|
163
168
|
if categ_feature_scaler is None:
|
|
@@ -169,6 +174,9 @@ class DatasetScaler:
|
|
|
169
174
|
categ_feature_scaler.fit(X_categ)
|
|
170
175
|
|
|
171
176
|
if self.selection[dataset_name].has_numer_columns:
|
|
177
|
+
if padding_mask is not None:
|
|
178
|
+
X = X[padding_mask]
|
|
179
|
+
|
|
172
180
|
numer_feature_scaler = self.dataset_scalers[dataset_name]["numer"][idx]
|
|
173
181
|
|
|
174
182
|
if numer_feature_scaler is None:
|
|
@@ -28,6 +28,10 @@ WeightedBatchType = tuple[
|
|
|
28
28
|
torch.Tensor,
|
|
29
29
|
dict[str, Any],
|
|
30
30
|
]
|
|
31
|
+
FullWeightedBatchType = tuple[
|
|
32
|
+
dict[str, tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]],
|
|
33
|
+
dict[str, Any],
|
|
34
|
+
]
|
|
31
35
|
|
|
32
36
|
|
|
33
37
|
@dataclass(frozen=True, slots=True, repr=False, eq=False)
|
|
@@ -165,6 +169,16 @@ class WeightedBatch:
|
|
|
165
169
|
if isinstance(self._y_aux, np.ndarray):
|
|
166
170
|
self._y_aux = torch.from_numpy(self._y_aux)
|
|
167
171
|
|
|
172
|
+
n_samples = self._X.shape[0]
|
|
173
|
+
if self._y is not None and self._y.shape[0] != n_samples:
|
|
174
|
+
raise RuntimeError("X and y must have the same first dimension!")
|
|
175
|
+
|
|
176
|
+
if self._w is not None and self._w.shape[0] != n_samples:
|
|
177
|
+
raise RuntimeError("X and w must have the same first dimension!")
|
|
178
|
+
|
|
179
|
+
if self._y_aux is not None and self._y_aux.shape[0] != n_samples:
|
|
180
|
+
raise RuntimeError("X and y_aux must have the same first dimension!")
|
|
181
|
+
|
|
168
182
|
@property
|
|
169
183
|
def X(self) -> torch.Tensor:
|
|
170
184
|
return self._X # type: ignore
|
|
@@ -185,8 +199,7 @@ class WeightedBatch:
|
|
|
185
199
|
def feature_shape(self) -> tuple[int, ...]:
|
|
186
200
|
return tuple(self._X.shape)
|
|
187
201
|
|
|
188
|
-
|
|
189
|
-
def batch(self) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
|
202
|
+
def make_batch(self) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
|
190
203
|
return self._X, self._y, self._w, self._y_aux # type: ignore
|
|
191
204
|
|
|
192
205
|
def mask(self, mask: np.ndarray | torch.Tensor) -> WeightedBatch:
|
|
@@ -283,6 +296,14 @@ class WeightedDatasetBatch:
|
|
|
283
296
|
|
|
284
297
|
return self.__class__(_masked_datasets)
|
|
285
298
|
|
|
299
|
+
def make_batch(
|
|
300
|
+
self,
|
|
301
|
+
) -> dict[str, tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]]:
|
|
302
|
+
batch = {}
|
|
303
|
+
for key, ds_batch in self._datasets.items():
|
|
304
|
+
batch[key] = ds_batch.make_batch()
|
|
305
|
+
return batch
|
|
306
|
+
|
|
286
307
|
def __repr__(self) -> str:
|
|
287
308
|
return f"WeightedDatasetBatch({self.keys()})"
|
|
288
309
|
|
|
@@ -393,7 +414,7 @@ class MLHdf5Iterator(Hdf5Iterator):
|
|
|
393
414
|
postfix: str | None = None,
|
|
394
415
|
extra_hash: str = "",
|
|
395
416
|
) -> tuple[NumericalFeatureScaler | None, CategoricalFeatureScaler | None]:
|
|
396
|
-
if scaler_path is None
|
|
417
|
+
if scaler_path is None or scaler_type is None:
|
|
397
418
|
return None, None
|
|
398
419
|
|
|
399
420
|
numer_feature_scaler: NumericalFeatureScaler | None
|
|
@@ -422,7 +443,7 @@ class MLHdf5Iterator(Hdf5Iterator):
|
|
|
422
443
|
|
|
423
444
|
return numer_feature_scaler, categ_feature_scaler
|
|
424
445
|
|
|
425
|
-
def _scale(self, X: np.ndarray, dataset_name: str, is_numer: bool) -> np.ndarray:
|
|
446
|
+
def _scale(self, X: np.ndarray, dataset_name: str, is_numer: bool, scale_categ_padding: bool = True) -> np.ndarray:
|
|
426
447
|
scalers: list[NumericalFeatureScaler | None] | list[CategoricalFeatureScaler | None]
|
|
427
448
|
|
|
428
449
|
if is_numer:
|
|
@@ -432,6 +453,9 @@ class MLHdf5Iterator(Hdf5Iterator):
|
|
|
432
453
|
|
|
433
454
|
n_objects = self.selection[dataset_name].n_objects
|
|
434
455
|
|
|
456
|
+
if len(scalers) != n_objects:
|
|
457
|
+
raise RuntimeError(f"Scaler list length mismatch for dataset {dataset_name}.")
|
|
458
|
+
|
|
435
459
|
if n_objects == 1:
|
|
436
460
|
scaler = scalers[0]
|
|
437
461
|
if scaler is not None:
|
|
@@ -443,7 +467,9 @@ class MLHdf5Iterator(Hdf5Iterator):
|
|
|
443
467
|
X_i = X[:, i, :]
|
|
444
468
|
pad_value = self.selection[dataset_name].pad_value
|
|
445
469
|
|
|
446
|
-
if pad_value is not None:
|
|
470
|
+
if pad_value is not None and scale_categ_padding and not is_numer:
|
|
471
|
+
X_objects.append(scaler.transform(X_i))
|
|
472
|
+
elif pad_value is not None:
|
|
447
473
|
mask = padding_mask(X_i, pad_value)
|
|
448
474
|
|
|
449
475
|
X_full = np.full_like(X_i, pad_value)
|
|
@@ -504,16 +530,16 @@ class MLHdf5Iterator(Hdf5Iterator):
|
|
|
504
530
|
|
|
505
531
|
if X_numer_dataset_stack is not None and X_categ_dataset_stack is not None:
|
|
506
532
|
X_dataset_stack = np.concatenate([X_numer_dataset_stack, X_categ_dataset_stack], axis=-1)
|
|
507
|
-
numer_idx = np.arange(numer_shape[1])
|
|
508
|
-
categ_idx = np.arange(numer_shape[1], numer_shape[1] + categ_shape[1])
|
|
533
|
+
numer_idx = np.arange(numer_shape[-1])
|
|
534
|
+
categ_idx = np.arange(numer_shape[-1], numer_shape[-1] + categ_shape[-1])
|
|
509
535
|
|
|
510
536
|
elif X_numer_dataset_stack is not None:
|
|
511
537
|
X_dataset_stack = X_numer_dataset_stack
|
|
512
|
-
categ_idx, numer_idx = None, np.arange(
|
|
538
|
+
categ_idx, numer_idx = None, np.arange(numer_shape[-1])
|
|
513
539
|
|
|
514
540
|
elif X_categ_dataset_stack is not None:
|
|
515
541
|
X_dataset_stack = X_categ_dataset_stack
|
|
516
|
-
categ_idx, numer_idx = np.arange(
|
|
542
|
+
categ_idx, numer_idx = np.arange(categ_shape[-1]), None
|
|
517
543
|
|
|
518
544
|
else:
|
|
519
545
|
raise RuntimeError("Both X_numer and X_categ are None!")
|
|
@@ -629,10 +655,24 @@ def events_collate_fn(batch: Any) -> WeightedBatchType:
|
|
|
629
655
|
return ds.X, ds.y, ds.w, ds.y_aux, reports
|
|
630
656
|
|
|
631
657
|
|
|
632
|
-
def
|
|
658
|
+
def full_collate_fn(batch: Any) -> Any:
|
|
659
|
+
datasets_batch, reports = batch[0].make_batch(), batch[1]
|
|
660
|
+
return datasets_batch, reports
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def default_setup_func(stacked_datasets: StackedDatasets, ml_iterator: MLHdf5Iterator) -> WeightedDatasetBatch | None:
|
|
633
664
|
weighted_datase_batch = WeightedDatasetBatch()
|
|
634
665
|
|
|
635
|
-
|
|
666
|
+
dataset_keys = stacked_datasets.keys()
|
|
667
|
+
if "events" in dataset_keys:
|
|
668
|
+
dataset_keys.remove("events")
|
|
669
|
+
dataset_keys.insert(0, "events")
|
|
670
|
+
|
|
671
|
+
mask_unmapped: np.ndarray | None = None
|
|
672
|
+
|
|
673
|
+
for ds_name in dataset_keys:
|
|
674
|
+
ds = stacked_datasets[ds_name]
|
|
675
|
+
|
|
636
676
|
X = ds.X.astype(np.float32)
|
|
637
677
|
y = ds.get_extra("label_type", None)
|
|
638
678
|
w = ds.get_extra("weights", None)
|
|
@@ -643,18 +683,14 @@ def default_setup_func(stacked_datasets: StackedDatasets, ml_iterator: MLHdf5Ite
|
|
|
643
683
|
if remap_labels is not None:
|
|
644
684
|
max_label = ml_iterator.dataset_kwargs["max_label"]
|
|
645
685
|
y, mask_unmapped = remap_labels_lookup(y, max_label, remap_labels)
|
|
646
|
-
X = X[mask_unmapped]
|
|
647
|
-
if w is not None:
|
|
648
|
-
w = w[mask_unmapped]
|
|
649
686
|
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
X[:] = X[shuffle_idx]
|
|
687
|
+
if y.shape[0] == 0:
|
|
688
|
+
return None
|
|
653
689
|
|
|
654
|
-
|
|
655
|
-
|
|
690
|
+
if mask_unmapped is not None:
|
|
691
|
+
X = X[mask_unmapped]
|
|
656
692
|
if w is not None:
|
|
657
|
-
w
|
|
693
|
+
w = w[mask_unmapped]
|
|
658
694
|
|
|
659
695
|
weighted_datase_batch[ds_name] = WeightedBatch(X, y, w, y_aux)
|
|
660
696
|
|
|
@@ -28,19 +28,26 @@ class ProcessorsCollection:
|
|
|
28
28
|
def __getitem__(self, name: str) -> Processor:
|
|
29
29
|
return self.objects[name]
|
|
30
30
|
|
|
31
|
+
def _add_processor(self, processor: Processor) -> None:
|
|
32
|
+
# Check if processor being added is unique
|
|
33
|
+
if processor in self.objects.values():
|
|
34
|
+
raise ValueError(f"{processor.name} already exists in the {self.collection_name} collection!")
|
|
35
|
+
|
|
36
|
+
# Check if name of unique processor being added is unique
|
|
37
|
+
if processor.name in self.objects:
|
|
38
|
+
raise ValueError(
|
|
39
|
+
f'Processor with name "{processor.name}" already exists in the {self.collection_name} collection! Choose a unique name.'
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
self.objects[processor.name] = processor
|
|
43
|
+
|
|
31
44
|
def add(self, *objs: Processor | ProcessorsCollection) -> ProcessorsCollection:
|
|
32
45
|
for obj in objs:
|
|
33
46
|
if isinstance(obj, ProcessorsCollection):
|
|
34
47
|
for v in obj.objects.values():
|
|
35
|
-
|
|
36
|
-
raise ValueError(f"{v.name} already exists in the {self.collection_name} collection!")
|
|
37
|
-
else:
|
|
38
|
-
self.objects[v.name] = v
|
|
48
|
+
self._add_processor(v)
|
|
39
49
|
elif isinstance(obj, Processor):
|
|
40
|
-
|
|
41
|
-
raise ValueError(f"{obj.name} already exists in the {self.collection_name} collection!")
|
|
42
|
-
else:
|
|
43
|
-
self.objects[obj.name] = obj
|
|
50
|
+
self._add_processor(obj)
|
|
44
51
|
else:
|
|
45
52
|
raise ValueError(f"Invalid object type {type(obj)}!")
|
|
46
53
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "f9columnar"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.4"
|
|
4
4
|
description = "Columnar analysis utils."
|
|
5
5
|
authors = ["Jan Gavranovic <jan.gavranovic@cern.ch>"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -9,7 +9,7 @@ license = "MIT"
|
|
|
9
9
|
|
|
10
10
|
[tool.poetry.dependencies]
|
|
11
11
|
python = ">=3.10,<3.13"
|
|
12
|
-
torch = { version = "2.
|
|
12
|
+
torch = { version = "2.8.0", optional = true }
|
|
13
13
|
numpy = "^2.0.2"
|
|
14
14
|
pandas = "^2.2.2"
|
|
15
15
|
scipy = "^1.14.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|