legend-pydataobj 1.5.0a1__tar.gz → 1.5.0a3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {legend_pydataobj-1.5.0a1/src/legend_pydataobj.egg-info → legend_pydataobj-1.5.0a3}/PKG-INFO +1 -1
  2. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3/src/legend_pydataobj.egg-info}/PKG-INFO +1 -1
  3. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/_version.py +1 -1
  4. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/cli.py +9 -3
  5. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/lh5/store.py +25 -3
  6. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/lh5_store.py +31 -0
  7. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/arrayofequalsizedarrays.py +2 -0
  8. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/struct.py +7 -1
  9. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/table.py +98 -88
  10. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/vectorofvectors.py +35 -21
  11. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/lh5/test_lh5_store.py +2 -29
  12. legend_pydataobj-1.5.0a3/tests/types/test_table_eval.py +62 -0
  13. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_vectorofvectors.py +3 -0
  14. legend_pydataobj-1.5.0a1/tests/types/test_table_eval.py +0 -96
  15. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/LICENSE +0 -0
  16. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/README.md +0 -0
  17. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/pyproject.toml +0 -0
  18. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/setup.cfg +0 -0
  19. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/setup.py +0 -0
  20. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/legend_pydataobj.egg-info/SOURCES.txt +0 -0
  21. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/legend_pydataobj.egg-info/dependency_links.txt +0 -0
  22. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/legend_pydataobj.egg-info/entry_points.txt +0 -0
  23. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/legend_pydataobj.egg-info/not-zip-safe +0 -0
  24. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/legend_pydataobj.egg-info/requires.txt +0 -0
  25. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/legend_pydataobj.egg-info/top_level.txt +0 -0
  26. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/__init__.py +0 -0
  27. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/compression/__init__.py +0 -0
  28. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/compression/base.py +0 -0
  29. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/compression/generic.py +0 -0
  30. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/compression/radware.py +0 -0
  31. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/compression/utils.py +0 -0
  32. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/compression/varlen.py +0 -0
  33. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/lgdo_utils.py +0 -0
  34. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/lh5/__init__.py +0 -0
  35. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/lh5/iterator.py +0 -0
  36. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/lh5/utils.py +0 -0
  37. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/logging.py +0 -0
  38. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/__init__.py +0 -0
  39. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/array.py +0 -0
  40. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/encoded.py +0 -0
  41. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/fixedsizearray.py +0 -0
  42. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/lgdo.py +0 -0
  43. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/scalar.py +0 -0
  44. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/types/waveformtable.py +0 -0
  45. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/units.py +0 -0
  46. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/src/lgdo/utils.py +0 -0
  47. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/compression/conftest.py +0 -0
  48. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/compression/sigcompress/LDQTA_r117_20200110T105115Z_cal_geds_raw-0.dat +0 -0
  49. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/compression/sigcompress/special-wf-clipped.dat +0 -0
  50. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/compression/test_compression.py +0 -0
  51. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/compression/test_radware_sigcompress.py +0 -0
  52. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/compression/test_str2wfcodec.py +0 -0
  53. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/compression/test_uleb128_zigzag_diff.py +0 -0
  54. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/conftest.py +0 -0
  55. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/lh5/test_lh5_iterator.py +0 -0
  56. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/lh5/test_lh5_utils.py +0 -0
  57. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/test_cli.py +0 -0
  58. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/test_lgdo_utils.py +0 -0
  59. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_array.py +0 -0
  60. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_arrayofequalsizedarrays.py +0 -0
  61. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_encoded.py +0 -0
  62. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_fixedsizearray.py +0 -0
  63. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_representations.py +0 -0
  64. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_scalar.py +0 -0
  65. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_struct.py +0 -0
  66. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_table.py +0 -0
  67. {legend_pydataobj-1.5.0a1 → legend_pydataobj-1.5.0a3}/tests/types/test_waveformtable.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: legend_pydataobj
3
- Version: 1.5.0a1
3
+ Version: 1.5.0a3
4
4
  Summary: LEGEND Python Data Objects
5
5
  Home-page: https://github.com/legend-exp/legend-pydataobj
6
6
  Author: The LEGEND Collaboration
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: legend_pydataobj
3
- Version: 1.5.0a1
3
+ Version: 1.5.0a3
4
4
  Summary: LEGEND Python Data Objects
5
5
  Home-page: https://github.com/legend-exp/legend-pydataobj
6
6
  Author: The LEGEND Collaboration
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '1.5.0a1'
15
+ __version__ = version = '1.5.0a3'
16
16
  __version_tuple__ = version_tuple = (1, 5, 0)
@@ -5,7 +5,7 @@ import sys
5
5
 
6
6
  import lgdo
7
7
  import lgdo.logging
8
- from lgdo import show
8
+ from lgdo.lh5 import show
9
9
 
10
10
 
11
11
  def lh5ls():
@@ -26,7 +26,6 @@ def lh5ls():
26
26
  )
27
27
  parser.add_argument(
28
28
  "--debug",
29
- "-d",
30
29
  action="store_true",
31
30
  help="""Increase the program verbosity to maximum""",
32
31
  )
@@ -39,6 +38,13 @@ def lh5ls():
39
38
  parser.add_argument(
40
39
  "--attributes", "-a", action="store_true", help="""Print HDF5 attributes too"""
41
40
  )
41
+ parser.add_argument(
42
+ "--depth",
43
+ "-d",
44
+ type=int,
45
+ default=None,
46
+ help="""Maximum tree depth of groups to print""",
47
+ )
42
48
 
43
49
  args = parser.parse_args()
44
50
 
@@ -53,4 +59,4 @@ def lh5ls():
53
59
  print(lgdo.__version__) # noqa: T201
54
60
  sys.exit()
55
61
 
56
- show(args.lh5_file, args.lh5_group, attrs=args.attributes)
62
+ show(args.lh5_file, args.lh5_group, attrs=args.attributes, depth=args.depth)
@@ -12,6 +12,7 @@ import sys
12
12
  from bisect import bisect_left
13
13
  from collections import defaultdict
14
14
  from typing import Any, Union
15
+ from warnings import warn
15
16
 
16
17
  import h5py
17
18
  import numba as nb
@@ -347,9 +348,7 @@ class LH5Store:
347
348
  default = not field_mask[list(field_mask.keys())[0]]
348
349
  field_mask = defaultdict(lambda: default, field_mask)
349
350
  elif isinstance(field_mask, (list, tuple)):
350
- field_mask = defaultdict(
351
- lambda: False, {field: True for field in field_mask}
352
- )
351
+ field_mask = defaultdict(bool, {field: True for field in field_mask})
353
352
  elif not isinstance(field_mask, defaultdict):
354
353
  raise RuntimeError("bad field_mask of type", type(field_mask).__name__)
355
354
  elif field_mask is not None:
@@ -1337,6 +1336,7 @@ def show(
1337
1336
  attrs: bool = False,
1338
1337
  indent: str = "",
1339
1338
  header: bool = True,
1339
+ depth: int | None = None,
1340
1340
  ) -> None:
1341
1341
  """Print a tree of LH5 file contents with LGDO datatype.
1342
1342
 
@@ -1352,6 +1352,8 @@ def show(
1352
1352
  indent the diagram with this string.
1353
1353
  header
1354
1354
  print `lh5_group` at the top of the diagram.
1355
+ depth
1356
+ maximum tree depth of groups to print
1355
1357
 
1356
1358
  Examples
1357
1359
  --------
@@ -1367,6 +1369,10 @@ def show(
1367
1369
  │ └── values · array_of_equalsized_arrays<1,1>{real}
1368
1370
  └── wf_std · array<1>{real}
1369
1371
  """
1372
+ # check tree depth if we are using it
1373
+ if depth is not None and depth <= 0:
1374
+ return
1375
+
1370
1376
  # open file
1371
1377
  if isinstance(lh5_file, str):
1372
1378
  lh5_file = h5py.File(expand_path(lh5_file), "r")
@@ -1422,6 +1428,7 @@ def show(
1422
1428
  indent=indent + (" " if killme else "│ "),
1423
1429
  header=False,
1424
1430
  attrs=attrs,
1431
+ depth=depth - 1 if depth else None,
1425
1432
  )
1426
1433
 
1427
1434
  # break or move to next key
@@ -1461,6 +1468,14 @@ def load_nda(
1461
1468
  Each entry contains the data for the specified parameter concatenated
1462
1469
  over all files in `f_list`.
1463
1470
  """
1471
+ warn(
1472
+ "load_nda() is deprecated. "
1473
+ "Please replace it with LH5Store.read(...).view_as('np'). "
1474
+ "load_nda() will be removed in a future release.",
1475
+ DeprecationWarning,
1476
+ stacklevel=2,
1477
+ )
1478
+
1464
1479
  if isinstance(f_list, str):
1465
1480
  f_list = [f_list]
1466
1481
  if idx_list is not None:
@@ -1515,6 +1530,13 @@ def load_dfs(
1515
1530
  all data for the associated parameters concatenated over all files in
1516
1531
  `f_list`.
1517
1532
  """
1533
+ warn(
1534
+ "load_dfs() is deprecated. "
1535
+ "Please replace it with LH5Store.read(...).view_as('pd'). "
1536
+ "load_dfs() will be removed in a future release.",
1537
+ DeprecationWarning,
1538
+ stacklevel=2,
1539
+ )
1518
1540
  return pd.DataFrame(
1519
1541
  load_nda(f_list, par_list, lh5_group=lh5_group, idx_list=idx_list)
1520
1542
  )
@@ -131,6 +131,37 @@ class LH5Store(lh5.LH5Store):
131
131
  )
132
132
  super().__init__(base_path, keep_open)
133
133
 
134
+ def read_object(
135
+ self,
136
+ name: str,
137
+ lh5_file: str | h5py.File | list[str | h5py.File],
138
+ **kwargs,
139
+ ) -> tuple[LGDO, int]:
140
+ warn(
141
+ "LH5Store.read_object() has been renamed to LH5Store.read(), "
142
+ "Please update your code."
143
+ "LH5Store.read_object() will be removed in a future release.",
144
+ DeprecationWarning,
145
+ stacklevel=2,
146
+ )
147
+ return super().read(self, name, lh5_file, **kwargs)
148
+
149
+ def write_object(
150
+ self,
151
+ obj: LGDO,
152
+ name: str,
153
+ lh5_file: str | h5py.File,
154
+ **kwargs,
155
+ ) -> tuple[LGDO, int]:
156
+ warn(
157
+ "LH5Store.write_object() has been renamed to LH5Store.write(), "
158
+ "Please update your code."
159
+ "LH5Store.write_object() will be removed in a future release.",
160
+ DeprecationWarning,
161
+ stacklevel=2,
162
+ )
163
+ return super().read(self, obj, name, lh5_file, **kwargs)
164
+
134
165
 
135
166
  def load_dfs(
136
167
  f_list: str | list[str],
@@ -70,6 +70,8 @@ class ArrayOfEqualSizedArrays(Array):
70
70
  if nda is None:
71
71
  s = shape
72
72
  else:
73
+ if not isinstance(nda, np.ndarray):
74
+ nda = np.array(nda)
73
75
  s = nda.shape
74
76
  self.dims = (1, len(s) - 1)
75
77
  else:
@@ -60,7 +60,13 @@ class Struct(LGDO, dict):
60
60
  return self.add_field(name, obj)
61
61
 
62
62
  def __getattr__(self, name: str) -> LGDO:
63
- return self.__getitem__(name)
63
+ if hasattr(super(), name):
64
+ return super().__getattr__(name)
65
+
66
+ if name in self.keys():
67
+ return super().__getitem__(name)
68
+
69
+ raise AttributeError(name)
64
70
 
65
71
  def remove_field(self, name: str | int, delete: bool = False) -> None:
66
72
  """Remove a field from the table.
@@ -5,8 +5,7 @@ equal length and corresponding utilities.
5
5
  from __future__ import annotations
6
6
 
7
7
  import logging
8
- import re
9
- from typing import Any
8
+ from typing import Any, Mapping
10
9
  from warnings import warn
11
10
 
12
11
  import awkward as ak
@@ -18,6 +17,7 @@ from pandas.io.formats import format as fmt
18
17
  from .array import Array
19
18
  from .arrayofequalsizedarrays import ArrayOfEqualSizedArrays
20
19
  from .lgdo import LGDO
20
+ from .scalar import Scalar
21
21
  from .struct import Struct
22
22
  from .vectorofvectors import VectorOfVectors
23
23
 
@@ -38,8 +38,6 @@ class Table(Struct):
38
38
  :meth:`__len__` to access valid data, which returns the ``size`` attribute.
39
39
  """
40
40
 
41
- # TODO: overload getattr to allow access to fields as object attributes?
42
-
43
41
  def __init__(
44
42
  self,
45
43
  size: int = None,
@@ -217,102 +215,114 @@ class Table(Struct):
217
215
  )
218
216
  return self.view_as(library="pd", cols=cols, prefix=prefix)
219
217
 
220
- def eval(self, expr_config: dict) -> Table:
221
- """Apply column operations to the table and return a new table holding
222
- the resulting columns.
218
+ def eval(
219
+ self,
220
+ expr: str,
221
+ parameters: Mapping[str, str] = None,
222
+ ) -> LGDO:
223
+ """Apply column operations to the table and return a new LGDO.
223
224
 
224
- Currently defers all the job to :meth:`numexpr.evaluate`. This
225
- might change in the future.
225
+ Internally uses :func:`numexpr.evaluate` if dealing with columns
226
+ representable as NumPy arrays or :func:`eval` if
227
+ :class:`.VectorOfVectors` are involved. In the latter case, the VoV
228
+ columns are viewed as :class:`ak.Array` and the respective routines are
229
+ therefore available.
226
230
 
227
231
  Parameters
228
232
  ----------
229
- expr_config
230
- dictionary that configures expressions according the following
231
- specification:
232
-
233
- .. code-block:: js
234
-
235
- {
236
- "O1": {
237
- "expression": "p1 + p2 * a**2",
238
- "parameters": {
239
- "p1": 2,
240
- "p2": 3
241
- }
242
- },
243
- "O2": {
244
- "expression": "O1 - b"
245
- }
246
- // ...
247
- }
248
-
249
- where:
250
-
251
- - ``expression`` is an expression string supported by
252
- :meth:`numexpr.evaluate` (see also `here
253
- <https://numexpr.readthedocs.io/projects/NumExpr3/en/latest/index.html>`_
254
- for documentation). Note: because of internal limitations,
255
- reduction operations must appear the last in the stack.
256
- - ``parameters`` is a dictionary of function parameters. Passed to
257
- :meth:`numexpr.evaluate`` as `local_dict` argument.
258
-
259
-
260
- Warning
261
- -------
262
- Blocks in `expr_config` must be ordered according to mutual dependency.
233
+ expr
234
+ if the expression only involves non-:class:`.VectorOfVectors`
235
+ columns, the syntax is the one supported by
236
+ :func:`numexpr.evaluate` (see `here
237
+ <https://numexpr.readthedocs.io/projects/NumExpr3/en/latest/index.html>`_
238
+ for documentation). Note: because of internal limitations,
239
+ reduction operations must appear the last in the stack. If at least
240
+ one considered column is a :class:`.VectorOfVectors`, plain
241
+ :func:`eval` is used and :class:`ak.Array` transforms can be used
242
+ through the ``ak.`` prefix. (NumPy functions are analogously
243
+ accessible through ``np.``). See also examples below.
244
+ parameters
245
+ a dictionary of function parameters. Passed to
246
+ :func:`numexpr.evaluate`` as `local_dict` argument or to
247
+ :func:`eval` as `locals` argument.
248
+
249
+ Examples
250
+ --------
251
+ >>> import lgdo
252
+ >>> tbl = lgdo.Table(
253
+ ... col_dict={
254
+ ... "a": lgdo.Array([1, 2, 3]),
255
+ ... "b": lgdo.VectorOfVectors([[5], [6, 7], [8, 9, 0]]),
256
+ ... }
257
+ ... )
258
+ >>> print(tbl.eval("a + b"))
259
+ [[6],
260
+ [8 9],
261
+ [11 12 3],
262
+ ]
263
+ >>> print(tbl.eval("np.sum(a) + ak.sum(b)"))
264
+ 41
263
265
  """
264
- out_tbl = Table(size=self.size)
265
- for out_var, spec in expr_config.items():
266
- in_vars = {}
267
- # Find all valid python variables in expression (e.g "a*b+sin(Cool)" --> ['a','b','sin','Cool'])
268
- for elem in re.findall(r"\s*[A-Za-z_]\w*\s*", spec["expression"]):
269
- elem = elem.strip()
270
- if elem in self: # check if the variable comes from dsp
271
- in_vars[elem] = self[elem]
272
- elif (
273
- elem in out_tbl.keys()
274
- ): # if not try from previously processed data, else ignore since it is e.g sin func
275
- in_vars[elem] = out_tbl[elem]
276
-
266
+ if parameters is None:
267
+ parameters = {}
268
+
269
+ # get the valid python variable names in the expression
270
+ c = compile(expr, "0vbb is real!", "eval")
271
+
272
+ # make a dictionary of low-level objects (numpy or awkward)
273
+ # for later computation
274
+ self_unwrap = {}
275
+ has_ak = False
276
+ for obj in c.co_names:
277
+ if obj in self.keys():
278
+ if isinstance(self[obj], VectorOfVectors):
279
+ self_unwrap[obj] = self[obj].view_as("ak", with_units=False)
280
+ has_ak = True
277
281
  else:
278
- continue
279
- # get the nda if it is an Array instance
280
- if isinstance(in_vars[elem], Array):
281
- in_vars[elem] = in_vars[elem].nda
282
- # No vector of vectors support yet
283
- elif isinstance(in_vars[elem], VectorOfVectors):
284
- raise TypeError("Data of type VectorOfVectors not supported (yet)")
282
+ self_unwrap[obj] = self[obj].view_as("np", with_units=False)
285
283
 
284
+ # use numexpr if we are only dealing with numpy data types
285
+ if not has_ak:
286
286
  out_data = ne.evaluate(
287
- f"{spec['expression']}",
288
- local_dict=dict(in_vars, **spec["parameters"])
289
- if "parameters" in spec
290
- else in_vars,
291
- ) # Division is chosen by __future__.division in the interpreter
292
-
293
- # smart way to find right LGDO data type:
294
-
295
- # out_data has one row and this row has a scalar (eg scalar product of two rows)
296
- if len(np.shape(out_data)) == 0:
297
- out_data = Array(nda=out_data)
298
-
299
- # out_data has scalar in each row
300
- elif len(np.shape(out_data)) == 1:
301
- out_data = Array(nda=out_data)
302
-
303
- # out_data is like
304
- elif len(np.shape(out_data)) == 2:
305
- out_data = ArrayOfEqualSizedArrays(nda=out_data)
306
-
307
- # higher order data (eg matrix product of ArrayOfEqualSizedArrays) not supported yet
287
+ expr,
288
+ local_dict=(self_unwrap | parameters),
289
+ )
290
+
291
+ # need to convert back to LGDO
292
+ # np.evaluate should always return a numpy thing?
293
+ if out_data.ndim == 0:
294
+ return Scalar(out_data.item())
295
+ elif out_data.ndim == 1:
296
+ return Array(out_data)
297
+ elif out_data.ndim == 2:
298
+ return ArrayOfEqualSizedArrays(nda=out_data)
308
299
  else:
309
- ValueError(
310
- f"Calculation resulted in {len(np.shape(out_data))-1}-D row which is not supported yet"
300
+ msg = (
301
+ f"evaluation resulted in {out_data.ndim}-dimensional data, "
302
+ "I don't know which LGDO this corresponds to"
311
303
  )
304
+ raise RuntimeError(msg)
312
305
 
313
- out_tbl.add_column(out_var, out_data)
306
+ else:
307
+ # resort to good ol' eval()
308
+ globs = {"ak": ak, "np": np}
309
+ out_data = eval(expr, globs, (self_unwrap | parameters))
310
+
311
+ # need to convert back to LGDO
312
+ if isinstance(out_data, ak.Array):
313
+ if out_data.ndim == 1:
314
+ return Array(out_data.to_numpy())
315
+ else:
316
+ return VectorOfVectors(out_data)
314
317
 
315
- return out_tbl
318
+ if np.isscalar(out_data):
319
+ return Scalar(out_data)
320
+ else:
321
+ msg = (
322
+ f"evaluation resulted in a {type(out_data)} object, "
323
+ "I don't know which LGDO this corresponds to"
324
+ )
325
+ raise RuntimeError(msg)
316
326
 
317
327
  def __str__(self):
318
328
  opts = fmt.get_dataframe_repr_params()
@@ -35,7 +35,7 @@ class VectorOfVectors(LGDO):
35
35
 
36
36
  def __init__(
37
37
  self,
38
- listoflists: list[list[int | float]] = None,
38
+ array: ak.Array | list[list[int | float]] = None,
39
39
  flattened_data: Array | NDArray = None,
40
40
  cumulative_length: Array | NDArray = None,
41
41
  shape_guess: tuple[int, int] = None,
@@ -46,9 +46,10 @@ class VectorOfVectors(LGDO):
46
46
  """
47
47
  Parameters
48
48
  ----------
49
- listoflists
50
- create a VectorOfVectors out of a Python list of lists. Takes
51
- priority over `flattened_data` and `cumulative_length`.
49
+ array
50
+ create a ``VectorOfVectors`` out of a Python list of lists or an
51
+ :class:`ak.Array`. Takes priority over `flattened_data` and
52
+ `cumulative_length`.
52
53
  flattened_data
53
54
  if not ``None``, used as the internal array for
54
55
  `self.flattened_data`. Otherwise, an internal `flattened_data` is
@@ -67,30 +68,43 @@ class VectorOfVectors(LGDO):
67
68
  of `flattened_data` if it was not supplied.
68
69
  dtype
69
70
  sets the type of data stored in `flattened_data`. Required if
70
- `flattened_data` and `listoflists` are ``None``.
71
+ `flattened_data` and `array` are ``None``.
71
72
  fill_val
72
73
  fill all of `self.flattened_data` with this value.
73
74
  attrs
74
75
  a set of user attributes to be carried along with this LGDO.
75
76
  """
76
- if listoflists is not None:
77
- cl_nda = np.cumsum([len(ll) for ll in listoflists])
78
- if dtype is None:
79
- if len(cl_nda) == 0 or cl_nda[-1] == 0:
80
- raise ValueError("listoflists can't be empty with dtype=None!")
81
- else:
82
- # Set dtype from the first element in the list
83
- # Find it efficiently, allowing for zero-length lists as some of the entries
84
- first_element = next(itertools.chain.from_iterable(listoflists))
85
- dtype = type(first_element)
77
+ if array is not None:
78
+ if isinstance(array, ak.Array):
79
+ if array.ndim != 2:
80
+ raise ValueError(
81
+ "cannot initialize a VectorOfVectors with "
82
+ f"{array.ndim}-dimensional data"
83
+ )
86
84
 
87
- self.dtype = np.dtype(dtype)
88
- self.cumulative_length = Array(cl_nda)
89
- self.flattened_data = Array(
90
- np.fromiter(
91
- itertools.chain.from_iterable(listoflists), dtype=self.dtype
85
+ form, length, container = ak.to_buffers(array)
86
+
87
+ return self.__init__(
88
+ flattened_data=container["node1-data"],
89
+ cumulative_length=container["node0-offsets"][1:],
90
+ )
91
+
92
+ else:
93
+ cl_nda = np.cumsum([len(ll) for ll in array])
94
+ if dtype is None:
95
+ if len(cl_nda) == 0 or cl_nda[-1] == 0:
96
+ raise ValueError("array can't be empty with dtype=None!")
97
+ else:
98
+ # Set dtype from the first element in the list
99
+ # Find it efficiently, allowing for zero-length lists as some of the entries
100
+ first_element = next(itertools.chain.from_iterable(array))
101
+ dtype = type(first_element)
102
+
103
+ self.dtype = np.dtype(dtype)
104
+ self.cumulative_length = Array(cl_nda)
105
+ self.flattened_data = Array(
106
+ np.fromiter(itertools.chain.from_iterable(array), dtype=self.dtype)
92
107
  )
93
- )
94
108
 
95
109
  else:
96
110
  if cumulative_length is None:
@@ -3,7 +3,6 @@ import os
3
3
 
4
4
  import h5py
5
5
  import numpy as np
6
- import pandas as pd
7
6
  import pytest
8
7
 
9
8
  import lgdo
@@ -69,21 +68,6 @@ def test_show(lgnd_file):
69
68
  lh5.show(lgnd_file, "geds/raw")
70
69
 
71
70
 
72
- def test_load_nda(lgnd_file):
73
- nda = lh5.load_nda(
74
- [lgnd_file, lgnd_file],
75
- ["baseline", "waveform/values"],
76
- lh5_group="/geds/raw",
77
- idx_list=[[1, 3, 5], [2, 6, 7]],
78
- )
79
-
80
- assert isinstance(nda, dict)
81
- assert isinstance(nda["baseline"], np.ndarray)
82
- assert nda["baseline"].shape == (6,)
83
- assert isinstance(nda["waveform/values"], np.ndarray)
84
- assert nda["waveform/values"].shape == (6, 5592)
85
-
86
-
87
71
  @pytest.fixture(scope="module")
88
72
  def lh5_file(tmptestdir):
89
73
  store = lh5.LH5Store()
@@ -780,8 +764,8 @@ def test_write_object_overwrite_lgdo(caplog, tmptestdir):
780
764
  assert scalar_dat.value == 1
781
765
 
782
766
  # Finally, try overwriting a vector of vectors
783
- vov1 = types.VectorOfVectors(listoflists=[np.zeros(1), np.ones(2), np.zeros(3)])
784
- vov2 = types.VectorOfVectors(listoflists=[np.ones(1), np.zeros(2), np.ones(3)])
767
+ vov1 = types.VectorOfVectors([np.zeros(1), np.ones(2), np.zeros(3)])
768
+ vov2 = types.VectorOfVectors([np.ones(1), np.zeros(2), np.ones(3)])
785
769
  store = lh5.LH5Store()
786
770
  store.write(vov1, "my_vector", f"{tmptestdir}/write_object_overwrite_test.lh5")
787
771
  store.write(
@@ -906,14 +890,3 @@ def test_write_object_append_column(tmptestdir):
906
890
  assert isinstance(tb_dat, types.Table)
907
891
  assert np.array_equal(tb_dat["dset1"].nda, np.zeros(10))
908
892
  assert np.array_equal(tb_dat["dset2"].nda, np.ones(10))
909
-
910
-
911
- def test_load_dfs(lgnd_file):
912
- dfs = lh5.load_dfs(
913
- [lgnd_file, lgnd_file],
914
- ["baseline", "waveform/t0"],
915
- lh5_group="/geds/raw",
916
- idx_list=[[1, 3, 5], [2, 6, 7]],
917
- )
918
-
919
- assert isinstance(dfs, pd.DataFrame)
@@ -0,0 +1,62 @@
1
+ import numpy as np
2
+
3
+ import lgdo
4
+
5
+
6
+ def test_eval_dependency():
7
+ obj = lgdo.Table(
8
+ col_dict={
9
+ "a": lgdo.Array([1, 2, 3, 4], attrs={"units": "s"}),
10
+ "b": lgdo.Array([5, 6, 7, 8]),
11
+ "c": lgdo.ArrayOfEqualSizedArrays(
12
+ nda=[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]
13
+ ),
14
+ "d": lgdo.ArrayOfEqualSizedArrays(
15
+ nda=[
16
+ [21, 22, 23, 24],
17
+ [25, 26, 27, 8],
18
+ [29, 210, 211, 212],
19
+ [213, 214, 215, 216],
20
+ ],
21
+ ),
22
+ "e": lgdo.VectorOfVectors([[1, 2, 3], [4], [], [8, 6]]),
23
+ }
24
+ )
25
+ r = obj.eval("sum(a)")
26
+ assert isinstance(r, lgdo.Scalar)
27
+
28
+ r = obj.eval("a + b")
29
+ assert isinstance(r, lgdo.Array)
30
+ assert np.array_equal(r.nda, obj.a.nda + obj.b.nda)
31
+
32
+ r = obj.eval("((a - b) > 1) & ((b - a) < -1)")
33
+ assert isinstance(r, lgdo.Array)
34
+ assert r.dtype == "bool"
35
+
36
+ r = obj.eval("a + d")
37
+ assert isinstance(r, lgdo.ArrayOfEqualSizedArrays)
38
+
39
+ assert obj.eval("a**2")
40
+ assert obj.eval("sin(a)")
41
+ assert obj.eval("log(d)")
42
+
43
+ r = obj.eval("a + e")
44
+ assert isinstance(r, lgdo.VectorOfVectors)
45
+ assert r == lgdo.VectorOfVectors([[2, 3, 4], [6], [], [12, 10]])
46
+
47
+ r = obj.eval("2*e + 1")
48
+ assert isinstance(r, lgdo.VectorOfVectors)
49
+ assert r == lgdo.VectorOfVectors([[3, 5, 7], [9], [], [17, 13]])
50
+
51
+ r = obj.eval("e > 2")
52
+ assert isinstance(r, lgdo.VectorOfVectors)
53
+ assert r == lgdo.VectorOfVectors([[False, False, True], [True], [], [True, True]])
54
+ assert r.dtype == "bool"
55
+
56
+ r = obj.eval("ak.sum(e, axis=-1)")
57
+ assert isinstance(r, lgdo.Array)
58
+
59
+ r = obj.eval("ak.sum(e)")
60
+ assert isinstance(r, lgdo.Scalar)
61
+
62
+ assert obj.eval("np.sum(a) + ak.sum(e)")
@@ -48,6 +48,9 @@ def test_init(lgdo_vov):
48
48
  v = VectorOfVectors(shape_guess=(5, 0), dtype="int32")
49
49
  assert v.cumulative_length == lgdo.Array([0, 0, 0, 0, 0])
50
50
 
51
+ v = VectorOfVectors(ak.Array([[1, 2], [3, 4, 5], [2], [4, 8, 9, 7], [5, 3, 1]]))
52
+ assert v == test
53
+
51
54
 
52
55
  def test_datatype_name(lgdo_vov):
53
56
  assert lgdo_vov.datatype_name() == "array"
@@ -1,96 +0,0 @@
1
- import numpy as np
2
-
3
- from lgdo import Array, ArrayOfEqualSizedArrays, Table
4
-
5
-
6
- def test_eval_dependency():
7
- obj = Table(
8
- col_dict={
9
- "a": Array(nda=np.array([1, 2, 3, 4], dtype=np.float32)),
10
- "b": Array(nda=np.array([5, 6, 7, 8], dtype=np.float32)),
11
- "c": ArrayOfEqualSizedArrays(
12
- nda=np.array(
13
- [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]],
14
- dtype=np.float32,
15
- )
16
- ),
17
- "d": ArrayOfEqualSizedArrays(
18
- nda=np.array(
19
- [
20
- [21, 22, 23, 24],
21
- [25, 26, 27, 8],
22
- [29, 210, 211, 212],
23
- [213, 214, 215, 216],
24
- ],
25
- dtype=np.float32,
26
- )
27
- ),
28
- }
29
- )
30
-
31
- expr_config = {
32
- "O1": {"expression": "p1 + p2 * a**2", "parameters": {"p1": 2, "p2": 3}},
33
- "O2": {"expression": "O1 - b"},
34
- "O3": {"expression": "p1 + p2 * c", "parameters": {"p1": 2, "p2": 3}},
35
- "O4": {"expression": "O3 - d", "parameters": {"p1": 2, "p2": 3}},
36
- "O5": {"expression": "sum(c,axis=1)"},
37
- "O6": {"expression": "a>p1", "parameters": {"p1": 2}},
38
- "O7": {"expression": "c>p1", "parameters": {"p1": 2}},
39
- }
40
-
41
- out_tbl = obj.eval(expr_config)
42
- assert list(out_tbl.keys()) == ["O1", "O2", "O3", "O4", "O5", "O6", "O7"]
43
- assert (out_tbl["O1"].nda == [5, 14, 29, 50]).all()
44
- assert (out_tbl["O2"].nda == [0, 8, 22, 42]).all()
45
- assert (
46
- out_tbl["O3"].nda
47
- == [[5, 8, 11, 14], [17, 20, 23, 26], [29, 32, 35, 38], [41, 44, 47, 50]]
48
- ).all()
49
- assert (
50
- out_tbl["O4"].nda
51
- == [
52
- [-16.0, -14.0, -12.0, -10.0],
53
- [-8.0, -6.0, -4.0, 18.0],
54
- [0.0, -178.0, -176.0, -174.0],
55
- [-172.0, -170.0, -168.0, -166.0],
56
- ]
57
- ).all()
58
- assert (out_tbl["O5"].nda == [10.0, 26.0, 42.0, 58.0]).all()
59
- assert (out_tbl["O6"].nda == [False, False, True, True]).all()
60
- assert (
61
- out_tbl["O7"].nda
62
- == [
63
- [False, False, True, True],
64
- [True, True, True, True],
65
- [True, True, True, True],
66
- [True, True, True, True],
67
- ]
68
- ).all()
69
-
70
-
71
- def test_eval_math_functions():
72
- obj = Table(
73
- col_dict={
74
- "a": Array(nda=np.array([1, 2, 3, 4], dtype=np.float32)),
75
- "b": Array(nda=np.array([5, 6, 7, 8], dtype=np.float32)),
76
- "c": ArrayOfEqualSizedArrays(
77
- nda=np.array(
78
- [[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]],
79
- dtype=np.float32,
80
- )
81
- ),
82
- }
83
- )
84
-
85
- expr_config = {
86
- "O1": {"expression": "exp(log(a))"},
87
- "O2": {"expression": "exp(log(c))"},
88
- }
89
-
90
- out_tbl = obj.eval(expr_config)
91
- assert list(out_tbl.keys()) == ["O1", "O2"]
92
- assert (out_tbl["O1"].nda == [1, 2, 3, 4]).all()
93
- assert (
94
- out_tbl["O2"].nda
95
- == np.array([[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]])
96
- ).all()