legend-pydataobj 1.12.0a1__py3-none-any.whl → 1.12.0a3__py3-none-any.whl

{legend_pydataobj-1.12.0a1.dist-info → legend_pydataobj-1.12.0a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend_pydataobj
-Version: 1.12.0a1
+Version: 1.12.0a3
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.12.0a1.dist-info → legend_pydataobj-1.12.0a3.dist-info}/RECORD

@@ -1,9 +1,8 @@
-legend_pydataobj-1.12.0a1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lgdo/__init__.py,sha256=QMYK9HhoMi0pbahPN8mPD18gyTxscFgo7QKfCxVhy-0,3196
-lgdo/_version.py,sha256=kTYHwRhTzZEJHpwJeVgXBi4yFTeQDpnR6MYkvCMA06Q,515
+legend_pydataobj-1.12.0a3.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lgdo/__init__.py,sha256=fkRv79kdtBasw31gPVK9SdLQ2vEEajTV2t3UPDvFg9o,3206
+lgdo/_version.py,sha256=0ujGt6htU-oZxWpmS96KCcv4YV2Uo7Akjbaoi2K7Od8,521
 lgdo/cli.py,sha256=s_EWTBWW76l7zWb6gaTSTjiT-0RzzcYEmjeFEQCVxfk,4647
 lgdo/lgdo_utils.py,sha256=6a2YWEwpyEMXlAyTHZMO01aqxy6SxJzPZkGNWKNWuS0,2567
-lgdo/lh5_store.py,sha256=5BzbJA9sLcqjp8bJDc2olwOiw0VS6rmfg3cfh1kQkRY,8512
 lgdo/logging.py,sha256=82wIOj7l7xr3WYyeHdpSXbbjzHJsy-uRyKYUYx2vMfQ,1003
 lgdo/units.py,sha256=VQYME86_ev9S7Fq8RyCOQNqYr29MphTTYemmEouZafk,161
 lgdo/utils.py,sha256=WRTmXnaQ-h2hVxwJ27qiOigdsD3DHcaDrdDjvupCuZU,3940
@@ -13,15 +12,15 @@ lgdo/compression/generic.py,sha256=tF3UhLJbUDcovLxpIzgQRxFSjZ5Fz3uDRy9kI4mFntQ,2
 lgdo/compression/radware.py,sha256=GcNTtjuyL7VBBqziUBmSqNXuhqy1bJJgvcyvyumPtrc,23839
 lgdo/compression/utils.py,sha256=W2RkBrxPpXlat84dnU9Ad7d_tTws0irtGl7O1dNWjnk,1140
 lgdo/compression/varlen.py,sha256=6ZZUItyoOfygDdE0DyoISeFZfqdbH6xl7T0eclfarzg,15127
-lgdo/lh5/__init__.py,sha256=y1XE_mpFWwamrl7WVjAVSVB25X4PrEfdVXSneSQEmlQ,825
+lgdo/lh5/__init__.py,sha256=UTzKGmpgFoHwVB_yNULvJsHD_uQQGl-R87l-3QBkh7w,773
 lgdo/lh5/concat.py,sha256=BZCgK7TWPKK8fMmha8K83d3bC31FVO1b5LOW7x-Ru1s,6186
-lgdo/lh5/core.py,sha256=GjosZGUp4GSO5FtWV9eXUt_6DGU_OwJXODlj5K1j93M,13320
+lgdo/lh5/core.py,sha256=nULH5UoRjUCH0E3Z0-OH_DbFz2PRAQP73Qaf1kfnyPE,13481
 lgdo/lh5/datatype.py,sha256=O_7BqOlX8PFMyG0ppkfUT5aps5HEqX0bpuKcJO3jhu0,1691
 lgdo/lh5/exceptions.py,sha256=3kj8avXl4eBGvebl3LG12gJEmw91W0T8PYR0AfvUAyM,1211
 lgdo/lh5/iterator.py,sha256=1ob9B7Bf3ioGCtZkUZoL6ibTxAwLf4ld8_33ghVVEa4,20498
-lgdo/lh5/store.py,sha256=MYbMt-Mc7izELxuyLlSrrYrylCIzxc2CLzZYIVbZ33w,8455
-lgdo/lh5/tools.py,sha256=T9CgHA8A3_tVBMtiNJ6hATQKhdqI61m3cX4p2wGKc6c,9937
-lgdo/lh5/utils.py,sha256=ioz8DlyXZsejwnU2qYdIccdHcF12H62jgLkZsiDOLSM,6243
+lgdo/lh5/store.py,sha256=qkBm3gPbr1R2UlQpUuDR5sGRMzpYJBWFL8fDIry6tmQ,8474
+lgdo/lh5/tools.py,sha256=drtJWHY82wCFuFr6LVVnm2AQgs_wZuFmAvyOB4tcOHs,6431
+lgdo/lh5/utils.py,sha256=D5w-3fRLIuN971pMi3RttJZVVmaFJzt25Gfyyp6TUfc,6331
 lgdo/lh5/_serializers/__init__.py,sha256=eZzxMp1SeZWG0PkEXUiCz3XyprQ8EmelHUmJogC8xYE,1263
 lgdo/lh5/_serializers/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lgdo/lh5/_serializers/read/array.py,sha256=uWfMCihfAmW2DE2ewip2qCK_kvQC_mb2zvOv26uzijc,1000
@@ -33,11 +32,11 @@ lgdo/lh5/_serializers/read/utils.py,sha256=YfSqPO-83A1XvhhuULxQ0Qz2A5ODa3sb7ApNx
 lgdo/lh5/_serializers/read/vector_of_vectors.py,sha256=765P8mElGArAaEPkHTAUXFQ47t1_3-3BQAete0LckBQ,7207
 lgdo/lh5/_serializers/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lgdo/lh5/_serializers/write/array.py,sha256=66DKnW2yqIBlUGNBPWcE-m4W0B2-nTKusDHGX9m6GY0,3223
-lgdo/lh5/_serializers/write/composite.py,sha256=I6lH0nWFIpAfZyG4-0rLxzg3mfazZ_FEhQVp1FZ0aA4,9254
+lgdo/lh5/_serializers/write/composite.py,sha256=JYoLT9intT_Y4xPeL_l7CSd22O0ZKyEmd0flKkWWPFA,9268
 lgdo/lh5/_serializers/write/scalar.py,sha256=JPt_fcdTKOSFp5hfJdcKIfK4hxhcD8vhOlvDF-7btQ8,763
 lgdo/lh5/_serializers/write/vector_of_vectors.py,sha256=puGQX9XF5P_5DVbm_Cc6TvPrsDywgBLSYtkqFNltbB4,3493
 lgdo/types/__init__.py,sha256=DNfOErPiAZg-7Gygkp6ZKAi20Yrm1mfderZHvKo1Y4s,821
-lgdo/types/array.py,sha256=e3p93yrfzSmyBgWdGqqtETcKpM7_FxENaAErru15rvo,8904
+lgdo/types/array.py,sha256=vxViJScqKw4zGUrrIOuuU_9Y0oTfOkEEhs0TOyUYjwI,9284
 lgdo/types/arrayofequalsizedarrays.py,sha256=DOGJiTmc1QCdm7vLbE6uIRXoMPtt8uuCfmwQawgWf5s,4949
 lgdo/types/encoded.py,sha256=_e8u_BPfpjJbLnEdyTo9QG3kbNsGj0BN4gjdj3L1ndw,15640
 lgdo/types/fixedsizearray.py,sha256=7RjUwTz1bW0pcrdy27JlfrXPAuOU89Kj7pOuSUCojK8,1527
@@ -46,11 +45,11 @@ lgdo/types/lgdo.py,sha256=21YNtJCHnSO3M60rjsAdbMO5crDjL_0BtuFpudZ2xvU,4500
 lgdo/types/scalar.py,sha256=c5Es2vyDqyWTPV6mujzfIzMpC1jNWkEIcvYyWQUxH3Q,1933
 lgdo/types/struct.py,sha256=Q0OWLVd4B0ciLb8t6VsxU3MPbmGLZ7WfQNno1lSQS0Q,4918
 lgdo/types/table.py,sha256=FkWesoEA9bmGGSW8Ewig1Zs77ffUoR_nggfYSmkWpjU,20079
-lgdo/types/vectorofvectors.py,sha256=CtPR2WDBmJmzzfXwH4aUcNMB5LvTiGWmL_qRbFah3to,24756
-lgdo/types/vovutils.py,sha256=WjvPLEJrRNjktnbyfypfgxZX-K_aOvcwPygfzoknsyA,10701
-lgdo/types/waveformtable.py,sha256=f2tS4f1OEoYaTM5ldCX9zmw8iSISCT3t3wS1SrPdu_o,9901
-legend_pydataobj-1.12.0a1.dist-info/METADATA,sha256=55pMph32j8h4LKGnoVEdvHX27bHr8k__sdT4L9O5dIA,44445
-legend_pydataobj-1.12.0a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-legend_pydataobj-1.12.0a1.dist-info/entry_points.txt,sha256=0KWfnwbuwhNn0vPUqARukjp04Ca6lzfZBSirouRmk7I,76
-legend_pydataobj-1.12.0a1.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
-legend_pydataobj-1.12.0a1.dist-info/RECORD,,
+lgdo/types/vectorofvectors.py,sha256=GbAKV_ehXN4XdWSwnmKS_ErCiudRetcH_3wo7iDrVjw,26854
+lgdo/types/vovutils.py,sha256=LW3ZcwECxVYxxcFadAtY3nnK-9-rk8Xbg_m8hY30lo4,10708
+lgdo/types/waveformtable.py,sha256=9S_NMg894NZTGt2pLuskwH4-zQ5EbLnzWI6FVui6fXE,9827
+legend_pydataobj-1.12.0a3.dist-info/METADATA,sha256=6KWUi7oveauluZsDiYwncWRg5ix2LHmzOeomT_Or1TI,44445
+legend_pydataobj-1.12.0a3.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+legend_pydataobj-1.12.0a3.dist-info/entry_points.txt,sha256=0KWfnwbuwhNn0vPUqARukjp04Ca6lzfZBSirouRmk7I,76
+legend_pydataobj-1.12.0a3.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
+legend_pydataobj-1.12.0a3.dist-info/RECORD,,

{legend_pydataobj-1.12.0a1.dist-info → legend_pydataobj-1.12.0a3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

lgdo/__init__.py

@@ -45,7 +45,7 @@ browsed easily in python like any `HDF5 <https://www.hdfgroup.org>`_ file using
 from __future__ import annotations
 
 from ._version import version as __version__
-from .lh5_store import LH5Iterator, LH5Store, load_dfs, load_nda, ls, show
+from .lh5 import LH5Iterator, ls, read, read_as, read_n_rows, show, write
 from .types import (
     LGDO,
     Array,
@@ -69,7 +69,6 @@ __all__ = [
     "FixedSizeArray",
     "Histogram",
     "LH5Iterator",
-    "LH5Store",
     "Scalar",
     "Struct",
     "Table",
@@ -77,8 +76,10 @@ __all__ = [
     "VectorOfVectors",
     "WaveformTable",
     "__version__",
-    "load_dfs",
-    "load_nda",
     "ls",
+    "read",
+    "read_as",
+    "read_n_rows",
     "show",
+    "write",
 ]

lgdo/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.12.0a1'
-__version_tuple__ = version_tuple = (1, 12, 0)
+__version__ = version = '1.12.0a3'
+__version_tuple__ = version_tuple = (1, 12, 0, 'a3')

lgdo/lh5/__init__.py

@@ -11,7 +11,7 @@ from ._serializers.write.array import DEFAULT_HDF5_SETTINGS
 from .core import read, read_as, write
 from .iterator import LH5Iterator
 from .store import LH5Store
-from .tools import load_dfs, load_nda, ls, show
+from .tools import ls, show
 from .utils import read_n_rows
 
 __all__ = [
@@ -19,8 +19,6 @@ __all__ = [
     "LH5Iterator",
     "LH5Store",
     "concat",
-    "load_dfs",
-    "load_nda",
     "ls",
     "read",
     "read_as",

lgdo/lh5/_serializers/write/composite.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import logging
-import os
 from inspect import signature
+from pathlib import Path
 
 import h5py
 
@@ -53,7 +53,7 @@ def _h5_write_lgdo(
     # change any object in the file. So we use file:append for
     # write_object:overwrite.
     if not isinstance(lh5_file, h5py.File):
-        mode = "w" if wo_mode == "of" or not os.path.exists(lh5_file) else "a"
+        mode = "w" if wo_mode == "of" or not Path(lh5_file).exists() else "a"
         lh5_file = h5py.File(lh5_file, mode=mode, **file_kwargs)
 
     log.debug(

lgdo/lh5/core.py

@@ -113,7 +113,11 @@ def read(
         lh5_obj = lh5_file[name]
     elif isinstance(lh5_file, str):
         lh5_file = h5py.File(lh5_file, mode="r", locking=locking)
-        lh5_obj = lh5_file[name]
+        try:
+            lh5_obj = lh5_file[name]
+        except KeyError as ke:
+            err = f"Object {name} not found in file {lh5_file.filename}"
+            raise KeyError(err) from ke
     else:
         if obj_buf is not None:
             obj_buf.resize(obj_buf_start)

lgdo/lh5/store.py

@@ -6,11 +6,11 @@ HDF5 files.
 from __future__ import annotations
 
 import logging
-import os
 import sys
 from collections import OrderedDict
 from collections.abc import Mapping, Sequence
 from inspect import signature
+from pathlib import Path
 from typing import Any
 
 import h5py
@@ -92,16 +92,16 @@ class LH5Store:
             return self.files[lh5_file]
 
         if self.base_path != "":
-            full_path = os.path.join(self.base_path, lh5_file)
+            full_path = Path(self.base_path) / lh5_file
         else:
-            full_path = lh5_file
+            full_path = Path(lh5_file)
 
-        file_exists = os.path.exists(full_path)
+        file_exists = full_path.exists()
         if mode != "r":
-            directory = os.path.dirname(full_path)
-            if directory != "" and not os.path.exists(directory):
+            directory = full_path.parent
+            if directory != "" and not full_path.parent.exists():
                 log.debug(f"making path {directory}")
-                os.makedirs(directory)
+                directory.mkdir(parents=True, exist_ok=True)
 
         if mode == "r" and not file_exists:
             msg = f"file {full_path} not found"

lgdo/lh5/tools.py

@@ -1,16 +1,10 @@
 from __future__ import annotations
 
 import fnmatch
-import glob
 import logging
-import os
 from copy import copy
-from warnings import warn
 
 import h5py
-import numpy as np
-import pandas as pd
-from numpy.typing import NDArray
 
 from . import utils
 from .store import LH5Store
@@ -223,108 +217,3 @@ def show(
             break
 
         key = k_new
-
-
-def load_nda(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[NDArray | list | tuple] | None = None,
-) -> dict[str, NDArray]:
-    r"""Build a dictionary of :class:`numpy.ndarray`\ s from LH5 data.
-
-    Given a list of files, a list of LH5 table parameters, and an optional
-    group path, return a NumPy array with all values for each parameter.
-
-    Parameters
-    ----------
-    f_list
-        A list of files. Can contain wildcards.
-    par_list
-        A list of parameters to read from each file.
-    lh5_group
-        group path within which to find the specified parameters.
-    idx_list
-        for fancy-indexed reads. Must be one index array for each file in
-        `f_list`.
-
-    Returns
-    -------
-    par_data
-        A dictionary of the parameter data keyed by the elements of `par_list`.
-        Each entry contains the data for the specified parameter concatenated
-        over all files in `f_list`.
-    """
-    warn(
-        "load_nda() is deprecated. "
-        "Please replace it with LH5Store.read(...).view_as('np'), "
-        "or just read_as(..., 'np'). "
-        "load_nda() will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-
-    if isinstance(f_list, str):
-        f_list = [f_list]
-        if idx_list is not None:
-            idx_list = [idx_list]
-    if idx_list is not None and len(f_list) != len(idx_list):
-        msg = f"f_list length ({len(f_list)}) != idx_list length ({len(idx_list)})!"
-        raise ValueError(msg)
-
-    # Expand wildcards
-    f_list = [f for f_wc in f_list for f in sorted(glob.glob(os.path.expandvars(f_wc)))]
-
-    sto = LH5Store()
-    par_data = {par: [] for par in par_list}
-    for ii, ff in enumerate(f_list):
-        f = sto.gimme_file(ff, "r")
-        for par in par_list:
-            if f"{lh5_group}/{par}" not in f:
-                msg = f"'{lh5_group}/{par}' not in file {ff}"
-                raise RuntimeError(msg)
-
-            if idx_list is None:
-                data, _ = sto.read(f"{lh5_group}/{par}", f)
-            else:
-                data, _ = sto.read(f"{lh5_group}/{par}", f, idx=idx_list[ii])
-            if not data:
-                continue
-            par_data[par].append(data.nda)
-    return {par: np.concatenate(par_data[par]) for par in par_list}
-
-
-def load_dfs(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[NDArray | list | tuple] | None = None,
-) -> pd.DataFrame:
-    """Build a :class:`pandas.DataFrame` from LH5 data.
-
-    Given a list of files (can use wildcards), a list of LH5 columns, and
-    optionally the group path, return a :class:`pandas.DataFrame` with all
-    values for each parameter.
-
-    See Also
-    --------
-    :func:`load_nda`
-
-    Returns
-    -------
-    dataframe
-        contains columns for each parameter in `par_list`, and rows containing
-        all data for the associated parameters concatenated over all files in
-        `f_list`.
-    """
-    warn(
-        "load_dfs() is deprecated. "
-        "Please replace it with LH5Store.read(...).view_as('pd'), "
-        "or just read_as(..., 'pd'). "
-        "load_dfs() will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return pd.DataFrame(
-        load_nda(f_list, par_list, lh5_group=lh5_group, idx_list=idx_list)
-    )

lgdo/lh5/utils.py

@@ -7,6 +7,7 @@ import logging
 import os
 import string
 from collections.abc import Mapping, Sequence
+from pathlib import Path
 from typing import Any
 
 import h5py
@@ -183,14 +184,15 @@ def expand_path(
         Unique absolute path, or list of all absolute paths
     """
     if base_path is not None and base_path != "":
-        base_path = os.path.expanduser(os.path.expandvars(base_path))
-        path = os.path.join(base_path, path)
+        base_path = Path(os.path.expandvars(base_path)).expanduser()
+        path = base_path / path
 
     # first expand variables
     _path = expand_vars(path, substitute)
 
     # then expand wildcards
-    paths = sorted(glob.glob(os.path.expanduser(_path)))
+    # pathlib glob works differently so use glob for now
+    paths = sorted(glob.glob(str(Path(_path).expanduser())))  # noqa: PTH207
 
     if base_path is not None and base_path != "":
         paths = [os.path.relpath(p, base_path) for p in paths]

lgdo/types/array.py

@@ -6,7 +6,7 @@ corresponding utilities.
 from __future__ import annotations
 
 import logging
-from collections.abc import Iterator
+from collections.abc import Collection, Iterator
 from typing import Any
 
 import awkward as ak
@@ -126,19 +126,27 @@ class Array(LGDOCollection):
         "Set capacity to be minimum needed to support Array size"
         self.reserve_capacity(np.prod(self.shape))
 
-    def resize(self, new_size: int, trim=False) -> None:
+    def resize(self, new_size: int | Collection[int], trim=False) -> None:
         """Set size of Array in rows. Only change capacity if it must be
         increased to accommodate new rows; in this case double capacity.
-        If trim is True, capacity will be set to match size."""
+        If trim is True, capacity will be set to match size. If new_size
+        is an int, do not change size of inner dimensions.
 
-        self._size = new_size
+        If new_size is a collection, internal memory will be re-allocated, so
+        this should be done only rarely!"""
 
-        if trim and new_size != self.get_capacity:
-            self.reserve_capacity(new_size)
+        if isinstance(new_size, Collection):
+            self._size = new_size[0]
+            self._nda.resize(new_size)
+        else:
+            self._size = new_size
+
+            if trim and new_size != self.get_capacity:
+                self.reserve_capacity(new_size)
 
-        # If capacity is not big enough, set to next power of 2 big enough
-        if new_size > self.get_capacity():
-            self.reserve_capacity(int(2 ** (np.ceil(np.log2(new_size)))))
+            # If capacity is not big enough, set to next power of 2 big enough
+            if new_size > self.get_capacity():
+                self.reserve_capacity(int(2 ** (np.ceil(np.log2(new_size)))))
 
     def append(self, value: np.ndarray) -> None:
         "Append value to end of array (with copy)"

lgdo/types/vectorofvectors.py

@@ -130,20 +130,48 @@ class VectorOfVectors(LGDOCollection):
 
             # ak.to_buffer helps in de-serialization
             # NOTE: ak.to_packed() needed?
-            form, length, container = ak.to_buffers(ak.to_packed(data))
-
-            # NOTE: node#-data is not even in the dict if the awkward array is empty
-            # NOTE: if the data arg was a numpy array, to_buffers() preserves
-            # the original dtype
-            # FIXME: have to copy the buffers, otherwise self will not own the
-            # data and self.resize() will fail. Is it possible to avoid this?
-            flattened_data = np.copy(
-                container.pop(f"node{data.ndim - 1}-data", np.empty(0, dtype=dtype))
-            )
+            form, _, container = ak.to_buffers(ak.to_packed(data))
+
+            # check if bytestring
+            curr = form
+            for _ in range(data.ndim - 1):
+                curr = curr.content
+            if (
+                "__array__" in curr.parameters
+                and curr.parameters["__array__"] == "bytestring"
+            ):
+                diffs = np.diff(container[f"node{data.ndim - 1}-offsets"])
+                if (diffs != diffs[0]).all():
+                    err_msg = "Non uniform string lengths not supported"
+                    raise NotImplementedError(err_msg)
+                flattened_data = np.asarray(
+                    ak.enforce_type(
+                        ak.unflatten(
+                            container.pop(
+                                f"node{data.ndim}-data", np.empty(0, dtype=dtype)
+                            ),
+                            diffs[0],
+                        ),
+                        "bytes",
+                    )
+                )
 
-            # if user-provided dtype is different than dtype from Awkward, cast
-            # NOTE: makes a copy only if needed
-            flattened_data = np.asarray(flattened_data, dtype=dtype)
+                # if user-provided dtype is different than dtype from Awkward, cast
+                # NOTE: makes a copy only if needed
+                flattened_data = np.asarray(flattened_data, dtype=dtype)
+            else:
+                # NOTE: node#-data is not even in the dict if the awkward array is empty
+                # NOTE: if the data arg was a numpy array, to_buffers() preserves
+                # the original dtype
+                # FIXME: have to copy the buffers, otherwise self will not own the
+                # data and self.resize() will fail. Is it possible to avoid this?
+                flattened_data = np.copy(
+                    container.pop(f"node{data.ndim - 1}-data", np.empty(0, dtype=dtype))
+                )
+
+                # if user-provided dtype is different than dtype from Awkward, cast
+                # NOTE: makes a copy only if needed
+                flattened_data = np.asarray(flattened_data, dtype=dtype)
 
             # start from innermost VoV and build nested structure
             for i in range(data.ndim - 2, -1, -1):
@@ -476,7 +504,10 @@ class VectorOfVectors(LGDOCollection):
             else:
                 nan_val = np.nan
             vovutils._nb_fill(
-                vec, lens, nan_val, self.flattened_data.nda[start : cum_lens[-1]]
+                vec,
+                lens,
+                np.array([nan_val]).astype(self.flattened_data.nda.dtype),
+                self.flattened_data.nda[start : cum_lens[-1]],
             )
 
             # add new vector(s) length to cumulative_length
@@ -627,11 +658,25 @@ class VectorOfVectors(LGDOCollection):
             offsets[1:] = self.cumulative_length.nda
             offsets[0] = 0
 
-            content = (
-                ak.contents.NumpyArray(self.flattened_data.nda)
-                if self.ndim == 2
-                else self.flattened_data.view_as(library, with_units=with_units).layout
-            )
+            if self.ndim != 2:
+                content = self.flattened_data.view_as(
+                    library, with_units=with_units
+                ).layout
+            # need to handle strings separately
+            elif np.issubdtype(self.flattened_data.nda.dtype, np.bytes_):
+                byte_arrays = []
+                for s in self.flattened_data.nda:
+                    # Convert each string to array of bytes
+                    byte_array = np.frombuffer(s, dtype=np.uint8)
+                    byte_arrays.append(byte_array)
+                max_len = max(len(b) for b in byte_arrays)
+                raw_arrays = ak.contents.NumpyArray(np.concatenate(byte_arrays))
+                array_of_chars = ak.contents.RegularArray(
+                    raw_arrays, max_len, parameters={"__array__": "bytes"}
+                )
+                content = ak.enforce_type(array_of_chars, "bytes", highlevel=False)
+            else:
+                content = ak.contents.NumpyArray(self.flattened_data.nda)
 
             layout = ak.contents.ListOffsetArray(
                 offsets=ak.index.Index(offsets),

lgdo/types/vovutils.py

@@ -131,7 +131,7 @@ def _nb_fill(
     for i, ll in enumerate(len_in):
         stop = start + ll
         if ll > max_len:
-            flattened_array_out[start : start + max_len] = aoa_in[i, :]
+            flattened_array_out[start : start + max_len] = aoa_in[i, :max_len]
             flattened_array_out[start + max_len : stop] = nan_val
         else:
             flattened_array_out[start:stop] = aoa_in[i, :ll]

lgdo/types/waveformtable.py

@@ -112,12 +112,10 @@ class WaveformTable(Table):
         if not isinstance(t0, Array):
             shape = (size,)
             t0_dtype = t0.dtype if hasattr(t0, "dtype") else np.float32
-            nda = (
-                t0 if isinstance(t0, np.ndarray) else np.full(shape, t0, dtype=t0_dtype)
-            )
-            if nda.shape != shape:
-                nda.resize(shape, refcheck=True)
-            t0 = Array(nda=nda)
+            if isinstance(t0, np.ndarray):
+                t0 = Array(nda=t0, shape=shape, dtype=t0_dtype)
+            else:
+                t0 = Array(fill_val=t0, shape=shape, dtype=t0_dtype)
 
         if t0_units is not None:
             t0.attrs["units"] = f"{t0_units}"
@@ -125,12 +123,11 @@ class WaveformTable(Table):
         if not isinstance(dt, Array):
             shape = (size,)
             dt_dtype = dt.dtype if hasattr(dt, "dtype") else np.float32
-            nda = (
-                dt if isinstance(dt, np.ndarray) else np.full(shape, dt, dtype=dt_dtype)
-            )
-            if nda.shape != shape:
-                nda.resize(shape, refcheck=True)
-            dt = Array(nda=nda)
+            if isinstance(dt, np.ndarray):
+                dt = Array(nda=dt, shape=shape, dtype=dt_dtype)
+            else:
+                dt = Array(fill_val=dt, shape=shape, dtype=dt_dtype)
+
         if dt_units is not None:
             dt.attrs["units"] = f"{dt_units}"
 
@@ -174,14 +171,15 @@ class WaveformTable(Table):
                         if hasattr(values, "dtype")
                         else np.dtype(np.float64)
                     )
-                nda = (
-                    values
-                    if isinstance(values, np.ndarray)
-                    else np.zeros(shape, dtype=dtype)
-                )
-                if nda.shape != shape:
-                    nda.resize(shape, refcheck=True)
-                values = ArrayOfEqualSizedArrays(dims=(1, 1), nda=nda)
+                if isinstance(values, np.ndarray):
+                    values = ArrayOfEqualSizedArrays(
+                        dims=(1, 1), nda=values, shape=shape, dtype=dtype
+                    )
+                else:
+                    values = ArrayOfEqualSizedArrays(
+                        dims=(1, 1), fill_val=0, shape=shape, dtype=dtype
+                    )
+
         if values_units is not None:
             values.attrs["units"] = f"{values_units}"
 
@@ -215,7 +213,7 @@ class WaveformTable(Table):
             return
         shape = self.values.nda.shape
         shape = (shape[0], wf_len)
-        self.values.nda.resize(shape, refcheck=True)
+        self.values.resize(shape)
 
     def resize_wf_len(self, new_len: int) -> None:
         """Alias for `wf_len.setter`, for when we want to make it clear in

lgdo/lh5_store.py

@@ -1,284 +0,0 @@
-"""
-.. warning::
-    This subpackage is deprecated, use :mod:`lgdo.lh5`.
-"""
-
-from __future__ import annotations
-
-import sys
-from collections.abc import Iterator
-from typing import Union
-from warnings import warn
-
-import h5py
-import numpy as np
-import pandas as pd
-
-from . import lh5
-from .types import (
-    Array,
-    ArrayOfEncodedEqualSizedArrays,  # noqa: F401
-    ArrayOfEqualSizedArrays,  # noqa: F401
-    FixedSizeArray,  # noqa: F401
-    Histogram,  # noqa: F401
-    Scalar,
-    Struct,
-    Table,  # noqa: F401
-    VectorOfEncodedVectors,  # noqa: F401
-    VectorOfVectors,
-    WaveformTable,  # noqa: F401
-)
-
-LGDO = Union[Array, Scalar, Struct, VectorOfVectors]
-
-
-class LH5Iterator(lh5.LH5Iterator):
-    """
-    .. warning::
-        This class is deprecated, use :class:`lgdo.lh5.iterator.LH5Iterator`.
-
-    """
-
-    def __init__(
-        self,
-        lh5_files: str | list[str],
-        groups: str | list[str],
-        base_path: str = "",
-        entry_list: list[int] | list[list[int]] | None = None,
-        entry_mask: list[bool] | list[list[bool]] | None = None,
-        field_mask: dict[str, bool] | list[str] | tuple[str] | None = None,
-        buffer_len: int = 3200,
-        friend: Iterator | None = None,
-    ) -> None:
-        warn(
-            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Iterator."
-            "Please replace 'from lgdo.lh5_store import LH5Iterator' with 'from lgdo.lh5 import LH5Iterator'."
-            "lgdo.lh5_store will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        super().__init__(
-            lh5_files,
-            groups,
-            base_path,
-            entry_list,
-            entry_mask,
-            field_mask,
-            buffer_len,
-            friend,
-        )
-
-    def write_object(
-        self,
-        obj: LGDO,
-        name: str,
-        lh5_file: str | h5py.File,
-        group: str | h5py.Group = "/",
-        start_row: int = 0,
-        n_rows: int | None = None,
-        wo_mode: str = "append",
-        write_start: int = 0,
-        **h5py_kwargs,
-    ) -> None:
-        """
-        .. warning::
-            This method is deprecated, use :meth:`lgdo.lh5.iterator.LH5Iterator.write`.
-
-        """
-        warn(
-            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Iterator. "
-            "The object you are calling this function from uses the old LH5Iterator class."
-            "Please replace 'from lgdo.lh5_store import LH5Iterator' with 'from lgdo.lh5 import LH5Iterator'."
-            "lgdo.lh5_store will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        self.write(
-            obj,
-            name,
-            lh5_file,
-            group,
-            start_row,
-            n_rows,
-            wo_mode,
-            write_start,
-            h5py_kwargs,
-        )
-
-    def read_object(
-        self,
-        name: str,
-        lh5_file: str | h5py.File | list[str | h5py.File],
-        start_row: int = 0,
-        n_rows: int = sys.maxsize,
-        idx: np.ndarray | list | tuple | list[np.ndarray | list | tuple] = None,
-        field_mask: dict[str, bool] | list[str] | tuple[str] | None = None,
-        obj_buf: LGDO = None,
-        obj_buf_start: int = 0,
-        decompress: bool = True,
-    ) -> tuple[LGDO, int]:
-        """
-        .. warning::
-            This method is deprecated, use :meth:`lgdo.lh5.iterator.LH5Iterator.read`.
-
-        """
-        warn(
-            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Iterator. "
-            "The object you are calling this function from uses the old LH5Iterator class."
-            "Please replace 'from lgdo.lh5_store import LH5Iterator' with 'from lgdo.lh5 import LH5Iterator'."
-            "lgdo.lh5_store will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return self.read(
-            name,
-            lh5_file,
-            start_row,
-            n_rows,
-            idx,
-            field_mask,
-            obj_buf,
-            obj_buf_start,
-            decompress,
-        )
-
-
-class LH5Store(lh5.LH5Store):
-    """
-    .. warning::
-        This class is deprecated, use :class:`lgdo.lh5.iterator.LH5Store`.
-
-    """
-
-    def __init__(self, base_path: str = "", keep_open: bool = False):
-        warn(
-            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Store. "
-            "Please replace 'from lgdo.lh5_store import LH5Store' with 'from lgdo.lh5 import LH5Store'."
-            "lgdo.lh5_store will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        super().__init__(base_path, keep_open)
-
-    def read_object(
-        self,
-        name: str,
-        lh5_file: str | h5py.File | list[str | h5py.File],
-        **kwargs,
-    ) -> tuple[LGDO, int]:
-        """
-        .. warning::
-            This method is deprecated, use :meth:`lgdo.lh5.store.LH5Store.read`.
-
-        """
-        warn(
-            "LH5Store.read_object() has been renamed to LH5Store.read(), "
-            "Please update your code."
-            "LH5Store.read_object() will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return super().read(self, name, lh5_file, **kwargs)
-
-    def write_object(
-        self,
-        obj: LGDO,
-        name: str,
-        lh5_file: str | h5py.File,
-        **kwargs,
-    ) -> tuple[LGDO, int]:
-        """
-        .. warning::
-            This method is deprecated, use :meth:`lgdo.lh5.store.LH5Store.write`.
-
-        """
-        warn(
-            "LH5Store.write_object() has been renamed to LH5Store.write(), "
-            "Please update your code."
-            "LH5Store.write_object() will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return super().read(self, obj, name, lh5_file, **kwargs)
-
-
-def load_dfs(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[np.ndarray | list | tuple] | None = None,
-) -> pd.DataFrame:
-    """
-    .. warning::
-        This function is deprecated, use :meth:`lgdo.types.lgdo.LGDO.view_as` to
-        view LGDO data as a Pandas data structure.
-
-    """
-    warn(
-        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
-        "Please replace 'from lgdo.lh5_store import load_dfs' with 'from lgdo.lh5 import load_dfs'. "
-        "lgdo.lh5_store will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return lh5.load_dfs(f_list, par_list, lh5_group, idx_list)
-
-
-def load_nda(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[np.ndarray | list | tuple] | None = None,
-) -> dict[str, np.ndarray]:
-    """
-    .. warning::
-        This function is deprecated, use :meth:`lgdo.types.lgdo.LGDO.view_as` to
-        view LGDO data as a NumPy data structure.
-
-    """
-    warn(
-        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
-        "Please replace 'from lgdo.lh5_store import load_nda' with 'from lgdo.lh5 import load_nda'. "
-        "lgdo.lh5_store will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return lh5.load_nda(f_list, par_list, lh5_group, idx_list)
-
-
-def ls(lh5_file: str | h5py.Group, lh5_group: str = "") -> list[str]:
-    """
-    .. warning::
-        This function is deprecated, import :func:`lgdo.lh5.tools.ls`.
-
-    """
-    warn(
-        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
-        "Please replace 'from lgdo.lh5_store import ls' with 'from lgdo.lh5 import ls'. "
-        "lgdo.lh5_store will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return lh5.ls(lh5_file, lh5_group)
-
-
-def show(
-    lh5_file: str | h5py.Group,
-    lh5_group: str = "/",
-    attrs: bool = False,
-    indent: str = "",
-    header: bool = True,
-) -> None:
-    """
-    .. warning::
-        This function is deprecated, import :func:`lgdo.lh5.tools.show`.
-
-    """
-    warn(
-        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
-        "Please replace 'from lgdo.lh5_store import show' with 'from lgdo.lh5 import show'. "
-        "lgdo.lh5_store will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    lh5.show(lh5_file, lh5_group, attrs, indent, header)