legend-pydataobj 1.11.7__py3-none-any.whl → 1.12.0a2__py3-none-any.whl

lgdo/types/vectorofvectors.py

@@ -20,12 +20,12 @@ from .. import utils
 from . import arrayofequalsizedarrays as aoesa
 from . import vovutils
 from .array import Array
-from .lgdo import LGDO
+from .lgdo import LGDOCollection
 
 log = logging.getLogger(__name__)
 
 
-class VectorOfVectors(LGDO):
+class VectorOfVectors(LGDOCollection):
     """A n-dimensional variable-length 1D array of variable-length 1D arrays.
 
     If the vector is 2-dimensional, the internal representation is as two NumPy
@@ -210,20 +210,17 @@ class VectorOfVectors(LGDO):
             elif self.flattened_data is None:
                 self.flattened_data = flattened_data
 
-            # finally set dtype
-            self.dtype = self.flattened_data.dtype
-
-        # set ndim
-        self.ndim = 2
-        pointer = self.flattened_data
-        while True:
-            if isinstance(pointer, Array):
-                break
+        super().__init__(attrs)
 
-            self.ndim += 1
-            pointer = pointer.flattened_data
+    @property
+    def ndim(self):
+        return 1 + (
+            1 if isinstance(self.flattened_data, Array) else self.flattened_data.ndim
+        )
 
-        super().__init__(attrs)
+    @property
+    def dtype(self) -> np.dtype:
+        return self.flattened_data.dtype
 
     def datatype_name(self) -> str:
         return "array"
@@ -276,7 +273,30 @@ class VectorOfVectors(LGDO):
         else:
             raise NotImplementedError
 
-    def resize(self, new_size: int) -> None:
+    def reserve_capacity(self, cap_cl, *cap_args) -> None:
+        """Set capacity of internal data arrays. Expect number of args to
+        equal `self.n_dim`. First arg is capacity of cumulative length array.
+        If `self.n_dim` is 2, second argument is capacity of flattened data,
+        otherwise arguments are fed recursively to remaining dimensions.
+        """
+        self.cumulative_length.reserve_capacity(cap_cl)
+        self.flattened_data.reserve_capacity(*cap_args)
+
+    def get_capacity(self) -> tuple[int]:
+        """Get tuple containing capacity of each dimension. First dimension
+        is cumulative length array. Last dimension is flattened data.
+        """
+        fd_cap = self.flattened_data.get_capacity()
+        if isinstance(fd_cap, int):
+            return (self.cumulative_length.get_capacity(), fd_cap)
+        return (self.cumulative_length.get_capacity(), *fd_cap)
+
+    def trim_capacity(self) -> None:
+        "Set capacity for all dimensions to minimum needed to hold data"
+        self.cumulative_length.trim_capacity()
+        self.flattened_data.trim_capacity()
+
+    def resize(self, new_size: int, trim: bool = False) -> None:
         """Resize vector along the first axis.
 
         `self.flattened_data` is resized only if `new_size` is smaller than the
@@ -286,6 +306,8 @@ class VectorOfVectors(LGDO):
         `self.cumulative_length` is padded with its last element.  This
         corresponds to appending empty vectors.
 
+        If `trim` is ``True``, resize capacity to match new size
+
         Examples
         --------
         >>> vov = VectorOfVectors([[1, 2, 3], [4, 5]])
@@ -303,23 +325,22 @@ class VectorOfVectors(LGDO):
          [3],
         ]
         """
-        vidx = self.cumulative_length
         old_s = len(self)
-        dlen = new_size - old_s
-        csum = vidx[-1] if len(self) > 0 else 0
 
         # first resize the cumulative length
-        self.cumulative_length.resize(new_size)
+        self.cumulative_length.resize(new_size, trim)
 
         # if new_size > size, new elements are filled with zeros, let's fix
         # that
-        if dlen > 0:
-            self.cumulative_length[old_s:] = csum
+        if new_size > old_s:
+            self.cumulative_length[old_s:] = self.cumulative_length[old_s - 1]
 
         # then resize the data array
         # if dlen > 0 this has no effect
         if len(self.cumulative_length) > 0:
-            self.flattened_data.resize(self.cumulative_length[-1])
+            self.flattened_data.resize(self.cumulative_length[-1], trim)
+        else:
+            self.flattened_data.resize(0, trim)
 
     def append(self, new: NDArray) -> None:
         """Append a 1D vector `new` at the end.
@@ -334,20 +355,7 @@ class VectorOfVectors(LGDO):
          [8 9],
         ]
         """
-        if self.ndim == 2:
-            # first extend cumulative_length by +1
-            self.cumulative_length.resize(len(self) + 1)
-            # set it at the right value
-            newlen = (
-                self.cumulative_length[-2] + len(new) if len(self) > 1 else len(new)
-            )
-            self.cumulative_length[-1] = newlen
-            # then resize flattened_data to accommodate the new vector
-            self.flattened_data.resize(len(self.flattened_data) + len(new))
-            # finally set it
-            self[-1] = new
-        else:
-            raise NotImplementedError
+        self.insert(len(self), new)
 
     def insert(self, i: int, new: NDArray) -> None:
         """Insert a vector at index `i`.
@@ -364,23 +372,15 @@ class VectorOfVectors(LGDO):
          [8 9],
          [4 5],
         ]
-
-        Warning
-        -------
-        This method involves a significant amount of memory re-allocation and
-        is expected to perform poorly on large vectors.
         """
         if self.ndim == 2:
-            if i >= len(self):
-                msg = f"index {i} is out of bounds for vector owith size {len(self)}"
+            if i > len(self):
+                msg = f"index {i} is out of bounds for vector with size {len(self)}"
                 raise IndexError(msg)
 
-            self.flattened_data = Array(
-                np.insert(self.flattened_data, self.cumulative_length[i - 1], new)
-            )
-            self.cumulative_length = Array(
-                np.insert(self.cumulative_length, i, self.cumulative_length[i - 1])
-            )
+            i_start = 0 if i == 0 else self.cumulative_length[i - 1]
+            self.flattened_data.insert(i_start, new)
+            self.cumulative_length.insert(i, i_start)
             self.cumulative_length[i:] += np.uint32(len(new))
         else:
             raise NotImplementedError
@@ -400,11 +400,6 @@ class VectorOfVectors(LGDO):
         [[8 9],
          [4 5],
         ]
-
-        Warning
-        -------
-        This method involves a significant amount of memory re-allocation and
-        is expected to perform poorly on large vectors.
         """
         if self.ndim == 2:
             if i >= len(self):
@@ -414,27 +409,17 @@ class VectorOfVectors(LGDO):
             vidx = self.cumulative_length
             dlen = len(new) - len(self[i])
 
-            if dlen == 0:
-                # don't waste resources
-                self[i] = new
-            elif dlen < 0:
-                start = vidx[i - 1]
-                stop = start + len(new)
-                # set the already allocated indices
-                self.flattened_data[start:stop] = new
-                # then delete the extra indices
-                self.flattened_data = Array(
-                    np.delete(self.flattened_data, np.s_[stop : vidx[i]])
-                )
-            else:
-                # set the already allocated indices
-                self.flattened_data[vidx[i - 1] : vidx[i]] = new[: len(self[i])]
-                # then insert the remaining
-                self.flattened_data = Array(
-                    np.insert(self.flattened_data, vidx[i], new[len(self[i]) :])
-                )
-
-            vidx[i:] = vidx[i:] + dlen
+            if dlen != 0:
+                # move the subsequent entries
+                vidx[i:] += dlen
+                self.flattened_data.resize(vidx[-1])
+                self.flattened_data._nda[vidx[i] : vidx[-1]] = self.flattened_data._nda[
+                    vidx[i] - dlen : vidx[-1] - dlen
+                ]
+
+            # set the already allocated indices
+            start = vidx[i - 1] if i > 0 else 0
+            self.flattened_data[start : vidx[i]] = new
         else:
             raise NotImplementedError
 
@@ -484,7 +469,18 @@ class VectorOfVectors(LGDO):
             cum_lens = np.add(start, lens.cumsum(), dtype=int)
 
             # fill with fast vectorized routine
-            vovutils._nb_fill(vec, lens, self.flattened_data.nda[start : cum_lens[-1]])
+            if np.issubdtype(self.flattened_data.dtype, np.unsignedinteger):
+                nan_val = np.iinfo(self.flattened_data.dtype).max
+            if np.issubdtype(self.flattened_data.dtype, np.integer):
+                nan_val = np.iinfo(self.flattened_data.dtype).min
+            else:
+                nan_val = np.nan
+            vovutils._nb_fill(
+                vec,
+                lens,
+                np.array([nan_val]).astype(self.flattened_data.nda.dtype),
+                self.flattened_data.nda[start : cum_lens[-1]],
+            )
 
             # add new vector(s) length to cumulative_length
             self.cumulative_length[i : i + len(lens)] = cum_lens

lgdo/types/vovutils.py

@@ -81,7 +81,7 @@ def _nb_build_cl(sorted_array_in: NDArray, cumulative_length_out: NDArray) -> ND
 
 @numba.guvectorize(
     [
-        f"{data_type}[:,:],{size_type}[:],{data_type}[:]"
+        f"{data_type}[:,:],{size_type}[:],{data_type},{data_type}[:]"
         for data_type in [
             "b1",
             "i1",
@@ -99,10 +99,12 @@ def _nb_build_cl(sorted_array_in: NDArray, cumulative_length_out: NDArray) -> ND
         ]
         for size_type in ["i4", "i8", "u4", "u8"]
     ],
-    "(l,m),(l),(n)",
+    "(l,m),(l),(),(n)",
     **nb_kwargs,
 )
-def _nb_fill(aoa_in: NDArray, len_in: NDArray, flattened_array_out: NDArray):
+def _nb_fill(
+    aoa_in: NDArray, len_in: NDArray, nan_val: int | float, flattened_array_out: NDArray
+):
     """Vectorized function to fill flattened array from array of arrays and
     lengths. Values in aoa_in past lengths will not be copied.
 
@@ -112,6 +114,9 @@ def _nb_fill(aoa_in: NDArray, len_in: NDArray, flattened_array_out: NDArray):
         array of arrays containing values to be copied
     len_in
         array of vector lengths for each row of aoa_in
+    nan_val
+        value to use when len_in is longer than aoa_in. Should use
+        np.nan for floating point, and 0xfff... for integer types
     flattened_array_out
         flattened array to copy values into. Must be longer than sum of
         lengths in len_in
@@ -122,9 +127,14 @@ def _nb_fill(aoa_in: NDArray, len_in: NDArray, flattened_array_out: NDArray):
         raise ValueError(msg)
 
     start = 0
+    max_len = aoa_in.shape[1]
     for i, ll in enumerate(len_in):
         stop = start + ll
-        flattened_array_out[start:stop] = aoa_in[i, :ll]
+        if ll > max_len:
+            flattened_array_out[start : start + max_len] = aoa_in[i, :max_len]
+            flattened_array_out[start + max_len : stop] = nan_val
+        else:
+            flattened_array_out[start:stop] = aoa_in[i, :ll]
         start = stop
 
 

lgdo/types/waveformtable.py

@@ -112,12 +112,10 @@ class WaveformTable(Table):
         if not isinstance(t0, Array):
             shape = (size,)
             t0_dtype = t0.dtype if hasattr(t0, "dtype") else np.float32
-            nda = (
-                t0 if isinstance(t0, np.ndarray) else np.full(shape, t0, dtype=t0_dtype)
-            )
-            if nda.shape != shape:
-                nda.resize(shape, refcheck=True)
-            t0 = Array(nda=nda)
+            if isinstance(t0, np.ndarray):
+                t0 = Array(nda=t0, shape=shape, dtype=t0_dtype)
+            else:
+                t0 = Array(fill_val=t0, shape=shape, dtype=t0_dtype)
 
         if t0_units is not None:
             t0.attrs["units"] = f"{t0_units}"
@@ -125,12 +123,11 @@ class WaveformTable(Table):
         if not isinstance(dt, Array):
             shape = (size,)
             dt_dtype = dt.dtype if hasattr(dt, "dtype") else np.float32
-            nda = (
-                dt if isinstance(dt, np.ndarray) else np.full(shape, dt, dtype=dt_dtype)
-            )
-            if nda.shape != shape:
-                nda.resize(shape, refcheck=True)
-            dt = Array(nda=nda)
+            if isinstance(dt, np.ndarray):
+                dt = Array(nda=dt, shape=shape, dtype=dt_dtype)
+            else:
+                dt = Array(fill_val=dt, shape=shape, dtype=dt_dtype)
+
         if dt_units is not None:
             dt.attrs["units"] = f"{dt_units}"
 
@@ -174,14 +171,15 @@ class WaveformTable(Table):
                         if hasattr(values, "dtype")
                         else np.dtype(np.float64)
                     )
-                nda = (
-                    values
-                    if isinstance(values, np.ndarray)
-                    else np.zeros(shape, dtype=dtype)
-                )
-                if nda.shape != shape:
-                    nda.resize(shape, refcheck=True)
-                values = ArrayOfEqualSizedArrays(dims=(1, 1), nda=nda)
+                if isinstance(values, np.ndarray):
+                    values = ArrayOfEqualSizedArrays(
+                        dims=(1, 1), nda=values, shape=shape, dtype=dtype
+                    )
+                else:
+                    values = ArrayOfEqualSizedArrays(
+                        dims=(1, 1), fill_val=0, shape=shape, dtype=dtype
+                    )
+
         if values_units is not None:
             values.attrs["units"] = f"{values_units}"
 
@@ -215,7 +213,7 @@ class WaveformTable(Table):
             return
         shape = self.values.nda.shape
         shape = (shape[0], wf_len)
-        self.values.nda.resize(shape, refcheck=True)
+        self.values.resize(shape)
 
     def resize_wf_len(self, new_len: int) -> None:
         """Alias for `wf_len.setter`, for when we want to make it clear in

lgdo/lh5_store.py

@@ -1,284 +0,0 @@
-"""
-.. warning::
-    This subpackage is deprecated, use :mod:`lgdo.lh5`.
-"""
-
-from __future__ import annotations
-
-import sys
-from collections.abc import Iterator
-from typing import Union
-from warnings import warn
-
-import h5py
-import numpy as np
-import pandas as pd
-
-from . import lh5
-from .types import (
-    Array,
-    ArrayOfEncodedEqualSizedArrays,  # noqa: F401
-    ArrayOfEqualSizedArrays,  # noqa: F401
-    FixedSizeArray,  # noqa: F401
-    Histogram,  # noqa: F401
-    Scalar,
-    Struct,
-    Table,  # noqa: F401
-    VectorOfEncodedVectors,  # noqa: F401
-    VectorOfVectors,
-    WaveformTable,  # noqa: F401
-)
-
-LGDO = Union[Array, Scalar, Struct, VectorOfVectors]
-
-
-class LH5Iterator(lh5.LH5Iterator):
-    """
-    .. warning::
-        This class is deprecated, use :class:`lgdo.lh5.iterator.LH5Iterator`.
-
-    """
-
-    def __init__(
-        self,
-        lh5_files: str | list[str],
-        groups: str | list[str],
-        base_path: str = "",
-        entry_list: list[int] | list[list[int]] | None = None,
-        entry_mask: list[bool] | list[list[bool]] | None = None,
-        field_mask: dict[str, bool] | list[str] | tuple[str] | None = None,
-        buffer_len: int = 3200,
-        friend: Iterator | None = None,
-    ) -> None:
-        warn(
-            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Iterator."
-            "Please replace 'from lgdo.lh5_store import LH5Iterator' with 'from lgdo.lh5 import LH5Iterator'."
-            "lgdo.lh5_store will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        super().__init__(
-            lh5_files,
-            groups,
-            base_path,
-            entry_list,
-            entry_mask,
-            field_mask,
-            buffer_len,
-            friend,
-        )
-
-    def write_object(
-        self,
-        obj: LGDO,
-        name: str,
-        lh5_file: str | h5py.File,
-        group: str | h5py.Group = "/",
-        start_row: int = 0,
-        n_rows: int | None = None,
-        wo_mode: str = "append",
-        write_start: int = 0,
-        **h5py_kwargs,
-    ) -> None:
-        """
-        .. warning::
-            This method is deprecated, use :meth:`lgdo.lh5.iterator.LH5Iterator.write`.
-
-        """
-        warn(
-            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Iterator. "
-            "The object you are calling this function from uses the old LH5Iterator class."
-            "Please replace 'from lgdo.lh5_store import LH5Iterator' with 'from lgdo.lh5 import LH5Iterator'."
-            "lgdo.lh5_store will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        self.write(
-            obj,
-            name,
-            lh5_file,
-            group,
-            start_row,
-            n_rows,
-            wo_mode,
-            write_start,
-            h5py_kwargs,
-        )
-
-    def read_object(
-        self,
-        name: str,
-        lh5_file: str | h5py.File | list[str | h5py.File],
-        start_row: int = 0,
-        n_rows: int = sys.maxsize,
-        idx: np.ndarray | list | tuple | list[np.ndarray | list | tuple] = None,
-        field_mask: dict[str, bool] | list[str] | tuple[str] | None = None,
-        obj_buf: LGDO = None,
-        obj_buf_start: int = 0,
-        decompress: bool = True,
-    ) -> tuple[LGDO, int]:
-        """
-        .. warning::
-            This method is deprecated, use :meth:`lgdo.lh5.iterator.LH5Iterator.read`.
-
-        """
-        warn(
-            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Iterator. "
-            "The object you are calling this function from uses the old LH5Iterator class."
-            "Please replace 'from lgdo.lh5_store import LH5Iterator' with 'from lgdo.lh5 import LH5Iterator'."
-            "lgdo.lh5_store will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return self.read(
-            name,
-            lh5_file,
-            start_row,
-            n_rows,
-            idx,
-            field_mask,
-            obj_buf,
-            obj_buf_start,
-            decompress,
-        )
-
-
-class LH5Store(lh5.LH5Store):
-    """
-    .. warning::
-        This class is deprecated, use :class:`lgdo.lh5.iterator.LH5Store`.
-
-    """
-
-    def __init__(self, base_path: str = "", keep_open: bool = False):
-        warn(
-            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Store. "
-            "Please replace 'from lgdo.lh5_store import LH5Store' with 'from lgdo.lh5 import LH5Store'."
-            "lgdo.lh5_store will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        super().__init__(base_path, keep_open)
-
-    def read_object(
-        self,
-        name: str,
-        lh5_file: str | h5py.File | list[str | h5py.File],
-        **kwargs,
-    ) -> tuple[LGDO, int]:
-        """
-        .. warning::
-            This method is deprecated, use :meth:`lgdo.lh5.store.LH5Store.read`.
-
-        """
-        warn(
-            "LH5Store.read_object() has been renamed to LH5Store.read(), "
-            "Please update your code."
-            "LH5Store.read_object() will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return super().read(self, name, lh5_file, **kwargs)
-
-    def write_object(
-        self,
-        obj: LGDO,
-        name: str,
-        lh5_file: str | h5py.File,
-        **kwargs,
-    ) -> tuple[LGDO, int]:
-        """
-        .. warning::
-            This method is deprecated, use :meth:`lgdo.lh5.store.LH5Store.write`.
-
-        """
-        warn(
-            "LH5Store.write_object() has been renamed to LH5Store.write(), "
-            "Please update your code."
-            "LH5Store.write_object() will be removed in a future release.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return super().read(self, obj, name, lh5_file, **kwargs)
-
-
-def load_dfs(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[np.ndarray | list | tuple] | None = None,
-) -> pd.DataFrame:
-    """
-    .. warning::
-        This function is deprecated, use :meth:`lgdo.types.lgdo.LGDO.view_as` to
-        view LGDO data as a Pandas data structure.
-
-    """
-    warn(
-        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
-        "Please replace 'from lgdo.lh5_store import load_dfs' with 'from lgdo.lh5 import load_dfs'. "
-        "lgdo.lh5_store will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return lh5.load_dfs(f_list, par_list, lh5_group, idx_list)
-
-
-def load_nda(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[np.ndarray | list | tuple] | None = None,
-) -> dict[str, np.ndarray]:
-    """
-    .. warning::
-        This function is deprecated, use :meth:`lgdo.types.lgdo.LGDO.view_as` to
-        view LGDO data as a NumPy data structure.
-
-    """
-    warn(
-        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
-        "Please replace 'from lgdo.lh5_store import load_nda' with 'from lgdo.lh5 import load_nda'. "
-        "lgdo.lh5_store will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return lh5.load_nda(f_list, par_list, lh5_group, idx_list)
-
-
-def ls(lh5_file: str | h5py.Group, lh5_group: str = "") -> list[str]:
-    """
-    .. warning::
-        This function is deprecated, import :func:`lgdo.lh5.tools.ls`.
-
-    """
-    warn(
-        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
-        "Please replace 'from lgdo.lh5_store import ls' with 'from lgdo.lh5 import ls'. "
-        "lgdo.lh5_store will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return lh5.ls(lh5_file, lh5_group)
-
-
-def show(
-    lh5_file: str | h5py.Group,
-    lh5_group: str = "/",
-    attrs: bool = False,
-    indent: str = "",
-    header: bool = True,
-) -> None:
-    """
-    .. warning::
-        This function is deprecated, import :func:`lgdo.lh5.tools.show`.
-
-    """
-    warn(
-        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
-        "Please replace 'from lgdo.lh5_store import show' with 'from lgdo.lh5 import show'. "
-        "lgdo.lh5_store will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    lh5.show(lh5_file, lh5_group, attrs, indent, header)