legend-pydataobj 1.11.1__tar.gz → 1.11.3__tar.gz

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: legend_pydataobj
-Version: 1.11.1
+Version: 1.11.3
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/legend_pydataobj.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: legend_pydataobj
-Version: 1.11.1
+Version: 1.11.3
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/legend_pydataobj.egg-info/SOURCES.txt

@@ -69,6 +69,7 @@ tests/compression/sigcompress/LDQTA_r117_20200110T105115Z_cal_geds_raw-0.dat
 tests/compression/sigcompress/special-wf-clipped.dat
 tests/lh5/conftest.py
 tests/lh5/test_core.py
+tests/lh5/test_exceptions.py
 tests/lh5/test_lh5_datatype.py
 tests/lh5/test_lh5_iterator.py
 tests/lh5/test_lh5_store.py

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.11.1'
-__version_tuple__ = version_tuple = (1, 11, 1)
+__version__ = version = '1.11.3'
+__version_tuple__ = version_tuple = (1, 11, 3)

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/lh5/_serializers/read/vector_of_vectors.py

@@ -189,8 +189,9 @@ def _h5_read_vector_of_vectors(
         # if the buffer is partially filled, cumulative_length will be invalid
         # (i.e. non monotonically increasing). Let's fix that but filling the
         # rest of the array with the length of flattened_data
-        end = obj_buf_start + n_rows_read
-        obj_buf.cumulative_length.nda[end:] = obj_buf.cumulative_length.nda[end - 1]
+        if n_rows_read > 0:
+            end = obj_buf_start + n_rows_read
+            obj_buf.cumulative_length.nda[end:] = obj_buf.cumulative_length.nda[end - 1]
 
         return obj_buf, n_rows_read
 

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/lh5/exceptions.py

@@ -16,6 +16,9 @@ class LH5DecodeError(Exception):
             + super().__str__()
         )
 
+    def __reduce__(self) -> tuple:  # for pickling.
+        return self.__class__, (*self.args, self.file, self.obj)
+
 
 class LH5EncodeError(Exception):
     def __init__(
@@ -32,3 +35,6 @@ class LH5EncodeError(Exception):
             f"while writing object {self.group}/{self.name} to file {self.file}: "
             + super().__str__()
         )
+
+    def __reduce__(self) -> tuple:  # for pickling.
+        return self.__class__, (*self.args, self.file, self.group, self.name)

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/types/histogram.py

@@ -418,13 +418,18 @@ class Histogram(Struct):
 
     def __setitem__(self, name: str, obj: LGDO) -> None:
         # do not allow for new attributes on this
-        msg = "histogram fields cannot be mutated"
-        raise TypeError(msg)
+        known_keys = ("binning", "weights", "isdensity")
+        if name in known_keys and not dict.__contains__(self, name):
+            # but allow initialization while unpickling (after __init__() this is unreachable)
+            dict.__setitem__(self, name, obj)
+        else:
+            msg = "histogram fields cannot be mutated "
+            raise TypeError(msg)
 
     def __getattr__(self, name: str) -> None:
         # do not allow for new attributes on this
         msg = "histogram fields cannot be mutated"
-        raise TypeError(msg)
+        raise AttributeError(msg)
 
     def add_field(self, name: str | int, obj: LGDO) -> None:  # noqa: ARG002
         """

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/types/lgdo.py

@@ -11,6 +11,12 @@ import pandas as pd
 class LGDO(ABC):
     """Abstract base class representing a LEGEND Data Object (LGDO)."""
 
+    def __new__(cls, *_args, **_kwargs):
+        # allow for (un-)pickling LGDO objects.
+        obj = super().__new__(cls)
+        obj.attrs = {}
+        return obj
+
     @abstractmethod
     def __init__(self, attrs: dict[str, Any] | None = None) -> None:
         self.attrs = {} if attrs is None else dict(attrs)

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/types/table.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import logging
 from collections.abc import Mapping
+from types import ModuleType
 from typing import Any
 from warnings import warn
 
@@ -40,6 +41,12 @@ class Table(Struct):
     :meth:`__len__` to access valid data, which returns the ``size`` attribute.
     """
 
+    def __new__(cls, *args, **kwargs):
+        # allow for (un-)pickling LGDO objects.
+        obj = super().__new__(cls, *args, **kwargs)
+        obj.size = None
+        return obj
+
     def __init__(
         self,
         col_dict: Mapping[str, LGDO] | pd.DataFrame | ak.Array | None = None,
@@ -266,6 +273,7 @@ class Table(Struct):
         self,
         expr: str,
         parameters: Mapping[str, str] | None = None,
+        modules: Mapping[str, ModuleType] | None = None,
     ) -> LGDO:
         """Apply column operations to the table and return a new LGDO.
 
@@ -299,6 +307,10 @@ class Table(Struct):
             a dictionary of function parameters. Passed to
             :func:`numexpr.evaluate`` as `local_dict` argument or to
             :func:`eval` as `locals` argument.
+        modules
+            a dictionary of additional modules used by the expression. If this is not `None`
+            then :func:`eval`is used and the expression can depend on any modules from this dictionary in
+            addition to awkward and numpy. These are passed to :func:`eval` as `globals` argument.
 
         Examples
         --------
@@ -339,8 +351,8 @@ class Table(Struct):
         msg = f"evaluating {expr!r} with locals={(self_unwrap | parameters)} and {has_ak=}"
         log.debug(msg)
 
-        # use numexpr if we are only dealing with numpy data types
-        if not has_ak:
+        # use numexpr if we are only dealing with numpy data types (and no global dictionary)
+        if not has_ak and modules is None:
             out_data = ne.evaluate(
                 expr,
                 local_dict=(self_unwrap | parameters),
@@ -366,6 +378,9 @@ class Table(Struct):
 
         # resort to good ol' eval()
         globs = {"ak": ak, "np": np}
+        if modules is not None:
+            globs = globs | modules
+
         out_data = eval(expr, globs, (self_unwrap | parameters))
 
         msg = f"...the result is {out_data!r}"
@@ -380,6 +395,10 @@ class Table(Struct):
         if np.isscalar(out_data):
             return Scalar(out_data)
 
+        # if out_data is already an LGDO just return it
+        if isinstance(out_data, LGDO):
+            return out_data
+
         msg = (
             f"evaluation resulted in a {type(out_data)} object, "
             "I don't know which LGDO this corresponds to"

legend_pydataobj-1.11.3/tests/lh5/test_exceptions.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import pickle
+
+from lgdo.lh5.exceptions import LH5DecodeError, LH5EncodeError
+
+
+def test_pickle():
+    # test (un-)pickling of LH5 exceptions; e.g. for multiprocessing use.
+
+    ex = LH5EncodeError("message", "file", "group", "name")
+    ex = pickle.loads(pickle.dumps(ex))
+    assert isinstance(ex, LH5EncodeError)
+
+    ex = LH5DecodeError("message", "file", "obj")
+    ex = pickle.loads(pickle.dumps(ex))
+    assert isinstance(ex, LH5DecodeError)

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/tests/lh5/test_lh5_store.py

@@ -292,6 +292,10 @@ def test_read_table_fancy_idx(lh5_file):
     assert isinstance(lh5_obj, types.Table)
     assert n_rows == 2
 
+    lh5_obj, n_rows = store.read("/data/struct/table", lh5_file, idx=[])
+    assert isinstance(lh5_obj, types.Table)
+    assert n_rows == 0
+
 
 def test_read_empty_struct(lh5_file):
     store = lh5.LH5Store()
@@ -468,6 +472,16 @@ def test_read_lgnd_vov_fancy_idx(lgnd_file):
     assert (lh5_obj.cumulative_length.nda == [1, 2, 3, 4, 5, 6, 7]).all()
     assert (lh5_obj.flattened_data.nda == [40, 60, 64, 60, 64, 28, 60]).all()
 
+    lh5_obj, n_rows = store.read("/geds/raw/tracelist", lgnd_file, idx=[])
+    assert isinstance(lh5_obj, types.VectorOfVectors)
+    assert n_rows == 0
+    assert len(lh5_obj) == 0
+
+    lh5_obj, n_rows = store.read("/geds/raw/tracelist", [lgnd_file] * 3, idx=[250])
+    assert isinstance(lh5_obj, types.VectorOfVectors)
+    assert n_rows == 1
+    assert len(lh5_obj) == 1
+
 
 def test_read_array_concatenation(lgnd_file):
     store = lh5.LH5Store()

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/tests/types/test_array.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pickle
+
 import awkward as ak
 import numpy as np
 import pandas as pd
@@ -61,3 +63,14 @@ def test_view():
 
     with pytest.raises(ValueError):
         a.view_as("ak", with_units=True)
+
+
+def test_pickle():
+    obj = Array(nda=np.array([1, 2, 3, 4]))
+    obj.attrs["attr1"] = 1
+
+    ex = pickle.loads(pickle.dumps(obj))
+    assert isinstance(ex, Array)
+    assert ex.attrs["attr1"] == 1
+    assert ex.attrs["datatype"] == obj.attrs["datatype"]
+    assert np.all(ex.nda == np.array([1, 2, 3, 4]))

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/tests/types/test_encoded.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pickle
+
 import awkward as ak
 import awkward_pandas as akpd
 import numpy as np
@@ -285,3 +287,51 @@ def test_aoeesa_view_as():
 
     with pytest.raises(TypeError):
         df = voev.view_as("np")
+
+
+def test_aoeesa_pickle():
+    obj = ArrayOfEncodedEqualSizedArrays(
+        encoded_data=VectorOfVectors(
+            flattened_data=Array(nda=np.array([1, 2, 3, 4, 5, 2, 4, 8, 9, 7, 5, 3, 1])),
+            cumulative_length=Array(nda=np.array([2, 5, 6, 10, 13])),
+        ),
+        decoded_size=99,
+    )
+
+    ex = pickle.loads(pickle.dumps(obj))
+
+    desired = [
+        [1, 2],
+        [3, 4, 5],
+        [2],
+        [4, 8, 9, 7],
+        [5, 3, 1],
+    ]
+
+    for i, v in enumerate(ex):
+        assert np.array_equal(v, desired[i])
+
+
+def test_voev_pickle():
+    obj = VectorOfEncodedVectors(
+        encoded_data=VectorOfVectors(
+            flattened_data=Array(nda=np.array([1, 2, 3, 4, 5, 2, 4, 8, 9, 7, 5, 3, 1])),
+            cumulative_length=Array(nda=np.array([2, 5, 6, 10, 13])),
+        ),
+        decoded_size=Array(shape=5, fill_val=6),
+        attrs={"units": "s"},
+    )
+
+    ex = pickle.loads(pickle.dumps(obj))
+
+    desired = [
+        [1, 2],
+        [3, 4, 5],
+        [2],
+        [4, 8, 9, 7],
+        [5, 3, 1],
+    ]
+
+    for i, (v, s) in enumerate(ex):
+        assert np.array_equal(v, desired[i])
+        assert s == 6

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/tests/types/test_histogram.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import pickle
 
 import hist
 import numpy as np
@@ -266,7 +267,7 @@ def test_view_as_np():
 def test_not_like_table():
     h = Histogram(np.array([1, 1]), (np.array([0, 1, 2]),))
     assert h.form_datatype() == "struct{binning,weights,isdensity}"
-    with pytest.raises(TypeError):
+    with pytest.raises(AttributeError):
         x = h.x  # noqa: F841
     with pytest.raises(TypeError):
         h["x"] = Scalar(1.0)
@@ -392,3 +393,14 @@ def test_histogram_fill():
 
     with pytest.raises(ValueError, match="data must be"):
         h.fill(np.ones(shape=(5, 5)))
+
+
+def test_pickle():
+    obj = Histogram(np.array([1, 1]), (Histogram.Axis.from_range_edges([0, 1, 2]),))
+    obj.attrs["attr1"] = 1
+
+    ex = pickle.loads(pickle.dumps(obj))
+    assert isinstance(ex, Histogram)
+    assert ex.attrs["attr1"] == 1
+    assert ex.attrs["datatype"] == obj.attrs["datatype"]
+    assert np.all(ex.weights == obj.weights)

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/tests/types/test_scalar.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pickle
+
 import pytest
 
 import lgdo
@@ -33,3 +35,14 @@ def test_getattrs():
 
 def test_equality():
     assert lgdo.Scalar(value=42) == lgdo.Scalar(value=42)
+
+
+def test_pickle():
+    obj = lgdo.Scalar(value=10)
+    obj.attrs["attr1"] = 1
+
+    ex = pickle.loads(pickle.dumps(obj))
+    assert isinstance(ex, lgdo.Scalar)
+    assert ex.attrs["attr1"] == 1
+    assert ex.attrs["datatype"] == obj.attrs["datatype"]
+    assert ex.value == 10

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/tests/types/test_struct.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pickle
+
 import pytest
 
 import lgdo
@@ -78,3 +80,15 @@ def test_remove_field():
 
     struct.remove_field("array1", delete=True)
     assert list(struct.keys()) == []
+
+
+def test_pickle():
+    obj_dict = {"scalar1": lgdo.Scalar(value=10)}
+    attrs = {"attr1": 1}
+    struct = lgdo.Struct(obj_dict=obj_dict, attrs=attrs)
+
+    ex = pickle.loads(pickle.dumps(struct))
+    assert isinstance(ex, lgdo.Struct)
+    assert ex.attrs["attr1"] == 1
+    assert ex.attrs["datatype"] == struct.attrs["datatype"]
+    assert ex["scalar1"].value == 10

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/tests/types/test_table.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import pickle
 import warnings
 
 import awkward as ak
@@ -221,3 +222,20 @@ def test_remove_column():
 
     tbl.remove_column("c")
     assert list(tbl.keys()) == ["b"]
+
+
+def test_pickle():
+    col_dict = {
+        "a": lgdo.Array(nda=np.array([1, 2, 3, 4])),
+        "b": lgdo.Array(nda=np.array([5, 6, 7, 8])),
+        "c": lgdo.Array(nda=np.array([9, 10, 11, 12])),
+    }
+    obj = Table(col_dict=col_dict)
+    obj.attrs["attr1"] = 1
+
+    ex = pickle.loads(pickle.dumps(obj))
+    assert isinstance(ex, Table)
+    assert ex.attrs["attr1"] == 1
+    assert ex.attrs["datatype"] == obj.attrs["datatype"]
+    for key, val in col_dict.items():
+        assert ex[key] == val

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/tests/types/test_table_eval.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import hist
 import numpy as np
+import pytest
 
 import lgdo
 
@@ -31,6 +33,7 @@ def test_eval_dependency():
             ),
         }
     )
+
     r = obj.eval("sum(a)")
     assert isinstance(r, lgdo.Scalar)
 
@@ -77,3 +80,11 @@ def test_eval_dependency():
     assert isinstance(r, lgdo.Scalar)
 
     assert obj.eval("np.sum(a) + ak.sum(e)")
+
+    # test with modules argument, the simplest is using directly lgdo
+    res = obj.eval("lgdo.Array([1,2,3])", {}, modules={"lgdo": lgdo})
+    assert res == lgdo.Array([1, 2, 3])
+
+    # check bad type
+    with pytest.raises(RuntimeError):
+        obj.eval("hist.Hist()", modules={"hist": hist})

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/tests/types/test_vectorofvectors.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import copy
+import pickle
 from collections import namedtuple
 
 import awkward as ak
@@ -441,3 +442,19 @@ def test_lh5_iterator_view_as(lgnd_test_data):
 
     for obj, _, _ in it:
         assert ak.is_valid(obj.view_as("ak"))
+
+
+def test_pickle(testvov):
+    obj = testvov.v2d
+    ex = pickle.loads(pickle.dumps(obj))
+
+    desired = [
+        np.array([1, 2]),
+        np.array([3, 4, 5]),
+        np.array([2]),
+        np.array([4, 8, 9, 7]),
+        np.array([5, 3, 1]),
+    ]
+
+    for i in range(len(desired)):
+        assert np.array_equal(desired[i], ex[i])

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/__init__.py

@@ -62,23 +62,23 @@ from .types import (
 )
 
 __all__ = [
+    "LGDO",
     "Array",
-    "ArrayOfEqualSizedArrays",
     "ArrayOfEncodedEqualSizedArrays",
+    "ArrayOfEqualSizedArrays",
     "FixedSizeArray",
     "Histogram",
-    "LGDO",
+    "LH5Iterator",
+    "LH5Store",
     "Scalar",
     "Struct",
     "Table",
-    "VectorOfVectors",
     "VectorOfEncodedVectors",
+    "VectorOfVectors",
     "WaveformTable",
+    "__version__",
     "load_dfs",
     "load_nda",
     "ls",
     "show",
-    "LH5Iterator",
-    "LH5Store",
-    "__version__",
 ]

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/compression/__init__.py

@@ -30,9 +30,9 @@ from .radware import RadwareSigcompress
 from .varlen import ULEB128ZigZagDiff
 
 __all__ = [
-    "WaveformCodec",
-    "encode",
-    "decode",
     "RadwareSigcompress",
     "ULEB128ZigZagDiff",
+    "WaveformCodec",
+    "decode",
+    "encode",
 ]

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/lh5/__init__.py

@@ -20,10 +20,10 @@ __all__ = [
     "LH5Store",
     "load_dfs",
     "load_nda",
+    "ls",
     "read",
-    "write",
     "read_as",
-    "ls",
     "read_n_rows",
     "show",
+    "write",
 ]

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/lh5/_serializers/__init__.py

@@ -24,22 +24,22 @@ from .write.scalar import _h5_write_scalar
 from .write.vector_of_vectors import _h5_write_vector_of_vectors
 
 __all__ = [
-    "_h5_read_lgdo",
-    "_h5_read_vector_of_vectors",
-    "_h5_read_ndarray",
     "_h5_read_array",
+    "_h5_read_array_of_encoded_equalsized_arrays",
+    "_h5_read_array_of_equalsized_arrays",
     "_h5_read_encoded_array",
     "_h5_read_fixedsize_array",
-    "_h5_read_array_of_equalsized_arrays",
-    "_h5_read_struct",
-    "_h5_read_table",
     "_h5_read_histogram",
+    "_h5_read_lgdo",
+    "_h5_read_ndarray",
     "_h5_read_scalar",
-    "_h5_read_array_of_encoded_equalsized_arrays",
+    "_h5_read_struct",
+    "_h5_read_table",
     "_h5_read_vector_of_encoded_vectors",
-    "_h5_write_scalar",
+    "_h5_read_vector_of_vectors",
     "_h5_write_array",
-    "_h5_write_vector_of_vectors",
-    "_h5_write_struct",
     "_h5_write_lgdo",
+    "_h5_write_scalar",
+    "_h5_write_struct",
+    "_h5_write_vector_of_vectors",
 ]

{legend_pydataobj-1.11.1 → legend_pydataobj-1.11.3}/src/lgdo/types/__init__.py

@@ -15,17 +15,17 @@ from .vectorofvectors import VectorOfVectors
 from .waveformtable import WaveformTable
 
 __all__ = [
+    "LGDO",
     "Array",
-    "ArrayOfEqualSizedArrays",
     "ArrayOfEncodedEqualSizedArrays",
+    "ArrayOfEqualSizedArrays",
     "FixedSizeArray",
     "Histogram",
-    "LGDO",
     "Scalar",
     "Struct",
     "Table",
-    "VectorOfVectors",
     "VectorOfEncodedVectors",
+    "VectorOfVectors",
     "WaveformTable",
 ]