legend-pydataobj 1.0.0__py3-none-any.whl

legend_pydataobj-1.0.0.dist-info/METADATA

@@ -0,0 +1,63 @@
+Metadata-Version: 2.1
+Name: legend-pydataobj
+Version: 1.0.0
+Summary: LEGEND Python Data Objects
+Home-page: https://github.com/legend-exp/legend-pydataobj
+Author: The LEGEND Collaboration
+Maintainer: The LEGEND Collaboration
+License: GPL-3.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Classifier: Topic :: Scientific/Engineering :: Physics
+Classifier: Topic :: Software Development
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: h5py (>=3.2.0)
+Requires-Dist: hdf5plugin
+Requires-Dist: numba (!=0.53.*,!=0.54.*)
+Requires-Dist: numexpr
+Requires-Dist: numpy (>=1.21)
+Requires-Dist: pandas (>=1.4.4)
+Requires-Dist: parse
+Requires-Dist: pint
+Provides-Extra: all
+Requires-Dist: legend-pydataobj[docs,test] ; extra == 'all'
+Provides-Extra: docs
+Requires-Dist: furo ; extra == 'docs'
+Requires-Dist: jupyter ; extra == 'docs'
+Requires-Dist: myst-parser ; extra == 'docs'
+Requires-Dist: nbsphinx ; extra == 'docs'
+Requires-Dist: sphinx ; extra == 'docs'
+Requires-Dist: sphinx-copybutton ; extra == 'docs'
+Requires-Dist: sphinx-inline-tabs ; extra == 'docs'
+Provides-Extra: test
+Requires-Dist: pre-commit ; extra == 'test'
+Requires-Dist: pylegendtestdata ; extra == 'test'
+Requires-Dist: pytest (>=6.0) ; extra == 'test'
+Requires-Dist: pytest-cov ; extra == 'test'
+
+# legend-pydataobj
+
+[![PyPI](https://img.shields.io/pypi/v/legend-pydataobj?logo=pypi)](https://pypi.org/project/legend-pydataobj/)
+![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/legend-exp/legend-pydataobj?logo=git)
+[![GitHub Workflow Status](https://img.shields.io/github/checks-status/legend-exp/legend-pydataobj/main?label=main%20branch&logo=github)](https://github.com/legend-exp/legend-pydataobj/actions)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Codecov](https://img.shields.io/codecov/c/github/legend-exp/legend-pydataobj?logo=codecov)](https://app.codecov.io/gh/legend-exp/legend-pydataobj)
+![GitHub issues](https://img.shields.io/github/issues/legend-exp/legend-pydataobj?logo=github)
+![GitHub pull requests](https://img.shields.io/github/issues-pr/legend-exp/legend-pydataobj?logo=github)
+![License](https://img.shields.io/github/license/legend-exp/legend-pydataobj)
+[![Read the Docs](https://img.shields.io/readthedocs/legend-pydataobj?logo=readthedocs)](https://legend-pydataobj.readthedocs.io)

legend_pydataobj-1.0.0.dist-info/RECORD

@@ -0,0 +1,26 @@
+lgdo/__init__.py,sha256=mw2Xnk3eLo9W-jpOIkh8Xi9x_r4piK_ZC7r-H0OzDyE,2826
+lgdo/_version.py,sha256=FrtOfsGtY3HPkXwopVTLnBRPrKexJ-VIfXoRzWX47O4,160
+lgdo/lgdo_utils.py,sha256=i6zG4r5wnv2WHXfWv232C_ZxaET556Rft8315QnTTiQ,5596
+lgdo/lh5_store.py,sha256=ozql8fFtfL5qpOte3B_i9LscDOKLz75b9JHasPYU6qQ,68252
+lgdo/compression/__init__.py,sha256=oT9OXiDDxC7BZciWrQVfHZNkOxXfj4p8EpF2tF04w84,1091
+lgdo/compression/base.py,sha256=82cQJujfvoAOKBFx761dEcx_xM02TBCBBuBo6i78tuI,838
+lgdo/compression/generic.py,sha256=zWoZqR5H4x-xpbvBYKDZ0rnNqUh9pC9aZeoc8Z98OjI,2252
+lgdo/compression/radware.py,sha256=ZfdebK_N1r1t2YVqqBu5B6RN87OoN9_deeiR97mP0Zo,19312
+lgdo/compression/utils.py,sha256=YhxfJ_zvDu9I1BnjOsALR7SYbNohZNdm8HZ8IqTjd6w,1060
+lgdo/compression/varlen.py,sha256=SJtfRfP5T_YM4YsyuAXXZgsRtSps2_ERBE985_Eli3w,14497
+lgdo/types/__init__.py,sha256=5V2V_e8zLb7wmOGwnkSeuQBN_0yPq6zFyo06gBHa3_c,736
+lgdo/types/array.py,sha256=Z0RyxW3iXCKlnppN97hLde6Yoj5OFL_teOg5IhCmz68,4489
+lgdo/types/arrayofequalsizedarrays.py,sha256=vq8kjt7ckztX65duRkcSZTaFxz8AtdIRIvs4D9wmcCU,4492
+lgdo/types/encoded.py,sha256=M1Z4dKZ4Dq5VkB03-jYmK_jDEfrVi8O0u2w4lZaWHA0,11302
+lgdo/types/fixedsizearray.py,sha256=zDhj9GXcwPBcnwhlh3RTrJIdATJVZ-5If39PUWBy9mw,1250
+lgdo/types/lgdo.py,sha256=COil-sFRsUWe9jXN5PdOVF3w0h6KNw-CVnzFW91kYXM,1515
+lgdo/types/scalar.py,sha256=UnSfVqXOJyAMwInccWXFWpXKRyjMlpfgbMTTGJm1EWE,1651
+lgdo/types/struct.py,sha256=UxV0wnCHoQM5rSmzEC9EIKWYV6drHVyK5Ab7UQztuj4,2984
+lgdo/types/table.py,sha256=h5E4Keu-a3l25WXf7m_hm2Yr4RRzGjS_WCQ_Pla7SAI,12531
+lgdo/types/vectorofvectors.py,sha256=1oxKJDX8VVWpmvUUDHHEzEYw0RRWJrMjOB-jHRY12N4,21859
+lgdo/types/waveform_table.py,sha256=52vqjGudX5_ZR1-b087jx3vuTxJ_yEPO-dO8Dpi0ceg,9407
+legend_pydataobj-1.0.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+legend_pydataobj-1.0.0.dist-info/METADATA,sha256=75XcT2_CMrBJoZCYOu3gohPrpvPl6wwmhtLmcA2uYI4,3219
+legend_pydataobj-1.0.0.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+legend_pydataobj-1.0.0.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
+legend_pydataobj-1.0.0.dist-info/RECORD,,

legend_pydataobj-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.40.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

legend_pydataobj-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+lgdo

lgdo/__init__.py

@@ -0,0 +1,75 @@
+"""
+LEGEND Data Objects (LGDO) are defined in the `LEGEND data format specification
+<https://github.com/legend-exp/legend-data-format-specs>`_.  This package
+serves as the Python implementation of that specification. The general strategy
+for the implementation is to dress standard Python and NumPy objects with an
+``attr`` dictionary holding LGDO metadata, plus some convenience functions. The
+basic data object classes are:
+
+* :class:`.LGDO`: abstract base class for all LGDOs
+* :class:`.Scalar`: typed Python scalar. Access data via the :attr:`value`
+  attribute
+* :class:`.Array`: basic :class:`numpy.ndarray`. Access data via the
+  :attr:`nda` attribute.
+* :class:`.FixedSizeArray`: basic :class:`numpy.ndarray`. Access data via the
+  :attr:`nda` attribute.
+* :class:`.ArrayOfEqualSizedArrays`: multi-dimensional :class:`numpy.ndarray`.
+  Access data via the :attr:`nda` attribute.
+* :class:`.VectorOfVectors`: a variable length array of variable length arrays.
+  Implemented as a pair of :class:`.Array`: :attr:`flattened_data` holding the
+  raw data, and :attr:`cumulative_length` whose ith element is the sum of the
+  lengths of the vectors with ``index <= i``
+* :class:`.VectorOfEncodedVectors`: an array of variable length *encoded*
+  arrays. Implemented as a :class:`.VectorOfVectors` :attr:`encoded_data`
+  holding the encoded vectors and an :class:`.Array` :attr:`decoded_size`
+  specifying the size of each decoded vector. Mainly used to represent a list
+  of compressed waveforms.
+* :class:`.ArrayOfEncodedEqualSizedArrays`: an array of equal sized encoded
+  arrays. Similar to :class:`.VectorOfEncodedVectors` except for
+  :attr:`decoded_size`, which is now a scalar.
+* :class:`.Struct`: a dictionary containing LGDO objects. Derives from
+  :class:`dict`
+* :class:`.Table`: a :class:`.Struct` whose elements ("columns") are all array
+  types with the same length (number of rows)
+
+Currently the primary on-disk format for LGDO object is LEGEND HDF5 (LH5) files. IO
+is done via the class :class:`.lh5_store.LH5Store`. LH5 files can also be
+browsed easily in python like any `HDF5 <https://www.hdfgroup.org>`_ file using
+`h5py <https://www.h5py.org>`_.
+"""
+
+from .lh5_store import LH5Iterator, LH5Store, load_dfs, load_nda, ls, show
+from .types import (
+    LGDO,
+    Array,
+    ArrayOfEncodedEqualSizedArrays,
+    ArrayOfEqualSizedArrays,
+    FixedSizeArray,
+    Scalar,
+    Struct,
+    Table,
+    VectorOfEncodedVectors,
+    VectorOfVectors,
+    WaveformTable,
+)
+
+__all__ = [
+    "Array",
+    "ArrayOfEqualSizedArrays",
+    "ArrayOfEncodedEqualSizedArrays",
+    "FixedSizeArray",
+    "LGDO",
+    "Scalar",
+    "Struct",
+    "Table",
+    "VectorOfVectors",
+    "VectorOfEncodedVectors",
+    "WaveformTable",
+    "LH5Iterator",
+    "LH5Store",
+    "load_dfs",
+    "load_nda",
+    "ls",
+    "show",
+    "copy",
+]

lgdo/_version.py

@@ -0,0 +1,4 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+__version__ = version = '1.0.0'
+__version_tuple__ = version_tuple = (1, 0, 0)

lgdo/compression/__init__.py

@@ -0,0 +1,36 @@
+r"""Data compression utilities.
+
+This subpackage collects all LEGEND custom data compression (encoding) and
+decompression (decoding) algorithms.
+
+Available lossless waveform compression algorithms:
+
+* :class:`.RadwareSigcompress`, a Python port of the C algorithm
+  `radware-sigcompress` by D. Radford.
+* :class:`.ULEB128ZigZagDiff` variable-length base-128 encoding of waveform
+  differences.
+
+All waveform compression algorithms inherit from the :class:`.WaveformCodec`
+abstract class.
+
+:func:`~.generic.encode` and :func:`~.generic.decode` provide a high-level
+interface for encoding/decoding :class:`~.lgdo.LGDO`\ s.
+
+>>> from lgdo import WaveformTable, compression
+>>> wftbl = WaveformTable(...)
+>>> enc_wft = compression.encode(wftable, RadwareSigcompress(codec_shift=-23768)
+>>> compression.decode(enc_wft) # == wftbl
+"""
+
+from .base import WaveformCodec
+from .generic import decode, encode
+from .radware import RadwareSigcompress
+from .varlen import ULEB128ZigZagDiff
+
+__all__ = [
+    "WaveformCodec",
+    "encode",
+    "decode",
+    "RadwareSigcompress",
+    "ULEB128ZigZagDiff",
+]

lgdo/compression/base.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import re
+from dataclasses import asdict, dataclass
+
+
+@dataclass(frozen=True)
+class WaveformCodec:
+    """Base class identifying a waveform compression algorithm.
+
+    The `self.codec` property returns a string identifier suitable for labeling
+    encoded data on disk. This identifier is constant for all class instances.
+
+    Note
+    ----
+    This is an abstract type. The user must provided a concrete subclass.
+    """
+
+    @property
+    def codec(self):
+        """The waveform codec string identifier.
+
+        Will be attached as an attribute to the encoded Waveform values.
+        """
+        return re.sub("(?<!^)(?=[A-Z])", "_", type(self).__name__).lower()
+
+    def asdict(self):
+        """Return the dataclass fields as dictionary."""
+        return {"codec": self.codec} | asdict(self)

lgdo/compression/generic.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import logging
+
+from .. import types as lgdo
+from . import radware, varlen
+from .base import WaveformCodec
+
+log = logging.getLogger(__name__)
+
+
+def encode(
+    obj: lgdo.VectorOfVectors | lgdo.ArrayOfEqualsizedArrays,
+    codec: WaveformCodec | str = None,
+) -> lgdo.VectorOfEncodedVectors | lgdo.ArrayOfEncodedEqualSizedArrays:
+    """Encode LGDOs with `codec`.
+
+    Defines behaviors for each implemented waveform encoding algorithm.
+
+    Parameters
+    ----------
+    obj
+        LGDO array type.
+    codec
+        algorithm to be used for encoding.
+    """
+    log.debug(f"encoding {repr(obj)} with {codec}")
+
+    if _is_codec(codec, radware.RadwareSigcompress):
+        enc_obj = radware.encode(obj, shift=codec.codec_shift)
+    elif _is_codec(codec, varlen.ULEB128ZigZagDiff):
+        enc_obj = varlen.encode(obj)
+    else:
+        raise ValueError(f"'{codec}' not supported")
+
+    enc_obj.attrs |= codec.asdict()
+
+    return enc_obj
+
+
+def decode(
+    obj: lgdo.VectorOfEncodedVectors | lgdo.ArrayOfEncodedEqualSizedArrays,
+) -> lgdo.VectorOfVectors | lgdo.ArrayOfEqualsizedArrays:
+    """Decode encoded LGDOs.
+
+    Defines decoding behaviors for each implemented waveform encoding
+    algorithm. Expects to find the codec (and its parameters) the arrays where
+    encoded with among the LGDO attributes.
+
+    Parameters
+    ----------
+    obj
+        LGDO array type.
+    """
+    if "codec" not in obj.attrs:
+        raise RuntimeError(
+            "object does not carry any 'codec' attribute, I don't know how to decode it"
+        )
+
+    codec = obj.attrs["codec"]
+    log.debug(f"decoding {repr(obj)} with {codec}")
+
+    if _is_codec(codec, radware.RadwareSigcompress):
+        return radware.decode(obj, shift=int(obj.attrs.get("codec_shift", 0)))
+    elif _is_codec(codec, varlen.ULEB128ZigZagDiff):
+        return varlen.decode(obj)
+    else:
+        raise ValueError(f"'{codec}' not supported")
+
+
+def _is_codec(ident: WaveformCodec | str, codec) -> bool:
+    if isinstance(ident, WaveformCodec):
+        return isinstance(ident, codec)
+    elif isinstance(ident, str):
+        return ident == codec().codec
+    else:
+        raise ValueError("input must be WaveformCodec object or string identifier")