legend-daq2lh5 1.0.2__tar.gz → 1.2.0a1__tar.gz

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: legend_daq2lh5
-Version: 1.0.2
+Version: 1.2.0a1
 Summary: Convert digitizer data to LH5
 Home-page: https://github.com/legend-exp/legend-daq2lh5
 Author: Jason Detwiler
@@ -26,10 +26,10 @@ Classifier: Topic :: Software Development
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dspeed~=1.1
+Requires-Dist: dspeed>=1.3.0a4
 Requires-Dist: h5py>=3.2.0
 Requires-Dist: hdf5plugin
-Requires-Dist: legend-pydataobj~=1.1
+Requires-Dist: legend-pydataobj>=1.5.0a1
 Requires-Dist: numpy>=1.21
 Requires-Dist: pyfcutils
 Requires-Dist: tqdm>=4.27
@@ -60,3 +60,16 @@ Requires-Dist: pytest-cov; extra == "test"
 ![GitHub pull requests](https://img.shields.io/github/issues-pr/legend-exp/legend-daq2lh5?logo=github)
 ![License](https://img.shields.io/github/license/legend-exp/legend-daq2lh5)
 [![Read the Docs](https://img.shields.io/readthedocs/legend-daq2lh5?logo=readthedocs)](https://legend-daq2lh5.readthedocs.io)
+
+JSON-configurable conversion of digitized data into
+[LEGEND HDF5](https://legend-exp.github.io/legend-data-format-specs/dev/hdf5/),
+with optional data pre-processing via [dspeed](https://dspeed.readthedocs.io)
+and data compression via [legend-pydataobj](https://legend-pydataobj.readthedocs.io).
+
+Currently supported DAQ data formats:
+* [FlashCam](https://www.mizzi-computer.de/home)
+* [CoMPASS](https://www.caen.it/products/compass)
+* [ORCA](https://github.com/unc-enap/Orca), reading out:
+  - FlashCam
+  - [Struck SIS3302](https://www.struck.de/sis3302.htm)
+  - [Struck SIS3316](https://www.struck.de/sis3316.html)

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/README.md

@@ -10,3 +10,16 @@
 ![GitHub pull requests](https://img.shields.io/github/issues-pr/legend-exp/legend-daq2lh5?logo=github)
 ![License](https://img.shields.io/github/license/legend-exp/legend-daq2lh5)
 [![Read the Docs](https://img.shields.io/readthedocs/legend-daq2lh5?logo=readthedocs)](https://legend-daq2lh5.readthedocs.io)
+
+JSON-configurable conversion of digitized data into
+[LEGEND HDF5](https://legend-exp.github.io/legend-data-format-specs/dev/hdf5/),
+with optional data pre-processing via [dspeed](https://dspeed.readthedocs.io)
+and data compression via [legend-pydataobj](https://legend-pydataobj.readthedocs.io).
+
+Currently supported DAQ data formats:
+* [FlashCam](https://www.mizzi-computer.de/home)
+* [CoMPASS](https://www.caen.it/products/compass)
+* [ORCA](https://github.com/unc-enap/Orca), reading out:
+  - FlashCam
+  - [Struck SIS3302](https://www.struck.de/sis3302.htm)
+  - [Struck SIS3316](https://www.struck.de/sis3316.html)

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/setup.cfg

@@ -31,10 +31,10 @@ classifiers =
 [options]
 packages = find:
 install_requires = 
-	dspeed~=1.1
+	dspeed>=1.3.0a4
 	h5py>=3.2.0
 	hdf5plugin
-	legend-pydataobj~=1.1
+	legend-pydataobj>=1.5.0a1
 	numpy>=1.21
 	pyfcutils
 	tqdm>=4.27

legend_daq2lh5-1.2.0a1/src/daq2lh5/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '1.2.0a1'
+__version_tuple__ = version_tuple = (1, 2, 0)

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/src/daq2lh5/buffer_processor/buffer_processor.py

@@ -48,10 +48,17 @@ def buffer_processor(rb: RawBuffer) -> Table:
     ``"dtype_conv": {"lgdo": "dtype" [, ...]}`` `(dict)`
       Casts `lgdo` to the requested data type.
 
-    ``"compression": { "lgdo": "codec_name" [, ...]}`` `(dict)`
+    ``"compression": {"lgdo": "codec_name" [, ...]}`` `(dict)`
       Updates the `compression` attribute of `lgdo` to `codec_name`. The
       attribute sets the compression algorithm applied by
-      :func:`~.lgdo.lh5_store.LH5Store.read_object` before writing `lgdo` to
+      :func:`~lgdo.lh5.store.LH5Store.read` before writing `lgdo` to
+      disk. Can be used to apply custom waveform compression algorithms from
+      :mod:`lgdo.compression`.
+
+    ``"hdf5_settings": {"lgdo": { <HDF5 settings> }}`` `(dict)`
+      Updates the `hdf5_settings` attribute of `lgdo`. The attribute sets the
+      HDF5 dataset options applied by
+      :func:`~lgdo.lh5.store.LH5Store.read` before writing `lgdo` to
       disk.
 
     Parameters
@@ -102,7 +109,9 @@ def buffer_processor(rb: RawBuffer) -> Table:
                 ,}
                 "compression": {
                   "windowed_waveform/values": RadwareSigcompress(codec_shift=-32768),
-                  "presummed_waveform/values": ULEB128ZigZagDiff(),
+                }
+                "hdf5_settings": {
+                  "presummed_waveform/values": {"shuffle": True, "compression": "lzf"},
                 }
               }
             },
@@ -143,7 +152,7 @@ def buffer_processor(rb: RawBuffer) -> Table:
     if "drop" in rb.proc_spec.keys():
         process_drop(rb, tmp_table)
 
-    # at last, assign compression attributes
+    # assign compression attributes
     if "compression" in rb.proc_spec.keys():
         for name, codec in rb.proc_spec["compression"].items():
             ptr = tmp_table
@@ -154,6 +163,15 @@ def buffer_processor(rb: RawBuffer) -> Table:
                 codec if isinstance(codec, WaveformCodec) else str2wfcodec(codec)
             )
 
+    # and HDF5 settings
+    if "hdf5_settings" in rb.proc_spec.keys():
+        for name, settings in rb.proc_spec["hdf5_settings"].items():
+            ptr = tmp_table
+            for word in name.split("/"):
+                ptr = ptr[word]
+
+            ptr.attrs["hdf5_settings"] = settings
+
     return tmp_table
 
 
@@ -277,7 +295,7 @@ def process_windowed_t0(t0s: Array, dts: Array, start_index: int) -> Array:
 
 
 def process_dsp(rb: RawBuffer, tmp_table: Table) -> None:
-    r"""Run a DSP processing chain.
+    r"""Run a DSP processing chain with :mod:`dspeed`.
 
     Run a provided DSP config from `rb.proc_spec` using
     :func:`.dsp.build_processing_chain`, and add specified outputs to the

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/src/daq2lh5/buffer_processor/lh5_buffer_processor.py

@@ -6,7 +6,7 @@ import os
 
 import h5py
 import lgdo
-from lgdo import LH5Store
+from lgdo import lh5
 
 from ..buffer_processor.buffer_processor import buffer_processor
 from ..raw_buffer import RawBuffer, RawBufferLibrary
@@ -54,14 +54,14 @@ def lh5_buffer_processor(
     """
 
     # Initialize the input raw file
-    raw_store = LH5Store()
+    raw_store = lh5.LH5Store()
     lh5_file = raw_store.gimme_file(lh5_raw_file_in, "r")
     if lh5_file is None:
         raise ValueError(f"input file not found: {lh5_raw_file_in}")
         return
 
     # List the groups in the raw file
-    lh5_groups = lgdo.ls(lh5_raw_file_in)
+    lh5_groups = lh5.ls(lh5_raw_file_in)
     lh5_tables = []
 
     # check if group points to raw data; sometimes 'raw' is nested, e.g g024/raw
@@ -69,21 +69,19 @@ def lh5_buffer_processor(
         # Make sure that the upper level key isn't a dataset
         if isinstance(lh5_file[tb], h5py.Dataset):
             lh5_tables.append(f"{tb}")
-        elif "raw" not in tb and lgdo.ls(lh5_file, f"{tb}/raw"):
+        elif "raw" not in tb and lh5.ls(lh5_file, f"{tb}/raw"):
             lh5_tables.append(f"{tb}/raw")
         # Look one layer deeper for a :meth:`lgdo.Table` if necessary
-        elif lgdo.ls(lh5_file, f"{tb}"):
+        elif lh5.ls(lh5_file, f"{tb}"):
             # Check to make sure that this isn't a table itself
-            maybe_table, _ = raw_store.read_object(f"{tb}", lh5_file)
+            maybe_table, _ = raw_store.read(f"{tb}", lh5_file)
             if isinstance(maybe_table, lgdo.Table):
                 lh5_tables.append(f"{tb}")
                 del maybe_table
             # otherwise, go deeper
             else:
-                for sub_table in lgdo.ls(lh5_file, f"{tb}"):
-                    maybe_table, _ = raw_store.read_object(
-                        f"{tb}/{sub_table}", lh5_file
-                    )
+                for sub_table in lh5.ls(lh5_file, f"{tb}"):
+                    maybe_table, _ = raw_store.read(f"{tb}/{sub_table}", lh5_file)
                     if isinstance(maybe_table, lgdo.Table):
                         lh5_tables.append(f"{tb}/{sub_table}")
                     del maybe_table
@@ -114,7 +112,7 @@ def lh5_buffer_processor(
 
     # Write everything in the raw file to the new file, check for proc_spec under either the group name, out_name, or the name
     for tb in lh5_tables:
-        lgdo_obj, _ = raw_store.read_object(f"{tb}", lh5_file)
+        lgdo_obj, _ = raw_store.read(f"{tb}", lh5_file)
 
         # Find the out_name.
         # If the top level group has an lgdo table in it, then the out_name is group
@@ -198,6 +196,4 @@ def lh5_buffer_processor(
                 pass
 
         # Write the (possibly processed) lgdo_obj to a file
-        raw_store.write_object(
-            lgdo_obj, out_name, lh5_file=proc_file_name, group=group_name
-        )
+        raw_store.write(lgdo_obj, out_name, lh5_file=proc_file_name, group=group_name)

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/src/daq2lh5/build_raw.py

@@ -6,10 +6,8 @@ import logging
 import os
 import time
 
-import hdf5plugin
-import lgdo
 import numpy as np
-from lgdo.lh5_store import DEFAULT_HDF5_COMPRESSION
+from lgdo import lh5
 from tqdm.auto import tqdm
 
 from .compass.compass_streamer import CompassStreamer
@@ -28,7 +26,7 @@ def build_raw(
     n_max: int = np.inf,
     overwrite: bool = False,
     compass_config_file: str = None,
-    hdf5_compression: str | dict | hdf5plugin.filters.Filter = DEFAULT_HDF5_COMPRESSION,
+    hdf5_settings: dict[str, ...] = None,
     **kwargs,
 ) -> None:
     """Convert data into LEGEND HDF5 raw-tier format.
@@ -77,12 +75,16 @@ def build_raw(
           json-shorthand for the output specification (see
           :mod:`.compass.compass_event_decoder`).
 
-    hdf5_compression
-        forwarded to :meth:`~.lgdo.lh5_store.LH5Store.write_object`.
+    hdf5_settings
+        keyword arguments (as a dict) forwarded to
+        :meth:`lgdo.lh5.store.LH5Store.write`.
 
     **kwargs
         sent to :class:`.RawBufferLibrary` generation as `kw_dict` argument.
     """
+    if hdf5_settings is None:
+        hdf5_settings = {}
+
     # convert any environment variables in in_stream so that we can check for readability
     in_stream = os.path.expandvars(in_stream)
     # later: fix if in_stream is not a file
@@ -222,8 +224,8 @@ def build_raw(
         os.remove(out_file_glob[0])
 
     # Write header data
-    lh5_store = lgdo.LH5Store(keep_open=True)
-    write_to_lh5_and_clear(header_data, lh5_store, hdf5_compression=hdf5_compression)
+    lh5_store = lh5.LH5Store(keep_open=True)
+    write_to_lh5_and_clear(header_data, lh5_store, **hdf5_settings)
 
     # Now loop through the data
     n_bytes_last = streamer.n_bytes_read
@@ -248,7 +250,7 @@ def build_raw(
         if log.getEffectiveLevel() <= logging.INFO and n_max < np.inf:
             progress_bar.update(n_read)
 
-        write_to_lh5_and_clear(chunk_list, lh5_store, hdf5_compression=hdf5_compression)
+        write_to_lh5_and_clear(chunk_list, lh5_store, **hdf5_settings)
 
         if n_max <= 0:
             log.info(f"Wrote {n_max} rows, exiting...")

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/src/daq2lh5/data_decoder.py

@@ -3,13 +3,10 @@ Base classes for decoding data into raw LGDO Tables or files
 """
 from __future__ import annotations
 
-from typing import Union
-
 import lgdo
 import numpy as np
-from lgdo import LH5Store
-
-LGDO = Union[lgdo.Scalar, lgdo.Struct, lgdo.Array, lgdo.VectorOfVectors]
+from lgdo import LGDO
+from lgdo.lh5 import LH5Store
 
 
 class DataDecoder:
@@ -18,37 +15,39 @@ class DataDecoder:
     Most decoders will repeatedly decode the same set of values from each
     packet.  The values that get decoded need to be described by a dict stored
     in `self.decoded_values` that helps determine how to set up the buffers and
-    write them to file as :class:`~.lgdo.LGDO`\ s. :class:`~.lgdo.table.Table`\ s
-    are made whose columns correspond to the elements of `decoded_values`, and
-    packet data gets pushed to the end of the table one row at a time.
+    write them to file as :class:`~lgdo.types.lgdo.LGDO`\ s.
+    :class:`~lgdo.types.table.Table`\ s are made whose columns correspond to
+    the elements of `decoded_values`, and packet data gets pushed to the end of
+    the table one row at a time.
 
     Any key-value entry in a configuration dictionary attached to an element
     of `decoded_values` is typically interpreted as an attribute to be attached
     to the corresponding LGDO. This feature can be for example exploited to
-    specify the data compression algorithm used by
-    :meth:`~.lgdo.lh5_store.LH5Store.write_object` to write LGDOs to disk.
+    specify HDF5 dataset settings used by
+    :meth:`~lgdo.lh5.store.LH5Store.write` to write LGDOs to disk.
 
     For example ::
 
       from lgdo.compression import RadwareSigcompress
 
       FCEventDecoder.decoded_values = {
-        "packet_id": {"dtype": "uint32", "compression": "gzip"},
+        "packet_id": {"dtype": "uint32", "hdf5_settings": {"compression": "gzip"}},
         # ...
         "waveform": {
           "dtype": "uint16",
           "datatype": "waveform",
           # ...
           "compression": {"values": RadwareSigcompress(codec_shift=-32768)},
+          "hdf5_settings": {"t0": {"compression": "lzf", shuffle: True}},
         }
       }
 
-    LGDOs corresponding to ``packet_id`` and ``waveform`` will have their
-    `compression` attribute set as ``"gzip"`` and
-    ``RadwareSigcompress(codec_shift=-32768)``, respectively. Before being
-    written to disk, they will compressed with the HDF5 built-in Gzip filter
-    and with the :class:`~.lgdo.compression.radware.RadwareSigcompress`
-    waveform compressor.
+    The LGDO corresponding to ``packet_id`` will have its `hdf5_settings`
+    attribute set as ``{"compression": "gzip"}``, while ``waveform.values``
+    will have its `compression` attribute set to
+    ``RadwareSigcompress(codec_shift=-32768)``.  Before being written to disk,
+    they will be compressed with the HDF5 built-in Gzip filter and with the
+    :class:`~lgdo.compression.radware.RadwareSigcompress` waveform compressor.
 
     Examples
     --------
@@ -118,7 +117,7 @@ class DataDecoder:
         """Make an LGDO for this :class:`DataDecoder` to fill.
 
         This default version of this function allocates a
-        :class:`~.lgdo.table.Table` using the `decoded_values` for key. If a
+        :class:`~lgdo.types.table.Table` using the `decoded_values` for key. If a
         different type of LGDO object is required for this decoder, overload
         this function.
 
@@ -178,7 +177,10 @@ class DataDecoder:
                 dt = attrs.pop("dt")
                 dt_units = attrs.pop("dt_units")
                 wf_len = attrs.pop("wf_len")
-                compression = attrs.pop("compression", None)
+                settings = {
+                    "compression": attrs.pop("compression", {}),
+                    "hdf5_settings": attrs.pop("hdf5_settings", {}),
+                }
 
                 wf_table = lgdo.WaveformTable(
                     size=size,
@@ -190,24 +192,20 @@ class DataDecoder:
                     dtype=dtype,
                     attrs=attrs,
                 )
-                if compression is not None:
-                    if not isinstance(compression, dict):
-                        raise RuntimeError(
-                            "waveform/compression attribute must be a dictionary"
-                        )
-
-                    if "values" in compression:
-                        wf_table.values.attrs["compression"] = compression["values"]
-                    if "t0" in compression:
-                        wf_table.t0.attrs["compression"] = compression["t0"]
-                    if "dt" in compression:
-                        wf_table.dt.attrs["compression"] = compression["dt"]
+
+                # attach compression/hdf5_settings to sub-fields
+                for el in ["values", "t0", "dt"]:
+                    for settings_name in ("hdf5_settings", "compression"):
+                        if el in settings[settings_name]:
+                            wf_table[el].attrs[settings_name] = settings[settings_name][
+                                el
+                            ]
 
                 data_obj.add_field(field, wf_table)
                 continue
 
             # Parse datatype for remaining lgdos
-            datatype, shape, elements = lgdo.lgdo_utils.parse_datatype(datatype)
+            datatype, shape, elements = lgdo.lh5.utils.parse_datatype(datatype)
 
             # ArrayOfEqualSizedArrays
             if datatype == "array_of_equalsized_arrays":
@@ -258,7 +256,7 @@ class DataDecoder:
         n_rows = self.garbage_table.loc
         if n_rows == 0:
             return
-        lh5_store.write_object(
+        lh5_store.write(
             self.garbage_table, "garbage", filename, group, n_rows=n_rows, append=True
         )
         self.garbage_table.clear()

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/src/daq2lh5/fc/fc_config_decoder.py

@@ -28,7 +28,7 @@ class FCConfigDecoder(DataDecoder):
     >>> decoder = FCConfigDecoder()
     >>> config = decoder.decode_config(fc)
     >>> type(config)
-    lgdo.struct.Struct
+    lgdo.types.struct.Struct
     """
 
     def __init__(self, *args, **kwargs) -> None:

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/src/daq2lh5/orca/orca_digitizers.py

@@ -98,11 +98,11 @@ class ORSIS3302DecoderForEnergy(OrcaDecoder):
                         sys.exit()
                     self.decoded_values[ccc]["waveform"]["wf_len"] = trace_length
 
-    def get_key_lists(self) -> list[list[str]]:
+    def get_key_lists(self) -> list[list[int]]:
         key_lists = []
         for key in self.decoded_values.keys():
             key_lists.append([key])
-        return [key_lists]
+        return key_lists
 
     def get_decoded_values(self, key: int = None) -> dict[str, Any]:
         if key is None:

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/src/daq2lh5/orca/orca_flashcam.py

@@ -326,6 +326,7 @@ class ORFlashCamListenerStatusDecoder(OrcaDecoder):
     def decode_packet(
         self, packet: OrcaPacket, packet_id: int, rbl: RawBufferLibrary
     ) -> bool:
+        return False  # FIXME: skip decoding until pyfcutils is updated
         """Decode the ORCA FlashCam Status packet."""
         # aliases for brevity
         if len(rbl) != 1:

{legend_daq2lh5-1.0.2 → legend_daq2lh5-1.2.0a1}/src/daq2lh5/orca/orca_packet.py

@@ -47,11 +47,9 @@ def hex_dump(
     as_short: bool = False,
     id_dict: dict = None,
     use_logging: bool = True,
+    return_output=False,
 ) -> None:
-    dump_cmd = print  # noqa: T202
-    if use_logging:
-        dump_cmd = log.debug
-
+    output = []
     data_id = get_data_id(packet, shift=shift_data_id)
     n_words = get_n_words(packet)
     if id_dict is not None:
@@ -62,9 +60,9 @@ def hex_dump(
     else:
         heading = f"data ID = {data_id}"
     if print_n_words:
-        dump_cmd(f"{heading}: {n_words} words")
+        output.append(f"{heading}: {n_words} words")
     else:
-        dump_cmd(f"{heading}:")
+        output.append(f"{heading}:")
         n_to_print = int(np.minimum(n_words, max_words))
         pad = int(np.ceil(np.log10(n_to_print)))
         for i in range(n_to_print):
@@ -76,4 +74,13 @@ def hex_dump(
                 line += f" {packet[i]}"
             if as_short:
                 line += f" {np.frombuffer(packet[i:i+1].tobytes(), dtype='uint16')}"
-            dump_cmd(line)
+            output.append(line)
+
+    dump_cmd = print  # noqa: T202
+    if use_logging:
+        dump_cmd = log.debug
+    for line in output:
+        dump_cmd(line)
+
+    if return_output:
+        return output