legend-daq2lh5 1.0.2__py3-none-any.whl → 1.2.0a1__py3-none-any.whl

daq2lh5/_version.py

@@ -2,7 +2,15 @@
 # don't change, don't track in version control
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
 
-__version__ = version = '1.0.2'  # type: str
-__version_tuple__ = version_tuple = (1, 0, 2)  # type: Tuple[int | str, ...]
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '1.2.0a1'
+__version_tuple__ = version_tuple = (1, 2, 0)

daq2lh5/buffer_processor/buffer_processor.py

@@ -48,10 +48,17 @@ def buffer_processor(rb: RawBuffer) -> Table:
     ``"dtype_conv": {"lgdo": "dtype" [, ...]}`` `(dict)`
       Casts `lgdo` to the requested data type.
 
-    ``"compression": { "lgdo": "codec_name" [, ...]}`` `(dict)`
+    ``"compression": {"lgdo": "codec_name" [, ...]}`` `(dict)`
       Updates the `compression` attribute of `lgdo` to `codec_name`. The
       attribute sets the compression algorithm applied by
-      :func:`~.lgdo.lh5_store.LH5Store.read_object` before writing `lgdo` to
+      :func:`~lgdo.lh5.store.LH5Store.read` before writing `lgdo` to
+      disk. Can be used to apply custom waveform compression algorithms from
+      :mod:`lgdo.compression`.
+
+    ``"hdf5_settings": {"lgdo": { <HDF5 settings> }}`` `(dict)`
+      Updates the `hdf5_settings` attribute of `lgdo`. The attribute sets the
+      HDF5 dataset options applied by
+      :func:`~lgdo.lh5.store.LH5Store.read` before writing `lgdo` to
       disk.
 
     Parameters
@@ -102,7 +109,9 @@ def buffer_processor(rb: RawBuffer) -> Table:
                 ,}
                 "compression": {
                   "windowed_waveform/values": RadwareSigcompress(codec_shift=-32768),
-                  "presummed_waveform/values": ULEB128ZigZagDiff(),
+                }
+                "hdf5_settings": {
+                  "presummed_waveform/values": {"shuffle": True, "compression": "lzf"},
                 }
               }
             },
@@ -143,7 +152,7 @@ def buffer_processor(rb: RawBuffer) -> Table:
     if "drop" in rb.proc_spec.keys():
         process_drop(rb, tmp_table)
 
-    # at last, assign compression attributes
+    # assign compression attributes
     if "compression" in rb.proc_spec.keys():
         for name, codec in rb.proc_spec["compression"].items():
             ptr = tmp_table
@@ -154,6 +163,15 @@ def buffer_processor(rb: RawBuffer) -> Table:
                 codec if isinstance(codec, WaveformCodec) else str2wfcodec(codec)
             )
 
+    # and HDF5 settings
+    if "hdf5_settings" in rb.proc_spec.keys():
+        for name, settings in rb.proc_spec["hdf5_settings"].items():
+            ptr = tmp_table
+            for word in name.split("/"):
+                ptr = ptr[word]
+
+            ptr.attrs["hdf5_settings"] = settings
+
     return tmp_table
 
 
@@ -277,7 +295,7 @@ def process_windowed_t0(t0s: Array, dts: Array, start_index: int) -> Array:
 
 
 def process_dsp(rb: RawBuffer, tmp_table: Table) -> None:
-    r"""Run a DSP processing chain.
+    r"""Run a DSP processing chain with :mod:`dspeed`.
 
     Run a provided DSP config from `rb.proc_spec` using
     :func:`.dsp.build_processing_chain`, and add specified outputs to the

daq2lh5/buffer_processor/lh5_buffer_processor.py

@@ -6,7 +6,7 @@ import os
 
 import h5py
 import lgdo
-from lgdo import LH5Store
+from lgdo import lh5
 
 from ..buffer_processor.buffer_processor import buffer_processor
 from ..raw_buffer import RawBuffer, RawBufferLibrary
@@ -54,14 +54,14 @@ def lh5_buffer_processor(
     """
 
     # Initialize the input raw file
-    raw_store = LH5Store()
+    raw_store = lh5.LH5Store()
     lh5_file = raw_store.gimme_file(lh5_raw_file_in, "r")
     if lh5_file is None:
         raise ValueError(f"input file not found: {lh5_raw_file_in}")
         return
 
     # List the groups in the raw file
-    lh5_groups = lgdo.ls(lh5_raw_file_in)
+    lh5_groups = lh5.ls(lh5_raw_file_in)
     lh5_tables = []
 
     # check if group points to raw data; sometimes 'raw' is nested, e.g g024/raw
@@ -69,21 +69,19 @@ def lh5_buffer_processor(
         # Make sure that the upper level key isn't a dataset
         if isinstance(lh5_file[tb], h5py.Dataset):
             lh5_tables.append(f"{tb}")
-        elif "raw" not in tb and lgdo.ls(lh5_file, f"{tb}/raw"):
+        elif "raw" not in tb and lh5.ls(lh5_file, f"{tb}/raw"):
             lh5_tables.append(f"{tb}/raw")
         # Look one layer deeper for a :meth:`lgdo.Table` if necessary
-        elif lgdo.ls(lh5_file, f"{tb}"):
+        elif lh5.ls(lh5_file, f"{tb}"):
             # Check to make sure that this isn't a table itself
-            maybe_table, _ = raw_store.read_object(f"{tb}", lh5_file)
+            maybe_table, _ = raw_store.read(f"{tb}", lh5_file)
             if isinstance(maybe_table, lgdo.Table):
                 lh5_tables.append(f"{tb}")
                 del maybe_table
             # otherwise, go deeper
             else:
-                for sub_table in lgdo.ls(lh5_file, f"{tb}"):
-                    maybe_table, _ = raw_store.read_object(
-                        f"{tb}/{sub_table}", lh5_file
-                    )
+                for sub_table in lh5.ls(lh5_file, f"{tb}"):
+                    maybe_table, _ = raw_store.read(f"{tb}/{sub_table}", lh5_file)
                     if isinstance(maybe_table, lgdo.Table):
                         lh5_tables.append(f"{tb}/{sub_table}")
                     del maybe_table
@@ -114,7 +112,7 @@ def lh5_buffer_processor(
 
     # Write everything in the raw file to the new file, check for proc_spec under either the group name, out_name, or the name
     for tb in lh5_tables:
-        lgdo_obj, _ = raw_store.read_object(f"{tb}", lh5_file)
+        lgdo_obj, _ = raw_store.read(f"{tb}", lh5_file)
 
         # Find the out_name.
         # If the top level group has an lgdo table in it, then the out_name is group
@@ -198,6 +196,4 @@ def lh5_buffer_processor(
                 pass
 
         # Write the (possibly processed) lgdo_obj to a file
-        raw_store.write_object(
-            lgdo_obj, out_name, lh5_file=proc_file_name, group=group_name
-        )
+        raw_store.write(lgdo_obj, out_name, lh5_file=proc_file_name, group=group_name)

daq2lh5/build_raw.py

@@ -6,10 +6,8 @@ import logging
 import os
 import time
 
-import hdf5plugin
-import lgdo
 import numpy as np
-from lgdo.lh5_store import DEFAULT_HDF5_COMPRESSION
+from lgdo import lh5
 from tqdm.auto import tqdm
 
 from .compass.compass_streamer import CompassStreamer
@@ -28,7 +26,7 @@ def build_raw(
     n_max: int = np.inf,
     overwrite: bool = False,
     compass_config_file: str = None,
-    hdf5_compression: str | dict | hdf5plugin.filters.Filter = DEFAULT_HDF5_COMPRESSION,
+    hdf5_settings: dict[str, ...] = None,
     **kwargs,
 ) -> None:
     """Convert data into LEGEND HDF5 raw-tier format.
@@ -77,12 +75,16 @@ def build_raw(
           json-shorthand for the output specification (see
           :mod:`.compass.compass_event_decoder`).
 
-    hdf5_compression
-        forwarded to :meth:`~.lgdo.lh5_store.LH5Store.write_object`.
+    hdf5_settings
+        keyword arguments (as a dict) forwarded to
+        :meth:`lgdo.lh5.store.LH5Store.write`.
 
     **kwargs
         sent to :class:`.RawBufferLibrary` generation as `kw_dict` argument.
     """
+    if hdf5_settings is None:
+        hdf5_settings = {}
+
     # convert any environment variables in in_stream so that we can check for readability
     in_stream = os.path.expandvars(in_stream)
     # later: fix if in_stream is not a file
@@ -222,8 +224,8 @@ def build_raw(
         os.remove(out_file_glob[0])
 
     # Write header data
-    lh5_store = lgdo.LH5Store(keep_open=True)
-    write_to_lh5_and_clear(header_data, lh5_store, hdf5_compression=hdf5_compression)
+    lh5_store = lh5.LH5Store(keep_open=True)
+    write_to_lh5_and_clear(header_data, lh5_store, **hdf5_settings)
 
     # Now loop through the data
     n_bytes_last = streamer.n_bytes_read
@@ -248,7 +250,7 @@ def build_raw(
         if log.getEffectiveLevel() <= logging.INFO and n_max < np.inf:
             progress_bar.update(n_read)
 
-        write_to_lh5_and_clear(chunk_list, lh5_store, hdf5_compression=hdf5_compression)
+        write_to_lh5_and_clear(chunk_list, lh5_store, **hdf5_settings)
 
         if n_max <= 0:
             log.info(f"Wrote {n_max} rows, exiting...")

daq2lh5/data_decoder.py

@@ -3,13 +3,10 @@ Base classes for decoding data into raw LGDO Tables or files
 """
 from __future__ import annotations
 
-from typing import Union
-
 import lgdo
 import numpy as np
-from lgdo import LH5Store
-
-LGDO = Union[lgdo.Scalar, lgdo.Struct, lgdo.Array, lgdo.VectorOfVectors]
+from lgdo import LGDO
+from lgdo.lh5 import LH5Store
 
 
 class DataDecoder:
@@ -18,37 +15,39 @@ class DataDecoder:
     Most decoders will repeatedly decode the same set of values from each
     packet.  The values that get decoded need to be described by a dict stored
     in `self.decoded_values` that helps determine how to set up the buffers and
-    write them to file as :class:`~.lgdo.LGDO`\ s. :class:`~.lgdo.table.Table`\ s
-    are made whose columns correspond to the elements of `decoded_values`, and
-    packet data gets pushed to the end of the table one row at a time.
+    write them to file as :class:`~lgdo.types.lgdo.LGDO`\ s.
+    :class:`~lgdo.types.table.Table`\ s are made whose columns correspond to
+    the elements of `decoded_values`, and packet data gets pushed to the end of
+    the table one row at a time.
 
     Any key-value entry in a configuration dictionary attached to an element
     of `decoded_values` is typically interpreted as an attribute to be attached
     to the corresponding LGDO. This feature can be for example exploited to
-    specify the data compression algorithm used by
-    :meth:`~.lgdo.lh5_store.LH5Store.write_object` to write LGDOs to disk.
+    specify HDF5 dataset settings used by
+    :meth:`~lgdo.lh5.store.LH5Store.write` to write LGDOs to disk.
 
     For example ::
 
       from lgdo.compression import RadwareSigcompress
 
       FCEventDecoder.decoded_values = {
-        "packet_id": {"dtype": "uint32", "compression": "gzip"},
+        "packet_id": {"dtype": "uint32", "hdf5_settings": {"compression": "gzip"}},
         # ...
         "waveform": {
           "dtype": "uint16",
           "datatype": "waveform",
           # ...
           "compression": {"values": RadwareSigcompress(codec_shift=-32768)},
+          "hdf5_settings": {"t0": {"compression": "lzf", shuffle: True}},
         }
       }
 
-    LGDOs corresponding to ``packet_id`` and ``waveform`` will have their
-    `compression` attribute set as ``"gzip"`` and
-    ``RadwareSigcompress(codec_shift=-32768)``, respectively. Before being
-    written to disk, they will compressed with the HDF5 built-in Gzip filter
-    and with the :class:`~.lgdo.compression.radware.RadwareSigcompress`
-    waveform compressor.
+    The LGDO corresponding to ``packet_id`` will have its `hdf5_settings`
+    attribute set as ``{"compression": "gzip"}``, while ``waveform.values``
+    will have its `compression` attribute set to
+    ``RadwareSigcompress(codec_shift=-32768)``.  Before being written to disk,
+    they will be compressed with the HDF5 built-in Gzip filter and with the
+    :class:`~lgdo.compression.radware.RadwareSigcompress` waveform compressor.
 
     Examples
     --------
@@ -118,7 +117,7 @@ class DataDecoder:
         """Make an LGDO for this :class:`DataDecoder` to fill.
 
         This default version of this function allocates a
-        :class:`~.lgdo.table.Table` using the `decoded_values` for key. If a
+        :class:`~lgdo.types.table.Table` using the `decoded_values` for key. If a
         different type of LGDO object is required for this decoder, overload
         this function.
 
@@ -178,7 +177,10 @@ class DataDecoder:
                 dt = attrs.pop("dt")
                 dt_units = attrs.pop("dt_units")
                 wf_len = attrs.pop("wf_len")
-                compression = attrs.pop("compression", None)
+                settings = {
+                    "compression": attrs.pop("compression", {}),
+                    "hdf5_settings": attrs.pop("hdf5_settings", {}),
+                }
 
                 wf_table = lgdo.WaveformTable(
                     size=size,
@@ -190,24 +192,20 @@ class DataDecoder:
                     dtype=dtype,
                     attrs=attrs,
                 )
-                if compression is not None:
-                    if not isinstance(compression, dict):
-                        raise RuntimeError(
-                            "waveform/compression attribute must be a dictionary"
-                        )
-
-                    if "values" in compression:
-                        wf_table.values.attrs["compression"] = compression["values"]
-                    if "t0" in compression:
-                        wf_table.t0.attrs["compression"] = compression["t0"]
-                    if "dt" in compression:
-                        wf_table.dt.attrs["compression"] = compression["dt"]
+
+                # attach compression/hdf5_settings to sub-fields
+                for el in ["values", "t0", "dt"]:
+                    for settings_name in ("hdf5_settings", "compression"):
+                        if el in settings[settings_name]:
+                            wf_table[el].attrs[settings_name] = settings[settings_name][
+                                el
+                            ]
 
                 data_obj.add_field(field, wf_table)
                 continue
 
             # Parse datatype for remaining lgdos
-            datatype, shape, elements = lgdo.lgdo_utils.parse_datatype(datatype)
+            datatype, shape, elements = lgdo.lh5.utils.parse_datatype(datatype)
 
             # ArrayOfEqualSizedArrays
             if datatype == "array_of_equalsized_arrays":
@@ -258,7 +256,7 @@ class DataDecoder:
         n_rows = self.garbage_table.loc
         if n_rows == 0:
             return
-        lh5_store.write_object(
+        lh5_store.write(
             self.garbage_table, "garbage", filename, group, n_rows=n_rows, append=True
         )
         self.garbage_table.clear()

daq2lh5/fc/fc_config_decoder.py

@@ -28,7 +28,7 @@ class FCConfigDecoder(DataDecoder):
     >>> decoder = FCConfigDecoder()
     >>> config = decoder.decode_config(fc)
     >>> type(config)
-    lgdo.struct.Struct
+    lgdo.types.struct.Struct
     """
 
     def __init__(self, *args, **kwargs) -> None:

daq2lh5/orca/orca_digitizers.py

@@ -98,11 +98,11 @@ class ORSIS3302DecoderForEnergy(OrcaDecoder):
                         sys.exit()
                     self.decoded_values[ccc]["waveform"]["wf_len"] = trace_length
 
-    def get_key_lists(self) -> list[list[str]]:
+    def get_key_lists(self) -> list[list[int]]:
         key_lists = []
         for key in self.decoded_values.keys():
             key_lists.append([key])
-        return [key_lists]
+        return key_lists
 
     def get_decoded_values(self, key: int = None) -> dict[str, Any]:
         if key is None:

daq2lh5/orca/orca_flashcam.py

@@ -326,6 +326,7 @@ class ORFlashCamListenerStatusDecoder(OrcaDecoder):
     def decode_packet(
         self, packet: OrcaPacket, packet_id: int, rbl: RawBufferLibrary
     ) -> bool:
+        return False  # FIXME: skip decoding until pyfcutils is updated
         """Decode the ORCA FlashCam Status packet."""
         # aliases for brevity
         if len(rbl) != 1:

daq2lh5/orca/orca_packet.py

@@ -47,11 +47,9 @@ def hex_dump(
     as_short: bool = False,
     id_dict: dict = None,
     use_logging: bool = True,
+    return_output=False,
 ) -> None:
-    dump_cmd = print  # noqa: T202
-    if use_logging:
-        dump_cmd = log.debug
-
+    output = []
     data_id = get_data_id(packet, shift=shift_data_id)
     n_words = get_n_words(packet)
     if id_dict is not None:
@@ -62,9 +60,9 @@ def hex_dump(
     else:
         heading = f"data ID = {data_id}"
     if print_n_words:
-        dump_cmd(f"{heading}: {n_words} words")
+        output.append(f"{heading}: {n_words} words")
     else:
-        dump_cmd(f"{heading}:")
+        output.append(f"{heading}:")
         n_to_print = int(np.minimum(n_words, max_words))
         pad = int(np.ceil(np.log10(n_to_print)))
         for i in range(n_to_print):
@@ -76,4 +74,13 @@ def hex_dump(
                 line += f" {packet[i]}"
             if as_short:
                 line += f" {np.frombuffer(packet[i:i+1].tobytes(), dtype='uint16')}"
-            dump_cmd(line)
+            output.append(line)
+
+    dump_cmd = print  # noqa: T202
+    if use_logging:
+        dump_cmd = log.debug
+    for line in output:
+        dump_cmd(line)
+
+    if return_output:
+        return output

daq2lh5/orca/orca_streamer.py

@@ -32,30 +32,138 @@ class OrcaStreamer(DataStreamer):
     def __init__(self) -> None:
         super().__init__()
         self.in_stream = None
+        self.packet_locs = []
         self.buffer = np.empty(1024, dtype="uint32")  # start with a 4 kB packet buffer
         self.header = None
         self.header_decoder = OrcaHeaderDecoder()
         self.decoder_id_dict = {}  # dict of data_id to decoder object
         self.rbl_id_dict = {}  # dict of RawBufferLists for each data_id
+        self.missing_decoders = []
+
+    def load_packet_header(self) -> np.uint32 | None:
+        """Loads the packet header at the current read location into the buffer
+
+        and updates internal variables.
+        """
+        pkt_hdr = self.buffer[:1]
+        n_bytes_read = self.in_stream.readinto(pkt_hdr)  # buffer is at least 4 kB long
+        self.n_bytes_read += n_bytes_read
+        if n_bytes_read == 0:  # EOF
+            return None
+        if n_bytes_read != 4:
+            raise RuntimeError(f"only got {n_bytes_read} bytes for packet header")
+
+        # packet is valid. Can set the packet_id and log its location
+        self.packet_id += 1
+        filepos = self.in_stream.tell() - n_bytes_read
+        if self.packet_id < len(self.packet_locs):
+            if self.packet_locs[self.packet_id] != filepos:
+                raise RuntimeError(
+                    f"filepos for packet {self.packet_id} was {filepos} but {self.packet_locs[self.packet_id]} was expected"
+                )
+        else:
+            if len(self.packet_locs) != self.packet_id:
+                raise RuntimeError(
+                    f"loaded packet {self.packet_id} after packet {len(self.packet_locs)-1}"
+                )
+            self.packet_locs.append(filepos)
+
+        return pkt_hdr
+
+    def skip_packet(self, n: int = 1) -> bool:
+        """Skip a packets without loading it into the internal buffer.
+
+        Requires loading the header. Optionally skips n packets.
+
+        Returns
+        ----------
+        succeeded
+            returns False if reached EOF, otherwise returns true
+        """
+        if self.in_stream is None:
+            raise RuntimeError("self.in_stream is None")
+        if not int(n) >= 0:
+            raise ValueError(f"n must be a non-negative int, can't be {n}")
+        n = int(n)
+        while n > 0:
+            pkt_hdr = self.load_packet_header()
+            if pkt_hdr is None:
+                return False
+            self.in_stream.seek((orca_packet.get_n_words(pkt_hdr) - 1) * 4, 1)
+            n -= 1
+        return True
+
+    def build_packet_locs(self, saveloc=True) -> None:
+        loc = self.in_stream.tell()
+        pid = self.packet_id
+        if len(self.packet_locs) > 0:
+            self.in_stream.seek(self.packet_locs[-1])
+            self.packet_id = len(self.packet_locs) - 2
+        while self.skip_packet():
+            pass  # builds the rest of the packet_locs list
+        if saveloc:
+            self.in_stream.seek(loc)
+            self.packet_id = pid
+
+    def count_packets(self, saveloc=True) -> None:
+        self.build_packet_locs(saveloc=saveloc)
+        return len(self.packet_locs)
 
     # TODO: need to correct for endianness?
-    def load_packet(self, skip_unknown_ids: bool = False) -> np.uint32 | None:
+    def load_packet(
+        self, index: int = None, whence: int = 0, skip_unknown_ids: bool = False
+    ) -> np.uint32 | None:
         """Loads the next packet into the internal buffer.
 
         Returns packet as a :class:`numpy.uint32` view of the buffer (a slice),
         returns ``None`` at EOF.
+
+        Parameters
+        ----------
+        index
+            Optionally give an index of packet to skip to, relative to the
+            "whence" location. Can be positive or negative. If out-of-range for
+            the file, None will be returned.
+        whence
+            used when an index is supplied. Follows the file.seek() convention:
+            whence = 0 (default) means index is relative to the beginning of the
+            file; whence = 1 means index is relative to the current position in
+            the file; whence = 2 means relative to the end of the file.
+
+        Returns
+        ----------
+        packet
+            a view of the internal buffer spanning the packet data (uint32
+            ndarray). If you want to hold on to the packet data while you load
+            more packets, you can call copy() on the view to make a copy.
         """
         if self.in_stream is None:
             raise RuntimeError("self.in_stream is None")
 
-        # read packet header
-        pkt_hdr = self.buffer[:1]
-        n_bytes_read = self.in_stream.readinto(pkt_hdr)  # buffer is at least 4 kB long
-        self.n_bytes_read += n_bytes_read
-        if n_bytes_read == 0:
+        if index is not None:
+            if whence not in [0, 1, 2]:
+                raise ValueError(f"whence can't be {whence}")
+            index = int(index)
+            # convert whence 1 or 2 to whence = 0
+            if whence == 1:  # index is relative to current position
+                index += self.packet_id - 1
+            elif whence == 2:  # index is relative to end of file
+                self.build_packet_locs(saveloc=False)
+                index += len(self.packet_locs) - 2
+            if index < 0:
+                self.in_stream.seek(0)
+                self.packet_id = -1
+                return None
+            while index >= len(self.packet_locs):
+                if not self.skip_packet():
+                    return None
+            self.in_stream.seek(self.packet_locs[index])
+            self.packet_id = index - 1
+
+        # load packet header
+        pkt_hdr = self.load_packet_header()
+        if pkt_hdr is None:
             return None
-        if n_bytes_read != 4:
-            raise RuntimeError(f"only got {n_bytes_read} bytes for packet header")
 
         # if it's a short packet, we are done
         if orca_packet.is_short(pkt_hdr):
@@ -69,7 +177,6 @@ class OrcaStreamer(DataStreamer):
             not in self.decoder_id_dict
         ):
             self.in_stream.seek((n_words - 1) * 4, 1)
-            self.n_bytes_read += (n_words - 1) * 4  # well, we didn't really read it...
             return pkt_hdr
 
         # load into buffer, resizing as necessary
@@ -204,15 +311,17 @@ class OrcaStreamer(DataStreamer):
         """
 
         self.set_in_stream(stream_name)
+        self.packet_id = -1
 
         # read in the header
         packet = self.load_packet()
+        if packet is None:
+            raise RuntimeError(f"no orca data in file {stream_name}")
         if orca_packet.get_data_id(packet) != 0:
             raise RuntimeError(
                 f"got data id {orca_packet.get_data_id(packet)} for header"
             )
 
-        self.packet_id = 0
         self.any_full |= self.header_decoder.decode_packet(packet, self.packet_id)
         self.header = self.header_decoder.header
 
@@ -240,9 +349,7 @@ class OrcaStreamer(DataStreamer):
             name = id_to_dec_name_dict[data_id]
             if name not in instantiated_decoders:
                 if name not in globals():
-                    log.warning(
-                        f"no implementation of {name}, corresponding packets will be skipped"
-                    )
+                    self.missing_decoders.append(data_id)
                     continue
                 decoder = globals()[name]
                 instantiated_decoders[name] = decoder(header=self.header)
@@ -296,13 +403,18 @@ class OrcaStreamer(DataStreamer):
             packet = self.load_packet(skip_unknown_ids=True)
             if packet is None:
                 return False
-            self.packet_id += 1
 
             # look up the data id, decoder, and rbl
             data_id = orca_packet.get_data_id(packet, shift=False)
             log.debug(
                 f"packet {self.packet_id}: data_id = {data_id}, decoder = {'None' if data_id not in self.decoder_id_dict else type(self.decoder_id_dict[data_id]).__name__}"
             )
+            if data_id in self.missing_decoders:
+                name = self.header.get_id_to_decoder_name_dict(shift_data_id=False)[
+                    data_id
+                ]
+                log.warning(f"no implementation of {name}, packets were skipped")
+                continue
             if data_id in self.rbl_id_dict:
                 break
 

daq2lh5/raw_buffer.py

@@ -65,21 +65,19 @@ keys.
 from __future__ import annotations
 
 import os
-from typing import Union
 
 import lgdo
-from lgdo import LH5Store
+from lgdo import LGDO
+from lgdo.lh5 import LH5Store
 
 from .buffer_processor.buffer_processor import buffer_processor
 
-LGDO = Union[lgdo.Scalar, lgdo.Struct, lgdo.Array, lgdo.VectorOfVectors]
-
 
 class RawBuffer:
     r"""Base class to represent a buffer of raw data.
 
     A :class:`RawBuffer` is in essence a an LGDO object (typically a
-    :class:`~.lgdo.table.Table`) to which decoded data will be written, along
+    :class:`~lgdo.types.table.Table`) to which decoded data will be written, along
     with some meta-data distinguishing what data goes into it, and where the
     LGDO gets written out. Also holds on to the current location in the buffer
     for writing.
@@ -88,7 +86,7 @@ class RawBuffer:
     ----------
     lgdo
         the LGDO used as the actual buffer. Typically a
-        :class:`~.lgdo.table.Table`. Set to ``None`` upon creation so that the
+        :class:`~lgdo.types.table.Table`. Set to ``None`` upon creation so that the
         user or a decoder can initialize it later.
     key_list
         a list of keys (e.g. channel numbers) identifying data to be written
@@ -107,7 +105,7 @@ class RawBuffer:
     proc_spec
         a dictionary containing the following:
         - a DSP config file, passed as a dictionary, or as a path to a JSON file
-        - an array containing: the name of an :class:`~.lgdo` object stored in the :class:`.RawBuffer` to be sliced,
+        - an array containing: the name of an LGDO object stored in the :class:`.RawBuffer` to be sliced,
         the start and end indices of the slice, and the new name for the sliced object
         - a dictionary of fields to drop
         - a dictionary of new fields and their return datatype
@@ -440,11 +438,11 @@ def write_to_lh5_and_clear(
         files (saves some time opening / closing files).
     **kwargs
         keyword-arguments forwarded to
-        :meth:`.lgdo.lh5_store.LH5Store.write_object`.
+        :meth:`lgdo.lh5.store.LH5Store.write`.
 
     See Also
     --------
-    .lgdo.lh5_store.LH5Store.write_object
+    lgdo.lh5.store.LH5Store.write
     """
     if lh5_store is None:
         lh5_store = lgdo.LH5Store()
@@ -470,7 +468,7 @@ def write_to_lh5_and_clear(
 
         # write if requested...
         if filename != "":
-            lh5_store.write_object(
+            lh5_store.write(
                 lgdo_to_write,
                 rb.out_name,
                 filename,

{legend_daq2lh5-1.0.2.dist-info → legend_daq2lh5-1.2.0a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: legend-daq2lh5
-Version: 1.0.2
+Name: legend_daq2lh5
+Version: 1.2.0a1
 Summary: Convert digitizer data to LH5
 Home-page: https://github.com/legend-exp/legend-daq2lh5
 Author: Jason Detwiler
@@ -26,10 +26,10 @@ Classifier: Topic :: Software Development
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dspeed ~=1.1
+Requires-Dist: dspeed >=1.3.0a4
 Requires-Dist: h5py >=3.2.0
 Requires-Dist: hdf5plugin
-Requires-Dist: legend-pydataobj ~=1.1
+Requires-Dist: legend-pydataobj >=1.5.0a1
 Requires-Dist: numpy >=1.21
 Requires-Dist: pyfcutils
 Requires-Dist: tqdm >=4.27
@@ -60,3 +60,16 @@ Requires-Dist: pytest-cov ; extra == 'test'
 ![GitHub pull requests](https://img.shields.io/github/issues-pr/legend-exp/legend-daq2lh5?logo=github)
 ![License](https://img.shields.io/github/license/legend-exp/legend-daq2lh5)
 [![Read the Docs](https://img.shields.io/readthedocs/legend-daq2lh5?logo=readthedocs)](https://legend-daq2lh5.readthedocs.io)
+
+JSON-configurable conversion of digitized data into
+[LEGEND HDF5](https://legend-exp.github.io/legend-data-format-specs/dev/hdf5/),
+with optional data pre-processing via [dspeed](https://dspeed.readthedocs.io)
+and data compression via [legend-pydataobj](https://legend-pydataobj.readthedocs.io).
+
+Currently supported DAQ data formats:
+* [FlashCam](https://www.mizzi-computer.de/home)
+* [CoMPASS](https://www.caen.it/products/compass)
+* [ORCA](https://github.com/unc-enap/Orca), reading out:
+  - FlashCam
+  - [Struck SIS3302](https://www.struck.de/sis3302.htm)
+  - [Struck SIS3316](https://www.struck.de/sis3316.html)

{legend_daq2lh5-1.0.2.dist-info → legend_daq2lh5-1.2.0a1.dist-info}/RECORD

@@ -1,36 +1,36 @@
 daq2lh5/__init__.py,sha256=VPmwKuZSA0icpce05ojhnsKWhR4_QUgD0oVXUoN9wks,975
-daq2lh5/_version.py,sha256=37405taMQ5GWqqWhBXb1DerL4bDz062TZx0QfP2a0bw,274
-daq2lh5/build_raw.py,sha256=6uMuDp2N9Rjqh3JCt2oHNFMMWlrQuiYozAvHx-si6oI,10585
+daq2lh5/_version.py,sha256=k5PS9p0a5Ey36DDxagN4mnTZow7bHSa0Oh_ycx0FrX4,413
+daq2lh5/build_raw.py,sha256=JFXC5ln9u353TUZMksY3zydLiV2HlxqdI6_Y2_ZMCIE,10524
 daq2lh5/cli.py,sha256=HCZ9Vyg-gqvairN9zJIpBjw5vLpp9ZUOOQYLFxloLL8,2912
-daq2lh5/data_decoder.py,sha256=hl4kebdH22K3hImvWFeNf6C43YC03DCK7HlWCdwHhhw,10670
+daq2lh5/data_decoder.py,sha256=ka2WIJuPvsG892__HCW1SagCEzyiZJ2kQP6zGDMtlr0,10641
 daq2lh5/data_streamer.py,sha256=6SEAekOHyfC4k3E0df0lW37ap6ZemVFbH8PYMl6UvCU,14130
 daq2lh5/logging.py,sha256=Nu3wgIoWN7cyUxuzPom5rMwFvTlBu8p8d9uONHDquRg,965
-daq2lh5/raw_buffer.py,sha256=yVLUYhxLe6KOjwxq_k91MAekx3ZUYmWL32GAhu3ahls,17784
+daq2lh5/raw_buffer.py,sha256=dyPUok0N3MP41oP9F8sO_PrH7-SWs9UdPh7dqCF729g,17687
 daq2lh5/buffer_processor/__init__.py,sha256=7k6v_KPximtv7805QnX4-xp_S3vqvqwDfdV3q95oZJo,84
-daq2lh5/buffer_processor/buffer_processor.py,sha256=Ka4NbYOBCYdqyg3C0VZEJRbCVcjPEX9xlRbclhQjJVw,13758
-daq2lh5/buffer_processor/lh5_buffer_processor.py,sha256=Jf-lVM6t9Ui-6-0fr9tXkMsX6ButHZB7I0gyJcRQqw8,8323
+daq2lh5/buffer_processor/buffer_processor.py,sha256=GUxpNDbqGLuUEZmXjeratipbzmki12RFNYZkxgMtesg,14483
+daq2lh5/buffer_processor/lh5_buffer_processor.py,sha256=yL1ru0_GTsZx099oi45sXL-FxPfdChtStd_IFtZNI_Q,8222
 daq2lh5/compass/__init__.py,sha256=mOXHWp7kRDgNTPQty3E8k2KPSy_vAzjneKfAcCVaPyE,132
 daq2lh5/compass/compass_config_parser.py,sha256=zeAsOo1dOJPGLL8-zkAcdYRkqt8BodtOPi96n7fWsl4,12300
 daq2lh5/compass/compass_event_decoder.py,sha256=kiPOaEu8SgLD2wbSPbBahcbTBBRAIw35wtVLBcwPcXY,7386
 daq2lh5/compass/compass_header_decoder.py,sha256=AA-Md2FIT3nD4mXX9CrWvbbfmKiA436-BTmzcU3_XOY,2823
 daq2lh5/compass/compass_streamer.py,sha256=zSl7IqO0ID0wcixkLE9QVEG3bF9hfGVITVPomCeOFTM,8841
 daq2lh5/fc/__init__.py,sha256=bB1j6r-bDmylNi0iutQeAJGjsDSjLSoXMqFfXWwfb8I,141
-daq2lh5/fc/fc_config_decoder.py,sha256=6PA4AGxfoI9S22lfoMFgb4L_tMloT1TF7HWi2r7OJaM,1990
+daq2lh5/fc/fc_config_decoder.py,sha256=RLRfUOZN0vYbAprqTymP7TGg641IiP9rgCGIOwWVKzU,1996
 daq2lh5/fc/fc_event_decoder.py,sha256=JIRsySnxeuY3wmxjJOrTXo6wpelVup8WIvxU-fkPL-A,8131
 daq2lh5/fc/fc_status_decoder.py,sha256=o_3vTAgYXelZxIsreCYioVYid2mY-wqloYKlxoCqX5Q,3390
 daq2lh5/fc/fc_streamer.py,sha256=S0imXdVsiyolPvxI1uiBngpC58DporSNZPqx1HeVi5o,5737
 daq2lh5/orca/__init__.py,sha256=Xf6uOIOzk_QkKH_7VizGlCo3iuiAgLtUE3A07x_HXC0,175
 daq2lh5/orca/orca_base.py,sha256=-XIolXsHj-1EdewaGxyvJTZvRGZsDyZe-5PzVOd-LFY,1333
-daq2lh5/orca/orca_digitizers.py,sha256=rpk2SSDQgE681FB_iaewAuTXCVEqhUTlhLAm0RGJDfo,20869
-daq2lh5/orca/orca_flashcam.py,sha256=2zJUkuhR7MsZ14stW3VI67DE2UIwN6LSZ80K0_7EpuI,33016
+daq2lh5/orca/orca_digitizers.py,sha256=BsAA3OgQ13YIirDM8pd_xDY3F5FqEY4YjSHviflmov8,20867
+daq2lh5/orca/orca_flashcam.py,sha256=gsvPorUXk1Jn-U93GsxXJ5z6pbTK2yjsYDqZFVCm57U,33088
 daq2lh5/orca/orca_header.py,sha256=1tDRG8l9Gqu4c0K4BjXBSC5eiLTzY_HaCsgNBiv5EgI,4283
 daq2lh5/orca/orca_header_decoder.py,sha256=ORIIyfx22ybyKc-uyWy5ER49-dl3BGpHdfV8OCDmjIw,1632
-daq2lh5/orca/orca_packet.py,sha256=FCB-toCjqQUwJmcYRDjnHoxo8m_xra9jr6rEfuawnh0,2340
+daq2lh5/orca/orca_packet.py,sha256=TcdfuYN8_gcug_Xdjz98KqjHw1MqJ4J98zc7WI2xtf4,2488
 daq2lh5/orca/orca_run_decoder.py,sha256=3atKXC6mDi8_PK6ICUBBJ-LyaTM8OU31kKWIpmttRr4,2065
-daq2lh5/orca/orca_streamer.py,sha256=lU2ZkxKyxgk650Zdw9aanFthW93P-FIzraPmlv-nwIA,11307
-legend_daq2lh5-1.0.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-legend_daq2lh5-1.0.2.dist-info/METADATA,sha256=OPBk1lZxJjOJby-YlH72YWABmuCKDQygRh4ItS5VzJw,3127
-legend_daq2lh5-1.0.2.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-legend_daq2lh5-1.0.2.dist-info/entry_points.txt,sha256=R08R4NrHi0ab5MJN_qKqzePVzrLSsw5WpmbiwwduYjw,59
-legend_daq2lh5-1.0.2.dist-info/top_level.txt,sha256=MJQVLyLqMgMKBdVfNXFaCKCjHKakAs19VLbC9ctXZ7A,8
-legend_daq2lh5-1.0.2.dist-info/RECORD,,
+daq2lh5/orca/orca_streamer.py,sha256=VbD9PF-rx_Rk-rEy7XECPmgxr6kZSUf0tC7Qbol3Qeg,15693
+legend_daq2lh5-1.2.0a1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+legend_daq2lh5-1.2.0a1.dist-info/METADATA,sha256=QiBKAO0ycatdNK5W8HlhHXA28pUnqFr2iPnjyjr2RAE,3755
+legend_daq2lh5-1.2.0a1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+legend_daq2lh5-1.2.0a1.dist-info/entry_points.txt,sha256=R08R4NrHi0ab5MJN_qKqzePVzrLSsw5WpmbiwwduYjw,59
+legend_daq2lh5-1.2.0a1.dist-info/top_level.txt,sha256=MJQVLyLqMgMKBdVfNXFaCKCjHKakAs19VLbC9ctXZ7A,8
+legend_daq2lh5-1.2.0a1.dist-info/RECORD,,

{legend_daq2lh5-1.0.2.dist-info → legend_daq2lh5-1.2.0a1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any