legend-daq2lh5 1.2.1__tar.gz → 1.3.0__tar.gz

{legend_daq2lh5-1.2.1/src/legend_daq2lh5.egg-info → legend_daq2lh5-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: legend_daq2lh5
-Version: 1.2.1
+Version: 1.3.0
 Summary: Convert digitizer data to LH5
 Home-page: https://github.com/legend-exp/legend-daq2lh5
 Author: Jason Detwiler
@@ -32,6 +32,7 @@ Requires-Dist: hdf5plugin
 Requires-Dist: legend-pydataobj>=1.6
 Requires-Dist: numpy>=1.21
 Requires-Dist: pyfcutils
+Requires-Dist: pyyaml
 Requires-Dist: tqdm>=4.27
 Requires-Dist: xmltodict
 Provides-Extra: all

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0}/setup.cfg

@@ -37,6 +37,7 @@ install_requires =
 	legend-pydataobj>=1.6
 	numpy>=1.21
 	pyfcutils
+	pyyaml
 	tqdm>=4.27
 	xmltodict
 python_requires = >=3.9

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0}/src/daq2lh5/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.2.1'
-__version_tuple__ = version_tuple = (1, 2, 1)
+__version__ = version = '1.3.0'
+__version_tuple__ = version_tuple = (1, 3, 0)

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0}/src/daq2lh5/build_raw.py

@@ -12,6 +12,7 @@ from tqdm.auto import tqdm
 
 from .compass.compass_streamer import CompassStreamer
 from .fc.fc_streamer import FCStreamer
+from .llama.llama_streamer import LLAMAStreamer
 from .orca.orca_streamer import OrcaStreamer
 from .raw_buffer import RawBufferLibrary, write_to_lh5_and_clear
 
@@ -180,7 +181,7 @@ def build_raw(
     elif in_stream_type == "Compass":
         streamer = CompassStreamer(compass_config_file)
     elif in_stream_type == "LlamaDaq":
-        raise NotImplementedError("LlamaDaq streaming not yet implemented")
+        streamer = LLAMAStreamer()
     elif in_stream_type == "MGDO":
         raise NotImplementedError("MGDO streaming not yet implemented")
     else:

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0}/src/daq2lh5/data_decoder.py

@@ -74,7 +74,7 @@ class DataDecoder:
     def __init__(
         self, garbage_length: int = 256, packet_size_guess: int = 1024
     ) -> None:
-        self.garbage_table = lgdo.Table(garbage_length)
+        self.garbage_table = lgdo.Table(size=garbage_length)
         shape_guess = (garbage_length, packet_size_guess)
         self.garbage_table.add_field(
             "packets", lgdo.VectorOfVectors(shape_guess=shape_guess, dtype="uint8")

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0}/src/daq2lh5/data_streamer.py

@@ -350,7 +350,7 @@ class DataStreamer(ABC):
                     if len(key_list) == 1:
                         this_name = f"{dec_key}_{key_list[0]}"
                     else:
-                        this_name = f"{dec_key}_{ii}"
+                        this_name = f"{dec_key}_{ii}"  # this can cause a name clash e.g. for [[1],[2,3]] ...
                 rb = RawBuffer(
                     key_list=key_list, out_stream=out_stream, out_name=this_name
                 )

legend_daq2lh5-1.3.0/src/daq2lh5/llama/llama_base.py

@@ -0,0 +1,14 @@
+"""
+General utilities for llamaDAQ data decoding
+"""
+
+from __future__ import annotations
+
+import logging
+
+log = logging.getLogger(__name__)
+
+
+# build a unique flat identifier for fadc and channel together
+def join_fadcid_chid(fadcid: int, chid: int) -> int:
+    return (fadcid << 4) + chid

legend_daq2lh5-1.3.0/src/daq2lh5/llama/llama_event_decoder.py

@@ -0,0 +1,328 @@
+from __future__ import annotations
+
+import copy
+import logging
+from typing import Any
+
+import lgdo
+import numpy as np
+
+from ..data_decoder import DataDecoder
+from .llama_header_decoder import LLAMA_Channel_Configs_t
+
+log = logging.getLogger(__name__)
+
+# put decoded values here
+llama_decoded_values_template = {
+    # packet index in file
+    "packet_id": {"dtype": "uint32"},
+    # combined index of FADC and channel
+    "fch_id": {"dtype": "uint32"},
+    # time since epoch
+    "timestamp": {"dtype": "uint64", "units": "clock_ticks"},
+    "status_flag": {"dtype": "uint32"},
+    # waveform data --> not always present
+    # "waveform": {
+    #    "dtype": "uint16",
+    #    "datatype": "waveform",
+    #    "wf_len": 65532,  # max value. override this before initializing buffers to save RAM
+    #    "dt": 8,  # override if a different clock rate is used
+    #    "dt_units": "ns",
+    #    "t0_units": "ns",
+    # }
+}
+# """Default llamaDAQ SIS3316 Event decoded values.
+#
+# Warning
+# -------
+# This configuration will be dynamically modified by the decoder at runtime.
+# """
+
+
+def check_dict_spec_equal(
+    d1: dict[str, Any], d2: dict[str, Any], specs: list[str]
+) -> bool:
+    for spec in specs:
+        if d1.get(spec) != d2.get(spec):
+            return False
+    return True
+
+
+class LLAMAEventDecoder(DataDecoder):
+    """Decode llamaDAQ SIS3316 digitizer event data."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        # these are read for every event (decode_event)
+        # One set of settings per fch, since settings can be different per channel group
+        self.decoded_values: dict[int, dict[str, Any]] = {}
+        super().__init__(*args, **kwargs)
+        self.skipped_channels = {}
+        self.channel_configs = None
+        self.dt_raw: dict[int, float] = (
+            {}
+        )  # need to buffer that to update t0 for avg waveforms per event
+        self.t0_raw: dict[int, float] = (
+            {}
+        )  # store when receiving channel configs and use for each waveform
+        self.t0_avg_const: dict[int, float] = (
+            {}
+        )  # constant part of the t0 of averaged waveforms
+
+    def set_channel_configs(self, channel_configs: LLAMA_Channel_Configs_t) -> None:
+        """Receive channel configurations from llama_streamer after header was parsed
+        Adapt self.decoded_values dict based on read configuration
+        """
+        self.channel_configs = channel_configs
+        for fch, config in self.channel_configs.items():
+            self.decoded_values[fch] = copy.deepcopy(llama_decoded_values_template)
+            format_bits = config["format_bits"]
+            sample_clock_freq = config["sample_freq"]
+            avg_mode = config["avg_mode"]
+            dt_raw: float = 1 / sample_clock_freq * 1000
+            dt_avg: float = dt_raw * (1 << (avg_mode + 1))
+            # t0 generation functions from llamaDAQ -> EventConfig.hh
+            t0_raw: float = (
+                float(config["sample_start_index"]) - float(config["sample_pretrigger"])
+            ) * dt_raw  # location of the trigger is at t = 0
+            t0_avg: float = (
+                -float(config["sample_pretrigger"]) * float(dt_raw)
+                - float(config["avg_sample_pretrigger"]) * dt_avg
+            )  # additional offset to be added independently for every event
+            self.dt_raw[fch] = dt_raw
+            self.t0_raw[fch] = t0_raw
+            self.t0_avg_const[fch] = t0_avg
+            if config["sample_length"] > 0:
+                self.__add_waveform(
+                    self.decoded_values[fch], False, config["sample_length"], dt_raw
+                )
+            if config["avg_sample_length"] > 0 and avg_mode > 0:
+                self.__add_waveform(
+                    self.decoded_values[fch], True, config["avg_sample_length"], dt_avg
+                )
+            if format_bits & 0x01:
+                self.__add_accum1till6(self.decoded_values[fch])
+            if format_bits & 0x02:
+                self.__add_accum7and8(self.decoded_values[fch])
+            if format_bits & 0x04:
+                self.__add_maw(self.decoded_values[fch])
+            if format_bits & 0x08:
+                self.__add_energy(self.decoded_values[fch])
+
+    def get_key_lists(self) -> list[list[int | str]]:
+        """
+        Return a list of lists of keys available for this decoder.
+        Each inner list are the fch_id's which share the exact same settings (trace lengths, avg mode, ...),
+        so they can end up in the same buffer.
+        """
+        if self.channel_configs is None:
+            raise RuntimeError(
+                "Identification of key lists requires channel configs to be set!"
+            )
+
+        params_for_equality = ["sample_length", "avg_sample_length", "avg_mode"]
+
+        def check_equal(c1, c2):
+            return check_dict_spec_equal(c1, c2, params_for_equality)
+
+        kll: list[list[int]] = []  # key-list-list
+        for fch_id, config in self.channel_configs.items():
+            for kl in kll:
+                # use 1st entry of a list of list as "archetype"
+                if check_equal(config, self.channel_configs[kl[0]]):
+                    kl.append(fch_id)
+                    break
+            else:
+                kll.append([fch_id])
+        log.debug(f"key lists are: {repr(kll)}")
+        return kll
+
+    # copied from ORCA SIS3316
+    def get_decoded_values(self, key: int = None) -> dict[str, Any]:
+        if key is None:
+            raise RuntimeError("Key is None!")
+            dec_vals_list = self.decoded_values.values()
+            if len(dec_vals_list) == 0:
+                raise RuntimeError("decoded_values not built yet!")
+
+            return dec_vals_list  # Get first thing we find
+        else:
+            dec_vals_list = self.decoded_values[key]
+            return dec_vals_list
+
+    def decode_packet(
+        self,
+        packet: bytes,
+        evt_rbkd: lgdo.Table | dict[int, lgdo.Table],
+        packet_id: int,
+        fch_id: int,
+        # header: lgdo.Table | dict[int, lgdo.Table]
+    ) -> bool:
+        """
+        Decodes a single packet, which is a single SIS3316 event, as specified in the Struck manual.
+        A single packet corresponds to a single event and channel, and has a unique timestamp.
+        packets of different channel groups can vary in size!
+        """
+
+        # Check if this fch_id should be recorded.
+        if fch_id not in evt_rbkd:
+            if fch_id not in self.skipped_channels:
+                self.skipped_channels[fch_id] = 0
+                log.info(f"Skipping channel: {fch_id}")
+                log.debug(f"evt_rbkd: {evt_rbkd.keys()}")
+            self.skipped_channels[fch_id] += 1
+            return False
+
+        tbl = evt_rbkd[fch_id].lgdo
+        ii = evt_rbkd[fch_id].loc
+
+        # parse the raw event data into numpy arrays of 16 and 32 bit ints
+        evt_data_32 = np.frombuffer(packet, dtype=np.uint32)
+        evt_data_16 = np.frombuffer(packet, dtype=np.uint16)
+
+        # e sti gran binaries non ce li metti
+        # fch_id = (evt_data_32[0] >> 4) & 0x00000fff  --> to be read earlier, since we need size for chopping out the event from the stream
+        timestamp = ((evt_data_32[0] & 0xFFFF0000) << 16) + evt_data_32[1]
+        format_bits = (evt_data_32[0]) & 0x0000000F
+        tbl["fch_id"].nda[ii] = fch_id
+        tbl["packet_id"].nda[ii] = packet_id
+        tbl["timestamp"].nda[ii] = timestamp
+        offset = 2
+        if format_bits & 0x1:
+            tbl["peakHighValue"].nda[ii] = evt_data_16[4]
+            tbl["peakHighIndex"].nda[ii] = evt_data_16[5]
+            tbl["information"].nda[ii] = (evt_data_32[offset + 1] >> 24) & 0xFF
+            tbl["accSum1"].nda[ii] = evt_data_32[offset + 2]
+            tbl["accSum2"].nda[ii] = evt_data_32[offset + 3]
+            tbl["accSum3"].nda[ii] = evt_data_32[offset + 4]
+            tbl["accSum4"].nda[ii] = evt_data_32[offset + 5]
+            tbl["accSum5"].nda[ii] = evt_data_32[offset + 6]
+            tbl["accSum6"].nda[ii] = evt_data_32[offset + 7]
+            offset += 7
+        if format_bits & 0x2:
+            tbl["accSum7"].nda[ii] = evt_data_32[offset + 0]
+            tbl["accSum8"].nda[ii] = evt_data_32[offset + 1]
+            offset += 2
+        if format_bits & 0x4:
+            tbl["mawMax"].nda[ii] = evt_data_32[offset + 0]
+            tbl["mawBefore"].nda[ii] = evt_data_32[offset + 1]
+            tbl["mawAfter"].nda[ii] = evt_data_32[offset + 2]
+            offset += 3
+        if format_bits & 0x8:
+            tbl["startEnergy"].nda[ii] = evt_data_32[offset + 0]
+            tbl["maxEnergy"].nda[ii] = evt_data_32[offset + 1]
+            offset += 2
+
+        raw_length_32 = (evt_data_32[offset + 0]) & 0x03FFFFFF
+        tbl["status_flag"].nda[ii] = (
+            (evt_data_32[offset + 0]) & 0x04000000
+        ) >> 26  # bit 26
+        maw_test_flag = ((evt_data_32[offset + 0]) & 0x08000000) >> 27  # bit 27
+        avg_data_coming = False
+        if evt_data_32[offset + 0] & 0xF0000000 == 0xE0000000:
+            avg_data_coming = False
+        elif evt_data_32[offset + 0] & 0xF0000000 == 0xA0000000:
+            avg_data_coming = True
+        else:
+            raise RuntimeError("Data corruption 1!")
+        offset += 1
+        avg_length_32 = 0
+        if avg_data_coming:
+            avg_count_status = (
+                evt_data_32[offset + 0] & 0x00FF0000
+            ) >> 16  # bits 23 - 16
+            avg_length_32 = evt_data_32[offset + 0] & 0x0000FFFF
+            if evt_data_32[offset + 0] & 0xF0000000 != 0xE0000000:
+                raise RuntimeError("Data corruption 2!")
+            offset += 1
+
+        # --- now the offset points to the raw wf data ---
+
+        if maw_test_flag:
+            raise RuntimeError("Cannot handle data with MAW test data!")
+
+        # compute expected and actual array dimensions
+        raw_length_16 = 2 * raw_length_32
+        avg_length_16 = 2 * avg_length_32
+        header_length_16 = offset * 2
+        expected_wf_length = len(evt_data_16) - header_length_16
+
+        # error check: waveform size must match expectations
+        if raw_length_16 + avg_length_16 != expected_wf_length:
+            raise RuntimeError(
+                f"Waveform sizes {raw_length_16} (raw) and {avg_length_16} (avg) doesn't match expected size {expected_wf_length}."
+            )
+
+        # store waveform if available:
+        if raw_length_16 > 0:
+            tbl["waveform"]["values"].nda[ii] = evt_data_16[
+                offset * 2 : offset * 2 + raw_length_16
+            ]
+            offset += raw_length_32
+            tbl["waveform"]["t0"].nda[ii] = self.t0_raw[fch_id]
+
+        # store pre-averaged (avg) waveform if available:
+        if avg_length_16 > 0:
+            tbl["avgwaveform"]["values"].nda[ii] = evt_data_16[
+                offset * 2 : offset * 2 + avg_length_16
+            ]
+            offset += avg_length_32
+            # need to update avg waveform t0 based on the offset I get per event
+            tbl["avgwaveform"]["t0"].nda[ii] = (
+                self.t0_avg_const[fch_id]
+                + float(avg_count_status) * self.dt_raw[fch_id]
+            )
+
+        if offset != len(evt_data_32):
+            raise RuntimeError("I messed up...")
+
+        evt_rbkd[fch_id].loc += 1
+
+        return evt_rbkd[fch_id].is_full()
+
+    def __add_waveform(
+        self,
+        decoded_values_fch: dict[str, Any],
+        is_avg: bool,
+        max_samples: int,
+        dt: float,
+    ) -> None:
+        """
+        Averaged samples are available from the 125 MHz (16 bit) variatnt of the SIS3316 and can be stored independently of raw samples.
+        I use waveform for raw samples (dt from clock itself) and avgwaveform from averaged samples (dt from clock * avg number).
+
+        GERDA used to have the low-frequency (waveform) & the high-frequency (aux waveform); here: LF = avgwaveform & HF = waveform.
+        """
+        name: str = "avgwaveform" if is_avg else "waveform"
+        decoded_values_fch[name] = {
+            "dtype": "uint16",
+            "datatype": "waveform",
+            "wf_len": max_samples,  # max value. override this before initializing buffers to save RAM
+            "dt": dt,  # the sample pitch (inverse of clock speed)
+            # "t0": t0, # Adding t0 here does not work
+            "dt_units": "ns",
+            "t0_units": "ns",
+        }
+
+    def __add_accum1till6(self, decoded_values_fch: dict[str, Any]) -> None:
+        decoded_values_fch["peakHighValue"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["peakHighIndex"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["information"] = {"dtype": "uint32"}
+        decoded_values_fch["accSum1"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum2"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum3"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum4"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum5"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum6"] = {"dtype": "uint32", "units": "adc"}
+
+    def __add_accum7and8(self, decoded_values_fch: dict[str, Any]) -> None:
+        decoded_values_fch["accSum7"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum8"] = {"dtype": "uint32", "units": "adc"}
+
+    def __add_maw(self, decoded_values_fch: dict[str, Any]) -> None:
+        decoded_values_fch["mawMax"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["mawBefore"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["mawAfter"] = {"dtype": "uint32", "units": "adc"}
+
+    def __add_energy(self, decoded_values_fch: dict[str, Any]) -> None:
+        decoded_values_fch["startEnergy"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["maxEnergy"] = {"dtype": "uint32", "units": "adc"}

legend_daq2lh5-1.3.0/src/daq2lh5/llama/llama_header_decoder.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import io
+import logging
+from typing import Any, Dict
+
+import lgdo
+import numpy as np
+
+from ..data_decoder import DataDecoder
+from .llama_base import join_fadcid_chid
+
+log = logging.getLogger(__name__)
+
+LLAMA_Channel_Configs_t = Dict[int, Dict[str, Any]]
+
+
+class LLAMAHeaderDecoder(DataDecoder):  # DataDecoder currently unused
+    """
+    Decode llamaDAQ header data. Includes the file header as well as all available ("open") channel configurations.
+    """
+
+    @staticmethod
+    def magic_bytes() -> int:
+        return 0x4972414C
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.config = lgdo.Struct()
+        self.channel_configs = None
+
+    def decode_header(self, f_in: io.BufferedReader) -> lgdo.Struct:
+        n_bytes_read = 0
+
+        f_in.seek(0)  # should be there anyhow, but re-set if not
+        header = f_in.read(16)  # read 16 bytes
+        n_bytes_read += 16
+        evt_data_32 = np.frombuffer(header, dtype=np.uint32)
+        evt_data_16 = np.frombuffer(header, dtype=np.uint16)
+
+        # line0: magic bytes
+        magic = evt_data_32[0]
+        # print(hex(magic))
+        if magic == self.magic_bytes():
+            log.info("Read in file as llamaDAQ-SIS3316, magic bytes correct.")
+        else:
+            log.error("Magic bytes not matching for llamaDAQ file!")
+            raise RuntimeError("wrong file type")
+
+        self.version_major = evt_data_16[4]
+        self.version_minor = evt_data_16[3]
+        self.version_patch = evt_data_16[2]
+        self.length_econf = evt_data_16[5]
+        self.number_chOpen = evt_data_32[3]
+
+        log.debug(
+            f"File version: {self.version_major}.{self.version_minor}.{self.version_patch}"
+        )
+        log.debug(
+            f"{self.number_chOpen} channels open, each config {self.length_econf} bytes long"
+        )
+
+        n_bytes_read += self.__decode_channel_configs(f_in)
+
+        # print(self.channel_configs[0]["MAW3_offset"])
+
+        # assemble LGDO struct:
+        self.config.add_field("version_major", lgdo.Scalar(self.version_major))
+        self.config.add_field("version_minor", lgdo.Scalar(self.version_minor))
+        self.config.add_field("version_patch", lgdo.Scalar(self.version_patch))
+        self.config.add_field("length_econf", lgdo.Scalar(self.length_econf))
+        self.config.add_field("number_chOpen", lgdo.Scalar(self.number_chOpen))
+
+        for fch_id, fch_content in self.channel_configs.items():
+            fch_lgdo = lgdo.Struct()
+            for key, value in fch_content.items():
+                fch_lgdo.add_field(key, lgdo.Scalar(value))
+            self.config.add_field(f"fch_{fch_id:02d}", fch_lgdo)
+
+        return self.config, n_bytes_read
+
+    # override from DataDecoder
+    def make_lgdo(self, key: int = None, size: int = None) -> lgdo.Struct:
+        return self.config
+
+    def get_channel_configs(self) -> LLAMA_Channel_Configs_t:
+        return self.channel_configs
+
+    def __decode_channel_configs(self, f_in: io.BufferedReader) -> int:
+        """
+        Reads the metadata from the beginning of the file (the "channel configuration" part, directly after the file header).
+        Creates a dictionary of the metadata for each FADC/channel combination, which is returned
+
+        FADC-ID and channel-ID are combined into a single id for flattening:
+            (fadcid << 4) + chid
+
+        returns number of bytes read
+        """
+        # f_in.seek(16)    #should be after file header anyhow, but re-set if not
+        n_bytes_read = 0
+        self.channel_configs = {}
+
+        if self.length_econf != 88:
+            raise RuntimeError("Invalid channel configuration format")
+
+        for _i in range(0, self.number_chOpen):
+            # print("reading in channel config {}".format(i))
+
+            channel = f_in.read(self.length_econf)
+            n_bytes_read += self.length_econf
+            ch_dpf = channel[16:32]
+            evt_data_32 = np.frombuffer(channel, dtype=np.uint32)
+            evt_data_dpf = np.frombuffer(ch_dpf, dtype=np.float64)
+
+            fadc_index = evt_data_32[0]
+            channel_index = evt_data_32[1]
+            fch_id = join_fadcid_chid(fadc_index, channel_index)
+
+            if fch_id in self.channel_configs:
+                raise RuntimeError(
+                    f"duplicate channel configuration in file: FADCID: {fadc_index}, ChannelID: {channel_index}"
+                )
+            else:
+                self.channel_configs[fch_id] = {}
+
+            self.channel_configs[fch_id]["14BitFlag"] = evt_data_32[2] & 0x00000001
+            if evt_data_32[2] & 0x00000002 == 0:
+                log.warning("Channel in configuration marked as non-open!")
+            self.channel_configs[fch_id]["ADC_offset"] = evt_data_32[3]
+            self.channel_configs[fch_id]["sample_freq"] = evt_data_dpf[
+                0
+            ]  # 64 bit float
+            self.channel_configs[fch_id]["gain"] = evt_data_dpf[1]
+            self.channel_configs[fch_id]["format_bits"] = evt_data_32[8]
+            self.channel_configs[fch_id]["sample_start_index"] = evt_data_32[9]
+            self.channel_configs[fch_id]["sample_pretrigger"] = evt_data_32[10]
+            self.channel_configs[fch_id]["avg_sample_pretrigger"] = evt_data_32[11]
+            self.channel_configs[fch_id]["avg_mode"] = evt_data_32[12]
+            self.channel_configs[fch_id]["sample_length"] = evt_data_32[13]
+            self.channel_configs[fch_id]["avg_sample_length"] = evt_data_32[14]
+            self.channel_configs[fch_id]["MAW_buffer_length"] = evt_data_32[15]
+            self.channel_configs[fch_id]["event_length"] = evt_data_32[16]
+            self.channel_configs[fch_id]["event_header_length"] = evt_data_32[17]
+            self.channel_configs[fch_id]["accum6_offset"] = evt_data_32[18]
+            self.channel_configs[fch_id]["accum2_offset"] = evt_data_32[19]
+            self.channel_configs[fch_id]["MAW3_offset"] = evt_data_32[20]
+            self.channel_configs[fch_id]["energy_offset"] = evt_data_32[21]
+
+        return n_bytes_read

legend_daq2lh5-1.3.0/src/daq2lh5/llama/llama_streamer.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import logging
+
+import numpy as np
+
+from ..data_decoder import DataDecoder
+from ..data_streamer import DataStreamer
+from ..raw_buffer import RawBuffer, RawBufferLibrary
+from .llama_event_decoder import LLAMAEventDecoder
+from .llama_header_decoder import LLAMAHeaderDecoder
+
+log = logging.getLogger(__name__)
+
+
+class LLAMAStreamer(DataStreamer):
+    """
+    Decode SIS3316 data acquired using llamaDAQ.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.in_stream = None
+        self.event_rbkd = None
+        self.header_decoder = LLAMAHeaderDecoder()
+        self.event_decoder = LLAMAEventDecoder()
+
+    def get_decoder_list(self) -> list[DataDecoder]:
+        dec_list = []
+        dec_list.append(self.header_decoder)
+        dec_list.append(self.event_decoder)
+        return dec_list
+
+    def open_stream(
+        self,
+        llama_filename: str,
+        rb_lib: RawBufferLibrary = None,
+        buffer_size: int = 8192,
+        chunk_mode: str = "any_full",
+        out_stream: str = "",
+    ) -> list[RawBuffer]:
+        """Initialize the LLAMA data stream.
+
+        Refer to the documentation for
+        :meth:`.data_streamer.DataStreamer.open_stream` for a description
+        of the parameters.
+        """
+
+        if self.in_stream is not None:
+            raise RuntimeError("tried to open stream while previous one still open")
+        self.in_stream = open(llama_filename.encode("utf-8"), "rb")
+        self.n_bytes_read = 0
+        self.packet_id = 0
+
+        # read header info here
+        header, n_bytes_hdr = self.header_decoder.decode_header(self.in_stream)
+        self.n_bytes_read += n_bytes_hdr
+
+        self.event_decoder.set_channel_configs(
+            self.header_decoder.get_channel_configs()
+        )
+
+        # as far as I can tell, this happens if a user does not specify output.
+        # Then I can still get a rb_lib, but that misses keys entirely, which I need since channels can have different setups.
+        # So I try to hack my own here in case there is none provided.
+        # if rb_lib is None:
+        #    rb_lib = self.__hack_rb_lib(self.header_decoder.get_channel_configs())
+
+        # initialize the buffers in rb_lib. Store them for fast lookup
+        # Docu tells me to use initialize instead, but that does not exits (?)
+        super().open_stream(
+            llama_filename,
+            rb_lib,
+            buffer_size=buffer_size,
+            chunk_mode=chunk_mode,
+            out_stream=out_stream,
+        )
+        if rb_lib is None:
+            rb_lib = self.rb_lib
+
+        self.event_rbkd = (
+            rb_lib["LLAMAEventDecoder"].get_keyed_dict()
+            if "LLAMAEventDecoder" in rb_lib
+            else None
+        )
+
+        if "LLAMAHeaderDecoder" in rb_lib:
+            config_rb_list = rb_lib["LLAMAHeaderDecoder"]
+            if len(config_rb_list) != 1:
+                log.warning(
+                    f"config_rb_list had length {len(config_rb_list)}, ignoring all but the first"
+                )
+            rb = config_rb_list[0]
+        else:
+            rb = RawBuffer(lgdo=header)
+        rb.loc = 1  # we have filled this buffer
+        return [rb]
+
+    def close_stream(self) -> None:
+        if self.in_stream is None:
+            raise RuntimeError("tried to close an unopened stream")
+        self.in_stream.close()
+        self.in_stream = None
+
+    def read_packet(self) -> bool:
+        """Reads a single packet's worth of data in to the :class:`.RawBufferLibrary`.
+
+        Returns
+        -------
+        still_has_data
+            returns `True` while there is still data to read.
+        """
+
+        packet, fch_id = self.__read_bytes()
+        if packet is None:
+            return False  # EOF
+        self.packet_id += 1
+        self.n_bytes_read += len(packet)
+
+        self.any_full |= self.event_decoder.decode_packet(
+            packet, self.event_rbkd, self.packet_id, fch_id
+        )
+
+        return True
+
+    def __read_bytes(self) -> tuple[bytes | None, int]:
+        """
+        return bytes if read successful or None if EOF.
+        int is the fch_id (needs to be fetched to obtain the size of the event)
+        """
+        if self.in_stream is None:
+            raise RuntimeError("No stream open!")
+
+        position = self.in_stream.tell()  # save position of the event header's 1st byte
+        data1 = self.in_stream.read(
+            4
+        )  # read the first (32 bit) word of the event's header: channelID & format bits
+        if len(data1) < 4:
+            return None, -1  # EOF, I guess
+        self.in_stream.seek(position)  # go back to 1st position of event header
+
+        header_data_32 = np.frombuffer(data1, dtype=np.uint32)
+        fch_id = (header_data_32[0] >> 4) & 0x00000FFF
+
+        event_length_32 = self.header_decoder.get_channel_configs()[fch_id][
+            "event_length"
+        ]
+        event_length_8 = event_length_32 * 4
+
+        packet = self.in_stream.read(event_length_8)
+        if len(packet) < event_length_8:
+            raise RuntimeError(
+                f"Tried to read {event_length_8} bytes but got {len(packet)}"
+            )
+
+        return packet, fch_id

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0}/src/daq2lh5/logging.py

@@ -11,6 +11,8 @@ ERROR = logging.ERROR
 FATAL = logging.FATAL
 CRITICAL = logging.CRITICAL
 
+root = logging.root
+
 
 def setup(level: int = logging.INFO, logger: logging.Logger = None) -> None:
     """Setup a colorful logging output.

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0/src/legend_daq2lh5.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: legend_daq2lh5
-Version: 1.2.1
+Version: 1.3.0
 Summary: Convert digitizer data to LH5
 Home-page: https://github.com/legend-exp/legend-daq2lh5
 Author: Jason Detwiler
@@ -32,6 +32,7 @@ Requires-Dist: hdf5plugin
 Requires-Dist: legend-pydataobj>=1.6
 Requires-Dist: numpy>=1.21
 Requires-Dist: pyfcutils
+Requires-Dist: pyyaml
 Requires-Dist: tqdm>=4.27
 Requires-Dist: xmltodict
 Provides-Extra: all

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0}/src/legend_daq2lh5.egg-info/SOURCES.txt

@@ -24,6 +24,10 @@ src/daq2lh5/fc/fc_config_decoder.py
 src/daq2lh5/fc/fc_event_decoder.py
 src/daq2lh5/fc/fc_status_decoder.py
 src/daq2lh5/fc/fc_streamer.py
+src/daq2lh5/llama/llama_base.py
+src/daq2lh5/llama/llama_event_decoder.py
+src/daq2lh5/llama/llama_header_decoder.py
+src/daq2lh5/llama/llama_streamer.py
 src/daq2lh5/orca/__init__.py
 src/daq2lh5/orca/orca_base.py
 src/daq2lh5/orca/orca_digitizers.py
@@ -62,6 +66,10 @@ tests/fc/test_fc_config_decoder.py
 tests/fc/test_fc_event_decoder.py
 tests/fc/test_fc_status_decoder.py
 tests/fc/test_fc_streamer.py
+tests/llama/conftest.py
+tests/llama/test_llama_event_decoder.py
+tests/llama/test_llama_header_decoder.py
+tests/llama/test_llama_streamer.py
 tests/orca/conftest.py
 tests/orca/test_or_run_decoder_for_run.py
 tests/orca/test_orca_fc.py

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0}/src/legend_daq2lh5.egg-info/requires.txt

@@ -4,6 +4,7 @@ hdf5plugin
 legend-pydataobj>=1.6
 numpy>=1.21
 pyfcutils
+pyyaml
 tqdm>=4.27
 xmltodict
 

{legend_daq2lh5-1.2.1 → legend_daq2lh5-1.3.0}/tests/conftest.py

@@ -30,7 +30,7 @@ def pytest_sessionfinish(session, exitstatus):
 @pytest.fixture(scope="session")
 def lgnd_test_data():
     ldata = LegendTestData()
-    ldata.checkout("ecb370e")
+    ldata.checkout("88d8c01")
     return ldata
 
 

legend_daq2lh5-1.3.0/tests/llama/conftest.py

@@ -0,0 +1,7 @@
+import pytest
+
+
+# lgnd_test_data (LegendTestData) from fixture in root conftest.py
+@pytest.fixture(scope="module")
+def test_data_path(lgnd_test_data):
+    return lgnd_test_data.get_path("llamaDAQ/20241218-150158-pulser.bin")

legend_daq2lh5-1.3.0/tests/llama/test_llama_event_decoder.py

@@ -0,0 +1,127 @@
+import lgdo
+import pytest
+
+from daq2lh5.llama.llama_event_decoder import LLAMAEventDecoder, check_dict_spec_equal
+from daq2lh5.llama.llama_streamer import LLAMAStreamer
+
+
+def test_check_dict_spec_equal():
+    d1 = {"X": "1", "Y": "2", "Z": "3"}
+    d2 = {"X": "2", "Y": "2", "Z": "3"}
+    assert check_dict_spec_equal(d1, d2, ["Y", "Z"])
+    assert not check_dict_spec_equal(d1, d2, ["X", "Y"])
+
+
+@pytest.fixture(scope="module")
+def open_stream(test_data_path):
+    streamer = LLAMAStreamer()
+    streamer.open_stream(test_data_path)
+    yield streamer
+    streamer.close_stream()
+
+
+def test_get_key_lists(open_stream):
+    evt_dec: LLAMAEventDecoder = open_stream.event_decoder
+    assert evt_dec.get_key_lists() == [[0], [4]]
+
+
+def test_get_decoded_values(open_stream):
+    evt_dec: LLAMAEventDecoder = open_stream.event_decoder
+    dec_vals_0 = evt_dec.get_decoded_values(0)
+    assert dec_vals_0["waveform"]["wf_len"] == 2000
+    assert dec_vals_0["avgwaveform"]["wf_len"] == 10000
+    dec_vals_4 = evt_dec.get_decoded_values(4)
+    assert dec_vals_4["waveform"]["wf_len"] == 2000
+    assert dec_vals_4["avgwaveform"]["wf_len"] == 500
+
+
+def test_first_packet(open_stream):
+    good_packet = open_stream.read_packet()
+    assert good_packet
+    evt_dec: LLAMAEventDecoder = open_stream.event_decoder
+    assert evt_dec is not None
+    evt_rbkd = open_stream.event_rbkd
+    tbl = evt_rbkd[0].lgdo
+    assert isinstance(tbl, lgdo.Table)
+    ii = evt_rbkd[0].loc
+    assert ii == 1
+    ii = ii - 1  # use the last written entry (which is the only one, actually)
+    assert tbl["fch_id"].nda[ii] == 0
+    assert tbl["packet_id"].nda[ii] == 1
+    assert tbl["timestamp"].nda[ii] == 757530
+    assert tbl["peakHighValue"].nda[ii] == 9454
+    assert tbl["peakHighIndex"].nda[ii] == 1968
+    assert tbl["information"].nda[ii] == 0
+    assert tbl["accSum1"].nda[ii] == 7826
+    assert tbl["accSum2"].nda[ii] == 7826
+    assert tbl["accSum3"].nda[ii] == 7826
+    assert tbl["accSum4"].nda[ii] == 7826
+    assert tbl["accSum5"].nda[ii] == 7826
+    assert tbl["accSum6"].nda[ii] == 7826
+    assert tbl["accSum7"].nda[ii] == 7826
+    assert tbl["accSum8"].nda[ii] == 7826
+    assert (
+        tbl["waveform"]["dt"].nda[ii] > 3.999 and tbl["waveform"]["dt"].nda[ii] < 4.001
+    )
+    assert (
+        tbl["avgwaveform"]["dt"].nda[ii] > 15.999
+        and tbl["avgwaveform"]["dt"].nda[ii] < 16.001
+    )
+    assert (
+        tbl["waveform"]["t0"].nda[ii] > -4000.1
+        and tbl["waveform"]["t0"].nda[ii] < -3999.9
+    )
+    assert (
+        tbl["avgwaveform"]["t0"].nda[ii] > -8000.1
+        and tbl["avgwaveform"]["t0"].nda[ii] < -7999.9
+    )
+
+
+def test_first_packet_ch4(open_stream):
+    evt_rbkd = open_stream.event_rbkd
+    while True:
+        good_packet = open_stream.read_packet()
+        if not good_packet:
+            break
+    tbl = evt_rbkd[4].lgdo
+    assert evt_rbkd[4].loc > 0, "Not a single event of channel 4"
+    ii = 0
+    assert tbl["fch_id"].nda[ii] == 4
+    assert tbl["packet_id"].nda[ii] == 10
+    assert tbl["timestamp"].nda[ii] == 757530
+    assert tbl["peakHighValue"].nda[ii] == 7923
+    assert tbl["peakHighIndex"].nda[ii] == 371
+    assert tbl["information"].nda[ii] == 0
+    assert tbl["accSum1"].nda[ii] == 7912
+    assert tbl["accSum2"].nda[ii] == 7912
+    assert tbl["accSum3"].nda[ii] == 7912
+    assert tbl["accSum4"].nda[ii] == 7912
+    assert tbl["accSum5"].nda[ii] == 7912
+    assert tbl["accSum6"].nda[ii] == 7912
+    assert tbl["accSum7"].nda[ii] == 7912
+    assert tbl["accSum8"].nda[ii] == 7912
+    assert (
+        tbl["waveform"]["dt"].nda[ii] > 3.999 and tbl["waveform"]["dt"].nda[ii] < 4.001
+    )
+    assert (
+        tbl["avgwaveform"]["dt"].nda[ii] > 31.999
+        and tbl["avgwaveform"]["dt"].nda[ii] < 32.001
+    )
+    assert (
+        tbl["waveform"]["t0"].nda[ii] > -4000.1
+        and tbl["waveform"]["t0"].nda[ii] < -3999.9
+    )
+    assert (
+        tbl["avgwaveform"]["t0"].nda[ii] > -4000.1
+        and tbl["avgwaveform"]["t0"].nda[ii] < -3999.9
+    )
+
+
+def test_event_count(open_stream):
+    evt_rbkd = open_stream.event_rbkd
+    while True:
+        good_packet = open_stream.read_packet()
+        if not good_packet:
+            break
+    assert evt_rbkd[0].loc == 37
+    assert evt_rbkd[4].loc == 37

legend_daq2lh5-1.3.0/tests/llama/test_llama_header_decoder.py

@@ -0,0 +1,16 @@
+from daq2lh5.llama.llama_header_decoder import LLAMAHeaderDecoder
+from daq2lh5.llama.llama_streamer import LLAMAStreamer
+
+
+def test_read_header(test_data_path):
+    streamer = LLAMAStreamer()
+    streamer.open_stream(test_data_path)
+    header = streamer.header_decoder
+    assert isinstance(header, LLAMAHeaderDecoder)
+    # following data is specific to the particular test file:
+    assert header.version_major == 2
+    assert header.version_minor == 0
+    assert header.version_patch == 0
+    assert header.length_econf == 88  # 22 words of 4 bytes
+    assert header.number_chOpen == 2
+    streamer.close_stream()

legend_daq2lh5-1.3.0/tests/llama/test_llama_streamer.py

@@ -0,0 +1,36 @@
+from daq2lh5.llama.llama_event_decoder import LLAMAEventDecoder
+from daq2lh5.llama.llama_header_decoder import LLAMAHeaderDecoder
+from daq2lh5.llama.llama_streamer import LLAMAStreamer
+from daq2lh5.raw_buffer import RawBuffer
+
+
+def test_get_decoder_list():
+    streamer = LLAMAStreamer()
+    assert len(streamer.get_decoder_list()) == 2
+    assert isinstance(streamer.get_decoder_list()[0], LLAMAHeaderDecoder)
+    assert isinstance(streamer.get_decoder_list()[1], LLAMAEventDecoder)
+
+
+# test_data_path (str) from fixture in ./conftest.py
+def test_open_stream(test_data_path):
+    streamer = LLAMAStreamer()
+    rbl: list[RawBuffer] = streamer.open_stream(test_data_path)
+    assert len(rbl) == 1
+    assert isinstance(rbl[0], RawBuffer)
+    assert streamer.rb_lib is not None
+    nbytes_hdr = streamer.n_bytes_read
+    assert nbytes_hdr > 0
+    assert streamer.read_packet()  # there has to be at last a single good packet
+    assert streamer.packet_id == 1
+    assert streamer.n_bytes_read > nbytes_hdr
+    streamer.close_stream()
+
+
+def test_open_stream_multiple(test_data_path):
+    streamer = LLAMAStreamer()
+    rbl: list[RawBuffer] = streamer.open_stream(test_data_path)
+    assert len(rbl) == 1
+    streamer.close_stream()
+    rbl: list[RawBuffer] = streamer.open_stream(test_data_path)
+    assert len(rbl) == 1
+    streamer.close_stream()