legend-daq2lh5 1.2.2__py3-none-any.whl → 1.3.0__py3-none-any.whl

daq2lh5/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.2.2'
-__version_tuple__ = version_tuple = (1, 2, 2)
+__version__ = version = '1.3.0'
+__version_tuple__ = version_tuple = (1, 3, 0)

daq2lh5/build_raw.py

@@ -12,6 +12,7 @@ from tqdm.auto import tqdm
 
 from .compass.compass_streamer import CompassStreamer
 from .fc.fc_streamer import FCStreamer
+from .llama.llama_streamer import LLAMAStreamer
 from .orca.orca_streamer import OrcaStreamer
 from .raw_buffer import RawBufferLibrary, write_to_lh5_and_clear
 
@@ -180,7 +181,7 @@ def build_raw(
     elif in_stream_type == "Compass":
         streamer = CompassStreamer(compass_config_file)
     elif in_stream_type == "LlamaDaq":
-        raise NotImplementedError("LlamaDaq streaming not yet implemented")
+        streamer = LLAMAStreamer()
     elif in_stream_type == "MGDO":
         raise NotImplementedError("MGDO streaming not yet implemented")
     else:

daq2lh5/data_streamer.py

@@ -350,7 +350,7 @@ class DataStreamer(ABC):
                     if len(key_list) == 1:
                         this_name = f"{dec_key}_{key_list[0]}"
                     else:
-                        this_name = f"{dec_key}_{ii}"
+                        this_name = f"{dec_key}_{ii}"  # this can cause a name clash e.g. for [[1],[2,3]] ...
                 rb = RawBuffer(
                     key_list=key_list, out_stream=out_stream, out_name=this_name
                 )

daq2lh5/llama/llama_base.py

@@ -0,0 +1,14 @@
+"""
+General utilities for llamaDAQ data decoding
+"""
+
+from __future__ import annotations
+
+import logging
+
+log = logging.getLogger(__name__)
+
+
+# build a unique flat identifier for fadc and channel together
+def join_fadcid_chid(fadcid: int, chid: int) -> int:
+    return (fadcid << 4) + chid

daq2lh5/llama/llama_event_decoder.py

@@ -0,0 +1,328 @@
+from __future__ import annotations
+
+import copy
+import logging
+from typing import Any
+
+import lgdo
+import numpy as np
+
+from ..data_decoder import DataDecoder
+from .llama_header_decoder import LLAMA_Channel_Configs_t
+
+log = logging.getLogger(__name__)
+
+# put decoded values here
+llama_decoded_values_template = {
+    # packet index in file
+    "packet_id": {"dtype": "uint32"},
+    # combined index of FADC and channel
+    "fch_id": {"dtype": "uint32"},
+    # time since epoch
+    "timestamp": {"dtype": "uint64", "units": "clock_ticks"},
+    "status_flag": {"dtype": "uint32"},
+    # waveform data --> not always present
+    # "waveform": {
+    #    "dtype": "uint16",
+    #    "datatype": "waveform",
+    #    "wf_len": 65532,  # max value. override this before initializing buffers to save RAM
+    #    "dt": 8,  # override if a different clock rate is used
+    #    "dt_units": "ns",
+    #    "t0_units": "ns",
+    # }
+}
+# """Default llamaDAQ SIS3316 Event decoded values.
+#
+# Warning
+# -------
+# This configuration will be dynamically modified by the decoder at runtime.
+# """
+
+
+def check_dict_spec_equal(
+    d1: dict[str, Any], d2: dict[str, Any], specs: list[str]
+) -> bool:
+    for spec in specs:
+        if d1.get(spec) != d2.get(spec):
+            return False
+    return True
+
+
+class LLAMAEventDecoder(DataDecoder):
+    """Decode llamaDAQ SIS3316 digitizer event data."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        # these are read for every event (decode_event)
+        # One set of settings per fch, since settings can be different per channel group
+        self.decoded_values: dict[int, dict[str, Any]] = {}
+        super().__init__(*args, **kwargs)
+        self.skipped_channels = {}
+        self.channel_configs = None
+        self.dt_raw: dict[int, float] = (
+            {}
+        )  # need to buffer that to update t0 for avg waveforms per event
+        self.t0_raw: dict[int, float] = (
+            {}
+        )  # store when receiving channel configs and use for each waveform
+        self.t0_avg_const: dict[int, float] = (
+            {}
+        )  # constant part of the t0 of averaged waveforms
+
+    def set_channel_configs(self, channel_configs: LLAMA_Channel_Configs_t) -> None:
+        """Receive channel configurations from llama_streamer after header was parsed
+        Adapt self.decoded_values dict based on read configuration
+        """
+        self.channel_configs = channel_configs
+        for fch, config in self.channel_configs.items():
+            self.decoded_values[fch] = copy.deepcopy(llama_decoded_values_template)
+            format_bits = config["format_bits"]
+            sample_clock_freq = config["sample_freq"]
+            avg_mode = config["avg_mode"]
+            dt_raw: float = 1 / sample_clock_freq * 1000
+            dt_avg: float = dt_raw * (1 << (avg_mode + 1))
+            # t0 generation functions from llamaDAQ -> EventConfig.hh
+            t0_raw: float = (
+                float(config["sample_start_index"]) - float(config["sample_pretrigger"])
+            ) * dt_raw  # location of the trigger is at t = 0
+            t0_avg: float = (
+                -float(config["sample_pretrigger"]) * float(dt_raw)
+                - float(config["avg_sample_pretrigger"]) * dt_avg
+            )  # additional offset to be added independently for every event
+            self.dt_raw[fch] = dt_raw
+            self.t0_raw[fch] = t0_raw
+            self.t0_avg_const[fch] = t0_avg
+            if config["sample_length"] > 0:
+                self.__add_waveform(
+                    self.decoded_values[fch], False, config["sample_length"], dt_raw
+                )
+            if config["avg_sample_length"] > 0 and avg_mode > 0:
+                self.__add_waveform(
+                    self.decoded_values[fch], True, config["avg_sample_length"], dt_avg
+                )
+            if format_bits & 0x01:
+                self.__add_accum1till6(self.decoded_values[fch])
+            if format_bits & 0x02:
+                self.__add_accum7and8(self.decoded_values[fch])
+            if format_bits & 0x04:
+                self.__add_maw(self.decoded_values[fch])
+            if format_bits & 0x08:
+                self.__add_energy(self.decoded_values[fch])
+
+    def get_key_lists(self) -> list[list[int | str]]:
+        """
+        Return a list of lists of keys available for this decoder.
+        Each inner list are the fch_id's which share the exact same settings (trace lengths, avg mode, ...),
+        so they can end up in the same buffer.
+        """
+        if self.channel_configs is None:
+            raise RuntimeError(
+                "Identification of key lists requires channel configs to be set!"
+            )
+
+        params_for_equality = ["sample_length", "avg_sample_length", "avg_mode"]
+
+        def check_equal(c1, c2):
+            return check_dict_spec_equal(c1, c2, params_for_equality)
+
+        kll: list[list[int]] = []  # key-list-list
+        for fch_id, config in self.channel_configs.items():
+            for kl in kll:
+                # use 1st entry of a list of list as "archetype"
+                if check_equal(config, self.channel_configs[kl[0]]):
+                    kl.append(fch_id)
+                    break
+            else:
+                kll.append([fch_id])
+        log.debug(f"key lists are: {repr(kll)}")
+        return kll
+
+    # copied from ORCA SIS3316
+    def get_decoded_values(self, key: int = None) -> dict[str, Any]:
+        if key is None:
+            raise RuntimeError("Key is None!")
+            dec_vals_list = self.decoded_values.values()
+            if len(dec_vals_list) == 0:
+                raise RuntimeError("decoded_values not built yet!")
+
+            return dec_vals_list  # Get first thing we find
+        else:
+            dec_vals_list = self.decoded_values[key]
+            return dec_vals_list
+
+    def decode_packet(
+        self,
+        packet: bytes,
+        evt_rbkd: lgdo.Table | dict[int, lgdo.Table],
+        packet_id: int,
+        fch_id: int,
+        # header: lgdo.Table | dict[int, lgdo.Table]
+    ) -> bool:
+        """
+        Decodes a single packet, which is a single SIS3316 event, as specified in the Struck manual.
+        A single packet corresponds to a single event and channel, and has a unique timestamp.
+        packets of different channel groups can vary in size!
+        """
+
+        # Check if this fch_id should be recorded.
+        if fch_id not in evt_rbkd:
+            if fch_id not in self.skipped_channels:
+                self.skipped_channels[fch_id] = 0
+                log.info(f"Skipping channel: {fch_id}")
+                log.debug(f"evt_rbkd: {evt_rbkd.keys()}")
+            self.skipped_channels[fch_id] += 1
+            return False
+
+        tbl = evt_rbkd[fch_id].lgdo
+        ii = evt_rbkd[fch_id].loc
+
+        # parse the raw event data into numpy arrays of 16 and 32 bit ints
+        evt_data_32 = np.frombuffer(packet, dtype=np.uint32)
+        evt_data_16 = np.frombuffer(packet, dtype=np.uint16)
+
+        # e sti gran binaries non ce li metti
+        # fch_id = (evt_data_32[0] >> 4) & 0x00000fff  --> to be read earlier, since we need size for chopping out the event from the stream
+        timestamp = ((evt_data_32[0] & 0xFFFF0000) << 16) + evt_data_32[1]
+        format_bits = (evt_data_32[0]) & 0x0000000F
+        tbl["fch_id"].nda[ii] = fch_id
+        tbl["packet_id"].nda[ii] = packet_id
+        tbl["timestamp"].nda[ii] = timestamp
+        offset = 2
+        if format_bits & 0x1:
+            tbl["peakHighValue"].nda[ii] = evt_data_16[4]
+            tbl["peakHighIndex"].nda[ii] = evt_data_16[5]
+            tbl["information"].nda[ii] = (evt_data_32[offset + 1] >> 24) & 0xFF
+            tbl["accSum1"].nda[ii] = evt_data_32[offset + 2]
+            tbl["accSum2"].nda[ii] = evt_data_32[offset + 3]
+            tbl["accSum3"].nda[ii] = evt_data_32[offset + 4]
+            tbl["accSum4"].nda[ii] = evt_data_32[offset + 5]
+            tbl["accSum5"].nda[ii] = evt_data_32[offset + 6]
+            tbl["accSum6"].nda[ii] = evt_data_32[offset + 7]
+            offset += 7
+        if format_bits & 0x2:
+            tbl["accSum7"].nda[ii] = evt_data_32[offset + 0]
+            tbl["accSum8"].nda[ii] = evt_data_32[offset + 1]
+            offset += 2
+        if format_bits & 0x4:
+            tbl["mawMax"].nda[ii] = evt_data_32[offset + 0]
+            tbl["mawBefore"].nda[ii] = evt_data_32[offset + 1]
+            tbl["mawAfter"].nda[ii] = evt_data_32[offset + 2]
+            offset += 3
+        if format_bits & 0x8:
+            tbl["startEnergy"].nda[ii] = evt_data_32[offset + 0]
+            tbl["maxEnergy"].nda[ii] = evt_data_32[offset + 1]
+            offset += 2
+
+        raw_length_32 = (evt_data_32[offset + 0]) & 0x03FFFFFF
+        tbl["status_flag"].nda[ii] = (
+            (evt_data_32[offset + 0]) & 0x04000000
+        ) >> 26  # bit 26
+        maw_test_flag = ((evt_data_32[offset + 0]) & 0x08000000) >> 27  # bit 27
+        avg_data_coming = False
+        if evt_data_32[offset + 0] & 0xF0000000 == 0xE0000000:
+            avg_data_coming = False
+        elif evt_data_32[offset + 0] & 0xF0000000 == 0xA0000000:
+            avg_data_coming = True
+        else:
+            raise RuntimeError("Data corruption 1!")
+        offset += 1
+        avg_length_32 = 0
+        if avg_data_coming:
+            avg_count_status = (
+                evt_data_32[offset + 0] & 0x00FF0000
+            ) >> 16  # bits 23 - 16
+            avg_length_32 = evt_data_32[offset + 0] & 0x0000FFFF
+            if evt_data_32[offset + 0] & 0xF0000000 != 0xE0000000:
+                raise RuntimeError("Data corruption 2!")
+            offset += 1
+
+        # --- now the offset points to the raw wf data ---
+
+        if maw_test_flag:
+            raise RuntimeError("Cannot handle data with MAW test data!")
+
+        # compute expected and actual array dimensions
+        raw_length_16 = 2 * raw_length_32
+        avg_length_16 = 2 * avg_length_32
+        header_length_16 = offset * 2
+        expected_wf_length = len(evt_data_16) - header_length_16
+
+        # error check: waveform size must match expectations
+        if raw_length_16 + avg_length_16 != expected_wf_length:
+            raise RuntimeError(
+                f"Waveform sizes {raw_length_16} (raw) and {avg_length_16} (avg) doesn't match expected size {expected_wf_length}."
+            )
+
+        # store waveform if available:
+        if raw_length_16 > 0:
+            tbl["waveform"]["values"].nda[ii] = evt_data_16[
+                offset * 2 : offset * 2 + raw_length_16
+            ]
+            offset += raw_length_32
+            tbl["waveform"]["t0"].nda[ii] = self.t0_raw[fch_id]
+
+        # store pre-averaged (avg) waveform if available:
+        if avg_length_16 > 0:
+            tbl["avgwaveform"]["values"].nda[ii] = evt_data_16[
+                offset * 2 : offset * 2 + avg_length_16
+            ]
+            offset += avg_length_32
+            # need to update avg waveform t0 based on the offset I get per event
+            tbl["avgwaveform"]["t0"].nda[ii] = (
+                self.t0_avg_const[fch_id]
+                + float(avg_count_status) * self.dt_raw[fch_id]
+            )
+
+        if offset != len(evt_data_32):
+            raise RuntimeError("I messed up...")
+
+        evt_rbkd[fch_id].loc += 1
+
+        return evt_rbkd[fch_id].is_full()
+
+    def __add_waveform(
+        self,
+        decoded_values_fch: dict[str, Any],
+        is_avg: bool,
+        max_samples: int,
+        dt: float,
+    ) -> None:
+        """
+        Averaged samples are available from the 125 MHz (16 bit) variatnt of the SIS3316 and can be stored independently of raw samples.
+        I use waveform for raw samples (dt from clock itself) and avgwaveform from averaged samples (dt from clock * avg number).
+
+        GERDA used to have the low-frequency (waveform) & the high-frequency (aux waveform); here: LF = avgwaveform & HF = waveform.
+        """
+        name: str = "avgwaveform" if is_avg else "waveform"
+        decoded_values_fch[name] = {
+            "dtype": "uint16",
+            "datatype": "waveform",
+            "wf_len": max_samples,  # max value. override this before initializing buffers to save RAM
+            "dt": dt,  # the sample pitch (inverse of clock speed)
+            # "t0": t0, # Adding t0 here does not work
+            "dt_units": "ns",
+            "t0_units": "ns",
+        }
+
+    def __add_accum1till6(self, decoded_values_fch: dict[str, Any]) -> None:
+        decoded_values_fch["peakHighValue"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["peakHighIndex"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["information"] = {"dtype": "uint32"}
+        decoded_values_fch["accSum1"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum2"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum3"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum4"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum5"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum6"] = {"dtype": "uint32", "units": "adc"}
+
+    def __add_accum7and8(self, decoded_values_fch: dict[str, Any]) -> None:
+        decoded_values_fch["accSum7"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["accSum8"] = {"dtype": "uint32", "units": "adc"}
+
+    def __add_maw(self, decoded_values_fch: dict[str, Any]) -> None:
+        decoded_values_fch["mawMax"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["mawBefore"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["mawAfter"] = {"dtype": "uint32", "units": "adc"}
+
+    def __add_energy(self, decoded_values_fch: dict[str, Any]) -> None:
+        decoded_values_fch["startEnergy"] = {"dtype": "uint32", "units": "adc"}
+        decoded_values_fch["maxEnergy"] = {"dtype": "uint32", "units": "adc"}

daq2lh5/llama/llama_header_decoder.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import io
+import logging
+from typing import Any, Dict
+
+import lgdo
+import numpy as np
+
+from ..data_decoder import DataDecoder
+from .llama_base import join_fadcid_chid
+
+log = logging.getLogger(__name__)
+
+LLAMA_Channel_Configs_t = Dict[int, Dict[str, Any]]
+
+
+class LLAMAHeaderDecoder(DataDecoder):  # DataDecoder currently unused
+    """
+    Decode llamaDAQ header data. Includes the file header as well as all available ("open") channel configurations.
+    """
+
+    @staticmethod
+    def magic_bytes() -> int:
+        return 0x4972414C
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.config = lgdo.Struct()
+        self.channel_configs = None
+
+    def decode_header(self, f_in: io.BufferedReader) -> lgdo.Struct:
+        n_bytes_read = 0
+
+        f_in.seek(0)  # should be there anyhow, but re-set if not
+        header = f_in.read(16)  # read 16 bytes
+        n_bytes_read += 16
+        evt_data_32 = np.frombuffer(header, dtype=np.uint32)
+        evt_data_16 = np.frombuffer(header, dtype=np.uint16)
+
+        # line0: magic bytes
+        magic = evt_data_32[0]
+        # print(hex(magic))
+        if magic == self.magic_bytes():
+            log.info("Read in file as llamaDAQ-SIS3316, magic bytes correct.")
+        else:
+            log.error("Magic bytes not matching for llamaDAQ file!")
+            raise RuntimeError("wrong file type")
+
+        self.version_major = evt_data_16[4]
+        self.version_minor = evt_data_16[3]
+        self.version_patch = evt_data_16[2]
+        self.length_econf = evt_data_16[5]
+        self.number_chOpen = evt_data_32[3]
+
+        log.debug(
+            f"File version: {self.version_major}.{self.version_minor}.{self.version_patch}"
+        )
+        log.debug(
+            f"{self.number_chOpen} channels open, each config {self.length_econf} bytes long"
+        )
+
+        n_bytes_read += self.__decode_channel_configs(f_in)
+
+        # print(self.channel_configs[0]["MAW3_offset"])
+
+        # assemble LGDO struct:
+        self.config.add_field("version_major", lgdo.Scalar(self.version_major))
+        self.config.add_field("version_minor", lgdo.Scalar(self.version_minor))
+        self.config.add_field("version_patch", lgdo.Scalar(self.version_patch))
+        self.config.add_field("length_econf", lgdo.Scalar(self.length_econf))
+        self.config.add_field("number_chOpen", lgdo.Scalar(self.number_chOpen))
+
+        for fch_id, fch_content in self.channel_configs.items():
+            fch_lgdo = lgdo.Struct()
+            for key, value in fch_content.items():
+                fch_lgdo.add_field(key, lgdo.Scalar(value))
+            self.config.add_field(f"fch_{fch_id:02d}", fch_lgdo)
+
+        return self.config, n_bytes_read
+
+    # override from DataDecoder
+    def make_lgdo(self, key: int = None, size: int = None) -> lgdo.Struct:
+        return self.config
+
+    def get_channel_configs(self) -> LLAMA_Channel_Configs_t:
+        return self.channel_configs
+
+    def __decode_channel_configs(self, f_in: io.BufferedReader) -> int:
+        """
+        Reads the metadata from the beginning of the file (the "channel configuration" part, directly after the file header).
+        Creates a dictionary of the metadata for each FADC/channel combination, which is returned
+
+        FADC-ID and channel-ID are combined into a single id for flattening:
+            (fadcid << 4) + chid
+
+        returns number of bytes read
+        """
+        # f_in.seek(16)    #should be after file header anyhow, but re-set if not
+        n_bytes_read = 0
+        self.channel_configs = {}
+
+        if self.length_econf != 88:
+            raise RuntimeError("Invalid channel configuration format")
+
+        for _i in range(0, self.number_chOpen):
+            # print("reading in channel config {}".format(i))
+
+            channel = f_in.read(self.length_econf)
+            n_bytes_read += self.length_econf
+            ch_dpf = channel[16:32]
+            evt_data_32 = np.frombuffer(channel, dtype=np.uint32)
+            evt_data_dpf = np.frombuffer(ch_dpf, dtype=np.float64)
+
+            fadc_index = evt_data_32[0]
+            channel_index = evt_data_32[1]
+            fch_id = join_fadcid_chid(fadc_index, channel_index)
+
+            if fch_id in self.channel_configs:
+                raise RuntimeError(
+                    f"duplicate channel configuration in file: FADCID: {fadc_index}, ChannelID: {channel_index}"
+                )
+            else:
+                self.channel_configs[fch_id] = {}
+
+            self.channel_configs[fch_id]["14BitFlag"] = evt_data_32[2] & 0x00000001
+            if evt_data_32[2] & 0x00000002 == 0:
+                log.warning("Channel in configuration marked as non-open!")
+            self.channel_configs[fch_id]["ADC_offset"] = evt_data_32[3]
+            self.channel_configs[fch_id]["sample_freq"] = evt_data_dpf[
+                0
+            ]  # 64 bit float
+            self.channel_configs[fch_id]["gain"] = evt_data_dpf[1]
+            self.channel_configs[fch_id]["format_bits"] = evt_data_32[8]
+            self.channel_configs[fch_id]["sample_start_index"] = evt_data_32[9]
+            self.channel_configs[fch_id]["sample_pretrigger"] = evt_data_32[10]
+            self.channel_configs[fch_id]["avg_sample_pretrigger"] = evt_data_32[11]
+            self.channel_configs[fch_id]["avg_mode"] = evt_data_32[12]
+            self.channel_configs[fch_id]["sample_length"] = evt_data_32[13]
+            self.channel_configs[fch_id]["avg_sample_length"] = evt_data_32[14]
+            self.channel_configs[fch_id]["MAW_buffer_length"] = evt_data_32[15]
+            self.channel_configs[fch_id]["event_length"] = evt_data_32[16]
+            self.channel_configs[fch_id]["event_header_length"] = evt_data_32[17]
+            self.channel_configs[fch_id]["accum6_offset"] = evt_data_32[18]
+            self.channel_configs[fch_id]["accum2_offset"] = evt_data_32[19]
+            self.channel_configs[fch_id]["MAW3_offset"] = evt_data_32[20]
+            self.channel_configs[fch_id]["energy_offset"] = evt_data_32[21]
+
+        return n_bytes_read

daq2lh5/llama/llama_streamer.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import logging
+
+import numpy as np
+
+from ..data_decoder import DataDecoder
+from ..data_streamer import DataStreamer
+from ..raw_buffer import RawBuffer, RawBufferLibrary
+from .llama_event_decoder import LLAMAEventDecoder
+from .llama_header_decoder import LLAMAHeaderDecoder
+
+log = logging.getLogger(__name__)
+
+
+class LLAMAStreamer(DataStreamer):
+    """
+    Decode SIS3316 data acquired using llamaDAQ.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.in_stream = None
+        self.event_rbkd = None
+        self.header_decoder = LLAMAHeaderDecoder()
+        self.event_decoder = LLAMAEventDecoder()
+
+    def get_decoder_list(self) -> list[DataDecoder]:
+        dec_list = []
+        dec_list.append(self.header_decoder)
+        dec_list.append(self.event_decoder)
+        return dec_list
+
+    def open_stream(
+        self,
+        llama_filename: str,
+        rb_lib: RawBufferLibrary = None,
+        buffer_size: int = 8192,
+        chunk_mode: str = "any_full",
+        out_stream: str = "",
+    ) -> list[RawBuffer]:
+        """Initialize the LLAMA data stream.
+
+        Refer to the documentation for
+        :meth:`.data_streamer.DataStreamer.open_stream` for a description
+        of the parameters.
+        """
+
+        if self.in_stream is not None:
+            raise RuntimeError("tried to open stream while previous one still open")
+        self.in_stream = open(llama_filename.encode("utf-8"), "rb")
+        self.n_bytes_read = 0
+        self.packet_id = 0
+
+        # read header info here
+        header, n_bytes_hdr = self.header_decoder.decode_header(self.in_stream)
+        self.n_bytes_read += n_bytes_hdr
+
+        self.event_decoder.set_channel_configs(
+            self.header_decoder.get_channel_configs()
+        )
+
+        # as far as I can tell, this happens if a user does not specify output.
+        # Then I can still get a rb_lib, but that misses keys entirely, which I need since channels can have different setups.
+        # So I try to hack my own here in case there is none provided.
+        # if rb_lib is None:
+        #    rb_lib = self.__hack_rb_lib(self.header_decoder.get_channel_configs())
+
+        # initialize the buffers in rb_lib. Store them for fast lookup
+        # Docu tells me to use initialize instead, but that does not exits (?)
+        super().open_stream(
+            llama_filename,
+            rb_lib,
+            buffer_size=buffer_size,
+            chunk_mode=chunk_mode,
+            out_stream=out_stream,
+        )
+        if rb_lib is None:
+            rb_lib = self.rb_lib
+
+        self.event_rbkd = (
+            rb_lib["LLAMAEventDecoder"].get_keyed_dict()
+            if "LLAMAEventDecoder" in rb_lib
+            else None
+        )
+
+        if "LLAMAHeaderDecoder" in rb_lib:
+            config_rb_list = rb_lib["LLAMAHeaderDecoder"]
+            if len(config_rb_list) != 1:
+                log.warning(
+                    f"config_rb_list had length {len(config_rb_list)}, ignoring all but the first"
+                )
+            rb = config_rb_list[0]
+        else:
+            rb = RawBuffer(lgdo=header)
+        rb.loc = 1  # we have filled this buffer
+        return [rb]
+
+    def close_stream(self) -> None:
+        if self.in_stream is None:
+            raise RuntimeError("tried to close an unopened stream")
+        self.in_stream.close()
+        self.in_stream = None
+
+    def read_packet(self) -> bool:
+        """Reads a single packet's worth of data in to the :class:`.RawBufferLibrary`.
+
+        Returns
+        -------
+        still_has_data
+            returns `True` while there is still data to read.
+        """
+
+        packet, fch_id = self.__read_bytes()
+        if packet is None:
+            return False  # EOF
+        self.packet_id += 1
+        self.n_bytes_read += len(packet)
+
+        self.any_full |= self.event_decoder.decode_packet(
+            packet, self.event_rbkd, self.packet_id, fch_id
+        )
+
+        return True
+
+    def __read_bytes(self) -> tuple[bytes | None, int]:
+        """
+        return bytes if read successful or None if EOF.
+        int is the fch_id (needs to be fetched to obtain the size of the event)
+        """
+        if self.in_stream is None:
+            raise RuntimeError("No stream open!")
+
+        position = self.in_stream.tell()  # save position of the event header's 1st byte
+        data1 = self.in_stream.read(
+            4
+        )  # read the first (32 bit) word of the event's header: channelID & format bits
+        if len(data1) < 4:
+            return None, -1  # EOF, I guess
+        self.in_stream.seek(position)  # go back to 1st position of event header
+
+        header_data_32 = np.frombuffer(data1, dtype=np.uint32)
+        fch_id = (header_data_32[0] >> 4) & 0x00000FFF
+
+        event_length_32 = self.header_decoder.get_channel_configs()[fch_id][
+            "event_length"
+        ]
+        event_length_8 = event_length_32 * 4
+
+        packet = self.in_stream.read(event_length_8)
+        if len(packet) < event_length_8:
+            raise RuntimeError(
+                f"Tried to read {event_length_8} bytes but got {len(packet)}"
+            )
+
+        return packet, fch_id

daq2lh5/logging.py

@@ -11,6 +11,8 @@ ERROR = logging.ERROR
 FATAL = logging.FATAL
 CRITICAL = logging.CRITICAL
 
+root = logging.root
+
 
 def setup(level: int = logging.INFO, logger: logging.Logger = None) -> None:
     """Setup a colorful logging output.

{legend_daq2lh5-1.2.2.dist-info → legend_daq2lh5-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: legend_daq2lh5
-Version: 1.2.2
+Version: 1.3.0
 Summary: Convert digitizer data to LH5
 Home-page: https://github.com/legend-exp/legend-daq2lh5
 Author: Jason Detwiler
@@ -26,27 +26,28 @@ Classifier: Topic :: Software Development
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dspeed >=1.3.0a4
-Requires-Dist: h5py >=3.2.0
+Requires-Dist: dspeed>=1.3.0a4
+Requires-Dist: h5py>=3.2.0
 Requires-Dist: hdf5plugin
-Requires-Dist: legend-pydataobj >=1.6
-Requires-Dist: numpy >=1.21
+Requires-Dist: legend-pydataobj>=1.6
+Requires-Dist: numpy>=1.21
 Requires-Dist: pyfcutils
-Requires-Dist: tqdm >=4.27
+Requires-Dist: pyyaml
+Requires-Dist: tqdm>=4.27
 Requires-Dist: xmltodict
 Provides-Extra: all
-Requires-Dist: legend-daq2lh5[docs,test] ; extra == 'all'
+Requires-Dist: legend-daq2lh5[docs,test]; extra == "all"
 Provides-Extra: docs
-Requires-Dist: furo ; extra == 'docs'
-Requires-Dist: myst-parser ; extra == 'docs'
-Requires-Dist: sphinx ; extra == 'docs'
-Requires-Dist: sphinx-copybutton ; extra == 'docs'
-Requires-Dist: sphinx-inline-tabs ; extra == 'docs'
+Requires-Dist: furo; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-copybutton; extra == "docs"
+Requires-Dist: sphinx-inline-tabs; extra == "docs"
 Provides-Extra: test
-Requires-Dist: pre-commit ; extra == 'test'
-Requires-Dist: pylegendtestdata ; extra == 'test'
-Requires-Dist: pytest >=6.0 ; extra == 'test'
-Requires-Dist: pytest-cov ; extra == 'test'
+Requires-Dist: pre-commit; extra == "test"
+Requires-Dist: pylegendtestdata; extra == "test"
+Requires-Dist: pytest>=6.0; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
 
 # legend-daq2lh5
 

{legend_daq2lh5-1.2.2.dist-info → legend_daq2lh5-1.3.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
 daq2lh5/__init__.py,sha256=VPmwKuZSA0icpce05ojhnsKWhR4_QUgD0oVXUoN9wks,975
-daq2lh5/_version.py,sha256=XEVwqOPlIChKtEnSO5v_SvghWXnn9WeQSoJ436w3v9Y,411
-daq2lh5/build_raw.py,sha256=JFXC5ln9u353TUZMksY3zydLiV2HlxqdI6_Y2_ZMCIE,10524
+daq2lh5/_version.py,sha256=HGwtpza1HCPtlyqElUvIyH97K44TO13CYiYVZNezQ1M,411
+daq2lh5/build_raw.py,sha256=w0sS8MEN-UhAHSjlPbqDlMBfzjeznIIrKxNxuRxPFqA,10531
 daq2lh5/cli.py,sha256=7bPfH1XbyAS48wZn_0unj4Y5MD5kF7V34Q5srn4jKVM,2913
 daq2lh5/data_decoder.py,sha256=Cn40fodfKs7pKa2odzG1j806iw9IyQVfbbWObNGmof8,10677
-daq2lh5/data_streamer.py,sha256=uv_pCFR45Yoi8GK7p_MYqGk_lzWspeg8cWZensCDqAQ,14131
-daq2lh5/logging.py,sha256=_1gq0S1PyzfQsSTiNm--ERJy-4FRfborTz0-vkMelyE,966
+daq2lh5/data_streamer.py,sha256=WuyqneVg8Kf7V072aUWYT5q3vmPZoobjAZ7oSw0sC9k,14187
+daq2lh5/logging.py,sha256=rYNeToaZBTCaIiC42a4CUroAo1PCOreTXbpEZyMO8Fo,987
 daq2lh5/raw_buffer.py,sha256=2jJvH7BbVkJCzF5A2nOIdcgwUs68eLnpvc0FvTjckaI,17688
 daq2lh5/buffer_processor/__init__.py,sha256=7k6v_KPximtv7805QnX4-xp_S3vqvqwDfdV3q95oZJo,84
 daq2lh5/buffer_processor/buffer_processor.py,sha256=GUxpNDbqGLuUEZmXjeratipbzmki12RFNYZkxgMtesg,14483
@@ -19,6 +19,10 @@ daq2lh5/fc/fc_config_decoder.py,sha256=RLRfUOZN0vYbAprqTymP7TGg641IiP9rgCGIOwWVK
 daq2lh5/fc/fc_event_decoder.py,sha256=JIRsySnxeuY3wmxjJOrTXo6wpelVup8WIvxU-fkPL-A,8131
 daq2lh5/fc/fc_status_decoder.py,sha256=o_3vTAgYXelZxIsreCYioVYid2mY-wqloYKlxoCqX5Q,3390
 daq2lh5/fc/fc_streamer.py,sha256=S0imXdVsiyolPvxI1uiBngpC58DporSNZPqx1HeVi5o,5737
+daq2lh5/llama/llama_base.py,sha256=B-NCBjE_FQE1WxijWi4Z1XBy4rqLHm2XhkC40s7Sdms,290
+daq2lh5/llama/llama_event_decoder.py,sha256=NGzUA1DLChcfnEUNqMAPWq4uqXwUd-FH2e-xXcj2lzM,13894
+daq2lh5/llama/llama_header_decoder.py,sha256=NB7wVH2r99hveQ2KJ-9YMkJMZ6ccNfFsjYa5HbuThAU,6114
+daq2lh5/llama/llama_streamer.py,sha256=Bmcj5Bs28KSV4y08TeJcntzUAkqz6HqlSNm7Kffgloc,5203
 daq2lh5/orca/__init__.py,sha256=Xf6uOIOzk_QkKH_7VizGlCo3iuiAgLtUE3A07x_HXC0,175
 daq2lh5/orca/orca_base.py,sha256=-XIolXsHj-1EdewaGxyvJTZvRGZsDyZe-5PzVOd-LFY,1333
 daq2lh5/orca/orca_digitizers.py,sha256=BsAA3OgQ13YIirDM8pd_xDY3F5FqEY4YjSHviflmov8,20867
@@ -28,9 +32,9 @@ daq2lh5/orca/orca_header_decoder.py,sha256=ORIIyfx22ybyKc-uyWy5ER49-dl3BGpHdfV8O
 daq2lh5/orca/orca_packet.py,sha256=nOHuBXsTI1SzTjHZtff0txSQYvkwo4XGx3fpk7XfYj8,2489
 daq2lh5/orca/orca_run_decoder.py,sha256=3atKXC6mDi8_PK6ICUBBJ-LyaTM8OU31kKWIpmttRr4,2065
 daq2lh5/orca/orca_streamer.py,sha256=VbD9PF-rx_Rk-rEy7XECPmgxr6kZSUf0tC7Qbol3Qeg,15693
-legend_daq2lh5-1.2.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-legend_daq2lh5-1.2.2.dist-info/METADATA,sha256=3fISxtb-PvR-D2YG03ZuQYi95ORmLDCPk0w2f0e-wDc,3950
-legend_daq2lh5-1.2.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-legend_daq2lh5-1.2.2.dist-info/entry_points.txt,sha256=R08R4NrHi0ab5MJN_qKqzePVzrLSsw5WpmbiwwduYjw,59
-legend_daq2lh5-1.2.2.dist-info/top_level.txt,sha256=MJQVLyLqMgMKBdVfNXFaCKCjHKakAs19VLbC9ctXZ7A,8
-legend_daq2lh5-1.2.2.dist-info/RECORD,,
+legend_daq2lh5-1.3.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+legend_daq2lh5-1.3.0.dist-info/METADATA,sha256=3fR5Z-61fjaRaAEObVk9k8HpTGxiDF0wL4f6djIX8aQ,3956
+legend_daq2lh5-1.3.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+legend_daq2lh5-1.3.0.dist-info/entry_points.txt,sha256=R08R4NrHi0ab5MJN_qKqzePVzrLSsw5WpmbiwwduYjw,59
+legend_daq2lh5-1.3.0.dist-info/top_level.txt,sha256=MJQVLyLqMgMKBdVfNXFaCKCjHKakAs19VLbC9ctXZ7A,8
+legend_daq2lh5-1.3.0.dist-info/RECORD,,

{legend_daq2lh5-1.2.2.dist-info → legend_daq2lh5-1.3.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any