legend-daq2lh5 1.5.0__py3-none-any.whl → 1.6.1__py3-none-any.whl

daq2lh5/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.5.0'
-__version_tuple__ = version_tuple = (1, 5, 0)
+__version__ = version = '1.6.1'
+__version_tuple__ = version_tuple = (1, 6, 1)

daq2lh5/buffer_processor/buffer_processor.py

@@ -18,7 +18,7 @@ if TYPE_CHECKING:
 log = logging.getLogger(__name__)
 
 
-def buffer_processor(rb: RawBuffer) -> Table:
+def buffer_processor(rb: RawBuffer, db_dict: dict = None) -> Table:
     r"""Process raw data buffers.
 
     Takes in a :class:`.RawBuffer`, performs any processes specified in the
@@ -142,7 +142,7 @@ def buffer_processor(rb: RawBuffer) -> Table:
 
     # Read in and perform the DSP routine
     if "dsp_config" in rb.proc_spec.keys():
-        process_dsp(rb, tmp_table)
+        process_dsp(rb, tmp_table, db_dict)
 
     # Cast as requested dtype before writing to the table
     if "dtype_conv" in rb.proc_spec.keys():
@@ -294,7 +294,7 @@ def process_windowed_t0(t0s: Array, dts: Array, start_index: int) -> Array:
     return copy_t0s
 
 
-def process_dsp(rb: RawBuffer, tmp_table: Table) -> None:
+def process_dsp(rb: RawBuffer, tmp_table: Table, db_dict: dict = None) -> None:
     r"""Run a DSP processing chain with :mod:`dspeed`.
 
     Run a provided DSP config from `rb.proc_spec` using
@@ -309,6 +309,8 @@ def process_dsp(rb: RawBuffer, tmp_table: Table) -> None:
     tmp_table
         a :class:`lgdo.Table` that is temporarily created to be written
         to the raw file.
+    db_dict
+        a database dictionary storing parameters for each channel
 
     Notes
     -----
@@ -322,10 +324,11 @@ def process_dsp(rb: RawBuffer, tmp_table: Table) -> None:
     try:
         # execute the processing chain
         # This checks that the rb.lgdo is a table and that the field_name is present in the table
-        proc_chain, mask, dsp_out = bpc(rb.lgdo, dsp_dict)
+        proc_chain, mask, dsp_out = bpc(rb.lgdo, dsp_dict, db_dict=db_dict)
     # Allow for exceptions, in the case of "*" key expansion in the build_raw out_spec
-    except ProcessingChainError:
+    except ProcessingChainError as e:
         log.info("DSP could not be performed")
+        log.info(f"Error: {e}")
         return None
 
     proc_chain.execute()

daq2lh5/buffer_processor/lh5_buffer_processor.py

@@ -19,6 +19,7 @@ def lh5_buffer_processor(
     overwrite: bool = False,
     out_spec: dict = None,
     proc_file_name: str = None,
+    db_dict: dict = None,
 ) -> None:
     r"""Process raw buffers from an LH5 file.
 
@@ -74,14 +75,14 @@ def lh5_buffer_processor(
         # Look one layer deeper for a :meth:`lgdo.Table` if necessary
         elif lh5.ls(lh5_file, f"{tb}"):
             # Check to make sure that this isn't a table itself
-            maybe_table = raw_store.read(f"{tb}", lh5_file)
+            maybe_table = lh5.read(f"{tb}", lh5_file)
             if isinstance(maybe_table, lgdo.Table):
                 lh5_tables.append(f"{tb}")
                 del maybe_table
             # otherwise, go deeper
             else:
                 for sub_table in lh5.ls(lh5_file, f"{tb}"):
-                    maybe_table, _ = raw_store.read(f"{tb}/{sub_table}", lh5_file)
+                    maybe_table = lh5.read(f"{tb}/{sub_table}", lh5_file)
                     if isinstance(maybe_table, lgdo.Table):
                         lh5_tables.append(f"{tb}/{sub_table}")
                     del maybe_table
@@ -114,7 +115,7 @@ def lh5_buffer_processor(
 
     # Write everything in the raw file to the new file, check for proc_spec under either the group name, out_name, or the name
     for tb in lh5_tables:
-        lgdo_obj = raw_store.read(f"{tb}", lh5_file)
+        lgdo_obj = lh5.read(f"{tb}", lh5_file)
 
         # Find the out_name.
         # If the top level group has an lgdo table in it, then the out_name is group
@@ -138,7 +139,9 @@ def lh5_buffer_processor(
                         out_name=out_name,
                         proc_spec=out_spec[decoder_name][group_name]["proc_spec"],
                     )
-                    tmp_table = buffer_processor(rb)
+                    tmp_table = buffer_processor(
+                        rb, db_dict=db_dict[tb] if db_dict is not None else None
+                    )
                     # Update the lgdo_obj to be written to the processed file
                     lgdo_obj = tmp_table
                 else:
@@ -198,4 +201,4 @@ def lh5_buffer_processor(
                 pass
 
         # Write the (possibly processed) lgdo_obj to a file
-        raw_store.write(lgdo_obj, out_name, lh5_file=proc_file_name, group=group_name)
+        lh5.write(lgdo_obj, out_name, lh5_file=proc_file_name, group=group_name)

daq2lh5/build_raw.py

@@ -28,6 +28,7 @@ def build_raw(
     overwrite: bool = False,
     compass_config_file: str = None,
     hdf5_settings: dict[str, ...] = None,
+    db_dict: dict = None,
     **kwargs,
 ) -> None:
     """Convert data into LEGEND HDF5 raw-tier format.
@@ -200,7 +201,7 @@ def build_raw(
         in_stream,
         rb_lib=rb_lib,
         buffer_size=buffer_size,
-        chunk_mode="full_only",
+        chunk_mode="only_full",
         out_stream=out_stream,
     )
     rb_lib = streamer.rb_lib
@@ -230,7 +231,7 @@ def build_raw(
 
     # Write header data
     lh5_store = lh5.LH5Store(keep_open=True)
-    write_to_lh5_and_clear(header_data, lh5_store, **hdf5_settings)
+    write_to_lh5_and_clear(header_data, lh5_store, db_dict, **hdf5_settings)
 
     # Now loop through the data
     n_bytes_last = streamer.n_bytes_read
@@ -255,7 +256,7 @@ def build_raw(
         if log.getEffectiveLevel() <= logging.INFO and n_max < np.inf:
             progress_bar.update(n_read)
 
-        write_to_lh5_and_clear(chunk_list, lh5_store, **hdf5_settings)
+        write_to_lh5_and_clear(chunk_list, lh5_store, db_dict, **hdf5_settings)
 
         if n_max <= 0:
             log.info(f"Wrote {n_max} rows, exiting...")

daq2lh5/data_decoder.py

@@ -253,12 +253,10 @@ class DataDecoder:
     def write_out_garbage(
         self, filename: str, group: str = "/", lh5_store: LH5Store = None
     ) -> None:
-        if lh5_store is None:
-            lh5_store = lgdo.LH5Store()
         n_rows = self.garbage_table.loc
         if n_rows == 0:
             return
-        lh5_store.write(
+        lgdo.lh5.write(
             self.garbage_table, "garbage", filename, group, n_rows=n_rows, append=True
         )
         self.garbage_table.clear()

daq2lh5/data_streamer.py

@@ -97,7 +97,6 @@ class DataStreamer(ABC):
         dec_names = []
         for decoder in decoders:
             dec_name = type(decoder).__name__
-
             # set up wildcard decoder buffers
             if dec_name not in rb_lib:
                 if "*" not in rb_lib:
@@ -128,33 +127,78 @@ class DataStreamer(ABC):
             # dec_name is in rb_lib: store the name, and initialize its buffer lgdos
             dec_names.append(dec_name)
 
-            # set up wildcard key buffers
+            # Parse wildcard keys in RawBuffers and replace with known keys of the decoder.
+            dec_key_list = set(sum(decoder.get_key_lists(), []))
+            log.debug(f"{dec_name} offers keys: {dec_key_list}")
+
+            # track keys which are already used
+            matched_keys = set()
+            only_wildcard_rb = None
+            wildcard_rbs = []
+            # find wildcard key buffers
             for rb in rb_lib[dec_name]:
-                if (
-                    len(rb.key_list) == 1
-                    and isinstance(rb.key_list[0], str)
-                    and "*" in rb.key_list[0]
-                ):
-                    matched_key_lists = []
-                    for key_list in decoder.get_key_lists():
-                        # special case: decoders without keys
-                        if rb.key_list[0] == "*" and key_list == [None]:
-                            matched_key_lists.append(key_list)
-                            continue
-                        key_type = type(key_list[0])
-                        for ik in range(len(key_list)):
-                            key_list[ik] = str(key_list[ik])
-                        matched_keys = fnmatch.filter(key_list, rb.key_list[0])
-                        if len(matched_keys) > 1:
-                            for ik in range(len(matched_keys)):
-                                matched_keys[ik] = key_type(key_list[ik])
-                            matched_key_lists.append(matched_keys)
-                    if len(matched_key_lists) == 0:
-                        log.warning(
-                            f"no matched keys for key_list {rb.key_list[0]} in {dec_name}.{rb.out_name}"
-                        )
+                log.debug(f"rb {rb.out_name} seeks keys: {rb.key_list}")
+                for key in rb.key_list:
+                    # only string can contain wildcard *
+                    if not isinstance(key, str):
+                        matched_keys.add(key)
                         continue
-                    rb.key_list = sum(matched_key_lists, [])
+                    if key == "*":
+                        if only_wildcard_rb is None:
+                            only_wildcard_rb = rb
+                        else:
+                            raise KeyError(
+                                f"Only one '*' wildcard key allowed for decoder {dec_name}"
+                            )
+
+                    elif "*" in key:
+                        wildcard_rbs.append(rb)
+                    else:
+                        matched_keys.add(key)
+
+            # append pure wildcard, so it matches last
+            if only_wildcard_rb is not None:
+                wildcard_rbs.append(only_wildcard_rb)
+
+            # remove already matched keys with original key type
+            dec_key_list = dec_key_list.difference(matched_keys)
+            dec_key_list = set(map(str, dec_key_list))
+            # remove already matched keys with str key type
+            dec_key_list = dec_key_list.difference(matched_keys)
+
+            log.debug(f"{dec_name} remaining keys: {dec_key_list}")
+
+            for rb in wildcard_rbs:
+                matched_keys = set()
+                for key in rb.key_list:
+                    # find matching keys in the decoder list
+                    matches = set(fnmatch.filter(dec_key_list, key))
+                    dec_key_list = dec_key_list.difference(matches)
+
+                    log.debug(f"{dec_name} {key} matched keys: {matches}")
+                    log.debug(f"{dec_name} remaining keys: {dec_key_list}")
+                    matched_keys |= matches
+
+                # Construct the new key_list for the RawBuffer
+                # Expect anything that can be cast to int wants to be cast
+                rb.key_list = []
+                for key in matched_keys:
+                    if key == "None":
+                        rb.key_list.append(None)
+                    try:
+                        new_key = int(key)
+                        rb.key_list.append(new_key)
+                    except ValueError:
+                        rb.key_list.append(key)
+
+                if len(rb.key_list) == 0:
+                    log.warning(
+                        f"no matched keys for key_list {rb.key_list} in {dec_name}.{rb.out_name}"
+                    )
+                log.debug(
+                    f"{dec_name}:{rb.out_stream}/{rb.out_name} matched wildcards to {rb.key_list}"
+                )
+
             keyed_name_rbs = []
             ii = 0
             while ii < len(rb_lib[dec_name]):
@@ -175,7 +219,6 @@ class DataStreamer(ABC):
                         else:
                             key = str(key)
                         expanded_name = rb.out_name.format(key=key)
-
                     new_rb = RawBuffer(
                         key_list=[key],
                         out_stream=rb.out_stream,
@@ -191,7 +234,7 @@ class DataStreamer(ABC):
                 rb.fill_safety = decoder.get_max_rows_in_packet()
                 if buffer_size < rb.fill_safety:
                     raise ValueError(
-                        f"{dec_name} requires a buffer of at least length"
+                        f"{dec_name} requires a buffer of at least length "
                         f"{rb.fill_safety} but buffer size is only {buffer_size}"
                     )
 
@@ -333,6 +376,7 @@ class DataStreamer(ABC):
         """
         rb_lib = RawBufferLibrary()
         decoders = self.get_decoder_list()
+        log.debug(f"Default rb_lib knows about: {decoders}")
         if len(decoders) == 0:
             log.warning(
                 f"no decoders returned by get_decoder_list() for {type(self).__name__}"
@@ -344,6 +388,7 @@ class DataStreamer(ABC):
             if dec_key.endswith("Decoder"):
                 dec_key = dec_key.removesuffix("Decoder")
             key_lists = decoder.get_key_lists()
+            log.debug(f"{dec_key} supports keys {key_lists}")
             for ii, key_list in enumerate(key_lists):
                 this_name = dec_key
                 if len(key_lists) > 1:

daq2lh5/fc/fc_config_decoder.py

@@ -1,15 +1,47 @@
 from __future__ import annotations
 
+import copy
 import logging
+from typing import Any
 
-import fcutils
 import lgdo
 import numpy as np
+from fcio import FCIO, Limits
 
 from ..data_decoder import DataDecoder
 
 log = logging.getLogger(__name__)
 
+fc_config_decoded_values = {
+    "packet_id": {
+        "dtype": "uint32",
+        "description": "The index of this decoded packet in the file.",
+    },
+    "nsamples": {"dtype": "int32", "description": "samples per channel"},
+    "nadcs": {"dtype": "int32", "description": "number of adc channels"},
+    "ntriggers": {"dtype": "int32", "description": "number of triggertraces"},
+    "streamid": {"dtype": "int32", "description": "id of stream"},
+    "adcbits": {"dtype": "int32", "description": "bit range of the adc channels"},
+    "sumlength": {"dtype": "int32", "description": "length of the fpga integrator"},
+    "blprecision": {"dtype": "int32", "description": "precision of the fpga baseline"},
+    "mastercards": {"dtype": "int32", "description": "number of attached mastercards"},
+    "triggercards": {
+        "dtype": "int32",
+        "description": "number of attached triggercards",
+    },
+    "adccards": {"dtype": "int32", "description": "number of attached fadccards"},
+    "gps": {
+        "dtype": "int32",
+        "description": "gps mode (0: not used, >0: external pps and 10MHz)",
+    },
+    "tracemap": {
+        "dtype": "uint32",
+        "datatype": "array<1>{array<1>{real}}",
+        "length": Limits.MaxChannels,
+        "description": "",
+    },
+}
+
 
 class FCConfigDecoder(DataDecoder):
     """Decode FlashCam config data.
@@ -22,9 +54,9 @@ class FCConfigDecoder(DataDecoder):
 
     Example
     -------
-    >>> import fcutils
-    >>> from daq2lh5.fc.fc_config_decoder import FCConfigDecoder
-    >>> fc = fcutils.fcio('file.fcio')
+    >>> from fcio import fcio_open
+    >>> from daq2lh5.fc.config_decoder import FCConfigDecoder
+    >>> fc = fcio_open('file.fcio')
     >>> decoder = FCConfigDecoder()
     >>> config = decoder.decode_config(fc)
     >>> type(config)
@@ -33,29 +65,68 @@ class FCConfigDecoder(DataDecoder):
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
-        self.config = lgdo.Struct()
-
-    def decode_config(self, fcio: fcutils.fcio) -> lgdo.Struct:
-        config_names = [
-            "nsamples",  # samples per channel
-            "nadcs",  # number of adc channels
-            "ntriggers",  # number of triggertraces
-            "telid",  # id of telescope
-            "adcbits",  # bit range of the adc channels
-            "sumlength",  # length of the fpga integrator
-            "blprecision",  # precision of the fpga baseline
-            "mastercards",  # number of attached mastercards
-            "triggercards",  # number of attached triggercards
-            "adccards",  # number of attached fadccards
-            "gps",  # gps mode (0: not used, 1: external pps and 10MHz)
-        ]
-        for name in config_names:
-            if name in self.config:
-                log.warning(f"{name} already in self.config. skipping...")
+        self.decoded_values = copy.deepcopy(fc_config_decoded_values)
+
+    def decode_packet(
+        self,
+        fcio: FCIO,
+        config_rb: lgdo.Table,
+        packet_id: int,
+    ) -> bool:
+
+        tbl = config_rb.lgdo
+        loc = config_rb.loc
+
+        tbl["packet_id"].nda[loc] = packet_id
+
+        tbl["nsamples"].nda[loc] = fcio.config.eventsamples
+        tbl["nadcs"].nda[loc] = fcio.config.adcs
+        tbl["ntriggers"].nda[loc] = fcio.config.triggers
+        tbl["streamid"].nda[loc] = fcio.config.streamid
+        tbl["adcbits"].nda[loc] = fcio.config.adcbits
+        tbl["sumlength"].nda[loc] = fcio.config.sumlength
+        tbl["blprecision"].nda[loc] = fcio.config.blprecision
+        tbl["mastercards"].nda[loc] = fcio.config.mastercards
+        tbl["triggercards"].nda[loc] = fcio.config.triggercards
+        tbl["adccards"].nda[loc] = fcio.config.adccards
+        tbl["gps"].nda[loc] = fcio.config.gps
+        ntraces = fcio.config.adcs + fcio.config.triggers
+        tbl["tracemap"]._set_vector_unsafe(loc, fcio.config.tracemap[:ntraces])
+
+        config_rb.loc += 1
+
+        return config_rb.is_full()
+
+    def decode_config(self, fcio: FCIO) -> lgdo.Struct:
+
+        tbl = lgdo.Struct()
+
+        fcio_attr_names_map = {
+            "nsamples": "eventsamples",
+            "nadcs": "adcs",
+            "ntriggers": "triggers",
+            "streamid": "streamid",
+            "adcbits": "adcbits",
+            "sumlength": "sumlength",
+            "blprecision": "blprecision",
+            "mastercards": "mastercards",
+            "triggercards": "triggercards",
+            "adccards": "adccards",
+            "gps": "gps",
+        }
+
+        for name, fcio_attr_name in fcio_attr_names_map.items():
+            if name in tbl:
+                log.warning(f"{name} already in tbl. skipping...")
                 continue
-            value = np.int32(getattr(fcio, name))  # all config fields are int32
-            self.config.add_field(name, lgdo.Scalar(value))
-        return self.config
+            value = np.int32(
+                getattr(fcio.config, fcio_attr_name)
+            )  # all config fields are int32
+            tbl.add_field(name, lgdo.Scalar(value))
+        ntraces = fcio.config.adcs + fcio.config.triggers
+        tbl.add_field("tracemap", lgdo.Array(fcio.config.tracemap[:ntraces]))
+
+        return tbl
 
-    def make_lgdo(self, key: int = None, size: int = None) -> lgdo.Struct:
-        return self.config
+    def get_decoded_values(self, key: int | str = None) -> dict[str, dict[str, Any]]:
+        return self.decoded_values