reboost 0.6.2__py3-none-any.whl → 0.8.0__py3-none-any.whl

reboost/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.6.2'
-__version_tuple__ = version_tuple = (0, 6, 2)
+__version__ = version = '0.8.0'
+__version_tuple__ = version_tuple = (0, 8, 0)
+
+__commit_id__ = commit_id = None

reboost/build_hit.py

@@ -7,11 +7,16 @@ A :func:`build_hit` to parse the following configuration file:
     # dictionary of objects useful for later computation. they are constructed with
     # auxiliary data (e.g. metadata). They can be accessed later as OBJECTS (all caps)
     objects:
-     lmeta: LegendMetadata(ARGS.legendmetadata)
+     lmeta: legendmeta.LegendMetadata(ARGS.legendmetadata)
      geometry: pyg4ometry.load(ARGS.gdml)
      user_pars: dbetto.TextDB(ARGS.par)
      dataprod_pars: dbetto.TextDB(ARGS.dataprod_cycle)
 
+     _spms: OBJECTS.lmeta.channelmap(on=ARGS.timestamp)
+        .group("system").spms
+        .map("name")
+     spms: "{name: spm.daq.rawid for name, spm in OBJECTS._spms.items()}"
+
     # processing chain is defined to act on a group of detectors
     processing_groups:
 
@@ -107,9 +112,10 @@ A :func:`build_hit` to parse the following configuration file:
           outputs:
             - evtid
             - tot_edep_wlsr
+            - num_scint_ph_lar
 
           operations:
-            tot_edep_wlsr: ak.sum(HITS[(HITS.__detuid == 0) & (HITS.__zloc < 3000)].__edep, axis=-1)
+            tot_edep_wlsr: ak.sum(HITS.edep[np.abs(HITS.zloc) < 3000], axis=-1)
 
         - name: spms
 
@@ -117,11 +123,14 @@ A :func:`build_hit` to parse the following configuration file:
           # same name as the current detector. This can be overridden for special processors
 
           detector_mapping:
-           - output: OBJECTS.lmeta.channglmap(on=ARGS.timestamp)
-            .group("system").spms
-            .group("analysis.status").on
-            .map("name").keys()
-           - input: lar
+           - output: OBJECTS.spms.keys()
+             input: lar
+
+          hit_table_layout: reboost.shape.group_by_time(STEPS, window=10)
+
+          pre_operations:
+            num_scint_ph_lar: reboost.spms.emitted_scintillation_photons(HITS.edep, HITS.particle, "lar")
+            # num_scint_ph_pen: ...
 
           outputs:
             - t0
@@ -130,22 +139,23 @@ A :func:`build_hit` to parse the following configuration file:
 
           detector_objects:
              meta: pygeomtools.get_sensvol_metadata(OBJECTS.geometry, DETECTOR)
-             optmap_lar: lgdo.lh5.read(DETECTOR, "optmaps/pen", ARGS.optmap_path)
-             optmap_pen: lgdo.lh5.read(DETECTOR, "optmaps/lar", ARGS.optmap_path)
-
-          hit_table_layout: reboost.shape.group_by_time(STEPS, window=10)
+             spm_uid: OBJECTS.spms[DETECTOR]
+             optmap_lar: reboost.spms.load_optmap(ARGS.optmap_path_pen, DETECTOR_OBJECTS.spm_uid)
+             optmap_pen: reboost.spms.load_optmap(ARGS.optmap_path_lar, DETECTOR_OBJECTS.spm_uid)
 
           operations:
             pe_times_lar: reboost.spms.detected_photoelectrons(
-                STEPS,
+                HITS.num_scint_ph_lar, HITS.particle, HITS.time, HITS.xloc, HITS.yloc, HITS.zloc,
                 DETECTOR_OBJECTS.optmap_lar,
-                0
+                "lar",
+                DETECTOR_OBJECTS.spm_uid
              )
 
             pe_times_pen: reboost.spms.detected_photoelectrons(
-                STEPS,
+                HITS.num_scint_ph_pen, HITS.particle, HITS.time, HITS.xloc, HITS.yloc, HITS.zloc,
                 DETECTOR_OBJECTS.optmap_pen,
-                1
+                "pen",
+                DETECTOR_OBJECTS.spm_uid
              )
 
             pe_times: ak.concatenate([HITS.pe_times_lar, HITS.pe_times_pen], axis=-1)
@@ -189,6 +199,7 @@ def build_hit(
     out_field: str = "hit",
     buffer: int = int(5e6),
     overwrite: bool = False,
+    allow_missing_inputs=True,
 ) -> None | ak.Array:
     """Build the hit tier from the remage step files.
 
@@ -215,6 +226,8 @@ def build_hit(
         buffer size for use in the `LH5Iterator`.
     overwrite
         flag to overwrite the existing output.
+    allow_missing_inputs
+        Flag to allow an input table to be missing, generally when there were no events.
     """
     # extract the config file
     if isinstance(config, str):
@@ -238,7 +251,7 @@ def build_hit(
     output_tables_names = set()
 
     # iterate over files
-    for file_idx, (stp_file, glm_file) in enumerate(zip(files.stp, files.glm)):
+    for file_idx, (stp_file, glm_file) in enumerate(zip(files.stp, files.glm, strict=True)):
         msg = (
             f"starting processing of {stp_file} to {files.hit[file_idx]} "
             if files.hit[file_idx] is not None
@@ -257,7 +270,7 @@ def build_hit(
 
             # extract the output detectors and the mapping to input detectors
             detectors_mapping = core.get_detector_mapping(
-                proc_group.get("detector_mapping"), global_objects
+                proc_group.get("detector_mapping"), global_objects, args
             )
 
             # loop over detectors
@@ -276,20 +289,30 @@ def build_hit(
 
                 lh5_group = proc_group.get("lh5_group", "stp")
                 if lh5_group is None:
-                    lh5_group = "/"
+                    lh5_group = ""
+                    table = in_detector
+                else:
+                    table = f"{lh5_group}/{in_detector}"
 
                 # begin iterating over the glm
-                iterator = GLMIterator(
-                    glm_file,
-                    stp_file,
-                    lh5_group=in_detector,
-                    start_row=start_evtid,
-                    stp_field=lh5_group,
-                    n_rows=n_evtid,
-                    buffer=buffer,
-                    time_dict=time_dict[proc_name],
-                    reshaped_files="hit_table_layout" not in proc_group,
-                )
+                # check if the in_detector is in the file
+                if table in lh5.ls(stp_file, lh5_group + "/"):
+                    iterator = GLMIterator(
+                        glm_file,
+                        stp_file,
+                        lh5_group=in_detector,
+                        start_row=start_evtid,
+                        stp_field=lh5_group,
+                        n_rows=n_evtid,
+                        buffer=buffer,
+                        time_dict=time_dict[proc_name],
+                        reshaped_files="hit_table_layout" not in proc_group,
+                    )
+                elif allow_missing_inputs:
+                    continue
+                else:
+                    msg = f"Requested input detector {in_detector} is not present in the group {lh5_group} and missing inputs were not allowed"
+                    raise ValueError(msg)
 
                 for stps, chunk_idx, _ in iterator:
                     # converting to awkward
@@ -305,6 +328,21 @@ def build_hit(
                     if time_dict is not None:
                         time_dict[proc_name].update_field("conv", start_time)
 
+                    if "hit_table_layout" in proc_group:
+                        hit_table_layouted = core.evaluate_hit_table_layout(
+                            copy.deepcopy(ak_obj),
+                            expression=proc_group["hit_table_layout"],
+                            time_dict=time_dict[proc_name],
+                        )
+                    else:
+                        hit_table_layouted = copy.deepcopy(stps)
+
+                    local_dict = {"OBJECTS": global_objects}
+                    for field, info in proc_group.get("pre_operations", {}).items():
+                        _evaluate_operation(
+                            hit_table_layouted, field, info, local_dict, time_dict[proc_name]
+                        )
+
                     # produce the hit table
                     for out_det_idx, out_detector in enumerate(out_detectors):
                         # loop over the rows
@@ -314,14 +352,12 @@ def build_hit(
                         # get the attributes
                         attrs = utils.copy_units(stps)
 
-                        if "hit_table_layout" in proc_group:
-                            hit_table = core.evaluate_hit_table_layout(
-                                copy.deepcopy(ak_obj),
-                                expression=proc_group["hit_table_layout"],
-                                time_dict=time_dict[proc_name],
-                            )
-                        else:
-                            hit_table = copy.deepcopy(stps)
+                        # if we have more than one output detector, make an independent copy.
+                        hit_table = (
+                            copy.deepcopy(hit_table_layouted)
+                            if len(out_detectors) > 1
+                            else hit_table_layouted
+                        )
 
                         local_dict = {
                             "DETECTOR_OBJECTS": det_objects[out_detector],
@@ -330,28 +366,10 @@ def build_hit(
                         }
                         # add fields
                         for field, info in proc_group.get("operations", {}).items():
-                            if isinstance(info, str):
-                                expression = info
-                                units = None
-                            else:
-                                expression = info["expression"]
-                                units = info.get("units", None)
-
-                            # evaluate the expression
-                            col = core.evaluate_output_column(
-                                hit_table,
-                                table_name="HITS",
-                                expression=expression,
-                                local_dict=local_dict,
-                                time_dict=time_dict[proc_name],
-                                name=field,
+                            _evaluate_operation(
+                                hit_table, field, info, local_dict, time_dict[proc_name]
                             )
 
-                            if units is not None:
-                                col.attrs["units"] = units
-
-                            core.add_field_with_nesting(hit_table, field, col)
-
                         # remove unwanted fields
                         if "outputs" in proc_group:
                             hit_table = core.remove_columns(
@@ -414,9 +432,35 @@ def build_hit(
                     raise RuntimeError(msg) from e
 
     # return output table or nothing
-    log.debug(time_dict)
+    log.info(time_dict)
 
     if output_tables == {}:
         output_tables = None
 
     return output_tables, time_dict
+
+
+def _evaluate_operation(
+    hit_table, field: str, info: str | dict, local_dict: dict, time_dict: ProfileDict
+) -> None:
+    if isinstance(info, str):
+        expression = info
+        units = None
+    else:
+        expression = info["expression"]
+        units = info.get("units", None)
+
+    # evaluate the expression
+    col = core.evaluate_output_column(
+        hit_table,
+        table_name="HITS",
+        expression=expression,
+        local_dict=local_dict,
+        time_dict=time_dict,
+        name=field,
+    )
+
+    if units is not None:
+        col.attrs["units"] = units
+
+    core.add_field_with_nesting(hit_table, field, col)

reboost/cli.py

@@ -191,3 +191,4 @@ def cli(args=None) -> None:
             buffer=args.buffer,
             overwrite=args.overwrite,
         )
+        log.info("successfully finished post-processing")

reboost/core.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import logging
 import time
 from typing import Any
@@ -145,11 +146,14 @@ def evaluate_output_column(
     if globals_dict == {}:
         globals_dict = None
 
+    ctx = contextlib.nullcontext()
     if globals_dict is not None and "pyg4ometry" in globals_dict:
-        with utils.filter_logging(logging.CRITICAL):
-            res = hit_table.eval(func_call, local_dict, modules=globals_dict)
-    else:
-        res = hit_table.eval(func_call, local_dict, modules=globals_dict)
+        ctx = utils.filter_logging(logging.CRITICAL)
+
+    with ctx:
+        res = hit_table.eval(
+            func_call, local_dict, modules=globals_dict, library="ak", with_units=True
+        )
 
     # how long did it take
     if time_dict is not None:
@@ -227,7 +231,9 @@ def get_global_objects(
     return AttrsDict(res)
 
 
-def get_detector_mapping(detector_mapping: dict, global_objects: AttrsDict) -> dict:
+def get_detector_mapping(
+    detector_mapping: dict, global_objects: AttrsDict, args: AttrsDict
+) -> dict:
     """Get all the detector mapping using :func:`get_one_detector_mapping`.
 
     Parameters
@@ -236,6 +242,8 @@ def get_detector_mapping(detector_mapping: dict, global_objects: AttrsDict) -> d
         dictionary of detector mapping
     global_objects
         dictionary of global objects to use in evaluating the mapping.
+    args
+        any arguments the expression can depend on, is passed as `locals` to `eval()`.
     """
     return utils.merge_dicts(
         [
@@ -243,6 +251,7 @@ def get_detector_mapping(detector_mapping: dict, global_objects: AttrsDict) -> d
                 mapping["output"],
                 input_detector_name=mapping.get("input", None),
                 objects=global_objects,
+                args=args,
             )
             for mapping in detector_mapping
         ]
@@ -253,6 +262,7 @@ def get_one_detector_mapping(
     output_detector_expression: str | list,
     objects: AttrsDict | None = None,
     input_detector_name: str | None = None,
+    args: AttrsDict | None = None,
 ) -> dict:
     """Extract the output detectors and the list of input to outputs by parsing the expressions.
 
@@ -282,7 +292,8 @@ def get_one_detector_mapping(
         dictionary of objects that can be referenced in the expression.
     input_detector_name
         Optional input detector name for all the outputs.
-
+    args
+        any arguments the expression can depend on, is passed as `locals` to `eval()`.
 
     Returns
     -------
@@ -314,11 +325,9 @@ def get_one_detector_mapping(
         out_list = list(output_detector_expression)
 
     for expression_tmp in out_list:
-        func, globs = utils.get_function_string(expression_tmp)
-
         # if no package was imported its just a name
         try:
-            objs = evaluate_object(expression_tmp, local_dict={"OBJECTS": objects})
+            objs = evaluate_object(expression_tmp, local_dict={"ARGS": args, "OBJECTS": objects})
             out_names.extend(objs)
         except Exception:
             out_names.append(expression_tmp)

reboost/daq/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .core import run_daq_non_sparse
+
+__all__ = ["run_daq_non_sparse"]

reboost/daq/core.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+import awkward as ak
+import numba
+import numpy as np
+
+from .utils import print_random_crash_msg
+
+
+def run_daq_non_sparse(
+    evt: ak.Array,
+    n_sim_events: int,
+    source_activity: float,
+    *,
+    tau_preamp: float = 500,
+    noise_threshold: float = 5,
+    baseline_slope_threshold: float = 0.01,
+    trigger_threshold: float = 25,
+    waveform_length: float = 100,
+    trigger_position: float = 50,
+):
+    r"""Run the DAQ in non-sparse mode.
+
+    Pipe simulated HPGe events through the DAQ system in non-sparse mode.
+    Return a table where each row represents an event that was actually
+    recorded by the DAQ. for each event and each channel, determine the
+    characteristics of the waveform.
+
+    Warning
+    -------
+    This code assumes that the simulated events are time-independent.
+
+    The returned Awkward array (the table) has the following fields:
+
+    - ``evtid`` (int): event ID in the simulation.
+    - ``timestamp`` (float): timestamp of the event.
+    - ``has_trigger`` (array of bools): this waveform triggered the DAQ.
+    - ``has_pre_pulse`` (array of bools): the waveform has a signal below
+      trigger_threshold in the first part of the waveform, before the trigger
+      position.
+    - ``has_post_pulse`` (array of bools): the waveform has a signal in the
+      second part of the waveform, after the trigger position.
+    - ``has_slope`` (array of bools): waveform has decaying tail of earlier
+      signals, that came before this waveform.
+
+    The table sits in a tuple, together with a list of the channel
+    identifiers, with the same order as in the data array.
+
+    Parameters
+    ----------
+    evt
+        simulated events.
+    source_activity
+        source activity in Bq.
+    n_sim_events
+        total number of simulated events.
+    tau_preamp
+        pre-amplification RC constant in microseconds. the signal model is an
+        exponential:
+
+        .. math::
+
+          f(t) = E_i * e^{((t - t_i) / \tau)}
+
+        where :math:`E_i` is the energy of the signal and :math:`t_i` is the
+        time it occurred.
+    noise_threshold
+        threshold (in keV) for a signal to be "visible" above noise. In
+        LEGEND-200, the "energy" of forced trigger events is
+        gauss-distributed around 0.5 keV with a standard deviation of about
+        0.5 keV.
+    baseline_slope_threshold
+        threshold (in keV/us) on the baseline slope to be tagged as not flat.
+        in LEGEND-200, the slope of waveforms in force-triggered events is
+        gauss-distributed around 0 with a standard deviation of about 2 keV/ms.
+    trigger_threshold
+        amplitude (in keV) needed for the DAQ to trigger on a signal.
+    waveform_length
+        length of the waveform in microseconds stored on disk.
+    trigger_position
+        location (offset) in microseconds of the triggered signal in the
+        waveform.
+    """
+    # random engine
+    rng = np.random.default_rng()
+
+    # simulate ORCA
+    print_random_crash_msg(rng)
+
+    # add time of each simulated event, in microseconds, according to the expected event rate
+    detected_rate = source_activity * len(evt) / n_sim_events
+    evt["t0"] = np.cumsum(rng.exponential(scale=1e6 / detected_rate, size=len(evt)))
+
+    # get rawids of detectors present in the simulation
+    channel_ids = np.sort(np.unique(ak.flatten(evt.geds_rawid_active))).to_list()
+
+    daq_records = _run_daq_non_sparse_impl(
+        evt,
+        channel_ids,
+        tau_preamp,
+        noise_threshold,
+        baseline_slope_threshold,
+        trigger_threshold,
+        waveform_length,
+        trigger_position,
+    )
+
+    fields = ["evtid", "timestamp", "has_trigger", "has_pre_pulse", "has_post_pulse", "has_slope"]
+
+    daq_data = ak.Array(dict(zip(fields, daq_records, strict=False)))
+
+    return daq_data, channel_ids
+
+
+@numba.njit(cache=True)
+def _run_daq_non_sparse_impl(
+    evt: ak.Array,
+    chids: list,
+    tau_preamp: float,
+    noise_threshold: float,
+    baseline_slope_threshold: float,
+    trigger_threshold: float,
+    waveform_length: float,
+    trigger_position: float,
+):
+    """Numba-accelerated implementation of :func:`run_daq_non_sparse`."""
+    o_evtid = np.full(len(evt), dtype=np.int64, fill_value=-1)
+    o_timestamp = np.full(len(evt), dtype=np.float64, fill_value=-1)
+
+    def _init_data(dtype=np.bool_):
+        return np.zeros((len(evt), len(chids)), dtype=dtype)
+
+    o_has_trigger = _init_data()
+    o_has_pre_pulse = _init_data()
+    o_has_post_pulse = _init_data()
+    o_has_slope = _init_data()
+
+    # this is the index of the current daq record
+    r_idx = -1
+
+    # list of event indices (fifo) to keep track of past events that have still
+    # an effect on the baseline
+    evt_idx_buffer = []
+
+    # TODO: this will need to be updated
+    # {
+    #  evtid: 5.37e+03,
+    #  geds_energy_active: [756, 152],  in keV, "geds" means "HPGe detectors"
+    #  geds_multiplicity_active: 2,
+    #  geds_rawid_active: [1110400, 1112000],
+    #  geds_t0_active: [1.53e+12, 1.53e+12]
+    # }
+
+    # loop over simulated events
+    for s_idx, ev in enumerate(evt):
+        # loop over the event_buffer and remove events that occurred more
+        # than 10 times the tau_preamp ago
+        cutoff = ev.t0 - 10 * tau_preamp
+        while evt_idx_buffer:
+            first_idx = evt_idx_buffer[0]
+            if evt[first_idx].t0 < cutoff:
+                evt_idx_buffer.pop(0)
+            else:
+                break
+
+        # add current event to the buffer for later baseline analysis
+        evt_idx_buffer.append(s_idx)
+
+        # don't do any of this if there is no last record yet
+        if r_idx != -1:
+            # check if the last trigger was less than (waveform_length - trigger_position)
+            # ago. if yes, this is not a trigger but there is a hard post-pile-up
+            # on the previous trigger. so we need to check for each channel if the
+            # energy deposited is above the noise_threshold. if yes, get the last
+            # trigger and set the hard_post_pileup flag of that channel to true.
+            # then continue to the next event.
+            dt = ev.t0 - o_timestamp[r_idx]
+            if dt < (waveform_length - trigger_position):
+                for rawid, ene in zip(ev.geds_rawid_active, ev.geds_energy_active):  # noqa: B905
+                    if ene >= noise_threshold:
+                        o_has_post_pulse[r_idx, chids.index(rawid)] = True
+                continue
+
+            # check if the last trigger was less than waveform_length but more than
+            # (waveform_length - trigger_position) ago. if yes, this event is not
+            # recorded by the daq (dead time), so do nothing and just continue to the
+            # next event.
+            if dt > (waveform_length - trigger_position) and dt < waveform_length:
+                continue
+
+        # if we are here it means that we can actively look for new triggers.
+        # check if we have a trigger by checking energy in each detector against
+        # the trigger_threshold. if not, continue to the next event
+        triggered_rawids = []
+        for rawid, ene in zip(ev.geds_rawid_active, ev.geds_energy_active):  # noqa: B905
+            if ene >= trigger_threshold:
+                triggered_rawids.append(int(rawid))
+
+        if not triggered_rawids:
+            continue
+
+        # if we are here, it means we found a trigger. first, we save the last
+        # daq record to disk and we initialize a new one
+        r_idx += 1
+
+        o_evtid[r_idx] = ev.evtid
+        o_timestamp[r_idx] = ev.t0
+
+        # then, let's log which channels triggered the daq in the daq_record
+        for rawid in triggered_rawids:
+            o_has_trigger[r_idx, chids.index(rawid)] = True
+
+        # time of the start of the waveform
+        t0_start = ev.t0 - trigger_position
+        # now we need to peek into the event_buffer to check if the baseline is
+        # affected by the tails of previous events (soft pile-up) or includes a
+        # small in-trace signal (pre-hard-pileup). to take a decision, we use the
+        # baseline slope threshold
+        for rawid in chids:
+            abs_baseline_slope = 0
+            for j in evt_idx_buffer:
+                _ev = evt[j]
+
+                # for each event in the buffer get timestamp and energy
+                tj = _ev.t0
+                ej = 0
+                for k, e in zip(_ev.geds_rawid_active, _ev.geds_energy_active):  # noqa: B905
+                    if k == rawid:
+                        ej = e
+                        break
+
+                # if the event occurred before the current waveform window, we
+                # account for its tail in the baseline of the current waveform
+                # the baseline slope is calculated at the start of the waveform
+                if tj <= t0_start:
+                    abs_baseline_slope += ej / tau_preamp * np.exp(-(t0_start - tj) / tau_preamp)
+
+                # if there was any energy in a channel that occurred less than
+                # (timestamp - trigger_position) ago, this channel has a hard
+                # pre-pile-up in the current waveform
+                elif tj < ev.t0 and ej >= noise_threshold:
+                    o_has_pre_pulse[r_idx, chids.index(rawid)] = True
+
+            # now we have computed the baseline and we can check against the the
+            # noise threshold if it's significantly non-flat
+            if abs_baseline_slope >= baseline_slope_threshold:
+                o_has_slope[r_idx, chids.index(rawid)] = True
+
+    # the timestamp should refer to the start of the waveform, like in our DAQ
+    o_timestamp -= trigger_position
+
+    # the last event was recorded too
+    r_idx += 1
+
+    return (
+        o_evtid[:r_idx],
+        o_timestamp[:r_idx],
+        o_has_trigger[:r_idx],
+        o_has_pre_pulse[:r_idx],
+        o_has_post_pulse[:r_idx],
+        o_has_slope[:r_idx],
+    )