reboost 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

reboost/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.6.1'
-__version_tuple__ = version_tuple = (0, 6, 1)
+__version__ = version = '0.7.0'
+__version_tuple__ = version_tuple = (0, 7, 0)
+
+__commit_id__ = commit_id = None

reboost/build_hit.py

@@ -7,11 +7,16 @@ A :func:`build_hit` to parse the following configuration file:
     # dictionary of objects useful for later computation. they are constructed with
     # auxiliary data (e.g. metadata). They can be accessed later as OBJECTS (all caps)
     objects:
-     lmeta: LegendMetadata(ARGS.legendmetadata)
+     lmeta: legendmeta.LegendMetadata(ARGS.legendmetadata)
      geometry: pyg4ometry.load(ARGS.gdml)
      user_pars: dbetto.TextDB(ARGS.par)
      dataprod_pars: dbetto.TextDB(ARGS.dataprod_cycle)
 
+     _spms: OBJECTS.lmeta.channelmap(on=ARGS.timestamp)
+        .group("system").spms
+        .map("name")
+     spms: "{name: spm.daq.rawid for name, spm in OBJECTS._spms.items()}"
+
     # processing chain is defined to act on a group of detectors
     processing_groups:
 
@@ -107,9 +112,10 @@ A :func:`build_hit` to parse the following configuration file:
           outputs:
             - evtid
             - tot_edep_wlsr
+            - num_scint_ph_lar
 
           operations:
-            tot_edep_wlsr: ak.sum(HITS[(HITS.__detuid == 0) & (HITS.__zloc < 3000)].__edep, axis=-1)
+            tot_edep_wlsr: ak.sum(HITS.edep[np.abs(HITS.zloc) < 3000], axis=-1)
 
         - name: spms
 
@@ -117,11 +123,14 @@ A :func:`build_hit` to parse the following configuration file:
           # same name as the current detector. This can be overridden for special processors
 
           detector_mapping:
-           - output: OBJECTS.lmeta.channglmap(on=ARGS.timestamp)
-            .group("system").spms
-            .group("analysis.status").on
-            .map("name").keys()
-           - input: lar
+           - output: OBJECTS.spms.keys()
+             input: lar
+
+          hit_table_layout: reboost.shape.group_by_time(STEPS, window=10)
+
+          pre_operations:
+            num_scint_ph_lar: reboost.spms.emitted_scintillation_photons(HITS.edep, HITS.particle, "lar")
+            # num_scint_ph_pen: ...
 
           outputs:
             - t0
@@ -130,22 +139,23 @@ A :func:`build_hit` to parse the following configuration file:
 
           detector_objects:
              meta: pygeomtools.get_sensvol_metadata(OBJECTS.geometry, DETECTOR)
-             optmap_lar: lgdo.lh5.read(DETECTOR, "optmaps/pen", ARGS.optmap_path)
-             optmap_pen: lgdo.lh5.read(DETECTOR, "optmaps/lar", ARGS.optmap_path)
-
-          hit_table_layout: reboost.shape.group_by_time(STEPS, window=10)
+             spm_uid: OBJECTS.spms[DETECTOR]
+             optmap_lar: reboost.spms.load_optmap(ARGS.optmap_path_pen, DETECTOR_OBJECTS.spm_uid)
+             optmap_pen: reboost.spms.load_optmap(ARGS.optmap_path_lar, DETECTOR_OBJECTS.spm_uid)
 
           operations:
             pe_times_lar: reboost.spms.detected_photoelectrons(
-                STEPS,
+                HITS.num_scint_ph_lar, HITS.particle, HITS.time, HITS.xloc, HITS.yloc, HITS.zloc,
                 DETECTOR_OBJECTS.optmap_lar,
-                0
+                "lar",
+                DETECTOR_OBJECTS.spm_uid
              )
 
             pe_times_pen: reboost.spms.detected_photoelectrons(
-                STEPS,
+                HITS.num_scint_ph_pen, HITS.particle, HITS.time, HITS.xloc, HITS.yloc, HITS.zloc,
                 DETECTOR_OBJECTS.optmap_pen,
-                1
+                "pen",
+                DETECTOR_OBJECTS.spm_uid
              )
 
             pe_times: ak.concatenate([HITS.pe_times_lar, HITS.pe_times_pen], axis=-1)
@@ -189,6 +199,7 @@ def build_hit(
     out_field: str = "hit",
     buffer: int = int(5e6),
     overwrite: bool = False,
+    allow_missing_inputs=True,
 ) -> None | ak.Array:
     """Build the hit tier from the remage step files.
 
@@ -215,6 +226,8 @@ def build_hit(
         buffer size for use in the `LH5Iterator`.
     overwrite
         flag to overwrite the existing output.
+    allow_missing_inputs
+        Flag to allow an input table to be missing, generally when there were no events.
     """
     # extract the config file
     if isinstance(config, str):
@@ -235,9 +248,10 @@ def build_hit(
     files = utils.get_file_dict(stp_files=stp_files, glm_files=glm_files, hit_files=hit_files)
 
     output_tables = {}
+    output_tables_names = set()
 
     # iterate over files
-    for file_idx, (stp_file, glm_file) in enumerate(zip(files.stp, files.glm)):
+    for file_idx, (stp_file, glm_file) in enumerate(zip(files.stp, files.glm, strict=True)):
         msg = (
             f"starting processing of {stp_file} to {files.hit[file_idx]} "
             if files.hit[file_idx] is not None
@@ -256,7 +270,7 @@ def build_hit(
 
             # extract the output detectors and the mapping to input detectors
             detectors_mapping = core.get_detector_mapping(
-                proc_group.get("detector_mapping"), global_objects
+                proc_group.get("detector_mapping"), global_objects, args
             )
 
             # loop over detectors
@@ -275,20 +289,30 @@ def build_hit(
 
                 lh5_group = proc_group.get("lh5_group", "stp")
                 if lh5_group is None:
-                    lh5_group = "/"
+                    lh5_group = ""
+                    table = in_detector
+                else:
+                    table = f"{lh5_group}/{in_detector}"
 
                 # begin iterating over the glm
-                iterator = GLMIterator(
-                    glm_file,
-                    stp_file,
-                    lh5_group=in_detector,
-                    start_row=start_evtid,
-                    stp_field=lh5_group,
-                    n_rows=n_evtid,
-                    buffer=buffer,
-                    time_dict=time_dict[proc_name],
-                    reshaped_files="hit_table_layout" not in proc_group,
-                )
+                # check if the in_detector is in the file
+                if table in lh5.ls(stp_file, lh5_group + "/"):
+                    iterator = GLMIterator(
+                        glm_file,
+                        stp_file,
+                        lh5_group=in_detector,
+                        start_row=start_evtid,
+                        stp_field=lh5_group,
+                        n_rows=n_evtid,
+                        buffer=buffer,
+                        time_dict=time_dict[proc_name],
+                        reshaped_files="hit_table_layout" not in proc_group,
+                    )
+                elif allow_missing_inputs:
+                    continue
+                else:
+                    msg = f"Requested input detector {in_detector} is not present in the group {lh5_group} and missing inputs were not allowed"
+                    raise ValueError(msg)
 
                 for stps, chunk_idx, _ in iterator:
                     # converting to awkward
@@ -304,6 +328,21 @@ def build_hit(
                     if time_dict is not None:
                         time_dict[proc_name].update_field("conv", start_time)
 
+                    if "hit_table_layout" in proc_group:
+                        hit_table_layouted = core.evaluate_hit_table_layout(
+                            copy.deepcopy(ak_obj),
+                            expression=proc_group["hit_table_layout"],
+                            time_dict=time_dict[proc_name],
+                        )
+                    else:
+                        hit_table_layouted = copy.deepcopy(stps)
+
+                    local_dict = {"OBJECTS": global_objects}
+                    for field, info in proc_group.get("pre_operations", {}).items():
+                        _evaluate_operation(
+                            hit_table_layouted, field, info, local_dict, time_dict[proc_name]
+                        )
+
                     # produce the hit table
                     for out_det_idx, out_detector in enumerate(out_detectors):
                         # loop over the rows
@@ -313,14 +352,12 @@ def build_hit(
                         # get the attributes
                         attrs = utils.copy_units(stps)
 
-                        if "hit_table_layout" in proc_group:
-                            hit_table = core.evaluate_hit_table_layout(
-                                copy.deepcopy(ak_obj),
-                                expression=proc_group["hit_table_layout"],
-                                time_dict=time_dict[proc_name],
-                            )
-                        else:
-                            hit_table = copy.deepcopy(stps)
+                        # if we have more than one output detector, make an independent copy.
+                        hit_table = (
+                            copy.deepcopy(hit_table_layouted)
+                            if len(out_detectors) > 1
+                            else hit_table_layouted
+                        )
 
                         local_dict = {
                             "DETECTOR_OBJECTS": det_objects[out_detector],
@@ -329,28 +366,10 @@ def build_hit(
                         }
                         # add fields
                         for field, info in proc_group.get("operations", {}).items():
-                            if isinstance(info, str):
-                                expression = info
-                                units = None
-                            else:
-                                expression = info["expression"]
-                                units = info.get("units", None)
-
-                            # evaluate the expression
-                            col = core.evaluate_output_column(
-                                hit_table,
-                                table_name="HITS",
-                                expression=expression,
-                                local_dict=local_dict,
-                                time_dict=time_dict[proc_name],
-                                name=field,
+                            _evaluate_operation(
+                                hit_table, field, info, local_dict, time_dict[proc_name]
                             )
 
-                            if units is not None:
-                                col.attrs["units"] = units
-
-                            core.add_field_with_nesting(hit_table, field, col)
-
                         # remove unwanted fields
                         if "outputs" in proc_group:
                             hit_table = core.remove_columns(
@@ -363,16 +382,12 @@ def build_hit(
                         # now write
                         if files.hit[file_idx] is not None:
                             # get modes to write with
-                            new_hit_file = (file_idx == 0) or (
-                                files.hit[file_idx] != files.hit[file_idx - 1]
-                            )
-
                             wo_mode = utils.get_wo_mode(
                                 group=group_idx,
                                 out_det=out_det_idx,
                                 in_det=in_det_idx,
                                 chunk=chunk_idx,
-                                new_hit_file=new_hit_file,
+                                new_hit_file=utils.is_new_hit_file(files, file_idx),
                                 overwrite=overwrite,
                             )
                             # write the file
@@ -390,6 +405,8 @@ def build_hit(
                                 hit_table, output_tables[out_detector]
                             )
 
+                        output_tables_names.add(out_detector)
+
         # forward some data, if requested
         # possible improvement: iterate over data if it's a lot
         if "forward" in config and files.hit[file_idx] is not None:
@@ -400,20 +417,50 @@ def build_hit(
 
             for obj in obj_list:
                 try:
+                    wo_mode = utils.get_wo_mode_forwarded(
+                        output_tables_names, utils.is_new_hit_file(files, file_idx), overwrite
+                    )
                     lh5.write(
                         lh5.read(obj, stp_file),
                         obj,
                         files.hit[file_idx],
-                        wo_mode="write_safe" if file_idx == 0 else "append",
+                        wo_mode=wo_mode,
                     )
+                    output_tables_names.add(obj)
                 except LH5EncodeError as e:
                     msg = f"cannot forward object {obj} as it has been already processed by reboost"
                     raise RuntimeError(msg) from e
 
     # return output table or nothing
-    log.debug(time_dict)
+    log.info(time_dict)
 
     if output_tables == {}:
         output_tables = None
 
     return output_tables, time_dict
+
+
+def _evaluate_operation(
+    hit_table, field: str, info: str | dict, local_dict: dict, time_dict: ProfileDict
+) -> None:
+    if isinstance(info, str):
+        expression = info
+        units = None
+    else:
+        expression = info["expression"]
+        units = info.get("units", None)
+
+    # evaluate the expression
+    col = core.evaluate_output_column(
+        hit_table,
+        table_name="HITS",
+        expression=expression,
+        local_dict=local_dict,
+        time_dict=time_dict,
+        name=field,
+    )
+
+    if units is not None:
+        col.attrs["units"] = units
+
+    core.add_field_with_nesting(hit_table, field, col)

reboost/cli.py

@@ -191,3 +191,4 @@ def cli(args=None) -> None:
             buffer=args.buffer,
             overwrite=args.overwrite,
         )
+        log.info("successfully finished post-processing")

reboost/core.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import logging
 import time
 from typing import Any
@@ -145,11 +146,14 @@ def evaluate_output_column(
     if globals_dict == {}:
         globals_dict = None
 
+    ctx = contextlib.nullcontext()
     if globals_dict is not None and "pyg4ometry" in globals_dict:
-        with utils.filter_logging(logging.CRITICAL):
-            res = hit_table.eval(func_call, local_dict, modules=globals_dict)
-    else:
-        res = hit_table.eval(func_call, local_dict, modules=globals_dict)
+        ctx = utils.filter_logging(logging.CRITICAL)
+
+    with ctx:
+        res = hit_table.eval(
+            func_call, local_dict, modules=globals_dict, library="ak", with_units=True
+        )
 
     # how long did it take
     if time_dict is not None:
@@ -227,7 +231,9 @@ def get_global_objects(
     return AttrsDict(res)
 
 
-def get_detector_mapping(detector_mapping: dict, global_objects: AttrsDict) -> dict:
+def get_detector_mapping(
+    detector_mapping: dict, global_objects: AttrsDict, args: AttrsDict
+) -> dict:
     """Get all the detector mapping using :func:`get_one_detector_mapping`.
 
     Parameters
@@ -236,6 +242,8 @@ def get_detector_mapping(detector_mapping: dict, global_objects: AttrsDict) -> d
         dictionary of detector mapping
     global_objects
         dictionary of global objects to use in evaluating the mapping.
+    args
+        any arguments the expression can depend on, is passed as `locals` to `eval()`.
     """
     return utils.merge_dicts(
         [
@@ -243,6 +251,7 @@ def get_detector_mapping(detector_mapping: dict, global_objects: AttrsDict) -> d
                 mapping["output"],
                 input_detector_name=mapping.get("input", None),
                 objects=global_objects,
+                args=args,
             )
             for mapping in detector_mapping
         ]
@@ -253,6 +262,7 @@ def get_one_detector_mapping(
     output_detector_expression: str | list,
     objects: AttrsDict | None = None,
     input_detector_name: str | None = None,
+    args: AttrsDict | None = None,
 ) -> dict:
     """Extract the output detectors and the list of input to outputs by parsing the expressions.
 
@@ -282,7 +292,8 @@ def get_one_detector_mapping(
         dictionary of objects that can be referenced in the expression.
     input_detector_name
         Optional input detector name for all the outputs.
-
+    args
+        any arguments the expression can depend on, is passed as `locals` to `eval()`.
 
     Returns
     -------
@@ -318,7 +329,7 @@ def get_one_detector_mapping(
 
         # if no package was imported its just a name
         try:
-            objs = evaluate_object(expression_tmp, local_dict={"OBJECTS": objects})
+            objs = evaluate_object(expression_tmp, local_dict={"ARGS": args, "OBJECTS": objects})
             out_names.extend(objs)
         except Exception:
             out_names.append(expression_tmp)

reboost/daq/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .core import run_daq_non_sparse
+
+__all__ = ["run_daq_non_sparse"]