reboost 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

reboost/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.6.0'
-__version_tuple__ = version_tuple = (0, 6, 0)
+__version__ = version = '0.6.2'
+__version_tuple__ = version_tuple = (0, 6, 2)

reboost/build_hit.py

@@ -235,6 +235,7 @@ def build_hit(
     files = utils.get_file_dict(stp_files=stp_files, glm_files=glm_files, hit_files=hit_files)
 
     output_tables = {}
+    output_tables_names = set()
 
     # iterate over files
     for file_idx, (stp_file, glm_file) in enumerate(zip(files.stp, files.glm)):
@@ -363,16 +364,12 @@ def build_hit(
                         # now write
                         if files.hit[file_idx] is not None:
                             # get modes to write with
-                            new_hit_file = (file_idx == 0) or (
-                                files.hit[file_idx] != files.hit[file_idx - 1]
-                            )
-
                             wo_mode = utils.get_wo_mode(
                                 group=group_idx,
                                 out_det=out_det_idx,
                                 in_det=in_det_idx,
                                 chunk=chunk_idx,
-                                new_hit_file=new_hit_file,
+                                new_hit_file=utils.is_new_hit_file(files, file_idx),
                                 overwrite=overwrite,
                             )
                             # write the file
@@ -390,6 +387,8 @@ def build_hit(
                                 hit_table, output_tables[out_detector]
                             )
 
+                        output_tables_names.add(out_detector)
+
         # forward some data, if requested
         # possible improvement: iterate over data if it's a lot
         if "forward" in config and files.hit[file_idx] is not None:
@@ -400,12 +399,16 @@ def build_hit(
 
             for obj in obj_list:
                 try:
+                    wo_mode = utils.get_wo_mode_forwarded(
+                        output_tables_names, utils.is_new_hit_file(files, file_idx), overwrite
+                    )
                     lh5.write(
                         lh5.read(obj, stp_file),
                         obj,
                         files.hit[file_idx],
-                        wo_mode="write_safe" if file_idx == 0 else "append",
+                        wo_mode=wo_mode,
                     )
+                    output_tables_names.add(obj)
                 except LH5EncodeError as e:
                     msg = f"cannot forward object {obj} as it has been already processed by reboost"
                     raise RuntimeError(msg) from e

reboost/optmap/cli.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
 import argparse
-import json
 import logging
-from pathlib import Path
+
+import dbetto
 
 from ..log_utils import setup_log
 from ..utils import _check_input_file, _check_output_file
@@ -35,14 +35,18 @@ def optical_cli() -> None:
 
     subparsers = parser.add_subparsers(dest="command", required=True)
 
-    # STEP 1: build evt file from hit tier
-    evt_parser = subparsers.add_parser("evt", help="build evt file from remage hit file")
-    evt_parser.add_argument(
+    # STEP 1: build evt file from stp tier
+    evt_parser = subparsers.add_parser("evt", help="build optmap-evt file from remage stp file")
+    evt_parser_det_group = evt_parser.add_mutually_exclusive_group(required=True)
+    evt_parser_det_group.add_argument(
+        "--geom",
+        help="GDML geometry file",
+    )
+    evt_parser_det_group.add_argument(
         "--detectors",
-        help="file that contains a list of detector ids that are part of the input file",
-        required=True,
+        help="file with detector ids of all optical channels.",
     )
-    evt_parser.add_argument("input", help="input hit LH5 file", metavar="INPUT_HIT")
+    evt_parser.add_argument("input", help="input stp LH5 file", metavar="INPUT_STP")
     evt_parser.add_argument("output", help="output evt LH5 file", metavar="OUTPUT_EVT")
 
     # STEP 2a: build map file from evt tier
@@ -55,7 +59,20 @@ def optical_cli() -> None:
     )
     map_parser.add_argument(
         "--detectors",
-        help="file that contains a list of detector ids that will be produced as additional output maps.",
+        help=(
+            "file that contains a list of detector ids that will be produced as additional output maps."
+            + "By default, all channels will be included."
+        ),
+    )
+    map_parser_det_group = map_parser.add_mutually_exclusive_group(required=True)
+    map_parser_det_group.add_argument(
+        "--geom",
+        help="GDML geometry file",
+    )
+    map_parser_det_group.add_argument(
+        "--evt",
+        action="store_true",
+        help="the input file is already an optmap-evt file.",
     )
     map_parser.add_argument(
         "--n-procs",
@@ -69,7 +86,9 @@ def optical_cli() -> None:
         action="store_true",
         help="""Check map statistics after creation. default: %(default)s""",
     )
-    map_parser.add_argument("input", help="input evt LH5 file", metavar="INPUT_EVT", nargs="+")
+    map_parser.add_argument(
+        "input", help="input stp or optmap-evt LH5 file", metavar="INPUT_EVT", nargs="+"
+    )
     map_parser.add_argument("output", help="output map LH5 file", metavar="OUTPUT_MAP")
 
     # STEP 2b: view maps
@@ -171,7 +190,7 @@ def optical_cli() -> None:
     convolve_parser.add_argument(
         "--dist-mode",
         action="store",
-        default="multinomial+no-fano",
+        default="poisson+no-fano",
     )
     convolve_parser.add_argument("--output", help="output hit LH5 file", metavar="OUTPUT_HIT")
 
@@ -188,15 +207,18 @@ def optical_cli() -> None:
 
     # STEP 1: build evt file from hit tier
     if args.command == "evt":
-        from .evt import build_optmap_evt
+        from .evt import build_optmap_evt, get_optical_detectors_from_geom
 
-        _check_input_file(parser, args.detectors)
         _check_input_file(parser, args.input)
         _check_output_file(parser, args.output)
 
-        # load detector ids from a JSON array
-        with Path.open(Path(args.detectors)) as detectors_f:
-            detectors = json.load(detectors_f)
+        # load detector ids from the geometry.
+        if args.geom is not None:
+            _check_input_file(parser, args.geom, "geometry")
+            detectors = get_optical_detectors_from_geom(args.geom)
+        else:
+            _check_input_file(parser, args.detectors, "detectors")
+            detectors = dbetto.utils.load_dict(args.detectors)
 
         build_optmap_evt(args.input, args.output, detectors, args.bufsize)
 
@@ -209,23 +231,23 @@ def optical_cli() -> None:
 
         # load settings for binning from config file.
         _check_input_file(parser, args.input, "settings")
-        with Path.open(Path(args.settings)) as settings_f:
-            settings = json.load(settings_f)
+        settings = dbetto.utils.load_dict(args.settings)
 
-        chfilter = ()
+        chfilter = "*"
         if args.detectors is not None:
-            # load detector ids from a JSON array
-            with Path.open(Path(args.detectors)) as detectors_f:
-                chfilter = json.load(detectors_f)
+            # load detector ids from a JSON/YAML array
+            chfilter = dbetto.utils.load_dict(args.detectors)
 
         create_optical_maps(
             args.input,
             settings,
             args.bufsize,
+            is_stp_file=(not args.evt),
             chfilter=chfilter,
             output_lh5_fn=args.output,
             check_after_create=args.check,
             n_procs=args.n_procs,
+            geom_fn=args.geom,
         )
 
     # STEP 2b: view maps
@@ -251,8 +273,7 @@ def optical_cli() -> None:
 
         # load settings for binning from config file.
         _check_input_file(parser, args.input, "settings")
-        with Path.open(Path(args.settings)) as settings_f:
-            settings = json.load(settings_f)
+        settings = dbetto.utils.load_dict(args.settings)
 
         _check_input_file(parser, args.input)
         _check_output_file(parser, args.output)

reboost/optmap/convolve.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import re
 
 import legendoptics.scintillate as sc
 import numba
@@ -24,8 +25,8 @@ OPTMAP_SUM_CH = -2
 
 def open_optmap(optmap_fn: str):
     maps = lh5.ls(optmap_fn)
-    # TODO: rewrite logic to only accept _<number> instead of a blacklist
-    det_ntuples = [m for m in maps if m not in ("all", "_hitcounts", "_hitcounts_exp", "all_orig")]
+    # only accept _<number> (/all is read separately)
+    det_ntuples = [m for m in maps if re.match(r"_\d+$", m)]
     detids = np.array([int(m.lstrip("_")) for m in det_ntuples])
     detidx = np.arange(0, detids.shape[0])
 
@@ -53,17 +54,25 @@ def open_optmap(optmap_fn: str):
 
     # give this check some numerical slack.
     if np.any(
-        ow[OPTMAP_SUM_CH][ow[OPTMAP_ANY_CH] >= 0] - ow[OPTMAP_ANY_CH][ow[OPTMAP_ANY_CH] >= 0]
+        np.abs(
+            ow[OPTMAP_SUM_CH][ow[OPTMAP_ANY_CH] >= 0] - ow[OPTMAP_ANY_CH][ow[OPTMAP_ANY_CH] >= 0]
+        )
         < -1e-15
     ):
         msg = "optical map does not fulfill relation sum(p_i) >= p_any"
         raise ValueError(msg)
 
-    # get the exponent from the optical map file
-    optmap_multi_det_exp = lh5.read("/_hitcounts_exp", optmap_fn).value
-    assert isinstance(optmap_multi_det_exp, float)
+    try:
+        # check the exponent from the optical map file
+        optmap_multi_det_exp = lh5.read("/_hitcounts_exp", optmap_fn).value
+        assert isinstance(optmap_multi_det_exp, float)
+        if np.isfinite(optmap_multi_det_exp):
+            msg = f"found finite _hitcounts_exp {optmap_multi_det_exp} which is not supported any more"
+            raise RuntimeError(msg)
+    except KeyError:  # the _hitcounts_exp might not be always present.
+        pass
 
-    return detids, detidx, optmap_edges, ow, optmap_multi_det_exp
+    return detids, detidx, optmap_edges, ow
 
 
 def iterate_stepwise_depositions(
@@ -71,7 +80,7 @@ def iterate_stepwise_depositions(
     optmap_for_convolve,
     scint_mat_params: sc.ComputedScintParams,
     rng: np.random.Generator = None,
-    dist: str = "multinomial",
+    dist: str = "poisson",
     mode: str = "no-fano",
 ):
     # those np functions are not supported by numba, but needed for efficient array access below.
@@ -144,7 +153,6 @@ def _iterate_stepwise_depositions(
     detidx,
     optmap_edges,
     optmap_weights,
-    optmap_multi_det_exp,
     scint_mat_params: sc.ComputedScintParams,
     dist: str,
     mode: str,
@@ -223,8 +231,6 @@ def _iterate_stepwise_depositions(
                 # we detect this energy deposition; we should at least get one photon out here!
 
                 detsel_size = 1
-                if np.isfinite(optmap_multi_det_exp):
-                    detsel_size = rng.geometric(1 - np.exp(-optmap_multi_det_exp))
 
                 px_sum = optmap_weights[OPTMAP_SUM_CH, cur_bins[0], cur_bins[1], cur_bins[2]]
                 assert px_sum >= 0.0  # should not be negative.
@@ -238,7 +244,7 @@ def _iterate_stepwise_depositions(
                         detp[d] = 0.0
                 det_no_stats += had_det_no_stats
 
-                # should be equivalent to rng.choice(detidx, size=(detsel_size, p=detp)
+                # should be equivalent to rng.choice(detidx, size=detsel_size, p=detp)
                 detsel = detidx[
                     np.searchsorted(np.cumsum(detp), rng.random(size=(detsel_size,)), side="right")
                 ]
@@ -339,7 +345,7 @@ def convolve(
     material: str,
     output_file: str | None = None,
     buffer_len: int = int(1e6),
-    dist_mode: str = "multinomial+no-fano",
+    dist_mode: str = "poisson+no-fano",
 ):
     if material not in ["lar", "pen"]:
         msg = f"unknown material {material} for scintillation"
@@ -356,13 +362,18 @@ def convolve(
             (1 * pint.get_application_registry().ns),  # dummy!
         )
 
-    log.info("opening map %s", map_file)
-    optmap_for_convolve = open_optmap(map_file)
-
     # special handling of distributions and flags.
     dist, mode = dist_mode.split("+")
-    assert dist in ("multinomial", "poisson")
-    assert mode in ("", "no-fano")
+    if (
+        dist not in ("multinomial", "poisson")
+        or mode not in ("", "no-fano")
+        or (dist == "poisson" and mode != "no-fano")
+    ):
+        msg = f"unsupported statistical distribution {dist_mode} for scintillation emission"
+        raise ValueError(msg)
+
+    log.info("opening map %s", map_file)
+    optmap_for_convolve = open_optmap(map_file)
 
     log.info("opening energy deposition hit output %s", edep_file)
     it = LH5Iterator(edep_file, edep_path, buffer_len=buffer_len)

reboost/optmap/create.py

@@ -14,7 +14,12 @@ from numba import njit
 from numpy.typing import NDArray
 
 from ..log_utils import setup_log
-from .evt import EVT_TABLE_NAME, read_optmap_evt
+from .evt import (
+    EVT_TABLE_NAME,
+    generate_optmap_evt,
+    get_optical_detectors_from_geom,
+    read_optmap_evt,
+)
 from .optmap import OpticalMap
 
 log = logging.getLogger(__name__)
@@ -109,12 +114,13 @@ def _create_optical_maps_process_init(optmaps, log_level) -> None:
 
 
 def _create_optical_maps_process(
-    optmap_events_fn, buffer_len, all_det_ids, ch_idx_to_map_idx
+    optmap_events_fn, buffer_len, is_stp_file, all_det_ids, ch_idx_to_map_idx
 ) -> None:
     log.info("started worker task for %s", optmap_events_fn)
     x = _create_optical_maps_chunk(
         optmap_events_fn,
         buffer_len,
+        is_stp_file,
         all_det_ids,
         _shared_optmaps,
         ch_idx_to_map_idx,
@@ -124,9 +130,12 @@ def _create_optical_maps_process(
 
 
 def _create_optical_maps_chunk(
-    optmap_events_fn, buffer_len, all_det_ids, optmaps, ch_idx_to_map_idx
+    optmap_events_fn, buffer_len, is_stp_file, all_det_ids, optmaps, ch_idx_to_map_idx
 ) -> None:
-    optmap_events_it = read_optmap_evt(optmap_events_fn, buffer_len)
+    if not is_stp_file:
+        optmap_events_it = read_optmap_evt(optmap_events_fn, buffer_len)
+    else:
+        optmap_events_it = generate_optmap_evt(optmap_events_fn, all_det_ids, buffer_len)
 
     hits_per_primary = np.zeros(10, dtype=np.int64)
     hits_per_primary_len = 0
@@ -156,19 +165,24 @@ def create_optical_maps(
     optmap_events_fn: list[str],
     settings,
     buffer_len: int = int(5e6),
+    is_stp_file: bool = True,
     chfilter: tuple[str | int] | Literal["*"] = (),
     output_lh5_fn: str | None = None,
     after_save: Callable[[int, str, OpticalMap]] | None = None,
     check_after_create: bool = False,
     n_procs: int | None = 1,
+    geom_fn: str | None = None,
 ) -> None:
     """Create optical maps.
 
     Parameters
     ----------
     optmap_events_fn
-        list of filenames to lh5 files with a table ``/optmap_evt`` with columns ``{x,y,z}loc``
-        and one column (with numeric header) for each SiPM channel.
+        list of filenames to lh5 files, that can either be stp files from remage or "optmap-evt"
+        files with a table ``/optmap_evt`` with columns ``{x,y,z}loc`` and one column (with numeric
+        header) for each SiPM channel.
+    is_stp_file
+        if true, do convert a remage output file (stp file) on-the-fly to an optmap-evt file.
     chfilter
         tuple of detector ids that will be included in the resulting optmap. Those have to match
         the column names in ``optmap_events_fn``.
@@ -181,9 +195,13 @@ def create_optical_maps(
 
     use_shmem = n_procs is None or n_procs > 1
 
-    optmap_evt_columns = list(
-        lh5.read(EVT_TABLE_NAME, optmap_events_fn[0], start_row=0, n_rows=1).keys()
-    )  # peek into the (first) file to find column names.
+    if not is_stp_file:
+        optmap_evt_columns = list(
+            lh5.read(EVT_TABLE_NAME, optmap_events_fn[0], start_row=0, n_rows=1).keys()
+        )  # peek into the (first) file to find column names.
+    else:
+        optmap_evt_columns = [str(i) for i in get_optical_detectors_from_geom(geom_fn)]
+
     all_det_ids, optmaps, optmap_det_ids = _optmaps_for_channels(
         optmap_evt_columns, settings, chfilter=chfilter, use_shmem=use_shmem
     )
@@ -202,7 +220,9 @@ def create_optical_maps(
     if not use_shmem:
         for fn in optmap_events_fn:
             q.append(
-                _create_optical_maps_chunk(fn, buffer_len, all_det_ids, optmaps, ch_idx_to_map_idx)
+                _create_optical_maps_chunk(
+                    fn, buffer_len, is_stp_file, all_det_ids, optmaps, ch_idx_to_map_idx
+                )
             )
     else:
         ctx = mp.get_context("forkserver")
@@ -222,7 +242,7 @@ def create_optical_maps(
         for fn in optmap_events_fn:
             r = pool.apply_async(
                 _create_optical_maps_process,
-                args=(fn, buffer_len, all_det_ids, ch_idx_to_map_idx),
+                args=(fn, buffer_len, is_stp_file, all_det_ids, ch_idx_to_map_idx),
             )
             pool_results.append((r, fn))
 
@@ -401,6 +421,9 @@ def merge_optical_maps(
     hits_per_primary = np.zeros(10, dtype=np.int64)
     hits_per_primary_len = 0
     for optmap_fn in map_l5_files:
+        if "_hitcounts" not in lh5.ls(optmap_fn):
+            log.warning("skipping _hitcounts calculations, missing in file %s", optmap_fn)
+            return
         hitcounts = lh5.read("/_hitcounts", optmap_fn)
         assert isinstance(hitcounts, Array)
         hits_per_primary[0 : len(hitcounts)] += hitcounts
@@ -417,13 +440,18 @@ def merge_optical_maps(
 def check_optical_map(map_l5_file: str):
     """Run a health check on the map file.
 
-    This checks for consistency, and output details on map statistics.
+    This checks for consistency, and outputs details on map statistics.
     """
-    if lh5.read("_hitcounts_exp", lh5_file=map_l5_file).value != np.inf:
-        log.error("unexpected hitcount exp not equal to positive infinity")
+    if "_hitcounts_exp" not in lh5.ls(map_l5_file):
+        log.info("no _hitcounts_exp found")
+    elif lh5.read("_hitcounts_exp", lh5_file=map_l5_file).value != np.inf:
+        log.error("unexpected _hitcounts_exp not equal to positive infinity")
         return
-    if lh5.read("_hitcounts", lh5_file=map_l5_file).nda.shape != (2,):
-        log.error("unexpected hitcount shape")
+
+    if "_hitcounts" not in lh5.ls(map_l5_file):
+        log.info("no _hitcounts found")
+    elif lh5.read("_hitcounts", lh5_file=map_l5_file).nda.shape != (2,):
+        log.error("unexpected _hitcounts shape")
         return
 
     all_binning = None
@@ -477,4 +505,5 @@ def rebin_optical_maps(map_l5_file: str, output_lh5_file: str, factor: int):
 
     # just copy hitcounts exponent.
     for dset in ("_hitcounts_exp", "_hitcounts"):
-        lh5.write(lh5.read(dset, lh5_file=map_l5_file), dset, lh5_file=output_lh5_file)
+        if dset in lh5.ls(map_l5_file):
+            lh5.write(lh5.read(dset, lh5_file=map_l5_file), dset, lh5_file=output_lh5_file)

reboost/optmap/evt.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import logging
-from collections.abc import Iterable
+from collections.abc import Generator, Iterable
 from pathlib import Path
 
 import numpy as np
@@ -14,20 +14,18 @@ log = logging.getLogger(__name__)
 EVT_TABLE_NAME = "optmap_evt"
 
 
-def build_optmap_evt(
-    lh5_in_file: str, lh5_out_file: str, detectors: Iterable[str | int], buffer_len: int = int(5e6)
-) -> None:
+def generate_optmap_evt(
+    lh5_in_file: str, detectors: Iterable[str | int], buffer_len: int = int(5e6)
+) -> Generator[Table, None, None]:
     """Create a faster map for lookup of the hits in each detector, for each primary event."""
     log.info("reading file %s", lh5_in_file)
 
-    lh5_out_file = Path(lh5_out_file)
-    lh5_out_file_tmp = lh5_out_file.with_stem(".evt-tmp." + lh5_out_file.stem)
-    if lh5_out_file_tmp.exists():
-        msg = f"temporary output file {lh5_out_file_tmp} already exists"
-        raise RuntimeError(msg)
     vert_it = LH5Iterator(lh5_in_file, "vtx", buffer_len=buffer_len)
     opti_it = LH5Iterator(lh5_in_file, "stp/optical", buffer_len=buffer_len)
 
+    if len(detectors) == 0:
+        msg = "detector array cannot be empty for optmap-evt building"
+        raise ValueError(msg)
     detectors = [str(d) for d in detectors]
     for d in detectors:
         if not d.isnumeric():
@@ -35,11 +33,11 @@ def build_optmap_evt(
 
     vert_df = None
     vert_df_bounds = None
-    vert_it_count = 0
     hits_expected = 0
+    had_last_chunk = False
 
-    def _store_vert_df():
-        nonlocal vert_df
+    def _store_vert_df(last_chunk: bool) -> Generator[Table, None, None]:
+        nonlocal vert_df, had_last_chunk
         if vert_df is None:
             return
 
@@ -49,8 +47,8 @@ def build_optmap_evt(
             hits_sum += np.sum(vert_df[d])
         assert hits_sum == hits_expected
 
-        log.info("store evt file %s (%d)", lh5_out_file_tmp, vert_it_count - 1)
-        lh5.write(Table(vert_df), name=EVT_TABLE_NAME, lh5_file=lh5_out_file_tmp, wo_mode="append")
+        yield Table(vert_df)
+        had_last_chunk = last_chunk
         vert_df = None
 
     # helper function for "windowed join". while iterating the optical hits, we have to
@@ -59,8 +57,8 @@ def build_optmap_evt(
     # This function follows the assumption, that the output event ids are at least "somewhat"
     # monotonic, i.e. later chunks do not contain lower evtids than the previous chunk(s).
     # Going back is not implemented.
-    def _ensure_vert_df(vert_it: LH5Iterator, evtid: int) -> None:
-        nonlocal vert_df, vert_df_bounds, vert_it_count, hits_expected
+    def _ensure_vert_df(vert_it: LH5Iterator, evtid: int) -> Generator[Table, None, None]:
+        nonlocal vert_df, vert_df_bounds, hits_expected
 
         # skipping multiple chunks is possible in sparsely populated simulations.
         while vert_df_bounds is None or evtid > vert_df_bounds[1] or evtid < vert_df_bounds[0]:
@@ -74,9 +72,8 @@ def build_optmap_evt(
             # here, evtid > vert_df_bounds[1] (or vert_df_bounds is still None). We need to fetch
             # the next event table chunk.
 
-            vert_it_count += 1
             # we might have filled a dataframe, save it to disk.
-            _store_vert_df()
+            yield from _store_vert_df(last_chunk=False)
 
             # read the next vertex chunk into memory.
             vert_df = next(vert_it).view_as("pd")
@@ -98,11 +95,28 @@ def build_optmap_evt(
         log.info("build evt table (%d)", opti_it_count)
 
         for t in opti_df[["evtid", "det_uid"]].itertuples(name=None, index=False):
-            _ensure_vert_df(vert_it, t[0])
+            yield from _ensure_vert_df(vert_it, t[0])
             vert_df.loc[t[0], str(t[1])] += 1
             hits_expected += 1
 
-    _store_vert_df()  # store the last chunk.
+    yield from _store_vert_df(last_chunk=True)  # store the last chunk.
+
+    assert had_last_chunk, "did not reach last chunk in optmap-evt building"
+
+
+def build_optmap_evt(
+    lh5_in_file: str, lh5_out_file: str, detectors: Iterable[str | int], buffer_len: int = int(5e6)
+) -> None:
+    """Create a faster map for lookup of the hits in each detector, for each primary event."""
+    lh5_out_file = Path(lh5_out_file)
+    lh5_out_file_tmp = lh5_out_file.with_stem(".evt-tmp." + lh5_out_file.stem)
+    if lh5_out_file_tmp.exists():
+        msg = f"temporary output file {lh5_out_file_tmp} already exists"
+        raise RuntimeError(msg)
+
+    for vert_it_count, chunk in enumerate(generate_optmap_evt(lh5_in_file, detectors, buffer_len)):
+        log.info("store evt file %s (%d)", lh5_out_file_tmp, vert_it_count - 1)
+        lh5.write(Table(chunk), name=EVT_TABLE_NAME, lh5_file=lh5_out_file_tmp, wo_mode="append")
 
     # after finishing the output file, rename to the actual output file name.
     if lh5_out_file.exists():
@@ -111,5 +125,14 @@ def build_optmap_evt(
     lh5_out_file_tmp.rename(lh5_out_file)
 
 
+def get_optical_detectors_from_geom(geom_fn) -> list[int]:
+    import pyg4ometry
+    import pygeomtools
+
+    geom_registry = pyg4ometry.gdml.Reader(geom_fn).getRegistry()
+    detectors = pygeomtools.get_all_sensvols(geom_registry)
+    return [d.uid for d in detectors.values() if d.detector_type == "optical"]
+
+
 def read_optmap_evt(lh5_file: str, buffer_len: int = int(5e6)) -> LH5Iterator:
     return LH5Iterator(lh5_file, EVT_TABLE_NAME, buffer_len=buffer_len)

reboost/utils.py

@@ -51,7 +51,9 @@ def get_wo_mode(
     chunk
         the chunk index
     new_hit_file
-        a flag of whether we are writing a new hit file
+        a flag of whether we are writing a new hit file. This does not indicate whether
+        the file already exists on disk, but whether the file name is different from the
+        last written chunk for this detector.
     overwrite
         a flag of whether to overwrite the old file.
 
@@ -75,6 +77,49 @@ def get_wo_mode(
     return "append"
 
 
+def get_wo_mode_forwarded(
+    written_tables: set[str], new_hit_file: bool, overwrite: bool = False
+) -> str:
+    """Get the mode for lh5 file writing for forwarded tables tahat will be copied without chunking.
+
+    If we are writing a new output file and no other tables had been written yet, then
+    the mode "overwrite_file" is used if the overwrite flag is set, otherwise the mode
+    "write_safe" is used.
+
+    Otherwise "append" is used.
+
+    Parameters
+    ----------
+    written_tables
+        a set of already written table names, also including other table names of
+        non-forwarded (i.e. processed) tables.
+    new_hit_file
+        a flag of whether we are writing a new hit file. This does not indicate whether
+        the file already exists on disk, but whether the file name is different from the
+        last written chunk for this forwarded table.
+    overwrite
+        a flag of whether to overwrite the old file.
+
+    Returns
+    -------
+    the mode for IO
+    """
+    if not new_hit_file:
+        return "append"
+    if overwrite and len(written_tables) == 0:
+        return "overwrite_file"
+    return "write_safe"
+
+
+def is_new_hit_file(files: AttrsDict, file_idx: int) -> bool:
+    """Return whether the hit file with the given index is a "new" hit file.
+
+    A new file is either the first file written, or when the previous file index has a
+    different file name.
+    """
+    return (file_idx == 0) or (files.hit[file_idx] != files.hit[file_idx - 1])
+
+
 def get_file_dict(
     stp_files: list[str] | str,
     glm_files: list[str] | str | None,

{reboost-0.6.0.dist-info → reboost-0.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: reboost
-Version: 0.6.0
+Version: 0.6.2
 Summary: New LEGEND Monte-Carlo simulation post-processing
 Author-email: Manuel Huber <info@manuelhu.de>, Toby Dixon <toby.dixon.23@ucl.ac.uk>, Luigi Pertoldi <gipert@pm.me>
 Maintainer: The LEGEND Collaboration
@@ -703,6 +703,7 @@ Requires-Dist: scipy
 Requires-Dist: numba
 Requires-Dist: legend-pydataobj>=1.14
 Requires-Dist: legend-pygeom-optics>=0.9.2
+Requires-Dist: legend-pygeom-tools>=0.0.11
 Requires-Dist: hist
 Requires-Dist: dbetto
 Requires-Dist: particle
@@ -722,8 +723,6 @@ Requires-Dist: pre-commit; extra == "test"
 Requires-Dist: pytest>=6.0; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: legend-pygeom-hpges; extra == "test"
-Requires-Dist: legend-pygeom-tools; extra == "test"
-Requires-Dist: pyg4ometry; extra == "test"
 Requires-Dist: pylegendtestdata>=0.6; extra == "test"
 Dynamic: license-file
 

{reboost-0.6.0.dist-info → reboost-0.6.2.dist-info}/RECORD

@@ -1,15 +1,15 @@
 reboost/__init__.py,sha256=VZz9uo7i2jgAx8Zi15SptLZnE_qcnGuNWwqkD3rYHFA,278
-reboost/_version.py,sha256=jF9TuoEIJRaca3ScKo6qaz6PzaMlu7jjuSQIrJ3nX4U,511
+reboost/_version.py,sha256=DhO-RAVJdQTjYPaMf0M6q_WAXdD6jcx2tWl_B9FtuhM,511
 reboost/build_evt.py,sha256=VXIfK_pfe_Cgym6gI8dESwONZi-v_4fll0Pn09vePQY,3767
 reboost/build_glm.py,sha256=IerSLQfe51ZO7CQP2kmfPnOIVaDtcfw3byOM02Vaz6o,9472
-reboost/build_hit.py,sha256=23JL5B7qThdHZqAK_HWoytqcEOWDhGsk4n5UMtojJ1c,15513
+reboost/build_hit.py,sha256=BwcAzr7hkhLqQpyULYPqM70VEvENaw-j90fvBYKh-68,15649
 reboost/cli.py,sha256=HZgqUZK0tSmnlGqoXjrbmLitW_i001TzibxvDrRxLLg,6324
 reboost/core.py,sha256=WGbWe2rcfMDEaehVyw7peqAHoTFWoCu5J6CdWHC5aWA,14974
 reboost/iterator.py,sha256=fATFDxu2PUc0e48OdJJujZo2kwykfRLH1oBtcB-s5pM,6905
 reboost/log_utils.py,sha256=VqS_9OC5NeNU3jcowVOBB0NJ6ssYvNWnirEY-JVduEA,766
 reboost/profile.py,sha256=EOTmjmS8Rm_nYgBWNh6Rntl2XDsxdyed7yEdWtsZEeg,2598
 reboost/units.py,sha256=3EH8XlpbsObdu5vLgxhm1600L6UNYD5jng4SjJT_1QE,2202
-reboost/utils.py,sha256=WT2jlyRT3LEMBEBRJtYIat3KIIZKocv4rHlTViBbhYM,10938
+reboost/utils.py,sha256=dzDiQWwv_snoWoRBDnDNZ27hG0CpCRo8V4jSEG2b82c,12592
 reboost/hpge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reboost/hpge/psd.py,sha256=jAUAoQ_PMz76wyA1NXYHNKtOwoCnRT3My8_LCFrKi-U,13860
 reboost/hpge/surface.py,sha256=lbWcFnFFWKxtFKs755GyM9US_IfyxaoM6MpOIZgIMM0,7478
@@ -18,10 +18,10 @@ reboost/math/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reboost/math/functions.py,sha256=OymiYTcA0NXxxm-MBDw5kqyNwHoLCmuv4J48AwnSrbU,5633
 reboost/math/stats.py,sha256=cG-6mQx33Dzpv3ABkHLEwC104WJ_PMgbWtmjg37SBj4,3164
 reboost/optmap/__init__.py,sha256=imvuyld-GLw8qdwqW-lXCg2feptcTyQo3wIzPvDHwmY,93
-reboost/optmap/cli.py,sha256=SzbPRgsbR5Llm3aSJubH02Ym8FQyTH7kvuLjK7faLiY,9572
-reboost/optmap/convolve.py,sha256=_volpLmhW5mOPA0KkzXRyHyqkj4_zDSnvfHv1Dtuxm8,14390
-reboost/optmap/create.py,sha256=B-MWurmnzl4Y62N2Pj7IwM1IaEEt2fydpZa_t0gmsxo,17048
-reboost/optmap/evt.py,sha256=UYESkMAwDbE_ap4Jb-a2n0uWxHRnYmHzQiXh0vexaPQ,4513
+reboost/optmap/cli.py,sha256=N8J2Hd8m_csYU9CtpAp7Rc3LHy6eNzZ26gWZgHCiUso,10250
+reboost/optmap/convolve.py,sha256=x7boLDcBJIsontoB0yemvzHSE2hlRpUomlDXc3jqdr4,14668
+reboost/optmap/create.py,sha256=_6GZbdRvmjDFs6DDbWC-THZxaNPUiLAOIDNaigMKJSQ,18139
+reboost/optmap/evt.py,sha256=UrjjNNeS7Uie4Ah9y_f5PyroFutLGo5aOFcwReOEy7o,5556
 reboost/optmap/mapview.py,sha256=73kpe0_SKDj9bIhEx1ybX1sBP8TyvufiLfps84A_ijA,6798
 reboost/optmap/numba_pdg.py,sha256=y8cXR5PWE2Liprp4ou7vl9do76dl84vXU52ZJD9_I7A,731
 reboost/optmap/optmap.py,sha256=j4rfbQ84PYSpE-BvP4Rdt96ZjPdwy8P4e4eZz1mATys,12817
@@ -29,9 +29,9 @@ reboost/shape/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reboost/shape/cluster.py,sha256=RIvBlhHzp88aaUZGofp5SD9bimnoiqIOddhQ84jiwoM,8135
 reboost/shape/group.py,sha256=gOCYgir2gZqmW1JXtbNRPlQqP0gmUcbe7RVb9CbY1pU,5540
 reboost/shape/reduction.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reboost-0.6.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-reboost-0.6.0.dist-info/METADATA,sha256=QGF9b0vthakVY3Wl9OUQnPIyAv3IYpui4WK9Zkp7Nwo,44274
-reboost-0.6.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-reboost-0.6.0.dist-info/entry_points.txt,sha256=DxhD6BidSWNot9BrejHJjQ7RRLmrMaBIl52T75oWTwM,93
-reboost-0.6.0.dist-info/top_level.txt,sha256=q-IBsDepaY_AbzbRmQoW8EZrITXRVawVnNrB-_zyXZs,8
-reboost-0.6.0.dist-info/RECORD,,
+reboost-0.6.2.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+reboost-0.6.2.dist-info/METADATA,sha256=o0DAQXxkMCG68Uf-WHTHeVsIEMKMsj044lWWMJLAx_k,44222
+reboost-0.6.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+reboost-0.6.2.dist-info/entry_points.txt,sha256=DxhD6BidSWNot9BrejHJjQ7RRLmrMaBIl52T75oWTwM,93
+reboost-0.6.2.dist-info/top_level.txt,sha256=q-IBsDepaY_AbzbRmQoW8EZrITXRVawVnNrB-_zyXZs,8
+reboost-0.6.2.dist-info/RECORD,,