legend-dataflow-scripts 0.1.0__py3-none-any.whl

legend_dataflow_scripts-0.1.0.dist-info/METADATA

@@ -0,0 +1,57 @@
+Metadata-Version: 2.4
+Name: legend-dataflow-scripts
+Version: 0.1.0
+Summary: Python package for the processing scripts for LEGEND-200 data
+Author-email: George Marshall <ggmarsh@uw.edu>, Luigi Pertoldi <gipert@pm.me>
+Maintainer: The LEGEND Collaboration
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: colorlog
+Requires-Dist: dbetto>=1.2.3
+Requires-Dist: pygama>=2.1.1a3
+Requires-Dist: dspeed>=1.6.6a1
+Requires-Dist: pylegendmeta>=1.2.5
+Requires-Dist: legend-pydataobj>=1.12.0a4
+Requires-Dist: legend-daq2lh5>=1.5.0
+Requires-Dist: pip
+Provides-Extra: test
+Requires-Dist: legend-dataflow-scripts; extra == "test"
+Requires-Dist: pytest>=6; extra == "test"
+Requires-Dist: pytest-cov>=3; extra == "test"
+Provides-Extra: dev
+Requires-Dist: legend-dataflow-scripts[test]; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.0; extra == "docs"
+Requires-Dist: myst_parser>=0.13; extra == "docs"
+Requires-Dist: sphinx_inline_tabs; extra == "docs"
+Requires-Dist: sphinx_copybutton; extra == "docs"
+Requires-Dist: sphinx_autodoc_typehints; extra == "docs"
+Requires-Dist: furo>=2023.08.17; extra == "docs"
+
+# LEGEND dataflow scripts
+
+[![PyPI](https://img.shields.io/pypi/v/legend-dataflow-scripts?logo=pypi)](https://pypi.org/project/legend-dataflow-scripts/)
+![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/legend-exp/legend-dataflow-scripts?logo=git)
+[![GitHub Workflow Status](https://img.shields.io/github/checks-status/legend-exp/legend-dataflow-scripts/main?label=main%20branch&logo=github)](https://github.com/legend-exp/legend-dataflow-scripts/actions)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Codecov](https://img.shields.io/codecov/c/github/legend-exp/legend-dataflow-scripts?logo=codecov)](https://app.codecov.io/gh/legend-exp/legend-dataflow-scripts)
+![GitHub issues](https://img.shields.io/github/issues/legend-exp/legend-dataflow-scripts?logo=github)
+![GitHub pull requests](https://img.shields.io/github/issues-pr/legend-exp/legend-dataflow-scripts?logo=github)
+![License](https://img.shields.io/github/license/legend-exp/legend-dataflow-scripts)
+[![Read the Docs](https://img.shields.io/readthedocs/legend-dataflow-scripts?logo=readthedocs)](https://legend-dataflow-scripts.readthedocs.io)
+
+Scripts used in the LEGEND data processing. These scripts are general enough to be used in
+test stand processings also.

legend_dataflow_scripts-0.1.0.dist-info/RECORD

@@ -0,0 +1,36 @@
+legenddataflowscripts/__init__.py,sha256=hlpvTxSBjOyXlZUyOyYx3VwT5LS6zNzhAZnTmfT3NjU,303
+legenddataflowscripts/_version.py,sha256=-LyU5F1uZDjn6Q8_Z6-_FJt_8RE4Kq9zcKdg1abSSps,511
+legenddataflowscripts/par/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+legenddataflowscripts/par/geds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+legenddataflowscripts/par/geds/dsp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+legenddataflowscripts/par/geds/dsp/dplms.py,sha256=OOvnN_OcU046zhuyNTGxcmGYLCZ7YseK4rNyMDaRsC4,5314
+legenddataflowscripts/par/geds/dsp/eopt.py,sha256=UklMozVw57X8EhOVjOxwKSXtuHkX45iJ2YMLx69TQps,13680
+legenddataflowscripts/par/geds/dsp/evtsel.py,sha256=qheFqGezjRxW9UNf9DV1n8VuZjCLGZT-WaWyhrfgPJ4,16485
+legenddataflowscripts/par/geds/dsp/nopt.py,sha256=K7R4OBfO0wek03VvhBoYXDP7lURi_Xoph9Kb3iMkfx4,4263
+legenddataflowscripts/par/geds/dsp/pz.py,sha256=6tkXvL1ND-j4ulxfsRJadiNXPWqqadT7k99eti9MK-k,8265
+legenddataflowscripts/par/geds/dsp/svm.py,sha256=eDneRB_PQZp8Q4n2VheTX3kbu4ufZQ-jnuCCjvtwFpk,826
+legenddataflowscripts/par/geds/dsp/svm_build.py,sha256=8K0NUpQqL2HUWRKjMpM5H-TienVPUzIuKED94ZZnIzA,2227
+legenddataflowscripts/par/geds/hit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+legenddataflowscripts/par/geds/hit/aoe.py,sha256=0reLWzYxau09DKyu_KrywC56njSLOVGAT6QfDBSNfvM,8499
+legenddataflowscripts/par/geds/hit/ecal.py,sha256=tsIL9FP-aXgM2B1K8ygjcsLeNZ9Yv7je84fT-K7m3vQ,26194
+legenddataflowscripts/par/geds/hit/lq.py,sha256=CWasjmdaBNVk7AFNrV-RZ8UcvUzfww7wxvhrRdGSobc,7379
+legenddataflowscripts/par/geds/hit/qc.py,sha256=yT922UcNdw9X0yQ-JqoABeDuT-NMpKE2dvKO7qUGLes,11821
+legenddataflowscripts/tier/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+legenddataflowscripts/tier/dsp.py,sha256=J_ABEqbQurZHJOg8LV2aporNjZQpPg2bDZG6DkmuAL4,8633
+legenddataflowscripts/tier/hit.py,sha256=-LCfsqYAYPAuN2OKbPuh4G2P5B4uA4YmQGcqUtbZxis,4989
+legenddataflowscripts/utils/__init__.py,sha256=_eYTjqTMgjeQGO4HKmiqVAkyW9fh9FEz-_lQtcM-Lfg,366
+legenddataflowscripts/utils/alias_table.py,sha256=e0wRiDcpje8VVr_PlcbkA0kPz0qSGpGX4vJ3tfGefMA,742
+legenddataflowscripts/utils/cfgtools.py,sha256=_1yxw_eJ08AihONhJ9sWet5HQZpOagj8Yg8y9LS3zX4,381
+legenddataflowscripts/utils/convert_np.py,sha256=8q_K3w3jAREG-9CJlErP3Vca0OYTYAYJIK4g_TClvBU,819
+legenddataflowscripts/utils/log.py,sha256=5v8J7PZJHHoSy12w9e9DkYmqdPvYXj6YfVaMbhvX614,2230
+legenddataflowscripts/utils/pulser_removal.py,sha256=kuARdp1jf-lsUWcb0_KRDp-ZXzkHNrDCXUc3h7TJm7Q,424
+legenddataflowscripts/workflow/__init__.py,sha256=p-57OklNpVYqwMaLctLO8zMvhpXaTILkd3j4CaS_Juk,394
+legenddataflowscripts/workflow/execenv.py,sha256=34V7euhKiTXDDBDuX4t2Oi_zPx6lAcCRYiHqVCz6Xps,9032
+legenddataflowscripts/workflow/filedb.py,sha256=rbvOcXUxLbHz177QuDIDAL3aysz-bZDjHOiMsRHssZo,3434
+legenddataflowscripts/workflow/pre_compile_catalog.py,sha256=cEK0KXh-ClSE2Bo9MK471o79XG22bMY5r-2tIihtCfk,790
+legenddataflowscripts/workflow/utils.py,sha256=eKE8KIG2ffynZt9fTbI1SVQV85i3aW9GFGh1Nio1iDo,3118
+legend_dataflow_scripts-0.1.0.dist-info/METADATA,sha256=fzaP59xzQtj2rxNOiq0K29b36LYQgtbo-I44z24-7ew,3134
+legend_dataflow_scripts-0.1.0.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+legend_dataflow_scripts-0.1.0.dist-info/entry_points.txt,sha256=B197waSm-orA_ZS-9rkxNDsmOHdCn8CbWodnlqXQKRg,1313
+legend_dataflow_scripts-0.1.0.dist-info/top_level.txt,sha256=s8E2chjJNYUbrN6whFG_VCsJKySFp1IOXLcUefA7DB0,22
+legend_dataflow_scripts-0.1.0.dist-info/RECORD,,

legend_dataflow_scripts-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.7.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

legend_dataflow_scripts-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,18 @@
+[console_scripts]
+build-filedb = legenddataflowscripts.workflow.filedb:build_filedb
+build-tier-dsp = legenddataflowscripts.tier.dsp:build_tier_dsp
+build-tier-dsp-single-channel = legenddataflowscripts.tier.dsp:build_tier_dsp_single_channel
+build-tier-hit = legenddataflowscripts.tier.hit:build_tier_hit
+build-tier-hit-single-channel = legenddataflowscripts.tier.hit:build_tier_hit_single_channel
+dataflow = legenddataflowscripts.workflow.execenv:dataflow
+par-geds-dsp-dplms = legenddataflowscripts.par.geds.dsp.dplms:par_geds_dsp_dplms
+par-geds-dsp-eopt = legenddataflowscripts.par.geds.dsp.eopt:par_geds_dsp_eopt
+par-geds-dsp-evtsel = legenddataflowscripts.par.geds.dsp.evtsel:par_geds_dsp_evtsel
+par-geds-dsp-nopt = legenddataflowscripts.par.geds.dsp.nopt:par_geds_dsp_nopt
+par-geds-dsp-pz = legenddataflowscripts.par.geds.dsp.pz:par_geds_dsp_pz
+par-geds-dsp-svm = legenddataflowscripts.par.geds.dsp.svm:par_geds_dsp_svm
+par-geds-dsp-svm-build = legenddataflowscripts.par.geds.dsp.svm_build:par_geds_dsp_svm_build
+par-geds-hit-aoe = legenddataflowscripts.par.geds.hit.aoe:par_geds_hit_aoe
+par-geds-hit-ecal = legenddataflowscripts.par.geds.hit.ecal:par_geds_hit_ecal
+par-geds-hit-lq = legenddataflowscripts.par.geds.hit.lq:par_geds_hit_lq
+par-geds-hit-qc = legenddataflowscripts.par.geds.hit.qc:par_geds_hit_qc

legend_dataflow_scripts-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+legenddataflowscripts

legenddataflowscripts/__init__.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from .workflow import (
+    as_ro,
+    set_last_rule_name,
+    subst_vars,
+    subst_vars_impl,
+    subst_vars_in_snakemake_config,
+)
+
+__all__ = [
+    "as_ro",
+    "set_last_rule_name",
+    "subst_vars",
+    "subst_vars_impl",
+    "subst_vars_in_snakemake_config",
+]

legenddataflowscripts/_version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)

legenddataflowscripts/par/geds/dsp/dplms.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import argparse
+import pickle as pkl
+import time
+from pathlib import Path
+
+import numpy as np
+from dbetto import TextDB
+from dbetto.catalog import Props
+from lgdo import Array, Table, lh5
+from pygama.pargen.dplms_ge_dict import dplms_ge_dict
+
+from ....utils import build_log, convert_dict_np_to_float
+
+
+def par_geds_dsp_dplms() -> None:
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument("--fft-raw-filelist", help="fft_raw_filelist", type=str)
+    argparser.add_argument("--peak-file", help="tcm_filelist", type=str, required=True)
+    argparser.add_argument("--inplots", help="in_plot_path", type=str)
+    argparser.add_argument("--database", help="database", type=str, required=True)
+
+    argparser.add_argument("--log", help="log_file", type=str)
+    argparser.add_argument("--configs", help="configs", type=str, required=True)
+
+    argparser.add_argument("--datatype", help="Datatype", type=str, required=True)
+    argparser.add_argument("--timestamp", help="Timestamp", type=str, required=True)
+    argparser.add_argument("--channel", help="Channel", type=str, required=True)
+    argparser.add_argument(
+        "--raw-table-name", help="raw table name", type=str, required=True
+    )
+
+    argparser.add_argument("--dsp-pars", help="dsp_pars", type=str, required=True)
+    argparser.add_argument("--lh5-path", help="lh5_path", type=str, required=True)
+    argparser.add_argument("--plot-path", help="plot_path", type=str)
+
+    args = argparser.parse_args()
+
+    configs = TextDB(args.configs, lazy=True).on(args.timestamp, system=args.datatype)
+    config_dict = configs["snakemake_rules"]["pars_dsp_dplms"]
+
+    log = build_log(config_dict, args.log)
+
+    configs = TextDB(args.configs).on(args.timestamp, system=args.datatype)
+    dsp_config = config_dict["inputs"]["proc_chain"][args.channel]
+
+    dplms_json = config_dict["inputs"]["dplms_pars"][args.channel]
+    dplms_dict = Props.read_from(dplms_json)
+
+    db_dict = Props.read_from(args.database)
+
+    if dplms_dict["run_dplms"] is True:
+        with Path(args.fft_raw_filelist).open() as f:
+            fft_files = sorted(f.read().splitlines())
+
+        t0 = time.time()
+        log.info("\nLoad fft data")
+        energies = lh5.read_as(
+            f"{args.raw_table_name}/daqenergy", fft_files, library="np"
+        )
+        idxs = np.where(energies == 0)[0]
+        raw_fft = lh5.read(
+            args.raw_table_name,
+            fft_files,
+            n_rows=dplms_dict["n_baselines"],
+            idx=idxs,
+        )
+        t1 = time.time()
+        msg = f"Time to load fft data {(t1 - t0):.2f} s, total events {len(raw_fft)}"
+        log.info(msg)
+
+        log.info("\nRunning event selection")
+        peaks_kev = np.array(dplms_dict["peaks_kev"])
+        # kev_widths = [tuple(kev_width) for kev_width in dplms_dict["kev_widths"]]
+
+        peaks_rounded = [int(peak) for peak in peaks_kev]
+        peaks = lh5.read_as(f"{args.raw_table_name}/peak", args.peak_file, library="np")
+        ids = np.isin(peaks, peaks_rounded)
+        peaks = peaks[ids]
+        # idx_list = [np.where(peaks == peak)[0] for peak in peaks_rounded]
+
+        raw_cal = lh5.read(args.raw_table_name, args.peak_file, idx=ids)
+        msg = f"Time to run event selection {(time.time() - t1):.2f} s, total events {len(raw_cal)}"
+        log.info(msg)
+
+        if isinstance(dsp_config, str | list):
+            dsp_config = Props.read_from(dsp_config)
+
+        if args.plot_path:
+            out_dict, plot_dict = dplms_ge_dict(
+                raw_fft,
+                raw_cal,
+                dsp_config,
+                db_dict,
+                dplms_dict,
+                display=1,
+            )
+            if args.inplots:
+                with Path(args.inplots).open("rb") as r:
+                    inplot_dict = pkl.load(r)
+                inplot_dict.update({"dplms": plot_dict})
+
+        else:
+            out_dict = dplms_ge_dict(
+                raw_fft,
+                raw_cal,
+                dsp_config,
+                db_dict,
+                dplms_dict,
+            )
+
+        coeffs = out_dict["dplms"].pop("coefficients")
+        dplms_pars = Table(col_dict={"coefficients": Array(coeffs)})
+        out_dict["dplms"]["coefficients"] = (
+            f"loadlh5('{args.lh5_path}', '{args.channel}/dplms/coefficients')"
+        )
+        msg = f"DPLMS creation finished in {(time.time() - t0) / 60} minutes"
+        log.info(msg)
+    else:
+        out_dict = {}
+        dplms_pars = Table(col_dict={"coefficients": Array([])})
+        if args.inplots:
+            with Path(args.inplots).open("rb") as r:
+                inplot_dict = pkl.load(r)
+        else:
+            inplot_dict = {}
+
+    db_dict.update(out_dict)
+
+    Path(args.lh5_path).parent.mkdir(parents=True, exist_ok=True)
+    lh5.write(
+        Table(col_dict={"dplms": dplms_pars}),
+        name=args.channel,
+        lh5_file=args.lh5_path,
+        wo_mode="overwrite",
+    )
+
+    Path(args.dsp_pars).parent.mkdir(parents=True, exist_ok=True)
+    Props.write_to(args.dsp_pars, convert_dict_np_to_float(db_dict))
+
+    if args.plot_path:
+        Path(args.plot_path).parent.mkdir(parents=True, exist_ok=True)
+        with Path(args.plot_path).open("wb") as f:
+            pkl.dump(inplot_dict, f, protocol=pkl.HIGHEST_PROTOCOL)