legend-daq2lh5 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

daq2lh5/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.3.0'
-__version_tuple__ = version_tuple = (1, 3, 0)
+__version__ = version = '1.4.0'
+__version_tuple__ = version_tuple = (1, 4, 0)

daq2lh5/buffer_processor/lh5_buffer_processor.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import json
 import logging
 import os
 
@@ -8,6 +7,7 @@ import h5py
 import lgdo
 from lgdo import lh5
 
+from .. import utils
 from ..buffer_processor.buffer_processor import buffer_processor
 from ..raw_buffer import RawBuffer, RawBufferLibrary
 
@@ -104,11 +104,13 @@ def lh5_buffer_processor(
     raw_store.gimme_file(proc_file_name, "a")
 
     # Do key expansion on the out_spec
-    if isinstance(out_spec, str) and out_spec.endswith(".json"):
-        with open(out_spec) as json_file:
-            out_spec = json.load(json_file)
+    allowed_exts = [ext for exts in utils.__file_extensions__.values() for ext in exts]
+    if isinstance(out_spec, str) and any(
+        [out_spec.endswith(ext) for ext in allowed_exts]
+    ):
+        out_spec = utils.load_dict(out_spec)
     if isinstance(out_spec, dict):
-        RawBufferLibrary(json_dict=out_spec)
+        RawBufferLibrary(config=out_spec)
 
     # Write everything in the raw file to the new file, check for proc_spec under either the group name, out_name, or the name
     for tb in lh5_tables:

daq2lh5/build_raw.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import glob
-import json
 import logging
 import os
 import time
@@ -10,6 +9,7 @@ import numpy as np
 from lgdo import lh5
 from tqdm.auto import tqdm
 
+from . import utils
 from .compass.compass_streamer import CompassStreamer
 from .fc.fc_streamer import FCStreamer
 from .llama.llama_streamer import LLAMAStreamer
@@ -49,12 +49,14 @@ def build_raw(
         Specification for the output stream.
 
         - if None, uses ``{in_stream}.lh5`` as the output filename.
-        - if a str not ending in ``.json``, interpreted as the output filename.
-        - if a str ending in ``.json``, interpreted as a filename containing
-          json-shorthand for the output specification (see :mod:`.raw_buffer`).
-        - if a JSON dict, should be a dict loaded from the json shorthand
-          notation for RawBufferLibraries (see :mod:`.raw_buffer`), which is
-          then used to build a :class:`.RawBufferLibrary`.
+        - if a str not ending with a config file extension, interpreted as the
+          output filename.
+        - if a str ending with a config file extension, interpreted as a
+          filename containing shorthand for the output specification (see
+          :mod:`.raw_buffer`).
+        - if a dict, should be a dict loaded from the shorthand notation for
+          RawBufferLibraries (see :mod:`.raw_buffer`), which is then used to
+          build a :class:`.RawBufferLibrary`.
         - if a :class:`.RawBufferLibrary`, the mapping of data to output file /
           group is taken from that.
 
@@ -72,8 +74,8 @@ def build_raw(
 
         - if None, CompassDecoder will sacrifice the first packet to determine
           waveform length
-        - if a str ending in ``.json``, interpreted as a filename containing
-          json-shorthand for the output specification (see
+        - if a str ending with a config file extension, interpreted as a
+          filename containing shorthand for the output specification (see
           :mod:`.compass.compass_event_decoder`).
 
     hdf5_settings
@@ -120,11 +122,13 @@ def build_raw(
 
     # process out_spec and setup rb_lib if specified
     rb_lib = None
-    if isinstance(out_spec, str) and out_spec.endswith(".json"):
-        with open(out_spec) as json_file:
-            out_spec = json.load(json_file)
+    allowed_exts = [ext for exts in utils.__file_extensions__.values() for ext in exts]
+    if isinstance(out_spec, str) and any(
+        [out_spec.endswith(ext) for ext in allowed_exts]
+    ):
+        out_spec = utils.load_dict(out_spec)
     if isinstance(out_spec, dict):
-        out_spec = RawBufferLibrary(json_dict=out_spec, kw_dict=kwargs)
+        out_spec = RawBufferLibrary(config=out_spec, kw_dict=kwargs)
     if isinstance(out_spec, RawBufferLibrary):
         rb_lib = out_spec
     # if no rb_lib, write all data to file

daq2lh5/raw_buffer.py

@@ -19,48 +19,49 @@ output to a :class:`.RawBufferList`.
 :class:`.RawBufferLibrary`: a dictionary of :class:`RawBufferList`\ s, e.g. one
 for each :class:`~.data_decoder.DataDecoder`. Keyed by the decoder name.
 
-:class:`.RawBuffer` supports a JSON short-hand notation, see
-:meth:`.RawBufferLibrary.set_from_json_dict` for full specification.
-
-Example JSON yielding a valid :class:`.RawBufferLibrary` is below. In the
-example, the user would call ``RawBufferLibrary.set_from_json_dict(json_dict,
-kw_dict)`` with ``kw_dict`` containing an entry for ``'file_key'``. The other
-keywords ``{key}`` and ``{name}`` are understood by and filled in during
-:meth:`.RawBufferLibrary.set_from_json_dict` unless overloaded in ``kw_dict``.
+:class:`.RawBuffer` supports a config file short-hand notation, see
+:meth:`.RawBufferLibrary.set_from_dict` for full specification.
+
+Example YAML yielding a valid :class:`.RawBufferLibrary` is below (other
+formats like JSON are also supported). In the example, the user would call
+``RawBufferLibrary.set_from_dict(config, kw_dict)`` with ``kw_dict``
+containing an entry for ``'file_key'``. The other keywords ``{key}`` and
+``{name}`` are understood by and filled in during
+:meth:`.RawBufferLibrary.set_from_dict` unless overloaded in ``kw_dict``.
 Note the use of the wildcard ``*``: this will match all other decoder names /
 keys.
 
-.. code-block :: json
-
-    {
-      "FCEventDecoder" : {
-        "g{key:0>3d}" : {
-          "key_list" : [ [24,64] ],
-          "out_stream" : "$DATADIR/{file_key}_geds.lh5:/geds",
-          "proc_spec": {
-            "window": {"waveform", 10, 100, "windowed_waveform"}
-          }
-        },
-        "spms" : {
-          "key_list" : [ [6,23] ],
-          "out_stream" : "$DATADIR/{file_key}_spms.lh5:/spms"
-        },
-        "puls" : {
-          "key_list" : [ 0 ],
-          "out_stream" : "$DATADIR/{file_key}_auxs.lh5:/auxs"
-        },
-        "muvt" : {
-          "key_list" : [ 1, 5 ],
-          "out_stream" : "$DATADIR/{file_key}_auxs.lh5:/auxs"
-        }
-      },
-      "*" : {
-        "{name}" : {
-          "key_list" : [ "*" ],
-          "out_stream" : "$DATADIR/{file_key}_{name}.lh5"
-        }
-      }
-    }
+.. code-block :: yaml
+
+    FCEventDecoder:
+      "g{key:0>3d}":
+        key_list:
+          - [24, 64]
+        out_stream: "$DATADIR/{file_key}_geds.lh5:/geds"
+        proc_spec:
+          window:
+            - waveform
+            - 10
+            - 100
+            - windowed_waveform
+      spms:
+        key_list:
+          - [6, 23]
+        out_stream: "$DATADIR/{file_key}_spms.lh5:/spms"
+      puls:
+        key_list:
+          - 0
+        out_stream: "$DATADIR/{file_key}_auxs.lh5:/auxs"
+      muvt:
+        key_list:
+          - 1
+          - 5
+        out_stream: "$DATADIR/{file_key}_auxs.lh5:/auxs"
+
+    "*":
+      "{name}":
+        key_list: ["*"]
+        out_stream: "$DATADIR/{file_key}_{name}.lh5"
 """
 
 from __future__ import annotations
@@ -105,7 +106,7 @@ class RawBuffer:
         the name or identifier of the object in the output stream.
     proc_spec
         a dictionary containing the following:
-        - a DSP config file, passed as a dictionary, or as a path to a JSON file
+        - a DSP config file, passed as a dictionary, or as a path to a config file
         - an array containing: the name of an LGDO object stored in the :class:`.RawBuffer` to be sliced,
         the start and end indices of the slice, and the new name for the sliced object
         - a dictionary of fields to drop
@@ -185,29 +186,27 @@ class RawBufferList(list):
                     self.keyed_dict[key] = rb
         return self.keyed_dict
 
-    def set_from_json_dict(
-        self, json_dict: dict, kw_dict: dict[str, str] = None
-    ) -> None:
-        """Set up a :class:`.RawBufferList` from a dictionary written in JSON
-        shorthand. See :meth:`.RawBufferLibrary.set_from_json_dict` for details.
+    def set_from_dict(self, config: dict, kw_dict: dict[str, str] = None) -> None:
+        """Set up a :class:`.RawBufferList` from a dictionary. See
+        :meth:`.RawBufferLibrary.set_from_dict` for details.
 
         Notes
         -----
-        `json_dict` is changed by this function.
+        `config` is changed by this function.
         """
-        expand_rblist_json_dict(json_dict, {} if kw_dict is None else kw_dict)
-        for name in json_dict:
+        expand_rblist_dict(config, {} if kw_dict is None else kw_dict)
+        for name in config:
             rb = RawBuffer()
-            if "key_list" in json_dict[name]:
-                rb.key_list = json_dict[name]["key_list"]
-            if "out_stream" in json_dict[name]:
-                rb.out_stream = json_dict[name]["out_stream"]
-            if "proc_spec" in json_dict[name]:
-                rb.proc_spec = json_dict[name][
+            if "key_list" in config[name]:
+                rb.key_list = config[name]["key_list"]
+            if "out_stream" in config[name]:
+                rb.out_stream = config[name]["out_stream"]
+            if "proc_spec" in config[name]:
+                rb.proc_spec = config[name][
                     "proc_spec"
                 ]  # If you swap this with the next line, then key expansion doesn't work
-            if "out_name" in json_dict[name]:
-                rb.out_name = json_dict[name]["out_name"]
+            if "out_name" in config[name]:
+                rb.out_name = config[name]["out_name"]
             else:
                 rb.out_name = name
             self.append(rb)
@@ -250,15 +249,12 @@ class RawBufferLibrary(dict):
     write to them.
     """
 
-    def __init__(self, json_dict: dict = None, kw_dict: dict[str, str] = None) -> None:
-        if json_dict is not None:
-            self.set_from_json_dict(json_dict, kw_dict)
+    def __init__(self, config: dict = None, kw_dict: dict[str, str] = None) -> None:
+        if config is not None:
+            self.set_from_dict(config, kw_dict)
 
-    def set_from_json_dict(
-        self, json_dict: dict, kw_dict: dict[str, str] = None
-    ) -> None:
-        r"""Set up a :class:`.RawBufferLibrary` from a dictionary written in
-        JSON shorthand.
+    def set_from_dict(self, config: dict, kw_dict: dict[str, str] = None) -> None:
+        r"""Set up a :class:`.RawBufferLibrary` from a dictionary.
 
         Basic structure:
 
@@ -288,7 +284,7 @@ class RawBufferLibrary(dict):
           to the first and last integer keys in a contiguous range (e.g. of
           channels) that get stored to the same buffer. These simply get
           replaced with the explicit list of integers in the range. We use
-          lists not tuples for JSON compliance.
+          lists not tuples for config file format compliance.
         * The ``name`` can include ``{key:xxx}`` format specifiers, indicating
           that each key in ``key_list`` should be given its own buffer with the
           corresponding name.  The same specifier can appear in ``out_path`` to
@@ -307,18 +303,18 @@ class RawBufferLibrary(dict):
 
         Parameters
         ----------
-        json_dict
-            loaded from a JSON file written in the allowed shorthand.
-            `json_dict` is changed by this function.
+        config
+            loaded from a config file written in the allowed shorthand.
+            `config` is changed by this function.
         kw_dict
             dictionary of keyword-value pairs for substitutions into the
             ``out_stream`` and ``out_name`` fields.
         """
-        for list_name in json_dict:
+        for list_name in config:
             if list_name not in self:
                 self[list_name] = RawBufferList()
-            self[list_name].set_from_json_dict(
-                json_dict[list_name], {} if kw_dict is None else kw_dict
+            self[list_name].set_from_dict(
+                config[list_name], {} if kw_dict is None else kw_dict
             )
 
     def get_list_of(self, attribute: str, unique: bool = True) -> list:
@@ -352,26 +348,26 @@ class RawBufferLibrary(dict):
             rb_list.clear_full()
 
 
-def expand_rblist_json_dict(json_dict: dict, kw_dict: dict[str, str]) -> None:
-    """Expand shorthands in a JSON dictionary representing a
+def expand_rblist_dict(config: dict, kw_dict: dict[str, str]) -> None:
+    """Expand shorthands in a dictionary representing a
     :class:`.RawBufferList`.
 
-    See :meth:`.RawBufferLibrary.set_from_json_dict` for details.
+    See :meth:`.RawBufferLibrary.set_from_dict` for details.
 
     Notes
     -----
-    The input JSON dictionary is changed by this function.
+    The input dictionary is changed by this function.
     """
     # get the original list of groups because we are going to change the
-    # dict.keys() of json_dict inside the next list. Note: we have to convert
+    # dict.keys() of config inside the next list. Note: we have to convert
     # from dict_keys to list here otherwise the loop complains about changing
     # the dictionary during iteration
-    buffer_names = list(json_dict.keys())
+    buffer_names = list(config.keys())
     for name in buffer_names:
         if name == "":
             raise ValueError("buffer name can't be empty")
 
-        info = json_dict[name]  # changes to info will change json_dict[name]
+        info = config[name]  # changes to info will change config[name]
         # make sure we have a key list
         if "key_list" not in info:
             raise ValueError(f"'{name}' is missing key_list")
@@ -399,12 +395,12 @@ def expand_rblist_json_dict(json_dict: dict, kw_dict: dict[str, str]) -> None:
                 continue  # will be handled later, once the key_list is known
             for key in info["key_list"]:
                 expanded_name = name.format(key=key)
-                json_dict[expanded_name] = info.copy()
-                json_dict[expanded_name]["key_list"] = [key]
-            json_dict.pop(name)
+                config[expanded_name] = info.copy()
+                config[expanded_name]["key_list"] = [key]
+            config.pop(name)
 
     # now re-iterate and expand out_paths
-    for name, info in json_dict.items():
+    for name, info in config.items():
         if len(info["key_list"]) == 1 and not (
             isinstance(info["key_list"][0], str) and "*" in info["key_list"][0]
         ):

daq2lh5/utils.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import yaml
+
+log = logging.getLogger(__name__)
+
+
+__file_extensions__ = {"json": [".json"], "yaml": [".yaml", ".yml"]}
+
+
+def load_dict(fname: str, ftype: str | None = None) -> dict:
+    """Load a text file as a Python dict."""
+    fname = Path(fname)
+
+    # determine file type from extension
+    if ftype is None:
+        for _ftype, exts in __file_extensions__.items():
+            if fname.suffix in exts:
+                ftype = _ftype
+
+    msg = f"loading {ftype} dict from: {fname}"
+    log.debug(msg)
+
+    with fname.open() as f:
+        if ftype == "json":
+            return json.load(f)
+        if ftype == "yaml":
+            return yaml.safe_load(f)
+
+        msg = f"unsupported file format {ftype}"
+        raise NotImplementedError(msg)

{legend_daq2lh5-1.3.0.dist-info → legend_daq2lh5-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: legend_daq2lh5
-Version: 1.3.0
+Version: 1.4.0
 Summary: Convert digitizer data to LH5
 Home-page: https://github.com/legend-exp/legend-daq2lh5
 Author: Jason Detwiler

{legend_daq2lh5-1.3.0.dist-info → legend_daq2lh5-1.4.0.dist-info}/RECORD

@@ -1,14 +1,15 @@
 daq2lh5/__init__.py,sha256=VPmwKuZSA0icpce05ojhnsKWhR4_QUgD0oVXUoN9wks,975
-daq2lh5/_version.py,sha256=HGwtpza1HCPtlyqElUvIyH97K44TO13CYiYVZNezQ1M,411
-daq2lh5/build_raw.py,sha256=w0sS8MEN-UhAHSjlPbqDlMBfzjeznIIrKxNxuRxPFqA,10531
+daq2lh5/_version.py,sha256=R8-T9fmURjcuoxYpHTAjyNAhgJPDtI2jogCjqYYkfCU,411
+daq2lh5/build_raw.py,sha256=SDpdOU8qpfzMtx8gtFu-RZYqxutQo1smJSkv-LrH9YE,10672
 daq2lh5/cli.py,sha256=7bPfH1XbyAS48wZn_0unj4Y5MD5kF7V34Q5srn4jKVM,2913
 daq2lh5/data_decoder.py,sha256=Cn40fodfKs7pKa2odzG1j806iw9IyQVfbbWObNGmof8,10677
 daq2lh5/data_streamer.py,sha256=WuyqneVg8Kf7V072aUWYT5q3vmPZoobjAZ7oSw0sC9k,14187
 daq2lh5/logging.py,sha256=rYNeToaZBTCaIiC42a4CUroAo1PCOreTXbpEZyMO8Fo,987
-daq2lh5/raw_buffer.py,sha256=2jJvH7BbVkJCzF5A2nOIdcgwUs68eLnpvc0FvTjckaI,17688
+daq2lh5/raw_buffer.py,sha256=LfkVAOZa4cWz227Ef22rKi57Shk7GbENoxUEkxC6IgU,17403
+daq2lh5/utils.py,sha256=Pc8Oh0ZVBqwwVehyhSJttHnX5tWbOyEZPk-rVg8mb0c,839
 daq2lh5/buffer_processor/__init__.py,sha256=7k6v_KPximtv7805QnX4-xp_S3vqvqwDfdV3q95oZJo,84
 daq2lh5/buffer_processor/buffer_processor.py,sha256=GUxpNDbqGLuUEZmXjeratipbzmki12RFNYZkxgMtesg,14483
-daq2lh5/buffer_processor/lh5_buffer_processor.py,sha256=yL1ru0_GTsZx099oi45sXL-FxPfdChtStd_IFtZNI_Q,8222
+daq2lh5/buffer_processor/lh5_buffer_processor.py,sha256=einRDLI6EVR-U_TT2GdCZPctFnosJ774eMiUj-ahn6c,8316
 daq2lh5/compass/__init__.py,sha256=mOXHWp7kRDgNTPQty3E8k2KPSy_vAzjneKfAcCVaPyE,132
 daq2lh5/compass/compass_config_parser.py,sha256=zeAsOo1dOJPGLL8-zkAcdYRkqt8BodtOPi96n7fWsl4,12300
 daq2lh5/compass/compass_event_decoder.py,sha256=kiPOaEu8SgLD2wbSPbBahcbTBBRAIw35wtVLBcwPcXY,7386
@@ -32,9 +33,9 @@ daq2lh5/orca/orca_header_decoder.py,sha256=ORIIyfx22ybyKc-uyWy5ER49-dl3BGpHdfV8O
 daq2lh5/orca/orca_packet.py,sha256=nOHuBXsTI1SzTjHZtff0txSQYvkwo4XGx3fpk7XfYj8,2489
 daq2lh5/orca/orca_run_decoder.py,sha256=3atKXC6mDi8_PK6ICUBBJ-LyaTM8OU31kKWIpmttRr4,2065
 daq2lh5/orca/orca_streamer.py,sha256=VbD9PF-rx_Rk-rEy7XECPmgxr6kZSUf0tC7Qbol3Qeg,15693
-legend_daq2lh5-1.3.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-legend_daq2lh5-1.3.0.dist-info/METADATA,sha256=3fR5Z-61fjaRaAEObVk9k8HpTGxiDF0wL4f6djIX8aQ,3956
-legend_daq2lh5-1.3.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-legend_daq2lh5-1.3.0.dist-info/entry_points.txt,sha256=R08R4NrHi0ab5MJN_qKqzePVzrLSsw5WpmbiwwduYjw,59
-legend_daq2lh5-1.3.0.dist-info/top_level.txt,sha256=MJQVLyLqMgMKBdVfNXFaCKCjHKakAs19VLbC9ctXZ7A,8
-legend_daq2lh5-1.3.0.dist-info/RECORD,,
+legend_daq2lh5-1.4.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+legend_daq2lh5-1.4.0.dist-info/METADATA,sha256=SKEW8u7wRrCK5zTY01iMGDYGPmrmg1ED5g9fsbwLVVA,3956
+legend_daq2lh5-1.4.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+legend_daq2lh5-1.4.0.dist-info/entry_points.txt,sha256=R08R4NrHi0ab5MJN_qKqzePVzrLSsw5WpmbiwwduYjw,59
+legend_daq2lh5-1.4.0.dist-info/top_level.txt,sha256=MJQVLyLqMgMKBdVfNXFaCKCjHKakAs19VLbC9ctXZ7A,8
+legend_daq2lh5-1.4.0.dist-info/RECORD,,