data-manipulation-utilities 0.2.8.dev714__py3-none-any.whl → 0.2.8.dev725__py3-none-any.whl

{data_manipulation_utilities-0.2.8.dev714.dist-info → data_manipulation_utilities-0.2.8.dev725.dist-info}/METADATA

@@ -1,7 +1,36 @@
-Metadata-Version: 2.4
-Name: data_manipulation_utilities
-Version: 0.2.8.dev714
+Metadata-Version: 2.3
+Name: data-manipulation-utilities
+Version: 0.2.8.dev725
 Summary: Project storing utilities needed to reduce boilerplate code when analyzing data
+Requires-Python: >=3.10,<3.13
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: dev
+Provides-Extra: fit
+Provides-Extra: ml
+Requires-Dist: PyYAML
+Requires-Dist: awkward
+Requires-Dist: awkward-pandas
+Requires-Dist: dask[dataframe,distributed]
+Requires-Dist: hist[plot]
+Requires-Dist: joblib ; extra == "ml"
+Requires-Dist: logzero
+Requires-Dist: matplotlib
+Requires-Dist: mplhep
+Requires-Dist: numpy
+Requires-Dist: omegaconf
+Requires-Dist: optuna ; extra == "ml"
+Requires-Dist: pandas
+Requires-Dist: pytest ; extra == "dev"
+Requires-Dist: scikit-learn ; extra == "ml"
+Requires-Dist: scipy
+Requires-Dist: tensorflow
+Requires-Dist: toml
+Requires-Dist: tqdm
+Requires-Dist: uproot
+Requires-Dist: zfit (==0.26.0) ; extra == "fit"
 Description-Content-Type: text/markdown
 
 [TOC]
@@ -1793,3 +1822,4 @@ lxplus:
 ```
 
 and should be placed in `$HOME/.config/dmu/ssh/servers.yaml`
+

{data_manipulation_utilities-0.2.8.dev714.dist-info → data_manipulation_utilities-0.2.8.dev725.dist-info}/RECORD

@@ -1,4 +1,3 @@
-data_manipulation_utilities-0.2.8.dev714.data/scripts/publish,sha256=-3K_Y2_4CfWCV50rPB8CRuhjxDu7xMGswinRwPovgLs,1976
 dmu/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dmu/arrays/utilities.py,sha256=PKoYyybPptA2aU-V3KLnJXBudWxTXu4x1uGdIMQ49HY,1722
 dmu/generic/hashing.py,sha256=QR5Gbv6-ANvi5hL232UNMrw9DONpU27BWTynXGxQLGU,1806
@@ -15,7 +14,7 @@ dmu/ml/utilities.py,sha256=A9j3tBh-jfaFdwwLUleo1QnttfawN7XDiQRh4VTvqVY,4597
 dmu/pdataframe/utilities.py,sha256=xl6iLVKUccqVXYjuHsDUZ6UrCKQPw1k8D-f6407Yq30,2742
 dmu/plotting/fwhm.py,sha256=4e8n6624pxWLcOOtayCQ_hDSSMKU21-3UsdmbkX1ojk,1949
 dmu/plotting/matrix.py,sha256=s_5W8O3yXF3u8OX3f4J4hCoxIVZt1TF8S-qJsFBh2Go,5005
-dmu/plotting/plotter.py,sha256=oc_n9ug0JPaQZycrW_TJkgNxjr0LHNrVJcijqmiLUR4,8136
+dmu/plotting/plotter.py,sha256=5N5mLdQAqOUs43ukX5mT9nRaYD5dkn_sED5NoZJV5A0,8483
 dmu/plotting/plotter_1d.py,sha256=Kyoyh-QyZLXXqX19wqEDUWCD1nJEvEonGp9nlgEaoZE,10936
 dmu/plotting/plotter_2d.py,sha256=dXC-7Rsquibe5cn7622ryoKpuv7KCAmouIIXwQ_VEFM,3172
 dmu/plotting/utilities.py,sha256=SI9dvtZq2gr-PXVz71KE4o0i09rZOKgqJKD1jzf6KXk,1167
@@ -77,17 +76,7 @@ dmu_data/text/transform.txt,sha256=EX760da6Vkf-_EPxnQlC5hGSkfFhJCCGCD19NU-1Qto,4
 dmu_data/text/transform_set.toml,sha256=Jeh7BTz82idqvbOQJtl9-ur56mZkzDn5WtvmIb48LoE,150
 dmu_data/text/transform_set.txt,sha256=1KivMoP9LxPn9955QrRmOzjEqduEjhTetQ9MXykO5LY,46
 dmu_data/text/transform_trf.txt,sha256=zxBRTgcSmX7RdqfmWF88W1YqbyNHa4Ccruf1MmnYv2A,74
-dmu_scripts/git/publish,sha256=-3K_Y2_4CfWCV50rPB8CRuhjxDu7xMGswinRwPovgLs,1976
-dmu_scripts/kerberos/check_expiration,sha256=PRJopcyFSeiAHdWpLEZp6mu_OKctUdIJj0HZfC0EWxg,308
-dmu_scripts/kerberos/convert_certificate,sha256=_4k4fmxpK-MbSLkkRYEPLQc9twfYBqOIiYZqL9yAXKE,445
-dmu_scripts/ml/compare_classifiers.py,sha256=XuHdcVyDLFGoKfvfv6YrgIavRpjpMrnBSqUnlliD7ew,2312
-dmu_scripts/physics/check_truth.py,sha256=b1P_Pa9ef6VcFtyY6Y9KS9Om9L-QrCBjDKp4dqca0PQ,3964
-dmu_scripts/rfile/compare_root_files.py,sha256=T8lDnQxsRNMr37x1Y7YvWD8ySHrJOWZki7ZQynxXX9Q,9540
-dmu_scripts/rfile/print_trees.py,sha256=Ze4Ccl_iUldl4eVEDVnYBoe4amqBT1fSBR1zN5WSztk,941
-dmu_scripts/ssh/coned.py,sha256=lhilYNHWRCGxC-jtyJ3LQ4oUgWW33B2l1tYCcyHHsR0,4858
-dmu_scripts/text/transform_text.py,sha256=9akj1LB0HAyopOvkLjNOJiptZw5XoOQLe17SlcrGMD0,1456
-data_manipulation_utilities-0.2.8.dev714.dist-info/METADATA,sha256=M5n-tPUt3o_0kY4viuQj6lbP4JQxWhpxkSnWCW29PFg,50263
-data_manipulation_utilities-0.2.8.dev714.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-data_manipulation_utilities-0.2.8.dev714.dist-info/entry_points.txt,sha256=-02cr8ibY6L_reX-_Owz2N7OUQyTAwydRIvLr9kKZK0,332
-data_manipulation_utilities-0.2.8.dev714.dist-info/top_level.txt,sha256=n_x5J6uWtSqy9mRImKtdA2V2NJNyU8Kn3u8DTOKJix0,25
-data_manipulation_utilities-0.2.8.dev714.dist-info/RECORD,,
+data_manipulation_utilities-0.2.8.dev725.dist-info/METADATA,sha256=_4bxAW7aoKgPY3H1rKhp626lTEYrD2UWrwZX9avU750,51153
+data_manipulation_utilities-0.2.8.dev725.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+data_manipulation_utilities-0.2.8.dev725.dist-info/entry_points.txt,sha256=M0C8_u9B_xSmyfemdPwdIBh9QuPIkjhEpG060Y5_Pjw,321
+data_manipulation_utilities-0.2.8.dev725.dist-info/RECORD,,

{data_manipulation_utilities-0.2.8.dev714.dist-info → data_manipulation_utilities-0.2.8.dev725.dist-info}/WHEEL

@@ -1,5 +1,4 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any
-

data_manipulation_utilities-0.2.8.dev725.dist-info/entry_points.txt

@@ -0,0 +1,8 @@
+[console_scripts]
+check_truth=dmu_scripts.physics.check_truth:main
+compare_classifiers=dmu_scripts.ml.compare_classifiers:main
+compare_root_files=dmu_scripts.rfile.compare_root_files:main
+coned=dmu_scripts.ssh.coned:main
+print_trees=dmu_scripts.rfile.print_trees:main
+transform_text=dmu_scripts.text.transform_text:main
+

dmu/plotting/plotter.py

@@ -5,12 +5,12 @@ Module containing plotter class
 import os
 import json
 import math
-from typing import Union
 
 import numpy
 import matplotlib.pyplot as plt
 
-from ROOT                  import RDataFrame
+from ROOT                  import RDataFrame, RDF
+from omegaconf             import DictConfig
 from dmu.logging.log_store import LogStore
 
 log = LogStore.add_logger('dmu:plotting:Plotter')
@@ -20,16 +20,28 @@ class Plotter:
     Base class of Plotter1D and Plotter2D
     '''
     #-------------------------------------
-    def __init__(self, d_rdf=None, cfg=None):
-        if not isinstance(  cfg, dict):
+    def __init__(
+            self,
+            d_rdf: dict|None            =None,
+            cfg  : dict|DictConfig|None =None):
+        '''
+        Parameters
+        --------------
+        d_rdf: Dictionary where
+            key  : Identifier of dataset
+            value: ROOT dataframe representing dataset
+
+        cfg  : Dictionary or DictConfig instance holding configuration
+        '''
+        if not isinstance(  cfg, (dict,DictConfig)):
             raise ValueError('Config dictionary not passed')
 
         if not isinstance(d_rdf, dict):
             raise ValueError('Dataframe dictionary not passed')
 
         self._d_cfg = cfg
-        self._d_rdf : dict[str, RDataFrame]    = { name : self._preprocess_rdf(rdf) for name, rdf in d_rdf.items()}
-        self._d_wgt : Union[dict[str, Union[numpy.ndarray, None]], None]
+        self._d_rdf : dict[str, RDF.RNode]    = { name : self._preprocess_rdf(rdf) for name, rdf in d_rdf.items()}
+        self._d_wgt : dict[str, numpy.ndarray|None] | None
 
         self._title : str = ''
     #-------------------------------------
@@ -68,9 +80,9 @@ class Plotter:
 
         return minx, maxx
     #-------------------------------------
-    def _preprocess_rdf(self, rdf : RDataFrame) -> RDataFrame:
+    def _preprocess_rdf(self, rdf : RDF.RNode) -> RDF.RNode:
         '''
-        rdf (RDataFrame): ROOT dataframe
+        rdf (RDF.RNode): ROOT dataframe
 
         returns preprocessed dataframe
         '''
@@ -146,7 +158,7 @@ class Plotter:
 
         return rdf
     # --------------------------------------------
-    def _print_weights(self, arr_wgt : Union[numpy.ndarray, None], var : str, sample : str) -> None:
+    def _print_weights(self, arr_wgt : numpy.ndarray|None, var : str, sample : str) -> None:
         if arr_wgt is None:
             log.debug(f'Not using weights for {sample}:{var}')
             return
@@ -171,7 +183,7 @@ class Plotter:
 
         return fig_size
     #-------------------------------------
-    def _get_weights(self, var) -> Union[dict[str, Union[numpy.ndarray, None]], None]:
+    def _get_weights(self, var) -> dict[str, numpy.ndarray|None]| None:
         d_cfg = self._d_cfg['plots'][var]
         if 'weights' not in d_cfg:
             return None
@@ -186,7 +198,7 @@ class Plotter:
 
         return d_weight
     # --------------------------------------------
-    def _read_weights(self, name : str, rdf : RDataFrame) -> numpy.ndarray:
+    def _read_weights(self, name : str, rdf : RDF.RNode) -> numpy.ndarray:
         v_col = rdf.GetColumnNames()
         l_col = [ col.c_str() for col in v_col ]
 

data_manipulation_utilities-0.2.8.dev714.data/scripts/publish

@@ -1,89 +0,0 @@
-#!/usr/bin/env bash
-
-# --------------------------
-display_help()
-{
-    echo "Script meant to:"
-    echo ""
-    echo "1. Check if version in pyproject.toml has been modified"
-    echo "2. If it has create new tag following version name"
-    echo "3. Push to remote "
-}
-# --------------------------
-get_opts()
-{
-    while getopts :hf: option; do 
-        case "${option}" in
-            h)  
-                display_help
-                exit 0
-                ;;  
-           \?)  echo "Invalid option: -${OPTARG}"
-                display_help
-                exit 1
-                ;;  
-            :)  echo "$0: Arguments needed"
-                display_help
-                exit 1
-                ;;  
-        esac
-    done
-}
-# --------------------------
-# Picks up version from pyproject.toml
-get_version()
-{
-    if [[ ! -f pyproject.toml ]];then
-        echo "Cannot find pyproject.toml"
-        exit 1
-    fi
-
-    VERSION_LINE=$(grep version pyproject.toml)
-
-    if [[ $? -ne 0 ]];then
-        ehco "Could not extract version from pyproject.toml"
-        exit 1
-    fi
-
-    if [[ "$VERSION_LINE" =~ .*([0-9]\.[0-9]\.[0-9]).* ]];then
-        VERSION=${BASH_REMATCH[1]}
-        echo "Using version: $VERSION"
-        return
-    fi
-
-    echo "Could not extract version from: $VERSION_LINE"
-    exit 1
-}
-# --------------------------
-create_tag()
-{
-    git tag -n | grep $VERSION
-
-    if [[ $? -eq 0 ]];then
-        echo "Version found among tags, not tagging"
-        return
-    fi
-
-    echo "Version $VERSION not found among tags, creating new tag"
-
-    git tag -a $VERSION
-}
-# --------------------------
-push_all()
-{
-    for REMOTE in $(git remote);do
-        echo "Pushing tags and commits to remote: $REMOTE"
-        git add pyproject.toml
-        git commit -m "Publication commit"
-
-        git pull    $REMOTE HEAD
-        git push -u $REMOTE HEAD
-        git push    $REMOTE --tags
-    done
-}
-# --------------------------
-get_opts "$@"
-
-get_version
-create_tag
-push_all

data_manipulation_utilities-0.2.8.dev714.dist-info/entry_points.txt

@@ -1,7 +0,0 @@
-[console_scripts]
-check_truth = dmu_scripts.physics.check_truth:main
-compare_classifiers = dmu_scripts.ml.compare_classifiers:main
-compare_root_files = dmu_scripts.rfile.compare_root_files:main
-coned = dmu_scripts.ssh.coned:main
-print_trees = dmu_scripts.rfile.print_trees:main
-transform_text = dmu_scripts.text.transform_text:main

data_manipulation_utilities-0.2.8.dev714.dist-info/top_level.txt

@@ -1,3 +0,0 @@
-dmu
-dmu_data
-dmu_scripts

dmu_scripts/git/publish

@@ -1,89 +0,0 @@
-#!/usr/bin/env bash
-
-# --------------------------
-display_help()
-{
-    echo "Script meant to:"
-    echo ""
-    echo "1. Check if version in pyproject.toml has been modified"
-    echo "2. If it has create new tag following version name"
-    echo "3. Push to remote "
-}
-# --------------------------
-get_opts()
-{
-    while getopts :hf: option; do 
-        case "${option}" in
-            h)  
-                display_help
-                exit 0
-                ;;  
-           \?)  echo "Invalid option: -${OPTARG}"
-                display_help
-                exit 1
-                ;;  
-            :)  echo "$0: Arguments needed"
-                display_help
-                exit 1
-                ;;  
-        esac
-    done
-}
-# --------------------------
-# Picks up version from pyproject.toml
-get_version()
-{
-    if [[ ! -f pyproject.toml ]];then
-        echo "Cannot find pyproject.toml"
-        exit 1
-    fi
-
-    VERSION_LINE=$(grep version pyproject.toml)
-
-    if [[ $? -ne 0 ]];then
-        ehco "Could not extract version from pyproject.toml"
-        exit 1
-    fi
-
-    if [[ "$VERSION_LINE" =~ .*([0-9]\.[0-9]\.[0-9]).* ]];then
-        VERSION=${BASH_REMATCH[1]}
-        echo "Using version: $VERSION"
-        return
-    fi
-
-    echo "Could not extract version from: $VERSION_LINE"
-    exit 1
-}
-# --------------------------
-create_tag()
-{
-    git tag -n | grep $VERSION
-
-    if [[ $? -eq 0 ]];then
-        echo "Version found among tags, not tagging"
-        return
-    fi
-
-    echo "Version $VERSION not found among tags, creating new tag"
-
-    git tag -a $VERSION
-}
-# --------------------------
-push_all()
-{
-    for REMOTE in $(git remote);do
-        echo "Pushing tags and commits to remote: $REMOTE"
-        git add pyproject.toml
-        git commit -m "Publication commit"
-
-        git pull    $REMOTE HEAD
-        git push -u $REMOTE HEAD
-        git push    $REMOTE --tags
-    done
-}
-# --------------------------
-get_opts "$@"
-
-get_version
-create_tag
-push_all

dmu_scripts/kerberos/check_expiration

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-: '
-This script is meant to check the expiration date of a grid certificate
-
-Usage:
-
-./check_expiration
-'
-
-check()
-{
-    PEMFILE=$1
-    if [[ ! -f $PEMFILE ]];then
-        echo "Cannot find PEM file: $PEMFILE"
-    fi
-
-    openssl x509 -enddate -noout -in $PEMFILE 
-}
-
-check usercert.pem

dmu_scripts/kerberos/convert_certificate

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-: '
-This script is used to convert p12 grid certificate files into PEM files
-
-Usage:
-
-./convert_certificate cert.p12
-'
-
-CERTIFICATE=$1
-
-if [[ ! -f $CERTIFICATE  ]];then
-    echo "ERROR::Certificate \"$CERTIFICATE\" does not exist."
-    kill -INT $$
-fi
-
-openssl pkcs12 -in $CERTIFICATE -clcerts -nokeys -out usercert.pem
-openssl pkcs12 -in $CERTIFICATE -nocerts -out userkey.pem
-chmod 400 userkey.pem
-chmod 444 usercert.pem
-

dmu_scripts/ml/compare_classifiers.py

@@ -1,85 +0,0 @@
-'''
-Script used to compare performance of classifiers
-'''
-import os
-import argparse
-import yaml
-import mplhep
-import matplotlib.pyplot as plt
-import pandas            as pnd
-
-from sklearn.metrics       import auc
-from dmu.logging.log_store import LogStore
-
-log=LogStore.add_logger('dmu:ml:compare_classifiers')
-# ------------------------------
-class Data:
-    '''
-    Data class
-    '''
-    out_path : str
-    cfg_path : str
-    logl     : int
-    cfg      : dict
-
-    plt.style.use(mplhep.style.LHCb2)
-# ------------------------------
-def _initialize() -> None:
-    log.info(f'Loading settings from: {Data.cfg_path}')
-    with open(Data.cfg_path, encoding='utf-8') as ifile:
-        Data.cfg = yaml.safe_load(ifile)
-
-    Data.out_path = Data.cfg['out_dir']
-    os.makedirs(Data.out_path, exist_ok=True)
-# ------------------------------
-def _parse_args():
-    parser = argparse.ArgumentParser(description='Used to perform comparisons of classifier performances')
-    parser.add_argument('-c', '--conf' , help='Path to configuration path', required=True)
-    parser.add_argument('-l', '--logl' , help='Logging level', choices=[10, 20, 30], default=20)
-    args = parser.parse_args()
-
-    Data.cfg_path = args.conf
-    Data.logl     = args.logl
-# ------------------------------
-def _plot_roc(name : str, path : str) -> None:
-    roc_path = f'{path}/fold_all/roc.json'
-    df = pnd.read_json(roc_path)
-
-    plt.figure(num='ROC')
-    xval = df['x'].to_numpy()
-    yval = df['y'].to_numpy()
-    area = auc(xval, yval)
-
-    plt.plot(xval, yval, label=f'{name}: {area:.3f}')
-# ------------------------------
-def _compare():
-    for name, cls_path in Data.cfg['classifiers'].items():
-        _plot_roc(name=name, path=cls_path)
-
-    _save_roc()
-# ------------------------------
-def _save_roc():
-    d_set = Data.cfg['roc']
-    if 'xrange' in d_set:
-        plt.xlim(d_set['xrange'])
-
-    if 'yrange' in d_set:
-        plt.ylim(d_set['yrange'])
-
-    plt.figure(num='ROC')
-    plt.legend()
-    plt.grid()
-    plt.xlabel('Signal Efficiency')
-    plt.ylabel('Background Rejection')
-    plt.savefig(f'{Data.out_path}/roc.png')
-# ------------------------------
-def main():
-    '''
-    Start here
-    '''
-    _parse_args()
-    _initialize()
-    _compare()
-# ------------------------------
-if __name__ == '__main__':
-    main()

dmu_scripts/physics/check_truth.py

@@ -1,121 +0,0 @@
-'''
-Script meant to do truth matching checks
-'''
-import os
-import copy
-import argparse
-
-import yaml
-import mplhep
-import matplotlib.pyplot as plt
-
-from ROOT import RDataFrame
-
-from dmu.logging.log_store   import LogStore
-from dmu.plotting.plotter_1d import Plotter1D as Plotter
-
-log=LogStore.add_logger('dmu:physics:check_truth')
-# ----------------------------------
-def _set_logs() -> None:
-    LogStore.set_level('dmu:plotting:Plotter'  , 30)
-    LogStore.set_level('dmu:plotting:Plotter1D', 30)
-# ----------------------------------
-def _get_args() -> argparse.Namespace:
-    '''
-    Parse args
-    '''
-    parser = argparse.ArgumentParser(description='Script used to carry out checks on truth matching mechanisms for MC')
-    parser.add_argument('-c', '--conf' , type=str, help='Path to config file', required=True)
-    args = parser.parse_args()
-
-    return args
-# ----------------------------------
-def _get_config(args : argparse.Namespace) -> dict:
-    path = args.conf
-    if not os.path.isfile(path):
-        raise FileNotFoundError(f'Cannot find {path}')
-
-    with open(path, encoding='utf-8') as ifile:
-        cfg = yaml.safe_load(ifile)
-
-    return cfg
-# ----------------------------------
-def _get_rdf(file_path : str, tree_path : str) -> RDataFrame:
-    log.debug(f'Picking inputs from: {file_path}/{tree_path}')
-    rdf = RDataFrame(tree_path, file_path)
-
-    nentries = rdf.Count().GetValue()
-    log.debug(f'Found {nentries} entries')
-
-    return rdf
-# ----------------------------------
-def _preprocess_rdf(rdf : RDataFrame, cfg : dict) -> RDataFrame:
-    if 'max_entries' in cfg:
-        max_entries = cfg['max_entries']
-        rdf = rdf.Range(max_entries)
-
-    return rdf
-# ----------------------------------
-def _check(cfg : dict) -> None:
-    log.info(110 * '-')
-    log.info(f'{"Sample":<20}{"Method":<20}{"Initial":<15}{"":<15}{"Final":<15}{"":15}{"Efficiency":<10}')
-    log.info(110 * '-')
-
-    for sample_name in cfg['samples']:
-        file_path = cfg['samples'][sample_name]['file_path']
-        tree_path = cfg['samples'][sample_name]['tree_path']
-        rdf = _get_rdf(file_path, tree_path)
-        rdf = _preprocess_rdf(rdf, cfg)
-
-        d_cut_true = {}
-        d_cut_fake = {}
-        for method, cut in cfg['samples'][sample_name]['methods'].items():
-            _check_kind(rdf, sample_name, method, cut)
-
-            d_cut_true[method] = cut
-            d_cut_fake[method] = f'({cut}) == 0'
-        log.info('')
-
-        _plot_distributions(cfg, sample_name, rdf, d_cut_true, kind='matched')
-        _plot_distributions(cfg, sample_name, rdf, d_cut_fake, kind='anti_matched')
-# ----------------------------------
-def _plot_distributions(cfg : dict, sample_name : str, rdf : RDataFrame, d_cut : dict[str,str], kind : str) -> None:
-    cfg     = copy.deepcopy(cfg)
-    cfg_plt = cfg['samples'][sample_name]['plot']
-    cfg_plt = _add_suffix(cfg_plt, sample_name, kind)
-    d_rdf   = { method : rdf.Filter(cut) for method, cut in d_cut.items() }
-
-    ptr=Plotter(d_rdf=d_rdf, cfg=cfg_plt)
-    ptr.run()
-# ----------------------------------
-def _add_suffix(cfg : dict, sample_name : str, kind : str) -> dict:
-    d_var = cfg['plots']
-    for var in d_var:
-        d_var[var]['name']  = f'{var}_{kind}'
-        d_var[var]['title'] = f'{sample_name}; {kind}'
-
-    cfg['plots'] = d_var
-
-    return cfg
-# ----------------------------------
-def _check_kind(rdf : RDataFrame, sample : str, name : str, cut : str) -> RDataFrame:
-    nini = rdf.Count().GetValue()
-    rdf  = rdf.Filter(cut, name)
-    nfnl = rdf.Count().GetValue()
-    eff  = nfnl / nini * 100
-
-    log.info(f'{sample:<20}{name:<20}{nini:<15}{"":<15}{nfnl:<15}{"-->":15}{eff:10.2f}')
-# ----------------------------------
-def main():
-    '''
-    Script starts here
-    '''
-    _set_logs()
-    args = _get_args()
-    cfg  = _get_config(args)
-    plt.style.use(mplhep.style.LHCb2)
-
-    _check(cfg)
-# ----------------------------------
-if __name__ == '__main__':
-    main()

dmu_scripts/rfile/compare_root_files.py

@@ -1,299 +0,0 @@
-'''
-Script used to compare ROOT files
-'''
-
-import re
-import os
-from dataclasses import dataclass
-from typing import ClassVar
-
-import argparse
-
-import yaml
-import numpy
-from dmu.logging.log_store import LogStore
-
-from ROOT import TFile, TTree, RDataFrame
-
-import dmu.rfile.utilities   as rfut
-
-
-log=LogStore.add_logger('rx_scripts:compare_files')
-#------------------
-@dataclass
-class Data:
-    '''
-    Class used to store shared attributes
-    '''
-    max_entries   : int
-    max_trees     : int
-    l_exclude     : list[str]
-    raise_if_diff : bool
-    file_name_1   : str
-    file_name_2   : str
-
-    d_summary     : ClassVar[dict]= {}
-#------------------
-def _print_trees_difference(l_val_1 : list[str], l_val_2 : list[str]) -> None:
-    s_val_1 = set(l_val_1)
-    s_val_2 = set(l_val_2)
-
-    s_only_1 = s_val_1 - s_val_2
-    s_only_2 = s_val_2 - s_val_1
-
-    Data.d_summary[f'Trees only in {Data.file_name_1}'] = list(s_only_1)
-    Data.d_summary[f'Trees only in {Data.file_name_2}'] = list(s_only_2)
-
-    nonly_1  = len(s_only_1)
-    nonly_2  = len(s_only_2)
-
-    if nonly_1 > 0:
-        log.info(f'Found {nonly_1} trees in first file but not second')
-        for name in s_only_1:
-            log.info(f'{"":<4}{name:<20}')
-
-    if nonly_2 > 0:
-        log.info(f'Found {nonly_2} trees in second file but not first')
-        for name in s_only_2:
-            log.info(f'{"":<4}{name:<20}')
-#------------------
-def _check_trees(d_tree_1 : dict[str, TTree], d_tree_2 : dict[str, TTree]):
-    '''
-    Check if dictionaries have same trees
-    For corresponding trees, check if number of entries is the same
-    '''
-    l_treename_1 = list(d_tree_1.keys())
-    l_treename_2 = list(d_tree_2.keys())
-
-    if l_treename_1 != l_treename_2:
-        log.warning('Files contain different trees')
-        _print_trees_difference(l_treename_1, l_treename_2)
-
-    s_treename_1 = set(l_treename_1)
-    s_treename_2 = set(l_treename_2)
-    s_treename   = s_treename_1 & s_treename_2
-
-    for treename in s_treename:
-        if treename in Data.l_exclude:
-            continue
-
-        tree_1 = d_tree_1[treename]
-        tree_2 = d_tree_2[treename]
-
-        entries_1 = tree_1.GetEntries()
-        entries_2 = tree_2.GetEntries()
-
-        if entries_1 != entries_2:
-            raise ValueError(f'Tree {treename} differs in entries {entries_1}/{entries_2}')
-
-    return list(s_treename)
-#------------------
-def _get_data(tree : TTree) -> dict[str, numpy.ndarray]:
-    rdf = RDataFrame(tree)
-    if Data.max_entries > 0:
-        log.warning(f'Limiting to {Data.max_entries} entries')
-        rdf = rdf.Range(Data.max_entries)
-
-    d_data = rdf.AsNumpy(exclude=[])
-
-    return d_data
-#------------------
-def _check_branches(tree_name : str, l_branch_1 : list[str], l_branch_2 : list[str]) -> None:
-    '''
-    Takes lists of branch names
-    Checks if they are the same, if not print differences
-
-    if raise_if_diff is True, will raise exception if branches are not the same
-    '''
-    if l_branch_1 == l_branch_2:
-        return
-
-    s_branch_1 = set(l_branch_1)
-    s_branch_2 = set(l_branch_2)
-
-    s_branch_1_m_2 = s_branch_1.difference(s_branch_2)
-    log.info(f'Found len({s_branch_1_m_2}) branches in first tree but not second')
-    for branch_name in s_branch_1_m_2:
-        log.debug(f'{"":<4}{branch_name:<20}')
-
-    s_branch_2_m_1 = s_branch_2.difference(s_branch_1)
-    log.info(f'Found len({s_branch_2_m_1}) branches in second tree but not first')
-    for branch_name in s_branch_2_m_1:
-        log.debug(f'{"":<4}{branch_name:<20}')
-
-    Data.d_summary[tree_name] = {
-            f'Only {Data.file_name_1}' : list(s_branch_1_m_2),
-            f'Only {Data.file_name_2}' : list(s_branch_2_m_1),
-            }
-
-    if Data.raise_if_diff:
-        raise ValueError('Branches differ')
-#------------------
-def _compare_branches(tree_name : str, d_data_1 : dict[str, list], d_data_2 : dict[str, list]) -> list[str]:
-    '''
-    Will check for different branches in trees
-    Will return list of branch names for common branches
-    '''
-    l_branch_1 = list(d_data_1.keys())
-    l_branch_2 = list(d_data_2.keys())
-
-    l_branch_1.sort()
-    l_branch_2.sort()
-    _check_branches(tree_name, l_branch_1, l_branch_2)
-
-    s_branch_1 = set(l_branch_1)
-    s_branch_2 = set(l_branch_2)
-
-    s_branch = s_branch_1.intersection(s_branch_2)
-
-    return list(s_branch)
-#------------------
-def _compare(tree_name : str, d_data_1, d_data_2) -> None:
-    log.info('')
-    log.debug('Comparing branches')
-    l_branch_name = _compare_branches(tree_name, d_data_1, d_data_2)
-
-    log.debug('Comparing contents of branches')
-    l_diff_branch = []
-    for branch_name in l_branch_name:
-        arr_val_1 = d_data_1[branch_name]
-        arr_val_2 = d_data_2[branch_name]
-
-        if _contents_differ(tree_name, branch_name, arr_val_1, arr_val_2):
-            l_diff_branch.append(branch_name)
-
-    ndiff = len(l_diff_branch)
-    ntot  = len(l_branch_name)
-
-    Data.d_summary[f'Branches that differ for tree: {tree_name}'] = l_diff_branch
-
-    if ndiff == 0:
-        log.debug(f'Trees {tree_name} have same contents')
-        return
-
-    log.warning(f'{ndiff:<10}{"differing branches out of":<20}{ntot:<10}{"in":<10}{tree_name:<50}')
-    for branch_name in l_diff_branch:
-        log.debug(f'{"":<4}{branch_name:<20}')
-#------------------
-def _contents_differ(tree_name : str, branch_name : str, arr_val_1 : numpy.ndarray, arr_val_2 : numpy.ndarray) -> bool:
-    is_different = False
-    str_type = str(arr_val_1.dtype)
-    if   str_type == 'object':
-        return is_different
-
-    if str_type not in ['bool', 'int32', 'uint32', 'uint64', 'float64', 'float32']:
-        log.info(f'Skipping {branch_name}, {str_type}')
-        return is_different
-
-    if not numpy.array_equal(arr_val_1, arr_val_2):
-        is_different = True
-
-        log.debug(20 * '-')
-        log.debug(f'Branch {branch_name} in tree {tree_name} differ')
-        log.debug(20 * '-')
-        log.debug(arr_val_1)
-        log.debug(arr_val_2)
-        log.debug(20 * '-')
-
-    return is_different
-#------------------
-def _update_keys(d_tree):
-    d_out = {}
-
-    for key, val in d_tree.items():
-        #Remove everything before .root/ and use it as new key
-        new_key = re.sub(r'^.*\.root/', '', key)
-        d_out[new_key] = val
-
-    return d_out
-#------------------
-def _check_file_existence(path : str) -> None:
-    if not os.path.isfile(path):
-        raise FileNotFoundError(f'Cannot find {path}')
-#------------------
-def _validate(file_1 : str, file_2 : str) -> None:
-    _check_file_existence(file_1)
-    _check_file_existence(file_2)
-
-    ifile_1     = TFile(file_1)
-    ifile_2     = TFile(file_2)
-
-    d_tree_1    =  rfut.get_trees_from_file(ifile_1)
-    d_tree_1    = _update_keys(d_tree_1)
-
-    d_tree_2    =  rfut.get_trees_from_file(ifile_2)
-    d_tree_2    = _update_keys(d_tree_2)
-
-    l_tree_name = _check_trees(d_tree_1, d_tree_2)
-
-    if Data.max_trees > -1:
-        log.warning(f'Limiting to {Data.max_trees} trees')
-        l_tree_name = l_tree_name[:Data.max_trees]
-
-    ncommon = len(l_tree_name)
-    log.debug(f'Found common {ncommon} trees')
-    for name in l_tree_name:
-        log.debug(f'{"":<4}{name}')
-
-    log.info('Checking trees')
-    for treename in l_tree_name:
-        if treename in Data.l_exclude:
-            log.debug(f'Skipping {treename}')
-            continue
-
-        log.debug(f'{"":<4}{treename}')
-
-        tree_1 = d_tree_1[treename]
-        tree_2 = d_tree_2[treename]
-
-        log.debug('Getting data from reference')
-        d_data_1= _get_data(tree_1)
-
-        log.debug('Getting data from new')
-        d_data_2= _get_data(tree_2)
-
-        log.debug(f'Comparing {treename}')
-        _compare(treename, d_data_1, d_data_2)
-
-    ifile_1.Close()
-    ifile_2.Close()
-#------------------
-def _save_summary() -> None:
-    '''
-    Saves Data.d_summary to summary.yaml
-    '''
-
-    with open('summary.yaml', 'w', encoding='utf-8') as ofile:
-        yaml.dump(Data.d_summary, ofile, indent=2, default_flow_style=False)
-#------------------
-def main():
-    '''
-    Script starts here
-    '''
-    parser = argparse.ArgumentParser(description='Used to validate versions of code that produce potentially different files')
-    parser.add_argument('-f', '--files'         , nargs=  2, help='List of files to compare')
-    parser.add_argument('-n', '--max_entries'   , type=int , help='Limit running over this number of entries. By default will run over everything', default=-1)
-    parser.add_argument('-t', '--max_trees'     , type=int , help='Limit running over this number of trees. By default will run over everything'  , default=-1)
-    parser.add_argument('-l', '--log_level'     , type=int , help='Logging level'                              , default=20, choices=[10, 20, 30, 40])
-    parser.add_argument('-e', '--exclude'       , nargs='+', help='List of trees that should not be compared'  , default=[], )
-    parser.add_argument('-r', '--raise_if_diff' ,            help='If used, will fail as soon as it finds trees with different branches.', action='store_true')
-
-    args = parser.parse_args()
-
-    LogStore.set_level('rx_scripts:compare_files', args.log_level)
-
-    Data.max_entries   = args.max_entries
-    Data.max_trees     = args.max_trees
-    Data.l_exclude     = args.exclude
-    Data.raise_if_diff = args.raise_if_diff
-
-    [file_1, file_2] = args.files
-
-    Data.file_name_1 = file_1
-    Data.file_name_2 = file_2
-
-    _validate(file_1, file_2)
-    _save_summary()
-#------------------
-if __name__ == '__main__':
-    main()

dmu_scripts/rfile/print_trees.py

@@ -1,35 +0,0 @@
-'''
-Script used to print contents of root files
-'''
-
-import argparse
-
-from dmu.rfile.rfprinter import RFPrinter
-
-# -----------------------------
-class Data:
-    '''
-    Data class holding shared attributes
-    '''
-    path   : str
-    screen : bool
-# -----------------------------
-def _get_args():
-    parser = argparse.ArgumentParser(description='Script used to print information about ROOT files and dump it to text')
-    parser.add_argument('-p', '--path'  , type=str, help='Path to ROOT file')
-    parser.add_argument('-s', '--screen', help='If used, will dump output to screen', action='store_true')
-    args = parser.parse_args()
-
-    Data.path  = args.path
-    Data.screen= args.screen
-# -----------------------------
-def main():
-    '''
-    Execution starts here
-    '''
-    _get_args()
-    prt = RFPrinter(path = Data.path)
-    prt.save(to_screen = Data.screen)
-# -----------------------------
-if __name__ == '__main__':
-    main()

dmu_scripts/ssh/coned.py

@@ -1,168 +0,0 @@
-'''
-Script used to implement connection to servers
-'''
-
-import os
-import copy
-import argparse
-
-import yaml
-from dmu.logging.log_store import LogStore
-
-log = LogStore.add_logger('dmu:scripts:coned')
-#---------------------------------------
-class Data:
-    '''
-    Class used to store shared data
-    '''
-    logl : int
-    dry  : bool
-    prnt : bool
-    cfg  : dict
-    l_ad : list[str]
-    l_rm : list[str]
-#----------------------------
-def _print_configs():
-    '''
-    Prints configuration
-    '''
-
-    yaml_output = yaml.dump(Data.cfg, default_flow_style=False)
-    print(yaml_output)
-#----------------------------
-def _initialize():
-    _load_config()
-
-    LogStore.set_level('dmu:scripts:coned', Data.logl)
-
-    log.debug(f'Running at {Data.logl} logging level')
-#----------------------------
-def _get_args():
-    '''
-    Will parse arguments
-    '''
-    parser = argparse.ArgumentParser(description='Used to edit and print server list specified by ~/.config/connect/servers.yaml')
-    parser.add_argument('-p', '--print'  , help ='Prints config settings and exits', action='store_true')
-    parser.add_argument('-l', '--log_lvl', type =int, help='Logging level', default=20, choices=[10,20,30])
-    parser.add_argument('-a', '--add'    , nargs=3  , help='Adds task to given server, e.g. task 123 server'     , default=[])
-    parser.add_argument('-r', '--rem'    , nargs=3  , help='Removes task from given server, e.g. task 123 server', default=[])
-    parser.add_argument('-d', '--dry'    , help='Run dry run, for adding and removing entries', action='store_true')
-    args   = parser.parse_args()
-
-    Data.prnt = args.print
-    Data.logl = args.log_lvl
-    Data.l_ad = args.add
-    Data.l_rm = args.rem
-    Data.dry  = args.dry
-#---------------------------------------
-def _load_config():
-    home_dir    = os.environ['HOME']
-    config_path = f'{home_dir}/.config/dmu/ssh/servers.yaml'
-    if not os.path.isfile(config_path):
-        raise FileNotFoundError(f'Config not found: {config_path}')
-
-    with open(config_path, encoding='utf-8') as ifile:
-        Data.cfg = yaml.safe_load(ifile)
-#---------------------------------------
-def _dump_config(cfg : dict):
-    if cfg == Data.cfg:
-        log.debug('Config was not modified, will not save it')
-        return
-
-    home_dir    = os.environ['HOME']
-    config_path = f'{home_dir}/.config/dmu/ssh/servers.yaml'
-    if not os.path.isfile(config_path):
-        raise FileNotFoundError(f'Config not found: {config_path}')
-
-    if Data.dry:
-        content = yaml.dump(cfg, default_flow_style=False)
-        print(content)
-        return
-
-    with open(config_path, 'w', encoding='utf-8') as ofile:
-        yaml.dump(cfg, ofile, default_flow_style=False)
-#---------------------------------------
-def _get_updated_config() -> dict:
-    log.debug('Getting updated config')
-
-    cfg = copy.deepcopy(Data.cfg)
-    cfg = _add_task(cfg)
-    cfg = _remove_task(cfg)
-
-    return cfg
-#---------------------------------------
-def _add_task(cfg : dict) -> dict:
-    if len(Data.l_ad) == 0:
-        log.debug('No task added')
-        return cfg
-
-    [task, machine, server] = Data.l_ad
-    if server not in cfg:
-        cfg[server] = {}
-
-    if machine not in cfg[server]:
-        cfg[server][machine] = []
-
-    cfg[server][machine].append(task)
-
-    log.info(f'{"Added":<10}{server:<20}{machine:<10}{task:<20}')
-
-    return cfg
-#---------------------------------------
-def _remove_task(cfg : dict) -> dict:
-    if len(Data.l_rm) == 0:
-        log.debug('No task removed')
-        return cfg
-
-    [task, machine, server] = Data.l_rm
-    if server not in cfg:
-        log.warning(f'Server {server} not found')
-        return cfg
-
-    if machine not in cfg[server]:
-        log.warning(f'Machine {machine} not found in server {server}')
-        return cfg
-
-    l_task = cfg[server][machine]
-    if task not in l_task:
-        log.warning(f'Task {task} not found in {server}:{machine}')
-        return cfg
-
-    index = l_task.index(task)
-    del l_task[index]
-    cfg[server][machine] = l_task
-
-    log.info(f'{"Removed":<10}{server:<20}{machine:<10}{task:<20}')
-
-    cfg = _trim_config(cfg, machine, server)
-
-    return cfg
-#---------------------------------------
-def _trim_config(cfg : dict, machine : str, server : str) -> dict:
-    if cfg[server][machine] == []:
-        log.debug(f'Trimming {server}:{machine}')
-        del cfg[server][machine]
-
-    if cfg[server] == {}:
-        log.debug(f'Trimming {server}')
-        del cfg[server]
-
-    return cfg
-#---------------------------------------
-def main():
-    '''
-    Starts here
-    '''
-    _get_args()
-    _initialize()
-
-    if Data.prnt:
-        log.debug('Printing and returning')
-        _print_configs()
-        return
-
-    cfg = _get_updated_config()
-    _dump_config(cfg)
-#---------------------------------------
-if __name__ == '__main__':
-    main()

dmu_scripts/text/transform_text.py

@@ -1,46 +0,0 @@
-#!/usr/bin/env python3
-
-from dmu.text.transformer import transformer as txt_trf
-
-import argparse
-import logging
-
-log = logging.getLogger('dmu_scripts:text:transformer')
-#---------------------------------
-class data:
-    txt = None
-    out = None
-    cfg = None
-    lvl = None
-#---------------------------------
-def get_args():
-    parser=argparse.ArgumentParser(description='Will transform a text file following a set of rules')
-    parser.add_argument('-i', '--input' , type=str, help='Path to input file' , required=True) 
-    parser.add_argument('-o', '--output', type=str, help='Path to output file, if not passed, it will be same as input, but with trf before extension')
-    parser.add_argument('-c', '--config', type=str, help='Path to config file', required=True) 
-    parser.add_argument('-l', '--loglvl', type=int, help='Log level'          , default=20, choices=[10, 20, 30, 40]) 
-    args = parser.parse_args()
-
-    data.txt = args.input
-    data.out = args.output
-    data.cfg = args.config
-    data.lvl = args.loglvl
-#---------------------------------
-def set_logs():
-    logging.basicConfig()
-
-    log_tr = logging.getLogger('dmu:text:transformer')
-
-    log_tr.setLevel(data.lvl)
-    log.setLevel(data.lvl)
-#---------------------------------
-def main():
-    get_args()
-    set_logs()
-
-    trf = txt_trf(txt_path=data.txt, cfg_path=data.cfg)
-    trf.save_as(data.out)
-#---------------------------------
-if __name__ == '__main__':
-    main()
-