data-manipulation-utilities 0.2.7__py3-none-any.whl → 0.2.8.dev720__py3-none-any.whl

data_manipulation_utilities-0.2.8.dev720.dist-info/RECORD

@@ -0,0 +1,45 @@
+dmu/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dmu/arrays/utilities.py,sha256=PKoYyybPptA2aU-V3KLnJXBudWxTXu4x1uGdIMQ49HY,1722
+dmu/generic/hashing.py,sha256=QR5Gbv6-ANvi5hL232UNMrw9DONpU27BWTynXGxQLGU,1806
+dmu/generic/utilities.py,sha256=0tT93vF_x0q8STRrTD0GvBEpALz-mqE-vJyen4zWCO8,6861
+dmu/generic/version_management.py,sha256=j0ImlAq6SVNjTh3xRsF6G7DSoyr1w8kTRY84dNriGRE,3750
+dmu/logging/log_store.py,sha256=eRSy8Y4fuiDFJK02Z6fq67XQzOrhQ7GMr2LvvJQbJ40,5172
+dmu/logging/messages.py,sha256=Oj3O5EO2KOPtffyVq2P7RPzjpoXtxZ6yXO5HwTftVcM,2903
+dmu/ml/cv_classifier.py,sha256=6rjezMahwL-WzLGKU-fzMzNxJZAGbM7YAbhaZVcJ3F0,4258
+dmu/ml/cv_diagnostics.py,sha256=PLh41mSVE8Kagp9KcuRDN_7tDL9MjPxQzuewY8jDnNo,7600
+dmu/ml/cv_performance.py,sha256=q9sLxIx7GP-dand3tnhHCBJnT6xqssNdRYv_TVjYWUM,1910
+dmu/ml/cv_predict.py,sha256=0sc_OqwOewKvipcMyi3QqkgG30nkpZZjE-SOhHWHMd0,10778
+dmu/ml/train_mva.py,sha256=7KAFX_zOx8MGbYx62U81JbdBkrZvqclSSkgmYvWX-60,34861
+dmu/ml/utilities.py,sha256=A9j3tBh-jfaFdwwLUleo1QnttfawN7XDiQRh4VTvqVY,4597
+dmu/pdataframe/utilities.py,sha256=xl6iLVKUccqVXYjuHsDUZ6UrCKQPw1k8D-f6407Yq30,2742
+dmu/plotting/fwhm.py,sha256=4e8n6624pxWLcOOtayCQ_hDSSMKU21-3UsdmbkX1ojk,1949
+dmu/plotting/matrix.py,sha256=s_5W8O3yXF3u8OX3f4J4hCoxIVZt1TF8S-qJsFBh2Go,5005
+dmu/plotting/plotter.py,sha256=oc_n9ug0JPaQZycrW_TJkgNxjr0LHNrVJcijqmiLUR4,8136
+dmu/plotting/plotter_1d.py,sha256=Kyoyh-QyZLXXqX19wqEDUWCD1nJEvEonGp9nlgEaoZE,10936
+dmu/plotting/plotter_2d.py,sha256=dXC-7Rsquibe5cn7622ryoKpuv7KCAmouIIXwQ_VEFM,3172
+dmu/plotting/utilities.py,sha256=SI9dvtZq2gr-PXVz71KE4o0i09rZOKgqJKD1jzf6KXk,1167
+dmu/rdataframe/atr_mgr.py,sha256=FdhaQWVpsm4OOe1IRbm7rfrq8VenTNdORyI-lZ2Bs1M,2386
+dmu/rdataframe/utilities.py,sha256=cY1Na8HbJ7kB2dwmBagRdsRyCA4ZT_vyIU86ewREj2Y,5322
+dmu/rfile/ddfgetter.py,sha256=0jfNzpv72_NQUKOK5SBsn289rUqVt2BMvuL-Ro5oY7I,3316
+dmu/rfile/rfprinter.py,sha256=mp5jd-oCJAnuokbdmGyL9i6tK2lY72jEfROuBIZ_ums,3941
+dmu/rfile/utilities.py,sha256=XuYY7HuSBj46iSu3c60UYBHtI6KIPoJU_oofuhb-be0,945
+dmu/stats/fit_stats.py,sha256=wzkQT9U32ljGe4azUj1Fj0ECF3zmnH2Ncn0O-_Pl1zQ,4070
+dmu/stats/fitter.py,sha256=rm_fwjkq-0LSjXB_gt3y6BnHoK8Xvd4gHYwKBUJaItQ,19603
+dmu/stats/function.py,sha256=yzi_Fvp_ASsFzbWFivIf-comquy21WoeY7is6dgY0Go,9491
+dmu/stats/gof_calculator.py,sha256=63zNJJGKPy-j_hPNPfu9qNlhrHjYIgJOyL8-VDtbwuI,4894
+dmu/stats/minimizers.py,sha256=db9R2G0SOV-k0BKi6m4EyB_yp6AtZdP23_28B0315oo,7094
+dmu/stats/model_factory.py,sha256=0_o5OmiX0cNhp9_cNqBOYfasBgKlQkQPiy5nqi9qQKA,18966
+dmu/stats/parameters.py,sha256=9lycexTT5ZcxXciiQY9HoJV8O1ahrTEkagd7dYXcfj8,3224
+dmu/stats/utilities.py,sha256=7_tr1j-dl3lLNpxIMWruZs4yUtlNuUTknwGMERpfLhs,17338
+dmu/stats/wdata.py,sha256=IbjZFU9SHTLSYfaBgqamDvqy1K7-3-SaKbU4bGsamK0,6799
+dmu/stats/zfit.py,sha256=aSZj_4IHi9IBthfqlNJeA8YSoMmXO5WipgiKnXKGbnM,286
+dmu/stats/zfit_models.py,sha256=SI61KJ-OG1UAabDICU1iTh6JPKM3giR2ErDraRjkCV8,1842
+dmu/stats/zfit_plotter.py,sha256=gbN5KxhJcP4ItCi98c-fj5_UtvVWL_NA9jkTHiRjvnE,23854
+dmu/testing/utilities.py,sha256=WYlz7Ve5lQjuWhhNL4gWe6_qcByBLV762Lhrc6A0P9E,7421
+dmu/text/transformer.py,sha256=4lrGknbAWRm0-rxbvgzOO-eR1-9bkYk61boJUEV3cQ0,6100
+dmu/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dmu/workflow/cache.py,sha256=CtkGwxuF4UJlD55SmUJcRgWYLsbZOyUvYLI8oTVzk_g,8768
+data_manipulation_utilities-0.2.8.dev720.dist-info/METADATA,sha256=RuHltvo8DQctnGYdFssfMv92oU6b7tgn3haFZ2HVk0E,51153
+data_manipulation_utilities-0.2.8.dev720.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+data_manipulation_utilities-0.2.8.dev720.dist-info/entry_points.txt,sha256=M0C8_u9B_xSmyfemdPwdIBh9QuPIkjhEpG060Y5_Pjw,321
+data_manipulation_utilities-0.2.8.dev720.dist-info/RECORD,,

{data_manipulation_utilities-0.2.7.dist-info → data_manipulation_utilities-0.2.8.dev720.dist-info}/WHEEL

@@ -1,5 +1,4 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any
-

data_manipulation_utilities-0.2.8.dev720.dist-info/entry_points.txt

@@ -0,0 +1,8 @@
+[console_scripts]
+check_truth=dmu_scripts.physics.check_truth:main
+compare_classifiers=dmu_scripts.ml.compare_classifiers:main
+compare_root_files=dmu_scripts.rfile.compare_root_files:main
+coned=dmu_scripts.ssh.coned:main
+print_trees=dmu_scripts.rfile.print_trees:main
+transform_text=dmu_scripts.text.transform_text:main
+

dmu/generic/hashing.py

@@ -2,6 +2,7 @@
 Module with functions needed to provide hashes
 '''
 
+import os
 import json
 import hashlib
 from typing import Any
@@ -12,12 +13,10 @@ from dmu.logging.log_store import LogStore
 log=LogStore.add_logger('dmu:generic.hashing')
 # ------------------------------------
 def _object_to_string(obj : Any) -> str:
-    try:
-        string = json.dumps(obj)
-    except Exception as exc:
-        raise ValueError(f'Cannot hash object: {obj}') from exc
+    def default_encoder(x):
+        raise TypeError(f"Unserializable type: {type(x)}")
 
-    return string
+    return json.dumps(obj, sort_keys=True, default=default_encoder)
 # ------------------------------------
 def _dataframe_to_hash(df : pnd.DataFrame) -> str:
     sr_hash = pnd.util.hash_pandas_object(df, index=True)
@@ -29,16 +28,43 @@ def _dataframe_to_hash(df : pnd.DataFrame) -> str:
 # ------------------------------------
 def hash_object(obj : Any) -> str:
     '''
-    Function taking a python object and returning 
+    Function taking a python object and returning
     a string representing the hash
     '''
 
     if isinstance(obj, pnd.DataFrame):
-        return _dataframe_to_hash(df=obj)
+        value = _dataframe_to_hash(df=obj)
+        value = value[:10]
+
+        return value
 
     string     = _object_to_string(obj=obj)
     string_bin = string.encode('utf-8')
     hsh        = hashlib.sha256(string_bin)
+    value      = hsh.hexdigest()
+    value      = value[:10]
+
+    return value
+# ------------------------------------
+def hash_file(path : str) -> str:
+    '''
+    Parameters
+    ----------------
+    path: Path to file whose content has to be hashed
+
+    Returns
+    ----------------
+    A string representing the hash
+    '''
+    if not os.path.isfile(path):
+        raise FileNotFoundError(f'Cannot find: {path}')
+
+    h = hashlib.sha256()
+    with open(path, 'rb') as f:
+        for chunk in iter(lambda: f.read(8192), b''):
+            h.update(chunk)
+
+    value = h.hexdigest()
 
-    return hsh.hexdigest()
+    return value[:10]
 # ------------------------------------

dmu/generic/utilities.py

@@ -4,17 +4,69 @@ Module containing generic utility functions
 import os
 import time
 import json
+import pickle
 import inspect
-
-from typing import Callable
-
+from importlib.resources   import files
+from typing                import Callable, Any
 from functools             import wraps
+from contextlib            import contextmanager
+
+import yaml
+from omegaconf             import OmegaConf, DictConfig
+from dmu.generic           import hashing
+from dmu.generic           import utilities as gut
 from dmu.logging.log_store import LogStore
 
 TIMER_ON=False
 
 log = LogStore.add_logger('dmu:generic:utilities')
+# --------------------------------
+class BlockStyleDumper(yaml.SafeDumper):
+    '''
+    Class needed to specify proper indentation when
+    dumping data to YAML files
+    '''
+    def increase_indent(self, flow=False, indentless=False):
+        return super().increase_indent(flow=flow, indentless=False)
+# ---------------------------------
+def load_data(package : str, fpath : str) -> Any:
+    '''
+    This function will load a YAML or JSON file from a data package
+
+    Parameters
+    ---------------------
+    package: Data package, e.g. `dmu_data`
+    path   : Path to YAML/JSON file, relative to the data package
+
+    Returns
+    ---------------------
+    Dictionary or whatever structure the file is holding
+    '''
+
+    cpath = files(package).joinpath(fpath)
+    cpath = str(cpath)
+    data  = load_json(cpath)
+
+    return data
+# --------------------------------
+def load_conf(package : str, fpath : str) -> DictConfig:
+    '''
+    This function will load a YAML or JSON file from a data package
+
+    Parameters
+    ---------------------
+    package: Data package, e.g. `dmu_data`
+    path   : Path to YAML/JSON file, relative to the data package
+
+    Returns
+    ---------------------
+    DictConfig class from the OmegaConf package
+    '''
+
+    cpath = files(package).joinpath(fpath)
+    cfg   = OmegaConf.load(cpath)
 
+    return cfg
 # --------------------------------
 def _get_module_name( fun : Callable) -> str:
     mod = inspect.getmodule(fun)
@@ -28,7 +80,7 @@ def timeit(f):
     Decorator used to time functions, it is turned off by default, can be turned on with:
 
     from dmu.generic.utilities import TIMER_ON
-    from dmu.generic.utilities import timeit 
+    from dmu.generic.utilities import timeit
 
     TIMER_ON=True
 
@@ -54,29 +106,130 @@ def timeit(f):
 # --------------------------------
 def dump_json(data, path : str, sort_keys : bool = False) -> None:
     '''
-    Saves data as JSON
+    Saves data as JSON or YAML, depending on the extension, supported .json, .yaml, .yml
 
     Parameters
     data     : dictionary, list, etc
-    path     : Path to JSON file where to save it
-    sort_keys: Will set sort_keys argument of json.dump function 
+    path     : Path to output file where to save it
+    sort_keys: Will set sort_keys argument of json.dump function
     '''
     dir_name = os.path.dirname(path)
     os.makedirs(dir_name, exist_ok=True)
 
     with open(path, 'w', encoding='utf-8') as ofile:
-        json.dump(data, ofile, indent=4, sort_keys=sort_keys)
+        if path.endswith('.json'):
+            json.dump(data, ofile, indent=4, sort_keys=sort_keys)
+            return
+
+        if path.endswith('.yaml') or path.endswith('.yml'):
+            yaml.dump(data, ofile, Dumper=BlockStyleDumper, sort_keys=sort_keys)
+            return
+
+        raise NotImplementedError(f'Cannot deduce format from extension in path: {path}')
 # --------------------------------
 def load_json(path : str):
     '''
-    Loads data from JSON
+    Loads data from JSON or YAML, depending on extension of files, supported .json, .yaml, .yml
 
     Parameters
-    path     : Path to JSON file where data is saved 
+    path     : Path to outut file where data is saved
     '''
 
     with open(path, encoding='utf-8') as ofile:
-        data = json.load(ofile)
+        if path.endswith('.json'):
+            data = json.load(ofile)
+            return data
+
+        if path.endswith('.yaml') or path.endswith('.yml'):
+            data = yaml.safe_load(ofile)
+            return data
+
+        raise NotImplementedError(f'Cannot deduce format from extension in path: {path}')
+# --------------------------------
+def dump_pickle(data, path : str) -> None:
+    '''
+    Saves data as pickle file
+
+    Parameters
+    data     : dictionary, list, etc
+    path     : Path to output file where to save it
+    '''
+    dir_name = os.path.dirname(path)
+    os.makedirs(dir_name, exist_ok=True)
+
+    with open(path, 'wb') as ofile:
+        pickle.dump(data, ofile)
+# --------------------------------
+def load_pickle(path : str) -> None:
+    '''
+    loads data file
+
+    Parameters
+    path     : Path to output file where to save it
+    '''
+    with open(path, 'rb') as ofile:
+        data = pickle.load(ofile)
 
     return data
 # --------------------------------
+@contextmanager
+def silent_import():
+    '''
+    In charge of suppressing messages
+    of imported modules
+    '''
+    saved_stdout_fd = os.dup(1)
+    saved_stderr_fd = os.dup(2)
+
+    with open(os.devnull, 'w', encoding='utf-8') as devnull:
+        os.dup2(devnull.fileno(), 1)
+        os.dup2(devnull.fileno(), 2)
+        try:
+            yield
+        finally:
+            os.dup2(saved_stdout_fd, 1)
+            os.dup2(saved_stderr_fd, 2)
+            os.close(saved_stdout_fd)
+            os.close(saved_stderr_fd)
+# --------------------------------
+# Caching
+# --------------------------------
+def cache_data(obj : Any, hash_obj : Any) -> None:
+    '''
+    Will save data to a text file using a name from a hash
+
+    Parameters
+    -----------
+    obj      : Object that can be saved to a text file, e.g. list, number, dictionary
+    hash_obj : Object that can be used to get hash e.g. immutable
+    '''
+    try:
+        json.dumps(obj)
+    except Exception as exc:
+        raise ValueError('Object is not JSON serializable') from exc
+
+    val  = hashing.hash_object(hash_obj)
+    path = f'/tmp/dmu/cache/{val}.json'
+    gut.dump_json(obj, path)
+# --------------------------------
+def load_cached(hash_obj : Any, on_fail : Any = None) -> Any:
+    '''
+    Loads data corresponding to hash from hash_obj
+
+    Parameters
+    ---------------
+    hash_obj: Object used to calculate hash, which is in the file name
+    on_fail : Value returned if no data was found.
+              By default None, and it will just raise a FileNotFoundError
+    '''
+    val  = hashing.hash_object(hash_obj)
+    path = f'/tmp/dmu/cache/{val}.json'
+    if os.path.isfile(path):
+        data = gut.load_json(path)
+        return data
+
+    if on_fail is not None:
+        return on_fail
+
+    raise FileNotFoundError(f'Cannot find cached data at: {path}')
+# --------------------------------

dmu/logging/log_store.py

@@ -1,10 +1,11 @@
 '''
 Module holding LogStore 
 '''
-
 import logging
-from logging import Logger
+import contextlib
+from typing import Union
 
+from logging import Logger
 import logzero
 
 #------------------------------------------------------------
@@ -40,6 +41,36 @@ class LogStore:
     backend       = 'logging'
     #--------------------------
     @staticmethod
+    @contextlib.contextmanager
+    def level(name : str, lvl : int) -> None:
+        '''
+        Context manager used to set the logging level of a given logger
+
+        Parameters
+        ------------------
+        name : Name of logger
+        lvl  : Integer representing logging level
+        '''
+        log       = LogStore.get_logger(name=name)
+        if log is None:
+            raise ValueError(f'Cannot find logger {name}')
+
+        old_lvl = log.getEffectiveLevel()
+
+        LogStore.set_level(name, lvl)
+        try:
+            yield
+        finally:
+            LogStore.set_level(name, old_lvl)
+    #--------------------------
+    @staticmethod
+    def get_logger(name : str) -> Union[Logger,None]:
+        '''
+        Returns logger for a given name or None, if no logger found for that name
+        '''
+        return LogStore.d_logger.get(name)
+    #--------------------------
+    @staticmethod
     def add_logger(name : str, exists_ok : bool = False) -> Logger:
         '''
         Will use underlying logging library logzero/logging, etc to make logger
@@ -78,6 +109,7 @@ class LogStore:
     @staticmethod
     def _get_logging_logger(name : str, level : int) -> Logger:
         logger = logging.getLogger(name=name)
+        logger.propagate = False
 
         logger.setLevel(level)
 

dmu/logging/messages.py

@@ -0,0 +1,96 @@
+'''
+Module containing code meant to deal with logging of
+third party tools
+'''
+import os
+import sys
+import time
+import threading
+from io                    import StringIO
+from contextlib            import contextmanager
+from dmu.logging.log_store import LogStore
+
+log = LogStore.add_logger('dmu:logging:messages')
+# --------------------------------
+class FilteredStderr:
+    '''
+    This class is meant to be used to filter the messages
+    in the error stream by substrings
+    '''
+    # --------------------------------
+    def __init__(
+            self,
+            banned_substrings : list[str],
+            capture_stream    : StringIO):
+        '''
+        Parameters
+        -------------
+        banned_substrings : List of substrings that, if found in error message, will drop error
+        capture_stream    : Used to store error stream filtered messages, expected to be sys.__stderr__
+        '''
+        self._banned         = banned_substrings
+        self._capture_stream = capture_stream
+    # --------------------------------
+    def write(self, message : str):
+        '''
+        Should allow filtering error messages
+        '''
+        if not any(bad in message for bad in self._banned):
+            # This will make it to the error messages
+            self._capture_stream.write(message)
+    # --------------------------------
+    def flush(self):
+        '''
+        Should override the error stream's flush method
+        '''
+        self._capture_stream.flush()
+# --------------------------------
+@contextmanager
+def filter_stderr(
+        banned_substrings : list[str],
+        capture_stream    : StringIO|None=None):
+    '''
+    This contextmanager will suppress error messages
+
+    Parameters
+    -----------------
+    banned_substrings : List of substrings that need to be found in error messages
+                        in order for them to be suppressed
+    capture_stream    : Buffer needed to run tests, not needed for normal use
+    '''
+    if capture_stream is None:
+        capture_stream = sys.__stderr__
+
+    read_fd, write_fd = os.pipe()
+    saved_fd          = os.dup(2)
+
+    os.dup2(write_fd, 2)
+    os.close(write_fd)
+
+    filtered        = FilteredStderr(banned_substrings, capture_stream)
+    reader_finished = threading.Event()
+
+    def reader():
+        try:
+            with os.fdopen(read_fd, 'r', buffering=1) as pipe:
+                while True:
+                    line = pipe.readline()
+                    if not line:
+                        break
+                    filtered.write(line)
+                filtered.flush()
+        finally:
+            reader_finished.set()
+
+    thread = threading.Thread(target=reader, daemon=True)
+    thread.start()
+
+    try:
+        yield
+    finally:
+        os.dup2(saved_fd, 2)
+        os.close(saved_fd)
+
+        time.sleep(0.1)
+        reader_finished.wait(timeout=1.0)
+# --------------------------------

dmu/ml/cv_classifier.py

@@ -37,17 +37,17 @@ class CVClassifier(GradientBoostingClassifier):
 
         self._s_hash    = set()
         self._data      = {}
-        self._l_ft_name = None
+        self._l_ft_name : list[str]
     # ----------------------------------
     @property
-    def features(self):
+    def features(self) -> list[str]:
         '''
         Returns list of feature names used in training dataset
         '''
         return self._l_ft_name
     # ----------------------------------
     @property
-    def hashes(self):
+    def hashes(self) -> set[str]:
         '''
         Will return set with hashes of training data
         '''

dmu/ml/cv_diagnostics.py

@@ -186,6 +186,9 @@ class CVDiagnostics:
         raise NotImplementedError(f'Correlation coefficient {method} not implemented')
     # -------------------------
     def _plot_cutflow(self) -> None:
+        '''
+        Plot the 'mass' column for different values of working point
+        '''
         if 'overlay' not in self._cfg['correlations']['target']:
             log.debug('Not plotting cutflow of target distribution')
             return

dmu/ml/cv_performance.py

@@ -0,0 +1,58 @@
+'''
+This module contains the class CVPerformance
+'''
+# pylint: disable=too-many-positional-arguments, too-many-arguments
+
+from ROOT                  import RDataFrame
+from dmu.ml.cv_classifier  import CVClassifier
+from dmu.ml.cv_predict     import CVPredict
+from dmu.ml.train_mva      import TrainMva
+from dmu.logging.log_store import LogStore
+
+log=LogStore.add_logger('dmu:ml:cv_performance')
+# -----------------------------------------------------
+class CVPerformance:
+    '''
+    This class is meant to:
+
+    - Compare the classifier performance, through the ROC curve, of a model, for a given background and signal sample
+    '''
+    # ---------------------------
+    def plot_roc(
+            self,
+            name  : str,
+            color : str,
+            sig   : RDataFrame,
+            bkg   : RDataFrame,
+            model : list[CVClassifier] ) -> float:
+        '''
+        Method in charge of picking up model and data and plotting ROC curve
+
+        Parameters
+        --------------------------
+        name : Label of combination, used for plots
+        sig  : ROOT dataframe storing signal samples
+        bkg  : ROOT dataframe storing background samples
+        model: List of instances of the CVClassifier
+
+        Returns
+        --------------------------
+        Area under the ROC curve
+        '''
+        log.info(f'Loading {name}')
+
+        cvp_sig = CVPredict(models=model, rdf=sig)
+        arr_sig = cvp_sig.predict()
+
+        cvp_bkg = CVPredict(models=model, rdf=bkg)
+        arr_bkg = cvp_bkg.predict()
+
+        _, _, auc = TrainMva.plot_roc_from_prob(
+                arr_sig_prb=arr_sig,
+                arr_bkg_prb=arr_bkg,
+                kind       =   name,
+                color      =  color, # This should allow the function to pick kind
+                ifold      =    999) # for the label
+
+        return auc
+# -----------------------------------------------------