data-manipulation-utilities 0.2.6__py3-none-any.whl → 0.2.8.dev714__py3-none-any.whl

dmu/workflow/cache.py

@@ -0,0 +1,266 @@
+'''
+This module contains
+'''
+import os
+import sys
+import shutil
+from types      import NoneType
+from pathlib    import Path
+from contextlib import contextmanager
+
+from dmu.generic           import hashing
+from dmu.logging.log_store import LogStore
+
+log=LogStore.add_logger('dmu:workflow:cache')
+# ---------------------------
+class Cache:
+    '''
+    Class meant to wrap other classes in order to
+
+    - Keep track of the inputs through hashes
+    - Load cached data, if found, and prevent calculations
+
+    The following directories will be important:
+
+    out_dir  : Directory where the outputs will go, specified by the user
+    cache_dir: Subdirectory of out_dir, ${out_dir}/.cache
+    hash_dir : Subdirectory of out_dir, ${out_dir}/.cache/{hash}
+               Where {hash} is a 10 alphanumeric representing the has of the inputs
+
+    # On skipping caching
+
+    This is controlled by `_l_skip_class` which is a list of class names:
+
+    - These classes will have the caching turned off
+    - If the list is empty, caching runs for everything
+    - If the list is None, caching is turned off for everything
+    '''
+    _cache_root     : str|None = None
+    _l_skip_class   : list[str]|None = []
+    # ---------------------------
+    def __init__(self, out_path : str, **kwargs):
+        '''
+        Parameters
+        ---------------
+        out_path: Path to directory where outputs will go
+        kwargs  : Key word arguments symbolizing identity of inputs, used for hashing
+        '''
+        if Cache._cache_root is None:
+            raise ValueError('Caching directory not set')
+
+        log.debug(f'Using {Cache._cache_root} root directory for caching')
+        if 'code' in kwargs:
+            raise ValueError('Cannot append hashing data with key "code", already used')
+
+        kwargs['code']  = self._get_code_hash()
+
+        self._out_path  = os.path.normpath(f'{Cache._cache_root}/{out_path}')
+        log.debug(f'Using {self._out_path} output path')
+        os.makedirs(self._out_path, exist_ok=True)
+
+        self._dat_hash  = kwargs
+
+        self._cache_dir = self._get_dir(kind='cache')
+        self._hash_dir  : str
+    # ---------------------------
+    @classmethod
+    def set_cache_root(cls, root : str) -> None:
+        '''
+        Sets the path to the directory WRT which the _out_path_
+        will be placed
+        '''
+        if cls._cache_root is not None:
+            raise ValueError(f'Trying to set {root}, but already found {cls._cache_root}')
+
+        os.makedirs(root, exist_ok=True)
+
+        cls._cache_root = root
+    # ---------------------------
+    def _get_code_hash(self) -> str:
+        '''
+        If `MyTool` inherits from `Cache`. `mytool.py` git commit hash
+        should be returned
+        '''
+        cls   = self.__class__
+        mod   = sys.modules.get(cls.__module__)
+        if mod is None:
+            raise ValueError(f'Module not found: {cls.__module__}')
+
+        if mod.__file__ is None:
+            raise ValueError(f'Cannot extract file path for module: {cls.__module__}')
+
+        fname = mod.__file__
+        fpath = os.path.abspath(fname)
+        val   = hashing.hash_file(path=fpath)
+
+        log.debug(f'Using hash for: {fpath} = {val}')
+
+        return val
+    # ---------------------------
+    def _get_dir(
+            self,
+            kind : str,
+            make : bool = True) -> str:
+        '''
+        Parameters
+        --------------
+        kind : Kind of directory, cash, hash
+        make : If True (default) will try to make directory
+        '''
+        if   kind == 'cache':
+            dir_path  = f'{self._out_path}/.cache'
+        elif kind == 'hash':
+            cache_dir = self._get_dir(kind='cache')
+            hsh       = hashing.hash_object(self._dat_hash)
+            dir_path  = f'{cache_dir}/{hsh}'
+        else:
+            raise ValueError(f'Invalid directory kind: {kind}')
+
+        if make:
+            os.makedirs(dir_path, exist_ok=True)
+
+        return dir_path
+    # ---------------------------
+    def _cache(self) -> None:
+        '''
+        Meant to be called after all the calculations finish
+        It will copy all the outputs of the processing
+        to a hashed directory
+        '''
+        self._hash_dir  = self._get_dir(kind= 'hash')
+        log.info(f'Caching outputs to: {self._hash_dir}')
+
+        for source in Path(self._out_path).glob('*'):
+            if str(source) == self._cache_dir:
+                continue
+
+            log.debug(str(source))
+            log.debug('-->')
+            log.debug(self._hash_dir)
+            log.debug('')
+
+            if source.is_dir():
+                shutil.copytree(source, self._hash_dir+'/'+source.name, dirs_exist_ok=True)
+            else:
+                shutil.copy2(source, self._hash_dir)
+
+        self._delete_from_output(only_links=False)
+        self._copy_from_hashdir()
+    # ---------------------------
+    def _delete_from_output(self, only_links : bool) -> None:
+        '''
+        Delete all objects from _out_path directory, except for `.cache`
+
+        only_links: If true will only delete links
+        '''
+        for path in Path(self._out_path).iterdir():
+            if str(path) == self._cache_dir:
+                log.debug(f'Skipping cache dir: {self._cache_dir}')
+                continue
+
+            # These will always be symbolic links
+            if only_links and not path.is_symlink():
+                log.warning(f'Found a non-symlink not deleting: {path}')
+                continue
+
+            log.debug(f'Deleting {path}')
+            if path.is_dir() and not path.is_symlink():
+                shutil.rmtree(path)
+            else:
+                path.unlink()
+    # ---------------------------
+    def _copy_from_hashdir(self) -> None:
+        '''
+        Copies all the objects from _hash_dir to _out_path
+        '''
+        for source in Path(self._hash_dir).iterdir():
+            target = f'{self._out_path}/{source.name}'
+            log.debug(f'{str(source):<50}{"-->"}{target}')
+
+            os.symlink(source, target)
+    # ---------------------------
+    def _dont_cache(self) -> bool:
+        '''
+        Returns
+        ---------------
+        Flag that if:
+
+        True : Will stop the derived class from using caching (i.e. caching is off)
+        False: Cache
+        '''
+        if Cache._l_skip_class is None:
+            log.info('No class will be cached')
+            return True
+
+        if len(Cache._l_skip_class) == 0:
+            log.debug('All classes will be cached')
+            return False
+
+        class_name = self.__class__.__name__
+
+        skip = class_name in Cache._l_skip_class
+
+        if skip:
+            log.warning(f'Caching turned off for {class_name}')
+        else:
+            log.debug(f'Caching turned on  for {class_name}')
+
+        return skip
+    # ---------------------------
+    def _copy_from_cache(self) -> bool:
+        '''
+        Checks if hash directory exists:
+
+        No : Returns False
+        Yes:
+            - Removes contents of `out_path`, except for .cache
+            - Copies the contents of `hash_dir` to `out_dir`
+
+        Returns
+        ---------------
+        True if the object, cached was found, false otherwise.
+        '''
+        if self._dont_cache():
+            # If not copying from cache, will need to remove what is
+            # in the output directory, so that it gets replaced with
+            # new outputs
+            self._delete_from_output(only_links=False)
+            log.info('Not picking already cached outputs, remaking them')
+            return False
+
+        hash_dir = self._get_dir(kind='hash', make=False)
+        if not os.path.isdir(hash_dir):
+            log.debug(f'Hash directory {hash_dir} not found, not caching')
+            self._delete_from_output(only_links=False)
+            return False
+
+        self._hash_dir = hash_dir
+        log.debug(f'Data found in hash directory: {self._hash_dir}')
+
+        self._delete_from_output(only_links=False)
+        self._copy_from_hashdir()
+
+        return True
+    # ---------------------------
+    @contextmanager
+    @staticmethod
+    def turn_off_cache(val : list[str]|None):
+        '''
+        Parameters
+        ------------------
+        val: List of names of classes that inherit from `Cache`.
+        If None, will not cache for any class.
+        By default this is an empty list and it will cache for every class
+        '''
+        if not isinstance(val, (NoneType, list)):
+            log.error('This manager expects: list[str]|None')
+            raise ValueError(f'Invalid value: {val}')
+
+        old_val = Cache._l_skip_class
+
+        Cache._l_skip_class = val
+        try:
+            yield
+        finally:
+            Cache._l_skip_class = old_val
+# ---------------------------

dmu_data/ml/tests/diagnostics_from_file.yaml

@@ -0,0 +1,13 @@
+output         : /tmp/tests/dmu/ml/cv_diagnostics/from_rdf
+  # Will assume that the target is already in the input dataframe
+  # and will use it, instead of evaluating models
+score_from_rdf : w 
+correlations:
+  # Variables with respect to which the correlations with the features will be measured
+  target :
+    name : z
+  methods:
+    - Pearson
+  figure:
+    title: Scores from file 
+    size : [10, 8]

dmu_data/ml/tests/diagnostics_from_model.yaml

@@ -0,0 +1,10 @@
+output      : /tmp/tests/dmu/ml/cv_diagnostics/from_model
+correlations:
+  # Variables with respect to which the correlations with the features will be measured
+  target    : 
+    name    : z
+  methods:
+    - Pearson
+  figure:
+    size  : [10, 8]
+    rotate: 90

dmu_data/ml/tests/diagnostics_multiple_methods.yaml

@@ -0,0 +1,10 @@
+output : /tmp/tests/dmu/ml/cv_diagnostics/multiple_methods
+correlations:
+  # Variables with respect to which the correlations with the features will be measured
+  target    : 
+    name : z
+  methods:
+    - Pearson
+    - Kendall-$\tau$ 
+  figure:
+    size : [10, 8]

dmu_data/ml/tests/diagnostics_overlay.yaml

@@ -0,0 +1,33 @@
+output         : /tmp/tests/dmu/ml/cv_diagnostics/overlay
+  # Will assume that the target is already in the input dataframe
+  # and will use it, instead of evaluating models
+score_from_rdf : w
+correlations:
+  # Variables with respect to which the correlations with the features will be measured
+  target :
+    name : z
+    overlay :
+      wp :
+        - 0.2
+        - 0.5
+        - 0.7
+        - 0.9
+      general:
+        size : [12, 10]
+      saving:
+        plt_dir : /tmp/tests/dmu/ml/cv_diagnostics/overlay
+      plots:
+        z :
+          binning    : [-4, 4, 10]
+          yscale     : 'linear'
+          labels     : ['$z$', 'Entries']
+          normalized : true
+          styling :          
+            linestyle: '-'
+  methods:
+    - Pearson
+    - Kendall-$\tau$ 
+  figure:
+    title     : Scores from file
+    size      : [12, 10]
+    xlabelsize: 30 

dmu_data/ml/tests/train_mva.yaml

@@ -1,19 +1,26 @@
 dataset:
+  define :
+    r : z + x
   nan :
     x : -3
-    y : -3 
+    y : -3
 training :
     nfold    : 3
-    features : [x, y, z]
+    features :
+      - x
+      - y
+      - r
     rdm_stat : 1
     hyper    :
       loss              : log_loss
+      max_features      : sqrt 
       n_estimators      : 100
-      max_depth         : 3
-      learning_rate     : 0.1
+      max_depth         : 5
       min_samples_split : 2
+      subsample         : 0.8
+      learning_rate     : 0.1
 saving:
-    path : '/tmp/dmu/ml/tests/train_mva/model.pkl'
+    output : /tmp/tests/dmu/ml/train_mva
 plotting:
     roc     :
         min : [0.0, 0.0]
@@ -29,24 +36,25 @@ plotting:
       title      : 'Correlation matrix'
       size       : [10, 10]
       mask_value : 0
-    val_dir : '/tmp/dmu/ml/tests/train_mva'
     features:
-        saving:
-            plt_dir : '/tmp/dmu/ml/tests/train_mva/features'
         plots:
+          r :
+            binning : [-6, 6, 100]
+            yscale  : 'linear'
+            labels  : ['$r$', '']
           w :
             binning : [-4, 4, 100]
             yscale  : 'linear'
-            labels  : ['w', '']
+            labels  : ['$w$', '']
           x :
             binning : [-4, 4, 100]
             yscale  : 'linear'
-            labels  : ['x', '']
+            labels  : ['$x$', '']
           y :
             binning : [-4, 4, 100]
             yscale  : 'linear'
-            labels  : ['y', '']
+            labels  : ['$y$', '']
           z :
             binning : [-4, 4, 100]
             yscale  : 'linear'
-            labels  : ['z', '']
+            labels  : ['$z$', '']

dmu_data/ml/tests/train_mva_def.yaml

@@ -0,0 +1,75 @@
+# This config file is used for testing training and evaluation
+# when there is a variable that is defined in different ways
+# for the `sig` and `bkg` samples
+
+dataset:
+  samples:
+    sig :
+      definitions:
+        n : x + y
+    bkg :
+      definitions:
+        n : x - y
+  define :
+    r : z + x
+  nan :
+    n : -3
+    y : -3
+training :
+    nfold    : 3
+    features :
+      - n
+      - y
+      - r
+    rdm_stat : 1
+    hyper    :
+      loss              : log_loss
+      max_features      : sqrt
+      n_estimators      : 100
+      max_depth         : 5
+      min_samples_split : 2
+      subsample         : 0.8
+      learning_rate     : 0.1
+saving:
+    output : /tmp/tests/dmu/ml/train_mva
+plotting:
+    roc     :
+        min : [0.0, 0.0]
+        max : [1.2, 1.2]
+        annotate:
+          sig_eff : [0.5, 0.6, 0.7, 0.8, 0.9]
+          form : '{:.2f}'
+          color: 'green'
+          xoff : -15
+          yoff : -15
+          size :  10
+    correlation:
+      title      : 'Correlation matrix'
+      size       : [10, 10]
+      mask_value : 0
+    features:
+        plots:
+          r :
+            binning : [-6, 6, 100]
+            yscale  : 'linear'
+            labels  : ['$r$', '']
+          n :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$n$', '']
+          w :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$w$', '']
+          x :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$x$', '']
+          y :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$y$', '']
+          z :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$z$', '']

dmu_data/ml/tests/train_mva_with_diagnostics.yaml

@@ -0,0 +1,87 @@
+dataset:
+  define :
+    r : z + x
+  nan :
+    x : -3
+    y : -3
+training :
+    nfold    : 3
+    features : [x, y, r]
+    rdm_stat : 1
+    hyper    :
+      loss              : log_loss
+      n_estimators      : 100
+      max_depth         : 3
+      learning_rate     : 0.1
+      min_samples_split : 2
+saving:
+    output : /tmp/tests/dmu/ml/train_mva
+plotting:
+    roc     :
+        min : [0.0, 0.0]
+        max : [1.2, 1.2]
+        annotate:
+          sig_eff : [0.5, 0.6, 0.7, 0.8, 0.9]
+          form : '{:.2f}'
+          color: 'green'
+          xoff : -15
+          yoff : -15
+          size :  10
+    correlation:
+      title      : 'Correlation matrix'
+      size       : [10, 10]
+      mask_value : 0
+    val_dir : '/tmp/tests/dmu/ml/train_mva'
+    features:
+        saving:
+            plt_dir : '/tmp/tests/dmu/ml/train_mva/features'
+        plots:
+          r :
+            binning : [-6, 6, 100]
+            yscale  : 'linear'
+            labels  : ['$r$', '']
+          w :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$w$', '']
+          x :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$x$', '']
+          y :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$y$', '']
+          z :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$z$', '']
+diagnostics:
+  output         : /tmp/tests/dmu/ml/train_mva/diagnostics
+  correlations:
+    target :
+      name : z
+      overlay :
+        wp :
+          - 0.2
+          - 0.6
+          - 0.8
+          - 0.9
+        general:
+          size : [20, 10]
+        saving:
+          plt_dir : /tmp/tests/dmu/ml/train_mva/diagnostics
+        plots:
+          z :
+            binning    : [-4, +4, 30]
+            yscale     : 'linear'
+            labels     : ['z', 'Entries']
+            normalized : true
+            styling :
+              linestyle: '-'
+    methods:
+      - Pearson
+      - Kendall-$\tau$
+    figure:
+      title: Training diagnostics
+      size : [10, 8]

dmu_data/ml/tests/train_mva_with_preffix.yaml

@@ -0,0 +1,58 @@
+dataset:
+  define :
+    r : z + preffix.x.suffix
+  nan :
+    preffix.x.suffix : -3
+    y : -3
+training :
+    nfold    : 2
+    features :
+      - preffix.x.suffix
+      - y
+      - r
+    rdm_stat : 1
+    hyper    :
+      loss              : log_loss
+      n_estimators      : 100
+      max_depth         : 3
+      learning_rate     : 0.1
+      min_samples_split : 2
+saving:
+    output : /tmp/tests/dmu/ml/train_mva
+plotting:
+    roc     :
+        min : [0.0, 0.0]
+        max : [1.2, 1.2]
+        annotate:
+          sig_eff : [0.5, 0.6, 0.7, 0.8, 0.9]
+          form : '{:.2f}'
+          color: 'green'
+          xoff : -15
+          yoff : -15
+          size :  10
+    correlation:
+      title      : 'Correlation matrix'
+      size       : [10, 10]
+      mask_value : 0
+    features:
+        plots:
+          r :
+            binning : [-6, 6, 100]
+            yscale  : 'linear'
+            labels  : ['$r$', '']
+          w :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$w$', '']
+          preffix.x.suffix :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$x$', '']
+          y :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$y$', '']
+          z :
+            binning : [-4, 4, 100]
+            yscale  : 'linear'
+            labels  : ['$z$', '']

dmu_data/plotting/tests/2d.yaml

@@ -1,5 +1,5 @@
 saving:
-    plt_dir : /tmp/dmu/tests/plotting/2d_weighted
+    plt_dir : /tmp/tests/dmu/plotting
 selection:
   cuts:
     xlow : x > -1.5
@@ -8,10 +8,10 @@ definitions:
 general:
     size : [20, 10]
 plots_2d:
-    - [x, y, weights, 'xy_wgt', false]
-    - [x, y,    null, 'xy_raw', false]
-    - [x, z,    null, 'xz_raw', false]
-    - [x, z,    null, 'xz_log',  true]
+    - [x, y, weights, 'xy_wgt_lin', false]
+    - [x, z, weights, 'xz_wgt_log',  true]
+    - [x, y,    null, 'xy_raw_lin', false]
+    - [x, z,    null, 'xz_raw_log',  true]
 axes:
     x :
         binning : [-3.0, 3.0, 40]

dmu_data/plotting/tests/line.yaml

@@ -0,0 +1,15 @@
+saving:
+    plt_dir : tests/plotting/line
+plots:
+    x :
+      binning : [-5.0, 8.0, 40]
+      title   : x distribution
+      vline   :
+        x     : 0
+        label : label
+        ls    : --
+        c     : blue
+        lw    : 1
+    y :
+      binning : [-5.0, 8.0, 40]
+      title   : y distribution

dmu_data/plotting/tests/plug_fwhm.yaml

@@ -0,0 +1,24 @@
+saving:
+    plt_dir : plotting/pluggins/fwhm
+plots:
+    x : 
+      binning : [-5.0, 8.0, 40]
+      title   : x distribution
+    y : 
+      binning : [-5.0, 8.0, 40]
+      title   : y distribution
+plugin:
+  fwhm:
+    x :
+      plot   : true
+      obs    : [-2, 4]
+      plot   : true
+      format : FWHM={:.3f}
+      add_std: True
+    y :
+      plot   : true
+      obs    : [-4, 8]
+      plot   : true
+      format : FWHM={:.3f}
+      add_std: True
+