data-manipulation-utilities 0.2.7__py3-none-any.whl → 0.2.8.dev720__py3-none-any.whl

dmu/workflow/cache.py

@@ -0,0 +1,266 @@
+'''
+This module contains
+'''
+import os
+import sys
+import shutil
+from types      import NoneType
+from pathlib    import Path
+from contextlib import contextmanager
+
+from dmu.generic           import hashing
+from dmu.logging.log_store import LogStore
+
+log=LogStore.add_logger('dmu:workflow:cache')
+# ---------------------------
+class Cache:
+    '''
+    Class meant to wrap other classes in order to
+
+    - Keep track of the inputs through hashes
+    - Load cached data, if found, and prevent calculations
+
+    The following directories will be important:
+
+    out_dir  : Directory where the outputs will go, specified by the user
+    cache_dir: Subdirectory of out_dir, ${out_dir}/.cache
+    hash_dir : Subdirectory of out_dir, ${out_dir}/.cache/{hash}
+               Where {hash} is a 10 alphanumeric representing the has of the inputs
+
+    # On skipping caching
+
+    This is controlled by `_l_skip_class` which is a list of class names:
+
+    - These classes will have the caching turned off
+    - If the list is empty, caching runs for everything
+    - If the list is None, caching is turned off for everything
+    '''
+    _cache_root     : str|None = None
+    _l_skip_class   : list[str]|None = []
+    # ---------------------------
+    def __init__(self, out_path : str, **kwargs):
+        '''
+        Parameters
+        ---------------
+        out_path: Path to directory where outputs will go
+        kwargs  : Key word arguments symbolizing identity of inputs, used for hashing
+        '''
+        if Cache._cache_root is None:
+            raise ValueError('Caching directory not set')
+
+        log.debug(f'Using {Cache._cache_root} root directory for caching')
+        if 'code' in kwargs:
+            raise ValueError('Cannot append hashing data with key "code", already used')
+
+        kwargs['code']  = self._get_code_hash()
+
+        self._out_path  = os.path.normpath(f'{Cache._cache_root}/{out_path}')
+        log.debug(f'Using {self._out_path} output path')
+        os.makedirs(self._out_path, exist_ok=True)
+
+        self._dat_hash  = kwargs
+
+        self._cache_dir = self._get_dir(kind='cache')
+        self._hash_dir  : str
+    # ---------------------------
+    @classmethod
+    def set_cache_root(cls, root : str) -> None:
+        '''
+        Sets the path to the directory WRT which the _out_path_
+        will be placed
+        '''
+        if cls._cache_root is not None:
+            raise ValueError(f'Trying to set {root}, but already found {cls._cache_root}')
+
+        os.makedirs(root, exist_ok=True)
+
+        cls._cache_root = root
+    # ---------------------------
+    def _get_code_hash(self) -> str:
+        '''
+        If `MyTool` inherits from `Cache`. `mytool.py` git commit hash
+        should be returned
+        '''
+        cls   = self.__class__
+        mod   = sys.modules.get(cls.__module__)
+        if mod is None:
+            raise ValueError(f'Module not found: {cls.__module__}')
+
+        if mod.__file__ is None:
+            raise ValueError(f'Cannot extract file path for module: {cls.__module__}')
+
+        fname = mod.__file__
+        fpath = os.path.abspath(fname)
+        val   = hashing.hash_file(path=fpath)
+
+        log.debug(f'Using hash for: {fpath} = {val}')
+
+        return val
+    # ---------------------------
+    def _get_dir(
+            self,
+            kind : str,
+            make : bool = True) -> str:
+        '''
+        Parameters
+        --------------
+        kind : Kind of directory, cash, hash
+        make : If True (default) will try to make directory
+        '''
+        if   kind == 'cache':
+            dir_path  = f'{self._out_path}/.cache'
+        elif kind == 'hash':
+            cache_dir = self._get_dir(kind='cache')
+            hsh       = hashing.hash_object(self._dat_hash)
+            dir_path  = f'{cache_dir}/{hsh}'
+        else:
+            raise ValueError(f'Invalid directory kind: {kind}')
+
+        if make:
+            os.makedirs(dir_path, exist_ok=True)
+
+        return dir_path
+    # ---------------------------
+    def _cache(self) -> None:
+        '''
+        Meant to be called after all the calculations finish
+        It will copy all the outputs of the processing
+        to a hashed directory
+        '''
+        self._hash_dir  = self._get_dir(kind= 'hash')
+        log.info(f'Caching outputs to: {self._hash_dir}')
+
+        for source in Path(self._out_path).glob('*'):
+            if str(source) == self._cache_dir:
+                continue
+
+            log.debug(str(source))
+            log.debug('-->')
+            log.debug(self._hash_dir)
+            log.debug('')
+
+            if source.is_dir():
+                shutil.copytree(source, self._hash_dir+'/'+source.name, dirs_exist_ok=True)
+            else:
+                shutil.copy2(source, self._hash_dir)
+
+        self._delete_from_output(only_links=False)
+        self._copy_from_hashdir()
+    # ---------------------------
+    def _delete_from_output(self, only_links : bool) -> None:
+        '''
+        Delete all objects from _out_path directory, except for `.cache`
+
+        only_links: If true will only delete links
+        '''
+        for path in Path(self._out_path).iterdir():
+            if str(path) == self._cache_dir:
+                log.debug(f'Skipping cache dir: {self._cache_dir}')
+                continue
+
+            # These will always be symbolic links
+            if only_links and not path.is_symlink():
+                log.warning(f'Found a non-symlink not deleting: {path}')
+                continue
+
+            log.debug(f'Deleting {path}')
+            if path.is_dir() and not path.is_symlink():
+                shutil.rmtree(path)
+            else:
+                path.unlink()
+    # ---------------------------
+    def _copy_from_hashdir(self) -> None:
+        '''
+        Copies all the objects from _hash_dir to _out_path
+        '''
+        for source in Path(self._hash_dir).iterdir():
+            target = f'{self._out_path}/{source.name}'
+            log.debug(f'{str(source):<50}{"-->"}{target}')
+
+            os.symlink(source, target)
+    # ---------------------------
+    def _dont_cache(self) -> bool:
+        '''
+        Returns
+        ---------------
+        Flag that if:
+
+        True : Will stop the derived class from using caching (i.e. caching is off)
+        False: Cache
+        '''
+        if Cache._l_skip_class is None:
+            log.info('No class will be cached')
+            return True
+
+        if len(Cache._l_skip_class) == 0:
+            log.debug('All classes will be cached')
+            return False
+
+        class_name = self.__class__.__name__
+
+        skip = class_name in Cache._l_skip_class
+
+        if skip:
+            log.warning(f'Caching turned off for {class_name}')
+        else:
+            log.debug(f'Caching turned on  for {class_name}')
+
+        return skip
+    # ---------------------------
+    def _copy_from_cache(self) -> bool:
+        '''
+        Checks if hash directory exists:
+
+        No : Returns False
+        Yes:
+            - Removes contents of `out_path`, except for .cache
+            - Copies the contents of `hash_dir` to `out_dir`
+
+        Returns
+        ---------------
+        True if the object, cached was found, false otherwise.
+        '''
+        if self._dont_cache():
+            # If not copying from cache, will need to remove what is
+            # in the output directory, so that it gets replaced with
+            # new outputs
+            self._delete_from_output(only_links=False)
+            log.info('Not picking already cached outputs, remaking them')
+            return False
+
+        hash_dir = self._get_dir(kind='hash', make=False)
+        if not os.path.isdir(hash_dir):
+            log.debug(f'Hash directory {hash_dir} not found, not caching')
+            self._delete_from_output(only_links=False)
+            return False
+
+        self._hash_dir = hash_dir
+        log.debug(f'Data found in hash directory: {self._hash_dir}')
+
+        self._delete_from_output(only_links=False)
+        self._copy_from_hashdir()
+
+        return True
+    # ---------------------------
+    @contextmanager
+    @staticmethod
+    def turn_off_cache(val : list[str]|None):
+        '''
+        Parameters
+        ------------------
+        val: List of names of classes that inherit from `Cache`.
+        If None, will not cache for any class.
+        By default this is an empty list and it will cache for every class
+        '''
+        if not isinstance(val, (NoneType, list)):
+            log.error('This manager expects: list[str]|None')
+            raise ValueError(f'Invalid value: {val}')
+
+        old_val = Cache._l_skip_class
+
+        Cache._l_skip_class = val
+        try:
+            yield
+        finally:
+            Cache._l_skip_class = old_val
+# ---------------------------

data_manipulation_utilities-0.2.7.data/scripts/publish

@@ -1,89 +0,0 @@
-#!/usr/bin/env bash
-
-# --------------------------
-display_help()
-{
-    echo "Script meant to:"
-    echo ""
-    echo "1. Check if version in pyproject.toml has been modified"
-    echo "2. If it has create new tag following version name"
-    echo "3. Push to remote "
-}
-# --------------------------
-get_opts()
-{
-    while getopts :hf: option; do 
-        case "${option}" in
-            h)  
-                display_help
-                exit 0
-                ;;  
-           \?)  echo "Invalid option: -${OPTARG}"
-                display_help
-                exit 1
-                ;;  
-            :)  echo "$0: Arguments needed"
-                display_help
-                exit 1
-                ;;  
-        esac
-    done
-}
-# --------------------------
-# Picks up version from pyproject.toml
-get_version()
-{
-    if [[ ! -f pyproject.toml ]];then
-        echo "Cannot find pyproject.toml"
-        exit 1
-    fi
-
-    VERSION_LINE=$(grep version pyproject.toml)
-
-    if [[ $? -ne 0 ]];then
-        ehco "Could not extract version from pyproject.toml"
-        exit 1
-    fi
-
-    if [[ "$VERSION_LINE" =~ .*([0-9]\.[0-9]\.[0-9]).* ]];then
-        VERSION=${BASH_REMATCH[1]}
-        echo "Using version: $VERSION"
-        return
-    fi
-
-    echo "Could not extract version from: $VERSION_LINE"
-    exit 1
-}
-# --------------------------
-create_tag()
-{
-    git tag -n | grep $VERSION
-
-    if [[ $? -eq 0 ]];then
-        echo "Version found among tags, not tagging"
-        return
-    fi
-
-    echo "Version $VERSION not found among tags, creating new tag"
-
-    git tag -a $VERSION
-}
-# --------------------------
-push_all()
-{
-    for REMOTE in $(git remote);do
-        echo "Pushing tags and commits to remote: $REMOTE"
-        git add pyproject.toml
-        git commit -m "Publication commit"
-
-        git pull    $REMOTE HEAD
-        git push -u $REMOTE HEAD
-        git push    $REMOTE --tags
-    done
-}
-# --------------------------
-get_opts "$@"
-
-get_version
-create_tag
-push_all

data_manipulation_utilities-0.2.7.dist-info/RECORD

@@ -1,69 +0,0 @@
-data_manipulation_utilities-0.2.7.data/scripts/publish,sha256=-3K_Y2_4CfWCV50rPB8CRuhjxDu7xMGswinRwPovgLs,1976
-dmu/arrays/utilities.py,sha256=PKoYyybPptA2aU-V3KLnJXBudWxTXu4x1uGdIMQ49HY,1722
-dmu/generic/hashing.py,sha256=Mxwr88jyGuEP5Bhqtw1gDva9fNO-l11jb2qA0ceOysw,1167
-dmu/generic/utilities.py,sha256=dBvrucgdM3vyyETharLvPAC7qEbgF9cL1mLVQ0nxNJM,2020
-dmu/generic/version_management.py,sha256=j0ImlAq6SVNjTh3xRsF6G7DSoyr1w8kTRY84dNriGRE,3750
-dmu/logging/log_store.py,sha256=umdvjNDuV3LdezbG26b0AiyTglbvkxST19CQu9QATbA,4184
-dmu/ml/cv_classifier.py,sha256=ZbzEm_jW9yoTC7k_xBA7hFpc1bDNayiVR3tbaj1_ieE,4228
-dmu/ml/cv_diagnostics.py,sha256=T6wkNcuLkdLrZ9z1YtAYMeb-M_bxnSChRHtdPeQv_To,7507
-dmu/ml/cv_predict.py,sha256=4wwYL_jcUExDqLJVfClxEUWSd_QAx8yKHO3rX-mx4vw,6711
-dmu/ml/train_mva.py,sha256=tkdSODahXG9R77wmcckKFxYceMmc-3O-Y4pJXMe8WAI,21137
-dmu/ml/utilities.py,sha256=PK_61fW7gBV9aGZyez3PI8zAT7_Fc6IlQzDB7f8iBTM,4133
-dmu/pdataframe/utilities.py,sha256=kwv9PuSmccz518e5bkrASacewCkMLTSPLtLiMH3olBc,2011
-dmu/plotting/fwhm.py,sha256=Vn0TWlxgNMNk_wb2CD_At-ltVmzI49fyhoJXa0CT0I4,1924
-dmu/plotting/matrix.py,sha256=pXuUJn-LgOvrI9qGkZQw16BzLjOjeikYQ_ll2VIcIXU,4978
-dmu/plotting/plotter.py,sha256=f3hoZF1n3G-yjHsQ_2huxqz5cqFelZGs0RiaHYjF0JM,7391
-dmu/plotting/plotter_1d.py,sha256=zygY6XUk4_wmaAK-bietjYdhkA03TZyft5IYX5acoYI,9142
-dmu/plotting/plotter_2d.py,sha256=mZhp3D5I-JodOnFTEF1NqHtcLtuI-2WNpCQsrsoXNtw,3017
-dmu/plotting/utilities.py,sha256=SI9dvtZq2gr-PXVz71KE4o0i09rZOKgqJKD1jzf6KXk,1167
-dmu/rdataframe/atr_mgr.py,sha256=FdhaQWVpsm4OOe1IRbm7rfrq8VenTNdORyI-lZ2Bs1M,2386
-dmu/rdataframe/utilities.py,sha256=pNcQARMP7txMhy6k27UnDcYf0buNy5U2fshaJDl_h8o,3661
-dmu/rfile/rfprinter.py,sha256=mp5jd-oCJAnuokbdmGyL9i6tK2lY72jEfROuBIZ_ums,3941
-dmu/rfile/utilities.py,sha256=XuYY7HuSBj46iSu3c60UYBHtI6KIPoJU_oofuhb-be0,945
-dmu/stats/fitter.py,sha256=gmSejQmoqsHmg6QPypbNb0MTfDFkSMy-DsDLKk1H3KE,18387
-dmu/stats/function.py,sha256=yzi_Fvp_ASsFzbWFivIf-comquy21WoeY7is6dgY0Go,9491
-dmu/stats/gof_calculator.py,sha256=4EN6OhULcztFvsAZ00rxgohJemnjtDNB5o0IBcv6kbk,4657
-dmu/stats/minimizers.py,sha256=db9R2G0SOV-k0BKi6m4EyB_yp6AtZdP23_28B0315oo,7094
-dmu/stats/model_factory.py,sha256=N0Q5R7GDfy2r_08xGduzydYO-ALvNTIRE1g0KKcRFyc,15872
-dmu/stats/utilities.py,sha256=LQy4kd3xSXqpApcWuYfZxkGQyjowaXv2Wr1c4Bj-4ys,4523
-dmu/stats/zfit_models.py,sha256=SI61KJ-OG1UAabDICU1iTh6JPKM3giR2ErDraRjkCV8,1842
-dmu/stats/zfit_plotter.py,sha256=bnkqFKyxhOEb5hVv4mxh4gpg-MrGeTrCh_5QKIT9hL0,19928
-dmu/testing/utilities.py,sha256=c91g_PEzIZfS3Kxj2EOh05mc5ia7vFpZM1VEhjh4DA0,4661
-dmu/text/transformer.py,sha256=4lrGknbAWRm0-rxbvgzOO-eR1-9bkYk61boJUEV3cQ0,6100
-dmu_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dmu_data/ml/tests/diagnostics_from_file.yaml,sha256=quvXOPkRducnBsctyape_Rn5_aqMEpPo6nO_UweMORo,404
-dmu_data/ml/tests/diagnostics_from_model.yaml,sha256=rtCQlmGS9ld2xoQJEE35nA07yfRMklEfQEW0w3gRv2A,261
-dmu_data/ml/tests/diagnostics_multiple_methods.yaml,sha256=w8Fpmr7kX1Jsb_h6LL2hiuYKf5lYpckFCpYKzWetbA0,265
-dmu_data/ml/tests/diagnostics_overlay.yaml,sha256=ZVOsxLL8_JQtf41n8Ct-M9Ch10xBwHK54q1fttWPDlE,866
-dmu_data/ml/tests/train_mva.yaml,sha256=AFUJV5yrXIdHHzA9TzfJv3RWhvk0GUB_CFubePCzfxE,1437
-dmu_data/ml/tests/train_mva_with_diagnostics.yaml,sha256=ZyKU7W_mDGVvg8xP_wQX0NtO6N_tLSnKO1CyHAaYhuk,2036
-dmu_data/plotting/tests/2d.yaml,sha256=HSAtER-8CEqIGBY_jdcIdSVOHMfYPYhmgeZghTpVYh8,516
-dmu_data/plotting/tests/fig_size.yaml,sha256=7ROq49nwZ1A2EbPiySmu6n3G-Jq6YAOkc3d2X3YNZv0,294
-dmu_data/plotting/tests/high_stat.yaml,sha256=bLglBLCZK6ft0xMhQ5OltxE76cWsBMPMjO6GG0OkDr8,522
-dmu_data/plotting/tests/legend.yaml,sha256=wGpj58ig-GOlqbWoN894zrCet2Fj9f5QtY0rig_UC-c,213
-dmu_data/plotting/tests/name.yaml,sha256=mkcPAVg8wBAmlSbSRQ1bcaMl4vOS6LXMtpqQeDrrtO4,312
-dmu_data/plotting/tests/no_bounds.yaml,sha256=8e1QdphBjz-suDr857DoeUC2DXiy6SE-gvkORJQYv80,257
-dmu_data/plotting/tests/normalized.yaml,sha256=Y0eKtyV5pvlSxvqfsLjytYtv8xYF3HZ5WEdCJdeHGQI,193
-dmu_data/plotting/tests/plug_fwhm.yaml,sha256=xl5LXc9Nt66anM-HOXAxCtlaxWNM7zzIXf1Y6U8M4Wg,449
-dmu_data/plotting/tests/plug_stats.yaml,sha256=ROO8soYXBbZIFYZcGngA_K5XHgIAFCmuAGfZCJgMmd0,384
-dmu_data/plotting/tests/simple.yaml,sha256=Xc59Pjfb3BKMicLVBxODVqomHFupcb5GvefKbKHCQWQ,195
-dmu_data/plotting/tests/stats.yaml,sha256=fSZjoV-xPnukpCH2OAXsz_SNPjI113qzDg8Ln3spaaA,165
-dmu_data/plotting/tests/styling.yaml,sha256=yUV75HewT2tFofS-5i-mU-3s4PSHcpZOZRn5R_iSRAw,218
-dmu_data/plotting/tests/title.yaml,sha256=bawKp9aGpeRrHzv69BOCbFX8sq9bb3Es9tdsPTE7jIk,333
-dmu_data/plotting/tests/weights.yaml,sha256=RWQ1KxbCq-uO62WJ2AoY4h5Umc37zG35s-TpKnNMABI,312
-dmu_data/text/transform.toml,sha256=R-832BZalzHZ6c5gD6jtT_Hj8BCsM5vxa1v6oeiwaP4,94
-dmu_data/text/transform.txt,sha256=EX760da6Vkf-_EPxnQlC5hGSkfFhJCCGCD19NU-1Qto,44
-dmu_data/text/transform_set.toml,sha256=Jeh7BTz82idqvbOQJtl9-ur56mZkzDn5WtvmIb48LoE,150
-dmu_data/text/transform_set.txt,sha256=1KivMoP9LxPn9955QrRmOzjEqduEjhTetQ9MXykO5LY,46
-dmu_data/text/transform_trf.txt,sha256=zxBRTgcSmX7RdqfmWF88W1YqbyNHa4Ccruf1MmnYv2A,74
-dmu_scripts/git/publish,sha256=-3K_Y2_4CfWCV50rPB8CRuhjxDu7xMGswinRwPovgLs,1976
-dmu_scripts/physics/check_truth.py,sha256=b1P_Pa9ef6VcFtyY6Y9KS9Om9L-QrCBjDKp4dqca0PQ,3964
-dmu_scripts/rfile/compare_root_files.py,sha256=T8lDnQxsRNMr37x1Y7YvWD8ySHrJOWZki7ZQynxXX9Q,9540
-dmu_scripts/rfile/print_trees.py,sha256=Ze4Ccl_iUldl4eVEDVnYBoe4amqBT1fSBR1zN5WSztk,941
-dmu_scripts/ssh/coned.py,sha256=lhilYNHWRCGxC-jtyJ3LQ4oUgWW33B2l1tYCcyHHsR0,4858
-dmu_scripts/text/transform_text.py,sha256=9akj1LB0HAyopOvkLjNOJiptZw5XoOQLe17SlcrGMD0,1456
-data_manipulation_utilities-0.2.7.dist-info/METADATA,sha256=aaUGi8PUMDuJCdslarv5Rn10iI-Jvbu4WkWbEOwmtzw,34464
-data_manipulation_utilities-0.2.7.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-data_manipulation_utilities-0.2.7.dist-info/entry_points.txt,sha256=1TIZDed651KuOH-DgaN5AoBdirKmrKE_oM1b6b7zTUU,270
-data_manipulation_utilities-0.2.7.dist-info/top_level.txt,sha256=n_x5J6uWtSqy9mRImKtdA2V2NJNyU8Kn3u8DTOKJix0,25
-data_manipulation_utilities-0.2.7.dist-info/RECORD,,

data_manipulation_utilities-0.2.7.dist-info/entry_points.txt

@@ -1,6 +0,0 @@
-[console_scripts]
-check_truth = dmu_scripts.physics.check_truth:main
-compare_root_files = dmu_scripts.rfile.compare_root_files:main
-coned = dmu_scripts.ssh.coned:main
-print_trees = dmu_scripts.rfile.print_trees:main
-transform_text = dmu_scripts.text.transform_text:main

data_manipulation_utilities-0.2.7.dist-info/top_level.txt

@@ -1,3 +0,0 @@
-dmu
-dmu_data
-dmu_scripts

dmu_data/ml/tests/diagnostics_from_file.yaml

@@ -1,13 +0,0 @@
-output         : /tmp/tests/dmu/ml/cv_diagnostics/from_rdf
-  # Will assume that the target is already in the input dataframe
-  # and will use it, instead of evaluating models
-score_from_rdf : w 
-correlations:
-  # Variables with respect to which the correlations with the features will be measured
-  target :
-    name : z
-  methods:
-    - Pearson
-  figure:
-    title: Scores from file 
-    size : [10, 8]

dmu_data/ml/tests/diagnostics_from_model.yaml

@@ -1,10 +0,0 @@
-output      : /tmp/tests/dmu/ml/cv_diagnostics/from_model
-correlations:
-  # Variables with respect to which the correlations with the features will be measured
-  target    : 
-    name    : z
-  methods:
-    - Pearson
-  figure:
-    size  : [10, 8]
-    rotate: 90

dmu_data/ml/tests/diagnostics_multiple_methods.yaml

@@ -1,10 +0,0 @@
-output : /tmp/tests/dmu/ml/cv_diagnostics/multiple_methods
-correlations:
-  # Variables with respect to which the correlations with the features will be measured
-  target    : 
-    name : z
-  methods:
-    - Pearson
-    - Kendall-$\tau$ 
-  figure:
-    size : [10, 8]

dmu_data/ml/tests/diagnostics_overlay.yaml

@@ -1,33 +0,0 @@
-output         : /tmp/tests/dmu/ml/cv_diagnostics/overlay
-  # Will assume that the target is already in the input dataframe
-  # and will use it, instead of evaluating models
-score_from_rdf : w
-correlations:
-  # Variables with respect to which the correlations with the features will be measured
-  target :
-    name : z
-    overlay :
-      wp :
-        - 0.2
-        - 0.5
-        - 0.7
-        - 0.9
-      general:
-        size : [12, 10]
-      saving:
-        plt_dir : /tmp/tests/dmu/ml/cv_diagnostics/overlay
-      plots:
-        z :
-          binning    : [-4, 4, 10]
-          yscale     : 'linear'
-          labels     : ['$z$', 'Entries']
-          normalized : true
-          styling :          
-            linestyle: '-'
-  methods:
-    - Pearson
-    - Kendall-$\tau$ 
-  figure:
-    title     : Scores from file
-    size      : [12, 10]
-    xlabelsize: 30 

dmu_data/ml/tests/train_mva.yaml

@@ -1,58 +0,0 @@
-dataset:
-  define :
-    r : z + x
-  nan :
-    x : -3
-    y : -3
-training :
-    nfold    : 3
-    features : [x, y, r]
-    rdm_stat : 1
-    hyper    :
-      loss              : log_loss
-      n_estimators      : 100
-      max_depth         : 3
-      learning_rate     : 0.1
-      min_samples_split : 2
-saving:
-    path : '/tmp/tests/dmu/ml/train_mva/model.pkl'
-plotting:
-    roc     :
-        min : [0.0, 0.0]
-        max : [1.2, 1.2]
-        annotate:
-          sig_eff : [0.5, 0.6, 0.7, 0.8, 0.9]
-          form : '{:.2f}'
-          color: 'green'
-          xoff : -15
-          yoff : -15
-          size :  10
-    correlation:
-      title      : 'Correlation matrix'
-      size       : [10, 10]
-      mask_value : 0
-    val_dir : '/tmp/tests/dmu/ml/train_mva'
-    features:
-        saving:
-            plt_dir : '/tmp/tests/dmu/ml/train_mva/features'
-        plots:
-          r :
-            binning : [-6, 6, 100]
-            yscale  : 'linear'
-            labels  : ['$r$', '']
-          w :
-            binning : [-4, 4, 100]
-            yscale  : 'linear'
-            labels  : ['$w$', '']
-          x :
-            binning : [-4, 4, 100]
-            yscale  : 'linear'
-            labels  : ['$x$', '']
-          y :
-            binning : [-4, 4, 100]
-            yscale  : 'linear'
-            labels  : ['$y$', '']
-          z :
-            binning : [-4, 4, 100]
-            yscale  : 'linear'
-            labels  : ['$z$', '']

dmu_data/ml/tests/train_mva_with_diagnostics.yaml

@@ -1,82 +0,0 @@
-dataset:
-  define :
-    r : z + x
-  nan :
-    x : -3
-    y : -3
-training :
-    nfold    : 3
-    features : [x, y, r]
-    rdm_stat : 1
-    hyper    :
-      loss              : log_loss
-      n_estimators      : 100
-      max_depth         : 3
-      learning_rate     : 0.1
-      min_samples_split : 2
-saving:
-    path : '/tmp/tests/dmu/ml/train_mva/model.pkl'
-plotting:
-    roc     :
-        min : [0.0, 0.0]
-        max : [1.2, 1.2]
-        annotate:
-          sig_eff : [0.5, 0.6, 0.7, 0.8, 0.9]
-          form : '{:.2f}'
-          color: 'green'
-          xoff : -15
-          yoff : -15
-          size :  10
-    correlation:
-      title      : 'Correlation matrix'
-      size       : [10, 10]
-      mask_value : 0
-    val_dir : '/tmp/tests/dmu/ml/train_mva'
-    features:
-        saving:
-            plt_dir : '/tmp/tests/dmu/ml/train_mva/features'
-        plots:
-          r :
-            binning : [-6, 6, 100]
-            yscale  : 'linear'
-            labels  : ['$r$', '']
-          w :
-            binning : [-4, 4, 100]
-            yscale  : 'linear'
-            labels  : ['$w$', '']
-          x :
-            binning : [-4, 4, 100]
-            yscale  : 'linear'
-            labels  : ['$x$', '']
-          y :
-            binning : [-4, 4, 100]
-            yscale  : 'linear'
-            labels  : ['$y$', '']
-          z :
-            binning : [-4, 4, 100]
-            yscale  : 'linear'
-            labels  : ['$z$', '']
-diagnostics:
-  output         : /tmp/tests/dmu/ml/train_mva/diagnostics
-  correlations:
-    target :
-      name : z 
-      overlay :  
-        general:
-          size : [20, 10]
-        saving:
-          plt_dir : /tmp/tests/dmu/ml/train_mva/diagnostics
-        plots:
-          z :
-            binning    : [-4, +4, 30] 
-            yscale     : 'linear'
-            labels     : ['z', 'Entries']
-            normalized : true
-            styling :
-              linestyle: '-'
-    methods:
-      - Pearson
-      - Kendall-$\tau$
-    figure:
-      title: Training diagnostics 
-      size : [10, 8]

dmu_data/plotting/tests/2d.yaml

@@ -1,24 +0,0 @@
-saving:
-    plt_dir : /tmp/dmu/tests/plotting/2d_weighted
-selection:
-  cuts:
-    xlow : x > -1.5
-definitions:
-  z : x + y
-general:
-    size : [20, 10]
-plots_2d:
-    - [x, y, weights, 'xy_wgt', false]
-    - [x, y,    null, 'xy_raw', false]
-    - [x, z,    null, 'xz_raw', false]
-    - [x, z,    null, 'xz_log',  true]
-axes:
-    x :
-        binning : [-3.0, 3.0, 40]
-        label   : 'x'
-    y :
-        binning : [-5.0, 8.0, 40]
-        label   : 'y'
-    z :
-        binning : [-5.0, 16.0, 40]
-        label   : 'z'

dmu_data/plotting/tests/fig_size.yaml

@@ -1,13 +0,0 @@
-saving:
-    plt_dir : tests/plotting/fig_size
-general:
-    size : [20, 10]
-plots:
-    x : 
-        binning : [-5.0, 8.0, 40]
-        yscale  : 'linear' 
-        labels  : ['x', 'Entries']
-    y : 
-        binning : [-5.0, 8.0, 40]
-        yscale  : 'linear' 
-        labels  : ['y', 'Entries']