data-manipulation-utilities 0.0.1__py3-none-any.whl

data_manipulation_utilities-0.0.1.dist-info/RECORD

@@ -0,0 +1,45 @@
+dmu/arrays/utilities.py,sha256=PKoYyybPptA2aU-V3KLnJXBudWxTXu4x1uGdIMQ49HY,1722
+dmu/dataframe/dataframe.py,sha256=ZgRCw5hN18gOXGL9nHDc4eNi0P8lOIAIEILmbEiTlXw,1088
+dmu/generic/utilities.py,sha256=0Xnq9t35wuebAqKxbyAiMk1ISB7IcXK4cFH25MT1fgw,1741
+dmu/logging/log_store.py,sha256=v0tiNz-6ktT_afD5DuvCZ8Nmr82JKQOPli8hgd28P1Q,3960
+dmu/ml/cv_classifier.py,sha256=n81m7i2M6Zq96AEd9EZGwXSrbG5m9jkS5RdeXvbsAXU,3712
+dmu/ml/cv_predict.py,sha256=Bqxu-f6qquKJokFljhCzL_kiGcjLJLQFhVBD130fsyw,4893
+dmu/ml/train_mva.py,sha256=d_n-A07DFweikz5nXap4OE_Mqx8VprFT7zbxmnQAbac,9638
+dmu/ml/utilities.py,sha256=Nue7O9zi1QXgjGRPH6wnSAW9jusMQ2ZOSDJzBqJKIi0,3687
+dmu/plotting/plotter.py,sha256=laa6Kl7P-ZOIhaOFBVjOH4XQ4kPCV7wBNvLIMBnyCwM,7181
+dmu/plotting/plotter_1d.py,sha256=G-i94uzm2TjNaog1A4agAKar_G0qNdkAqIPCmzhe85Y,3660
+dmu/plotting/plotter_2d.py,sha256=SWPKns-CfpUZHgBXvwm3gceH3k2eL_mKGXQ8sWpZJB0,2919
+dmu/rdataframe/atr_mgr.py,sha256=FdhaQWVpsm4OOe1IRbm7rfrq8VenTNdORyI-lZ2Bs1M,2386
+dmu/rdataframe/utilities.py,sha256=a31PdUz12sC2bx78LK6gvACh1M_eFaIVwuZEvOTcvcc,2084
+dmu/rfile/rfprinter.py,sha256=vGdqyHT_GwGBhrY7KG63EAUGWEOqobz_5yTL6goXbfk,2722
+dmu/rfile/utilities.py,sha256=XuYY7HuSBj46iSu3c60UYBHtI6KIPoJU_oofuhb-be0,945
+dmu/stats/fitter.py,sha256=LDvFNyhgO0OzXN7aH3kfHe6LzuPqdQfPcKR_IegDcaU,18204
+dmu/stats/function.py,sha256=yzi_Fvp_ASsFzbWFivIf-comquy21WoeY7is6dgY0Go,9491
+dmu/stats/utilities.py,sha256=LQy4kd3xSXqpApcWuYfZxkGQyjowaXv2Wr1c4Bj-4ys,4523
+dmu/testing/utilities.py,sha256=WbMM4e9Cn3-B-12Vr64mB5qTKkV32joStlRkD-48lG0,3460
+dmu/text/transformer.py,sha256=4lrGknbAWRm0-rxbvgzOO-eR1-9bkYk61boJUEV3cQ0,6100
+dmu_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dmu_data/ml/tests/train_mva.yaml,sha256=TCniCVpXMEFxZcHa8IIqollKA7ci4OkBnRznLEkXM9o,925
+dmu_data/plotting/tests/2d.yaml,sha256=lTMNheK3DB8klp4O5QjMDwBI1A1Oh2_Wp2F2Ro9VQKM,282
+dmu_data/plotting/tests/fig_size.yaml,sha256=7ROq49nwZ1A2EbPiySmu6n3G-Jq6YAOkc3d2X3YNZv0,294
+dmu_data/plotting/tests/high_stat.yaml,sha256=bLglBLCZK6ft0xMhQ5OltxE76cWsBMPMjO6GG0OkDr8,522
+dmu_data/plotting/tests/name.yaml,sha256=mkcPAVg8wBAmlSbSRQ1bcaMl4vOS6LXMtpqQeDrrtO4,312
+dmu_data/plotting/tests/no_bounds.yaml,sha256=8e1QdphBjz-suDr857DoeUC2DXiy6SE-gvkORJQYv80,257
+dmu_data/plotting/tests/simple.yaml,sha256=N_TvNBh_2dU0-VYgu_LMrtY0kV_hg2HxVuEoDlr1HX8,138
+dmu_data/plotting/tests/title.yaml,sha256=bawKp9aGpeRrHzv69BOCbFX8sq9bb3Es9tdsPTE7jIk,333
+dmu_data/plotting/tests/weights.yaml,sha256=RWQ1KxbCq-uO62WJ2AoY4h5Umc37zG35s-TpKnNMABI,312
+dmu_data/text/transform.toml,sha256=R-832BZalzHZ6c5gD6jtT_Hj8BCsM5vxa1v6oeiwaP4,94
+dmu_data/text/transform.txt,sha256=EX760da6Vkf-_EPxnQlC5hGSkfFhJCCGCD19NU-1Qto,44
+dmu_data/text/transform_set.toml,sha256=Jeh7BTz82idqvbOQJtl9-ur56mZkzDn5WtvmIb48LoE,150
+dmu_data/text/transform_set.txt,sha256=1KivMoP9LxPn9955QrRmOzjEqduEjhTetQ9MXykO5LY,46
+dmu_data/text/transform_trf.txt,sha256=zxBRTgcSmX7RdqfmWF88W1YqbyNHa4Ccruf1MmnYv2A,74
+dmu_scripts/physics/check_truth.py,sha256=b1P_Pa9ef6VcFtyY6Y9KS9Om9L-QrCBjDKp4dqca0PQ,3964
+dmu_scripts/rfile/compare_root_files.py,sha256=T8lDnQxsRNMr37x1Y7YvWD8ySHrJOWZki7ZQynxXX9Q,9540
+dmu_scripts/rfile/print_trees.py,sha256=Ze4Ccl_iUldl4eVEDVnYBoe4amqBT1fSBR1zN5WSztk,941
+dmu_scripts/ssh/coned.py,sha256=lhilYNHWRCGxC-jtyJ3LQ4oUgWW33B2l1tYCcyHHsR0,4858
+dmu_scripts/text/transform_text.py,sha256=9akj1LB0HAyopOvkLjNOJiptZw5XoOQLe17SlcrGMD0,1456
+data_manipulation_utilities-0.0.1.dist-info/METADATA,sha256=chAPGy68TwWTweqqzGgXvrD6-xia1Bq9XDgvgS1qLEE,19714
+data_manipulation_utilities-0.0.1.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+data_manipulation_utilities-0.0.1.dist-info/entry_points.txt,sha256=1TIZDed651KuOH-DgaN5AoBdirKmrKE_oM1b6b7zTUU,270
+data_manipulation_utilities-0.0.1.dist-info/top_level.txt,sha256=n_x5J6uWtSqy9mRImKtdA2V2NJNyU8Kn3u8DTOKJix0,25
+data_manipulation_utilities-0.0.1.dist-info/RECORD,,

data_manipulation_utilities-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.5.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

data_manipulation_utilities-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,6 @@
+[console_scripts]
+check_truth = dmu_scripts.physics.check_truth:main
+compare_root_files = dmu_scripts.rfile.compare_root_files:main
+coned = dmu_scripts.ssh.coned:main
+print_trees = dmu_scripts.rfile.print_trees:main
+transform_text = dmu_scripts.text.transform_text:main

data_manipulation_utilities-0.0.1.dist-info/top_level.txt

@@ -0,0 +1,3 @@
+dmu
+dmu_data
+dmu_scripts

dmu/arrays/utilities.py

@@ -0,0 +1,55 @@
+'''
+Module with utility functions to handle numpy arrays
+'''
+
+import math
+import numpy
+
+#-----------------------------------------------
+def _check_ftimes(ftimes : float) -> None:
+    '''
+    Check if floating scale factor makes sense
+    '''
+    if not isinstance(ftimes, float):
+        raise TypeError(f'Scaling factor is not a float, but: {ftimes}')
+
+    if ftimes <= 1.0:
+        raise ValueError(f'Scaling factor needs to be larger than 1.0, found: {ftimes}')
+#-----------------------------------------------
+def repeat_arr(arr_val : numpy.ndarray, ftimes : float) -> numpy.ndarray:
+    '''
+    Will repeat elements in an array a non integer number of times.
+
+    arr_val: 1D array of objects
+    ftimes (float): Number of times to repeat it.
+    '''
+
+    _check_ftimes(ftimes)
+
+    a = math.floor(ftimes)
+    b = math.ceil(ftimes)
+    if numpy.isclose(a, b):
+        return numpy.repeat(arr_val, a)
+
+    # Will split randomly data in arr_val, such that one set will get increased
+    # by floor(ftimes) and the other by ceiling(ftimes)
+
+    # Get probability that given element belongs to dataset weighted by "a"
+    p         = b - ftimes
+    size_t    = len(arr_val)
+    size_1    = int(p * size_t)
+
+    # Find subset to weight by "a"
+    arr_ind_1 = numpy.random.choice(size_t, size=size_1, replace=False)
+    arr_val_1 = arr_val[arr_ind_1]
+
+    # Find subset to weight by "b"
+    arr_ind_2 = numpy.setdiff1d(numpy.arange(size_t), arr_ind_1)
+    arr_val_2 = arr_val[arr_ind_2]
+
+    # Repeat them an integer number of times
+    arr_val_1 = numpy.repeat(arr_val_1, a)
+    arr_val_2 = numpy.repeat(arr_val_2, b)
+
+    return numpy.concatenate([arr_val_1, arr_val_2])
+#-----------------------------------------------

dmu/dataframe/dataframe.py

@@ -0,0 +1,36 @@
+'''
+Class re-implementing dataframe, meant to be a thin layer on top of polars
+'''
+
+import polars as pl
+
+# ------------------------------------------
+class DataFrame(pl.DataFrame):
+    '''
+    Class reimplementing dataframes from polars
+    '''
+    # ------------------------------------------
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+    # ------------------------------------------
+    def define(self, name  : str, expr : str):
+        '''
+        Function will define new column in dataframe
+
+        name (str): Name of new column
+        expr (str): Expression depending on existing columns
+        '''
+
+        for col in self.columns:
+            expr = expr.replace(col, f' {col} ')
+
+        for col in self.columns:
+            expr = expr.replace(f' {col} ',  f' pl.col("{col}") ')
+
+        try:
+            df = self.with_columns(eval(expr).alias(name))
+        except TypeError as exc:
+            raise TypeError(f'Cannot define {expr} -> {name}') from exc
+
+        return DataFrame(df)
+# ------------------------------------------

dmu/generic/utilities.py

@@ -0,0 +1,69 @@
+'''
+Module containing generic utility functions
+'''
+import os
+import time
+import json
+import inspect
+
+from typing import Callable
+
+from functools             import wraps
+from dmu.logging.log_store import LogStore
+
+TIMER_ON=False
+
+log = LogStore.add_logger('dmu:generic:utilities')
+
+# --------------------------------
+def _get_module_name( fun : Callable) -> str:
+    mod = inspect.getmodule(fun)
+    if mod is None:
+        raise ValueError(f'Cannot determine module name for function: {fun}')
+
+    return mod.__name__
+# --------------------------------
+def timeit(f):
+    '''
+    Decorator used to time functions, it is turned off by default, can be turned on with:
+
+    from dmu.generic.utilities import TIMER_ON
+    from dmu.generic.utilities import timeit 
+
+    TIMER_ON=True
+
+    @timeit
+    def fun():
+        ...
+    '''
+    @wraps(f)
+    def wrap(*args, **kw):
+        if not TIMER_ON:
+            result = f(*args, **kw)
+            return result
+
+        ts = time.time()
+        result = f(*args, **kw)
+        te = time.time()
+        mod_nam = _get_module_name(f)
+        fun_nam = f.__name__
+        log.info(f'{mod_nam}.py:{fun_nam}; Time: {te-ts:.3f}s')
+
+        return result
+    return wrap
+# --------------------------------
+def dump_json(data, path : str, sort_keys : bool = False):
+    '''
+    Saves data as JSON
+
+    Parameters
+    data     : dictionary, list, etc
+    path     : Path to JSON file where to save it
+    sort_keys: Will set sort_keys argument of json.dump function 
+    '''
+    dir_name = os.path.dirname(path)
+    os.makedirs(dir_name, exist_ok=True)
+
+    with open(path, 'w', encoding='utf-8') as ofile:
+        json.dump(data, ofile, indent=4, sort_keys=sort_keys)
+# --------------------------------

dmu/logging/log_store.py

@@ -0,0 +1,129 @@
+'''
+Module holding LogStore 
+'''
+
+import logging
+import logzero
+
+#------------------------------------------------------------
+class StoreFormater(logging.Formatter):
+    '''
+    Custom formatter
+    '''
+    LOG_COLORS = {
+        logging.DEBUG   : '\033[94m',     # Gray
+        logging.INFO    : '\033[37m',     # White
+        logging.WARNING : '\033[93m',     # Yellow
+        logging.ERROR   : '\033[91m',     # Red
+        logging.CRITICAL: '\033[1;91m'    # Bold Red
+    }
+
+    RESET_COLOR = '\033[0m'  # Reset color to default
+
+    def format(self, record):
+        log_color = self.LOG_COLORS.get(record.levelno, self.RESET_COLOR)
+        message   = super().format(record)
+
+        return f'{log_color}{message}{self.RESET_COLOR}'
+#------------------------------------------------------------
+class LogStore:
+    '''
+    Class used to make loggers, set log levels, print loggers, e.g. interface to logging/logzero, etc.
+    '''
+    #pylint: disable = invalid-name
+    d_logger      = {}
+    d_levels      = {}
+    log_level     = logging.INFO
+    is_configured = False
+    backend       = 'logging'
+    #--------------------------
+    @staticmethod
+    def add_logger(name=None):
+        '''
+        Will use underlying logging library logzero/logging, etc to make logger
+
+        name (str): Name of logger
+        '''
+
+        if   name is None:
+            raise ValueError('Logger name missing')
+
+        if name in LogStore.d_logger:
+            raise ValueError(f'Logger name {name} already found')
+
+        level  = LogStore.log_level if name not in LogStore.d_levels else LogStore.d_levels[name]
+
+        if   LogStore.backend == 'logging':
+            logger = LogStore._get_logging_logger(name, level)
+        elif LogStore.backend == 'logzero':
+            logger = LogStore._get_logzero_logger(name, level)
+        else:
+            raise ValueError(f'Invalid backend: {LogStore.backend}')
+
+        LogStore.d_logger[name] = logger
+
+        return logger
+    #--------------------------
+    @staticmethod
+    def _get_logzero_logger(name : str, level : int):
+        log = logzero.setup_logger(name=name)
+        log.setLevel(level)
+
+        return log
+    #--------------------------
+    @staticmethod
+    def _get_logging_logger(name : str, level : int):
+        logger = logging.getLogger(name=name)
+
+        logger.setLevel(level)
+
+        hnd= logging.StreamHandler()
+        hnd.setLevel(level)
+
+        fmt= StoreFormater('%(asctime)s - %(filename)s:%(lineno)d - %(message)s', datefmt='%H:%M:%S')
+        hnd.setFormatter(fmt)
+
+        if logger.hasHandlers():
+            logger.handlers.clear()
+
+        logger.addHandler(hnd)
+
+        return logger
+    #--------------------------
+    @staticmethod
+    def set_level(name, value):
+        '''
+        Will set the level of a logger, it not present yet, it will store the level and set it when created.
+        Parameters:
+        -----------------
+        name (str): Name of logger
+        value (int): 10 debug, 20 info, 30 warning
+        '''
+
+        if name in LogStore.d_logger:
+            lgr=LogStore.d_logger[name]
+            lgr.handlers[0].setLevel(value)
+            lgr.setLevel(value)
+        else:
+            LogStore.d_levels[name] = value
+    #--------------------------
+    @staticmethod
+    def show_loggers():
+        '''
+        Will print loggers and log levels in two columns
+        '''
+        print(80 * '-')
+        print(f'{"Name":<60}{"Level":<20}')
+        print(80 * '-')
+        for name, logger in LogStore.d_logger.items():
+            print(f'{name:<60}{logger.level:<20}')
+    #--------------------------
+    @staticmethod
+    def set_all_levels(level):
+        '''
+        Will set all loggers to this level (int)
+        '''
+        for name, logger in LogStore.d_logger.items():
+            logger.setLevel(level)
+            print(f'{name:<60}{"->":20}{logger.level:<20}')
+#------------------------------------------------------------

dmu/ml/cv_classifier.py

@@ -0,0 +1,122 @@
+'''
+Module holding cv_classifier class
+'''
+
+from sklearn.ensemble        import GradientBoostingClassifier
+
+from dmu.logging.log_store import LogStore
+import dmu.ml.utilities    as ut
+
+log = LogStore.add_logger('dmu:ml:CVClassifier')
+
+# ---------------------------------------
+class CVSameData(Exception):
+    '''
+    Will be raised if a model is been evaluated with a dataset such that at least one of the
+    samples was also used for the training
+    '''
+# ---------------------------------------
+class CVClassifier(GradientBoostingClassifier):
+    '''
+    Derived class meant to implement features needed for cross-validation
+    '''
+    # pylint: disable = too-many-ancestors, abstract-method
+    # ----------------------------------
+    def __init__(self, cfg : dict | None = None):
+        '''
+        cfg (dict) : Dictionary with configuration, specially the hyperparameters set in the `hyper` field
+        '''
+        if cfg is None:
+            raise ValueError('No configuration was passed')
+
+        self._cfg = cfg
+
+        d_hyp = self._cfg['training']['hyper']
+        super().__init__(**d_hyp)
+
+        self._s_hash    = set()
+        self._data      = {}
+        self._l_ft_name = None
+    # ----------------------------------
+    @property
+    def features(self):
+        '''
+        Returns list of feature names used in training dataset
+        '''
+        return self._l_ft_name
+    # ----------------------------------
+    @property
+    def hashes(self):
+        '''
+        Will return set with hashes of training data
+        '''
+        return self._s_hash
+    # ----------------------------------
+    @property
+    def cfg(self):
+        '''
+        Will return dictionary with configuration
+        '''
+
+        return self._cfg
+    # ----------------------------------
+    def __str__(self):
+        nhash = len(self._s_hash)
+
+        msg = 40 * '-' + '\n'
+        msg+= f'{"Attribute":<20}{"Value":<20}\n'
+        msg+= 40 * '-' + '\n'
+        msg += f'{"Hashes":<20}{nhash:<20}\n'
+        msg+= 40 * '-'
+
+        return msg
+    # ----------------------------------
+    def fit(self, *args, **kwargs):
+        '''
+        Runs the training of the model
+        '''
+        log.debug('Fitting')
+
+        df_ft           = args[0]
+        self._l_ft_name = list(df_ft.columns)
+
+        self._s_hash = ut.get_hashes(df_ft)
+        log.debug(f'Saving {len(self._s_hash)} hashes')
+
+        super().fit(*args, **kwargs)
+
+        return self
+    # ----------------------------------
+    def _check_hashes(self, df_ft):
+        '''
+        Will check that the hashes of the passed features do not intersect with the
+        hashes of the features used for the training.
+        Else it will raise CVSameData exception
+        '''
+
+        if len(self._s_hash) == 0:
+            raise ValueError('Found no hashes in model')
+
+        s_hash  = ut.get_hashes(df_ft)
+        s_inter = self._s_hash.intersection(s_hash)
+
+        nh1 = len(self._s_hash)
+        nh2 = len(      s_hash)
+        nh3 = len(s_inter)
+
+        if nh3 > 0:
+            raise CVSameData(f'Found non empty intersection of size: {nh1} ^ {nh2} = {nh3}')
+    # ----------------------------------
+    def predict_proba(self, X, on_training_ok=False):
+        '''
+        Takes pandas dataframe with features
+        Will first check hashes to make sure none of the events/samples
+        used for the training of this model are in the prediction
+
+        on_training_ok (bool): True if the dataset is expected to contain samples used for training, default is False
+        '''
+        if not on_training_ok:
+            self._check_hashes(X)
+
+        return super().predict_proba(X)
+# ---------------------------------------

dmu/ml/cv_predict.py

@@ -0,0 +1,152 @@
+'''
+Module holding CVPredict class
+'''
+from typing import Optional
+
+import pandas as pnd
+import numpy
+import tqdm
+
+from ROOT import RDataFrame
+
+import dmu.ml.utilities     as ut
+import dmu.ml.cv_classifier as CVClassifier
+
+from dmu.logging.log_store import LogStore
+
+log = LogStore.add_logger('dmu:ml:cv_predict')
+# ---------------------------------------
+class CVPredict:
+    '''
+    Class used to get classification probabilities from ROOT
+    dataframe and a set of models. The models were trained with CVClassifier
+    '''
+    def __init__(self, models : Optional[list] = None, rdf : Optional[RDataFrame] = None):
+        '''
+        Will take a list of CVClassifier models and a ROOT dataframe
+        '''
+
+        if models is None:
+            raise ValueError('No list of models passed')
+
+        if rdf is None:
+            raise ValueError('No ROOT dataframe passed')
+
+        self._l_model = models
+        self._rdf     = rdf
+
+        self._arr_patch : numpy.ndarray
+    # --------------------------------------------
+    def _get_df(self):
+        '''
+        Will make ROOT rdf into dataframe and return it
+        '''
+        model = self._l_model[0]
+        l_ft  = model.features
+        d_data= self._rdf.AsNumpy(l_ft)
+        df_ft = pnd.DataFrame(d_data)
+        df_ft = ut.patch_and_tag(df_ft)
+
+        if 'patched_indices' in df_ft.attrs:
+            self._arr_patch = df_ft.attrs['patched_indices']
+
+        nfeat = len(l_ft)
+        log.info(f'Found {nfeat} features')
+        for name in l_ft:
+            log.debug(name)
+
+        return df_ft
+    # --------------------------------------------
+    def _non_overlapping_hashes(self, model, df_ft):
+        '''
+        Will return True if hashes of model and data do not overlap
+        '''
+
+        s_mod_hash = model.hashes
+        s_dff_hash = ut.get_hashes(df_ft)
+
+        s_int = s_mod_hash.intersection(s_dff_hash)
+        if len(s_int) == 0:
+            return True
+
+        return False
+    # --------------------------------------------
+    def _predict_with_overlap(self, df_ft : pnd.DataFrame) -> numpy.ndarray:
+        '''
+        Takes pandas dataframe with features
+
+        Will return numpy array of prediction probabilities when there is an overlap
+        of data and model hashes
+        '''
+        df_ft      = ut.index_with_hashes(df_ft)
+        d_prob     = {}
+        ntotal     = len(df_ft)
+        log.debug(30 * '-')
+        log.info(f'Total size: {ntotal}')
+        log.debug(30 * '-')
+        for model in tqdm.tqdm(self._l_model, ascii=' -'):
+            d_prob_tmp = self._evaluate_model(model, df_ft)
+            d_prob.update(d_prob_tmp)
+
+        ndata  = len(df_ft)
+        nprob  = len(d_prob)
+        if ndata != nprob:
+            log.warning(f'Dataset size ({ndata}) and probabilities size ({nprob}) differ, likely there are repeated entries')
+
+        l_prob = [ d_prob[hsh] for hsh in df_ft.index ]
+
+        return numpy.array(l_prob)
+    # --------------------------------------------
+    def _evaluate_model(self, model : CVClassifier, df_ft : pnd.DataFrame) -> dict[str, float]:
+        '''
+        Evaluate the dataset for one of the folds, by taking the model and the full dataset
+        '''
+        s_dat_hash = set(df_ft.index)
+        s_mod_hash = model.hashes
+
+        s_dif_hash = s_dat_hash - s_mod_hash
+
+        ndif = len(s_dif_hash)
+        ndat = len(s_dat_hash)
+        nmod = len(s_mod_hash)
+        log.debug(f'{ndif:<20}{"=":10}{ndat:<20}{"-":10}{nmod:<20}')
+
+        df_ft_group= df_ft.loc[df_ft.index.isin(s_dif_hash)]
+
+        l_prob = model.predict_proba(df_ft_group)
+        l_hash = list(df_ft_group.index)
+        d_prob = dict(zip(l_hash, l_prob))
+        nfeat  = len(df_ft_group)
+        nprob  = len(l_prob)
+        log.debug(f'{nfeat:<10}{"->":10}{nprob:<10}')
+
+        return d_prob
+    # --------------------------------------------
+    def _patch_probabilities(self, arr_prb : numpy.ndarray) -> numpy.ndarray:
+        if not hasattr(self, '_arr_patch'):
+            return arr_prb
+
+        nentries = len(self._arr_patch)
+        log.warning(f'Patching {nentries} probabilities')
+        arr_prb[self._arr_patch] = -1
+
+        return arr_prb
+    # --------------------------------------------
+    def predict(self) -> numpy.ndarray:
+        '''
+        Will return array of prediction probabilities for the signal category
+        '''
+        df_ft = self._get_df()
+        model = self._l_model[0]
+
+        if self._non_overlapping_hashes(model, df_ft):
+            log.debug('No intersecting hashes found between model and data')
+            arr_prb = model.predict_proba(df_ft)
+        else:
+            log.info('Intersecting hashes found between model and data')
+            arr_prb = self._predict_with_overlap(df_ft)
+
+        arr_prb = self._patch_probabilities(arr_prb)
+
+        return arr_prb
+# ---------------------------------------