virgo-modules 0.0.75__py3-none-any.whl → 0.0.76__py3-none-any.whl

virgo_modules/src/aws_utils.py

@@ -6,14 +6,36 @@ import pandas as pd
 
 
 def upload_file_to_aws(bucket,key,input_path, aws_credentials):
-    
+    '''
+    upload file from a folder to an s3 folder
+
+            Parameters:
+                    bucket (str): bucket name
+                    key (str): key pattern or folder in s3 e.g. path/to/upload/
+                    input_path (str): input path of the file to upload e.g. path/to/upload.txt
+                    aws_credentials (dict): aws credentials dictionary
+                    
+            Returns:
+                    None
+    '''
     session = boto3.Session(aws_access_key_id=aws_credentials['AWS_ACCESS_KEY_ID'],aws_secret_access_key=aws_credentials['AWS_SECRET_ACCESS_KEY'])
     bucket = aws_credentials[bucket]
     s3 = session.resource('s3')
     s3.meta.client.upload_file(Filename=input_path , Bucket=bucket, Key=key)
 
 def upload_pandas_to_s3(data_frame,bucket,key, aws_credentials):
+    '''
+    upload dataframe as csv to an s3 folder
 
+            Parameters:
+                    data_frame (pd.DataFrame): data
+                    bucket (str): bucket name
+                    key (str): key pattern or folder in s3 e.g. path/to/upload/
+                    aws_credentials (dict): aws credentials dictionary
+                    
+            Returns:
+                    None
+    '''
     csv_buffer = StringIO()
     data_frame.to_csv(csv_buffer)
     csv_buffer.seek(0)
@@ -23,7 +45,17 @@ def upload_pandas_to_s3(data_frame,bucket,key, aws_credentials):
     s3.put_object(Bucket=bucket, Body=csv_buffer.getvalue(), Key= key)
 
 def download_file_to_aws(bucket,key, aws_credentials):
-    
+    '''
+    download csv file from s3 folder
+
+            Parameters:
+                    bucket (str): bucket name
+                    key (str): key pattern or folder in s3 e.g. path/to/download/file.csv
+                    aws_credentials (dict): aws credentials dictionary
+                    
+            Returns:
+                    None
+    '''
     s3c = boto3.client(
             's3', 
             region_name = aws_credentials['AWS_DEFAULT_REGION'],

virgo_modules/src/edge_utils.py

@@ -12,11 +12,54 @@ from feature_engine.discretisation import EqualWidthDiscretiser
 from .ticketer_source import VirgoWinsorizerFeature, InverseHyperbolicSine
 
 class produce_model_wrapper:
+    """
+    class that wraps a pipeline and a machine learning model. it also provides data spliting train/validation
+
+    Attributes
+    ----------
+    data : pd.DataFrame
+        list of features to apply the transformation
+    X_train : pd.DataFrame
+    y_train : pd.DataFrame
+    X_val : pd.DataFrame
+    y_val : pd.DataFrame
+    self.pipeline: obj
+        sklearn pipeline including model and pipleline
+
+    Methods
+    -------
+    preprocess(validation_size=int, target=list):
+        ingest data and split data between train and validation data and X and Y data
+    train_model(pipe=obj, model=obj, cv_=boolean):
+        merge and train pipeline and machine learning model
+    """
     def __init__(self,data):
+        """
+        Initialize object
+
+        Parameters
+        ----------
+        data (pd.DataFrame): data
+
+        Returns
+        -------
+        None
+        """
         self.data = data.copy()
     
     def preprocess(self, validation_size, target):
-        
+        """
+        ingest data and split data between train and validation data and X and Y data
+
+        Parameters
+        ----------
+        validation_size (int): validation data size, the remaining is taken as training data
+        target (list): target column list
+
+        Returns
+        -------
+        None
+        """
         val_date = self.data.groupby('Date', as_index = False).agg(target_down = (target[0],'count')).sort_values('Date').iloc[-validation_size:,].head(1)['Date'].values[0]
         
         train_data = self.data[self.data['Date'] < val_date].dropna()
@@ -31,6 +74,18 @@ class produce_model_wrapper:
         self.y_val = y_val
     
     def train_model(self, pipe, model, cv_ = False):
+        """
+        merge and train pipeline and machine learning model
+
+        Parameters
+        ----------
+        pipe (int): sklearn pipeline object
+        model (list): model
+
+        Returns
+        -------
+        None
+        """
         self.model = model
         self.pipe_transform = pipe
         self.pipeline = Pipeline([('pipe_transform',self.pipe_transform), ('model',self.model)])
@@ -38,11 +93,53 @@ class produce_model_wrapper:
         self.pipeline.fit(self.X_train, self.y_train)
 
 class register_results():
+    """
+    class that collects model metrics
+
+    Attributes
+    ----------
+    model_name : str
+        model name
+    metric_logger : diot
+        dictionary that collect model metrics
+
+    Methods
+    -------
+    eval_metrics(pipeline=obj, X=pd.DataFrame, y=pd.DataFrame, type_data=str, phase=str):
+        register model metrics
+    print_metric_logger():
+        print logger results
+    """
     def __init__(self, model_name):
+        """
+        Initialize object
+
+        Parameters
+        ----------
+        model_name (str): model name
+
+        Returns
+        -------
+        None
+        """
         self.model_name = model_name
         self.metric_logger = dict()
     def eval_metrics(self, pipeline, X, y, type_data, phase):
-        
+        """
+        register model metrics
+
+        Parameters
+        ----------
+        pipeline (obj): model pipeline
+        X (pd.DataFrame): input data
+        Y (pd.DataFrame): target data
+        type_data (str): data type, either train, test or validation
+        phase (str): model phase, either baseline, feature selection, tunned model
+
+        Returns
+        -------
+        None
+        """
         preds_proba = pipeline.predict_proba(X)
         preds = pipeline.predict(X)
     
@@ -56,6 +153,17 @@ class register_results():
         self.metric_logger[f'{phase}//{self.model_name}//{type_data}'] = {'roc':roc, 'precision':precision, 'recall':recall}
 
     def print_metric_logger(self):
+        """
+        print logger results
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        None
+        """
         parts = list(self.metric_logger.keys())
         phase_parts = [ x.split('//')[0] for x in parts]
     
@@ -74,7 +182,19 @@ class register_results():
 
 
 def eval_metrics(pipeline, X, y, type_data, model_name):
-    
+    '''
+    print metrics from a model pipeline
+
+            Parameters:
+                    pipeline (obj): model pipeline
+                    X (pd.DataFrame): input data
+                    Y (pd.DataFrame): target data
+                    type_data (str): data type, either train, test or validation
+                    model_name (str): model name
+
+            Returns:
+                    objects (dict): that contains ml artifacts, data , configs and models
+    '''
     preds_proba = pipeline.predict_proba(X)
     preds = pipeline.predict(X)
 
@@ -97,7 +217,22 @@ def data_processing_pipeline_classifier(
         invhypervolsin_features = False,
         pipeline_order = 'selector//winzorizer//discretizer//median_inputer//drop//correlation'):
 
+    '''
+    pipeline builder
 
+            Parameters:
+                    features_base (list): model pipeline
+                    features_to_drop (list): features to drop list
+                    winsorizer_conf (dict): winsorising configuration dictionary
+                    discretize_columns (list): feature list to discretize
+                    bins_discretize (int): number of bins to discretize
+                    correlation (float): correlation threshold to discard correlated features
+                    fillna (boolean): if true to fill na features
+                    invhypervolsin_features (list): list of features to apply inverse hyperbolic sine
+                    pipeline_order (str): custom pipeline order eg. selector//winzorizer//discretizer//median_inputer//drop//correlation
+            Returns:
+                    pipe (obj): pipeline object
+    '''
     select_pipe = [('selector', FeatureSelector(features_base))] if features_base else []
     winzorizer_pipe = [('winzorized_features', VirgoWinsorizerFeature(winsorizer_conf))] if winsorizer_conf else []
     drop_pipe = [('drop_features' , DropFeatures(features_to_drop=features_to_drop))] if features_to_drop else []
@@ -130,15 +265,62 @@ def data_processing_pipeline_classifier(
 
 
 class ExpandingMultipleTimeSeriesKFold:
-    """increasing training window where the test can be overlap"""
+    """
+    class that creates a custom cv schema that is compatible with sklearn cv arguments.
+
+    Attributes
+    ----------
+    df : pd.DataFrame
+        dataset
+    number_window : int
+        number of train splits
+    window_size : int
+        window size data
+    overlap_size : int 
+        overlap size
+
+    Methods
+    -------
+    split(X=pd.DataFrame, y=pd.DataFrame, groups=boolean):
+        custom split procedure
+    get_n_splits(X=pd.DataFrame, y=pd.DataFrame, groups=boolean):
+        get number of splits
+    """
+    
     def __init__(self, df, window_size = 100, number_window=3, overlap_size = 0):
+        """
+        Initialize object
+
+        Parameters
+        ----------
+        df (pd.DataFrame): dataset
+        number_window (int): number of train splits
+        window_size (int): window size data
+        overlap_size (int): overlap size
+
+        Returns
+        -------
+        None
+        """
         self.df = df
         self.number_window = number_window
         self.window_size = window_size
         self.overlap_size = overlap_size
         
     def split(self, X, y, groups=None):
-        
+        """
+        custom split procedure
+
+        Parameters
+        ----------
+        X (pd.DataFrame): input data (required for sklearn classes)
+        y (pd.DataFrame): target data (required for sklearn classes)
+        groups (boolean): to apply groups (required for sklearn classes)
+
+        Returns
+        -------
+        None
+        """
         if 'Date_i' not in self.df.index.names or 'i' not in self.df.index.names:
             raise Exception('no date and/or index in the index dataframe')
         
@@ -175,4 +357,17 @@ class ExpandingMultipleTimeSeriesKFold:
             yield train_index, test_index
 
     def get_n_splits(self, X, y, groups=None):
+        """
+        get number of splits
+
+        Parameters
+        ----------
+        X (pd.DataFrame): input data (required for sklearn classes)
+        y (pd.DataFrame): target data (required for sklearn classes)
+        groups (boolean): to apply groups (required for sklearn classes)
+
+        Returns
+        -------
+        number_window (int): number of splits
+        """
         return self.number_window