virgo-modules 0.5.0__tar.gz → 0.6.1__tar.gz

{virgo_modules-0.5.0 → virgo_modules-0.6.1}/PKG-INFO

@@ -1,19 +1,19 @@
 Metadata-Version: 2.1
 Name: virgo_modules
-Version: 0.5.0
+Version: 0.6.1
 Summary: data processing and statistical modeling using stock market data
 Home-page: https://github.com/miguelmayhem92/virgo_module
 Author: Miguel Mayhuire
 Author-email: miguelmayhem92@gmail.com
 License: MIT
-Platform: UNKNOWN
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Provides-Extra: dev
 License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
 
 # Virgo Package
 
@@ -34,4 +34,3 @@ obj = stock_eda_panel(stock_code = 'PEP', n_days = 20)
 obj.get_data()
 print(obj.df.shape)
 ```
-

{virgo_modules-0.5.0 → virgo_modules-0.6.1}/setup.py

@@ -5,7 +5,7 @@ with open("virgo_app/README.md", "r") as f:
 
 setup(
     name="virgo_modules",
-    version="0.5.0",
+    version="0.6.1",
     description="data processing and statistical modeling using stock market data",
     package_dir={"": "virgo_app"},
     packages=find_packages(where="virgo_app"),

virgo_modules-0.6.1/virgo_app/virgo_modules/src/edge_utils/conformal_utils.py

@@ -0,0 +1,106 @@
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from sklearn.pipeline import Pipeline
+import mlflow
+import pandas as pd
+import numpy as np
+from sklearn.base import BaseEstimator, ClassifierMixin
+from mapie.classification import SplitConformalClassifier
+
+class ConformalStack(mlflow.pyfunc.PythonModel):
+    def __init__(self, model,targets, alphas):
+        self.model = model
+        self.targets = targets
+        self.alphas = alphas
+    def fit(self, data):
+        self.classifiers = dict()
+        for i,target in enumerate(self.targets):
+            st = SingleStack(self.model["model"],i)
+            st.fit()
+            seg_model = Pipeline([ 
+                ('pipe',self.model['pipe_transform']),
+                ('modelbase',st)
+            ])
+            mapie_class = SplitConformalClassifier(seg_model, prefit=True, random_state=123, conformity_score="lac", confidence_level=1-np.array(self.alphas))
+            mapie_class.conformalize(data, data[self.targets[i]].values)
+            self.classifiers[target] = mapie_class
+    def predict_conformal(self, data, ):
+        for target in self.targets:
+            prefix = target+"_conf"
+            _, y_pis = self.classifiers[target].predict_set(data)
+            for i,alpha in enumerate(self.alphas):
+                data[f'{prefix}-{alpha}'] = y_pis[:,1,i]
+                data[f'{prefix}-{alpha}'] = np.where(data[f'{prefix}-{alpha}'] == True,alpha,0)
+        return data
+    
+
+class SingleStack(ClassifierMixin, BaseEstimator):
+    def __init__(self, model, estimator_index):
+        self.model = model
+        self.estimator_index = estimator_index
+        
+    def fit(self):
+        self._is_fitted = True
+        self.classes_ = [0,1]
+
+    def predict_proba(self, X):
+        metas_pred = dict()
+        for i,cont in enumerate(self.model.estimators, start=1):
+            _,estimator = cont
+            meta_pred = estimator.predict_proba(X)
+            metas_pred[f"meta{i}0"] = meta_pred[0][:,1]
+            metas_pred[f"meta{i}1"] = meta_pred[1][:,1]
+        self.meta_preds_df__ = pd.DataFrame(metas_pred)
+
+        prediction_vector = list()
+        for i,cont in enumerate(self.model.meta_estimators, start=0):
+            _,estimator = cont
+            metacols = [f"meta{j}{i}" for j in range(1,len(self.model.estimators)+1)]
+            preds = estimator.predict_proba(self.meta_preds_df__[metacols].values)
+            prediction_vector.append(preds)
+        return prediction_vector[self.estimator_index]
+        
+    def predict(self, X):
+        prediction_vector = list()
+        _ = self.predict_proba(X)
+        for i,cont in enumerate(self.model.meta_estimators, start=0):
+            _,estimator = cont
+            metacols = [f"meta{j}{i}" for j in range(1,len(self.model.estimators)+1)]
+            preds = estimator.predict(self.meta_preds_df__[metacols].values)
+            prediction_vector.append(preds) 
+        
+        p = np.array(tuple(prediction_vector))
+        return p.reshape((p.shape[1],p.shape[0]))[:,self.estimator_index]
+    
+    def __sklearn_is_fitted__(self):
+        return hasattr(self, "_is_fitted") and self._is_fitted
+
+def edge_conformal_lines(data, alphas,threshold = 0.6, plot = False, look_back = 750, offset = 0.08):
+    ### corect labels ####
+    df = data.sort_values('Date').iloc[-look_back:]
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(color = 'blue'),showlegend=True,legendgroup='go up', name='go up'),secondary_y=True)
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),showlegend=True,legendgroup='go down',name='go down'),secondary_y=True)
+    for i,alpha in enumerate(alphas, start=1):
+        try:
+            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_up' in x][0]
+            df_ = df[df[col_alpha] != 0]
+            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_up + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'blue')
+                                     ,showlegend=False,legendgroup='go up',name='go up', text=df_[col_alpha],textposition="bottom center")
+                                     , secondary_y=True)
+        except:
+            pass
+        try:
+            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_down' in x][0]
+            df_ = df[df[col_alpha] != 0]
+            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_down + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'coral')
+                                     ,showlegend=False,legendgroup='go down', name='go down',text=df_[col_alpha].astype(str),textposition="bottom center")
+                                     , secondary_y=True)
+        except:
+            pass
+    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"))
+    fig.update_layout(title_text="sirius - edge probabilities conformal",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig

{virgo_modules-0.5.0 → virgo_modules-0.6.1}/virgo_app/virgo_modules/src/edge_utils/edge_utils.py

@@ -1,5 +1,7 @@
 import numpy as np
 import itertools
+import random
+import math
 
 from sklearn.metrics import roc_auc_score, precision_score, recall_score
 from sklearn.pipeline import Pipeline
@@ -310,7 +312,7 @@ class ExpandingMultipleTimeSeriesKFold:
         get number of splits
     """
     
-    def __init__(self, df, window_size = 100, number_window=3, overlap_size = 0):
+    def __init__(self, df, window_size = 100, number_window=3, overlap_size = 0, sample_parts = None):
         """
         Initialize object
 
@@ -320,6 +322,7 @@ class ExpandingMultipleTimeSeriesKFold:
         number_window (int): number of train splits
         window_size (int): window size data
         overlap_size (int): overlap size
+        sample_individuals (tuple(float, str)): sample partition units to remove from the train set, tuple()
 
         Returns
         -------
@@ -329,6 +332,7 @@ class ExpandingMultipleTimeSeriesKFold:
         self.number_window = number_window
         self.window_size = window_size
         self.overlap_size = overlap_size
+        self.sample_parts = sample_parts
         
     def split(self, X, y, groups=None):
         """
@@ -372,9 +376,21 @@ class ExpandingMultipleTimeSeriesKFold:
             max_train_date = max(train_dates)
             min_test_date, max_test_date = min(test_dates), max(test_dates)
             
-            cut = cut - (self.window_size - self.overlap_size) 
-        
-            train_index = self.df[self.df.index.get_level_values('Date_i') <= max_train_date].index.get_level_values('i')
+            cut = cut - (self.window_size - self.overlap_size)
+
+            if self.sample_parts:
+                sample_part = self.sample_parts[0]
+                part_col = self.sample_parts[1]
+                unique_parts = list(self.df.index.get_level_values(part_col).unique())
+                random.shuffle(unique_parts)
+                n_select = math.ceil(len(unique_parts)*sample_part)
+                to_drop = unique_parts[0:n_select]
+                train_index = self.df[
+                    (self.df.index.get_level_values('Date_i') <= max_train_date) 
+                    & 
+                    (~self.df.index.get_level_values(part_col).isin(to_drop))].index.get_level_values('i')
+            else:
+                train_index = self.df[self.df.index.get_level_values('Date_i') <= max_train_date].index.get_level_values('i')
             test_index = self.df[(self.df.index.get_level_values('Date_i') >= min_test_date) & (self.df.index.get_level_values('Date_i') <= max_test_date)].index.get_level_values('i')
         
             yield train_index, test_index

virgo_modules-0.6.1/virgo_app/virgo_modules/src/edge_utils/shap_utils.py

@@ -0,0 +1,54 @@
+import shap
+import mlflow
+import pandas as pd
+import numpy as np
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+
+class StackInterpretor(mlflow.pyfunc.PythonModel):
+    def __init__(self, model, targets):
+        self.base_estimators = model.estimators_
+        self.targets = targets
+    def fit_interpretor(self, data):
+        interpretors = {}
+        for label, predictor in zip(self.targets,self.base_estimators):
+            explainer = shap.Explainer(predictor, data)
+            interpretors[label] = explainer
+        self.interpretors = interpretors
+    def get_shap_values(self, data):
+        shap_values = dict()
+        for label, interpretor in self.interpretors.items():
+            shap_value = interpretor(data)
+            shap_values[label] = shap_value
+        return shap_values
+    def register_map(self, mapping):
+        self.mapping = mapping
+
+def mean_shap(data, explainers, pipe_transform):
+    t_data = pipe_transform.transform(data)
+    input_features = t_data.columns
+    shap_results = explainers.get_shap_values(t_data)
+    dict_shap_values = explainers.mapping
+    arrays_ = list()
+    for k,_ in shap_results.items():
+        arrays_.append(shap_results.get(k).values)
+    shap_results_mean = np.mean(np.array(arrays_), axis = 0)
+    df_shap = pd.DataFrame(shap_results_mean, columns=input_features, index=data.index)
+    df_shap['Close'] = data['Close']
+    df_shap['Date'] = data['Date']
+    df_shap = df_shap[['Date','Close']+list(dict_shap_values.keys())]
+    df_shap = df_shap.rename(columns =dict_shap_values)
+    return df_shap
+
+def edge_shap_lines(data, plot = False, look_back = 750):
+    ### corect labels ####
+    shap_cols = [col for col in data.columns if col not in ['Date','Close']]
+    df = data.sort_values('Date').iloc[-look_back:]
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
+    for col in shap_cols:
+        fig.add_trace(go.Scatter(x=df.Date, y=df[col],mode='lines+markers',name=col),secondary_y=True)
+    fig.update_layout(title_text="sirius - feature power",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig

{virgo_modules-0.5.0 → virgo_modules-0.6.1}/virgo_app/virgo_modules/src/edge_utils/stack_model.py

@@ -3,7 +3,7 @@ import pandas as pd
 
 from sklearn.base import BaseEstimator, ClassifierMixin
 
-class MyStackingClassifierMultiClass(BaseEstimator, ClassifierMixin):
+class MyStackingClassifierMultiClass(ClassifierMixin, BaseEstimator):
     def __init__(self,  estimators, meta_estimators,targets,perc=None,stack_size=None, **kwargs):
         self.estimators = estimators
         self.meta_estimators = meta_estimators
@@ -14,6 +14,7 @@ class MyStackingClassifierMultiClass(BaseEstimator, ClassifierMixin):
             raise Exception('set one option')
         self.stack_size = stack_size
         self.perc = perc
+        
     def get_index_training(self, X):
         if self.stack_size:
             unique_dates = list(X.index.get_level_values('Date_i').unique())
@@ -53,6 +54,8 @@ class MyStackingClassifierMultiClass(BaseEstimator, ClassifierMixin):
                 meta_preds_df[metacols],
                 y[X.index.get_level_values('i').isin(meta_indexes)][self.targets[i]]
             )
+        self.is_fitted_ = True
+        self.classes_ = np.array([[0,1],[0,1]])
         
     def predict_proba(self, X):
         metas_pred = dict()

{virgo_modules-0.5.0 → virgo_modules-0.6.1}/virgo_app/virgo_modules.egg-info/PKG-INFO

@@ -1,19 +1,19 @@
 Metadata-Version: 2.1
 Name: virgo-modules
-Version: 0.5.0
+Version: 0.6.1
 Summary: data processing and statistical modeling using stock market data
 Home-page: https://github.com/miguelmayhem92/virgo_module
 Author: Miguel Mayhuire
 Author-email: miguelmayhem92@gmail.com
 License: MIT
-Platform: UNKNOWN
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Provides-Extra: dev
 License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
 
 # Virgo Package
 
@@ -34,4 +34,3 @@ obj = stock_eda_panel(stock_code = 'PEP', n_days = 20)
 obj.get_data()
 print(obj.df.shape)
 ```
-

virgo_modules-0.5.0/virgo_app/virgo_modules/src/edge_utils/conformal_utils.py

@@ -1,71 +0,0 @@
-from plotly.subplots import make_subplots
-import plotly.graph_objects as go
-from mapie.classification import MapieClassifier
-from sklearn.pipeline import Pipeline
-import mlflow
-import numpy as np
-
-
-def get_conformal_classifiers(model, data, targets):
-    classfiers = list()
-    for i, _ in enumerate(model['model'].estimators_):
-        seg_model = Pipeline([ 
-            ('pipe',model['pipe_transform']),
-            ('model',model['model'].estimators_[i])
-        ])
-        mapie_class = MapieClassifier(seg_model, cv='prefit', random_state=123, method="lac")
-        mapie_class.fit(data, data[targets[i]].values)
-        classfiers.append(mapie_class)
-    return classfiers
-
-def log_confmodels(runid, classifiers):
-    with mlflow.start_run(run_id=runid) as run:
-        for i,classifier in enumerate(classifiers):
-            mlflow.sklearn.log_model(classifier,name = f"conformal_model-{i}")
-        print('models were logged')
-
-def load_confmodel(runid, target_variables):
-    classifiers = list()
-    for i in range(len(target_variables)):
-        folder = f"conformal_model-{i}"
-        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}",)
-        classifiers.append(model)
-    return classifiers
-
-
-def get_conformal_prediction(classifier, alphas, data, prefix='conf'):
-    _, y_pis = classifier.predict(data, alpha=alphas)
-    for i,alpha in enumerate(alphas):
-        data[f'{prefix}-{alpha}'] = y_pis[:,1,i]
-        data[f'{prefix}-{alpha}'] = np.where(data[f'{prefix}-{alpha}'] == True,alpha,0)
-    return data
-
-def edge_conformal_lines(data, alphas,threshold = 0.6, plot = False, look_back = 750, offset = 0.08):
-    ### corect labels ####
-    df = data.sort_values('Date').iloc[-look_back:]
-    fig = make_subplots(specs=[[{"secondary_y": True}]])
-    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
-    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(color = 'blue'),showlegend=True,legendgroup='go up', name='go up'),secondary_y=True)
-    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),showlegend=True,legendgroup='go down',name='go down'),secondary_y=True)
-    for i,alpha in enumerate(alphas, start=1):
-        try:
-            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_up' in x][0]
-            df_ = df[df[col_alpha] != 0]
-            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_up + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'blue')
-                                     ,showlegend=False,legendgroup='go up',name='go up', text=df_[col_alpha],textposition="bottom center")
-                                     , secondary_y=True)
-        except:
-            pass
-        try:
-            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_down' in x][0]
-            df_ = df[df[col_alpha] != 0]
-            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_down + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'coral')
-                                     ,showlegend=False,legendgroup='go down', name='go down',text=df_[col_alpha].astype(str),textposition="bottom center")
-                                     , secondary_y=True)
-        except:
-            pass
-    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"))
-    fig.update_layout(title_text="sirius - edge probabilities conformal",width=1200,height = 500)
-    if plot:
-        fig.show()
-    return fig

virgo_modules-0.5.0/virgo_app/virgo_modules/src/edge_utils/shap_utils.py

@@ -1,83 +0,0 @@
-import shap
-import mlflow
-import pandas as pd
-import numpy as np
-from plotly.subplots import make_subplots
-import plotly.graph_objects as go
-
-def get_explainers(model, data):
-    explainers = list()
-    for i, _ in enumerate(model['model'].estimators_):
-        transf_data = model['pipe_transform'].transform(data)
-        predictor = model['model'].estimators_[i]
-        explainer= shap.Explainer(predictor, transf_data)
-        explainers.append(explainer)
-    return explainers
-
-def log_explainer(runid, classifiers):
-    with mlflow.start_run(run_id=runid) as run:
-        for i,classifier in enumerate(classifiers):
-            mlflow.sklearn.log_model(classifier,f"explainer/explainer-{i}")
-        print('models were logged')
-
-def load_explainer(runid, target_variables):
-    explainers = list()
-    for i in range(len(target_variables)):
-        folder = f"explainer/explainer-{i}"
-        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}")
-        explainers.append(model)
-    return explainers
-
-def get_shapvalues(explainers, data):
-    shap_values = {}
-    for i,explainer in enumerate(explainers):
-        shap_value_i = explainer(data)
-        shap_values[i] = shap_value_i
-    return shap_values
-
-def get_explainerclusters(model, data, targets):
-    clustermodels = list()
-    for i, _ in enumerate(model['model'].estimators_):
-        transf_data = model['pipe_transform'].transform(data)
-        Y = data[targets[i]]
-        cluster_model = shap.utils.hclust(transf_data, Y)
-        clustermodels.append(cluster_model)
-    return clustermodels
-
-def mean_shap(data, explainers, pipe_transform, dict_shap_values):
-    t_data = pipe_transform.transform(data)
-    input_features = t_data.columns
-    shap_results = get_shapvalues(explainers,t_data)
-    arrays_ = list()
-    for k,_ in shap_results.items():
-        arrays_.append(shap_results.get(k).values)
-    shap_results_mean = np.mean(np.array(arrays_), axis = 0)
-    df_shap = pd.DataFrame(shap_results_mean, columns=input_features, index=data.index)
-    df_shap['Close'] = data['Close']
-    df_shap['Date'] = data['Date']
-    df_shap = df_shap[['Date','Close']+list(dict_shap_values.keys())]
-    df_shap = df_shap.rename(columns =dict_shap_values)
-    return df_shap
-
-def edge_shap_lines(data, plot = False, look_back = 750):
-    ### corect labels ####
-    shap_cols = [col for col in data.columns if col not in ['Date','Close']]
-    df = data.sort_values('Date').iloc[-look_back:]
-    fig = make_subplots(specs=[[{"secondary_y": True}]])
-    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
-    for col in shap_cols:
-        fig.add_trace(go.Scatter(x=df.Date, y=df[col],mode='lines+markers',name=col),secondary_y=True)
-    fig.update_layout(title_text="sirius - feature power",width=1200,height = 500)
-    if plot:
-        fig.show()
-    return fig
-
-def log_top_shap(runid, top_shap):
-    with mlflow.start_run(run_id=runid) as run:
-        mlflow.log_dict(top_shap,f"explainer/top_shap.json")
-        print('artifact was logged')
-
-def load_top_shap(runid):
-    folder = f"explainer/top_shap.json"
-    top_shap = mlflow.artifacts.load_dict(f"runs:/{runid}/{folder}")
-    return top_shap