virgo-modules 0.2.10__tar.gz → 0.3.1__tar.gz

{virgo_modules-0.2.10 → virgo_modules-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virgo_modules
-Version: 0.2.10
+Version: 0.3.1
 Summary: data processing and statistical modeling using stock market data
 Home-page: https://github.com/miguelmayhem92/virgo_module
 Author: Miguel Mayhuire

{virgo_modules-0.2.10 → virgo_modules-0.3.1}/setup.py

@@ -5,7 +5,7 @@ with open("virgo_app/README.md", "r") as f:
 
 setup(
     name="virgo_modules",
-    version="0.2.10",
+    version="0.3.1",
     description="data processing and statistical modeling using stock market data",
     package_dir={"": "virgo_app"},
     packages=find_packages(where="virgo_app"),

virgo_modules-0.3.1/virgo_app/virgo_modules/src/edge_utils/conformal_utils.py

@@ -0,0 +1,71 @@
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from mapie.classification import MapieClassifier
+from sklearn.pipeline import Pipeline
+import mlflow
+import numpy as np
+
+
+def get_conformal_classifiers(model, data, targets):
+    classfiers = list()
+    for i, _ in enumerate(model['model'].estimators_):
+        seg_model = Pipeline([ 
+            ('pipe',model['pipe_transform']),
+            ('model',model['model'].estimators_[i])
+        ])
+        mapie_class = MapieClassifier(seg_model, cv='prefit', random_state=123, method="lac")
+        mapie_class.fit(data, data[targets[i]].values)
+        classfiers.append(mapie_class)
+    return classfiers
+
+def log_confmodels(runid, classifiers):
+    with mlflow.start_run(run_id=runid) as run:
+        for i,classifier in enumerate(classifiers):
+            mlflow.sklearn.log_model(classifier,f"conformal/conformal_model-{i}")
+        print('models were logged')
+
+def load_confmodel(runid, target_variables):
+    classifiers = list()
+    for i in range(len(target_variables)):
+        folder = f"conformal/conformal_model-{i}"
+        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}")
+        classifiers.append(model)
+    return classifiers
+
+
+def get_conformal_prediction(classifier, alphas, data, prefix='conf'):
+    _, y_pis = classifier.predict(data, alpha=alphas)
+    for i,alpha in enumerate(alphas):
+        data[f'{prefix}-{alpha}'] = y_pis[:,1,i]
+        data[f'{prefix}-{alpha}'] = np.where(data[f'{prefix}-{alpha}'] == True,alpha,0)
+    return data
+
+def edge_conformal_lines(data, alphas,threshold = 0.6, plot = False, look_back = 750, offset = 0.08):
+    ### corect labels ####
+    df = data.sort_values('Date').iloc[-look_back:]
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(color = 'blue'),showlegend=True,legendgroup='go up', name='go up'),secondary_y=True)
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),showlegend=True,legendgroup='go down',name='go down'),secondary_y=True)
+    for i,alpha in enumerate(alphas, start=1):
+        try:
+            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_up' in x][0]
+            df_ = df[df[col_alpha] != 0]
+            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_up + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'blue')
+                                     ,showlegend=False,legendgroup='go up',name='go up', text=df_[col_alpha],textposition="bottom center")
+                                     , secondary_y=True)
+        except:
+            pass
+        try:
+            col_alpha = [x for x in df.columns if str(alpha) in x and 'target_down' in x][0]
+            df_ = df[df[col_alpha] != 0]
+            fig.add_trace(go.Scatter(x=df_.Date, y=df_.proba_target_down + (offset*i),mode='markers',marker = dict(opacity=0.7,size=10, color = 'coral')
+                                     ,showlegend=False,legendgroup='go down', name='go down',text=df_[col_alpha].astype(str),textposition="bottom center")
+                                     , secondary_y=True)
+        except:
+            pass
+    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"))
+    fig.update_layout(title_text="sirius - edge probabilities conformal",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig

{virgo_modules-0.2.10/virgo_app/virgo_modules/src → virgo_modules-0.3.1/virgo_app/virgo_modules/src/edge_utils}/edge_utils.py

@@ -9,7 +9,10 @@ from feature_engine.imputation import  MeanMedianImputer
 from feature_engine.discretisation import EqualWidthDiscretiser
 from feature_engine.datetime import DatetimeFeatures
 
-from .transformer_utils import VirgoWinsorizerFeature, InverseHyperbolicSine, FeaturesEntropy, FeatureSelector
+from ..transformer_utils import VirgoWinsorizerFeature, InverseHyperbolicSine, FeaturesEntropy, FeatureSelector
+
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
 
 class produce_model_wrapper:
     """
@@ -386,4 +389,29 @@ class ExpandingMultipleTimeSeriesKFold:
         -------
         number_window (int): number of splits
         """
-        return self.number_window
+        return self.number_window
+    
+def edge_probas_lines(data, threshold, plot = False, look_back = 750):
+    """
+    produce a plotly plot of edges and closing prices
+
+            Parameters:
+                    data (pd.DataFrame): asset data with edge probabilities
+                    plot (boolean): if true, display plot
+                    threshold (float): edge threshold
+                    look_back (int): number of rows back to display
+
+            Returns:
+                    fig (obj): plotly go object
+    """
+    df = data[['Date','Close','proba_target_down','proba_target_up']].iloc[-look_back:]
+    
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',name='Close price'))
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),name='go down'),secondary_y=True)
+    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(opacity=0.1,size=80), name='go up'),secondary_y=True)
+    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"),)
+    fig.update_layout(title_text="sirius - edge probabilities",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig

virgo_modules-0.3.1/virgo_app/virgo_modules/src/edge_utils/shap_utils.py

@@ -0,0 +1,83 @@
+import shap
+import mlflow
+import pandas as pd
+import numpy as np
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+
+def get_explainers(model, data):
+    explainers = list()
+    for i, _ in enumerate(model['model'].estimators_):
+        transf_data = model['pipe_transform'].transform(data)
+        predictor = model['model'].estimators_[i]
+        explainer= shap.Explainer(predictor, transf_data)
+        explainers.append(explainer)
+    return explainers
+
+def log_explainer(runid, classifiers):
+    with mlflow.start_run(run_id=runid) as run:
+        for i,classifier in enumerate(classifiers):
+            mlflow.sklearn.log_model(classifier,f"explainer/explainer-{i}")
+        print('models were logged')
+
+def load_explainer(runid, target_variables):
+    explainers = list()
+    for i in range(len(target_variables)):
+        folder = f"explainer/explainer-{i}"
+        model = mlflow.sklearn.load_model(f"runs:/{runid}/{folder}")
+        explainers.append(model)
+    return explainers
+
+def get_shapvalues(explainers, data):
+    shap_values = {}
+    for i,explainer in enumerate(explainers):
+        shap_value_i = explainer(data)
+        shap_values[i] = shap_value_i
+    return shap_values
+
+def get_explainerclusters(model, data, targets):
+    clustermodels = list()
+    for i, _ in enumerate(model['model'].estimators_):
+        transf_data = model['pipe_transform'].transform(data)
+        Y = data[targets[i]]
+        cluster_model = shap.utils.hclust(transf_data, Y)
+        clustermodels.append(cluster_model)
+    return clustermodels
+
+def mean_shap(data, explainers, pipe_transform, dict_shap_values):
+    t_data = pipe_transform.transform(data)
+    input_features = t_data.columns
+    shap_results = get_shapvalues(explainers,t_data)
+    arrays_ = list()
+    for k,_ in shap_results.items():
+        arrays_.append(shap_results.get(k).values)
+    shap_results_mean = np.mean(np.array(arrays_), axis = 0)
+    df_shap = pd.DataFrame(shap_results_mean, columns=input_features, index=data.index)
+    df_shap['Close'] = data['Close']
+    df_shap['Date'] = data['Date']
+    df_shap = df_shap[['Date','Close']+list(dict_shap_values.keys())]
+    df_shap = df_shap.rename(columns =dict_shap_values)
+    return df_shap
+
+def edge_shap_lines(data, plot = False, look_back = 750):
+    ### corect labels ####
+    shap_cols = [col for col in data.columns if col not in ['Date','Close']]
+    df = data.sort_values('Date').iloc[-look_back:]
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',marker = dict(color = 'grey'),line = dict(color = 'grey'),name='Close price'))
+    for col in shap_cols:
+        fig.add_trace(go.Scatter(x=df.Date, y=df[col],mode='lines+markers',name=col),secondary_y=True)
+    fig.update_layout(title_text="sirius - feature power",width=1200,height = 500)
+    if plot:
+        fig.show()
+    return fig
+
+def log_top_shap(runid, top_shap):
+    with mlflow.start_run(run_id=runid) as run:
+        mlflow.log_dict(top_shap,f"explainer/top_shap.json")
+        print('artifact was logged')
+
+def load_top_shap(runid):
+    folder = f"explainer/top_shap.json"
+    top_shap = mlflow.artifacts.load_dict(f"runs:/{runid}/{folder}")
+    return top_shap

{virgo_modules-0.2.10 → virgo_modules-0.3.1}/virgo_app/virgo_modules/src/re_utils.py

@@ -1574,8 +1574,9 @@ def produce_probas(model,data, target_variables):
     """
     label_prediction = ['proba_'+x for x in target_variables]
     predictions = model.predict_proba(data)
+    if isinstance(predictions, list):
+        predictions = np.array([ x[:,1].T for x in predictions]).T
     predictions = pd.DataFrame(predictions, columns = label_prediction, index = data.index)
-    
     result_df = pd.concat([data, predictions], axis=1)
     result_df = result_df[['Date'] + target_variables + label_prediction]
 
@@ -1604,27 +1605,9 @@ def produce_signals(result_df, feature_name, threshold, label_prediction):
 
     return result_df
 
-def edge_probas_lines(data, threshold, plot = False, look_back = 750):
-    """
-    produce a plotly plot of edges and closing prices
-
-            Parameters:
-                    data (pd.DataFrame): asset data with edge probabilities
-                    plot (boolean): if true, display plot
-                    threshold (float): edge threshold
-                    look_back (int): number of rows back to display
-
-            Returns:
-                    fig (obj): plotly go object
-    """
-    df = data[['Date','Close','proba_target_down','proba_target_up']].iloc[-look_back:]
-    
-    fig = make_subplots(specs=[[{"secondary_y": True}]])
-    fig.add_trace(go.Scatter(x=df.Date, y=df.Close,mode='lines+markers',name='Close price'))
-    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_down,mode='lines',marker = dict(color = 'coral'),name='go down'),secondary_y=True)
-    fig.add_trace(go.Scatter(x=df.Date, y=df.proba_target_up,mode='lines',marker = dict(opacity=0.1,size=80), name='go up'),secondary_y=True)
-    fig.add_shape(type="line", xref="paper", yref="y2",x0=0.02, y0=threshold, x1=0.9, y1=threshold,line=dict(color="red",dash="dash"),)
-    fig.update_layout(title_text="sirius - edge probabilities",width=1200,height = 500)
-    if plot:
-        fig.show()
-    return fig
+def clean_cols(data, patterns):
+    drop_cols = list()
+    for pattern in patterns:
+        drop_cols = drop_cols + [ x for x in data.columns if pattern in x]
+    data = data.drop(columns = drop_cols)
+    return data

{virgo_modules-0.2.10 → virgo_modules-0.3.1}/virgo_app/virgo_modules.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virgo-modules
-Version: 0.2.10
+Version: 0.3.1
 Summary: data processing and statistical modeling using stock market data
 Home-page: https://github.com/miguelmayhem92/virgo_module
 Author: Miguel Mayhuire

{virgo_modules-0.2.10 → virgo_modules-0.3.1}/virgo_app/virgo_modules.egg-info/SOURCES.txt

@@ -10,9 +10,12 @@ virgo_app/virgo_modules.egg-info/top_level.txt
 virgo_app/virgo_modules/src/__init__.py
 virgo_app/virgo_modules/src/aws_utils.py
 virgo_app/virgo_modules/src/backtester.py
-virgo_app/virgo_modules/src/edge_utils.py
 virgo_app/virgo_modules/src/hmm_utils.py
 virgo_app/virgo_modules/src/pull_artifacts.py
 virgo_app/virgo_modules/src/re_utils.py
 virgo_app/virgo_modules/src/ticketer_source.py
-virgo_app/virgo_modules/src/transformer_utils.py
+virgo_app/virgo_modules/src/transformer_utils.py
+virgo_app/virgo_modules/src/edge_utils/__init__.py
+virgo_app/virgo_modules/src/edge_utils/conformal_utils.py
+virgo_app/virgo_modules/src/edge_utils/edge_utils.py
+virgo_app/virgo_modules/src/edge_utils/shap_utils.py